From cd6808ba9f937f30716e4b844358189a49105f6c Mon Sep 17 00:00:00 2001 From: Julien Moutinho Date: Fri, 13 May 2022 00:53:16 +0200 Subject: [PATCH] sourcehut: fix Unix socket support for Redis --- .../services/misc/sourcehut/default.nix | 2 +- .../services/misc/sourcehut/service.nix | 2 +- nixos/tests/sourcehut.nix | 1 + .../version-management/sourcehut/builds.nix | 4 -- .../version-management/sourcehut/core.nix | 2 + .../version-management/sourcehut/git.nix | 4 -- .../version-management/sourcehut/lists.nix | 4 -- .../version-management/sourcehut/meta.nix | 4 -- ...uild-submission-and-queue-monitoring.patch | 69 ------------------- ...ocket-support-in-RedisQueueCollector.patch | 42 +++++++++++ ...-Revert-Add-webhook-queue-monitoring.patch | 33 --------- ...-Revert-Add-webhook-queue-monitoring.patch | 48 ------------- ...-Revert-Add-webhook-queue-monitoring.patch | 48 ------------- ...-Revert-Add-webhook-queue-monitoring.patch | 50 -------------- .../version-management/sourcehut/todo.nix | 5 -- 15 files changed, 47 insertions(+), 271 deletions(-) delete mode 100644 pkgs/applications/version-management/sourcehut/patches/redis-socket/build/0001-Revert-Add-build-submission-and-queue-monitoring.patch create mode 100644 pkgs/applications/version-management/sourcehut/patches/redis-socket/core/0001-Fix-Unix-socket-support-in-RedisQueueCollector.patch delete mode 100644 pkgs/applications/version-management/sourcehut/patches/redis-socket/git/0001-Revert-Add-webhook-queue-monitoring.patch delete mode 100644 pkgs/applications/version-management/sourcehut/patches/redis-socket/lists/0001-Revert-Add-webhook-queue-monitoring.patch delete mode 100644 pkgs/applications/version-management/sourcehut/patches/redis-socket/meta/0001-Revert-Add-webhook-queue-monitoring.patch delete mode 100644 pkgs/applications/version-management/sourcehut/patches/redis-socket/todo/0001-Revert-Add-webhook-queue-monitoring.patch diff --git a/nixos/modules/services/misc/sourcehut/default.nix b/nixos/modules/services/misc/sourcehut/default.nix index 70f6ea7ce97..f5e4742d902 100644 --- a/nixos/modules/services/misc/sourcehut/default.nix +++ b/nixos/modules/services/misc/sourcehut/default.nix @@ -83,7 +83,7 @@ let python = pkgs.sourcehut.python.withPackages (ps: with ps; [ gunicorn eventlet - # For monitoring Celery: sudo -u listssrht celery --app listssrht.process -b redis+socket:///run/redis-sourcehut/redis.sock?virtual_host=5 flower + # For monitoring Celery: sudo -u listssrht celery --app listssrht.process -b redis+socket:///run/redis-sourcehut/redis.sock?virtual_host=1 flower flower # Sourcehut services srht diff --git a/nixos/modules/services/misc/sourcehut/service.nix b/nixos/modules/services/misc/sourcehut/service.nix index f1706ad0a6a..4ecc7a72669 100644 --- a/nixos/modules/services/misc/sourcehut/service.nix +++ b/nixos/modules/services/misc/sourcehut/service.nix @@ -148,7 +148,7 @@ in redis = { host = mkOption { type = types.str; - default = "unix:/run/redis-sourcehut-${srvsrht}/redis.sock?db=0"; + default = "unix:///run/redis-sourcehut-${srvsrht}/redis.sock?db=0"; example = "redis://shared.wireguard:6379/0"; description = '' The redis host URL. This is used for caching and temporary storage, and must diff --git a/nixos/tests/sourcehut.nix b/nixos/tests/sourcehut.nix index 34a60247e00..6998f4f0c10 100644 --- a/nixos/tests/sourcehut.nix +++ b/nixos/tests/sourcehut.nix @@ -195,6 +195,7 @@ in # Testing metasrht machine.wait_for_unit("metasrht-api.service") machine.wait_for_unit("metasrht.service") + machine.wait_for_unit("metasrht-webhooks.service") machine.wait_for_open_port(5000) machine.succeed("curl -sL http://localhost:5000 | grep meta.${domain}") machine.succeed("curl -sL http://meta.${domain} | grep meta.${domain}") diff --git a/pkgs/applications/version-management/sourcehut/builds.nix b/pkgs/applications/version-management/sourcehut/builds.nix index bef1b175cd6..99d6a05c4eb 100644 --- a/pkgs/applications/version-management/sourcehut/builds.nix +++ b/pkgs/applications/version-management/sourcehut/builds.nix @@ -39,10 +39,6 @@ buildPythonPackage rec { inherit src version; pname = "buildsrht"; - patches = [ - # Revert change breaking Unix socket support for Redis - patches/redis-socket/build/0001-Revert-Add-build-submission-and-queue-monitoring.patch - ]; postPatch = '' substituteInPlace Makefile \ --replace "all: api worker" "" diff --git a/pkgs/applications/version-management/sourcehut/core.nix b/pkgs/applications/version-management/sourcehut/core.nix index 610d5e2647c..0648f21910f 100644 --- a/pkgs/applications/version-management/sourcehut/core.nix +++ b/pkgs/applications/version-management/sourcehut/core.nix @@ -48,6 +48,8 @@ buildPythonPackage rec { patches = [ # Disable check for npm ./disable-npm-install.patch + # Fix Unix socket support in RedisQueueCollector + patches/redis-socket/core/0001-Fix-Unix-socket-support-in-RedisQueueCollector.patch ]; nativeBuildInputs = [ diff --git a/pkgs/applications/version-management/sourcehut/git.nix b/pkgs/applications/version-management/sourcehut/git.nix index e56022604fa..a721da0ecd3 100644 --- a/pkgs/applications/version-management/sourcehut/git.nix +++ b/pkgs/applications/version-management/sourcehut/git.nix @@ -58,10 +58,6 @@ buildPythonPackage rec { inherit src version; pname = "gitsrht"; - patches = [ - # Revert change breaking Unix socket support for Redis - patches/redis-socket/git/0001-Revert-Add-webhook-queue-monitoring.patch - ]; postPatch = '' substituteInPlace Makefile \ --replace "all: api gitsrht-dispatch gitsrht-keys gitsrht-shell gitsrht-update-hook" "" diff --git a/pkgs/applications/version-management/sourcehut/lists.nix b/pkgs/applications/version-management/sourcehut/lists.nix index 213ef3a8cfd..b9a00ffd213 100644 --- a/pkgs/applications/version-management/sourcehut/lists.nix +++ b/pkgs/applications/version-management/sourcehut/lists.nix @@ -30,10 +30,6 @@ buildPythonPackage rec { vendorSha256 = "sha256-xnmMkRSokbhWD+kz0XQ9AinYdm6/50FRBISURPvlzD0="; } // import ./fix-gqlgen-trimpath.nix {inherit unzip;}); - patches = [ - # Revert change breaking Unix socket support for Redis - patches/redis-socket/lists/0001-Revert-Add-webhook-queue-monitoring.patch - ]; postPatch = '' substituteInPlace Makefile \ --replace "all: api" "" diff --git a/pkgs/applications/version-management/sourcehut/meta.nix b/pkgs/applications/version-management/sourcehut/meta.nix index 79293736e4a..896279addd3 100644 --- a/pkgs/applications/version-management/sourcehut/meta.nix +++ b/pkgs/applications/version-management/sourcehut/meta.nix @@ -40,10 +40,6 @@ buildPythonPackage rec { pname = "metasrht"; inherit version src; - patches = [ - # Revert change breaking Unix socket support for Redis - patches/redis-socket/meta/0001-Revert-Add-webhook-queue-monitoring.patch - ]; postPatch = '' substituteInPlace Makefile \ --replace "all: api" "" diff --git a/pkgs/applications/version-management/sourcehut/patches/redis-socket/build/0001-Revert-Add-build-submission-and-queue-monitoring.patch b/pkgs/applications/version-management/sourcehut/patches/redis-socket/build/0001-Revert-Add-build-submission-and-queue-monitoring.patch deleted file mode 100644 index ae8e95ddc0c..00000000000 --- a/pkgs/applications/version-management/sourcehut/patches/redis-socket/build/0001-Revert-Add-build-submission-and-queue-monitoring.patch +++ /dev/null @@ -1,69 +0,0 @@ -From 069b03f85847ed4a9223183b62ee53f420838911 Mon Sep 17 00:00:00 2001 -From: Julien Moutinho -Date: Thu, 16 Dec 2021 04:54:24 +0100 -Subject: [PATCH builds.sr.ht] Revert "Add build submission and queue - monitoring" - -This reverts commit 690f1aa16c77e418dc40109cd5e8fdf4a7ed947a. - -This has broken Unix socket support for Redis -See https://lists.sr.ht/~sircmpwn/sr.ht-dev/%3C20211208082636.65665-1-me%40ignaskiela.eu%3E#%3C20211216033723.wefibfulfjhqnhem@sourcephile.fr%3E ---- - buildsrht/app.py | 3 --- - buildsrht/runner.py | 9 +-------- - 2 files changed, 1 insertion(+), 11 deletions(-) - -diff --git a/buildsrht/app.py b/buildsrht/app.py -index e5321a2..7c9977c 100644 ---- a/buildsrht/app.py -+++ b/buildsrht/app.py -@@ -36,9 +36,6 @@ class BuildApp(SrhtFlask): - self.register_blueprint(secrets) - self.register_blueprint(gql_blueprint) - -- from buildsrht.runner import builds_queue_metrics_collector -- self.metrics_registry.register(builds_queue_metrics_collector) -- - @self.context_processor - def inject(): - return { -diff --git a/buildsrht/runner.py b/buildsrht/runner.py -index 7773452..0389c8e 100644 ---- a/buildsrht/runner.py -+++ b/buildsrht/runner.py -@@ -5,13 +5,10 @@ from srht.config import cfg - from srht.database import db - from srht.email import send_email - from srht.oauth import UserType --from srht.metrics import RedisQueueCollector --from prometheus_client import Counter - - allow_free = cfg("builds.sr.ht", "allow-free", default="no") == "yes" - --builds_broker = cfg("builds.sr.ht", "redis") --runner = Celery('builds', broker=builds_broker, config_source={ -+runner = Celery('builds', broker=cfg("builds.sr.ht", "redis"), config_source={ - "CELERY_TASK_SERIALIZER": "json", - "CELERY_ACCEPT_CONTENT": ["json"], - "CELERY_RESULT_SERIALIZER": "json", -@@ -19,9 +16,6 @@ runner = Celery('builds', broker=builds_broker, config_source={ - "CELERY_TASK_PROTOCOL": 1 - }) - --builds_queue_metrics_collector = RedisQueueCollector(builds_broker, "buildsrht_builds", "Number of builds currently in queue") --builds_submitted = Counter("buildsrht_builds_submited", "Number of builds submitted") -- - def queue_build(job, manifest): - from buildsrht.types import JobStatus - job.status = JobStatus.queued -@@ -34,7 +28,6 @@ def queue_build(job, manifest): - cfg("sr.ht", "owner-email"), - "Cryptocurrency mining attempt on builds.sr.ht") - else: -- builds_submitted.inc() - run_build.delay(job.id, manifest.to_dict()) - - def requires_payment(user): --- -2.34.0 - diff --git a/pkgs/applications/version-management/sourcehut/patches/redis-socket/core/0001-Fix-Unix-socket-support-in-RedisQueueCollector.patch b/pkgs/applications/version-management/sourcehut/patches/redis-socket/core/0001-Fix-Unix-socket-support-in-RedisQueueCollector.patch new file mode 100644 index 00000000000..889331f3e07 --- /dev/null +++ b/pkgs/applications/version-management/sourcehut/patches/redis-socket/core/0001-Fix-Unix-socket-support-in-RedisQueueCollector.patch @@ -0,0 +1,42 @@ +From 76dd636151735671be74ba9d55f773e190e22827 Mon Sep 17 00:00:00 2001 +From: Julien Moutinho +Date: Fri, 13 May 2022 22:40:46 +0200 +Subject: [PATCH core.sr.ht] Fix Unix socket support in RedisQueueCollector + +The broker URL is not necessarily in the format expected by Redis.from_url + +Especially, Redis.from_url supports this format for Unix sockets: + unix:///run/redis-sourcehut-metasrht/redis.sock?db=0 +See https://redis-py.readthedocs.io/en/stable/#redis.ConnectionPool.from_url + +Whereas Celery+Kombu support Redis but also other transports +and thus expect another scheme: + redis+socket:///run/redis-sourcehut-metasrht/redis.sock?virtual_host=1 +See https://docs.celeryproject.org/en/stable/userguide/configuration.html#redis-backend-settings +and https://github.com/celery/celery/blob/e5d99801e4b56a02af4a2e183879c767228d2817/celery/backends/redis.py#L299-L352 +and https://github.com/celery/kombu/blob/master/kombu/utils/url.py +--- + srht/metrics.py | 3 ++- + 1 file changed, 2 insertions(+), 1 deletion(-) + +diff --git a/srht/metrics.py b/srht/metrics.py +index 68caf8e..2df5777 100644 +--- a/srht/metrics.py ++++ b/srht/metrics.py +@@ -1,11 +1,12 @@ + import time ++from celery import Celery + from prometheus_client.metrics_core import GaugeMetricFamily + from redis import Redis, ResponseError + + + class RedisQueueCollector: + def __init__(self, broker, name, documentation, queue_name="celery"): +- self.redis = Redis.from_url(broker) ++ self.redis = Celery("collector", broker=broker).connection_for_read().channel().client + self.queue_name = queue_name + self.name = name + self.documentation = documentation +-- +2.35.1 + diff --git a/pkgs/applications/version-management/sourcehut/patches/redis-socket/git/0001-Revert-Add-webhook-queue-monitoring.patch b/pkgs/applications/version-management/sourcehut/patches/redis-socket/git/0001-Revert-Add-webhook-queue-monitoring.patch deleted file mode 100644 index f9235102b95..00000000000 --- a/pkgs/applications/version-management/sourcehut/patches/redis-socket/git/0001-Revert-Add-webhook-queue-monitoring.patch +++ /dev/null @@ -1,33 +0,0 @@ -diff --git a/gitsrht/app.py b/gitsrht/app.py -index eaae395..731bb03 100644 ---- a/gitsrht/app.py -+++ b/gitsrht/app.py -@@ -55,9 +55,6 @@ class GitApp(SrhtFlask): - self.add_template_filter(url_quote) - self.add_template_filter(commit_links) - -- from gitsrht.webhooks import webhook_metrics_collector -- self.metrics_registry.register(webhook_metrics_collector) -- - @self.context_processor - def inject(): - notice = session.get("notice") -diff --git a/gitsrht/webhooks.py b/gitsrht/webhooks.py -index 36486d0..0109428 100644 ---- a/gitsrht/webhooks.py -+++ b/gitsrht/webhooks.py -@@ -7,12 +7,10 @@ if not hasattr(db, "session"): - db.init() - from srht.webhook import Event - from srht.webhook.celery import CeleryWebhook, make_worker --from srht.metrics import RedisQueueCollector -+from scmsrht.webhooks import UserWebhook - import sqlalchemy as sa - --webhook_broker = cfg("git.sr.ht", "webhooks") --worker = make_worker(broker=webhook_broker) --webhook_metrics_collector = RedisQueueCollector(webhook_broker, "srht_webhooks", "Webhook queue length") -+worker = make_worker(broker=cfg("git.sr.ht", "webhooks")) - - class UserWebhook(CeleryWebhook): - events = [ diff --git a/pkgs/applications/version-management/sourcehut/patches/redis-socket/lists/0001-Revert-Add-webhook-queue-monitoring.patch b/pkgs/applications/version-management/sourcehut/patches/redis-socket/lists/0001-Revert-Add-webhook-queue-monitoring.patch deleted file mode 100644 index 872d285b9bb..00000000000 --- a/pkgs/applications/version-management/sourcehut/patches/redis-socket/lists/0001-Revert-Add-webhook-queue-monitoring.patch +++ /dev/null @@ -1,48 +0,0 @@ -From 730e090f31b150d42be4b4722751f8e4610835b0 Mon Sep 17 00:00:00 2001 -From: Julien Moutinho -Date: Sat, 12 Feb 2022 00:38:12 +0100 -Subject: [PATCH lists.sr.ht] Revert "Add webhook queue monitoring" - -This reverts commit e74e344808e8d523a9786cefcbf64c9a247d7a0e. - -Which has broken Unix socket support for Redis. -See https://lists.sr.ht/~sircmpwn/sr.ht-dev/%3C20211208082636.65665-1-me%40ignaskiela.eu%3E#%3C20211216033723.wefibfulfjhqnhem@sourcephile.fr%3E ---- - listssrht/app.py | 3 --- - listssrht/webhooks.py | 5 +---- - 2 files changed, 1 insertion(+), 7 deletions(-) - -diff --git a/listssrht/app.py b/listssrht/app.py -index aec59f3..83a355d 100644 ---- a/listssrht/app.py -+++ b/listssrht/app.py -@@ -29,9 +29,6 @@ class ListsApp(SrhtFlask): - self.register_blueprint(user) - self.register_blueprint(gql_blueprint) - -- from listssrht.webhooks import webhook_metrics_collector -- self.metrics_registry.register(webhook_metrics_collector) -- - @self.context_processor - def inject(): - from listssrht.types import ListAccess -diff --git a/listssrht/webhooks.py b/listssrht/webhooks.py -index ae5b1cb..86421ba 100644 ---- a/listssrht/webhooks.py -+++ b/listssrht/webhooks.py -@@ -8,11 +8,8 @@ if not hasattr(db, "session"): - db.init() - from srht.webhook import Event - from srht.webhook.celery import CeleryWebhook, make_worker --from srht.metrics import RedisQueueCollector - --webhook_broker = cfg("lists.sr.ht", "webhooks") --worker = make_worker(broker=webhook_broker) --webhook_metrics_collector = RedisQueueCollector(webhook_broker, "srht_webhooks", "Webhook queue length") -+worker = make_worker(broker=cfg("lists.sr.ht", "webhooks")) - - class ListWebhook(CeleryWebhook): - events = [ --- -2.34.1 - diff --git a/pkgs/applications/version-management/sourcehut/patches/redis-socket/meta/0001-Revert-Add-webhook-queue-monitoring.patch b/pkgs/applications/version-management/sourcehut/patches/redis-socket/meta/0001-Revert-Add-webhook-queue-monitoring.patch deleted file mode 100644 index 9ec37670b06..00000000000 --- a/pkgs/applications/version-management/sourcehut/patches/redis-socket/meta/0001-Revert-Add-webhook-queue-monitoring.patch +++ /dev/null @@ -1,48 +0,0 @@ -From d88bee195797c6c294320617ff14798da94cd0f3 Mon Sep 17 00:00:00 2001 -From: Julien Moutinho -Date: Thu, 16 Dec 2021 04:52:08 +0100 -Subject: [PATCH meta.sr.ht] Revert "Add webhook queue monitoring" - -This reverts commit 9931df3c23094af5179df9ef019ca732b8125dac. - -This has broken Unix socket support for Redis. -See https://lists.sr.ht/~sircmpwn/sr.ht-dev/%3C20211208082636.65665-1-me%40ignaskiela.eu%3E#%3C20211216033723.wefibfulfjhqnhem@sourcephile.fr%3E ---- - metasrht/app.py | 3 --- - metasrht/webhooks.py | 5 +---- - 2 files changed, 1 insertion(+), 7 deletions(-) - -diff --git a/metasrht/app.py b/metasrht/app.py -index b190875..89c59bc 100644 ---- a/metasrht/app.py -+++ b/metasrht/app.py -@@ -49,9 +49,6 @@ class MetaApp(SrhtFlask): - from metasrht.blueprints.billing import billing - self.register_blueprint(billing) - -- from metasrht.webhooks import webhook_metrics_collector -- self.metrics_registry.register(webhook_metrics_collector) -- - @self.context_processor - def inject(): - return { -diff --git a/metasrht/webhooks.py b/metasrht/webhooks.py -index 3e1149e..3f0ba01 100644 ---- a/metasrht/webhooks.py -+++ b/metasrht/webhooks.py -@@ -7,11 +7,8 @@ if not hasattr(db, "session"): - db.init() - from srht.webhook import Event - from srht.webhook.celery import CeleryWebhook, make_worker --from srht.metrics import RedisQueueCollector - --webhook_broker = cfg("meta.sr.ht", "webhooks", "redis://") --worker = make_worker(broker=webhook_broker) --webhook_metrics_collector = RedisQueueCollector(webhook_broker, "srht_webhooks", "Webhook queue length") -+worker = make_worker(broker=cfg("meta.sr.ht", "webhooks", "redis://")) - - class UserWebhook(CeleryWebhook): - events = [ --- -2.34.0 - diff --git a/pkgs/applications/version-management/sourcehut/patches/redis-socket/todo/0001-Revert-Add-webhook-queue-monitoring.patch b/pkgs/applications/version-management/sourcehut/patches/redis-socket/todo/0001-Revert-Add-webhook-queue-monitoring.patch deleted file mode 100644 index 861608c1184..00000000000 --- a/pkgs/applications/version-management/sourcehut/patches/redis-socket/todo/0001-Revert-Add-webhook-queue-monitoring.patch +++ /dev/null @@ -1,50 +0,0 @@ -From 42a27ea60d8454552d54e1f51f1b976d1067fc32 Mon Sep 17 00:00:00 2001 -From: Julien Moutinho -Date: Sat, 12 Feb 2022 00:30:29 +0100 -Subject: [PATCH todo.sr.ht] Revert "Add webhook queue monitoring" - -This reverts commit 320a5e8f7cd16ca43928c36f0320593f84d986fa. - -Which has broken Unix socket support for Redis. -See https://lists.sr.ht/~sircmpwn/sr.ht-dev/%3C20211208082636.65665-1-me%40ignaskiela.eu%3E#%3C20211216033723.wefibfulfjhqnhem@sourcephile.fr%3E ---- - todosrht/flask.py | 3 --- - todosrht/webhooks.py | 6 +----- - 2 files changed, 1 insertion(+), 8 deletions(-) - -diff --git a/todosrht/flask.py b/todosrht/flask.py -index 5e8ac66..9d0fd27 100644 ---- a/todosrht/flask.py -+++ b/todosrht/flask.py -@@ -43,9 +43,6 @@ class TodoApp(SrhtFlask): - self.add_template_filter(urls.tracker_url) - self.add_template_filter(urls.user_url) - -- from todosrht.webhooks import webhook_metrics_collector -- self.metrics_registry.register(webhook_metrics_collector) -- - @self.context_processor - def inject(): - return { -diff --git a/todosrht/webhooks.py b/todosrht/webhooks.py -index eb8e08a..950047f 100644 ---- a/todosrht/webhooks.py -+++ b/todosrht/webhooks.py -@@ -7,13 +7,9 @@ if not hasattr(db, "session"): - db.init() - from srht.webhook import Event - from srht.webhook.celery import CeleryWebhook, make_worker --from srht.metrics import RedisQueueCollector - import sqlalchemy as sa - -- --webhooks_broker = cfg("todo.sr.ht", "webhooks") --worker = make_worker(broker=webhooks_broker) --webhook_metrics_collector = RedisQueueCollector(webhooks_broker, "srht_webhooks", "Webhook queue length") -+worker = make_worker(broker=cfg("todo.sr.ht", "webhooks")) - - import todosrht.tracker_import - --- -2.34.1 - diff --git a/pkgs/applications/version-management/sourcehut/todo.nix b/pkgs/applications/version-management/sourcehut/todo.nix index 9dbe86e6f63..67367b71ff9 100644 --- a/pkgs/applications/version-management/sourcehut/todo.nix +++ b/pkgs/applications/version-management/sourcehut/todo.nix @@ -23,16 +23,11 @@ buildPythonPackage rec { sha256 = "sha256-m7FY+jXpwPnK1+b1iQiDGe8JPfAFQp65BzGH6WvNwhM="; }; - patches = [ - # Revert change breaking Unix socket support for Redis - patches/redis-socket/todo/0001-Revert-Add-webhook-queue-monitoring.patch - ]; postPatch = '' substituteInPlace Makefile \ --replace "all: api" "" ''; - todosrht-api = buildGoModule ({ inherit src version; pname = "todosrht-api";