From d6c4d41c2b9ff16967190e57cb1bad213711e6ab Mon Sep 17 00:00:00 2001 From: Slavi Pantaleev Date: Thu, 18 Feb 2021 18:19:51 +0200 Subject: [PATCH] Define instanceId property on workers This give us the possibility to run multiple instances of workers that that don't expose a port. Right now, we don't support that, but in the future we could run multiple `federation_sender` or `pusher` workers, without them fighting over naming (previously, they'd all be named something like `matrix-synapse-worker-pusher-0`, because they'd all define `port` as `0`). --- .../nginx/conf.d/matrix-synapse.conf.j2 | 8 +++--- roles/matrix-synapse/defaults/main.yml | 28 +++++++++++-------- .../tasks/synapse/workers/init.yml | 6 ++++ .../inject_systemd_services_for_worker.yml | 14 +++++++++- .../workers/util/setup_files_for_worker.yml | 5 ++-- 5 files changed, 42 insertions(+), 19 deletions(-) diff --git a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 index 6801f4f9..0b44678c 100644 --- a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 +++ b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 @@ -14,7 +14,7 @@ ip_hash; {% for worker in generic_workers %} - server "matrix-synapse-worker-{{ worker.type }}-{{ worker.port }}:{{ worker.port }}"; + server "matrix-synapse-worker-{{ worker.type }}-{{ worker.instanceId }}:{{ worker.port }}"; {% endfor %} } {% endif %} @@ -22,7 +22,7 @@ {% if frontend_proxy_workers %} upstream frontend_proxy_upstream { {% for worker in frontend_proxy_workers %} - server "matrix-synapse-worker-{{ worker.type }}-{{ worker.port }}:{{ worker.port }}"; + server "matrix-synapse-worker-{{ worker.type }}-{{ worker.instanceId }}:{{ worker.port }}"; {% endfor %} } {% endif %} @@ -30,7 +30,7 @@ {% if media_repository_workers %} upstream media_repository_upstream { {% for worker in media_repository_workers %} - server "matrix-synapse-worker-{{ worker.type }}-{{ worker.port }}:{{ worker.port }}"; + server "matrix-synapse-worker-{{ worker.type }}-{{ worker.instanceId }}:{{ worker.port }}"; {% endfor %} } {% endif %} @@ -38,7 +38,7 @@ {% if user_dir_workers %} upstream user_dir_upstream { {% for worker in user_dir_workers %} - server "matrix-synapse-worker-{{ worker.type }}-{{ worker.port }}:{{ worker.port }}"; + server "matrix-synapse-worker-{{ worker.type }}-{{ worker.instanceId }}:{{ worker.port }}"; {% endfor %} } {% endif %} diff --git a/roles/matrix-synapse/defaults/main.yml b/roles/matrix-synapse/defaults/main.yml index 17f2556a..dc7f4347 100644 --- a/roles/matrix-synapse/defaults/main.yml +++ b/roles/matrix-synapse/defaults/main.yml @@ -353,18 +353,24 @@ matrix_synapse_workers_frontend_proxy_workers_metrics_range_start: 19771 # Doing that is simpler and also protects you from shooting yourself in the foot, # as certain workers can only be spawned just once. # -# Example of what this needs to look like: +# Each worker instance in the list defines the following fields: +# - `type` - the type of worker (`generic_worker`, etc.) +# - `instanceId` - a string that identifies the worker. The combination of (`type` + `instanceId`) represents the name of the worker and must be unique. +# - `port` - an HTTP port where the worker listens for requests (can be `0` for workers that don't do HTTP request processing) +# - `metrics_port` - an HTTP port where the worker exports Prometheus metrics +# +# Example of what this needs to look like, if you're defining it manually: # matrix_synapse_workers_enabled_list: -# - { type: generic_worker, port: 18111, metrics_port: 19111 } -# - { type: generic_worker, port: 18112, metrics_port: 19112 } -# - { type: generic_worker, port: 18113, metrics_port: 19113 } -# - { type: generic_worker, port: 18114, metrics_port: 19114 } -# - { type: generic_worker, port: 18115, metrics_port: 19115 } -# - { type: generic_worker, port: 18116, metrics_port: 19116 } -# - { type: pusher, port: 0, metrics_port: 19200 } -# - { type: appservice, port: 0, metrics_port: 19300 } -# - { type: federation_sender, port: 0, metrics_port: 19400 } -# - { type: media_repository, port: 18551, metrics_port: 19551 } +# - { type: generic_worker, instanceId: '18111', port: 18111, metrics_port: 19111 } +# - { type: generic_worker, instanceId: '18112', port: 18112, metrics_port: 19112 } +# - { type: generic_worker, instanceId: '18113', port: 18113, metrics_port: 19113 } +# - { type: generic_worker, instanceId: '18114', port: 18114, metrics_port: 19114 } +# - { type: generic_worker, instanceId: '18115', port: 18115, metrics_port: 19115 } +# - { type: generic_worker, instanceId: '18116', port: 18116, metrics_port: 19116 } +# - { type: pusher, instanceId: '0', port: 0, metrics_port: 19200 } +# - { type: appservice, instanceId: '0', port: 0, metrics_port: 19300 } +# - { type: federation_sender, instanceId: '0', port: 0, metrics_port: 19400 } +# - { type: media_repository, instanceId: '18551', port: 18551, metrics_port: 19551 } matrix_synapse_workers_enabled_list: [] # Redis information diff --git a/roles/matrix-synapse/tasks/synapse/workers/init.yml b/roles/matrix-synapse/tasks/synapse/workers/init.yml index a4c4f36f..204a0841 100644 --- a/roles/matrix-synapse/tasks/synapse/workers/init.yml +++ b/roles/matrix-synapse/tasks/synapse/workers/init.yml @@ -7,6 +7,7 @@ set_fact: worker: type: 'generic_worker' + instanceId: "{{ matrix_synapse_workers_generic_workers_port_range_start + item }}" port: "{{ matrix_synapse_workers_generic_workers_port_range_start + item }}" metrics_port: "{{ matrix_synapse_workers_generic_workers_metrics_range_start + item }}" register: "matrix_synapse_workers_list_results_generic_workers" @@ -16,6 +17,7 @@ set_fact: worker: type: 'federation_sender' + instanceId: "{{ item }}" port: 0 metrics_port: "{{ matrix_synapse_workers_federation_sender_workers_metrics_range_start + item }}" register: "matrix_synapse_workers_list_results_federation_sender_workers" @@ -26,6 +28,7 @@ set_fact: worker: type: 'pusher' + instanceId: "{{ item }}" port: 0 metrics_port: "{{ matrix_synapse_workers_pusher_workers_metrics_range_start + item }}" register: "matrix_synapse_workers_list_results_pusher_workers" @@ -36,6 +39,7 @@ set_fact: worker: type: 'appservice' + instanceId: "{{ item }}" port: 0 metrics_port: "{{ matrix_synapse_workers_appservice_workers_metrics_range_start + item }}" register: "matrix_synapse_workers_list_results_appservice_workers" @@ -45,6 +49,7 @@ set_fact: worker: type: 'media_repository' + instanceId: "{{ matrix_synapse_workers_media_repository_workers_port_range_start + item }}" port: "{{ matrix_synapse_workers_media_repository_workers_port_range_start + item }}" metrics_port: "{{ matrix_synapse_workers_media_repository_workers_metrics_range_start + item }}" register: "matrix_synapse_workers_list_results_media_repository_workers" @@ -54,6 +59,7 @@ set_fact: worker: type: 'frontend_proxy' + instanceId: "{{ matrix_synapse_workers_frontend_proxy_workers_port_range_start + item }}" port: "{{ matrix_synapse_workers_frontend_proxy_workers_port_range_start + item }}" metrics_port: "{{ matrix_synapse_workers_frontend_proxy_workers_metrics_range_start + item }}" register: "matrix_synapse_workers_list_results_frontend_proxy_workers" diff --git a/roles/matrix-synapse/tasks/synapse/workers/util/inject_systemd_services_for_worker.yml b/roles/matrix-synapse/tasks/synapse/workers/util/inject_systemd_services_for_worker.yml index c95f881a..62b42625 100644 --- a/roles/matrix-synapse/tasks/synapse/workers/util/inject_systemd_services_for_worker.yml +++ b/roles/matrix-synapse/tasks/synapse/workers/util/inject_systemd_services_for_worker.yml @@ -1,6 +1,18 @@ +# The tasks below run before `validate_config.yml`. +# To avoid failing with a cryptic error message, we'll do validation here. +# +# This check is mostly relevant to people who explicitly define `matrix_synapse_workers_enabled_list` +# (Synapse Workers users from the earlier days of this PR - https://github.com/spantaleev/matrix-docker-ansible-deploy/pull/456). +# +# In the future, it should be possible to remove this check. +# Our own code which dynamically builds `matrix_synapse_workers_enabled_list` does things right. +- name: Fail if instanceId not defined for worker + fail: + msg: "Synapse workers (like {{ matrix_synapse_worker_details|to_json }}) need to define an instanceId property (type + instanceId must be unique)" + when: "'instanceId' not in matrix_synapse_worker_details" - set_fact: - matrix_synapse_worker_systemd_service_name: "matrix-synapse-worker-{{ matrix_synapse_worker_details.type }}-{{ matrix_synapse_worker_details.port }}.service" + matrix_synapse_worker_systemd_service_name: "matrix-synapse-worker-{{ matrix_synapse_worker_details.type }}-{{ matrix_synapse_worker_details.instanceId }}.service" - set_fact: matrix_systemd_services_list: "{{ matrix_systemd_services_list + [matrix_synapse_worker_systemd_service_name] }}" diff --git a/roles/matrix-synapse/tasks/synapse/workers/util/setup_files_for_worker.yml b/roles/matrix-synapse/tasks/synapse/workers/util/setup_files_for_worker.yml index 6a15e048..93ed6575 100644 --- a/roles/matrix-synapse/tasks/synapse/workers/util/setup_files_for_worker.yml +++ b/roles/matrix-synapse/tasks/synapse/workers/util/setup_files_for_worker.yml @@ -1,12 +1,11 @@ - - set_fact: - matrix_synapse_worker_systemd_service_name: "matrix-synapse-worker-{{ matrix_synapse_worker_details.type }}-{{ matrix_synapse_worker_details.port }}" + matrix_synapse_worker_systemd_service_name: "matrix-synapse-worker-{{ matrix_synapse_worker_details.type }}-{{ matrix_synapse_worker_details.instanceId }}" - set_fact: matrix_synapse_worker_container_name: "{{ matrix_synapse_worker_systemd_service_name }}" - set_fact: - matrix_synapse_worker_config_file_name: "worker.{{ matrix_synapse_worker_details.type }}_{{ matrix_synapse_worker_details.port }}.yaml" + matrix_synapse_worker_config_file_name: "worker.{{ matrix_synapse_worker_details.type }}_{{ matrix_synapse_worker_details.instanceId }}.yaml" - name: Ensure configuration exists for {{ matrix_synapse_worker_systemd_service_name }} template: