From 353bc7c3624f6b7186d65757e7b66bc8e6e56e2c Mon Sep 17 00:00:00 2001 From: Marcel Partap Date: Fri, 10 Apr 2020 23:44:53 +0200 Subject: [PATCH] Add initial support for synapse workers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit · needs documentation; no checks yet for port clashes or typos in worker name · according to https://github.com/matrix-org/synapse/wiki/Workers-setup-with-nginx#results about 90% of requests go to the synchrotron endpoint · thus, the synchrotron worker is especially suited to be load-balanced · most of the other workers are documented to support only a single instance · https://github.com/matrix-org/synapse/blob/master/docs/workers.md --- roles/matrix-synapse/defaults/main.yml | 37 +++++++++++++++++ roles/matrix-synapse/tasks/setup_synapse.yml | 2 + roles/matrix-synapse/tasks/workers/setup.yml | 7 ++++ .../tasks/workers/setup_install.yml | 41 +++++++++++++++++++ .../tasks/workers/setup_uninstall.yml | 34 +++++++++++++++ .../templates/synapse/homeserver.yaml.j2 | 38 +++++++++++++++++ .../systemd/matrix-synapse-worker@.service.j2 | 29 +++++++++++++ .../templates/synapse/worker.yaml.j2 | 29 +++++++++++++ 8 files changed, 217 insertions(+) create mode 100644 roles/matrix-synapse/tasks/workers/setup.yml create mode 100644 roles/matrix-synapse/tasks/workers/setup_install.yml create mode 100644 roles/matrix-synapse/tasks/workers/setup_uninstall.yml create mode 100644 roles/matrix-synapse/templates/synapse/systemd/matrix-synapse-worker@.service.j2 create mode 100644 roles/matrix-synapse/templates/synapse/worker.yaml.j2 diff --git a/roles/matrix-synapse/defaults/main.yml b/roles/matrix-synapse/defaults/main.yml index f117fe93..dcd42ab8 100644 --- a/roles/matrix-synapse/defaults/main.yml +++ b/roles/matrix-synapse/defaults/main.yml @@ -258,6 +258,43 @@ matrix_synapse_metrics_port: 9100 # See https://github.com/matrix-org/synapse/blob/master/docs/manhole.md matrix_synapse_manhole_enabled: false +# Enable support for Synapse workers +matrix_synapse_workers_enabled: false + +# List of workers to spawn +matrix_synapse_workers_enabled_list: [] + +# Default list of workers to spawn +matrix_synapse_workers_enabled_list: + - { worker: synchrotron, port: 8082 } + - { worker: synchrotron, port: 8083 } + - { worker: synchrotron, port: 8084 } + - { worker: appservice, port: 8085 } + - { worker: client_reader, port: 8086 } + - { worker: event_creator, port: 8087 } + - { worker: federation_reader, port: 8088 } + - { worker: federation_sender, port: 8089 } + - { worker: frontend_proxy, port: 8090 } + - { worker: media_repository, port: 8091 } + - { worker: pusher, port: 8092 } + - { worker: user_dir, port: 8093 } + +# The list of available workers (2020-04-14) +matrix_synapse_workers_avail_list: + - appservice + - client_reader + - event_creator + - federation_reader + - federation_sender + - frontend_proxy + - media_repository + - pusher + - synchrotron + - user_dir + +# Ports used for communication between main synapse process and workers +matrix_synapse_replication_tcp_port: 9092 +matrix_synapse_replication_http_port: 9093 # Send ERROR logs to sentry.io for easier tracking # To set this up: go to sentry.io, create a python project, and set diff --git a/roles/matrix-synapse/tasks/setup_synapse.yml b/roles/matrix-synapse/tasks/setup_synapse.yml index c40ae170..ad1b24e6 100644 --- a/roles/matrix-synapse/tasks/setup_synapse.yml +++ b/roles/matrix-synapse/tasks/setup_synapse.yml @@ -19,6 +19,8 @@ - import_tasks: "{{ role_path }}/tasks/ext/setup.yml" +- import_tasks: "{{ role_path }}/tasks/workers/setup.yml" + - import_tasks: "{{ role_path }}/tasks/synapse/setup.yml" - import_tasks: "{{ role_path }}/tasks/goofys/setup.yml" diff --git a/roles/matrix-synapse/tasks/workers/setup.yml b/roles/matrix-synapse/tasks/workers/setup.yml new file mode 100644 index 00000000..faf2899e --- /dev/null +++ b/roles/matrix-synapse/tasks/workers/setup.yml @@ -0,0 +1,7 @@ +--- + +- import_tasks: "{{ role_path }}/tasks/workers/setup_install.yml" + when: "matrix_synapse_enabled|bool and matrix_synapse_workers_enabled|bool" + +- import_tasks: "{{ role_path }}/tasks/workers/setup_uninstall.yml" + when: "not matrix_synapse_workers_enabled|bool" diff --git a/roles/matrix-synapse/tasks/workers/setup_install.yml b/roles/matrix-synapse/tasks/workers/setup_install.yml new file mode 100644 index 00000000..1f23d0c5 --- /dev/null +++ b/roles/matrix-synapse/tasks/workers/setup_install.yml @@ -0,0 +1,41 @@ +--- + +- name: Ensure synapse worker base service file installed + template: + src: "{{ role_path }}/templates/synapse/systemd/matrix-synapse-worker@.service.j2" + dest: "{{ matrix_systemd_path }}/matrix-synapse-worker@.service" + mode: 0644 + register: matrix_synapse_worker_systemd_service_result + +- name: Ensure previous worker service symlinks are cleaned (FIXME) + file: + path: "{{ item.root + '/' + item.path }}" + state: absent + when: + - item.state == 'link' + - item.path is match('matrix-synapse-worker@*.service') + with_filetree: + - "{{ matrix_systemd_path }}/matrix-synapse.service.wants" + +- name: Ensure systemd reloaded the worker service unit + service: + daemon_reload: yes + +- name: Ensure individual worker service symlinks exist + service: + name: "matrix-synapse-worker@{{ item.worker }}:{{ item.port }}.service" + enabled: true + with_items: "{{ matrix_synapse_workers_enabled_list }}" + +- name: Ensure creation of specific worker configs + template: + src: "{{ role_path }}/templates/synapse/worker.yaml.j2" + dest: "{{ matrix_synapse_config_dir_path }}/worker.{{ item.worker }}:{{ item.port }}.yaml" + with_list: "{{ matrix_synapse_workers_enabled_list }}" + +- name: Add workers to synapse.wants list + set_fact: + matrix_synapse_systemd_wanted_services_list: > + {{ matrix_synapse_systemd_wanted_services_list + + ['matrix-synapse-worker@' + item.worker + ':' + item.port|string + '.service'] }} + with_items: "{{ matrix_synapse_workers_enabled_list }}" diff --git a/roles/matrix-synapse/tasks/workers/setup_uninstall.yml b/roles/matrix-synapse/tasks/workers/setup_uninstall.yml new file mode 100644 index 00000000..86430879 --- /dev/null +++ b/roles/matrix-synapse/tasks/workers/setup_uninstall.yml @@ -0,0 +1,34 @@ +--- + +- name: Ensure individual worker services are stopped + service: + name: "matrix-synapse-worker@{{ item.worker }}:{{ item.port }}.service" + state: stopped + with_items: "{{ matrix_synapse_workers_enabled_list }}" + +# As we cannot know the ports of workers removed from the enabled_list.. +# => .. just kill them all (FIXME?) +- name: Ensure previous worker service symlinks are cleaned + file: + path: "{{ item.root + '/' + item.path }}" + state: absent + when: + - item.state == 'link' + - item.path is match('matrix-synapse-worker@*.service') + with_filetree: + - "{{ matrix_systemd_path }}/matrix-synapse.service.wants" + +- name: Ensure synapse worker base service file gets removed + file: + path: "{{ matrix_systemd_path }}/matrix-synapse-worker@.service" + state: absent + register: matrix_synapse_worker_systemd_service_result + +- name: Remove workers from synapse.wants list + set_fact: + matrix_synapse_systemd_wanted_services_list: "{{ matrix_synapse_systemd_wanted_services_list | reject('search', item) | list }}" + with_items: "{{ matrix_synapse_workers_avail_list }}" + +- name: Ensure systemd noticed removal of worker service units + service: + daemon_reload: yes diff --git a/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 b/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 index d41313e3..fde097f0 100644 --- a/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 +++ b/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 @@ -251,6 +251,44 @@ listeners: type: manhole {% endif %} +{% if matrix_synapse_workers_enabled %} + # c.f. https://github.com/matrix-org/synapse/tree/master/docs/workers.md + # TCP replication: streaming data from the master to the workers + - port: {{ matrix_synapse_replication_tcp_port }} + bind_addresses: ['0.0.0.0'] + type: replication + + # HTTP replication: for the workers to send data to the main synapse process + - port: {{ matrix_synapse_replication_http_port }} + bind_addresses: ['0.0.0.0'] + type: http + resources: + - names: [replication] + +# c.f. https://github.com/matrix-org/synapse/tree/master/contrib/systemd-with-workers/README.md +worker_app: synapse.app.homeserver + +# thx https://oznetnerd.com/2017/04/18/jinja2-selectattr-filter/ +# reduce the main worker's offerings to core homeserver business +{% if matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'appservice')|list %} +notify_appservices: false +{% endif %} +{% if matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'federation_sender')|list %} +send_federation: false +{% endif %} +{% if matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'media_repository')|list %} +enable_media_repo: false +{% endif %} +{% if matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'pusher')|list %} +start_pushers: false +{% endif %} +{% if matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'user_dir')|list %} +update_user_directory: false +{% endif %} + +# rather let systemd handle the forking +daemonize: false +{% endif %} ## Homeserver blocking ## diff --git a/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse-worker@.service.j2 b/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse-worker@.service.j2 new file mode 100644 index 00000000..a46517c4 --- /dev/null +++ b/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse-worker@.service.j2 @@ -0,0 +1,29 @@ +#jinja2: lstrip_blocks: "True" +# c.f. https://github.com/matrix-org/synapse/pull/4662 +[Unit] +Description=Synapse Matrix Worker +After=matrix-synapse.service +BindsTo=matrix-synapse.service + +[Service] +Type=simple + +# Intentional delay, so that the homeserver (we likely depend on) can manage to start. +ExecStartPre=/bin/sleep 5 + +# systemd ftw 🤦‍♂️ +# https://github.com/systemd/systemd/issues/14895#issuecomment-594123923 +ExecStart=/bin/sh -c "WORKER=%i; WORKER=$${WORKER%%:*}; \ + exec /usr/bin/docker exec \ + --user={{ matrix_user_uid }}:{{ matrix_user_gid }} \ + matrix-synapse \ + python -m synapse.app.$${WORKER} -c /data/homeserver.yaml -c /data/worker.%i.yaml" + +ExecStop=/usr/bin/docker exec matrix-synapse pkill -f %i +KillMode=process +Restart=always +RestartSec=10 +SyslogIdentifier=matrix-synapse-%i + +[Install] +WantedBy=matrix-synapse.service diff --git a/roles/matrix-synapse/templates/synapse/worker.yaml.j2 b/roles/matrix-synapse/templates/synapse/worker.yaml.j2 new file mode 100644 index 00000000..37a5f87b --- /dev/null +++ b/roles/matrix-synapse/templates/synapse/worker.yaml.j2 @@ -0,0 +1,29 @@ +#jinja2: lstrip_blocks: "True" +worker_app: synapse.app.{{ item.worker }} + +worker_replication_host: 127.0.0.1 +worker_replication_port: {{ matrix_synapse_replication_tcp_port }} +worker_replication_http_port: {{ matrix_synapse_replication_http_port }} + +{% if item.worker not in [ 'appservice', 'federation_sender', 'pusher' ] %} +worker_listeners: + - type: http + port: {{ item.port }} + resources: + - names: +{% if item.worker in [ 'synchrotron', 'client_reader', 'event_creator', 'frontend_proxy', 'user_dir' ] %} + - client +{% elif item.worker in [ 'federation_reader' ] %} + - federation +{% elif item.worker in [ 'media_repository' ] %} + - media +{% endif %} +{% endif %} + +{% if item.worker == 'frontend_proxy' %} +worker_main_http_uri: http://127.0.0.1:8008 +{% endif %} + +worker_daemonize: false +worker_pid_file: /matrix-run/{{ item.worker }}.port{{ item.port }}.pid +worker_log_config: /data/{{ matrix_server_fqn_matrix }}.log.config