synapse workers: supply systemd with actual worker PIDs (requires jq)

also, worker.yaml.j2:
  - hone worker_name
  - remove worker_pid_file entry (would only be used if worker_daemonize
    set to true; also, synapse only knows about the container namespace
    and thus can not provide the required host-view PID)
This commit is contained in:
Marcel Partap 2020-10-22 20:53:41 +02:00
parent d2e61af224
commit 501efee07e
5 changed files with 50 additions and 6 deletions

View file

@ -0,0 +1,30 @@
#!/bin/bash
# Find a synapse worker's PID and write it to a file so systemd can manage it as a service
# example invocation:
# matrix-synapse-worker-write-pid user_dir:18700 /run/matrix-synapse-worker.user_dir:18700.pid
docker_api_call() { curl --silent --unix-socket /var/run/docker.sock ${@}; }
TARGETCONTAINER=matrix-synapse
TARGETWORKER=${1}
PIDFILE=${2}
# get ID list of subprocesses executed in $TARGETCONTAINER, and for each..
for EXECID in $(docker_api_call http://localhost/containers/${TARGETCONTAINER}/json | jq --raw-output '.ExecIDs[]')
do
# fetch detailed process info
EXECINFO=$(docker_api_call http://localhost/exec/${EXECID}/json)
# extract config file path from last command argument
WORKERCONFIGFILE=$(echo ${EXECINFO} | jq --raw-output .ProcessConfig.arguments[-1])
# reconstruct worker name
WORKERNAME=${WORKERCONFIGFILE#*/worker.}
WORKERNAME=${WORKERNAME%.yaml}
# if name matches the target worker: write out most recent PID & quit
[ "${WORKERNAME}" = "${TARGETWORKER}" ] \
&& echo ${EXECINFO} | jq --raw-output .Pid > ${PIDFILE} \
&& exit 0
done

View file

@ -40,3 +40,9 @@
{{ matrix_synapse_systemd_wanted_services_list + {{ matrix_synapse_systemd_wanted_services_list +
['matrix-synapse-worker@' + item.worker + ':' + item.port|string + '.service'] }} ['matrix-synapse-worker@' + item.worker + ':' + item.port|string + '.service'] }}
with_items: "{{ matrix_synapse_workers_enabled_list }}" with_items: "{{ matrix_synapse_workers_enabled_list }}"
- name: Ensure matrix-synapse-worker-write-pid script is created
copy:
src: "{{ role_path }}/files/usr-local-bin/matrix-synapse-worker-write-pid"
dest: "{{ matrix_local_bin_path }}/matrix-synapse-worker-write-pid"
mode: 0750

View file

@ -36,3 +36,8 @@
- name: Ensure systemd noticed removal of worker service units - name: Ensure systemd noticed removal of worker service units
service: service:
daemon_reload: yes daemon_reload: yes
- name: Ensure matrix-synapse-worker-write-pid script is removed
file:
path: "{{ matrix_local_bin_path }}/matrix-synapse-worker-write-pid"
state: absent

View file

@ -3,8 +3,8 @@
# alongside the homeserver main process. # alongside the homeserver main process.
# c.f. https://github.com/matrix-org/synapse/pull/4662 # c.f. https://github.com/matrix-org/synapse/pull/4662
[Unit] [Unit]
Description=Synapse Matrix Worker Description=Matrix worker synapse.app.%i
AssertPathExists={{matrix_synapse_config_dir_path }}/worker.%i.yaml AssertPathExists={{ matrix_synapse_config_dir_path }}/worker.%i.yaml
After=matrix-synapse.service After=matrix-synapse.service
BindsTo=matrix-synapse.service BindsTo=matrix-synapse.service
@ -23,9 +23,13 @@ ExecStart=/bin/sh -c "WORKER=%i; WORKER=$${WORKER%%:*}; \
matrix-synapse \ matrix-synapse \
python -m synapse.app.$${WORKER} -c /data/homeserver.yaml -c /data/worker.%i.yaml" python -m synapse.app.$${WORKER} -c /data/homeserver.yaml -c /data/worker.%i.yaml"
# wait for worker startup & write out PID of actual worker process so systemd can handle it
ExecStartPost=/bin/sleep 5
ExecStartPost=/usr/local/bin/matrix-synapse-worker-write-pid %i /run/matrix-synapse-worker.%i.pid
ExecReload=/bin/kill -HUP $MAINPID ExecReload=/bin/kill -HUP $MAINPID
ExecStop=/usr/bin/docker exec matrix-synapse pkill -f %i ExecStop=/bin/kill $MAINPID
PIDFile=/matrix-run/{{ item.worker }}.port{{ item.port }}.pid PIDFile=/run/matrix-synapse-worker.%i.pid
KillMode=process KillMode=process
Restart=always Restart=always
RestartSec=10 RestartSec=10

View file

@ -1,6 +1,6 @@
#jinja2: lstrip_blocks: "True" #jinja2: lstrip_blocks: "True"
worker_app: synapse.app.{{ item.worker }} worker_app: synapse.app.{{ item.worker }}
worker_name: {{ item.worker ~ '_' ~ item.port }} worker_name: {{ item.worker ~ ':' ~ item.port }}
worker_replication_host: 127.0.0.1 worker_replication_host: 127.0.0.1
worker_replication_http_port: {{ matrix_synapse_replication_http_port }} worker_replication_http_port: {{ matrix_synapse_replication_http_port }}
@ -26,5 +26,4 @@ worker_main_http_uri: http://127.0.0.1:8008
{% endif %} {% endif %}
worker_daemonize: false worker_daemonize: false
worker_pid_file: /matrix-run/{{ item.worker }}.port{{ item.port }}.pid
worker_log_config: /data/{{ matrix_server_fqn_matrix }}.log.config worker_log_config: /data/{{ matrix_server_fqn_matrix }}.log.config