From 06bc430c7c59cff0235b98cce37e49ceb6eeaad7 Mon Sep 17 00:00:00 2001 From: Max Klenk Date: Fri, 28 Aug 2020 13:53:39 +0200 Subject: [PATCH] refactor to use new workers and routes they serve --- roles/matrix-nginx-proxy/defaults/main.yml | 78 ++++++ .../nginx/conf.d/matrix-synapse.conf.j2 | 240 ++++++++---------- roles/matrix-synapse/defaults/main.yml | 39 ++- .../templates/synapse/worker.yaml.j2 | 5 +- 4 files changed, 206 insertions(+), 156 deletions(-) diff --git a/roles/matrix-nginx-proxy/defaults/main.yml b/roles/matrix-nginx-proxy/defaults/main.yml index 8ba0c532..d6a3d3a7 100644 --- a/roles/matrix-nginx-proxy/defaults/main.yml +++ b/roles/matrix-nginx-proxy/defaults/main.yml @@ -247,3 +247,81 @@ matrix_ssl_log_dir_path: "{{ matrix_ssl_base_path }}/log" # nginx status page configurations. matrix_nginx_proxy_proxy_matrix_nginx_status_enabled: false matrix_nginx_proxy_proxy_matrix_nginx_status_allowed_addresses: ['{{ ansible_default_ipv4.address }}'] + + +# worker +matrix_nginx_proxy_synapse_workers_enabled: "{{ matrix_synapse_workers_enabled }}" +matrix_nginx_proxy_synapse_workers_enabled_list: "{{ matrix_synapse_workers_enabled_list }}" +matrix_nginx_proxy_synapse_generic_worker_locations: [ + # Sync requests + '^/_matrix/client/(v2_alpha|r0)/sync$', + '^/_matrix/client/(api/v1|v2_alpha|r0)/events$', + '^/_matrix/client/(api/v1|r0)/initialSync$', + '^/_matrix/client/(api/v1|r0)/rooms/[^/]+/initialSync$', + + # Federation requests + '^/_matrix/federation/v1/event/', + '^/_matrix/federation/v1/state/', + '^/_matrix/federation/v1/state_ids/', + '^/_matrix/federation/v1/backfill/', + '^/_matrix/federation/v1/get_missing_events/', + '^/_matrix/federation/v1/publicRooms', + '^/_matrix/federation/v1/query/', + '^/_matrix/federation/v1/make_join/', + '^/_matrix/federation/v1/make_leave/', + '^/_matrix/federation/v1/send_join/', + '^/_matrix/federation/v2/send_join/', + '^/_matrix/federation/v1/send_leave/', + '^/_matrix/federation/v2/send_leave/', + '^/_matrix/federation/v1/invite/', + '^/_matrix/federation/v2/invite/', + '^/_matrix/federation/v1/query_auth/', + '^/_matrix/federation/v1/event_auth/', + '^/_matrix/federation/v1/exchange_third_party_invite/', + '^/_matrix/federation/v1/user/devices/', + '^/_matrix/federation/v1/get_groups_publicised$', + '^/_matrix/key/v2/query', + + # Inbound federation transaction request + '^/_matrix/federation/v1/send/', + + # Client API requests + '^/_matrix/client/(api/v1|r0|unstable)/publicRooms$', + '^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/joined_members$', + '^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/context/.*$', + '^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/members$', + '^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/state$', + '^/_matrix/client/(api/v1|r0|unstable)/account/3pid$', + '^/_matrix/client/(api/v1|r0|unstable)/keys/query$', + '^/_matrix/client/(api/v1|r0|unstable)/keys/changes$', + '^/_matrix/client/versions$', + '^/_matrix/client/(api/v1|r0|unstable)/voip/turnServer$', + '^/_matrix/client/(api/v1|r0|unstable)/joined_groups$', + '^/_matrix/client/(api/v1|r0|unstable)/publicised_groups$', + '^/_matrix/client/(api/v1|r0|unstable)/publicised_groups/', + + # Registration/login requests + '^/_matrix/client/(api/v1|r0|unstable)/login$', + '^/_matrix/client/(r0|unstable)/register$', + '^/_matrix/client/(r0|unstable)/auth/.*/fallback/web$', + + # Event sending requests + '^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/send', + '^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/state/', + '^/_matrix/client/(api/v1|r0|unstable)/rooms/.*/(join|invite|leave|ban|unban|kick)$', + '^/_matrix/client/(api/v1|r0|unstable)/join/', + '^/_matrix/client/(api/v1|r0|unstable)/profile/', +] + +matrix_nginx_proxy_synapse_media_repository_locations: [ + '^/_matrix/media/*$', + '^/_synapse/admin/v1/purge_media_cache$', + '^/_synapse/admin/v1/room/.*/media.*$', + '^/_synapse/admin/v1/user/.*/media.*$', + '^/_synapse/admin/v1/media/.*$', + '^/_synapse/admin/v1/quarantine_media/.*$', +] + +matrix_nginx_proxy_synapse_user_dir_locations: [ + 'matrix_nginx_proxy_synapse_media_workers_endpoints', +] diff --git a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 index bb67ff56..a49bd8b6 100644 --- a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 +++ b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 @@ -101,115 +101,60 @@ } {% endif %} - {% if matrix_synapse_workers_enabled %} - {% if synchrotron_workers %} - {# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L134 #} - location /_matrix/client/r0/sync { - proxy_pass http://synchrotron$request_uri; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $remote_addr; - } - location /_matrix/client/r0/events { - proxy_pass http://synchrotron$request_uri; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $remote_addr; - } - location /_matrix/client/r0/initialSync { - proxy_pass http://synchrotron$request_uri; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $remote_addr; - } - location ~ ^/_matrix/client/r0/rooms/[^/]+/initialSync$ { - proxy_pass http://synchrotron$request_uri; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $remote_addr; - } + {% if matrix_nginx_proxy_synapse_workers_enabled %} + {# Synapse Workers #} + + {% if generic_worker_workers %} + {# https://github.com/matrix-org/synapse/blob/master/docs/workers.md#synapseappgeneric_worker #} + {% for location in matrix_nginx_proxy_synapse_generic_worker_locations %} + location ~ {{ location }} { + proxy_pass http://generic_worker_upstream$request_uri; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $remote_addr; + } + {% endfor %} + {# ToDo: add GET ^/_matrix/federation/v1/groups/ #} + {% endif %} + + {% if media_repository_workers %} + {# https://github.com/matrix-org/synapse/blob/master/docs/workers.md#synapseappmedia_repository #} + {% for location in matrix_nginx_proxy_synapse_media_repository_locations %} + location ~ {{ location }} { + proxy_pass http://media_repository_upstream$request_uri; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $remote_addr; + } + {% endfor %} + {% endif %} + + {% if user_dir_workers %} + {# https://github.com/matrix-org/synapse/blob/master/docs/workers.md#synapseappuser_dir #} + {% for location in matrix_nginx_proxy_synapse_user_dir_locations %} + location ~ {{ location }} { + proxy_pass http://user_dir_upstream$request_uri; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $remote_addr; + } + {% endfor %} + {% endif %} + + {% if frontend_proxy_workers %} + {# https://github.com/matrix-org/synapse/blob/master/docs/workers.md#synapseappfrontend_proxy #} + location ~ ^/_matrix/client/(api/v1|r0|unstable)/keys/upload { + proxy_pass http://frontend_proxy_upstream$request_uri; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $remote_addr; + } + {% if not matrix_synapse_use_presence %} + location ~ ^/_matrix/client/(api/v1|r0|unstable)/presence/[^/]+/status { + proxy_pass http://frontend_proxy_upstream$request_uri; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $remote_addr; + } + {% endif %} + {% endif %} {% endif %} - {% set client_reader_worker = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'client_reader')|first %} - {% if client_reader_worker %} - {# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L252 #} - location ^/_matrix/client/(versions$|(api/v1|r0|unstable)/(publicRooms$|rooms/.*/joined_me|rooms/.*/context/.|rooms/.*/members$|rooms/.*/messages$|rooms/.*/state$|login$|account/3pid$|keys/query$|keys/changes$|voip/turnServer$|joined_groups$|publicised_groups$|publicised_groups/|pushrules/.*$|groups/.*$|register$|auth/.*/fallback/web$)) { - {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; - set $backend "matrix-synapse:{{ client_reader_worker.port }}"; - proxy_pass http://$backend$request_uri; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $remote_addr; - } - {% endif %} - - {% set media_repository_worker = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'media_repository')|first %} - {% if media_repository_worker %} - {# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L219 #} - location /_matrix/media/ { - {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; - set $backend "matrix-synapse:{{ media_repository_worker.port }}"; - proxy_pass http://$backend$request_uri; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $remote_addr; - } - {# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L223 #} - location ~ ^/_synapse/admin/v1/(purge_media_cache|room/.*/media.*|user/.*/media.*|media/.*|quarantine_media/.*)$ { - {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; - set $backend "matrix-synapse:{{ media_repository_worker.port }}"; - proxy_pass http://$backend$request_uri; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $remote_addr; - } - {% endif %} - - {% set event_creator_worker = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'event_creator')|first %} - {% if event_creator_worker %} - {# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L323 #} - location ~ ^/_matrix/client/(api/v1|r0|unstable)/(rooms/.*/send|rooms/.*/state/|rooms/.*/(join|invite|leave|ban|unban|kick)$|join/|profile/) { - {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; - set $backend "matrix-synapse:{{ event_creator_worker.port }}"; - proxy_pass http://$backend$request_uri; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $remote_addr; - } - {% endif %} - - {% set frontend_proxy_worker = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'frontend_proxy')|first %} - {% if frontend_proxy_worker %} - {# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L302 #} - location ~ ^/_matrix/client/(api/v1|r0|unstable)/keys/upload { - {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; - set $backend "matrix-synapse:{{ frontend_proxy_worker.port }}"; - proxy_pass http://$backend$request_uri; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $remote_addr; - } - {% if not matrix_synapse_use_presence %} - location ~ ^/_matrix/client/(api/v1|r0|unstable)/presence/[^/]+/status { - {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; - set $backend "matrix-synapse:{{ frontend_proxy_worker.port }}"; - proxy_pass http://$backend$request_uri; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $remote_addr; - } - {% endif %} - {% endif %} - - {% set user_dir_worker = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'user_dir')|first %} - {% if user_dir_worker %} - {# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L290 #} - location ~ ^/_matrix/client/(api/v1|r0|unstable)/user_directory/search$ { - {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; - set $backend "matrix-synapse:{{ user_dir_worker.port }}"; - proxy_pass http://$backend$request_uri; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $remote_addr; - } - {% endif %} - {% endif %} {% for configuration_block in matrix_nginx_proxy_proxy_matrix_additional_server_configuration_blocks %} {{- configuration_block }} @@ -284,19 +229,51 @@ } {% endmacro %} -{% if matrix_synapse_workers_enabled %} -{% set synchrotron_workers = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'synchrotron')|list %} -{% if synchrotron_workers %} -upstream synchrotron { - # ensures that requests from the same client will always be passed - # to the same server (except when this server is unavailable) - ip_hash; +{% set generic_worker_workers = matrix_nginx_proxy_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'generic_worker')|list %} +{% set media_repository_workers = matrix_nginx_proxy_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'media_repository')|list %} +{% set user_dir_workers = matrix_nginx_proxy_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'user_dir')|list %} +{% set frontend_proxy_workers = matrix_nginx_proxy_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'frontend_proxy')|list %} +{% if matrix_nginx_proxy_synapse_workers_enabled %} + {# Setup upstream for groups of workers #} - {% for synchrotron_worker in synchrotron_workers %} - server "matrix-synapse:{{ synchrotron_worker.port }}"; - {% endfor %} -} -{% endif %} + {% if generic_worker_workers %} + upstream generic_worker_upstream { + # ensures that requests from the same client will always be passed + # to the same server (except when this server is unavailable) + ip_hash; + + {% for worker in generic_worker_workers %} + server "matrix-synapse:{{ worker.port }}"; + {% endfor %} + } + {% endif %} + + {% if frontend_proxy_workers %} + upstream frontend_proxy_upstream { + # Round Robin + {% for worker in frontend_proxy_workers %} + server "matrix-synapse:{{ worker.port }}"; + {% endfor %} + } + {% endif %} + + {% if media_repository_workers %} + upstream media_repository_upstream { + # Round Robin + {% for worker in media_repository_workers %} + server "matrix-synapse:{{ worker.port }}"; + {% endfor %} + } + {% endif %} + + {% if user_dir_workers %} + upstream user_dir_upstream { + # Round Robin + {% for worker in user_dir_workers %} + server "matrix-synapse:{{ worker.port }}"; + {% endfor %} + } + {% endif %} {% endif %} server { @@ -380,19 +357,18 @@ server { ssl_ciphers "EECDH+AESGCM:EDH+AESGCM:AES256+EECDH:AES256+EDH"; {% endif %} - {% if matrix_synapse_workers_enabled } - {% set federation_reader_worker = matrix_synapse_workers_enabled_list|selectattr('worker', 'equalto', 'federation_reader')|first %} - {% if federation_reader_worker %} - {# c.f. https://github.com/matrix-org/synapse/blame/master/docs/workers.md#L160 #} - location ~ ^(/_matrix/federation/v1/event/|/_matrix/federation/v1/state/|/_matrix/federation/v1/state_ids/|/_matrix/federation/v1/backfill/|/_matrix/federation/v1/get_missing_events/|/_matrix/federation/v1/publicRooms|/_matrix/federation/v1/query/|/_matrix/federation/v1/make_join/|/_matrix/federation/v1/make_leave/|/_matrix/federation/v1/send_join/|/_matrix/federation/v2/send_join/|/_matrix/federation/v1/send_leave/|/_matrix/federation/v2/send_leave/|/_matrix/federation/v1/invite/|/_matrix/federation/v2/invite/|/_matrix/federation/v1/query_auth/|/_matrix/federation/v1/event_auth/|/_matrix/federation/v1/exchange_third_party_invite/|/_matrix/federation/v1/user/devices/|/_matrix/federation/v1/send/|/_matrix/federation/v1/get_groups_publicised$|/_matrix/key/v2/query|/_matrix/federation/v1/groups/) { - {# Use the embedded DNS resolver in Docker containers to discover the service #} - resolver 127.0.0.11 valid=5s; - set $backend "matrix-synapse:{{ federation_reader_worker.port }}"; - proxy_pass http://$backend$request_uri; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-For $remote_addr; - } - {% endif %} + {% if matrix_nginx_proxy_synapse_workers_enabled %} + {% if generic_worker_workers %} + {# https://github.com/matrix-org/synapse/blob/master/docs/workers.md#synapseappgeneric_worker #} + {% for location in matrix_nginx_proxy_synapse_generic_worker_locations %} + location ~ {{ location }} { + proxy_pass http://generic_worker_upstream$request_uri; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $remote_addr; + } + {% endfor %} + {# ToDo: add GET ^/_matrix/federation/v1/groups/ #} + {% endif %} {% endif %} location / { diff --git a/roles/matrix-synapse/defaults/main.yml b/roles/matrix-synapse/defaults/main.yml index efddec1b..aad9fad3 100644 --- a/roles/matrix-synapse/defaults/main.yml +++ b/roles/matrix-synapse/defaults/main.yml @@ -265,31 +265,28 @@ matrix_synapse_workers_enabled: false # List of workers to spawn matrix_synapse_workers_enabled_list: - - { worker: synchrotron, port: 18082 } - - { worker: synchrotron, port: 18083 } - - { worker: synchrotron, port: 18084 } - - { worker: appservice, port: 18085 } - - { worker: client_reader, port: 18086 } - - { worker: event_creator, port: 18087 } - - { worker: federation_reader, port: 18088 } - - { worker: federation_sender, port: 18089 } - - { worker: frontend_proxy, port: 18090 } - - { worker: media_repository, port: 18091 } - - { worker: pusher, port: 18092 } - - { worker: user_dir, port: 18093 } + - { worker: generic_worker, port: 18101 } + - { worker: generic_worker, port: 18102 } + - { worker: generic_worker, port: 18103 } + - { worker: generic_worker, port: 18104 } + - { worker: generic_worker, port: 18105 } + - { worker: generic_worker, port: 18106 } + - { worker: pusher, port: 18201 } + - { worker: appservice, port: 18301 } + - { worker: federation_sender, port: 18401 } + - { worker: media_repository, port: 18501 } + - { worker: user_dir, port: 18502 } + - { worker: frontend_proxy, port: 18503 } -# The list of available workers (2020-04-14) +# The list of available workers (2020-08-28) matrix_synapse_workers_avail_list: - - appservice - - client_reader - - event_creator - - federation_reader - - federation_sender - - frontend_proxy - - media_repository + - generic_worker - pusher - - synchrotron + - appservice + - federation_sender + - media_repository - user_dir + - frontend_proxy # Ports used for communication between main synapse process and workers matrix_synapse_replication_tcp_port: 9092 diff --git a/roles/matrix-synapse/templates/synapse/worker.yaml.j2 b/roles/matrix-synapse/templates/synapse/worker.yaml.j2 index 37a5f87b..d5f78fdb 100644 --- a/roles/matrix-synapse/templates/synapse/worker.yaml.j2 +++ b/roles/matrix-synapse/templates/synapse/worker.yaml.j2 @@ -11,9 +11,8 @@ worker_listeners: port: {{ item.port }} resources: - names: -{% if item.worker in [ 'synchrotron', 'client_reader', 'event_creator', 'frontend_proxy', 'user_dir' ] %} +{% if item.worker in [ 'generic_worker', 'frontend_proxy', 'user_dir' ] %} - client -{% elif item.worker in [ 'federation_reader' ] %} - federation {% elif item.worker in [ 'media_repository' ] %} - media @@ -23,7 +22,7 @@ worker_listeners: {% if item.worker == 'frontend_proxy' %} worker_main_http_uri: http://127.0.0.1:8008 {% endif %} - + worker_daemonize: false worker_pid_file: /matrix-run/{{ item.worker }}.port{{ item.port }}.pid worker_log_config: /data/{{ matrix_server_fqn_matrix }}.log.config