Allow Synapse worker list to be generated dynamically

This leads to much easier management and potential safety
features (validation). In the future, we could try to avoid port
conflicts as well, but it didn't seem worth the effort to do it now.
Our port ranges seem large enough.

This can also pave the way for a "presets" feature
(similar to `matrix_nginx_proxy_ssl_presets`) which makes it even easier
for people to configure worker counts.

roles/matrix-synapse/defaults/main.yml

@@ -301,7 +301,6 @@ matrix_synapse_manhole_enabled: false
 # Enable support for Synapse workers
 matrix_synapse_workers_enabled: false
 
-
 # Controls whether the matrix-synapse container exposes the various worker ports
 # (see `port` and `metrics_port` in `matrix_synapse_workers_enabled_list`) outside of the container.
 #
@@ -309,23 +308,59 @@ matrix_synapse_workers_enabled: false
 # It takes "*" to signify "bind on all interfaces" ("0.0.0.0" is IPv4-only).
 matrix_synapse_workers_container_host_bind_address: ''
 
-# Default list of workers to spawn (order in accord to docs)
+matrix_synapse_workers_generic_workers_count: 3
-# - no endpoints / doesn't need port mapping if port ends on 0
+matrix_synapse_workers_generic_workers_port_range_start: 18111
-# - single-instance-only if 2nd last digit of port number is 0
+matrix_synapse_workers_generic_workers_metrics_range_start: 19111
-matrix_synapse_workers_enabled_list:
+
-  - { type: generic_worker, port: 18111, metrics_port: 19111 }
+# matrix_synapse_workers_pusher_workers_count can only be 0 or 1. More instances are not supported.
-  - { type: generic_worker, port: 18112, metrics_port: 19112 }
+matrix_synapse_workers_pusher_workers_count: 1
-  - { type: generic_worker, port: 18113, metrics_port: 19113 }
+matrix_synapse_workers_pusher_workers_metrics_range_start: 19200
-  - { type: generic_worker, port: 18114, metrics_port: 19114 }
+
-  - { type: generic_worker, port: 18115, metrics_port: 19115 }
+# matrix_synapse_workers_appservice_workers_count can only be 0 or 1. More instances are not supported.
-  - { type: generic_worker, port: 18116, metrics_port: 19116 }
+matrix_synapse_workers_appservice_workers_count: 1
-  - { type: pusher, port: 00, metrics_port: 19200 }
+matrix_synapse_workers_appservice_workers_metrics_range_start: 19300
-  - { type: appservice, port: 00, metrics_port: 19300 }
+
-  - { type: federation_sender, port: 0, metrics_port: 19400 }
+matrix_synapse_workers_federation_sender_workers_count: 1
-  - { type: media_repository, port: 18551, metrics_port: 19551 }
+matrix_synapse_workers_federation_sender_workers_metrics_range_start: 19400
-# disable until https://github.com/matrix-org/synapse/issues/8787 resolved
+
-# - { type: user_dir, port: 18661, metrics_port: 19661 }
+matrix_synapse_workers_media_repository_workers_count: 1
-  - { type: frontend_proxy, port: 18771, metrics_port: 19771 }
+matrix_synapse_workers_media_repository_workers_port_range_start: 18551
+matrix_synapse_workers_media_repository_workers_metrics_range_start: 19551
+
+# Disabled until https://github.com/matrix-org/synapse/issues/8787 is resolved.
+matrix_synapse_workers_user_dir_workers_count: 0
+matrix_synapse_workers_user_dir_workers_port_range_start: 18661
+matrix_synapse_workers_user_dir_workers_metrics_range_start: 19661
+
+matrix_synapse_workers_frontend_proxy_workers_count: 1
+matrix_synapse_workers_frontend_proxy_workers_port_range_start: 18771
+matrix_synapse_workers_frontend_proxy_workers_metrics_range_start: 19771
+
+# Default list of workers to spawn.
+#
+# Unless you populate this manually, this list is dynamically generated
+# based on other variables above:
+# - `matrix_synapse_workers_*_workers_count`
+# - `matrix_synapse_workers_*_workers_port_range_start`
+# - `matrix_synapse_workers_*_workers_port_metrics_range_start`
+#
+# We advise that you use those variables and let this list be populated dynamically.
+# Doing that is simpler and also protects you from shooting yourself in the foot,
+# as certain workers can only be spawned just once.
+#
+# Example of what this needs to look like:
+# matrix_synapse_workers_enabled_list:
+#   - { type: generic_worker, port: 18111, metrics_port: 19111 }
+#   - { type: generic_worker, port: 18112, metrics_port: 19112 }
+#   - { type: generic_worker, port: 18113, metrics_port: 19113 }
+#   - { type: generic_worker, port: 18114, metrics_port: 19114 }
+#   - { type: generic_worker, port: 18115, metrics_port: 19115 }
+#   - { type: generic_worker, port: 18116, metrics_port: 19116 }
+#   - { type: pusher, port: 0, metrics_port: 19200 }
+#   - { type: appservice, port: 0, metrics_port: 19300 }
+#   - { type: federation_sender, port: 0, metrics_port: 19400 }
+#   - { type: media_repository, port: 18551, metrics_port: 19551 }
+matrix_synapse_workers_enabled_list: []
 
 # Redis information
 matrix_synapse_redis_enabled: false

roles/matrix-synapse/tasks/init.yml

@@ -1,3 +1,8 @@
+# Unless `matrix_synapse_workers_enabled_list` is explicitly defined,
+# we'll generate it dynamically.
+- import_tasks: "{{ role_path }}/tasks/synapse/workers/init.yml"
+  when: "matrix_synapse_enabled and matrix_synapse_workers_enabled and matrix_synapse_workers_enabled_list|length == 0"
+
 - set_fact:
     matrix_systemd_services_list: "{{ matrix_systemd_services_list + ['matrix-synapse.service'] }}"
   when: matrix_synapse_enabled|bool

roles/matrix-synapse/tasks/synapse/workers/init.yml

@@ -0,0 +1,80 @@
+# Below is a huge hack for dynamically building a list of workers and finally assigning it to `matrix_synapse_workers_enabled_list`.
+#
+# set_fact within a loop does not work reliably in Ansible (it only executes on the first iteration for some reason),
+# so we're forced to do something much uglier.
+
+- name: Build generic workers
+  set_fact:
+    worker:
+      type: 'generic_worker'
+      port: "{{ matrix_synapse_workers_generic_workers_port_range_start + item }}"
+      metrics_port: "{{ matrix_synapse_workers_generic_workers_metrics_range_start + item }}"
+  register: "matrix_synapse_workers_list_results_generic_workers"
+  loop: "{{ range(0, matrix_synapse_workers_generic_workers_count)|list }}"
+
+- name: Build federation sender workers
+  set_fact:
+    worker:
+      type: 'federation_sender'
+      port: 0
+      metrics_port: "{{ matrix_synapse_workers_federation_sender_workers_metrics_range_start + item }}"
+  register: "matrix_synapse_workers_list_results_federation_sender_workers"
+  loop: "{{ range(0, matrix_synapse_workers_federation_sender_workers_count)|list }}"
+
+# This type of worker can only have a count of 1, at most
+- name: Build pusher workers
+  set_fact:
+    worker:
+      type: 'pusher'
+      port: 0
+      metrics_port: "{{ matrix_synapse_workers_pusher_workers_metrics_range_start + item }}"
+  register: "matrix_synapse_workers_list_results_pusher_workers"
+  loop: "{{ range(0, matrix_synapse_workers_pusher_workers_count)|list }}"
+
+# This type of worker can only have a count of 1, at most
+- name: Build appservice workers
+  set_fact:
+    worker:
+      type: 'appservice'
+      port: 0
+      metrics_port: "{{ matrix_synapse_workers_appservice_workers_metrics_range_start + item }}"
+  register: "matrix_synapse_workers_list_results_appservice_workers"
+  loop: "{{ range(0, matrix_synapse_workers_appservice_workers_count)|list }}"
+
+- name: Build media_repository workers
+  set_fact:
+    worker:
+      type: 'media_repository'
+      port: "{{ matrix_synapse_workers_media_repository_workers_port_range_start + item }}"
+      metrics_port: "{{ matrix_synapse_workers_media_repository_workers_metrics_range_start + item }}"
+  register: "matrix_synapse_workers_list_results_media_repository_workers"
+  loop: "{{ range(0, matrix_synapse_workers_media_repository_workers_count)|list }}"
+
+- name: Build frontend_proxy workers
+  set_fact:
+    worker:
+      type: 'frontend_proxy'
+      port: "{{ matrix_synapse_workers_frontend_proxy_workers_port_range_start + item }}"
+      metrics_port: "{{ matrix_synapse_workers_frontend_proxy_workers_metrics_range_start + item }}"
+  register: "matrix_synapse_workers_list_results_frontend_proxy_workers"
+  loop: "{{ range(0, matrix_synapse_workers_frontend_proxy_workers_count)|list }}"
+
+- set_fact:
+    matrix_synapse_dynamic_workers_list: "{{ matrix_synapse_dynamic_workers_list|default([]) + [item.ansible_facts.worker] }}"
+  with_items: |
+    {{
+      matrix_synapse_workers_list_results_generic_workers.results
+      +
+      matrix_synapse_workers_list_results_federation_sender_workers.results
+      +
+      matrix_synapse_workers_list_results_pusher_workers.results
+      +
+      matrix_synapse_workers_list_results_appservice_workers.results
+      +
+      matrix_synapse_workers_list_results_media_repository_workers.results
+      +
+      matrix_synapse_workers_list_results_frontend_proxy_workers.results
+    }}
+
+- set_fact:
+    matrix_synapse_workers_enabled_list: "{{ matrix_synapse_dynamic_workers_list }}"

roles/matrix-synapse/tasks/validate_config.yml

@@ -12,6 +12,15 @@
     - "matrix_synapse_database_password"
     - "matrix_synapse_database_database"
 
+- name: Fail if asking for more than 1 instance of single-instance workers
+  fail:
+    msg: >-
+      `{{ item }}` cannot be more than 1. This is a single-instance worker.
+  when: "vars[item] > 1"
+  with_items:
+    - "matrix_synapse_workers_appservice_workers_count"
+    - "matrix_synapse_workers_pusher_workers_count"
+
 - name: (Deprecation) Catch and report renamed settings
   fail:
     msg: >-