Add service priorities - try to stop/start them in an optimal order

group_vars/matrix_servers

@@ -22,139 +22,159 @@ matrix_playbook_docker_installation_enabled: true
 # This list is not exhaustive and final.
 # Synapse workers are still injected into the list at runtime.
 # Additional JVB workers (playbooks/jitsi_jvb.yml -- roles/custom/matrix-jitsi/tasks/init_additional_jvb.yml) override this variable at runtime as well.
-matrix_systemd_services_list: |
+#
+# Priority levels are like this:
+# - core services (the homeserver) get a level of ~1000
+# - services that core services depend on (database, Redis, ntfy, etc.) get a lower level - between 500 and 1000
+# - reverse-proxying services get level 3000
+# - Matrix utility services (bridges, bots) get a level of 2000/2200, so that:
+#   - they can start before the reverse-proxy
+#   - so that, when the reverse-proxy is up (Matrix is up), all bots and bridges can be interacted with
+# - monitoring services (Prometheus, Grafana, ..) get a level of 4000 - they can start later than all-of-Matrix
+# - services which aren't time-sensitive (various crons and timers) get a level of 5000 - they can start later than all-of-Matrix
+matrix_systemd_services_list_auto: |
   {{
-    (['matrix-backup-borg.timer'] if matrix_backup_borg_enabled else [])
+    ([{'name': 'matrix-backup-borg.timer', 'priority': 5000}] if matrix_backup_borg_enabled else [])
     +
-    (['matrix-bot-buscarron.service'] if matrix_bot_buscarron_enabled else [])
+    ([{'name': 'matrix-bot-buscarron.service', 'priority': 2200}] if matrix_bot_buscarron_enabled else [])
     +
-    (['matrix-bot-go-neb.service'] if matrix_bot_go_neb_enabled else [])
+    ([{'name': 'matrix-bot-go-neb.service', 'priority': 2200}] if matrix_bot_go_neb_enabled else [])
     +
-    (['matrix-bot-honoroit.service'] if matrix_bot_honoroit_enabled else [])
+    ([{'name': 'matrix-bot-honoroit.service', 'priority': 2200}] if matrix_bot_honoroit_enabled else [])
     +
-    (['matrix-bot-matrix-registration-bot.service'] if matrix_bot_matrix_registration_bot_enabled else [])
+    ([{'name': 'matrix-bot-matrix-registration-bot.service', 'priority': 2200}] if matrix_bot_matrix_registration_bot_enabled else [])
     +
-    (['matrix-bot-matrix-reminder-bot.service'] if matrix_bot_matrix_reminder_bot_enabled else [])
+    ([{'name': 'matrix-bot-matrix-reminder-bot.service', 'priority': 2200}] if matrix_bot_matrix_reminder_bot_enabled else [])
     +
-    (['matrix-bot-maubot.service'] if matrix_bot_maubot_enabled else [])
+    ([{'name': 'matrix-bot-maubot.service', 'priority': 2200}] if matrix_bot_maubot_enabled else [])
     +
-    (['matrix-bot-mjolnir.service'] if matrix_bot_mjolnir_enabled else [])
+    ([{'name': 'matrix-bot-mjolnir.service', 'priority': 2200}] if matrix_bot_mjolnir_enabled else [])
     +
-    (['matrix-bot-postmoogle.service'] if matrix_bot_postmoogle_enabled else [])
+    ([{'name': 'matrix-bot-postmoogle.service', 'priority': 2200}] if matrix_bot_postmoogle_enabled else [])
     +
-    (['matrix-appservice-discord.service'] if matrix_appservice_discord_enabled else [])
+    ([{'name': 'matrix-appservice-discord.service', 'priority': 2000}] if matrix_appservice_discord_enabled else [])
     +
-    (['matrix-appservice-irc.service'] if matrix_appservice_irc_enabled else [])
+    ([{'name': 'matrix-appservice-irc.service', 'priority': 2000}] if matrix_appservice_irc_enabled else [])
     +
-    (['matrix-appservice-kakaotalk.service', 'matrix-appservice-kakaotalk-node.service'] if matrix_appservice_kakaotalk_enabled else [])
+    ([{'name': 'matrix-appservice-kakaotalk.service', 'priority': 2000}] if matrix_appservice_kakaotalk_enabled else [])
     +
-    (['matrix-appservice-slack.service'] if matrix_appservice_slack_enabled else [])
+    ([{'name': 'matrix-appservice-kakaotalk-node.service', 'priority': 1900}] if matrix_appservice_kakaotalk_enabled else [])
     +
-    (['matrix-appservice-webhooks.service'] if matrix_appservice_webhooks_enabled else [])
+    ([{'name': 'matrix-appservice-slack.service', 'priority': 2000}] if matrix_appservice_slack_enabled else [])
     +
-    (['matrix-beeper-linkedin.service'] if matrix_beeper_linkedin_enabled else [])
+    ([{'name': 'matrix-appservice-webhooks.service', 'priority': 2000}] if matrix_appservice_webhooks_enabled else [])
     +
-    (['matrix-go-skype-bridge.service'] if matrix_go_skype_bridge_enabled else [])
+    ([{'name': 'matrix-beeper-linkedin.service', 'priority': 2000}] if matrix_beeper_linkedin_enabled else [])
     +
-    (['matrix-heisenbridge.service'] if matrix_heisenbridge_enabled else [])
+    ([{'name': 'matrix-go-skype-bridge.service', 'priority': 2000}] if matrix_go_skype_bridge_enabled else [])
     +
-    (['matrix-hookshot.service'] if matrix_hookshot_enabled else [])
+    ([{'name': 'matrix-heisenbridge.service', 'priority': 2000}] if matrix_heisenbridge_enabled else [])
     +
-    (['matrix-mautrix-discord.service'] if matrix_mautrix_discord_enabled else [])
+    ([{'name': 'matrix-hookshot.service', 'priority': 2000}] if matrix_hookshot_enabled else [])
     +
-    (['matrix-mautrix-facebook.service'] if matrix_mautrix_facebook_enabled else [])
+    ([{'name': 'matrix-mautrix-discord.service', 'priority': 2000}] if matrix_mautrix_discord_enabled else [])
     +
-    (['matrix-mautrix-googlechat.service'] if matrix_mautrix_googlechat_enabled else [])
+    ([{'name': 'matrix-mautrix-facebook.service', 'priority': 2000}] if matrix_mautrix_facebook_enabled else [])
     +
-    (['matrix-mautrix-hangouts.service'] if matrix_mautrix_hangouts_enabled else [])
+    ([{'name': 'matrix-mautrix-googlechat.service', 'priority': 2000}] if matrix_mautrix_googlechat_enabled else [])
     +
-    (['matrix-mautrix-instagram.service'] if matrix_mautrix_instagram_enabled else [])
+    ([{'name': 'matrix-mautrix-hangouts.service', 'priority': 2000}] if matrix_mautrix_hangouts_enabled else [])
     +
-    (['matrix-mautrix-signal.service', 'matrix-mautrix-signal-daemon.service'] if matrix_mautrix_signal_enabled else [])
+    ([{'name': 'matrix-mautrix-instagram.service', 'priority': 2000}] if matrix_mautrix_instagram_enabled else [])
     +
-    (['matrix-mautrix-telegram.service'] if matrix_mautrix_telegram_enabled else [])
+    ([{'name': 'matrix-mautrix-signal.service', 'priority': 2000}] if matrix_mautrix_signal_enabled else [])
     +
-    (['matrix-mautrix-twitter.service'] if matrix_mautrix_twitter_enabled else [])
+    ([{'name': 'matrix-mautrix-signal-daemon.service', 'priority': 1900}] if matrix_mautrix_signal_enabled else [])
     +
-    (['matrix-mautrix-whatsapp.service'] if matrix_mautrix_whatsapp_enabled else [])
+    ([{'name': 'matrix-mautrix-telegram.service', 'priority': 2000}] if matrix_mautrix_telegram_enabled else [])
     +
-    (['matrix-mx-puppet-discord.service'] if matrix_mx_puppet_discord_enabled else [])
+    ([{'name': 'matrix-mautrix-twitter.service', 'priority': 2000}] if matrix_mautrix_twitter_enabled else [])
     +
-    (['matrix-mx-puppet-groupme.service'] if matrix_mx_puppet_groupme_enabled else [])
+    ([{'name': 'matrix-mautrix-whatsapp.service', 'priority': 2000}] if matrix_mautrix_whatsapp_enabled else [])
     +
-    (['matrix-mx-puppet-instagram.service'] if matrix_mx_puppet_instagram_enabled else [])
+    ([{'name': 'matrix-mx-puppet-discord.service', 'priority': 2000}] if matrix_mx_puppet_discord_enabled else [])
     +
-    (['matrix-mx-puppet-slack.service'] if matrix_mx_puppet_slack_enabled else [])
+    ([{'name': 'matrix-mx-puppet-groupme.service', 'priority': 2000}] if matrix_mx_puppet_groupme_enabled else [])
     +
-    (['matrix-mx-puppet-steam.service'] if matrix_mx_puppet_steam_enabled else [])
+    ([{'name': 'matrix-mx-puppet-instagram.service', 'priority': 2000}] if matrix_mx_puppet_instagram_enabled else [])
     +
-    (['matrix-mx-puppet-twitter.service'] if matrix_mx_puppet_twitter_enabled else [])
+    ([{'name': 'matrix-mx-puppet-slack.service', 'priority': 2000}] if matrix_mx_puppet_slack_enabled else [])
     +
-    (['matrix-sms-bridge.service'] if matrix_sms_bridge_enabled else [])
+    ([{'name': 'matrix-mx-puppet-steam.service', 'priority': 2000}] if matrix_mx_puppet_steam_enabled else [])
     +
-    (['matrix-cactus-comments.service'] if matrix_cactus_comments_enabled else [])
+    ([{'name': 'matrix-mx-puppet-twitter.service', 'priority': 2000}] if matrix_mx_puppet_twitter_enabled else [])
     +
-    (['matrix-client-cinny.service'] if matrix_client_cinny_enabled else [])
+    ([{'name': 'matrix-sms-bridge.service', 'priority': 2000}] if matrix_sms_bridge_enabled else [])
     +
-    (['matrix-client-element.service'] if matrix_client_element_enabled else [])
+    ([{'name': 'matrix-cactus-comments.service', 'priority': 2000}] if matrix_cactus_comments_enabled else [])
     +
-    (['matrix-client-hydrogen.service'] if matrix_client_hydrogen_enabled else [])
+    ([{'name': 'matrix-client-cinny.service', 'priority': 2000}] if matrix_client_cinny_enabled else [])
     +
-    (['matrix-' + matrix_homeserver_implementation + '.service'])
+    ([{'name': 'matrix-client-element.service', 'priority': 2000}] if matrix_client_element_enabled else [])
     +
-    (['matrix-corporal.service'] if matrix_corporal_enabled else [])
+    ([{'name': 'matrix-client-hydrogen.service', 'priority': 2000}] if matrix_client_hydrogen_enabled else [])
     +
-    (['matrix-coturn.service'] if matrix_coturn_enabled else [])
+    ([{'name': ('matrix-' + matrix_homeserver_implementation + '.service'), 'priority': 1000}])
     +
-    (['matrix-coturn-reload.timer'] if (matrix_coturn_enabled and matrix_coturn_tls_enabled) else [])
+    ([{'name': 'matrix-corporal.service', 'priority': 1500}] if matrix_corporal_enabled else [])
     +
-    (['matrix-dimension.service'] if matrix_dimension_enabled else [])
+    ([{'name': 'matrix-coturn.service', 'priority': 4000}] if matrix_coturn_enabled else [])
     +
-    (['matrix-dynamic-dns.service'] if matrix_dynamic_dns_enabled else [])
+    ([{'name': 'matrix-coturn-reload.timer', 'priority': 5000}] if (matrix_coturn_enabled and matrix_coturn_tls_enabled) else [])
     +
-    (['matrix-email2matrix.service'] if matrix_email2matrix_enabled else [])
+    ([{'name': 'matrix-dimension.service', 'priority': 2500}] if matrix_dimension_enabled else [])
     +
-    (['matrix-etherpad.service'] if matrix_etherpad_enabled else [])
+    ([{'name': 'matrix-dynamic-dns.service', 'priority': 5000}] if matrix_dynamic_dns_enabled else [])
     +
-    (['matrix-grafana.service'] if matrix_grafana_enabled else [])
+    ([{'name': 'matrix-email2matrix.service', 'priority': 2000}] if matrix_email2matrix_enabled else [])
     +
-    (['matrix-jitsi-web.service', 'matrix-jitsi-prosody.service', 'matrix-jitsi-jicofo.service', 'matrix-jitsi-jvb.service'] if matrix_jitsi_enabled else [])
+    ([{'name': 'matrix-etherpad.service', 'priority': 4000}] if matrix_etherpad_enabled else [])
     +
-    (['matrix-ldap-registration-proxy.service'] if matrix_ldap_registration_proxy_enabled else [])
+    ([{'name': 'matrix-grafana.service', 'priority': 4000}] if matrix_grafana_enabled else [])
     +
-    (['matrix-ma1sd.service'] if matrix_ma1sd_enabled else [])
+    ([{'name': 'matrix-jitsi-web.service', 'priority': 4200}] if matrix_jitsi_enabled else [])
     +
-    (['matrix-mailer.service'] if matrix_mailer_enabled else [])
+    ([{'name': 'matrix-jitsi-prosody.service', 'priority': 4000}] if matrix_jitsi_enabled else [])
     +
-    (['matrix-nginx-proxy.service'] if matrix_nginx_proxy_enabled else [])
+    ([{'name': 'matrix-jitsi-jicofo.service', 'priority': 4100}] if matrix_jitsi_enabled else [])
     +
-    (matrix_ssl_renewal_systemd_units_list | selectattr('applicable') | map(attribute='name'))
+    ([{'name': 'matrix-jitsi-jvb.service', 'priority': 4100}] if matrix_jitsi_enabled else [])
     +
-    (['matrix-ntfy.service'] if matrix_ntfy_enabled else [])
+    ([{'name': 'matrix-ldap-registration-proxy.service', 'priority': 2000}] if matrix_ldap_registration_proxy_enabled else [])
     +
-    (['matrix-postgres.service'] if matrix_postgres_enabled else [])
+    ([{'name': 'matrix-ma1sd.service', 'priority': 2000}] if matrix_ma1sd_enabled else [])
     +
-    (['matrix-postgres-backup.service'] if matrix_postgres_backup_enabled else [])
+    ([{'name': 'matrix-mailer.service', 'priority': 2000}] if matrix_mailer_enabled else [])
     +
-    (['matrix-prometheus.service'] if matrix_prometheus_enabled else [])
+    ([{'name': 'matrix-nginx-proxy.service', 'priority': 3000}] if matrix_nginx_proxy_enabled else [])
     +
-    (['matrix-prometheus-node-exporter.service'] if matrix_prometheus_node_exporter_enabled else [])
+    (matrix_ssl_renewal_systemd_units_list | selectattr('applicable'))
     +
-    (['matrix-prometheus-postgres-exporter.service'] if matrix_prometheus_postgres_exporter_enabled else [])
+    ([{'name': 'matrix-ntfy.service', 'priority': 800}] if matrix_ntfy_enabled else [])
     +
-    (['matrix-redis'] if matrix_redis_enabled else [])
+    ([{'name': 'matrix-postgres.service', 'priority': 500}] if matrix_postgres_enabled else [])
     +
-    (['matrix-registration.service'] if matrix_registration_enabled else [])
+    ([{'name': 'matrix-postgres-backup.service', 'priority': 3000}] if matrix_postgres_backup_enabled else [])
     +
-    (['matrix-sygnal.service'] if matrix_sygnal_enabled else [])
+    ([{'name': 'matrix-prometheus.service', 'priority': 4000}] if matrix_prometheus_enabled else [])
     +
-    (['matrix-goofys.service'] if matrix_s3_media_store_enabled else [])
+    ([{'name': 'matrix-prometheus-node-exporter.service', 'priority': 3900}] if matrix_prometheus_node_exporter_enabled else [])
     +
-    (['matrix-synapse-s3-storage-provider-migrate.timer'] if matrix_synapse_ext_synapse_s3_storage_provider_enabled else [])
+    ([{'name': 'matrix-prometheus-postgres-exporter.service', 'priority': 3900}] if matrix_prometheus_postgres_exporter_enabled else [])
     +
-    (['matrix-synapse-admin.service'] if matrix_synapse_admin_enabled else [])
+    ([{'name': 'matrix-redis', 'priority': 750}] if matrix_redis_enabled else [])
     +
-    (['matrix-synapse-reverse-proxy-companion.service'] if matrix_synapse_reverse_proxy_companion_enabled else [])
+    ([{'name': 'matrix-registration.service', 'priority': 4000}] if matrix_registration_enabled else [])
+    +
+    ([{'name': 'matrix-sygnal.service', 'priority': 800}] if matrix_sygnal_enabled else [])
+    +
+    ([{'name': 'matrix-goofys.service', 'priority': 800}] if matrix_s3_media_store_enabled else [])
+    +
+    ([{'name': 'matrix-synapse-s3-storage-provider-migrate.timer', 'priority': 5000}] if matrix_synapse_ext_synapse_s3_storage_provider_enabled else [])
+    +
+    ([{'name': 'matrix-synapse-admin.service', 'priority': 4000}] if matrix_synapse_admin_enabled else [])
+    +
+    ([{'name': 'matrix-synapse-reverse-proxy-companion.service', 'priority': 1500}] if matrix_synapse_reverse_proxy_companion_enabled else [])
   }}
 
 matrix_homeserver_app_service_config_files_auto: |

roles/custom/matrix-base/defaults/main.yml

@@ -253,9 +253,26 @@ matrix_well_known_matrix_server_enabled: true
 # See `matrix_homeserver_admin_contacts`, `matrix_homeserver_support_url`, etc.
 matrix_well_known_matrix_support_enabled: false
 
-# This will contain a list of enabled services that the playbook is managing.
-# Each component is expected to append its service name to this list.
-matrix_systemd_services_list: []
+# matrix_systemd_services_list_auto contains a list of systemd services and their priorities.
+# This list is managed by the playbook. You're not meant to override this variable.
+# To add your own items to the list, use `matrix_systemd_services_list_additional`
+matrix_systemd_services_list_auto: []
+
+# matrix_systemd_services_list_additional contains your own list of systemd services and their priorities.
+#
+# Example:
+# matrix_systemd_services_list_additional:
+#   - name: some-service.service
+#     priority: 1250
+#   - name: another-service.service
+#     priority: 3500
+matrix_systemd_services_list_additional: []
+
+# matrix_systemd_services_list contains a list of systemd services and their priorities.
+matrix_systemd_services_list: "{{ matrix_systemd_services_list_auto + matrix_systemd_services_list_additional }}"
+
+# matrix_systemd_services_autostart_enabled controls whether systemd services should auto-start when the system reboots
+matrix_systemd_services_autostart_enabled: true
 
 matrix_homeserver_container_extra_arguments_auto: []
 matrix_homeserver_app_service_config_files_auto: []

roles/custom/matrix-common-after/tasks/start.yml

@@ -1,26 +1,22 @@
 ---
 
-- name: Determine whether we should make services autostart
-  ansible.builtin.set_fact:
-    matrix_services_autostart_enabled_bool: "{{ true if matrix_services_autostart_enabled | default('') == '' else matrix_services_autostart_enabled | bool }}"
-
 - name: Ensure systemd is reloaded
   ansible.builtin.service:
     daemon_reload: true
 
 - name: Ensure Matrix services are stopped
   ansible.builtin.service:
-    name: "{{ item }}"
+    name: "{{ item.name }}"
     state: stopped
-  with_items: "{{ matrix_systemd_services_list }}"
+  with_items: "{{ matrix_systemd_services_list | sort (attribute='priority,name', reverse=true) }}"
   when: not ansible_check_mode
 
 - name: Ensure Matrix services are started
   ansible.builtin.service:
-    name: "{{ item }}"
-    enabled: "{{ matrix_services_autostart_enabled_bool }}"
+    name: "{{ item.name }}"
     state: started
-  with_items: "{{ matrix_systemd_services_list }}"
+    enabled: "{{ matrix_systemd_services_autostart_enabled }}"
+  with_items: "{{ matrix_systemd_services_list | sort (attribute='priority,name') }}"
   when: not ansible_check_mode
 
 # If we check service state immediately, we may succeed,
@@ -48,7 +44,7 @@
           If you're on a slow or overloaded server, it may be that services take a longer time to start and that this error is a false-positive.
           You can consider raising the value of the `matrix_common_after_systemd_service_start_wait_for_timeout_seconds` variable.
           See `roles/custom/matrix-common-after/defaults/main.yml` for more details about that.
-      with_items: "{{ matrix_systemd_services_list }}"
+      with_items: "{{ matrix_systemd_services_list | map(attribute='name') }}"
       when:
         - "item.endswith('.service') and (ansible_facts.services[item] | default(none) is none or ansible_facts.services[item].state != 'running')"
 
@@ -59,7 +55,7 @@
     # Therefore iterating here manually
     - name: Fetch systemd information
       ansible.builtin.systemd:
-        name: "{{ item }}"
+        name: "{{ item.name }}"
       register: systemdstatus
       with_items: "{{ matrix_systemd_services_list }}"
 

roles/custom/matrix-common-after/tasks/stop.yml

@@ -2,6 +2,6 @@
 
 - name: Ensure Matrix services stopped
   ansible.builtin.service:
-    name: "{{ item }}"
+    name: "{{ item.name }}"
     state: stopped
-  with_items: "{{ matrix_systemd_services_list }}"
+  with_items: "{{ matrix_systemd_services_list | sort (attribute='priority,name', reverse=true) }}"

roles/custom/matrix-jitsi/tasks/init_additional_jvb.yml

@@ -1,5 +1,5 @@
 ---
 
 - ansible.builtin.set_fact:
-    matrix_systemd_services_list: "{{ ['matrix-jitsi-jvb.service'] }}"
+    matrix_systemd_services_list: "{{ [{'name': 'matrix-jitsi-jvb.service', 'priority': 1000}] }}"
   when: matrix_jitsi_enabled | bool

roles/custom/matrix-nginx-proxy/vars/main.yml

@@ -7,12 +7,16 @@ matrix_ssl_renewal_systemd_units_list:
   - name: matrix-ssl-lets-encrypt-certificates-renew.service
     applicable: "{{ matrix_ssl_retrieval_method == 'lets-encrypt' }}"
     enableable: false
+    priority: 5000
   - name: matrix-ssl-lets-encrypt-certificates-renew.timer
     applicable: "{{ matrix_ssl_retrieval_method == 'lets-encrypt' }}"
     enableable: true
+    priority: 5000
   - name: matrix-ssl-nginx-proxy-reload.service
     applicable: "{{ matrix_ssl_retrieval_method == 'lets-encrypt' and matrix_nginx_proxy_enabled | bool }}"
     enableable: false
+    priority: 5000
   - name: matrix-ssl-nginx-proxy-reload.timer
     applicable: "{{ matrix_ssl_retrieval_method == 'lets-encrypt' and matrix_nginx_proxy_enabled | bool }}"
     enableable: true
+    priority: 5000

roles/custom/matrix-synapse/tasks/synapse/workers/setup_install.yml

@@ -27,7 +27,7 @@
     state: stopped
     enabled: false
   with_items: "{{ matrix_synapse_workers_current_systemd_services.files }}"
-  when: "not ansible_check_mode and item.path | basename not in matrix_systemd_services_list"
+  when: "not ansible_check_mode and item.path | basename not in matrix_systemd_services_list | map(attribute='name')"
 
 - name: Ensure unnecessary worker systemd services are cleaned
   ansible.builtin.file:

roles/custom/matrix-synapse/tasks/synapse/workers/util/inject_worker.yml

@@ -53,7 +53,7 @@
       when: "'replication_port' not in matrix_synapse_worker_details"
 
 - ansible.builtin.set_fact:
-    matrix_systemd_services_list: "{{ matrix_systemd_services_list + [matrix_synapse_worker_details.name + '.service'] }}"
+    matrix_systemd_services_list_auto: "{{ matrix_systemd_services_list_auto + [{'name': (matrix_synapse_worker_details.name + '.service'), 'priority': 1100}] }}"
 
 - ansible.builtin.set_fact:
     matrix_synapse_webserving_workers_systemd_services_list: "{{ matrix_synapse_webserving_workers_systemd_services_list + [matrix_synapse_worker_details.name + '.service'] }}"