Add service priorities - try to stop/start them in an optimal order

This commit is contained in:
Slavi Pantaleev 2022-11-23 08:43:46 +02:00
parent 0ea7cb5d18
commit 360e643f84
8 changed files with 123 additions and 86 deletions

View file

@ -22,139 +22,159 @@ matrix_playbook_docker_installation_enabled: true
# This list is not exhaustive and final.
# Synapse workers are still injected into the list at runtime.
# Additional JVB workers (playbooks/jitsi_jvb.yml -- roles/custom/matrix-jitsi/tasks/init_additional_jvb.yml) override this variable at runtime as well.
matrix_systemd_services_list: |
#
# Priority levels are like this:
# - core services (the homeserver) get a level of ~1000
# - services that core services depend on (database, Redis, ntfy, etc.) get a lower level - between 500 and 1000
# - reverse-proxying services get level 3000
# - Matrix utility services (bridges, bots) get a level of 2000/2200, so that:
# - they can start before the reverse-proxy
# - so that, when the reverse-proxy is up (Matrix is up), all bots and bridges can be interacted with
# - monitoring services (Prometheus, Grafana, ..) get a level of 4000 - they can start later than all-of-Matrix
# - services which aren't time-sensitive (various crons and timers) get a level of 5000 - they can start later than all-of-Matrix
matrix_systemd_services_list_auto: |
{{
(['matrix-backup-borg.timer'] if matrix_backup_borg_enabled else [])
([{'name': 'matrix-backup-borg.timer', 'priority': 5000}] if matrix_backup_borg_enabled else [])
+
(['matrix-bot-buscarron.service'] if matrix_bot_buscarron_enabled else [])
([{'name': 'matrix-bot-buscarron.service', 'priority': 2200}] if matrix_bot_buscarron_enabled else [])
+
(['matrix-bot-go-neb.service'] if matrix_bot_go_neb_enabled else [])
([{'name': 'matrix-bot-go-neb.service', 'priority': 2200}] if matrix_bot_go_neb_enabled else [])
+
(['matrix-bot-honoroit.service'] if matrix_bot_honoroit_enabled else [])
([{'name': 'matrix-bot-honoroit.service', 'priority': 2200}] if matrix_bot_honoroit_enabled else [])
+
(['matrix-bot-matrix-registration-bot.service'] if matrix_bot_matrix_registration_bot_enabled else [])
([{'name': 'matrix-bot-matrix-registration-bot.service', 'priority': 2200}] if matrix_bot_matrix_registration_bot_enabled else [])
+
(['matrix-bot-matrix-reminder-bot.service'] if matrix_bot_matrix_reminder_bot_enabled else [])
([{'name': 'matrix-bot-matrix-reminder-bot.service', 'priority': 2200}] if matrix_bot_matrix_reminder_bot_enabled else [])
+
(['matrix-bot-maubot.service'] if matrix_bot_maubot_enabled else [])
([{'name': 'matrix-bot-maubot.service', 'priority': 2200}] if matrix_bot_maubot_enabled else [])
+
(['matrix-bot-mjolnir.service'] if matrix_bot_mjolnir_enabled else [])
([{'name': 'matrix-bot-mjolnir.service', 'priority': 2200}] if matrix_bot_mjolnir_enabled else [])
+
(['matrix-bot-postmoogle.service'] if matrix_bot_postmoogle_enabled else [])
([{'name': 'matrix-bot-postmoogle.service', 'priority': 2200}] if matrix_bot_postmoogle_enabled else [])
+
(['matrix-appservice-discord.service'] if matrix_appservice_discord_enabled else [])
([{'name': 'matrix-appservice-discord.service', 'priority': 2000}] if matrix_appservice_discord_enabled else [])
+
(['matrix-appservice-irc.service'] if matrix_appservice_irc_enabled else [])
([{'name': 'matrix-appservice-irc.service', 'priority': 2000}] if matrix_appservice_irc_enabled else [])
+
(['matrix-appservice-kakaotalk.service', 'matrix-appservice-kakaotalk-node.service'] if matrix_appservice_kakaotalk_enabled else [])
([{'name': 'matrix-appservice-kakaotalk.service', 'priority': 2000}] if matrix_appservice_kakaotalk_enabled else [])
+
(['matrix-appservice-slack.service'] if matrix_appservice_slack_enabled else [])
([{'name': 'matrix-appservice-kakaotalk-node.service', 'priority': 1900}] if matrix_appservice_kakaotalk_enabled else [])
+
(['matrix-appservice-webhooks.service'] if matrix_appservice_webhooks_enabled else [])
([{'name': 'matrix-appservice-slack.service', 'priority': 2000}] if matrix_appservice_slack_enabled else [])
+
(['matrix-beeper-linkedin.service'] if matrix_beeper_linkedin_enabled else [])
([{'name': 'matrix-appservice-webhooks.service', 'priority': 2000}] if matrix_appservice_webhooks_enabled else [])
+
(['matrix-go-skype-bridge.service'] if matrix_go_skype_bridge_enabled else [])
([{'name': 'matrix-beeper-linkedin.service', 'priority': 2000}] if matrix_beeper_linkedin_enabled else [])
+
(['matrix-heisenbridge.service'] if matrix_heisenbridge_enabled else [])
([{'name': 'matrix-go-skype-bridge.service', 'priority': 2000}] if matrix_go_skype_bridge_enabled else [])
+
(['matrix-hookshot.service'] if matrix_hookshot_enabled else [])
([{'name': 'matrix-heisenbridge.service', 'priority': 2000}] if matrix_heisenbridge_enabled else [])
+
(['matrix-mautrix-discord.service'] if matrix_mautrix_discord_enabled else [])
([{'name': 'matrix-hookshot.service', 'priority': 2000}] if matrix_hookshot_enabled else [])
+
(['matrix-mautrix-facebook.service'] if matrix_mautrix_facebook_enabled else [])
([{'name': 'matrix-mautrix-discord.service', 'priority': 2000}] if matrix_mautrix_discord_enabled else [])
+
(['matrix-mautrix-googlechat.service'] if matrix_mautrix_googlechat_enabled else [])
([{'name': 'matrix-mautrix-facebook.service', 'priority': 2000}] if matrix_mautrix_facebook_enabled else [])
+
(['matrix-mautrix-hangouts.service'] if matrix_mautrix_hangouts_enabled else [])
([{'name': 'matrix-mautrix-googlechat.service', 'priority': 2000}] if matrix_mautrix_googlechat_enabled else [])
+
(['matrix-mautrix-instagram.service'] if matrix_mautrix_instagram_enabled else [])
([{'name': 'matrix-mautrix-hangouts.service', 'priority': 2000}] if matrix_mautrix_hangouts_enabled else [])
+
(['matrix-mautrix-signal.service', 'matrix-mautrix-signal-daemon.service'] if matrix_mautrix_signal_enabled else [])
([{'name': 'matrix-mautrix-instagram.service', 'priority': 2000}] if matrix_mautrix_instagram_enabled else [])
+
(['matrix-mautrix-telegram.service'] if matrix_mautrix_telegram_enabled else [])
([{'name': 'matrix-mautrix-signal.service', 'priority': 2000}] if matrix_mautrix_signal_enabled else [])
+
(['matrix-mautrix-twitter.service'] if matrix_mautrix_twitter_enabled else [])
([{'name': 'matrix-mautrix-signal-daemon.service', 'priority': 1900}] if matrix_mautrix_signal_enabled else [])
+
(['matrix-mautrix-whatsapp.service'] if matrix_mautrix_whatsapp_enabled else [])
([{'name': 'matrix-mautrix-telegram.service', 'priority': 2000}] if matrix_mautrix_telegram_enabled else [])
+
(['matrix-mx-puppet-discord.service'] if matrix_mx_puppet_discord_enabled else [])
([{'name': 'matrix-mautrix-twitter.service', 'priority': 2000}] if matrix_mautrix_twitter_enabled else [])
+
(['matrix-mx-puppet-groupme.service'] if matrix_mx_puppet_groupme_enabled else [])
([{'name': 'matrix-mautrix-whatsapp.service', 'priority': 2000}] if matrix_mautrix_whatsapp_enabled else [])
+
(['matrix-mx-puppet-instagram.service'] if matrix_mx_puppet_instagram_enabled else [])
([{'name': 'matrix-mx-puppet-discord.service', 'priority': 2000}] if matrix_mx_puppet_discord_enabled else [])
+
(['matrix-mx-puppet-slack.service'] if matrix_mx_puppet_slack_enabled else [])
([{'name': 'matrix-mx-puppet-groupme.service', 'priority': 2000}] if matrix_mx_puppet_groupme_enabled else [])
+
(['matrix-mx-puppet-steam.service'] if matrix_mx_puppet_steam_enabled else [])
([{'name': 'matrix-mx-puppet-instagram.service', 'priority': 2000}] if matrix_mx_puppet_instagram_enabled else [])
+
(['matrix-mx-puppet-twitter.service'] if matrix_mx_puppet_twitter_enabled else [])
([{'name': 'matrix-mx-puppet-slack.service', 'priority': 2000}] if matrix_mx_puppet_slack_enabled else [])
+
(['matrix-sms-bridge.service'] if matrix_sms_bridge_enabled else [])
([{'name': 'matrix-mx-puppet-steam.service', 'priority': 2000}] if matrix_mx_puppet_steam_enabled else [])
+
(['matrix-cactus-comments.service'] if matrix_cactus_comments_enabled else [])
([{'name': 'matrix-mx-puppet-twitter.service', 'priority': 2000}] if matrix_mx_puppet_twitter_enabled else [])
+
(['matrix-client-cinny.service'] if matrix_client_cinny_enabled else [])
([{'name': 'matrix-sms-bridge.service', 'priority': 2000}] if matrix_sms_bridge_enabled else [])
+
(['matrix-client-element.service'] if matrix_client_element_enabled else [])
([{'name': 'matrix-cactus-comments.service', 'priority': 2000}] if matrix_cactus_comments_enabled else [])
+
(['matrix-client-hydrogen.service'] if matrix_client_hydrogen_enabled else [])
([{'name': 'matrix-client-cinny.service', 'priority': 2000}] if matrix_client_cinny_enabled else [])
+
(['matrix-' + matrix_homeserver_implementation + '.service'])
([{'name': 'matrix-client-element.service', 'priority': 2000}] if matrix_client_element_enabled else [])
+
(['matrix-corporal.service'] if matrix_corporal_enabled else [])
([{'name': 'matrix-client-hydrogen.service', 'priority': 2000}] if matrix_client_hydrogen_enabled else [])
+
(['matrix-coturn.service'] if matrix_coturn_enabled else [])
([{'name': ('matrix-' + matrix_homeserver_implementation + '.service'), 'priority': 1000}])
+
(['matrix-coturn-reload.timer'] if (matrix_coturn_enabled and matrix_coturn_tls_enabled) else [])
([{'name': 'matrix-corporal.service', 'priority': 1500}] if matrix_corporal_enabled else [])
+
(['matrix-dimension.service'] if matrix_dimension_enabled else [])
([{'name': 'matrix-coturn.service', 'priority': 4000}] if matrix_coturn_enabled else [])
+
(['matrix-dynamic-dns.service'] if matrix_dynamic_dns_enabled else [])
([{'name': 'matrix-coturn-reload.timer', 'priority': 5000}] if (matrix_coturn_enabled and matrix_coturn_tls_enabled) else [])
+
(['matrix-email2matrix.service'] if matrix_email2matrix_enabled else [])
([{'name': 'matrix-dimension.service', 'priority': 2500}] if matrix_dimension_enabled else [])
+
(['matrix-etherpad.service'] if matrix_etherpad_enabled else [])
([{'name': 'matrix-dynamic-dns.service', 'priority': 5000}] if matrix_dynamic_dns_enabled else [])
+
(['matrix-grafana.service'] if matrix_grafana_enabled else [])
([{'name': 'matrix-email2matrix.service', 'priority': 2000}] if matrix_email2matrix_enabled else [])
+
(['matrix-jitsi-web.service', 'matrix-jitsi-prosody.service', 'matrix-jitsi-jicofo.service', 'matrix-jitsi-jvb.service'] if matrix_jitsi_enabled else [])
([{'name': 'matrix-etherpad.service', 'priority': 4000}] if matrix_etherpad_enabled else [])
+
(['matrix-ldap-registration-proxy.service'] if matrix_ldap_registration_proxy_enabled else [])
([{'name': 'matrix-grafana.service', 'priority': 4000}] if matrix_grafana_enabled else [])
+
(['matrix-ma1sd.service'] if matrix_ma1sd_enabled else [])
([{'name': 'matrix-jitsi-web.service', 'priority': 4200}] if matrix_jitsi_enabled else [])
+
(['matrix-mailer.service'] if matrix_mailer_enabled else [])
([{'name': 'matrix-jitsi-prosody.service', 'priority': 4000}] if matrix_jitsi_enabled else [])
+
(['matrix-nginx-proxy.service'] if matrix_nginx_proxy_enabled else [])
([{'name': 'matrix-jitsi-jicofo.service', 'priority': 4100}] if matrix_jitsi_enabled else [])
+
(matrix_ssl_renewal_systemd_units_list | selectattr('applicable') | map(attribute='name'))
([{'name': 'matrix-jitsi-jvb.service', 'priority': 4100}] if matrix_jitsi_enabled else [])
+
(['matrix-ntfy.service'] if matrix_ntfy_enabled else [])
([{'name': 'matrix-ldap-registration-proxy.service', 'priority': 2000}] if matrix_ldap_registration_proxy_enabled else [])
+
(['matrix-postgres.service'] if matrix_postgres_enabled else [])
([{'name': 'matrix-ma1sd.service', 'priority': 2000}] if matrix_ma1sd_enabled else [])
+
(['matrix-postgres-backup.service'] if matrix_postgres_backup_enabled else [])
([{'name': 'matrix-mailer.service', 'priority': 2000}] if matrix_mailer_enabled else [])
+
(['matrix-prometheus.service'] if matrix_prometheus_enabled else [])
([{'name': 'matrix-nginx-proxy.service', 'priority': 3000}] if matrix_nginx_proxy_enabled else [])
+
(['matrix-prometheus-node-exporter.service'] if matrix_prometheus_node_exporter_enabled else [])
(matrix_ssl_renewal_systemd_units_list | selectattr('applicable'))
+
(['matrix-prometheus-postgres-exporter.service'] if matrix_prometheus_postgres_exporter_enabled else [])
([{'name': 'matrix-ntfy.service', 'priority': 800}] if matrix_ntfy_enabled else [])
+
(['matrix-redis'] if matrix_redis_enabled else [])
([{'name': 'matrix-postgres.service', 'priority': 500}] if matrix_postgres_enabled else [])
+
(['matrix-registration.service'] if matrix_registration_enabled else [])
([{'name': 'matrix-postgres-backup.service', 'priority': 3000}] if matrix_postgres_backup_enabled else [])
+
(['matrix-sygnal.service'] if matrix_sygnal_enabled else [])
([{'name': 'matrix-prometheus.service', 'priority': 4000}] if matrix_prometheus_enabled else [])
+
(['matrix-goofys.service'] if matrix_s3_media_store_enabled else [])
([{'name': 'matrix-prometheus-node-exporter.service', 'priority': 3900}] if matrix_prometheus_node_exporter_enabled else [])
+
(['matrix-synapse-s3-storage-provider-migrate.timer'] if matrix_synapse_ext_synapse_s3_storage_provider_enabled else [])
([{'name': 'matrix-prometheus-postgres-exporter.service', 'priority': 3900}] if matrix_prometheus_postgres_exporter_enabled else [])
+
(['matrix-synapse-admin.service'] if matrix_synapse_admin_enabled else [])
([{'name': 'matrix-redis', 'priority': 750}] if matrix_redis_enabled else [])
+
(['matrix-synapse-reverse-proxy-companion.service'] if matrix_synapse_reverse_proxy_companion_enabled else [])
([{'name': 'matrix-registration.service', 'priority': 4000}] if matrix_registration_enabled else [])
+
([{'name': 'matrix-sygnal.service', 'priority': 800}] if matrix_sygnal_enabled else [])
+
([{'name': 'matrix-goofys.service', 'priority': 800}] if matrix_s3_media_store_enabled else [])
+
([{'name': 'matrix-synapse-s3-storage-provider-migrate.timer', 'priority': 5000}] if matrix_synapse_ext_synapse_s3_storage_provider_enabled else [])
+
([{'name': 'matrix-synapse-admin.service', 'priority': 4000}] if matrix_synapse_admin_enabled else [])
+
([{'name': 'matrix-synapse-reverse-proxy-companion.service', 'priority': 1500}] if matrix_synapse_reverse_proxy_companion_enabled else [])
}}
matrix_homeserver_app_service_config_files_auto: |

View file

@ -253,9 +253,26 @@ matrix_well_known_matrix_server_enabled: true
# See `matrix_homeserver_admin_contacts`, `matrix_homeserver_support_url`, etc.
matrix_well_known_matrix_support_enabled: false
# This will contain a list of enabled services that the playbook is managing.
# Each component is expected to append its service name to this list.
matrix_systemd_services_list: []
# matrix_systemd_services_list_auto contains a list of systemd services and their priorities.
# This list is managed by the playbook. You're not meant to override this variable.
# To add your own items to the list, use `matrix_systemd_services_list_additional`
matrix_systemd_services_list_auto: []
# matrix_systemd_services_list_additional contains your own list of systemd services and their priorities.
#
# Example:
# matrix_systemd_services_list_additional:
# - name: some-service.service
# priority: 1250
# - name: another-service.service
# priority: 3500
matrix_systemd_services_list_additional: []
# matrix_systemd_services_list contains a list of systemd services and their priorities.
matrix_systemd_services_list: "{{ matrix_systemd_services_list_auto + matrix_systemd_services_list_additional }}"
# matrix_systemd_services_autostart_enabled controls whether systemd services should auto-start when the system reboots
matrix_systemd_services_autostart_enabled: true
matrix_homeserver_container_extra_arguments_auto: []
matrix_homeserver_app_service_config_files_auto: []

View file

@ -1,26 +1,22 @@
---
- name: Determine whether we should make services autostart
ansible.builtin.set_fact:
matrix_services_autostart_enabled_bool: "{{ true if matrix_services_autostart_enabled | default('') == '' else matrix_services_autostart_enabled | bool }}"
- name: Ensure systemd is reloaded
ansible.builtin.service:
daemon_reload: true
- name: Ensure Matrix services are stopped
ansible.builtin.service:
name: "{{ item }}"
name: "{{ item.name }}"
state: stopped
with_items: "{{ matrix_systemd_services_list }}"
with_items: "{{ matrix_systemd_services_list | sort (attribute='priority,name', reverse=true) }}"
when: not ansible_check_mode
- name: Ensure Matrix services are started
ansible.builtin.service:
name: "{{ item }}"
enabled: "{{ matrix_services_autostart_enabled_bool }}"
name: "{{ item.name }}"
state: started
with_items: "{{ matrix_systemd_services_list }}"
enabled: "{{ matrix_systemd_services_autostart_enabled }}"
with_items: "{{ matrix_systemd_services_list | sort (attribute='priority,name') }}"
when: not ansible_check_mode
# If we check service state immediately, we may succeed,
@ -48,7 +44,7 @@
If you're on a slow or overloaded server, it may be that services take a longer time to start and that this error is a false-positive.
You can consider raising the value of the `matrix_common_after_systemd_service_start_wait_for_timeout_seconds` variable.
See `roles/custom/matrix-common-after/defaults/main.yml` for more details about that.
with_items: "{{ matrix_systemd_services_list }}"
with_items: "{{ matrix_systemd_services_list | map(attribute='name') }}"
when:
- "item.endswith('.service') and (ansible_facts.services[item] | default(none) is none or ansible_facts.services[item].state != 'running')"
@ -59,7 +55,7 @@
# Therefore iterating here manually
- name: Fetch systemd information
ansible.builtin.systemd:
name: "{{ item }}"
name: "{{ item.name }}"
register: systemdstatus
with_items: "{{ matrix_systemd_services_list }}"

View file

@ -2,6 +2,6 @@
- name: Ensure Matrix services stopped
ansible.builtin.service:
name: "{{ item }}"
name: "{{ item.name }}"
state: stopped
with_items: "{{ matrix_systemd_services_list }}"
with_items: "{{ matrix_systemd_services_list | sort (attribute='priority,name', reverse=true) }}"

View file

@ -1,5 +1,5 @@
---
- ansible.builtin.set_fact:
matrix_systemd_services_list: "{{ ['matrix-jitsi-jvb.service'] }}"
matrix_systemd_services_list: "{{ [{'name': 'matrix-jitsi-jvb.service', 'priority': 1000}] }}"
when: matrix_jitsi_enabled | bool

View file

@ -7,12 +7,16 @@ matrix_ssl_renewal_systemd_units_list:
- name: matrix-ssl-lets-encrypt-certificates-renew.service
applicable: "{{ matrix_ssl_retrieval_method == 'lets-encrypt' }}"
enableable: false
priority: 5000
- name: matrix-ssl-lets-encrypt-certificates-renew.timer
applicable: "{{ matrix_ssl_retrieval_method == 'lets-encrypt' }}"
enableable: true
priority: 5000
- name: matrix-ssl-nginx-proxy-reload.service
applicable: "{{ matrix_ssl_retrieval_method == 'lets-encrypt' and matrix_nginx_proxy_enabled | bool }}"
enableable: false
priority: 5000
- name: matrix-ssl-nginx-proxy-reload.timer
applicable: "{{ matrix_ssl_retrieval_method == 'lets-encrypt' and matrix_nginx_proxy_enabled | bool }}"
enableable: true
priority: 5000

View file

@ -27,7 +27,7 @@
state: stopped
enabled: false
with_items: "{{ matrix_synapse_workers_current_systemd_services.files }}"
when: "not ansible_check_mode and item.path | basename not in matrix_systemd_services_list"
when: "not ansible_check_mode and item.path | basename not in matrix_systemd_services_list | map(attribute='name')"
- name: Ensure unnecessary worker systemd services are cleaned
ansible.builtin.file:

View file

@ -53,7 +53,7 @@
when: "'replication_port' not in matrix_synapse_worker_details"
- ansible.builtin.set_fact:
matrix_systemd_services_list: "{{ matrix_systemd_services_list + [matrix_synapse_worker_details.name + '.service'] }}"
matrix_systemd_services_list_auto: "{{ matrix_systemd_services_list_auto + [{'name': (matrix_synapse_worker_details.name + '.service'), 'priority': 1100}] }}"
- ansible.builtin.set_fact:
matrix_synapse_webserving_workers_systemd_services_list: "{{ matrix_synapse_webserving_workers_systemd_services_list + [matrix_synapse_worker_details.name + '.service'] }}"