From d9fa2f7ed45114fdfaaa5c3e205a091a0d8e8191 Mon Sep 17 00:00:00 2001 From: HarHarLinks Date: Mon, 4 Oct 2021 19:53:38 +0200 Subject: [PATCH 1/6] add auto proxy synapse worker metrics when matrix_nginx_proxy_proxy_synapse_metrics is enabled --- roles/matrix-nginx-proxy/defaults/main.yml | 2 +- .../nginx/conf.d/matrix-synapse.conf.j2 | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/roles/matrix-nginx-proxy/defaults/main.yml b/roles/matrix-nginx-proxy/defaults/main.yml index 79211a23..07e84ace 100644 --- a/roles/matrix-nginx-proxy/defaults/main.yml +++ b/roles/matrix-nginx-proxy/defaults/main.yml @@ -222,7 +222,7 @@ matrix_nginx_proxy_proxy_matrix_client_api_forwarded_location_prefix_regexes: | + (['/_synapse/admin'] if matrix_nginx_proxy_proxy_matrix_client_api_forwarded_location_synapse_admin_api_enabled else []) + - (['/_synapse/metrics'] if matrix_nginx_proxy_proxy_synapse_metrics else []) + (['/_synapse.*/metrics'] if matrix_nginx_proxy_proxy_synapse_metrics else []) }} # Specifies where requests for the root URI (`/`) on the `matrix.` domain should be redirected. diff --git a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 index db111090..343f04e5 100644 --- a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 +++ b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 @@ -153,6 +153,24 @@ server { } {% endif %} + {% if matrix_nginx_proxy_enabled and matrix_nginx_proxy_proxy_synapse_metrics %} + {% for worker in matrix_prometheus_scraper_synapse_workers_enabled_list %} + {% if worker.metrics_port != 0 %} + location /_synapse-worker-{{ worker.type }}-{{ worker.instanceId }}/metrics { + resolver 127.0.0.11 valid=5s; + set $backend "matrix-synapse-worker-{{ worker.type }}-{{ worker.instanceId }}:{{ worker.metrics_port }}"; + proxy_pass http://$backend/_synapse/metrics; + proxy_set_header Host $host; + + {% if matrix_nginx_proxy_proxy_synapse_metrics_basic_auth_enabled %} + auth_basic "protected"; + auth_basic_user_file /nginx-data/matrix-synapse-metrics-htpasswd; + {% endif %} + } + {% endif %} + {% endfor %} + {% endif %} + {# Everything else just goes to the API server ##} location / { {% if matrix_nginx_proxy_enabled %} From 4209c4208c1d6ae5f1fa4a6c064d991e060972d5 Mon Sep 17 00:00:00 2001 From: HarHarLinks Date: Tue, 19 Oct 2021 23:14:14 +0200 Subject: [PATCH 2/6] add own variable for worker metrics https://github.com/spantaleev/matrix-docker-ansible-deploy/pull/1311#issuecomment-945718866 --- group_vars/matrix_servers | 2 ++ roles/matrix-nginx-proxy/defaults/main.yml | 1 + .../templates/nginx/conf.d/matrix-synapse.conf.j2 | 2 +- 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index acdb7b75..23833875 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -1296,6 +1296,8 @@ matrix_nginx_proxy_synapse_media_repository_locations: "{{matrix_synapse_workers matrix_nginx_proxy_synapse_user_dir_locations: "{{ matrix_synapse_workers_user_dir_endpoints|default([]) }}" matrix_nginx_proxy_synapse_frontend_proxy_locations: "{{ matrix_synapse_workers_frontend_proxy_endpoints|default([]) }}" +matrix_nginx_proxy_proxy_synapse_workers_enabled_list: "{{ matrix_synapse_workers_enabled_list }}" + matrix_nginx_proxy_systemd_wanted_services_list: | {{ (['matrix-synapse.service'] if matrix_synapse_enabled else []) diff --git a/roles/matrix-nginx-proxy/defaults/main.yml b/roles/matrix-nginx-proxy/defaults/main.yml index 07e84ace..8e633875 100644 --- a/roles/matrix-nginx-proxy/defaults/main.yml +++ b/roles/matrix-nginx-proxy/defaults/main.yml @@ -182,6 +182,7 @@ matrix_nginx_proxy_proxy_matrix_identity_api_addr_sans_container: "127.0.0.1:809 # Controls whether proxying for metrics (`/_synapse/metrics`) should be done (on the matrix domain) matrix_nginx_proxy_proxy_synapse_metrics: false +matrix_nginx_proxy_synapse_workers_enabled_list: [] matrix_nginx_proxy_proxy_synapse_metrics_basic_auth_enabled: false matrix_nginx_proxy_proxy_synapse_metrics_basic_auth_key: "" diff --git a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 index 343f04e5..6933e967 100644 --- a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 +++ b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 @@ -154,7 +154,7 @@ server { {% endif %} {% if matrix_nginx_proxy_enabled and matrix_nginx_proxy_proxy_synapse_metrics %} - {% for worker in matrix_prometheus_scraper_synapse_workers_enabled_list %} + {% for worker in matrix_nginx_proxy_proxy_synapse_workers_enabled_list %} {% if worker.metrics_port != 0 %} location /_synapse-worker-{{ worker.type }}-{{ worker.instanceId }}/metrics { resolver 127.0.0.11 valid=5s; From dd4dc22619ea15319e0335d704b05251d5b77a70 Mon Sep 17 00:00:00 2001 From: HarHarLinks Date: Tue, 19 Oct 2021 23:32:28 +0200 Subject: [PATCH 3/6] add worker metrics documentation --- docs/configuring-playbook-prometheus-grafana.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/configuring-playbook-prometheus-grafana.md b/docs/configuring-playbook-prometheus-grafana.md index b7f3caae..529104d2 100644 --- a/docs/configuring-playbook-prometheus-grafana.md +++ b/docs/configuring-playbook-prometheus-grafana.md @@ -58,6 +58,10 @@ Name | Description `matrix_nginx_proxy_proxy_synapse_metrics_basic_auth_key`|Set this to a password to use for HTTP Basic Auth for protecting `https://matrix.DOMAIN/_synapse/metrics` (the username is always `prometheus` - it's not configurable) `matrix_server_fqn_grafana`|Use this variable to override the domain at which the Grafana web user-interface is at (defaults to `stats.DOMAIN`) +### Collecting worker metrics to an external Prometheus server + +If you are using workers (`matrix_synapse_workers_enabled`) and have enabled `matrix_nginx_proxy_proxy_synapse_metrics` as described above, the playbook will also automatically proxy the all worker threads's metrics to `https://matrix.DOMAIN/_synapse-worker-TYPE-ID/metrics`, where `TYPE` corresponds to the type and `ID` to the instanceId of a worker as exemplified in `matrix_synapse_workers_enabled_list`. + ### Collecting system and Postgres metrics to an external Prometheus server (advanced) When you normally enable the Prometheus and Grafana via the playbook, it will also show general system (via node-exporter) and Postgres (via postgres-exporter) stats. If you are instead collecting your metrics to an external Prometheus server, you can follow this advanced configuration example to also export these stats. From ce41674e611917dcd608fb67d5ec9379b0926cf1 Mon Sep 17 00:00:00 2001 From: HarHarLinks Date: Wed, 20 Oct 2021 12:31:49 +0200 Subject: [PATCH 4/6] auto-generate prometheus.yml for workers metrics --- ...configuring-playbook-prometheus-grafana.md | 28 +++++++++++++ .../tasks/setup_nginx_proxy.yml | 15 +++++++ .../external_prometheus.yml.example.j2 | 40 +++++++++++++++++++ 3 files changed, 83 insertions(+) create mode 100644 roles/matrix-nginx-proxy/templates/prometheus/external_prometheus.yml.example.j2 diff --git a/docs/configuring-playbook-prometheus-grafana.md b/docs/configuring-playbook-prometheus-grafana.md index 529104d2..4edc2a81 100644 --- a/docs/configuring-playbook-prometheus-grafana.md +++ b/docs/configuring-playbook-prometheus-grafana.md @@ -62,6 +62,34 @@ Name | Description If you are using workers (`matrix_synapse_workers_enabled`) and have enabled `matrix_nginx_proxy_proxy_synapse_metrics` as described above, the playbook will also automatically proxy the all worker threads's metrics to `https://matrix.DOMAIN/_synapse-worker-TYPE-ID/metrics`, where `TYPE` corresponds to the type and `ID` to the instanceId of a worker as exemplified in `matrix_synapse_workers_enabled_list`. +The playbook also generates an exemplary prometheus.yml config file (`matrix_base_data_path/external_prometheus.yml.template`) with all the correct paths which you can copy to your Prometheus server and adapt to your needs, especially edit the specified `password_file` path and contents and path to your `synapse-v2.rules`. +It will look a bit like this: +```yaml +scrape_configs: + - job_name: 'synapse' + metrics_path: /_synapse/metrics + scheme: https + basic_auth: + username: prometheus + password_file: /etc/prometheus/password.pwd + static_configs: + - targets: ['matrix.DOMAIN:443'] + labels: + job: "master" + index: 1 + - job_name: 'synapse-generic_worker-1' + metrics_path: /_synapse-worker-generic_worker-18111/metrics + scheme: https + basic_auth: + username: prometheus + password_file: /etc/prometheus/password.pwd + static_configs: + - targets: ['matrix.DOMAIN:443'] + labels: + job: "generic_worker" + index: 18111 +``` + ### Collecting system and Postgres metrics to an external Prometheus server (advanced) When you normally enable the Prometheus and Grafana via the playbook, it will also show general system (via node-exporter) and Postgres (via postgres-exporter) stats. If you are instead collecting your metrics to an external Prometheus server, you can follow this advanced configuration example to also export these stats. diff --git a/roles/matrix-nginx-proxy/tasks/setup_nginx_proxy.yml b/roles/matrix-nginx-proxy/tasks/setup_nginx_proxy.yml index 149fadab..3dd828d2 100644 --- a/roles/matrix-nginx-proxy/tasks/setup_nginx_proxy.yml +++ b/roles/matrix-nginx-proxy/tasks/setup_nginx_proxy.yml @@ -38,6 +38,15 @@ mode: 0400 when: "matrix_nginx_proxy_proxy_synapse_metrics_basic_auth_enabled|bool and matrix_nginx_proxy_proxy_synapse_metrics|bool" +- name: Generate sample prometheus.yml for external scraping + template: + src: "{{ role_path }}/templates/prometheus/external_prometheus.yml.example.j2" + dest: "{{ matrix_base_data_path }}/external_prometheus.yml.example" + owner: "{{ matrix_user_username }}" + group: "{{ matrix_user_groupname }}" + mode: 0400 + when: matrix_nginx_proxy_proxy_synapse_metrics|bool + - name: Ensure Matrix nginx-proxy configured (generic) template: src: "{{ role_path }}/templates/nginx/conf.d/nginx-http.conf.j2" @@ -270,3 +279,9 @@ path: "{{ matrix_nginx_proxy_data_path }}/matrix-synapse-metrics-htpasswd" state: absent when: "not matrix_nginx_proxy_proxy_synapse_metrics_basic_auth_enabled|bool or not matrix_nginx_proxy_proxy_synapse_metrics|bool" + +- name: Ensure sample prometheus.yml for external scraping is deleted + file: + path: "{{ matrix_base_data_path }}/external_prometheus.yml.example" + state: absent + when: "not matrix_nginx_proxy_proxy_synapse_metrics|bool" diff --git a/roles/matrix-nginx-proxy/templates/prometheus/external_prometheus.yml.example.j2 b/roles/matrix-nginx-proxy/templates/prometheus/external_prometheus.yml.example.j2 new file mode 100644 index 00000000..36d61c05 --- /dev/null +++ b/roles/matrix-nginx-proxy/templates/prometheus/external_prometheus.yml.example.j2 @@ -0,0 +1,40 @@ +global: + scrape_interval: 5s + + # Attach these labels to any time series or alerts when communicating with + # external systems (federation, remote storage, Alertmanager). + external_labels: + monitor: 'synapse-{{ matrix_domain }}' + +rule_files: + - /etc/prometheus/synapse-v2.rules + +scrape_configs: + - job_name: 'synapse' + metrics_path: /_synapse/metrics + scheme: {{ 'https' if matrix_nginx_proxy_https_enabled else 'http' }} +{% if matrix_nginx_proxy_proxy_synapse_metrics_basic_auth_enabled %} + basic_auth: + username: prometheus + password_file: /path/to/your/passwordfile.pwd +{% endif %} + static_configs: + - targets: ['{{ matrix_server_fqn_matrix }}:{{ matrix_nginx_proxy_container_https_host_bind_port if matrix_nginx_proxy_https_enabled else matrix_nginx_proxy_container_http_host_bind_port }}'] + labels: + job: "master" + index: 1 +{% for worker in matrix_nginx_proxy_proxy_synapse_workers_enabled_list %} + - job_name: 'synapse-{{ worker.type }}-{{ worker.instanceId }}' + metrics_path: /_synapse-worker-{{ worker.type }}-{{ worker.instanceId }}/metrics + scheme: {{ 'https' if matrix_nginx_proxy_https_enabled else 'http' }} +{% if matrix_nginx_proxy_proxy_synapse_metrics_basic_auth_enabled %} + basic_auth: + username: prometheus + password_file: /path/to/your/passwordfile.pwd +{% endif %} + static_configs: + - targets: [{{ matrix_server_fqn_matrix }:{{ matrix_nginx_proxy_container_https_host_bind_port if matrix_nginx_proxy_https_enabled else matrix_nginx_proxy_container_http_host_bind_port }}'] + labels: + job: "{{ worker.type }}" + index: {{ worker.instanceId }} +{% endfor %} From 7b33fc8e1971a509e5aaee1af8d9781dab677e40 Mon Sep 17 00:00:00 2001 From: HarHarLinks Date: Wed, 20 Oct 2021 12:57:55 +0200 Subject: [PATCH 5/6] fixup! auto-generate prometheus.yml for workers metrics --- roles/matrix-nginx-proxy/tasks/setup_nginx_proxy.yml | 2 +- .../templates/prometheus/external_prometheus.yml.example.j2 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/matrix-nginx-proxy/tasks/setup_nginx_proxy.yml b/roles/matrix-nginx-proxy/tasks/setup_nginx_proxy.yml index 3dd828d2..e577491b 100644 --- a/roles/matrix-nginx-proxy/tasks/setup_nginx_proxy.yml +++ b/roles/matrix-nginx-proxy/tasks/setup_nginx_proxy.yml @@ -44,7 +44,7 @@ dest: "{{ matrix_base_data_path }}/external_prometheus.yml.example" owner: "{{ matrix_user_username }}" group: "{{ matrix_user_groupname }}" - mode: 0400 + mode: 0644 when: matrix_nginx_proxy_proxy_synapse_metrics|bool - name: Ensure Matrix nginx-proxy configured (generic) diff --git a/roles/matrix-nginx-proxy/templates/prometheus/external_prometheus.yml.example.j2 b/roles/matrix-nginx-proxy/templates/prometheus/external_prometheus.yml.example.j2 index 36d61c05..01a39ffa 100644 --- a/roles/matrix-nginx-proxy/templates/prometheus/external_prometheus.yml.example.j2 +++ b/roles/matrix-nginx-proxy/templates/prometheus/external_prometheus.yml.example.j2 @@ -33,7 +33,7 @@ scrape_configs: password_file: /path/to/your/passwordfile.pwd {% endif %} static_configs: - - targets: [{{ matrix_server_fqn_matrix }:{{ matrix_nginx_proxy_container_https_host_bind_port if matrix_nginx_proxy_https_enabled else matrix_nginx_proxy_container_http_host_bind_port }}'] + - targets: ['{{ matrix_server_fqn_matrix }}:{{ matrix_nginx_proxy_container_https_host_bind_port if matrix_nginx_proxy_https_enabled else matrix_nginx_proxy_container_http_host_bind_port }}'] labels: job: "{{ worker.type }}" index: {{ worker.instanceId }} From 5f6bbafa17731eed8148cff5c22d99af3014b224 Mon Sep 17 00:00:00 2001 From: Kim Brose Date: Sun, 24 Oct 2021 16:00:42 +0200 Subject: [PATCH 6/6] fix space before tab in indent --- .../templates/nginx/conf.d/matrix-synapse.conf.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 index 6933e967..6f569327 100644 --- a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 +++ b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 @@ -168,7 +168,7 @@ server { {% endif %} } {% endif %} - {% endfor %} + {% endfor %} {% endif %} {# Everything else just goes to the API server ##}