Merge pull request #1311 from HarHarLinks/master

add auto proxy synapse worker metrics
This commit is contained in:
Slavi Pantaleev 2021-10-25 09:21:11 +03:00 committed by GitHub
commit 06bcdcf9d2
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 109 additions and 1 deletions

View file

@ -58,6 +58,38 @@ Name | Description
`matrix_nginx_proxy_proxy_synapse_metrics_basic_auth_key`|Set this to a password to use for HTTP Basic Auth for protecting `https://matrix.DOMAIN/_synapse/metrics` (the username is always `prometheus` - it's not configurable)
`matrix_server_fqn_grafana`|Use this variable to override the domain at which the Grafana web user-interface is at (defaults to `stats.DOMAIN`)
### Collecting worker metrics to an external Prometheus server
If you are using workers (`matrix_synapse_workers_enabled`) and have enabled `matrix_nginx_proxy_proxy_synapse_metrics` as described above, the playbook will also automatically proxy the all worker threads's metrics to `https://matrix.DOMAIN/_synapse-worker-TYPE-ID/metrics`, where `TYPE` corresponds to the type and `ID` to the instanceId of a worker as exemplified in `matrix_synapse_workers_enabled_list`.
The playbook also generates an exemplary prometheus.yml config file (`matrix_base_data_path/external_prometheus.yml.template`) with all the correct paths which you can copy to your Prometheus server and adapt to your needs, especially edit the specified `password_file` path and contents and path to your `synapse-v2.rules`.
It will look a bit like this:
```yaml
scrape_configs:
- job_name: 'synapse'
metrics_path: /_synapse/metrics
scheme: https
basic_auth:
username: prometheus
password_file: /etc/prometheus/password.pwd
static_configs:
- targets: ['matrix.DOMAIN:443']
labels:
job: "master"
index: 1
- job_name: 'synapse-generic_worker-1'
metrics_path: /_synapse-worker-generic_worker-18111/metrics
scheme: https
basic_auth:
username: prometheus
password_file: /etc/prometheus/password.pwd
static_configs:
- targets: ['matrix.DOMAIN:443']
labels:
job: "generic_worker"
index: 18111
```
### Collecting system and Postgres metrics to an external Prometheus server (advanced)
When you normally enable the Prometheus and Grafana via the playbook, it will also show general system (via node-exporter) and Postgres (via postgres-exporter) stats. If you are instead collecting your metrics to an external Prometheus server, you can follow this advanced configuration example to also export these stats.

View file

@ -1339,6 +1339,8 @@ matrix_nginx_proxy_synapse_media_repository_locations: "{{matrix_synapse_workers
matrix_nginx_proxy_synapse_user_dir_locations: "{{ matrix_synapse_workers_user_dir_endpoints|default([]) }}"
matrix_nginx_proxy_synapse_frontend_proxy_locations: "{{ matrix_synapse_workers_frontend_proxy_endpoints|default([]) }}"
matrix_nginx_proxy_proxy_synapse_workers_enabled_list: "{{ matrix_synapse_workers_enabled_list }}"
matrix_nginx_proxy_systemd_wanted_services_list: |
{{
(['matrix-synapse.service'] if matrix_synapse_enabled else [])

View file

@ -182,6 +182,7 @@ matrix_nginx_proxy_proxy_matrix_identity_api_addr_sans_container: "127.0.0.1:809
# Controls whether proxying for metrics (`/_synapse/metrics`) should be done (on the matrix domain)
matrix_nginx_proxy_proxy_synapse_metrics: false
matrix_nginx_proxy_synapse_workers_enabled_list: []
matrix_nginx_proxy_proxy_synapse_metrics_basic_auth_enabled: false
# The following value will be written verbatim to the htpasswd file that stores the password for nginx to check against and needs to be encoded appropriately.
# Read the manpage at `man 1 htpasswd` to learn more, then encrypt your password, and paste the encrypted value here.
@ -226,7 +227,7 @@ matrix_nginx_proxy_proxy_matrix_client_api_forwarded_location_prefix_regexes: |
+
(['/_synapse/admin'] if matrix_nginx_proxy_proxy_matrix_client_api_forwarded_location_synapse_admin_api_enabled else [])
+
(['/_synapse/metrics'] if matrix_nginx_proxy_proxy_synapse_metrics else [])
(['/_synapse.*/metrics'] if matrix_nginx_proxy_proxy_synapse_metrics else [])
}}
# Specifies where requests for the root URI (`/`) on the `matrix.` domain should be redirected.

View file

@ -38,6 +38,15 @@
mode: 0400
when: "matrix_nginx_proxy_proxy_synapse_metrics_basic_auth_enabled|bool and matrix_nginx_proxy_proxy_synapse_metrics|bool"
- name: Generate sample prometheus.yml for external scraping
template:
src: "{{ role_path }}/templates/prometheus/external_prometheus.yml.example.j2"
dest: "{{ matrix_base_data_path }}/external_prometheus.yml.example"
owner: "{{ matrix_user_username }}"
group: "{{ matrix_user_groupname }}"
mode: 0644
when: matrix_nginx_proxy_proxy_synapse_metrics|bool
- name: Ensure Matrix nginx-proxy configured (generic)
template:
src: "{{ role_path }}/templates/nginx/conf.d/nginx-http.conf.j2"
@ -270,3 +279,9 @@
path: "{{ matrix_nginx_proxy_data_path }}/matrix-synapse-metrics-htpasswd"
state: absent
when: "not matrix_nginx_proxy_proxy_synapse_metrics_basic_auth_enabled|bool or not matrix_nginx_proxy_proxy_synapse_metrics|bool"
- name: Ensure sample prometheus.yml for external scraping is deleted
file:
path: "{{ matrix_base_data_path }}/external_prometheus.yml.example"
state: absent
when: "not matrix_nginx_proxy_proxy_synapse_metrics|bool"

View file

@ -153,6 +153,24 @@ server {
}
{% endif %}
{% if matrix_nginx_proxy_enabled and matrix_nginx_proxy_proxy_synapse_metrics %}
{% for worker in matrix_nginx_proxy_proxy_synapse_workers_enabled_list %}
{% if worker.metrics_port != 0 %}
location /_synapse-worker-{{ worker.type }}-{{ worker.instanceId }}/metrics {
resolver 127.0.0.11 valid=5s;
set $backend "matrix-synapse-worker-{{ worker.type }}-{{ worker.instanceId }}:{{ worker.metrics_port }}";
proxy_pass http://$backend/_synapse/metrics;
proxy_set_header Host $host;
{% if matrix_nginx_proxy_proxy_synapse_metrics_basic_auth_enabled %}
auth_basic "protected";
auth_basic_user_file /nginx-data/matrix-synapse-metrics-htpasswd;
{% endif %}
}
{% endif %}
{% endfor %}
{% endif %}
{# Everything else just goes to the API server ##}
location / {
{% if matrix_nginx_proxy_enabled %}

View file

@ -0,0 +1,40 @@
global:
scrape_interval: 5s
# Attach these labels to any time series or alerts when communicating with
# external systems (federation, remote storage, Alertmanager).
external_labels:
monitor: 'synapse-{{ matrix_domain }}'
rule_files:
- /etc/prometheus/synapse-v2.rules
scrape_configs:
- job_name: 'synapse'
metrics_path: /_synapse/metrics
scheme: {{ 'https' if matrix_nginx_proxy_https_enabled else 'http' }}
{% if matrix_nginx_proxy_proxy_synapse_metrics_basic_auth_enabled %}
basic_auth:
username: prometheus
password_file: /path/to/your/passwordfile.pwd
{% endif %}
static_configs:
- targets: ['{{ matrix_server_fqn_matrix }}:{{ matrix_nginx_proxy_container_https_host_bind_port if matrix_nginx_proxy_https_enabled else matrix_nginx_proxy_container_http_host_bind_port }}']
labels:
job: "master"
index: 1
{% for worker in matrix_nginx_proxy_proxy_synapse_workers_enabled_list %}
- job_name: 'synapse-{{ worker.type }}-{{ worker.instanceId }}'
metrics_path: /_synapse-worker-{{ worker.type }}-{{ worker.instanceId }}/metrics
scheme: {{ 'https' if matrix_nginx_proxy_https_enabled else 'http' }}
{% if matrix_nginx_proxy_proxy_synapse_metrics_basic_auth_enabled %}
basic_auth:
username: prometheus
password_file: /path/to/your/passwordfile.pwd
{% endif %}
static_configs:
- targets: ['{{ matrix_server_fqn_matrix }}:{{ matrix_nginx_proxy_container_https_host_bind_port if matrix_nginx_proxy_https_enabled else matrix_nginx_proxy_container_http_host_bind_port }}']
labels:
job: "{{ worker.type }}"
index: {{ worker.instanceId }}
{% endfor %}