Merge pull request #1311 from HarHarLinks/master

add auto proxy synapse worker metrics
This commit is contained in:
Slavi Pantaleev 2021-10-25 09:21:11 +03:00 committed by GitHub
commit 06bcdcf9d2
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 109 additions and 1 deletions

View file

@ -58,6 +58,38 @@ Name | Description
`matrix_nginx_proxy_proxy_synapse_metrics_basic_auth_key`|Set this to a password to use for HTTP Basic Auth for protecting `https://matrix.DOMAIN/_synapse/metrics` (the username is always `prometheus` - it's not configurable) `matrix_nginx_proxy_proxy_synapse_metrics_basic_auth_key`|Set this to a password to use for HTTP Basic Auth for protecting `https://matrix.DOMAIN/_synapse/metrics` (the username is always `prometheus` - it's not configurable)
`matrix_server_fqn_grafana`|Use this variable to override the domain at which the Grafana web user-interface is at (defaults to `stats.DOMAIN`) `matrix_server_fqn_grafana`|Use this variable to override the domain at which the Grafana web user-interface is at (defaults to `stats.DOMAIN`)
### Collecting worker metrics to an external Prometheus server
If you are using workers (`matrix_synapse_workers_enabled`) and have enabled `matrix_nginx_proxy_proxy_synapse_metrics` as described above, the playbook will also automatically proxy the all worker threads's metrics to `https://matrix.DOMAIN/_synapse-worker-TYPE-ID/metrics`, where `TYPE` corresponds to the type and `ID` to the instanceId of a worker as exemplified in `matrix_synapse_workers_enabled_list`.
The playbook also generates an exemplary prometheus.yml config file (`matrix_base_data_path/external_prometheus.yml.template`) with all the correct paths which you can copy to your Prometheus server and adapt to your needs, especially edit the specified `password_file` path and contents and path to your `synapse-v2.rules`.
It will look a bit like this:
```yaml
scrape_configs:
- job_name: 'synapse'
metrics_path: /_synapse/metrics
scheme: https
basic_auth:
username: prometheus
password_file: /etc/prometheus/password.pwd
static_configs:
- targets: ['matrix.DOMAIN:443']
labels:
job: "master"
index: 1
- job_name: 'synapse-generic_worker-1'
metrics_path: /_synapse-worker-generic_worker-18111/metrics
scheme: https
basic_auth:
username: prometheus
password_file: /etc/prometheus/password.pwd
static_configs:
- targets: ['matrix.DOMAIN:443']
labels:
job: "generic_worker"
index: 18111
```
### Collecting system and Postgres metrics to an external Prometheus server (advanced) ### Collecting system and Postgres metrics to an external Prometheus server (advanced)
When you normally enable the Prometheus and Grafana via the playbook, it will also show general system (via node-exporter) and Postgres (via postgres-exporter) stats. If you are instead collecting your metrics to an external Prometheus server, you can follow this advanced configuration example to also export these stats. When you normally enable the Prometheus and Grafana via the playbook, it will also show general system (via node-exporter) and Postgres (via postgres-exporter) stats. If you are instead collecting your metrics to an external Prometheus server, you can follow this advanced configuration example to also export these stats.

View file

@ -1339,6 +1339,8 @@ matrix_nginx_proxy_synapse_media_repository_locations: "{{matrix_synapse_workers
matrix_nginx_proxy_synapse_user_dir_locations: "{{ matrix_synapse_workers_user_dir_endpoints|default([]) }}" matrix_nginx_proxy_synapse_user_dir_locations: "{{ matrix_synapse_workers_user_dir_endpoints|default([]) }}"
matrix_nginx_proxy_synapse_frontend_proxy_locations: "{{ matrix_synapse_workers_frontend_proxy_endpoints|default([]) }}" matrix_nginx_proxy_synapse_frontend_proxy_locations: "{{ matrix_synapse_workers_frontend_proxy_endpoints|default([]) }}"
matrix_nginx_proxy_proxy_synapse_workers_enabled_list: "{{ matrix_synapse_workers_enabled_list }}"
matrix_nginx_proxy_systemd_wanted_services_list: | matrix_nginx_proxy_systemd_wanted_services_list: |
{{ {{
(['matrix-synapse.service'] if matrix_synapse_enabled else []) (['matrix-synapse.service'] if matrix_synapse_enabled else [])

View file

@ -182,6 +182,7 @@ matrix_nginx_proxy_proxy_matrix_identity_api_addr_sans_container: "127.0.0.1:809
# Controls whether proxying for metrics (`/_synapse/metrics`) should be done (on the matrix domain) # Controls whether proxying for metrics (`/_synapse/metrics`) should be done (on the matrix domain)
matrix_nginx_proxy_proxy_synapse_metrics: false matrix_nginx_proxy_proxy_synapse_metrics: false
matrix_nginx_proxy_synapse_workers_enabled_list: []
matrix_nginx_proxy_proxy_synapse_metrics_basic_auth_enabled: false matrix_nginx_proxy_proxy_synapse_metrics_basic_auth_enabled: false
# The following value will be written verbatim to the htpasswd file that stores the password for nginx to check against and needs to be encoded appropriately. # The following value will be written verbatim to the htpasswd file that stores the password for nginx to check against and needs to be encoded appropriately.
# Read the manpage at `man 1 htpasswd` to learn more, then encrypt your password, and paste the encrypted value here. # Read the manpage at `man 1 htpasswd` to learn more, then encrypt your password, and paste the encrypted value here.
@ -226,7 +227,7 @@ matrix_nginx_proxy_proxy_matrix_client_api_forwarded_location_prefix_regexes: |
+ +
(['/_synapse/admin'] if matrix_nginx_proxy_proxy_matrix_client_api_forwarded_location_synapse_admin_api_enabled else []) (['/_synapse/admin'] if matrix_nginx_proxy_proxy_matrix_client_api_forwarded_location_synapse_admin_api_enabled else [])
+ +
(['/_synapse/metrics'] if matrix_nginx_proxy_proxy_synapse_metrics else []) (['/_synapse.*/metrics'] if matrix_nginx_proxy_proxy_synapse_metrics else [])
}} }}
# Specifies where requests for the root URI (`/`) on the `matrix.` domain should be redirected. # Specifies where requests for the root URI (`/`) on the `matrix.` domain should be redirected.

View file

@ -38,6 +38,15 @@
mode: 0400 mode: 0400
when: "matrix_nginx_proxy_proxy_synapse_metrics_basic_auth_enabled|bool and matrix_nginx_proxy_proxy_synapse_metrics|bool" when: "matrix_nginx_proxy_proxy_synapse_metrics_basic_auth_enabled|bool and matrix_nginx_proxy_proxy_synapse_metrics|bool"
- name: Generate sample prometheus.yml for external scraping
template:
src: "{{ role_path }}/templates/prometheus/external_prometheus.yml.example.j2"
dest: "{{ matrix_base_data_path }}/external_prometheus.yml.example"
owner: "{{ matrix_user_username }}"
group: "{{ matrix_user_groupname }}"
mode: 0644
when: matrix_nginx_proxy_proxy_synapse_metrics|bool
- name: Ensure Matrix nginx-proxy configured (generic) - name: Ensure Matrix nginx-proxy configured (generic)
template: template:
src: "{{ role_path }}/templates/nginx/conf.d/nginx-http.conf.j2" src: "{{ role_path }}/templates/nginx/conf.d/nginx-http.conf.j2"
@ -270,3 +279,9 @@
path: "{{ matrix_nginx_proxy_data_path }}/matrix-synapse-metrics-htpasswd" path: "{{ matrix_nginx_proxy_data_path }}/matrix-synapse-metrics-htpasswd"
state: absent state: absent
when: "not matrix_nginx_proxy_proxy_synapse_metrics_basic_auth_enabled|bool or not matrix_nginx_proxy_proxy_synapse_metrics|bool" when: "not matrix_nginx_proxy_proxy_synapse_metrics_basic_auth_enabled|bool or not matrix_nginx_proxy_proxy_synapse_metrics|bool"
- name: Ensure sample prometheus.yml for external scraping is deleted
file:
path: "{{ matrix_base_data_path }}/external_prometheus.yml.example"
state: absent
when: "not matrix_nginx_proxy_proxy_synapse_metrics|bool"

View file

@ -153,6 +153,24 @@ server {
} }
{% endif %} {% endif %}
{% if matrix_nginx_proxy_enabled and matrix_nginx_proxy_proxy_synapse_metrics %}
{% for worker in matrix_nginx_proxy_proxy_synapse_workers_enabled_list %}
{% if worker.metrics_port != 0 %}
location /_synapse-worker-{{ worker.type }}-{{ worker.instanceId }}/metrics {
resolver 127.0.0.11 valid=5s;
set $backend "matrix-synapse-worker-{{ worker.type }}-{{ worker.instanceId }}:{{ worker.metrics_port }}";
proxy_pass http://$backend/_synapse/metrics;
proxy_set_header Host $host;
{% if matrix_nginx_proxy_proxy_synapse_metrics_basic_auth_enabled %}
auth_basic "protected";
auth_basic_user_file /nginx-data/matrix-synapse-metrics-htpasswd;
{% endif %}
}
{% endif %}
{% endfor %}
{% endif %}
{# Everything else just goes to the API server ##} {# Everything else just goes to the API server ##}
location / { location / {
{% if matrix_nginx_proxy_enabled %} {% if matrix_nginx_proxy_enabled %}

View file

@ -0,0 +1,40 @@
global:
scrape_interval: 5s
# Attach these labels to any time series or alerts when communicating with
# external systems (federation, remote storage, Alertmanager).
external_labels:
monitor: 'synapse-{{ matrix_domain }}'
rule_files:
- /etc/prometheus/synapse-v2.rules
scrape_configs:
- job_name: 'synapse'
metrics_path: /_synapse/metrics
scheme: {{ 'https' if matrix_nginx_proxy_https_enabled else 'http' }}
{% if matrix_nginx_proxy_proxy_synapse_metrics_basic_auth_enabled %}
basic_auth:
username: prometheus
password_file: /path/to/your/passwordfile.pwd
{% endif %}
static_configs:
- targets: ['{{ matrix_server_fqn_matrix }}:{{ matrix_nginx_proxy_container_https_host_bind_port if matrix_nginx_proxy_https_enabled else matrix_nginx_proxy_container_http_host_bind_port }}']
labels:
job: "master"
index: 1
{% for worker in matrix_nginx_proxy_proxy_synapse_workers_enabled_list %}
- job_name: 'synapse-{{ worker.type }}-{{ worker.instanceId }}'
metrics_path: /_synapse-worker-{{ worker.type }}-{{ worker.instanceId }}/metrics
scheme: {{ 'https' if matrix_nginx_proxy_https_enabled else 'http' }}
{% if matrix_nginx_proxy_proxy_synapse_metrics_basic_auth_enabled %}
basic_auth:
username: prometheus
password_file: /path/to/your/passwordfile.pwd
{% endif %}
static_configs:
- targets: ['{{ matrix_server_fqn_matrix }}:{{ matrix_nginx_proxy_container_https_host_bind_port if matrix_nginx_proxy_https_enabled else matrix_nginx_proxy_container_http_host_bind_port }}']
labels:
job: "{{ worker.type }}"
index: {{ worker.instanceId }}
{% endfor %}