From 87e3deebfd59143d3255d3aadd88811f8267145f Mon Sep 17 00:00:00 2001
From: dhose <4066579+dhose@users.noreply.github.com>
Date: Wed, 30 Jan 2019 19:31:50 +0100
Subject: [PATCH] Enable exposure of Prometheus metrics.

---
 group_vars/matrix-servers                     |  6 +++++
 roles/matrix-nginx-proxy/defaults/main.yml    |  5 +++++
 .../tasks/setup_nginx_proxy.yml               | 15 +++++++++++++
 .../nginx/conf.d/matrix-synapse.conf.j2       | 22 +++++++++++++++++++
 .../nginx/matrix-synapse-metrics-htpasswd.j2  |  2 ++
 .../systemd/matrix-nginx-proxy.service.j2     |  3 +++
 roles/matrix-synapse/defaults/main.yml        |  8 +++++++
 .../templates/synapse/homeserver.yaml.j2      |  8 ++++++-
 .../synapse/systemd/matrix-synapse.service.j2 |  3 +++
 9 files changed, 71 insertions(+), 1 deletion(-)
 create mode 100644 roles/matrix-nginx-proxy/templates/nginx/matrix-synapse-metrics-htpasswd.j2

diff --git a/group_vars/matrix-servers b/group_vars/matrix-servers
index 30e55131..9f1d37de 100644
--- a/group_vars/matrix-servers
+++ b/group_vars/matrix-servers
@@ -181,6 +181,10 @@ matrix_nginx_proxy_proxy_matrix_identity_api_enabled: "{{ matrix_mxisd_enabled }
 matrix_nginx_proxy_proxy_matrix_identity_api_addr_with_container: "matrix-mxisd:8090"
 matrix_nginx_proxy_proxy_matrix_identity_api_addr_sans_container: "localhost:8090"
 
+matrix_nginx_proxy_proxy_synapse_metrics: "{{ matrix_synapse_metrics_enabled }}"
+matrix_nginx_proxy_proxy_synapse_metrics_addr_with_container: "matrix-synapse:{{ matrix_synapse_metrics_port }}"
+matrix_nginx_proxy_proxy_synapse_metrics_addr_sans_container: "localhost:{{ matrix_synapse_metrics_port }}"
+
 matrix_nginx_proxy_proxy_matrix_user_directory_search_enabled: "{{ matrix_mxisd_enabled }}"
 matrix_nginx_proxy_proxy_matrix_user_directory_search_addr_with_container: "{{ matrix_nginx_proxy_proxy_matrix_identity_api_addr_with_container }}"
 matrix_nginx_proxy_proxy_matrix_user_directory_search_addr_sans_container: "{{ matrix_nginx_proxy_proxy_matrix_identity_api_addr_sans_container }}"
@@ -272,6 +276,8 @@ matrix_synapse_trusted_third_party_id_servers: "{{ [hostname_matrix] if matrix_m
 # the Client/Server API's port to the local host (`127.0.0.1:8008`).
 matrix_synapse_container_expose_client_server_api_port: "{{ not matrix_nginx_proxy_enabled }}"
 
+matrix_synapse_container_expose_metrics_port: "{{ not matrix_nginx_proxy_enabled }}"
+
 matrix_synapse_database_host: "{{ matrix_postgres_connection_hostname }}"
 matrix_synapse_database_user: "{{ matrix_postgres_connection_username }}"
 matrix_synapse_database_password: "{{ matrix_postgres_connection_password }}"
diff --git a/roles/matrix-nginx-proxy/defaults/main.yml b/roles/matrix-nginx-proxy/defaults/main.yml
index 0bf7955b..7ae4bd00 100644
--- a/roles/matrix-nginx-proxy/defaults/main.yml
+++ b/roles/matrix-nginx-proxy/defaults/main.yml
@@ -39,6 +39,11 @@ matrix_nginx_proxy_proxy_matrix_identity_api_enabled: false
 matrix_nginx_proxy_proxy_matrix_identity_api_addr_with_container: "matrix-mxisd:8090"
 matrix_nginx_proxy_proxy_matrix_identity_api_addr_sans_container: "localhost:8090"
 
+# Controls whether proxying for metrics (`/_synapse/metrics`) should be done (on the matrix domain)
+matrix_nginx_proxy_proxy_synapse_metrics: false
+matrix_nginx_proxy_proxy_synapse_metrics_basic_auth_enabled: false
+matrix_nginx_proxy_proxy_synapse_metrics_basic_auth_key: ""
+
 # The addresses where the Matrix Client API is.
 # Certain extensions (like matrix-corporal) may override this in order to capture all traffic.
 matrix_nginx_proxy_proxy_matrix_client_api_addr_with_container: "matrix-synapse:8008"
diff --git a/roles/matrix-nginx-proxy/tasks/setup_nginx_proxy.yml b/roles/matrix-nginx-proxy/tasks/setup_nginx_proxy.yml
index 434cfad8..97530ec2 100644
--- a/roles/matrix-nginx-proxy/tasks/setup_nginx_proxy.yml
+++ b/roles/matrix-nginx-proxy/tasks/setup_nginx_proxy.yml
@@ -28,6 +28,15 @@
     mode: 0644
   when: "matrix_nginx_proxy_enabled"
 
+- name: Ensure matrix-synapse-metrics-htpasswd is present (protecting /_synapse/metrics URI)
+  template:
+    src: "{{ role_path }}/templates/nginx/matrix-synapse-metrics-htpasswd.j2"
+    dest: "{{ matrix_nginx_proxy_data_path }}/matrix-synapse-metrics-htpasswd"
+    owner: "{{ matrix_user_username }}"
+    group: "{{ matrix_user_username }}"
+    mode: 0400
+  when: "matrix_nginx_proxy_proxy_synapse_metrics_basic_auth_enabled and matrix_nginx_proxy_proxy_synapse_metrics"
+
 - name: Ensure Matrix nginx-proxy configured (generic)
   template:
     src: "{{ role_path }}/templates/nginx/conf.d/nginx-http.conf.j2"
@@ -116,3 +125,9 @@
     path: "{{ matrix_nginx_proxy_data_path }}/nginx.conf"
     state: absent
   when: "not matrix_nginx_proxy_enabled"
+
+- name: Ensure Matrix nginx-proxy htpasswd is deleted (protecting /_synapse/metrics URI)
+  file:
+    path: "{{ matrix_nginx_proxy_data_path }}/matrix-synapse-metrics-htpasswd"
+    state: absent
+  when: "not matrix_nginx_proxy_proxy_synapse_metrics_basic_auth_enabled or not matrix_nginx_proxy_proxy_synapse_metrics"
diff --git a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2 b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2
index 016b9414..734ddb5b 100644
--- a/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2
+++ b/roles/matrix-nginx-proxy/templates/nginx/conf.d/matrix-synapse.conf.j2
@@ -120,6 +120,28 @@ server {
 		proxy_max_temp_file_size 0;
 	}
 
+	{% if matrix_nginx_proxy_proxy_synapse_metrics %}
+	location /_synapse/metrics {
+		{% if matrix_nginx_proxy_enabled %}
+			{# Use the embedded DNS resolver in Docker containers to discover the service #}
+			resolver 127.0.0.11 valid=5s;
+			set $backend "{{ matrix_nginx_proxy_proxy_synapse_metrics_addr_with_container }}";
+			proxy_pass http://$backend;
+		{% else %}
+			{# Generic configuration for use outside of our container setup #}
+			proxy_pass http://{{ matrix_nginx_proxy_proxy_synapse_metrics_addr_sans_container }};
+		{% endif %}
+
+		proxy_set_header Host $host;
+		proxy_set_header X-Forwarded-For $remote_addr;
+
+		{% if matrix_nginx_proxy_proxy_synapse_metrics_basic_auth_enabled %}
+			auth_basic "protected";
+			auth_basic_user_file .matrix-synapse-metrics-htpasswd;
+		{% endif %}
+	}
+	{% endif %}
+
 	location / {
 		rewrite ^/$ /_matrix/static/ last;
 	}
diff --git a/roles/matrix-nginx-proxy/templates/nginx/matrix-synapse-metrics-htpasswd.j2 b/roles/matrix-nginx-proxy/templates/nginx/matrix-synapse-metrics-htpasswd.j2
new file mode 100644
index 00000000..27cf298f
--- /dev/null
+++ b/roles/matrix-nginx-proxy/templates/nginx/matrix-synapse-metrics-htpasswd.j2
@@ -0,0 +1,2 @@
+# User and password for protecting /_synapse/metrics URI
+prometheus:{{ matrix_nginx_proxy_proxy_synapse_metrics_basic_auth_key }}
diff --git a/roles/matrix-nginx-proxy/templates/systemd/matrix-nginx-proxy.service.j2 b/roles/matrix-nginx-proxy/templates/systemd/matrix-nginx-proxy.service.j2
index 410d6a01..7f05ed26 100644
--- a/roles/matrix-nginx-proxy/templates/systemd/matrix-nginx-proxy.service.j2
+++ b/roles/matrix-nginx-proxy/templates/systemd/matrix-nginx-proxy.service.j2
@@ -26,6 +26,9 @@ ExecStart=/usr/bin/docker run --rm --name matrix-nginx-proxy \
 			-v {{ matrix_nginx_proxy_confd_path }}:/etc/nginx/conf.d:ro \
 			-v {{ matrix_ssl_config_dir_path }}:{{ matrix_ssl_config_dir_path }}:ro \
 			-v {{ matrix_static_files_base_path }}:{{ matrix_static_files_base_path }}:ro \
+			{% if (matrix_nginx_proxy_proxy_synapse_metrics and matrix_nginx_proxy_proxy_synapse_metrics_basic_auth_enabled) %}
+			-v {{ matrix_nginx_proxy_data_path }}/matrix-synapse-metrics-htpasswd:/etc/nginx/.matrix-synapse-metrics-htpasswd:ro \
+			{% endif %}
 			{{ matrix_nginx_proxy_docker_image }}
 
 ExecStop=-/usr/bin/docker kill matrix-nginx-proxy
diff --git a/roles/matrix-synapse/defaults/main.yml b/roles/matrix-synapse/defaults/main.yml
index ba737dc6..13273f78 100644
--- a/roles/matrix-synapse/defaults/main.yml
+++ b/roles/matrix-synapse/defaults/main.yml
@@ -13,6 +13,9 @@ matrix_synapse_container_expose_client_server_api_port: false
 # Controls whether the Appservice IRC container exposes the Client/Server API port (tcp/9999).
 matrix_appservice_irc_container_expose_client_server_api_port: false
 
+# Controls whether the matrix-synapse container exposes the metrics port (tcp/9100).
+matrix_synapse_container_expose_metrics_port: false
+
 # List of systemd services that matrix-synapse.service depends on
 matrix_synapse_systemd_required_services_list: ['docker.service']
 
@@ -118,6 +121,11 @@ matrix_synapse_app_service_config_files: []
 # any password providers have been enabled or not.
 matrix_synapse_password_providers_enabled: false
 
+# Enable exposure of metrics to Prometheus
+# See https://github.com/matrix-org/synapse/blob/master/docs/metrics-howto.rst
+matrix_synapse_metrics_enabled: false
+matrix_synapse_metrics_port: 9100
+
 # Postgres database information
 matrix_synapse_database_host: ""
 matrix_synapse_database_user: ""
diff --git a/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2 b/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2
index a123f319..e6537ef0 100644
--- a/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2
+++ b/roles/matrix-synapse/templates/synapse/homeserver.yaml.j2
@@ -127,6 +127,12 @@ federation_domain_whitelist: {{ matrix_synapse_federation_domain_whitelist|to_js
 # List of ports that Synapse should listen on, their purpose and their
 # configuration.
 listeners:
+{% if matrix_synapse_metrics_enabled %}
+  - type: metrics
+    port: {{ matrix_synapse_metrics_port }}
+    bind_addresses:
+      - '0.0.0.0'
+{% endif %}
 {% if matrix_synapse_federation_enabled %}
   # Main HTTPS listener
   # For when matrix traffic is sent directly to synapse.
@@ -496,7 +502,7 @@ autocreate_auto_join_rooms: {{ matrix_synapse_autocreate_auto_join_rooms }}
 ## Metrics ###
 
 # Enable collection and rendering of performance metrics
-enable_metrics: False
+enable_metrics: {{ matrix_synapse_metrics_enabled }}
 report_stats: {{ matrix_synapse_report_stats|to_json }}
 
 
diff --git a/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse.service.j2 b/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse.service.j2
index a369190d..9ba918d6 100644
--- a/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse.service.j2
+++ b/roles/matrix-synapse/templates/synapse/systemd/matrix-synapse.service.j2
@@ -34,6 +34,9 @@ ExecStart=/usr/bin/docker run --rm --name matrix-synapse \
 			{% if matrix_synapse_container_expose_client_server_api_port %}
 			-p 127.0.0.1:8008:8008 \
 			{% endif %}
+			{% if matrix_synapse_container_expose_metrics_port %}
+			-p 127.0.0.1:{{ matrix_synapse_metrics_port }}:{{ matrix_synapse_metrics_port }} \
+			{% endif %}
 			-v {{ matrix_synapse_config_dir_path }}:/data:ro \
 			-v {{ matrix_synapse_run_path }}:/matrix-run:rw \
 			-v {{ matrix_synapse_base_path }}/storage:/matrix-media-store-parent:slave \