Add support for a prometheus postgres exporter

This commit introduces a new role that downloads and installs the
prometheus community postgres exporter  https://github.com/prometheus-community/postgres_exporter.
A new credential is added to matrix_postgres_additional_databases that
allows the exporter access to the database to gather statistics.
A new dashboard was added to the grafana role, with some refactoring
to enable the dashboard only if the new role is enabled.
I've included some basic instructions for how to enable the role in
the Docs section.

In terms of testing, I've tested enabling the role, and disabling
it to make sure it cleans up the container and systemd role.

docs/configuring-playbook-prometheus-postgres.md

@@ -0,0 +1,29 @@
+# Enabling metrics and graphs for Postgres (optional)
+
+Expanding on the metrics exposed by the [syapse exporter and the node exporter](configuring-playbook-prometheus-grafana.md), the playbook enables the [postgres exporter](https://github.com/prometheus-community/postgres_exporter) that exposes more detailed information about what's happening on your postgres database.
+
+You can enable this with the following settings in your configuration file (`inventory/host_vars/matrix.<your-domain>/vars.yml`):
+
+
+```yaml
+matrix_prometheus_postgres_exporter_enabled: true
+
+# the role creates a postgres user as credential. You can configure these if required:
+matrix_prometheus_postgres_exporter_database_username: 'matrix_prometheus_postgres_exporter'
+matrix_prometheus_postgres_exporter_database_password: 'some-password'
+
+```
+
+## What does it do?
+
+Name | Description
+-----|----------
+`matrix_prometheus_postgres_exporter_enabled`|Enable the postgres prometheus exporter. This sets up the docker container, connects it to the database and adds a 'job' to the prometheus config which tells prometheus about this new exporter. The default is 'false'
+`matrix_prometheus_postgres_exporter_database_username`| The 'username' for the user that the exporter uses to connect to the database. The default is 'matrix_prometheus_postgres_exporter'
+`matrix_prometheus_postgres_exporter_database_password`| The 'username' for the user that the exporter uses to connect to the database.
+
+
+## More information
+
+- [The PostgresSQL dashboard](https://grafana.com/grafana/dashboards/9628) (generic postgres dashboard)
+

group_vars/matrix_servers

@@ -1466,6 +1466,13 @@ matrix_postgres_additional_databases: |
       'username': matrix_sygnal_database_username,
       'password': matrix_sygnal_database_password,
     }] if (matrix_sygnal_enabled and matrix_sygnal_database_engine == 'postgres' and matrix_sygnal_database_hostname == 'matrix-postgres') else [])
+     +
+    ([{
+      'name': matrix_prometheus_postgres_exporter_database_name,
+      'username': matrix_prometheus_postgres_exporter_database_username,
+      'password': matrix_prometheus_postgres_exporter_database_password,
+    }] if (matrix_prometheus_postgres_exporter_enabled and matrix_prometheus_postgres_exporter_database_hostname == 'matrix-postgres') else [])
+   
    }}
 
 matrix_postgres_import_roles_to_ignore: |
@@ -1766,6 +1773,10 @@ matrix_prometheus_scraper_synapse_rules_synapse_tag: "{{ matrix_synapse_docker_i
 matrix_prometheus_scraper_node_enabled: "{{ matrix_prometheus_node_exporter_enabled }}"
 matrix_prometheus_scraper_node_targets: "{{ ['matrix-prometheus-node-exporter:9100'] if matrix_prometheus_node_exporter_enabled else [] }}"
 
+matrix_prometheus_scraper_postgres_enabled: "{{ matrix_prometheus_postgres_exporter_enabled }}"
+matrix_prometheus_scraper_postgres_targets: "{{ ['matrix-prometheus-postgres-exporter:'+ matrix_prometheus_postgres_exporter_port|string] if matrix_prometheus_scraper_postgres_enabled else [] }}"
+
+
 ######################################################################
 #
 # /matrix-prometheus
@@ -1773,6 +1784,27 @@ matrix_prometheus_scraper_node_targets: "{{ ['matrix-prometheus-node-exporter:91
 ######################################################################
 
 
+######################################################################
+#
+# matrix-prometheus-postgres-exporter
+#
+######################################################################
+
+matrix_prometheus_postgres_exporter_enabled: false
+matrix_prometheus_postgres_exporter_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'prometheus.pg.db') | to_uuid }}"
+
+matrix_prometheus_postgres_exporter_systemd_required_services_list: |
+  {{
+    ['docker.service']
+    +
+    (['matrix-postgres.service'] if matrix_postgres_enabled else [])
+  }}
+
+######################################################################
+#
+# /matrix-prometheus-postgres-exporter
+#
+######################################################################
 
 ######################################################################
 #
@@ -1787,6 +1819,14 @@ matrix_grafana_enabled: false
 # Grafana's HTTP port to the local host.
 matrix_grafana_container_http_host_bind_port: "{{ '' if matrix_nginx_proxy_enabled else '127.0.0.1:3000' }}"
 
+matrix_grafana_dashboard_download_urls_all: |
+  {{
+    matrix_grafana_dashboard_download_urls
+    +
+    (matrix_prometheus_postgres_exporter_dashboard_urls if matrix_prometheus_postgres_exporter_enabled else [])
+  }}
+
+
 ######################################################################
 #
 # /matrix-grafana

roles/matrix-grafana/tasks/setup.yml

@@ -64,7 +64,7 @@
     mode: 0440
     owner: "{{ matrix_user_username }}"
     group: "{{ matrix_user_groupname }}"
-  with_items: "{{ matrix_grafana_dashboard_download_urls }}"
+  with_items: "{{ matrix_grafana_dashboard_download_urls_all }}"
   when: matrix_grafana_enabled|bool
 
 - name: Ensure matrix-grafana.service installed

roles/matrix-prometheus-postgres-exporter/defaults/main.yml

@@ -0,0 +1,49 @@
+# matrix-prometheus-postgres-exporter is an Prometheus exporter for postgres metrics
+# See: https://github.com/prometheus-community/postgres_exporter
+
+matrix_prometheus_postgres_exporter_enabled: false
+
+matrix_prometheus_postgres_exporter_version: v0.9.0
+matrix_prometheus_postgres_exporter_port: 9187
+
+matrix_prometheus_postgres_exporter_docker_image: "quay.io/prometheuscommunity/postgres-exporter:{{ matrix_prometheus_postgres_exporter_version }}"
+matrix_prometheus_postgres_exporter_docker_image_force_pull: "{{ matrix_prometheus_postgres_exporter_docker_image.endswith(':latest') }}"
+
+# A list of extra arguments to pass to the container
+matrix_prometheus_postgres_exporter_container_extra_arguments: ["-e PG_EXPORTER_AUTO_DISCOVER_DATABASES=true",
+ "-e PG_EXPORTER_WEB_LISTEN_ADDRESS=\":{{matrix_prometheus_postgres_exporter_port}}\"",
+ "-e DATA_SOURCE_NAME=\"postgresql://{{matrix_prometheus_postgres_exporter_database_username}}:{{matrix_prometheus_postgres_exporter_database_password}}@{{matrix_prometheus_postgres_exporter_database_hostname}}:5432/{{matrix_prometheus_postgres_exporter_database_name}}?sslmode=disable\"" ]
+
+# List of systemd services that matrix-prometheus-postgres-exporter.service depends on
+matrix_prometheus_postgres_exporter_systemd_required_services_list: ['docker.service']
+
+# List of systemd services that matrix-prometheus-postgres-exporter.service wants
+matrix_prometheus_postgres_exporter_systemd_wanted_services_list: []
+
+# details for connecting to the database
+matrix_prometheus_postgres_exporter_database_username: 'matrix_prometheus_postgres_exporter'
+matrix_prometheus_postgres_exporter_database_password: 'some-password'
+matrix_prometheus_postgres_exporter_database_hostname: 'matrix-postgres'
+matrix_prometheus_postgres_exporter_database_port: 5432
+matrix_prometheus_postgres_exporter_database_name: 'matrix_prometheus_postgres_exporter'
+
+
+# Controls whether the matrix-prometheus container exposes its HTTP port (tcp/9100 in the container).
+#
+# Takes an "<ip>:<port>" value (e.g. "127.0.0.1:9100"), or empty string to not expose.
+#
+# Official recommendations are to run this container with `--net=host`,
+# but we don't do that, since it:
+# - likely exposes the metrics web server way too publicly (before applying https://github.com/spantaleev/matrix-docker-ansible-deploy/pull/1008)
+# - or listens on a loopback interface only (--net=host and 127.0.0.1:9100), which is not reachable from another container (like `matrix-prometheus`)
+#
+# Using `--net=host` and binding to Docker's `matrix` bridge network may be a solution to both,
+# but that's trickier to accomplish and won't necessarily work (hasn't been tested).
+#
+# Not using `--net=host` means that our network statistic reports are likely broken (inaccurate),
+# because node-exporter can't see all interfaces, etc.
+# For now, we'll live with that, until someone develops a better solution.
+matrix_prometheus_postgres_exporter_container_http_host_bind_port: ''
+
+matrix_prometheus_postgres_exporter_dashboard_urls:
+- "https://grafana.com/api/dashboards/9628/revisions/7/download"

roles/matrix-prometheus-postgres-exporter/tasks/init.yml

@@ -0,0 +1,5 @@
+- set_fact:
+    matrix_systemd_services_list: "{{ matrix_systemd_services_list + ['matrix-prometheus-postgres-exporter.service'] }}"
+  when: matrix_prometheus_postgres_exporter_enabled|bool
+
+

roles/matrix-prometheus-postgres-exporter/tasks/main.yml

@@ -0,0 +1,8 @@
+- import_tasks: "{{ role_path }}/tasks/init.yml"
+  tags:
+    - always
+
+- import_tasks: "{{ role_path }}/tasks/setup.yml"
+  tags:
+    - setup-all
+    - setup-prometheus-postgres-exporter

roles/matrix-prometheus-postgres-exporter/tasks/setup.yml

@@ -0,0 +1,54 @@
+---
+
+#
+# Tasks related to setting up matrix-prometheus-postgres-exporter
+#
+
+- name: Ensure matrix-prometheus-postgres-exporter image is pulled
+  docker_image:
+    name: "{{ matrix_prometheus_postgres_exporter_docker_image }}"
+    source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}"
+    force_source: "{{ matrix_prometheus_postgres_exporter_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}"
+    force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_prometheus_postgres_exporter_docker_image_force_pull }}"
+  when: "matrix_prometheus_postgres_exporter_enabled|bool"
+
+- name: Ensure matrix-prometheus-postgres-exporter.service installed
+  template:
+    src: "{{ role_path }}/templates/systemd/matrix-prometheus-postgres-exporter.service.j2"
+    dest: "{{ matrix_systemd_path }}/matrix-prometheus-postgres-exporter.service"
+    mode: 0644
+  register: matrix_prometheus_postgres_exporter_systemd_service_result
+  when: matrix_prometheus_postgres_exporter_enabled|bool
+
+- name: Ensure systemd reloaded after matrix-prometheus.service installation
+  service:
+    daemon_reload: yes
+  when: "matrix_prometheus_postgres_exporter_enabled|bool and matrix_prometheus_postgres_exporter_systemd_service_result.changed"
+
+#
+# Tasks related to getting rid of matrix-prometheus-postgres-exporter (if it was previously enabled)
+#
+
+- name: Check existence of matrix-prometheus-postgres-exporter service
+  stat:
+    path: "{{ matrix_systemd_path }}/matrix-prometheus-postgres-exporter.service"
+  register: matrix_prometheus_postgres_exporter_service_stat
+
+- name: Ensure matrix-prometheus-postgres-exporter is stopped
+  service:
+    name: matrix-prometheus-postgres-exporter
+    state: stopped
+    daemon_reload: yes
+  register: stopping_result
+  when: "not matrix_prometheus_postgres_exporter_enabled|bool and matrix_prometheus_postgres_exporter_service_stat.stat.exists"
+
+- name: Ensure matrix-prometheus-postgres-exporter.service doesn't exist
+  file:
+    path: "{{ matrix_systemd_path }}/matrix-prometheus-postgres-exporter.service"
+    state: absent
+  when: "not matrix_prometheus_postgres_exporter_enabled|bool and matrix_prometheus_postgres_exporter_service_stat.stat.exists"
+
+- name: Ensure systemd reloaded after matrix-prometheus-postgres-exporter.service removal
+  service:
+    daemon_reload: yes
+  when: "not matrix_prometheus_postgres_exporter_enabled|bool and matrix_prometheus_postgres_exporter_service_stat.stat.exists"

roles/matrix-prometheus-postgres-exporter/templates/systemd/matrix-prometheus-postgres-exporter.service.j2

@@ -0,0 +1,42 @@
+#jinja2: lstrip_blocks: "True"
+[Unit]
+Description=matrix-prometheus-postgres-exporter
+{% for service in matrix_prometheus_postgres_exporter_systemd_required_services_list %}
+Requires={{ service }}
+After={{ service }}
+{% endfor %}
+{% for service in matrix_prometheus_postgres_exporter_systemd_wanted_services_list %}
+Wants={{ service }}
+{% endfor %}
+DefaultDependencies=no
+
+[Service]
+Type=simple
+Environment="HOME={{ matrix_systemd_unit_home_path }}"
+ExecStartPre=-{{ matrix_host_command_sh }} -c '{{ matrix_host_command_docker }} kill matrix-prometheus-postgres-exporter 2>/dev/null'
+ExecStartPre=-{{ matrix_host_command_sh }} -c '{{ matrix_host_command_docker }} rm matrix-prometheus-postgres-exporter 2>/dev/null'
+
+
+ExecStart={{ matrix_host_command_docker }} run --rm --name matrix-prometheus-postgres-exporter \
+			--log-driver=none \
+			--user={{ matrix_user_uid }}:{{ matrix_user_gid }} \
+			--cap-drop=ALL \
+			--read-only \
+			{% for arg in matrix_prometheus_postgres_exporter_container_extra_arguments %}
+			{{ arg }} \
+			{% endfor %}
+			--network={{ matrix_docker_network }} \
+			{% if matrix_prometheus_postgres_exporter_container_http_host_bind_port %}
+			-p {{ matrix_prometheus_postgres_exporter_container_http_host_bind_port }}:{{matrix_prometheus_postgres_exporter_port}} \
+			{% endif %}
+			--pid=host \
+			{{ matrix_prometheus_postgres_exporter_docker_image }} 
+
+ExecStop=-{{ matrix_host_command_sh }} -c '{{ matrix_host_command_docker }} kill matrix-prometheus-postgres-exporter 2>/dev/null'
+ExecStop=-{{ matrix_host_command_sh }} -c '{{ matrix_host_command_docker }} rm matrix-prometheus-postgres-exporter 2>/dev/null'
+Restart=always
+RestartSec=30
+SyslogIdentifier=matrix-prometheus-postgres-exporter
+
+[Install]
+WantedBy=multi-user.target

roles/matrix-prometheus/templates/prometheus.yml.j2

@@ -38,3 +38,9 @@ scrape_configs:
     static_configs:
       - targets: {{ matrix_prometheus_scraper_node_targets|to_json }}
   {% endif %}
+
+  {% if matrix_prometheus_scraper_postgres_enabled %}
+  - job_name: postgres
+    static_configs:
+      - targets: {{ matrix_prometheus_scraper_postgres_targets|to_json }}
+  {% endif %}

setup.yml

@@ -54,5 +54,5 @@
     - matrix-coturn
     - matrix-aux
     - matrix-postgres-backup
-    - matrix-common-after
-
+    - matrix-prometheus-postgres-exporter
+    - matrix-common-after