Add support for a prometheus postgres exporter

This commit introduces a new role that downloads and installs the
prometheus community postgres exporter  https://github.com/prometheus-community/postgres_exporter.
A new credential is added to matrix_postgres_additional_databases that
allows the exporter access to the database to gather statistics.
A new dashboard was added to the grafana role, with some refactoring
to enable the dashboard only if the new role is enabled.
I've included some basic instructions for how to enable the role in
the Docs section.

In terms of testing, I've tested enabling the role, and disabling
it to make sure it cleans up the container and systemd role.
This commit is contained in:
Raymond Coetzee 2021-05-27 20:13:29 +01:00
parent 1cb6a0c96f
commit 4e2780ff88
10 changed files with 236 additions and 3 deletions

View file

@ -0,0 +1,29 @@
# Enabling metrics and graphs for Postgres (optional)
Expanding on the metrics exposed by the [syapse exporter and the node exporter](configuring-playbook-prometheus-grafana.md), the playbook enables the [postgres exporter](https://github.com/prometheus-community/postgres_exporter) that exposes more detailed information about what's happening on your postgres database.
You can enable this with the following settings in your configuration file (`inventory/host_vars/matrix.<your-domain>/vars.yml`):
```yaml
matrix_prometheus_postgres_exporter_enabled: true
# the role creates a postgres user as credential. You can configure these if required:
matrix_prometheus_postgres_exporter_database_username: 'matrix_prometheus_postgres_exporter'
matrix_prometheus_postgres_exporter_database_password: 'some-password'
```
## What does it do?
Name | Description
-----|----------
`matrix_prometheus_postgres_exporter_enabled`|Enable the postgres prometheus exporter. This sets up the docker container, connects it to the database and adds a 'job' to the prometheus config which tells prometheus about this new exporter. The default is 'false'
`matrix_prometheus_postgres_exporter_database_username`| The 'username' for the user that the exporter uses to connect to the database. The default is 'matrix_prometheus_postgres_exporter'
`matrix_prometheus_postgres_exporter_database_password`| The 'username' for the user that the exporter uses to connect to the database.
## More information
- [The PostgresSQL dashboard](https://grafana.com/grafana/dashboards/9628) (generic postgres dashboard)

View file

@ -1466,6 +1466,13 @@ matrix_postgres_additional_databases: |
'username': matrix_sygnal_database_username,
'password': matrix_sygnal_database_password,
}] if (matrix_sygnal_enabled and matrix_sygnal_database_engine == 'postgres' and matrix_sygnal_database_hostname == 'matrix-postgres') else [])
+
([{
'name': matrix_prometheus_postgres_exporter_database_name,
'username': matrix_prometheus_postgres_exporter_database_username,
'password': matrix_prometheus_postgres_exporter_database_password,
}] if (matrix_prometheus_postgres_exporter_enabled and matrix_prometheus_postgres_exporter_database_hostname == 'matrix-postgres') else [])
}}
matrix_postgres_import_roles_to_ignore: |
@ -1766,6 +1773,10 @@ matrix_prometheus_scraper_synapse_rules_synapse_tag: "{{ matrix_synapse_docker_i
matrix_prometheus_scraper_node_enabled: "{{ matrix_prometheus_node_exporter_enabled }}"
matrix_prometheus_scraper_node_targets: "{{ ['matrix-prometheus-node-exporter:9100'] if matrix_prometheus_node_exporter_enabled else [] }}"
matrix_prometheus_scraper_postgres_enabled: "{{ matrix_prometheus_postgres_exporter_enabled }}"
matrix_prometheus_scraper_postgres_targets: "{{ ['matrix-prometheus-postgres-exporter:'+ matrix_prometheus_postgres_exporter_port|string] if matrix_prometheus_scraper_postgres_enabled else [] }}"
######################################################################
#
# /matrix-prometheus
@ -1773,6 +1784,27 @@ matrix_prometheus_scraper_node_targets: "{{ ['matrix-prometheus-node-exporter:91
######################################################################
######################################################################
#
# matrix-prometheus-postgres-exporter
#
######################################################################
matrix_prometheus_postgres_exporter_enabled: false
matrix_prometheus_postgres_exporter_database_password: "{{ matrix_synapse_macaroon_secret_key | password_hash('sha512', 'prometheus.pg.db') | to_uuid }}"
matrix_prometheus_postgres_exporter_systemd_required_services_list: |
{{
['docker.service']
+
(['matrix-postgres.service'] if matrix_postgres_enabled else [])
}}
######################################################################
#
# /matrix-prometheus-postgres-exporter
#
######################################################################
######################################################################
#
@ -1787,6 +1819,14 @@ matrix_grafana_enabled: false
# Grafana's HTTP port to the local host.
matrix_grafana_container_http_host_bind_port: "{{ '' if matrix_nginx_proxy_enabled else '127.0.0.1:3000' }}"
matrix_grafana_dashboard_download_urls_all: |
{{
matrix_grafana_dashboard_download_urls
+
(matrix_prometheus_postgres_exporter_dashboard_urls if matrix_prometheus_postgres_exporter_enabled else [])
}}
######################################################################
#
# /matrix-grafana

View file

@ -64,7 +64,7 @@
mode: 0440
owner: "{{ matrix_user_username }}"
group: "{{ matrix_user_groupname }}"
with_items: "{{ matrix_grafana_dashboard_download_urls }}"
with_items: "{{ matrix_grafana_dashboard_download_urls_all }}"
when: matrix_grafana_enabled|bool
- name: Ensure matrix-grafana.service installed

View file

@ -0,0 +1,49 @@
# matrix-prometheus-postgres-exporter is an Prometheus exporter for postgres metrics
# See: https://github.com/prometheus-community/postgres_exporter
matrix_prometheus_postgres_exporter_enabled: false
matrix_prometheus_postgres_exporter_version: v0.9.0
matrix_prometheus_postgres_exporter_port: 9187
matrix_prometheus_postgres_exporter_docker_image: "quay.io/prometheuscommunity/postgres-exporter:{{ matrix_prometheus_postgres_exporter_version }}"
matrix_prometheus_postgres_exporter_docker_image_force_pull: "{{ matrix_prometheus_postgres_exporter_docker_image.endswith(':latest') }}"
# A list of extra arguments to pass to the container
matrix_prometheus_postgres_exporter_container_extra_arguments: ["-e PG_EXPORTER_AUTO_DISCOVER_DATABASES=true",
"-e PG_EXPORTER_WEB_LISTEN_ADDRESS=\":{{matrix_prometheus_postgres_exporter_port}}\"",
"-e DATA_SOURCE_NAME=\"postgresql://{{matrix_prometheus_postgres_exporter_database_username}}:{{matrix_prometheus_postgres_exporter_database_password}}@{{matrix_prometheus_postgres_exporter_database_hostname}}:5432/{{matrix_prometheus_postgres_exporter_database_name}}?sslmode=disable\"" ]
# List of systemd services that matrix-prometheus-postgres-exporter.service depends on
matrix_prometheus_postgres_exporter_systemd_required_services_list: ['docker.service']
# List of systemd services that matrix-prometheus-postgres-exporter.service wants
matrix_prometheus_postgres_exporter_systemd_wanted_services_list: []
# details for connecting to the database
matrix_prometheus_postgres_exporter_database_username: 'matrix_prometheus_postgres_exporter'
matrix_prometheus_postgres_exporter_database_password: 'some-password'
matrix_prometheus_postgres_exporter_database_hostname: 'matrix-postgres'
matrix_prometheus_postgres_exporter_database_port: 5432
matrix_prometheus_postgres_exporter_database_name: 'matrix_prometheus_postgres_exporter'
# Controls whether the matrix-prometheus container exposes its HTTP port (tcp/9100 in the container).
#
# Takes an "<ip>:<port>" value (e.g. "127.0.0.1:9100"), or empty string to not expose.
#
# Official recommendations are to run this container with `--net=host`,
# but we don't do that, since it:
# - likely exposes the metrics web server way too publicly (before applying https://github.com/spantaleev/matrix-docker-ansible-deploy/pull/1008)
# - or listens on a loopback interface only (--net=host and 127.0.0.1:9100), which is not reachable from another container (like `matrix-prometheus`)
#
# Using `--net=host` and binding to Docker's `matrix` bridge network may be a solution to both,
# but that's trickier to accomplish and won't necessarily work (hasn't been tested).
#
# Not using `--net=host` means that our network statistic reports are likely broken (inaccurate),
# because node-exporter can't see all interfaces, etc.
# For now, we'll live with that, until someone develops a better solution.
matrix_prometheus_postgres_exporter_container_http_host_bind_port: ''
matrix_prometheus_postgres_exporter_dashboard_urls:
- "https://grafana.com/api/dashboards/9628/revisions/7/download"

View file

@ -0,0 +1,5 @@
- set_fact:
matrix_systemd_services_list: "{{ matrix_systemd_services_list + ['matrix-prometheus-postgres-exporter.service'] }}"
when: matrix_prometheus_postgres_exporter_enabled|bool

View file

@ -0,0 +1,8 @@
- import_tasks: "{{ role_path }}/tasks/init.yml"
tags:
- always
- import_tasks: "{{ role_path }}/tasks/setup.yml"
tags:
- setup-all
- setup-prometheus-postgres-exporter

View file

@ -0,0 +1,54 @@
---
#
# Tasks related to setting up matrix-prometheus-postgres-exporter
#
- name: Ensure matrix-prometheus-postgres-exporter image is pulled
docker_image:
name: "{{ matrix_prometheus_postgres_exporter_docker_image }}"
source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}"
force_source: "{{ matrix_prometheus_postgres_exporter_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}"
force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_prometheus_postgres_exporter_docker_image_force_pull }}"
when: "matrix_prometheus_postgres_exporter_enabled|bool"
- name: Ensure matrix-prometheus-postgres-exporter.service installed
template:
src: "{{ role_path }}/templates/systemd/matrix-prometheus-postgres-exporter.service.j2"
dest: "{{ matrix_systemd_path }}/matrix-prometheus-postgres-exporter.service"
mode: 0644
register: matrix_prometheus_postgres_exporter_systemd_service_result
when: matrix_prometheus_postgres_exporter_enabled|bool
- name: Ensure systemd reloaded after matrix-prometheus.service installation
service:
daemon_reload: yes
when: "matrix_prometheus_postgres_exporter_enabled|bool and matrix_prometheus_postgres_exporter_systemd_service_result.changed"
#
# Tasks related to getting rid of matrix-prometheus-postgres-exporter (if it was previously enabled)
#
- name: Check existence of matrix-prometheus-postgres-exporter service
stat:
path: "{{ matrix_systemd_path }}/matrix-prometheus-postgres-exporter.service"
register: matrix_prometheus_postgres_exporter_service_stat
- name: Ensure matrix-prometheus-postgres-exporter is stopped
service:
name: matrix-prometheus-postgres-exporter
state: stopped
daemon_reload: yes
register: stopping_result
when: "not matrix_prometheus_postgres_exporter_enabled|bool and matrix_prometheus_postgres_exporter_service_stat.stat.exists"
- name: Ensure matrix-prometheus-postgres-exporter.service doesn't exist
file:
path: "{{ matrix_systemd_path }}/matrix-prometheus-postgres-exporter.service"
state: absent
when: "not matrix_prometheus_postgres_exporter_enabled|bool and matrix_prometheus_postgres_exporter_service_stat.stat.exists"
- name: Ensure systemd reloaded after matrix-prometheus-postgres-exporter.service removal
service:
daemon_reload: yes
when: "not matrix_prometheus_postgres_exporter_enabled|bool and matrix_prometheus_postgres_exporter_service_stat.stat.exists"

View file

@ -0,0 +1,42 @@
#jinja2: lstrip_blocks: "True"
[Unit]
Description=matrix-prometheus-postgres-exporter
{% for service in matrix_prometheus_postgres_exporter_systemd_required_services_list %}
Requires={{ service }}
After={{ service }}
{% endfor %}
{% for service in matrix_prometheus_postgres_exporter_systemd_wanted_services_list %}
Wants={{ service }}
{% endfor %}
DefaultDependencies=no
[Service]
Type=simple
Environment="HOME={{ matrix_systemd_unit_home_path }}"
ExecStartPre=-{{ matrix_host_command_sh }} -c '{{ matrix_host_command_docker }} kill matrix-prometheus-postgres-exporter 2>/dev/null'
ExecStartPre=-{{ matrix_host_command_sh }} -c '{{ matrix_host_command_docker }} rm matrix-prometheus-postgres-exporter 2>/dev/null'
ExecStart={{ matrix_host_command_docker }} run --rm --name matrix-prometheus-postgres-exporter \
--log-driver=none \
--user={{ matrix_user_uid }}:{{ matrix_user_gid }} \
--cap-drop=ALL \
--read-only \
{% for arg in matrix_prometheus_postgres_exporter_container_extra_arguments %}
{{ arg }} \
{% endfor %}
--network={{ matrix_docker_network }} \
{% if matrix_prometheus_postgres_exporter_container_http_host_bind_port %}
-p {{ matrix_prometheus_postgres_exporter_container_http_host_bind_port }}:{{matrix_prometheus_postgres_exporter_port}} \
{% endif %}
--pid=host \
{{ matrix_prometheus_postgres_exporter_docker_image }}
ExecStop=-{{ matrix_host_command_sh }} -c '{{ matrix_host_command_docker }} kill matrix-prometheus-postgres-exporter 2>/dev/null'
ExecStop=-{{ matrix_host_command_sh }} -c '{{ matrix_host_command_docker }} rm matrix-prometheus-postgres-exporter 2>/dev/null'
Restart=always
RestartSec=30
SyslogIdentifier=matrix-prometheus-postgres-exporter
[Install]
WantedBy=multi-user.target

View file

@ -38,3 +38,9 @@ scrape_configs:
static_configs:
- targets: {{ matrix_prometheus_scraper_node_targets|to_json }}
{% endif %}
{% if matrix_prometheus_scraper_postgres_enabled %}
- job_name: postgres
static_configs:
- targets: {{ matrix_prometheus_scraper_postgres_targets|to_json }}
{% endif %}

View file

@ -54,5 +54,5 @@
- matrix-coturn
- matrix-aux
- matrix-postgres-backup
- matrix-prometheus-postgres-exporter
- matrix-common-after