From e525970b393350f9280bdf8ddcb78f9d55c1da24 Mon Sep 17 00:00:00 2001 From: Peetz0r Date: Fri, 29 Jan 2021 06:17:45 +0100 Subject: [PATCH] Prometheus Node Exporter Basic system stats, to show stuff the synapse metrics can't show such as resource usage by bridges, etc Seems to work fine as well. This too has only been tested on debian amd64 so far --- group_vars/matrix_servers | 21 +++++++ .../defaults/main.yml | 21 +++++++ .../tasks/init.yml | 5 ++ .../tasks/main.yml | 8 +++ .../tasks/setup.yml | 60 +++++++++++++++++++ .../tasks/validate_config.yml | 7 +++ ...matrix-prometheus-node-exporter.service.j2 | 40 +++++++++++++ .../tasks/validate_config.yml | 6 +- .../templates/prometheus.yml.j2 | 5 ++ setup.yml | 1 + 10 files changed, 171 insertions(+), 3 deletions(-) create mode 100644 roles/matrix-prometheus-node-exporter/defaults/main.yml create mode 100644 roles/matrix-prometheus-node-exporter/tasks/init.yml create mode 100644 roles/matrix-prometheus-node-exporter/tasks/main.yml create mode 100644 roles/matrix-prometheus-node-exporter/tasks/setup.yml create mode 100644 roles/matrix-prometheus-node-exporter/tasks/validate_config.yml create mode 100644 roles/matrix-prometheus-node-exporter/templates/systemd/matrix-prometheus-node-exporter.service.j2 diff --git a/group_vars/matrix_servers b/group_vars/matrix_servers index 833089a2..4f2cfa6a 100755 --- a/group_vars/matrix_servers +++ b/group_vars/matrix_servers @@ -1368,6 +1368,27 @@ matrix_synapse_admin_container_self_build: "{{ matrix_architecture != 'amd64' }} +###################################################################### +# +# matrix-prometheus-node-exporter +# +###################################################################### + +matrix_prometheus_node_exporter_enabled: false + +# Normally, matrix-nginx-proxy is enabled and nginx can reach Prometheus Node Exporter over the container network. +# If matrix-nginx-proxy is not enabled, or you otherwise have a need for it, you can expose +# Prometheus' HTTP port to the local host. +matrix_prometheus_node_exporter_container_http_host_bind_port: "{{ '' if matrix_nginx_proxy_enabled else '127.0.0.1:9100' }}" + +###################################################################### +# +# /matrix-prometheus-node-exporter +# +###################################################################### + + + ###################################################################### # # matrix-prometheus diff --git a/roles/matrix-prometheus-node-exporter/defaults/main.yml b/roles/matrix-prometheus-node-exporter/defaults/main.yml new file mode 100644 index 00000000..a5664b83 --- /dev/null +++ b/roles/matrix-prometheus-node-exporter/defaults/main.yml @@ -0,0 +1,21 @@ +# matrix-prometheus-node-exporter is an Prometheus exporter for machine metrics +# See: https://prometheus.io/docs/guides/node-exporter/ + +matrix_prometheus_node_exporter_enabled: false + +matrix_prometheus_node_exporter_docker_image: "docker.io/prom/node-exporter:v1.0.1" +matrix_prometheus_node_exporter_docker_image_force_pull: "{{ matrix_prometheus_node_exporter_docker_image.endswith(':latest') }}" + +# A list of extra arguments to pass to the container +matrix_prometheus_node_exporter_container_extra_arguments: [] + +# List of systemd services that matrix-prometheus.service depends on +matrix_prometheus_node_exporter_systemd_required_services_list: ['docker.service'] + +# List of systemd services that matrix-prometheus.service wants +matrix_prometheus_node_exporter_systemd_wanted_services_list: [] + +# Controls whether the matrix-prometheus container exposes its HTTP port (tcp/9100 in the container). +# +# Takes an ":" or "" value (e.g. "127.0.0.1:9100"), or empty string to not expose. +matrix_prometheus_node_exporter_container_http_host_bind_port: '' diff --git a/roles/matrix-prometheus-node-exporter/tasks/init.yml b/roles/matrix-prometheus-node-exporter/tasks/init.yml new file mode 100644 index 00000000..2894b717 --- /dev/null +++ b/roles/matrix-prometheus-node-exporter/tasks/init.yml @@ -0,0 +1,5 @@ +- set_fact: + matrix_systemd_services_list: "{{ matrix_systemd_services_list + ['matrix-prometheus-node-exporter.service'] }}" + when: matrix_prometheus_node_exporter_enabled|bool + + diff --git a/roles/matrix-prometheus-node-exporter/tasks/main.yml b/roles/matrix-prometheus-node-exporter/tasks/main.yml new file mode 100644 index 00000000..172b5721 --- /dev/null +++ b/roles/matrix-prometheus-node-exporter/tasks/main.yml @@ -0,0 +1,8 @@ +- import_tasks: "{{ role_path }}/tasks/init.yml" + tags: + - always + +- import_tasks: "{{ role_path }}/tasks/setup.yml" + tags: + - setup-all + - setup-prometheus-node-exporter diff --git a/roles/matrix-prometheus-node-exporter/tasks/setup.yml b/roles/matrix-prometheus-node-exporter/tasks/setup.yml new file mode 100644 index 00000000..6f03fbaa --- /dev/null +++ b/roles/matrix-prometheus-node-exporter/tasks/setup.yml @@ -0,0 +1,60 @@ +--- + +# +# Tasks related to setting up matrix-prometheus-node-exporter +# + +- name: Ensure matrix-prometheus-node-exporter image is pulled + docker_image: + name: "{{ matrix_prometheus_node_exporter_docker_image }}" + source: "{{ 'pull' if ansible_version.major > 2 or ansible_version.minor > 7 else omit }}" + force_source: "{{ matrix_prometheus_node_exporter_docker_image_force_pull if ansible_version.major > 2 or ansible_version.minor >= 8 else omit }}" + force: "{{ omit if ansible_version.major > 2 or ansible_version.minor >= 8 else matrix_prometheus_node_exporter_docker_image_force_pull }}" + when: "matrix_prometheus_node_exporter_enabled|bool" + +- name: Ensure matrix-prometheus-node-exporter.service installed + template: + src: "{{ role_path }}/templates/systemd/matrix-prometheus-node-exporter.service.j2" + dest: "{{ matrix_systemd_path }}/matrix-prometheus-node-exporter.service" + mode: 0644 + register: matrix_prometheus_node_exporter_systemd_service_result + when: matrix_prometheus_node_exporter_enabled|bool + +- name: Ensure systemd reloaded after matrix-prometheus.service installation + service: + daemon_reload: yes + when: "matrix_prometheus_node_exporter_enabled|bool and matrix_prometheus_node_exporter_systemd_service_result.changed" + +# +# Tasks related to getting rid of matrix-prometheus-node-exporter (if it was previously enabled) +# + +- name: Check existence of matrix-prometheus-node-exporter service + stat: + path: "{{ matrix_systemd_path }}/matrix-prometheus-node-exporter.service" + register: matrix_prometheus_node_exporter_service_stat + +- name: Ensure matrix-prometheus-node-exporter is stopped + service: + name: matrix-prometheus-node-exporter + state: stopped + daemon_reload: yes + register: stopping_result + when: "not matrix_prometheus_node_exporter_enabled|bool and matrix_prometheus_node_exporter_service_stat.stat.exists" + +- name: Ensure matrix-prometheus-node-exporter.service doesn't exist + file: + path: "{{ matrix_systemd_path }}/matrix-prometheus-node-exporter.service" + state: absent + when: "not matrix_prometheus_node_exporter_enabled|bool and matrix_prometheus_node_exporter_service_stat.stat.exists" + +- name: Ensure systemd reloaded after matrix-prometheus-node-exporter.service removal + service: + daemon_reload: yes + when: "not matrix_prometheus_node_exporter_enabled|bool and matrix_prometheus_node_exporter_service_stat.stat.exists" + +- name: Ensure matrix-prometheus-node-exporter Docker image doesn't exist + docker_image: + name: "{{ matrix_prometheus_node_exporter_docker_image }}" + state: absent + when: "not matrix_prometheus_node_exporter_enabled|bool" diff --git a/roles/matrix-prometheus-node-exporter/tasks/validate_config.yml b/roles/matrix-prometheus-node-exporter/tasks/validate_config.yml new file mode 100644 index 00000000..713646ae --- /dev/null +++ b/roles/matrix-prometheus-node-exporter/tasks/validate_config.yml @@ -0,0 +1,7 @@ +--- + +- name: Fail if Synapse metrics or Prometheus Node Exporter not enabled + fail: + msg: > + You need to enable `matrix_synapse_metrics_enabled` and/or `matrix_prometheus_node_exporter_enabled` for Prometheus grab metrics. + when: "not matrix_synapse_metrics_enabled and not matrix_prometheus_node_exporter_enabled" diff --git a/roles/matrix-prometheus-node-exporter/templates/systemd/matrix-prometheus-node-exporter.service.j2 b/roles/matrix-prometheus-node-exporter/templates/systemd/matrix-prometheus-node-exporter.service.j2 new file mode 100644 index 00000000..622947d0 --- /dev/null +++ b/roles/matrix-prometheus-node-exporter/templates/systemd/matrix-prometheus-node-exporter.service.j2 @@ -0,0 +1,40 @@ +#jinja2: lstrip_blocks: "True" +[Unit] +Description=matrix-prometheus-node-exporter +{% for service in matrix_prometheus_node_exporter_systemd_required_services_list %} +Requires={{ service }} +After={{ service }} +{% endfor %} +{% for service in matrix_prometheus_node_exporter_systemd_wanted_services_list %} +Wants={{ service }} +{% endfor %} +DefaultDependencies=no + +[Service] +Type=simple +Environment="HOME={{ matrix_systemd_unit_home_path }}" +ExecStartPre=-{{ matrix_host_command_sh }} -c '{{ matrix_host_command_docker }} kill matrix-prometheus-node-exporter 2>/dev/null' +ExecStartPre=-{{ matrix_host_command_sh }} -c '{{ matrix_host_command_docker }} rm matrix-prometheus-node-exporter 2>/dev/null' + + +ExecStart={{ matrix_host_command_docker }} run --rm --name matrix-prometheus-node-exporter \ + --log-driver=none \ + --user={{ matrix_user_uid }}:{{ matrix_user_gid }} \ + --cap-drop=ALL \ + --network={{ matrix_docker_network }} \ + {% if matrix_prometheus_node_exporter_container_http_host_bind_port %} + -p {{ matrix_prometheus_node_exporter_container_http_host_bind_port }}:9100 \ + {% endif %} + {% for arg in matrix_prometheus_node_exporter_container_extra_arguments %} + {{ arg }} \ + {% endfor %} + {{ matrix_prometheus_node_exporter_docker_image }} + +ExecStop=-{{ matrix_host_command_sh }} -c '{{ matrix_host_command_docker }} kill matrix-prometheus-node-exporter 2>/dev/null' +ExecStop=-{{ matrix_host_command_sh }} -c '{{ matrix_host_command_docker }} rm matrix-prometheus-node-exporter 2>/dev/null' +Restart=always +RestartSec=30 +SyslogIdentifier=matrix-prometheus-node-exporter + +[Install] +WantedBy=multi-user.target diff --git a/roles/matrix-prometheus/tasks/validate_config.yml b/roles/matrix-prometheus/tasks/validate_config.yml index b614b438..713646ae 100644 --- a/roles/matrix-prometheus/tasks/validate_config.yml +++ b/roles/matrix-prometheus/tasks/validate_config.yml @@ -1,7 +1,7 @@ --- -- name: Fail if Synapse metrics not enabled +- name: Fail if Synapse metrics or Prometheus Node Exporter not enabled fail: msg: > - You need to enable `matrix_synapse_metrics_enabled` for Prometheus grab metrics. - when: "not matrix_synapse_metrics_enabled" + You need to enable `matrix_synapse_metrics_enabled` and/or `matrix_prometheus_node_exporter_enabled` for Prometheus grab metrics. + when: "not matrix_synapse_metrics_enabled and not matrix_prometheus_node_exporter_enabled" diff --git a/roles/matrix-prometheus/templates/prometheus.yml.j2 b/roles/matrix-prometheus/templates/prometheus.yml.j2 index 7b90baeb..317dcd16 100644 --- a/roles/matrix-prometheus/templates/prometheus.yml.j2 +++ b/roles/matrix-prometheus/templates/prometheus.yml.j2 @@ -33,3 +33,8 @@ scrape_configs: - targets: ['matrix-synapse:{{ matrix_synapse_metrics_port }}'] {% endif %} + {% if matrix_prometheus_node_exporter_enabled %} + - job_name: node + static_configs: + - targets: ['matrix-prometheus-node-exporter:9100'] + {% endif %} diff --git a/setup.yml b/setup.yml index 7965185c..838e08c8 100755 --- a/setup.yml +++ b/setup.yml @@ -28,6 +28,7 @@ - matrix-bot-matrix-reminder-bot - matrix-synapse - matrix-synapse-admin + - matrix-prometheus-node-exporter - matrix-prometheus - matrix-registration - matrix-client-element