From 37f9419aef31c125140ef3a8d767e85a374323a3 Mon Sep 17 00:00:00 2001 From: Pablo Ovelleiro Corral Date: Sun, 25 Aug 2024 17:17:54 +0200 Subject: [PATCH 1/5] Add blackbox-exporter config --- modules/prometheus/alert-rules.nix | 8 ++--- modules/prometheus/default.nix | 55 ++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 4 deletions(-) diff --git a/modules/prometheus/alert-rules.nix b/modules/prometheus/alert-rules.nix index a483c2a..1d5fd88 100644 --- a/modules/prometheus/alert-rules.nix +++ b/modules/prometheus/alert-rules.nix @@ -198,10 +198,10 @@ lib.mapAttrsToList description = "{{$labels.instance}}: healtcheck {{$labels.job}} fails!"; }; */ - #cert_expiry = { - # condition = "(probe_ssl_earliest_cert_expiry - time())/(3600*24) < 30"; - # description = "{{$labels.instance}}: The TLS certificate will expire in less than 30 days: {{$value}}s"; - #}; + cert_expiry = { + condition = "(probe_ssl_earliest_cert_expiry - time())/(3600*24) < 30"; + description = "{{$labels.instance}}: The TLS certificate will expire in less than 30 days: {{$value}}s"; + }; # ignore devices that disabled S.M.A.R.T (example if attached via USB) diff --git a/modules/prometheus/default.nix b/modules/prometheus/default.nix index 709f6a2..bdc7891 100644 --- a/modules/prometheus/default.nix +++ b/modules/prometheus/default.nix @@ -5,6 +5,10 @@ flake, ... }: +let + # TODO add hosts here + blackboxTargets = [ "https://pablo.tools" ]; +in { age.secrets.alertmanager-envfile = { file = "${flake.self}/secrets/alertmanager-envfile.age"; @@ -27,6 +31,33 @@ enable = true; port = 9001; exporters = { + blackbox = { + enable = true; + # Default port is 9115 + # Listen on 0.0.0.0, bet we only open the firewall for wg0 + openFirewall = false; + + configFile = pkgs.writeTextFile { + name = "blackbox-exporter-config"; + text = '' + modules: + http_2xx: + prober: http + timeout: 5s + http: + valid_http_versions: ["HTTP/1.1", "HTTP/2.0"] + valid_status_codes: [] # Defaults to 2xx + method: GET + no_follow_redirects: false + fail_if_ssl: false + fail_if_not_ssl: false + tls_config: + insecure_skip_verify: false + preferred_ip_protocol: "ip4" # defaults to "ip6" + ip_protocol_fallback: true # fallback to "ip6" + ''; + }; + }; node = { enable = true; enabledCollectors = [ "systemd" ]; @@ -38,6 +69,30 @@ scrape_timeout = "9s"; }; scrapeConfigs = [ + { + job_name = "blackbox"; + scrape_interval = "2m"; + metrics_path = "/probe"; + params = { + module = [ "http_2xx" ]; + }; + static_configs = [ { targets = blackboxTargets; } ]; + + relabel_configs = [ + { + source_labels = [ "__address__" ]; + target_label = "__param_target"; + } + { + source_labels = [ "__param_target" ]; + target_label = "instance"; + } + { + target_label = "__address__"; + replacement = "127.0.0.1:9115"; # The blackbox exporter's real hostname:port. + } + ]; + } { job_name = "node-exporter"; static_configs = [ -- 2.44.1 From 21df30e9b48803a26f87e21bcf42046eb3dc006c Mon Sep 17 00:00:00 2001 From: Pablo Ovelleiro Corral Date: Sun, 25 Aug 2024 17:23:04 +0200 Subject: [PATCH 2/5] fix formatting --- modules/prometheus/alert-rules.nix | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/prometheus/alert-rules.nix b/modules/prometheus/alert-rules.nix index 1d5fd88..b2f2a11 100644 --- a/modules/prometheus/alert-rules.nix +++ b/modules/prometheus/alert-rules.nix @@ -199,8 +199,8 @@ lib.mapAttrsToList }; */ cert_expiry = { - condition = "(probe_ssl_earliest_cert_expiry - time())/(3600*24) < 30"; - description = "{{$labels.instance}}: The TLS certificate will expire in less than 30 days: {{$value}}s"; + condition = "(probe_ssl_earliest_cert_expiry - time())/(3600*24) < 30"; + description = "{{$labels.instance}}: The TLS certificate will expire in less than 30 days: {{$value}}s"; }; # ignore devices that disabled S.M.A.R.T (example if attached via USB) -- 2.44.1 From 77757372021b8190a76534729de5dc8c5745db2f Mon Sep 17 00:00:00 2001 From: teutat3s Date: Mon, 26 Aug 2024 10:04:29 +0200 Subject: [PATCH 3/5] grafana: add blackbox exporter dashboard --- modules/grafana/default.nix | 5 + .../blackbox-exporter_rev3.json | 1126 +++++++++++++++++ 2 files changed, 1131 insertions(+) create mode 100644 modules/grafana/grafana-dashboards/blackbox-exporter_rev3.json diff --git a/modules/grafana/default.nix b/modules/grafana/default.nix index b62789e..3525985 100644 --- a/modules/grafana/default.nix +++ b/modules/grafana/default.nix @@ -28,6 +28,11 @@ group = "grafana"; user = "grafana"; }; + "grafana-dashboards/blackbox-exporter_rev3.json" = { + source = ./grafana-dashboards/blackbox-exporter_rev3.json; + group = "grafana"; + user = "grafana"; + }; "grafana-dashboards/synapse.json" = { source = ./grafana-dashboards/synapse.json; group = "grafana"; diff --git a/modules/grafana/grafana-dashboards/blackbox-exporter_rev3.json b/modules/grafana/grafana-dashboards/blackbox-exporter_rev3.json new file mode 100644 index 0000000..f9f1a7b --- /dev/null +++ b/modules/grafana/grafana-dashboards/blackbox-exporter_rev3.json @@ -0,0 +1,1126 @@ +{ + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "signcl-prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "5.2.2" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "5.0.0" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "5.0.0" + }, + { + "type": "panel", + "id": "singlestat", + "name": "Singlestat", + "version": "5.0.0" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Prometheus Blackbox Exporter Overview", + "editable": true, + "gnetId": 7587, + "graphTooltip": 0, + "id": null, + "iteration": 1534695504413, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "fill": 1, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 138, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "probe_duration_seconds{instance=~\"$target\"}", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "{{ instance }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Global Probe Duration", + "tooltip": { + "shared": true, + "sort": 1, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 15, + "panels": [], + "repeat": "target", + "title": "$target status", + "type": "row" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "#d44a3a", + "rgba(237, 129, 40, 0.89)", + "#299c46" + ], + "datasource": "${DS_PROMETHEUS}", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 0, + "y": 9 + }, + "id": 2, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "minSpan": 3, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "repeat": null, + "repeatDirection": "v", + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "probe_success{instance=~\"$target\"}", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "1,1", + "title": "Status", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + }, + { + "op": "=", + "text": "UP", + "value": "1" + }, + { + "op": "=", + "text": "DOWN", + "value": "0" + } + ], + "valueName": "current" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "fill": 1, + "gridPos": { + "h": 6, + "w": 10, + "x": 4, + "y": 9 + }, + "id": 25, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "probe_http_duration_seconds{instance=~\"$target\"}", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "{{ phase }}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "HTTP Duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "fill": 1, + "gridPos": { + "h": 6, + "w": 10, + "x": 14, + "y": 9 + }, + "id": 17, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "probe_duration_seconds{instance=~\"$target\"}", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "seconds", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Probe Duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "${DS_PROMETHEUS}", + "decimals": 0, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 0, + "y": 11 + }, + "id": 20, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "minSpan": 3, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "repeat": null, + "repeatDirection": "h", + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "probe_http_status_code{instance=~\"$target\"}", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "201, 399", + "title": "HTTP Status Code", + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + }, + { + "op": "=", + "text": "YES", + "value": "1" + }, + { + "op": "=", + "text": "N/A", + "value": "0" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "${DS_PROMETHEUS}", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 0, + "y": 13 + }, + "id": 27, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "probe_http_version{instance=~\"$target\"}", + "format": "time_series", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "", + "title": "HTTP Version", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "#d44a3a", + "rgba(237, 129, 40, 0.89)", + "#299c46" + ], + "datasource": "${DS_PROMETHEUS}", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 0, + "y": 15 + }, + "id": 18, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "minSpan": 3, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "repeat": null, + "repeatDirection": "v", + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "probe_http_ssl{instance=~\"$target\"}", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "0, 1", + "title": "SSL", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + }, + { + "op": "=", + "text": "YES", + "value": "1" + }, + { + "op": "=", + "text": "NO", + "value": "0" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "#d44a3a", + "rgba(237, 129, 40, 0.89)", + "#299c46" + ], + "datasource": "${DS_PROMETHEUS}", + "decimals": 2, + "format": "dtdurations", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 10, + "x": 4, + "y": 15 + }, + "id": 19, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "minSpan": 3, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "repeat": null, + "repeatDirection": "h", + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "probe_ssl_earliest_cert_expiry{instance=~\"$target\"} - time()", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "0,1209600", + "timeFrom": null, + "title": "SSL Expiry", + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + }, + { + "op": "=", + "text": "YES", + "value": "1" + }, + { + "op": "=", + "text": "NO", + "value": "0" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "${DS_PROMETHEUS}", + "format": "s", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 5, + "x": 14, + "y": 15 + }, + "id": 23, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "repeat": null, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "avg(probe_duration_seconds{instance=~\"$target\"})", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "", + "title": "Average Probe Duration", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "${DS_PROMETHEUS}", + "format": "s", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 5, + "x": 19, + "y": 15 + }, + "id": 24, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "repeat": null, + "repeatDirection": "h", + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "avg(probe_dns_lookup_time_seconds{instance=~\"$target\"})", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "refId": "A" + } + ], + "thresholds": "", + "title": "Average DNS Lookup", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + } + ], + "refresh": "10s", + "schemaVersion": 16, + "style": "dark", + "tags": [ + "blackbox", + "prometheus" + ], + "templating": { + "list": [ + { + "auto": true, + "auto_count": 10, + "auto_min": "10s", + "current": { + "text": "10s", + "value": "10s" + }, + "hide": 0, + "label": "Interval", + "name": "interval", + "options": [ + { + "selected": false, + "text": "auto", + "value": "$__auto_interval_interval" + }, + { + "selected": false, + "text": "5s", + "value": "5s" + }, + { + "selected": true, + "text": "10s", + "value": "10s" + }, + { + "selected": false, + "text": "30s", + "value": "30s" + }, + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": false, + "text": "10m", + "value": "10m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + }, + { + "selected": false, + "text": "6h", + "value": "6h" + }, + { + "selected": false, + "text": "12h", + "value": "12h" + }, + { + "selected": false, + "text": "1d", + "value": "1d" + }, + { + "selected": false, + "text": "7d", + "value": "7d" + }, + { + "selected": false, + "text": "14d", + "value": "14d" + }, + { + "selected": false, + "text": "30d", + "value": "30d" + } + ], + "query": "5s,10s,30s,1m,10m,30m,1h,6h,12h,1d,7d,14d,30d", + "refresh": 2, + "type": "interval" + }, + { + "allValue": null, + "current": {}, + "datasource": "${DS_PROMETHEUS}", + "hide": 0, + "includeAll": true, + "label": null, + "multi": true, + "name": "target", + "options": [], + "query": "label_values(probe_success, instance)", + "refresh": 1, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "current": { + "selected": false, + "text": "default", + "value": "default" + }, + "hide": 0, + "includeAll": false, + "multi": false, + "name": "DS_PROMETHEUS", + "label": "Datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Prometheus Blackbox Exporter", + "uid": "xtkCtBkiz", + "version": 2 +} -- 2.44.1 From 36c5073022c8ab834d96da83e76f92ded401cd8f Mon Sep 17 00:00:00 2001 From: teutat3s Date: Mon, 26 Aug 2024 13:43:05 +0200 Subject: [PATCH 4/5] monitoring: scrape blackbox every 5m, alert 21 days before certificate expiry --- modules/prometheus/alert-rules.nix | 4 ++-- modules/prometheus/default.nix | 3 +-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/modules/prometheus/alert-rules.nix b/modules/prometheus/alert-rules.nix index b2f2a11..1df8a95 100644 --- a/modules/prometheus/alert-rules.nix +++ b/modules/prometheus/alert-rules.nix @@ -199,8 +199,8 @@ lib.mapAttrsToList }; */ cert_expiry = { - condition = "(probe_ssl_earliest_cert_expiry - time())/(3600*24) < 30"; - description = "{{$labels.instance}}: The TLS certificate will expire in less than 30 days: {{$value}}s"; + condition = "(probe_ssl_earliest_cert_expiry - time())/(3600*24) < 21"; + description = "{{$labels.instance}}: The TLS certificate will expire in less than 21 days: {{$value}}s"; }; # ignore devices that disabled S.M.A.R.T (example if attached via USB) diff --git a/modules/prometheus/default.nix b/modules/prometheus/default.nix index bdc7891..8b857d6 100644 --- a/modules/prometheus/default.nix +++ b/modules/prometheus/default.nix @@ -34,7 +34,6 @@ in blackbox = { enable = true; # Default port is 9115 - # Listen on 0.0.0.0, bet we only open the firewall for wg0 openFirewall = false; configFile = pkgs.writeTextFile { @@ -71,7 +70,7 @@ in scrapeConfigs = [ { job_name = "blackbox"; - scrape_interval = "2m"; + scrape_interval = "5m"; metrics_path = "/probe"; params = { module = [ "http_2xx" ]; -- 2.44.1 From d395d893f872bc7aa38ccbce5869e42eeec90689 Mon Sep 17 00:00:00 2001 From: teutat3s Date: Mon, 26 Aug 2024 13:47:31 +0200 Subject: [PATCH 5/5] style: fix formatting --- .../blackbox-exporter_rev3.json | 59 +++---------------- 1 file changed, 9 insertions(+), 50 deletions(-) diff --git a/modules/grafana/grafana-dashboards/blackbox-exporter_rev3.json b/modules/grafana/grafana-dashboards/blackbox-exporter_rev3.json index f9f1a7b..2e203d5 100644 --- a/modules/grafana/grafana-dashboards/blackbox-exporter_rev3.json +++ b/modules/grafana/grafana-dashboards/blackbox-exporter_rev3.json @@ -159,11 +159,7 @@ "cacheTimeout": null, "colorBackground": true, "colorValue": false, - "colors": [ - "#d44a3a", - "rgba(237, 129, 40, 0.89)", - "#299c46" - ], + "colors": ["#d44a3a", "rgba(237, 129, 40, 0.89)", "#299c46"], "datasource": "${DS_PROMETHEUS}", "format": "none", "gauge": { @@ -424,11 +420,7 @@ "cacheTimeout": null, "colorBackground": false, "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], + "colors": ["#299c46", "rgba(237, 129, 40, 0.89)", "#d44a3a"], "datasource": "${DS_PROMETHEUS}", "decimals": 0, "format": "none", @@ -520,11 +512,7 @@ "cacheTimeout": null, "colorBackground": false, "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], + "colors": ["#299c46", "rgba(237, 129, 40, 0.89)", "#d44a3a"], "datasource": "${DS_PROMETHEUS}", "format": "none", "gauge": { @@ -600,11 +588,7 @@ "cacheTimeout": null, "colorBackground": false, "colorValue": true, - "colors": [ - "#d44a3a", - "rgba(237, 129, 40, 0.89)", - "#299c46" - ], + "colors": ["#d44a3a", "rgba(237, 129, 40, 0.89)", "#299c46"], "datasource": "${DS_PROMETHEUS}", "format": "none", "gauge": { @@ -694,11 +678,7 @@ "cacheTimeout": null, "colorBackground": false, "colorValue": true, - "colors": [ - "#d44a3a", - "rgba(237, 129, 40, 0.89)", - "#299c46" - ], + "colors": ["#d44a3a", "rgba(237, 129, 40, 0.89)", "#299c46"], "datasource": "${DS_PROMETHEUS}", "decimals": 2, "format": "dtdurations", @@ -791,11 +771,7 @@ "cacheTimeout": null, "colorBackground": false, "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], + "colors": ["#299c46", "rgba(237, 129, 40, 0.89)", "#d44a3a"], "datasource": "${DS_PROMETHEUS}", "format": "s", "gauge": { @@ -873,11 +849,7 @@ "cacheTimeout": null, "colorBackground": false, "colorValue": false, - "colors": [ - "#299c46", - "rgba(237, 129, 40, 0.89)", - "#d44a3a" - ], + "colors": ["#299c46", "rgba(237, 129, 40, 0.89)", "#d44a3a"], "datasource": "${DS_PROMETHEUS}", "format": "s", "gauge": { @@ -956,10 +928,7 @@ "refresh": "10s", "schemaVersion": 16, "style": "dark", - "tags": [ - "blackbox", - "prometheus" - ], + "tags": ["blackbox", "prometheus"], "templating": { "list": [ { @@ -1107,17 +1076,7 @@ "2h", "1d" ], - "time_options": [ - "5m", - "15m", - "1h", - "6h", - "12h", - "24h", - "2d", - "7d", - "30d" - ] + "time_options": ["5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d"] }, "timezone": "", "title": "Prometheus Blackbox Exporter", -- 2.44.1