diff --git a/nixos/modules/module-list.nix b/nixos/modules/module-list.nix index e9a5b8b9445..c97e9f01ad7 100644 --- a/nixos/modules/module-list.nix +++ b/nixos/modules/module-list.nix @@ -491,6 +491,7 @@ ./services/monitoring/prometheus/default.nix ./services/monitoring/prometheus/alertmanager.nix ./services/monitoring/prometheus/exporters.nix + ./services/monitoring/prometheus/pushgateway.nix ./services/monitoring/riemann.nix ./services/monitoring/riemann-dash.nix ./services/monitoring/riemann-tools.nix diff --git a/nixos/modules/services/monitoring/prometheus/pushgateway.nix b/nixos/modules/services/monitoring/prometheus/pushgateway.nix new file mode 100644 index 00000000000..f8fcc3eb97e --- /dev/null +++ b/nixos/modules/services/monitoring/prometheus/pushgateway.nix @@ -0,0 +1,166 @@ +{ config, pkgs, lib, ... }: + +with lib; + +let + cfg = config.services.prometheus.pushgateway; + + cmdlineArgs = + opt "web.listen-address" cfg.web.listen-address + ++ opt "web.telemetry-path" cfg.web.telemetry-path + ++ opt "web.external-url" cfg.web.external-url + ++ opt "web.route-prefix" cfg.web.route-prefix + ++ optional cfg.persistMetrics ''--persistence.file="/var/lib/${cfg.stateDir}/metrics"'' + ++ opt "persistence.interval" cfg.persistence.interval + ++ opt "log.level" cfg.log.level + ++ opt "log.format" cfg.log.format + ++ cfg.extraFlags; + + opt = k : v : optional (v != null) ''--${k}="${v}"''; + +in { + options = { + services.prometheus.pushgateway = { + enable = mkEnableOption "Prometheus Pushgateway"; + + package = mkOption { + type = types.package; + default = pkgs.prometheus-pushgateway; + defaultText = "pkgs.prometheus-pushgateway"; + description = '' + Package that should be used for the prometheus pushgateway. + ''; + }; + + web.listen-address = mkOption { + type = types.nullOr types.str; + default = null; + description = '' + Address to listen on for the web interface, API and telemetry. + + null will default to :9091. + ''; + }; + + web.telemetry-path = mkOption { + type = types.nullOr types.str; + default = null; + description = '' + Path under which to expose metrics. + + null will default to /metrics. + ''; + }; + + web.external-url = mkOption { + type = types.nullOr types.str; + default = null; + description = '' + The URL under which Pushgateway is externally reachable. + ''; + }; + + web.route-prefix = mkOption { + type = types.nullOr types.str; + default = null; + description = '' + Prefix for the internal routes of web endpoints. + + Defaults to the path of + . + ''; + }; + + persistence.interval = mkOption { + type = types.nullOr types.str; + default = null; + example = "10m"; + description = '' + The minimum interval at which to write out the persistence file. + + null will default to 5m. + ''; + }; + + log.level = mkOption { + type = types.nullOr (types.enum ["debug" "info" "warn" "error" "fatal"]); + default = null; + description = '' + Only log messages with the given severity or above. + + null will default to info. + ''; + }; + + log.format = mkOption { + type = types.nullOr types.str; + default = null; + example = "logger:syslog?appname=bob&local=7"; + description = '' + Set the log target and format. + + null will default to logger:stderr. + ''; + }; + + extraFlags = mkOption { + type = types.listOf types.str; + default = []; + description = '' + Extra commandline options when launching the Pushgateway. + ''; + }; + + persistMetrics = mkOption { + type = types.bool; + default = false; + description = '' + Whether to persist metrics to a file. + + When enabled metrics will be saved to a file called + metrics in the directory + /var/lib/pushgateway. The directory below + /var/lib can be set using + . + ''; + }; + + stateDir = mkOption { + type = types.str; + default = "pushgateway"; + description = '' + Directory below /var/lib to store metrics. + + This directory will be created automatically using systemd's + StateDirectory mechanism when + + is enabled. + ''; + }; + }; + }; + + config = mkIf cfg.enable { + assertions = [ + { + assertion = !hasPrefix "/" cfg.stateDir; + message = + "The option services.prometheus.pushgateway.stateDir" + + " shouldn't be an absolute directory." + + " It should be a directory relative to /var/lib."; + } + ]; + systemd.services.pushgateway = { + wantedBy = [ "multi-user.target" ]; + after = [ "network.target" ]; + serviceConfig = { + Restart = "always"; + DynamicUser = true; + ExecStart = "${cfg.package}/bin/pushgateway" + + optionalString (length cmdlineArgs != 0) (" \\\n " + + concatStringsSep " \\\n " cmdlineArgs); + StateDirectory = if cfg.persistMetrics then cfg.stateDir else null; + }; + }; + }; +} diff --git a/nixos/tests/prometheus-2.nix b/nixos/tests/prometheus-2.nix index 5a4d8668cb8..d7035d49ad4 100644 --- a/nixos/tests/prometheus-2.nix +++ b/nixos/tests/prometheus-2.nix @@ -3,15 +3,29 @@ import ./make-test.nix { nodes = { one = { pkgs, ... }: { + environment.systemPackages = [ pkgs.jq ]; services.prometheus2 = { enable = true; - scrapeConfigs = [{ - job_name = "prometheus"; - static_configs = [{ - targets = [ "127.0.0.1:9090" ]; - labels = { instance = "localhost"; }; - }]; - }]; + scrapeConfigs = [ + { + job_name = "prometheus"; + static_configs = [ + { + targets = [ "127.0.0.1:9090" ]; + labels = { instance = "localhost"; }; + } + ]; + } + { + job_name = "pushgateway"; + scrape_interval = "1s"; + static_configs = [ + { + targets = [ "127.0.0.1:9091" ]; + } + ]; + } + ]; rules = [ '' groups: @@ -22,6 +36,12 @@ import ./make-test.nix { '' ]; }; + services.prometheus.pushgateway = { + enable = true; + persistMetrics = true; + persistence.interval = "1s"; + stateDir = "prometheus-pushgateway"; + }; }; }; @@ -30,5 +50,18 @@ import ./make-test.nix { $one->waitForUnit("prometheus2.service"); $one->waitForOpenPort(9090); $one->succeed("curl -s http://127.0.0.1:9090/metrics"); + + # Let's test if pushing a metric to the pushgateway succeeds + # and whether that metric gets ingested by prometheus. + $one->waitForUnit("pushgateway.service"); + $one->succeed( + "echo 'some_metric 3.14' | " . + "curl --data-binary \@- http://127.0.0.1:9091/metrics/job/some_job"); + $one->waitUntilSucceeds( + "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=some_metric' " . + "| jq '.data.result[0].value[1]' | grep '\"3.14\"'"); + + # Let's test if the pushgateway persists metrics to the configured location. + $one->waitUntilSucceeds("test -e /var/lib/prometheus-pushgateway/metrics"); ''; }