From d539517f2b4e20973d2046ca16ed83d9bc5b34ae Mon Sep 17 00:00:00 2001 From: WilliButz Date: Fri, 1 Jan 2021 23:54:13 +0100 Subject: [PATCH 1/3] prometheus-json-exporter: unstable-2017-10-06 -> 0.2.0 --- .../monitoring/prometheus/json-exporter.nix | 26 ++-- .../prometheus/json-exporter_deps.nix | 111 ------------------ 2 files changed, 9 insertions(+), 128 deletions(-) delete mode 100644 pkgs/servers/monitoring/prometheus/json-exporter_deps.nix diff --git a/pkgs/servers/monitoring/prometheus/json-exporter.nix b/pkgs/servers/monitoring/prometheus/json-exporter.nix index afa61080931..796eb065e63 100644 --- a/pkgs/servers/monitoring/prometheus/json-exporter.nix +++ b/pkgs/servers/monitoring/prometheus/json-exporter.nix @@ -1,31 +1,23 @@ -# This file was generated by go2nix. -{ buildGoPackage, fetchFromGitHub, fetchpatch, lib, nixosTests }: +{ lib, buildGoModule, fetchFromGitHub, nixosTests }: -buildGoPackage { +buildGoModule rec { pname = "prometheus-json-exporter"; - version = "unstable-2017-10-06"; - - goPackagePath = "github.com/kawamuray/prometheus-json-exporter"; + version = "0.2.0"; src = fetchFromGitHub { - owner = "kawamuray"; - repo = "prometheus-json-exporter"; - rev = "51e3dc02a30ab818bb73e5c98c3853231c2dbb5f"; - sha256 = "1v1p4zcqnb3d3rm55r695ydn61h6gz95f55cpa22hzw18dasahdh"; + owner = "prometheus-community"; + repo = "json_exporter"; + rev = "v${version}"; + sha256 = "1aabvd75a2223x5wnbyryigj7grw6l05jhr3g3s97b1f1hfigz6d"; }; - goDeps = ./json-exporter_deps.nix; - - patches = [(fetchpatch { # adds bool support - url = "https://patch-diff.githubusercontent.com/raw/kawamuray/prometheus-json-exporter/pull/17.patch"; - sha256 = "0mc5axhd2bykci41dgswl4r1552d70jsmb17lbih7czhsy6rgmrm"; - })]; + vendorSha256 = "03kb0gklq08krl7qnvs7267siw1pkyy346b42dsk1d9gr5302wsw"; passthru.tests = { inherit (nixosTests.prometheus-exporters) json; }; meta = with lib; { description = "A prometheus exporter which scrapes remote JSON by JSONPath"; - homepage = "https://github.com/kawamuray/prometheus-json-exporter"; + homepage = "https://github.com/prometheus-community/json_exporter"; license = licenses.asl20; maintainers = with maintainers; [ willibutz ]; }; diff --git a/pkgs/servers/monitoring/prometheus/json-exporter_deps.nix b/pkgs/servers/monitoring/prometheus/json-exporter_deps.nix deleted file mode 100644 index 9416125bc0c..00000000000 --- a/pkgs/servers/monitoring/prometheus/json-exporter_deps.nix +++ /dev/null @@ -1,111 +0,0 @@ -# This file was generated by go2nix. -[ - { - goPackagePath = "github.com/Sirupsen/logrus"; - fetch = { - type = "git"; - url = "https://github.com/Sirupsen/logrus"; - rev = "a437dfd2463eaedbec3dfe443e477d3b0a810b3f"; - sha256 = "1904s2bbc7p88anzjp6fyj3jrbm5p6wbb8j4490674dq10kkcfbj"; - }; - } - { - goPackagePath = "github.com/beorn7/perks"; - fetch = { - type = "git"; - url = "https://github.com/beorn7/perks"; - rev = "4c0e84591b9aa9e6dcfdf3e020114cd81f89d5f9"; - sha256 = "1hrybsql68xw57brzj805xx2mghydpdiysv3gbhr7f5wlxj2514y"; - }; - } - { - goPackagePath = "github.com/golang/protobuf"; - fetch = { - type = "git"; - url = "https://github.com/golang/protobuf"; - rev = "8ee79997227bf9b34611aee7946ae64735e6fd93"; - sha256 = "0qm1lpdhf97k2hxgivq2cpjgawhlmmz39y230kgxijhm96xijxb8"; - }; - } - { - goPackagePath = "github.com/kawamuray/jsonpath"; - fetch = { - type = "git"; - url = "https://github.com/kawamuray/jsonpath"; - rev = "5c448ebf973557834ef870e788b0b2d95ac68d12"; - sha256 = "1i1id1i85rf09rldp03h0ibyi6zg0fz9p9l5qj7i8dfs6h6y8f0a"; - }; - } - { - goPackagePath = "github.com/kawamuray/prometheus-exporter-harness"; - fetch = { - type = "git"; - url = "https://github.com/kawamuray/prometheus-exporter-harness"; - rev = "97eeea7b8b0619ea2a065d75c0f0f5909327afa6"; - sha256 = "12al8irf8jb6p2xmm4wbh0wjqsyrdywynr4w122wxxnsx2n56xyv"; - }; - } - { - goPackagePath = "github.com/matttproud/golang_protobuf_extensions"; - fetch = { - type = "git"; - url = "https://github.com/matttproud/golang_protobuf_extensions"; - rev = "c12348ce28de40eed0136aa2b644d0ee0650e56c"; - sha256 = "1d0c1isd2lk9pnfq2nk0aih356j30k3h1gi2w0ixsivi5csl7jya"; - }; - } - { - goPackagePath = "github.com/prometheus/client_golang"; - fetch = { - type = "git"; - url = "https://github.com/prometheus/client_golang"; - rev = "575f371f7862609249a1be4c9145f429fe065e32"; - sha256 = "0hyvszjv5m6i40k2fn21c3bjr8jhlfdqavk1r6g2v5dybyj47vps"; - }; - } - { - goPackagePath = "github.com/prometheus/client_model"; - fetch = { - type = "git"; - url = "https://github.com/prometheus/client_model"; - rev = "fa8ad6fec33561be4280a8f0514318c79d7f6cb6"; - sha256 = "11a7v1fjzhhwsl128znjcf5v7v6129xjgkdpym2lial4lac1dhm9"; - }; - } - { - goPackagePath = "github.com/prometheus/common"; - fetch = { - type = "git"; - url = "https://github.com/prometheus/common"; - rev = "0d5de9d6d8629cb8bee6d4674da4127cd8b615a3"; - sha256 = "1zlvvgw67p66fz9nswkydq15j4a5z5kkiskl0jxps8i27ya1baq0"; - }; - } - { - goPackagePath = "github.com/prometheus/procfs"; - fetch = { - type = "git"; - url = "https://github.com/prometheus/procfs"; - rev = "abf152e5f3e97f2fafac028d2cc06c1feb87ffa5"; - sha256 = "0cp8lznv1b4zhi3wnbjkfxwzhkqd3wbmiy6mwgjanip8l9l3ykws"; - }; - } - { - goPackagePath = "github.com/urfave/cli"; - fetch = { - type = "git"; - url = "https://github.com/urfave/cli"; - rev = "0bdeddeeb0f650497d603c4ad7b20cfe685682f6"; - sha256 = "1ny63c7bfwfrsp7vfkvb4i0xhq4v7yxqnwxa52y4xlfxs4r6v6fg"; - }; - } - { - goPackagePath = "gopkg.in/yaml.v2"; - fetch = { - type = "git"; - url = "https://gopkg.in/yaml.v2"; - rev = "a5b47d31c556af34a302ce5d659e6fea44d90de0"; - sha256 = "0v6l48fshdjrqzyq1kwn22gy7vy434xdr1i0lm3prsf6jbln9fam"; - }; - } -] From 3f94c66ee121a2fdb136741c3685195c6d1db643 Mon Sep 17 00:00:00 2001 From: WilliButz Date: Fri, 1 Jan 2021 23:59:06 +0100 Subject: [PATCH 2/3] nixos/prometheus-json-exporter: update modules & tests, add release notes --- nixos/doc/manual/release-notes/rl-2103.xml | 26 ++++ .../monitoring/prometheus/exporters.nix | 2 - .../monitoring/prometheus/exporters/json.nix | 30 +++-- .../prometheus/exporters/rspamd.nix | 122 ++++++++++-------- nixos/tests/prometheus-exporters.nix | 15 ++- 5 files changed, 120 insertions(+), 75 deletions(-) diff --git a/nixos/doc/manual/release-notes/rl-2103.xml b/nixos/doc/manual/release-notes/rl-2103.xml index 38262b50899..0475c365504 100644 --- a/nixos/doc/manual/release-notes/rl-2103.xml +++ b/nixos/doc/manual/release-notes/rl-2103.xml @@ -295,6 +295,32 @@ Based on , existing installations will continue to work. + + + The prometheus json exporter is now managed by the prometheus community. Together with additional features + some backwards incompatibilities were introduced. + Most importantly the exporter no longer accepts a fixed command-line parameter to specify the URL of the + endpoint serving JSON. It now expects this URL to be passed as an URL parameter, when scraping the exporter's + /probe endpoint. + In the prometheus scrape configuration the scrape target might look like this: + + http://some.json-exporter.host:7979/probe?target=https://example.com/some/json/endpoint + + + + Existing configuration for the exporter needs to be updated, but can partially be re-used. + Documentation is available in the upstream repository and a small example for NixOS is available + in the corresponding NixOS test. + + + These changes also affect services.prometheus.exporters.rspamd, which is + just a preconfigured instance of the json exporter. + + + For more information, take a look at the + official documentation of the json_exporter. + + diff --git a/nixos/modules/services/monitoring/prometheus/exporters.nix b/nixos/modules/services/monitoring/prometheus/exporters.nix index fe9d1a65907..1fd85c66f84 100644 --- a/nixos/modules/services/monitoring/prometheus/exporters.nix +++ b/nixos/modules/services/monitoring/prometheus/exporters.nix @@ -236,8 +236,6 @@ in services.prometheus.exporters.minio.minioAddress = mkDefault "http://localhost:9000"; services.prometheus.exporters.minio.minioAccessKey = mkDefault config.services.minio.accessKey; services.prometheus.exporters.minio.minioAccessSecret = mkDefault config.services.minio.secretKey; - })] ++ [(mkIf config.services.rspamd.enable { - services.prometheus.exporters.rspamd.url = mkDefault "http://localhost:11334/stat"; })] ++ [(mkIf config.services.prometheus.exporters.rtl_433.enable { hardware.rtl-sdr.enable = mkDefault true; })] ++ [(mkIf config.services.nginx.enable { diff --git a/nixos/modules/services/monitoring/prometheus/exporters/json.nix b/nixos/modules/services/monitoring/prometheus/exporters/json.nix index bd0026b55f7..24364aababd 100644 --- a/nixos/modules/services/monitoring/prometheus/exporters/json.nix +++ b/nixos/modules/services/monitoring/prometheus/exporters/json.nix @@ -4,32 +4,42 @@ with lib; let cfg = config.services.prometheus.exporters.json; + in { port = 7979; extraOpts = { - url = mkOption { - type = types.str; - description = '' - URL to scrape JSON from. - ''; - }; configFile = mkOption { type = types.path; description = '' Path to configuration file. ''; }; - listenAddress = {}; # not used }; serviceOpts = { serviceConfig = { ExecStart = '' - ${pkgs.prometheus-json-exporter}/bin/prometheus-json-exporter \ - --port ${toString cfg.port} \ - ${cfg.url} ${escapeShellArg cfg.configFile} \ + ${pkgs.prometheus-json-exporter}/bin/json_exporter \ + --config.file ${escapeShellArg cfg.configFile} \ + --web.listen-address="${cfg.listenAddress}:${toString cfg.port}" \ ${concatStringsSep " \\\n " cfg.extraFlags} ''; }; }; + imports = [ + (mkRemovedOptionModule [ "url" ] '' + This option was removed. The URL of the endpoint serving JSON + must now be provided to the exporter by prometheus via the url + parameter target. + + In prometheus a scrape URL would look like this: + + http://some.json-exporter.host:7979/probe?target=https://example.com/some/json/endpoint + + + For more information, take a look at the + official documentation of the json_exporter. + '') + ({ options.warnings = options.warnings; options.assertions = options.assertions; }) + ]; } diff --git a/nixos/modules/services/monitoring/prometheus/exporters/rspamd.nix b/nixos/modules/services/monitoring/prometheus/exporters/rspamd.nix index 1f02ae20724..84e7ba3554b 100644 --- a/nixos/modules/services/monitoring/prometheus/exporters/rspamd.nix +++ b/nixos/modules/services/monitoring/prometheus/exporters/rspamd.nix @@ -10,64 +10,55 @@ let echo '${builtins.toJSON conf}' | ${pkgs.buildPackages.jq}/bin/jq '.' > $out ''; - generateConfig = extraLabels: (map (path: { - name = "rspamd_${replaceStrings [ "." " " ] [ "_" "_" ] path}"; - path = "$.${path}"; - labels = extraLabels; - }) [ - "actions.'add header'" - "actions.'no action'" - "actions.'rewrite subject'" - "actions.'soft reject'" - "actions.greylist" - "actions.reject" - "bytes_allocated" - "chunks_allocated" - "chunks_freed" - "chunks_oversized" - "connections" - "control_connections" - "ham_count" - "learned" - "pools_allocated" - "pools_freed" - "read_only" - "scanned" - "shared_chunks_allocated" - "spam_count" - "total_learns" - ]) ++ [{ - name = "rspamd_statfiles"; - type = "object"; - path = "$.statfiles[*]"; - labels = recursiveUpdate { - symbol = "$.symbol"; - type = "$.type"; - } extraLabels; - values = { - revision = "$.revision"; - size = "$.size"; - total = "$.total"; - used = "$.used"; - languages = "$.languages"; - users = "$.users"; - }; - }]; + generateConfig = extraLabels: { + metrics = (map (path: { + name = "rspamd_${replaceStrings [ "." " " ] [ "_" "_" ] path}"; + path = "$.${path}"; + labels = extraLabels; + }) [ + "actions.'add header'" + "actions.'no action'" + "actions.'rewrite subject'" + "actions.'soft reject'" + "actions.greylist" + "actions.reject" + "bytes_allocated" + "chunks_allocated" + "chunks_freed" + "chunks_oversized" + "connections" + "control_connections" + "ham_count" + "learned" + "pools_allocated" + "pools_freed" + "read_only" + "scanned" + "shared_chunks_allocated" + "spam_count" + "total_learns" + ]) ++ [{ + name = "rspamd_statfiles"; + type = "object"; + path = "$.statfiles[*]"; + labels = recursiveUpdate { + symbol = "$.symbol"; + type = "$.type"; + } extraLabels; + values = { + revision = "$.revision"; + size = "$.size"; + total = "$.total"; + used = "$.used"; + languages = "$.languages"; + users = "$.users"; + }; + }]; + }; in { port = 7980; extraOpts = { - listenAddress = {}; # not used - - url = mkOption { - type = types.str; - description = '' - URL to the rspamd metrics endpoint. - Defaults to http://localhost:11334/stat when - is true. - ''; - }; - extraLabels = mkOption { type = types.attrsOf types.str; default = { @@ -84,9 +75,26 @@ in }; }; serviceOpts.serviceConfig.ExecStart = '' - ${pkgs.prometheus-json-exporter}/bin/prometheus-json-exporter \ - --port ${toString cfg.port} \ - ${cfg.url} ${prettyJSON (generateConfig cfg.extraLabels)} \ + ${pkgs.prometheus-json-exporter}/bin/json_exporter \ + --config.file ${prettyJSON (generateConfig cfg.extraLabels)} \ + --web.listen-address "${cfg.listenAddress}:${toString cfg.port}" \ ${concatStringsSep " \\\n " cfg.extraFlags} ''; + + imports = [ + (mkRemovedOptionModule [ "url" ] '' + This option was removed. The URL of the rspamd metrics endpoint + must now be provided to the exporter by prometheus via the url + parameter target. + + In prometheus a scrape URL would look like this: + + http://some.rspamd-exporter.host:7980/probe?target=http://some.rspamd.host:11334/stat + + + For more information, take a look at the + official documentation of the json_exporter. + '') + ({ options.warnings = options.warnings; options.assertions = options.assertions; }) + ]; } diff --git a/nixos/tests/prometheus-exporters.nix b/nixos/tests/prometheus-exporters.nix index f611d961769..8fcb0a7aa2c 100644 --- a/nixos/tests/prometheus-exporters.nix +++ b/nixos/tests/prometheus-exporters.nix @@ -222,10 +222,11 @@ let exporterConfig = { enable = true; url = "http://localhost"; - configFile = pkgs.writeText "json-exporter-conf.json" (builtins.toJSON [{ - name = "json_test_metric"; - path = "$.test"; - }]); + configFile = pkgs.writeText "json-exporter-conf.json" (builtins.toJSON { + metrics = [ + { name = "json_test_metric"; path = "$.test"; } + ]; + }); }; metricProvider = { systemd.services.prometheus-json-exporter.after = [ "nginx.service" ]; @@ -241,7 +242,9 @@ let wait_for_open_port(80) wait_for_unit("prometheus-json-exporter.service") wait_for_open_port(7979) - succeed("curl -sSf localhost:7979/metrics | grep -q 'json_test_metric 1'") + succeed( + "curl -sSf 'localhost:7979/probe?target=http://localhost' | grep -q 'json_test_metric 1'" + ) ''; }; @@ -659,7 +662,7 @@ let wait_for_open_port(11334) wait_for_open_port(7980) wait_until_succeeds( - "curl -sSf localhost:7980/metrics | grep -q 'rspamd_scanned{host=\"rspamd\"} 0'" + "curl -sSf 'localhost:7980/probe?target=http://localhost:11334/stat' | grep -q 'rspamd_scanned{host=\"rspamd\"} 0'" ) ''; }; From 6e43cc7fb0f22a88a989583ccf50e56f7f72ad6a Mon Sep 17 00:00:00 2001 From: Maximilian Bosch Date: Sun, 3 Jan 2021 19:04:33 +0100 Subject: [PATCH 3/3] nixos/prometheus-exporters: minor doc improvements * Content of `programlisting` shouldn't be indented, otherwise it's weirdly indented in the output. * Use `` in the release notes: then users can directly go to the option documentation when reading release notes. * Don't use docbook tags in `mkRemovedOptionModule`: it's only used during evaluation where docbook isn't rendered. --- nixos/doc/manual/release-notes/rl-2103.xml | 4 ++-- .../monitoring/prometheus/exporters/json.nix | 12 +++++------- .../monitoring/prometheus/exporters/rspamd.nix | 11 +++++------ 3 files changed, 12 insertions(+), 15 deletions(-) diff --git a/nixos/doc/manual/release-notes/rl-2103.xml b/nixos/doc/manual/release-notes/rl-2103.xml index 0475c365504..76aa3d839bf 100644 --- a/nixos/doc/manual/release-notes/rl-2103.xml +++ b/nixos/doc/manual/release-notes/rl-2103.xml @@ -304,7 +304,7 @@ /probe endpoint. In the prometheus scrape configuration the scrape target might look like this: - http://some.json-exporter.host:7979/probe?target=https://example.com/some/json/endpoint +http://some.json-exporter.host:7979/probe?target=https://example.com/some/json/endpoint @@ -313,7 +313,7 @@ in the corresponding NixOS test. - These changes also affect services.prometheus.exporters.rspamd, which is + These changes also affect , which is just a preconfigured instance of the json exporter. diff --git a/nixos/modules/services/monitoring/prometheus/exporters/json.nix b/nixos/modules/services/monitoring/prometheus/exporters/json.nix index 24364aababd..1800da69a25 100644 --- a/nixos/modules/services/monitoring/prometheus/exporters/json.nix +++ b/nixos/modules/services/monitoring/prometheus/exporters/json.nix @@ -4,7 +4,6 @@ with lib; let cfg = config.services.prometheus.exporters.json; - in { port = 7979; @@ -30,15 +29,14 @@ in (mkRemovedOptionModule [ "url" ] '' This option was removed. The URL of the endpoint serving JSON must now be provided to the exporter by prometheus via the url - parameter target. + parameter `target'. In prometheus a scrape URL would look like this: - - http://some.json-exporter.host:7979/probe?target=https://example.com/some/json/endpoint - - For more information, take a look at the - official documentation of the json_exporter. + http://some.json-exporter.host:7979/probe?target=https://example.com/some/json/endpoint + + For more information, take a look at the official documentation + (https://github.com/prometheus-community/json_exporter) of the json_exporter. '') ({ options.warnings = options.warnings; options.assertions = options.assertions; }) ]; diff --git a/nixos/modules/services/monitoring/prometheus/exporters/rspamd.nix b/nixos/modules/services/monitoring/prometheus/exporters/rspamd.nix index 84e7ba3554b..78fe120e4d9 100644 --- a/nixos/modules/services/monitoring/prometheus/exporters/rspamd.nix +++ b/nixos/modules/services/monitoring/prometheus/exporters/rspamd.nix @@ -85,15 +85,14 @@ in (mkRemovedOptionModule [ "url" ] '' This option was removed. The URL of the rspamd metrics endpoint must now be provided to the exporter by prometheus via the url - parameter target. + parameter `target'. In prometheus a scrape URL would look like this: - - http://some.rspamd-exporter.host:7980/probe?target=http://some.rspamd.host:11334/stat - - For more information, take a look at the - official documentation of the json_exporter. + http://some.rspamd-exporter.host:7980/probe?target=http://some.rspamd.host:11334/stat + + For more information, take a look at the official documentation + (https://github.com/prometheus-community/json_exporter) of the json_exporter. '') ({ options.warnings = options.warnings; options.assertions = options.assertions; }) ];