Merge pull request #58255 from jbgi/prometheus2

Add Prometheus 2 service in parallel with 1.x version (continuation)
This commit is contained in:
Bas van Dijk 2019-04-09 14:14:18 +02:00 committed by GitHub
commit 2f2e2971d6
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 352 additions and 32 deletions

View file

@ -37,7 +37,52 @@
<itemizedlist>
<listitem>
<para />
<para>
Besides the existing <option>services.prometheus</option> module which
targets Prometheus-1 a new <option>services.prometheus2</option> module
has been added which targets Prometheus-2.
</para>
<para>
Both modules can be enabled at the same time. In fact
<link xlink:href="https://prometheus.io/docs/prometheus/latest/migration/#storage">
this is needed for upgrading existing Prometheus-1 data to Prometheus-2
</link>.
</para>
</listitem>
</itemizedlist>
</section>
<section xmlns="http://docbook.org/ns/docbook"
xmlns:xlink="http://www.w3.org/1999/xlink"
xmlns:xi="http://www.w3.org/2001/XInclude"
version="5.0"
xml:id="sec-release-19.09-incompatibilities">
<title>Backward Incompatibilities</title>
<para>
When upgrading from a previous release, please be aware of the following
incompatible changes:
</para>
<itemizedlist>
<listitem>
<para>
The directory where Prometheus will store its metric data is now
managed by systemd's StateDirectory mechanism. It still defaults
to <literal>/var/lib/prometheus</literal>.
</para>
<para>
Its location can be specified by the new
<option>services.prometheus.stateDir</option> option which
defaults to <literal>prometheus</literal>. Note that this should
be a directory relative to <literal>/var/lib/</literal>.
</para>
<para>
The option <option>services.prometheus.dataDir</option> has been
deprecated. You can still set it but it's now required to have
<literal>/var/lib/</literal> as a prefix and you can't set
<option>services.prometheus.stateDir</option> at the same time.
</para>
</listitem>
</itemizedlist>
</section>

View file

@ -4,9 +4,24 @@ with lib;
let
cfg = config.services.prometheus;
cfg2 = config.services.prometheus2;
promUser = "prometheus";
promGroup = "prometheus";
stateDir =
if cfg.stateDir != null
then cfg.stateDir
else
if cfg.dataDir != null
then
# This assumes /var/lib/ is a prefix of cfg.dataDir.
# This is checked as an assertion below.
removePrefix stateDirBase cfg.dataDir
else "prometheus";
stateDirBase = "/var/lib/";
workingDir = stateDirBase + stateDir;
workingDir2 = stateDirBase + cfg2.stateDir;
# Get a submodule without any embedded metadata:
_filter = x: filterAttrs (k: v: k != "_module") x;
@ -17,13 +32,23 @@ let
promtool ${what} $out
'';
# a wrapper that verifies that the configuration is valid for
# prometheus 2
prom2toolCheck = what: name: file:
pkgs.runCommand
"${name}-${replaceStrings [" "] [""] what}-checked"
{ buildInputs = [ cfg2.package ]; } ''
ln -s ${file} $out
promtool ${what} $out
'';
# Pretty-print JSON to a file
writePrettyJSON = name: x:
pkgs.runCommand name { preferLocalBuild = true; } ''
echo '${builtins.toJSON x}' | ${pkgs.jq}/bin/jq . > $out
'';
# This becomes the main config file
# This becomes the main config file for Prometheus 1
promConfig = {
global = cfg.globalConfig;
rule_files = map (promtoolCheck "check-rules" "rules") (cfg.ruleFiles ++ [
@ -35,20 +60,53 @@ let
generatedPrometheusYml = writePrettyJSON "prometheus.yml" promConfig;
prometheusYml = let
yml = if cfg.configText != null then
yml = if cfg.configText != null then
pkgs.writeText "prometheus.yml" cfg.configText
else generatedPrometheusYml;
in promtoolCheck "check-config" "prometheus.yml" yml;
cmdlineArgs = cfg.extraFlags ++ [
"-storage.local.path=${cfg.dataDir}/metrics"
"-storage.local.path=${workingDir}/metrics"
"-config.file=${prometheusYml}"
"-web.listen-address=${cfg.listenAddress}"
"-alertmanager.notification-queue-capacity=${toString cfg.alertmanagerNotificationQueueCapacity}"
"-alertmanager.timeout=${toString cfg.alertmanagerTimeout}s"
(optionalString (cfg.alertmanagerURL != []) "-alertmanager.url=${concatStringsSep "," cfg.alertmanagerURL}")
(optionalString (cfg.webExternalUrl != null) "-web.external-url=${cfg.webExternalUrl}")
];
] ++
optional (cfg.alertmanagerURL != []) "-alertmanager.url=${concatStringsSep "," cfg.alertmanagerURL}" ++
optional (cfg.webExternalUrl != null) "-web.external-url=${cfg.webExternalUrl}";
# This becomes the main config file for Prometheus 2
promConfig2 = {
global = cfg2.globalConfig;
rule_files = map (prom2toolCheck "check rules" "rules") (cfg2.ruleFiles ++ [
(pkgs.writeText "prometheus.rules" (concatStringsSep "\n" cfg2.rules))
]);
scrape_configs = cfg2.scrapeConfigs;
alerting = optionalAttrs (cfg2.alertmanagerURL != []) {
alertmanagers = [{
static_configs = [{
targets = cfg2.alertmanagerURL;
}];
}];
};
};
generatedPrometheus2Yml = writePrettyJSON "prometheus.yml" promConfig2;
prometheus2Yml = let
yml = if cfg2.configText != null then
pkgs.writeText "prometheus.yml" cfg2.configText
else generatedPrometheus2Yml;
in prom2toolCheck "check config" "prometheus.yml" yml;
cmdlineArgs2 = cfg2.extraFlags ++ [
"--storage.tsdb.path=${workingDir2}/data/"
"--config.file=${prometheus2Yml}"
"--web.listen-address=${cfg2.listenAddress}"
"--alertmanager.notification-queue-capacity=${toString cfg2.alertmanagerNotificationQueueCapacity}"
"--alertmanager.timeout=${toString cfg2.alertmanagerTimeout}s"
] ++
optional (cfg2.webExternalUrl != null) "--web.external-url=${cfg2.webExternalUrl}";
promTypes.globalConfig = types.submodule {
options = {
@ -403,10 +461,21 @@ in {
};
dataDir = mkOption {
type = types.path;
default = "/var/lib/prometheus";
type = types.nullOr types.path;
default = null;
description = ''
Directory to store Prometheus metrics data.
This option is deprecated, please use <option>services.prometheus.stateDir</option>.
'';
};
stateDir = mkOption {
type = types.nullOr types.str;
default = null;
description = ''
Directory below <literal>${stateDirBase}</literal> to store Prometheus metrics data.
This directory will be created automatically using systemd's StateDirectory mechanism.
Defaults to <literal>prometheus</literal>.
'';
};
@ -497,30 +566,201 @@ in {
'';
};
};
};
services.prometheus2 = {
config = mkIf cfg.enable {
users.groups.${promGroup}.gid = config.ids.gids.prometheus;
users.users.${promUser} = {
description = "Prometheus daemon user";
uid = config.ids.uids.prometheus;
group = promGroup;
home = cfg.dataDir;
createHome = true;
};
systemd.services.prometheus = {
wantedBy = [ "multi-user.target" ];
after = [ "network.target" ];
script = ''
#!/bin/sh
exec ${cfg.package}/bin/prometheus \
${concatStringsSep " \\\n " cmdlineArgs}
'';
serviceConfig = {
User = promUser;
Restart = "always";
WorkingDirectory = cfg.dataDir;
enable = mkOption {
type = types.bool;
default = false;
description = ''
Enable the Prometheus 2 monitoring daemon.
'';
};
package = mkOption {
type = types.package;
default = pkgs.prometheus_2;
defaultText = "pkgs.prometheus_2";
description = ''
The prometheus2 package that should be used.
'';
};
listenAddress = mkOption {
type = types.str;
default = "0.0.0.0:9090";
description = ''
Address to listen on for the web interface, API, and telemetry.
'';
};
stateDir = mkOption {
type = types.str;
default = "prometheus2";
description = ''
Directory below <literal>${stateDirBase}</literal> to store Prometheus metrics data.
This directory will be created automatically using systemd's StateDirectory mechanism.
Defaults to <literal>prometheus2</literal>.
'';
};
extraFlags = mkOption {
type = types.listOf types.str;
default = [];
description = ''
Extra commandline options when launching Prometheus 2.
'';
};
configText = mkOption {
type = types.nullOr types.lines;
default = null;
description = ''
If non-null, this option defines the text that is written to
prometheus.yml. If null, the contents of prometheus.yml is generated
from the structured config options.
'';
};
globalConfig = mkOption {
type = promTypes.globalConfig;
default = {};
apply = _filter;
description = ''
Parameters that are valid in all configuration contexts. They
also serve as defaults for other configuration sections
'';
};
rules = mkOption {
type = types.listOf types.str;
default = [];
description = ''
Alerting and/or Recording rules to evaluate at runtime.
'';
};
ruleFiles = mkOption {
type = types.listOf types.path;
default = [];
description = ''
Any additional rules files to include in this configuration.
'';
};
scrapeConfigs = mkOption {
type = types.listOf promTypes.scrape_config;
default = [];
apply = x: map _filter x;
description = ''
A list of scrape configurations.
'';
};
alertmanagerURL = mkOption {
type = types.listOf types.str;
default = [];
description = ''
List of Alertmanager URLs to send notifications to.
'';
};
alertmanagerNotificationQueueCapacity = mkOption {
type = types.int;
default = 10000;
description = ''
The capacity of the queue for pending alert manager notifications.
'';
};
alertmanagerTimeout = mkOption {
type = types.int;
default = 10;
description = ''
Alert manager HTTP API timeout (in seconds).
'';
};
webExternalUrl = mkOption {
type = types.nullOr types.str;
default = null;
example = "https://example.com/";
description = ''
The URL under which Prometheus is externally reachable (for example,
if Prometheus is served via a reverse proxy).
'';
};
};
};
};
config = mkMerge [
(mkIf (cfg.enable || cfg2.enable) {
users.groups.${promGroup}.gid = config.ids.gids.prometheus;
users.users.${promUser} = {
description = "Prometheus daemon user";
uid = config.ids.uids.prometheus;
group = promGroup;
};
})
(mkIf cfg.enable {
warnings =
optional (cfg.dataDir != null) ''
The option services.prometheus.dataDir is deprecated, please use
services.prometheus.stateDir.
'';
assertions = [
{
assertion = !(cfg.dataDir != null && cfg.stateDir != null);
message =
"The options services.prometheus.dataDir and services.prometheus.stateDir" +
" can't both be set at the same time! It's recommended to only set the latter" +
" since the former is deprecated.";
}
{
assertion = cfg.dataDir != null -> hasPrefix stateDirBase cfg.dataDir;
message =
"The option services.prometheus.dataDir should have ${stateDirBase} as a prefix!";
}
{
assertion = cfg.stateDir != null -> !hasPrefix "/" cfg.stateDir;
message =
"The option services.prometheus.stateDir shouldn't be an absolute directory." +
" It should be a directory relative to ${stateDirBase}.";
}
{
assertion = cfg2.stateDir != null -> !hasPrefix "/" cfg2.stateDir;
message =
"The option services.prometheus2.stateDir shouldn't be an absolute directory." +
" It should be a directory relative to ${stateDirBase}.";
}
];
systemd.services.prometheus = {
wantedBy = [ "multi-user.target" ];
after = [ "network.target" ];
serviceConfig = {
ExecStart = "${cfg.package}/bin/prometheus" +
optionalString (length cmdlineArgs != 0) (" \\\n " +
concatStringsSep " \\\n " cmdlineArgs);
User = promUser;
Restart = "always";
WorkingDirectory = workingDir;
StateDirectory = stateDir;
};
};
})
(mkIf cfg2.enable {
systemd.services.prometheus2 = {
wantedBy = [ "multi-user.target" ];
after = [ "network.target" ];
serviceConfig = {
ExecStart = "${cfg2.package}/bin/prometheus" +
optionalString (length cmdlineArgs2 != 0) (" \\\n " +
concatStringsSep " \\\n " cmdlineArgs2);
User = promUser;
Restart = "always";
WorkingDirectory = workingDir2;
StateDirectory = cfg2.stateDir;
};
};
})
];
}

View file

@ -194,6 +194,7 @@ in
predictable-interface-names = handleTest ./predictable-interface-names.nix {};
printing = handleTest ./printing.nix {};
prometheus = handleTest ./prometheus.nix {};
prometheus2 = handleTest ./prometheus-2.nix {};
prometheus-exporters = handleTest ./prometheus-exporters.nix {};
prosody = handleTest ./prosody.nix {};
proxy = handleTest ./proxy.nix {};

View file

@ -0,0 +1,34 @@
import ./make-test.nix {
name = "prometheus-2";
nodes = {
one = { pkgs, ... }: {
services.prometheus2 = {
enable = true;
scrapeConfigs = [{
job_name = "prometheus";
static_configs = [{
targets = [ "127.0.0.1:9090" ];
labels = { instance = "localhost"; };
}];
}];
rules = [
''
groups:
- name: test
rules:
- record: testrule
expr: count(up{job="prometheus"})
''
];
};
};
};
testScript = ''
startAll;
$one->waitForUnit("prometheus2.service");
$one->waitForOpenPort(9090);
$one->succeed("curl -s http://127.0.0.1:9090/metrics");
'';
}