infra/modules/prometheus/default.nix

Ignoring revisions in .git-blame-ignore-revs. Click here to bypass and see the normal blame view.

192 lines
5 KiB
Nix
Raw Normal View History

2024-04-28 17:05:12 +00:00
{
config,
lib,
pkgs,
flake,
...
}:
{
2024-05-15 15:15:46 +00:00
age.secrets.alertmanager-envfile = {
file = "${flake.self}/secrets/alertmanager-envfile.age";
mode = "600";
owner = "alertmanager";
};
2024-04-28 17:05:12 +00:00
services.nginx.virtualHosts."alerts.${config.pub-solar-os.networking.domain}" = {
enableACME = true;
forceSSL = true;
listenAddresses = [
"10.7.6.5"
"[fd00:fae:fae:fae:fae:5::]"
];
locations."/" = {
proxyPass = "http://127.0.0.1:${toString config.services.prometheus.alertmanager.port}";
};
};
2024-04-28 17:05:12 +00:00
services.prometheus = {
enable = true;
port = 9001;
exporters = {
node = {
enable = true;
enabledCollectors = [ "systemd" ];
port = 9002;
};
};
globalConfig = {
scrape_interval = "10s";
scrape_timeout = "9s";
};
scrapeConfigs = [
{
job_name = "node-exporter";
2024-04-28 17:05:12 +00:00
static_configs = [
{
targets = [ "nachtigall.wg.${config.pub-solar-os.networking.domain}" ];
2024-04-28 17:05:12 +00:00
labels = {
instance = "nachtigall";
};
}
{
2024-06-06 10:56:55 +00:00
targets = [
"metronom.wg.${config.pub-solar-os.networking.domain}:${toString config.services.prometheus.exporters.node.port}"
];
labels = {
instance = "metronom";
};
}
{
2024-06-06 10:56:55 +00:00
targets = [
"tankstelle.wg.${config.pub-solar-os.networking.domain}:${toString config.services.prometheus.exporters.node.port}"
];
labels = {
instance = "tankstelle";
};
}
{
targets = [
"trinkgenossin.wg.${config.pub-solar-os.networking.domain}:${toString config.services.prometheus.exporters.node.port}"
];
labels = {
instance = "trinkgenossin";
};
}
{
targets = [
"delite.wg.${config.pub-solar-os.networking.domain}:${toString config.services.prometheus.exporters.node.port}"
];
labels = {
instance = "delite";
};
}
{
targets = [
"blue-shell.wg.${config.pub-solar-os.networking.domain}:${toString config.services.prometheus.exporters.node.port}"
];
labels = {
instance = "blue-shell";
};
}
2024-04-28 17:05:12 +00:00
];
}
{
job_name = "matrix-synapse";
metrics_path = "/_synapse/metrics";
static_configs = [
{
targets = [ "nachtigall.wg.${config.pub-solar-os.networking.domain}" ];
2024-04-28 17:05:12 +00:00
labels = {
instance = "nachtigall";
};
}
];
}
{
job_name = "garage";
static_configs = [
{
targets = [
"trinkgenossin.wg.${config.pub-solar-os.networking.domain}:3903"
];
labels = {
instance = "trinkgenossin";
};
}
{
targets = [
"delite.wg.${config.pub-solar-os.networking.domain}:3903"
];
labels = {
instance = "delite";
};
}
{
targets = [
"blue-shell.wg.${config.pub-solar-os.networking.domain}:3903"
];
labels = {
instance = "blue-shell";
};
}
];
}
2024-04-28 17:05:12 +00:00
];
2024-04-26 22:08:23 +00:00
ruleFiles = [
2024-05-12 19:17:49 +00:00
(pkgs.writeText "prometheus-rules.yml" (
builtins.toJSON {
groups = [
{
name = "alerting-rules";
rules = import ./alert-rules.nix { inherit lib; };
}
];
}
))
2024-04-26 22:08:23 +00:00
];
2024-05-12 19:17:49 +00:00
alertmanagers = [ { static_configs = [ { targets = [ "localhost:9093" ]; } ]; } ];
2024-04-26 22:08:23 +00:00
alertmanager = {
enable = true;
# port = 9093; # Default
2024-05-12 20:17:58 +00:00
webExternalUrl = "https://alerts.pub.solar";
2024-05-15 15:15:46 +00:00
environmentFile = "${config.age.secrets.alertmanager-envfile.path}";
2024-04-26 22:08:23 +00:00
configuration = {
route = {
receiver = "all";
group_by = [ "instance" ];
group_wait = "30s";
group_interval = "2m";
repeat_interval = "24h";
};
2024-05-12 19:17:49 +00:00
receivers = [
{
name = "all";
# Email config documentation: https://prometheus.io/docs/alerting/latest/configuration/#email_config
email_configs = [
{
send_resolved = true;
2024-05-12 20:17:58 +00:00
to = "admins@pub.solar";
2024-05-12 19:17:49 +00:00
from = "alerts@pub.solar";
smarthost = "mail.pub.solar:465";
2024-05-12 20:17:58 +00:00
auth_username = "admins@pub.solar";
2024-05-15 15:15:46 +00:00
auth_password = "$SMTP_AUTH_PASSWORD";
require_tls = false;
2024-05-12 19:17:49 +00:00
}
];
# TODO:
# For matrix notifications, look into: https://github.com/pinpox/matrix-hook and add a webhook
# webhook_configs = [ { url = "http://127.0.0.1:11000/alert"; } ];
}
];
2024-04-26 22:08:23 +00:00
};
};
2024-04-28 17:05:12 +00:00
};
}