Merge pull request 'feat: grafana + prometheus + loki on flora-6' (#77) from feat/grafana into main

Reviewed-on: #77
Reviewed-by: Akshay Mankar <axeman@noreply.git.pub.solar>

hosts/flora-6/apps/caddy.nix

@@ -27,6 +27,25 @@
           reverse_proxy :4000
         '';
       };
+      "flora-6.pub.solar" = {
+        logFormat = lib.mkForce ''
+          output discard
+        '';
+        extraConfig = ''
+          basicauth * {
+	    hakkonaut $2a$14$mmIAy/Ezm6YGohUtXa2mWeW6Bcw1MQXPhrRbz14jAD2iUu3oob/t.
+          }
+          reverse_proxy :${toString config.services.loki.configuration.server.http_listen_port}
+        '';
+      };
+      "grafana.pub.solar" = {
+        logFormat = lib.mkForce ''
+          output discard
+        '';
+        extraConfig = ''
+          reverse_proxy :${toString config.services.grafana.settings.server.http_port}
+        '';
+      };
       "obs-portal.pub.solar" = {
         logFormat = lib.mkForce ''
           output discard

hosts/flora-6/apps/forgejo-actions-runner.nix

@@ -10,6 +10,10 @@
     mode = "644";
   };
 
+  # Trust docker bridge interface traffic
+  # Needed for the docker runner to communicate with the act_runner cache
+  networking.firewall.trustedInterfaces = [ "br-+" ];
+
   # forgejo actions runner
   # https://forgejo.org/docs/latest/admin/actions/
   # https://docs.gitea.com/usage/actions/quickstart

hosts/flora-6/apps/grafana.nix

@@ -0,0 +1,108 @@
+{
+  config,
+  lib,
+  pkgs,
+  flake,
+  ...
+}: {
+  age.secrets.grafana-admin-password = {
+    file = "${flake.self}/secrets/grafana-admin-password.age";
+    mode = "644";
+    owner = "grafana";
+  };
+  age.secrets.grafana-smtp-password = {
+    file = "${flake.self}/secrets/grafana-smtp-password.age";
+    mode = "644";
+    owner = "grafana";
+  };
+  age.secrets.grafana-keycloak-client-secret = {
+    file = "${flake.self}/secrets/grafana-keycloak-client-secret.age";
+    mode = "644";
+    owner = "grafana";
+  };
+
+  environment.etc = {
+    "grafana-dashboards/node-exporter-full_rev33.json" = {
+      source = ./grafana-dashboards/node-exporter-full_rev33.json;
+      group = "grafana";
+      user = "grafana";
+    };
+  };
+
+  services.grafana = {
+    enable = true;
+    settings = {
+      server = {
+        # Listening Address
+        http_addr = "127.0.0.1";
+        # and Port
+        http_port = 3000;
+        # Grafana needs to know on which domain and URL it's running
+        domain = "grafana.pub.solar";
+        root_url = "https://grafana.pub.solar";
+        enable_gzip = true;
+      };
+      smtp = {
+        enabled = true;
+        host = "mail.greenbaum.zone:465";
+        user = "admins@pub.solar";
+        password = "\$__file{${config.age.secrets.grafana-smtp-password.path}}";
+        from_address = "no-reply@pub.solar";
+        from_name = "grafana.pub.solar";
+        ehlo_identity = "flora-6.pub.solar";
+      };
+      security = {
+        admin_email = "crew@pub.solar";
+        admin_password = "\$__file{${config.age.secrets.grafana-admin-password.path}}";
+      };
+      "auth.generic_oauth" = {
+        enabled = true;
+        name = "pub.solar ID";
+        allow_sign_up = true;
+        client_id = "grafana";
+        client_secret = "\$__file{${config.age.secrets.grafana-keycloak-client-secret.path}}";
+        scopes = "openid email profile offline_access roles";
+        email_attribute_path = "email";
+        login_attribute_path = "preferred_username";
+        name_attribute_path = "full_name";
+        auth_url = "https://auth.pub.solar/realms/pub.solar/protocol/openid-connect/auth";
+        token_url = "https://auth.pub.solar/realms/pub.solar/protocol/openid-connect/token";
+        api_url = "https://auth.pub.solar/realms/pub.solar/protocol/openid-connect/userinfo";
+        role_attribute_path = "contains(roles[*], 'admin') && 'GrafanaAdmin' || 'Viewer'";
+        allow_assign_grafana_admin = true;
+      };
+    };
+    provision = {
+      enable = true;
+      datasources = {
+        settings = {
+          datasources = [
+            {
+              name = "Prometheus";
+              type = "prometheus";
+              access = "proxy";
+              url = "http://127.0.0.1:${toString config.services.prometheus.port}";
+              isDefault = true;
+            }
+            {
+              name = "Loki";
+              type = "loki";
+              access = "proxy";
+              url = "http://127.0.0.1:${toString config.services.loki.configuration.server.http_listen_port}";
+            }
+          ];
+        };
+      };
+      dashboards = {
+        settings = {
+          providers = [
+            {
+              name = "pub.solar Dashboards";
+              options.path = "/etc/grafana-dashboards";
+            }
+          ];
+        };
+      };
+    };
+  };
+}

hosts/flora-6/apps/loki.nix

@@ -0,0 +1,85 @@
+{
+  config,
+  lib,
+  pkgs,
+  flake,
+  ...
+}: {
+  # source: https://gist.github.com/rickhull/895b0cb38fdd537c1078a858cf15d63e
+  # https://grafana.com/docs/loki/latest/configure/examples/#1-local-configuration-exampleyaml
+  services.loki = {
+    enable = true;
+    configuration = {
+      server.http_listen_port = 3100;
+      auth_enabled = false;
+      common = {
+        ring = {
+          instance_addr = "127.0.0.1";
+          kvstore = {
+            store = "inmemory";
+          };
+        };
+        replication_factor = 1;
+        path_prefix = "/var/lib/loki";
+        storage = {
+          filesystem = {
+            chunks_directory = "chunks/";
+            rules_directory = "rules/";
+          };
+        };
+      };
+      # Keep logs for 4 weeks
+      # https://grafana.com/docs/loki/latest/operations/storage/retention/
+      limits_config.retention_period = "4w";
+      compactor = {
+        shared_store = "filesystem";
+        compaction_interval = "10m";
+        retention_enabled = true;
+        retention_delete_delay = "2h";
+        retention_delete_worker_count = 150;
+      };
+      schema_config = {
+        configs = [{
+          from = "2020-05-15";
+          store = "boltdb-shipper";
+          object_store = "filesystem";
+          schema = "v11";
+          index = {
+            prefix = "index_";
+            period = "24h";
+          };
+        }];
+      };
+    };
+  };
+
+  services.promtail = {
+    enable = true;
+    configuration = {
+      server = {
+        http_listen_port = 9080;
+        grpc_listen_port = 0;
+      };
+      positions = {
+        filename = "/tmp/positions.yaml";
+      };
+      clients = [{
+        url = "http://127.0.0.1:${toString config.services.loki.configuration.server.http_listen_port}/loki/api/v1/push";
+      }];
+      scrape_configs = [{
+        job_name = "journal";
+        journal = {
+          max_age = "24h";
+          labels = {
+            job = "systemd-journal";
+            host = "flora-6";
+          };
+        };
+        relabel_configs = [{
+          source_labels = [ "__journal__systemd_unit" ];
+          target_label = "unit";
+        }];
+      }];
+    };
+  };
+}

hosts/flora-6/apps/prometheus.nix

@@ -0,0 +1,55 @@
+{
+  config,
+  lib,
+  pkgs,
+  flake,
+  ...
+}: {
+  age.secrets.nachtigall-metrics-prometheus-basic-auth-password = {
+    file = "${flake.self}/secrets/nachtigall-metrics-prometheus-basic-auth-password.age";
+    mode = "600";
+    owner = "prometheus";
+  };
+
+  services.prometheus = {
+    enable = true;
+    port = 9001;
+    exporters = {
+      node = {
+        enable = true;
+        enabledCollectors = [ "systemd" ];
+        port = 9002;
+      };
+    };
+    globalConfig = {
+      scrape_interval = "10s";
+      scrape_timeout = "9s";
+    };
+    scrapeConfigs = [
+      {
+        job_name = "http-targets";
+        static_configs = [{
+          targets = [ "127.0.0.1:${toString config.services.prometheus.exporters.node.port}" ];
+          labels = {
+            instance = "flora-6";
+          };
+        }];
+      }
+      {
+        job_name = "https-targets";
+        scheme = "https";
+        metrics_path = "/metrics";
+        basic_auth = {
+          username = "hakkonaut";
+          password_file = "${config.age.secrets.nachtigall-metrics-prometheus-basic-auth-password.path}";
+        };
+        static_configs = [{
+          targets = [ "nachtigall.pub.solar" ];
+          labels = {
+            instance = "nachtigall";
+          };
+        }];
+      }
+    ];
+  };
+}

hosts/flora-6/configuration.nix

@@ -29,6 +29,13 @@ in {
     # Force getting the hostname from cloud-init
     networking.hostName = lib.mkDefault "";
 
+    # We use cloud-init to configure networking, this option should fix
+    # systemd-networkd-wait-online timeouts
+    #systemd.services."systemd-networkd".environment.SYSTEMD_LOG_LEVEL = "debug";
+    systemd.network.wait-online.ignoredInterfaces = [
+     "docker0"
+    ];
+
     # List services that you want to enable:
     services.cloud-init.enable = true;
     services.cloud-init.ext4.enable = true;

hosts/flora-6/default.nix

@@ -11,5 +11,8 @@
 
       ./apps/drone.nix
       ./apps/forgejo-actions-runner.nix
+      ./apps/grafana.nix
+      ./apps/prometheus.nix
+      ./apps/loki.nix
     ];
 }

hosts/nachtigall/apps/nginx-prometheus-exporters.nix

@@ -0,0 +1,19 @@
+{ config, flake, ... }:
+
+{
+  age.secrets.nachtigall-metrics-nginx-basic-auth = {
+    file = "${flake.self}/secrets/nachtigall-metrics-nginx-basic-auth.age";
+    mode = "600";
+    owner = "nginx";
+  };
+  services.nginx.virtualHosts = {
+    "nachtigall.pub.solar" = {
+      enableACME = true;
+      addSSL = true;
+      basicAuthFile = "${config.age.secrets.nachtigall-metrics-nginx-basic-auth.path}";
+      locations."/metrics" = {
+        proxyPass = "http://127.0.0.1:${toString(config.services.prometheus.exporters.node.port)}";
+      };
+    };
+  };
+}

hosts/nachtigall/apps/prometheus-exporters.nix

@@ -0,0 +1,14 @@
+{
+  config,
+  ...
+}: {
+  services.prometheus = {
+    exporters = {
+      node = {
+        enable = true;
+        enabledCollectors = [ "systemd" ];
+        port = 9002;
+      };
+    };
+  };
+}

hosts/nachtigall/apps/promtail.nix

@@ -0,0 +1,47 @@
+{
+  config,
+  lib,
+  pkgs,
+  flake,
+  ...
+}: {
+  age.secrets.nachtigall-metrics-prometheus-basic-auth-password = {
+    file = "${flake.self}/secrets/nachtigall-metrics-prometheus-basic-auth-password.age";
+    mode = "600";
+    owner = "promtail";
+  };
+
+  services.promtail = {
+    enable = true;
+    configuration = {
+      server = {
+        http_listen_port = 9080;
+        grpc_listen_port = 0;
+      };
+      positions = {
+        filename = "/tmp/positions.yaml";
+      };
+      clients = [{
+        url = "https://flora-6.pub.solar/loki/api/v1/push";
+        basic_auth = {
+          username = "hakkonaut";
+          password_file = "${config.age.secrets.nachtigall-metrics-prometheus-basic-auth-password.path}";
+        };
+      }];
+      scrape_configs = [{
+        job_name = "journal";
+        journal = {
+          max_age = "24h";
+          labels = {
+            job = "systemd-journal";
+            host = "nachtigall";
+          };
+        };
+        relabel_configs = [{
+          source_labels = [ "__journal__systemd_unit" ];
+          target_label = "unit";
+        }];
+      }];
+    };
+  };
+}

hosts/nachtigall/default.nix

@@ -18,17 +18,20 @@
       ./apps/mastodon.nix
       ./apps/mediawiki.nix
       ./apps/nextcloud.nix
-      ./apps/owncast.nix
       ./apps/nginx-mastodon.nix
       ./apps/nginx-mastodon-files.nix
+      ./apps/nginx-prometheus-exporters.nix
       ./apps/nginx-website.nix
       ./apps/opensearch.nix
+      ./apps/owncast.nix
       ./apps/postgresql.nix
+      ./apps/prometheus-exporters.nix
+      ./apps/promtail.nix
       ./apps/searx.nix
 
+      ./apps/matrix/irc.nix
       ./apps/matrix/mautrix-telegram.nix
       ./apps/matrix/synapse.nix
-      ./apps/matrix/irc.nix
       ./apps/nginx-matrix.nix
     ];
 }

secrets/grafana-keycloak-client-secret.age

@@ -0,0 +1,28 @@
+age-encryption.org/v1
+-> ssh-ed25519 Y0ZZaw M6ha3gQ4Oq4PdymYZ5ZG0qGwFlpCYfJdhOBwH9n1gxg
+zCtB0PJanufNdV0ShynDT0Z/2jxMFDRby8xsfv6YPaA
+-> ssh-ed25519 uYcDNw V89Ll4HJ3ZkQegiCI6gswz736domVgDGSDCA8bZBwHs
+W7IrEL+1xUXuVdy6A61z6P+pS/ajTGPL+qv+9Jh8UxI
+-> ssh-rsa kFDS0A
+SV6QVIW8MCQVB8ABiOGxLTXEMO6rfeG82CktBFtf76WeIYzlkho/IaGgWXoqoIQ0
+KC/ev7vNGnB01AOWe/xkuMZDRvK+qGaOLB7wpZG1cJhqSon9oZtztoDjd/Crp5K0
+nfeHjY9E/jgFr0KYeaLedw5OJuaOw4YiuKyTThVbpRZwbof30nvHXqrYKPZJi1gq
+s5spoWYH2ijZi9mrJojP2ZqK5DJjCteXqP1YHdz3LjxomoDyl5cv/tLNsvrptfxD
+FvZMcPrvrC/IWqJ8qGW+f8ENUGyjXxx6jFQ2WN9IMIdJYk5bz458ip3GKqnAlwi3
+SZbaxRuEYEoy6ikKGRuXMAwpJd3YXcRcaRdetw0a4grdD6hF21bTl2+LnTb1ydnb
+frzeoXaqbBdhEyLpZFAmGLydteIyA/Kl/D/PEJ0MHc0G0EGofMm6YsNJJrP3mQgi
+mXC2Kto6WV/JLVEnURayf12rPR1T/VPIyYZ/Xi9HfPh0p3Y21nadPAcEq/PltWgR
+AqELfBbVpNtcxTP2pjEJqGskJCYKAmMeM+yQ0moKVmuMWicahMqjQRJO1jnvTwwd
+GhJlUO32EuI6Fn6sApthv2FfLrle+x0H4/v9xvHDJIVSmLYtzK+9ueUPn/A1x8X1
+lGeJh+ecEV2r630insGAp8WQzyXhraHrn3lgyacwRmA
+-> ssh-ed25519 YFSOsg KKhXh/XW7iF7wMA7JD9fbgmty5yVPaSS1vGdHz0Xh0M
+eLJc+F/yIR1ckZX/npLI+l3I2iB+OrKBkJAQTkbWVF4
+-> ssh-ed25519 iHV63A xoJ7Tr8mKgYVPPeJYBnOHLBY5E0i34vEQR3pMVKxbAc
+TKqc9Y/RpnfTP3CNvCearB4FuvNmW0mcGVLh7Ebjzeo
+-> ssh-ed25519 BVsyTA LaMK6X/MJyQTQ24p9uHXh75leMcp/akCA2YZACEG03M
+psw6sVlNGT8WsG3L9kbXdrhqxp8hIdSF7s4o60jTYgY
+-> vcxmk`-grease 8^p$~+LB -G)+N&$^ P)7#7[wX
+8TyK2RrSHFuMyFy9YY7ZI6RSduF5hw6xZKhiysVkif4Husb1flN8QVmWtoW8laWz
+n8772TmNTcfq5ebUp+UA+S6MVgf75D1GnDumEDH/LbM4LNjRZzyw3nBGu/Q
+--- Ouu56e69gTpAY1ouLPlzI/n6geKz1CMmTl8wAVyIDPM
+Ÿ·¢5¿ä7W>J@°óðj’Á–€l_ƒ¥«­Ï/œö÷ú=ßÕ»‰4(²<18>²K» µÅÑ¥„zSÌsæ

secrets/nachtigall-metrics-nginx-basic-auth.age

@@ -0,0 +1,31 @@
+age-encryption.org/v1
+-> ssh-ed25519 Y0ZZaw FWuk2kYGB+GfoY3rWfeCosoBOLvUHrH7SR8Fv18o+XI
+YyOTULtyOJ3vfAOnYSMzeCCyipJ4Fqrr3PJgRtbElJg
+-> ssh-ed25519 iDKjwg Bq6lNuS5MOhsU/7ypHw/E70BktIA+SmN6e3pvrIqRBQ
+Xo0OOUXfOkPQfArhqSJyiAkH5lxcJIAO7M5krkCZNfc
+-> ssh-ed25519 uYcDNw EfB1B4CSNk8Oe5B7T+KSl9O5OsCrulaLOjR3PBtxpSk
+xJxkmBSENc5JosdRiEAC3a41WI6TmTlTxm+lclup+g4
+-> ssh-rsa kFDS0A
+dYH3A43wClFnDQp8m3ZnhTK5d8LeG6ZkqDQ5dS1yB//4G5TaUnMqOp5Q2G1gbgXY
+Zu9qYOHdUydn5HIRSwBXj/KbBm5xJ1zFImOszn7S5mk4iReHFyTnSzAi4utatQcY
+DEjGnvKKRoc7ih08+F44kq6DYnhUBFqF8eigQZIsyeWpiW6C1FzasL0KnXoedPG2
+AYJForNB8zKp7a2Evxi0MY7a+ldHAekktz1Fta2u9MvrWUtqP/yLqJhCwCNvos7J
+kG+XO4j0kiOQCIO9TOeLAu59+VCVM64mY+dp+xc8tX0fWuu7ItSAh6jRHzfgSKjC
+qDJc/1YpUG1EnYSH39mfVox3ndeMuVrG6Q1h509jZuxsw/zoDsbY3bbhTaUQ3X8Y
+5ShCponnEGBLqeSm1gALCAnlgu8IS4gL6ePKuAhN0qMYj6iiXP/Ugp3lTcv1TvFD
+KINnV/tas1CO3PApQm6JgijHEPT9zyUbqR/xN06+OCWbg4hHuEix+0OhM1T5w2xC
+KvKF30iUK0tU2hZvKdku2MpbP4N0cQLqBEWiyrUKHRMCdXi3kyO5D84UdWXvETAt
+BfEvZ8ZG5fiSXzbPLxVqObXFZUirLuWomWtstqkDuadL9xJkTcsbr8ZCCNpPhxdL
+oOfao+tox3RBilAS3AfQVhrPvD2rVUptm+0nPtnO3rY
+-> ssh-ed25519 YFSOsg T2OdtA0kY4DqDIxE1QxMV5aCygvKlI5LgXQ+QYYuOko
+l0Kzo02jGISCT1zrGf5soXYj7FMVrN/9REF3Zscbmik
+-> ssh-ed25519 iHV63A 75daRGD2TQ/mXRsckaH9sGGkHMkLxgHFhn0eDdkDsU8
+TXeoLqfU0ywQucPayYoG43Gr56uZoYIWaK9F2YJJ0FM
+-> ssh-ed25519 BVsyTA J/xNtG1CAzfoiKPsnWwDp4pId7d3MywXpfhKAmpze3I
+8uMO07Se/6krP79flt+XZfjIsw12kWsoD6LqZyLG70M
+-> B-grease y3$t@ ; Bs *w
+dUrvWB09znCDyvO7RnduMguc9pWTn19q1fc0MHFUXk7WQWns+4kpJIX1qljB5hz/
+NPAbNzwMDQKj6awHAth1iFLaEw
+--- rI4jrrXCiUpV/EzGsla+lxONmL5/Eel/LODoIM80jcM
+˜_°0àÆ7Jˆq•[÷ç<>è'/ù‘õŽi„Ü<E2809E>Òl°mÙ
+ÌÂ!JPþ¼>œ…wk¡ž·³¤+ é™)ÚÈPhUÜóç²O=>k=?ÂTÐ

secrets/nachtigall-metrics-prometheus-basic-auth-password.age

@@ -0,0 +1,30 @@
+age-encryption.org/v1
+-> ssh-ed25519 Y0ZZaw CxhF1nK1+6OmJb/68UQ4mBIqxGgr8ngkNsL9dfaPN2s
+jZ/JBaTCjFcL0SAGVx5ECDanVn4TGt0g2yn2OQOP9iY
+-> ssh-ed25519 iDKjwg D/xqqA53Lw2UQJesg27wmK/UNCV+s914mvMlbKN1rhg
+AOg0SkPvSotuSHk33zVfRxB0wn67a29YWc/itDUZ/LQ
+-> ssh-ed25519 uYcDNw /QdfQUJmBMQZ+KRCst1gA0LqFGvM1K91ZL/RIRP+qBc
+Ttksa44OdwLuRmgYPC2rIn+wy/SooRPUq8gQTR+pF0k
+-> ssh-rsa kFDS0A
+L9MGJFRceqbge3EF/rqXdT13jt9faxP1NmfRB3i2mrTasvCaovc/62bA0UmlsB/9
+Y3hIzo28d6pZRcMm91l6PhWV0M33YNwPQf87vd7klv++1aMIdZ6/jHsQiohIBkRd
+4pBe6rrx/lUqEqfQVYUFPfRE50ufkw+hRw/NJCvcBgHgNhhDoeb8keWRPZhhuv0Z
+f0eP9ORKjeKxjv3tsIPjiE7aqxE1zTdrnSr7FuqklJhMYRdwVv+2ofNEh05hU6pR
+VL4AS7d6Di/0dWTWc/Je2ytsrdio2v0rPAUXN1fyTh4AtrAmGQzUXNWnr4sB5xH9
+QlL0Ea3IwndJSDNkqc4qI3JL0vx6QMUbsuNcMmVWSMkODP+gNQYXQNbnwNfeMAnE
+V++WBfyrA8+V+ES+usqeWoOXjApzShn+gnrV0DHHXDAzNR+M647rQcsLePSyNjf/
+NKd7Z8VfEq7m65AxmSHPezSGdICMf63WLG/Bffj9rWiQxaoiayGF8jbALpXlu93X
+txOw8pK7zA8xFEBujmkrDPH3sJFPLOgOMYa0uuCMbrCGxeJ34nuQMhSUTamESSXb
+AD3AgUrRvte1iXwy2PoZGolRLZfdq9zcAfFyq9KvIhvz/8b2F+KbqHQlAiKVPw8p
+XQo4sXcDAmF251WSCJGN1C6Doxj/6XLuWILbkobQqoI
+-> ssh-ed25519 YFSOsg FtIvWeEXI9blJIFAWMacXgPym5ePGXsuiOR+Gh3b3R8
+0rp/NIu4kCCt05Is2+eRdUmgNX8QPMsDPhZWIejnBDA
+-> ssh-ed25519 iHV63A 85G1w54UHS/gFcLvsXyYLPXvLHkJl3YQCi8ehb+ZrU8
+lXDaMXlPw5ohaaYpiEkCNAmE2tJ2824ydmp9EakPtD8
+-> ssh-ed25519 BVsyTA XimcaonVCGGyyCfn3BSX/a7zjJkWeaVY/xAcdNDrl1U
+RaqpXzUd54qrkYYRbRTUclTpZdZx2us42lkP6wBxjBM
+-> CWM8^B-grease
+HvBgzYx54YVP0M6pk1bp9qegLscQ4tHIV9DZhr7jnrW41adgY0D39wnE2IgIRc6g
+keRHAr7QVqdPy/kr+u0GwQ1MGFKI8Jss8vRxKwv/UgQfmg
+--- dJWXhQRYjxWchTW1u3TrF7KvQIOdrOvkEC7oUtFcGeE
+l>qTðFÞ®/®â
@tË\Å&Zò	êÄ:„Þ@ òÚKÏx©ªr¾áHKûĦEûb0ÊÖ—5Ëm¸/

secrets/secrets.nix

@@ -62,4 +62,11 @@ in {
   "mediawiki-secret-key.age".publicKeys = nachtigallKeys ++ baseKeys;
 
   "coturn-static-auth-secret.age".publicKeys = nachtigallKeys ++ baseKeys;
+
+  "grafana-admin-password.age".publicKeys = flora6Keys ++ baseKeys;
+  "grafana-keycloak-client-secret.age".publicKeys = flora6Keys ++ baseKeys;
+  "grafana-smtp-password.age".publicKeys = flora6Keys ++ baseKeys;
+
+  "nachtigall-metrics-nginx-basic-auth.age".publicKeys = nachtigallKeys ++ baseKeys;
+  "nachtigall-metrics-prometheus-basic-auth-password.age".publicKeys = flora6Keys ++ nachtigallKeys ++ baseKeys;
 }

terraform/dns.tf

@@ -38,13 +38,11 @@ resource "namecheap_domain_records" "pub-solar" {
     hostname = "obs-portal"
     type = "A"
     address = "80.71.153.210"
-    ttl = 60
   }
   record {
     hostname = "vpn"
     type = "A"
     address = "80.71.153.210"
-    ttl = 60
   }
   record {
     hostname = "cache"
@@ -81,13 +79,16 @@ resource "namecheap_domain_records" "pub-solar" {
     hostname = "turn"
     type = "A"
     address = "138.201.80.102"
-    ttl = 300
+  }
+  record {
+    hostname = "grafana"
+    type = "A"
+    address = "80.71.153.210"
   }
   record {
     hostname = "hpb"
     type = "A"
     address = "80.71.153.239"
-    ttl = 60
   }
   record {
     hostname = "files"