Merge pull request 'loki, prometheus, promtail should connect via wireguard' (#200) from loki-prometheus-via-wireguard into main

Reviewed-on: pub-solar/infra#200 Reviewed-by: Hendrik Sokolowski <hensoko@noreply.git.pub.solar> Reviewed-by: b12f <b12f@noreply.git.pub.solar>
2024-06-05 00:04:40 +00:00 · 2024-06-05 00:04:40 +00:00 · 10ed117dfe
parent a10027ed21 e93a56e594
commit 10ed117dfe
11 changed files with 83 additions and 169 deletions
--- a/docs/unlocking-root.md
+++ b/docs/unlocking-root.md
@ -1,17 +0,0 @@
-# Unlocking the root partition on boot
-
-After a reboot, the encrypted ZFS pool will have to be unlocked. This is done by accessing the server via SSH with user `root` on port 2222.
-
-Nachtigall:
-
-```
-ssh root@138.201.80.102 -p2222
-```
-
-Metronom:
-
-```
-ssh root@49.13.236.167 -p2222
-```
-
-After connecting, paste the crypt passphrase you can find in the shared keepass. This will disconnect the SSH session right away and the server will keep booting into stage 2.
--- a/docs/unlocking-zfs-pool.md
+++ b/docs/unlocking-zfs-pool.md
@ -0,0 +1,20 @@
+# Unlocking the ZFS pool on boot
+
+After a reboot, the encrypted ZFS pool will have to be unlocked. This is done by
+accessing the server via SSH as user `root` on port 2222.
+
+Nachtigall:
+
+```
+ssh root@nachtigall.pub.solar -p2222
+```
+
+Metronom:
+
+```
+ssh root@metronom.pub.solar -p2222
+```
+
+After connecting, paste the encryption passphrase you can find in the shared
+keepass. This will disconnect the SSH session immediately and the server will
+continue to boot into stage 2.
--- a/modules/core/networking.nix
+++ b/modules/core/networking.nix
@ -28,8 +28,14 @@
    networking.firewall.interfaces.wg-ssh.allowedTCPPorts = [ 22 ];

    networking.hosts = {
-      "10.7.6.1" = [ "nachtigall.${config.pub-solar-os.networking.domain}" ];
-      "10.7.6.2" = [ "flora-6.${config.pub-solar-os.networking.domain}" ];
+      "10.7.6.1" = [ "nachtigall.wg.${config.pub-solar-os.networking.domain}" ];
+      "10.7.6.2" = [ "flora-6.wg.${config.pub-solar-os.networking.domain}" ];
+      "10.7.6.3" = [ "metronom.wg.${config.pub-solar-os.networking.domain}" ];
+      "10.7.6.4" = [ "tankstelle.wg.${config.pub-solar-os.networking.domain}" ];
+      "fd00:fae:fae:fae:fae:1::" = [ "nachtigall.wg.${config.pub-solar-os.networking.domain}" ];
+      "fd00:fae:fae:fae:fae:2::" = [ "flora-6.wg.${config.pub-solar-os.networking.domain}" ];
+      "fd00:fae:fae:fae:fae:3::" = [ "metronom.wg.${config.pub-solar-os.networking.domain}" ];
+      "fd00:fae:fae:fae:fae:4::" = [ "tankstelle.wg.${config.pub-solar-os.networking.domain}" ];
    };

    services.openssh = {
--- a/modules/loki/default.nix
+++ b/modules/loki/default.nix
@ -6,19 +6,9 @@
  ...
 }:
 {
-  services.caddy.virtualHosts = {
-    "flora-6.${config.pub-solar-os.networking.domain}" = {
-      logFormat = lib.mkForce ''
-        output discard
-      '';
-      extraConfig = ''
-        basicauth * {
-          ${config.pub-solar-os.authentication.robot.username} $2a$14$mmIAy/Ezm6YGohUtXa2mWeW6Bcw1MQXPhrRbz14jAD2iUu3oob/t.
-        }
-        reverse_proxy :${toString config.services.loki.configuration.server.http_listen_port}
-      '';
-    };
-  };
+  # Only expose loki port via wireguard interface
+  networking.firewall.interfaces.wg-ssh.allowedTCPPorts = [ 3100 ];
+
  # source: https://gist.github.com/rickhull/895b0cb38fdd537c1078a858cf15d63e
  # https://grafana.com/docs/loki/latest/configure/examples/#1-local-configuration-exampleyaml
  services.loki = {
@ -28,7 +18,8 @@
      auth_enabled = false;
      common = {
        ring = {
-          instance_addr = "127.0.0.1";
+          instance_interface_names = [ "wg-ssh" ];
+          instance_enable_ipv6 = true;
          kvstore = {
            store = "inmemory";
          };
@ -81,7 +72,7 @@
      };
      clients = [
        {
-          url = "http://127.0.0.1:${toString config.services.loki.configuration.server.http_listen_port}/loki/api/v1/push";
+          url = "http://flora-6.wg.pub.solar:${toString config.services.loki.configuration.server.http_listen_port}/loki/api/v1/push";
        }
      ];
      scrape_configs = [
--- a/modules/nginx-prometheus-exporters/default.nix
+++ b/modules/nginx-prometheus-exporters/default.nix
@ -14,16 +14,12 @@ let
  synapseMetricsPort = "${toString listenerWithMetrics.port}";
 in
 {
-  age.secrets.nachtigall-metrics-nginx-basic-auth = {
-    file = "${flake.self}/secrets/nachtigall-metrics-nginx-basic-auth.age";
-    mode = "600";
-    owner = "nginx";
-  };
  services.nginx.virtualHosts = {
-    "nachtigall.${config.pub-solar-os.networking.domain}" = {
-      enableACME = true;
-      addSSL = true;
-      basicAuthFile = "${config.age.secrets.nachtigall-metrics-nginx-basic-auth.path}";
+    "nachtigall.wg.${config.pub-solar-os.networking.domain}" = {
+      listenAddresses = [
+        "10.7.6.1"
+        "[fd00:fae:fae:fae:fae:1::]"
+      ];
      locations."/metrics" = {
        proxyPass = "http://127.0.0.1:${toString (config.services.prometheus.exporters.node.port)}";
      };
--- a/modules/prometheus/default.nix
+++ b/modules/prometheus/default.nix
@ -6,11 +6,6 @@
  ...
 }:
 {
-  age.secrets.nachtigall-metrics-prometheus-basic-auth-password = {
-    file = "${flake.self}/secrets/nachtigall-metrics-prometheus-basic-auth-password.age";
-    mode = "600";
-    owner = "prometheus";
-  };
  age.secrets.alertmanager-envfile = {
    file = "${flake.self}/secrets/alertmanager-envfile.age";
    mode = "600";
@ -44,7 +39,7 @@
    };
    scrapeConfigs = [
      {
-        job_name = "node-exporter-http";
+        job_name = "node-exporter";
        static_configs = [
          {
            targets = [ "127.0.0.1:${toString config.services.prometheus.exporters.node.port}" ];
@ -52,19 +47,8 @@
              instance = "flora-6";
            };
          }
-        ];
-      }
          {
-        job_name = "node-exporter-https";
-        scheme = "https";
-        metrics_path = "/metrics";
-        basic_auth = {
-          username = "hakkonaut";
-          password_file = "${config.age.secrets.nachtigall-metrics-prometheus-basic-auth-password.path}";
-        };
-        static_configs = [
-          {
-            targets = [ "nachtigall.${config.pub-solar-os.networking.domain}" ];
+            targets = [ "nachtigall.wg.${config.pub-solar-os.networking.domain}" ];
            labels = {
              instance = "nachtigall";
            };
@ -73,15 +57,10 @@
      }
      {
        job_name = "matrix-synapse";
-        scheme = "https";
        metrics_path = "/_synapse/metrics";
-        basic_auth = {
-          username = "hakkonaut";
-          password_file = "${config.age.secrets.nachtigall-metrics-prometheus-basic-auth-password.path}";
-        };
        static_configs = [
          {
-            targets = [ "nachtigall.${config.pub-solar-os.networking.domain}" ];
+            targets = [ "nachtigall.wg.${config.pub-solar-os.networking.domain}" ];
            labels = {
              instance = "nachtigall";
            };
--- a/modules/promtail/default.nix
+++ b/modules/promtail/default.nix
@ -6,12 +6,6 @@
  ...
 }:
 {
-  age.secrets.nachtigall-metrics-prometheus-basic-auth-password = {
-    file = "${flake.self}/secrets/nachtigall-metrics-prometheus-basic-auth-password.age";
-    mode = "600";
-    owner = "promtail";
-  };
-
  services.promtail = {
    enable = true;
    configuration = {
@ -24,11 +18,7 @@
      };
      clients = [
        {
-          url = "https://flora-6.${config.pub-solar-os.networking.domain}/loki/api/v1/push";
-          basic_auth = {
-            username = "hakkonaut";
-            password_file = "${config.age.secrets.nachtigall-metrics-prometheus-basic-auth-password.path}";
-          };
+          url = "http://flora-6.wg.pub.solar:${toString flake.self.nixosConfigurations.flora-6.config.services.loki.configuration.server.http_listen_port}/loki/api/v1/push";
        }
      ];
      scrape_configs = [
--- a/secrets/nachtigall-metrics-nginx-basic-auth.age
+++ b/secrets/nachtigall-metrics-nginx-basic-auth.age
@ -1,43 +0,0 @@
-age-encryption.org/v1
-> ssh-ed25519 iDKjwg iFrOyGN0zSpptFEy3mRmzFH/SpqvmQZRhMHaOvHggSc
-HRTI1y0eUK0nAWO0Q/YVNYOyLU0OwY9KH0a3elGk1fs
-> ssh-ed25519 uYcDNw ojnoOpd7HElVjSlgSxrS53yz5ecb0ZZbZ4ZRa/C4vjc
-YoBa3whKDyeOsdXFdzUJAIElTL/8o1blYlltNsvWCjs
-> ssh-rsa f5THog
-j2mjjmsw8yj5gd6B6hHNiJrP2IICrupcaHcuPZHID5Bq9WbXcFlU9bsvLVtneBbD
-YyGgpgUzejokeRT8EKieQSzcRCt99qVSO0cJWlvtVMpY5kNL7L6q9v3hlgOgAHPH
-WgtnkHkXrGTiQQWSTaymt1dxtWBOfA3RvLnRubwrSzkIynqHuX1AqjXqQy3RL7BJ
-nfpp9ctviR2CXyBgF2VvFXLUB7dV+SWe+Sp09293/sx3lTDAJOs5DTL32I+suNl7
-g1VVgE+kgVt3B6aXqrIe1T/bDjb4IMu7saXL3q9dz7aZNysLcQgGI254HR7VkE3o
-GFlMb6PWj9oHa0R0PqCzyL0NV+VfKEXkdYFebCUI2p9jKajy8VCcNfRmekf5ZBHP
-tAmyjnKE8uO4qYyhcK7eZJHAMwIYC8LW+xcEo1ym27K0t6M9Ph2QbRslqPf8nWsP
-9a/Ca1cSKBc0IXhG88ulsDCHIFpiAegLPTdZL5GFe0VwyfyLukG4I8fXNndRVhK+
-RMxWl1ZGWYTBiQi/4a4JZvXP14JpTfC8DzvcZHXl8o2GqS/TEk7zAOsoGffwzqpO
-Fid11Axy0BY1iPfH6S44W8uxQz9b9AUVrJD53f9YIOTGjfMOUrOCwTHv2DcN+LC7
-02LmoCkSTsCqpnpJPDOXcGYh3nk75orQYqW5lnkwc8g
-> ssh-rsa kFDS0A
-FeZXachOnQfqnotkRdNFtoiZL02DViImVhkIizJAUh1VgUXiCHlQX+8epshgP3dL
-xYBf4yPx5RBKN/jKfNsjS0KyxwDlApemyD73JW83LJ5cm2JuUwvtGXVCBFrkD9OI
-I4oeuBdl8oBQgjvUbp4BkXvqh+0Ymw7rMs5IWJDjwMOUgnsrpvp363IbVY5wc2Cp
-tI9OeiP4Jx9zUVKTpeIXdH5U54tjBAr/n0D4OXRZC79CW2Sw475z0wbXzKkQMYL7
-XidTyBpvj9b2IdaswhQpx21nDIlNKSQy1+gVgQTljxuHBcs/tOulTM+DC/UbA/hy
-blKAs0HPOkodYGwl1VytIg6Qr1cczSUCUrgmZ4CxcFF/6earOT9uscjbT73jeyil
-JSuzBjyULh59tueYqmuPcq5wCcsvCEYJrUtg/vrU6JhWvLjmOk6HKMls6KcB+qeg
-pgkjSsSqgdN0k2mZaUOAe88bMC+z5oGL1Gi9dFEYmdN/gN8CFVaULxwrL/IXPnkw
-O7LBeVSV31et2iGKE9Mf1GjyCZV4xSaYdtuSTSOPsRuctTIW2y7FyU0MdUGhZmIl
-faEWPpnuBqDm6m8RUFuxy8un2k9mQzE2iroKWimj49kftqVdSAgUMgHws2G8GH/y
-MrRkarMtyVFgzHF/4WkO1FPdsBWy9pVdRhFdr7BSeQc
-> piv-p256 vRzPNw A9xaGL246GekLk5G2Jy6+AdtmVoBc101XDkGdqmCU0Ow
-NvuqIsu7dexWjLOJY8vCcZgyHjs9o9z8N2RrjjOGFDQ
-> piv-p256 zqq/iw A7A1tGYE+5KhtcWXQ5kE1FjY9teRnWb0HrmqkX5qqanK
-t+ViJ41AuFrL6CH2cYnWx3XLB6iR0fxgp9TK1zt3DNE
-> ssh-ed25519 YFSOsg O2M/GJ0nXaCtasaqdZCzHwOPlnKoxjrEyhZsWcjrCTw
-ZKQEI098YcHWNL6VBJ6JmRN7QLC1sQd3zUTQi1o3dbE
-> ssh-ed25519 iHV63A nARCFmD6Q9rj+ebUFckSf6rM0jTKRgHtDRS4qzCd9iE
-peM7be/ngP+HQYPgpQruhdL9D2QArUrJWao0L++Y1js
-> ssh-ed25519 BVsyTA U6fvbra/fd4P6r7bUFCN5bwqiDBF0h+V5AB94ZOBtwI
-UzDdo8fw7Ya7vHmPNLXSzOnAV4FVj3+2Ci3pStIuu/U
-> ssh-ed25519 +3V2lQ 8rvmvG/jd72rp0mhx+biUCihJcK7WjnkTPgwvcJYJEM
-785YAEjC6xaTLZPzgcLhQPFigh6TVYbSkhn1aVc5PKg
--- X3mEGGX4yRgEZLBHEnFT2P59pGYxEKQCqBntP8OM24Q
-×R(»Ü‘Þ5Ö5~,ëÓÝõ?ÇÆ]¬¼s\i8`—9G[¡?ðíÞ<C3AD>ÕÅÓ$LÚD:š´w3¼N{FB1Xü,zvÏ@a{²™å
--- a/secrets/nachtigall-metrics-prometheus-basic-auth-password.age
+++ b/secrets/nachtigall-metrics-prometheus-basic-auth-password.age
@ -1,45 +0,0 @@
-age-encryption.org/v1
-> ssh-ed25519 Y0ZZaw nTNUxIC9LkrJ9hUdbihbpeHVMmLJxAvJ1owTGipKUSE
-axyLEKraFg2oYLh28QyKxb5R+ao9Q374iqg0OcPKfao
-> ssh-ed25519 iDKjwg htWAMOoRqftyzvn7uCmsrF80MdFwmomqvB+UMJ/NVTU
-Wqe9W++Slv5ITX3C+89bsVWWytOM+SD3vISPmwVh87k
-> ssh-ed25519 uYcDNw yBxYg49sXazNjQbX6v9Vah6StIw8mrVG/yjgxFesLhE
-iDh8pDLGhmlTYkg3ESaM7P58gBbPn+tjFkr/+UthYos
-> ssh-rsa f5THog
-Rv+2zwwON/S9Ph3ZhC0oERqbaUw9r4mlJ+FfhOxt45fdy+DmcMRpZoUe/3Rb1LqE
-VTXpYlcG3FScRt2u+MOYywCu3E5ForqUjHKKXKeK5JwvSOdrOZWgDmg9kc9GA0io
-St+6EEQbBVXQ/l57+i8VQ/mSi+RlYBCVxoCvWm22i5cYV72SobAaJbITS4XWAdPb
-hQbOBD+5X5Laj5ixDNsc1wxdU47S+uY/uFm1Mpw/eJYG+cUlYw1/Kd/UpoJVSdT+
-EQN+WUPmDYEHJSn3VVoYVF4969MLONb+9X3w5KITYr9r7lpc+uKvqPicDPpRdTAw
-gtRPUDpz/MoBvP29NOsITFACavfiKJjYH443pn6JEQF7vtPdjyvCMLf/PxWmpIzw
-2BPZmllvqGwYxeVcjzRSDbbsNG85RE+tSVM5p37lVYF6AZfxHG0tLPJt68AT5n36
-fu2mvkEhRZR84/iUuNRGhemma4CuhTZk82MZGefSHlaCI03Bl8VmHlfKLlEEoCTq
-7EovI0mVyHzhfnRJyqcSm7rD3RKU2zH8K7aAB/zd9x4m2bk6mDnUJViObOcfMRjF
-GUy2RHO/FuRgQtD3ZTsQ+eG37fvhb8dSDMfAIP9ug04pl55co3L18JlUMEwktq8m
-AD+DDa0pXwLU1zminQRZwJIe7RU0li44lmqihxIlXGo
-> ssh-rsa kFDS0A
-jbDwJLKASE8aNqmgoyV8BO572dc7PoS1AMWnULJwv8JglL+KeYxU3HwlLulKQ1Ej
-pDC/BVONirMx1KE8qm8RTgo/xhoA/GVognpR4T19Z9yslD6E2mtGozCi+zlAjn0u
-BgThEp1pE9CCY54enXS9ADnTYYwZene+i2OkJsRpZ0qM3ULLRqrIl7otwvgHu7S3
-x5C9YJNTGPUE33aDwWFblAApgelQ9p7erXJOW35FVAs50WFcAeIh8FoV8AAgVXVL
-/4LADst6xxkT/jGBZcilO/W2Yj/k+sG+FBMtsat+u57CHLzp5G0KFNWpej9fzUFB
-xavyLn7HXhjhT9GmtFY3TT71mqKmbj1syNn19rs2liZwdeLfgYBKS0xRKDGmHLtn
-2JpElmKGM9qRZXYsPgq/NR5TsLEG2o/v0CxYT0wAbJnSfZJniiwJs4E+rrh78F4X
-0YzUzPbAsCs3G7SCEz/ow4EmQkOZkJjFkHb/bIXIAqgz8AaFWuaVJVeSEGexTUy5
-nXCOy9JOXJJC1O1CP/GwjmKKvqvYus/UBcCgVH+lQoxKWak1CD59ao+taCADevMu
-BtL+KaLSwfrHpVZ/CTf5JqPKl8aYoQeubWdQttmF/DRyCsEDsiHAJFwgp4NC73zh
-w1js8L5tt29ty2x3M7yY4bGQeC450+OwYsi50YpXE3Q
-> piv-p256 vRzPNw AwvMDdyTEURDqHbfoq5odnWJYvfneezIuvpMP1UQRKWg
-fil4sICJnowY8rRbxQouXUZdUwAoe9smsMw0lcKtSbA
-> piv-p256 zqq/iw Aq5f+a77FpRI4Xe3zQe8If5aPkH2SJ0BHkWdlsrOtc4u
-roBw1kwrU3OqKZZ38aVKdioUzfQ7d4ztwXgh/Icyni4
-> ssh-ed25519 YFSOsg 1c0L+d2frinozItIJB3NNOmdkttv9GLBhJTStTzG6Hg
-Xy4TN3qZL1FF+thpQw/mRZq4jv4odgDjBK9/Wcc2QrE
-> ssh-ed25519 iHV63A 8l9cP+kW+MfGiN3rXOh2rJQPf8g8bCAirBTz/jYTtw4
-w5FlcJiyDSN9D8GNNumLtWvv/E+0a2eoQPx81v/YzmU
-> ssh-ed25519 BVsyTA q7aLkPRcT8rPKXbEiwn+w300j20WO8rNfCIt6oLcUXk
-O9V5q98TG6UKFQJooUrVfX/Icab5UPYONvSH7mKa/pA
-> ssh-ed25519 +3V2lQ NxpGLFMboFSAztflSWw+NFjByFfkBL/IG4r/hFvMjkQ
-0uWTKEG3TAsNsrPcooLsrINmDTWKlVIx1/OAL2rlcgc
--- VrkwgHMM0SXQKvH6I1oz35B391zF9QHysr3AZxGTpxw
-M’°°<>l0<6C>â!wÏú™Þ+–‹B¼<s¤à`ÚEÂ*_<>Û„ÂÝ˜Ò1þÁó¥Jâ¡[¥?ì¾Î|»‹
--- a/secrets/secrets.nix
+++ b/secrets/secrets.nix
@ -70,9 +70,6 @@ in
  "grafana-smtp-password.age".publicKeys = flora6Keys ++ adminKeys;

  "alertmanager-envfile.age".publicKeys = flora6Keys ++ adminKeys;
-  "nachtigall-metrics-nginx-basic-auth.age".publicKeys = nachtigallKeys ++ adminKeys;
-  "nachtigall-metrics-prometheus-basic-auth-password.age".publicKeys =
-    flora6Keys ++ nachtigallKeys ++ adminKeys;

  "obs-portal-env.age".publicKeys = nachtigallKeys ++ adminKeys;
  "obs-portal-database-env.age".publicKeys = nachtigallKeys ++ adminKeys;
--- a/terraform/dns.tf
+++ b/terraform/dns.tf
@ -4,6 +4,46 @@ resource "namecheap_domain_records" "pub-solar" {
  mode       = "OVERWRITE"
  email_type = "MX"

+  record {
+    hostname = "nachtigall.wg"
+    type     = "A"
+    address  = "10.7.6.1"
+  }
+  record {
+    hostname = "flora-6.wg"
+    type     = "A"
+    address  = "10.7.6.2"
+  }
+  record {
+    hostname = "metronom.wg"
+    type     = "A"
+    address  = "10.7.6.3"
+  }
+  record {
+    hostname = "tankstelle.wg"
+    type     = "A"
+    address  = "10.7.6.4"
+  }
+  record {
+    hostname = "nachtigall.wg"
+    type     = "AAAA"
+    address  = "fd00:fae:fae:fae:fae:1::"
+  }
+  record {
+    hostname = "flora-6.wg"
+    type     = "AAAA"
+    address  = "fd00:fae:fae:fae:fae:2::"
+  }
+  record {
+    hostname = "metronom.wg"
+    type     = "AAAA"
+    address  = "fd00:fae:fae:fae:fae:3::"
+  }
+  record {
+    hostname = "tankstelle.wg"
+    type     = "AAAA"
+    address  = "fd00:fae:fae:fae:fae:4::"
+  }
  record {
    hostname = "flora-6"
    type     = "A"