Merge pull request 'loki, prometheus, promtail should connect via wireguard' (#200) from loki-prometheus-via-wireguard into main

Reviewed-on: pub-solar/infra#200
Reviewed-by: Hendrik Sokolowski <hensoko@noreply.git.pub.solar>
Reviewed-by: b12f <b12f@noreply.git.pub.solar>

docs/unlocking-root.md

@@ -1,17 +0,0 @@
-# Unlocking the root partition on boot
-
-After a reboot, the encrypted ZFS pool will have to be unlocked. This is done by accessing the server via SSH with user `root` on port 2222.
-
-Nachtigall:
-
-```
-ssh root@138.201.80.102 -p2222
-```
-
-Metronom:
-
-```
-ssh root@49.13.236.167 -p2222
-```
-
-After connecting, paste the crypt passphrase you can find in the shared keepass. This will disconnect the SSH session right away and the server will keep booting into stage 2.

docs/unlocking-zfs-pool.md

@@ -0,0 +1,20 @@
+# Unlocking the ZFS pool on boot
+
+After a reboot, the encrypted ZFS pool will have to be unlocked. This is done by
+accessing the server via SSH as user `root` on port 2222.
+
+Nachtigall:
+
+```
+ssh root@nachtigall.pub.solar -p2222
+```
+
+Metronom:
+
+```
+ssh root@metronom.pub.solar -p2222
+```
+
+After connecting, paste the encryption passphrase you can find in the shared
+keepass. This will disconnect the SSH session immediately and the server will
+continue to boot into stage 2.

modules/core/networking.nix

@@ -28,8 +28,14 @@
     networking.firewall.interfaces.wg-ssh.allowedTCPPorts = [ 22 ];
 
     networking.hosts = {
-      "10.7.6.1" = [ "nachtigall.${config.pub-solar-os.networking.domain}" ];
+      "10.7.6.1" = [ "nachtigall.wg.${config.pub-solar-os.networking.domain}" ];
-      "10.7.6.2" = [ "flora-6.${config.pub-solar-os.networking.domain}" ];
+      "10.7.6.2" = [ "flora-6.wg.${config.pub-solar-os.networking.domain}" ];
+      "10.7.6.3" = [ "metronom.wg.${config.pub-solar-os.networking.domain}" ];
+      "10.7.6.4" = [ "tankstelle.wg.${config.pub-solar-os.networking.domain}" ];
+      "fd00:fae:fae:fae:fae:1::" = [ "nachtigall.wg.${config.pub-solar-os.networking.domain}" ];
+      "fd00:fae:fae:fae:fae:2::" = [ "flora-6.wg.${config.pub-solar-os.networking.domain}" ];
+      "fd00:fae:fae:fae:fae:3::" = [ "metronom.wg.${config.pub-solar-os.networking.domain}" ];
+      "fd00:fae:fae:fae:fae:4::" = [ "tankstelle.wg.${config.pub-solar-os.networking.domain}" ];
     };
 
     services.openssh = {

modules/loki/default.nix

@@ -6,19 +6,9 @@
   ...
 }:
 {
-  services.caddy.virtualHosts = {
+  # Only expose loki port via wireguard interface
-    "flora-6.${config.pub-solar-os.networking.domain}" = {
+  networking.firewall.interfaces.wg-ssh.allowedTCPPorts = [ 3100 ];
-      logFormat = lib.mkForce ''
+
-        output discard
-      '';
-      extraConfig = ''
-        basicauth * {
-          ${config.pub-solar-os.authentication.robot.username} $2a$14$mmIAy/Ezm6YGohUtXa2mWeW6Bcw1MQXPhrRbz14jAD2iUu3oob/t.
-        }
-        reverse_proxy :${toString config.services.loki.configuration.server.http_listen_port}
-      '';
-    };
-  };
   # source: https://gist.github.com/rickhull/895b0cb38fdd537c1078a858cf15d63e
   # https://grafana.com/docs/loki/latest/configure/examples/#1-local-configuration-exampleyaml
   services.loki = {
@@ -28,7 +18,8 @@
       auth_enabled = false;
       common = {
         ring = {
-          instance_addr = "127.0.0.1";
+          instance_interface_names = [ "wg-ssh" ];
+          instance_enable_ipv6 = true;
           kvstore = {
             store = "inmemory";
           };
@@ -81,7 +72,7 @@
       };
       clients = [
         {
-          url = "http://127.0.0.1:${toString config.services.loki.configuration.server.http_listen_port}/loki/api/v1/push";
+          url = "http://flora-6.wg.pub.solar:${toString config.services.loki.configuration.server.http_listen_port}/loki/api/v1/push";
         }
       ];
       scrape_configs = [

modules/nginx-prometheus-exporters/default.nix

@@ -14,16 +14,12 @@ let
   synapseMetricsPort = "${toString listenerWithMetrics.port}";
 in
 {
-  age.secrets.nachtigall-metrics-nginx-basic-auth = {
-    file = "${flake.self}/secrets/nachtigall-metrics-nginx-basic-auth.age";
-    mode = "600";
-    owner = "nginx";
-  };
   services.nginx.virtualHosts = {
-    "nachtigall.${config.pub-solar-os.networking.domain}" = {
+    "nachtigall.wg.${config.pub-solar-os.networking.domain}" = {
-      enableACME = true;
+      listenAddresses = [
-      addSSL = true;
+        "10.7.6.1"
-      basicAuthFile = "${config.age.secrets.nachtigall-metrics-nginx-basic-auth.path}";
+        "[fd00:fae:fae:fae:fae:1::]"
+      ];
       locations."/metrics" = {
         proxyPass = "http://127.0.0.1:${toString (config.services.prometheus.exporters.node.port)}";
       };

modules/prometheus/default.nix

@@ -6,11 +6,6 @@
   ...
 }:
 {
-  age.secrets.nachtigall-metrics-prometheus-basic-auth-password = {
-    file = "${flake.self}/secrets/nachtigall-metrics-prometheus-basic-auth-password.age";
-    mode = "600";
-    owner = "prometheus";
-  };
   age.secrets.alertmanager-envfile = {
     file = "${flake.self}/secrets/alertmanager-envfile.age";
     mode = "600";
@@ -44,7 +39,7 @@
     };
     scrapeConfigs = [
       {
-        job_name = "node-exporter-http";
+        job_name = "node-exporter";
         static_configs = [
           {
             targets = [ "127.0.0.1:${toString config.services.prometheus.exporters.node.port}" ];
@@ -52,19 +47,8 @@
               instance = "flora-6";
             };
           }
-        ];
-      }
           {
-        job_name = "node-exporter-https";
+            targets = [ "nachtigall.wg.${config.pub-solar-os.networking.domain}" ];
-        scheme = "https";
-        metrics_path = "/metrics";
-        basic_auth = {
-          username = "hakkonaut";
-          password_file = "${config.age.secrets.nachtigall-metrics-prometheus-basic-auth-password.path}";
-        };
-        static_configs = [
-          {
-            targets = [ "nachtigall.${config.pub-solar-os.networking.domain}" ];
             labels = {
               instance = "nachtigall";
             };
@@ -73,15 +57,10 @@
       }
       {
         job_name = "matrix-synapse";
-        scheme = "https";
         metrics_path = "/_synapse/metrics";
-        basic_auth = {
-          username = "hakkonaut";
-          password_file = "${config.age.secrets.nachtigall-metrics-prometheus-basic-auth-password.path}";
-        };
         static_configs = [
           {
-            targets = [ "nachtigall.${config.pub-solar-os.networking.domain}" ];
+            targets = [ "nachtigall.wg.${config.pub-solar-os.networking.domain}" ];
             labels = {
               instance = "nachtigall";
             };

modules/promtail/default.nix

@@ -6,12 +6,6 @@
   ...
 }:
 {
-  age.secrets.nachtigall-metrics-prometheus-basic-auth-password = {
-    file = "${flake.self}/secrets/nachtigall-metrics-prometheus-basic-auth-password.age";
-    mode = "600";
-    owner = "promtail";
-  };
-
   services.promtail = {
     enable = true;
     configuration = {
@@ -24,11 +18,7 @@
       };
       clients = [
         {
-          url = "https://flora-6.${config.pub-solar-os.networking.domain}/loki/api/v1/push";
+          url = "http://flora-6.wg.pub.solar:${toString flake.self.nixosConfigurations.flora-6.config.services.loki.configuration.server.http_listen_port}/loki/api/v1/push";
-          basic_auth = {
-            username = "hakkonaut";
-            password_file = "${config.age.secrets.nachtigall-metrics-prometheus-basic-auth-password.path}";
-          };
         }
       ];
       scrape_configs = [

secrets/nachtigall-metrics-nginx-basic-auth.age

@@ -1,43 +0,0 @@
-age-encryption.org/v1
--> ssh-ed25519 iDKjwg iFrOyGN0zSpptFEy3mRmzFH/SpqvmQZRhMHaOvHggSc
-HRTI1y0eUK0nAWO0Q/YVNYOyLU0OwY9KH0a3elGk1fs
--> ssh-ed25519 uYcDNw ojnoOpd7HElVjSlgSxrS53yz5ecb0ZZbZ4ZRa/C4vjc
-YoBa3whKDyeOsdXFdzUJAIElTL/8o1blYlltNsvWCjs
--> ssh-rsa f5THog
-j2mjjmsw8yj5gd6B6hHNiJrP2IICrupcaHcuPZHID5Bq9WbXcFlU9bsvLVtneBbD
-YyGgpgUzejokeRT8EKieQSzcRCt99qVSO0cJWlvtVMpY5kNL7L6q9v3hlgOgAHPH
-WgtnkHkXrGTiQQWSTaymt1dxtWBOfA3RvLnRubwrSzkIynqHuX1AqjXqQy3RL7BJ
-nfpp9ctviR2CXyBgF2VvFXLUB7dV+SWe+Sp09293/sx3lTDAJOs5DTL32I+suNl7
-g1VVgE+kgVt3B6aXqrIe1T/bDjb4IMu7saXL3q9dz7aZNysLcQgGI254HR7VkE3o
-GFlMb6PWj9oHa0R0PqCzyL0NV+VfKEXkdYFebCUI2p9jKajy8VCcNfRmekf5ZBHP
-tAmyjnKE8uO4qYyhcK7eZJHAMwIYC8LW+xcEo1ym27K0t6M9Ph2QbRslqPf8nWsP
-9a/Ca1cSKBc0IXhG88ulsDCHIFpiAegLPTdZL5GFe0VwyfyLukG4I8fXNndRVhK+
-RMxWl1ZGWYTBiQi/4a4JZvXP14JpTfC8DzvcZHXl8o2GqS/TEk7zAOsoGffwzqpO
-Fid11Axy0BY1iPfH6S44W8uxQz9b9AUVrJD53f9YIOTGjfMOUrOCwTHv2DcN+LC7
-02LmoCkSTsCqpnpJPDOXcGYh3nk75orQYqW5lnkwc8g
--> ssh-rsa kFDS0A
-FeZXachOnQfqnotkRdNFtoiZL02DViImVhkIizJAUh1VgUXiCHlQX+8epshgP3dL
-xYBf4yPx5RBKN/jKfNsjS0KyxwDlApemyD73JW83LJ5cm2JuUwvtGXVCBFrkD9OI
-I4oeuBdl8oBQgjvUbp4BkXvqh+0Ymw7rMs5IWJDjwMOUgnsrpvp363IbVY5wc2Cp
-tI9OeiP4Jx9zUVKTpeIXdH5U54tjBAr/n0D4OXRZC79CW2Sw475z0wbXzKkQMYL7
-XidTyBpvj9b2IdaswhQpx21nDIlNKSQy1+gVgQTljxuHBcs/tOulTM+DC/UbA/hy
-blKAs0HPOkodYGwl1VytIg6Qr1cczSUCUrgmZ4CxcFF/6earOT9uscjbT73jeyil
-JSuzBjyULh59tueYqmuPcq5wCcsvCEYJrUtg/vrU6JhWvLjmOk6HKMls6KcB+qeg
-pgkjSsSqgdN0k2mZaUOAe88bMC+z5oGL1Gi9dFEYmdN/gN8CFVaULxwrL/IXPnkw
-O7LBeVSV31et2iGKE9Mf1GjyCZV4xSaYdtuSTSOPsRuctTIW2y7FyU0MdUGhZmIl
-faEWPpnuBqDm6m8RUFuxy8un2k9mQzE2iroKWimj49kftqVdSAgUMgHws2G8GH/y
-MrRkarMtyVFgzHF/4WkO1FPdsBWy9pVdRhFdr7BSeQc
--> piv-p256 vRzPNw A9xaGL246GekLk5G2Jy6+AdtmVoBc101XDkGdqmCU0Ow
-NvuqIsu7dexWjLOJY8vCcZgyHjs9o9z8N2RrjjOGFDQ
--> piv-p256 zqq/iw A7A1tGYE+5KhtcWXQ5kE1FjY9teRnWb0HrmqkX5qqanK
-t+ViJ41AuFrL6CH2cYnWx3XLB6iR0fxgp9TK1zt3DNE
--> ssh-ed25519 YFSOsg O2M/GJ0nXaCtasaqdZCzHwOPlnKoxjrEyhZsWcjrCTw
-ZKQEI098YcHWNL6VBJ6JmRN7QLC1sQd3zUTQi1o3dbE
--> ssh-ed25519 iHV63A nARCFmD6Q9rj+ebUFckSf6rM0jTKRgHtDRS4qzCd9iE
-peM7be/ngP+HQYPgpQruhdL9D2QArUrJWao0L++Y1js
--> ssh-ed25519 BVsyTA U6fvbra/fd4P6r7bUFCN5bwqiDBF0h+V5AB94ZOBtwI
-UzDdo8fw7Ya7vHmPNLXSzOnAV4FVj3+2Ci3pStIuu/U
--> ssh-ed25519 +3V2lQ 8rvmvG/jd72rp0mhx+biUCihJcK7WjnkTPgwvcJYJEM
-785YAEjC6xaTLZPzgcLhQPFigh6TVYbSkhn1aVc5PKg
---- X3mEGGX4yRgEZLBHEnFT2P59pGYxEKQCqBntP8OM24Q
-×R(»Ü‘Þ5Ö5~,ëÓÝõ?ÇÆ]¬¼s\i8`—9G[¡?ðíÞ<C3AD>ÕÅÓ$LÚD:š´w3¼N{FB1Xü,zvÏ@a{²™å

secrets/nachtigall-metrics-prometheus-basic-auth-password.age

@@ -1,45 +0,0 @@
-age-encryption.org/v1
--> ssh-ed25519 Y0ZZaw nTNUxIC9LkrJ9hUdbihbpeHVMmLJxAvJ1owTGipKUSE
-axyLEKraFg2oYLh28QyKxb5R+ao9Q374iqg0OcPKfao
--> ssh-ed25519 iDKjwg htWAMOoRqftyzvn7uCmsrF80MdFwmomqvB+UMJ/NVTU
-Wqe9W++Slv5ITX3C+89bsVWWytOM+SD3vISPmwVh87k
--> ssh-ed25519 uYcDNw yBxYg49sXazNjQbX6v9Vah6StIw8mrVG/yjgxFesLhE
-iDh8pDLGhmlTYkg3ESaM7P58gBbPn+tjFkr/+UthYos
--> ssh-rsa f5THog
-Rv+2zwwON/S9Ph3ZhC0oERqbaUw9r4mlJ+FfhOxt45fdy+DmcMRpZoUe/3Rb1LqE
-VTXpYlcG3FScRt2u+MOYywCu3E5ForqUjHKKXKeK5JwvSOdrOZWgDmg9kc9GA0io
-St+6EEQbBVXQ/l57+i8VQ/mSi+RlYBCVxoCvWm22i5cYV72SobAaJbITS4XWAdPb
-hQbOBD+5X5Laj5ixDNsc1wxdU47S+uY/uFm1Mpw/eJYG+cUlYw1/Kd/UpoJVSdT+
-EQN+WUPmDYEHJSn3VVoYVF4969MLONb+9X3w5KITYr9r7lpc+uKvqPicDPpRdTAw
-gtRPUDpz/MoBvP29NOsITFACavfiKJjYH443pn6JEQF7vtPdjyvCMLf/PxWmpIzw
-2BPZmllvqGwYxeVcjzRSDbbsNG85RE+tSVM5p37lVYF6AZfxHG0tLPJt68AT5n36
-fu2mvkEhRZR84/iUuNRGhemma4CuhTZk82MZGefSHlaCI03Bl8VmHlfKLlEEoCTq
-7EovI0mVyHzhfnRJyqcSm7rD3RKU2zH8K7aAB/zd9x4m2bk6mDnUJViObOcfMRjF
-GUy2RHO/FuRgQtD3ZTsQ+eG37fvhb8dSDMfAIP9ug04pl55co3L18JlUMEwktq8m
-AD+DDa0pXwLU1zminQRZwJIe7RU0li44lmqihxIlXGo
--> ssh-rsa kFDS0A
-jbDwJLKASE8aNqmgoyV8BO572dc7PoS1AMWnULJwv8JglL+KeYxU3HwlLulKQ1Ej
-pDC/BVONirMx1KE8qm8RTgo/xhoA/GVognpR4T19Z9yslD6E2mtGozCi+zlAjn0u
-BgThEp1pE9CCY54enXS9ADnTYYwZene+i2OkJsRpZ0qM3ULLRqrIl7otwvgHu7S3
-x5C9YJNTGPUE33aDwWFblAApgelQ9p7erXJOW35FVAs50WFcAeIh8FoV8AAgVXVL
-/4LADst6xxkT/jGBZcilO/W2Yj/k+sG+FBMtsat+u57CHLzp5G0KFNWpej9fzUFB
-xavyLn7HXhjhT9GmtFY3TT71mqKmbj1syNn19rs2liZwdeLfgYBKS0xRKDGmHLtn
-2JpElmKGM9qRZXYsPgq/NR5TsLEG2o/v0CxYT0wAbJnSfZJniiwJs4E+rrh78F4X
-0YzUzPbAsCs3G7SCEz/ow4EmQkOZkJjFkHb/bIXIAqgz8AaFWuaVJVeSEGexTUy5
-nXCOy9JOXJJC1O1CP/GwjmKKvqvYus/UBcCgVH+lQoxKWak1CD59ao+taCADevMu
-BtL+KaLSwfrHpVZ/CTf5JqPKl8aYoQeubWdQttmF/DRyCsEDsiHAJFwgp4NC73zh
-w1js8L5tt29ty2x3M7yY4bGQeC450+OwYsi50YpXE3Q
--> piv-p256 vRzPNw AwvMDdyTEURDqHbfoq5odnWJYvfneezIuvpMP1UQRKWg
-fil4sICJnowY8rRbxQouXUZdUwAoe9smsMw0lcKtSbA
--> piv-p256 zqq/iw Aq5f+a77FpRI4Xe3zQe8If5aPkH2SJ0BHkWdlsrOtc4u
-roBw1kwrU3OqKZZ38aVKdioUzfQ7d4ztwXgh/Icyni4
--> ssh-ed25519 YFSOsg 1c0L+d2frinozItIJB3NNOmdkttv9GLBhJTStTzG6Hg
-Xy4TN3qZL1FF+thpQw/mRZq4jv4odgDjBK9/Wcc2QrE
--> ssh-ed25519 iHV63A 8l9cP+kW+MfGiN3rXOh2rJQPf8g8bCAirBTz/jYTtw4
-w5FlcJiyDSN9D8GNNumLtWvv/E+0a2eoQPx81v/YzmU
--> ssh-ed25519 BVsyTA q7aLkPRcT8rPKXbEiwn+w300j20WO8rNfCIt6oLcUXk
-O9V5q98TG6UKFQJooUrVfX/Icab5UPYONvSH7mKa/pA
--> ssh-ed25519 +3V2lQ NxpGLFMboFSAztflSWw+NFjByFfkBL/IG4r/hFvMjkQ
-0uWTKEG3TAsNsrPcooLsrINmDTWKlVIx1/OAL2rlcgc
---- VrkwgHMM0SXQKvH6I1oz35B391zF9QHysr3AZxGTpxw
-M’°°<>l0<6C>â!wÏú™Þ+–­‹B¼<s¤à`ÚEÂ*_<>Û„ÂݘÒ1þÁó¥Jâ¡[¥?ì¾Î|»‹

secrets/secrets.nix

@@ -70,9 +70,6 @@ in
   "grafana-smtp-password.age".publicKeys = flora6Keys ++ adminKeys;
 
   "alertmanager-envfile.age".publicKeys = flora6Keys ++ adminKeys;
-  "nachtigall-metrics-nginx-basic-auth.age".publicKeys = nachtigallKeys ++ adminKeys;
-  "nachtigall-metrics-prometheus-basic-auth-password.age".publicKeys =
-    flora6Keys ++ nachtigallKeys ++ adminKeys;
 
   "obs-portal-env.age".publicKeys = nachtigallKeys ++ adminKeys;
   "obs-portal-database-env.age".publicKeys = nachtigallKeys ++ adminKeys;

terraform/dns.tf

@@ -4,6 +4,46 @@ resource "namecheap_domain_records" "pub-solar" {
   mode       = "OVERWRITE"
   email_type = "MX"
 
+  record {
+    hostname = "nachtigall.wg"
+    type     = "A"
+    address  = "10.7.6.1"
+  }
+  record {
+    hostname = "flora-6.wg"
+    type     = "A"
+    address  = "10.7.6.2"
+  }
+  record {
+    hostname = "metronom.wg"
+    type     = "A"
+    address  = "10.7.6.3"
+  }
+  record {
+    hostname = "tankstelle.wg"
+    type     = "A"
+    address  = "10.7.6.4"
+  }
+  record {
+    hostname = "nachtigall.wg"
+    type     = "AAAA"
+    address  = "fd00:fae:fae:fae:fae:1::"
+  }
+  record {
+    hostname = "flora-6.wg"
+    type     = "AAAA"
+    address  = "fd00:fae:fae:fae:fae:2::"
+  }
+  record {
+    hostname = "metronom.wg"
+    type     = "AAAA"
+    address  = "fd00:fae:fae:fae:fae:3::"
+  }
+  record {
+    hostname = "tankstelle.wg"
+    type     = "AAAA"
+    address  = "fd00:fae:fae:fae:fae:4::"
+  }
   record {
     hostname = "flora-6"
     type     = "A"