blackbox-exporter: scrape every 60m, add URLs

before certificate expiry

docs/administrative-access.md

@@ -28,18 +28,18 @@ People with admin access to the infrastructure are added to [`logins/admins.nix`
 SSH is not reachable from the open internet. Instead, SSH Port 22 is protected by a wireguard VPN network. Thus, to get root access on the servers, at least two pieces of information have to be added to the admins config:
 
 1. **SSH Public key**: self-explanatory. Add your public key to your user attrset under `sshPubKeys`.
-2. **Wireguard device**: each wireguard device has two parts: the public key and the IP addresses it should have in the wireguard network. The pub.solar wireguard network uses the subnets `10.7.6.0/24` and `fd00:fae:fae:fae:fae::/80`. To add your device, it's best to choose a free number between 200 and 255 and use that in both the ipv4 and ipv6 ranges: `10.7.6.<ip-address>/32` `fd00:fae:fae:fae:fae:<ip-address>::/96`. For more information on how to generate keypairs, see [the NixOS Wireguard docs](https://nixos.wiki/wiki/WireGuard#Generate_keypair).
+2. **Wireguard device**: each wireguard device has two parts: the public key and the IP addresses it should have in the wireguard network. The pub.solar wireguard network is spaced under `10.7.6.0/24` and `fd00:fae:fae:fae:fae::/80`. To add your device, it's best to choose a free number between 200 and 255 and use that in both the ipv4 and ipv6 ranges: `10.7.6.<ip-address>/32` `fd00:fae:fae:fae:fae:<ip-address>::/96`. For more information on how to generate keypairs, see [the NixOS Wireguard docs](https://nixos.wiki/wiki/WireGuard#Generate_keypair).
 
 One can access our hosts using this domain scheme:
 
 ```
-ssh <unix-username>@<hostname>.wg.pub.solar
+ssh barkeeper@<hostname>.wg.pub.solar
 ```
 
 So, for example for `nachtigall`:
 
 ```
-ssh teutat3s@nachtigall.wg.pub.solar
+ssh barkeeper@nachtigall.wg.pub.solar
 ```
 
 Example NixOS snippet for WireGuard client config

docs/deploying.md

@@ -7,29 +7,16 @@ be manually deployed.
 To deploy, make sure you have a [working development shell](./development-shell.md).
 Then, run `deploy-rs` with the hostname of the server you want to deploy:
 
-### Dry-run
-
-Use `--dry-activate` to show a diff of updated packages and all services that
-would be restarted by the update. This will also put all files in place without
-switching to the new generation, enabling a quick switch to the new config at a
-later moment.
-
 For nachtigall.pub.solar:
 
 ```
-deploy --targets '.#nachtigall' --ssh-user <unix-username> --magic-rollback false --auto-rollback false --keep-result --result-path ./results --dry-activate
+deploy --targets '.#nachtigall' --magic-rollback false --auto-rollback false --keep-result --result-path ./results
-```
-
-After reviewing the changes, apply the update with:
-
-```
-deploy --targets '.#nachtigall' --ssh-user <unix-username> --magic-rollback false --auto-rollback false --keep-result --result-path ./results
 ```
 
 For metronom.pub.solar (aarch64-linux):
 
 ```
-deploy --targets '.#metronom' --ssh-user <unix-username> --magic-rollback false --auto-rollback false --keep-result --result-path ./results --remote-build
+deploy --targets '.#metronom' --magic-rollback false --auto-rollback false --keep-result --result-path ./results --remote-build
 ```
 
 Usually we skip all rollback functionality, but if you want to deploy a change
@@ -38,6 +25,9 @@ that might lock you out, e.g. to SSH, it might make sense to set these to `true`
 To skip flake checks, e.g. because you already ran them manually before
 deployment, add the flag `--skip-checks` at the end of the command.
 
+`--dry-activate` can be used to only put all files in place without switching,
+to enable switching to the new config quickly at a later moment.
+
 We use `--keep-result --result-path ./results` to keep the last `result`
 symlink of each `deploy` from being garbage collected. That way, we keep builds
 cached in the Nix store. This is optional and both flags can be removed if disk

docs/garage.md

@@ -8,7 +8,7 @@ Requirements:
 - [Setup WireGuard](./administrative-access.md#ssh-access) for hosts: `trinkgenossin`, optionally: `delite`, `blue-shell`
 
 ```
-ssh <unix-username>@trinkgenossin.wg.pub.solar
+ssh barkeeper@trinkgenossin.wg.pub.solar
 ```
 
 ```
@@ -58,7 +58,7 @@ Further reading:
 ### Notes on manual setup steps
 
 ```
-ssh <unix-username>@trinkgenossin.wg.pub.solar
+ssh barkeeper@trinkgenossin.wg.pub.solar
 
 # Add a few spaces to avoid leaking the secret to the shell history
     export GARAGE_RPC_SECRET=<secret-in-keepass>

docs/nix-flake-updates.md

@@ -41,7 +41,3 @@ wrapped-ruby-mastodon-gems: 4.2.1 → 4.2.3
 zfs-kernel: 2.2.1-6.1.64 → 2.2.2-6.1.66
 zfs-user: 2.2.1 → 2.2.2
 ```
-
-### Deploying updates
-
-See [deploying.md](./deploying.md).

flake.lock

@@ -94,11 +94,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1731895210,
+        "lastModified": 1729712798,
-        "narHash": "sha256-z76Q/OXLxO/RxMII3fIt/TG665DANiE2lVvnolK2lXk=",
+        "narHash": "sha256-a+Aakkb+amHw4biOZ0iMo8xYl37uUL48YEXIC5PYJ/8=",
         "owner": "nix-community",
         "repo": "disko",
-        "rev": "639d1520df9417ca2761536c3072688569e83c80",
+        "rev": "09a776702b004fdf9c41a024e1299d575ee18a7d",
         "type": "github"
       },
       "original": {
@@ -185,11 +185,11 @@
         "nixpkgs-lib": "nixpkgs-lib"
       },
       "locked": {
-        "lastModified": 1730504689,
+        "lastModified": 1727826117,
-        "narHash": "sha256-hgmguH29K2fvs9szpq2r3pz2/8cJd2LPS+b4tfNFCwE=",
+        "narHash": "sha256-K5ZLCyfO/Zj9mPFldf3iwS6oZStJcU4tSpiXTMYaaL0=",
         "owner": "hercules-ci",
         "repo": "flake-parts",
-        "rev": "506278e768c2a08bec68eb62932193e341f55c90",
+        "rev": "3d04084d54bedc3d6b8b736c70ef449225c361b1",
         "type": "github"
       },
       "original": {
@@ -280,11 +280,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1731518114,
+        "lastModified": 1730041422,
-        "narHash": "sha256-h9Wb3VjmXBZwTO3prRweUKwp2H9hZHCQKrkbU+2WPQs=",
+        "narHash": "sha256-aEz5/yUJN/PSEXwPBuKMs2FbAmz68fDIQ9B0tVRVmTo=",
         "ref": "main",
-        "rev": "060ecccc5f8c92a0705ab91ff047811efd559468",
+        "rev": "09f7b1ed16c99f5fb5c5f9a2a73ccc9ff0645b35",
-        "revCount": 36,
+        "revCount": 32,
         "type": "git",
         "url": "https://git.pub.solar/pub-solar/keycloak-theme"
       },
@@ -320,11 +320,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1732016537,
+        "lastModified": 1729757100,
-        "narHash": "sha256-XwXUK+meYnlhdQz2TVE4Wv+tsx1CkdGbDPt1tRzCNH4=",
+        "narHash": "sha256-x+8uGaX66V5+fUBHY23Q/OQyibQ38nISzxgj7A7Jqds=",
         "owner": "lnl7",
         "repo": "nix-darwin",
-        "rev": "61cee20168a3ebb71a9efd70a55adebaadfbe4d4",
+        "rev": "04193f188e4144d7047f83ad1de81d6034d175cd",
         "type": "github"
       },
       "original": {
@@ -336,11 +336,11 @@
     },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1731797254,
+        "lastModified": 1729449015,
-        "narHash": "sha256-df3dJApLPhd11AlueuoN0Q4fHo/hagP75LlM5K1sz9g=",
+        "narHash": "sha256-Gf04dXB0n4q0A9G5nTGH3zuMGr6jtJppqdeljxua1fo=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "e8c38b73aeb218e27163376a2d617e61a2ad9b59",
+        "rev": "89172919243df199fe237ba0f776c3e3e3d72367",
         "type": "github"
       },
       "original": {
@@ -352,14 +352,14 @@
     },
     "nixpkgs-lib": {
       "locked": {
-        "lastModified": 1730504152,
+        "lastModified": 1727825735,
-        "narHash": "sha256-lXvH/vOfb4aGYyvFmZK/HlsNsr/0CVWlwYvo2rxJk3s=",
+        "narHash": "sha256-0xHYkMkeLVQAMa7gvkddbPqpxph+hDzdu1XdGPJR+Os=",
         "type": "tarball",
-        "url": "https://github.com/NixOS/nixpkgs/archive/cc2f28000298e1269cea6612cd06ec9979dd5d7f.tar.gz"
+        "url": "https://github.com/NixOS/nixpkgs/archive/fb192fec7cc7a4c26d51779e9bab07ce6fa5597a.tar.gz"
       },
       "original": {
         "type": "tarball",
-        "url": "https://github.com/NixOS/nixpkgs/archive/cc2f28000298e1269cea6612cd06ec9979dd5d7f.tar.gz"
+        "url": "https://github.com/NixOS/nixpkgs/archive/fb192fec7cc7a4c26d51779e9bab07ce6fa5597a.tar.gz"
       }
     },
     "root": {
@@ -484,11 +484,11 @@
     },
     "unstable": {
       "locked": {
-        "lastModified": 1731676054,
+        "lastModified": 1729665710,
-        "narHash": "sha256-OZiZ3m8SCMfh3B6bfGC/Bm4x3qc1m2SVEAlkV6iY7Yg=",
+        "narHash": "sha256-AlcmCXJZPIlO5dmFzV3V2XF6x/OpNWUV8Y/FMPGd8Z4=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "5e4fbfb6b3de1aa2872b76d49fafc942626e2add",
+        "rev": "2768c7d042a37de65bb1b5b3268fc987e534c49d",
         "type": "github"
       },
       "original": {

flake.nix

@@ -120,43 +120,56 @@
           devShells.ci = pkgs.mkShell { buildInputs = with pkgs; [ nodejs ]; };
         };
 
-      flake = {
+      flake =
-        nixosModules = builtins.listToAttrs (
+        let
-          map (x: {
+          username = "barkeeper";
-            name = x;
+        in
-            value = import (./modules + "/${x}");
+        {
-          }) (builtins.attrNames (builtins.readDir ./modules))
+          inherit username;
-        );
 
-        checks = builtins.mapAttrs (
+          nixosModules = builtins.listToAttrs (
-          system: deployLib: deployLib.deployChecks self.deploy
+            map (x: {
-        ) inputs.deploy-rs.lib;
+              name = x;
+              value = import (./modules + "/${x}");
+            }) (builtins.attrNames (builtins.readDir ./modules))
+          );
 
-        formatter."x86_64-linux" = inputs.unstable.legacyPackages."x86_64-linux".nixfmt-rfc-style;
+          checks = builtins.mapAttrs (
+            system: deployLib: deployLib.deployChecks self.deploy
+          ) inputs.deploy-rs.lib;
 
-        deploy.nodes = self.lib.deploy.mkDeployNodes self.nixosConfigurations {
+          formatter."x86_64-linux" = inputs.unstable.legacyPackages."x86_64-linux".nixfmt-rfc-style;
-          nachtigall = {
+
-            hostname = "nachtigall.wg.pub.solar";
+          deploy.nodes = self.lib.deploy.mkDeployNodes self.nixosConfigurations {
-          };
+            nachtigall = {
-          metronom = {
+              hostname = "nachtigall.wg.pub.solar";
-            hostname = "metronom.wg.pub.solar";
+              sshUser = username;
-          };
+            };
-          tankstelle = {
+            metronom = {
-            hostname = "tankstelle.wg.pub.solar";
+              hostname = "metronom.wg.pub.solar";
-          };
+              sshUser = username;
-          underground = {
+            };
-            hostname = "80.244.242.3";
+            tankstelle = {
-          };
+              hostname = "tankstelle.wg.pub.solar";
-          trinkgenossin = {
+              sshUser = username;
-            hostname = "trinkgenossin.wg.pub.solar";
+            };
-          };
+            underground = {
-          delite = {
+              hostname = "80.244.242.3";
-            hostname = "delite.wg.pub.solar";
+              sshUser = username;
-          };
+            };
-          blue-shell = {
+            trinkgenossin = {
-            hostname = "blue-shell.wg.pub.solar";
+              hostname = "trinkgenossin.wg.pub.solar";
+              sshUser = username;
+            };
+            delite = {
+              hostname = "delite.wg.pub.solar";
+              sshUser = username;
+            };
+            blue-shell = {
+              hostname = "blue-shell.wg.pub.solar";
+              sshUser = username;
+            };
           };
         };
-      };
     };
 }

hosts/blue-shell/wireguard.nix

@@ -22,7 +22,7 @@ in
         "${wireguardIPv6}/96"
       ];
       privateKeyFile = config.age.secrets.wg-private-key.path;
-      peers = flake.self.logins.wireguardDevices ++ [
+      peers = flake.self.logins.admins.wireguardDevices ++ [
         {
           # trinkgenossin.pub.solar
           publicKey = "QWgHovHxtqiQhnHLouSWiT6GIoQDmuvnThYL5c/rvU4=";

hosts/delite/wireguard.nix

@@ -22,7 +22,7 @@ in
         "${wireguardIPv6}/96"
       ];
       privateKeyFile = config.age.secrets.wg-private-key.path;
-      peers = flake.self.logins.wireguardDevices ++ [
+      peers = flake.self.logins.admins.wireguardDevices ++ [
         {
           # trinkgenossin.pub.solar
           publicKey = "QWgHovHxtqiQhnHLouSWiT6GIoQDmuvnThYL5c/rvU4=";

hosts/metronom/wireguard.nix

@@ -18,7 +18,7 @@
         "fd00:fae:fae:fae:fae:3::/96"
       ];
       privateKeyFile = config.age.secrets.wg-private-key.path;
-      peers = flake.self.logins.wireguardDevices ++ [
+      peers = flake.self.logins.admins.wireguardDevices ++ [
         {
           # nachtigall.pub.solar
           endpoint = "138.201.80.102:51820";

hosts/nachtigall/wireguard.nix

@@ -18,7 +18,7 @@
         "fd00:fae:fae:fae:fae:1::/96"
       ];
       privateKeyFile = config.age.secrets.wg-private-key.path;
-      peers = flake.self.logins.wireguardDevices ++ [
+      peers = flake.self.logins.admins.wireguardDevices ++ [
         {
           # tankstelle.pub.solar
           endpoint = "80.244.242.5:51820";

hosts/tankstelle/wireguard.nix

@@ -18,7 +18,7 @@
         "fd00:fae:fae:fae:fae:4::/96"
       ];
       privateKeyFile = config.age.secrets.wg-private-key.path;
-      peers = flake.self.logins.wireguardDevices ++ [
+      peers = flake.self.logins.admins.wireguardDevices ++ [
         {
           # nachtigall.pub.solar
           endpoint = "138.201.80.102:51820";

hosts/trinkgenossin/wireguard.nix

@@ -22,7 +22,7 @@ in
         "${wireguardIPv6}/96"
       ];
       privateKeyFile = config.age.secrets.wg-private-key.path;
-      peers = flake.self.logins.wireguardDevices ++ [
+      peers = flake.self.logins.admins.wireguardDevices ++ [
         {
           # nachtigall.pub.solar
           endpoint = "138.201.80.102:51820";

logins/default.nix

@@ -6,16 +6,19 @@ in
 {
   flake = {
     logins = {
-      admins = admins;
+      admins =
-      wireguardDevices = lib.lists.foldl (
+        lib.lists.foldl
-        wireguardDevices: adminConfig:
+          (logins: adminConfig: {
-        wireguardDevices ++ (if adminConfig ? "wireguardDevices" then adminConfig.wireguardDevices else [ ])
+            sshPubKeys = logins.sshPubKeys ++ (lib.attrsets.attrValues adminConfig.sshPubKeys);
-      ) [ ] (lib.attrsets.attrValues admins);
+            wireguardDevices =
-      sshPubKeys = lib.lists.foldl (
+              logins.wireguardDevices
-        sshPubKeys: adminConfig:
+              ++ (if adminConfig ? "wireguardDevices" then adminConfig.wireguardDevices else [ ]);
-        sshPubKeys
+          })
-        ++ (if adminConfig ? "sshPubKeys" then lib.attrsets.attrValues adminConfig.sshPubKeys else [ ])
+          {
-      ) [ ] (lib.attrsets.attrValues admins);
+            sshPubKeys = [ ];
+            wireguardDevices = [ ];
+          }
+          (lib.attrsets.attrValues admins);
       robots.sshPubKeys = lib.attrsets.attrValues robots;
     };
   };

modules/core/default.nix

@@ -54,5 +54,9 @@
     };
 
     time.timeZone = "Etc/UTC";
+
+    home-manager.users.${config.pub-solar-os.authentication.username} = {
+      home.stateVersion = "23.05";
+    };
   };
 }

modules/core/nix.nix

@@ -11,17 +11,6 @@
     permittedInsecurePackages = [ "olm-3.2.16" ];
   };
 
-  system.activationScripts.diff-closures = {
-    text = ''
-      if [[ -e /run/current-system ]]; then
-        ${config.nix.package}/bin/nix store diff-closures \
-          /run/current-system "$systemConfig" \
-          --extra-experimental-features nix-command
-      fi
-    '';
-    supportsDryActivation = true;
-  };
-
   nix = {
     # Use default version alias for nix package
     package = pkgs.nix;

modules/core/terminal-tooling.nix

@@ -1,27 +1,19 @@
-{ flake, lib, ... }:
+{ flake, config, ... }:
 {
-  home-manager.users = (
+  home-manager.users.${config.pub-solar-os.authentication.username} = {
-    lib.attrsets.foldlAttrs (
+    programs.git.enable = true;
-      acc: name: value:
+    programs.starship.enable = true;
-      acc
+    programs.bash.enable = true;
-      // {
+    programs.neovim = {
-        ${name} = {
+      enable = true;
-          programs.git.enable = true;
+      vimAlias = true;
-          programs.starship.enable = true;
+      viAlias = true;
-          programs.bash.enable = true;
+      defaultEditor = true;
-          programs.neovim = {
+      # configure = {
-            enable = true;
+      #   packages.myVimPackages = with pkgs.vimPlugins; {
-            vimAlias = true;
+      #     start = [vim-nix vim-surrund rainbow];
-            viAlias = true;
+      #   };
-            defaultEditor = true;
+      # };
-            # configure = {
+    };
-            #   packages.myVimPackages = with pkgs.vimPlugins; {
+  };
-            #     start = [vim-nix vim-surrund rainbow];
-            #   };
-            # };
-          };
-        };
-      }
-    ) { } flake.self.logins.admins
-  );
 }

modules/core/users.nix

@@ -11,6 +11,18 @@
       inherit (lib) mkOption types;
     in
     {
+      username = mkOption {
+        description = "Username for the adminstrative user";
+        type = types.str;
+        default = flake.self.username;
+      };
+
+      sshPubKeys = mkOption {
+        description = "SSH Keys that should have administrative root access";
+        type = types.listOf types.str;
+        default = flake.self.logins.admins.sshPubKeys;
+      };
+
       root.initialHashedPassword = mkOption {
         description = "Hashed password of the root account";
         type = types.str;
@@ -31,60 +43,36 @@
     };
 
   config = {
-    users.users =
+    users.users.${config.pub-solar-os.authentication.username} = {
-      (lib.attrsets.foldlAttrs (
+      name = config.pub-solar-os.authentication.username;
-        acc: name: value:
+      group = config.pub-solar-os.authentication.username;
-        acc
+      extraGroups = [
-        // {
+        "wheel"
-          ${name} = {
+        "docker"
-            name = name;
+      ];
-            group = name;
+      isNormalUser = true;
-            extraGroups = [
+      openssh.authorizedKeys.keys = config.pub-solar-os.authentication.sshPubKeys;
-              "wheel"
+    };
-              "docker"
+    users.groups.${config.pub-solar-os.authentication.username} = { };
-            ];
-            isNormalUser = true;
-            openssh.authorizedKeys.keys = lib.attrsets.attrValues value.sshPubKeys;
-          };
-        }
-      ) { } flake.self.logins.admins)
-      // {
-        # TODO: Remove when we stop locking ourselves out.
-        root.openssh.authorizedKeys.keys = flake.self.logins.sshPubKeys;
-        root.initialHashedPassword = config.pub-solar-os.authentication.root.initialHashedPassword;
 
-        ${config.pub-solar-os.authentication.robot.username} = {
+    # TODO: Remove when we stop locking ourselves out.
-          description = "CI and automation user";
+    users.users.root.openssh.authorizedKeys.keys = config.pub-solar-os.authentication.sshPubKeys;
-          home = "/home/${config.pub-solar-os.authentication.robot.username}";
-          createHome = true;
-          useDefaultShell = true;
-          uid = 998;
-          group = "${config.pub-solar-os.authentication.robot.username}";
-          isSystemUser = true;
-          openssh.authorizedKeys.keys = config.pub-solar-os.authentication.robot.sshPubKeys;
-        };
-      };
 
-    home-manager.users = (
+    users.users.${config.pub-solar-os.authentication.robot.username} = {
-      lib.attrsets.foldlAttrs (
+      description = "CI and automation user";
-        acc: name: value:
+      home = "/home/${config.pub-solar-os.authentication.robot.username}";
-        acc
+      createHome = true;
-        // {
+      useDefaultShell = true;
-          ${name} = {
+      uid = 998;
-            home.stateVersion = "23.05";
+      group = "${config.pub-solar-os.authentication.robot.username}";
-          };
+      isSystemUser = true;
-        }
+      openssh.authorizedKeys.keys = config.pub-solar-os.authentication.robot.sshPubKeys;
-      ) { } flake.self.logins.admins
+    };
-    );
 
-    users.groups =
+    users.groups.${config.pub-solar-os.authentication.robot.username} = { };
-      (lib.attrsets.foldlAttrs (
+
-        acc: name: value:
+    users.users.root.initialHashedPassword =
-        acc // { "${name}" = { }; }
+      config.pub-solar-os.authentication.root.initialHashedPassword;
-      ) { } flake.self.logins.admins)
-      // {
-        ${config.pub-solar-os.authentication.robot.username} = { };
-      };
 
     security.sudo.wheelNeedsPassword = false;
   };

modules/coturn/default.nix

@@ -5,8 +5,9 @@
   ...
 }:
 {
-  age.secrets."coturn-static-auth-secret" = {
+  age.secrets."nachtigall-coturn-static-auth-secret" = {
-    file = "${flake.self}/secrets/coturn-static-auth-secret.age";
+    file = "${flake.self}/secrets/nachtigall-coturn-static-auth-secret.age";
+    path = "/run/agenix/coturn-static-auth-secret";
     mode = "400";
     owner = "turnserver";
   };
@@ -18,7 +19,7 @@
     min-port = 49000;
     max-port = 50000;
     use-auth-secret = true;
-    static-auth-secret-file = config.age.secrets."coturn-static-auth-secret".path;
+    static-auth-secret-file = config.age.secrets."nachtigall-coturn-static-auth-secret".path;
     realm = "turn.${config.pub-solar-os.networking.domain}";
     cert = "${config.security.acme.certs.${realm}.directory}/full.pem";
     pkey = "${config.security.acme.certs.${realm}.directory}/key.pem";

modules/forgejo/default.nix

@@ -141,12 +141,6 @@
         LOGIN_REMEMBER_DAYS = 365;
       };
 
-      # See https://docs.gitea.com/administration/config-cheat-sheet#migrations-migrations
-      migrations = {
-        # This allows migrations from the same forgejo instance
-        ALLOW_LOCALNETWORKS = true;
-      };
-
       # https://forgejo.org/docs/next/admin/config-cheat-sheet/#indexer-indexer
       indexer = {
         REPO_INDEXER_ENABLED = true;

modules/grafana/default.nix

@@ -28,6 +28,11 @@
       group = "grafana";
       user = "grafana";
     };
+    "grafana-dashboards/blackbox-exporter_rev3.json" = {
+      source = ./grafana-dashboards/blackbox-exporter_rev3.json;
+      group = "grafana";
+      user = "grafana";
+    };
     "grafana-dashboards/synapse.json" = {
       source = ./grafana-dashboards/synapse.json;
       group = "grafana";

modules/matrix-irc/default.nix

@@ -35,8 +35,7 @@ in
           port = 1113;
         };
         logging = {
-          # set to debug for debugging
+          level = "debug";
-          level = "warn";
           maxFiles = 5;
           toCosole = true;
         };

modules/matrix/default.nix

@@ -124,17 +124,6 @@ in
         enable_room_list_search = true;
         encryption_enabled_by_default_for_room_type = "off";
         event_cache_size = "100K";
-
-        # https://github.com/element-hq/synapse/issues/11203
-        # No YAML deep-merge, so this needs to be in secret extraConfigFiles
-        # together with msc3861
-        #experimental_features = {
-        #  # Room summary API
-        #  msc3266_enabled = true;
-        #  # Rendezvous server for QR Code generation
-        #  msc4108_enabled = true;
-        #};
-
         federation_rr_transactions_per_room_per_second = 50;
         federation_client_minimum_tls_version = "1.2";
         forget_rooms_on_leave = true;
@@ -294,9 +283,7 @@ in
       createDatabase = true;
       extraConfigFiles = config.pub-solar-os.matrix.matrix-authentication-service.extra-config-files;
 
-      # https://element-hq.github.io/matrix-authentication-service/reference/configuration.html
       settings = {
-        account.email_change_allowed = false;
         http.public_base = "https://mas.${config.pub-solar-os.networking.domain}";
         http.issuer = "https://mas.${config.pub-solar-os.networking.domain}";
         http.listeners = [

modules/nginx-matrix/default.nix

@@ -24,6 +24,14 @@ let
       secure_backup_required = false;
       secure_backup_setup_methods = [ ];
     };
+    "m.integrations" = {
+      managers = [
+        {
+          api_url = "https://dimension.${domain}/api/v1/scalar";
+          ui_url = "https://dimension.${domain}/element";
+        }
+      ];
+    };
   };
   wellKnownServer = domain: { "m.server" = "matrix.${domain}:8448"; };
   wellKnownSupport = {

modules/prometheus/alert-rules.nix

@@ -142,8 +142,8 @@ lib.mapAttrsToList
 
     cpu_using_90percent = {
       condition = ''100 - (avg by (instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) >= 90'';
-      time = "20m";
+      time = "10m";
-      description = "{{$labels.instance}} is running with cpu usage > 90% for at least 20 minutes: {{$value}}";
+      description = "{{$labels.instance}} is running with cpu usage > 90% for at least 10 minutes: {{$value}}";
     };
 
     reboot = {
@@ -198,10 +198,10 @@ lib.mapAttrsToList
       description = "{{$labels.instance}}: healtcheck {{$labels.job}} fails!";
       };
     */
-    #cert_expiry = {
+    cert_expiry = {
-    #  condition = "(probe_ssl_earliest_cert_expiry - time())/(3600*24) < 30";
+      condition = "(probe_ssl_earliest_cert_expiry - time())/(3600*24) < 21";
-    #  description = "{{$labels.instance}}: The TLS certificate will expire in less than 30 days: {{$value}}s";
+      description = "{{$labels.instance}}: The TLS certificate will expire in less than 21 days: {{$value}}s";
-    #};
+    };
 
     # ignore devices that disabled S.M.A.R.T (example if attached via USB)
 
@@ -234,10 +234,10 @@ lib.mapAttrsToList
       };
     */
 
-    #host_memory_under_memory_pressure = {
+    host_memory_under_memory_pressure = {
-    #  condition = "rate(node_vmstat_pgmajfault[1m]) > 1000";
+      condition = "rate(node_vmstat_pgmajfault[1m]) > 1000";
-    #  description = "{{$labels.instance}}: The node is under heavy memory pressure. High rate of major page faults: {{$value}}";
+      description = "{{$labels.instance}}: The node is under heavy memory pressure. High rate of major page faults: {{$value}}";
-    #};
+    };
 
     # ext4_errors = {
     #   condition = "ext4_errors_value > 0";

modules/prometheus/default.nix

@@ -5,6 +5,22 @@
   flake,
   ...
 }:
+let
+  blackboxTargets = [
+    "https://pub.solar"
+    "https://chat.pub.solar"
+    "https://cloud.pub.solar"
+    "https://collabora.pub.solar"
+    "https://git.pub.solar"
+    "https://grafana.pub.solar"
+    "https://list.pub.solar"
+    "https://mastodon.pub.solar"
+    "https://obs-portal.pub.solar"
+    "https://stream.pub.solar"
+    "https://wiki.pub.solar"
+    "https://www.pub.solar"
+  ];
+in
 {
   age.secrets.alertmanager-envfile = {
     file = "${flake.self}/secrets/alertmanager-envfile.age";
@@ -39,6 +55,32 @@
     enable = true;
     port = 9001;
     exporters = {
+      blackbox = {
+        enable = true;
+        # Default port is 9115
+        openFirewall = false;
+
+        configFile = pkgs.writeTextFile {
+          name = "blackbox-exporter-config";
+          text = ''
+            modules:
+              http_2xx:
+                prober: http
+                timeout: 5s
+                http:
+                  valid_http_versions: ["HTTP/1.1", "HTTP/2.0"]
+                  valid_status_codes: []  # Defaults to 2xx
+                  method: GET
+                  no_follow_redirects: false
+                  fail_if_ssl: false
+                  fail_if_not_ssl: false
+                  tls_config:
+                    insecure_skip_verify: false
+                  preferred_ip_protocol: "ip4" # defaults to "ip6"
+                  ip_protocol_fallback: true  # fallback to "ip6"
+          '';
+        };
+      };
       node = {
         enable = true;
         enabledCollectors = [ "systemd" ];
@@ -50,6 +92,30 @@
       scrape_timeout = "9s";
     };
     scrapeConfigs = [
+      {
+        job_name = "blackbox";
+        scrape_interval = "60m";
+        metrics_path = "/probe";
+        params = {
+          module = [ "http_2xx" ];
+        };
+        static_configs = [ { targets = blackboxTargets; } ];
+
+        relabel_configs = [
+          {
+            source_labels = [ "__address__" ];
+            target_label = "__param_target";
+          }
+          {
+            source_labels = [ "__param_target" ];
+            target_label = "instance";
+          }
+          {
+            target_label = "__address__";
+            replacement = "127.0.0.1:9115"; # The blackbox exporter's real hostname:port.
+          }
+        ];
+      }
       {
         job_name = "node-exporter";
         static_configs = [

modules/unlock-luks-on-boot/default.nix

@@ -10,7 +10,7 @@
 
       # Please create this manually the first time.
       hostKeys = [ "/etc/secrets/initrd/ssh_host_ed25519_key" ];
-      authorizedKeys = flake.self.logins.sshPubKeys;
+      authorizedKeys = config.pub-solar-os.authentication.sshPubKeys;
     };
     postCommands = ''
       # Automatically ask for the password on SSH login

modules/unlock-zfs-on-boot/default.nix

@@ -11,7 +11,7 @@
 
       # Please create this manually the first time.
       hostKeys = [ "/etc/secrets/initrd/ssh_host_ed25519_key" ];
-      authorizedKeys = flake.self.logins.sshPubKeys;
+      authorizedKeys = config.pub-solar-os.authentication.sshPubKeys;
     };
     # this will automatically load the zfs password prompt on login
     # and kill the other prompt so boot can continue

secrets/drone-db-secrets.age

@@ -0,0 +1,43 @@
+age-encryption.org/v1
+-> ssh-ed25519 Y0ZZaw 5slOxDM4xGALMpYxFm1WBX4Sds7itgPBMIiY97d7Egk
+mZNzn4I6obUHAdox6eVR4H82EZagZ1IrCcq0CDtK44Y
+-> ssh-ed25519 uYcDNw w5lzhmA8wIMXihKF25d5jx4/Cc5BFE3Lw6ad60b0wBg
+v9z03cpts6oVlcTQ48hMw8rjWHp1JUOov2qCUjFN4bs
+-> ssh-rsa f5THog
+A93Usdjf6yJFLFqDiy6UUIJ4faBgQXIvk0pZlABlj9M5n7fSf9uzR6sSih4HNCvm
+sMkZ4wKyQHJnUB4Uc2jGrdcWqpmP5MLYHhj74Rxsi6heZuCRf94KH7sE/03A958w
+jAV4v9z4EqmkvWLNQi/hxMVMs5A61Vs63WIX/TA6vhL8Yrn0FeIKlRZYUVIeMu42
+pbEuLWeIzbUioAuEA1ZKV5VDx+6ack8TS/Dj5bTNEnzFWpjnHjO0/GeQU+aaQZTe
+Zy325TcRosT0V7PIh0tDQZKKRpOH/e9LnDkd8NIjyfEsGdDYaP1EVOYVxPCqUDAh
+A0kV1kkTiBzaXDkuakc+HDCIxtYXLWthsmbD+vI3D7FlTl0CY4fOP0wwO/0rS5Yp
+KDuxjz89II1H4+ZvlcPUihyW7OEj4d+NwFQy+7Qq0Y9Ii0NONXNsnx17FKXJwOMo
+NKyLo097FvHV7k8F9wv9mmZboRulDAoRyDngeO0+SJA90uJass04DuiZvK+g3Hry
+xVzbkk59j9EQqUogopW/oSeSbUP0pvcKOahGcSIW8vmadDTgnN7zzqf3fq+dJ2TM
+QD2IXAwvoTFBE+9DnPOtptk1X1D2umZuAWTzGAseXOImrPFZ+bEr5MV3qLGlg7sL
+yA7Mvbp4diVdH5aePzeBefhxrYphz+yfCbELFTYam9g
+-> ssh-rsa kFDS0A
+Ng0FhTDjASWJkrlNh+UZxU/dU/wfmoV1/fwTv6Xg69k/2qU9lk0oR6e5xAimvX6u
+h7rKAYt3zSRIFveGczPCflC1nycG9wLSpaoJghav+q+muoDQ/fbSKSgHFXITC7Me
+f/wblyWvJsUQbjxSW3g6/8EGz6FvpTnycPtD2vbRj+Ctq72GPA2ZWg/OC4jAUlDs
+r6X0Ql2jwWzy3Y12v0mPknlBezN8cIfjBmoNOWokUeGJIBjujlS7loA1yif09BLg
+PTSLCY1YH3QYcm6lCXK0HaNcMjSSk/ZK9D0wROriF9PBbkpWgg5NlIrqGaeqPN9z
+QwRR2DvhuCa1br57F36Y2LKGphYjmhWAtzCyQ0h9YQ+AzEy9uFCbK0IFyyeVl/fN
++HBGgxacJBcEGsNV3mbJvh6dn1348eex0GgaQEf1B/lu/y66WHbmSqVyUDfWkqEz
+IytAC43VT2rKgg+B5u0d/JhLDLwXTp7iVDy52ul1n7keJHk8t1GDaufAXbWqalQ4
+vuyxs6ghSIXUi27IZrYblg/OEPFTBfcoMXkmCgyx5a+eK+DhnBazWjy5j+vgp2so
+ZQRQurbG02qpZasTwBM3iy4ZklX/uFjsKnk0c/YcmK4YcMviHcQQjdjKruEE93u+
+Za1KE+qZGLkhFCd9O3ZPMtEjRjpN10XIs5ylKQ9MKU4
+-> piv-p256 vRzPNw AiNjNIR0OGHBu5Qn+bvn+Lk5VnpI2BQ3eJ3+2/FTJfZC
+elT3acRVdmtBl0qC5YbvfntxkJrsZwEJqlF6aN5hhWw
+-> piv-p256 zqq/iw AjIzSibkqG+YcP894QekM61Wsty6MaKBghlWapHfU0Jn
+HyXBp8DxtnNsfuzZq13bwgma5CzLTf3UB5Eht6XUwe8
+-> ssh-ed25519 YFSOsg WRBQZZYM+X26hfoH4zvNWQulZvVWP/Ha5OgkUmGK/Q4
+5Hw4ZDNawn5YRC673Op/sbpexOKeL3gez2B7oZxUKhA
+-> ssh-ed25519 iHV63A wyr8R4DlqLAu0XypddVoFimK2ZMncWaa+KWV7vMEQm8
+puV3g1t5AbnEgC0S1U4ft1evB7KuNppEi1g/AtxHgWE
+-> ssh-ed25519 BVsyTA 0N3iyyGqTCRAHHcK7QfN5xRttorc2E2GL0RDTIVIBU4
+Bph0OujqmXzi9IswduX9Mbh+yRdPKOwCf3fBv2zUzqI
+-> ssh-ed25519 +3V2lQ 0p90VtsxWyGFaeeoTISIxQRyeKVk0HoGGq71tjpIPjg
+sRf73Tp3BJ0DsTnJO2xVGyCKjaX7C7oydXj+39dKMUg
+--- +/HCG0s/x+c03NG5qrgliJ+5EXXI6UnuJz5XDv2aphY
+ÞšÂ<>™Ý@»=£L¬“7*®„ÐFq<46>UÒ*ûU꿉»È$e=þLgJ|*1Ͻ
BÚE ZG—_Ü5ê²ð—²ŽíÂ,òöÛi<C39B>_'¸d7
Ý3Ú“Nä3ãç¡*»ðªê<C2AA>£ŽáŽòqýŸ‰Oy#¶([l³†pÄf¼õ¾¥ö

secrets/forgejo-actions-runner-token.age

@@ -0,0 +1,43 @@
+age-encryption.org/v1
+-> ssh-ed25519 Y0ZZaw tm4AmC8yXPgR82lgsQR4VZn4xfGiK8o6fIn8pKPY6XQ
+IDnsYVD8noh2HdPNvjY/M5G+meR5rwvVI5SVN/cHEVM
+-> ssh-ed25519 uYcDNw ZB7GyOvD8S8XLqE1AeMXWTPcJnvEntWbZ7TGg8CJVxc
+Bfb/+P2DEiKI9ZWH105rLAYQXTUwWftWtudUGnVtjSU
+-> ssh-rsa f5THog
+Q3Hqks6BMGmP8TXnUkbblO8btrVdls7AUdxDW8e/w5biis/4awZVBHuZCLpiasM7
+7RWXcUep2VVyfCMb+8tedaf5a1MpGPDkZvdbxhfDVWZakh7vsEnth+gK2QsV0h8e
+eIgfDMA9J6DHXNCr4EYSf22PxY12KPqGqsMpVBhOZQuXoJwJy0ob3jbJEOYfPlu6
+V+TLYQNGQ2UQ+A7zjpUrGz1L+u7rUejY4Rv1BmCakg1bLEs8oSDmIVmsuVmFPqOI
+wqQJvnYlRAdioVQZwYCiqJech2QJ0ZhtC/ZeVp4c6TgFwB1ottxiAI8l7Bz1nFzW
+/E85qU3Jkh1tcNcLsVHj8+tnxwn1SSQ3xQxcOT9l1Po98sNapK7mwd/xx3pJ1hH8
+5YUQAtG80e5YmOBxkabVI4s612wACfK9JrIdL+uyIIzGeNHgoimjx7GuOCQ2ut6L
+gj/+Rcv3e0ERkNaXTXGkcdrsMTt45lGmyxUgxz6lbHgtqq+r+BHogiQ9cdPKwXuK
+wom4AvaMOBKCxtB4qVsuNHRd0I8OaA9Ab9SUvHCRvzCkJRHP4qc0zRJif7Rk4qRl
+rAGYwVxq3DRk2HHCQCYC26VqLU6B5LuAAqOipVhOeTfbgaSGD6Wfrt+XBBPmWB7o
+i9zDrk5GKehsPeDKgjh9uVd1y+IBHcWoYBxR5WPTYnA
+-> ssh-rsa kFDS0A
+HS3y8/5wAej0jv0sQqhdGwWd79vUwkrLkoKKPmqo9HlJaO70Cr1bnAIdyA8PBphs
+NIRjIcdClbUBuelZudzuhHuEzH8/JMAVwgHoIiUxviEIr6JpJVBagtvSHp1nfDdn
+x/hkpt2isSYb6fVzYgewqkdD3tv9XEm/WR2JmzlfaNTV9N9x/HNJYy5iYoTWRxKr
+e8R7txdmgRaDDxpbkJdWBcoV9HVgytTMtvBkqGViWzaFDopb9nDlfN/C7/BkCp6H
+9b65JqznpIChoJV7+sK5SEw8VcFj7ikIHzREWscEn8XBb7Kth2iVukaEPM+BgGZz
+Irk0IdkSb/XmQFwsOLnQViwUjkFXGXwHyMdHIcU2qEzZ4PN0PgEengILt9vqWJs0
+qHxrA7sKiC1D8S0i1+Dn+DiI//1s5Bbmp1jk626tH6fNKqSOlpwM47IGArTxCAFd
+NMinIBnR47DUCXWheirsWF6yP7kwX7vOW1dR4UVJJnVPKkgjklwCZvJiNAo0Soo+
+95zuugaeobsJ+qz2Pv+l8BGYriOFpRIAu7YTy9yY7mqHwC5MoeY0G8eNg5UmEzFz
+JsEbKPsZcsMg4WdywzLU2aufK4M/Cd7lVPGZzuZ8hJHBF/EvTFov7L/HK5VnhZar
+CYtILdyiVvmMZ7dhEARG1GG988W9wMendikmKpM4dTA
+-> piv-p256 vRzPNw AuCJRxHGmvv25VTHpnbfMLyLIj8K+daFD97wwHvFAqHl
+m4lPR+5h3+xmdL0OBfmNoxSM/O5Ca+2lVRLwITUtVmQ
+-> piv-p256 zqq/iw ApNbp/6seWw6gCj/QWKLYlmuHaqdaSKVI+Hup1fKAO2O
+xpNXgDXMVFe15eS+L9lGaI0Ip2F13SuhjCTQLDtBIr0
+-> ssh-ed25519 YFSOsg v9BtvUZh5HIvN7nsnErVrHbWTwRhWpj/SlHoiiJSIR0
+ol9z9juHfOCuZsSpuRl/zGFuJg5RzpKK1YnX+VDLDTA
+-> ssh-ed25519 iHV63A yfa9P22C7+wCMqtRRSyiOhcFnLWPI50jMWxWpLarMT8
+VpU+Uv/20JJGkTWTATiF5JImMsDKwyHMj+Wp0mMC/MQ
+-> ssh-ed25519 BVsyTA dUj/mHSyOm61h2ETa4tSX7Cyf+KRN8AMXCVKwMbJTE0
+E1EEPqVQSqlxSAi7DaMlaS1Az1D5XsasrPrIdwylSAE
+-> ssh-ed25519 +3V2lQ AWBlXeuJq76UgQR0xQVPMN0NUq/mqa2iDlIDBBp/Wjo
+XiliN4FB5YQ7qmTyV3AIbvoCY8UoGS6Vi5LpVWrH9kg
+--- V4Tp84/WJUM+/l+eEjlypE4Lx47BtkGFpEnNIdIB35E
+ô¡qzîL¡çzYNŽv×0[¿AöKN<ÛÀ<C39B>ö¦÷:,D<>¢S¹:!$`Õw©^‡àŽhi†O'(ž7!=ÿC(ÿï>´ûDоÙFÐ=J

secrets/restic-password-droppie.age

@@ -0,0 +1,29 @@
+age-encryption.org/v1
+-> ssh-ed25519 iDKjwg tSrbeVPpTxjeuCg6JiOdeIcaGWq/PUOyXrt+ZNIWmzI
+fz4B2nMu36G1DYPyNiu30Txx4cOLGyUreuBLqktORhY
+-> ssh-ed25519 uYcDNw zySKp8qUnCKOCbIP17bWYvJ/TsKxLl/v/AEsfVQpHTc
+SYqaUEXJ7mSA2wY6I3i/hirEiqvXVnWH1NYtXbaMzbw
+-> ssh-rsa kFDS0A
+MiyJjq7nn9B+PF9Lf6vD496FZUnUy5zI9H2kunJLhmYnr+7ZJaPGI+MW0cuPjsmM
+1XhmUC72zA9WTiuReGijl9GhIyU0D6vPx+ezQrx18dUGRJEqnvk5C75SCUuPGtUi
+DamPxJc781mnDSyW91MgRrqlBVwhfJ9Zmws3+/j8LODWb0T9Jvy0ywmFX4ilapr9
+6dqbNmwBZkfh2CKB9LtMG/DuSNicgHscpt4Mt7EogJRdgBrGIcA1u8BXUOwf+R8G
+Ya8VzoQV0iVhGtbff3g/6/0+UX9KEZBQc/aiT2ItrgUmgMyjAZo13Dj3pD2wPdfa
+t346symp7bcYxRea/U0hfmPdnixIKgBLTUZkzrMFza26QOvnSwiMxWY+//Vmbp3z
+0YCsyOQTQJVB+Q0pYE0+9EVI9bInP0LxHhM7mIdJroscRtY1YphjZUlFmcUXk0iC
+5g3/wdLr8KdT4aZ93m4WVgkBUzg8chhfMHb9uFvpHT9xeZHrV3u30TuJpmmuqMXK
+JD3+9xRzNVHwvfZuhD8B3vkK2tYO7pTxJytkJtdolx/uSju8SCm9F4lwt5y8rQd4
+7676QS0IWS1WItcpLrsgk9nWZUNzy78gBc39jtJUOsFDqJSdAWACq/jgYpnx6Bja
+5aZonRAp3fyCEavUpS4AQUJZGVx9EFp9LqXeUiIkCJU
+-> ssh-ed25519 YFSOsg vJ7xw4zDSKBFuv8FT6ZnB83uGwHDnDsEwsvQ6urZkEk
+gOkuwjOX8U/qmPuGrPPb+FYy6PuJjrgCNFYrukkaIPA
+-> ssh-ed25519 iHV63A FzZRHQB39iDTLm4y1QLWtfDi3jZn6i59nuSlf0d5mww
+5kiyKN7KFTv8pJgRAZTUMgR2+TInByFotDhAw307P6g
+-> ssh-ed25519 BVsyTA rTjBaEI6AwwmpK9Q8elVJaSN60deXOMEmZPrsH9O1nY
+HikNbHzx3BEJ3hn4YPFr2p/cnSu3qr9cvgdKCxN6xY8
+-> h|2u%UF\-grease Gcp2NG
+y6OjgzrOYEVgrY2+fv8a1xrVuoHQIPncGuGeLI0zHh4xf1qbzD2vKYp+W1fRBE8d
+/yPEGUe/T/ZjO+F4oF266HUR23wancFeFoGpfJgWQVS2oc0Z4aMuNebf/+Kw087W
+
+--- GJBLNxBoH7vI1mkcCmfbm9UiG4xuwIOWS7IaJKuO6cc
+C<EFBFBD>3=¼,¦{-TçÄéȵ	™-ˆ]äÌ·»æq=‡“û
Òú–=~}Î1®˜ lß„¿fúì_[žU_å›A€G.ÒdŽêÖrëa`×åUyÉPt+̽òö‡¯!<

secrets/secrets.nix

@@ -92,7 +92,7 @@ in
   "mediawiki-oidc-client-secret.age".publicKeys = nachtigallKeys ++ adminKeys;
   "mediawiki-secret-key.age".publicKeys = nachtigallKeys ++ adminKeys;
 
-  "coturn-static-auth-secret.age".publicKeys = nachtigallKeys ++ adminKeys;
+  "nachtigall-coturn-static-auth-secret.age".publicKeys = nachtigallKeys ++ adminKeys;
 
   "grafana-admin-password.age".publicKeys = trinkgenossinKeys ++ adminKeys;
   "grafana-keycloak-client-secret.age".publicKeys = trinkgenossinKeys ++ adminKeys;

tests/keycloak.nix

@@ -66,7 +66,7 @@ in
   testScript =
     { nodes, ... }:
     let
-      user = nodes.client.users.users.b12f;
+      user = nodes.client.users.users.${nodes.client.pub-solar-os.authentication.username};
       #uid = toString user.uid;
       bus = "DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/$(id -u ${user.name})/bus";
       gdbus = "${bus} gdbus";

tests/support/client.nix

@@ -11,7 +11,7 @@
   services.xserver.displayManager.gdm.enable = true;
   services.xserver.desktopManager.gnome.enable = true;
   services.xserver.displayManager.autoLogin.enable = true;
-  services.xserver.displayManager.autoLogin.user = "b12f";
+  services.xserver.displayManager.autoLogin.user = config.pub-solar-os.authentication.username;
 
   systemd.user.services = {
     "org.gnome.Shell@wayland" = {