obs-portal: add backups to garage bucket + storagebox

Restic backups to garage S3 bucket nachtigall-backups
Merge pull request 'devshell: add terraform-ls' (#227 ) from terraform-devshell into main
2024-08-29 10:09:04 +02:00 · 2024-08-28 15:55:29 +00:00 · 2024-08-28 15:55:16 +00:00 · 2024-08-28 15:54:33 +00:00 · 2024-08-28 17:13:34 +02:00 · 2024-08-28 17:13:34 +02:00
60 changed files with 3168 additions and 36 deletions
--- a/.forgejo/workflows/check.yml
+++ b/.forgejo/workflows/check.yml
@ -18,14 +18,20 @@ jobs:
          # Prevent cache garbage collection by creating GC roots
          mkdir -p /var/lib/gitea-runner/tankstelle/.local/state/nix/results

-          for target in $(nix flake show --json --all-systems | jq '
+          for target in $(nix flake show --json --all-systems | jq --raw-output '
            .["nixosConfigurations"] |
            to_entries[] |
-            .key
-            ' | tr -d '"'
+            .key'
          ); do
            nix --print-build-logs --verbose --accept-flake-config --access-tokens '' \
              build --out-link /var/lib/gitea-runner/tankstelle/.local/state/nix/results/"$target" ".#nixosConfigurations.${target}.config.system.build.toplevel"
          done

-          nix --print-build-logs --verbose --accept-flake-config --access-tokens '' flake check
+          for check in $(nix flake show --json --all-systems | jq --raw-output '
+            .checks."x86_64-linux" |
+            to_entries[] |
+            .key'
+          ); do
+            nix --print-build-logs --verbose --accept-flake-config --access-tokens '' \
+              build --out-link /var/lib/gitea-runner/tankstelle/.local/state/nix/results/"$check" ".#checks.x86_64-linux.${check}"
+          done
--- a/docs/administrative-access.md
+++ b/docs/administrative-access.md
@ -85,6 +85,39 @@ Example NixOS snippet for WireGuard client config
            #endpoint = "80.244.242.5:51820";
            persistentKeepalive = 15;
          }
+          {
+            # trinkgenossin.pub.solar
+            publicKey = "QWgHovHxtqiQhnHLouSWiT6GIoQDmuvnThYL5c/rvU4=";
+            allowedIPs = [
+              "10.7.6.5/32"
+              "fd00:fae:fae:fae:fae:5::/96"
+            ];
+            #endpoint = "85.215.152.22:51820";
+            endpoint = "[2a01:239:35d:f500::1]:51820";
+            persistentKeepalive = 15;
+          }
+          {
+            # delite.pub.solar
+            publicKey = "ZT2qGWgMPwHRUOZmTQHWCRX4m14YwOsiszjsA5bpc2k=";
+            allowedIPs = [
+              "10.7.6.6/32"
+              "fd00:fae:fae:fae:fae:6::/96"
+            ];
+            #endpoint = "5.255.119.132:51820";
+            endpoint = "[2a04:52c0:124:9d8c::2]:51820";
+            persistentKeepalive = 15;
+          }
+          {
+            # blue-shell.pub.solar
+            publicKey = "bcrIpWrKc1M+Hq4ds3aN1lTaKE26f2rvXhd+93QrzR8=";
+            allowedIPs = [
+              "10.7.6.7/32"
+              "fd00:fae:fae:fae:fae:7::/96"
+            ];
+            #endpoint = "194.13.83.205:51820";
+            endpoint = "[2a03:4000:43:24e::1]:51820";
+            persistentKeepalive = 15;
+          }
        ];
      };
    };
--- a/docs/garage.md
+++ b/docs/garage.md
@ -0,0 +1,49 @@
+# Garage
+
+### How-To create a new bucket + keys
+
+Requirements:
+
+- [Setup WireGuard](./administrative-access.md#ssh-access) for hosts: `trinkgenossin`, optionally: `delite`, `blue-shell`
+
+```
+ssh barkeeper@trinkgenossin.wg.pub.solar
+
+export GARAGE_RPC_SECRET=<secret-in-keepass>
+
+garage bucket create <bucket-name>
+garage key create <bucket-name>-key
+garage bucket allow <bucket-name> --read --write --key <bucket-name>-key
+```
+
+Docs: https://garagehq.deuxfleurs.fr/documentation/quick-start/#creating-buckets-and-keys
+
+Then [setup your favourite S3 client](https://garagehq.deuxfleurs.fr/documentation/connect/cli/)
+or use the bucket with any [S3 compatible software](https://garagehq.deuxfleurs.fr/documentation/connect/).
+
+### Notes on manual setup steps
+
+```
+ssh barkeeper@trinkgenossin.wg.pub.solar
+
+export GARAGE_RPC_SECRET=<secret-in-keepass>
+
+# Uses the default config /etc/garage.toml
+garage node id
+
+garage node connect <node-id2>
+garage node connect <node-id3>
+
+garage status
+
+#Zones
+#DE-1 DE-2 NL-1
+
+garage layout assign fdaa -z DE-1 -c 800G -t trinkgenossin
+garage layout assign 8835 -z DE-2 -c 800G -t blue-shell
+garage layout assign 73da -z NL-1 -c 800G -t delite
+garage layout show
+garage layout apply --version 1
+```
+
+Source: https://garagehq.deuxfleurs.fr/documentation/cookbook/real-world/#creating-a-cluster-layout
--- a/docs/nixos-anywhere.md
+++ b/docs/nixos-anywhere.md
@ -0,0 +1,13 @@
+```
+curl -L https://github.com/nix-community/nixos-images/releases/download/nixos-unstable/nixos-kexec-installer-noninteractive-x86_64-linux.tar.gz | tar -xzf- -C /root
+/root/kexec/run
+```
+
+```
+mkdir -p /etc/secrets/initrd
+ssh-keygen -t ed25519 -f /etc/secrets/initrd/ssh_host_ed25519_key
+```
+
+```
+nix run github:nix-community/nixos-anywhere -- --flake .#blue-shell root@194.13.83.205
+```
--- a/flake.lock
+++ b/flake.lock
@ -87,6 +87,26 @@
        "type": "github"
      }
    },
+    "disko": {
+      "inputs": {
+        "nixpkgs": [
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1724349583,
+        "narHash": "sha256-zgB1Cfk46irIsto8666yLdKjqKdBrjR48Dd3lhQ0CnQ=",
+        "owner": "nix-community",
+        "repo": "disko",
+        "rev": "435737144be0259559ca3b43f7d72252b1fdcc1b",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "repo": "disko",
+        "type": "github"
+      }
+    },
    "element-stickers": {
      "inputs": {
        "maunium-stickerpicker": [
@ -376,6 +396,7 @@
      "inputs": {
        "agenix": "agenix",
        "deploy-rs": "deploy-rs",
+        "disko": "disko",
        "element-stickers": "element-stickers",
        "element-themes": "element-themes",
        "flake-parts": "flake-parts",
--- a/flake.nix
+++ b/flake.nix
@ -18,6 +18,9 @@
    deploy-rs.url = "github:serokell/deploy-rs";
    deploy-rs.inputs.nixpkgs.follows = "nixpkgs";

+    disko.url = "github:nix-community/disko";
+    disko.inputs.nixpkgs.follows = "nixpkgs";
+
    agenix.url = "github:ryantm/agenix";
    agenix.inputs.nixpkgs.follows = "nixpkgs";
    agenix.inputs.darwin.follows = "nix-darwin";
@ -65,6 +68,7 @@
          system,
          pkgs,
          config,
+          lib,
          ...
        }:
        {
@ -77,6 +81,27 @@
            unstable = import inputs.unstable { inherit system; };
            master = import inputs.master { inherit system; };
          };
+
+          checks =
+            let
+              nixos-lib = import (inputs.nixpkgs + "/nixos/lib") { };
+              testDir = builtins.attrNames (builtins.readDir ./tests);
+              testFiles = builtins.filter (n: builtins.match "^.*.nix$" n != null) testDir;
+            in
+            builtins.listToAttrs (
+              map (x: {
+                name = "test-${lib.strings.removeSuffix ".nix" x}";
+                value = nixos-lib.runTest (
+                  import (./tests + "/${x}") {
+                    inherit self;
+                    inherit pkgs;
+                    inherit lib;
+                    inherit config;
+                  }
+                );
+              }) testFiles
+            );
+
          devShells.default = pkgs.mkShell {
            buildInputs = with pkgs; [
              deploy-rs
@ -92,9 +117,11 @@
              treefmt
              nixos-generators
              inputs.nixpkgs-2205.legacyPackages.${system}.terraform
+              terraform-ls
              jq
            ];
          };
+
          devShells.ci = pkgs.mkShell { buildInputs = with pkgs; [ nodejs ]; };
        };

@ -135,6 +162,18 @@
              hostname = "tankstelle.wg.pub.solar";
              sshUser = username;
            };
+            trinkgenossin = {
+              hostname = "trinkgenossin.wg.pub.solar";
+              sshUser = username;
+            };
+            delite = {
+              hostname = "delite.wg.pub.solar";
+              sshUser = username;
+            };
+            blue-shell = {
+              hostname = "blue-shell.wg.pub.solar";
+              sshUser = username;
+            };
          };
        };
    };
--- a/hosts/blue-shell/configuration.nix
+++ b/hosts/blue-shell/configuration.nix
@ -0,0 +1,33 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+{
+  boot.loader.grub.enable = true;
+
+  boot.kernelParams = [
+    "boot.shell_on_fail=1"
+    "ip=dhcp"
+  ];
+
+  # This option defines the first version of NixOS you have installed on this particular machine,
+  # and is used to maintain compatibility with application data (e.g. databases) created on older NixOS versions.
+  #
+  # Most users should NEVER change this value after the initial install, for any reason,
+  # even if you've upgraded your system to a new NixOS release.
+  #
+  # This value does NOT affect the Nixpkgs version your packages and OS are pulled from,
+  # so changing it will NOT upgrade your system - see https://nixos.org/manual/nixos/stable/#sec-upgrading for how
+  # to actually do that.
+  #
+  # This value being lower than the current NixOS release does NOT mean your system is
+  # out of date, out of support, or vulnerable.
+  #
+  # Do NOT change this value unless you have manually inspected all the changes it would make to your configuration,
+  # and migrated your data accordingly.
+  #
+  # For more information, see `man configuration.nix` or https://nixos.org/manual/nixos/stable/options#opt-system.stateVersion .
+  system.stateVersion = "24.05"; # Did you read the comment?
+}
--- a/hosts/blue-shell/default.nix
+++ b/hosts/blue-shell/default.nix
@ -0,0 +1,13 @@
+{ flake, ... }:
+
+{
+  imports = [
+    ./hardware-configuration.nix
+    ./configuration.nix
+    ./disk-config.nix
+
+    ./networking.nix
+    ./wireguard.nix
+    #./backups.nix
+  ];
+}
--- a/hosts/blue-shell/disk-config.nix
+++ b/hosts/blue-shell/disk-config.nix
@ -0,0 +1,101 @@
+{
+  disko.devices = {
+    disk = {
+      main = {
+        type = "disk";
+        device = "/dev/vdb";
+        content = {
+          type = "gpt";
+          partitions = {
+            bios = {
+              size = "1M";
+              type = "EF02"; # for grub MBR
+            };
+            boot = {
+              size = "1G";
+              type = "8300";
+              content = {
+                type = "filesystem";
+                format = "ext4";
+                mountpoint = "/boot";
+                mountOptions = [ "defaults" ];
+              };
+            };
+            luks = {
+              size = "100%";
+              content = {
+                type = "luks";
+                name = "cryptroot";
+                extraOpenArgs = [ ];
+                # if you want to use the key for interactive login be sure there is no trailing newline
+                # for example use `echo -n "password" > /tmp/secret.key`
+                passwordFile = "/tmp/luks-password";
+                content = {
+                  type = "lvm_pv";
+                  vg = "vg0";
+                };
+              };
+            };
+          };
+        };
+      };
+      data = {
+        type = "disk";
+        device = "/dev/vdc";
+        content = {
+          type = "gpt";
+          partitions = {
+            luks = {
+              size = "100%";
+              content = {
+                type = "luks";
+                name = "cryptdata";
+                extraOpenArgs = [ ];
+                # if you want to use the key for interactive login be sure there is no trailing newline
+                # for example use `echo -n "password" > /tmp/secret.key`
+                passwordFile = "/tmp/luks-password";
+                content = {
+                  type = "filesystem";
+                  format = "xfs";
+                  mountpoint = "/var/lib/garage/data";
+                  mountOptions = [ "defaults" ];
+                };
+              };
+            };
+          };
+        };
+      };
+    };
+    lvm_vg = {
+      vg0 = {
+        type = "lvm_vg";
+        lvs = {
+          root = {
+            size = "100G";
+            content = {
+              type = "filesystem";
+              format = "ext4";
+              mountpoint = "/";
+              mountOptions = [ "defaults" ];
+            };
+          };
+          swap = {
+            size = "16G";
+            content = {
+              type = "swap";
+            };
+          };
+          metadata = {
+            size = "50G";
+            content = {
+              type = "filesystem";
+              format = "btrfs";
+              mountpoint = "/var/lib/garage/meta";
+              mountOptions = [ "defaults" ];
+            };
+          };
+        };
+      };
+    };
+  };
+}
--- a/hosts/blue-shell/hardware-configuration.nix
+++ b/hosts/blue-shell/hardware-configuration.nix
@ -0,0 +1,27 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{
+  config,
+  lib,
+  pkgs,
+  modulesPath,
+  ...
+}:
+
+{
+  imports = [ (modulesPath + "/profiles/qemu-guest.nix") ];
+
+  boot.initrd.availableKernelModules = [
+    "ata_piix"
+    "uhci_hcd"
+    "virtio_pci"
+    "sr_mod"
+    "virtio_blk"
+  ];
+  boot.initrd.kernelModules = [ ];
+  boot.kernelModules = [ ];
+  boot.extraModulePackages = [ ];
+
+  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
+}
--- a/hosts/blue-shell/networking.nix
+++ b/hosts/blue-shell/networking.nix
@ -0,0 +1,26 @@
+{
+  config,
+  pkgs,
+  flake,
+  ...
+}:
+{
+  services.garage.settings.rpc_public_addr = "[2a03:4000:43:24e::1]:3901";
+
+  networking.hostName = "blue-shell";
+  networking.hostId = "00000005";
+
+  networking.useDHCP = false;
+  systemd.network.enable = true;
+  systemd.network.networks."10-wan" = {
+    matchConfig.Name = "ens3";
+    address = [
+      "194.13.83.205/22"
+      "2a03:4000:43:24e::1/64"
+    ];
+    gateway = [
+      "194.13.80.1"
+      "fe80::1"
+    ];
+  };
+}
--- a/hosts/blue-shell/wireguard.nix
+++ b/hosts/blue-shell/wireguard.nix
@ -0,0 +1,49 @@
+{
+  config,
+  pkgs,
+  flake,
+  ...
+}:
+let
+  wireguardIPv4 = "10.7.6.7";
+  wireguardIPv6 = "fd00:fae:fae:fae:fae:7::";
+in
+{
+  networking.firewall.allowedUDPPorts = [ 51820 ];
+
+  age.secrets.wg-private-key.file = "${flake.self}/secrets/blue-shell-wg-private-key.age";
+
+  networking.wireguard.interfaces = {
+    wg-ssh = {
+      listenPort = 51820;
+      mtu = 1300;
+      ips = [
+        "${wireguardIPv4}/32"
+        "${wireguardIPv6}/96"
+      ];
+      privateKeyFile = config.age.secrets.wg-private-key.path;
+      peers = flake.self.logins.admins.wireguardDevices ++ [
+        {
+          # flora-6.pub.solar
+          endpoint = "80.71.153.210:51820";
+          publicKey = "jtSR5G2P/nm9s8WrVc26Xc/SQLupRxyXE+5eIeqlsTU=";
+          allowedIPs = [
+            "10.7.6.2/32"
+            "fd00:fae:fae:fae:fae:2::/96"
+          ];
+        }
+      ];
+    };
+  };
+
+  services.openssh.listenAddresses = [
+    {
+      addr = wireguardIPv4;
+      port = 22;
+    }
+    {
+      addr = "[${wireguardIPv6}]";
+      port = 22;
+    }
+  ];
+}
--- a/hosts/default.nix
+++ b/hosts/default.nix
@ -11,6 +11,7 @@
          self.nixosModules.unlock-zfs-on-boot
          self.nixosModules.core
          self.nixosModules.docker
+          self.nixosModules.backups

          self.nixosModules.nginx
          self.nixosModules.collabora
@ -49,6 +50,7 @@
          ./flora-6
          self.nixosModules.overlays
          self.nixosModules.core
+          self.nixosModules.backups

          self.nixosModules.keycloak
          self.nixosModules.caddy
@ -68,6 +70,7 @@
          self.nixosModules.overlays
          self.nixosModules.unlock-zfs-on-boot
          self.nixosModules.core
+          self.nixosModules.backups
          self.nixosModules.mail
          self.nixosModules.prometheus-exporters
          self.nixosModules.promtail
@ -83,10 +86,61 @@
          ./tankstelle
          self.nixosModules.overlays
          self.nixosModules.core
+          self.nixosModules.backups
          self.nixosModules.prometheus-exporters
          self.nixosModules.promtail
        ];
      };
+
+      trinkgenossin = self.nixos-flake.lib.mkLinuxSystem {
+        imports = [
+          self.inputs.agenix.nixosModules.default
+          self.nixosModules.home-manager
+          ./trinkgenossin
+          self.nixosModules.overlays
+          self.nixosModules.unlock-luks-on-boot
+          self.nixosModules.core
+          self.nixosModules.prometheus-exporters
+          self.nixosModules.promtail
+
+          self.nixosModules.garage
+          self.nixosModules.nginx
+        ];
+      };
+
+      delite = self.nixos-flake.lib.mkLinuxSystem {
+        imports = [
+          self.inputs.agenix.nixosModules.default
+          self.inputs.disko.nixosModules.disko
+          self.nixosModules.home-manager
+          ./delite
+          self.nixosModules.overlays
+          self.nixosModules.unlock-luks-on-boot
+          self.nixosModules.core
+          #self.nixosModules.prometheus-exporters
+          #self.nixosModules.promtail
+
+          self.nixosModules.garage
+          self.nixosModules.nginx
+        ];
+      };
+
+      blue-shell = self.nixos-flake.lib.mkLinuxSystem {
+        imports = [
+          self.inputs.agenix.nixosModules.default
+          self.inputs.disko.nixosModules.disko
+          self.nixosModules.home-manager
+          ./blue-shell
+          self.nixosModules.overlays
+          self.nixosModules.unlock-luks-on-boot
+          self.nixosModules.core
+          #self.nixosModules.prometheus-exporters
+          #self.nixosModules.promtail
+
+          self.nixosModules.garage
+          self.nixosModules.nginx
+        ];
+      };
    };
  };
 }
--- a/hosts/delite/configuration.nix
+++ b/hosts/delite/configuration.nix
@ -0,0 +1,33 @@
+{
+  flake,
+  config,
+  pkgs,
+  ...
+}:
+{
+  boot.loader.grub.enable = true;
+
+  boot.kernelParams = [
+    "boot.shell_on_fail=1"
+    "ip=dhcp"
+  ];
+
+  # This option defines the first version of NixOS you have installed on this particular machine,
+  # and is used to maintain compatibility with application data (e.g. databases) created on older NixOS versions.
+  #
+  # Most users should NEVER change this value after the initial install, for any reason,
+  # even if you've upgraded your system to a new NixOS release.
+  #
+  # This value does NOT affect the Nixpkgs version your packages and OS are pulled from,
+  # so changing it will NOT upgrade your system - see https://nixos.org/manual/nixos/stable/#sec-upgrading for how
+  # to actually do that.
+  #
+  # This value being lower than the current NixOS release does NOT mean your system is
+  # out of date, out of support, or vulnerable.
+  #
+  # Do NOT change this value unless you have manually inspected all the changes it would make to your configuration,
+  # and migrated your data accordingly.
+  #
+  # For more information, see `man configuration.nix` or https://nixos.org/manual/nixos/stable/options#opt-system.stateVersion .
+  system.stateVersion = "24.05"; # Did you read the comment?
+}
--- a/hosts/delite/default.nix
+++ b/hosts/delite/default.nix
@ -0,0 +1,13 @@
+{ flake, ... }:
+
+{
+  imports = [
+    ./hardware-configuration.nix
+    ./configuration.nix
+    ./disk-config.nix
+
+    ./networking.nix
+    ./wireguard.nix
+    #./backups.nix
+  ];
+}
--- a/hosts/delite/disk-config.nix
+++ b/hosts/delite/disk-config.nix
@ -0,0 +1,84 @@
+{
+  disko.devices = {
+    disk = {
+      main = {
+        type = "disk";
+        device = "/dev/vda";
+        content = {
+          type = "gpt";
+          partitions = {
+            bios = {
+              size = "1M";
+              type = "EF02"; # for grub MBR
+            };
+            boot = {
+              size = "1G";
+              type = "8300";
+              content = {
+                type = "filesystem";
+                format = "ext4";
+                mountpoint = "/boot";
+                mountOptions = [ "defaults" ];
+              };
+            };
+            luks = {
+              size = "100%";
+              content = {
+                type = "luks";
+                name = "cryptroot";
+                extraOpenArgs = [ ];
+                # if you want to use the key for interactive login be sure there is no trailing newline
+                # for example use `echo -n "password" > /tmp/secret.key`
+                passwordFile = "/tmp/luks-password";
+                content = {
+                  type = "lvm_pv";
+                  vg = "vg0";
+                };
+              };
+            };
+          };
+        };
+      };
+    };
+    lvm_vg = {
+      vg0 = {
+        type = "lvm_vg";
+        lvs = {
+          root = {
+            size = "40G";
+            content = {
+              type = "filesystem";
+              format = "ext4";
+              mountpoint = "/";
+              mountOptions = [ "defaults" ];
+            };
+          };
+          swap = {
+            size = "8G";
+            content = {
+              type = "swap";
+            };
+          };
+          data = {
+            size = "800G";
+            content = {
+              type = "filesystem";
+              format = "xfs";
+              mountpoint = "/var/lib/garage/data";
+              mountOptions = [ "defaults" ];
+            };
+          };
+          metadata = {
+            size = "50G";
+            content = {
+              type = "filesystem";
+              format = "btrfs";
+              mountpoint = "/var/lib/garage/meta";
+              mountOptions = [ "defaults" ];
+            };
+          };
+        };
+      };
+    };
+  };
+}
--- a/hosts/delite/hardware-configuration.nix
+++ b/hosts/delite/hardware-configuration.nix
@ -0,0 +1,26 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{
+  config,
+  lib,
+  pkgs,
+  modulesPath,
+  ...
+}:
+
+{
+  imports = [ (modulesPath + "/profiles/qemu-guest.nix") ];
+
+  boot.initrd.availableKernelModules = [
+    "ata_piix"
+    "uhci_hcd"
+    "virtio_pci"
+    "virtio_blk"
+  ];
+  boot.initrd.kernelModules = [ "dm-snapshot" ];
+  boot.kernelModules = [ ];
+  boot.extraModulePackages = [ ];
+
+  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
+}
--- a/hosts/delite/networking.nix
+++ b/hosts/delite/networking.nix
@ -0,0 +1,26 @@
+{
+  config,
+  pkgs,
+  flake,
+  ...
+}:
+{
+  services.garage.settings.rpc_public_addr = "[2a04:52c0:124:9d8c::2]:3901";
+
+  networking.hostName = "delite";
+  networking.hostId = "00000004";
+
+  networking.useDHCP = false;
+  systemd.network.enable = true;
+  systemd.network.networks."10-wan" = {
+    matchConfig.Name = "ens3";
+    address = [
+      "5.255.119.132/24"
+      "2a04:52c0:124:9d8c::2/48"
+    ];
+    gateway = [
+      "5.255.119.1"
+      "2a04:52c0:124::1"
+    ];
+  };
+}
--- a/hosts/delite/wireguard.nix
+++ b/hosts/delite/wireguard.nix
@ -0,0 +1,49 @@
+{
+  config,
+  pkgs,
+  flake,
+  ...
+}:
+let
+  wireguardIPv4 = "10.7.6.6";
+  wireguardIPv6 = "fd00:fae:fae:fae:fae:6::";
+in
+{
+  networking.firewall.allowedUDPPorts = [ 51820 ];
+
+  age.secrets.wg-private-key.file = "${flake.self}/secrets/delite-wg-private-key.age";
+
+  networking.wireguard.interfaces = {
+    wg-ssh = {
+      listenPort = 51820;
+      mtu = 1300;
+      ips = [
+        "${wireguardIPv4}/32"
+        "${wireguardIPv6}/96"
+      ];
+      privateKeyFile = config.age.secrets.wg-private-key.path;
+      peers = flake.self.logins.admins.wireguardDevices ++ [
+        {
+          # flora-6.pub.solar
+          endpoint = "80.71.153.210:51820";
+          publicKey = "jtSR5G2P/nm9s8WrVc26Xc/SQLupRxyXE+5eIeqlsTU=";
+          allowedIPs = [
+            "10.7.6.2/32"
+            "fd00:fae:fae:fae:fae:2::/96"
+          ];
+        }
+      ];
+    };
+  };
+
+  services.openssh.listenAddresses = [
+    {
+      addr = wireguardIPv4;
+      port = 22;
+    }
+    {
+      addr = "[${wireguardIPv6}]";
+      port = 22;
+    }
+  ];
+}
--- a/hosts/flora-6/wireguard.nix
+++ b/hosts/flora-6/wireguard.nix
@ -47,6 +47,33 @@
            "fd00:fae:fae:fae:fae:4::/96"
          ];
        }
+        {
+          # trinkgenossin.pub.solar
+          endpoint = "85.215.152.22:51820";
+          publicKey = "QWgHovHxtqiQhnHLouSWiT6GIoQDmuvnThYL5c/rvU4=";
+          allowedIPs = [
+            "10.7.6.5/32"
+            "fd00:fae:fae:fae:fae:5::/96"
+          ];
+        }
+        {
+          # delite.pub.solar
+          endpoint = "5.255.119.132:51820";
+          publicKey = "ZT2qGWgMPwHRUOZmTQHWCRX4m14YwOsiszjsA5bpc2k=";
+          allowedIPs = [
+            "10.7.6.6/32"
+            "fd00:fae:fae:fae:fae:6::/96"
+          ];
+        }
+        {
+          # blue-shell.pub.solar
+          endpoint = "194.13.83.205:51820";
+          publicKey = "bcrIpWrKc1M+Hq4ds3aN1lTaKE26f2rvXhd+93QrzR8=";
+          allowedIPs = [
+            "10.7.6.7/32"
+            "fd00:fae:fae:fae:fae:7::/96"
+          ];
+        }
      ];
    };
  };
--- a/hosts/nachtigall/backups.nix
+++ b/hosts/nachtigall/backups.nix
@ -1,4 +1,4 @@
-{ flake, ... }:
+{ config, flake, ... }:
 {
  age.secrets."restic-repo-droppie" = {
    file = "${flake.self}/secrets/restic-repo-droppie.age";
@ -10,4 +10,25 @@
    mode = "400";
    owner = "root";
  };
+  age.secrets.restic-repo-garage-nachtigall = {
+    file = "${flake.self}/secrets/restic-repo-garage-nachtigall.age";
+    mode = "400";
+    owner = "root";
+  };
+  age.secrets.restic-repo-garage-nachtigall-env = {
+    file = "${flake.self}/secrets/restic-repo-garage-nachtigall-env.age";
+    mode = "400";
+    owner = "root";
+  };
+
+  pub-solar-os.backups.repos.storagebox = {
+    passwordFile = config.age.secrets."restic-repo-storagebox".path;
+    repository = "sftp:u377325@u377325.your-storagebox.de:/backups";
+  };
+
+  pub-solar-os.backups.repos.garage = {
+    passwordFile = config.age.secrets."restic-repo-garage-nachtigall".path;
+    environmentFile = config.age.secrets."restic-repo-garage-nachtigall-env".path;
+    repository = "s3:https://buckets.pub.solar/nachtigall-backups";
+  };
 }
--- a/hosts/nachtigall/configuration.nix
+++ b/hosts/nachtigall/configuration.nix
@ -48,9 +48,21 @@
    owner = "root";
  };

-  pub-solar-os.auth.enable = true;
+  age.secrets.keycloak-database-password = {
+    file = "${flake.self}/secrets/keycloak-database-password.age";
+    mode = "600";
+    #owner = "keycloak";
+  };

-  nixpkgs.config.permittedInsecurePackages = [ "keycloak-23.0.6" ];
+  pub-solar-os.auth = {
+    enable = true;
+    database-password-file = config.age.secrets.keycloak-database-password.path;
+  };
+
+  systemd.services.postgresql = {
+    after = [ "var-lib-postgresql.mount" ];
+    requisite = [ "var-lib-postgresql.mount" ];
+  };

  # This value determines the NixOS release with which your system is to be
  # compatible, in order to avoid breaking some software such as database
--- a/hosts/trinkgenossin/configuration.nix
+++ b/hosts/trinkgenossin/configuration.nix
@ -0,0 +1,35 @@
+{
+  flake,
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+{
+  boot.loader.grub.enable = true;
+  boot.loader.grub.devices = [ "/dev/vda" ];
+
+  boot.kernelParams = [
+    "boot.shell_on_fail=1"
+    "ip=dhcp"
+  ];
+
+  # This option defines the first version of NixOS you have installed on this particular machine,
+  # and is used to maintain compatibility with application data (e.g. databases) created on older NixOS versions.
+  #
+  # Most users should NEVER change this value after the initial install, for any reason,
+  # even if you've upgraded your system to a new NixOS release.
+  #
+  # This value does NOT affect the Nixpkgs version your packages and OS are pulled from,
+  # so changing it will NOT upgrade your system - see https://nixos.org/manual/nixos/stable/#sec-upgrading for how
+  # to actually do that.
+  #
+  # This value being lower than the current NixOS release does NOT mean your system is
+  # out of date, out of support, or vulnerable.
+  #
+  # Do NOT change this value unless you have manually inspected all the changes it would make to your configuration,
+  # and migrated your data accordingly.
+  #
+  # For more information, see `man configuration.nix` or https://nixos.org/manual/nixos/stable/options#opt-system.stateVersion .
+  system.stateVersion = "24.05"; # Did you read the comment?
+}
--- a/hosts/trinkgenossin/default.nix
+++ b/hosts/trinkgenossin/default.nix
@ -0,0 +1,12 @@
+{ flake, ... }:
+
+{
+  imports = [
+    ./hardware-configuration.nix
+    ./configuration.nix
+
+    ./networking.nix
+    ./wireguard.nix
+    #./backups.nix
+  ];
+}
--- a/hosts/trinkgenossin/hardware-configuration.nix
+++ b/hosts/trinkgenossin/hardware-configuration.nix
@ -0,0 +1,54 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{
+  config,
+  lib,
+  pkgs,
+  modulesPath,
+  ...
+}:
+
+{
+  imports = [ ];
+
+  boot.initrd.availableKernelModules = [
+    "ata_piix"
+    "uhci_hcd"
+    "virtio_pci"
+    "sr_mod"
+    "virtio_blk"
+  ];
+  boot.initrd.kernelModules = [ "dm-snapshot" ];
+  boot.kernelModules = [ "kvm-amd" ];
+  boot.extraModulePackages = [ ];
+
+  boot.initrd.luks.devices."cryptroot" = {
+    device = "/dev/disk/by-uuid/52a1fd17-63d7-4d0a-b7ff-74aceaf6085a";
+  };
+
+  fileSystems."/" = {
+    device = "/dev/disk/by-label/nixos";
+    fsType = "ext4";
+  };
+
+  fileSystems."/boot" = {
+    device = "/dev/disk/by-label/boot";
+    fsType = "ext4";
+  };
+
+  fileSystems."/var/lib/garage/data" = {
+    device = "/dev/disk/by-label/data";
+    fsType = "xfs";
+  };
+
+  fileSystems."/var/lib/garage/meta" = {
+    device = "/dev/disk/by-label/metadata";
+    fsType = "btrfs";
+  };
+
+  swapDevices = [ { device = "/dev/disk/by-label/swap"; } ];
+
+  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
+  virtualisation.hypervGuest.enable = true;
+}
--- a/hosts/trinkgenossin/networking.nix
+++ b/hosts/trinkgenossin/networking.nix
@ -0,0 +1,15 @@
+{
+  config,
+  pkgs,
+  flake,
+  ...
+}:
+{
+  services.garage.settings.rpc_public_addr = "[2a01:239:35d:f500::1]:3901";
+
+  networking.hostName = "trinkgenossin";
+  networking.hostId = "00000003";
+
+  networking.enableIPv6 = true;
+  networking.useDHCP = true;
+}
--- a/hosts/trinkgenossin/wireguard.nix
+++ b/hosts/trinkgenossin/wireguard.nix
@ -0,0 +1,49 @@
+{
+  config,
+  pkgs,
+  flake,
+  ...
+}:
+let
+  wireguardIPv4 = "10.7.6.5";
+  wireguardIPv6 = "fd00:fae:fae:fae:fae:5::";
+in
+{
+  networking.firewall.allowedUDPPorts = [ 51820 ];
+
+  age.secrets.wg-private-key.file = "${flake.self}/secrets/trinkgenossin-wg-private-key.age";
+
+  networking.wireguard.interfaces = {
+    wg-ssh = {
+      listenPort = 51820;
+      mtu = 1300;
+      ips = [
+        "${wireguardIPv4}/32"
+        "${wireguardIPv6}/96"
+      ];
+      privateKeyFile = config.age.secrets.wg-private-key.path;
+      peers = flake.self.logins.admins.wireguardDevices ++ [
+        {
+          # flora-6.pub.solar
+          endpoint = "80.71.153.210:51820";
+          publicKey = "jtSR5G2P/nm9s8WrVc26Xc/SQLupRxyXE+5eIeqlsTU=";
+          allowedIPs = [
+            "10.7.6.2/32"
+            "fd00:fae:fae:fae:fae:2::/96"
+          ];
+        }
+      ];
+    };
+  };
+
+  services.openssh.listenAddresses = [
+    {
+      addr = wireguardIPv4;
+      port = 22;
+    }
+    {
+      addr = "[${wireguardIPv6}]";
+      port = 22;
+    }
+  ];
+}
--- a/modules/backups/default.nix
+++ b/modules/backups/default.nix
@ -0,0 +1,284 @@
+{
+  flake,
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  utils = import "${flake.inputs.nixpkgs}/nixos/lib/utils.nix" {
+    inherit lib;
+    inherit config;
+    inherit pkgs;
+  };
+  # Type for a valid systemd unit option. Needed for correctly passing "timerConfig" to "systemd.timers"
+  inherit (utils.systemdUtils.unitOptions) unitOption;
+  inherit (lib)
+    literalExpression
+    mkOption
+    mkPackageOption
+    types
+    ;
+in
+{
+  options.pub-solar-os.backups = {
+    repos = mkOption {
+      description = ''
+        Configuration of Restic repositories.
+      '';
+      type = types.attrsOf (
+        types.submodule (
+          { name, ... }:
+          {
+            options = {
+              passwordFile = mkOption {
+                type = types.str;
+                description = ''
+                  Read the repository password from a file.
+                '';
+                example = "/etc/nixos/restic-password";
+              };
+
+              environmentFile = mkOption {
+                type = with types; nullOr str;
+                default = null;
+                description = ''
+                  Read repository secrets as environment variables from a file.
+                '';
+                example = "/etc/nixos/restic-env";
+              };
+
+              repository = mkOption {
+                type = with types; nullOr str;
+                default = null;
+                description = ''
+                  repository to backup to.
+                '';
+                example = "sftp:backup@192.168.1.100:/backups/${name}";
+              };
+            };
+          }
+        )
+      );
+
+      default = { };
+      example = {
+        remotebackup = {
+          repository = "sftp:backup@host:/backups/home";
+          passwordFile = "/etc/nixos/secrets/restic-password";
+          environmentFile = "/etc/nixos/secrets/restic-env";
+        };
+      };
+    };
+
+    restic = mkOption {
+      description = ''
+        Periodic backups to create with Restic.
+      '';
+      type = types.attrsOf (
+        types.submodule (
+          { name, ... }:
+          {
+            options = {
+              paths = mkOption {
+                # This is nullable for legacy reasons only. We should consider making it a pure listOf
+                # after some time has passed since this comment was added.
+                type = types.nullOr (types.listOf types.str);
+                default = [ ];
+                description = ''
+                  Which paths to backup, in addition to ones specified via
+                  `dynamicFilesFrom`.  If null or an empty array and
+                  `dynamicFilesFrom` is also null, no backup command will be run.
+                   This can be used to create a prune-only job.
+                '';
+                example = [
+                  "/var/lib/postgresql"
+                  "/home/user/backup"
+                ];
+              };
+
+              exclude = mkOption {
+                type = types.listOf types.str;
+                default = [ ];
+                description = ''
+                  Patterns to exclude when backing up. See
+                  https://restic.readthedocs.io/en/latest/040_backup.html#excluding-files for
+                  details on syntax.
+                '';
+                example = [
+                  "/var/cache"
+                  "/home/*/.cache"
+                  ".git"
+                ];
+              };
+
+              timerConfig = mkOption {
+                type = types.nullOr (types.attrsOf unitOption);
+                default = {
+                  OnCalendar = "daily";
+                  Persistent = true;
+                };
+                description = ''
+                  When to run the backup. See {manpage}`systemd.timer(5)` for
+                  details. If null no timer is created and the backup will only
+                  run when explicitly started.
+                '';
+                example = {
+                  OnCalendar = "00:05";
+                  RandomizedDelaySec = "5h";
+                  Persistent = true;
+                };
+              };
+
+              user = mkOption {
+                type = types.str;
+                default = "root";
+                description = ''
+                  As which user the backup should run.
+                '';
+                example = "postgresql";
+              };
+
+              extraBackupArgs = mkOption {
+                type = types.listOf types.str;
+                default = [ ];
+                description = ''
+                  Extra arguments passed to restic backup.
+                '';
+                example = [ "--exclude-file=/etc/nixos/restic-ignore" ];
+              };
+
+              extraOptions = mkOption {
+                type = types.listOf types.str;
+                default = [ ];
+                description = ''
+                  Extra extended options to be passed to the restic --option flag.
+                '';
+                example = [ "sftp.command='ssh backup@192.168.1.100 -i /home/user/.ssh/id_rsa -s sftp'" ];
+              };
+
+              initialize = mkOption {
+                type = types.bool;
+                default = false;
+                description = ''
+                  Create the repository if it doesn't exist.
+                '';
+              };
+
+              pruneOpts = mkOption {
+                type = types.listOf types.str;
+                default = [ ];
+                description = ''
+                  A list of options (--keep-\* et al.) for 'restic forget
+                  --prune', to automatically prune old snapshots.  The
+                  'forget' command is run *after* the 'backup' command, so
+                  keep that in mind when constructing the --keep-\* options.
+                '';
+                example = [
+                  "--keep-daily 7"
+                  "--keep-weekly 5"
+                  "--keep-monthly 12"
+                  "--keep-yearly 75"
+                ];
+              };
+
+              runCheck = mkOption {
+                type = types.bool;
+                default = (builtins.length config.pub-solar-os.backups.restic.${name}.checkOpts > 0);
+                defaultText = literalExpression ''builtins.length config.services.backups.${name}.checkOpts > 0'';
+                description = "Whether to run the `check` command with the provided `checkOpts` options.";
+                example = true;
+              };
+
+              checkOpts = mkOption {
+                type = types.listOf types.str;
+                default = [ ];
+                description = ''
+                  A list of options for 'restic check'.
+                '';
+                example = [ "--with-cache" ];
+              };
+
+              dynamicFilesFrom = mkOption {
+                type = with types; nullOr str;
+                default = null;
+                description = ''
+                  A script that produces a list of files to back up.  The
+                  results of this command are given to the '--files-from'
+                  option. The result is merged with paths specified via `paths`.
+                '';
+                example = "find /home/matt/git -type d -name .git";
+              };
+
+              backupPrepareCommand = mkOption {
+                type = with types; nullOr str;
+                default = null;
+                description = ''
+                  A script that must run before starting the backup process.
+                '';
+              };
+
+              backupCleanupCommand = mkOption {
+                type = with types; nullOr str;
+                default = null;
+                description = ''
+                  A script that must run after finishing the backup process.
+                '';
+              };
+
+              package = mkPackageOption pkgs "restic" { };
+
+              createWrapper = lib.mkOption {
+                type = lib.types.bool;
+                default = true;
+                description = ''
+                  Whether to generate and add a script to the system path, that has the same environment variables set
+                  as the systemd service. This can be used to e.g. mount snapshots or perform other opterations, without
+                  having to manually specify most options.
+                '';
+              };
+            };
+          }
+        )
+      );
+      default = { };
+      example = {
+        localbackup = {
+          paths = [ "/home" ];
+          exclude = [ "/home/*/.cache" ];
+          initialize = true;
+        };
+        remotebackup = {
+          paths = [ "/home" ];
+          extraOptions = [
+            "sftp.command='ssh backup@host -i /etc/nixos/secrets/backup-private-key -s sftp'"
+          ];
+          timerConfig = {
+            OnCalendar = "00:05";
+            RandomizedDelaySec = "5h";
+          };
+        };
+      };
+    };
+  };
+
+  config = {
+    services.restic.backups =
+      let
+        repos = config.pub-solar-os.backups.repos;
+        restic = config.pub-solar-os.backups.restic;
+
+        repoNames = builtins.attrNames repos;
+        backupNames = builtins.attrNames restic;
+
+        createBackups =
+          backupName:
+          map (repoName: {
+            name = "${backupName}-${repoName}";
+            value = repos."${repoName}" // restic."${backupName}";
+          }) repoNames;
+
+      in
+      builtins.listToAttrs (lib.lists.flatten (map createBackups backupNames));
+  };
+}
--- a/modules/core/nix.nix
+++ b/modules/core/nix.nix
@ -6,7 +6,7 @@
  ...
 }:
 {
-  nixpkgs.config.allowUnfreePredicate = pkg: builtins.elem (lib.getName pkg) [ ];
+  nixpkgs.config = lib.mkDefault { allowUnfreePredicate = pkg: builtins.elem (lib.getName pkg) [ ]; };

  nix = {
    # Use default version alias for nix package
--- a/modules/garage/default.nix
+++ b/modules/garage/default.nix
@ -0,0 +1,141 @@
+{
+  config,
+  lib,
+  pkgs,
+  flake,
+  ...
+}:
+{
+  age.secrets."garage-rpc-secret" = {
+    file = "${flake.self}/secrets/garage-rpc-secret.age";
+    mode = "400";
+  };
+
+  age.secrets."garage-admin-token" = {
+    file = "${flake.self}/secrets/garage-admin-token.age";
+    mode = "400";
+  };
+
+  age.secrets."acme-namecheap-env" = {
+    file = "${flake.self}/secrets/acme-namecheap-env.age";
+    mode = "400";
+  };
+
+  networking.firewall.allowedTCPPorts = [
+    3900
+    3901
+    3902
+  ];
+
+  networking.firewall.interfaces.wg-ssh.allowedTCPPorts = [ 3903 ];
+
+  security.acme = {
+    defaults = {
+      environmentFile = config.age.secrets.acme-namecheap-env.path;
+    };
+    certs = {
+      # Wildcard certificate gets created automatically
+      "buckets.${config.pub-solar-os.networking.domain}" = {
+        # disable http challenge
+        webroot = null;
+        # enable dns challenge
+        dnsProvider = "namecheap";
+        dnsPropagationCheck = false;
+      };
+      # Wildcard certificate gets created automatically
+      "web.${config.pub-solar-os.networking.domain}" = {
+        # disable http challenge
+        webroot = null;
+        # enable dns challenge
+        dnsProvider = "namecheap";
+        dnsPropagationCheck = false;
+      };
+    };
+  };
+
+  services.nginx = {
+    upstreams.s3_backend.servers = {
+      "[::1]:3900" = { };
+    };
+    upstreams.web_backend.servers = {
+      "[::1]:3902" = { };
+    };
+    virtualHosts."buckets.${config.pub-solar-os.networking.domain}" = {
+      serverAliases = [ "*.buckets.${config.pub-solar-os.networking.domain}" ];
+
+      enableACME = true;
+      forceSSL = true;
+
+      locations."/" = {
+        proxyPass = "http://s3_backend";
+        extraConfig = ''
+          proxy_max_temp_file_size 0;
+        '';
+      };
+    };
+    virtualHosts."web.${config.pub-solar-os.networking.domain}" = {
+      serverAliases = [ "*.web.${config.pub-solar-os.networking.domain}" ];
+
+      enableACME = true;
+      forceSSL = true;
+
+      locations."/" = {
+        proxyPass = "http://web_backend";
+      };
+    };
+  };
+
+  services.garage = {
+    enable = true;
+    package = pkgs.garage_1_0_0;
+    settings = {
+      data_dir = "/var/lib/garage/data";
+      metadata_dir = "/var/lib/garage/meta";
+      db_engine = "lmdb";
+      replication_factor = 3;
+      compression_level = 2;
+      rpc_bind_addr = "[::]:3901";
+      s3_api = {
+        s3_region = "eu-central";
+        api_bind_addr = "[::]:3900";
+        root_domain = ".s3.${config.pub-solar-os.networking.domain}";
+      };
+      s3_web = {
+        bind_addr = "[::]:3902";
+        root_domain = ".web.${config.pub-solar-os.networking.domain}";
+        index = "index.html";
+      };
+      admin = {
+        api_bind_addr = "[::]:3903";
+      };
+    };
+  };
+
+  users.users.garage = {
+    isSystemUser = true;
+    home = "/var/lib/garage";
+    group = "garage";
+  };
+
+  users.groups.garage = { };
+
+  # Adapted from https://git.clan.lol/clan/clan-core/src/commit/23a9e35c665ff531fe1193dcc47056432fbbeacf/clanModules/garage/default.nix
+  # Disabled DynamicUser https://github.com/NixOS/nixpkgs/blob/nixos-24.05/nixos/modules/services/web-servers/garage.nix
+  # for mounts + permissions to work
+  systemd.services.garage = {
+    serviceConfig = {
+      user = "garage";
+      group = "garage";
+      DynamicUser = false;
+      LoadCredential = [
+        "rpc_secret_path:${config.age.secrets.garage-rpc-secret.path}"
+        "admin_token_path:${config.age.secrets.garage-admin-token.path}"
+      ];
+      Environment = [
+        "GARAGE_ALLOW_WORLD_READABLE_SECRETS=true"
+        "GARAGE_RPC_SECRET_FILE=%d/rpc_secret_path"
+        "GARAGE_ADMIN_TOKEN_FILE=%d/admin_token_path"
+      ];
+    };
+  };
+}
--- a/modules/grafana/default.nix
+++ b/modules/grafana/default.nix
@ -33,6 +33,11 @@
      group = "grafana";
      user = "grafana";
    };
+    "grafana-dashboards/grafana-garage-dashboard-prometheus.json" = {
+      source = ./grafana-dashboards/grafana-garage-dashboard-prometheus.json;
+      group = "grafana";
+      user = "grafana";
+    };
  };

  services.caddy.virtualHosts."grafana.${config.pub-solar-os.networking.domain}" = {
--- a/modules/grafana/grafana-dashboards/grafana-garage-dashboard-prometheus.json
+++ b/modules/grafana/grafana-dashboards/grafana-garage-dashboard-prometheus.json
--- a/modules/keycloak/default.nix
+++ b/modules/keycloak/default.nix
@ -6,23 +6,22 @@
  ...
 }:
 {
-  options.pub-solar-os.auth = {
-    enable = lib.mkEnableOption "Enable keycloak to run on the node";
+  options.pub-solar-os.auth = with lib; {
+    enable = mkEnableOption "Enable keycloak to run on the node";

-    realm = lib.mkOption {
+    realm = mkOption {
      description = "Name of the realm";
-      type = lib.types.str;
+      type = types.str;
      default = config.pub-solar-os.networking.domain;
    };
+
+    database-password-file = mkOption {
+      description = "Database password file path";
+      type = types.str;
+    };
  };

  config = lib.mkIf config.pub-solar-os.auth.enable {
-    age.secrets.keycloak-database-password = {
-      file = "${flake.self}/secrets/keycloak-database-password.age";
-      mode = "600";
-      #owner = "keycloak";
-    };
-
    services.nginx.virtualHosts."auth.${config.pub-solar-os.networking.domain}" = {
      enableACME = true;
      forceSSL = true;
@ -46,7 +45,7 @@
    # keycloak
    services.keycloak = {
      enable = true;
-      database.passwordFile = config.age.secrets.keycloak-database-password.path;
+      database.passwordFile = config.pub-solar-os.auth.database-password-file;
      settings = {
        hostname = "auth.${config.pub-solar-os.networking.domain}";
        http-host = "127.0.0.1";
@ -59,14 +58,12 @@
      };
    };

-    services.restic.backups.keycloak-storagebox = {
+    pub-solar-os.backups.restic.keycloak = {
      paths = [ "/tmp/keycloak-backup.sql" ];
      timerConfig = {
        OnCalendar = "*-*-* 03:00:00 Etc/UTC";
      };
      initialize = true;
-      passwordFile = config.age.secrets."restic-repo-storagebox".path;
-      repository = "sftp:u377325@u377325.your-storagebox.de:/backups";
      backupPrepareCommand = ''
        ${pkgs.sudo}/bin/sudo -u postgres ${pkgs.postgresql}/bin/pg_dump -d keycloak > /tmp/keycloak-backup.sql
      '';
--- a/modules/mediawiki/default.nix
+++ b/modules/mediawiki/default.nix
@ -232,4 +232,27 @@ in
      };
    };
  };
+
+  pub-solar-os.backups.restic.mediawiki = {
+    paths = [
+      "/var/lib/mediawiki/images"
+      "/var/lib/mediawiki/uploads"
+      "/tmp/mediawiki-backup.sql"
+    ];
+    timerConfig = {
+      OnCalendar = "*-*-* 00:30:00 Etc/UTC";
+    };
+    initialize = true;
+    backupPrepareCommand = ''
+      ${pkgs.sudo}/bin/sudo -u postgres ${pkgs.postgresql}/bin/pg_dump -d mediawiki > /tmp/mediawiki-backup.sql
+    '';
+    backupCleanupCommand = ''
+      rm /tmp/mediawiki-backup.sql
+    '';
+    pruneOpts = [
+      "--keep-daily 7"
+      "--keep-weekly 4"
+      "--keep-monthly 3"
+    ];
+  };
 }
--- a/modules/nginx-website/default.nix
+++ b/modules/nginx-website/default.nix
@ -7,7 +7,7 @@
  services.nginx.virtualHosts = {
    "www.${config.pub-solar-os.networking.domain}" = {
      enableACME = true;
-      addSSL = true;
+      forceSSL = true;

      extraConfig = ''
        error_log /dev/null;
--- a/modules/obs-portal/default.nix
+++ b/modules/obs-portal/default.nix
@ -147,4 +147,26 @@ in
      };
    };
  };
+
+  pub-solar-os.backups.restic.obs-portal = {
+    paths = [
+      "/var/lib/obs-portal/data"
+      "/tmp/obs-portal-backup.sql"
+    ];
+    timerConfig = {
+      OnCalendar = "*-*-* 00:30:00 Etc/UTC";
+    };
+    initialize = true;
+    backupPrepareCommand = ''
+      ${pkgs.docker}/bin/docker exec -ti --user postgres obs-portal-db pg_dump obs > /tmp/obs-portal-backup.sql
+    '';
+    backupCleanupCommand = ''
+      rm /tmp/obs-portal-backup.sql
+    '';
+    pruneOpts = [
+      "--keep-daily 7"
+      "--keep-weekly 4"
+      "--keep-monthly 3"
+    ];
+  };
 }
--- a/modules/postgresql/default.nix
+++ b/modules/postgresql/default.nix
@ -25,9 +25,4 @@
      full_page_writes = false;
    };
  };
-
-  systemd.services.postgresql = {
-    after = [ "var-lib-postgresql.mount" ];
-    requisite = [ "var-lib-postgresql.mount" ];
-  };
 }
--- a/modules/prometheus/default.nix
+++ b/modules/prometheus/default.nix
@ -69,6 +69,14 @@
              instance = "tankstelle";
            };
          }
+          {
+            targets = [
+              "trinkgenossin.wg.${config.pub-solar-os.networking.domain}:${toString config.services.prometheus.exporters.node.port}"
+            ];
+            labels = {
+              instance = "trinkgenossin";
+            };
+          }
        ];
      }
      {
@ -83,6 +91,18 @@
          }
        ];
      }
+      {
+        job_name = "garage";
+        static_configs = [
+          {
+            targets = [
+              "trinkgenossin.wg.${config.pub-solar-os.networking.domain}:3903"
+              "delite.wg.${config.pub-solar-os.networking.domain}:3903"
+              "blue-shell.wg.${config.pub-solar-os.networking.domain}:3903"
+            ];
+          }
+        ];
+      }
    ];

    ruleFiles = [
--- a/modules/unlock-luks-on-boot/default.nix
+++ b/modules/unlock-luks-on-boot/default.nix
@ -0,0 +1,20 @@
+{ flake, config, ... }:
+{
+  boot.initrd.network = {
+    enable = true;
+    ssh = {
+      enable = true;
+      # To prevent ssh clients from freaking out because a different host key is used,
+      # a different port for ssh is useful (assuming the same host has also a regular sshd running)
+      port = 2222;
+
+      # Please create this manually the first time.
+      hostKeys = [ "/etc/secrets/initrd/ssh_host_ed25519_key" ];
+      authorizedKeys = config.pub-solar-os.authentication.sshPubKeys;
+    };
+    postCommands = ''
+      # Automatically ask for the password on SSH login
+      echo 'cryptsetup-askpass || echo "Unlock was successful; exiting SSH session" && exit 1' >> /root/.profile
+    '';
+  };
+}
--- a/secrets/acme-namecheap-env.age
+++ b/secrets/acme-namecheap-env.age
--- a/secrets/blue-shell-wg-private-key.age
+++ b/secrets/blue-shell-wg-private-key.age
@ -0,0 +1,43 @@
+age-encryption.org/v1
+-> ssh-ed25519 eP5MMw xAObv9OBtCMHWnbnO9b9w5fiG3tkJJTvjFNWmYmGfhQ
+sXmgq1drzY3rap8aD8/iMbMgjGkajfENkBQDdK/2TKQ
+-> ssh-ed25519 uYcDNw sxw3K1xYG+OZQy+4U2UfX//ZElPaCieANzFHanJxfxw
+VyhqjYppfHVb2jTceXLL/yYwEJE2uD9TY4PliHu7c0E
+-> ssh-rsa f5THog
+hRIMIg4P7SOOS3IGr2vF/TAdvgAXjJ8CbjKQt+Bd8MUjxf769rD3bln3lF3DlfIU
+RMkicdnwRdWTnqp+HyV0/UsD4ZzVb0YY+ntK6wujqgEwhpef9NOE2Hsiyvv228l5
+pu6eUTrosmb0ysnw8zRmr4RYdv1+MsD9gbnLOnnuHkA+i28jAE6o4gXIdnKfDcIk
+ptJISTFwyzz+q8UvdfO4YGRBL4zSoWM5VEQS0IPjrt4+qBlOyC2IlXz1/aaGZk41
+ODgPUO68USpzFsLqsmRJJQOMxxkdnYWy4DoeHKUyZI4YO0t94zVEHIS3x6w2l+/Q
+9r7TdKXxniLFYW9t5+28ez5XfLruapXroA8kp2hy3S7ybuOyB2MIfCXVvK0kiOfw
+2kZnv7LvL4BLUMtgPKoAxj8Pzpi8HzFGxQudqNwSkjb4bvFg8eej0oP/WhvFalsY
+MGSdlFJkKKeLWh8MzmD2WXHJ5yVcRFHydyWGMHlZJ4soi0I3gTSQaNSPUxWtFSd3
+Trk3Jz4Qrd7EA6y6wn2Jc4xCX+cWb8Q3nrXVZJL/FSxss+zstQit5O3BwWupJQLt
+ubyqLxQI/PKP/gIU8sGhwKDWGjZUlwvjPBG+EdWJAxoI7S7+4UxkGgsNvx67VKgi
+xxz3ANlHPbGLgAZTtEk5AqHWLWE6ZRn/7i3Qv1HWZJ4
+-> ssh-rsa kFDS0A
+OqNv7KThlRd0aD1gixsI2LVb+Zj2r7OVm4lUD/UJ0WVe/ihlcBAeEj8noqNA3zgH
+djO53WpAkGrbt8CejuQcPYLyw8Bdk++cBu9Po+X0dAp2cCPqakgIpEI6qG1uiEty
+LOOF13TivCB802UrJmX/8HLDV3yv7IWIr2XV0V8s5UvasYlgCt1eg4suhXIngnJG
+wd7WFnXwtNtR/UcnWtpE8c6p5kaAn4wSAtx6hFqnkN5ANjKXHy29NkFR1Lgu55OU
+qoJPpcZpvEESYMQfPvfuCUSZgcSvJE8B1MJZzxdKBHFiaWa7BkkWOL2KNIARFji5
+nBrGzGLV9IaHWsS0UfIWixvM3OPl4wgwpdLtVJsnLX/ggFZrZmj+iR5DdqurfW7E
+0a8Ie1NKK7FU+HnzEk6+8aiDb0QozEhrmaE5olc8dBOKK13e1idDTCsquaQcSOEh
+DBP8r0LhqU7YwnNF6UpQiYs3Zs2HyPegfz2GwB3yeaHqc7+MnXm/j4B5qUcX5naz
+llQoNfddAtoXGeK3G0yjugkSluValw6o2hQ+4iRx4n1f3dVcurXkBr9fjCKMmjC4
+qbOHhsBU4dHCvFIOXY27Xvq5qZ4/ceNb6fq/NXvkD87eePMLg9R9hmtTaCDlEN2d
+a350/FxlWOZXEox+mRZhE+mE5qSIg+LbnFZ+zjWq+yc
+-> piv-p256 vRzPNw AzwIBCtS5Tx/zuFHRYsYSOffxAE79O5foV+ndpw0hR1t
+HDmWtvUS9wSnlNjbkD0Rc0jQ4tNhqpcqpeztW1GXC3g
+-> piv-p256 zqq/iw Av5ZYxbCJrjUImhX7hoO8nxtWEtd7mPWhofwCxtW5GNI
+tB/mFmw4U2NbeDKdasi/Z99VggQYhnv6+n9+VJekq/k
+-> ssh-ed25519 YFSOsg nl0SHBFBylYgoy4qrZ851AQ6NLuDpXtIQ5WffqQPckU
+/yk/gT3enujLcjHkYuE0XGUDrYUEEzvyPvIlKhHtf5c
+-> ssh-ed25519 iHV63A 2Gq6dIvLDJQmwgQwxhqrPpubkToiseczLkobeCZiOSA
+IXddfsh84BrA0v6X/SjqoFbUfJfw3v+zD3Dk5RdsfAE
+-> ssh-ed25519 BVsyTA 9oRVFqCqPoQ35/u+Cg4dPkG4eXw7vSRaPwhel430TGE
+C54Ofc94lPFMGLljqY4Ag0AhM/MHWeZjZ6x1fmyMmqI
+-> ssh-ed25519 +3V2lQ 2g1xRrQZy30nCaDq6RtfXQfUchtD8oOnmGYX+A2venQ
+oop5rNpGKvTUOLGN2HGc7B63H/8XYrhO+XsCjsKfPgA
+--- cMgwwO4kfMX17njkjYczc4R6FVRwC+cpK37g2cFAapc
+È
`Ñ<>Xéø¶Ó“¡;êR…Vàâ]tbM"N4×Éa§êš/æI×pšBâ7Qòe‡MÑ
§ºÀnpÊ±£†ç2Å}Éz-R4E<34>‚(éÀN
--- a/secrets/delite-wg-private-key.age
+++ b/secrets/delite-wg-private-key.age
--- a/secrets/garage-admin-token.age
+++ b/secrets/garage-admin-token.age
@ -0,0 +1,47 @@
+age-encryption.org/v1
+-> ssh-ed25519 NID4eA MqbdKdPSNLU32j+ZFTT1bmSFk4ZQowSuYcuP/FRtKgs
+m0UaJclORGOrQDLIvdIv4Aq4U3XNOet9CzU8pX/pGsI
+-> ssh-ed25519 9RQHxg WyTt2Lqd2mRtwsx2PLajmk68IB1yP0DkgXR/xfIMERM
+4JaJ+DB97XxOuI2G/qnk+NX8Xv1ruY28NPRl7aUPWEc
+-> ssh-ed25519 eP5MMw HUf/aXRbaIEjQFdWpo8vWtNSqzYcQKaNKJ7y+ewMVz0
+6IcRiPufkMLjOV5mSaOBclPmFDVKII/8mqGN7jmfkYk
+-> ssh-ed25519 uYcDNw gHg0z1q+4AGZtuMzbV1dZjWojmYNE0JO1ZhMnGqAbR0
+sFE6mIV3ZgRMlsuHFltkC4M+aAhEDzVQQ3rkgiR5GSY
+-> ssh-rsa f5THog
+BFJZWIMurARpA6XJMlxnDYFUTGCX5gEb6IycqUQbIdu9+91mpDR3WxlJYLLZC3vD
+j7e5xFvr4bOXQg56N59k4AD5CnC7CoZld/PqGtP5JqUcoXPFHgq95GzfiuTLedkZ
+esPsPvXfdeyqfzcdMhxTz8pO0RRPUk9Xc2wnvHrDyeNO0vS5N+bMsNMvFfRaMZTi
+xV2Xl98VFl90+gzuPACN0zpfoEqajcJhPtMqoaOjOzD7sGaAz5UYLAIyOHdh3t9w
+DwVorUheXKLqq1kxZRZ5QCC7N+TCbi7x6B5xCezz9hF2vXkzunDh/d7maQG34AXJ
+x1sQ4R9dXn778j14RqBBNbsFPDg8WIS4Zs+ypdqev8w+aOhZkcDf5unKQJLdeu3N
+W+IUJTN5zFObMX0TXtj0yI7xUSkZKSLuZs43MhdVkNo1YewHalMpKetctnw2Lz+z
+ZcDCRLmD91U5BL7xp9KYQQ9EBjEn8dyw9hn8RmCaRKsvi71cIStv4OtuGxzIvTOJ
+phqDJdjz53E6L/0NDmhgN7bxolYZJsPC2o/ca0lG1rx3SZdfI18WaTsQrOO2NaeZ
+i4u9vkw/k7xT65mPxPsN7s67niE6lVGyTOLMwjvfO1sAHABbtto2GL0zRc454b9v
+gObMCi/ZbOUo6wndMMPnrQh9SsSGKB6EIOEzmg+8M+Y
+-> ssh-rsa kFDS0A
+h9TH7j+1hquLDwCIRDvQiN7UJaO6rJ/NtWdeWTuqTqTGSS+aVvm9gDZBZiLzWyqb
+GLd2guhepOCuSQkxocD7KAz1hp5pqf++XI8/yqepnHuU+CmInsOBmSZsG8QGktbk
+wjY3ZqZdcfCorgAxjIaw4aNW3zeoq7ATC3rC3aDeOsZC6jImdgEeCvd8BczWSqOa
+9acfRqd+QNDjqq3MIGIepVBlQ0nNniRtQoh4XtUey8jVb6kjow4TqBo9V33UhO7x
+r3WMgk5q1onECAhaAlYFxQ3DzLjUxAVdulnkIoSBqdE1b6Njyc70CI9oYwZcGmbD
+gC3qdQWmcoeOQ95YkdJAxaFIl3VNOTyn9rSXtn3a1PsVygaT+98qd1zwGPY+3ZDk
+B1chqWp3IJ4y2oPqt1mO4j0am2NAChpcWlEhgeldD0HoUixoXvp2Gu0igD5ENfyH
+BopesdhSoIxw8PbjKvnuwPFNSa9ysS6mpdmChTdBGpOCSpwMEcEE/m3pLo4FkzJf
+nhQaxav7VmvZ2jxnmEgP/NUgxmZPHKOzlI3vqWfTsIwz/chHDwekhb6dB0O5nrjO
+LzmKVrkVH0EEa3R1/BIfF674RSPXOmW89NNU7tCax3IOuPWC0QYv+2r/zjmAEq+Q
+LJMPf1fUjLlsCmkOlxDYK4E/EyiG5dcjxdyXk8J+l6s
+-> piv-p256 vRzPNw AzRQKtxg/bXdDoLXM4IBQaPXZ7Tg/6GSg0LeQQ3GNUnw
+xf8jnJPg8nqUt26lvtemwdvV8h0nT8k/H4iGO8fBa1k
+-> piv-p256 zqq/iw AtpFzxs9a5N87BOsagEIMnJFxyBv8PZwsF4orrWS7dlP
+8w3nDD2dIXKRKtnz/xnFW3it99ZmfY9X4ZiQhjXnaPM
+-> ssh-ed25519 YFSOsg E3q3wZk2o3+Gmv0xMuokFWqhckCEZxVomrj61M2U21w
+CUlB8ynJwO5JmHND3+x/NN/PyV0HJXfoxI3TgPjOD/4
+-> ssh-ed25519 iHV63A 4tjRDQ0kSZZYda0V3pEs6teT7n8r0WFQ1iHj90XIP1c
+OMGjdTkZo8LOojtNDhIw5OXaWpC21f8hIOcnv1suLA4
+-> ssh-ed25519 BVsyTA 5BzUJ7kmv/3ZwxRGOq6QNohlWn5nuHgCBTjbcGtA1jU
+vaN4i/JBvrDtdb9IX+F4uLG8v6wiaF0zjTyBXJ7B0Ts
+-> ssh-ed25519 +3V2lQ eS/DpYwsqIkwm1JENRu1kIHqe+qAcMssglt/DHIAuBk
+JYdv3kTgpF+3b4a9eycCzl3FWyLSkunas4IYqfw9z48
+--- iffZKXnN9jbIrfnm0NrUmqfNhsBtd0FCBEc9Zp0YSj8
+Ñ[4aÏ“ Ü¥¹ŒçG”Žž(˜É\<5C>)YQÍ
'~@L—KñTw¬;æ–jó~,‰¤¤¦îÎµó½‡Ô*áüq?o}ª' ? ¸×¿
--- a/secrets/garage-rpc-secret.age
+++ b/secrets/garage-rpc-secret.age
--- a/secrets/restic-repo-garage-nachtigall-env.age
+++ b/secrets/restic-repo-garage-nachtigall-env.age
--- a/secrets/restic-repo-garage-nachtigall.age
+++ b/secrets/restic-repo-garage-nachtigall.age
@ -0,0 +1,43 @@
+age-encryption.org/v1
+-> ssh-ed25519 iDKjwg yk6WgkJbk16Yqc/aWojwDePfNZN3XgaiacpZqufw1hc
+qx3x0zCUB6l7qPS9R9xGO41s6ESGsNd/ixglY4gYDtY
+-> ssh-ed25519 uYcDNw nDZXlykiDEZIA+Srg6imZFRLAzhykR3PDVrfWb693Sg
+MBJrrXbtLzBozetgfZaE52o1ixsrPK6Ojdp8sXhI3dc
+-> ssh-rsa f5THog
+WngnjCboeUafoR5NOg/+NFeW/ILxonIrcl7iUQLXoObcFWtmgzEebcogo3kqnkdk
+cj7NnqQ/sFiHdPCBr+VyiWcP4BJHJeWF/w+Ht9SYP5+WlHUkC2IPxlcMzJFyy2ro
+9PDq8FaPX5ZqarOfBW+U1pQsegxG9wkfoSzoF8NVL0jU5mwubPc+1s0ycbjMcGzP
+CSCpWY0OqqRbCXj8gVBNOOoELVhgbNCO0oRWh6Iafqjrx/rO4MNAdZmUF9DiTZML
+8QQjBFR7G8/+4ehGHn/9PF/0yVVsnPaJTEo3juS/By+NVcCihwP3b0SgQkjtp73+
+2QZL4XImwR0bBLDla86IjvZWH7GkLzR8iAw75FZ/FGIxZ/XgwRcUqtcI8m105KAD
+iX1v7ai0LUYko4RPSqdDItEdGjK31Dl4bCdaLakO+aZdtc+60c7hx8uEoTp7JY9W
+Qmyv4rt5bVUUAeH0IyR72vTDpTtXSkbp0vypk6Zq+yhYjbe0wmakAPCF4WbIV9gQ
+YIoSyicQ3NYxfrQGcQvmct2dNXXZcCr8BW704/J5w7ngw44FiZDUIC2y562sHD84
+9rFO0obVGERJa55+MnZB4UYMqU3eWs+mkxHgWotx5yEtO0wMpTP9IFk92scJeV8h
+bdr6fq1p6gtFlSNlA7P8A0go1SWs+qIoO/PEPyznoss
+-> ssh-rsa kFDS0A
+RM1VIV11yDWaG7KDnXbVt+Qo9kIC5kPO1JwpROuxbTH8tDGbu7bC2mJ6qsZo1R5x
+ewtmDVwv+Gogs2drjQeQgdZH3qN0cMt34micbW2lvBW4NmvAGKEQ+dkkUgIWIhy9
+dNuFBCHAGqfw7FDqvX1Mm3e50CsbIVqpurBXe882seMjm/nSVzGgjOOQVaQKmNSg
+04s0gsLzN1f98kQtKuNPlG8Fun9y1bKLzWpBT2iriQInkhcWSwYqg1M0yM7U01j3
+Kcxn4LJmgB9qNkloEKNpHCze5fgIGyOf+MsG4VN90CvwpNxy/EHrEyncXDxSxiCt
+A81PVHCZfC7fQTR+hF2s1xGcW9mmtRpPPSEUrGY84cW2k8m4E5A9J7MJZ7FOKe6C
+OY3LoU5KCV292ujHqYs1c7JSJRqVq4IMCupsNL6afzB/Fe+cV6GX7bXh1ZUl9HT7
+B6j1QH2xA9OGz+6VDrB+B6cdnxp02zRZpuS19uPTJqpIg4Sgc4vvw7YFQfz8AfUD
+/SafkBpVKznEHl1/gO7bOMa1nTkNBicUp6d2Z/zNtJ75NJPb3qYc4aVsaEuyMPts
+ieMR/iieeiDOMPhedtibfCWZ+0YrtyIrkkUsPK5yG76VbJgMjSRyEP+bXTedih/b
+bTkE78BoV5DmNxGEAva2BJpMKtn3Or8dCgSudoSlI14
+-> piv-p256 vRzPNw Al8mhhyZ/0YFf7OSKeGm3LZhowpCdcITOhtjmky7Ygnq
+CxFNUst8+6e6Cra/j7Pa2lZs3lHGLCOWRInb1VZTr2M
+-> piv-p256 zqq/iw A+r/W9OhuPjuGXRuhp1vW45k/QzCFO6VLfp1W6l7RJVU
+GxunFSBPJnzd5t5Ar42vr06tyJkJvZhljlGkHVZOFOQ
+-> ssh-ed25519 YFSOsg e7udSQwtwKETHB0Re59fb1DdiBBPLDbV/JHmUUI4GSU
+054wi1iKJm8lnWDjONCk+h2vea0setKqdCpXHuJaecI
+-> ssh-ed25519 iHV63A S2sle86zYVPjtCozODRjqtTs6a5GksTpJHkmO/WYzHU
+7h5kS8Hc3BiyhiWerEa9xPX6o+D/bxoJLK1fXFq1jWc
+-> ssh-ed25519 BVsyTA 7meQJdKDB4JLKsYmBPgDBUnWhil1fKnoijm+uzHDemY
+1bxklkYv5KYab9fXjMtz/w5QTUYMlZFTQG/khBftlWo
+-> ssh-ed25519 +3V2lQ n1bF6+o/16zx7dEt/Um1gL30mARiuPaE6z3N+qjFZx4
+7ZjmhkuJDYXLcMoUEA0wosWcWZ1T1oR45kVhFyQwN1E
+--- TbpcaLv/1jF23nynpaw1XjSKsO2t9hAxovPQHKhXAoQ
+P‰^¢f«E~¢÷ß6ÿO[Ö-cf]–øÄ9eÓ6Ñû‘4Eÿl2=sÿßW‡…O¿QFCzcõbÖTšÎÉÿ;[a
--- a/secrets/secrets.nix
+++ b/secrets/secrets.nix
@ -5,6 +5,9 @@ let
  flora-6-host = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGP1InpTBN4AlF/4V8HHumAMLJzeO8DpzjUv9Co/+J09 root@flora-6";
  metronom-host = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICLX6UvvrKALKL0xsNnytLPHryzZF5evUnxAgGokf14i root@metronom";
  tankstelle-host = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJdF6cJKPDiloWiDja1ZtqkXDdXOCHPs10HD+JMzgeU4 root@tankstelle";
+  trinkgenossin-host = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDZXRDpom/LtyoCxvRuoONARKxIT6wNUwEyUjzHRE7DG root@trinkgenossin";
+  delite-host = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAKo7zlfQhcJ5/okFTOoOstZtmEL1iNlHxQ4q2baEcWT root@delite";
+  blue-shell-host = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIP9g9X0a/MaVtbh44IeLxcq+McuYec0GYAdLsseBpk5f root@blue-shell";

  adminKeys = builtins.foldl' (
    keys: login: keys ++ (builtins.attrValues login.secretEncryptionKeys)
@ -17,6 +20,18 @@ let
  flora6Keys = [ flora-6-host ];

  metronomKeys = [ metronom-host ];
+
+  trinkgenossinKeys = [ trinkgenossin-host ];
+
+  deliteKeys = [ delite-host ];
+
+  blueshellKeys = [ blue-shell-host ];
+
+  garageKeys = [
+    trinkgenossin-host
+    delite-host
+    blue-shell-host
+  ];
 in
 {
  # ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBB5XaH02a6+TchnyQED2VwaltPgeFCbildbE2h6nF5e root@nachtigall
@ -26,6 +41,9 @@ in
  "tankstelle-wg-private-key.age".publicKeys = tankstelleKeys ++ adminKeys;
  "flora6-wg-private-key.age".publicKeys = flora6Keys ++ adminKeys;
  "metronom-wg-private-key.age".publicKeys = metronomKeys ++ adminKeys;
+  "trinkgenossin-wg-private-key.age".publicKeys = trinkgenossinKeys ++ adminKeys;
+  "delite-wg-private-key.age".publicKeys = deliteKeys ++ adminKeys;
+  "blue-shell-wg-private-key.age".publicKeys = blueshellKeys ++ adminKeys;

  "mastodon-secret-key-base.age".publicKeys = nachtigallKeys ++ adminKeys;
  "mastodon-otp-secret.age".publicKeys = nachtigallKeys ++ adminKeys;
@ -54,6 +72,8 @@ in

  "restic-repo-droppie.age".publicKeys = nachtigallKeys ++ adminKeys;
  "restic-repo-storagebox.age".publicKeys = nachtigallKeys ++ adminKeys;
+  "restic-repo-garage-nachtigall.age".publicKeys = nachtigallKeys ++ adminKeys;
+  "restic-repo-garage-nachtigall-env.age".publicKeys = nachtigallKeys ++ adminKeys;

  "drone-db-secrets.age".publicKeys = flora6Keys ++ adminKeys;
  "drone-secrets.age".publicKeys = flora6Keys ++ adminKeys;
@ -87,4 +107,10 @@ in
  "mail/crew.age".publicKeys = metronomKeys ++ adminKeys;
  "mail/erpnext.age".publicKeys = metronomKeys ++ adminKeys;
  "mail/hakkonaut.age".publicKeys = metronomKeys ++ adminKeys;
+
+  # garage
+  "garage-rpc-secret.age".publicKeys = garageKeys ++ adminKeys;
+  "garage-admin-token.age".publicKeys = garageKeys ++ adminKeys;
+
+  "acme-namecheap-env.age".publicKeys = garageKeys ++ adminKeys;
 }
--- a/secrets/trinkgenossin-wg-private-key.age
+++ b/secrets/trinkgenossin-wg-private-key.age
@ -0,0 +1,43 @@
+age-encryption.org/v1
+-> ssh-ed25519 NID4eA Q3E8hBMDQRxoJx6UGzECMZmkffdgyYlhGaloKFNmxw0
+37DKT5sgmAEritSoPuW+O5dvjCH23pOAdFyJG0TnY6w
+-> ssh-ed25519 uYcDNw zgfSabCBntiTnc0fdfDzpkq/AwUXSpyvsA14gkatP3w
+tLbZDE6UB/xvC153mSGcGSSWKH+Ph1Ek5D+JTrWjzdk
+-> ssh-rsa f5THog
+0THw5q/Aa/wCzfqO/9YFBOvSfISS/O2cMHBlQ7NXzF2hlj+hzGjJeL2USmL8iZ7X
+YToH0oy8lreVRYxfi/LUMyg14hQf11hTekT/sKc8m5eBm+8WiHbWJsQJKdRg6WVO
+B2Ju+3QIZXBk7ajCIcVjgoJQy1JydXm5YQkZnI69icdtAEnYSEoVEpaPHkT7Et65
+UUC/eegltWFSeJl4bzgceVXO3VtszoG/KkL7ToT2WX2Hbnb4R3U8cWLOTr4I6hzM
+90h6mNaorm3bd5fysOoU3G531/eAqcC7QZQQGyCOEBBHgx0w32ZKpjqu8q8c/f0M
+VMOgE0JfK4/iB5E3dwGqRZ2G8iXu/cx0CQY98YAFCDOYExMsQzFXzqKq8KecVd+g
+vaj598KJFxYskwrkGNXwIds2lPCte3HIa9XcGeQ4svaLYdQw5zsSIF32zOwAbxRn
+1ABzp/T7V3BYyykJbeJi3UMoHUo3nsq75xClmXRnlTvQ7I0x62DrXdMNE9tJtqAF
+lVUeD7gKlDPmYMK0QKbxFHYTVbdJ3d7UbJUgb2SxHpm5I8J7Wx29p6gLN7+swdIg
+y0Z8+qcABkeVHQ9OWwV0XYdMdLFovnc0pDAEvHL4rxv3E8W2Nv2mm0xW7I4HcG0
+h9uJ2lU4mn60YqBtso/cu+LzUHIPcHji5sRK8/qu+Tg
+-> ssh-rsa kFDS0A
+kXXvKZebwQLFofayT/0SFzdFl0e8xQbUJf+q47YHmPMlJY7nsQBs1fvmQsp7fsfW
+TRdh14uVImErEQsuqNYgYDR/jBUVtRkySOXde8Q9QM/CZwwR7NMu8P2vBzZ6uXKL
+amlZS8iYXWJKRgmxsgiONFZwfcB9TWhaDYsmeqFxV6ui2LGgtCBllttYzvyVNWZI
+NXMg0bbkLd+I2svBSBX/p8rECxq5gUmr8PB2k+yrHuXJvnR8Hop7YjvbrC2qSy5r
+6OOTButBV7cILTf03DPvd3f61uUqm/NapxN4UdEZDTmOYud2dF8Eqw+BCNIT2wZr
+/KD49ElPlcWM8cCxBGaSTTT66mP3FWwIcCZVfdVrhf//TN/SAo+lcoN7m/p9Uj0i
+Y5nM3JR7ZuiLHfXu+fTHBiWnWBtLkPYUDlGIGlFGinMQwi4CMqoUY7jROdjHuPkt
+S0VK2ViRxBB5Z5tQSnL02+TNsDd+CDURRiBgWfdmk1kkh9o2SrSiGcxCV03UVEaE
+4Q07ZjKJF9HeC6goK+QjSOvLHS2qHyJznUty7nAiAS+yPDlq6m13/dFTvFii7H2h
+UJ+5MJcVHLd6VQhzjmwTQXCrbTn/FI2LkZgR4HPRFDElkmnMUV1NU/2gkwm7Z0gu
+RaEAuYMoKZNBQ6eQgANst+LFA7ctwpz/d0PB8Gvjf6g
+-> piv-p256 vRzPNw AmCpZNeI8ggIr211niro2CalG6ELXYubjXj2J01eSwL+
+IPXLB01UKFj7tptbB7FmNbbjDGrqbEoizjNzzJvNsXs
+-> piv-p256 zqq/iw Auwb+rr9JfTX9VoTKaDobEGFPIwJZUFAMolG8SvZ5ix6
+e5IWW7Sxy1T2F0Ykm1tKQIvGGJFODIrNdvrCKREvrI
+-> ssh-ed25519 YFSOsg 5Eeo3PIUgfRgPrY/eR6aps8UB2NNNr9YJswZ5mPj5Fk
+cN4Dwp+ZFN8UCptfVsAp3iGesYhry7umwGc77jom2Vo
+-> ssh-ed25519 iHV63A RMH4ezLwPxlf9cLgFlWSrGMDdlySpIr22O3Nr4ESgkg
+8Ll31aL8PCOFp5+TIhv7qYVzjnBMepWJSlT6PGBMtdM
+-> ssh-ed25519 BVsyTA 9yQZMVRpIitqx4ggP9pswC2VBmtKHR8FqIiLAq6wdCs
+g/dA5SXBUNyLFGuOEVwsPIu2sEyAS4y+5RlccymLfL0
+-> ssh-ed25519 +3V2lQ unqrENNB2tJvICc/nAi31TZ8c7ilbFreww51f/Mi4nQ
+y7QWtFtEUq8elK+Q2HxpGav3nx3dxDQd5ikn9DpIJ20
+--- SJo44grQsKFl8WMnva4kMp88kMZ9D3EWnm3mN0Oe15c
+~°ÙìÚŒxÌ†Il`<60>u²”˜ôÿÁÙ@«Ž'B#ÅRaˆÙ£;«ã(»ÃÎïŠ<C3AF>Šò˜åŽ³&G3<Q<>sH‰ª~Gò)½¡®
--- a/terraform/dns.tf
+++ b/terraform/dns.tf
@ -24,6 +24,21 @@ resource "namecheap_domain_records" "pub-solar" {
    type     = "A"
    address  = "10.7.6.4"
  }
+  record {
+    hostname = "trinkgenossin.wg"
+    type     = "A"
+    address  = "10.7.6.5"
+  }
+  record {
+    hostname = "delite.wg"
+    type     = "A"
+    address  = "10.7.6.6"
+  }
+  record {
+    hostname = "blue-shell.wg"
+    type     = "A"
+    address  = "10.7.6.7"
+  }
  record {
    hostname = "nachtigall.wg"
    type     = "AAAA"
@ -44,6 +59,21 @@ resource "namecheap_domain_records" "pub-solar" {
    type     = "AAAA"
    address  = "fd00:fae:fae:fae:fae:4::"
  }
+  record {
+    hostname = "trinkgenossin.wg"
+    type     = "AAAA"
+    address  = "fd00:fae:fae:fae:fae:5::"
+  }
+  record {
+    hostname = "delite.wg"
+    type     = "AAAA"
+    address  = "fd00:fae:fae:fae:fae:6::"
+  }
+  record {
+    hostname = "blue-shell.wg"
+    type     = "AAAA"
+    address  = "fd00:fae:fae:fae:fae:7::"
+  }
  record {
    hostname = "flora-6"
    type     = "A"
@ -59,6 +89,36 @@ resource "namecheap_domain_records" "pub-solar" {
    type     = "A"
    address  = "49.13.236.167"
  }
+  record {
+    hostname = "trinkgenossin"
+    type     = "A"
+    address  = "85.215.152.22"
+  }
+  record {
+    hostname = "trinkgenossin"
+    type     = "AAAA"
+    address  = "2a01:239:35d:f500::1"
+  }
+  record {
+    hostname = "delite"
+    type     = "A"
+    address  = "5.255.119.132"
+  }
+  record {
+    hostname = "delite"
+    type     = "AAAA"
+    address  = "2a04:52c0:124:9d8c::2"
+  }
+  record {
+    hostname = "blue-shell"
+    type     = "A"
+    address  = "194.13.83.205"
+  }
+  record {
+    hostname = "blue-shell"
+    type     = "AAAA"
+    address  = "2a03:4000:43:24e::1"
+  }
  record {
    hostname = "auth"
    type     = "CNAME"
@ -69,6 +129,51 @@ resource "namecheap_domain_records" "pub-solar" {
    type     = "A"
    address  = "80.71.153.210"
  }
+  record {
+    hostname = "buckets"
+    type     = "A"
+    address  = "85.215.152.22"
+  }
+  record {
+    hostname = "buckets"
+    type     = "A"
+    address  = "5.255.119.132"
+  }
+  record {
+    hostname = "buckets"
+    type     = "A"
+    address  = "194.13.83.205"
+  }
+  record {
+    hostname = "buckets"
+    type     = "AAAA"
+    address  = "2a01:239:35d:f500::1"
+  }
+  record {
+    hostname = "buckets"
+    type     = "AAAA"
+    address  = "2a04:52c0:124:9d8c::2"
+  }
+  record {
+    hostname = "buckets"
+    type     = "AAAA"
+    address  = "2a03:4000:43:24e::1"
+  }
+  record {
+    hostname = "*.buckets"
+    type     = "CNAME"
+    address  = "buckets.pub.solar."
+  }
+  record {
+    hostname = "web"
+    type     = "CNAME"
+    address  = "buckets.pub.solar."
+  }
+  record {
+    hostname = "*.web"
+    type     = "CNAME"
+    address  = "buckets.pub.solar."
+  }
  record {
    hostname = "tankstelle"
    type     = "A"
--- a/tests/keycloak.nix
+++ b/tests/keycloak.nix
@ -0,0 +1,92 @@
+{
+  self,
+  pkgs,
+  lib,
+  config,
+  ...
+}:
+let
+in
+{
+  name = "keycloak";
+
+  hostPkgs = pkgs;
+
+  node.pkgs = pkgs;
+  node.specialArgs = self.outputs.nixosConfigurations.nachtigall._module.specialArgs;
+
+  nodes = {
+    acme-server = {
+      imports = [
+        self.nixosModules.home-manager
+        self.nixosModules.core
+        ./support/ca.nix
+      ];
+    };
+
+    client = {
+      imports = [
+        self.nixosModules.home-manager
+        self.nixosModules.core
+        ./support/client.nix
+      ];
+    };
+
+    nachtigall = {
+      imports = [
+        self.inputs.agenix.nixosModules.default
+        self.nixosModules.home-manager
+        self.nixosModules.core
+        self.nixosModules.backups
+        self.nixosModules.nginx
+        self.nixosModules.keycloak
+        self.nixosModules.postgresql
+        ./support/global.nix
+      ];
+
+      systemd.tmpfiles.rules = [ "f /tmp/dbf 1777 root root 10d password" ];
+
+      virtualisation.memorySize = 4096;
+
+      pub-solar-os.auth = {
+        enable = true;
+        database-password-file = "/tmp/dbf";
+      };
+      services.keycloak.database.createLocally = true;
+
+      networking.interfaces.eth0.ipv4.addresses = [
+        {
+          address = "192.168.1.3";
+          prefixLength = 32;
+        }
+      ];
+    };
+  };
+
+  testScript =
+    { nodes, ... }:
+    let
+      user = nodes.client.users.users.${nodes.client.pub-solar-os.authentication.username};
+      #uid = toString user.uid;
+      bus = "DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/$(id -u ${user.name})/bus";
+      gdbus = "${bus} gdbus";
+      su = command: "su - ${user.name} -c '${command}'";
+      gseval = "call --session -d org.gnome.Shell -o /org/gnome/Shell -m org.gnome.Shell.Eval";
+      wmClass = su "${gdbus} ${gseval} global.display.focus_window.wm_class";
+    in
+    ''
+      start_all()
+
+      nachtigall.wait_for_unit("system.slice")
+      nachtigall.succeed("ping 127.0.0.1 -c 2")
+      nachtigall.wait_for_unit("nginx.service")
+      nachtigall.wait_for_unit("keycloak.service")
+      nachtigall.wait_until_succeeds("curl http://127.0.0.1:8080/")
+      nachtigall.wait_until_succeeds("curl https://auth.test.pub.solar/")
+
+      client.wait_for_unit("system.slice")
+      client.sleep(30)
+      # client.wait_until_succeeds("${wmClass} | grep -q 'firefox'")
+      client.screenshot("screen")
+    '';
+}
--- a/tests/support/ca.nix
+++ b/tests/support/ca.nix
@ -0,0 +1,47 @@
+{
+  pkgs,
+  lib,
+  config,
+  ...
+}:
+{
+  imports = [ ./global.nix ];
+
+  systemd.tmpfiles.rules = [ "f /tmp/step-ca-intermediate-pw 1777 root root 10d password" ];
+
+  networking.interfaces.eth0.ipv4.addresses = [
+    {
+      address = "192.168.1.1";
+      prefixLength = 32;
+    }
+  ];
+
+  services.step-ca =
+    let
+      certificates = pkgs.stdenv.mkDerivation {
+        name = "certificates";
+        src = ./step;
+        installPhase = ''
+          mkdir -p $out;
+          cp -r certs $out/
+          cp -r secrets $out/
+        '';
+      };
+    in
+    {
+      enable = true;
+      openFirewall = true;
+      intermediatePasswordFile = "/tmp/step-ca-intermediate-pw";
+      port = 443;
+      address = "0.0.0.0";
+      settings = (builtins.fromJSON (builtins.readFile ./step/config/ca.json)) // {
+        root = "${certificates}/certs/root_ca.crt";
+        crt = "${certificates}/certs/intermediate_ca.crt";
+        key = "${certificates}/secrets/intermediate_ca_key";
+        db = {
+          type = "badgerv2";
+          dataSource = "/var/lib/step-ca/db";
+        };
+      };
+    };
+}
--- a/tests/support/client.nix
+++ b/tests/support/client.nix
@ -0,0 +1,35 @@
+{
+  pkgs,
+  lib,
+  config,
+  ...
+}:
+{
+  imports = [ ./global.nix ];
+
+  services.xserver.enable = true;
+  services.xserver.displayManager.gdm.enable = true;
+  services.xserver.desktopManager.gnome.enable = true;
+  services.xserver.displayManager.autoLogin.enable = true;
+  services.xserver.displayManager.autoLogin.user = config.pub-solar-os.authentication.username;
+
+  systemd.user.services = {
+    "org.gnome.Shell@wayland" = {
+      serviceConfig = {
+        ExecStart = [
+          # Clear the list before overriding it.
+          ""
+          # Eval API is now internal so Shell needs to run in unsafe mode.
+          "${pkgs.gnome.gnome-shell}/bin/gnome-shell --unsafe-mode"
+        ];
+      };
+    };
+  };
+
+  networking.interfaces.eth0.ipv4.addresses = [
+    {
+      address = "192.168.1.2";
+      prefixLength = 32;
+    }
+  ];
+}
--- a/tests/support/global.nix
+++ b/tests/support/global.nix
@ -0,0 +1,50 @@
+{
+  pkgs,
+  lib,
+  config,
+  ...
+}:
+{
+  pub-solar-os.networking.domain = "test.pub.solar";
+
+  security.acme.defaults.server = "https://ca.${config.pub-solar-os.networking.domain}/acme/acme/directory";
+
+  security.pki.certificates = [ (builtins.readFile ./step/certs/root_ca.crt) ];
+
+  services.openssh = {
+    enable = true;
+    openFirewall = true;
+    settings = {
+      PermitRootLogin = lib.mkForce "yes";
+      PermitEmptyPasswords = lib.mkForce "yes";
+      PasswordAuthentication = lib.mkForce true;
+    };
+  };
+
+  security.pam.services.sshd.allowNullPassword = true;
+
+  virtualisation.forwardPorts =
+    let
+      address = (builtins.elemAt config.networking.interfaces.eth0.ipv4.addresses 0).address;
+      lastAddressPart = builtins.elemAt (lib.strings.splitString "." address) 3;
+    in
+    [
+      {
+        from = "host";
+        host.port = 2000 + (lib.strings.toInt lastAddressPart);
+        guest.port = 22;
+      }
+    ];
+
+  networking.interfaces.eth0.useDHCP = false;
+
+  networking.hosts = {
+    "192.168.1.1" = [ "ca.${config.pub-solar-os.networking.domain}" ];
+    "192.168.1.2" = [ "client.${config.pub-solar-os.networking.domain}" ];
+    "192.168.1.3" = [
+      "${config.pub-solar-os.networking.domain}"
+      "www.${config.pub-solar-os.networking.domain}"
+      "auth.${config.pub-solar-os.networking.domain}"
+    ];
+  };
+}
--- a/tests/support/step/certs/intermediate_ca.crt
+++ b/tests/support/step/certs/intermediate_ca.crt
@ -0,0 +1,13 @@
+-----BEGIN CERTIFICATE-----
+MIIB4DCCAYagAwIBAgIQVR/3c0swvc/ifeYqLQn3HTAKBggqhkjOPQQDAjA6MRcw
+FQYDVQQKEw5wdWIuc29sYXItdGVzdDEfMB0GA1UEAxMWcHViLnNvbGFyLXRlc3Qg
+Um9vdCBDQTAeFw0yNDA4MjQwMTI3MTBaFw0zNDA4MjIwMTI3MTBaMEIxFzAVBgNV
+BAoTDnB1Yi5zb2xhci10ZXN0MScwJQYDVQQDEx5wdWIuc29sYXItdGVzdCBJbnRl
+cm1lZGlhdGUgQ0EwWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAATpCjy3PAiawAeb
+47ZZ9kPXuuV0EavOfFlgnlZBkOc2AXY0R6P1jK06US0SiPo17rqyNgUWH0oV4v8i
+/HbZYNXYo2YwZDAOBgNVHQ8BAf8EBAMCAQYwEgYDVR0TAQH/BAgwBgEB/wIBADAd
+BgNVHQ4EFgQU1hueYsLAH6+wxjArqCM3IHFqnIEwHwYDVR0jBBgwFoAUxg/BmKK7
+9Zs+b1bvlpYwggy5lnswCgYIKoZIzj0EAwIDSAAwRQIgfxkjyC4HHADRmNDLqZ5L
+0po+JD5/9b1L//JoXG+vgXECIQDgkRe8r8/0Ep/NWgBtbkA3oTYq8vCwo1FewBZZ
+43fo5w==
+-----END CERTIFICATE-----
--- a/tests/support/step/certs/root_ca.crt
+++ b/tests/support/step/certs/root_ca.crt
@ -0,0 +1,12 @@
+-----BEGIN CERTIFICATE-----
+MIIBuDCCAV2gAwIBAgIQMXg7xoEIrVjgvKcrRaxo0DAKBggqhkjOPQQDAjA6MRcw
+FQYDVQQKEw5wdWIuc29sYXItdGVzdDEfMB0GA1UEAxMWcHViLnNvbGFyLXRlc3Qg
+Um9vdCBDQTAeFw0yNDA4MjQwMTI3MDlaFw0zNDA4MjIwMTI3MDlaMDoxFzAVBgNV
+BAoTDnB1Yi5zb2xhci10ZXN0MR8wHQYDVQQDExZwdWIuc29sYXItdGVzdCBSb290
+IENBMFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEYNxMcHclQP/zv2y6LJIGx9pg
+Q2337Zb8TuPY+DnL1MjuCMoeTaMwngzjU/DSbKL0Vx/y+I+PBjhHmPrYcGDcSKNF
+MEMwDgYDVR0PAQH/BAQDAgEGMBIGA1UdEwEB/wQIMAYBAf8CAQEwHQYDVR0OBBYE
+FMYPwZiiu/WbPm9W75aWMIIMuZZ7MAoGCCqGSM49BAMCA0kAMEYCIQDcgr9WyR1C
+806aEQ38alYgGg3PhQdT14q47tWIUOnpygIhAM0x/QK/mm7VvQxBLAA4DT6X730m
+k/tBvh9SHNbwPxCt
+-----END CERTIFICATE-----
--- a/tests/support/step/config/ca.json
+++ b/tests/support/step/config/ca.json
@ -0,0 +1,44 @@
+{
+  "federatedRoots": null,
+  "address": ":443",
+  "insecureAddress": "",
+  "dnsNames": ["ca.test.pub.solar"],
+  "logger": {
+    "format": "text"
+  },
+  "db": {
+    "type": "badgerv2",
+    "badgerFileLoadingMode": ""
+  },
+  "authority": {
+    "provisioners": [
+      {
+        "name": "acme",
+        "type": "ACME"
+      },
+      {
+        "type": "JWK",
+        "name": "test.pub.solar",
+        "key": {
+          "use": "sig",
+          "kty": "EC",
+          "kid": "lM-BJXRwwQcdgxLqAS4Za23A2YatZpwXx-PP5NIt8JM",
+          "crv": "P-256",
+          "alg": "ES256",
+          "x": "ouB2mP04Kt8rDa10C8ZzYyzA36rrz-k0c4_ud1hVjyg",
+          "y": "RbXKcudQRPEFqjG_5AxuqCQXn7pyRToQCwC4MrwLVUQ"
+        },
+        "encryptedKey": "eyJhbGciOiJQQkVTMi1IUzI1NitBMTI4S1ciLCJjdHkiOiJqd2sranNvbiIsImVuYyI6IkEyNTZHQ00iLCJwMmMiOjYwMDAwMCwicDJzIjoiNWR5T2puR2Y5aFFNRlc1U25fRWhzUSJ9.a3xtSBuMmzZCMsdfHAXMgFpe9bq8A6bGGOoW9F2Gw7AhxL4bG-AlgA.IA68rSJSGTAKnaVS.XDQc4da-8D9Ykfw-8S4uphsauq5gsEm4qp7zKQUIvcjUlnPAtiHP3xiiBie29ncdg8rKmyzprEEOpTNvXtQl7LsPsHXyKV3SqsTnJecvim9YXGDneAHyWe-XF6hyCZAfSoFbFMgLDKR6d44hMht3ueazL_TPlkFUBLrJbsW782MfdfF3nzcaDf_JDuhKsKHDmKqZyNXDzwf6rINe8adrf5gqaLM2_sGhk7i3XyXygn8HHVw1Dj_w2gPOVm4MS7CO_NgikPqAtGuXDhpWZfXte-FlnMO6d9xQF67b0cwB8kmColPSp1zRiCKPAk9vof8Nn-gGE_aw8zxPi0CJkoY.xbuqSSspgLc_Uw17uiRF7Q"
+      }
+    ]
+  },
+  "tls": {
+    "cipherSuites": [
+      "TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256",
+      "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256"
+    ],
+    "minVersion": 1.2,
+    "maxVersion": 1.3,
+    "renegotiation": false
+  }
+}
--- a/tests/support/step/config/defaults.json
+++ b/tests/support/step/config/defaults.json
@ -0,0 +1,6 @@
+{
+  "ca-url": "https://ca.test.pub.solar",
+  "ca-config": "/home/b12f/.step/config/ca.json",
+  "fingerprint": "4d6a1a918355380acbd0256a2203d0a0da8436bb788e8f19326589045c3cd842",
+  "root": "/home/b12f/.step/certs/root_ca.crt"
+}
--- a/tests/support/step/secrets/intermediate_ca_key
+++ b/tests/support/step/secrets/intermediate_ca_key
@ -0,0 +1,8 @@
+-----BEGIN EC PRIVATE KEY-----
+Proc-Type: 4,ENCRYPTED
+DEK-Info: AES-256-CBC,0b34c00cb76ffc16441f5fe762d8d915
+
+xJQ5r5kGiaG6rCsmVnONxm99sqceb62dO8/YvgdZ/ouHAxz1OlXYpTJNd2GvezAc
+XA6Zx6eGzNCOyhgMNJTXEn8QmcJcMd6OjVLxQ9Tr2Mi3LShcBzMPs30/X2XYsM22
+5G4fRhQD0L4nQ08B3GG6FjPe/HYmkRNZmAeDc2wE5Fg=
+-----END EC PRIVATE KEY-----
--- a/tests/support/step/secrets/root_ca_key
+++ b/tests/support/step/secrets/root_ca_key
@ -0,0 +1,8 @@
+-----BEGIN EC PRIVATE KEY-----
+Proc-Type: 4,ENCRYPTED
+DEK-Info: AES-256-CBC,48f59a57e5a2b81359e0a3668161b61e
+
+jMZbpiHSFa74ns30QrAnIlcguqWp+FE20cXbiIVPpLAJpzGskc3k5vRFTpPM8geg
+sZ6bVvq3APbKmkopxZHWpd4ly6uHkolbtR1NFxTNKymaJZuSuKspUmDohkIyZN6c
+KG0upERMZIOg6Ky1JiM5pLJMHBTsCmzJBmdFCW7GSww=
+-----END EC PRIVATE KEY-----
--- a/tests/website.nix
+++ b/tests/website.nix
@ -8,17 +8,52 @@
 {
  name = "website";

-  nodes.nachtigall-test = self.nixosConfigurations.nachtigall-test;
-
-  node.specialArgs = self.outputs.nixosConfigurations.nachtigall._module.specialArgs;
  hostPkgs = pkgs;

-  enableOCR = true;
+  node.pkgs = pkgs;
+  node.specialArgs = self.outputs.nixosConfigurations.nachtigall._module.specialArgs;
+
+  nodes = {
+    acme-server = {
+      imports = [
+        self.nixosModules.home-manager
+        self.nixosModules.core
+        ./support/ca.nix
+      ];
+    };
+
+    nachtigall = {
+      imports = [
+        self.nixosModules.home-manager
+        self.nixosModules.core
+        self.nixosModules.nginx
+        self.nixosModules.nginx-website
+        ./support/global.nix
+      ];
+
+      virtualisation.memorySize = 4096;
+
+      networking.interfaces.eth0.ipv4.addresses = [
+        {
+          address = "192.168.1.3";
+          prefixLength = 32;
+        }
+      ];
+    };
+  };

  testScript = ''
-    machine.wait_for_unit("system.slice")
-    machine.succeed("ping 127.0.0.1 -c 2")
-    machine.wait_for_unit("nginx.service")
-    machine.succeed("curl -H 'Host:pub.solar' http://127.0.0.1/")
+    start_all()
+
+    acme_server.wait_for_unit("system.slice")
+    acme_server.wait_for_unit("step-ca.service")
+    acme_server.succeed("ping ca.test.pub.solar -c 2")
+    acme_server.wait_until_succeeds("curl 127.0.0.1:443")
+
+    nachtigall.wait_for_unit("system.slice")
+    nachtigall.succeed("ping test.pub.solar -c 2")
+    nachtigall.succeed("ping ca.test.pub.solar -c 2")
+    nachtigall.wait_for_unit("nginx.service")
+    nachtigall.wait_until_succeeds("curl https://test.pub.solar/")
  '';
 }
Author	SHA1	Message	Date
teutat3s	27dc20dd04	obs-portal: add backups to garage bucket + storagebox All checks were successful Flake checks / Check (pull_request) Successful in 23m21s Details Restic backups to garage S3 bucket nachtigall-backups	2024-08-29 10:09:04 +02:00
teutat3s	a0fb6a60c3	Merge pull request 'devshell: add terraform-ls' (#227 ) from terraform-devshell into main Reviewed-on: #227 Reviewed-by: b12f <b12f@noreply.git.pub.solar> Reviewed-by: Hendrik Sokolowski <hensoko@noreply.git.pub.solar>	2024-08-28 15:55:29 +00:00
teutat3s	d2389497c2	Merge pull request 'garage: initial cluster' (#222 ) from garage-cluster into main Reviewed-on: #222 Reviewed-by: b12f <b12f@noreply.git.pub.solar> Reviewed-by: Hendrik Sokolowski <hensoko@noreply.git.pub.solar>	2024-08-28 15:55:16 +00:00
teutat3s	c056d9c35e	Merge pull request 'mediawiki: add backups' (#225 ) from mediawiki-backups into main Reviewed-on: #225 Reviewed-by: Hendrik Sokolowski <hensoko@noreply.git.pub.solar>	2024-08-28 15:54:33 +00:00
teutat3s	4626fd85c0	mediawiki: add backups to garage bucket + storagebox All checks were successful Flake checks / Check (pull_request) Successful in 1m56s Details Restic backups to garage S3 bucket nachtigall-backups https://garagehq.deuxfleurs.fr/documentation/connect/backup/#restic	2024-08-28 17:13:34 +02:00
teutat3s	c0a3d90d63	backups: add environmentFile option	2024-08-28 17:13:34 +02:00
teutat3s	1d92ef53ca	backups: storeName -> repoName	2024-08-28 17:13:33 +02:00
teutat3s	751d82f7e3	backups: rename pub-solar-os.backups.backups -> pub-solar-os.backups.restic	2024-08-28 17:12:22 +02:00
teutat3s	fb8ee1278a	Merge pull request 'feat/tests' (#224 ) from feat/tests into main Reviewed-on: #224 Reviewed-by: teutat3s <teutat3s@noreply.git.pub.solar>	2024-08-27 10:45:56 +00:00
teutat3s	66ed87e666	ci: avoid garbage collection of checks All checks were successful Flake checks / Check (pull_request) Successful in 17m36s Details	2024-08-27 12:37:37 +02:00
teutat3s	88b76beb5c	keycloak: use backups module All checks were successful Flake checks / Check (pull_request) Successful in 19m4s Details Co-authored-by: b12f <b12f@noreply.git.pub.solar> Co-authored-by: Hendrik Sokolowski <hensoko@noreply.git.pub.solar>	2024-08-27 10:09:07 +02:00
teutat3s	e857c6198b	modules/backup: init Co-authored-by: b12f <b12f@noreply.git.pub.solar> Co-authored-by: Hendrik Sokolowski <hensoko@noreply.git.pub.solar>	2024-08-27 10:04:10 +02:00
teutat3s	998cf4c63d	website: force HTTPS Co-authored-by: b12f <b12f@noreply.git.pub.solar> Co-authored-by: Hendrik Sokolowski <hensoko@noreply.git.pub.solar>	2024-08-27 10:03:43 +02:00
teutat3s	a0b52d51e5	nachtigall: make postgres wait for zfs mount Co-authored-by: b12f <b12f@noreply.git.pub.solar> Co-authored-by: Hendrik Sokolowski <hensoko@noreply.git.pub.solar>	2024-08-27 10:00:42 +02:00
teutat3s	701c62dd69	tests: create keycloak test, add working test for website Co-authored-by: b12f <b12f@noreply.git.pub.solar> Co-authored-by: Hendrik Sokolowski <hensoko@noreply.git.pub.solar>	2024-08-27 09:55:25 +02:00
teutat3s	711347abe6	docs: add how-to create garage bucket + key All checks were successful Flake checks / Check (pull_request) Successful in 6m38s Details add new hosts to WireGuard example config	2024-08-26 11:56:45 +02:00
teutat3s	13bf3f5beb	docs: SSH to trinkgenossin for garage command All checks were successful Flake checks / Check (pull_request) Successful in 6m45s Details	2024-08-25 03:27:42 +02:00
teutat3s	f639fbe050	devshell: add terraform-ls All checks were successful Flake checks / Check (pull_request) Successful in 5m49s Details	2024-08-25 02:37:36 +02:00
teutat3s	f236962e17	garage: add monitoring, connect to grafana + loki All checks were successful Flake checks / Check (pull_request) Successful in 7m10s Details https://garagehq.deuxfleurs.fr/documentation/reference-manual/monitoring/	2024-08-25 00:18:09 +02:00
teutat3s	d32abd7a7f	wireguard: add trinkgenossin, delite, blue-shell	2024-08-25 00:13:53 +02:00
teutat3s	15b507904f	garage: init buckets.pub.solar, use nginx as reverse proxy https://garagehq.deuxfleurs.fr/documentation/cookbook/reverse-proxy/	2024-08-24 21:48:48 +02:00
teutat3s	b0790876ec	style: format using nixfmt-rfc-style All checks were successful Flake checks / Check (pull_request) Successful in 7m42s Details	2024-08-24 17:39:49 +02:00
teutat3s	b6070d0f75	docs: initial docs for nixos-anywhere and garage setup Some checks failed Flake checks / Check (pull_request) Failing after 2m26s Details	2024-08-24 03:06:17 +02:00
teutat3s	25827a97d3	modules: add unlock-luks-on-boot	2024-08-24 03:05:28 +02:00
teutat3s	4a3d3ce84b	garage: init module	2024-08-24 03:05:16 +02:00
teutat3s	9eb746313e	flake: add disko https://github.com/nix-community/disko And add hosts to deploy-rs	2024-08-24 03:04:19 +02:00
teutat3s	83b7e3e11e	hosts: init blue-shell	2024-08-24 03:02:15 +02:00
teutat3s	4ef9781d10	hosts: init delite	2024-08-24 03:01:46 +02:00
teutat3s	ca8e578b11	hosts: init trinkgenossin	2024-08-24 03:00:01 +02:00
teutat3s	49c21fe740	dns: add trinkgenossin, delite, blue-shell	2024-08-24 02:58:37 +02:00