fix: rekey all secrets

Reviewed-on: #248

.drone.yml

@@ -17,7 +17,7 @@ steps:
       - nix $$NIX_FLAGS develop --command nix flake show
       - nix $$NIX_FLAGS develop --command treefmt --fail-on-change
       - nix $$NIX_FLAGS develop --command editorconfig-checker
-      - nix $$NIX_FLAGS build ".#nixosConfigurations.PubSolarOS.config.system.build.toplevel"
+      - nix $$NIX_FLAGS build ".#nixosConfigurations.flora-6.config.system.build.toplevel"
 
 ---
 kind: pipeline
@@ -44,7 +44,7 @@ steps:
         from_secret: private_ssh_key
       MANTA_USER: pub_solar
       MANTA_URL: https://eu-central.manta.greenbaum.cloud
-      MANTA_KEY_ID: "5d:5f:3d:22:8d:37:1f:e6:d6:ab:06:18:d9:a2:04:67"
+      MANTA_KEY_ID: "59:9f:5a:6f:c4:e2:3b:32:7f:13:1f:de:b7:59:80:85"
     commands:
       - export TARGET_DIR="ci/$${DRONE_REPO}/$${DRONE_BUILD_NUMBER}"
       - echo env var TARGET_DIR is set to $$TARGET_DIR
@@ -149,6 +149,6 @@ volumes:
 
 ---
 kind: signature
-hmac: a116f78a0b22188052893bdb46aa40f8de66438826c10ced362ea183d7644d67
+hmac: 17811add241edae457584ba78389886df02b5e51820d826ef5fb2d97de2430e2
 
 ...

flake.lock

@@ -89,6 +89,28 @@
         "type": "github"
       }
     },
+    "devshell_2": {
+      "inputs": {
+        "nixpkgs": [
+          "keycloak-theme-pub-solar",
+          "nixpkgs"
+        ],
+        "systems": "systems"
+      },
+      "locked": {
+        "lastModified": 1688380630,
+        "narHash": "sha256-8ilApWVb1mAi4439zS3iFeIT0ODlbrifm/fegWwgHjA=",
+        "owner": "numtide",
+        "repo": "devshell",
+        "rev": "f9238ec3d75cefbb2b42a44948c4e8fb1ae9a205",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "devshell",
+        "type": "github"
+      }
+    },
     "digga": {
       "inputs": {
         "darwin": [
@@ -199,7 +221,25 @@
     },
     "flake-utils_3": {
       "inputs": {
-        "systems": "systems"
+        "systems": "systems_2"
+      },
+      "locked": {
+        "lastModified": 1689068808,
+        "narHash": "sha256-6ixXo3wt24N/melDWjq70UuHQLxGV8jZvooRanIHXw0=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "919d646de7be200f3bf08cb76ae1f09402b6f9b4",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
+    "flake-utils_4": {
+      "inputs": {
+        "systems": "systems_3"
       },
       "locked": {
         "lastModified": 1687171271,
@@ -215,6 +255,21 @@
         "type": "github"
       }
     },
+    "flake-utils_5": {
+      "locked": {
+        "lastModified": 1653893745,
+        "narHash": "sha256-0jntwV3Z8//YwuOjzhV2sgJJPt+HY6KhU7VZUL0fKZQ=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "1ed9fb1935d260de5fe1c2f7ee0ebaae17ed2fa1",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
+    },
     "home": {
       "inputs": {
         "nixpkgs": [
@@ -236,34 +291,36 @@
         "type": "github"
       }
     },
-    "home-manager": {
+    "keycloak-theme-pub-solar": {
       "inputs": {
+        "devshell": "devshell_2",
+        "flake-utils": "flake-utils_3",
         "nixpkgs": [
-          "agenix",
+          "nixos"
-          "nixpkgs"
         ]
       },
       "locked": {
-        "lastModified": 1682203081,
+        "lastModified": 1689875310,
-        "narHash": "sha256-kRL4ejWDhi0zph/FpebFYhzqlOBrk0Pl3dzGEKSAlEw=",
+        "narHash": "sha256-gJxh8fVX24nZXBxstZcrzZhMRFG9jyOnQEfkgoRr39I=",
-        "owner": "nix-community",
+        "ref": "main",
-        "repo": "home-manager",
+        "rev": "c2c86bbf9855f16a231a596b75b443232a7b9395",
-        "rev": "32d3e39c491e2f91152c84f8ad8b003420eab0a1",
+        "revCount": 24,
-        "type": "github"
+        "type": "git",
+        "url": "https://git.pub.solar/pub-solar/keycloak-theme"
       },
       "original": {
-        "owner": "nix-community",
+        "ref": "main",
-        "repo": "home-manager",
+        "type": "git",
-        "type": "github"
+        "url": "https://git.pub.solar/pub-solar/keycloak-theme"
       }
     },
     "latest": {
       "locked": {
-        "lastModified": 1688231357,
+        "lastModified": 1689192006,
-        "narHash": "sha256-ZOn16X5jZ6X5ror58gOJAxPfFLAQhZJ6nOUeS4tfFwo=",
+        "narHash": "sha256-QM0f0d8oPphOTYJebsHioR9+FzJcy1QNIzREyubB91U=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "645ff62e09d294a30de823cb568e9c6d68e92606",
+        "rev": "2de8efefb6ce7f5e4e75bdf57376a96555986841",
         "type": "github"
       },
       "original": {
@@ -275,11 +332,11 @@
     },
     "nixos": {
       "locked": {
-        "lastModified": 1688109178,
+        "lastModified": 1689209875,
-        "narHash": "sha256-BSdeYp331G4b1yc7GIRgAnfUyaktW2nl7k0C577Tttk=",
+        "narHash": "sha256-8AVcBV1DiszaZzHFd5iLc8HSLfxRAuqcU0QdfBEF3Ag=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "b72aa95f7f096382bff3aea5f8fde645bca07422",
+        "rev": "fcc147b1e9358a8386b2c4368bd928e1f63a7df2",
         "type": "github"
       },
       "original": {
@@ -325,7 +382,7 @@
         "flake-compat": [
           "flake-compat"
         ],
-        "flake-utils": "flake-utils_3",
+        "flake-utils": "flake-utils_4",
         "nixpkgs": [
           "nixos"
         ]
@@ -352,10 +409,12 @@
         "digga": "digga",
         "flake-compat": "flake-compat",
         "home": "home",
+        "keycloak-theme-pub-solar": "keycloak-theme-pub-solar",
         "latest": "latest",
         "nixos": "nixos",
         "nixos-hardware": "nixos-hardware",
-        "nvfetcher": "nvfetcher"
+        "nvfetcher": "nvfetcher",
+        "triton-vmtools": "triton-vmtools"
       }
     },
     "systems": {
@@ -373,6 +432,60 @@
         "type": "github"
       }
     },
+    "systems_2": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    },
+    "systems_3": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    },
+    "triton-vmtools": {
+      "inputs": {
+        "flake-utils": "flake-utils_5",
+        "nixpkgs": [
+          "nixos"
+        ]
+      },
+      "locked": {
+        "dir": "vmtools",
+        "lastModified": 1669648111,
+        "narHash": "sha256-EKh7iM4fCyZ7L6+HmGn3QkZ1HuG9zMEkziOH3K13SbY=",
+        "ref": "main",
+        "rev": "d78c4afe040440437949ce581ae0dcdc5893553c",
+        "revCount": 28,
+        "type": "git",
+        "url": "https://git.b12f.io/pub-solar/infra?dir=vmtools"
+      },
+      "original": {
+        "dir": "vmtools",
+        "ref": "main",
+        "type": "git",
+        "url": "https://git.b12f.io/pub-solar/infra?dir=vmtools"
+      }
+    },
     "utils": {
       "locked": {
         "lastModified": 1667395993,

flake.nix

@@ -38,6 +38,12 @@
     nvfetcher.url = "github:berberman/nvfetcher";
     nvfetcher.inputs.nixpkgs.follows = "nixos";
     nvfetcher.inputs.flake-compat.follows = "flake-compat";
+
+    triton-vmtools.url = "git+https://git.b12f.io/pub-solar/infra?ref=main&dir=vmtools";
+    triton-vmtools.inputs.nixpkgs.follows = "nixos";
+
+    keycloak-theme-pub-solar.url = "git+https://git.pub.solar/pub-solar/keycloak-theme?ref=main";
+    keycloak-theme-pub-solar.inputs.nixpkgs.follows = "nixos";
   };
 
   outputs = {
@@ -49,6 +55,8 @@
     agenix,
     deploy,
     nvfetcher,
+    triton-vmtools,
+    keycloak-theme-pub-solar,
     ...
   } @ inputs:
     digga.lib.mkFlake
@@ -117,12 +125,14 @@
             ];
           };
           PubSolarOS = {
-            tests = [
+            # Broken since https://github.com/NixOS/nixpkgs/commit/5bcef4224928fe45312f0ee321ddf0f0e8feeb7b
-              #(import ./tests/first-test.nix {
+            # Needs a fix in https://github.com/divnix/digga/blob/main/src/tests.nix#L12-L21
+            #tests = [
+            #  (import ./tests/first-test.nix {
             #    pkgs = nixos.legacyPackages.x86_64-linux;
             #    lib = nixos.lib;
-              #})
+            #  })
-            ];
+            #];
           };
         };
         importables = rec {
@@ -153,6 +163,11 @@
           pub-solar = {suites, ...}: {
             imports = suites.base;
 
+            home.stateVersion = "21.03";
+          };
+          barkeeper = {suites, ...}: {
+            imports = suites.base;
+
             home.stateVersion = "21.03";
           };
         }; # digga.lib.importers.rakeLeaves ./users/hm;
@@ -163,6 +178,27 @@
       homeConfigurations = digga.lib.mkHomeConfigurations self.nixosConfigurations;
 
       deploy.nodes = digga.lib.mkDeployNodes self.nixosConfigurations {
+        flora-6 = {
+          sshUser = "barkeeper";
+          hostname = "flora-6.pub.solar";
+          fastConnect = true;
+          profilesOrder = ["system" "direnv"];
+          profiles.direnv = {
+            user = "barkeeper";
+            path = self.pkgs.x86_64-linux.nixos.deploy-rs.lib.activate.home-manager self.homeConfigurationsPortable.x86_64-linux.barkeeper;
+          };
+        };
+
+        nougat-2 = {
+          sshUser = "barkeeper";
+          hostname = "nougat-2.b12f.io";
+          fastConnect = true;
+          profilesOrder = ["system" "direnv"];
+          profiles.direnv = {
+            user = "barkeeper";
+            path = self.pkgs.x86_64-linux.nixos.deploy-rs.lib.activate.home-manager self.homeConfigurationsPortable.x86_64-linux.barkeeper;
+          };
+        };
         #example = {
         #  hostname = "example.com:22";
         #  sshUser = "bartender";

hosts/flora-6/README.md

@@ -0,0 +1,16 @@
+# Mailman on NixOS docs
+
+- add reverse DNS record for IP
+
+Manual setup done for mailman, adapted from https://nixos.wiki/wiki/Mailman:
+
+```
+# Add DNS records in infra repo using terraform:
+# https://git.pub.solar/pub-solar/infra/commit/db234cdb5b55758a3d74387ada0760e06e166b9d
+
+# Generate initial postfix_domains.db and postfix_lmtp.db databases for Postfix
+sudo -u mailman mailman aliases
+# Create a django superuser account
+sudo -u mailman-web mailman-web createsuperuser
+# Followed outlined steps in web UI
+```

hosts/flora-6/caddy.nix

@@ -0,0 +1,140 @@
+{
+  config,
+  lib,
+  pkgs,
+  self,
+  ...
+}: {
+  systemd.tmpfiles.rules = [
+    "d '/data/srv/www/os/download/' 0750 hakkonaut hakkonaut - -"
+  ];
+
+  services.caddy = {
+    enable = lib.mkForce true;
+    group = "hakkonaut";
+    email = "admins@pub.solar";
+    enableReload = true;
+    globalConfig = lib.mkForce ''
+      grace_period 60s
+    '';
+    virtualHosts = {
+      "pub.solar" = {
+        logFormat = lib.mkForce ''
+          output discard
+        '';
+        extraConfig = ''
+          # Named matcher, used below for Mastodon webfinger
+          @query query resource=*
+
+          # PubSolarOS images
+          handle /os/download/* {
+            root * /data/srv/www
+            file_server /os/download/* browse
+          }
+          # serve base domain pub.solar for mastodon.pub.solar
+          # https://masto.host/mastodon-usernames-different-from-the-domain-used-for-installation/
+          handle /.well-known/host-meta {
+            redir https://mastodon.pub.solar{uri}
+          }
+
+          # Tailscale OIDC webfinger requirement plus Mastodon webfinger redirect
+          handle /.well-known/webfinger {
+            # Redirect requests that match /.well-known/webfinger?resource=* to Mastodon
+            handle @query {
+              redir https://mastodon.pub.solar{uri}
+            }
+            respond 200 {
+              body `{
+                "subject": "acct:admins@pub.solar",
+                "links": [
+                  {
+                    "rel": "http://openid.net/specs/connect/1.0/issuer",
+                    "href": "https://auth.pub.solar/realms/pub.solar"
+                  }
+                ]
+              }`
+            }
+          }
+
+          # redirect to statutes
+          redir /satzung https://cloud.pub.solar/s/2tRCP9aZFCiWxQy temporary
+
+          # pub.solar website
+          handle {
+            root * /srv/www/pub.solar
+            try_files {path}.html {path}
+            file_server
+          }
+          # minimal error handling, respond with status code and text
+          handle_errors {
+            respond "{http.error.status_code} {http.error.status_text}"
+          }
+        '';
+      };
+      "www.pub.solar" = {
+        logFormat = lib.mkForce ''
+          output discard
+        '';
+        extraConfig = ''
+          redir https://pub.solar{uri}
+        '';
+      };
+      "auth.pub.solar" = {
+        logFormat = lib.mkForce ''
+          output discard
+        '';
+        extraConfig = ''
+          redir / /realms/pub.solar/account temporary
+          reverse_proxy :8080
+        '';
+      };
+      "git.pub.solar" = {
+        logFormat = lib.mkForce ''
+          output discard
+        '';
+        extraConfig = ''
+          redir /user/login /user/oauth2/keycloak temporary
+          reverse_proxy :3000
+        '';
+      };
+      "ci.pub.solar" = {
+        logFormat = lib.mkForce ''
+          output discard
+        '';
+        extraConfig = ''
+          reverse_proxy :4000
+        '';
+      };
+      "stream.pub.solar" = {
+        logFormat = lib.mkForce ''
+          output discard
+        '';
+        extraConfig = ''
+          reverse_proxy :5000
+        '';
+      };
+      "list.pub.solar" = {
+        logFormat = lib.mkForce ''
+          output discard
+        '';
+        extraConfig = ''
+          handle_path /static/* {
+            root * /var/lib/mailman-web-static
+            file_server
+          }
+
+          reverse_proxy :18507
+        '';
+      };
+      "obs-portal.pub.solar" = {
+        logFormat = lib.mkForce ''
+          output discard
+        '';
+        extraConfig = ''
+          reverse_proxy obs-portal.svc.e5756d08-36fd-424b-f8bc-acdb92ca7b82.lev-1.int.greenbaum.zone:3000
+        '';
+      };
+    };
+  };
+  networking.firewall.allowedTCPPorts = [80 443];
+}

hosts/flora-6/default.nix

@@ -0,0 +1,5 @@
+{...}: {
+  imports = [
+    ./flora-6.nix
+  ];
+}

hosts/flora-6/drone.nix

@@ -0,0 +1,114 @@
+{
+  config,
+  lib,
+  pkgs,
+  self,
+  ...
+}: {
+  age.secrets.drone-secrets = {
+    file = "${self}/secrets/drone-secrets.age";
+    mode = "600";
+    owner = "drone";
+  };
+  age.secrets.drone-db-secrets = {
+    file = "${self}/secrets/drone-db-secrets.age";
+    mode = "600";
+    owner = "drone";
+  };
+
+  users.users.drone = {
+    description = "Drone Service";
+    home = "/var/lib/drone";
+    useDefaultShell = true;
+    uid = 994;
+    group = "drone";
+    isSystemUser = true;
+  };
+
+  users.groups.drone = {};
+
+  systemd.tmpfiles.rules = [
+    "d '/var/lib/drone-db' 0750 drone drone - -"
+  ];
+
+  systemd.services."docker-network-drone" = let
+    docker = config.virtualisation.oci-containers.backend;
+    dockerBin = "${pkgs.${docker}}/bin/${docker}";
+  in {
+    serviceConfig.Type = "oneshot";
+    before = ["docker-drone-server.service"];
+    script = ''
+      ${dockerBin} network inspect drone-net >/dev/null 2>&1 || ${dockerBin} network create drone-net --subnet 172.20.0.0/24
+    '';
+  };
+
+  virtualisation = {
+    docker = {
+      enable = true; # sadly podman is not supported rightnow
+      extraOptions = ''
+        --data-root /data/docker
+      '';
+    };
+
+    oci-containers = {
+      backend = "docker";
+      containers."drone-db" = {
+        image = "postgres:14";
+        autoStart = true;
+        user = "994";
+        volumes = [
+          "/var/lib/drone-db:/var/lib/postgresql/data"
+        ];
+        extraOptions = [
+          "--network=drone-net"
+        ];
+        environmentFiles = [
+          config.age.secrets.drone-db-secrets.path
+        ];
+      };
+      containers."drone-server" = {
+        image = "drone/drone:2";
+        autoStart = true;
+        user = "994";
+        ports = [
+          "4000:80"
+        ];
+        dependsOn = ["drone-db"];
+        extraOptions = [
+          "--network=drone-net"
+        ];
+        environment = {
+          DRONE_GITEA_SERVER = "https://git.pub.solar";
+          DRONE_SERVER_HOST = "ci.pub.solar";
+          DRONE_SERVER_PROTO = "https";
+          DRONE_DATABASE_DRIVER = "postgres";
+        };
+        environmentFiles = [
+          config.age.secrets.drone-secrets.path
+        ];
+      };
+      containers."drone-docker-runner" = {
+        image = "drone/drone-runner-docker:1";
+        autoStart = true;
+        # needs to run as root
+        #user = "994";
+        volumes = [
+          "/var/run/docker.sock:/var/run/docker.sock"
+        ];
+        dependsOn = ["drone-db"];
+        extraOptions = [
+          "--network=drone-net"
+        ];
+        environment = {
+          DRONE_RPC_HOST = "ci.pub.solar";
+          DRONE_RPC_PROTO = "https";
+          DRONE_RUNNER_CAPACITY = "2";
+          DRONE_RUNNER_NAME = "flora-6-docker-runner";
+        };
+        environmentFiles = [
+          config.age.secrets.drone-secrets.path
+        ];
+      };
+    };
+  };
+}

hosts/flora-6/flora-6.nix

@@ -0,0 +1,167 @@
+{
+  config,
+  latestModulesPath,
+  lib,
+  inputs,
+  pkgs,
+  profiles,
+  self,
+  ...
+}: let
+  psCfg = config.pub-solar;
+in {
+  imports = [
+    # Include the results of the hardware scan.
+    ./hardware-configuration.nix
+    ./triton-vmtools.nix
+
+    ./caddy.nix
+    ./drone.nix
+    ./keycloak.nix
+    ./gitea.nix
+    ./mailman.nix
+    ./owncast.nix
+
+    profiles.base-user
+    profiles.users.root # make sure to configure ssh keys
+    profiles.users.barkeeper
+
+    "${latestModulesPath}/services/misc/gitea.nix"
+    "${latestModulesPath}/services/web-servers/caddy/default.nix"
+  ];
+  disabledModules = [
+    "services/misc/gitea.nix"
+    "services/web-servers/caddy/default.nix"
+  ];
+
+  config = {
+    # # #
+    # # # pub.solar options
+    # # #
+    pub-solar.core = {
+      disk-encryption-active = false;
+      iso-options.enable = true;
+      lite = true;
+    };
+
+    # Allow sudo without a password for the barkeeper user
+    security.sudo.extraRules = [
+      {
+        users = ["${psCfg.user.name}"];
+        commands = [
+          {
+            command = "ALL";
+            options = ["NOPASSWD"];
+          }
+        ];
+      }
+    ];
+
+    # Override nix.conf for more agressive garbage collection
+    nix.extraOptions = lib.mkForce ''
+      min-free = 536870912
+      keep-outputs = false
+      keep-derivations = false
+      fallback = true
+    '';
+
+    # Machine user for CI pipelines
+    users.users.hakkonaut = {
+      description = "CI and automation user";
+      home = "/var/nix/iso-cache";
+      useDefaultShell = true;
+      uid = 998;
+      group = "hakkonaut";
+      isSystemUser = true;
+      openssh.authorizedKeys.keys = [
+        "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGP5MvCwNRtCcP1pSDrn0XZTNlpOqYnjHDm9/OI4hECW hakkonaut@flora-6"
+      ];
+    };
+
+    users.groups.hakkonaut = {};
+
+    # # #
+    # # # Triton host specific options
+    # # # DO NOT ALTER below this line, changes might render system unbootable
+    # # #
+
+    # Use the systemd-boot EFI boot loader.
+    boot.loader.systemd-boot.enable = true;
+    boot.loader.efi.canTouchEfiVariables = true;
+
+    # Force getting the hostname from cloud-init
+    networking.hostName = lib.mkDefault "";
+
+    # Set your time zone.
+    time.timeZone = "Europe/Berlin";
+
+    # Select internationalisation properties.
+    console = {
+      font = "Lat2-Terminus16";
+      keyMap = "us";
+    };
+
+    # List packages installed in system profile. To search, run:
+    # $ nix search wget
+    environment.systemPackages = with pkgs; [
+      git
+      vim
+      wget
+    ];
+
+    # Some programs need SUID wrappers, can be configured further or are
+    # started in user sessions.
+    # programs.mtr.enable = true;
+    # programs.gnupg.agent = {
+    #   enable = true;
+    #   enableSSHSupport = true;
+    # };
+
+    # List services that you want to enable:
+    services.cloud-init.enable = true;
+    services.cloud-init.ext4.enable = true;
+    services.cloud-init.network.enable = true;
+    # use the default NixOS cloud-init config, but add some SmartOS customization to it
+    environment.etc."cloud/cloud.cfg.d/90_smartos.cfg".text = ''
+      datasource_list: [ SmartOS ]
+
+      # Do not create the centos/ubuntu/debian user
+      users: [ ]
+
+      # mount second disk with label ephemeral0, gets formated by cloud-init
+      # this will fail to get added to /etc/fstab as it's read-only, but should
+      # mount at boot anyway
+      mounts:
+      - [ vdb, /data, auto, "defaults,nofail" ]
+    '';
+
+    # Enable the OpenSSH daemon.
+    services.openssh = {
+      enable = true;
+      settings = {
+        PasswordAuthentication = false;
+        PermitRootLogin = "no";
+        Macs = [
+          "hmac-sha2-512-etm@openssh.com"
+          "hmac-sha2-256-etm@openssh.com"
+          "umac-128-etm@openssh.com"
+          "hmac-sha2-512"
+          "hmac-sha2-256"
+          "umac-128@openssh.com"
+        ];
+      };
+    };
+
+    # We manage the firewall with nix, too
+    # altough triton can also manage firewall rules via the triton fwrule subcommand
+    networking.firewall.enable = true;
+
+    # This value determines the NixOS release from which the default
+    # settings for stateful data, like file locations and database versions
+    # on your system were taken. It‘s perfectly fine and recommended to leave
+    # this value at the release version of the first install of this system.
+    # Before changing this value read the documentation for this option
+    # (e.g. man configuration.nix or on https://nixos.org/nixos/options.html).
+    system.stateVersion = "22.05"; # Did you read the comment?
+  };
+}

hosts/flora-6/gitea.nix

@@ -0,0 +1,82 @@
+{
+  config,
+  lib,
+  pkgs,
+  self,
+  ...
+}: {
+  age.secrets.gitea-database-password = {
+    file = "${self}/secrets/gitea-database-password.age";
+    mode = "600";
+    owner = "gitea";
+  };
+  age.secrets.gitea-mailer-password = {
+    file = "${self}/secrets/gitea-mailer-password.age";
+    mode = "600";
+    owner = "gitea";
+  };
+
+  # gitea
+  services.gitea = {
+    enable = true;
+    package = pkgs.forgejo;
+    appName = "pub.solar git server";
+    database = {
+      type = "postgres";
+      passwordFile = config.age.secrets.gitea-database-password.path;
+    };
+    lfs.enable = true;
+    mailerPasswordFile = config.age.secrets.gitea-mailer-password.path;
+    settings = {
+      server = {
+        ROOT_URL = "https://git.pub.solar";
+        DOMAIN = "git.pub.solar";
+        HTTP_ADDR = "127.0.0.1";
+        HTTP_PORT = 3000;
+      };
+      mailer = {
+        ENABLED = true;
+        PROTOCOL = "smtps";
+        SMTP_ADDR = "mx2.greenbaum.cloud";
+        SMTP_PORT = 465;
+        FROM = ''"pub.solar git server" <gitea@pub.solar>'';
+        USER = "admins@pub.solar";
+      };
+      "repository.signing" = {
+        SIGNING_KEY = "default";
+        MERGES = "always";
+      };
+      openid = {
+        ENABLE_OPENID_SIGNIN = true;
+        ENABLE_OPENID_SIGNUP = true;
+      };
+      # uncomment after initial deployment, first user is admin user
+      # required to setup SSO (oauth openid-connect, keycloak auth provider)
+      service.ALLOW_ONLY_EXTERNAL_REGISTRATION = true;
+      service.ENABLE_NOTIFY_MAIL = true;
+      session.COOKIE_SECURE = lib.mkForce true;
+    };
+  };
+
+  # See: https://docs.gitea.io/en-us/signing/#installing-and-generating-a-gpg-key-for-gitea
+  # Required for gitea server side gpg signatures
+  # configured/setup manually in:
+  # /var/lib/gitea/data/home/.gitconfig
+  # /var/lib/gitea/data/home/.gnupg/
+  # sudo su gitea
+  # export GNUPGHOME=/var/lib/gitea/data/home/.gnupg
+  # gpg --quick-gen-key 'pub.solar gitea <gitea@pub.solar>' ed25519
+  # TODO: implement declarative GPG key generation and
+  # gitea gitconfig
+  programs.gnupg.agent = {
+    enable = true;
+    pinentryFlavor = "curses";
+  };
+  # Required to make gpg work without a graphical environment?
+  # otherwise generating a new gpg key fails with this error:
+  # gpg: agent_genkey failed: No pinentry
+  # see: https://github.com/NixOS/nixpkgs/issues/97861#issuecomment-827951675
+  environment.variables = {
+    GPG_TTY = "$(tty)";
+  };
+}

hosts/flora-6/hardware-configuration.nix

@@ -0,0 +1,44 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{
+  config,
+  lib,
+  pkgs,
+  modulesPath,
+  ...
+}: {
+  imports = [];
+
+  boot.initrd.availableKernelModules = ["ahci" "virtio_pci" "xhci_pci" "sr_mod" "virtio_blk"];
+  boot.initrd.kernelModules = [];
+  boot.kernelModules = [];
+  boot.extraModulePackages = [];
+
+  fileSystems."/" = {
+    device = "/dev/disk/by-label/nixos";
+    autoResize = true;
+    fsType = "ext4";
+  };
+
+  fileSystems."/boot" = {
+    device = "/dev/disk/by-label/boot";
+    fsType = "vfat";
+  };
+
+  fileSystems."/data" = {
+    device = "/dev/disk/by-label/ephemeral0";
+    fsType = "ext4";
+    options = [
+      "defaults"
+      "nofail"
+    ];
+  };
+
+  swapDevices = [];
+
+  networking.useDHCP = lib.mkDefault false;
+  networking.networkmanager.enable = lib.mkForce false;
+
+  hardware.cpu.intel.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
+}

hosts/flora-6/keycloak.nix

@@ -0,0 +1,30 @@
+{
+  config,
+  lib,
+  inputs,
+  pkgs,
+  self,
+  ...
+}: {
+  age.secrets.keycloak-database-password = {
+    file = "${self}/secrets/keycloak-database-password.age";
+    mode = "700";
+    #owner = "keycloak";
+  };
+
+  # keycloak
+  services.keycloak = {
+    enable = true;
+    database.passwordFile = config.age.secrets.keycloak-database-password.path;
+    settings = {
+      hostname = "auth.pub.solar";
+      http-host = "127.0.0.1";
+      http-port = 8080;
+      proxy = "edge";
+      features = "declarative-user-profile";
+    };
+    themes = {
+      "pub.solar" = inputs.keycloak-theme-pub-solar.legacyPackages.${pkgs.system}.keycloak-theme-pub-solar;
+    };
+  };
+}

hosts/flora-6/mailman.nix

@@ -0,0 +1,102 @@
+{
+  config,
+  lib,
+  pkgs,
+  self,
+  ...
+}: let
+  # Source: https://github.com/NixOS/nixpkgs/blob/nixos-22.11/nixos/modules/services/mail/mailman.nix#L9C10-L10
+  # webEnv is required by the mailman-uwsgi systemd service
+  inherit (pkgs.mailmanPackages.buildEnvs {}) webEnv;
+in {
+  networking.firewall.allowedTCPPorts = [25];
+
+  services.postfix = {
+    enable = true;
+    relayDomains = ["hash:/var/lib/mailman/data/postfix_domains"];
+    # get TLS certs for list.pub.solar from caddy
+    # TODO: when caddy renews certs, postfix doesn't know about it
+    # implement custom built caddy with events exec handler or systemd-reload
+    # hook so postfix reloads, too
+    sslCert = "/var/lib/caddy/.local/share/caddy/certificates/acme-v02.api.letsencrypt.org-directory/list.pub.solar/list.pub.solar.crt";
+    sslKey = "/var/lib/caddy/.local/share/caddy/certificates/acme-v02.api.letsencrypt.org-directory/list.pub.solar/list.pub.solar.key";
+    config = {
+      transport_maps = ["hash:/var/lib/mailman/data/postfix_lmtp"];
+      local_recipient_maps = ["hash:/var/lib/mailman/data/postfix_lmtp"];
+    };
+    rootAlias = "admins@pub.solar";
+    postmasterAlias = "admins@pub.solar";
+    hostname = "list.pub.solar";
+  };
+
+  systemd.paths.watcher-caddy-ssl-file = {
+    description = "Watches for changes in caddy's TLS cert file (after renewals) to reload postfix";
+    documentation = ["systemd.path(5)"];
+    partOf = ["postfix-reload.service"];
+    pathConfig = {
+      PathChanged = "/var/lib/caddy/.local/share/caddy/certificates/acme-v02.api.letsencrypt.org-directory/list.pub.solar/list.pub.solar.crt";
+      Unit = "postfix-reload.service";
+    };
+    wantedBy = ["multi-user.target"];
+  };
+
+  systemd.services."postfix-reload" = {
+    description = "Reloads postfix config, e.g. after TLS certs change, notified by watcher-caddy-ssl-file.path";
+    documentation = ["systemd.path(5)"];
+    requires = ["postfix.service"];
+    after = ["postfix.service"];
+    startLimitIntervalSec = 10;
+    startLimitBurst = 5;
+    serviceConfig.Type = "oneshot";
+    script = ''
+      ${pkgs.systemd}/bin/systemctl reload postfix
+    '';
+    wantedBy = ["multi-user.target"];
+  };
+
+  services.mailman = {
+    enable = true;
+    # We use caddy instead of nginx
+    #serve.enable = true;
+    hyperkitty.enable = true;
+    webHosts = ["list.pub.solar"];
+    siteOwner = "admins@pub.solar";
+  };
+
+  # TODO add django-keycloak as auth provider
+  # https://django-keycloak.readthedocs.io/en/latest/
+  ## Extend settings.py directly since this can't be done via JSON
+  ## settings (services.mailman.webSettings)
+  #environment.etc."mailman3/settings.py".text = ''
+  #  INSTALLED_APPS.extend([
+  #    "allauth.socialaccount.providers.github",
+  #    "allauth.socialaccount.providers.gitlab"
+  #  ])
+  #'';
+
+  systemd.services.mailman-uwsgi = let
+    uwsgiConfig.uwsgi = {
+      type = "normal";
+      plugins = ["python3"];
+      home = webEnv;
+      manage-script-name = true;
+      mount = "/=mailman_web.wsgi:application";
+      http = "127.0.0.1:18507";
+    };
+    uwsgiConfigFile = pkgs.writeText "uwsgi-mailman.json" (builtins.toJSON uwsgiConfig);
+  in {
+    wantedBy = ["multi-user.target"];
+    after = ["postgresql.service"];
+    requires = ["mailman-web-setup.service" "postgresql.service"];
+    restartTriggers = [config.environment.etc."mailman3/settings.py".source];
+    serviceConfig = {
+      # Since the mailman-web settings.py obstinately creates a logs
+      # dir in the cwd, change to the (writable) runtime directory before
+      # starting uwsgi.
+      ExecStart = "${pkgs.coreutils}/bin/env -C $RUNTIME_DIRECTORY ${pkgs.uwsgi.override {plugins = ["python3"];}}/bin/uwsgi --json ${uwsgiConfigFile}";
+      User = "mailman-web";
+      Group = "mailman";
+      RuntimeDirectory = "mailman-uwsgi";
+    };
+  };
+}

hosts/flora-6/owncast.nix

@@ -0,0 +1,24 @@
+{
+  config,
+  lib,
+  pkgs,
+  self,
+  ...
+}: {
+  # owncast
+  services.owncast = {
+    enable = true;
+    user = "owncast";
+    group = "owncast";
+    # The directory where owncast stores its data files.
+    dataDir = "/var/lib/owncast";
+    # Open the appropriate ports in the firewall for owncast.
+    openFirewall = true;
+    # The IP address to bind the owncast web server to.
+    listen = "127.0.0.1";
+    # TCP port where owncast rtmp service listens.
+    rtmp-port = 1935;
+    # TCP port where owncast web-gui listens.
+    port = 5000;
+  };
+}

hosts/flora-6/triton-vmtools.nix

@@ -0,0 +1,9 @@
+{
+  pkgs,
+  inputs,
+  ...
+}: {
+  environment.systemPackages = with pkgs; [
+    inputs.triton-vmtools.packages.${pkgs.system}.default
+  ];
+}

hosts/nougat-2/acme.nix

@@ -0,0 +1,79 @@
+{
+  config,
+  lib,
+  pkgs,
+  self,
+  ...
+}: let
+  exDomain = (import ./ex-domain.nix) lib;
+  pubsolarDomain = import ./pubsolar-domain.nix;
+
+  hostingdeProviderConf = {
+    dnsProvider = "hostingde";
+    credentialsFile = "${pkgs.writeText "hostingde-creds" ''
+      HOSTINGDE_API_KEY_FILE=${config.age.secrets."hosting.de-api-key.age".path}
+    ''}";
+  };
+in {
+  age.secrets."hosting.de-api-key.age" = {
+    file = "${self}/secrets/hosting.de-api-key.age";
+    mode = "440";
+    group = "acme";
+  };
+
+  systemd.tmpfiles.rules = [
+    "d '/data/acme' 0750 root acme - -"
+  ];
+
+  users.groups.acme = {};
+  ids.uids.acme = 997;
+  ids.gids.acme = 997;
+
+  containers.acme = {
+    autoStart = true;
+    privateNetwork = true;
+    hostAddress = "192.168.101.0";
+    localAddress = "192.168.106.0";
+    hostAddress6 = "fc00::1";
+    localAddress6 = "fc00::6";
+
+    bindMounts = {
+      "/var/lib/acme" = {
+        hostPath = "/data/acme";
+        isReadOnly = false;
+      };
+
+      "${config.age.secrets."hosting.de-api-key.age".path}" = {
+        hostPath = "${config.age.secrets."hosting.de-api-key.age".path}";
+        isReadOnly = true;
+      };
+    };
+
+    config = {
+      networking.nameservers = ["1.1.1.1"];
+      users.groups.acme = config.users.groups.acme;
+
+      security.acme = {
+        acceptTerms = true;
+        defaults.email = "acme@benjaminbaedorf.eu";
+        defaults.server = "https://acme-staging-v02.api.letsencrypt.org/directory";
+        defaults.group = "acme";
+
+        certs."b12f.io" = hostingdeProviderConf;
+        certs."mail.b12f.io" = hostingdeProviderConf;
+        certs."transmission.b12f.io" = hostingdeProviderConf;
+
+        certs."${exDomain}" = hostingdeProviderConf;
+        certs."mail.${exDomain}" = hostingdeProviderConf;
+
+        certs."${pubsolarDomain}" = hostingdeProviderConf;
+        certs."www.${pubsolarDomain}" = hostingdeProviderConf;
+        certs."auth.${pubsolarDomain}" = hostingdeProviderConf;
+        certs."git.${pubsolarDomain}" = hostingdeProviderConf;
+        certs."ci.${pubsolarDomain}" = hostingdeProviderConf;
+        certs."list.${pubsolarDomain}" = hostingdeProviderConf;
+        certs."obs-portal.${pubsolarDomain}" = hostingdeProviderConf;
+      };
+    };
+  };
+}

hosts/nougat-2/caddy.nix

@@ -0,0 +1,181 @@
+{
+  config,
+  lib,
+  pkgs,
+  self,
+  ...
+}: let
+  pubsolarDomain = import ./pubsolar-domain.nix;
+  # Machine user for CI pipelines
+in {
+  networking.firewall.allowedTCPPorts = [80 443];
+  networking.networkmanager.unmanaged = ["interface-name:ve-caddy"];
+  networking.nat = {
+    enable = true;
+
+    internalInterfaces = ["ve-caddy"];
+    externalInterface = "enp0s31f6";
+
+    # Lazy IPv6 connectivity for the container
+    enableIPv6 = true;
+  };
+
+  systemd.tmpfiles.rules = [
+    "d '/data/www' 0750 root www - -"
+    "d '/data/caddy' 0750 root caddy - -"
+  ];
+
+  users.groups.caddy = {};
+  users.groups.www = {};
+  users.users.hakkonaut.extraGroups = ["www"];
+  ids.uids.www = 996;
+  ids.gids.www = 996;
+
+  fileSystems."/var/lib/caddy" = {
+    device = "/data/caddy";
+    options = ["bind"];
+  };
+
+  fileSystems."/srv/www" = {
+    device = "/data/www";
+    options = ["bind"];
+  };
+
+  containers.caddy = {
+    autoStart = true;
+    privateNetwork = true;
+    hostAddress = "192.168.101.0";
+    localAddress = "192.168.103.0";
+    hostAddress6 = "fc00::1";
+    localAddress6 = "fc00::3";
+
+    forwardPorts = [
+      {
+        containerPort = 443;
+        hostPort = 443;
+        protocol = "tcp";
+      }
+      {
+        containerPort = 80;
+        hostPort = 80;
+        protocol = "tcp";
+      }
+    ];
+
+    bindMounts = {
+      "/srv/www/" = {
+        hostPath = "/data/www";
+        isReadOnly = false;
+      };
+
+      "/var/lib/caddy/" = {
+        hostPath = "/data/caddy";
+        isReadOnly = false;
+      };
+
+      "/var/lib/caddy/.local/share/caddy/certificates/acme-v02.api.letsencrypt.org-directory" = {
+        hostPath = "/data/acme";
+        isReadOnly = false;
+      };
+    };
+
+    config = {
+      users.groups.caddy = {};
+      users.groups.www = {};
+      users.groups.acme = {};
+      users.users.caddy.extraGroups = ["www" "acme"];
+
+      networking.firewall.allowedTCPPorts = [80 443];
+      environment.etc."resolv.conf".text = "nameserver 1.1.1.0";
+
+      services.caddy = {
+        enable = lib.mkForce true;
+
+        globalConfig = lib.mkForce ''
+          auto_https disable_certs
+        '';
+
+        virtualHosts = {
+          "dashboard.nougat-2.b12f.io" = {
+            extraConfig = ''
+              reverse_proxy :2019
+            '';
+          };
+          "www.b12f.io" = {
+            extraConfig = ''
+              redir https://pub.solar{uri}
+            '';
+          };
+          "mail.b12f.io" = {
+            extraConfig = ''
+              redir / /realms/pub.solar/account temporary
+              reverse_proxy :8080
+            '';
+          };
+
+          "${pubsolarDomain}" = {
+            logFormat = lib.mkForce ''
+              output discard
+            '';
+            extraConfig = ''
+              # PubSolarOS images
+              handle /os/download/* {
+                root * /srv/www
+                file_server /os/download/* browse
+              }
+              # serve base domain pub.solar for mastodon.pub.solar
+              # https://masto.host/mastodon-usernames-different-from-the-domain-used-for-installation/
+              handle /.well-known/host-meta {
+                redir https://mastodon.${pubsolarDomain}{uri}
+              }
+              # pub.solar website
+              handle {
+                root * /srv/www/pub.solar
+                try_files {path}.html {path}
+                file_server
+              }
+              # minimal error handling, respond with status code and text
+              handle_errors {
+                respond "{http.error.status_code} {http.error.status_text}"
+              }
+            '';
+          };
+          "www.${pubsolarDomain}" = {
+            logFormat = lib.mkForce ''
+              output discard
+            '';
+            extraConfig = ''
+              redir https://${pubsolarDomain}{uri}
+            '';
+          };
+          "auth.${pubsolarDomain}" = {
+            logFormat = lib.mkForce ''
+              output discard
+            '';
+            extraConfig = ''
+              redir / /realms/${pubsolarDomain}/account temporary
+              reverse_proxy 192.168.104.0:8080
+            '';
+          };
+          "git.${pubsolarDomain}" = {
+            logFormat = lib.mkForce ''
+              output discard
+            '';
+            extraConfig = ''
+              redir /user/login /user/oauth2/keycloak temporary
+              reverse_proxy 192.168.105.0:3000
+            '';
+          };
+          "ci.${pubsolarDomain}" = {
+            logFormat = lib.mkForce ''
+              output discard
+            '';
+            extraConfig = ''
+              reverse_proxy 192.168.101.0:8080
+            '';
+          };
+        };
+      };
+    };
+  };
+}

hosts/nougat-2/concourse.nix

@@ -0,0 +1,137 @@
+{
+  config,
+  lib,
+  pkgs,
+  self,
+  ...
+}: let
+  pubsolarDomain = import ./pubsolar-domain.nix;
+
+  getSecret = name:
+    lib.attrsets.setAttrByPath [name] {
+      file = "${self}/secrets/${name}.age";
+      mode = "600";
+      owner = "concourse";
+    };
+
+  keys = [
+    "concourse-session-signing-key"
+    "concourse-worker-key"
+    "concourse-tsa-host-key"
+  ];
+
+  secrets =
+    [
+      "concourse-secrets"
+      "concourse-db-secrets"
+    ]
+    ++ keys;
+in {
+  age.secrets = lib.lists.foldl (a: b: a // getSecret b) {} secrets;
+
+  users.users.concourse = {
+    description = "Concourse Service";
+    home = "/var/lib/concourse";
+    useDefaultShell = true;
+    group = "concourse";
+    isSystemUser = true;
+  };
+
+  users.groups.concourse = {};
+  users.groups.postgres = {};
+  ids.uids.concourse = 995;
+  ids.gids.concourse = 995;
+
+  systemd.tmpfiles.rules = [
+    "d '/data/concourse/db' 0770 root postgres - -"
+  ];
+
+  system.activationScripts.mkConcourseNet = let
+    docker = config.virtualisation.oci-containers.backend;
+    dockerBin = "${pkgs.${docker}}/bin/${docker}";
+  in ''
+    ${dockerBin} network inspect concourse-net >/dev/null 2>&1 || ${dockerBin} network create concourse-net --subnet 172.20.0.0/24
+  '';
+
+  containers.concourse = {
+    autoStart = true;
+    privateNetwork = true;
+    hostAddress = "192.168.101.0";
+    localAddress = "192.168.107.0";
+    hostAddress6 = "fc00::1";
+    localAddress6 = "fc00::7";
+
+    bindMounts = {
+      "/var/lib/postgresql/14" = {
+        hostPath = "/data/concourse/db";
+        isReadOnly = false;
+      };
+
+      "${config.age.secrets.keycloak-database-password.path}" = {
+        hostPath = "${config.age.secrets.keycloak-database-password.path}";
+        isReadOnly = true;
+      };
+    };
+
+    config = {
+      networking.nameservers = ["1.1.1.1"];
+
+  virtualisation.oci-containers = {
+    containers."concourse-db" = {
+      image = "postgres:14";
+      autoStart = true;
+      user = builtins.toString config.ids.uids.postgres;
+      volumes = [
+        "/data/concourse/db:/var/lib/postgresql/data"
+      ];
+      extraOptions = [
+        "--network=concourse-net"
+      ];
+      environmentFiles = [
+        config.age.secrets.concourse-db-secrets.path
+      ];
+    };
+
+    containers."concourse" = {
+      image = "concourse/concourse:7.9.1";
+      autoStart = true;
+      user = builtins.toString config.ids.uids.concourse;
+      ports = [
+        "8080:8080"
+      ];
+      dependsOn = ["concourse-db"];
+      extraOptions = [
+        "--network=concourse-net"
+      ];
+      volumes = [
+        "${config.age.secrets.concourse-session-signing-key.path}:/keys/session_signing_key"
+        "${config.age.secrets.concourse-worker-key.path}:/keys/worker_key"
+        "${config.age.secrets.concourse-tsa-host-key.path}:/keys/tsa_host_key"
+      ];
+
+      environment = {
+        CONCOURSE_EXTERNAL_URL = "https://ci.${pubsolarDomain}";
+
+        CONCOURSE_ADD_LOCAL_USER = "crew:changeme";
+        CONCOURSE_MAIN_TEAM_LOCAL_USER = "crew";
+
+        # instead of relying on the default "detect"
+        CONCOURSE_WORKER_BAGGAGECLAIM_DRIVER = "overlay";
+        CONCOURSE_X_FRAME_OPTIONS = "allow";
+        CONCOURSE_CONTENT_SECURITY_POLICY = "*";
+        CONCOURSE_CLUSTER_NAME = "pub.solar";
+        CONCOURSE_WORKER_CONTAINERD_DNS_SERVER = "8.8.8.8";
+
+        CONCOURSE_SESSION_SIGNING_KEY = "/keys/session_signing_key";
+        CONCOURSE_TSA_HOST_KEY = "/keys/tsa_host_key";
+        CONCOURSE_TSA_AUTHORIZED_KEYS = "/keys/worker_key";
+
+        # For ARM-based machine, change the Concourse runtime to "houdini"
+        CONCOURSE_WORKER_RUNTIME = "containerd";
+      };
+      environmentFiles = [
+        config.age.secrets.concourse-secrets.path
+      ];
+    };
+  };
+}

hosts/nougat-2/configuration.nix

@@ -0,0 +1,136 @@
+# Edit this configuration file to define what should be installed on
+# your system.  Help is available in the configuration.nix(5) man page
+# and in the NixOS manual (accessible by running ‘nixos-help’).
+{
+  config,
+  pkgs,
+  lib,
+  ...
+}: let
+  psCfg = config.pub-solar;
+in {
+  imports = [
+    # Include the results of the hardware scan.
+    ./hardware-configuration.nix
+  ];
+
+  boot.loader.systemd-boot.enable = true;
+  boot.loader.efi.canTouchEfiVariables = true;
+  boot.kernelParams = [
+    "boot.shell_on_fail=1"
+    "ip=135.181.179.123::135.181.179.65:255.255.255.192:nougat-2.b12f.io::off"
+  ];
+  networking.hostName = "nougat-2";
+
+  # The mdadm RAID1s were created with 'mdadm --create ... --homehost=hetzner',
+  # but the hostname for each machine may be different, and mdadm's HOMEHOST
+  # setting defaults to '<system>' (using the system hostname).
+  # This results mdadm considering such disks as "foreign" as opposed to
+  # "local", and showing them as e.g. '/dev/md/hetzner:root0'
+  # instead of '/dev/md/root0'.
+  # This is mdadm's protection against accidentally putting a RAID disk
+  # into the wrong machine and corrupting data by accidental sync, see
+  # https://bugzilla.redhat.com/show_bug.cgi?id=606481#c14 and onward.
+  # We do not worry about plugging disks into the wrong machine because
+  # we will never exchange disks between machines, so we tell mdadm to
+  # ignore the homehost entirely.
+  environment.etc."mdadm.conf".text = ''
+    HOMEHOST <ignore>
+    ARRAY /dev/md/SSD metadata=1.2 name=nixos:SSD UUID=f8189c09:cb247cc7:22b79b5f:df888705
+    ARRAY /dev/md/HDD metadata=1.2 name=nixos:HDD UUID=85ed8a8e:9ddc5f09:c6ef6110:c00728fa
+  '';
+  # The RAIDs are assembled in stage1, so we need to make the config
+  # available there.
+  boot.initrd.services.swraid.enable = true;
+  boot.initrd.services.swraid.mdadmConf = config.environment.etc."mdadm.conf".text;
+
+  boot.initrd.network.enable = true;
+  boot.initrd.network.ssh = {
+    enable = true;
+    port = 22;
+    authorizedKeys =
+      if psCfg.user.publicKeys != null
+      then psCfg.user.publicKeys
+      else [];
+    hostKeys = ["/etc/secrets/initrd/ssh_host_ed25519_key"];
+  };
+
+  # Network (Hetzner uses static IP assignments, and we don't use DHCP here)
+  networking.useDHCP = false;
+  networking.interfaces."enp0s31f6".ipv4.addresses = [
+    {
+      address = "135.181.179.123";
+      prefixLength = 26;
+    }
+  ];
+  networking.defaultGateway = "135.181.179.65";
+
+  networking.interfaces."enp0s31f6".ipv6.addresses = [
+    {
+      address = "2a01:4f9:3a:2170::1";
+      prefixLength = 64;
+    }
+  ];
+  networking.defaultGateway6 = {
+    address = "fe80::1";
+    interface = "enp0s31f6";
+  };
+
+  networking.nameservers = ["1.1.1.1"];
+
+  # Initial empty root password for easy login:
+  users.users.root.initialHashedPassword = "";
+  users.users.root.openssh.authorizedKeys.keys =
+    if psCfg.user.publicKeys != null
+    then psCfg.user.publicKeys
+    else [];
+
+  users.users.hakkonaut = {
+    home = "/home/hakkonaut";
+    description = "CI and automation user";
+    useDefaultShell = true;
+    group = "hakkonaut";
+    isSystemUser = true;
+    openssh.authorizedKeys.keys = [
+      "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGP5MvCwNRtCcP1pSDrn0XZTNlpOqYnjHDm9/OI4hECW hakkonaut@flora-6"
+    ];
+  };
+
+  users.groups.hakkonaut = {};
+  ids.uids.hakkonaut = 998;
+  ids.gids.hakkonaut = 998;
+
+  services.openssh.enable = true;
+  services.openssh.settings.PermitRootLogin = "prohibit-password";
+
+  pub-solar.core.disk-encryption-active = false;
+  pub-solar.core.lite = true;
+
+  virtualisation = {
+    docker = {
+      enable = true;
+    };
+
+    oci-containers = {
+      backend = "docker";
+    };
+  };
+
+  security.sudo.extraRules = [
+    {
+      users = ["${psCfg.user.name}"];
+      commands = [
+        {
+          command = "ALL";
+          options = ["NOPASSWD"];
+        }
+      ];
+    }
+  ];
+
+  # This value determines the NixOS release with which your system is to be
+  # compatible, in order to avoid breaking some software such as database
+  # servers. You should change this only after NixOS release notes say you
+  # should.
+  system.stateVersion = "23.05"; # Did you read the comment?
+}

hosts/nougat-2/default.nix

@@ -0,0 +1,5 @@
+{...}: {
+  imports = [
+    ./nougat-2.nix
+  ];
+}

hosts/nougat-2/ex-domain.nix

@@ -0,0 +1 @@
+lib: lib.concatStrings (lib.lists.reverseList ["et" ".n" "zz" "wd" "h"])

hosts/nougat-2/gitea.nix

@@ -0,0 +1,124 @@
+{
+  config,
+  lib,
+  pkgs,
+  self,
+  ...
+}: let
+  pubsolarDomain = import ./pubsolar-domain.nix;
+in {
+  age.secrets.gitea-database-password = {
+    file = "${self}/secrets/gitea-database-password.age";
+    mode = "600";
+    group = "gitea";
+  };
+
+  # age.secrets.gitea-mailer-password = {
+  #   file = "${self}/secrets/gitea-mailer-password.age";
+  #   mode = "600";
+  #   owner = "gitea";
+  # };
+
+  systemd.tmpfiles.rules = [
+    "d '/data/gitea/db' 0770 root postgres - -"
+    "d '/data/gitea/gitea' 0770 root gitea - -"
+  ];
+
+  users.groups.postgres = {};
+  users.groups.gitea = {};
+  ids.uids.gitea = 994;
+  ids.gids.gitea = 994;
+
+  containers.gitea = {
+    autoStart = true;
+    privateNetwork = true;
+    hostAddress = "192.168.101.0";
+    localAddress = "192.168.105.0";
+    hostAddress6 = "fc00::1";
+    localAddress6 = "fc00::5";
+
+    bindMounts = {
+      "/var/lib/postgresql/14" = {
+        hostPath = "/data/gitea/db";
+        isReadOnly = false;
+      };
+
+      "/var/lib/gitea" = {
+        hostPath = "/data/gitea/gitea";
+        isReadOnly = false;
+      };
+
+      "${config.age.secrets.gitea-database-password.path}" = {
+        hostPath = "${config.age.secrets.gitea-database-password.path}";
+        isReadOnly = true;
+      };
+    };
+
+    config = {
+      networking.nameservers = ["1.1.1.1"];
+
+      services.gitea = {
+        enable = true;
+        package = pkgs.forgejo;
+        appName = "pub.solar git server";
+        database = {
+          type = "postgres";
+          passwordFile = config.age.secrets.gitea-database-password.path;
+        };
+        lfs.enable = true;
+        # mailerPasswordFile = config.age.secrets.gitea-mailer-password.path;
+        settings = {
+          server = {
+            DOMAIN = "git.${pubsolarDomain}";
+            HTTP_ADDR = "127.0.0.1";
+            HTTP_PORT = 3000;
+            ROOT_URL = "https://git.${pubsolarDomain}";
+          };
+          mailer = {
+            ENABLED = false;
+            PROTOCOL = "smtps";
+            SMTP_ADDR = "mx2.greenbaum.cloud";
+            SMTP_PORT = 465;
+            FROM = ''"pub.solar git server" <gitea@pub.solar>'';
+            USER = "admins@pub.solar";
+          };
+          "repository.signing" = {
+            SIGNING_KEY = "default";
+            MERGES = "always";
+          };
+          openid = {
+            ENABLE_OPENID_SIGNIN = true;
+            ENABLE_OPENID_SIGNUP = true;
+          };
+          # uncomment after initial deployment, first user is admin user
+          # required to setup SSO (oauth openid-connect, keycloak auth provider)
+          service.ALLOW_ONLY_EXTERNAL_REGISTRATION = true;
+          service.ENABLE_NOTIFY_MAIL = true;
+          session.COOKIE_SECURE = lib.mkForce true;
+        };
+      };
+
+      # See: https://docs.gitea.io/en-us/signing/#installing-and-generating-a-gpg-key-for-gitea
+      # Required for gitea server side gpg signatures
+      # configured/setup manually in:
+      # /var/lib/gitea/data/home/.gitconfig
+      # /var/lib/gitea/data/home/.gnupg/
+      # sudo su gitea
+      # export GNUPGHOME=/var/lib/gitea/data/home/.gnupg
+      # gpg --quick-gen-key 'pub.solar gitea <gitea@pub.solar>' ed25519
+      # TODO: implement declarative GPG key generation and
+      # gitea gitconfig
+      programs.gnupg.agent = {
+        enable = true;
+        pinentryFlavor = "curses";
+      };
+      # Required to make gpg work without a graphical environment?
+      # otherwise generating a new gpg key fails with this error:
+      # gpg: agent_genkey failed: No pinentry
+      # see: https://github.com/NixOS/nixpkgs/issues/97861#issuecomment-827951675
+      environment.variables = {
+        GPG_TTY = "$(tty)";
+      };
+    };
+  };
+}

hosts/nougat-2/hardware-configuration.nix

@@ -0,0 +1,64 @@
+{
+  config,
+  lib,
+  pkgs,
+  modulesPath,
+  ...
+}: {
+  imports = [
+    (modulesPath + "/installer/scan/not-detected.nix")
+  ];
+
+  boot.initrd.availableKernelModules = [
+    "dm-snapshot"
+    "xhci_pci"
+    "ahci"
+    "nvme"
+    "usbhid"
+    "usb_storage"
+    "sd_mod"
+    "dm-raid"
+    "e1000e"
+  ];
+  boot.initrd.kernelModules = [];
+  boot.kernelModules = ["kvm-intel"];
+  boot.extraModulePackages = [];
+
+  boot.initrd.luks.devices."ssd" = {
+    device = "/dev/disk/by-id/md-uuid-f8189c09:cb247cc7:22b79b5f:df888705";
+  };
+
+  boot.initrd.luks.devices."hdd" = {
+    device = "/dev/disk/by-id/md-uuid-85ed8a8e:9ddc5f09:c6ef6110:c00728fa";
+  };
+
+  fileSystems."/" = {
+    device = "/dev/disk/by-uuid/cb88e8b9-be51-43eb-a51a-cd021c90771c";
+    fsType = "ext4";
+  };
+
+  fileSystems."/boot" = {
+    device = "/dev/disk/by-uuid/3F6D-065E";
+    fsType = "vfat";
+  };
+
+  fileSystems."/data" = {
+    device = "/dev/disk/by-uuid/824341f0-fd56-4db7-bb7e-4f161d94144b";
+    fsType = "ext4";
+  };
+
+  swapDevices = [
+    {device = "/dev/disk/by-uuid/f37e9f96-0174-4cac-a0bb-b63b2a67a4ad";}
+  ];
+
+  # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
+  # (the default) this is the recommended approach. When using systemd-networkd it's
+  # still possible to use this option, but it's recommended to use it in conjunction
+  # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
+  networking.useDHCP = lib.mkDefault true;
+  # networking.interfaces.eno1.useDHCP = lib.mkDefault true;
+
+  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
+  powerManagement.cpuFreqGovernor = lib.mkDefault "powersave";
+  hardware.cpu.intel.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
+}

hosts/nougat-2/keycloak.nix

@@ -0,0 +1,64 @@
+{
+  config,
+  lib,
+  inputs,
+  pkgs,
+  self,
+  ...
+}: let
+  pubsolarDomain = import ./pubsolar-domain.nix;
+in {
+  age.secrets.keycloak-database-password = {
+    file = "${self}/secrets/keycloak-database-password.age";
+    mode = "770";
+    group = "keycloak";
+  };
+
+  systemd.tmpfiles.rules = [
+    "d '/data/keycloak/db' 0770 root postgres - -"
+  ];
+
+  users.groups.postgres = {};
+  users.groups.keycloak = {};
+  ids.uids.keycloak = 993;
+  ids.gids.keycloak = 993;
+
+  containers.keycloak = {
+    autoStart = true;
+    privateNetwork = true;
+    hostAddress = "192.168.101.0";
+    localAddress = "192.168.104.0";
+    hostAddress6 = "fc00::1";
+    localAddress6 = "fc00::4";
+
+    bindMounts = {
+      "/var/lib/postgresql/14" = {
+        hostPath = "/data/keycloak/db";
+        isReadOnly = false;
+      };
+
+      "${config.age.secrets.keycloak-database-password.path}" = {
+        hostPath = "${config.age.secrets.keycloak-database-password.path}";
+        isReadOnly = true;
+      };
+    };
+
+    config = {
+      networking.nameservers = ["1.1.1.1"];
+
+      services.keycloak = {
+        enable = true;
+        database.passwordFile = config.age.secrets.keycloak-database-password.path;
+        settings = {
+          hostname = "auth.${pubsolarDomain}";
+          http-host = "0.0.0.0";
+          http-port = 8080;
+          proxy = "edge";
+        };
+        themes = {
+          "pub.solar" = inputs.keycloak-theme-pub-solar.legacyPackages.${pkgs.system}.keycloak-theme-pub-solar;
+        };
+      };
+    };
+  };
+}

hosts/nougat-2/nougat-2.nix

@@ -0,0 +1,29 @@
+{
+  config,
+  pkgs,
+  lib,
+  self,
+  profiles,
+  fix-atomic-container-restartsModulesPath,
+  ...
+}:
+with lib; let
+  psCfg = config.pub-solar;
+  xdg = config.home-manager.users."${psCfg.user.name}".xdg;
+in {
+  imports = [
+    ./configuration.nix
+
+    profiles.base-user
+    profiles.users.root # make sure to configure ssh keys
+    profiles.users.barkeeper
+
+
+    ./acme.nix
+    ./caddy.nix
+    ./keycloak.nix
+    ./gitea.nix
+    # ./concourse.nix
+    # "${fix-atomic-container-restartsModulesPath}/virtualisation/nixos-containers.nix"
+  ];
+}

hosts/nougat-2/pubsolar-domain.nix

@@ -0,0 +1 @@
+"pub.solar.b12f.io"

modules/terminal-life/default.nix

@@ -25,6 +25,11 @@ in {
     programs.command-not-found.enable = false;
 
     environment.systemPackages = with pkgs; [
+      ack
+      bat
+      exa
+      fd
+      neovim
       screen
     ];
 

pkgs/_sources/generated.nix

@@ -3,17 +3,17 @@
 {
   blesh-nvfetcher = {
     pname = "blesh-nvfetcher";
-    version = "1afc616b890e487926897e5b3e3a33e0ad833cb3";
+    version = "4089c4e1cb411121472180189953664b978d8972";
     src = fetchFromGitHub {
       owner = "akinomyoga";
       repo = "ble.sh";
-      rev = "1afc616b890e487926897e5b3e3a33e0ad833cb3";
+      rev = "4089c4e1cb411121472180189953664b978d8972";
       fetchSubmodules = true;
       deepClone = false;
       leaveDotGit = true;
-      sha256 = "sha256-gDxx7nDleS2HWIJWc208gcTzuRUEu/JCyepTuOJDAGo=";
+      sha256 = "sha256-ZLkiBm3vsRe42crLffM9Z8F5yzKvNRV2/AqK9RkuU+8=";
     };
-    date = "2023-06-29";
+    date = "2023-07-18";
   };
   instant-nvim-nvfetcher = {
     pname = "instant-nvim-nvfetcher";

profiles/base-user/session-variables.nix

@@ -7,6 +7,13 @@
   psCfg = config.pub-solar;
   wlroots = psCfg.graphical.wayland;
   xdg = config.home-manager.users."${psCfg.user.name}".xdg;
+  globalVariables = {
+    EDITOR = "/run/current-system/sw/bin/nvim";
+    VISUAL = "/run/current-system/sw/bin/nvim";
+
+    # Make sure virsh runs without root
+    LIBVIRT_DEFAULT_URI = "qemu:///system";
+  };
   variables = {
     XDG_CONFIG_HOME = xdg.configHome;
     XDG_CACHE_HOME = xdg.cacheHome;
@@ -25,11 +32,8 @@
       then "pixman"
       else "gles2";
 
-    EDITOR = "/etc/profiles/per-user/${psCfg.user.name}/bin/nvim";
-    VISUAL = "/etc/profiles/per-user/${psCfg.user.name}/bin/nvim";
-
     # fix "xdg-open fork-bomb" your preferred browser from here
-    BROWSER = "${pkgs.firefox-wayland}/bin/firefox";
+    BROWSER = "firefox";
 
     # node
     NODE_REPL_HISTORY = "${xdg.dataHome}/node_repl_history";
@@ -41,9 +45,6 @@
     NPM_CONFIG_CACHE = "${xdg.configHome}/npm";
     # TODO: used to be XDG_RUNTIME_DIR NPM_CONFIG_TMP = "/tmp/npm";
 
-    # Make sure virsh runs without root
-    LIBVIRT_DEFAULT_URI = "qemu:///system";
-
     # wine
     WINEPREFIX = "${xdg.dataHome}/wineprefixes/default";
 
@@ -103,5 +104,5 @@ in {
     systemd.user.sessionVariables = variablesWithMeta;
   };
 
-  environment.variables = variablesWithMeta;
+  environment.variables = globalVariables;
 }

secrets/drone-db-secrets.age

@@ -0,0 +1,24 @@
+age-encryption.org/v1
+-> ssh-ed25519 Y0ZZaw 4IYDRUd6wQzWDLzyFLPzy/t8L1V1UT/KwgfMLvDn5GQ
+4lKiqrafTVNtmcbbWdDsEkPSaN0/1Ud1k+rW1p0Wi0I
+-> ssh-ed25519 BVsyTA 5kVXS829ZZONa7iqxXQXpcQ4eoKEH14Aah6Oo6plWjE
+Rv06OqEOnVjrlwBy8JtfV+v+arbqrO2Cv6paIx0Bzf0
+-> ssh-rsa kFDS0A
+dvnz72ZtLpuBJfJEoXBb9TyQuEtNK1VZBXtSq9X7RL4fg4rbvGWRiGwl+IH1u8Tp
+6TPephD603lGkxhglh9KlmVp7vqZ4ILRN9836b4Ic0kPttK7iCWoRQsLzHMpwQyD
+Lb8ViRCIj+a2ZWfThaxjSXjXgDR5ZJGrnSwHhxsK9R3A4YUhT8jLvVRCfrUYhtPu
+9AhhT3P9FceflBJzIrD0lozYyaFi5eV4dySAgGyuuBzPXmdWZiOoEbArV9M/N1ss
+LJ/Nf23Ki39qe8w6YelcbhZTi6D2zfOA34Rd+QO8xzZFKKk0iZVSJ0ODk0I42itY
+rxiOQFX3Mpv2/FoqOzJY9WeFHHw61pfZid5UkjFLaongel60a0QSrJvhNoz8J5Jp
+k1GitKbBJl9V69XDY8RqQyDspImOkf7M7497C3OjdUtQkzC2cHzIfbDIi36Oifyp
+254KLVyCtArCqKZClnwcXAl9KtP4d9FM1TL7vSsJM67wfSpWakm5gptSM3WyYsZy
+Y2NkVU/Mk1AQLyrYKz+jtEwTmcrGo8zUFwKQZrXkytNV+vlWxwUAjZupNef5Ih7F
+6okWpmRTjozIZzdJgAHSJ96nnbu5QZt0GmmJ+LtCfIZ+1W3M156hODwCaN9Qg6Ki
+30MR/njAjWE7o5TB+gI6iV2OYxd0/Yqyy9lIdEEllFk
+-> ssh-ed25519 cakP9w usfEkp11W/3dAIKp0EcTL0NJe8cMHLJamShjSEbc+EA
+cVK/YSYLIvcXeZWpqEUHkwufHxOIR4uYOvZGi2eMNz4
+-> ym-grease
+INN4gag+EYKDUsKFd8N2CrLBWtRGC/BKP4IEfaAt4a5L1FFUAeEaRRAfuQ5ZWvnG
+vV0T8ASMwsMJ07X2X2faghc
+--- 7BrGKq+40E31/Jz0C6jg7Jequ0k4W+71wjLGLdR+9vg
+†Ì"	чdRY$†ñ œîŽÌÚZüä“w2«	£AKà MŒÜ./ÀžÌàU!ûÐÖj'$¡û¥šâ 8n`<60>û5ÛDØñ0J<Û<>¸ßˆëÂ*¹YÎ>“†›³R¹fÍ•²Š|/ôlS‰ÄòJÚëÊ`2‘á[§ƒ í™ì

secrets/gitea-mailer-password.age

@@ -0,0 +1,23 @@
+age-encryption.org/v1
+-> ssh-ed25519 Y0ZZaw KuyePsvWNh4rYXs8Eiq7JXZdVOUzewtQ9jEcOqTOTCs
+69jfk3fZ9gTJyq1dIJmi9KGAhlZ307fQLSgPmDm7/yQ
+-> ssh-ed25519 BVsyTA V9RtiFn2g/7PPL3ragEn51VIiRyKlW889Uc1fAqkc0I
+IQv7b2Rei0rdp0MJEsSqwQmjUmWlNcUHjzvsEk/anro
+-> ssh-rsa kFDS0A
+RfWK33SKHaI2/gw3/1h76dhJq/0z1fgl8JVjsKmPgj/K+TPH05DoObBmKG2wAwcg
+xA48ejVh4zIHhQkbTrnfOYUo32WJvtxEzDZcgwi2ZOJlhrZ8p8aIcoGVgcNiPI8V
+S2hN52aCjKtnz6+zzBaEsbsNEyqlAD6hIvjnBDgQUNy9tVSWhPZbbkkLD7vZ24iH
+MilVTDV6jUAxlgq+Yb3bxIyYhFlb0ACkc9D3XxmKp4Ukcag9HJ1dn0OPw2NEdEnO
+KCydY9aHqcw8nbTpDLzwXPgh6HPKXa7aNrfvKRrM6B6nA4Cy8hr+QcxBCEYBsVdm
+dzojEhJm/SccBhEbzPRyla1dUsdfM8tQzy4hg/4bAvmSb3sSqZX3LUhE8NhPML4c
+Vb5+0GUOyQ8zPZfORoT6L7785d5eeWJ9SbKnr2rteEJfzOLeHdM/IdJNXgP0u6ee
+BMaf/Jo3A6JPpYurHvUCWPulyEsic7LuVrEi6HnQmzeX342B7Gn/srL6YzsfGeV+
+j1G1AIUv+QHdz4bX3oqGgqwVGaPbcRClZDnB/Gq+a3E+zJVEGnFagoBSDE2/jpC0
+KXAzaS3CaZ4wqKRZJA/GGWu+KZXA7F87Gr1I84jwdhyn6Dr1l51z0s4NFx6gmYW2
+Nz7QUZz15DNcIXLdpz3XJqSpJ1sinkcVkCdQykv+miI
+-> ssh-ed25519 cakP9w Ghr10m5/sD5g4yyMtOrPiO3SEbmhrXMjtRVQf8c4I3E
+hbb4UOdZ7E2u7ImAAkyr4FV/pACSEopucguPLjQYo98
+-> 7=-grease <]_?g0s
+
+--- 2SbSEW2lbLlhNC7fPB52ZLLfhV1kywGzsLKFhZHRZaE
+ÔîæqX›yónÃ}øqÍ
¥ïzYv1tc¢nÚp¨v“Þ!ðλOÑ觃â>

secrets/hosting.de-api-key.age

@@ -0,0 +1,24 @@
+age-encryption.org/v1
+-> ssh-ed25519 Y0ZZaw 20k6EMk3iFO98VaGMQVxAxLuCIKp8M8DN3SucCtNNlM
+Za5+geQFA/CVkj/IADKmrj3EIsKcj4l9jETuvQbApCE
+-> ssh-ed25519 BVsyTA vqpU3pzGierKMAKK0Gn/xvEkdebGgfahP5CmRE8tZTY
+ZsmYvYgFzOL2PL1mLJ8vwoWiT1bngJq1Y4P3MBYcbbI
+-> ssh-rsa kFDS0A
+chQyiXGW3eND86nYKKZVX3Tr3MKVYJXoQccbYn5yGEmXw17U67z2YgbgXcKJGszE
+FiOKe6fWCbrnCi/GkktM+bbnbJtfCOFK/J2VlVQrHScibel0z4+yd0aeH02EVHla
+34vcv2Tdi09BVn5Me2Gh94/CKgzAiIIHmvDfYFsBZNQOalAbn0PvpHFJcVnf2DsR
+bVgOKL8l4mHz/gihI5cCr5snz3ClhHv/aFbTR2QpHPKAfz5aRi0pe9phpqv9TDb9
+3taeIDZ2dIVpASyqIBxzxzep6liJrupldmQQGpzleKpRfqPWrM5BAmMHr6/LiP24
+caHaP7SOVTr1s1wo4Qh8x97LrXYe2AEmJogteoLJddYYgqYhJuoCS62dDbyry32D
+3BHJTcxl2OsgFDP9Q9uABhwNExdu9z7ohbvyOwZ3ZkgEbpFOKPAOExKK0FV6C5R3
+nKox0yPZGIsTP0wjjyWlZAwUkNIj1m5gXL6l3h4930EnPZFe7vg+lfC/4v2V21pE
+l5U1a+LqBzOk7qCgVgtbAZZEIvK8s0kZYLmPmtMyjIm/mnl6yxY80Kwf6oTlb6KN
+bFopzuBfmCZk3hVzkrVDmpxpWTcFclSG7H/R+1SSibHOL0RcLuULhrVHd4qOPYrQ
+I4jDt3nmUKU1si1IhOQe20NKG3DokMGDQk3balKIMfU
+-> ssh-ed25519 cakP9w zXYz3ME+JJqwpWGFlnHzTxNTvpGMXAZB00rrh0MN1wc
+tgNVf1JJWPkU7TxYcakGE+omp1fNqwXVzyQBpPAdVCU
+-> Sq|yWUT-grease N>}A9
+TvMfldFuvLXmFvEmgZTd8zXY2iH9Sw
+--- jyypo7T4RplBaZ94/suJgMliWbt4tQNMhrDUv/YMaYE
+ž ÒS=Š8¶¨å¹Æ¸k®C¶³UÜŽ<C39C>2eõX}$G-oØžCÖl=ýÛQ
¥û_§áÞ¡r)ädâyHûiÇ<69>[*aµÄ R®ã
+X

secrets/mailman-core-secrets.age

@@ -0,0 +1,23 @@
+age-encryption.org/v1
+-> ssh-ed25519 Y0ZZaw gkhWIVF4XyeitN9y4jhIplrFAzkT+vUIXyyQKh/7IwU
+DWi5ZfTdf9QzqTFLD++3ctckO9DiBOynHqWfhyc4InI
+-> ssh-ed25519 BVsyTA afwxuuU7CZIzOZeN0aNDsPm3GQc1/pK1PJ9Yc6DnoHk
+2wkHEvDZb8FoRygCn08arkh2GZh5nxMO2ag5y7nAdzA
+-> ssh-rsa kFDS0A
+KZRclxsJja35rXgVUlFqBe3tOEryUTePT748oNs5LJoAi/h9PnRwIW4K/pzgFqVL
+ADcMb5T8WBqDJ/JvZKkOXOERIj9K2KGx2ETCYat0wgB/ojB8ER7NIsEaVmcbKyY0
+5x7DP07DGdkdkzfPNaRhsFBL4xoF8LxrR3ZjKR++c0ICMQ6WKOuXB0J6spBv15rR
+IE4yOpV+vhYoUh1X9tGKfNthxKnV40FtJX6ucSmIoGbRAzdZUACRDRVjWgNVJc+x
+ucbOnJE1UA1T/ttxFRkjIFg7jPyv75YPAUO+/u+0xrxQfsDhXY4zuvz+2Ze9gwue
+WQWHR2kKrv5udYCMLCug1f1QpgeBiysW+EzseR0sgLfSNGH/sssB8RNnXU2hruc0
+oMvnsSNTfLCD+xzMbDhJbELI415sVfbJ9Lm29Sv60k03aW7HagTdNGwQJ8qJeeeX
+XM+z089encgbej/iuruH4JVDAdiNzuEZMgCKWhmcX2VKQxunWTWCdUGpuotL4mL/
+xHxjmz2eEKF3KuSSmEuvtqMkVVJYRbpIjj7p1cMGE2DsA/sTWEd49hSBkkfV12pt
+lxyk1/0bPRI90qQb6slRm9R9DRwJp1v9Cht/ZEmNvOdXskTgBT3hyi6/8ykYZlvT
+ogX3qz4OF7KftBA+5YHWvA99tvKmUKN6bcCWVbFwtDo
+-> ssh-ed25519 cakP9w hMZotYYfld14w9/8QGj7D0iTeXJ3CZ6h08PF1mjrqmQ
+9ZJMVHwwU/a0AJNmSXHf9q/4Ap1tJvhXCsga0nY2REw
+-> ukh-grease
+k5f1OA
+--- qmDVJB1ai7/Ps+Wb2fXtyM5rIvH9e/2u6dDbf1/EefE
+³lö—ƒj£C$ŽÁ”Πý£ïð·X™¤ü—mË)(-bôIÙ1
pÀÊý&ˆI¶DºÞQ?cvŠ	’y‹\†¾>cAïãÞwJ)Ÿ
xme^,¿y±ÉÓÚÂænv\ÍE®º`…¹ô³3²â™LÑÍxŸ„úÁ%ü6rŒ)
.öx.¶Bü Ó|W£6ù½Àœóüä0%t.YyÆ4u½¯R±Ãe±Ó*‚	/Wi!PKj3‚½tG‘¬nôüüIoVÍ-ÂIᄤ<E2809E>6e‰€G6$Õ|(>]ißt½ÎÚ<C38E>k1ue»ª·,jómÆl,~Ïß×#fY>quÏz¤`íø4{ÙÜÝVj¢ÊÒï/ž’:3zºÜãj	?y˦€Aq!ʪÁ\N3žò.¡á­ïë ŽW•®¢œ«  p˜9˜µ‘¼å®ŒÛ

secrets/mailman-db-secrets.age

@@ -0,0 +1,23 @@
+age-encryption.org/v1
+-> ssh-ed25519 Y0ZZaw vcObihtzXJ3+hQDnNQ37nWiv6I+aSfX6u5U/KAxO+Rk
+hrcPHjL0qMaFN6Rj0AqO73L0EpH6H1AjC6cHwoJiDa4
+-> ssh-ed25519 BVsyTA O7ALAsno95CTjlrHpYeUhw5avBHYzVZLP/c8nPg+BCY
+mx3H5mB6Q1WavsjIIuUIC6pe/YHTE45bRh6SDbTZEhQ
+-> ssh-rsa kFDS0A
+pj+13dL6H7gPkh/b3vZOztRM7VdSElkeB4SjsfZh2lZoMpjhuiQuUauxJl+pAAXZ
+37evEIcAmDnrl2+tBXq/9krry4DbpYRL7Mz4TYjYmSn2sRDDd4q/Tv1bXdLwuZ8G
+svq1qWhDkAsL6rxzpOV+EsE2EyZxtGx0Odgp7GMmAJq1Vrm8NUF/+Nh3V8LtDmtz
+08tgebfKoWDRFnDbWN7Du+TN9TUp4MlpMPUqjJPDVmsVsVsKTRvck+M8iTaf9HPv
+GJ9TNPwDN9ZvL+0jFd+v440MqXsejsrxIxPHQ/KsgnOciiW58Drn2vNyBL3FVYxG
+C+JzKsRl3AQpP6qoXv5I1pwitI9yLXf+7aSyVUANUWU3ZG8M1KIGtIBTMHAh4w+B
+yJl5qwY2Ty3FKbi3alT7SdWBlRK/m6Lqh6qFSa9kptpgIQ6D3nzWOhrs8tdRFuYm
+6ilwarYuDXD+eLukBJ63l1OL1jVicLpMJXp40v2iQisDR3ifW7dHRXyadGIBvbHl
+6c9ZQP6WritVR3CCZIHSBoxnUD5pGV+KbfbHh0Q1E2goV4SOGyivG4q9iXKIGk6A
+S1Yl5/kwA+/gzc0VdgQmcr9dFe8jC4n5SsaFRAzr+2e+IjH2EhinJlERFwxDJ6+q
+ADeLbCApDA2mKNBiZ2c1p12F+iL7Vk07xLV0owHvNOw
+-> ssh-ed25519 cakP9w ay0pLMA3UdI/7XtF9GgftyKavJe89NqdpzYVP4+Trgk
+ptOrYOEC7NgQPXF3EnuUaySQqJqP30Ez1lY+0RmXUTg
+-> ma-grease (S#' ?5>*(YT
+N6grE/WR7SJGCaVJcWYSVxeeiA
+--- +u1sOmxTLHDiBiOFP0aI7VQvxQX3NYnq586q4LhJk1Q
+»?S ÿ¿…¹aª“à_²Ñ­cÄY-ù Å®$¦{]s]<7F>…<EFBFBD>ª2µÎQbÁXˆ|Å^µÿ:Xþ"õðøã§rÐÅÝ­‰p”w±I=WUúØdBȉÄC òž
2¿¹Ê„+àI\a­¬ú¸ù5ì$TG[òè=ËÖ ?_¶ p™¢°‘â²™d^ó%Ô&èTΔ¥ÿàNÚ

secrets/secrets.nix

@@ -1,8 +1,31 @@
 let
   # set ssh public keys here for your system and user
-  system = "";
+  b12f-bbcom = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQCmXpOU6vzQiVSSYCoxHYv7wDxC63Qg3dxlAMR6AOzwIABCU5PFFNcO0NWYms/YR7MOViorl+19LCLRABar9JgHU1n+uqxKV6eGph3OPeMp5sN8LAh7C9N+TZj8iJzBxQ3ch+Z/LdmLRwYNJ7KSUI+gwGK6xRS3+z1022Y4P0G0sx7IeCBl4lealQEIIF10ZOfjUdBcLQar7XTc5AxyGKnHCerXHRtccCoadLQujk0AvPXbv3Ma4JwX9X++AnCWRWakqS5UInu2tGuZ/6Hrjd2a9AKWjTaBVDcbYqCvY4XVuMj2/A2bCceFBaoi41apybSk26FSFTU4qiEUNQ6lxeOwG4+1NCXyHe2bGI4VyoxinDYa8vLLzXIRfTRA0qoGfCweXNeWPf0jMqASkUKaSOH5Ot7O5ps34r0j9pWzavDid8QeKJPyhxKuF1a5G4iBEZ0O9vuti60dPSjJPci9oTxbune2/jb7Sa0yO06DtLFJ2ncr5f70s/BDxKk4XIwQLy+KsvzlQEGdY8yA6xv28bOGxL3sQ0HE2pDTsvIbAisVOKzdJeolStL9MM5W8Hg0r/KkGj2bg0TfoRp1xHV9hjKkvJrsQ6okaPvNFeZq0HXzPhWMOVQ+/46z80uaQ1ByRLr3FTwuWJ7F/73ndfxiq6bDE4z2Ji0vOjeWJm6HCxTdGw==";
-  user = "";
+  teutat3s-dumpyourvms = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHcU6KPy4b1MQXd6EJhcYwbJu7E+0IrBZF/IP6T7gbMf teutat3s@dumpyourvms";
-  allKeys = [system user];
+  flora-6 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGP1InpTBN4AlF/4V8HHumAMLJzeO8DpzjUv9Co/+J09 root@pub-solar-infra-vm-1";
+  nougat-2-host = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAINELr5Bvr15GqCHevg9QP8oYFgmaRUUHcPFf4MZho9gI root@nougat-2";
+
+  allKeys = [
+    flora-6
+    teutat3s-dumpyourvms
+    b12f-bbcom
+    nougat-2-host
+  ];
+
+  deployKeys = [
+    flora-6
+    teutat3s-dumpyourvms
+    b12f-bbcom
+    nougat-2-host
+  ];
 in {
-  "secret.age".publicKeys = allKeys;
+  "gitea-database-password.age".publicKeys = deployKeys;
+  "gitea-mailer-password.age".publicKeys = deployKeys;
+  "keycloak-database-password.age".publicKeys = deployKeys;
+  "drone-secrets.age".publicKeys = deployKeys;
+  "drone-db-secrets.age".publicKeys = deployKeys;
+  "mailman-core-secrets.age".publicKeys = deployKeys;
+  "mailman-web-secrets.age".publicKeys = deployKeys;
+  "mailman-db-secrets.age".publicKeys = deployKeys;
+  "hosting.de-api-key.age".publicKeys = deployKeys;
 }

users/barkeeper/default.nix

@@ -0,0 +1,42 @@
+{
+  config,
+  hmUsers,
+  pkgs,
+  lib,
+  ...
+}: let
+  psCfg = config.pub-solar;
+in {
+  config = {
+    home-manager.users = {inherit (hmUsers) barkeeper;};
+
+    pub-solar = {
+      # These are your personal settings
+      # The only required settings are `name` and `password`,
+      # The rest is used for programs like git
+      user = {
+        name = "barkeeper";
+        description = "pub.solar infra user";
+        password = "$6$MCJ28kLwfNl9SNDq$Oh9eT6Sn6z4xGrQsLlIBI7cvJzX3P5As59OSZ.hoeBWc79Un2YdwH/hRIC.4ZDOuwQp0lHI82dNn/xeTaCn631";
+        fullName = "pub.solar infra barkeeper";
+        email = "admins@pub.solar";
+        gpgKeyId = "";
+        publicKeys = [
+          "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCmiF8ndGhnx2YAWbPDq14fftAwcJ0xnjJIVTotI12OO4SPX/SwH5Yp8C8Kf002qN9FbFmaONzq3s8TYpej13JubhfsQywNuFKZuZvJeHzmOwxsANW86RVrWT0WZmYx9a/a1TF9rPQpibDVt60wX8yLdExaJc5F1SvIIuyz1kxYpz36wItfR6hcwoLGh1emFCmfCpebJmp3hsrMDTTtTW/YNhyeSZW74ckyvZyjCYtRCJ8uF0ZmOSKRdillv4Ztg8MsUubGn+vaMl6V6x/QuDuehEPoM/3wBx9o22nf+QVbk7S1PC8EdT/K5vskn4/pfR7mDCyQOq1hB4w4Oyn0dsfX pi@ssrtc"
+
+          "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBHx4A8rLYmFgTOp1fDGbbONN8SOT0l5wWrUSYFUcVzMPTyfdT23ZVIdVD5yZCySgi/7PSh5mVmyLIZVIXlNrZJg= @b12f Yubi Main"
+          "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEST9eyAY3nzGYNnqDYfWHu+89LZsOjyKHMqCFvtP7vrgB7F7JbbECjdjAXEOfPDSCVwtMMpq8JJXeRMjpsD0rw= @b12f Yubi Backup"
+
+          "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFro/k4Mgqyh8yV/7Zwjc0dv60ZM7bROBU9JNd99P/4co6fxPt1pJiU/pEz2Dax/HODxgcO+jFZfvPEuLMCeAl0= YubiKey #10593996 PIV Slot 9a @teutat3s"
+
+          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAII/58A18EtxnLYHu63c/+AyTSkJQSso/VVdHUFGp1CTk cardno:FFFE34353135 @hensoko"
+          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIAqkqMYgncrnczcW/0PY+Z+FmNXXpgw6D9JWTTwiainy hensoko@hensoko-tp-work"
+          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEbaQdxp7Flz6ttELe63rn+Nt9g43qJOLih6VCMP4gPb @hensoko"
+
+          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIIKa5elEXgBc2luVBOHVWZisJgt0epFQOercPi0tZzPU root@cloud.pub.solar"
+          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMNeQYLFauAbzDyIbKC86NUh9yZfiyBm/BtIdkcpZnSU axeman@tuxnix"
+        ];
+      };
+    };
+  };
+}