2024-05-31 15:22:28 +00:00
30 changed files with 767 additions and 30 deletions
--- a/docs/deploying.md
+++ b/docs/deploying.md
@ -10,13 +10,19 @@ Then, run `deploy-rs` with the hostname of the server you want to deploy:
 For nachtigall.pub.solar:

 ```
-deploy --targets '.#nachtigall' --magic-rollback false --auto-rollback false
+deploy --targets '.#nachtigall' --magic-rollback false --auto-rollback false --keep-result --result-path ./results
 ```

 For flora-6.pub.solar:

 ```
-deploy --targets '.#flora-6' --magic-rollback false --auto-rollback false
+deploy --targets '.#flora-6' --magic-rollback false --auto-rollback false --keep-result --result-path ./results
+```
+
+For metronom.pub.solar (aarch64-linux):
+
+```
+deploy --targets '.#metronom' --magic-rollback false --auto-rollback false --keep-result --result-path ./results --remote-build
 ```

 Usually we skip all rollback functionality, but if you want to deploy a change
@ -28,6 +34,11 @@ deployment, add the flag `--skip-checks` at the end of the command.
 `--dry-activate` can be used to only put all files in place without switching,
 to enable switching to the new config quickly at a later moment.

+We use `--keep-result --result-path ./results` to keep the last `result`
+symlink of each `deploy` from being garbage collected. That way, we keep builds
+cached in the Nix store. This is optional and both flags can be removed if disk
+space is a scarce resource on your machine.
+
 You'll need to have SSH Access to the boxes to be able to run `deploy`.

 ### Getting SSH access
--- a/docs/mail.md
+++ b/docs/mail.md
@ -0,0 +1,4 @@
+### Mail
+
+mail.pub.solar aka metronom.pub.solar hosts our internal mails.
+This is a small Hetzner cloud instance on https://console.hetzner.cloud.
--- a/docs/unlocking-root.md
+++ b/docs/unlocking-root.md
@ -1,9 +1,17 @@
 # Unlocking the root partition on boot

-After a boot, the encrypted root partition will have to be unlocked. This is done by accessing the server via SSH with user root on port 2222.
+After a reboot, the encrypted ZFS pool will have to be unlocked. This is done by accessing the server via SSH with user `root` on port 2222.
+
+Nachtigall:

 ```
-ssh root@nachtigall.pub.solar -p2222
+ssh root@138.201.80.102 -p2222
+```
+
+Metronom:
+
+```
+ssh root@49.13.236.167 -p2222
 ```

 After connecting, paste the crypt passphrase you can find in the shared keepass. This will disconnect the SSH session right away and the server will keep booting into stage 2.
--- a/flake.lock
+++ b/flake.lock
@ -27,6 +27,22 @@
        "type": "github"
      }
    },
+    "blobs": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1604995301,
+        "narHash": "sha256-wcLzgLec6SGJA8fx1OEN1yV/Py5b+U5iyYpksUY/yLw=",
+        "owner": "simple-nixos-mailserver",
+        "repo": "blobs",
+        "rev": "2cccdf1ca48316f2cfd1c9a0017e8de5a7156265",
+        "type": "gitlab"
+      },
+      "original": {
+        "owner": "simple-nixos-mailserver",
+        "repo": "blobs",
+        "type": "gitlab"
+      }
+    },
    "deploy-rs": {
      "inputs": {
        "flake-compat": "flake-compat",
@ -128,6 +144,22 @@
        "type": "github"
      }
    },
+    "flake-compat_2": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1668681692,
+        "narHash": "sha256-Ht91NGdewz8IQLtWZ9LCeNXMSXHUss+9COoqu6JLmXU=",
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "rev": "009399224d5e398d03b22badca40a37ac85412a1",
+        "type": "github"
+      },
+      "original": {
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "type": "github"
+      }
+    },
    "flake-parts": {
      "inputs": {
        "nixpkgs-lib": "nixpkgs-lib"
@ -328,6 +360,21 @@
        "type": "github"
      }
    },
+    "nixpkgs-23_05": {
+      "locked": {
+        "lastModified": 1704290814,
+        "narHash": "sha256-LWvKHp7kGxk/GEtlrGYV68qIvPHkU9iToomNFGagixU=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "70bdadeb94ffc8806c0570eb5c2695ad29f0e421",
+        "type": "github"
+      },
+      "original": {
+        "id": "nixpkgs",
+        "ref": "nixos-23.05",
+        "type": "indirect"
+      }
+    },
    "nixpkgs-lib": {
      "locked": {
        "lastModified": 1714640452,
@ -340,6 +387,21 @@
        "url": "https://github.com/NixOS/nixpkgs/archive/50eb7ecf4cd0a5756d7275c8ba36790e5bd53e33.tar.gz"
      }
    },
+    "nixpkgs_2": {
+      "locked": {
+        "lastModified": 1705856552,
+        "narHash": "sha256-JXfnuEf5Yd6bhMs/uvM67/joxYKoysyE3M2k6T3eWbg=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "612f97239e2cc474c13c9dafa0df378058c5ad8d",
+        "type": "github"
+      },
+      "original": {
+        "id": "nixpkgs",
+        "ref": "nixos-unstable",
+        "type": "indirect"
+      }
+    },
    "root": {
      "inputs": {
        "agenix": "agenix",
@ -354,10 +416,37 @@
        "nixos-flake": "nixos-flake",
        "nixpkgs": "nixpkgs",
        "nixpkgs-2205": "nixpkgs-2205",
+        "simple-nixos-mailserver": "simple-nixos-mailserver",
        "triton-vmtools": "triton-vmtools",
        "unstable": "unstable"
      }
    },
+    "simple-nixos-mailserver": {
+      "inputs": {
+        "blobs": "blobs",
+        "flake-compat": "flake-compat_2",
+        "nixpkgs": "nixpkgs_2",
+        "nixpkgs-23_05": "nixpkgs-23_05",
+        "nixpkgs-23_11": [
+          "nixpkgs"
+        ],
+        "utils": "utils_2"
+      },
+      "locked": {
+        "lastModified": 1706219574,
+        "narHash": "sha256-qO+8UErk+bXCq2ybHU4GzXG4Ejk4Tk0rnnTPNyypW4g=",
+        "owner": "simple-nixos-mailserver",
+        "repo": "nixos-mailserver",
+        "rev": "e47f3719f1db3e0961a4358d4cb234a0acaa7baf",
+        "type": "gitlab"
+      },
+      "original": {
+        "owner": "simple-nixos-mailserver",
+        "ref": "nixos-23.11",
+        "repo": "nixos-mailserver",
+        "type": "gitlab"
+      }
+    },
    "systems": {
      "locked": {
        "lastModified": 1681028828,
@ -475,6 +564,21 @@
        "repo": "flake-utils",
        "type": "github"
      }
+    },
+    "utils_2": {
+      "locked": {
+        "lastModified": 1605370193,
+        "narHash": "sha256-YyMTf3URDL/otKdKgtoMChu4vfVL3vCMkRqpGifhUn0=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "5021eac20303a61fafe17224c087f5519baed54d",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
    }
  },
  "root": "root",
--- a/flake.nix
+++ b/flake.nix
@ -38,6 +38,9 @@
    element-stickers.url = "git+https://git.pub.solar/pub-solar/maunium-stickerpicker-nix?ref=main";
    element-stickers.inputs.maunium-stickerpicker.follows = "maunium-stickerpicker";
    element-stickers.inputs.nixpkgs.follows = "nixpkgs";
+
+    simple-nixos-mailserver.url = "gitlab:simple-nixos-mailserver/nixos-mailserver/nixos-23.11";
+    simple-nixos-mailserver.inputs.nixpkgs-23_11.follows = "nixpkgs";
  };

  outputs =
@ -123,6 +126,10 @@
              hostname = "10.7.6.2";
              sshUser = username;
            };
+            metronom = {
+              hostname = "10.7.6.3";
+              sshUser = username;
+            };
            tankstelle = {
              hostname = "80.244.242.5";
              sshUser = username;
--- a/hosts/default.nix
+++ b/hosts/default.nix
@ -59,6 +59,19 @@
        ];
      };

+      metronom = self.nixos-flake.lib.mkLinuxSystem {
+        imports = [
+          self.inputs.agenix.nixosModules.default
+          self.nixosModules.home-manager
+          ./metronom
+          self.nixosModules.overlays
+          self.nixosModules.unlock-zfs-on-boot
+          self.nixosModules.core
+
+          self.inputs.simple-nixos-mailserver.nixosModule
+        ];
+      };
+
      tankstelle = self.nixos-flake.lib.mkLinuxSystem {
        imports = [
          self.inputs.agenix.nixosModules.default
--- a/hosts/metronom/backups.nix
+++ b/hosts/metronom/backups.nix
@ -0,0 +1,13 @@
+{ flake, ... }:
+{
+  age.secrets."restic-repo-droppie" = {
+    file = "${flake.self}/secrets/restic-repo-droppie.age";
+    mode = "400";
+    owner = "root";
+  };
+  age.secrets."restic-repo-storagebox" = {
+    file = "${flake.self}/secrets/restic-repo-storagebox.age";
+    mode = "400";
+    owner = "root";
+  };
+}
--- a/hosts/metronom/configuration.nix
+++ b/hosts/metronom/configuration.nix
@ -0,0 +1,34 @@
+{
+  flake,
+  config,
+  pkgs,
+  ...
+}:
+{
+  boot.loader.systemd-boot.enable = true;
+  boot.supportedFilesystems = [ "zfs" ];
+
+  boot.kernelParams = [
+    "boot.shell_on_fail=1"
+    "ip=dhcp"
+  ];
+
+  boot.initrd.availableKernelModules = [ "igb" ];
+
+  # https://nixos.wiki/wiki/ZFS#declarative_mounting_of_ZFS_datasets
+  systemd.services.zfs-mount.enable = false;
+
+  # Declarative SSH private key
+  #age.secrets."metronom-root-ssh-key" = {
+  #  file = "${flake.self}/secrets/metronom-root-ssh-key.age";
+  #  path = "/root/.ssh/id_ed25519";
+  #  mode = "400";
+  #  owner = "root";
+  #};
+
+  # This value determines the NixOS release with which your system is to be
+  # compatible, in order to avoid breaking some software such as database
+  # servers. You should change this only after NixOS release notes say you
+  # should.
+  system.stateVersion = "23.11"; # Did you read the comment?
+}
--- a/hosts/metronom/default.nix
+++ b/hosts/metronom/default.nix
@ -0,0 +1,12 @@
+{ flake, ... }:
+
+{
+  imports = [
+    ./hardware-configuration.nix
+    ./configuration.nix
+
+    ./networking.nix
+    ./wireguard.nix
+    #./backups.nix
+  ];
+}
--- a/hosts/metronom/hardware-configuration.nix
+++ b/hosts/metronom/hardware-configuration.nix
@ -0,0 +1,48 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{
+  config,
+  lib,
+  pkgs,
+  modulesPath,
+  ...
+}:
+
+{
+  imports = [ (modulesPath + "/profiles/qemu-guest.nix") ];
+
+  boot.initrd.availableKernelModules = [
+    "xhci_pci"
+    "virtio_pci"
+    "virtio_scsi"
+    "usbhid"
+    "sr_mod"
+  ];
+  boot.initrd.kernelModules = [ ];
+  boot.kernelModules = [ ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" = {
+    device = "root_pool/root";
+    fsType = "zfs";
+  };
+
+  fileSystems."/boot" = {
+    device = "/dev/disk/by-uuid/2083-C68E";
+    fsType = "vfat";
+  };
+
+  swapDevices = [ ];
+
+  # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
+  # (the default) this is the recommended approach. When using systemd-networkd it's
+  # still possible to use this option, but it's recommended to use it in conjunction
+  # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
+  networking.useDHCP = lib.mkDefault true;
+  # networking.interfaces.eth0.useDHCP = lib.mkDefault true;
+
+  nixpkgs.hostPlatform = lib.mkDefault "aarch64-linux";
+  powerManagement.cpuFreqGovernor = lib.mkDefault "ondemand";
+  hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
+}
--- a/hosts/metronom/networking.nix
+++ b/hosts/metronom/networking.nix
@ -0,0 +1,23 @@
+{
+  config,
+  pkgs,
+  flake,
+  ...
+}:
+{
+
+  networking.hostName = "metronom";
+  networking.extraHosts = ''
+    127.0.0.2 mail.pub.solar mail
+    ::1 mail.pub.solar mail
+  '';
+  networking.domain = "pub.solar";
+  networking.hostId = "00000002";
+
+  networking.enableIPv6 = true;
+  networking.useDHCP = false;
+  networking.interfaces."enp1s0".useDHCP = true;
+
+  # TODO: ssh via wireguard only
+  services.openssh.openFirewall = true;
+}
--- a/hosts/metronom/wireguard.nix
+++ b/hosts/metronom/wireguard.nix
@ -0,0 +1,54 @@
+{
+  config,
+  pkgs,
+  flake,
+  ...
+}:
+{
+  networking.firewall.allowedUDPPorts = [ 51820 ];
+
+  age.secrets.wg-private-key.file = "${flake.self}/secrets/metronom-wg-private-key.age";
+
+  networking.wireguard.interfaces = {
+    wg-ssh = {
+      listenPort = 51820;
+      mtu = 1300;
+      ips = [
+        "10.7.6.3/32"
+        "fd00:fae:fae:fae:fae:3::/96"
+      ];
+      privateKeyFile = config.age.secrets.wg-private-key.path;
+      peers = flake.self.logins.admins.wireguardDevices ++ [
+        {
+          # flora-6.pub.solar
+          endpoint = "80.71.153.210:51820";
+          publicKey = "jtSR5G2P/nm9s8WrVc26Xc/SQLupRxyXE+5eIeqlsTU=";
+          allowedIPs = [
+            "10.7.6.2/32"
+            "fd00:fae:fae:fae:fae:2::/96"
+          ];
+        }
+        {
+          # nachtigall.pub.solar
+          endpoint = "138.201.80.102:51820";
+          publicKey = "qzNywKY9RvqTnDO8eLik75/SHveaSk9OObilDzv+xkk=";
+          allowedIPs = [
+            "10.7.6.1/32"
+            "fd00:fae:fae:fae:fae:1::/96"
+          ];
+        }
+      ];
+    };
+  };
+
+  services.openssh.listenAddresses = [
+    {
+      addr = "10.7.6.3";
+      port = 22;
+    }
+    {
+      addr = "[fd00:fae:fae:fae:fae:3::]";
+      port = 22;
+    }
+  ];
+}
--- a/lib/deploy.nix
+++ b/lib/deploy.nix
@ -7,21 +7,6 @@

 { lib, inputs }:
 let
-  # https://github.com/serokell/deploy-rs#overall-usage
-  system = "x86_64-linux";
-  pkgs = import inputs.nixpkgs { inherit system; };
-  deployPkgs = import inputs.nixpkgs {
-    inherit system;
-    overlays = [
-      inputs.deploy-rs.overlay
-      (self: super: {
-        deploy-rs = {
-          inherit (pkgs) deploy-rs;
-          lib = super.deploy-rs.lib;
-        };
-      })
-    ];
-  };
  getFqdn =
    c:
    let
@ -66,7 +51,28 @@ in
    */
    lib.recursiveUpdate (lib.mapAttrs (_: c: {
      hostname = getFqdn c;
-      profiles.system = {
+      profiles.system =
+        let
+          system = c.pkgs.system;
+
+          # Unmodified nixpkgs
+          pkgs = import inputs.nixpkgs { inherit system; };
+
+          # nixpkgs with deploy-rs overlay but force the nixpkgs package
+          deployPkgs = import inputs.nixpkgs {
+            inherit system;
+            overlays = [
+              inputs.deploy-rs.overlay # or deploy-rs.overlays.default
+              (self: super: {
+                deploy-rs = {
+                  inherit (pkgs) deploy-rs;
+                  lib = super.deploy-rs.lib;
+                };
+              })
+            ];
+          };
+        in
+        {
          user = "root";
          path = deployPkgs.deploy-rs.lib.activate.nixos c;
        };
--- a/modules/forgejo/default.nix
+++ b/modules/forgejo/default.nix
@ -94,7 +94,7 @@
      mailer = {
        ENABLED = true;
        PROTOCOL = "smtps";
-        SMTP_ADDR = "mail.greenbaum.zone";
+        SMTP_ADDR = "mail.pub.solar";
        SMTP_PORT = 465;
        FROM = ''"pub.solar git server" <forgejo@pub.solar>'';
        USER = "admins@pub.solar";
--- a/modules/grafana/default.nix
+++ b/modules/grafana/default.nix
@ -59,7 +59,7 @@
      };
      smtp = {
        enabled = true;
-        host = "mail.greenbaum.zone:465";
+        host = "mail.pub.solar:465";
        user = "admins@pub.solar";
        password = "\$__file{${config.age.secrets.grafana-smtp-password.path}}";
        from_address = "no-reply@pub.solar";
--- a/modules/mail/default.nix
+++ b/modules/mail/default.nix
@ -0,0 +1,70 @@
+{ config, flake, ... }:
+
+{
+  age.secrets.mail-hensoko.file = "${flake.self}/secrets/mail/hensoko.age";
+  age.secrets.mail-teutat3s.file = "${flake.self}/secrets/mail/teutat3s.age";
+  age.secrets.mail-admins.file = "${flake.self}/secrets/mail/admins.age";
+  age.secrets.mail-bot.file = "${flake.self}/secrets/mail/bot.age";
+  age.secrets.mail-crew.file = "${flake.self}/secrets/mail/crew.age";
+  age.secrets.mail-erpnext.file = "${flake.self}/secrets/mail/erpnext.age";
+  age.secrets.mail-hakkonaut.file = "${flake.self}/secrets/mail/hakkonaut.age";
+
+  mailserver = {
+    enable = true;
+    fqdn = "mail.pub.solar";
+    domains = [ "pub.solar" ];
+
+    # A list of all login accounts. To create the password hashes, use
+    # nix-shell -p mkpasswd --run 'mkpasswd -R11 -m bcrypt'
+    loginAccounts = {
+      "hensoko@pub.solar" = {
+        hashedPasswordFile = config.age.secrets.mail-hensoko.path;
+        quota = "2G";
+      };
+      "teutat3s@pub.solar" = {
+        hashedPasswordFile = config.age.secrets.mail-teutat3s.path;
+        quota = "2G";
+      };
+      "admins@pub.solar" = {
+        hashedPasswordFile = config.age.secrets.mail-admins.path;
+        quota = "2G";
+        aliases = [
+          "abuse@pub.solar"
+          "alerts@pub.solar"
+          "forgejo@pub.solar"
+          "keycloak@pub.solar"
+          "mastodon-notifications@pub.solar"
+          "matrix@pub.solar"
+          "postmaster@pub.solar"
+          "nextcloud@pub.solar"
+          "no-reply@pub.solar"
+          "security@pub.solar"
+        ];
+      };
+      "bot@pub.solar" = {
+        hashedPasswordFile = config.age.secrets.mail-bot.path;
+        quota = "2G";
+        aliases = [ "hackernews-bot@pub.solar" ];
+      };
+      "crew@pub.solar" = {
+        hashedPasswordFile = config.age.secrets.mail-crew.path;
+        quota = "2G";
+        aliases = [ "moderation@pub.solar" ];
+      };
+      "erpnext@pub.solar" = {
+        hashedPasswordFile = config.age.secrets.mail-erpnext.path;
+        quota = "2G";
+      };
+      "hakkonaut@pub.solar" = {
+        hashedPasswordFile = config.age.secrets.mail-hakkonaut.path;
+        quota = "2G";
+      };
+    };
+
+    # Use Let's Encrypt certificates. Note that this needs to set up a stripped
+    # down nginx and opens port 80.
+    certificateScheme = "acme-nginx";
+  };
+  security.acme.acceptTerms = true;
+  security.acme.defaults.email = "security@pub.solar";
+}
--- a/modules/mastodon/default.nix
+++ b/modules/mastodon/default.nix
@ -60,7 +60,7 @@
    vapidPublicKeyFile = "/run/agenix/mastodon-vapid-public-key";
    smtp = {
      createLocally = false;
-      host = "mail.greenbaum.zone";
+      host = "mail.pub.solar";
      port = 587;
      authenticate = true;
      user = "admins@pub.solar";
--- a/modules/nextcloud/default.nix
+++ b/modules/nextcloud/default.nix
@ -63,7 +63,7 @@
      mail_smtpname = "admins@pub.solar";
      mail_smtpsecure = "tls";
      mail_smtpauth = 1;
-      mail_smtphost = "mail.greenbaum.zone";
+      mail_smtphost = "mail.pub.solar";
      mail_smtpport = "587";

      # This is to allow connections to collabora and keycloak, among other services
--- a/modules/prometheus/default.nix
+++ b/modules/prometheus/default.nix
@ -129,7 +129,7 @@
                send_resolved = true;
                to = "admins@pub.solar";
                from = "alerts@pub.solar";
-                smarthost = "mail.greenbaum.zone:465";
+                smarthost = "mail.pub.solar:465";
                auth_username = "admins@pub.solar";
                auth_password = "$SMTP_AUTH_PASSWORD";
                require_tls = false;
--- a/secrets/mail/admins.age
+++ b/secrets/mail/admins.age
@ -0,0 +1,43 @@
+age-encryption.org/v1
+-> ssh-ed25519 UE5Ceg 6rewUSyj9mZOZp1Oi+DvWxj7u6r7HWUAnp/zSDLmZyA
+OLBPwlUCqlVZqrZaqT/sfzslgcYRViuTt9yzJZRPIPI
+-> ssh-ed25519 uYcDNw JNpKkljIQIPKR/KNG9AF/DxbJjYoMeQdhOjmpig2Q3c
+bxu5hEvJi0ip74WUJNJhm6pAfdvVlFBbyCwQKYPkUXo
+-> ssh-rsa f5THog
+0Im1QWg1IHp5nYfo0OK908ohS+Mo0Jyyyimq3sc6q5WoDUzufaMVYfgVpHJxasO/
+SrVAwE6QLcHuTBZPeyr1HZ7chyQiWT+Lepp/MXhgS8nDOkgJaSNxY35PO6W/qtpE
+rxkgdNZdB2Orqq0wHo0is5+pfZdcD7n6O4VoiayUh6kv5Brk98BUCHrydXMfJv26
+0Kzwg3s+/kDwOeVOt7uy6n5VPhcSLiJgQlK4t0HkPB2rUoD8dfyVqUZV3YmgCoJM
+Km1lCxaS96xKGnvt0HklYy0OX5S7ActBGpQJjcNLTl7sb2M/U0XAF7O8teSKzdq4
+ejKOnzMdxFB+qOSZ3fGzHbjxNDwxPqyps0yhm72rT5tww3wOzYZXUebn7LwNKVwU
+99mA0CR9W3wg3Thv4nwmsrycTMFHh9jvGRXOYgIqXNDoo2oqqkzLnS+N2fx6Wush
+SNziOeZkgb25h0wrehxmqsEOVjlSE6C59E40XlmSj+MJf6siDLQGpLShE4Fz1tyx
+GXASxlTNcJ8TY0N4UmozdWRW8pyTOtl1MhiuaHdYLQGvd3Zlwkr9C7pV6eVBxPyF
+agSqbSZXprY5owp17fUc7HQUu5AcNJyQtDstwqOTPbaJFNfPnyaHU61jt52sk468
+W2d1hZ9SYxiN32rjYV6py2SiuOvHIWMz3ODkvhxQdAM
+-> ssh-rsa kFDS0A
+TRrrVhtSIhhR9OXVAEwfmVn44a/LIaYJZWndqPAcAEhQp1Z3kPpolkxtKskz982G
+wQgSbzU3py4VRpXdy/FBttoEdBrhRMKG0z9N0szKlagfLA+DHQjTlaMn/UkxmO2S
+4AdwO8jEJVe26h6Y/3ne7N+/Ji8QKO6tKeNVapBKHYsJ8qqscgYW1WgKOAfJ3M6c
+6lyavfn2prTkM0xz6hMrywm1Is9ahM4vh39iLRAaVonFHmNJE+dAse8ijvKzjcYM
+KAiZtabdJkWwjD/3x513fU/o9DQCnBTHfM8KLb7DTPC9Ro1K//O7LjcG+WiaERSh
+0+dBZstMD7fQWEyJ/CgnRf54juZs2A7yBdrT9TcQtcgPKYk9QjFqHCmKB0R+TUaX
+nNh4h33i5V/8JfPRQTLz/YYFdG+kG5Hvucs9I2HN1n/vaHL9UIH3zC8BmkUd5fnR
+cnKXPjFCfrPPKg4DMT4gT5lIVtIBRx/IKxvjgR/8c8M9M3jk4SZSYHUlKtnzFOLq
+ycGJopWX7kBWGliEQ8jC+nKYOXpSYH+mbHOV54zplmNOZKMdLJ9ek23WoX5/BD7i
+arp4EtwYiD2LN3M1TG24gFW9VCY3Ofil6HAn5ySM9AMtIHwy/8srUBSCtdpWWGx+
+0fk+wGVu/5lCn51RPXl1L2YRloyx3giKvappuUcpho4
+-> piv-p256 vRzPNw AjkP6Dy1dEQ58LVB01S/1stB6JMpl+q3EuqHQp6RCfH9
+cePnQF/DS9AJx0MJArNi/5b6tncv46lKpu/1SIb5X7Y
+-> piv-p256 zqq/iw A7cNqXWWA3Zd4vccwwW/Wgfq5cCOjnIPq/Et0qpeQUMw
+p/e2OBgHoHA06WR4h3k1GK65u3qYH2YGPYQ10jz+pvQ
+-> ssh-ed25519 YFSOsg +Tl7z0DL81uPhdBuEJG+9qnZ6eoAzyZfvJ5FtrtyRUE
+nfVzlc5NoSxHv+2tM3D444kH9fCjUEYD+7wE2h83qYk
+-> ssh-ed25519 iHV63A FgYN6w2aRUPpBBp6lV8pqSyopRaWwzhkGXxncU83HVc
+PcNQ0P2ZGCnumKWuHVo0wwF3KCz13JadNkAHWgqIfbc
+-> ssh-ed25519 BVsyTA X/VL2A5AlbG1m6uTqbYDJTJj0wVrYGx5w/geJTpgQR4
+zwlsYTehOA3oK92zFN2J+HhgaX1zYd3MP0vQ3W751Co
+-> ssh-ed25519 +3V2lQ Hk8tcLh85helo+DXrRDhCHkDja+sEkM1CTz01s0SXDQ
+ftNhb63/JDulFgTukVu76XG2Dfcorbdt47EV6kqXw9g
+--- 37wAuChTQKbjj/RCIh7ZRB2GOf2kT1we3D4bQKevM3A
+ñ(=ÂžÙ>¤jIM¡ÿú ”ÑyÂA¼|à“áÊ¯Nè<4E>„…‰f‰1Dgí5èËÛÜ	¬áßˆßg<C39F>6ì²#>ßÐ%UjÃX›ŠÈ@ÌÑðG*ªNó™äÀå\çJE
--- a/secrets/mail/bot.age
+++ b/secrets/mail/bot.age
@ -0,0 +1,43 @@
+age-encryption.org/v1
+-> ssh-ed25519 UE5Ceg iKhPkRjtE/7UadHCdLoQR0/fe1LhVF9wSp5DQUw0hV8
+o8BmKJxLYcxml+hq7l57nWQ8xAQFrROcX/BDCpZW7YE
+-> ssh-ed25519 uYcDNw It3n9bvJCC+H+r5VRrtjrga1S1TkhiHUTGL/ltQbk0c
+h/98devoPCP18pYqK7KcXaDspMzQMtvs5YxsoyodDes
+-> ssh-rsa f5THog
+xVi9l7vg34PJaGhjOzOtPtoRMePzlvdYKjNnzCXLd0g6Y4JXQZMoKCeeWrO++rtY
+7/PDxJ0kJjJAEY7q2BnfV+87nmrGxFFerldDcEO9pP8/sN/u393WQpngb0tMNx6M
+cjhwv0Y9ygAb858G1NzvnALVZGmbUxX1JIsq8QDcoP3kz5JmonIKLM3b4LrO735I
+bfu3T+wTRebOHdC9SOhz6iuhyTnu/RmU9w22AKK/IL19z+11NJB2Xoejkfw0c6ZU
+cW25i3TdwmiJAZ+lCDJQyBXtLctDes1/e6HtOkXoJSKQA5QLfEtPeCMyBmE4y0pR
+z1DPiP0wMd37YR8dMXoYDRfo3EvsDJkNR0SDTZj86kio9e2sXA3OtIx8BLM0y01F
+0Vnh0FwpY9kclflboeY9w3Uq33/TCvy9aZ29XD+X7HGdqqiqxeo5rcAMXO9xAx3h
+2fIwdVyWYTnLt8TDOH9ZKDw8vausEITQM/D73AbVlLRKDnXTd+YTkYBgzU1rJtR0
+4FQK4PL2qkWYKEK7qDTp+Hrhc4vOnxURaLsdexTub/A/TXHhGAKPxpGBOcBbCjc5
+4mHSRQsDTbTNNE7bcDbkBiUcXAdlPgvEhfLmmBw8sho45M+krSeSd7V5CJ1NENhJ
+3SO92RqIuyGR48lmvsuN5js4uLS4ntoyQvnmIQIVSQI
+-> ssh-rsa kFDS0A
+EsW7RlBeeV69UwczFANtxqmz2Et2jpUL378UuMydlzRznbp/TJjrzCStMTOBEDyC
+SuADuvcvLf1WsVbf+rxRuFgte0YMiqUNlijN7tsOFg92odk8tHVwXEA71SW8/ZWh
+zFqUJ8pPFXPA6DEYMGmdNLV+tEx3YsUFCrTvhRIBGPCFbuYJj9Ta2xg0KK3uR5/l
+xziM5xxc7NtJGpW3dA/qFyneuY6gPm17PWav2l7gjAge/6FvLFzfev9TuF82iPgc
+RkCNgHZqClWLRO9b0af8FMGWIak6kr/mqao40net2azrFqMxmeQFLIKJSxa6Agz+
+UtlOND1COQwHrogQkHVuanBRRdUZzGk4QdW8MN49JPkvwvVPGS2XZrkE5m4k66Nu
+rfMtlcoSGSA+GIZXTDiDPLpfpYV/XDe4IoPTpLcivRNb8i75GwCT/5vD39Qmlyyc
+GHOX+v5JXh8WYpgvTEPDYE/oeKnsq27QT1wt8q0hKuHcRO4BcdPuiaSMnn0kjvLd
+o473b6cHE96F3cTKhXerLqeMFs1+DsJhrxYCmRikZot6Iz8H5GnqT82Me1by6cYt
+GDcuVLIB0OzWfI9ibZB0ueMM8UfrLeGDq8hSF5M0rDCbFc6ZzQw8PgI97PNaDGg
+FdIMho7IXEQKXMV7ueZ2/PiQEA8vfBWRnxGKFRQLOTY
+-> piv-p256 vRzPNw AjWew9VSba/AQKQ69l/4OhvZUT/bawt7AOSe4/LjanOI
+wHkZs8QQAOE69dq0d/2PAMgsi3xDBqEEvEFB7WKMC1Q
+-> piv-p256 zqq/iw AkKV76ktPNKCS/KidRxBHdRQmtH3BNO2kbBz408ZJ+wu
+S8KdsoVZUgvW7E4mlVFpp7/wxBarAPTEBqsYoBXar+M
+-> ssh-ed25519 YFSOsg SQt87e1+Lza1kqQl+AyqOu47+en8H2AbjCasMjDLfRE
+vBO3eKJPzagd9NdPmVG1SvO3x9rnf4H/8oddfCwpjLY
+-> ssh-ed25519 iHV63A a1iFLv3FlMcfq6p8+dKlFB9cDPC8RFVc9DxtpNIXU3c
+eQW7PJ+eGgp2loZTMUf40D8V3LNAinBSXgxdlHEQq34
+-> ssh-ed25519 BVsyTA KNSZgJezH8bUbpFOWiyBN9kPL6EvG/L7Yh9ZRGUJkzg
+Fb4oMWqk3OfdKFkLd8qq2wGvq9Fz1D4A9HmA5a412r8
+-> ssh-ed25519 +3V2lQ z3vxaJYUXcqI6f6U85Oj0u6cqyarKTLidDHsURqaTh0
+HNC+nhMbrJOUUS5SAcqJDDjwhjvRxOibo7Xx911cyOg
+--- 6hftMRn4kD/f/ixMq2T+VnXZwyfpcV7zxZ7PBAAcsDM
+Êü÷å5lŠk—9Ë¡zÉRÏÓ©õ×¨MáFM.º}ÊD§¨%ŒXŽºlu]7íÞç‹"\¼û<C2BC>êœ(}-¼â.åÐÿÛê25§‹>06ã
h'±^jˆK‹/5<16>Í
--- a/secrets/mail/crew.age
+++ b/secrets/mail/crew.age
@ -0,0 +1,43 @@
+age-encryption.org/v1
+-> ssh-ed25519 UE5Ceg qBHHVskxlk6AOCGIusKKItMQVrJpjpyWXBfcmpx6Bn8
+RDGWdLn/D8h+dKixRk39zrMFuoaqjdbnUX+CiRq+TSA
+-> ssh-ed25519 uYcDNw K4nqUOfxtA3GDpg32ndobWATCQBN2ylzD3wyLlnT2nQ
+hRPPtWcxI/paVmOHT3J5SS7Ov8+gvXDAqtceJFn7o+s
+-> ssh-rsa f5THog
+n+B7fmdbS+uwPFyHhBCNAAuCsGh6nzA3Q1ttF7vtadi2yw6P940XKB9hXnCe1btz
+NBRvKkVtIzRqc/5xDTqbDJivIYzFu8StofWv4xRBFzpA3P9r1qQV1lHwxOCfrsdd
+296KHvqWVo4rdhkbd9Cye7cxndr2AWs0Gwn1uNvM1WQjTzUWzuKy6UsVztEcsB0J
+4avT6+S+yxpKkMIyLqlbis/VYe/CDpPJGnxeG2GN8POVQpSdyBCEL32qkj07wR17
+9rZFWU5WKfIr0XXJkhq+ewNdJzQKfWDFEhHrZYrg8LxKYsOWhydRBVEHkWVXnLin
+CSD1Cv4VNHnqCycJ1Dv2Lq2n7SHoGMLPyC1UPJudmpY1Z5XIvWOu5uxvv0674mdN
+WxOXgZpitwpgcmMC6K4mBZtqI8yqMP1Gijupoj4hFK7YGqKdn6+Q6ZFsttL97I00
+lU22H1kf/Rxh0ZxMPiT1JcTwAZdOHIuRG6xPhVIx1hNUOmdUpg3YZa8dMKeA3Yjz
+7YL7ZaYkwsIhMh6w+3xWUiYNkWfmGffRq0DfXIzTkKzapQtQJGLOpeot4wPkW51q
+fHoJ2MNvlB3Yo5AveAkIaJpofjFFZgy9XVPGH2XSAFRez3hixXkV2rWiM+GJAAnQ
+z45H8qWfGnRKSjgqEKVPDlfFEiG78Dtzjtl4oW1gfbY
+-> ssh-rsa kFDS0A
+bZc7lDzI0kG/lY1reQtVjggoWfLj9/zz+BxmbZfisxsEE18AkYGsk/Ki9ddXFxDW
+5EIbCHheFBvkq7eb5OKcTUf3AFTch2/8dY1hnmR6uPq1Zwgl4ATCpcQPY85+7bPb
+GBl0msNpRHuo6um895rL4omdv+DItmMdp3Lyf+CcFRvaXOpRnFmOqgatZ1bMePx4
+qJajnToar4YIEJBzc53oGWdAHfcmVrvEdOIUNoS3QoyCmusCkMNrSfqmvPfwqsWt
+g+pTrI3NqmTt3+L0EawcRLjRYb/qM/L9/nSFOnYOv3hLzWOhwSQU/gr1ZKMxYnaI
+GxqWzWg2dvkuHlRKVwwf8mNBrZlqQDV/ydOeyjJUKe48jM/PsIj8NVsqRhkgHrkH
+/lvQClYEBhrgHc9Wdxzy4KM3DPyKCQSYxBPnZpFVzuFBKML/cnYU84i7r4Gkb/z4
+Jxwy6jxRzjt+Sou6gTP9dIASaYfMKYnf4ijB3IZLNApkNMBd0qt5qptTCG0LylDX
+eTGGWjKQrC11znI/PWkSJQsKuBDHesL+QmjgJBhPdpl7Tk9ZaI/rJk2KYAjF6J9V
+add0KsLxAZbqlFo1CJO8HHysCRljXob0jYefmnDXO2x8xZvt3eSzVa8JsNLcMv5w
+4/tAdHBfH4mifA5mVdVbeRUDby54TdfIWGAZtyhgvYg
+-> piv-p256 vRzPNw A/0edIuqR6hf5WE2qoSGqX18sbslgSxxgmDOc6wNqfQD
+GT94xHQpPOdNorZOaSi7EPdaqSSVjJNB2qaSYA6qZhY
+-> piv-p256 zqq/iw A5bQxOBbSgsr6+TL8bgNWl287IF8Zvec6k9oAZPgIRt2
+z0ygD5ZRl3WZjfVA3Aku70mKddTZZ/W9rX2XOBJ9cco
+-> ssh-ed25519 YFSOsg R487ufjbfae0x3wSAYH9d4Yz0dW/ze3wXxQI/DCFuWw
+klWo+lmfAMaZVo/gDz07/ht+szuA7YSpvDc0yEe0bgo
+-> ssh-ed25519 iHV63A Ond1kPLFFFIC/lSpv6K1uobvXYFmw+yVwNUTN1HIUVw
+ElzaC1ho8F2X2jRZtmAdY9FUMiCs5XAEcFqEPTy6Ilc
+-> ssh-ed25519 BVsyTA F9U4uSI1sNELggtM7/VwlYOlg+ghBg0xAQLux5Fmvw8
+4PY2p7QneYIuumlciTmEbR/DwBKVMXxsfRoSuSgfmR4
+-> ssh-ed25519 +3V2lQ 6i+WKf5wToBT5vne7ACy51BTAZrzMHCyiQ4D65m5Ol0
+/kt6I4forttfn8SbZ/9K2mvZRh4Cbj+JqmlZ746Pqqw
+--- ufN6THtH8xQ83XVERTJFwO8Ti0AJyflJwZtA8V2mba4
+g[Ä&‡à»¹î|þjG#¡—ßúíJÎ<0F>bÆ¤T<C2A4>c@EˆÝÉë}Œñ><3E>m¥÷÷ÜÅÿc™D*ÙMèÛ,(Ï”Ò6Cê‚¥´ÞÓkÇ’°é=¸â”f
Q‹
--- a/secrets/mail/erpnext.age
+++ b/secrets/mail/erpnext.age
@ -0,0 +1,43 @@
+age-encryption.org/v1
+-> ssh-ed25519 UE5Ceg I6uUuN8666FFZt7t0Z/EyWpTALPQKjGT8BBtjrJL8Ro
+4Cy7GJ3RQqmrDpYocWTx31MV8yg5QKUCEfMjAaBunnU
+-> ssh-ed25519 uYcDNw x+wqWbE6v2rzDZ8oDP8a/80yMBn5LI+aqBsUO7QktHU
+1s7d1LfdY7bhXi6PJMi67RfxPDF8UWcLpS5cQzuiPvg
+-> ssh-rsa f5THog
+JQDnaZPrI5bw7OSCOo2d+C/4KsXOa7Dt0140G3/Snv7j/DPxkz+hC+jxLlt/GIY5
+Py6bV/wqeS9HRUlReB9Lr+5Q89yOZhxqQI08zYnpmn6Ipr+ALNWy2jHKTBDHHPJ7
+LSuv46ppPRDnZoy6NEUIlaIQ5EOXAGGVGi6nhS/R5I/fJIF4yk7B7MKur5Mhj731
+Np7pb2yAfAZGxqleYO5I1jTLIGcBIDpmCricg8W057cdXFG9DG3P4Wvi+Q9bvSH8
+cQwhCscUsxwZN4uVUvIAeavo06JqqOio4N3XJAwzY3syPfKhQ0xdAIMiOhl0TYYc
+eVy7llsbtFd7PSu0FTFfWyuqOZNOmDoKghns3H7HCUeFcp0II1+LS0v6QKAJCEIR
+CVtkNbfM8SxFioGaUTwSfxWIy9+usSX8oHYp0SYKYjBCoukq/N01yZIxVVrXgROK
+FjEbyHCyIwnJ/UsrWh3TldwsDSKWbFogO66m9K0d0wJEq26UcVADQi2GLt1YCXgS
+klNjHAdX1oodhr2p0ZURxngYaWuwMgEOjsMtxyA4M+4nbXfF1ds/uj7i7Btn3R6b
+AzlOo+tVKg1iHFGMn5AUTOV7DtltaMxeWM24l3W9v677aozu7BDZQK5VwSSjyywF
+Vq5p0Rsdif1Vywg0+AUxsPyTy4YqTvXRfQviEU/k9Qg
+-> ssh-rsa kFDS0A
+IVW5AyRKdS2zzPPZLt0qLS5aqb4+C+tFgHfD0mVtrYadn9ugn11+Wk+HKdDko43z
+0rLdqE9q+Hyg3jCVk7DbnsL7lzfLKt6JQVfdCN2qihHLofPqqGgjC9pp8C48EjP/
+ND/S1nrSTq8A9jF2/oja+ofcQCKGZKGC3u8E3UUdC2rmDrQF1CRZ6bW6kUxbEh7n
+fogXy8BP4WX3/LxJxRwaUSQuYMrnA/SvCbQP50Z235xgr6v2+Hfm4KxmgBpy9YV1
+BCuuS0Rgkkipa4SkDg4BdEyWcbTu4JaXTZPJ/6UKdNS9wEGkIaCIENkGIkl7ViTk
+DDHjbGKMQD7nOv42Y9bQJwwcAEW3gN+g7kgD22GW9cpZEFTcGESX1tkYclZiZOIs
+IC63gYk0o5fEuLsCYoE0Jld0D9Ja7JYbVH/ukzJ99rWgcLLKgkC5pEosPa0kex1y
+L2+YDmSKtqSY3YjTFv8q4DVTBKeoWjNHkNaDl5IInhzbJ3k4zZAvJ5av02ws5aM9
+i7WYk+tARjK/Bsl4pEOq5UwdAlQBuAOWUMhjLjR7BN5tWtA/wrz0LfCctTjpwxSE
+vuIUIeJENpjIv88OAWVqR2SYqyTyLnHO0YpreWfF0nj1GTGY//XdwA/kqekhj8dZ
+U70iXnquIhqzuwkMSC2cq1WL78pmh8kkmDbIgk8y1tw
+-> piv-p256 vRzPNw AiRbeKSGWFJXI93xQ2+yh+CwJKIl6w9XFvaf1QMo8lSN
+XjzQLjfA9e88kyGeBlLWqhYGSkcFhbEp2G0mthdYRyU
+-> piv-p256 zqq/iw Ay5OxlqOR1CuTnrkdN0DbZXU0X3XbwKjj138AO3+GEGh
+UqBjfcB5Xj829ZgvWk5eJk/5kXNE1oXBxOIo46SEqz0
+-> ssh-ed25519 YFSOsg g11+RyINzDuZtkWMDhq03pXFK/sI0rrvu1nRgt2lTi0
+KwhWvcS4dGb6usaNScrRUFtzaAbIHYNziY+E5tq/QBQ
+-> ssh-ed25519 iHV63A 18otcJyCfFTil0bJHQzHbnS1MktjeryOSI1OZXypki4
+vq7Og0UJmDgclm/MRFw77uGOiOatgPRhlTeEH7kjuS8
+-> ssh-ed25519 BVsyTA ISv3vLZ8DHSiiNrRIFPB7YZqcMKkecuG4U7OPAj7hU8
+8ANZ3bmxLZT+i0QCRQ2I/KgcKsdv0YBLX5FoGSw+M6M
+-> ssh-ed25519 +3V2lQ qNtNUsgkHIHXGEIjzjPuF3xKLOfeSCeMrNrIdkpjmxU
+OyS0yUzVdtpG+A+OvKVyX8vl7dUKysIosb5b+1qdH/Q
+--- ptU7IkkyEOB/9kxpGyi6TS/nx4zIrRnvtCqGiZi0NII
+8TxŒ˜úvþàJÄüƒ)&»ÍÕìkü—Çñ´ï•ÄÜ²‚¨‘úM–&.N¸ƒ½`ÏòS¯8|Âµw|Éí®2me/ð,¿…ôÜ@´3}³pÝ.oÅµÐ>Gvzô/‡½
--- a/secrets/mail/hakkonaut.age
+++ b/secrets/mail/hakkonaut.age
--- a/secrets/mail/hensoko.age
+++ b/secrets/mail/hensoko.age
@ -0,0 +1,44 @@
+age-encryption.org/v1
+-> ssh-ed25519 UE5Ceg F7J2BMCNuOUcZhcbEyXBbFHkOI4sVA0qXbRmCWYNBAE
+Na/iuNS8cxz0qEiosflBEB9TAF87sQgwBbUl0/fhmZo
+-> ssh-ed25519 uYcDNw Xd8D3eCNMcXrxlYef4kj1N4CD16b5Xs3pfA/J8RJQDk
+UoBSRBj4wS1cxnDV37JjW5kBP2XWWo7seJJsU0y0cEA
+-> ssh-rsa f5THog
+OxPFa8NRWqy2ShVfYtxqZWfJAmgkYd2xg2E8vNCPoWafo/6hBob7C+4hDiKRZPZa
+EVLw0wgTe/nlMzBLOO3FlgZ0Ceb/uA2n4nu7st6mjwYQpsmVXwZoap88B2b+GYCs
+GG4sgybkZ/BrfFgm94TIcC1lr2lMjA6C4xhC9Mphf2iEQf1wjL4N1msOC4gTAW8Q
+zaH+K+qNEbTXne5Pox9wp6FjApSx33ldqRxOSzcf7RUuL2ew/63fTywW8ZdHcUgm
+usKqBZX9vyhLdsHzZWSXwetybMfKWs1ry5kU3ekf9EmAAkSiukFxFdr7PON3l+VV
+hNFxi7RBKGC2u+ZE2Oh/MdXkKHMIVuJE1yhUJyiirH9/Mj2S6gOpSL7pjXIQdbC
+RoGoE4fHWtp14Yn5X2YQCeGYPS+y87md9qKlVTzf29u95UjVkN4V8xwquOssWp/P
+qlBJscmU3cp+U3W4Gzh1k1IwdBQ7B26rUOFEwa2/DI8VsBd/x4WmLQGiIe0VnOIB
+YCekxeLrl4AAf/XTEc/qNTaXcn3OguMMq6KzyeWMTdKsrcw7/P7j+06SbK+Co57D
+7zt/h2dDeAEz1eo7yGLu/zd2s2iyEBNxnzvSqvRpYAkcNNI7DvNfdotDYWj0kbuW
+rKfPKnXOUvf9tKsjbd1BRI563TpcoL3ebnokhBfu+v4
+-> ssh-rsa kFDS0A
+k8vywS465lFJyN/RvPMx3OUSl3UG2phrlZ0QY9BL2Gqf79tiSqMrWFCKqeZ8Djg6
+yDNC8F62IwWSQB030iWQMhQfI3FM9BFepmMpVE3zviyg1WRTNgLl9vdpjLP4FuNi
+Il5S3T49RmUgAzsPGMs0UWLhEudm9tJOU3tI3XD32tG7mYVrMcimtog8/1zasFf1
+GE3H3MyBiuawfSu0uMnQ267rxYiGF75bI8Er1nI7zIF55Lw7twHLjN+KOlSed3Vk
+VU7tNeRKfbircTrfxXo0I6SVPuX21SfBP5RWq4KrO/h4chW36OLxza2eiRvy74lY
+/MekrH3PgO0q7y+uqeSbiGAcvL1UXeZFFdItv5pKxMC95vpdsEhoywO8Rj6dd+9q
+iQjmy5RS/HC6uDzbqAl0HQSq1fZXO3UO0fQg5Rv3whpKMBHVMTU/PVimP93oAu4J
+rXnUUpqpKJqecVDYQT4XSuMDK5Iw+S+7RLxBk6hIYsg0jtywqgwD+zF1S8RHi9kK
+BEX5mR3NC/B+LdHAzphYQkHuY6UOk5AcgMO5jYCLtVK4vqlvTJPVbTSgdO86rmdy
+nZXZmi0Uqgz8QEdOgIp0ego8WdqGkZF0aQwMUw11Bi+78Asx5+hy+fUncw0qZndZ
+04ayMacztVL0cEaQ1AeOf85z0MPOugcVYFvih/XkgjE
+-> piv-p256 vRzPNw AyKY9szzF5MMfOBUISqtfu4EVk3GWOQ2WSqwgn8tCE9B
+uoSrnNdzVP1WO3uZflc+Va6cT8y5AfUpm8P3njiSQzo
+-> piv-p256 zqq/iw Atu7Vk8b6dyNLZcLFtnOkAlYxOMN033PV/bv8O77LORR
+jbYx5/YXY6LwoFvOfXHHPhTiMOMLwgbENvFzFmGf6ak
+-> ssh-ed25519 YFSOsg BCuhqDI2VVkG3gk927TjEOLLOQNeURfxVbGodW/Xh2c
+lUEeZrF5FSC/e6XRxWNQq5B7oC70mKit56AIrWMTKCY
+-> ssh-ed25519 iHV63A Job9bw0T6OJpmgeizCOyNGqA9YHrcbml8sj+9kadKVw
+4+pfaDyrgXuj8DKQzMj04nk2KRfobvQ6Z+E7RDOUm24
+-> ssh-ed25519 BVsyTA 2cN+HWBYc7mSbSEziFpyuDfHs7cbVd5Vdfj7NYNJ6Uk
+8+APjCiQmu9hoqffuqdJKk09wtk0Ywa3NqeURnP+n+M
+-> ssh-ed25519 +3V2lQ h+MbnwkJqmQbk2gtkyWvU/8gqJHYIG90lUH3AMENonk
+wXsXHxzIsP9kSsi3mxmr5oujWL0Grj7y5inECZNSuIk
+--- hkrqXuu9Lldhr675cyYUX5peiFT2s5ZMjIrOi7oRIyw
+ê®è( <¾i0þøÃk$bL
+ø+ë©€¯ï¬]–†úß…ÑÇEÄ¢¦wêíÆÈ	»µ¬YÞ†é!0$šiôKÜà0DXæJdBÍÕ¦O.V×S¿‚ºd€Ä8çSƒ©¢
--- a/secrets/mail/teutat3s.age
+++ b/secrets/mail/teutat3s.age
@ -0,0 +1,43 @@
+age-encryption.org/v1
+-> ssh-ed25519 UE5Ceg NVteAXOZyA8sjXpRU5/ttHLFvGnzD1k48gWWd70erwM
+u57XR4AZoHLagd1/6aiYyz8jNSEtnEGp9Kc2kOHwq3o
+-> ssh-ed25519 uYcDNw CDCJGqbJfqR+8REsogbO7z2Uy4VDiWlLdd7FVUIHYn8
+OV7rjh5kzbGzwcKYsfgZX4jMP2pudlKEH8biFLvkeZU
+-> ssh-rsa f5THog
+DTPOjmtjwHBIOxCcvDSu2cJBd9GHBD+0t25w6CaU8lQl3v1ZJE8eOpxV9Bs3u07Q
+BTjPeGp2qyXxvlLQ7hrQfJyhO7pN+Ngk01MRppFN2t83XiHi6VdAHTwZfxndNt/e
+elP72j5octVrPVJVjNsZSJH92LyZlD4/PGtr31VdzW0/jvjB8bjXqQDEhlhs7Qz8
+9gVT380VmZv4HvXoSgyCT2I/Rmij3zaRX6JQVkKV4YuNcuqoAHCmcG5SgEtesot1
+h2+zH5lewQVB00Airi/hnYbTanyv41vmvdejT6yxrLyCMUGHjX8zbKzr+kXpmywo
+AMraBh47mknL0XKAvqwsVRWh5JZI75sWI51Vs0o8N4k7J4FXc6TOvB2o2yGj+C+8
+4cHLqC967jec2wmDdC0K645Bdm0BdZmp3f70NYb9ts4O5naooYCIRqSGgl11J9Nx
+vfGDVsg+FtMTbk3UN5kikoYltBnR4wOW5TWYeZ6NaB+VTkB++lcFVTS+TyN1ejhF
+H5N0QRhG5NaEuTaTuDESudgB3Rmi3nkKCcGLWPpPnrqV+ID9zsoC85DFHNjM8eVO
+hzeMQUStpwp/AMfJm94GoO+x+6xXocB4+2Mq1hnv3CkrEdCFQGhH6zSTJCrRDayq
+WD/bqtJ6twBmnh+jUPUBxlmz42bGTROznoXjC3slVxU
+-> ssh-rsa kFDS0A
+ap5x1yM55tQyJZRa5EewQwQlN/8FJXZ2JaZhAxP7TuKE0X5OqLqdh5sfF88vG2FT
+RwDImVDgAbCH/EN5DPRReW9XetmI/zC8vpXiqL5kNPh+pC9P46lVqsA9N4SE3AYk
+4XV7V8Z7MYS14vi0d8DFXNEBtwXAM0s4ZfOfEngkjUvOqRC9qCpSemMjrfNhvovP
+xjlwsh/LlEf7WAM+xPzNnMJEgs9sC3wp0+RdBZhjwSBiUp6lpmCZOcUyxKgwqfPU
+mSiQarTx8FZjurF/QZCAIyRGc5vs2mgQpHGOduWrPgLLwEgaWmOCz4ymdI60RJ0K
+qTD9EVDB8HO34+uPQWPvEJbtNL0KsEKjltGW661MJbQtqTIlChnzCsO79aqdqtGW
+wmOPGJJc3NMocVII/IA4mi2N/Ev5fnKK20Q8vQdsLW0WD3cm4zCPyIg+jiisC2by
+MRafMALkVBwTZYvjntv+l6Dlq6Q9IPfKPPi43UHWCv89yDrh19WxuM1e9lwYkWVl
+GUB9ncT89ETHm7IHzl4wtiogrTJbzFr9A/oBQqdIBvUYHP2HwPdDiPV9NCFHnWke
+4BzU8QUetQWDCvYreIxZobuJ2ig4SkBNsqrfb9ZQGS1lRqmkUk4J/38s8xAJpBR0
+KwzkEhJt5Dc92Q9RLlIW+QujLUEh9KjQPua/qb/1TYs
+-> piv-p256 vRzPNw AiWs1Nt6wGKVg0MqB7tHu8E6Wscj2Eo1xhxhB+/BZL2b
+pRjLl1Ds2dhLXVf4Im3Xzr3lG8vq+VJ1/EaPSAD5oiQ
+-> piv-p256 zqq/iw A64X3dQLMlgBuY3E+NRYn1TSs+CYq9JNDTgyMk3bTK79
+/tjhPEv0KwN5dH93zRvMFzBZRayjXQaQZjSHeW2etHE
+-> ssh-ed25519 YFSOsg a9MTVbDi1sA36SeVRnR51T4G2X6Wx1lx6VBI1bNsjFY
+UDUkvNwDXiuWc8XsVeFAW+WATZpKlJsKc+6i6ot7Pvk
+-> ssh-ed25519 iHV63A YwhQZF/lcI1OosRxfJ66wTcTctwcRa0/zY66U52G9VI
+HMHAI6FmX1DDq5z41/VomhCvRkJ9fIrxPEcO+aUIVp0
+-> ssh-ed25519 BVsyTA JKIbjoFUd8CNYCjYjxwaLersAaDp4yi/eN/KvTOhXkk
+1u9t02DQFgL6iN6e8HylV/tc7KpDlv/6hkulcNisrWk
+-> ssh-ed25519 +3V2lQ JJJAo2PVKGLTAFMPBGOSNfYEGEjkCPlRtxqBjFR9yDk
+PWm5uatk8fzhr4gK5XRgtdvTlzYRBUIEBfH6+CROyks
+--- FZl+1vvJBe49ofX4ncsNpdtzFmG0upDcJ3j0KUmXxbI
+ )À+K¢\5Óö4$*Í8ÄïÖ®£»ŒÙ‰ªxWõBÁà<C381>DÜ@æôIŒr¬HÖF}æ:ôÞeL	¤ÍáûýÌ•´¾qÓ½YûNCºùçíùŒ3uN–Z<E28093>Gža3Òû|
--- a/secrets/matrix-synapse-secret-config.yaml.age
+++ b/secrets/matrix-synapse-secret-config.yaml.age
--- a/secrets/metronom-wg-private-key.age
+++ b/secrets/metronom-wg-private-key.age
@ -0,0 +1,43 @@
+age-encryption.org/v1
+-> ssh-ed25519 UE5Ceg 1YUuuRDXFkGG2ZNYrRUro+Bx2GNGVTTCha+P9+T46DE
+gTxW/j5xNSxjSq5wze7fhNJm1SB5/YEizO65jG4Q9Tw
+-> ssh-ed25519 uYcDNw 7lGPy/ykR0Vnye8NYSBKcTRR2UzJ0lw2EXY6d/5gBjQ
+SHbqjmcN4TNzFbQb3AgHgzzm8Yhr0LHSFQHXMLyTDVM
+-> ssh-rsa f5THog
+IKJVe3MhHIFyivBHwYuf+COke576b1h0ARtu44ycuLSS71C2kteigviIwstXz97M
+GIHz9+aC0xJCa/gZ4WWZ5t5qO4XSmkIYCHPsV5UhjCEj6AAL27rP5oqXZKCTvPV6
+7bEw4dNJmVyjAGYP0h4M+HaAFwe8nlKO291lyJ3NoyZcMR+KjEFiBK22W0oEqvS6
+tvh3GgPp1iiHUvhF5uSUTxOqu30S7ogY1jtPLxQvEEJZwbXdCKZ/0BltfRGqKUWu
+DKBcKERUeEa+fSYRtxZqd0GGGOi0Xq3UKjTSmt5w58cBkrntbQeRTNYfnvvqXJJ7
+a0uRylsK2vnMjLXjlZryvL3ug+Ylpup/BuIMwzwpNEjasCqQt97v066Ho0qB0uej
+rwslyXSjwlOsvblf6UovUzQ3GIG17X9POOavsW6md7wxZFCNtioo+qb7fegKK5Tr
+W/H5GoB7g79pCbBUCMJP6MgPpMUVGH+5jDkWAQbik4lTH9ehD4Wu9V2hnyBub6fW
+CjEtrWzpwH+yHFkm7R5IjI8DWoE4CWsb8KI+GUgr2R3AjdNuXINbJy+ya+wpuMLh
+d5Q5tQbteQ2uBKJxXRrR8nNiiLqtQvRYsyF5G+BdXmAqAB0cBuH8yMmjUKju5tH9
+lSmdqUScCcVY11T6Hccath065f8Jtvwj3nJE9f2iPfo
+-> ssh-rsa kFDS0A
+RVoy79ijvAmU9XlEsbmiOOWUfenL+hITb6tXELUGjZjYIg+JPDneg7m1plUnRpBM
+sfLrTSzOLisWfct5rbXWb4QbNnD7biX0/uAPk8Jk3tmUfJsM1oLmNaRGGgo7RkFh
+J28PG0n5+eumauoS0Yf11GIgWUpC8FeVJMrNM5r4yV65EJEyyjRxFHjIGl5Jh6Rq
+bkJWpDsuFb2eb2BdZACV/M/aDYn+XGJW0oozNW91rryrQfsAHc3GzKoX2HtqNxua
+3Z348+NTS7jCKKhEwwNwibgTSz1PT2ynyaXi2N60KZ8IDc1xwtn1Ybj2/S1no64h
+P1GCjzKmwizgINoWQ8LYQ3nHxRXQjFdS4X63YUSXKcZ2TKMNydlB3IGL9N+xKflo
+w5EMqFTuHInpyOfz73WDg2LKuzlWabjn8KIlx2bYG8Etn5alSX+oQGD5zTUkDt4p
+/J3b8kLCdRSfVxwBudftXnk8CDg5gzM7LD0NOQ8/VK8lyTVE1dCCty1NUcM0o4mc
+VgdlcJn9ISZSd3UAt6BDUHEMYdxktJnlPr8Gsw1iDU44Gu2fPUY2OpmAnIz6FshR
+KkSThN08FL2EgEO99fbJ/8NiD+bml5duUNJQnjlQ8NC9w1S/4ADXpHSrJARQY0pn
+DfTvCz2CJnPqojb2vDb0knqvhPNLu1lmtrlyqMygmLg
+-> piv-p256 vRzPNw AlRMMj08FZgVJAcUdKDVtQzrrZWqOah1fq0xeLFOFYh/
+fySXnGSZYyKOX75bwaByIAqaiatXpFF4zsuE7JEH//c
+-> piv-p256 zqq/iw A7dI4n0fDq3z6OG/iuU8z4euPvx77lJJC9OlZG/RMPRc
+waoyEH8qBDeUmCugy7ZnMj6tgLx/1+slhJTAJ4uXMNQ
+-> ssh-ed25519 YFSOsg 99jNRmoZlrfV1ytKu8Pj41vBTNHED3dG99mjWnYe9Ec
+p+Q3Dik27t8LRb5Mr17EzVwxdSQIZBeO+ezJVvFqg00
+-> ssh-ed25519 iHV63A 1V4hJI/P7TkMWDbZb0NMdCSULS8XddPl6gGvc1gJ91I
+CKzsgmbASOGWYRFSyYBvY90HrmLfQNKcrTPLvf5m0es
+-> ssh-ed25519 BVsyTA tJu2Y42CtsqGMLf5VObT+nEMYHyujU2nmJQfWOTZsg8
+MGxxNMPHyRNRDVurqovUkptzqfsemX9mCLSLu0RL7b4
+-> ssh-ed25519 +3V2lQ vHPgK6xOUrH/1fqjkw2rhg10O0izPSTPX7b02v7J22A
+A/V11elKo6YNiFHYMQrWBnUTsaz21MNH9jcY78dTlmU
+--- QV+btlc1pzitb681enVVR/tT/kwE3s2sV1qB7yYJ/3Q
+Y¥DgIx,ìµ´âÙËœ!à¢ptë m•ŠÂòä"$ú•‚™€¿¦aZTÔ4'Äû`õejüÊúKøAÕ£t×WÚS÷&){i–_íSŽ
--- a/secrets/secrets.nix
+++ b/secrets/secrets.nix
@ -3,6 +3,7 @@ let

  nachtigall-host = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIP7G0ufi+MNvaAZLDgpieHrABPGN7e/kD5kMFwSk4ABj root@nachtigall";
  flora-6-host = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGP1InpTBN4AlF/4V8HHumAMLJzeO8DpzjUv9Co/+J09 root@flora-6";
+  metronom-host = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICLX6UvvrKALKL0xsNnytLPHryzZF5evUnxAgGokf14i root@metronom";
  tankstelle-host = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJdF6cJKPDiloWiDja1ZtqkXDdXOCHPs10HD+JMzgeU4 root@tankstelle";

  adminKeys = builtins.foldl' (
@ -14,6 +15,8 @@ let
  tankstelleKeys = [ tankstelle-host ];

  flora6Keys = [ flora-6-host ];
+
+  metronomKeys = [ metronom-host ];
 in
 {
  # ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBB5XaH02a6+TchnyQED2VwaltPgeFCbildbE2h6nF5e root@nachtigall
@ -22,6 +25,7 @@ in
  "nachtigall-wg-private-key.age".publicKeys = nachtigallKeys ++ adminKeys;
  "tankstelle-wg-private-key.age".publicKeys = tankstelleKeys ++ adminKeys;
  "flora6-wg-private-key.age".publicKeys = flora6Keys ++ adminKeys;
+  "metronom-wg-private-key.age".publicKeys = metronomKeys ++ adminKeys;

  "mastodon-secret-key-base.age".publicKeys = nachtigallKeys ++ adminKeys;
  "mastodon-otp-secret.age".publicKeys = nachtigallKeys ++ adminKeys;
@ -72,4 +76,13 @@ in

  "obs-portal-env.age".publicKeys = nachtigallKeys ++ adminKeys;
  "obs-portal-database-env.age".publicKeys = nachtigallKeys ++ adminKeys;
+
+  # mail
+  "mail/hensoko.age".publicKeys = metronomKeys ++ adminKeys;
+  "mail/teutat3s.age".publicKeys = metronomKeys ++ adminKeys;
+  "mail/admins.age".publicKeys = metronomKeys ++ adminKeys;
+  "mail/bot.age".publicKeys = metronomKeys ++ adminKeys;
+  "mail/crew.age".publicKeys = metronomKeys ++ adminKeys;
+  "mail/erpnext.age".publicKeys = metronomKeys ++ adminKeys;
+  "mail/hakkonaut.age".publicKeys = metronomKeys ++ adminKeys;
 }
--- a/terraform/dns.tf
+++ b/terraform/dns.tf
@ -9,6 +9,16 @@ resource "namecheap_domain_records" "pub-solar" {
    type     = "A"
    address  = "80.71.153.210"
  }
+  record {
+    hostname = "metronom"
+    type     = "A"
+    address  = "49.13.236.167"
+  }
+  record {
+    hostname = "mail"
+    type     = "A"
+    address  = "49.13.236.167"
+  }
  record {
    hostname = "auth"
    type     = "CNAME"
@ -143,7 +153,7 @@ resource "namecheap_domain_records" "pub-solar" {
  record {
    hostname = "@"
    type     = "TXT"
-    address  = "v=spf1 include:spf.greenbaum.zone a:list.pub.solar ~all"
+    address  = "v=spf1 a:mail.pub.solar a:list.pub.solar ~all"
  }
  record {
    hostname = "list"
@ -160,6 +170,11 @@ resource "namecheap_domain_records" "pub-solar" {
    type     = "TXT"
    address  = "v=DMARC1; p=reject;"
  }
+  record {
+    hostname = "mail._domainkey"
+    type     = "TXT"
+    address  = "v=DKIM1;k=rsa;p=MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDI333HhjmVmDYc5hYTtmB6o9KYb782xw+ewH1eQlpFcCMyJ1giYFeGKviNki9uSm52tk34zUIthsqJMRlz2WsKGgk4oq3MRtgPtogxbh1ipJlynXejPU5WVetjjMnwr6AtV1DP1Sv4n5Vz0EV8cTi3tRZdgYpG6hlriiHXbrvlIwIDAQAB"
+  }
  record {
    hostname = "modoboa._domainkey"
    type     = "TXT"
@ -168,7 +183,7 @@ resource "namecheap_domain_records" "pub-solar" {
  record {
    hostname = "@"
    type     = "MX"
-    address  = "mail.greenbaum.zone."
+    address  = "mail.pub.solar."
    mx_pref  = "0"
  }
  record {