Merge pull request 'Init mail.pub.solar' (#196) from feat/mail into main

Reviewed-on: pub-solar/infra#196
Reviewed-by: teutat3s <teutat3s@noreply.git.pub.solar>

docs/deploying.md

@@ -10,13 +10,19 @@ Then, run `deploy-rs` with the hostname of the server you want to deploy:
 For nachtigall.pub.solar:
 
 ```
-deploy --targets '.#nachtigall' --magic-rollback false --auto-rollback false
+deploy --targets '.#nachtigall' --magic-rollback false --auto-rollback false --keep-result --result-path ./results
 ```
 
 For flora-6.pub.solar:
 
 ```
-deploy --targets '.#flora-6' --magic-rollback false --auto-rollback false
+deploy --targets '.#flora-6' --magic-rollback false --auto-rollback false --keep-result --result-path ./results
+```
+
+For metronom.pub.solar (aarch64-linux):
+
+```
+deploy --targets '.#metronom' --magic-rollback false --auto-rollback false --keep-result --result-path ./results --remote-build
 ```
 
 Usually we skip all rollback functionality, but if you want to deploy a change
@@ -28,6 +34,11 @@ deployment, add the flag `--skip-checks` at the end of the command.
 `--dry-activate` can be used to only put all files in place without switching,
 to enable switching to the new config quickly at a later moment.
 
+We use `--keep-result --result-path ./results` to keep the last `result`
+symlink of each `deploy` from being garbage collected. That way, we keep builds
+cached in the Nix store. This is optional and both flags can be removed if disk
+space is a scarce resource on your machine.
+
 You'll need to have SSH Access to the boxes to be able to run `deploy`.
 
 ### Getting SSH access

docs/mail.md

@@ -0,0 +1,4 @@
+### Mail
+
+mail.pub.solar aka metronom.pub.solar hosts our internal mails.
+This is a small Hetzner cloud instance on https://console.hetzner.cloud.

docs/unlocking-root.md

@@ -1,9 +1,17 @@
 # Unlocking the root partition on boot
 
-After a boot, the encrypted root partition will have to be unlocked. This is done by accessing the server via SSH with user root on port 2222.
+After a reboot, the encrypted ZFS pool will have to be unlocked. This is done by accessing the server via SSH with user `root` on port 2222.
+
+Nachtigall:
 
 ```
-ssh root@nachtigall.pub.solar -p2222
+ssh root@138.201.80.102 -p2222
+```
+
+Metronom:
+
+```
+ssh root@49.13.236.167 -p2222
 ```
 
 After connecting, paste the crypt passphrase you can find in the shared keepass. This will disconnect the SSH session right away and the server will keep booting into stage 2.

flake.lock

@@ -27,6 +27,22 @@
         "type": "github"
       }
     },
+    "blobs": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1604995301,
+        "narHash": "sha256-wcLzgLec6SGJA8fx1OEN1yV/Py5b+U5iyYpksUY/yLw=",
+        "owner": "simple-nixos-mailserver",
+        "repo": "blobs",
+        "rev": "2cccdf1ca48316f2cfd1c9a0017e8de5a7156265",
+        "type": "gitlab"
+      },
+      "original": {
+        "owner": "simple-nixos-mailserver",
+        "repo": "blobs",
+        "type": "gitlab"
+      }
+    },
     "deploy-rs": {
       "inputs": {
         "flake-compat": "flake-compat",
@@ -128,6 +144,22 @@
         "type": "github"
       }
     },
+    "flake-compat_2": {
+      "flake": false,
+      "locked": {
+        "lastModified": 1668681692,
+        "narHash": "sha256-Ht91NGdewz8IQLtWZ9LCeNXMSXHUss+9COoqu6JLmXU=",
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "rev": "009399224d5e398d03b22badca40a37ac85412a1",
+        "type": "github"
+      },
+      "original": {
+        "owner": "edolstra",
+        "repo": "flake-compat",
+        "type": "github"
+      }
+    },
     "flake-parts": {
       "inputs": {
         "nixpkgs-lib": "nixpkgs-lib"
@@ -328,6 +360,21 @@
         "type": "github"
       }
     },
+    "nixpkgs-23_05": {
+      "locked": {
+        "lastModified": 1704290814,
+        "narHash": "sha256-LWvKHp7kGxk/GEtlrGYV68qIvPHkU9iToomNFGagixU=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "70bdadeb94ffc8806c0570eb5c2695ad29f0e421",
+        "type": "github"
+      },
+      "original": {
+        "id": "nixpkgs",
+        "ref": "nixos-23.05",
+        "type": "indirect"
+      }
+    },
     "nixpkgs-lib": {
       "locked": {
         "lastModified": 1714640452,
@@ -340,6 +387,21 @@
         "url": "https://github.com/NixOS/nixpkgs/archive/50eb7ecf4cd0a5756d7275c8ba36790e5bd53e33.tar.gz"
       }
     },
+    "nixpkgs_2": {
+      "locked": {
+        "lastModified": 1705856552,
+        "narHash": "sha256-JXfnuEf5Yd6bhMs/uvM67/joxYKoysyE3M2k6T3eWbg=",
+        "owner": "NixOS",
+        "repo": "nixpkgs",
+        "rev": "612f97239e2cc474c13c9dafa0df378058c5ad8d",
+        "type": "github"
+      },
+      "original": {
+        "id": "nixpkgs",
+        "ref": "nixos-unstable",
+        "type": "indirect"
+      }
+    },
     "root": {
       "inputs": {
         "agenix": "agenix",
@@ -354,10 +416,37 @@
         "nixos-flake": "nixos-flake",
         "nixpkgs": "nixpkgs",
         "nixpkgs-2205": "nixpkgs-2205",
+        "simple-nixos-mailserver": "simple-nixos-mailserver",
         "triton-vmtools": "triton-vmtools",
         "unstable": "unstable"
       }
     },
+    "simple-nixos-mailserver": {
+      "inputs": {
+        "blobs": "blobs",
+        "flake-compat": "flake-compat_2",
+        "nixpkgs": "nixpkgs_2",
+        "nixpkgs-23_05": "nixpkgs-23_05",
+        "nixpkgs-23_11": [
+          "nixpkgs"
+        ],
+        "utils": "utils_2"
+      },
+      "locked": {
+        "lastModified": 1706219574,
+        "narHash": "sha256-qO+8UErk+bXCq2ybHU4GzXG4Ejk4Tk0rnnTPNyypW4g=",
+        "owner": "simple-nixos-mailserver",
+        "repo": "nixos-mailserver",
+        "rev": "e47f3719f1db3e0961a4358d4cb234a0acaa7baf",
+        "type": "gitlab"
+      },
+      "original": {
+        "owner": "simple-nixos-mailserver",
+        "ref": "nixos-23.11",
+        "repo": "nixos-mailserver",
+        "type": "gitlab"
+      }
+    },
     "systems": {
       "locked": {
         "lastModified": 1681028828,
@@ -475,6 +564,21 @@
         "repo": "flake-utils",
         "type": "github"
       }
+    },
+    "utils_2": {
+      "locked": {
+        "lastModified": 1605370193,
+        "narHash": "sha256-YyMTf3URDL/otKdKgtoMChu4vfVL3vCMkRqpGifhUn0=",
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "rev": "5021eac20303a61fafe17224c087f5519baed54d",
+        "type": "github"
+      },
+      "original": {
+        "owner": "numtide",
+        "repo": "flake-utils",
+        "type": "github"
+      }
     }
   },
   "root": "root",

flake.nix

@@ -38,6 +38,9 @@
     element-stickers.url = "git+https://git.pub.solar/pub-solar/maunium-stickerpicker-nix?ref=main";
     element-stickers.inputs.maunium-stickerpicker.follows = "maunium-stickerpicker";
     element-stickers.inputs.nixpkgs.follows = "nixpkgs";
+
+    simple-nixos-mailserver.url = "gitlab:simple-nixos-mailserver/nixos-mailserver/nixos-23.11";
+    simple-nixos-mailserver.inputs.nixpkgs-23_11.follows = "nixpkgs";
   };
 
   outputs =
@@ -123,6 +126,10 @@
               hostname = "10.7.6.2";
               sshUser = username;
             };
+            metronom = {
+              hostname = "10.7.6.3";
+              sshUser = username;
+            };
             tankstelle = {
               hostname = "80.244.242.5";
               sshUser = username;

hosts/default.nix

@@ -59,6 +59,19 @@
         ];
       };
 
+      metronom = self.nixos-flake.lib.mkLinuxSystem {
+        imports = [
+          self.inputs.agenix.nixosModules.default
+          self.nixosModules.home-manager
+          ./metronom
+          self.nixosModules.overlays
+          self.nixosModules.unlock-zfs-on-boot
+          self.nixosModules.core
+
+          self.inputs.simple-nixos-mailserver.nixosModule
+        ];
+      };
+
       tankstelle = self.nixos-flake.lib.mkLinuxSystem {
         imports = [
           self.inputs.agenix.nixosModules.default

hosts/metronom/backups.nix

@@ -0,0 +1,13 @@
+{ flake, ... }:
+{
+  age.secrets."restic-repo-droppie" = {
+    file = "${flake.self}/secrets/restic-repo-droppie.age";
+    mode = "400";
+    owner = "root";
+  };
+  age.secrets."restic-repo-storagebox" = {
+    file = "${flake.self}/secrets/restic-repo-storagebox.age";
+    mode = "400";
+    owner = "root";
+  };
+}

hosts/metronom/configuration.nix

@@ -0,0 +1,34 @@
+{
+  flake,
+  config,
+  pkgs,
+  ...
+}:
+{
+  boot.loader.systemd-boot.enable = true;
+  boot.supportedFilesystems = [ "zfs" ];
+
+  boot.kernelParams = [
+    "boot.shell_on_fail=1"
+    "ip=dhcp"
+  ];
+
+  boot.initrd.availableKernelModules = [ "igb" ];
+
+  # https://nixos.wiki/wiki/ZFS#declarative_mounting_of_ZFS_datasets
+  systemd.services.zfs-mount.enable = false;
+
+  # Declarative SSH private key
+  #age.secrets."metronom-root-ssh-key" = {
+  #  file = "${flake.self}/secrets/metronom-root-ssh-key.age";
+  #  path = "/root/.ssh/id_ed25519";
+  #  mode = "400";
+  #  owner = "root";
+  #};
+
+  # This value determines the NixOS release with which your system is to be
+  # compatible, in order to avoid breaking some software such as database
+  # servers. You should change this only after NixOS release notes say you
+  # should.
+  system.stateVersion = "23.11"; # Did you read the comment?
+}

hosts/metronom/default.nix

@@ -0,0 +1,12 @@
+{ flake, ... }:
+
+{
+  imports = [
+    ./hardware-configuration.nix
+    ./configuration.nix
+
+    ./networking.nix
+    ./wireguard.nix
+    #./backups.nix
+  ];
+}

hosts/metronom/hardware-configuration.nix

@@ -0,0 +1,48 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{
+  config,
+  lib,
+  pkgs,
+  modulesPath,
+  ...
+}:
+
+{
+  imports = [ (modulesPath + "/profiles/qemu-guest.nix") ];
+
+  boot.initrd.availableKernelModules = [
+    "xhci_pci"
+    "virtio_pci"
+    "virtio_scsi"
+    "usbhid"
+    "sr_mod"
+  ];
+  boot.initrd.kernelModules = [ ];
+  boot.kernelModules = [ ];
+  boot.extraModulePackages = [ ];
+
+  fileSystems."/" = {
+    device = "root_pool/root";
+    fsType = "zfs";
+  };
+
+  fileSystems."/boot" = {
+    device = "/dev/disk/by-uuid/2083-C68E";
+    fsType = "vfat";
+  };
+
+  swapDevices = [ ];
+
+  # Enables DHCP on each ethernet and wireless interface. In case of scripted networking
+  # (the default) this is the recommended approach. When using systemd-networkd it's
+  # still possible to use this option, but it's recommended to use it in conjunction
+  # with explicit per-interface declarations with `networking.interfaces.<interface>.useDHCP`.
+  networking.useDHCP = lib.mkDefault true;
+  # networking.interfaces.eth0.useDHCP = lib.mkDefault true;
+
+  nixpkgs.hostPlatform = lib.mkDefault "aarch64-linux";
+  powerManagement.cpuFreqGovernor = lib.mkDefault "ondemand";
+  hardware.cpu.amd.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
+}

hosts/metronom/networking.nix

@@ -0,0 +1,23 @@
+{
+  config,
+  pkgs,
+  flake,
+  ...
+}:
+{
+
+  networking.hostName = "metronom";
+  networking.extraHosts = ''
+    127.0.0.2 mail.pub.solar mail
+    ::1 mail.pub.solar mail
+  '';
+  networking.domain = "pub.solar";
+  networking.hostId = "00000002";
+
+  networking.enableIPv6 = true;
+  networking.useDHCP = false;
+  networking.interfaces."enp1s0".useDHCP = true;
+
+  # TODO: ssh via wireguard only
+  services.openssh.openFirewall = true;
+}

hosts/metronom/wireguard.nix

@@ -0,0 +1,54 @@
+{
+  config,
+  pkgs,
+  flake,
+  ...
+}:
+{
+  networking.firewall.allowedUDPPorts = [ 51820 ];
+
+  age.secrets.wg-private-key.file = "${flake.self}/secrets/metronom-wg-private-key.age";
+
+  networking.wireguard.interfaces = {
+    wg-ssh = {
+      listenPort = 51820;
+      mtu = 1300;
+      ips = [
+        "10.7.6.3/32"
+        "fd00:fae:fae:fae:fae:3::/96"
+      ];
+      privateKeyFile = config.age.secrets.wg-private-key.path;
+      peers = flake.self.logins.admins.wireguardDevices ++ [
+        {
+          # flora-6.pub.solar
+          endpoint = "80.71.153.210:51820";
+          publicKey = "jtSR5G2P/nm9s8WrVc26Xc/SQLupRxyXE+5eIeqlsTU=";
+          allowedIPs = [
+            "10.7.6.2/32"
+            "fd00:fae:fae:fae:fae:2::/96"
+          ];
+        }
+        {
+          # nachtigall.pub.solar
+          endpoint = "138.201.80.102:51820";
+          publicKey = "qzNywKY9RvqTnDO8eLik75/SHveaSk9OObilDzv+xkk=";
+          allowedIPs = [
+            "10.7.6.1/32"
+            "fd00:fae:fae:fae:fae:1::/96"
+          ];
+        }
+      ];
+    };
+  };
+
+  services.openssh.listenAddresses = [
+    {
+      addr = "10.7.6.3";
+      port = 22;
+    }
+    {
+      addr = "[fd00:fae:fae:fae:fae:3::]";
+      port = 22;
+    }
+  ];
+}

lib/deploy.nix

@@ -7,21 +7,6 @@
 
 { lib, inputs }:
 let
-  # https://github.com/serokell/deploy-rs#overall-usage
-  system = "x86_64-linux";
-  pkgs = import inputs.nixpkgs { inherit system; };
-  deployPkgs = import inputs.nixpkgs {
-    inherit system;
-    overlays = [
-      inputs.deploy-rs.overlay
-      (self: super: {
-        deploy-rs = {
-          inherit (pkgs) deploy-rs;
-          lib = super.deploy-rs.lib;
-        };
-      })
-    ];
-  };
   getFqdn =
     c:
     let
@@ -66,7 +51,28 @@ in
     */
     lib.recursiveUpdate (lib.mapAttrs (_: c: {
       hostname = getFqdn c;
-      profiles.system = {
+      profiles.system =
+        let
+          system = c.pkgs.system;
+
+          # Unmodified nixpkgs
+          pkgs = import inputs.nixpkgs { inherit system; };
+
+          # nixpkgs with deploy-rs overlay but force the nixpkgs package
+          deployPkgs = import inputs.nixpkgs {
+            inherit system;
+            overlays = [
+              inputs.deploy-rs.overlay # or deploy-rs.overlays.default
+              (self: super: {
+                deploy-rs = {
+                  inherit (pkgs) deploy-rs;
+                  lib = super.deploy-rs.lib;
+                };
+              })
+            ];
+          };
+        in
+        {
           user = "root";
           path = deployPkgs.deploy-rs.lib.activate.nixos c;
         };

modules/forgejo/default.nix

@@ -94,7 +94,7 @@
       mailer = {
         ENABLED = true;
         PROTOCOL = "smtps";
-        SMTP_ADDR = "mail.greenbaum.zone";
+        SMTP_ADDR = "mail.pub.solar";
         SMTP_PORT = 465;
         FROM = ''"pub.solar git server" <forgejo@pub.solar>'';
         USER = "admins@pub.solar";

modules/grafana/default.nix

@@ -59,7 +59,7 @@
       };
       smtp = {
         enabled = true;
-        host = "mail.greenbaum.zone:465";
+        host = "mail.pub.solar:465";
         user = "admins@pub.solar";
         password = "\$__file{${config.age.secrets.grafana-smtp-password.path}}";
         from_address = "no-reply@pub.solar";

modules/mail/default.nix

@@ -0,0 +1,70 @@
+{ config, flake, ... }:
+
+{
+  age.secrets.mail-hensoko.file = "${flake.self}/secrets/mail/hensoko.age";
+  age.secrets.mail-teutat3s.file = "${flake.self}/secrets/mail/teutat3s.age";
+  age.secrets.mail-admins.file = "${flake.self}/secrets/mail/admins.age";
+  age.secrets.mail-bot.file = "${flake.self}/secrets/mail/bot.age";
+  age.secrets.mail-crew.file = "${flake.self}/secrets/mail/crew.age";
+  age.secrets.mail-erpnext.file = "${flake.self}/secrets/mail/erpnext.age";
+  age.secrets.mail-hakkonaut.file = "${flake.self}/secrets/mail/hakkonaut.age";
+
+  mailserver = {
+    enable = true;
+    fqdn = "mail.pub.solar";
+    domains = [ "pub.solar" ];
+
+    # A list of all login accounts. To create the password hashes, use
+    # nix-shell -p mkpasswd --run 'mkpasswd -R11 -m bcrypt'
+    loginAccounts = {
+      "hensoko@pub.solar" = {
+        hashedPasswordFile = config.age.secrets.mail-hensoko.path;
+        quota = "2G";
+      };
+      "teutat3s@pub.solar" = {
+        hashedPasswordFile = config.age.secrets.mail-teutat3s.path;
+        quota = "2G";
+      };
+      "admins@pub.solar" = {
+        hashedPasswordFile = config.age.secrets.mail-admins.path;
+        quota = "2G";
+        aliases = [
+          "abuse@pub.solar"
+          "alerts@pub.solar"
+          "forgejo@pub.solar"
+          "keycloak@pub.solar"
+          "mastodon-notifications@pub.solar"
+          "matrix@pub.solar"
+          "postmaster@pub.solar"
+          "nextcloud@pub.solar"
+          "no-reply@pub.solar"
+          "security@pub.solar"
+        ];
+      };
+      "bot@pub.solar" = {
+        hashedPasswordFile = config.age.secrets.mail-bot.path;
+        quota = "2G";
+        aliases = [ "hackernews-bot@pub.solar" ];
+      };
+      "crew@pub.solar" = {
+        hashedPasswordFile = config.age.secrets.mail-crew.path;
+        quota = "2G";
+        aliases = [ "moderation@pub.solar" ];
+      };
+      "erpnext@pub.solar" = {
+        hashedPasswordFile = config.age.secrets.mail-erpnext.path;
+        quota = "2G";
+      };
+      "hakkonaut@pub.solar" = {
+        hashedPasswordFile = config.age.secrets.mail-hakkonaut.path;
+        quota = "2G";
+      };
+    };
+
+    # Use Let's Encrypt certificates. Note that this needs to set up a stripped
+    # down nginx and opens port 80.
+    certificateScheme = "acme-nginx";
+  };
+  security.acme.acceptTerms = true;
+  security.acme.defaults.email = "security@pub.solar";
+}

modules/mastodon/default.nix

@@ -60,7 +60,7 @@
     vapidPublicKeyFile = "/run/agenix/mastodon-vapid-public-key";
     smtp = {
       createLocally = false;
-      host = "mail.greenbaum.zone";
+      host = "mail.pub.solar";
       port = 587;
       authenticate = true;
       user = "admins@pub.solar";

modules/nextcloud/default.nix

@@ -63,7 +63,7 @@
       mail_smtpname = "admins@pub.solar";
       mail_smtpsecure = "tls";
       mail_smtpauth = 1;
-      mail_smtphost = "mail.greenbaum.zone";
+      mail_smtphost = "mail.pub.solar";
       mail_smtpport = "587";
 
       # This is to allow connections to collabora and keycloak, among other services

modules/prometheus/default.nix

@@ -129,7 +129,7 @@
                 send_resolved = true;
                 to = "admins@pub.solar";
                 from = "alerts@pub.solar";
-                smarthost = "mail.greenbaum.zone:465";
+                smarthost = "mail.pub.solar:465";
                 auth_username = "admins@pub.solar";
                 auth_password = "$SMTP_AUTH_PASSWORD";
                 require_tls = false;

secrets/mail/admins.age

@@ -0,0 +1,43 @@
+age-encryption.org/v1
+-> ssh-ed25519 UE5Ceg 6rewUSyj9mZOZp1Oi+DvWxj7u6r7HWUAnp/zSDLmZyA
+OLBPwlUCqlVZqrZaqT/sfzslgcYRViuTt9yzJZRPIPI
+-> ssh-ed25519 uYcDNw JNpKkljIQIPKR/KNG9AF/DxbJjYoMeQdhOjmpig2Q3c
+bxu5hEvJi0ip74WUJNJhm6pAfdvVlFBbyCwQKYPkUXo
+-> ssh-rsa f5THog
+0Im1QWg1IHp5nYfo0OK908ohS+Mo0Jyyyimq3sc6q5WoDUzufaMVYfgVpHJxasO/
+SrVAwE6QLcHuTBZPeyr1HZ7chyQiWT+Lepp/MXhgS8nDOkgJaSNxY35PO6W/qtpE
+rxkgdNZdB2Orqq0wHo0is5+pfZdcD7n6O4VoiayUh6kv5Brk98BUCHrydXMfJv26
+0Kzwg3s+/kDwOeVOt7uy6n5VPhcSLiJgQlK4t0HkPB2rUoD8dfyVqUZV3YmgCoJM
+Km1lCxaS96xKGnvt0HklYy0OX5S7ActBGpQJjcNLTl7sb2M/U0XAF7O8teSKzdq4
+ejKOnzMdxFB+qOSZ3fGzHbjxNDwxPqyps0yhm72rT5tww3wOzYZXUebn7LwNKVwU
+99mA0CR9W3wg3Thv4nwmsrycTMFHh9jvGRXOYgIqXNDoo2oqqkzLnS+N2fx6Wush
+SNziOeZkgb25h0wrehxmqsEOVjlSE6C59E40XlmSj+MJf6siDLQGpLShE4Fz1tyx
+GXASxlTNcJ8TY0N4UmozdWRW8pyTOtl1MhiuaHdYLQGvd3Zlwkr9C7pV6eVBxPyF
+agSqbSZXprY5owp17fUc7HQUu5AcNJyQtDstwqOTPbaJFNfPnyaHU61jt52sk468
+W2d1hZ9SYxiN32rjYV6py2SiuOvHIWMz3ODkvhxQdAM
+-> ssh-rsa kFDS0A
+TRrrVhtSIhhR9OXVAEwfmVn44a/LIaYJZWndqPAcAEhQp1Z3kPpolkxtKskz982G
+wQgSbzU3py4VRpXdy/FBttoEdBrhRMKG0z9N0szKlagfLA+DHQjTlaMn/UkxmO2S
+4AdwO8jEJVe26h6Y/3ne7N+/Ji8QKO6tKeNVapBKHYsJ8qqscgYW1WgKOAfJ3M6c
+6lyavfn2prTkM0xz6hMrywm1Is9ahM4vh39iLRAaVonFHmNJE+dAse8ijvKzjcYM
+KAiZtabdJkWwjD/3x513fU/o9DQCnBTHfM8KLb7DTPC9Ro1K//O7LjcG+WiaERSh
+0+dBZstMD7fQWEyJ/CgnRf54juZs2A7yBdrT9TcQtcgPKYk9QjFqHCmKB0R+TUaX
+nNh4h33i5V/8JfPRQTLz/YYFdG+kG5Hvucs9I2HN1n/vaHL9UIH3zC8BmkUd5fnR
+cnKXPjFCfrPPKg4DMT4gT5lIVtIBRx/IKxvjgR/8c8M9M3jk4SZSYHUlKtnzFOLq
+ycGJopWX7kBWGliEQ8jC+nKYOXpSYH+mbHOV54zplmNOZKMdLJ9ek23WoX5/BD7i
+arp4EtwYiD2LN3M1TG24gFW9VCY3Ofil6HAn5ySM9AMtIHwy/8srUBSCtdpWWGx+
+0fk+wGVu/5lCn51RPXl1L2YRloyx3giKvappuUcpho4
+-> piv-p256 vRzPNw AjkP6Dy1dEQ58LVB01S/1stB6JMpl+q3EuqHQp6RCfH9
+cePnQF/DS9AJx0MJArNi/5b6tncv46lKpu/1SIb5X7Y
+-> piv-p256 zqq/iw A7cNqXWWA3Zd4vccwwW/Wgfq5cCOjnIPq/Et0qpeQUMw
+p/e2OBgHoHA06WR4h3k1GK65u3qYH2YGPYQ10jz+pvQ
+-> ssh-ed25519 YFSOsg +Tl7z0DL81uPhdBuEJG+9qnZ6eoAzyZfvJ5FtrtyRUE
+nfVzlc5NoSxHv+2tM3D444kH9fCjUEYD+7wE2h83qYk
+-> ssh-ed25519 iHV63A FgYN6w2aRUPpBBp6lV8pqSyopRaWwzhkGXxncU83HVc
+PcNQ0P2ZGCnumKWuHVo0wwF3KCz13JadNkAHWgqIfbc
+-> ssh-ed25519 BVsyTA X/VL2A5AlbG1m6uTqbYDJTJj0wVrYGx5w/geJTpgQR4
+zwlsYTehOA3oK92zFN2J+HhgaX1zYd3MP0vQ3W751Co
+-> ssh-ed25519 +3V2lQ Hk8tcLh85helo+DXrRDhCHkDja+sEkM1CTz01s0SXDQ
+ftNhb63/JDulFgTukVu76XG2Dfcorbdt47EV6kqXw9g
+--- 37wAuChTQKbjj/RCIh7ZRB2GOf2kT1we3D4bQKevM3A
+ñ(=žÙ>¤jIM¡ÿú ”ÑyÂA¼|à“áʯNè<4E>„…‰f‰1Dgí5èËÛÜ	¬á߈ßg<C39F>6ì²#>ßÐ
%UjÃX›ŠÈ@ÌÑðG*ªNó™äÀå\çJE

secrets/mail/bot.age

@@ -0,0 +1,43 @@
+age-encryption.org/v1
+-> ssh-ed25519 UE5Ceg iKhPkRjtE/7UadHCdLoQR0/fe1LhVF9wSp5DQUw0hV8
+o8BmKJxLYcxml+hq7l57nWQ8xAQFrROcX/BDCpZW7YE
+-> ssh-ed25519 uYcDNw It3n9bvJCC+H+r5VRrtjrga1S1TkhiHUTGL/ltQbk0c
+h/98devoPCP18pYqK7KcXaDspMzQMtvs5YxsoyodDes
+-> ssh-rsa f5THog
+xVi9l7vg34PJaGhjOzOtPtoRMePzlvdYKjNnzCXLd0g6Y4JXQZMoKCeeWrO++rtY
+7/PDxJ0kJjJAEY7q2BnfV+87nmrGxFFerldDcEO9pP8/sN/u393WQpngb0tMNx6M
+cjhwv0Y9ygAb858G1NzvnALVZGmbUxX1JIsq8QDcoP3kz5JmonIKLM3b4LrO735I
+bfu3T+wTRebOHdC9SOhz6iuhyTnu/RmU9w22AKK/IL19z+11NJB2Xoejkfw0c6ZU
+cW25i3TdwmiJAZ+lCDJQyBXtLctDes1/e6HtOkXoJSKQA5QLfEtPeCMyBmE4y0pR
+z1DPiP0wMd37YR8dMXoYDRfo3EvsDJkNR0SDTZj86kio9e2sXA3OtIx8BLM0y01F
+0Vnh0FwpY9kclflboeY9w3Uq33/TCvy9aZ29XD+X7HGdqqiqxeo5rcAMXO9xAx3h
+2fIwdVyWYTnLt8TDOH9ZKDw8vausEITQM/D73AbVlLRKDnXTd+YTkYBgzU1rJtR0
+4FQK4PL2qkWYKEK7qDTp+Hrhc4vOnxURaLsdexTub/A/TXHhGAKPxpGBOcBbCjc5
+4mHSRQsDTbTNNE7bcDbkBiUcXAdlPgvEhfLmmBw8sho45M+krSeSd7V5CJ1NENhJ
+3SO92RqIuyGR48lmvsuN5js4uLS4ntoyQvnmIQIVSQI
+-> ssh-rsa kFDS0A
+EsW7RlBeeV69UwczFANtxqmz2Et2jpUL378UuMydlzRznbp/TJjrzCStMTOBEDyC
+SuADuvcvLf1WsVbf+rxRuFgte0YMiqUNlijN7tsOFg92odk8tHVwXEA71SW8/ZWh
+zFqUJ8pPFXPA6DEYMGmdNLV+tEx3YsUFCrTvhRIBGPCFbuYJj9Ta2xg0KK3uR5/l
+xziM5xxc7NtJGpW3dA/qFyneuY6gPm17PWav2l7gjAge/6FvLFzfev9TuF82iPgc
+RkCNgHZqClWLRO9b0af8FMGWIak6kr/mqao40net2azrFqMxmeQFLIKJSxa6Agz+
+UtlOND1COQwHrogQkHVuanBRRdUZzGk4QdW8MN49JPkvwvVPGS2XZrkE5m4k66Nu
+rfMtlcoSGSA+GIZXTDiDPLpfpYV/XDe4IoPTpLcivRNb8i75GwCT/5vD39Qmlyyc
+GHOX+v5JXh8WYpgvTEPDYE/oeKnsq27QT1wt8q0hKuHcRO4BcdPuiaSMnn0kjvLd
+o473b6cHE96F3cTKhXerLqeMFs1+DsJhrxYCmRikZot6Iz8H5GnqT82Me1by6cYt
++GDcuVLIB0OzWfI9ibZB0ueMM8UfrLeGDq8hSF5M0rDCbFc6ZzQw8PgI97PNaDGg
+FdIMho7IXEQKXMV7ueZ2/PiQEA8vfBWRnxGKFRQLOTY
+-> piv-p256 vRzPNw AjWew9VSba/AQKQ69l/4OhvZUT/bawt7AOSe4/LjanOI
+wHkZs8QQAOE69dq0d/2PAMgsi3xDBqEEvEFB7WKMC1Q
+-> piv-p256 zqq/iw AkKV76ktPNKCS/KidRxBHdRQmtH3BNO2kbBz408ZJ+wu
+S8KdsoVZUgvW7E4mlVFpp7/wxBarAPTEBqsYoBXar+M
+-> ssh-ed25519 YFSOsg SQt87e1+Lza1kqQl+AyqOu47+en8H2AbjCasMjDLfRE
+vBO3eKJPzagd9NdPmVG1SvO3x9rnf4H/8oddfCwpjLY
+-> ssh-ed25519 iHV63A a1iFLv3FlMcfq6p8+dKlFB9cDPC8RFVc9DxtpNIXU3c
+eQW7PJ+eGgp2loZTMUf40D8V3LNAinBSXgxdlHEQq34
+-> ssh-ed25519 BVsyTA KNSZgJezH8bUbpFOWiyBN9kPL6EvG/L7Yh9ZRGUJkzg
+Fb4oMWqk3OfdKFkLd8qq2wGvq9Fz1D4A9HmA5a412r8
+-> ssh-ed25519 +3V2lQ z3vxaJYUXcqI6f6U85Oj0u6cqyarKTLidDHsURqaTh0
+HNC+nhMbrJOUUS5SAcqJDDjwhjvRxOibo7Xx911cyOg
+--- 6hftMRn4kD/f/ixMq2T+VnXZwyfpcV7zxZ7PBAAcsDM
+Êü÷å5lŠk—9Ë¡zÉRÏÓ©õרMáFM.º}ÊD§¨%ŒXŽºlu]7íÞç‹"\¼û<C2BC>êœ(}-¼â.åÐÿÛê25§­‹>06ã
h'±^jˆK‹/5<16>Í

secrets/mail/crew.age

@@ -0,0 +1,43 @@
+age-encryption.org/v1
+-> ssh-ed25519 UE5Ceg qBHHVskxlk6AOCGIusKKItMQVrJpjpyWXBfcmpx6Bn8
+RDGWdLn/D8h+dKixRk39zrMFuoaqjdbnUX+CiRq+TSA
+-> ssh-ed25519 uYcDNw K4nqUOfxtA3GDpg32ndobWATCQBN2ylzD3wyLlnT2nQ
+hRPPtWcxI/paVmOHT3J5SS7Ov8+gvXDAqtceJFn7o+s
+-> ssh-rsa f5THog
+n+B7fmdbS+uwPFyHhBCNAAuCsGh6nzA3Q1ttF7vtadi2yw6P940XKB9hXnCe1btz
+NBRvKkVtIzRqc/5xDTqbDJivIYzFu8StofWv4xRBFzpA3P9r1qQV1lHwxOCfrsdd
+296KHvqWVo4rdhkbd9Cye7cxndr2AWs0Gwn1uNvM1WQjTzUWzuKy6UsVztEcsB0J
+4avT6+S+yxpKkMIyLqlbis/VYe/CDpPJGnxeG2GN8POVQpSdyBCEL32qkj07wR17
+9rZFWU5WKfIr0XXJkhq+ewNdJzQKfWDFEhHrZYrg8LxKYsOWhydRBVEHkWVXnLin
+CSD1Cv4VNHnqCycJ1Dv2Lq2n7SHoGMLPyC1UPJudmpY1Z5XIvWOu5uxvv0674mdN
+WxOXgZpitwpgcmMC6K4mBZtqI8yqMP1Gijupoj4hFK7YGqKdn6+Q6ZFsttL97I00
+lU22H1kf/Rxh0ZxMPiT1JcTwAZdOHIuRG6xPhVIx1hNUOmdUpg3YZa8dMKeA3Yjz
+7YL7ZaYkwsIhMh6w+3xWUiYNkWfmGffRq0DfXIzTkKzapQtQJGLOpeot4wPkW51q
+fHoJ2MNvlB3Yo5AveAkIaJpofjFFZgy9XVPGH2XSAFRez3hixXkV2rWiM+GJAAnQ
+z45H8qWfGnRKSjgqEKVPDlfFEiG78Dtzjtl4oW1gfbY
+-> ssh-rsa kFDS0A
+bZc7lDzI0kG/lY1reQtVjggoWfLj9/zz+BxmbZfisxsEE18AkYGsk/Ki9ddXFxDW
+5EIbCHheFBvkq7eb5OKcTUf3AFTch2/8dY1hnmR6uPq1Zwgl4ATCpcQPY85+7bPb
+GBl0msNpRHuo6um895rL4omdv+DItmMdp3Lyf+CcFRvaXOpRnFmOqgatZ1bMePx4
+qJajnToar4YIEJBzc53oGWdAHfcmVrvEdOIUNoS3QoyCmusCkMNrSfqmvPfwqsWt
+g+pTrI3NqmTt3+L0EawcRLjRYb/qM/L9/nSFOnYOv3hLzWOhwSQU/gr1ZKMxYnaI
+GxqWzWg2dvkuHlRKVwwf8mNBrZlqQDV/ydOeyjJUKe48jM/PsIj8NVsqRhkgHrkH
+/lvQClYEBhrgHc9Wdxzy4KM3DPyKCQSYxBPnZpFVzuFBKML/cnYU84i7r4Gkb/z4
+Jxwy6jxRzjt+Sou6gTP9dIASaYfMKYnf4ijB3IZLNApkNMBd0qt5qptTCG0LylDX
+eTGGWjKQrC11znI/PWkSJQsKuBDHesL+QmjgJBhPdpl7Tk9ZaI/rJk2KYAjF6J9V
+add0KsLxAZbqlFo1CJO8HHysCRljXob0jYefmnDXO2x8xZvt3eSzVa8JsNLcMv5w
+4/tAdHBfH4mifA5mVdVbeRUDby54TdfIWGAZtyhgvYg
+-> piv-p256 vRzPNw A/0edIuqR6hf5WE2qoSGqX18sbslgSxxgmDOc6wNqfQD
+GT94xHQpPOdNorZOaSi7EPdaqSSVjJNB2qaSYA6qZhY
+-> piv-p256 zqq/iw A5bQxOBbSgsr6+TL8bgNWl287IF8Zvec6k9oAZPgIRt2
+z0ygD5ZRl3WZjfVA3Aku70mKddTZZ/W9rX2XOBJ9cco
+-> ssh-ed25519 YFSOsg R487ufjbfae0x3wSAYH9d4Yz0dW/ze3wXxQI/DCFuWw
+klWo+lmfAMaZVo/gDz07/ht+szuA7YSpvDc0yEe0bgo
+-> ssh-ed25519 iHV63A Ond1kPLFFFIC/lSpv6K1uobvXYFmw+yVwNUTN1HIUVw
+ElzaC1ho8F2X2jRZtmAdY9FUMiCs5XAEcFqEPTy6Ilc
+-> ssh-ed25519 BVsyTA F9U4uSI1sNELggtM7/VwlYOlg+ghBg0xAQLux5Fmvw8
+4PY2p7QneYIuumlciTmEbR/DwBKVMXxsfRoSuSgfmR4
+-> ssh-ed25519 +3V2lQ 6i+WKf5wToBT5vne7ACy51BTAZrzMHCyiQ4D65m5Ol0
+/kt6I4forttfn8SbZ/9K2mvZRh4Cbj+JqmlZ746Pqqw
+--- ufN6THtH8xQ83XVERTJFwO8Ti0AJyflJwZtA8V2mba4
+g[Ä&‡໹î|þjG#¡—ßúíJÎ<0F>bƤT<C2A4>c@EˆÝÉë}Œñ><3E>m¥÷÷ÜÅÿc™D*ÙMèÛ,(Ï”Ò6Cê‚¥´ÞÓkÇ’°é=¸â”f
Q‹

secrets/mail/erpnext.age

@@ -0,0 +1,43 @@
+age-encryption.org/v1
+-> ssh-ed25519 UE5Ceg I6uUuN8666FFZt7t0Z/EyWpTALPQKjGT8BBtjrJL8Ro
+4Cy7GJ3RQqmrDpYocWTx31MV8yg5QKUCEfMjAaBunnU
+-> ssh-ed25519 uYcDNw x+wqWbE6v2rzDZ8oDP8a/80yMBn5LI+aqBsUO7QktHU
+1s7d1LfdY7bhXi6PJMi67RfxPDF8UWcLpS5cQzuiPvg
+-> ssh-rsa f5THog
+JQDnaZPrI5bw7OSCOo2d+C/4KsXOa7Dt0140G3/Snv7j/DPxkz+hC+jxLlt/GIY5
+Py6bV/wqeS9HRUlReB9Lr+5Q89yOZhxqQI08zYnpmn6Ipr+ALNWy2jHKTBDHHPJ7
+LSuv46ppPRDnZoy6NEUIlaIQ5EOXAGGVGi6nhS/R5I/fJIF4yk7B7MKur5Mhj731
+Np7pb2yAfAZGxqleYO5I1jTLIGcBIDpmCricg8W057cdXFG9DG3P4Wvi+Q9bvSH8
+cQwhCscUsxwZN4uVUvIAeavo06JqqOio4N3XJAwzY3syPfKhQ0xdAIMiOhl0TYYc
+eVy7llsbtFd7PSu0FTFfWyuqOZNOmDoKghns3H7HCUeFcp0II1+LS0v6QKAJCEIR
+CVtkNbfM8SxFioGaUTwSfxWIy9+usSX8oHYp0SYKYjBCoukq/N01yZIxVVrXgROK
+FjEbyHCyIwnJ/UsrWh3TldwsDSKWbFogO66m9K0d0wJEq26UcVADQi2GLt1YCXgS
+klNjHAdX1oodhr2p0ZURxngYaWuwMgEOjsMtxyA4M+4nbXfF1ds/uj7i7Btn3R6b
+AzlOo+tVKg1iHFGMn5AUTOV7DtltaMxeWM24l3W9v677aozu7BDZQK5VwSSjyywF
+Vq5p0Rsdif1Vywg0+AUxsPyTy4YqTvXRfQviEU/k9Qg
+-> ssh-rsa kFDS0A
+IVW5AyRKdS2zzPPZLt0qLS5aqb4+C+tFgHfD0mVtrYadn9ugn11+Wk+HKdDko43z
+0rLdqE9q+Hyg3jCVk7DbnsL7lzfLKt6JQVfdCN2qihHLofPqqGgjC9pp8C48EjP/
+ND/S1nrSTq8A9jF2/oja+ofcQCKGZKGC3u8E3UUdC2rmDrQF1CRZ6bW6kUxbEh7n
+fogXy8BP4WX3/LxJxRwaUSQuYMrnA/SvCbQP50Z235xgr6v2+Hfm4KxmgBpy9YV1
+BCuuS0Rgkkipa4SkDg4BdEyWcbTu4JaXTZPJ/6UKdNS9wEGkIaCIENkGIkl7ViTk
+DDHjbGKMQD7nOv42Y9bQJwwcAEW3gN+g7kgD22GW9cpZEFTcGESX1tkYclZiZOIs
+IC63gYk0o5fEuLsCYoE0Jld0D9Ja7JYbVH/ukzJ99rWgcLLKgkC5pEosPa0kex1y
+L2+YDmSKtqSY3YjTFv8q4DVTBKeoWjNHkNaDl5IInhzbJ3k4zZAvJ5av02ws5aM9
+i7WYk+tARjK/Bsl4pEOq5UwdAlQBuAOWUMhjLjR7BN5tWtA/wrz0LfCctTjpwxSE
+vuIUIeJENpjIv88OAWVqR2SYqyTyLnHO0YpreWfF0nj1GTGY//XdwA/kqekhj8dZ
+U70iXnquIhqzuwkMSC2cq1WL78pmh8kkmDbIgk8y1tw
+-> piv-p256 vRzPNw AiRbeKSGWFJXI93xQ2+yh+CwJKIl6w9XFvaf1QMo8lSN
+XjzQLjfA9e88kyGeBlLWqhYGSkcFhbEp2G0mthdYRyU
+-> piv-p256 zqq/iw Ay5OxlqOR1CuTnrkdN0DbZXU0X3XbwKjj138AO3+GEGh
+UqBjfcB5Xj829ZgvWk5eJk/5kXNE1oXBxOIo46SEqz0
+-> ssh-ed25519 YFSOsg g11+RyINzDuZtkWMDhq03pXFK/sI0rrvu1nRgt2lTi0
+KwhWvcS4dGb6usaNScrRUFtzaAbIHYNziY+E5tq/QBQ
+-> ssh-ed25519 iHV63A 18otcJyCfFTil0bJHQzHbnS1MktjeryOSI1OZXypki4
+vq7Og0UJmDgclm/MRFw77uGOiOatgPRhlTeEH7kjuS8
+-> ssh-ed25519 BVsyTA ISv3vLZ8DHSiiNrRIFPB7YZqcMKkecuG4U7OPAj7hU8
+8ANZ3bmxLZT+i0QCRQ2I/KgcKsdv0YBLX5FoGSw+M6M
+-> ssh-ed25519 +3V2lQ qNtNUsgkHIHXGEIjzjPuF3xKLOfeSCeMrNrIdkpjmxU
+OyS0yUzVdtpG+A+OvKVyX8vl7dUKysIosb5b+1qdH/Q
+--- ptU7IkkyEOB/9kxpGyi6TS/nx4zIrRnvtCqGiZi0NII
+8TxŒ˜úvþàJÄüƒ)&»ÍÕìk
ü—Çñ´ï•Äܲ‚¨‘úM–&.N¸ƒ½`ÏòS¯8|µw|Éí®2me/ð,¿…ôÜ@´3}³pÝ.oŵÐ>Gvzô/‡½

secrets/mail/hensoko.age

@@ -0,0 +1,44 @@
+age-encryption.org/v1
+-> ssh-ed25519 UE5Ceg F7J2BMCNuOUcZhcbEyXBbFHkOI4sVA0qXbRmCWYNBAE
+Na/iuNS8cxz0qEiosflBEB9TAF87sQgwBbUl0/fhmZo
+-> ssh-ed25519 uYcDNw Xd8D3eCNMcXrxlYef4kj1N4CD16b5Xs3pfA/J8RJQDk
+UoBSRBj4wS1cxnDV37JjW5kBP2XWWo7seJJsU0y0cEA
+-> ssh-rsa f5THog
+OxPFa8NRWqy2ShVfYtxqZWfJAmgkYd2xg2E8vNCPoWafo/6hBob7C+4hDiKRZPZa
+EVLw0wgTe/nlMzBLOO3FlgZ0Ceb/uA2n4nu7st6mjwYQpsmVXwZoap88B2b+GYCs
+GG4sgybkZ/BrfFgm94TIcC1lr2lMjA6C4xhC9Mphf2iEQf1wjL4N1msOC4gTAW8Q
+zaH+K+qNEbTXne5Pox9wp6FjApSx33ldqRxOSzcf7RUuL2ew/63fTywW8ZdHcUgm
+usKqBZX9vyhLdsHzZWSXwetybMfKWs1ry5kU3ekf9EmAAkSiukFxFdr7PON3l+VV
++hNFxi7RBKGC2u+ZE2Oh/MdXkKHMIVuJE1yhUJyiirH9/Mj2S6gOpSL7pjXIQdbC
+RoGoE4fHWtp14Yn5X2YQCeGYPS+y87md9qKlVTzf29u95UjVkN4V8xwquOssWp/P
+qlBJscmU3cp+U3W4Gzh1k1IwdBQ7B26rUOFEwa2/DI8VsBd/x4WmLQGiIe0VnOIB
+YCekxeLrl4AAf/XTEc/qNTaXcn3OguMMq6KzyeWMTdKsrcw7/P7j+06SbK+Co57D
+7zt/h2dDeAEz1eo7yGLu/zd2s2iyEBNxnzvSqvRpYAkcNNI7DvNfdotDYWj0kbuW
+rKfPKnXOUvf9tKsjbd1BRI563TpcoL3ebnokhBfu+v4
+-> ssh-rsa kFDS0A
+k8vywS465lFJyN/RvPMx3OUSl3UG2phrlZ0QY9BL2Gqf79tiSqMrWFCKqeZ8Djg6
+yDNC8F62IwWSQB030iWQMhQfI3FM9BFepmMpVE3zviyg1WRTNgLl9vdpjLP4FuNi
+Il5S3T49RmUgAzsPGMs0UWLhEudm9tJOU3tI3XD32tG7mYVrMcimtog8/1zasFf1
+GE3H3MyBiuawfSu0uMnQ267rxYiGF75bI8Er1nI7zIF55Lw7twHLjN+KOlSed3Vk
+VU7tNeRKfbircTrfxXo0I6SVPuX21SfBP5RWq4KrO/h4chW36OLxza2eiRvy74lY
+/MekrH3PgO0q7y+uqeSbiGAcvL1UXeZFFdItv5pKxMC95vpdsEhoywO8Rj6dd+9q
+iQjmy5RS/HC6uDzbqAl0HQSq1fZXO3UO0fQg5Rv3whpKMBHVMTU/PVimP93oAu4J
+rXnUUpqpKJqecVDYQT4XSuMDK5Iw+S+7RLxBk6hIYsg0jtywqgwD+zF1S8RHi9kK
+BEX5mR3NC/B+LdHAzphYQkHuY6UOk5AcgMO5jYCLtVK4vqlvTJPVbTSgdO86rmdy
+nZXZmi0Uqgz8QEdOgIp0ego8WdqGkZF0aQwMUw11Bi+78Asx5+hy+fUncw0qZndZ
+04ayMacztVL0cEaQ1AeOf85z0MPOugcVYFvih/XkgjE
+-> piv-p256 vRzPNw AyKY9szzF5MMfOBUISqtfu4EVk3GWOQ2WSqwgn8tCE9B
+uoSrnNdzVP1WO3uZflc+Va6cT8y5AfUpm8P3njiSQzo
+-> piv-p256 zqq/iw Atu7Vk8b6dyNLZcLFtnOkAlYxOMN033PV/bv8O77LORR
+jbYx5/YXY6LwoFvOfXHHPhTiMOMLwgbENvFzFmGf6ak
+-> ssh-ed25519 YFSOsg BCuhqDI2VVkG3gk927TjEOLLOQNeURfxVbGodW/Xh2c
+lUEeZrF5FSC/e6XRxWNQq5B7oC70mKit56AIrWMTKCY
+-> ssh-ed25519 iHV63A Job9bw0T6OJpmgeizCOyNGqA9YHrcbml8sj+9kadKVw
+4+pfaDyrgXuj8DKQzMj04nk2KRfobvQ6Z+E7RDOUm24
+-> ssh-ed25519 BVsyTA 2cN+HWBYc7mSbSEziFpyuDfHs7cbVd5Vdfj7NYNJ6Uk
+8+APjCiQmu9hoqffuqdJKk09wtk0Ywa3NqeURnP+n+M
+-> ssh-ed25519 +3V2lQ h+MbnwkJqmQbk2gtkyWvU/8gqJHYIG90lUH3AMENonk
+wXsXHxzIsP9kSsi3mxmr5oujWL0Grj7y5inECZNSuIk
+--- hkrqXuu9Lldhr675cyYUX5peiFT2s5ZMjIrOi7oRIyw
+ê®è( <¾i0þøÃk$bL
+ø+ë©€¯ï¬]–†úß…ÑÇEÄ¢¦wêíÆÈ	»µ¬YÞ†é!0$šiôKÜà0DXæJdBÍÕ¦O.V×S¿‚ºd€Ä8çSƒ©¢

secrets/mail/teutat3s.age

@@ -0,0 +1,43 @@
+age-encryption.org/v1
+-> ssh-ed25519 UE5Ceg NVteAXOZyA8sjXpRU5/ttHLFvGnzD1k48gWWd70erwM
+u57XR4AZoHLagd1/6aiYyz8jNSEtnEGp9Kc2kOHwq3o
+-> ssh-ed25519 uYcDNw CDCJGqbJfqR+8REsogbO7z2Uy4VDiWlLdd7FVUIHYn8
+OV7rjh5kzbGzwcKYsfgZX4jMP2pudlKEH8biFLvkeZU
+-> ssh-rsa f5THog
+DTPOjmtjwHBIOxCcvDSu2cJBd9GHBD+0t25w6CaU8lQl3v1ZJE8eOpxV9Bs3u07Q
+BTjPeGp2qyXxvlLQ7hrQfJyhO7pN+Ngk01MRppFN2t83XiHi6VdAHTwZfxndNt/e
+elP72j5octVrPVJVjNsZSJH92LyZlD4/PGtr31VdzW0/jvjB8bjXqQDEhlhs7Qz8
+9gVT380VmZv4HvXoSgyCT2I/Rmij3zaRX6JQVkKV4YuNcuqoAHCmcG5SgEtesot1
+h2+zH5lewQVB00Airi/hnYbTanyv41vmvdejT6yxrLyCMUGHjX8zbKzr+kXpmywo
+AMraBh47mknL0XKAvqwsVRWh5JZI75sWI51Vs0o8N4k7J4FXc6TOvB2o2yGj+C+8
+4cHLqC967jec2wmDdC0K645Bdm0BdZmp3f70NYb9ts4O5naooYCIRqSGgl11J9Nx
+vfGDVsg+FtMTbk3UN5kikoYltBnR4wOW5TWYeZ6NaB+VTkB++lcFVTS+TyN1ejhF
+H5N0QRhG5NaEuTaTuDESudgB3Rmi3nkKCcGLWPpPnrqV+ID9zsoC85DFHNjM8eVO
+hzeMQUStpwp/AMfJm94GoO+x+6xXocB4+2Mq1hnv3CkrEdCFQGhH6zSTJCrRDayq
+WD/bqtJ6twBmnh+jUPUBxlmz42bGTROznoXjC3slVxU
+-> ssh-rsa kFDS0A
+ap5x1yM55tQyJZRa5EewQwQlN/8FJXZ2JaZhAxP7TuKE0X5OqLqdh5sfF88vG2FT
+RwDImVDgAbCH/EN5DPRReW9XetmI/zC8vpXiqL5kNPh+pC9P46lVqsA9N4SE3AYk
+4XV7V8Z7MYS14vi0d8DFXNEBtwXAM0s4ZfOfEngkjUvOqRC9qCpSemMjrfNhvovP
+xjlwsh/LlEf7WAM+xPzNnMJEgs9sC3wp0+RdBZhjwSBiUp6lpmCZOcUyxKgwqfPU
+mSiQarTx8FZjurF/QZCAIyRGc5vs2mgQpHGOduWrPgLLwEgaWmOCz4ymdI60RJ0K
+qTD9EVDB8HO34+uPQWPvEJbtNL0KsEKjltGW661MJbQtqTIlChnzCsO79aqdqtGW
+wmOPGJJc3NMocVII/IA4mi2N/Ev5fnKK20Q8vQdsLW0WD3cm4zCPyIg+jiisC2by
+MRafMALkVBwTZYvjntv+l6Dlq6Q9IPfKPPi43UHWCv89yDrh19WxuM1e9lwYkWVl
+GUB9ncT89ETHm7IHzl4wtiogrTJbzFr9A/oBQqdIBvUYHP2HwPdDiPV9NCFHnWke
+4BzU8QUetQWDCvYreIxZobuJ2ig4SkBNsqrfb9ZQGS1lRqmkUk4J/38s8xAJpBR0
+KwzkEhJt5Dc92Q9RLlIW+QujLUEh9KjQPua/qb/1TYs
+-> piv-p256 vRzPNw AiWs1Nt6wGKVg0MqB7tHu8E6Wscj2Eo1xhxhB+/BZL2b
+pRjLl1Ds2dhLXVf4Im3Xzr3lG8vq+VJ1/EaPSAD5oiQ
+-> piv-p256 zqq/iw A64X3dQLMlgBuY3E+NRYn1TSs+CYq9JNDTgyMk3bTK79
+/tjhPEv0KwN5dH93zRvMFzBZRayjXQaQZjSHeW2etHE
+-> ssh-ed25519 YFSOsg a9MTVbDi1sA36SeVRnR51T4G2X6Wx1lx6VBI1bNsjFY
+UDUkvNwDXiuWc8XsVeFAW+WATZpKlJsKc+6i6ot7Pvk
+-> ssh-ed25519 iHV63A YwhQZF/lcI1OosRxfJ66wTcTctwcRa0/zY66U52G9VI
+HMHAI6FmX1DDq5z41/VomhCvRkJ9fIrxPEcO+aUIVp0
+-> ssh-ed25519 BVsyTA JKIbjoFUd8CNYCjYjxwaLersAaDp4yi/eN/KvTOhXkk
+1u9t02DQFgL6iN6e8HylV/tc7KpDlv/6hkulcNisrWk
+-> ssh-ed25519 +3V2lQ JJJAo2PVKGLTAFMPBGOSNfYEGEjkCPlRtxqBjFR9yDk
+PWm5uatk8fzhr4gK5XRgtdvTlzYRBUIEBfH6+CROyks
+--- FZl+1vvJBe49ofX4ncsNpdtzFmG0upDcJ3j0KUmXxbI
+ )À+K¢\5Óö4$*Í8ÄïÖ®£»ŒÙ‰ªxWõBÁà<C381>DÜ@æôIŒr¬HÖF}æ:ôÞeL	¤ÍáûýÌ•´¾qÓ½YûNCºùçíùŒ3uN–Z<E28093>Gža3Òû|

secrets/metronom-wg-private-key.age

@@ -0,0 +1,43 @@
+age-encryption.org/v1
+-> ssh-ed25519 UE5Ceg 1YUuuRDXFkGG2ZNYrRUro+Bx2GNGVTTCha+P9+T46DE
+gTxW/j5xNSxjSq5wze7fhNJm1SB5/YEizO65jG4Q9Tw
+-> ssh-ed25519 uYcDNw 7lGPy/ykR0Vnye8NYSBKcTRR2UzJ0lw2EXY6d/5gBjQ
+SHbqjmcN4TNzFbQb3AgHgzzm8Yhr0LHSFQHXMLyTDVM
+-> ssh-rsa f5THog
+IKJVe3MhHIFyivBHwYuf+COke576b1h0ARtu44ycuLSS71C2kteigviIwstXz97M
+GIHz9+aC0xJCa/gZ4WWZ5t5qO4XSmkIYCHPsV5UhjCEj6AAL27rP5oqXZKCTvPV6
+7bEw4dNJmVyjAGYP0h4M+HaAFwe8nlKO291lyJ3NoyZcMR+KjEFiBK22W0oEqvS6
+tvh3GgPp1iiHUvhF5uSUTxOqu30S7ogY1jtPLxQvEEJZwbXdCKZ/0BltfRGqKUWu
+DKBcKERUeEa+fSYRtxZqd0GGGOi0Xq3UKjTSmt5w58cBkrntbQeRTNYfnvvqXJJ7
+a0uRylsK2vnMjLXjlZryvL3ug+Ylpup/BuIMwzwpNEjasCqQt97v066Ho0qB0uej
+rwslyXSjwlOsvblf6UovUzQ3GIG17X9POOavsW6md7wxZFCNtioo+qb7fegKK5Tr
+W/H5GoB7g79pCbBUCMJP6MgPpMUVGH+5jDkWAQbik4lTH9ehD4Wu9V2hnyBub6fW
+CjEtrWzpwH+yHFkm7R5IjI8DWoE4CWsb8KI+GUgr2R3AjdNuXINbJy+ya+wpuMLh
+d5Q5tQbteQ2uBKJxXRrR8nNiiLqtQvRYsyF5G+BdXmAqAB0cBuH8yMmjUKju5tH9
+lSmdqUScCcVY11T6Hccath065f8Jtvwj3nJE9f2iPfo
+-> ssh-rsa kFDS0A
+RVoy79ijvAmU9XlEsbmiOOWUfenL+hITb6tXELUGjZjYIg+JPDneg7m1plUnRpBM
+sfLrTSzOLisWfct5rbXWb4QbNnD7biX0/uAPk8Jk3tmUfJsM1oLmNaRGGgo7RkFh
+J28PG0n5+eumauoS0Yf11GIgWUpC8FeVJMrNM5r4yV65EJEyyjRxFHjIGl5Jh6Rq
+bkJWpDsuFb2eb2BdZACV/M/aDYn+XGJW0oozNW91rryrQfsAHc3GzKoX2HtqNxua
+3Z348+NTS7jCKKhEwwNwibgTSz1PT2ynyaXi2N60KZ8IDc1xwtn1Ybj2/S1no64h
+P1GCjzKmwizgINoWQ8LYQ3nHxRXQjFdS4X63YUSXKcZ2TKMNydlB3IGL9N+xKflo
+w5EMqFTuHInpyOfz73WDg2LKuzlWabjn8KIlx2bYG8Etn5alSX+oQGD5zTUkDt4p
+/J3b8kLCdRSfVxwBudftXnk8CDg5gzM7LD0NOQ8/VK8lyTVE1dCCty1NUcM0o4mc
+VgdlcJn9ISZSd3UAt6BDUHEMYdxktJnlPr8Gsw1iDU44Gu2fPUY2OpmAnIz6FshR
+KkSThN08FL2EgEO99fbJ/8NiD+bml5duUNJQnjlQ8NC9w1S/4ADXpHSrJARQY0pn
+DfTvCz2CJnPqojb2vDb0knqvhPNLu1lmtrlyqMygmLg
+-> piv-p256 vRzPNw AlRMMj08FZgVJAcUdKDVtQzrrZWqOah1fq0xeLFOFYh/
+fySXnGSZYyKOX75bwaByIAqaiatXpFF4zsuE7JEH//c
+-> piv-p256 zqq/iw A7dI4n0fDq3z6OG/iuU8z4euPvx77lJJC9OlZG/RMPRc
+waoyEH8qBDeUmCugy7ZnMj6tgLx/1+slhJTAJ4uXMNQ
+-> ssh-ed25519 YFSOsg 99jNRmoZlrfV1ytKu8Pj41vBTNHED3dG99mjWnYe9Ec
+p+Q3Dik27t8LRb5Mr17EzVwxdSQIZBeO+ezJVvFqg00
+-> ssh-ed25519 iHV63A 1V4hJI/P7TkMWDbZb0NMdCSULS8XddPl6gGvc1gJ91I
+CKzsgmbASOGWYRFSyYBvY90HrmLfQNKcrTPLvf5m0es
+-> ssh-ed25519 BVsyTA tJu2Y42CtsqGMLf5VObT+nEMYHyujU2nmJQfWOTZsg8
+MGxxNMPHyRNRDVurqovUkptzqfsemX9mCLSLu0RL7b4
+-> ssh-ed25519 +3V2lQ vHPgK6xOUrH/1fqjkw2rhg10O0izPSTPX7b02v7J22A
+A/V11elKo6YNiFHYMQrWBnUTsaz21MNH9jcY78dTlmU
+--- QV+btlc1pzitb681enVVR/tT/kwE3s2sV1qB7yYJ/3Q
+Y¥DgIx,ìµ´âÙËœ!à¢ptë m•ŠÂòä"$ú•‚™€¿¦aZTÔ4'Äû`õejüÊúKøAÕ£t×
WÚS÷&){i–_íSŽ

secrets/secrets.nix

@@ -3,6 +3,7 @@ let
 
   nachtigall-host = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIP7G0ufi+MNvaAZLDgpieHrABPGN7e/kD5kMFwSk4ABj root@nachtigall";
   flora-6-host = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGP1InpTBN4AlF/4V8HHumAMLJzeO8DpzjUv9Co/+J09 root@flora-6";
+  metronom-host = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAICLX6UvvrKALKL0xsNnytLPHryzZF5evUnxAgGokf14i root@metronom";
   tankstelle-host = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJdF6cJKPDiloWiDja1ZtqkXDdXOCHPs10HD+JMzgeU4 root@tankstelle";
 
   adminKeys = builtins.foldl' (
@@ -14,6 +15,8 @@ let
   tankstelleKeys = [ tankstelle-host ];
 
   flora6Keys = [ flora-6-host ];
+
+  metronomKeys = [ metronom-host ];
 in
 {
   # ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBB5XaH02a6+TchnyQED2VwaltPgeFCbildbE2h6nF5e root@nachtigall
@@ -22,6 +25,7 @@ in
   "nachtigall-wg-private-key.age".publicKeys = nachtigallKeys ++ adminKeys;
   "tankstelle-wg-private-key.age".publicKeys = tankstelleKeys ++ adminKeys;
   "flora6-wg-private-key.age".publicKeys = flora6Keys ++ adminKeys;
+  "metronom-wg-private-key.age".publicKeys = metronomKeys ++ adminKeys;
 
   "mastodon-secret-key-base.age".publicKeys = nachtigallKeys ++ adminKeys;
   "mastodon-otp-secret.age".publicKeys = nachtigallKeys ++ adminKeys;
@@ -72,4 +76,13 @@ in
 
   "obs-portal-env.age".publicKeys = nachtigallKeys ++ adminKeys;
   "obs-portal-database-env.age".publicKeys = nachtigallKeys ++ adminKeys;
+
+  # mail
+  "mail/hensoko.age".publicKeys = metronomKeys ++ adminKeys;
+  "mail/teutat3s.age".publicKeys = metronomKeys ++ adminKeys;
+  "mail/admins.age".publicKeys = metronomKeys ++ adminKeys;
+  "mail/bot.age".publicKeys = metronomKeys ++ adminKeys;
+  "mail/crew.age".publicKeys = metronomKeys ++ adminKeys;
+  "mail/erpnext.age".publicKeys = metronomKeys ++ adminKeys;
+  "mail/hakkonaut.age".publicKeys = metronomKeys ++ adminKeys;
 }

terraform/dns.tf

@@ -9,6 +9,16 @@ resource "namecheap_domain_records" "pub-solar" {
     type     = "A"
     address  = "80.71.153.210"
   }
+  record {
+    hostname = "metronom"
+    type     = "A"
+    address  = "49.13.236.167"
+  }
+  record {
+    hostname = "mail"
+    type     = "A"
+    address  = "49.13.236.167"
+  }
   record {
     hostname = "auth"
     type     = "CNAME"
@@ -143,7 +153,7 @@ resource "namecheap_domain_records" "pub-solar" {
   record {
     hostname = "@"
     type     = "TXT"
-    address  = "v=spf1 include:spf.greenbaum.zone a:list.pub.solar ~all"
+    address  = "v=spf1 a:mail.pub.solar a:list.pub.solar ~all"
   }
   record {
     hostname = "list"
@@ -160,6 +170,11 @@ resource "namecheap_domain_records" "pub-solar" {
     type     = "TXT"
     address  = "v=DMARC1; p=reject;"
   }
+  record {
+    hostname = "mail._domainkey"
+    type     = "TXT"
+    address  = "v=DKIM1;k=rsa;p=MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQDI333HhjmVmDYc5hYTtmB6o9KYb782xw+ewH1eQlpFcCMyJ1giYFeGKviNki9uSm52tk34zUIthsqJMRlz2WsKGgk4oq3MRtgPtogxbh1ipJlynXejPU5WVetjjMnwr6AtV1DP1Sv4n5Vz0EV8cTi3tRZdgYpG6hlriiHXbrvlIwIDAQAB"
+  }
   record {
     hostname = "modoboa._domainkey"
     type     = "TXT"
@@ -168,7 +183,7 @@ resource "namecheap_domain_records" "pub-solar" {
   record {
     hostname = "@"
     type     = "MX"
-    address  = "mail.greenbaum.zone."
+    address  = "mail.pub.solar."
     mx_pref  = "0"
   }
   record {