Merge pull request 'mail: add backups' (#226) from mail-backups into main

Reviewed-on: #226 Reviewed-by: b12f <b12f@noreply.git.pub.solar> Reviewed-by: Hendrik Sokolowski <hensoko@noreply.git.pub.solar>
2024-08-31 19:39:25 +00:00 · 2024-08-31 19:39:25 +00:00 · 83e4bcd2df
parent a0fb6a60c3 09804f5c25
commit 83e4bcd2df
21 changed files with 235 additions and 17 deletions
--- a/docs/backups.md
+++ b/docs/backups.md
@ -0,0 +1,36 @@
+# Backups
+
+We use [Restic](https://restic.readthedocs.io/en/stable/) to create backups and push them to two repositories.
+Check `./modules/backups.nix` and `./hosts/nachtigall/backups.nix` for working examples.
+
+### Hetzner Storagebox
+
+- Uses SFTP for transfer of backups
+
+Adding a new host SSH public key to the storagebox:
+
+First, [SSH to nachtigall](./administrative-access.md#ssh-access), then become root and add the new SSH public key
+
+```
+sudo -i
+echo '<ssh-public-key>' | ssh -p23 u377325@u377325.your-storagebox.de install-ssh-key
+```
+
+[Link to Hetzner storagebox docs](https://docs.hetzner.com/robot/storage-box/backup-space-ssh-keys).
+
+### Garage S3 buckets
+
+- Uses S3 for transfer of backups
+- One bucket per host, e.g. `nachtigall-backups`, `metronom-backups`
+
+To start transfering backups from a new hosts, this is how to create a new bucket:
+
+First, [SSH to trinkgenossin](./administrative-access.md#ssh-access), then use the `garage` CLI to create a new key and bucket:
+
+```
+export GARAGE_RPC_SECRET=<secret-in-keepass>
+
+garage bucket create <hostname>-backups
+garage key create <hostname>-backups-key
+garage bucket allow <hostname>-backups --read --write --key <hostname>-backups-key
+```
--- a/hosts/metronom/backups.nix
+++ b/hosts/metronom/backups.nix
@ -1,13 +1,29 @@
-{ flake, ... }:
+{ config, flake, ... }:
 {
-  age.secrets."restic-repo-droppie" = {
-    file = "${flake.self}/secrets/restic-repo-droppie.age";
+  age.secrets."restic-repo-storagebox-metronom" = {
+    file = "${flake.self}/secrets/restic-repo-storagebox-metronom.age";
    mode = "400";
    owner = "root";
  };
-  age.secrets."restic-repo-storagebox" = {
-    file = "${flake.self}/secrets/restic-repo-storagebox.age";
+  age.secrets.restic-repo-garage-metronom = {
+    file = "${flake.self}/secrets/restic-repo-garage-metronom.age";
    mode = "400";
    owner = "root";
  };
+  age.secrets.restic-repo-garage-metronom-env = {
+    file = "${flake.self}/secrets/restic-repo-garage-metronom-env.age";
+    mode = "400";
+    owner = "root";
+  };
+
+  pub-solar-os.backups.repos.storagebox = {
+    passwordFile = config.age.secrets."restic-repo-storagebox-metronom".path;
+    repository = "sftp:u377325@u377325.your-storagebox.de:/metronom-backups";
+  };
+
+  pub-solar-os.backups.repos.garage = {
+    passwordFile = config.age.secrets."restic-repo-garage-metronom".path;
+    environmentFile = config.age.secrets."restic-repo-garage-metronom-env".path;
+    repository = "s3:https://buckets.pub.solar/metronom-backups";
+  };
 }
--- a/hosts/metronom/configuration.nix
+++ b/hosts/metronom/configuration.nix
@ -23,6 +23,14 @@
    pools = [ "root_pool" ];
  };

+  # Declarative SSH private key
+  age.secrets."metronom-root-ssh-key" = {
+    file = "${flake.self}/secrets/metronom-root-ssh-key.age";
+    path = "/root/.ssh/id_ed25519";
+    mode = "400";
+    owner = "root";
+  };
+
  # Declarative SSH private key
  #age.secrets."metronom-root-ssh-key" = {
  #  file = "${flake.self}/secrets/metronom-root-ssh-key.age";
--- a/hosts/metronom/default.nix
+++ b/hosts/metronom/default.nix
@ -7,6 +7,6 @@

    ./networking.nix
    ./wireguard.nix
-    #./backups.nix
+    ./backups.nix
  ];
 }
--- a/hosts/nachtigall/backups.nix
+++ b/hosts/nachtigall/backups.nix
@ -5,8 +5,8 @@
    mode = "400";
    owner = "root";
  };
-  age.secrets."restic-repo-storagebox" = {
-    file = "${flake.self}/secrets/restic-repo-storagebox.age";
+  age.secrets."restic-repo-storagebox-nachtigall" = {
+    file = "${flake.self}/secrets/restic-repo-storagebox-nachtigall.age";
    mode = "400";
    owner = "root";
  };
@ -22,7 +22,7 @@
  };

  pub-solar-os.backups.repos.storagebox = {
-    passwordFile = config.age.secrets."restic-repo-storagebox".path;
+    passwordFile = config.age.secrets."restic-repo-storagebox-nachtigall".path;
    repository = "sftp:u377325@u377325.your-storagebox.de:/backups";
  };

--- a/hosts/tankstelle/backups.nix
+++ b/hosts/tankstelle/backups.nix
@ -5,8 +5,8 @@
    mode = "400";
    owner = "root";
  };
-  age.secrets."restic-repo-storagebox" = {
-    file = "${flake.self}/secrets/restic-repo-storagebox.age";
+  age.secrets."restic-repo-storagebox-tankstelle" = {
+    file = "${flake.self}/secrets/restic-repo-storagebox-tankstelle.age";
    mode = "400";
    owner = "root";
  };
--- a/modules/backups/default.nix
+++ b/modules/backups/default.nix
@ -280,5 +280,11 @@ in

      in
      builtins.listToAttrs (lib.lists.flatten (map createBackups backupNames));
+
+    # Used for pub-solar-os.backups.repos.storagebox
+    programs.ssh.knownHosts = {
+      "u377325.your-storagebox.de".publicKey = "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA5EB5p/5Hp3hGW1oHok+PIOH9Pbn7cnUiGmUEBrCVjnAw+HrKyN8bYVV0dIGllswYXwkG/+bgiBlE6IVIBAq+JwVWu1Sss3KarHY3OvFJUXZoZyRRg/Gc/+LRCE7lyKpwWQ70dbelGRyyJFH36eNv6ySXoUYtGkwlU5IVaHPApOxe4LHPZa/qhSRbPo2hwoh0orCtgejRebNtW5nlx00DNFgsvn8Svz2cIYLxsPVzKgUxs8Zxsxgn+Q/UvR7uq4AbAhyBMLxv7DjJ1pc7PJocuTno2Rw9uMZi1gkjbnmiOh6TTXIEWbnroyIhwc8555uto9melEUmWNQ+C+PwAK+MPw==";
+      "[u377325.your-storagebox.de]:23".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIICf9svRenC/PLKIL9nk6K/pxQgoiFC41wTNvoIncOxs";
+    };
  };
 }
--- a/modules/forgejo/default.nix
+++ b/modules/forgejo/default.nix
@ -182,7 +182,7 @@
      OnCalendar = "*-*-* 00:00:00 Etc/UTC";
    };
    initialize = true;
-    passwordFile = config.age.secrets."restic-repo-storagebox".path;
+    passwordFile = config.age.secrets."restic-repo-storagebox-nachtigall".path;
    repository = "sftp:u377325@u377325.your-storagebox.de:/backups";
    backupPrepareCommand = ''
      ${pkgs.sudo}/bin/sudo -u postgres ${pkgs.postgresql}/bin/pg_dump -d gitea > /tmp/forgejo-backup.sql
--- a/modules/garage/default.nix
+++ b/modules/garage/default.nix
@ -69,6 +69,7 @@
      locations."/" = {
        proxyPass = "http://s3_backend";
        extraConfig = ''
+          client_max_body_size 64m;
          proxy_max_temp_file_size 0;
        '';
      };
--- a/modules/mail/default.nix
+++ b/modules/mail/default.nix
@ -67,4 +67,20 @@
  };
  security.acme.acceptTerms = true;
  security.acme.defaults.email = "security@pub.solar";
+
+  pub-solar-os.backups.restic.mail = {
+    paths = [
+      "/var/vmail"
+      "/var/dkim"
+    ];
+    timerConfig = {
+      OnCalendar = "*-*-* 02:00:00 Etc/UTC";
+    };
+    initialize = true;
+    pruneOpts = [
+      "--keep-daily 7"
+      "--keep-weekly 4"
+      "--keep-monthly 3"
+    ];
+  };
 }
--- a/modules/mailman/default.nix
+++ b/modules/mailman/default.nix
@ -91,7 +91,7 @@
      OnCalendar = "*-*-* 02:00:00 Etc/UTC";
    };
    initialize = true;
-    passwordFile = config.age.secrets."restic-repo-storagebox".path;
+    passwordFile = config.age.secrets."restic-repo-storagebox-nachtigall".path;
    repository = "sftp:u377325@u377325.your-storagebox.de:/backups";
    pruneOpts = [
      "--keep-daily 7"
--- a/modules/mastodon/default.nix
+++ b/modules/mastodon/default.nix
@ -106,7 +106,7 @@
      OnCalendar = "*-*-* 04:00:00 Etc/UTC";
    };
    initialize = true;
-    passwordFile = config.age.secrets."restic-repo-storagebox".path;
+    passwordFile = config.age.secrets."restic-repo-storagebox-nachtigall".path;
    repository = "sftp:u377325@u377325.your-storagebox.de:/backups";
    backupPrepareCommand = ''
      ${pkgs.sudo}/bin/sudo -u postgres ${pkgs.postgresql}/bin/pg_dump -d mastodon > /tmp/mastodon-backup.sql
--- a/modules/matrix/default.nix
+++ b/modules/matrix/default.nix
@ -295,7 +295,7 @@ in
      OnCalendar = "*-*-* 05:00:00 Etc/UTC";
    };
    initialize = true;
-    passwordFile = config.age.secrets."restic-repo-storagebox".path;
+    passwordFile = config.age.secrets."restic-repo-storagebox-nachtigall".path;
    repository = "sftp:u377325@u377325.your-storagebox.de:/backups";
    backupPrepareCommand = ''
      ${pkgs.sudo}/bin/sudo -u postgres ${pkgs.postgresql}/bin/pg_dump -d matrix > /tmp/matrix-synapse-backup.sql
--- a/modules/nextcloud/default.nix
+++ b/modules/nextcloud/default.nix
@ -145,7 +145,7 @@
      OnCalendar = "*-*-* 01:00:00 Etc/UTC";
    };
    initialize = true;
-    passwordFile = config.age.secrets."restic-repo-storagebox".path;
+    passwordFile = config.age.secrets."restic-repo-storagebox-nachtigall".path;
    repository = "sftp:u377325@u377325.your-storagebox.de:/backups";
    backupPrepareCommand = ''
      ${pkgs.sudo}/bin/sudo -u postgres ${pkgs.postgresql}/bin/pg_dump -d nextcloud > /tmp/nextcloud-backup.sql
--- a/secrets/metronom-root-ssh-key.age
+++ b/secrets/metronom-root-ssh-key.age
--- a/secrets/restic-repo-garage-metronom-env.age
+++ b/secrets/restic-repo-garage-metronom-env.age
@ -0,0 +1,44 @@
+age-encryption.org/v1
+-> ssh-ed25519 UE5Ceg rpN1FsYIOjsiqPAt3iwd6l3ZEDYNomnzcvgowqS1CAI
+A5+KU6SOzcZzTQPkEPp1wN6bq9junwauKDPhM1eKi+8
+-> ssh-ed25519 uYcDNw V/zOsw5KmaQUm1YsnJExXJThypfsxOu/CS+EQ2np7RQ
+vMGUU/OPOoWiyR70xsXarqWN/AgegeKgTz5lOPa04CI
+-> ssh-rsa f5THog
+Z3tZv8bK67z15PAp4RgMEi1Ph4y5IFBIVNHdhENVTt2zS2TnzTBoUSypjaioRlGj
+YKYuUl7+sFys6QRHOWTrUM6CFF6KQo/hYR5bsFG01xE9xoG7e4V5x0ts6sFp0Xme
+0nl8NBfjbORhKYyCEye6p/9EvPwJ7qpRrQt6TUpnShv9BLrZZpEyw9sy7dXS3Sjp
+btXgkOiRmIJqkYLyZ3fZF2uDlOiCVVQn/m0Bii+t0vsp4ZoyvMyc/ho3pN8i2GUV
+QvUPAWzps4LTIKUf/0IYpHV4adyEfXD09/L/ShPxXJjLrYpT+4JjJqzIg5Gutbyk
+QRBP52GFqyt3V6M2yM9THvdk88hhczsIH37VGLmdPH+vHDG1LIabgf9rJk14+FmM
+h7/TE7M4EG9YHG//zLVI4WaVf64G9Oxet4y80BhCF4kpILWRm108mpwwzPL48tR3
+VMkiX0NpP0iOe22vV6u5zzugHQYqMvR6dPtrc4yBNUPgHhOLf6GWDhX12y478o91
+ILUM08J0R0PCJhH+8LARfc7wx1fjoxeJq468sw6znHqcqbIh7WPxarKaiTbTA5bj
+06oA4YHzFbV53AbiWNHcrKCNvLaGWOw+2vtXRg8UMmbbGr8icqbLMYl7qY9kS7he
+wINMQgMKD79Q+V3AweMqLuIn1AyLpqwVmh9Qon3Wzdk
+-> ssh-rsa kFDS0A
+ZpHhJzIt2oAC5Z/xJabaunnhXCE5Ijx+Uq/s07uow2tpautkMhmP6SbdgR6zGLFV
+QJMgHmDgOqybYLhaP6t8KCygmeT9DjOB47H7mmZ8yvWAitPXTpbJzlFUls6YH3Ei
+C4lxsEoCjbH+znVTKFd4220Cb4GGvnMS7tXuAnQ9GFMAn/90LFBzYjbqSvkDyv/n
+9Ej1Nya5r0RQg/BcTKvppr0sfdk1wCEE5jDrHAR4zMmofFxuFi7V85IcRdsrU8ij
+JawhvCYGfDM6G1Yh6j8V4oaqo3gAqki0CYF5gXED42sfPrxXLV2qtYMRJSua9z0i
+Zo3SgDa9WVQslqL0VZoDXn/KyDqUYWYsfsVY0kXrMezlN9+Jm77MFVWMdXNI31eG
+EIAWMr0f6nsTuXV58lwXoijSLy9Ap45TPjbVbp7+1JkD2X543DuJD3ONiNq01gey
+a7aGLS492IByZx0mw6sb9xpTt8jP6enH+ltqcE6gMsEcxwXfmagVKTxtNrK0izWm
+g2GdcpGnVqioj42lchUJzNt/PtPqutaraEvo2oq2cw1zxCjY4zxdyNO1RdaFV71b
+fFj2JJCm67GFHWdlqbAePTx2SvUoFt3a3N8DMNFKThGQN/1LwOaKEd25ZSTNEuwO
+1exQgJfC2kxrfypEmQP/whSrk2kR13NW40bBHvrZgjo
+-> piv-p256 vRzPNw Awtb8p5KgsKIBUumqHnVMgux3dRS478DdNpCENgG3frB
+wcIPacn7KP7gl0Z5SvtoYK0pnIjWLwUB2UvVQdWJfso
+-> piv-p256 zqq/iw AgAk66eJ/xs+PqwTBzazW4HfK8dawj/3jx5opFOaGLSj
+xThgJOorp+YXS8DvaULIoszFubEfACcKSy+vwf9KMSA
+-> ssh-ed25519 YFSOsg p+/PUojxwOxpfBfaDOfEHMOGS1oVCrl9dskXgo+gOGI
+PPYr0WVPDwRiFGo14Mx+Wv+gkZ91S7CKyYslGjCI/lQ
+-> ssh-ed25519 iHV63A iXr8vgW9lHnX+rX/E9/NrKNbF+LyRpe0M44P0IxaBHo
+/odvSKNzyS8ondJ0Tcuiry09NM4ozFn2qeVMqRgR17w
+-> ssh-ed25519 BVsyTA CgGBOj8nDcfP7GBIMnFV89WF1CAoiOFbA/dUOWggmVE
+V4CUV5WZbVTPm3AnoW6WfIqIdcMW/Sm/FTljx1awdeo
+-> ssh-ed25519 +3V2lQ Jg+gASEMV3bi9eEB86rFfguh6Be/yOO2szI19Mk2BlY
+q7vBOf0CFOUfxbpvwD8rpJH3asQqqNqWBJSzwYTBErs
+--- KSBDnbS1GMq4I8FXEljleKo/pKvauq9T8vomtInIEOQ
+–€³~pë
+âÑžh¤AVÖ>Üm< 1|þùLyÈ<iïÒ<C3AF>‹·O!EËÄ[©ºt“›5?b
ùûKá\L@
ÿ„ú<ÔÖ·5Dõš÷Èws6êá„&Ó2ß¬ò¯0¿F‰G0¢„&¡®õe<C3B5>ü™«<0F>q“ºƒM	QÓt¶Ôk¥˜w-Ñ]‰6öÂ<C3B6>LW:ç¸°Å¢»ªÎUkíËŒ8%QEF€ÉËŽBâ‚‚Ngc|¡@þ"d<ÁÅíÒ§›Š4\Ù¥ÓV\1xw•ÿ
--- a/secrets/restic-repo-garage-metronom.age
+++ b/secrets/restic-repo-garage-metronom.age
@ -0,0 +1,43 @@
+age-encryption.org/v1
+-> ssh-ed25519 UE5Ceg ys38fGOhLJNLg9zx9T3v2VgF2IbOr/Y/rj2+dWkcAlU
+QwkMX8WKgcJeGUomDSLjijen2K5UcRnYYwtebrITDqU
+-> ssh-ed25519 uYcDNw wF0oWExIUjlP32CQzOvp6MyEvFw33Sm8pHhYn3Sb0zE
+RHslJJumyXoCLHLw4sGlSLK++UHmgq97KPkqCu77G3o
+-> ssh-rsa f5THog
+pFSH+qCW+oM9zn2j+830+bja2rTXFuzATqfMNAq3o38ssW8Nl7+0FpkdMam4iYXu
+sw4Pcaj1QPTO8PbhkEvjoOU4f0bUsVuJSIvcour4k8SUOBgEMiW/98AVSTIk6KBX
+PvA+4uZn2Is+bB2m9EGCguwLJ9zzzfbur+USMQvwkQexg0YRpSfhJsRbCplLXhE+
+ZU6ut4HjCP0XWwvxgFzKc6sY4X+/PeWFJOd+WkWy5lL6gcMqUz5DXoi1CeG11AR4
+/hQ5KSJBpVsxw/ib3lSkGjA/ktQzwp4hZTI0l/dH9VHOFQflM8/9hPCYT2gsLVpF
+7F2N++tMjgqbMI6Jve0gXLixpWFflr7X5UIBFW96k7/Aq2G+WUch/COQA6wTmfqw
+OeP1wGd4Ka7YsgGByH5kuL60xDvtHG6+fYlnPXZAB5Fn86Ct6vRmWw9KUvLC7LKU
+iBXDccJliY/y4vGFZH74EYlimurEfaBPiT5sxAk0Ke6hoJued3sZ39Qi+wuxMxFH
+pleoFR/n1gBq2bu8FqTQaaNXB2Rsy7q4r5Fy1FxRJqDPgHJEmPx1k4rmYPq1VIaP
+/ScOstPQgdMNBqVsBGoNYq7vewkzoPl6MkEwh6gP7IjtC1nvYxxwlGh0gESe3RFm
+4MRh78EZaY9pmqIRAf/sRzajky26Aw+DkphmWNUjMTA
+-> ssh-rsa kFDS0A
+XsOTwrszUoHm2k4XSxiLniJZNWYfJOEn7riuDaQSGSW6ZpjMloD8K1FsZ/ZbMoUP
+S/QD71rnETAhfQc8JAAHANOarxMXmSw3y1tSmlbL2h+TRnSoq74a1nK4Ble3aszu
+y7tlUuUn3vEX6BVPRNOWM3bGW3oWNe5m0sMUAc4YSUXryWF4V48c/GbUp3T0OrRS
+jm+5DWOPxt4VcLuCqe4Nv1jrjPnb7oui/7grMuottf3JRJJQxv9qZolRwlhkG2RN
+4fuUSuOYnFUuHuaF2cfuTpOPaowLbh5H/Y6ETzOp+z9yNSuxRsdNgA63GrTsAorI
+2axdnMakUsP7m3Xxu6YsVu8xP+Sso1xzPZoEQKA+2eol0fZpQvRPrZ59bqaf9p5U
+VTIKSqIAIxyr/XN/s8S4ygaNKQZW8yBColG7TlggTth5v3XqAZ8RhcFXUg6z5lSr
+RErV6Bio9JIZofvNEiJaqrl8uTo8dU4ymVuYZoEiT/mW3noqBrZlKUh6XZFMplmk
+5giRTDThA3mirSTTELFCsc08kJMXqgkOzkPk5xm5kgP7VD6t/0SfGxetVWXOlUNd
+dbprg8Oko1hdlO+LePY1n50TTFKBl9TeZWhvcLOhUizc0bTowUcXm+04Taf+MDwa
+TMxplrtahOdCTz8k38c/HwBeHtfXRevh8A8Y1qnJXJY
+-> piv-p256 vRzPNw AqccwzdKUA4RP2LzIfcTlAN9LsoEB/b7tGYyM8bk39Pn
+f0srD9t9HaGY8OIAVImqJSrvHZRhxfMXkYwot4LJGeM
+-> piv-p256 zqq/iw Aj8544WraFJMX2S6qyzi6CTal6sRnunmzbMO4KUQhJOO
+BFiQSdLgrmgPnynqmSLNBqiWkyBme3KavSbi86HHSck
+-> ssh-ed25519 YFSOsg Zece1bOI+mVc6079POREAnnzSG7ZytiTRDm+NzbbhVE
+alK4ODfwrgRSDGWzcZmIuyZ88axaiMzSNfeGspsgk70
+-> ssh-ed25519 iHV63A LwfUkisQGB3txmxYYLlZSG6ddxVNVC9+UokxPiXEjRc
+yRmtdHT9uM0YkS/s80jetMr1baDjGsaRubVKbJVPpCk
+-> ssh-ed25519 BVsyTA +8LVssLl+DiF2f3H0KhAhvzEvTjciIAcRM9ZYwrGQh0
+CcQxWwMBdyXXzDv10vUmXBifYLXsHKOFd2/L95RGT5U
+-> ssh-ed25519 +3V2lQ RWquIefIO5crVvrUxdatV7OvTv1Jabyq4IF209Ezkw0
+0SM43tcO7m7FQlNJe9QnhC9J9PwHoVxucRtZGpcACUE
+--- xx8BodL5hv2CyeZ8m0tGXNzmH2DGaCveUNobqbAQK8U
+)ŠÐÑ\³“9°c½ùt4Ê¿Á~ÁÆëºùeJ¥}<Ó¼Š§Å[‘¸Ø+*x’£>;m/ «&I»–÷Ò:Ï‘óÛ3
--- a/secrets/restic-repo-storagebox-metronom.age
+++ b/secrets/restic-repo-storagebox-metronom.age
@ -0,0 +1,43 @@
+age-encryption.org/v1
+-> ssh-ed25519 UE5Ceg Ut5S3qcz82qT9y6KnUG6WfgvhZ4Cq3akEnw6qkfOkHA
+uwaOk4WQxkj/R8rw31ClUm7nS0nz5OFVyyqNdNjSKY0
+-> ssh-ed25519 uYcDNw kwQz6q16sYba+q2r/lH6Z0kSSXSxVrjGpK/3tPj4CT0
+nTo6jrcpQ9niGGxhuS7mZva6KnrYdjqvobW3yiZsYU4
+-> ssh-rsa f5THog
+ugWQuX4fXVBJ/MWuVaM9Wj+kUFIpKV/+2C+Hxe4xg05/HmVWH95/TUE7QnhBKu1/
+Dda8oQL8xMSvThxUh0tY9pJjUMa/1ShfYpIAD6zE311bnPObDAZRBtS6fF5m5Vcj
+9KL1ILwj7/Vj5/OMDM4BuEL1cmTmX2ohj/ho7hoWpU3ejohQKxpsH/atXAPevAgz
+oajzoiQ2+qX18gW4tTLr9MqGuIQYDhj0f1YsPR5gMQEukwGJcQYaZZe78wRV/iGH
+n4xbdRJGAxWyBZqKqeNKmh/VxdoC6wxIpAM3h/tchWA7RB2kn5rJBuk+XOF5pJLj
+tOjMpoJClHwIgr8hmIMb5eI5bQdBZi8sIuwgaxe8GAjfy1fXt+XcR30pWW9xh/la
+yLSEnz7OtjNPmHObWxMLmYkIX16qKXmdA1IMJib74THqboHOIFI08GVKWs6xCAr7
+9Mk4tFejwqPurFskwmX5Ubls8b5hNIO8Cz72RKwvGHwTFTtOR5Lzp/gRmCTD1qlb
+aNYOZnduZ/ApxwG0OnNbC60C8+NB8EwRToiHmXLlnFP6WGlYKlSKy/WjJFjreFIJ
+9hdiKCcHE87gsD2dEWHW4Hq4oZq/SwVL2FGCHC5dgohkY//i+aQVeSqAtvOR3tsb
+CnKVhf+O6dMVZqNUeONGJ1oEaZwZJzBST9XXv6Yi2rY
+-> ssh-rsa kFDS0A
+EmZCGuxH9lUz/OuZKjKfkeone7oi+AETEpkUH909lc0LoTbk96Z9Q9IsRSxcrXh2
+CuQ9c8zZ3PgpwHRxLwwvCef4VMW2rgu98FLza3C41tww5ceudK/p/vwCPACUAV5e
+T8fUGg2OTUFOvMpFtPejoD1fjrTHWahB4DW4r+ODbQM94fQiHjzq1qPDsFf4WpKF
+1azagGfB6uejCewpQ+M+GnUZXDjOTi2IgA1+/dqQz5kGhVA9SnykfSxWGZKjcGPh
+10EaBdrO5bqIh+Wf/b6GktFWKKTOipD6VzxiMbppMqr4pfRpFYy+okunl29ky+xk
+6LTJHw5+19QB2YZMAbHCivmZpX+rfuCI/4JmUMfdAHrdkL56F/OWPx8j+2gFFkTm
+qOUVKkuAKWfJFaUgfFbXtwwuE+JH/RuY1flG+PJjMr1xxnttFs3IYP4CVEH1j3Yg
+APzd3PYpkW3fkCNT3PUJHDjVhzS3jvAjIgD2qDwc/AsQyMjJuyICSL8ciSb9PQMY
+JeGX+4FjDpqgZeNYD1CIEEraAkoHoEBi2puNrsQY11qgnC/XXAfubz2XDtF4NhZS
+Lr29oOuqwl9UglQq1yx0rmPyLvb5fstFvN1JELTuArOX9r7uyV4tEjhr1Emjcmlx
+20XOvDtSRQNyG0JypayZcng4sXM8yLZngsUR+9kMZjo
+-> piv-p256 vRzPNw AjEZFjCMWC+lzG6Mzn+yk1ylhtIS0rr8+uG3pONi8aUB
+0dJi7kX8PFsGGo2nKxJ9DAi1Psz/Z93xusQ6hvgfo88
+-> piv-p256 zqq/iw AmPSSktHHwoEtSPexUIp4gro8kbxyiBi1F4I9PZBJXUN
+SllB8/hq5mPkqidZnpSCktBs4IKyDn66Rq4Tn1CHjZ4
+-> ssh-ed25519 YFSOsg dQQ89fQbmnEc8ws1Ph1sBcnF9rLeOJHcT5aXzf3wdBQ
+7Wve8saqRX4bbskIxPqbN6+danJimre2tNm5Y/nLBkI
+-> ssh-ed25519 iHV63A 9Is8lLheIcDBr0A82rW+ercEGb4WOOHYu2ArrNuwWHQ
+koc4Tp5KNMWlvqIY2Q5wGo1RV4PLLjbqZDH/te2+9vM
+-> ssh-ed25519 BVsyTA SbXK3Qyz2KIN5+SuYQri6oQSVRFTsekvtCRissDF7nQ
+EOuZGw1k2Ql6co/WFeEn2TmfGWN1ThCkksa1RD30yTE
+-> ssh-ed25519 +3V2lQ HRGVqQxpU9SCs0tD2gSuqKz92HE3paG0JsHru3eliEE
+U1z/FTfrf4sb4/gpEjHmpX559JSn7zsaiQUeej8ofpA
+--- V+P4YcVeFP56hwKuk4ZLSzE/zCSvYyCTrKKRj48AuMA
+%ŽGYÜ5fÜv…µ]W1Út™]±ìóãüü/M8ÀU=«b<C2AB>KÁ[ÈP1iÏ—'ÝyÄPU]€6'è‹ÇßØìøãÁáó‡•½fmeyÚÚ
ý
--- a/secrets/restic-repo-storagebox-nachtigall.age
+++ b/secrets/restic-repo-storagebox-nachtigall.age
--- a/secrets/restic-repo-storagebox.age
+++ b/secrets/restic-repo-storagebox.age
--- a/secrets/secrets.nix
+++ b/secrets/secrets.nix
@ -36,6 +36,8 @@ in
 {
  # ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBB5XaH02a6+TchnyQED2VwaltPgeFCbildbE2h6nF5e root@nachtigall
  "nachtigall-root-ssh-key.age".publicKeys = nachtigallKeys ++ adminKeys;
+  # ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPDeKXqbhNzbXk15h2k8wGBByxMDCC6HE1/fwa4j6ECu root@metronom
+  "metronom-root-ssh-key.age".publicKeys = metronomKeys ++ adminKeys;

  "nachtigall-wg-private-key.age".publicKeys = nachtigallKeys ++ adminKeys;
  "tankstelle-wg-private-key.age".publicKeys = tankstelleKeys ++ adminKeys;
@ -70,8 +72,11 @@ in

  "searx-environment.age".publicKeys = nachtigallKeys ++ adminKeys;

+  "restic-repo-garage-metronom.age".publicKeys = metronomKeys ++ adminKeys;
+  "restic-repo-garage-metronom-env.age".publicKeys = metronomKeys ++ adminKeys;
  "restic-repo-droppie.age".publicKeys = nachtigallKeys ++ adminKeys;
-  "restic-repo-storagebox.age".publicKeys = nachtigallKeys ++ adminKeys;
+  "restic-repo-storagebox-nachtigall.age".publicKeys = nachtigallKeys ++ adminKeys;
+  "restic-repo-storagebox-metronom.age".publicKeys = metronomKeys ++ adminKeys;
  "restic-repo-garage-nachtigall.age".publicKeys = nachtigallKeys ++ adminKeys;
  "restic-repo-garage-nachtigall-env.age".publicKeys = nachtigallKeys ++ adminKeys;