docs: how-to add backups for new hosts

To make bigger garage uploads work well, avoiding error
HTTP 413 Entity Too Large

docs/backups.md

@@ -0,0 +1,37 @@
+# Backups
+
+We use [Restic](https://restic.readthedocs.io/en/stable/) to create backups and push them to two repositories.
+Check `./modules/backups.nix` and `./hosts/nachtigall/backups.nix` for working examples.
+
+### Hetzner Storagebox
+
+- Uses SFTP for transfer of backups
+
+Adding a new host SSH public key to the storagebox:
+
+First, [SSH to nachtigall](./administrative-access.md#ssh-access), then become root and add the new SSH public key
+
+```
+sudo -i
+echo '<ssh-public-key>' | ssh -p23 u377325@u377325.your-storagebox.de install-ssh-key
+```
+
+[Link to Hetzner storagebox docs](https://docs.hetzner.com/robot/storage-box/backup-space-ssh-keys).
+
+
+### Garage S3 buckets
+
+- Uses S3 for transfer of backups
+- One bucket per host, e.g. `nachtigall-backups`, `metronom-backups`
+
+To start transfering backups from a new hosts, this is how to create a new bucket:
+
+First, [SSH to trinkgenossin](./administrative-access.md#ssh-access), then use the `garage` CLI to create a new key and bucket:
+
+```
+export GARAGE_RPC_SECRET=<secret-in-keepass>
+
+garage bucket create <hostname>-backups
+garage key create <hostname>-backups-key
+garage bucket allow <hostname>-backups --read --write --key <hostname>-backups-key
+```

hosts/metronom/backups.nix

@@ -1,7 +1,7 @@
 { config, flake, ... }:
 {
-  age.secrets."restic-repo-storagebox" = {
-    file = "${flake.self}/secrets/restic-repo-storagebox.age";
+  age.secrets."restic-repo-storagebox-metronom" = {
+    file = "${flake.self}/secrets/restic-repo-storagebox-metronom.age";
     mode = "400";
     owner = "root";
   };
@@ -17,8 +17,8 @@
   };
 
   pub-solar-os.backups.repos.storagebox = {
-    passwordFile = config.age.secrets."restic-repo-storagebox".path;
-    repository = "sftp:u377325@u377325.your-storagebox.de:/backups";
+    passwordFile = config.age.secrets."restic-repo-storagebox-metronom".path;
+    repository = "sftp:u377325@u377325.your-storagebox.de:/metronom-backups";
   };
 
   pub-solar-os.backups.repos.garage = {

hosts/metronom/configuration.nix

@@ -23,6 +23,14 @@
     pools = [ "root_pool" ];
   };
 
+  # Declarative SSH private key
+  age.secrets."metronom-root-ssh-key" = {
+    file = "${flake.self}/secrets/metronom-root-ssh-key.age";
+    path = "/root/.ssh/id_ed25519";
+    mode = "400";
+    owner = "root";
+  };
+
   # Declarative SSH private key
   #age.secrets."metronom-root-ssh-key" = {
   #  file = "${flake.self}/secrets/metronom-root-ssh-key.age";

hosts/nachtigall/backups.nix

@@ -5,8 +5,8 @@
     mode = "400";
     owner = "root";
   };
-  age.secrets."restic-repo-storagebox" = {
-    file = "${flake.self}/secrets/restic-repo-storagebox.age";
+  age.secrets."restic-repo-storagebox-nachtigall" = {
+    file = "${flake.self}/secrets/restic-repo-storagebox-nachtigall.age";
     mode = "400";
     owner = "root";
   };
@@ -22,7 +22,7 @@
   };
 
   pub-solar-os.backups.repos.storagebox = {
-    passwordFile = config.age.secrets."restic-repo-storagebox".path;
+    passwordFile = config.age.secrets."restic-repo-storagebox-nachtigall".path;
     repository = "sftp:u377325@u377325.your-storagebox.de:/backups";
   };
 

hosts/tankstelle/backups.nix

@@ -5,8 +5,8 @@
     mode = "400";
     owner = "root";
   };
-  age.secrets."restic-repo-storagebox" = {
-    file = "${flake.self}/secrets/restic-repo-storagebox.age";
+  age.secrets."restic-repo-storagebox-tankstelle" = {
+    file = "${flake.self}/secrets/restic-repo-storagebox-tankstelle.age";
     mode = "400";
     owner = "root";
   };

modules/backups/default.nix

@@ -280,5 +280,11 @@ in
 
       in
       builtins.listToAttrs (lib.lists.flatten (map createBackups backupNames));
+
+      # Used for pub-solar-os.backups.repos.storagebox
+      programs.ssh.knownHosts = {
+        "u377325.your-storagebox.de".publicKey = "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA5EB5p/5Hp3hGW1oHok+PIOH9Pbn7cnUiGmUEBrCVjnAw+HrKyN8bYVV0dIGllswYXwkG/+bgiBlE6IVIBAq+JwVWu1Sss3KarHY3OvFJUXZoZyRRg/Gc/+LRCE7lyKpwWQ70dbelGRyyJFH36eNv6ySXoUYtGkwlU5IVaHPApOxe4LHPZa/qhSRbPo2hwoh0orCtgejRebNtW5nlx00DNFgsvn8Svz2cIYLxsPVzKgUxs8Zxsxgn+Q/UvR7uq4AbAhyBMLxv7DjJ1pc7PJocuTno2Rw9uMZi1gkjbnmiOh6TTXIEWbnroyIhwc8555uto9melEUmWNQ+C+PwAK+MPw==";
+        "[u377325.your-storagebox.de]:23".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIICf9svRenC/PLKIL9nk6K/pxQgoiFC41wTNvoIncOxs";
+      };
   };
 }

modules/forgejo/default.nix

@@ -182,7 +182,7 @@
       OnCalendar = "*-*-* 00:00:00 Etc/UTC";
     };
     initialize = true;
-    passwordFile = config.age.secrets."restic-repo-storagebox".path;
+    passwordFile = config.age.secrets."restic-repo-storagebox-nachtigall".path;
     repository = "sftp:u377325@u377325.your-storagebox.de:/backups";
     backupPrepareCommand = ''
       ${pkgs.sudo}/bin/sudo -u postgres ${pkgs.postgresql}/bin/pg_dump -d gitea > /tmp/forgejo-backup.sql

modules/garage/default.nix

@@ -69,6 +69,7 @@
       locations."/" = {
         proxyPass = "http://s3_backend";
         extraConfig = ''
+          client_max_body_size 64m;
           proxy_max_temp_file_size 0;
         '';
       };

modules/mail/default.nix

@@ -70,8 +70,8 @@
 
   pub-solar-os.backups.restic.mail = {
     paths = [
-      "/var/lib/vmail"
-      "/var/lib/dkim"
+      "/var/vmail"
+      "/var/dkim"
     ];
     timerConfig = {
       OnCalendar = "*-*-* 02:00:00 Etc/UTC";

modules/mailman/default.nix

@@ -91,7 +91,7 @@
       OnCalendar = "*-*-* 02:00:00 Etc/UTC";
     };
     initialize = true;
-    passwordFile = config.age.secrets."restic-repo-storagebox".path;
+    passwordFile = config.age.secrets."restic-repo-storagebox-nachtigall".path;
     repository = "sftp:u377325@u377325.your-storagebox.de:/backups";
     pruneOpts = [
       "--keep-daily 7"

modules/mastodon/default.nix

@@ -106,7 +106,7 @@
       OnCalendar = "*-*-* 04:00:00 Etc/UTC";
     };
     initialize = true;
-    passwordFile = config.age.secrets."restic-repo-storagebox".path;
+    passwordFile = config.age.secrets."restic-repo-storagebox-nachtigall".path;
     repository = "sftp:u377325@u377325.your-storagebox.de:/backups";
     backupPrepareCommand = ''
       ${pkgs.sudo}/bin/sudo -u postgres ${pkgs.postgresql}/bin/pg_dump -d mastodon > /tmp/mastodon-backup.sql

modules/matrix/default.nix

@@ -295,7 +295,7 @@ in
       OnCalendar = "*-*-* 05:00:00 Etc/UTC";
     };
     initialize = true;
-    passwordFile = config.age.secrets."restic-repo-storagebox".path;
+    passwordFile = config.age.secrets."restic-repo-storagebox-nachtigall".path;
     repository = "sftp:u377325@u377325.your-storagebox.de:/backups";
     backupPrepareCommand = ''
       ${pkgs.sudo}/bin/sudo -u postgres ${pkgs.postgresql}/bin/pg_dump -d matrix > /tmp/matrix-synapse-backup.sql

modules/nextcloud/default.nix

@@ -145,7 +145,7 @@
       OnCalendar = "*-*-* 01:00:00 Etc/UTC";
     };
     initialize = true;
-    passwordFile = config.age.secrets."restic-repo-storagebox".path;
+    passwordFile = config.age.secrets."restic-repo-storagebox-nachtigall".path;
     repository = "sftp:u377325@u377325.your-storagebox.de:/backups";
     backupPrepareCommand = ''
       ${pkgs.sudo}/bin/sudo -u postgres ${pkgs.postgresql}/bin/pg_dump -d nextcloud > /tmp/nextcloud-backup.sql

secrets/restic-repo-storagebox-metronom.age

@@ -0,0 +1,43 @@
+age-encryption.org/v1
+-> ssh-ed25519 UE5Ceg Ut5S3qcz82qT9y6KnUG6WfgvhZ4Cq3akEnw6qkfOkHA
+uwaOk4WQxkj/R8rw31ClUm7nS0nz5OFVyyqNdNjSKY0
+-> ssh-ed25519 uYcDNw kwQz6q16sYba+q2r/lH6Z0kSSXSxVrjGpK/3tPj4CT0
+nTo6jrcpQ9niGGxhuS7mZva6KnrYdjqvobW3yiZsYU4
+-> ssh-rsa f5THog
+ugWQuX4fXVBJ/MWuVaM9Wj+kUFIpKV/+2C+Hxe4xg05/HmVWH95/TUE7QnhBKu1/
+Dda8oQL8xMSvThxUh0tY9pJjUMa/1ShfYpIAD6zE311bnPObDAZRBtS6fF5m5Vcj
+9KL1ILwj7/Vj5/OMDM4BuEL1cmTmX2ohj/ho7hoWpU3ejohQKxpsH/atXAPevAgz
+oajzoiQ2+qX18gW4tTLr9MqGuIQYDhj0f1YsPR5gMQEukwGJcQYaZZe78wRV/iGH
+n4xbdRJGAxWyBZqKqeNKmh/VxdoC6wxIpAM3h/tchWA7RB2kn5rJBuk+XOF5pJLj
+tOjMpoJClHwIgr8hmIMb5eI5bQdBZi8sIuwgaxe8GAjfy1fXt+XcR30pWW9xh/la
+yLSEnz7OtjNPmHObWxMLmYkIX16qKXmdA1IMJib74THqboHOIFI08GVKWs6xCAr7
+9Mk4tFejwqPurFskwmX5Ubls8b5hNIO8Cz72RKwvGHwTFTtOR5Lzp/gRmCTD1qlb
+aNYOZnduZ/ApxwG0OnNbC60C8+NB8EwRToiHmXLlnFP6WGlYKlSKy/WjJFjreFIJ
+9hdiKCcHE87gsD2dEWHW4Hq4oZq/SwVL2FGCHC5dgohkY//i+aQVeSqAtvOR3tsb
+CnKVhf+O6dMVZqNUeONGJ1oEaZwZJzBST9XXv6Yi2rY
+-> ssh-rsa kFDS0A
+EmZCGuxH9lUz/OuZKjKfkeone7oi+AETEpkUH909lc0LoTbk96Z9Q9IsRSxcrXh2
+CuQ9c8zZ3PgpwHRxLwwvCef4VMW2rgu98FLza3C41tww5ceudK/p/vwCPACUAV5e
+T8fUGg2OTUFOvMpFtPejoD1fjrTHWahB4DW4r+ODbQM94fQiHjzq1qPDsFf4WpKF
+1azagGfB6uejCewpQ+M+GnUZXDjOTi2IgA1+/dqQz5kGhVA9SnykfSxWGZKjcGPh
+10EaBdrO5bqIh+Wf/b6GktFWKKTOipD6VzxiMbppMqr4pfRpFYy+okunl29ky+xk
+6LTJHw5+19QB2YZMAbHCivmZpX+rfuCI/4JmUMfdAHrdkL56F/OWPx8j+2gFFkTm
+qOUVKkuAKWfJFaUgfFbXtwwuE+JH/RuY1flG+PJjMr1xxnttFs3IYP4CVEH1j3Yg
+APzd3PYpkW3fkCNT3PUJHDjVhzS3jvAjIgD2qDwc/AsQyMjJuyICSL8ciSb9PQMY
+JeGX+4FjDpqgZeNYD1CIEEraAkoHoEBi2puNrsQY11qgnC/XXAfubz2XDtF4NhZS
+Lr29oOuqwl9UglQq1yx0rmPyLvb5fstFvN1JELTuArOX9r7uyV4tEjhr1Emjcmlx
+20XOvDtSRQNyG0JypayZcng4sXM8yLZngsUR+9kMZjo
+-> piv-p256 vRzPNw AjEZFjCMWC+lzG6Mzn+yk1ylhtIS0rr8+uG3pONi8aUB
+0dJi7kX8PFsGGo2nKxJ9DAi1Psz/Z93xusQ6hvgfo88
+-> piv-p256 zqq/iw AmPSSktHHwoEtSPexUIp4gro8kbxyiBi1F4I9PZBJXUN
+SllB8/hq5mPkqidZnpSCktBs4IKyDn66Rq4Tn1CHjZ4
+-> ssh-ed25519 YFSOsg dQQ89fQbmnEc8ws1Ph1sBcnF9rLeOJHcT5aXzf3wdBQ
+7Wve8saqRX4bbskIxPqbN6+danJimre2tNm5Y/nLBkI
+-> ssh-ed25519 iHV63A 9Is8lLheIcDBr0A82rW+ercEGb4WOOHYu2ArrNuwWHQ
+koc4Tp5KNMWlvqIY2Q5wGo1RV4PLLjbqZDH/te2+9vM
+-> ssh-ed25519 BVsyTA SbXK3Qyz2KIN5+SuYQri6oQSVRFTsekvtCRissDF7nQ
+EOuZGw1k2Ql6co/WFeEn2TmfGWN1ThCkksa1RD30yTE
+-> ssh-ed25519 +3V2lQ HRGVqQxpU9SCs0tD2gSuqKz92HE3paG0JsHru3eliEE
+U1z/FTfrf4sb4/gpEjHmpX559JSn7zsaiQUeej8ofpA
+--- V+P4YcVeFP56hwKuk4ZLSzE/zCSvYyCTrKKRj48AuMA
+%ŽGYÜ5fÜv…µ]W1Út™]­±ìóãüü/M8ÀU=«b<C2AB>KÁ[ÈP1iÏ—'ÝyÄPU]€6'è‹ÇßØìøãÁá󇕽fmeyÚÚ
ý

secrets/secrets.nix

@@ -36,6 +36,8 @@ in
 {
   # ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBB5XaH02a6+TchnyQED2VwaltPgeFCbildbE2h6nF5e root@nachtigall
   "nachtigall-root-ssh-key.age".publicKeys = nachtigallKeys ++ adminKeys;
+  # ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIPDeKXqbhNzbXk15h2k8wGBByxMDCC6HE1/fwa4j6ECu root@metronom
+  "metronom-root-ssh-key.age".publicKeys = metronomKeys ++ adminKeys;
 
   "nachtigall-wg-private-key.age".publicKeys = nachtigallKeys ++ adminKeys;
   "tankstelle-wg-private-key.age".publicKeys = tankstelleKeys ++ adminKeys;
@@ -73,7 +75,8 @@ in
   "restic-repo-garage-metronom.age".publicKeys = metronomKeys ++ adminKeys;
   "restic-repo-garage-metronom-env.age".publicKeys = metronomKeys ++ adminKeys;
   "restic-repo-droppie.age".publicKeys = nachtigallKeys ++ adminKeys;
-  "restic-repo-storagebox.age".publicKeys = metronomKeys ++ nachtigallKeys ++ adminKeys;
+  "restic-repo-storagebox-nachtigall.age".publicKeys = nachtigallKeys ++ adminKeys;
+  "restic-repo-storagebox-metronom.age".publicKeys = metronomKeys ++ adminKeys;
   "restic-repo-garage-nachtigall.age".publicKeys = nachtigallKeys ++ adminKeys;
   "restic-repo-garage-nachtigall-env.age".publicKeys = nachtigallKeys ++ adminKeys;