blackbox-exporter: scrape every 60m, add URLs

before certificate expiry

.forgejo/workflows/check.yml

@@ -10,7 +10,7 @@ jobs:
 
       - name: Check formatting
         run: |
-          nix --accept-flake-config --access-tokens '' develop --command treefmt --fail-on-change
+          nix --accept-flake-config --access-tokens '' develop --command treefmt --ci
 
       - name: Run flake checks
         run: |
@@ -18,14 +18,20 @@ jobs:
           # Prevent cache garbage collection by creating GC roots
           mkdir -p /var/lib/gitea-runner/tankstelle/.local/state/nix/results
 
-          for target in $(nix flake show --json --all-systems | jq '
+          for target in $(nix flake show --json --all-systems | jq --raw-output '
             .["nixosConfigurations"] |
             to_entries[] |
-            .key
-            ' | tr -d '"'
+            .key'
           ); do
             nix --print-build-logs --verbose --accept-flake-config --access-tokens '' \
               build --out-link /var/lib/gitea-runner/tankstelle/.local/state/nix/results/"$target" ".#nixosConfigurations.${target}.config.system.build.toplevel"
           done
 
-          nix --print-build-logs --verbose --accept-flake-config --access-tokens '' flake check
+          for check in $(nix flake show --json --all-systems | jq --raw-output '
+            .checks."x86_64-linux" |
+            to_entries[] |
+            .key'
+          ); do
+            nix --print-build-logs --verbose --accept-flake-config --access-tokens '' \
+              build --out-link /var/lib/gitea-runner/tankstelle/.local/state/nix/results/"$check" ".#checks.x86_64-linux.${check}"
+          done

docs/administrative-access.md

@@ -63,12 +63,6 @@ Example NixOS snippet for WireGuard client config
             #endpoint = "138.201.80.102:51820";
             persistentKeepalive = 15;
           }
-          { # flora-6.pub.solar
-            publicKey = "jtSR5G2P/nm9s8WrVc26Xc/SQLupRxyXE+5eIeqlsTU=";
-            allowedIPs = [ "10.7.6.2/32" "fd00:fae:fae:fae:fae:2::/96" ];
-            endpoint = "80.71.153.210:51820";
-            persistentKeepalive = 15;
-          }
           { # metronom.pub.solar
             publicKey = "zOSYGO7MfnOOUnzaTcWiKRQM0qqxR3JQrwx/gtEtHmo=";
             allowedIPs = [ "10.7.6.3/32" "fd00:fae:fae:fae:fae:3::/96" ];
@@ -85,6 +79,39 @@ Example NixOS snippet for WireGuard client config
             #endpoint = "80.244.242.5:51820";
             persistentKeepalive = 15;
           }
+          {
+            # trinkgenossin.pub.solar
+            publicKey = "QWgHovHxtqiQhnHLouSWiT6GIoQDmuvnThYL5c/rvU4=";
+            allowedIPs = [
+              "10.7.6.5/32"
+              "fd00:fae:fae:fae:fae:5::/96"
+            ];
+            #endpoint = "85.215.152.22:51820";
+            endpoint = "[2a01:239:35d:f500::1]:51820";
+            persistentKeepalive = 15;
+          }
+          {
+            # delite.pub.solar
+            publicKey = "ZT2qGWgMPwHRUOZmTQHWCRX4m14YwOsiszjsA5bpc2k=";
+            allowedIPs = [
+              "10.7.6.6/32"
+              "fd00:fae:fae:fae:fae:6::/96"
+            ];
+            #endpoint = "5.255.119.132:51820";
+            endpoint = "[2a04:52c0:124:9d8c::2]:51820";
+            persistentKeepalive = 15;
+          }
+          {
+            # blue-shell.pub.solar
+            publicKey = "bcrIpWrKc1M+Hq4ds3aN1lTaKE26f2rvXhd+93QrzR8=";
+            allowedIPs = [
+              "10.7.6.7/32"
+              "fd00:fae:fae:fae:fae:7::/96"
+            ];
+            #endpoint = "194.13.83.205:51820";
+            endpoint = "[2a03:4000:43:24e::1]:51820";
+            persistentKeepalive = 15;
+          }
         ];
       };
     };

docs/backups.md

@@ -0,0 +1,36 @@
+# Backups
+
+We use [Restic](https://restic.readthedocs.io/en/stable/) to create backups and push them to two repositories.
+Check `./modules/backups.nix` and `./hosts/nachtigall/backups.nix` for working examples.
+
+### Hetzner Storagebox
+
+- Uses SFTP for transfer of backups
+
+Adding a new host SSH public key to the storagebox:
+
+First, [SSH to nachtigall](./administrative-access.md#ssh-access), then become root and add the new SSH public key
+
+```
+sudo -i
+echo '<ssh-public-key>' | ssh -p23 u377325@u377325.your-storagebox.de install-ssh-key
+```
+
+[Link to Hetzner storagebox docs](https://docs.hetzner.com/robot/storage-box/backup-space-ssh-keys).
+
+### Garage S3 buckets
+
+- Uses S3 for transfer of backups
+- One bucket per host, e.g. `nachtigall-backups`, `metronom-backups`
+
+To start transfering backups from a new hosts, this is how to create a new bucket:
+
+First, [SSH to trinkgenossin](./administrative-access.md#ssh-access), then use the `garage` CLI to create a new key and bucket:
+
+```
+export GARAGE_RPC_SECRET=<secret-in-keepass>
+
+garage bucket create <hostname>-backups
+garage key create <hostname>-backups-key
+garage bucket allow <hostname>-backups --read --write --key <hostname>-backups-key
+```

docs/cachix.md

@@ -0,0 +1,55 @@
+# Cachix usage
+
+URL: https://pub-solar.cachix.org
+
+Requirements:
+
+- [Install cachix](https://docs.cachix.org/installation)
+- Optional: To push to the cache, you need to set `CACHIX_AUTH_TOKEN` in your environment. To generate one for you, follow the [Getting Started](https://docs.cachix.org/getting-started#authenticating) docs and login with your GitHub account.
+- Add our binary cache [to your nix config](https://docs.cachix.org/faq#cachix-use-effects). To add the pub-solar cache, run:
+
+```
+cachix use pub-solar
+```
+
+Example to build and push a custom package of a host in this flake (e.g. after creating an overlay):
+
+```
+nix build --json -f . '.#nixosConfigurations.nachtigall.pkgs.keycloak^*' \
+  | jq -r '.[].outputs | to_entries[].value' \
+  | cachix push pub-solar
+```
+
+Example to build and push a package in the `nixpkgs` repo:
+
+```
+cd nixpkgs
+nix build --json -f . 'pkgs.lix^*' \
+  | jq -r '.[].outputs | to_entries[].value' \
+  | cachix push pub-solar
+```
+
+Checking if a package has been correctly pushed to the cache:
+
+```
+❯ nix build --json '/nix/store/f76xi83z4xk9sn6pbh38rh97yvqhb5m0-noto-fonts-color-emoji-png-2.042.drv^*' | jq -r '.[].outputs | to_entries[].value'  | cachix push pub-solar
+Pushing 1 paths (0 are already present) using zstd to cache pub-solar ⏳
+
+✓ /nix/store/xpgpi84765dxqja3gd5pldj49xx2v0xl-noto-fonts-color-emoji-png-2.042 (10.30 MiB)
+
+All done.
+
+❯ curl -I https://pub-solar.cachix.org/xpgpi84765dxqja3gd5pldj49xx2v0xl.narinfo
+HTTP/2 200
+date: Mon, 26 Aug 2024 09:31:10 GMT
+content-type: text/x-nix-narinfo
+traceparent: 00-b99db37cc9c2581b8d226cdf81e54507-794fc49193659c03-01
+tracestate:
+cache-control: public, max-age=14400
+last-modified: Mon, 26 Aug 2024 09:31:10 GMT
+cf-cache-status: EXPIRED
+report-to: {"endpoints":[{"url":"https:\/\/a.nel.cloudflare.com\/report\/v4?s=A67KGsCIsYjoFdvndxJ0rkmb7BZ5ztIpm8WUJKAiUPRVWvbYeXU9gU27P7zryiUtArbwrLzHhhMija0yyXk0kwNa3suz8gNzKK6z1CX1FWDZiiP07rnq7zAg8nZbSBiEU%2FZrU9nSrR6mhuL9ihbmW1Hf"}],"group":"cf-nel","max_age":604800}
+nel: {"success_fraction":0,"report_to":"cf-nel","max_age":604800}
+server: cloudflare
+cf-ray: 8b92ceab0d19c80e-DUS
+```

docs/deletion-request.md

@@ -50,7 +50,7 @@ Docs: https://forgejo.org/docs/latest/admin/command-line/#delete
 ### Matrix
 
 ```
-curl --header "Authorization: Bearer <admin-access-token>" --request POST http://172.18.0.3:8008/_synapse/admin/v1/deactivate/@<username>:pub.solar --data '{"erase": true}'
+curl --header "Authorization: Bearer <admin-access-token>" --request POST http://127.0.0.1:8008/_synapse/admin/v1/deactivate/@<username>:pub.solar --data '{"erase": true}'
 ```
 
 Docs: https://matrix-org.github.io/synapse/latest/admin_api/user_admin_api.html#deactivate-account

docs/deploying.md

@@ -13,12 +13,6 @@ For nachtigall.pub.solar:
 deploy --targets '.#nachtigall' --magic-rollback false --auto-rollback false --keep-result --result-path ./results
 ```
 
-For flora-6.pub.solar:
-
-```
-deploy --targets '.#flora-6' --magic-rollback false --auto-rollback false --keep-result --result-path ./results
-```
-
 For metronom.pub.solar (aarch64-linux):
 
 ```

docs/dns.md

@@ -1,18 +1,10 @@
 # Changing DNS entries
 
 Our current DNS provider is [namecheap](https://www.namecheap.com/).
-We use [Terraform](https://www.terraform.io) to declaratively manage our pub.solar DNS records.
+We use [OpenTofu](https://opentofu.org) to declaratively manage our pub.solar DNS records.
 
 ### Initial setup
 
-Skip this step if you already have a `triton` profile setup.
-
-```
-triton profile create
-```
-
-Please follow https://docs.greenbaum.cloud/en/devops/triton-cli.html for the details.
-
 You will need to setup the following [namecheap API credentials](https://www.namecheap.com/support/api/intro),
 look for "namecheap API key" in the pub.solar Keepass database.
 
@@ -28,13 +20,15 @@ You will probably also need to add your external IP to the [API allow list](http
 dig -4 ip @dns.toys
 ```
 
-Now, change into the terraform directory and initialize the terraform providers.
+Now, change into the terraform directory and initialize the terraform providers. To decrypt existing state,
+search for "terraform state passphrase" in the pub.solar Keepass database.
 
 ```
 cd terraform
-export TRITON_KEY_ID=$(cat ~/.config/triton/profiles.d/lev-1-pub_solar.json | jq --raw-output .keyId)
+export TF_VAR_state_passphrase=$(secret-tool lookup pub.solar terraform-state-passphrase-dns)
 
-terraform init
+alias tofu="terraform-backend-git --access-logs --tf tofu git terraform"
+tofu init
 ```
 
 Make your changes, e.g. in `dns.tf`.
@@ -46,20 +40,21 @@ $EDITOR dns.tf
 Plan your changes using:
 
 ```
-terraform plan -out pub-solar-infra.plan
+tofu plan -out pub-solar-infra.plan
 ```
 
 After verification, apply your changes with:
 
 ```
-terraform apply "pub-solar-infra.plan"
+tofu apply "pub-solar-infra.plan"
 ```
 
 ### Useful links
 
-We use the Manta remote backend to save the terraform state for collaboration.
+We use terraform-backend-git remote backend with opentofu state encryption for collaboration.
 
-- https://www.terraform.io/language/v1.2.x/settings/backends/manta
+- https://github.com/plumber-cd/terraform-backend-git
+- https://opentofu.org/docs/language/state/encryption
 
 Namecheap Terraform provider docs:
 

docs/drone-ci.md

@@ -1,19 +0,0 @@
-# Drone CI
-
-We currently use two CI systems, [drone CI](https://drone.io), reachable via
-https://ci.pub.solar and [Forgejo Actions](https://forgejo.org/docs/latest/user/actions/),
-which UI is integrated into https://git.pub.solar, for example
-https://git.pub.solar/pub-solar/infra/actions.
-
-### Signing the `.drone.yml` file
-
-Login to https://ci.pub.solar by clicking on the user icon in the bottom left.
-After logging in, you can view your personal API token by clicking on the same
-icon. If you're using the nix [development-shell](./development-shell.md), the
-`drone` command will already be installed.
-
-```
-export DRONE_TOKEN=<your-drone-api-token>
-
-drone --token $DRONE_TOKEN sign --save pub-solar/os
-```

docs/garage.md

@@ -0,0 +1,84 @@
+# Garage
+
+### How-To create a new bucket + keys
+
+Requirements:
+
+- `garage` RPC credentials, in the shared keepass, search for 'garage rpc secret'.
+- [Setup WireGuard](./administrative-access.md#ssh-access) for hosts: `trinkgenossin`, optionally: `delite`, `blue-shell`
+
+```
+ssh barkeeper@trinkgenossin.wg.pub.solar
+```
+
+```
+# Add a few spaces to avoid leaking the secret to the shell history
+   export GARAGE_RPC_SECRET=<secret-in-keepass>
+```
+
+Now, you can run the following command to check the cluster status:
+
+```
+garage status
+```
+
+Command to list all existing buckets:
+
+```
+garage bucket list
+```
+
+Creating a new bucket and access keys:
+
+```
+garage bucket create <bucket-name>
+garage key create <bucket-name>-key
+garage bucket allow <bucket-name> --read --write --key <bucket-name>-key
+```
+
+Full example for `mastodon` bucket:
+
+```
+garage bucket create mastodon
+
+garage key create mastodon-key
+
+garage bucket allow mastodon --read --write --key mastodon-key
+```
+
+Then [setup your favourite S3 client](https://garagehq.deuxfleurs.fr/documentation/connect/cli/)
+or use the bucket with any [S3 compatible software](https://garagehq.deuxfleurs.fr/documentation/connect/).
+
+Further reading:
+
+- https://garagehq.deuxfleurs.fr/documentation/quick-start/
+- https://garagehq.deuxfleurs.fr/documentation/connect/
+- https://garagehq.deuxfleurs.fr/documentation/connect/apps/#mastodon
+
+### Notes on manual setup steps
+
+```
+ssh barkeeper@trinkgenossin.wg.pub.solar
+
+# Add a few spaces to avoid leaking the secret to the shell history
+    export GARAGE_RPC_SECRET=<secret-in-keepass>
+
+# Uses the default config /etc/garage.toml
+garage node id
+
+garage node connect <node-id2>
+garage node connect <node-id3>
+
+garage status
+
+#Zones
+#DE-1 DE-2 NL-1
+
+garage layout assign fdaa -z DE-1 -c 800G -t trinkgenossin
+garage layout assign 8835 -z DE-2 -c 800G -t blue-shell
+garage layout assign 73da -z NL-1 -c 800G -t delite
+garage layout show
+garage layout apply --version 1
+```
+
+Source: https://garagehq.deuxfleurs.fr/documentation/cookbook/real-world/#creating-a-cluster-layout

docs/nextcloud.md

@@ -0,0 +1,19 @@
+# Nextcloud debugging
+
+Set loglevel to `0` for debug logs:
+
+```nix
+services.nextcloud.settings.loglevel = 0;
+```
+
+Then, logs appear in the `phpfpm-nextcloud.service` logs:
+
+```bash
+sudo journalctl -fu phpfpm-nextcloud
+```
+
+Make sure to set the loglevel back to the default `2` warning after debugging:
+
+```nix
+services.nextcloud.settings.loglevel = 2;
+```

docs/nixos-anywhere.md

@@ -0,0 +1,13 @@
+```
+curl -L https://github.com/nix-community/nixos-images/releases/download/nixos-unstable/nixos-kexec-installer-noninteractive-x86_64-linux.tar.gz | tar -xzf- -C /root
+/root/kexec/run
+```
+
+```
+mkdir -p /etc/secrets/initrd
+ssh-keygen -t ed25519 -f /etc/secrets/initrd/ssh_host_ed25519_key
+```
+
+```
+nix run github:nix-community/nixos-anywhere -- --flake .#blue-shell root@194.13.83.205
+```

flake.lock

@@ -52,11 +52,11 @@
         "utils": "utils"
       },
       "locked": {
-        "lastModified": 1718194053,
-        "narHash": "sha256-FaGrf7qwZ99ehPJCAwgvNY5sLCqQ3GDiE/6uLhxxwSY=",
+        "lastModified": 1727447169,
+        "narHash": "sha256-3KyjMPUKHkiWhwR91J1YchF6zb6gvckCAY1jOE+ne0U=",
         "owner": "serokell",
         "repo": "deploy-rs",
-        "rev": "3867348fa92bc892eba5d9ddb2d7a97b9e127a8a",
+        "rev": "aa07eb05537d4cd025e2310397a6adcedfe72c76",
         "type": "github"
       },
       "original": {
@@ -87,6 +87,26 @@
         "type": "github"
       }
     },
+    "disko": {
+      "inputs": {
+        "nixpkgs": [
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1729712798,
+        "narHash": "sha256-a+Aakkb+amHw4biOZ0iMo8xYl37uUL48YEXIC5PYJ/8=",
+        "owner": "nix-community",
+        "repo": "disko",
+        "rev": "09a776702b004fdf9c41a024e1299d575ee18a7d",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "repo": "disko",
+        "type": "github"
+      }
+    },
     "element-stickers": {
       "inputs": {
         "maunium-stickerpicker": [
@@ -165,11 +185,11 @@
         "nixpkgs-lib": "nixpkgs-lib"
       },
       "locked": {
-        "lastModified": 1722555600,
-        "narHash": "sha256-XOQkdLafnb/p9ij77byFQjDf5m5QYl9b2REiVClC+x4=",
+        "lastModified": 1727826117,
+        "narHash": "sha256-K5ZLCyfO/Zj9mPFldf3iwS6oZStJcU4tSpiXTMYaaL0=",
         "owner": "hercules-ci",
         "repo": "flake-parts",
-        "rev": "8471fe90ad337a8074e957b69ca4d0089218391d",
+        "rev": "3d04084d54bedc3d6b8b736c70ef449225c361b1",
         "type": "github"
       },
       "original": {
@@ -214,18 +234,19 @@
         "type": "github"
       }
     },
-    "flake-utils_3": {
+    "fork": {
       "locked": {
-        "lastModified": 1653893745,
-        "narHash": "sha256-0jntwV3Z8//YwuOjzhV2sgJJPt+HY6KhU7VZUL0fKZQ=",
-        "owner": "numtide",
-        "repo": "flake-utils",
-        "rev": "1ed9fb1935d260de5fe1c2f7ee0ebaae17ed2fa1",
+        "lastModified": 1729963002,
+        "narHash": "sha256-2zrYfd/qdfExU5zVwvH80uJnKc/dMeK6zp3O1UtW2Mo=",
+        "owner": "teutat3s",
+        "repo": "nixpkgs",
+        "rev": "005faaacbeede0296dec5c844f508027ab8a3ff6",
         "type": "github"
       },
       "original": {
-        "owner": "numtide",
-        "repo": "flake-utils",
+        "owner": "teutat3s",
+        "ref": "init-matrix-authentication-service-module",
+        "repo": "nixpkgs",
         "type": "github"
       }
     },
@@ -236,11 +257,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1720042825,
-        "narHash": "sha256-A0vrUB6x82/jvf17qPCpxaM+ulJnD8YZwH9Ci0BsAzE=",
+        "lastModified": 1726989464,
+        "narHash": "sha256-Vl+WVTJwutXkimwGprnEtXc/s/s8sMuXzqXaspIGlwM=",
         "owner": "nix-community",
         "repo": "home-manager",
-        "rev": "e1391fb22e18a36f57e6999c7a9f966dc80ac073",
+        "rev": "2f23fa308a7c067e52dfcc30a0758f47043ec176",
         "type": "github"
       },
       "original": {
@@ -259,11 +280,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1707424749,
-        "narHash": "sha256-eTvts5E3zmD4/DoAI9KedQjRwica0cg36wwIVp1NWbM=",
+        "lastModified": 1730041422,
+        "narHash": "sha256-aEz5/yUJN/PSEXwPBuKMs2FbAmz68fDIQ9B0tVRVmTo=",
         "ref": "main",
-        "rev": "1202a23c205b3c07a5feb5caf6813f21b3c69307",
-        "revCount": 30,
+        "rev": "09f7b1ed16c99f5fb5c5f9a2a73ccc9ff0645b35",
+        "revCount": 32,
         "type": "git",
         "url": "https://git.pub.solar/pub-solar/keycloak-theme"
       },
@@ -299,11 +320,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1724299755,
-        "narHash": "sha256-P5zMA17kD9tqiqMuNXwupkM7buM3gMNtoZ1VuJTRDE4=",
+        "lastModified": 1729757100,
+        "narHash": "sha256-x+8uGaX66V5+fUBHY23Q/OQyibQ38nISzxgj7A7Jqds=",
         "owner": "lnl7",
         "repo": "nix-darwin",
-        "rev": "a8968d88e5a537b0491f68ce910749cd870bdbef",
+        "rev": "04193f188e4144d7047f83ad1de81d6034d175cd",
         "type": "github"
       },
       "original": {
@@ -313,28 +334,13 @@
         "type": "github"
       }
     },
-    "nixos-flake": {
-      "locked": {
-        "lastModified": 1721140942,
-        "narHash": "sha256-iEqZGdnkG+Hm0jZhS59NJwEyB6z9caVnudWPGHZ/FAE=",
-        "owner": "srid",
-        "repo": "nixos-flake",
-        "rev": "5734c1d9a5fe0bc8e8beaf389ad6227392ca0108",
-        "type": "github"
-      },
-      "original": {
-        "owner": "srid",
-        "repo": "nixos-flake",
-        "type": "github"
-      }
-    },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1724242322,
-        "narHash": "sha256-HMpK7hNjhEk4z5SFg5UtxEio9OWFocHdaQzCfW1pE7w=",
+        "lastModified": 1729449015,
+        "narHash": "sha256-Gf04dXB0n4q0A9G5nTGH3zuMGr6jtJppqdeljxua1fo=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "224042e9a3039291f22f4f2ded12af95a616cca0",
+        "rev": "89172919243df199fe237ba0f776c3e3e3d72367",
         "type": "github"
       },
       "original": {
@@ -344,50 +350,33 @@
         "type": "github"
       }
     },
-    "nixpkgs-2205": {
-      "locked": {
-        "lastModified": 1685573264,
-        "narHash": "sha256-Zffu01pONhs/pqH07cjlF10NnMDLok8ix5Uk4rhOnZQ=",
-        "owner": "nixos",
-        "repo": "nixpkgs",
-        "rev": "380be19fbd2d9079f677978361792cb25e8a3635",
-        "type": "github"
-      },
-      "original": {
-        "owner": "nixos",
-        "ref": "nixos-22.05",
-        "repo": "nixpkgs",
-        "type": "github"
-      }
-    },
     "nixpkgs-lib": {
       "locked": {
-        "lastModified": 1722555339,
-        "narHash": "sha256-uFf2QeW7eAHlYXuDktm9c25OxOyCoUOQmh5SZ9amE5Q=",
+        "lastModified": 1727825735,
+        "narHash": "sha256-0xHYkMkeLVQAMa7gvkddbPqpxph+hDzdu1XdGPJR+Os=",
         "type": "tarball",
-        "url": "https://github.com/NixOS/nixpkgs/archive/a5d394176e64ab29c852d03346c1fc9b0b7d33eb.tar.gz"
+        "url": "https://github.com/NixOS/nixpkgs/archive/fb192fec7cc7a4c26d51779e9bab07ce6fa5597a.tar.gz"
       },
       "original": {
         "type": "tarball",
-        "url": "https://github.com/NixOS/nixpkgs/archive/a5d394176e64ab29c852d03346c1fc9b0b7d33eb.tar.gz"
+        "url": "https://github.com/NixOS/nixpkgs/archive/fb192fec7cc7a4c26d51779e9bab07ce6fa5597a.tar.gz"
       }
     },
     "root": {
       "inputs": {
         "agenix": "agenix",
         "deploy-rs": "deploy-rs",
+        "disko": "disko",
         "element-stickers": "element-stickers",
         "element-themes": "element-themes",
         "flake-parts": "flake-parts",
+        "fork": "fork",
         "home-manager": "home-manager",
         "keycloak-theme-pub-solar": "keycloak-theme-pub-solar",
         "maunium-stickerpicker": "maunium-stickerpicker",
         "nix-darwin": "nix-darwin",
-        "nixos-flake": "nixos-flake",
         "nixpkgs": "nixpkgs",
-        "nixpkgs-2205": "nixpkgs-2205",
         "simple-nixos-mailserver": "simple-nixos-mailserver",
-        "triton-vmtools": "triton-vmtools",
         "unstable": "unstable"
       }
     },
@@ -493,37 +482,13 @@
         "type": "github"
       }
     },
-    "triton-vmtools": {
-      "inputs": {
-        "flake-utils": "flake-utils_3",
-        "nixpkgs": [
-          "nixpkgs"
-        ]
-      },
-      "locked": {
-        "dir": "vmtools",
-        "lastModified": 1698443513,
-        "narHash": "sha256-wX2JIJ3JmJn6MAurdyjwZU+FZjLCwBArMrVSeeCb/ZU=",
-        "ref": "main",
-        "rev": "0d039dcf06afb8cbddd7ac54bae4d0d185f3e88e",
-        "revCount": 85,
-        "type": "git",
-        "url": "https://git.pub.solar/pub-solar/infra-vintage?dir=vmtools"
-      },
-      "original": {
-        "dir": "vmtools",
-        "ref": "main",
-        "type": "git",
-        "url": "https://git.pub.solar/pub-solar/infra-vintage?dir=vmtools"
-      }
-    },
     "unstable": {
       "locked": {
-        "lastModified": 1724224976,
-        "narHash": "sha256-Z/ELQhrSd7bMzTO8r7NZgi9g5emh+aRKoCdaAv5fiO0=",
+        "lastModified": 1729665710,
+        "narHash": "sha256-AlcmCXJZPIlO5dmFzV3V2XF6x/OpNWUV8Y/FMPGd8Z4=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "c374d94f1536013ca8e92341b540eba4c22f9c62",
+        "rev": "2768c7d042a37de65bb1b5b3268fc987e534c49d",
         "type": "github"
       },
       "original": {

flake.nix

@@ -3,8 +3,7 @@
     # Track channels with commits tested and built by hydra
     nixpkgs.url = "github:nixos/nixpkgs/nixos-24.05";
     unstable.url = "github:nixos/nixpkgs/nixos-unstable";
-
-    nixpkgs-2205.url = "github:nixos/nixpkgs/nixos-22.05";
+    fork.url = "github:teutat3s/nixpkgs/init-matrix-authentication-service-module";
 
     nix-darwin.url = "github:lnl7/nix-darwin/master";
     nix-darwin.inputs.nixpkgs.follows = "nixpkgs";
@@ -13,11 +12,13 @@
     home-manager.inputs.nixpkgs.follows = "nixpkgs";
 
     flake-parts.url = "github:hercules-ci/flake-parts";
-    nixos-flake.url = "github:srid/nixos-flake";
 
     deploy-rs.url = "github:serokell/deploy-rs";
     deploy-rs.inputs.nixpkgs.follows = "nixpkgs";
 
+    disko.url = "github:nix-community/disko";
+    disko.inputs.nixpkgs.follows = "nixpkgs";
+
     agenix.url = "github:ryantm/agenix";
     agenix.inputs.nixpkgs.follows = "nixpkgs";
     agenix.inputs.darwin.follows = "nix-darwin";
@@ -26,9 +27,6 @@
     keycloak-theme-pub-solar.url = "git+https://git.pub.solar/pub-solar/keycloak-theme?ref=main";
     keycloak-theme-pub-solar.inputs.nixpkgs.follows = "nixpkgs";
 
-    triton-vmtools.url = "git+https://git.pub.solar/pub-solar/infra-vintage?ref=main&dir=vmtools";
-    triton-vmtools.inputs.nixpkgs.follows = "nixpkgs";
-
     element-themes.url = "github:aaronraimist/element-themes/master";
     element-themes.flake = false;
 
@@ -53,7 +51,6 @@
       ];
 
       imports = [
-        inputs.nixos-flake.flakeModule
         ./logins
         ./lib
         ./overlays
@@ -65,6 +62,7 @@
           system,
           pkgs,
           config,
+          lib,
           ...
         }:
         {
@@ -77,6 +75,27 @@
             unstable = import inputs.unstable { inherit system; };
             master = import inputs.master { inherit system; };
           };
+
+          checks =
+            let
+              nixos-lib = import (inputs.nixpkgs + "/nixos/lib") { };
+              testDir = builtins.attrNames (builtins.readDir ./tests);
+              testFiles = builtins.filter (n: builtins.match "^.*.nix$" n != null) testDir;
+            in
+            builtins.listToAttrs (
+              map (x: {
+                name = "test-${lib.strings.removeSuffix ".nix" x}";
+                value = nixos-lib.runTest (
+                  import (./tests + "/${x}") {
+                    inherit self;
+                    inherit pkgs;
+                    inherit lib;
+                    inherit config;
+                  }
+                );
+              }) testFiles
+            );
+
           devShells.default = pkgs.mkShell {
             buildInputs = with pkgs; [
               deploy-rs
@@ -89,12 +108,15 @@
               nvfetcher
               shellcheck
               shfmt
-              treefmt
+              inputs.unstable.legacyPackages.${system}.treefmt2
               nixos-generators
-              inputs.nixpkgs-2205.legacyPackages.${system}.terraform
+              inputs.unstable.legacyPackages.${system}.opentofu
+              terraform-backend-git
+              terraform-ls
               jq
             ];
           };
+
           devShells.ci = pkgs.mkShell { buildInputs = with pkgs; [ nodejs ]; };
         };
 
@@ -123,10 +145,6 @@
               hostname = "nachtigall.wg.pub.solar";
               sshUser = username;
             };
-            flora-6 = {
-              hostname = "flora-6.wg.pub.solar";
-              sshUser = username;
-            };
             metronom = {
               hostname = "metronom.wg.pub.solar";
               sshUser = username;
@@ -135,6 +153,22 @@
               hostname = "tankstelle.wg.pub.solar";
               sshUser = username;
             };
+            underground = {
+              hostname = "80.244.242.3";
+              sshUser = username;
+            };
+            trinkgenossin = {
+              hostname = "trinkgenossin.wg.pub.solar";
+              sshUser = username;
+            };
+            delite = {
+              hostname = "delite.wg.pub.solar";
+              sshUser = username;
+            };
+            blue-shell = {
+              hostname = "blue-shell.wg.pub.solar";
+              sshUser = username;
+            };
           };
         };
     };

hosts/blue-shell/configuration.nix

@@ -0,0 +1,33 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+{
+  boot.loader.grub.enable = true;
+
+  boot.kernelParams = [
+    "boot.shell_on_fail=1"
+    "ip=dhcp"
+  ];
+
+  # This option defines the first version of NixOS you have installed on this particular machine,
+  # and is used to maintain compatibility with application data (e.g. databases) created on older NixOS versions.
+  #
+  # Most users should NEVER change this value after the initial install, for any reason,
+  # even if you've upgraded your system to a new NixOS release.
+  #
+  # This value does NOT affect the Nixpkgs version your packages and OS are pulled from,
+  # so changing it will NOT upgrade your system - see https://nixos.org/manual/nixos/stable/#sec-upgrading for how
+  # to actually do that.
+  #
+  # This value being lower than the current NixOS release does NOT mean your system is
+  # out of date, out of support, or vulnerable.
+  #
+  # Do NOT change this value unless you have manually inspected all the changes it would make to your configuration,
+  # and migrated your data accordingly.
+  #
+  # For more information, see `man configuration.nix` or https://nixos.org/manual/nixos/stable/options#opt-system.stateVersion .
+  system.stateVersion = "24.05"; # Did you read the comment?
+}

hosts/blue-shell/default.nix

@@ -1,11 +1,13 @@
-{ ... }:
+{ flake, ... }:
 
 {
   imports = [
-    # Include the results of the hardware scan.
     ./hardware-configuration.nix
     ./configuration.nix
-    ./triton-vmtools.nix
+    ./disk-config.nix
+
+    ./networking.nix
     ./wireguard.nix
+    #./backups.nix
   ];
 }

hosts/blue-shell/disk-config.nix

@@ -0,0 +1,101 @@
+{
+  disko.devices = {
+    disk = {
+      main = {
+        type = "disk";
+        device = "/dev/vdb";
+        content = {
+          type = "gpt";
+          partitions = {
+            bios = {
+              size = "1M";
+              type = "EF02"; # for grub MBR
+            };
+            boot = {
+              size = "1G";
+              type = "8300";
+              content = {
+                type = "filesystem";
+                format = "ext4";
+                mountpoint = "/boot";
+                mountOptions = [ "defaults" ];
+              };
+            };
+            luks = {
+              size = "100%";
+              content = {
+                type = "luks";
+                name = "cryptroot";
+                extraOpenArgs = [ ];
+                # if you want to use the key for interactive login be sure there is no trailing newline
+                # for example use `echo -n "password" > /tmp/secret.key`
+                passwordFile = "/tmp/luks-password";
+                content = {
+                  type = "lvm_pv";
+                  vg = "vg0";
+                };
+              };
+            };
+          };
+        };
+      };
+      data = {
+        type = "disk";
+        device = "/dev/vdc";
+        content = {
+          type = "gpt";
+          partitions = {
+            luks = {
+              size = "100%";
+              content = {
+                type = "luks";
+                name = "cryptdata";
+                extraOpenArgs = [ ];
+                # if you want to use the key for interactive login be sure there is no trailing newline
+                # for example use `echo -n "password" > /tmp/secret.key`
+                passwordFile = "/tmp/luks-password";
+                content = {
+                  type = "filesystem";
+                  format = "xfs";
+                  mountpoint = "/var/lib/garage/data";
+                  mountOptions = [ "defaults" ];
+                };
+              };
+            };
+          };
+        };
+      };
+    };
+    lvm_vg = {
+      vg0 = {
+        type = "lvm_vg";
+        lvs = {
+          root = {
+            size = "100G";
+            content = {
+              type = "filesystem";
+              format = "ext4";
+              mountpoint = "/";
+              mountOptions = [ "defaults" ];
+            };
+          };
+          swap = {
+            size = "16G";
+            content = {
+              type = "swap";
+            };
+          };
+          metadata = {
+            size = "50G";
+            content = {
+              type = "filesystem";
+              format = "btrfs";
+              mountpoint = "/var/lib/garage/meta";
+              mountOptions = [ "defaults" ];
+            };
+          };
+        };
+      };
+    };
+  };
+}

hosts/blue-shell/hardware-configuration.nix

@@ -0,0 +1,27 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{
+  config,
+  lib,
+  pkgs,
+  modulesPath,
+  ...
+}:
+
+{
+  imports = [ (modulesPath + "/profiles/qemu-guest.nix") ];
+
+  boot.initrd.availableKernelModules = [
+    "ata_piix"
+    "uhci_hcd"
+    "virtio_pci"
+    "sr_mod"
+    "virtio_blk"
+  ];
+  boot.initrd.kernelModules = [ ];
+  boot.kernelModules = [ ];
+  boot.extraModulePackages = [ ];
+
+  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
+}

hosts/blue-shell/networking.nix

@@ -0,0 +1,26 @@
+{
+  config,
+  pkgs,
+  flake,
+  ...
+}:
+{
+  services.garage.settings.rpc_public_addr = "[2a03:4000:43:24e::1]:3901";
+
+  networking.hostName = "blue-shell";
+  networking.hostId = "00000005";
+
+  networking.useDHCP = false;
+  systemd.network.enable = true;
+  systemd.network.networks."10-wan" = {
+    matchConfig.Name = "ens3";
+    address = [
+      "194.13.83.205/22"
+      "2a03:4000:43:24e::1/64"
+    ];
+    gateway = [
+      "194.13.80.1"
+      "fe80::1"
+    ];
+  };
+}

hosts/blue-shell/wireguard.nix

@@ -0,0 +1,51 @@
+{
+  config,
+  pkgs,
+  flake,
+  ...
+}:
+let
+  wireguardIPv4 = "10.7.6.7";
+  wireguardIPv6 = "fd00:fae:fae:fae:fae:7::";
+in
+{
+  networking.firewall.allowedUDPPorts = [ 51820 ];
+
+  age.secrets.wg-private-key.file = "${flake.self}/secrets/blue-shell-wg-private-key.age";
+
+  networking.wireguard.interfaces = {
+    wg-ssh = {
+      listenPort = 51820;
+      mtu = 1300;
+      ips = [
+        "${wireguardIPv4}/32"
+        "${wireguardIPv6}/96"
+      ];
+      privateKeyFile = config.age.secrets.wg-private-key.path;
+      peers = flake.self.logins.admins.wireguardDevices ++ [
+        {
+          # trinkgenossin.pub.solar
+          publicKey = "QWgHovHxtqiQhnHLouSWiT6GIoQDmuvnThYL5c/rvU4=";
+          allowedIPs = [
+            "10.7.6.5/32"
+            "fd00:fae:fae:fae:fae:5::/96"
+          ];
+          #endpoint = "85.215.152.22:51820";
+          endpoint = "[2a01:239:35d:f500::1]:51820";
+          persistentKeepalive = 15;
+        }
+      ];
+    };
+  };
+
+  services.openssh.listenAddresses = [
+    {
+      addr = wireguardIPv4;
+      port = 22;
+    }
+    {
+      addr = "[${wireguardIPv6}]";
+      port = 22;
+    }
+  ];
+}

hosts/default.nix

@@ -1,9 +1,35 @@
-{ self, ... }:
+{
+  self,
+  inputs,
+  config,
+  ...
+}:
 {
   flake = {
-    nixosConfigurations = {
-      nachtigall = self.nixos-flake.lib.mkLinuxSystem {
+    nixosModules = {
+      home-manager = {
         imports = [
+          inputs.home-manager.nixosModules.home-manager
+          {
+            home-manager.useGlobalPkgs = true;
+            home-manager.useUserPackages = true;
+            home-manager.extraSpecialArgs = {
+              flake = {
+                inherit self inputs config;
+              };
+            };
+          }
+        ];
+      };
+    };
+    nixosConfigurations = {
+      nachtigall = self.inputs.nixpkgs.lib.nixosSystem {
+        specialArgs = {
+          flake = {
+            inherit self inputs config;
+          };
+        };
+        modules = [
           self.inputs.agenix.nixosModules.default
           self.nixosModules.home-manager
           ./nachtigall
@@ -11,6 +37,7 @@
           self.nixosModules.unlock-zfs-on-boot
           self.nixosModules.core
           self.nixosModules.docker
+          self.nixosModules.backups
 
           self.nixosModules.nginx
           self.nixosModules.collabora
@@ -42,32 +69,20 @@
         ];
       };
 
-      flora-6 = self.nixos-flake.lib.mkLinuxSystem {
-        imports = [
-          self.inputs.agenix.nixosModules.default
-          self.nixosModules.home-manager
-          ./flora-6
-          self.nixosModules.overlays
-          self.nixosModules.core
-
-          self.nixosModules.keycloak
-          self.nixosModules.caddy
-          self.nixosModules.drone
-          self.nixosModules.forgejo-actions-runner
-          self.nixosModules.grafana
-          self.nixosModules.prometheus
-          self.nixosModules.loki
-        ];
+      metronom = self.inputs.nixpkgs.lib.nixosSystem {
+        specialArgs = {
+          flake = {
+            inherit self inputs config;
           };
-
-      metronom = self.nixos-flake.lib.mkLinuxSystem {
-        imports = [
+        };
+        modules = [
           self.inputs.agenix.nixosModules.default
           self.nixosModules.home-manager
           ./metronom
           self.nixosModules.overlays
           self.nixosModules.unlock-zfs-on-boot
           self.nixosModules.core
+          self.nixosModules.backups
           self.nixosModules.mail
           self.nixosModules.prometheus-exporters
           self.nixosModules.promtail
@@ -76,17 +91,117 @@
         ];
       };
 
-      tankstelle = self.nixos-flake.lib.mkLinuxSystem {
-        imports = [
+      tankstelle = self.inputs.nixpkgs.lib.nixosSystem {
+        specialArgs = {
+          flake = {
+            inherit self inputs config;
+          };
+        };
+        modules = [
           self.inputs.agenix.nixosModules.default
           self.nixosModules.home-manager
           ./tankstelle
           self.nixosModules.overlays
           self.nixosModules.core
+          self.nixosModules.backups
           self.nixosModules.prometheus-exporters
           self.nixosModules.promtail
         ];
       };
+
+      trinkgenossin = self.inputs.nixpkgs.lib.nixosSystem {
+        specialArgs = {
+          flake = {
+            inherit self inputs config;
+          };
+        };
+        modules = [
+          self.inputs.agenix.nixosModules.default
+          self.nixosModules.home-manager
+          ./trinkgenossin
+          self.nixosModules.backups
+          self.nixosModules.overlays
+          self.nixosModules.unlock-luks-on-boot
+          self.nixosModules.core
+
+          self.nixosModules.garage
+          self.nixosModules.nginx
+
+          # This module is already using options, and those options are used by the grafana module
+          self.nixosModules.keycloak
+          self.nixosModules.grafana
+          self.nixosModules.prometheus
+          self.nixosModules.loki
+        ];
+      };
+
+      delite = self.inputs.nixpkgs.lib.nixosSystem {
+        specialArgs = {
+          flake = {
+            inherit self inputs config;
+          };
+        };
+        modules = [
+          self.inputs.agenix.nixosModules.default
+          self.inputs.disko.nixosModules.disko
+          self.nixosModules.home-manager
+          ./delite
+          self.nixosModules.overlays
+          self.nixosModules.unlock-luks-on-boot
+          self.nixosModules.core
+          self.nixosModules.prometheus-exporters
+          self.nixosModules.promtail
+
+          self.nixosModules.garage
+          self.nixosModules.nginx
+        ];
+      };
+
+      blue-shell = self.inputs.nixpkgs.lib.nixosSystem {
+        specialArgs = {
+          flake = {
+            inherit self inputs config;
+          };
+        };
+        modules = [
+          self.inputs.agenix.nixosModules.default
+          self.inputs.disko.nixosModules.disko
+          self.nixosModules.home-manager
+          ./blue-shell
+          self.nixosModules.overlays
+          self.nixosModules.unlock-luks-on-boot
+          self.nixosModules.core
+          self.nixosModules.prometheus-exporters
+          self.nixosModules.promtail
+
+          self.nixosModules.garage
+          self.nixosModules.nginx
+        ];
+      };
+
+      underground = self.inputs.nixpkgs.lib.nixosSystem {
+        specialArgs = {
+          flake = {
+            inherit self inputs config;
+          };
+        };
+        modules = [
+          self.inputs.agenix.nixosModules.default
+          self.nixosModules.home-manager
+          ./underground
+          self.nixosModules.overlays
+          self.nixosModules.unlock-luks-on-boot
+          self.nixosModules.core
+
+          self.nixosModules.backups
+          self.nixosModules.keycloak
+          self.nixosModules.postgresql
+          self.nixosModules.matrix
+          self.nixosModules.matrix-irc
+          self.nixosModules.nginx
+          self.nixosModules.nginx-matrix
+        ];
+      };
     };
   };
 }

hosts/delite/configuration.nix

@@ -0,0 +1,33 @@
+{
+  flake,
+  config,
+  pkgs,
+  ...
+}:
+{
+  boot.loader.grub.enable = true;
+
+  boot.kernelParams = [
+    "boot.shell_on_fail=1"
+    "ip=5.255.119.132::5.255.119.1:255.255.255.0:delite::off"
+  ];
+
+  # This option defines the first version of NixOS you have installed on this particular machine,
+  # and is used to maintain compatibility with application data (e.g. databases) created on older NixOS versions.
+  #
+  # Most users should NEVER change this value after the initial install, for any reason,
+  # even if you've upgraded your system to a new NixOS release.
+  #
+  # This value does NOT affect the Nixpkgs version your packages and OS are pulled from,
+  # so changing it will NOT upgrade your system - see https://nixos.org/manual/nixos/stable/#sec-upgrading for how
+  # to actually do that.
+  #
+  # This value being lower than the current NixOS release does NOT mean your system is
+  # out of date, out of support, or vulnerable.
+  #
+  # Do NOT change this value unless you have manually inspected all the changes it would make to your configuration,
+  # and migrated your data accordingly.
+  #
+  # For more information, see `man configuration.nix` or https://nixos.org/manual/nixos/stable/options#opt-system.stateVersion .
+  system.stateVersion = "24.05"; # Did you read the comment?
+}

hosts/delite/default.nix

@@ -0,0 +1,13 @@
+{ flake, ... }:
+
+{
+  imports = [
+    ./hardware-configuration.nix
+    ./configuration.nix
+    ./disk-config.nix
+
+    ./networking.nix
+    ./wireguard.nix
+    #./backups.nix
+  ];
+}

hosts/delite/disk-config.nix

@@ -0,0 +1,84 @@
+{
+  disko.devices = {
+    disk = {
+      main = {
+        type = "disk";
+        device = "/dev/vda";
+        content = {
+          type = "gpt";
+          partitions = {
+            bios = {
+              size = "1M";
+              type = "EF02"; # for grub MBR
+            };
+            boot = {
+              size = "1G";
+              type = "8300";
+              content = {
+                type = "filesystem";
+                format = "ext4";
+                mountpoint = "/boot";
+                mountOptions = [ "defaults" ];
+              };
+            };
+            luks = {
+              size = "100%";
+              content = {
+                type = "luks";
+                name = "cryptroot";
+                extraOpenArgs = [ ];
+                # if you want to use the key for interactive login be sure there is no trailing newline
+                # for example use `echo -n "password" > /tmp/secret.key`
+                passwordFile = "/tmp/luks-password";
+                content = {
+                  type = "lvm_pv";
+                  vg = "vg0";
+                };
+              };
+            };
+          };
+        };
+      };
+    };
+    lvm_vg = {
+      vg0 = {
+        type = "lvm_vg";
+        lvs = {
+          root = {
+            size = "40G";
+            content = {
+              type = "filesystem";
+              format = "ext4";
+              mountpoint = "/";
+              mountOptions = [ "defaults" ];
+            };
+          };
+          swap = {
+            size = "8G";
+            content = {
+              type = "swap";
+            };
+          };
+          data = {
+            size = "800G";
+            content = {
+              type = "filesystem";
+              format = "xfs";
+              mountpoint = "/var/lib/garage/data";
+              mountOptions = [ "defaults" ];
+            };
+          };
+          metadata = {
+            size = "50G";
+            content = {
+              type = "filesystem";
+              format = "btrfs";
+              mountpoint = "/var/lib/garage/meta";
+              mountOptions = [ "defaults" ];
+            };
+          };
+        };
+      };
+    };
+  };
+}

hosts/delite/hardware-configuration.nix

@@ -0,0 +1,26 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{
+  config,
+  lib,
+  pkgs,
+  modulesPath,
+  ...
+}:
+
+{
+  imports = [ (modulesPath + "/profiles/qemu-guest.nix") ];
+
+  boot.initrd.availableKernelModules = [
+    "ata_piix"
+    "uhci_hcd"
+    "virtio_pci"
+    "virtio_blk"
+  ];
+  boot.initrd.kernelModules = [ "dm-snapshot" ];
+  boot.kernelModules = [ ];
+  boot.extraModulePackages = [ ];
+
+  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
+}

hosts/delite/networking.nix

@@ -0,0 +1,26 @@
+{
+  config,
+  pkgs,
+  flake,
+  ...
+}:
+{
+  services.garage.settings.rpc_public_addr = "[2a04:52c0:124:9d8c::2]:3901";
+
+  networking.hostName = "delite";
+  networking.hostId = "00000004";
+
+  networking.useDHCP = false;
+  systemd.network.enable = true;
+  systemd.network.networks."10-wan" = {
+    matchConfig.Name = "ens3";
+    address = [
+      "5.255.119.132/24"
+      "2a04:52c0:124:9d8c::2/48"
+    ];
+    gateway = [
+      "5.255.119.1"
+      "2a04:52c0:124::1"
+    ];
+  };
+}

hosts/delite/wireguard.nix

@@ -0,0 +1,51 @@
+{
+  config,
+  pkgs,
+  flake,
+  ...
+}:
+let
+  wireguardIPv4 = "10.7.6.6";
+  wireguardIPv6 = "fd00:fae:fae:fae:fae:6::";
+in
+{
+  networking.firewall.allowedUDPPorts = [ 51820 ];
+
+  age.secrets.wg-private-key.file = "${flake.self}/secrets/delite-wg-private-key.age";
+
+  networking.wireguard.interfaces = {
+    wg-ssh = {
+      listenPort = 51820;
+      mtu = 1300;
+      ips = [
+        "${wireguardIPv4}/32"
+        "${wireguardIPv6}/96"
+      ];
+      privateKeyFile = config.age.secrets.wg-private-key.path;
+      peers = flake.self.logins.admins.wireguardDevices ++ [
+        {
+          # trinkgenossin.pub.solar
+          publicKey = "QWgHovHxtqiQhnHLouSWiT6GIoQDmuvnThYL5c/rvU4=";
+          allowedIPs = [
+            "10.7.6.5/32"
+            "fd00:fae:fae:fae:fae:5::/96"
+          ];
+          #endpoint = "85.215.152.22:51820";
+          endpoint = "[2a01:239:35d:f500::1]:51820";
+          persistentKeepalive = 15;
+        }
+      ];
+    };
+  };
+
+  services.openssh.listenAddresses = [
+    {
+      addr = wireguardIPv4;
+      port = 22;
+    }
+    {
+      addr = "[${wireguardIPv6}]";
+      port = 22;
+    }
+  ];
+}

hosts/flora-6/configuration.nix

@@ -1,72 +0,0 @@
-{
-  config,
-  lib,
-  pkgs,
-  flake,
-  ...
-}:
-let
-  psCfg = config.pub-solar;
-in
-{
-  config = {
-    # Override nix.conf for more agressive garbage collection
-    nix.extraOptions = lib.mkForce ''
-      experimental-features = flakes nix-command
-      min-free = 536870912
-      keep-outputs = false
-      keep-derivations = false
-      fallback = true
-    '';
-
-    # # #
-    # # # Triton host specific options
-    # # # DO NOT ALTER below this line, changes might render system unbootable
-    # # #
-
-    # Use the systemd-boot EFI boot loader.
-    boot.loader.systemd-boot.enable = true;
-    boot.loader.efi.canTouchEfiVariables = true;
-
-    # Force getting the hostname from cloud-init
-    networking.hostName = lib.mkDefault "";
-
-    # We use cloud-init to configure networking, this option should fix
-    # systemd-networkd-wait-online timeouts
-    #systemd.services."systemd-networkd".environment.SYSTEMD_LOG_LEVEL = "debug";
-    systemd.network.wait-online.ignoredInterfaces = [
-      "docker0"
-      "wg-ssh"
-    ];
-
-    # List services that you want to enable:
-    services.cloud-init.enable = true;
-    services.cloud-init.ext4.enable = true;
-    services.cloud-init.network.enable = true;
-    # use the default NixOS cloud-init config, but add some SmartOS customization to it
-    environment.etc."cloud/cloud.cfg.d/90_smartos.cfg".text = ''
-      datasource_list: [ SmartOS ]
-
-      # Do not create the centos/ubuntu/debian user
-      users: [ ]
-
-      # mount second disk with label ephemeral0, gets formated by cloud-init
-      # this will fail to get added to /etc/fstab as it's read-only, but should
-      # mount at boot anyway
-      mounts:
-      - [ vdb, /data, auto, "defaults,nofail" ]
-    '';
-
-    # We manage the firewall with nix, too
-    # altough triton can also manage firewall rules via the triton fwrule subcommand
-    networking.firewall.enable = true;
-
-    # This value determines the NixOS release from which the default
-    # settings for stateful data, like file locations and database versions
-    # on your system were taken. It‘s perfectly fine and recommended to leave
-    # this value at the release version of the first install of this system.
-    # Before changing this value read the documentation for this option
-    # (e.g. man configuration.nix or on https://nixos.org/nixos/options.html).
-    system.stateVersion = "22.05"; # Did you read the comment?
-  };
-}

hosts/flora-6/triton-vmtools.nix

@@ -1,6 +0,0 @@
-{ pkgs, flake, ... }:
-{
-  environment.systemPackages = with pkgs; [
-    flake.inputs.triton-vmtools.packages.${pkgs.system}.default
-  ];
-}

hosts/metronom/backups.nix

@@ -1,13 +1,29 @@
-{ flake, ... }:
+{ config, flake, ... }:
 {
-  age.secrets."restic-repo-droppie" = {
-    file = "${flake.self}/secrets/restic-repo-droppie.age";
+  age.secrets."restic-repo-storagebox-metronom" = {
+    file = "${flake.self}/secrets/restic-repo-storagebox-metronom.age";
     mode = "400";
     owner = "root";
   };
-  age.secrets."restic-repo-storagebox" = {
-    file = "${flake.self}/secrets/restic-repo-storagebox.age";
+  age.secrets.restic-repo-garage-metronom = {
+    file = "${flake.self}/secrets/restic-repo-garage-metronom.age";
     mode = "400";
     owner = "root";
   };
+  age.secrets.restic-repo-garage-metronom-env = {
+    file = "${flake.self}/secrets/restic-repo-garage-metronom-env.age";
+    mode = "400";
+    owner = "root";
+  };
+
+  pub-solar-os.backups.repos.storagebox = {
+    passwordFile = config.age.secrets."restic-repo-storagebox-metronom".path;
+    repository = "sftp:u377325@u377325.your-storagebox.de:/metronom-backups";
+  };
+
+  pub-solar-os.backups.repos.garage = {
+    passwordFile = config.age.secrets."restic-repo-garage-metronom".path;
+    environmentFile = config.age.secrets."restic-repo-garage-metronom-env".path;
+    repository = "s3:https://buckets.pub.solar/metronom-backups";
+  };
 }

hosts/metronom/configuration.nix

@@ -23,6 +23,14 @@
     pools = [ "root_pool" ];
   };
 
+  # Declarative SSH private key
+  age.secrets."metronom-root-ssh-key" = {
+    file = "${flake.self}/secrets/metronom-root-ssh-key.age";
+    path = "/root/.ssh/id_ed25519";
+    mode = "400";
+    owner = "root";
+  };
+
   # Declarative SSH private key
   #age.secrets."metronom-root-ssh-key" = {
   #  file = "${flake.self}/secrets/metronom-root-ssh-key.age";

hosts/metronom/default.nix

@@ -7,6 +7,6 @@
 
     ./networking.nix
     ./wireguard.nix
-    #./backups.nix
+    ./backups.nix
   ];
 }

hosts/metronom/wireguard.nix

@@ -19,15 +19,6 @@
       ];
       privateKeyFile = config.age.secrets.wg-private-key.path;
       peers = flake.self.logins.admins.wireguardDevices ++ [
-        {
-          # flora-6.pub.solar
-          endpoint = "80.71.153.210:51820";
-          publicKey = "jtSR5G2P/nm9s8WrVc26Xc/SQLupRxyXE+5eIeqlsTU=";
-          allowedIPs = [
-            "10.7.6.2/32"
-            "fd00:fae:fae:fae:fae:2::/96"
-          ];
-        }
         {
           # nachtigall.pub.solar
           endpoint = "138.201.80.102:51820";
@@ -37,6 +28,17 @@
             "fd00:fae:fae:fae:fae:1::/96"
           ];
         }
+        {
+          # trinkgenossin.pub.solar
+          publicKey = "QWgHovHxtqiQhnHLouSWiT6GIoQDmuvnThYL5c/rvU4=";
+          allowedIPs = [
+            "10.7.6.5/32"
+            "fd00:fae:fae:fae:fae:5::/96"
+          ];
+          #endpoint = "85.215.152.22:51820";
+          endpoint = "[2a01:239:35d:f500::1]:51820";
+          persistentKeepalive = 15;
+        }
       ];
     };
   };

hosts/nachtigall/backups.nix

@@ -1,13 +1,34 @@
-{ flake, ... }:
+{ config, flake, ... }:
 {
   age.secrets."restic-repo-droppie" = {
     file = "${flake.self}/secrets/restic-repo-droppie.age";
     mode = "400";
     owner = "root";
   };
-  age.secrets."restic-repo-storagebox" = {
-    file = "${flake.self}/secrets/restic-repo-storagebox.age";
+  age.secrets."restic-repo-storagebox-nachtigall" = {
+    file = "${flake.self}/secrets/restic-repo-storagebox-nachtigall.age";
     mode = "400";
     owner = "root";
   };
+  age.secrets.restic-repo-garage-nachtigall = {
+    file = "${flake.self}/secrets/restic-repo-garage-nachtigall.age";
+    mode = "400";
+    owner = "root";
+  };
+  age.secrets.restic-repo-garage-nachtigall-env = {
+    file = "${flake.self}/secrets/restic-repo-garage-nachtigall-env.age";
+    mode = "400";
+    owner = "root";
+  };
+
+  pub-solar-os.backups.repos.storagebox = {
+    passwordFile = config.age.secrets."restic-repo-storagebox-nachtigall".path;
+    repository = "sftp:u377325@u377325.your-storagebox.de:/backups";
+  };
+
+  pub-solar-os.backups.repos.garage = {
+    passwordFile = config.age.secrets."restic-repo-garage-nachtigall".path;
+    environmentFile = config.age.secrets."restic-repo-garage-nachtigall-env".path;
+    repository = "s3:https://buckets.pub.solar/nachtigall-backups";
+  };
 }

hosts/nachtigall/configuration.nix

@@ -48,9 +48,77 @@
     owner = "root";
   };
 
-  pub-solar-os.auth.enable = true;
+  # keycloak
+  age.secrets.keycloak-database-password = {
+    file = "${flake.self}/secrets/keycloak-database-password.age";
+    mode = "600";
+    #owner = "keycloak";
+  };
 
-  nixpkgs.config.permittedInsecurePackages = [ "keycloak-23.0.6" ];
+  pub-solar-os.auth = {
+    enable = true;
+    database-password-file = config.age.secrets.keycloak-database-password.path;
+  };
+
+  # matrix-synapse
+  age.secrets."matrix-synapse-signing-key" = {
+    file = "${flake.self}/secrets/matrix-synapse-signing-key.age";
+    mode = "400";
+    owner = "matrix-synapse";
+  };
+
+  age.secrets."matrix-synapse-secret-config.yaml" = {
+    file = "${flake.self}/secrets/matrix-synapse-secret-config.yaml.age";
+    mode = "400";
+    owner = "matrix-synapse";
+  };
+
+  age.secrets."matrix-synapse-sliding-sync-secret" = {
+    file = "${flake.self}/secrets/matrix-synapse-sliding-sync-secret.age";
+    mode = "400";
+    owner = "matrix-synapse";
+  };
+
+  age.secrets."matrix-authentication-service-secret-config.yml" = {
+    file = "${flake.self}/secrets/matrix-authentication-service-secret-config.yml.age";
+    mode = "400";
+    owner = "matrix-authentication-service";
+  };
+
+  pub-solar-os.matrix = {
+    enable = true;
+    synapse = {
+      sliding-sync.enable = false;
+      signing_key_path = config.age.secrets."matrix-synapse-signing-key".path;
+      extra-config-files = [
+        config.age.secrets."matrix-synapse-secret-config.yaml".path
+
+        # The registration file is automatically generated after starting the
+        # appservice for the first time.
+        # cp /var/lib/mautrix-telegram/telegram-registration.yaml \
+        #   /var/lib/matrix-synapse/
+        # chown matrix-synapse:matrix-synapse \
+        #   /var/lib/matrix-synapse/telegram-registration.yaml
+        "/var/lib/matrix-synapse/telegram-registration.yaml"
+      ];
+      app-service-config-files = [
+        "/var/lib/matrix-synapse/telegram-registration.yaml"
+        "/var/lib/matrix-appservice-irc/registration.yml"
+        # "/matrix-appservice-slack-registration.yaml"
+        # "/hookshot-registration.yml"
+        # "/matrix-mautrix-signal-registration.yaml"
+        # "/matrix-mautrix-telegram-registration.yaml"
+      ];
+    };
+    matrix-authentication-service.extra-config-files = [
+      config.age.secrets."matrix-authentication-service-secret-config.yml".path
+    ];
+  };
+
+  systemd.services.postgresql = {
+    after = [ "var-lib-postgresql.mount" ];
+    requisite = [ "var-lib-postgresql.mount" ];
+  };
 
   # This value determines the NixOS release with which your system is to be
   # compatible, in order to avoid breaking some software such as database

hosts/nachtigall/default.nix

@@ -9,5 +9,12 @@
     ./networking.nix
     ./wireguard.nix
     ./backups.nix
+    "${flake.inputs.fork}/nixos/modules/services//matrix/matrix-authentication-service.nix"
+    "${flake.inputs.unstable}/nixos/modules/services/web-apps/mastodon.nix"
+  ];
+
+  disabledModules = [
+    "services/matrix/matrix-authentication-service.nix "
+    "services/web-apps/mastodon.nix"
   ];
 }

hosts/nachtigall/wireguard.nix

@@ -19,15 +19,6 @@
       ];
       privateKeyFile = config.age.secrets.wg-private-key.path;
       peers = flake.self.logins.admins.wireguardDevices ++ [
-        {
-          # flora-6.pub.solar
-          endpoint = "80.71.153.210:51820";
-          publicKey = "jtSR5G2P/nm9s8WrVc26Xc/SQLupRxyXE+5eIeqlsTU=";
-          allowedIPs = [
-            "10.7.6.2/32"
-            "fd00:fae:fae:fae:fae:2::/96"
-          ];
-        }
         {
           # tankstelle.pub.solar
           endpoint = "80.244.242.5:51820";
@@ -37,6 +28,17 @@
             "fd00:fae:fae:fae:fae:4::/96"
           ];
         }
+        {
+          # trinkgenossin.pub.solar
+          publicKey = "QWgHovHxtqiQhnHLouSWiT6GIoQDmuvnThYL5c/rvU4=";
+          allowedIPs = [
+            "10.7.6.5/32"
+            "fd00:fae:fae:fae:fae:5::/96"
+          ];
+          #endpoint = "85.215.152.22:51820";
+          endpoint = "[2a01:239:35d:f500::1]:51820";
+          persistentKeepalive = 15;
+        }
       ];
     };
   };

hosts/tankstelle/backups.nix

@@ -5,8 +5,8 @@
     mode = "400";
     owner = "root";
   };
-  age.secrets."restic-repo-storagebox" = {
-    file = "${flake.self}/secrets/restic-repo-storagebox.age";
+  age.secrets."restic-repo-storagebox-tankstelle" = {
+    file = "${flake.self}/secrets/restic-repo-storagebox-tankstelle.age";
     mode = "400";
     owner = "root";
   };

hosts/tankstelle/wireguard.nix

@@ -29,13 +29,15 @@
           ];
         }
         {
-          # flora-6.pub.solar
-          endpoint = "80.71.153.210:51820";
-          publicKey = "jtSR5G2P/nm9s8WrVc26Xc/SQLupRxyXE+5eIeqlsTU=";
+          # trinkgenossin.pub.solar
+          publicKey = "QWgHovHxtqiQhnHLouSWiT6GIoQDmuvnThYL5c/rvU4=";
           allowedIPs = [
-            "10.7.6.2/32"
-            "fd00:fae:fae:fae:fae:2::/96"
+            "10.7.6.5/32"
+            "fd00:fae:fae:fae:fae:5::/96"
           ];
+          #endpoint = "85.215.152.22:51820";
+          endpoint = "[2a01:239:35d:f500::1]:51820";
+          persistentKeepalive = 15;
         }
       ];
     };

hosts/trinkgenossin/configuration.nix

@@ -0,0 +1,35 @@
+{
+  flake,
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+{
+  boot.loader.grub.enable = true;
+  boot.loader.grub.devices = [ "/dev/vda" ];
+
+  boot.kernelParams = [
+    "boot.shell_on_fail=1"
+    "ip=dhcp"
+  ];
+
+  # This option defines the first version of NixOS you have installed on this particular machine,
+  # and is used to maintain compatibility with application data (e.g. databases) created on older NixOS versions.
+  #
+  # Most users should NEVER change this value after the initial install, for any reason,
+  # even if you've upgraded your system to a new NixOS release.
+  #
+  # This value does NOT affect the Nixpkgs version your packages and OS are pulled from,
+  # so changing it will NOT upgrade your system - see https://nixos.org/manual/nixos/stable/#sec-upgrading for how
+  # to actually do that.
+  #
+  # This value being lower than the current NixOS release does NOT mean your system is
+  # out of date, out of support, or vulnerable.
+  #
+  # Do NOT change this value unless you have manually inspected all the changes it would make to your configuration,
+  # and migrated your data accordingly.
+  #
+  # For more information, see `man configuration.nix` or https://nixos.org/manual/nixos/stable/options#opt-system.stateVersion .
+  system.stateVersion = "24.05"; # Did you read the comment?
+}

hosts/trinkgenossin/default.nix

@@ -0,0 +1,12 @@
+{ flake, ... }:
+
+{
+  imports = [
+    ./hardware-configuration.nix
+    ./configuration.nix
+
+    ./networking.nix
+    ./wireguard.nix
+    #./backups.nix
+  ];
+}

hosts/trinkgenossin/hardware-configuration.nix

@@ -8,45 +8,47 @@
   modulesPath,
   ...
 }:
+
 {
   imports = [ ];
 
   boot.initrd.availableKernelModules = [
-    "ahci"
+    "ata_piix"
+    "uhci_hcd"
     "virtio_pci"
-    "xhci_pci"
     "sr_mod"
     "virtio_blk"
+    "virtio_net"
   ];
-  boot.initrd.kernelModules = [ ];
-  boot.kernelModules = [ ];
+  boot.initrd.kernelModules = [ "dm-snapshot" ];
+  boot.kernelModules = [ "kvm-amd" ];
   boot.extraModulePackages = [ ];
 
+  boot.initrd.luks.devices."cryptroot" = {
+    device = "/dev/disk/by-uuid/52a1fd17-63d7-4d0a-b7ff-74aceaf6085a";
+  };
+
   fileSystems."/" = {
     device = "/dev/disk/by-label/nixos";
-    autoResize = true;
     fsType = "ext4";
   };
 
   fileSystems."/boot" = {
     device = "/dev/disk/by-label/boot";
-    fsType = "vfat";
-  };
-
-  fileSystems."/data" = {
-    device = "/dev/disk/by-label/ephemeral0";
     fsType = "ext4";
-    options = [
-      "defaults"
-      "nofail"
-    ];
   };
 
-  swapDevices = [ ];
+  fileSystems."/var/lib/garage/data" = {
+    device = "/dev/disk/by-label/data";
+    fsType = "xfs";
+  };
 
-  networking.useDHCP = lib.mkDefault false;
-  networking.networkmanager.enable = lib.mkForce false;
+  fileSystems."/var/lib/garage/meta" = {
+    device = "/dev/disk/by-label/metadata";
+    fsType = "btrfs";
+  };
+
+  swapDevices = [ { device = "/dev/disk/by-label/swap"; } ];
 
   nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
-  hardware.cpu.intel.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
 }

hosts/trinkgenossin/networking.nix

@@ -0,0 +1,15 @@
+{
+  config,
+  pkgs,
+  flake,
+  ...
+}:
+{
+  services.garage.settings.rpc_public_addr = "[2a01:239:35d:f500::1]:3901";
+
+  networking.hostName = "trinkgenossin";
+  networking.hostId = "00000003";
+
+  networking.enableIPv6 = true;
+  networking.useDHCP = true;
+}

hosts/trinkgenossin/wireguard.nix

@@ -4,18 +4,22 @@
   flake,
   ...
 }:
+let
+  wireguardIPv4 = "10.7.6.5";
+  wireguardIPv6 = "fd00:fae:fae:fae:fae:5::";
+in
 {
   networking.firewall.allowedUDPPorts = [ 51820 ];
 
-  age.secrets.wg-private-key.file = "${flake.self}/secrets/flora6-wg-private-key.age";
+  age.secrets.wg-private-key.file = "${flake.self}/secrets/trinkgenossin-wg-private-key.age";
 
   networking.wireguard.interfaces = {
     wg-ssh = {
       listenPort = 51820;
       mtu = 1300;
       ips = [
-        "10.7.6.2/32"
-        "fd00:fae:fae:fae:fae:2::/96"
+        "${wireguardIPv4}/32"
+        "${wireguardIPv6}/96"
       ];
       privateKeyFile = config.age.secrets.wg-private-key.path;
       peers = flake.self.logins.admins.wireguardDevices ++ [
@@ -47,17 +51,35 @@
             "fd00:fae:fae:fae:fae:4::/96"
           ];
         }
+        {
+          # delite.pub.solar
+          endpoint = "5.255.119.132:51820";
+          publicKey = "ZT2qGWgMPwHRUOZmTQHWCRX4m14YwOsiszjsA5bpc2k=";
+          allowedIPs = [
+            "10.7.6.6/32"
+            "fd00:fae:fae:fae:fae:6::/96"
+          ];
+        }
+        {
+          # blue-shell.pub.solar
+          endpoint = "194.13.83.205:51820";
+          publicKey = "bcrIpWrKc1M+Hq4ds3aN1lTaKE26f2rvXhd+93QrzR8=";
+          allowedIPs = [
+            "10.7.6.7/32"
+            "fd00:fae:fae:fae:fae:7::/96"
+          ];
+        }
       ];
     };
   };
 
   services.openssh.listenAddresses = [
     {
-      addr = "10.7.6.2";
+      addr = wireguardIPv4;
       port = 22;
     }
     {
-      addr = "[fd00:fae:fae:fae:fae:2::]";
+      addr = "[${wireguardIPv6}]";
       port = 22;
     }
   ];

hosts/underground/configuration.nix

@@ -0,0 +1,72 @@
+{
+  flake,
+  config,
+  pkgs,
+  ...
+}:
+{
+  # Use GRUB2 as the boot loader.
+  boot.loader.grub = {
+    enable = true;
+    devices = [ "/dev/vda" ];
+  };
+
+  pub-solar-os.networking.domain = "test.pub.solar";
+
+  systemd.tmpfiles.rules = [ "f /tmp/dbf 1777 root root 10d password" ];
+
+  # keycloak
+  pub-solar-os.auth = {
+    enable = true;
+    database-password-file = "/tmp/dbf";
+  };
+  services.keycloak.database.createLocally = true;
+
+  # matrix-synapse
+  # test.pub.solar /.well-known is required for federation
+  services.nginx.virtualHosts."${config.pub-solar-os.networking.domain}" = {
+    default = true;
+    enableACME = true;
+    forceSSL = true;
+  };
+
+  age.secrets."staging-matrix-synapse-secret-config.yaml" = {
+    file = "${flake.self}/secrets/staging-matrix-synapse-secret-config.yaml.age";
+    mode = "400";
+    owner = "matrix-synapse";
+  };
+
+  age.secrets."staging-matrix-authentication-service-secret-config.yml" = {
+    file = "${flake.self}/secrets/staging-matrix-authentication-service-secret-config.yml.age";
+    mode = "400";
+    owner = "matrix-authentication-service";
+  };
+
+  pub-solar-os.matrix = {
+    enable = true;
+    synapse = {
+      extra-config-files = [
+        config.age.secrets."staging-matrix-synapse-secret-config.yaml".path
+
+        # The registration file is automatically generated after starting the
+        # appservice for the first time.
+        # cp /var/lib/mautrix-telegram/telegram-registration.yaml \
+        #   /var/lib/matrix-synapse/
+        # chown matrix-synapse:matrix-synapse \
+        #   /var/lib/matrix-synapse/telegram-registration.yaml
+        #"/var/lib/matrix-synapse/telegram-registration.yaml"
+      ];
+      app-service-config-files = [
+        "/var/lib/matrix-appservice-irc/registration.yml"
+        #"/var/lib/matrix-synapse/telegram-registration.yaml"
+      ];
+    };
+    matrix-authentication-service.extra-config-files = [
+      config.age.secrets."staging-matrix-authentication-service-secret-config.yml".path
+    ];
+  };
+
+  services.openssh.openFirewall = true;
+
+  system.stateVersion = "24.05";
+}

hosts/underground/default.nix

@@ -0,0 +1,16 @@
+{ flake, ... }:
+
+{
+  imports = [
+    # Include the results of the hardware scan.
+    ./hardware-configuration.nix
+    ./configuration.nix
+
+    ./networking.nix
+    "${flake.inputs.fork}/nixos/modules/services//matrix/matrix-authentication-service.nix"
+  ];
+
+  disabledModules = [
+    "services/matrix/matrix-authentication-service.nix "
+  ];
+}

hosts/underground/hardware-configuration.nix

@@ -0,0 +1,47 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{
+  config,
+  lib,
+  pkgs,
+  modulesPath,
+  ...
+}:
+
+{
+  imports = [
+    (modulesPath + "/profiles/qemu-guest.nix")
+  ];
+
+  boot.initrd.availableKernelModules = [
+    "ahci"
+    "xhci_pci"
+    "virtio_pci"
+    "virtio_scsi"
+    "sd_mod"
+    "sr_mod"
+  ];
+  boot.kernelModules = [ "kvm-intel" ];
+  boot.extraModulePackages = [ ];
+
+  boot.initrd.luks.devices."cryptroot" = {
+    device = "/dev/disk/by-label/cryptroot";
+  };
+
+  fileSystems."/" = {
+    device = "/dev/disk/by-label/root";
+    fsType = "ext4";
+  };
+
+  fileSystems."/boot" = {
+    device = "/dev/disk/by-label/boot";
+    fsType = "ext4";
+  };
+
+  swapDevices = [
+    { device = "/dev/disk/by-label/swap"; }
+  ];
+
+  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
+}

hosts/underground/networking.nix

@@ -0,0 +1,30 @@
+{
+  config,
+  pkgs,
+  flake,
+  ...
+}:
+{
+
+  networking.hostName = "underground";
+
+  networking = {
+    defaultGateway = {
+      address = "80.244.242.1";
+      interface = "enp1s0";
+    };
+    nameservers = [
+      "95.129.51.51"
+      "80.244.244.244"
+    ];
+    interfaces.enp1s0 = {
+      useDHCP = false;
+      ipv4.addresses = [
+        {
+          address = "80.244.242.3";
+          prefixLength = 29;
+        }
+      ];
+    };
+  };
+}

logins/admins.nix

@@ -63,6 +63,7 @@
   teutat3s = {
     sshPubKeys = {
       teutat3s-1 = "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFro/k4Mgqyh8yV/7Zwjc0dv60ZM7bROBU9JNd99P/4co6fxPt1pJiU/pEz2Dax/HODxgcO+jFZfvPEuLMCeAl0= YubiKey #10593996 PIV Slot 9a";
+      teutat3s-2 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHcU6KPy4b1MQXd6EJhcYwbJu7E+0IrBZF/IP6T7gbMf teutat3s@dumpyourvms";
     };
 
     secretEncryptionKeys = {

logins/robots.nix

@@ -2,6 +2,5 @@
   # Used for restic backups to droppie, a server run by @b12f
   "root@droppie" = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBZQSephFJU0NMbVbhwvVJ2/m6jcPYo1IsWCsoarqKin root@droppie";
 
-  # robot user on flora-6
-  "hakkonaut@flora-6" = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGP5MvCwNRtCcP1pSDrn0XZTNlpOqYnjHDm9/OI4hECW hakkonaut@flora-6";
+  "hakkonaut" = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGP5MvCwNRtCcP1pSDrn0XZTNlpOqYnjHDm9/OI4hECW hakkonaut";
 }

modules/backups/default.nix

@@ -0,0 +1,290 @@
+{
+  flake,
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  utils = import "${flake.inputs.nixpkgs}/nixos/lib/utils.nix" {
+    inherit lib;
+    inherit config;
+    inherit pkgs;
+  };
+  # Type for a valid systemd unit option. Needed for correctly passing "timerConfig" to "systemd.timers"
+  inherit (utils.systemdUtils.unitOptions) unitOption;
+  inherit (lib)
+    literalExpression
+    mkOption
+    mkPackageOption
+    types
+    ;
+in
+{
+  options.pub-solar-os.backups = {
+    repos = mkOption {
+      description = ''
+        Configuration of Restic repositories.
+      '';
+      type = types.attrsOf (
+        types.submodule (
+          { name, ... }:
+          {
+            options = {
+              passwordFile = mkOption {
+                type = types.str;
+                description = ''
+                  Read the repository password from a file.
+                '';
+                example = "/etc/nixos/restic-password";
+              };
+
+              environmentFile = mkOption {
+                type = with types; nullOr str;
+                default = null;
+                description = ''
+                  Read repository secrets as environment variables from a file.
+                '';
+                example = "/etc/nixos/restic-env";
+              };
+
+              repository = mkOption {
+                type = with types; nullOr str;
+                default = null;
+                description = ''
+                  repository to backup to.
+                '';
+                example = "sftp:backup@192.168.1.100:/backups/${name}";
+              };
+            };
+          }
+        )
+      );
+
+      default = { };
+      example = {
+        remotebackup = {
+          repository = "sftp:backup@host:/backups/home";
+          passwordFile = "/etc/nixos/secrets/restic-password";
+          environmentFile = "/etc/nixos/secrets/restic-env";
+        };
+      };
+    };
+
+    restic = mkOption {
+      description = ''
+        Periodic backups to create with Restic.
+      '';
+      type = types.attrsOf (
+        types.submodule (
+          { name, ... }:
+          {
+            options = {
+              paths = mkOption {
+                # This is nullable for legacy reasons only. We should consider making it a pure listOf
+                # after some time has passed since this comment was added.
+                type = types.nullOr (types.listOf types.str);
+                default = [ ];
+                description = ''
+                  Which paths to backup, in addition to ones specified via
+                  `dynamicFilesFrom`.  If null or an empty array and
+                  `dynamicFilesFrom` is also null, no backup command will be run.
+                   This can be used to create a prune-only job.
+                '';
+                example = [
+                  "/var/lib/postgresql"
+                  "/home/user/backup"
+                ];
+              };
+
+              exclude = mkOption {
+                type = types.listOf types.str;
+                default = [ ];
+                description = ''
+                  Patterns to exclude when backing up. See
+                  https://restic.readthedocs.io/en/latest/040_backup.html#excluding-files for
+                  details on syntax.
+                '';
+                example = [
+                  "/var/cache"
+                  "/home/*/.cache"
+                  ".git"
+                ];
+              };
+
+              timerConfig = mkOption {
+                type = types.nullOr (types.attrsOf unitOption);
+                default = {
+                  OnCalendar = "daily";
+                  Persistent = true;
+                };
+                description = ''
+                  When to run the backup. See {manpage}`systemd.timer(5)` for
+                  details. If null no timer is created and the backup will only
+                  run when explicitly started.
+                '';
+                example = {
+                  OnCalendar = "00:05";
+                  RandomizedDelaySec = "5h";
+                  Persistent = true;
+                };
+              };
+
+              user = mkOption {
+                type = types.str;
+                default = "root";
+                description = ''
+                  As which user the backup should run.
+                '';
+                example = "postgresql";
+              };
+
+              extraBackupArgs = mkOption {
+                type = types.listOf types.str;
+                default = [ ];
+                description = ''
+                  Extra arguments passed to restic backup.
+                '';
+                example = [ "--exclude-file=/etc/nixos/restic-ignore" ];
+              };
+
+              extraOptions = mkOption {
+                type = types.listOf types.str;
+                default = [ ];
+                description = ''
+                  Extra extended options to be passed to the restic --option flag.
+                '';
+                example = [ "sftp.command='ssh backup@192.168.1.100 -i /home/user/.ssh/id_rsa -s sftp'" ];
+              };
+
+              initialize = mkOption {
+                type = types.bool;
+                default = false;
+                description = ''
+                  Create the repository if it doesn't exist.
+                '';
+              };
+
+              pruneOpts = mkOption {
+                type = types.listOf types.str;
+                default = [ ];
+                description = ''
+                  A list of options (--keep-\* et al.) for 'restic forget
+                  --prune', to automatically prune old snapshots.  The
+                  'forget' command is run *after* the 'backup' command, so
+                  keep that in mind when constructing the --keep-\* options.
+                '';
+                example = [
+                  "--keep-daily 7"
+                  "--keep-weekly 5"
+                  "--keep-monthly 12"
+                  "--keep-yearly 75"
+                ];
+              };
+
+              runCheck = mkOption {
+                type = types.bool;
+                default = (builtins.length config.pub-solar-os.backups.restic.${name}.checkOpts > 0);
+                defaultText = literalExpression ''builtins.length config.services.backups.${name}.checkOpts > 0'';
+                description = "Whether to run the `check` command with the provided `checkOpts` options.";
+                example = true;
+              };
+
+              checkOpts = mkOption {
+                type = types.listOf types.str;
+                default = [ ];
+                description = ''
+                  A list of options for 'restic check'.
+                '';
+                example = [ "--with-cache" ];
+              };
+
+              dynamicFilesFrom = mkOption {
+                type = with types; nullOr str;
+                default = null;
+                description = ''
+                  A script that produces a list of files to back up.  The
+                  results of this command are given to the '--files-from'
+                  option. The result is merged with paths specified via `paths`.
+                '';
+                example = "find /home/matt/git -type d -name .git";
+              };
+
+              backupPrepareCommand = mkOption {
+                type = with types; nullOr str;
+                default = null;
+                description = ''
+                  A script that must run before starting the backup process.
+                '';
+              };
+
+              backupCleanupCommand = mkOption {
+                type = with types; nullOr str;
+                default = null;
+                description = ''
+                  A script that must run after finishing the backup process.
+                '';
+              };
+
+              package = mkPackageOption pkgs "restic" { };
+
+              createWrapper = lib.mkOption {
+                type = lib.types.bool;
+                default = true;
+                description = ''
+                  Whether to generate and add a script to the system path, that has the same environment variables set
+                  as the systemd service. This can be used to e.g. mount snapshots or perform other opterations, without
+                  having to manually specify most options.
+                '';
+              };
+            };
+          }
+        )
+      );
+      default = { };
+      example = {
+        localbackup = {
+          paths = [ "/home" ];
+          exclude = [ "/home/*/.cache" ];
+          initialize = true;
+        };
+        remotebackup = {
+          paths = [ "/home" ];
+          extraOptions = [
+            "sftp.command='ssh backup@host -i /etc/nixos/secrets/backup-private-key -s sftp'"
+          ];
+          timerConfig = {
+            OnCalendar = "00:05";
+            RandomizedDelaySec = "5h";
+          };
+        };
+      };
+    };
+  };
+
+  config = {
+    services.restic.backups =
+      let
+        repos = config.pub-solar-os.backups.repos;
+        restic = config.pub-solar-os.backups.restic;
+
+        repoNames = builtins.attrNames repos;
+        backupNames = builtins.attrNames restic;
+
+        createBackups =
+          backupName:
+          map (repoName: {
+            name = "${backupName}-${repoName}";
+            value = repos."${repoName}" // restic."${backupName}";
+          }) repoNames;
+
+      in
+      builtins.listToAttrs (lib.lists.flatten (map createBackups backupNames));
+
+    # Used for pub-solar-os.backups.repos.storagebox
+    programs.ssh.knownHosts = {
+      "u377325.your-storagebox.de".publicKey = "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA5EB5p/5Hp3hGW1oHok+PIOH9Pbn7cnUiGmUEBrCVjnAw+HrKyN8bYVV0dIGllswYXwkG/+bgiBlE6IVIBAq+JwVWu1Sss3KarHY3OvFJUXZoZyRRg/Gc/+LRCE7lyKpwWQ70dbelGRyyJFH36eNv6ySXoUYtGkwlU5IVaHPApOxe4LHPZa/qhSRbPo2hwoh0orCtgejRebNtW5nlx00DNFgsvn8Svz2cIYLxsPVzKgUxs8Zxsxgn+Q/UvR7uq4AbAhyBMLxv7DjJ1pc7PJocuTno2Rw9uMZi1gkjbnmiOh6TTXIEWbnroyIhwc8555uto9melEUmWNQ+C+PwAK+MPw==";
+      "[u377325.your-storagebox.de]:23".publicKey = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIICf9svRenC/PLKIL9nk6K/pxQgoiFC41wTNvoIncOxs";
+    };
+  };
+}

modules/core/networking.nix

@@ -31,13 +31,17 @@
 
     networking.hosts = {
       "10.7.6.1" = [ "nachtigall.wg.${config.pub-solar-os.networking.domain}" ];
-      "10.7.6.2" = [ "flora-6.wg.${config.pub-solar-os.networking.domain}" ];
       "10.7.6.3" = [ "metronom.wg.${config.pub-solar-os.networking.domain}" ];
       "10.7.6.4" = [ "tankstelle.wg.${config.pub-solar-os.networking.domain}" ];
+      "10.7.6.5" = [ "trinkgenossin.wg.${config.pub-solar-os.networking.domain}" ];
+      "10.7.6.6" = [ "delite.wg.${config.pub-solar-os.networking.domain}" ];
+      "10.7.6.7" = [ "blue-shell.wg.${config.pub-solar-os.networking.domain}" ];
       "fd00:fae:fae:fae:fae:1::" = [ "nachtigall.wg.${config.pub-solar-os.networking.domain}" ];
-      "fd00:fae:fae:fae:fae:2::" = [ "flora-6.wg.${config.pub-solar-os.networking.domain}" ];
       "fd00:fae:fae:fae:fae:3::" = [ "metronom.wg.${config.pub-solar-os.networking.domain}" ];
       "fd00:fae:fae:fae:fae:4::" = [ "tankstelle.wg.${config.pub-solar-os.networking.domain}" ];
+      "fd00:fae:fae:fae:fae:5::" = [ "trinkgenossin.wg.${config.pub-solar-os.networking.domain}" ];
+      "fd00:fae:fae:fae:fae:6::" = [ "delite.wg.${config.pub-solar-os.networking.domain}" ];
+      "fd00:fae:fae:fae:fae:7::" = [ "blue-shell.wg.${config.pub-solar-os.networking.domain}" ];
     };
 
     services.openssh = {

modules/core/nix.nix

@@ -6,7 +6,10 @@
   ...
 }:
 {
-  nixpkgs.config.allowUnfreePredicate = pkg: builtins.elem (lib.getName pkg) [ ];
+  nixpkgs.config = lib.mkDefault {
+    allowUnfreePredicate = pkg: builtins.elem (lib.getName pkg) [ ];
+    permittedInsecurePackages = [ "olm-3.2.16" ];
+  };
 
   nix = {
     # Use default version alias for nix package

modules/coturn/default.nix

@@ -5,8 +5,9 @@
   ...
 }:
 {
-  age.secrets."coturn-static-auth-secret" = {
-    file = "${flake.self}/secrets/coturn-static-auth-secret.age";
+  age.secrets."nachtigall-coturn-static-auth-secret" = {
+    file = "${flake.self}/secrets/nachtigall-coturn-static-auth-secret.age";
+    path = "/run/agenix/coturn-static-auth-secret";
     mode = "400";
     owner = "turnserver";
   };
@@ -18,7 +19,7 @@
     min-port = 49000;
     max-port = 50000;
     use-auth-secret = true;
-    static-auth-secret-file = "/run/agenix/coturn-static-auth-secret";
+    static-auth-secret-file = config.age.secrets."nachtigall-coturn-static-auth-secret".path;
     realm = "turn.${config.pub-solar-os.networking.domain}";
     cert = "${config.security.acme.certs.${realm}.directory}/full.pem";
     pkey = "${config.security.acme.certs.${realm}.directory}/key.pem";

modules/drone/default.nix

@@ -1,114 +0,0 @@
-{
-  config,
-  lib,
-  pkgs,
-  flake,
-  ...
-}:
-{
-  age.secrets.drone-secrets = {
-    file = "${flake.self}/secrets/drone-secrets.age";
-    mode = "600";
-    owner = "drone";
-  };
-  age.secrets.drone-db-secrets = {
-    file = "${flake.self}/secrets/drone-db-secrets.age";
-    mode = "600";
-    owner = "drone";
-  };
-
-  users.users.drone = {
-    description = "Drone Service";
-    home = "/var/lib/drone";
-    useDefaultShell = true;
-    uid = 994;
-    group = "drone";
-    isSystemUser = true;
-  };
-
-  users.groups.drone = { };
-
-  systemd.tmpfiles.rules = [ "d '/var/lib/drone-db' 0750 drone drone - -" ];
-
-  services.caddy.virtualHosts."ci.${config.pub-solar-os.networking.domain}" = {
-    logFormat = lib.mkForce ''
-      output discard
-    '';
-    extraConfig = ''
-      reverse_proxy :4000
-    '';
-  };
-
-  systemd.services."docker-network-drone" =
-    let
-      docker = config.virtualisation.oci-containers.backend;
-      dockerBin = "${pkgs.${docker}}/bin/${docker}";
-    in
-    {
-      serviceConfig.Type = "oneshot";
-      before = [ "docker-drone-server.service" ];
-      script = ''
-        ${dockerBin} network inspect drone-net >/dev/null 2>&1 || ${dockerBin} network create drone-net --subnet 172.20.0.0/24
-      '';
-    };
-
-  virtualisation = {
-    docker = {
-      enable = true; # sadly podman is not supported rightnow
-      extraOptions = ''
-        --data-root /data/docker
-      '';
-    };
-
-    oci-containers = {
-      backend = "docker";
-      containers."drone-db" = {
-        image = "postgres:14";
-        autoStart = true;
-        user = "994";
-        volumes = [ "/var/lib/drone-db:/var/lib/postgresql/data" ];
-        extraOptions = [ "--network=drone-net" ];
-        environmentFiles = [ config.age.secrets.drone-db-secrets.path ];
-      };
-      containers."drone-server" = {
-        image = "drone/drone:2";
-        autoStart = true;
-        user = "994";
-        ports = [ "127.0.0.1:4000:80" ];
-        dependsOn = [ "drone-db" ];
-        extraOptions = [
-          "--network=drone-net"
-          "--pull=always"
-          "--add-host=nachtigall.${config.pub-solar-os.networking.domain}:10.7.6.1"
-        ];
-        environment = {
-          DRONE_GITEA_SERVER = "https://git.${config.pub-solar-os.networking.domain}";
-          DRONE_SERVER_HOST = "ci.${config.pub-solar-os.networking.domain}";
-          DRONE_SERVER_PROTO = "https";
-          DRONE_DATABASE_DRIVER = "postgres";
-        };
-        environmentFiles = [ config.age.secrets.drone-secrets.path ];
-      };
-      containers."drone-docker-runner" = {
-        image = "drone/drone-runner-docker:1";
-        autoStart = true;
-        # needs to run as root
-        #user = "994";
-        volumes = [ "/var/run/docker.sock:/var/run/docker.sock" ];
-        dependsOn = [ "drone-db" ];
-        extraOptions = [
-          "--network=drone-net"
-          "--pull=always"
-          "--add-host=nachtigall.${config.pub-solar-os.networking.domain}:10.7.6.1"
-        ];
-        environment = {
-          DRONE_RPC_HOST = "ci.${config.pub-solar-os.networking.domain}";
-          DRONE_RPC_PROTO = "https";
-          DRONE_RUNNER_CAPACITY = "2";
-          DRONE_RUNNER_NAME = "flora-6-docker-runner";
-        };
-        environmentFiles = [ config.age.secrets.drone-secrets.path ];
-      };
-    };
-  };
-}

modules/forgejo-actions-runner/default.nix

@@ -1,67 +0,0 @@
-{
-  config,
-  lib,
-  pkgs,
-  flake,
-  ...
-}:
-{
-  age.secrets.forgejo-actions-runner-token = {
-    file = "${flake.self}/secrets/forgejo-actions-runner-token.age";
-    mode = "440";
-  };
-
-  # Trust docker bridge interface traffic
-  # Needed for the docker runner to communicate with the act_runner cache
-  networking.firewall.trustedInterfaces = [ "br-+" ];
-
-  users.users.gitea-runner = {
-    home = "/var/lib/gitea-runner/flora-6";
-    useDefaultShell = true;
-    group = "gitea-runner";
-    isSystemUser = true;
-  };
-
-  users.groups.gitea-runner = { };
-
-  systemd.services."gitea-runner-flora\\x2d6".serviceConfig = {
-    DynamicUser = lib.mkForce false;
-  };
-
-  systemd.tmpfiles.rules = [
-    "d '/data/gitea-actions-runner' 0750 gitea-runner gitea-runner - -"
-    "d '/var/lib/gitea-runner' 0750 gitea-runner gitea-runner - -"
-  ];
-
-  # forgejo actions runner
-  # https://forgejo.org/docs/latest/admin/actions/
-  # https://docs.gitea.com/usage/actions/quickstart
-  services.gitea-actions-runner = {
-    package = pkgs.forgejo-runner;
-    instances."flora-6" = {
-      enable = true;
-      name = config.networking.hostName;
-      url = "https://git.pub.solar";
-      tokenFile = config.age.secrets.forgejo-actions-runner-token.path;
-      settings = {
-        cache = {
-          enabled = true;
-          dir = "/data/gitea-actions-runner/actcache";
-          host = "";
-          port = 0;
-          external_server = "";
-        };
-      };
-      labels = [
-        # provide a debian 12 bookworm base with Node.js for actions
-        "debian-latest:docker://git.pub.solar/pub-solar/actions-base-image:20-bookworm"
-        # fake the ubuntu name, commonly used in actions examples
-        "ubuntu-latest:docker://git.pub.solar/pub-solar/actions-base-image:20-bookworm"
-        # alpine with Node.js
-        "alpine-latest:docker://node:20-alpine"
-        # nix flakes enabled image with Node.js
-        "nix-flakes:docker://git.pub.solar/pub-solar/nix-flakes-node:latest"
-      ];
-    };
-  };
-}

modules/forgejo/default.nix

@@ -182,7 +182,7 @@
       OnCalendar = "*-*-* 00:00:00 Etc/UTC";
     };
     initialize = true;
-    passwordFile = config.age.secrets."restic-repo-storagebox".path;
+    passwordFile = config.age.secrets."restic-repo-storagebox-nachtigall".path;
     repository = "sftp:u377325@u377325.your-storagebox.de:/backups";
     backupPrepareCommand = ''
       ${pkgs.sudo}/bin/sudo -u postgres ${pkgs.postgresql}/bin/pg_dump -d gitea > /tmp/forgejo-backup.sql

modules/garage/default.nix

@@ -0,0 +1,142 @@
+{
+  config,
+  lib,
+  pkgs,
+  flake,
+  ...
+}:
+{
+  age.secrets."garage-rpc-secret" = {
+    file = "${flake.self}/secrets/garage-rpc-secret.age";
+    mode = "400";
+  };
+
+  age.secrets."garage-admin-token" = {
+    file = "${flake.self}/secrets/garage-admin-token.age";
+    mode = "400";
+  };
+
+  age.secrets."acme-namecheap-env" = {
+    file = "${flake.self}/secrets/acme-namecheap-env.age";
+    mode = "400";
+  };
+
+  networking.firewall.allowedTCPPorts = [
+    3900
+    3901
+    3902
+  ];
+
+  networking.firewall.interfaces.wg-ssh.allowedTCPPorts = [ 3903 ];
+
+  security.acme = {
+    defaults = {
+      # LEGO_DISABLE_CNAME_SUPPORT=true set here to fix issues with CNAME
+      # detection, as we use wildcard DNS for garage
+      environmentFile = config.age.secrets.acme-namecheap-env.path;
+    };
+    certs = {
+      # Wildcard certificate gets created automatically
+      "buckets.${config.pub-solar-os.networking.domain}" = {
+        # disable http challenge
+        webroot = null;
+        # enable dns challenge
+        dnsProvider = "namecheap";
+      };
+      # Wildcard certificate gets created automatically
+      "web.${config.pub-solar-os.networking.domain}" = {
+        # disable http challenge
+        webroot = null;
+        # enable dns challenge
+        dnsProvider = "namecheap";
+      };
+    };
+  };
+
+  services.nginx = {
+    upstreams.s3_backend.servers = {
+      "[::1]:3900" = { };
+    };
+    upstreams.web_backend.servers = {
+      "[::1]:3902" = { };
+    };
+    virtualHosts."buckets.${config.pub-solar-os.networking.domain}" = {
+      serverAliases = [ "*.buckets.${config.pub-solar-os.networking.domain}" ];
+
+      enableACME = true;
+      forceSSL = true;
+
+      locations."/" = {
+        proxyPass = "http://s3_backend";
+        extraConfig = ''
+          client_max_body_size 64m;
+          proxy_max_temp_file_size 0;
+        '';
+      };
+    };
+    virtualHosts."web.${config.pub-solar-os.networking.domain}" = {
+      serverAliases = [ "*.web.${config.pub-solar-os.networking.domain}" ];
+
+      enableACME = true;
+      forceSSL = true;
+
+      locations."/" = {
+        proxyPass = "http://web_backend";
+      };
+    };
+  };
+
+  services.garage = {
+    enable = true;
+    package = pkgs.garage_1_0_1;
+    settings = {
+      data_dir = "/var/lib/garage/data";
+      metadata_dir = "/var/lib/garage/meta";
+      db_engine = "lmdb";
+      replication_factor = 3;
+      compression_level = 2;
+      rpc_bind_addr = "[::]:3901";
+      s3_api = {
+        s3_region = "eu-central";
+        api_bind_addr = "[::]:3900";
+        root_domain = ".buckets.${config.pub-solar-os.networking.domain}";
+      };
+      s3_web = {
+        bind_addr = "[::]:3902";
+        root_domain = ".web.${config.pub-solar-os.networking.domain}";
+        index = "index.html";
+      };
+      admin = {
+        api_bind_addr = "[::]:3903";
+      };
+    };
+  };
+
+  users.users.garage = {
+    isSystemUser = true;
+    home = "/var/lib/garage";
+    group = "garage";
+  };
+
+  users.groups.garage = { };
+
+  # Adapted from https://git.clan.lol/clan/clan-core/src/commit/23a9e35c665ff531fe1193dcc47056432fbbeacf/clanModules/garage/default.nix
+  # Disabled DynamicUser https://github.com/NixOS/nixpkgs/blob/nixos-24.05/nixos/modules/services/web-servers/garage.nix
+  # for mounts + permissions to work
+  systemd.services.garage = {
+    serviceConfig = {
+      user = "garage";
+      group = "garage";
+      DynamicUser = false;
+      LoadCredential = [
+        "rpc_secret_path:${config.age.secrets.garage-rpc-secret.path}"
+        "admin_token_path:${config.age.secrets.garage-admin-token.path}"
+      ];
+      Environment = [
+        "GARAGE_ALLOW_WORLD_READABLE_SECRETS=true"
+        "GARAGE_RPC_SECRET_FILE=%d/rpc_secret_path"
+        "GARAGE_ADMIN_TOKEN_FILE=%d/admin_token_path"
+      ];
+    };
+  };
+}

modules/grafana/default.nix

@@ -28,20 +28,27 @@
       group = "grafana";
       user = "grafana";
     };
+    "grafana-dashboards/blackbox-exporter_rev3.json" = {
+      source = ./grafana-dashboards/blackbox-exporter_rev3.json;
+      group = "grafana";
+      user = "grafana";
+    };
     "grafana-dashboards/synapse.json" = {
       source = ./grafana-dashboards/synapse.json;
       group = "grafana";
       user = "grafana";
     };
+    "grafana-dashboards/grafana-garage-dashboard-prometheus.json" = {
+      source = ./grafana-dashboards/grafana-garage-dashboard-prometheus.json;
+      group = "grafana";
+      user = "grafana";
+    };
   };
 
-  services.caddy.virtualHosts."grafana.${config.pub-solar-os.networking.domain}" = {
-    logFormat = lib.mkForce ''
-      output discard
-    '';
-    extraConfig = ''
-      reverse_proxy :${toString config.services.grafana.settings.server.http_port}
-    '';
+  services.nginx.virtualHosts."grafana.${config.pub-solar-os.networking.domain}" = {
+    enableACME = true;
+    forceSSL = true;
+    locations."/".proxyPass = "http://127.0.0.1:${toString config.services.grafana.settings.server.http_port}";
   };
 
   services.grafana = {
@@ -64,7 +71,7 @@
         password = "\$__file{${config.age.secrets.grafana-smtp-password.path}}";
         from_address = "no-reply@pub.solar";
         from_name = "grafana.pub.solar";
-        ehlo_identity = "flora-6.pub.solar";
+        ehlo_identity = "grafana.pub.solar";
       };
       security = {
         admin_email = "crew@pub.solar";

modules/keycloak/default.nix

@@ -6,23 +6,22 @@
   ...
 }:
 {
-  options.pub-solar-os.auth = {
-    enable = lib.mkEnableOption "Enable keycloak to run on the node";
+  options.pub-solar-os.auth = with lib; {
+    enable = mkEnableOption "Enable keycloak to run on the node";
 
-    realm = lib.mkOption {
+    realm = mkOption {
       description = "Name of the realm";
-      type = lib.types.str;
+      type = types.str;
       default = config.pub-solar-os.networking.domain;
     };
+
+    database-password-file = mkOption {
+      description = "Database password file path";
+      type = types.str;
+    };
   };
 
   config = lib.mkIf config.pub-solar-os.auth.enable {
-    age.secrets.keycloak-database-password = {
-      file = "${flake.self}/secrets/keycloak-database-password.age";
-      mode = "600";
-      #owner = "keycloak";
-    };
-
     services.nginx.virtualHosts."auth.${config.pub-solar-os.networking.domain}" = {
       enableACME = true;
       forceSSL = true;
@@ -46,7 +45,7 @@
     # keycloak
     services.keycloak = {
       enable = true;
-      database.passwordFile = config.age.secrets.keycloak-database-password.path;
+      database.passwordFile = config.pub-solar-os.auth.database-password-file;
       settings = {
         hostname = "auth.${config.pub-solar-os.networking.domain}";
         http-host = "127.0.0.1";
@@ -59,14 +58,12 @@
       };
     };
 
-    services.restic.backups.keycloak-storagebox = {
+    pub-solar-os.backups.restic.keycloak = {
       paths = [ "/tmp/keycloak-backup.sql" ];
       timerConfig = {
         OnCalendar = "*-*-* 03:00:00 Etc/UTC";
       };
       initialize = true;
-      passwordFile = config.age.secrets."restic-repo-storagebox".path;
-      repository = "sftp:u377325@u377325.your-storagebox.de:/backups";
       backupPrepareCommand = ''
         ${pkgs.sudo}/bin/sudo -u postgres ${pkgs.postgresql}/bin/pg_dump -d keycloak > /tmp/keycloak-backup.sql
       '';

modules/loki/default.nix

@@ -25,7 +25,7 @@
           };
         };
         replication_factor = 1;
-        path_prefix = "/data/loki";
+        path_prefix = "/var/lib/loki";
         storage = {
           filesystem = {
             chunks_directory = "chunks/";
@@ -108,7 +108,7 @@
       };
       clients = [
         {
-          url = "http://flora-6.wg.pub.solar:${toString config.services.loki.configuration.server.http_listen_port}/loki/api/v1/push";
+          url = "http://trinkgenossin.wg.pub.solar:${toString config.services.loki.configuration.server.http_listen_port}/loki/api/v1/push";
         }
       ];
       scrape_configs = [
@@ -118,7 +118,7 @@
             max_age = "24h";
             labels = {
               job = "systemd-journal";
-              host = "flora-6";
+              host = "trinkgenossin";
             };
           };
           relabel_configs = [

modules/mail/default.nix

@@ -67,4 +67,20 @@
   };
   security.acme.acceptTerms = true;
   security.acme.defaults.email = "security@pub.solar";
+
+  pub-solar-os.backups.restic.mail = {
+    paths = [
+      "/var/vmail"
+      "/var/dkim"
+    ];
+    timerConfig = {
+      OnCalendar = "*-*-* 02:00:00 Etc/UTC";
+    };
+    initialize = true;
+    pruneOpts = [
+      "--keep-daily 7"
+      "--keep-weekly 4"
+      "--keep-monthly 3"
+    ];
+  };
 }

modules/mailman/default.nix

@@ -91,7 +91,7 @@
       OnCalendar = "*-*-* 02:00:00 Etc/UTC";
     };
     initialize = true;
-    passwordFile = config.age.secrets."restic-repo-storagebox".path;
+    passwordFile = config.age.secrets."restic-repo-storagebox-nachtigall".path;
     repository = "sftp:u377325@u377325.your-storagebox.de:/backups";
     pruneOpts = [
       "--keep-daily 7"

modules/mastodon/default.nix

@@ -7,6 +7,21 @@
 }:
 
 {
+  age.secrets."mastodon-active-record-encryption-deterministic-key" = {
+    file = "${flake.self}/secrets//mastodon-active-record-encryption-deterministic-key.age";
+    mode = "400";
+    owner = config.services.mastodon.user;
+  };
+  age.secrets."mastodon-active-record-encryption-key-derivation-salt" = {
+    file = "${flake.self}/secrets//mastodon-active-record-encryption-key-derivation-salt.age";
+    mode = "400";
+    owner = config.services.mastodon.user;
+  };
+  age.secrets."mastodon-active-record-encryption-primary-key" = {
+    file = "${flake.self}/secrets//mastodon-active-record-encryption-primary-key.age";
+    mode = "400";
+    owner = config.services.mastodon.user;
+  };
   age.secrets."mastodon-secret-key-base" = {
     file = "${flake.self}/secrets/mastodon-secret-key-base.age";
     mode = "400";
@@ -54,6 +69,9 @@
     webProcesses = 2;
     # Threads per process used by the mastodon-web service
     webThreads = 5;
+    activeRecordEncryptionDeterministicKeyFile = "/run/agenix/mastodon-active-record-encryption-deterministic-key";
+    activeRecordEncryptionKeyDerivationSaltFile = "/run/agenix/mastodon-active-record-encryption-key-derivation-salt";
+    activeRecordEncryptionPrimaryKeyFile = "/run/agenix/mastodon-active-record-encryption-primary-key";
     secretKeyBaseFile = "/run/agenix/mastodon-secret-key-base";
     otpSecretFile = "/run/agenix/mastodon-otp-secret";
     vapidPrivateKeyFile = "/run/agenix/mastodon-vapid-private-key";
@@ -67,20 +85,20 @@
       passwordFile = "/run/agenix/mastodon-smtp-password";
       fromAddress = "mastodon-notifications@pub.solar";
     };
+    # Defined in ./opensearch.nix
+    elasticsearch.host = "127.0.0.1";
     mediaAutoRemove = {
       olderThanDays = 7;
     };
     extraEnvFiles = [ "/run/agenix/mastodon-extra-env-secrets" ];
     extraConfig = {
       WEB_DOMAIN = "mastodon.${config.pub-solar-os.networking.domain}";
-      # Defined in ./opensearch.nix
-      ES_HOST = "127.0.0.1";
       # S3 File storage (optional)
       # -----------------------
       S3_ENABLED = "true";
-      S3_BUCKET = "pub-solar-mastodon";
-      S3_REGION = "europe-west-1";
-      S3_ENDPOINT = "https://gateway.tardigradeshare.io";
+      S3_BUCKET = "mastodon";
+      S3_REGION = "eu-central";
+      S3_ENDPOINT = "https://buckets.pub.solar";
       S3_ALIAS_HOST = "files.${config.pub-solar-os.networking.domain}";
       # Translation (optional)
       # -----------------------
@@ -106,7 +124,7 @@
       OnCalendar = "*-*-* 04:00:00 Etc/UTC";
     };
     initialize = true;
-    passwordFile = config.age.secrets."restic-repo-storagebox".path;
+    passwordFile = config.age.secrets."restic-repo-storagebox-nachtigall".path;
     repository = "sftp:u377325@u377325.your-storagebox.de:/backups";
     backupPrepareCommand = ''
       ${pkgs.sudo}/bin/sudo -u postgres ${pkgs.postgresql}/bin/pg_dump -d mastodon > /tmp/mastodon-backup.sql

modules/matrix-irc/default.nix

@@ -16,11 +16,6 @@ let
   synapseClientPort = "${toString listenerWithClient.port}";
 in
 {
-  systemd.services.matrix-appservice-irc.serviceConfig.SystemCallFilter = lib.mkForce [
-    "@system-service @pkey"
-    "~@privileged @resources"
-    "@chown"
-  ];
   services.matrix-appservice-irc = {
     enable = true;
     localpart = "irc_bot";

modules/matrix/default.nix

@@ -1,6 +1,7 @@
 {
   flake,
   config,
+  lib,
   pkgs,
   ...
 }:
@@ -9,24 +10,46 @@ let
   serverDomain = "${config.pub-solar-os.networking.domain}";
 in
 {
-  age.secrets."matrix-synapse-signing-key" = {
-    file = "${flake.self}/secrets/matrix-synapse-signing-key.age";
-    mode = "400";
-    owner = "matrix-synapse";
+  options.pub-solar-os = {
+    matrix = {
+      enable = lib.mkEnableOption "Enable matrix-synapse and matrix-authentication-service to run on the node";
+
+      synapse = {
+        app-service-config-files = lib.mkOption {
+          description = "List of app service config files";
+          type = lib.types.listOf lib.types.str;
+          default = [ ];
         };
 
-  age.secrets."matrix-synapse-secret-config.yaml" = {
-    file = "${flake.self}/secrets/matrix-synapse-secret-config.yaml.age";
-    mode = "400";
-    owner = "matrix-synapse";
+        extra-config-files = lib.mkOption {
+          description = "List of extra synapse config files";
+          type = lib.types.listOf lib.types.str;
+          default = [ ];
         };
 
-  age.secrets."matrix-synapse-sliding-sync-secret" = {
-    file = "${flake.self}/secrets/matrix-synapse-sliding-sync-secret.age";
-    mode = "400";
-    owner = "matrix-synapse";
+        signing_key_path = lib.mkOption {
+          description = "Path to file containing the signing key";
+          type = lib.types.str;
+          default = "${config.services.matrix-synapse.dataDir}/homeserver.signing.key";
         };
 
+        sliding-sync.enable = lib.mkEnableOption {
+          description = "Whether to enable a sliding-sync proxy, no longer needed with synapse version 1.114+";
+          default = false;
+        };
+      };
+
+      matrix-authentication-service = {
+        extra-config-files = lib.mkOption {
+          description = "List of extra mas config files";
+          type = lib.types.listOf lib.types.str;
+          default = [ ];
+        };
+      };
+    };
+  };
+
+  config = lib.mkIf config.pub-solar-os.matrix.enable {
     services.matrix-synapse = {
       enable = true;
       settings = {
@@ -194,7 +217,7 @@ in
           }
         ];
 
-      signing_key_path = "/run/agenix/matrix-synapse-signing-key";
+        signing_key_path = config.pub-solar-os.matrix.synapse.signing_key_path;
 
         stream_writers = { };
         trusted_key_servers = [ { server_name = "matrix.org"; } ];
@@ -240,29 +263,12 @@ in
         };
         user_ips_max_age = "28d";
 
-      app_service_config_files = [
-        "/var/lib/matrix-synapse/telegram-registration.yaml"
-        "/var/lib/matrix-appservice-irc/registration.yml"
-        # "/matrix-appservice-slack-registration.yaml"
-        # "/hookshot-registration.yml"
-        # "/matrix-mautrix-signal-registration.yaml"
-        # "/matrix-mautrix-telegram-registration.yaml"
-      ];
+        app_service_config_files = config.pub-solar-os.matrix.synapse.app-service-config-files;
       };
 
       withJemalloc = true;
 
-    extraConfigFiles = [
-      "/run/agenix/matrix-synapse-secret-config.yaml"
-
-      # The registration file is automatically generated after starting the
-      # appservice for the first time.
-      # cp /var/lib/mautrix-telegram/telegram-registration.yaml \
-      #   /var/lib/matrix-synapse/
-      # chown matrix-synapse:matrix-synapse \
-      #   /var/lib/matrix-synapse/telegram-registration.yaml
-      "/var/lib/matrix-synapse/telegram-registration.yaml"
-    ];
+      extraConfigFiles = config.pub-solar-os.matrix.synapse.extra-config-files;
 
       extras = [
         "oidc"
@@ -272,8 +278,56 @@ in
       plugins = [ config.services.matrix-synapse.package.plugins.matrix-synapse-shared-secret-auth ];
     };
 
-  services.matrix-sliding-sync = {
+    services.matrix-authentication-service = {
       enable = true;
+      createDatabase = true;
+      extraConfigFiles = config.pub-solar-os.matrix.matrix-authentication-service.extra-config-files;
+
+      settings = {
+        http.public_base = "https://mas.${config.pub-solar-os.networking.domain}";
+        http.issuer = "https://mas.${config.pub-solar-os.networking.domain}";
+        http.listeners = [
+          {
+            name = "web";
+            resources = [
+              { name = "discovery"; }
+              { name = "human"; }
+              { name = "oauth"; }
+              { name = "compat"; }
+              { name = "graphql"; }
+              {
+                name = "assets";
+                path = "${config.services.matrix-authentication-service.package}/share/matrix-authentication-service/assets";
+              }
+            ];
+            binds = [
+              {
+                host = "0.0.0.0";
+                port = 8090;
+              }
+            ];
+            proxy_protocol = false;
+          }
+          {
+            name = "internal";
+            resources = [
+              { name = "health"; }
+            ];
+            binds = [
+              {
+                host = "0.0.0.0";
+                port = 8081;
+              }
+            ];
+            proxy_protocol = false;
+          }
+        ];
+        passwords.enabled = false;
+      };
+    };
+
+    services.matrix-sliding-sync = {
+      enable = config.pub-solar-os.matrix.synapse.sliding-sync.enable;
       settings = {
         SYNCV3_SERVER = "https://${publicDomain}";
         SYNCV3_BINDADDR = "127.0.0.1:8011";
@@ -284,7 +338,7 @@ in
       environmentFile = config.age.secrets."matrix-synapse-sliding-sync-secret".path;
     };
 
-  services.restic.backups.matrix-synapse-storagebox = {
+    pub-solar-os.backups.restic.matrix-synapse = {
       paths = [
         "/var/lib/matrix-synapse"
         "/var/lib/matrix-appservice-irc"
@@ -295,8 +349,6 @@ in
         OnCalendar = "*-*-* 05:00:00 Etc/UTC";
       };
       initialize = true;
-    passwordFile = config.age.secrets."restic-repo-storagebox".path;
-    repository = "sftp:u377325@u377325.your-storagebox.de:/backups";
       backupPrepareCommand = ''
         ${pkgs.sudo}/bin/sudo -u postgres ${pkgs.postgresql}/bin/pg_dump -d matrix > /tmp/matrix-synapse-backup.sql
       '';
@@ -309,4 +361,5 @@ in
         "--keep-monthly 3"
       ];
     };
+  };
 }

modules/mediawiki/default.nix

@@ -232,4 +232,27 @@ in
       };
     };
   };
+
+  pub-solar-os.backups.restic.mediawiki = {
+    paths = [
+      "/var/lib/mediawiki/images"
+      "/var/lib/mediawiki/uploads"
+      "/tmp/mediawiki-backup.sql"
+    ];
+    timerConfig = {
+      OnCalendar = "*-*-* 00:30:00 Etc/UTC";
+    };
+    initialize = true;
+    backupPrepareCommand = ''
+      ${pkgs.sudo}/bin/sudo -u postgres ${pkgs.postgresql}/bin/pg_dump -d mediawiki > /tmp/mediawiki-backup.sql
+    '';
+    backupCleanupCommand = ''
+      rm /tmp/mediawiki-backup.sql
+    '';
+    pruneOpts = [
+      "--keep-daily 7"
+      "--keep-weekly 4"
+      "--keep-monthly 3"
+    ];
+  };
 }

modules/nextcloud/default.nix

@@ -145,7 +145,7 @@
       OnCalendar = "*-*-* 01:00:00 Etc/UTC";
     };
     initialize = true;
-    passwordFile = config.age.secrets."restic-repo-storagebox".path;
+    passwordFile = config.age.secrets."restic-repo-storagebox-nachtigall".path;
     repository = "sftp:u377325@u377325.your-storagebox.de:/backups";
     backupPrepareCommand = ''
       ${pkgs.sudo}/bin/sudo -u postgres ${pkgs.postgresql}/bin/pg_dump -d nextcloud > /tmp/nextcloud-backup.sql

modules/nginx-mastodon-files/default.nix

@@ -1,8 +1,7 @@
 { config, ... }:
 
 let
-  objStorHost = "link.tardigradeshare.io";
-  objStorBucket = "s/jw24ad6l4a6zxsnd32cmf5hp5nsq/pub-solar-mastodon";
+  objStorHost = "mastodon.web.pub.solar";
 in
 {
   services.nginx.virtualHosts = {
@@ -10,6 +9,12 @@ in
       enableACME = true;
       forceSSL = true;
 
+      # Use variable to force nginx to perform a DNS resolution on its value,
+      # the IP of the object storage provider may not always remain the same.
+      extraConfig = ''
+        set $s3_backend 'https://${objStorHost}';
+      '';
+
       locations = {
         "= /" = {
           index = "index.html";
@@ -25,7 +30,6 @@ in
               deny all;
             }
 
-            resolver 8.8.8.8;
             proxy_set_header Host ${objStorHost};
             proxy_set_header Connection \'\';
             proxy_set_header Authorization \'\';
@@ -40,7 +44,7 @@ in
             proxy_hide_header x-amz-bucket-region;
             proxy_hide_header x-amzn-requestid;
             proxy_ignore_headers Set-Cookie;
-            proxy_pass https://${objStorHost}/${objStorBucket}$request_uri?download;
+            proxy_pass $s3_backend$request_uri;
             proxy_intercept_errors off;
             proxy_ssl_protocols TLSv1.2 TLSv1.3;
             proxy_ssl_server_name on;

modules/nginx-matrix/default.nix

@@ -10,11 +10,14 @@ let
     add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always;
     add_header X-XSS-Protection "1; mode=block";
   '';
-  clientConfig = import ./element-client-config.nix { inherit lib pkgs; };
+  clientConfig = import ./element-client-config.nix { inherit config lib pkgs; };
   wellKnownClient = domain: {
     "m.homeserver".base_url = "https://matrix.${domain}";
     "m.identity_server".base_url = "https://matrix.${domain}";
-    "org.matrix.msc3575.proxy".url = "https://matrix.${domain}";
+    "org.matrix.msc2965.authentication" = {
+      issuer = "https://mas.${domain}/";
+      account = "https://mas.${domain}/account";
+    };
     "im.vector.riot.e2ee".default = true;
     "io.element.e2ee" = {
       default = true;
@@ -85,6 +88,27 @@ in
       root = pkgs.element-stickerpicker;
     };
 
+    "mas.${config.pub-solar-os.networking.domain}" = {
+      root = "/dev/null";
+
+      forceSSL = lib.mkDefault true;
+      enableACME = lib.mkDefault true;
+
+      locations = {
+        "/" = {
+          proxyPass = "http://127.0.0.1:8090";
+
+          extraConfig = ''
+            ${commonHeaders}
+            proxy_http_version 1.1;
+
+            # Forward the client IP address
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+          '';
+        };
+      };
+    };
+
     "matrix.${config.pub-solar-os.networking.domain}" = {
       root = "/dev/null";
 
@@ -99,28 +123,41 @@ in
       locations = {
         # For telegram
         "/c3c3f34b-29fb-5feb-86e5-98c75ec8214b" = {
+          priority = 100;
           proxyPass = "http://127.0.0.1:8009";
           extraConfig = commonHeaders;
         };
 
-        # sliding-sync
-        "~ ^/(client/|_matrix/client/unstable/org.matrix.msc3575/sync)" = {
-          proxyPass = "http://127.0.0.1:8011";
-          extraConfig = commonHeaders;
+        # Forward to the auth service
+        "~ ^/_matrix/client/(.*)/(login|logout|refresh)" = {
+          priority = 100;
+          proxyPass = "http://127.0.0.1:8090";
+
+          extraConfig = ''
+            ${commonHeaders}
+            proxy_http_version 1.1;
+
+            # Forward the client IP address
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+          '';
         };
 
-        "~* ^(/_matrix|/_synapse/client|/_synapse/oidc)" = {
+        # Forward to Synapse
+        # as per https://element-hq.github.io/synapse/latest/reverse_proxy.html#nginx
+        "~ ^(/_matrix|/_synapse/client)" = {
+          priority = 200;
           proxyPass = "http://127.0.0.1:8008";
 
           extraConfig = ''
             ${commonHeaders}
+            proxy_set_header X-Forwarded-For $remote_addr;
+            proxy_set_header X-Forwarded-Proto $scheme;
             proxy_set_header Host $host;
-            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-            proxy_set_header X-Forwarded-Proto $http_x_forwarded_proto;
 
             client_body_buffer_size 25M;
             client_max_body_size 50M;
             proxy_max_temp_file_size 0;
+            proxy_http_version 1.1;
           '';
         };
       };

modules/nginx-matrix/element-client-config.nix

@@ -1,9 +1,14 @@
-{ pkgs, lib, ... }:
+{
+  config,
+  pkgs,
+  lib,
+  ...
+}:
 {
   default_server_config = {
     "m.homeserver" = {
-      base_url = "https://matrix.pub.solar";
-      server_name = "pub.solar";
+      base_url = "https://matrix.${config.pub-solar-os.networking.domain}";
+      server_name = "${config.pub-solar-os.networking.domain}";
     };
     "m.identity_server" = {
       base_url = "";

modules/nginx-website/default.nix

@@ -7,7 +7,7 @@
   services.nginx.virtualHosts = {
     "www.${config.pub-solar-os.networking.domain}" = {
       enableACME = true;
-      addSSL = true;
+      forceSSL = true;
 
       extraConfig = ''
         error_log /dev/null;

modules/nginx/default.nix

@@ -22,6 +22,13 @@ in
     recommendedOptimisation = true;
     recommendedProxySettings = true;
     recommendedTlsSettings = true;
+    resolver.addresses = [
+      # quad9.net
+      "9.9.9.9"
+      "149.112.112.112"
+      "[2620:fe::fe]"
+      "[2620:fe::9]"
+    ];
     appendHttpConfig = ''
       # https://my.f5.com/manage/s/article/K51798430
       proxy_headers_hash_bucket_size 128;

modules/obs-portal/default.nix

@@ -147,4 +147,26 @@ in
       };
     };
   };
+
+  pub-solar-os.backups.restic.obs-portal = {
+    paths = [
+      "/var/lib/obs-portal/data"
+      "/tmp/obs-portal-backup.sql"
+    ];
+    timerConfig = {
+      OnCalendar = "*-*-* 01:30:00 Etc/UTC";
+    };
+    initialize = true;
+    backupPrepareCommand = ''
+      ${pkgs.docker}/bin/docker exec -i --user postgres obs-portal-db pg_dump obs > /tmp/obs-portal-backup.sql
+    '';
+    backupCleanupCommand = ''
+      rm /tmp/obs-portal-backup.sql
+    '';
+    pruneOpts = [
+      "--keep-daily 7"
+      "--keep-weekly 4"
+      "--keep-monthly 3"
+    ];
+  };
 }

modules/postgresql/default.nix

@@ -25,9 +25,4 @@
       full_page_writes = false;
     };
   };
-
-  systemd.services.postgresql = {
-    after = [ "var-lib-postgresql.mount" ];
-    requisite = [ "var-lib-postgresql.mount" ];
-  };
 }

modules/prometheus/alert-rules.nix

@@ -198,10 +198,10 @@ lib.mapAttrsToList
       description = "{{$labels.instance}}: healtcheck {{$labels.job}} fails!";
       };
     */
-    #cert_expiry = {
-    #  condition = "(probe_ssl_earliest_cert_expiry - time())/(3600*24) < 30";
-    #  description = "{{$labels.instance}}: The TLS certificate will expire in less than 30 days: {{$value}}s";
-    #};
+    cert_expiry = {
+      condition = "(probe_ssl_earliest_cert_expiry - time())/(3600*24) < 21";
+      description = "{{$labels.instance}}: The TLS certificate will expire in less than 21 days: {{$value}}s";
+    };
 
     # ignore devices that disabled S.M.A.R.T (example if attached via USB)
 
@@ -250,4 +250,10 @@ lib.mapAttrsToList
     #   description =
     #     "alertmanager: number of active silences has changed: {{$value}}";
     # };
+
+    garage_cluster_healthy = {
+      condition = "cluster_healthy == 0";
+      time = "15m";
+      description = "garage cluster on {{$labels.instance}} is not healthy: {{$labels.result}}!";
+    };
   })

modules/prometheus/default.nix

@@ -5,6 +5,22 @@
   flake,
   ...
 }:
+let
+  blackboxTargets = [
+    "https://pub.solar"
+    "https://chat.pub.solar"
+    "https://cloud.pub.solar"
+    "https://collabora.pub.solar"
+    "https://git.pub.solar"
+    "https://grafana.pub.solar"
+    "https://list.pub.solar"
+    "https://mastodon.pub.solar"
+    "https://obs-portal.pub.solar"
+    "https://stream.pub.solar"
+    "https://wiki.pub.solar"
+    "https://www.pub.solar"
+  ];
+in
 {
   age.secrets.alertmanager-envfile = {
     file = "${flake.self}/secrets/alertmanager-envfile.age";
@@ -12,21 +28,59 @@
     owner = "alertmanager";
   };
 
-  services.caddy.virtualHosts."alerts.${config.pub-solar-os.networking.domain}" = {
-    logFormat = lib.mkForce ''
-      output discard
-    '';
-    extraConfig = ''
-      bind 10.7.6.2 fd00:fae:fae:fae:fae:2::
-      tls internal
-      reverse_proxy :${toString config.services.prometheus.alertmanager.port}
-    '';
+  security.acme.certs = {
+    "alerts.${config.pub-solar-os.networking.domain}" = {
+      # disable http challenge
+      webroot = null;
+      # enable dns challenge
+      dnsProvider = "namecheap";
+    };
+  };
+
+  services.nginx.virtualHosts."alerts.${config.pub-solar-os.networking.domain}" = {
+    enableACME = true;
+    forceSSL = true;
+
+    listenAddresses = [
+      "10.7.6.5"
+      "[fd00:fae:fae:fae:fae:5::]"
+    ];
+
+    locations."/" = {
+      proxyPass = "http://127.0.0.1:${toString config.services.prometheus.alertmanager.port}";
+    };
   };
 
   services.prometheus = {
     enable = true;
     port = 9001;
     exporters = {
+      blackbox = {
+        enable = true;
+        # Default port is 9115
+        openFirewall = false;
+
+        configFile = pkgs.writeTextFile {
+          name = "blackbox-exporter-config";
+          text = ''
+            modules:
+              http_2xx:
+                prober: http
+                timeout: 5s
+                http:
+                  valid_http_versions: ["HTTP/1.1", "HTTP/2.0"]
+                  valid_status_codes: []  # Defaults to 2xx
+                  method: GET
+                  no_follow_redirects: false
+                  fail_if_ssl: false
+                  fail_if_not_ssl: false
+                  tls_config:
+                    insecure_skip_verify: false
+                  preferred_ip_protocol: "ip4" # defaults to "ip6"
+                  ip_protocol_fallback: true  # fallback to "ip6"
+          '';
+        };
+      };
       node = {
         enable = true;
         enabledCollectors = [ "systemd" ];
@@ -38,15 +92,33 @@
       scrape_timeout = "9s";
     };
     scrapeConfigs = [
+      {
+        job_name = "blackbox";
+        scrape_interval = "60m";
+        metrics_path = "/probe";
+        params = {
+          module = [ "http_2xx" ];
+        };
+        static_configs = [ { targets = blackboxTargets; } ];
+
+        relabel_configs = [
+          {
+            source_labels = [ "__address__" ];
+            target_label = "__param_target";
+          }
+          {
+            source_labels = [ "__param_target" ];
+            target_label = "instance";
+          }
+          {
+            target_label = "__address__";
+            replacement = "127.0.0.1:9115"; # The blackbox exporter's real hostname:port.
+          }
+        ];
+      }
       {
         job_name = "node-exporter";
         static_configs = [
-          {
-            targets = [ "127.0.0.1:${toString config.services.prometheus.exporters.node.port}" ];
-            labels = {
-              instance = "flora-6";
-            };
-          }
           {
             targets = [ "nachtigall.wg.${config.pub-solar-os.networking.domain}" ];
             labels = {
@@ -69,6 +141,30 @@
               instance = "tankstelle";
             };
           }
+          {
+            targets = [
+              "trinkgenossin.wg.${config.pub-solar-os.networking.domain}:${toString config.services.prometheus.exporters.node.port}"
+            ];
+            labels = {
+              instance = "trinkgenossin";
+            };
+          }
+          {
+            targets = [
+              "delite.wg.${config.pub-solar-os.networking.domain}:${toString config.services.prometheus.exporters.node.port}"
+            ];
+            labels = {
+              instance = "delite";
+            };
+          }
+          {
+            targets = [
+              "blue-shell.wg.${config.pub-solar-os.networking.domain}:${toString config.services.prometheus.exporters.node.port}"
+            ];
+            labels = {
+              instance = "blue-shell";
+            };
+          }
         ];
       }
       {
@@ -83,6 +179,29 @@
           }
         ];
       }
+      {
+        job_name = "garage";
+        static_configs = [
+          {
+            targets = [ "trinkgenossin.wg.${config.pub-solar-os.networking.domain}:3903" ];
+            labels = {
+              instance = "trinkgenossin";
+            };
+          }
+          {
+            targets = [ "delite.wg.${config.pub-solar-os.networking.domain}:3903" ];
+            labels = {
+              instance = "delite";
+            };
+          }
+          {
+            targets = [ "blue-shell.wg.${config.pub-solar-os.networking.domain}:3903" ];
+            labels = {
+              instance = "blue-shell";
+            };
+          }
+        ];
+      }
     ];
 
     ruleFiles = [

modules/promtail/default.nix

@@ -18,7 +18,7 @@
       };
       clients = [
         {
-          url = "http://flora-6.wg.pub.solar:${toString flake.self.nixosConfigurations.flora-6.config.services.loki.configuration.server.http_listen_port}/loki/api/v1/push";
+          url = "http://trinkgenossin.wg.pub.solar:${toString flake.self.nixosConfigurations.trinkgenossin.config.services.loki.configuration.server.http_listen_port}/loki/api/v1/push";
         }
       ];
       scrape_configs = [

modules/tt-rss/default.nix

@@ -10,6 +10,7 @@ let
     version = "7ebfbc91e92bb133beb907c6bde79279ee5156df";
     src = pkgs.fetchgit {
       url = "https://git.tt-rss.org/fox/ttrss-auth-oidc.git";
+      rev = "7ebfbc91e92bb133beb907c6bde79279ee5156df";
       hash = "sha256-G6vZBvSWms6s6nHZWsxJjMGuubt/imiBvbp6ykwrZbg=";
     };
     installPhase = ''

modules/unlock-luks-on-boot/default.nix

@@ -0,0 +1,20 @@
+{ flake, config, ... }:
+{
+  boot.initrd.network = {
+    enable = true;
+    ssh = {
+      enable = true;
+      # To prevent ssh clients from freaking out because a different host key is used,
+      # a different port for ssh is useful (assuming the same host has also a regular sshd running)
+      port = 2222;
+
+      # Please create this manually the first time.
+      hostKeys = [ "/etc/secrets/initrd/ssh_host_ed25519_key" ];
+      authorizedKeys = config.pub-solar-os.authentication.sshPubKeys;
+    };
+    postCommands = ''
+      # Automatically ask for the password on SSH login
+      echo 'cryptsetup-askpass || echo "Unlock was successful; exiting SSH session" && exit 1' >> /root/.profile
+    '';
+  };
+}

overlays/default.nix

@@ -16,6 +16,8 @@
                 element-stickerpicker = prev.callPackage ./pkgs/element-stickerpicker {
                   inherit (inputs) element-stickers maunium-stickerpicker;
                 };
+                mastodon = unstable.mastodon;
+                matrix-authentication-service = unstable.matrix-authentication-service;
               }
             )
           ];

secrets/acme-namecheap-env.age

@@ -0,0 +1,48 @@
+age-encryption.org/v1
+-> ssh-ed25519 NID4eA WtfgDmnK5l9s9DMhWgmk+tel+/uqPx8SHBd0qfWY3jk
+ZS3Qu4v3pnA+lYzJ3kad7T3LhcY7oE8fPsGQ1uQH1AA
+-> ssh-ed25519 9RQHxg SpHG3ijNizTi1YXvZCJS79Uwt4oGkYzqIme+eqQi9AQ
+GqVhyfaTF6tLwuo0vIby0vBv3JufHz59IdNX9ifWtSA
+-> ssh-ed25519 eP5MMw 9uU7tlyOzOxlsW/bfUmzjgicU3i2J5uCGWEVIljnHiM
+tDJdTB1rBJTXVaGFOOmtG5n2Ae0XOCsi41S0EagRmeM
+-> ssh-ed25519 uYcDNw ge+lEVE8+pS/S+eO+6sPqo/czym30CJbQnhTp11NsW4
+jxL7Xhn/7JRylJ/JbeGkmhMMeJ8G2KPEKVVq1icQXKU
+-> ssh-rsa f5THog
+Ybod3f7gvCiBUcNyLV6AXoBchtRGspQah9JwygSGCtBKmWPOUSw3/DVva9nPVwHB
+q4t05bEHINMZIoWy4l3VQ1jw+GTxW+6OeWDHrxHOG2hlu1/OT0tZnsQIjWwT/6Sg
+fzy6X04yD2ADkwHH6VJYjC2Lxa7kEOeCeKOACyyab7rlXk+HauytUDlcF3Nl3nOc
+JQZzfwIORU0XWVy+gDocwVqDaRJXZxhMW8oDjlU8BKgf/DpvExLfuZ9AHHJBU0Y9
+HefbTbGO1s5J0T+HEkuIDce9iPQEe8ufaSVO6tKyHpgguIAiLIkjqrdLNRmXv/y8
+9W653Xqar7fimd/sykb4K/PpdwvQcB9Ogy23t6s3Qxz5yPtC2m8IC3lgR+N+/nJO
+n29QuXFBNUZu/QBXnWMS2QF09MGE2aav/CiwFuNiTf5D4UGGN3Y7XhX/KVOFJTZX
+r1GLtch6rvD9RtfyKxAdbtCqbBEQJmoiut9ia5EzG4TvdPAE4XK3QNTn2BSmfjvI
+3aXiXOFSbdJqkxyI6ZU2mUMMor3OWrXxWizDDYef6iHZxGlWFqA/kVXyZgdwTK9n
+8Re6SYR8roH7T35eILzP4sskElN32UO/A+JyGfP1lOclGTlOrtp4HYTfY0NhhRJT
+L7YIB0pNbaRxMBsxsxwU47j3qMkaO1uzP+DgpUacWJY
+-> ssh-rsa kFDS0A
+GJjiIApapBS6F8pmh6lblCHG3FlVWL+WKN1Gi2u/6Pa1YbkiBCgYFTQBwm5GsBMR
+4tQwRJcQQDGgGddIH4/QcMAl1fTYLm3N1w8rueywgAbOwaWktKnJFYTj7lS6PSNr
+bZyqyiGvgi0oYYSVjRnm7MmCrycuKmhcGHv1ijj5J8yOxe6qFsomsn9QZm1DmR/m
+EZmc5DIYXjhuauzGgqtPVmjHi6hXTN8NX7Fg81aegko79yA12hmyHmaBj4P96Kqv
+RyWZ9Moc3ccyxq74jNzp0eFuPNhUJuNBqrKozCc2Lo3KQAmoqI27THkF/HA8ECGP
+BJDK7JdHBXyHhf/Fc5O5xOxHieIU8tHR0LLJn7VEvQyqTlKmWkZ5J53AqE8UDmm9
+0gY6zFh7h3SjyBwqktzGJ9zXn3bp4fpg0M1+SaYp9Qf6hkJ9k79Zth4s4ggxgvOl
+veib2sg3PCmL1OCMPMtyW3JkKsq0J+PtJdlAC9cmVvfvAMHKy2+aADsLt0H8Cpt2
+cNOxbnU29eLWgG9uzcCXfqqNtmSia6LUMu71GahAuteZUV8RnDOZdCNW4U2Ohnq/
+9znMqERVo0d3LgjaB0P3HXCCqhVFYTTDWg31R6N2RzSh7mb02CFgt7N+vHleQqAo
+G/6Pb+kKYSEbU884z95+o56eQrvPunCN9Vu1CjEBfG4
+-> piv-p256 vRzPNw A2dcPImS0ih5CjePQP5oPrPfwns6zAMP0J72P7fyzD/A
+p46umKyZjbc1MjOQGnJIRu6V99O+/PmVXQvryX/9XW4
+-> piv-p256 zqq/iw A5nBHU2O+bxsFqplf2GV6pK5wQ+hJ9l7tyFIe57QVKzw
+Ik6aUY3t4geZ3yiWPqBGlBem9xNU83x7t3UA7pYB55I
+-> ssh-ed25519 YFSOsg OhynWXlurzqU3ohq1ecH018Ja4wyWazDLv6isajeBUE
+Xnjo8yS9IkMwCGNeLi6BABYxjXDLbpuTrVfwAxjDWdQ
+-> ssh-ed25519 iHV63A 5CVIOtSwima5gIvwoAYExcy1tfOo8942RQ+SsflPbAM
+4HV21GcuyddIjonOZZFgjgpR5smjce7OlMN3DCy0/sU
+-> ssh-ed25519 BVsyTA mkLu2Vpr16bAZWimh6sViq5HlB1+lNOc2WPCxzgfqAg
+cIDgWit139jipd7XmZcT8mTRDKK8rJV9xIxIaPVL9pM
+-> ssh-ed25519 +3V2lQ eqfktAyV2Pia7T7XEfcYiHN9Jd4zivMzJk3in4XOTx0
+gZzO+MTyBOJR1EgGn4Mhh4rnIyr3N9gmlFty83ou+GU
+--- yJrzTzStOkRCNRu3Y+knfqTqHrwW0S0Bsko7oG/s86o
+®,Bgm°þ÷€få‚T¾èä`1†&1³%7Q˜(¯•¸Ÿ:?ßÝ
+êÎø—æ‡ðj£ùÄO_rqwÃÏi£O®´D›·)@0•ZK'óô+apU§<Ö`ºõµœctª. þ¡<C3BE>–ÌXÇNæ+íŒÂh†Ù=‰'‡VÑn^HHöv±5aa²nKÝþD¦×™

secrets/alertmanager-envfile.age

@@ -1,43 +1,43 @@
 age-encryption.org/v1
--> ssh-ed25519 Y0ZZaw TsTaRLA+9WtN9+FJWpXeP12Af5EXMbo+ANTaLC9YlC8
-Yols084RY1C9gfOrDMwJcFRuGZ/5dgGuJey7RXqm7g0
--> ssh-ed25519 uYcDNw ZLAINtv10PGMtK5TL5Tf0NyK/r1iww+vTC09ElMGoX0
-EgBB3aiHHdaDue9+Zdxg6mTV2VHeLoDN9wT+hlAzVMk
+-> ssh-ed25519 NID4eA jIwfpP0rFLANj63MsJAse0R+TQbGf7mUStdusSLkkCg
+RHyxZqWGYMvhQYfZUc89GPly42u7MR9gSpR8aFWH6LI
+-> ssh-ed25519 uYcDNw JGsVrWwxwA8ftUM+Fo1jFigWfpvNUwoNkK5zKIu582Y
+BzM82Iqmta2Dtb8xey2nkoil7mDipn1iZtGMPKwPcPI
 -> ssh-rsa f5THog
-aiJqMs3/u06tzs8lx2ISlQm87TDatqEn47v3LB3HehPanRpZx9O1HUIRTeiWkMU9
-XroGe27HQCCPd63QunBHUH7WStA10IS4rHVpMcULB5IM4jwcbOhSYSiGyY2sbv8+
-Nn/04ZOwrfzTabC7moV1DqAw6hnlDqKWp/q5N6xMb780w5vn6Poni3OJfuLaBWaT
-r6WhE5evVt3F4jyYI64fB2hFw4AR2N/zIMOMvBncLFwJf9lbIFdbsENZf94cYceF
-Tj150xdMPuErBsSJQOlfDYSmyioNN3UJUWiYsDeM3nbPEVPHhfTk6b2/lMhSQkcY
-KcuMj/mN/7w7i4HSxW6mUcK2sUMV1BcSSGYRH9ZFf7kq++KpyiP7vB8vaZkcKbfJ
-qqrIcXTuXhR+/bWZWqf/GQOVwRwe1TnqN5MoZHipg3a/UCe0gMM617VwZcfhBzjA
-eW6VUdjSewwA8YHEuDrAeoQ4CMs7y56EaIlr2IlQy6uzJPX9eeO0auO9RZ5AR40a
-7un0FrlTJX9uorpCD/zi3tvd22W5qVoMGZ8vXJShZmT9he9K3Bv6XbzG4DJQ9/nv
-xZ676HUYhWeyYZFBvt6DnEBneiDJFeaV2AeuQY+juHBOfBrbYmlE0S4Pd8uRSJ7w
-u5UJTT+RV5TkZhpCqqYm7DphYocnrv7Ic+QKmvKE4ls
+LkPMatwkNWAElm+RQiCHtHH2QPgVsAAd5b4qF0R0O6r+0CYzEF2OAOZ0LDsytTB8
+7oHAHxA6kAga/pqKUaJl28xw7ujVIb1CunZFvVSxtOTYRrEy1Rxe3AKUOm+ZmfPL
+66Ef58HWMCHzK9sc/ojo7Us4okfRhJBklB9lnORkSfdkvEHLeq0R0FfDtDCnynRD
+SKqlx3VbdWe9k6UOJidA+dY8Wx0w2TQM1c21nDr4vXsXpZf6ttT4HvrqbSrS9V1c
+nGofWP+72WinOpFRDQdLvdKvaNbLPwhigqL1VqaIcsnye5zZjQDNn+SYf55byBkS
+CMXj238UqvdDxB4E3mBEgpFxOnyi6kLQXcPEBF/xQ5fER0RS5MkWkuH4Up+BCQ+/
+CppqZrw85OOa9jAyWxil3yLQNAnLGi/P+mesPxSI+i2Not9wbUTALr4COG+1qvfF
+2MbHiqREoajnQUJjhGhXaAA332X7hNuOF/DjmBr7i81oWVmKs6TjCDVL7Yo9xu3j
+BcFqMlaOgr8gObwnyJ9BbtW4sBtnOeD5onPxWluV1+Ql8Idjmu/BKeuqIyGX6wFl
+606lbprSTRVjLZWvg3gWaIMlXdcnat9PmHeRk/yzTrHke2aFSkvUKLymnRCHETae
+Rh8ILeQTq36Ul9r7qklBNu4M7/f+jeX7gYPH/yDUCXc
 -> ssh-rsa kFDS0A
-HhilpvIiUps80SXYUXg5vqNmcy8SACvxpC5dTVBU2n+4OVXQY/35Il5ZOrUX3U7a
-arfVp/KaQF7Oncu3x8F6Tp1ibUwmoyAV6OYqqs128nEPwkNbJvwrLY3aEBm+NIzm
-gMlLRjj6EP84TVWgOsenQCS4l957f0QoNVxQ3f+GWdOiZZJFsv//ndsflng8zPlF
-bGZy8c1TxDZfOD0/kW3Nx05c9X0EHKOEoDUc0p4qntrWlflxcvLONCgv1gZuPMF+
-jMsPFP81eu3rkEUxefJ1qbvvGuW0cbzfwiStv7iGQ+Skh/vcoM0qw6p+csNKyHVO
-8nYFcs9kD8067zMnyuqiUHASfZ4rPqTji0iiPC5kZn6N0YSgz2bybkXcoqmy3m6y
-qs0S+RD99o2vCLhW46hZyKAgUyTU1DW42EmnZkPrLoqV7uin8fAwPO/98Q/b3Rkr
-zBRtyTEbooHvOCL8limiRtDl+5LMcjRFNWk8AN+9vHMsYurXPNOCnd8n2Z4MbT2U
-AhpoAD/+8HXp0InBJ/sclITVAc6tPb2CbJW6mrFezH8Ri+/6u+zSF84JDd9ZrCOz
-oIshiGZmhP5mIuspVrxgKlm78a56vQrygpqzvuSSYk3zIJxmhEkZhw09/ga+rhyB
-pkKn7GRyZTfKjwt5nnvW5/bmQndTa13j+7RhkRgBSvU
--> piv-p256 vRzPNw Awpc8paUfKnP6r0bYsaoeDE9GVSnads4/a3jCVScgS4V
-YydKOS09kyZDYN843SHIsYUimtSQKvGhIuycPWOFojc
--> piv-p256 zqq/iw A54xbcufPkLpTD+N47AiIe/xZ/0vA5kDJ4p3rIZw0a4A
-1WFP2K3tfUxtdKDBEmT3cx/u1i5nCzFR7cK4kN3WjC4
--> ssh-ed25519 YFSOsg L0lPSkoPVRKGlJ9MzkJx+cQvnZw/5m/j/JO4aRzd52Q
-o/N7zQkvbGGoadiJSvL6lfuP63uqzxEIxDtIg4tgKIo
--> ssh-ed25519 iHV63A qfLWZhbDisCSJ4vFFTR+XpRUR0WViuAqarf56M0ekT4
-ZSWW34pFRr0M2jFhnphIPJ5ch37ASM6OgTzyHSo0KAs
--> ssh-ed25519 BVsyTA JcFezSIfTF+AP8LYfFqz+wIpUrE0aoc1usiLtWxAPQE
-F9uhFyCPK46kIy+ud4V5/ESacQgc9R0JV+JTEZO6nBI
--> ssh-ed25519 +3V2lQ G4yT1e7B5O2Gy6tusRMxuWOFScynWfFY5AjrJvxMK1o
-n1OVFRqzijWlc+B93cBNdFPz+8CBYOsI5hpF1wz7xr0
---- 61u55uUc7z59iHF1IeyBLmcR6u7STUhpOPb/ODf75Vc
-<$kxpû´ÚH:}ò*ä/Tâ®Ñ$‹ÕbÀJ\F*ðòWîzÉ6 Ý	±Âì<î̹>e?ñ¼<C3B1>Ÿ6ÚµÌ~Ô!
+X1vrBlpHkWOVyhBokgO0yNDQk57S92xADIi88w2UU+nTYFgo/RsyTCCFAFMaDvR3
+kQdtorCowxQpKLnTzER8i2ABZAgAmUzGQuRPDKcqZuZH9oypNkBs6qeVI3TA5GKu
+V/IfKLeR57K3cpZT+TcOzKUqm/AAZO+rwdnrfW9qVAb7vlo3TWawfBHb+Fl7y9JL
+pEjhDMhnA7na67Ktz1MFm80XRneMTW+0NGtcTd1iQfjfHe0WfFuYU4H6aZ8ZpZYw
+2rLa+EmFqUpv0ELwdGViqmjUNwJunsJ6rhJZlMn43v5/XPLpapQr0zwtXzzfzZHd
+HnI6/X97zPYUFDsUeI6x2CiVKHVWMGjJ9VPAexpJepZSkgI6On2/mfs4++XnDWLv
+qsvsJqVzM075eH0LUyXq4WGu7oJc4OdfMm1CrEBKAaqdKRl0HnBZGSER3C/qAhLK
+Ihbk+kti5C7GTzRyUlzkwINVFV0pePClLP7AC5vdKMhXysGQlxNJsTeUTdAOhrAm
+UeWnd0Xp+K8OBsUgyGktKBMofNAJ+MilSKt3x7tJk3QuQIGjqHCshpkMf0ckixrh
+aDN9Rj+s0A8C3hrVv8z602jBrM5tfYOZv+q1/yFQo+ieic6Y9WEzzrTMJEHxFSzX
+KadqfZii8HCGQODcXh8VRpWDVjzt/pDVR/zu/0YCP+0
+-> piv-p256 vRzPNw A4KCXAKoTYy8euaKXot9+c5N21WG9/9uLPomiiI6rZ5W
+vbTcLpDNM1qVdTBCUPMrlX2GpyeMUsKaKLFl5GVVdYY
+-> piv-p256 zqq/iw ArX1s306JaaWVPiTA7XyzyTKcsBDHjeIiSoOg+5PhsBj
+zzmU1T5q5ff6TzIXhWqiVM0Oxxo/ln3uExBXBlLFcME
+-> ssh-ed25519 YFSOsg tgHAr/emB3i+9Hd+q9oYCjkPO+RuXv22kimdXz06Zys
+p8sYz3j5I95ZBJroWxUSzWljcj8E3Ic9uwwyrUWm1+E
+-> ssh-ed25519 iHV63A 9UXBAcuwIfuoTHcWYhLVa9qtJ7UsLsIQsH2Bn0T/Wy8
+OvfX4cOKJYv9pwaQp3yD/QPZdDnGSC6f1qemtKENtpE
+-> ssh-ed25519 BVsyTA nC+YMVK5YyCM79iNijTaBgIZDPi7Bvlunuzl2s9SrRw
+xVUpZwdIcszqsRdZw74fJrSduzxqrO25EMfuypipys0
+-> ssh-ed25519 +3V2lQ 4n/lkQ1nwcXD7mNc3DzIfC5xGF2mn27AoO36Chei8AA
+vDe0RU8Xm3L+/nFM0lKK3jv6hqiUE/YxZUFyHUsqAfI
+--- D4n9aVPWABXpzO9DI20yHf11MRJ5ACWVhT16bDls5pA
+iR	ÅÏÿ%µÙØY^Ï	Ýñý’µ¶{“²°Ý”#®Z0´P6šÿ+ÛÄR!i›J-\ul°9!å¬`Z÷¿Nh²

secrets/blue-shell-wg-private-key.age

@@ -0,0 +1,43 @@
+age-encryption.org/v1
+-> ssh-ed25519 eP5MMw 3H1XEr/Vt2TOQUvGu3K54CxbigkVpaC6hofMOfFR60U
+hqFTOoMhyhb/Fsywzu4nYXmEACOunenO/4NwPaVdrZs
+-> ssh-ed25519 uYcDNw raghSMdCaiQrfGviMlc9Pwh8cx33IWh+mnsxL8jgTQM
+aOMrh/746UulH7hkOV6XRiwEszgJtrI33mmzY5S2Ipg
+-> ssh-rsa f5THog
+Hm/2fpGDwqKG9K6zLYXuSDwOppDtDfw665ppaVzRvnNppizkilCohBzCrwXTMyeH
+KKZKqaEt+n35wrurfMfqQf1AkamEimjlXCMmr9IwrBHbJeIJuHn6vGSOssQ0Sq4R
+dr002QrKsGDzlL8dCLmaKU6VPTXcSbCOgDnBW/AyU7bvN51jtgm+jOIey7jH9Bm4
+0nU0UNcPiShgSbXLPp0V6O/1zQBOVeFGyGenblAKlqLILPvc04f4703alqSbKwXF
+q6GoS0Dipzr4N8X4Thzgc8H/YQ6cBVGQTebVGHqFEngLQ2C0yZWlDfJsKnKOGUce
+xSxSskfzEv2s9VHDrXeiTAzSYaLoZI0JEDHOSICmZT2wqT1voFokIQV8twORGpOo
+RAlNX0BULPEg5Mi2k3V4ZBcG5EwUjEcHAg+0cQ82D0f4OJEqMVIa3dse/lBbrFzC
+/Gkr8+LPWVv7f+mRr3sdKtZ1nBwz1uTP5qIA9m92UeLVdVjmp20hixmlgftAbTBm
+MtIH2xqkitpb6bhImGIbnxpg7U8IqQZNfavvOM9yqj0uud/Nh8ruUuwxci4RS7yP
+YlIcompHudeirCLPvYx1T6nYRB2GB1tpTcyQN60pb6YC7lH9w/tLhZf8GFcMdIY8
+eLx7FoWfNj2dIp8EpBrRUEaQlea6Eb1r2DYTjmNunnI
+-> ssh-rsa kFDS0A
+SPaIIABa0Wja26CIyeUvrlt+LWJY+Iw2+OouRZkMfi5YxDflb6SATesWh3zMD5+A
+SOiA/oUiRfGRVMmqiEznd2BLyUO7FQDv8tIT38R3m8Nj2Lb/kWtuhPKqrk0PPvr+
+kYR3iyM4klqccCzwOYwWfpO4ZkFACBApTOFYbU4WX50WDRQRco7M8funj10BAIsL
+H7hOKtigx2RJaJDcU3MPHV4beECpKbeH9scICEDmi098UP3850g1lN5AR7NjWvLi
+zr3Z8Up4UjEwH2mvuNnAbvA+0HOu5geIae+VUqy3EN7XFC5HnMHB6eK12Q1VAmF5
+MbgkppMeoqLnxIdLT9xxRNhD50B3QR8MNHgaNWuBqfl86XPfXjxYP5kRaCAraego
+nbE1gItlgIPESpf0ocfIEyXb0tSup8c+99ezgxBMrzNP6UtVGRhZvwLTpBZmrLoc
+5P/hTr0WQE7d/6lmY9VyDsXasPU758tUhM/cJxmmUyZcCOhjAiJdWSYHe7AUjSQy
+rUSKUSYwqM/epFNqdnanj4pmmRkSk/0AiEKikvb5DvWpHrnsL/EuQqtT382IGv6P
+VA6ORv3BmL8IjNnTppZsG8k91WWHLEqxoDKUHg6WHfJ5XqnEM4maFsRZs53QLqLB
+vjAxAeJHjFg13wkfJmo9mZNbw/0WXS/K+xmeOTYuH/8
+-> piv-p256 vRzPNw AwPYD0NiFDZ3/0L0+BEUS0hm7RddL3sPXUshz7XtIQVi
+7rzoQuAQQHxkuYFx5TrLEXZbGsERg78mAXcgQySwHGw
+-> piv-p256 zqq/iw A0Ec624/7FOTPVAbZDjhsBy0i5L1Tw9LwYfH/7DeKHi9
+djfKQINL2LVAAueovp/V1IGyhuy5LGQtOws5Dtih9sw
+-> ssh-ed25519 YFSOsg 6EeEfNtlQ7/a5Rc5iShfSa2ZjIoN6QcLDI0hJgpF8AY
+Tcp4iqFjBTTzSUAZrxRWe8QkvuEoPWVagNL4EiZLMIA
+-> ssh-ed25519 iHV63A P8IDXAspyflmLqtPOqPWE+J9s9e3OccKc5+8s/Wi9H8
+iRZba5723Ux5oo8YA2TDyiaWyGzHlAcvEiD7I99vq4o
+-> ssh-ed25519 BVsyTA LB7gg2/eozH+f9BNC4Q1m6Pl7b6znkO5rPVgvKSjen4
+AjNzM/44dMy7JyUcAT7c4pAFTtOuapiGtiqLdBPGrKA
+-> ssh-ed25519 +3V2lQ NHbovTrC4cTSsqb3AfmVOJ/pL0QQbK9GpMUpQMAW7w8
+iwAoDSQnucAzQPOgZZtl2bnJQ1mU19aoruItkQqJuZ8
+--- itqKtiBSCvkVJ5boq7PeY3uRMemElImzWvSeTwbz3y4
+×ZP38†¶0¿Òe¯8W’j–Ž÷[ªø#;ñHjÀëÏwïYÂœp¨µ“6W`ôhŒ²ªs§õvbÈ·èÓWu·ÔœxZ5f5½ 

secrets/coturn-static-auth-secret.age

@@ -1,44 +0,0 @@
-age-encryption.org/v1
--> ssh-ed25519 iDKjwg ZUEOvf7JnWeFNohEAhloJ0+YL2SwHujjm2YG85NLHyU
-HwrrqLMlNmfSlZVt/lCkIwqmCYLARbDOBhIm+AYmDEM
--> ssh-ed25519 uYcDNw Lrek6ru/vb2JIZyALem40oNZCf3ia/U6sb5hRyDaakA
-N34LLq2+qJOlbyaYXUtNP17fDPjF+evgZ6kOs7mVhYI
--> ssh-rsa f5THog
-jmwJ+hV1/50cWemVUhPTkTFgnd7iJ0YLtjU4fEKXghWIlie/OR3AK++1f2UJxKT4
-Z/32ALRBnmb7FlAPyYbxIns3IUJP+Z/Il5SCeDrtwaUxmtluwXwwO07WlztqZJlO
-bvZ0ifDazxOFZO6QfXQE2SaPDOqcH2AAiiL50eXMgbdY5lARYW2Qbai/2a4t/PuT
-Qn8WAwjyXOIdOnaYb/MZWyp0GQYsa3nEhYyOWvTSjSROEfR5qBtGNYkUBBTYF+YO
-DGOYStbPSkIhnYYQmNajlcy9wMW5fH36ujGdnMH7C6DgcSCY2iTDTdE3cyCxAuaJ
-bRThKyXYsvhMKgrFzbhlgt68taESb4KcKcNO5r7lqqID/I0b/fVltsKpkXrSCB53
-Th/aXLXPUrYEkbdP6nqDBbUjeA8RDid1raIF1O29Ok72oU48q50QXqP8GF+honkg
-HSdXmhPtlZyArlJNWogDaU9FkWp81E0JS8G0OnoNilCmiu7sF717GG4pkA6GTnaB
-hlJSiVWBPhmhVURIOKkRl5bIWLUvJESPLVVog/vsW7OJETOb2u+AlwvaBNY8w1wE
-An+m/qNO/H9Nksw0B4C9nLfasE/nDvbOT/Igc7k6jP0sw6/PAWnosJY5vDyIpR8k
-7q3rBPnsZRXUr213ue8xs0G7SsbLheYNu3/D4YdB1tg
--> ssh-rsa kFDS0A
-BwaozSAR0Lcn3ZOHhC/OuOYRZqW0ayV4kL7CSLgaw6x9WqA7NLcsE+HDr7aDx/lP
-K7TmFGYMrOiIk3siZ4Qc/JwZXPiayxGITcwoY82L+FrJKJmQd6c/3exggsHlc7B9
-1ijXoQgjnorlopI70Cyt3QLQyMCPFb7tuZFEKR0NqBzcFTi5fKVYcMrfa1WVxzMO
-0Ic+mhwMIAst6SQqOkqaVbtUYxATupQx+9FwThk+9NDety1vacb+lQ7hvCnImpTd
-uENry/G68I7zWhNuCeE6wj8lCplFkW7dvrJyoxUVokWheFnUKjziA3ZybfMyAmI9
-vJZnTvTc/7UxJCnuk/pB89q3ttm8LFT6AFAwZ1PY2ndWBMRlnOaB0JXSBKXZCYYV
-bmJ/NSNdzyO9Q4MrKwYO+O8SOkVWM9EqKYv+FMO5CksU/N9EOUkpZeLpMYh1WXPX
-BMKmXzRWp3YEsFH0g74ZBjFpTo+FK0bbRfYfTj7wtS9LpOFPr51qRDwv0zocM9cQ
-MkpNtuSqpXboCLGytJE34pAsDY1BHJpdAOwlwavwK8N/yxlF89ktIAtHpOaV5QNF
-r8oW2DLERj/s2yunrjZ5kQXaxbn2GBeml5gFyYWPnKVIa5x0PA6LgT2OMYd2x4vA
-r7UGlMktJLosJGjJEUVLUHXarKkTz8Xwrw4vtaaLIyc
--> piv-p256 vRzPNw ApWXG3ayudUSrW8zw38cU6hYVeCVZhIQm/ZbjKpZqgnb
-NqaQ7bjTAuMei08uNpVaK23uVmspjlkGyleF8phudVM
--> piv-p256 zqq/iw AxdOZ9zfYgKZJY9HhQokUHwSKbfKl7i7X+FPO30EADcr
-qsniaELyEVrTeSaJG/lp3sCPCmbTUA7CWdMxA9tsBXc
--> ssh-ed25519 YFSOsg 64fhQVd3dvwHCBXa0QiK6E8rYA1jScm0UiBvJVuL6Eo
-YAvXqNw6kQkTzBpDIboqa9gOoTgHE8hcaIMTg6UkODs
--> ssh-ed25519 iHV63A BlO/mSeyxTFBIa77g0Ce2CcaVf9SAiw9/OzkgnaHEV0
-sjmnXCpwe5KTgIJ1ZaM8j1U4fYi2Y5/WpwpUfAe8Dbk
--> ssh-ed25519 BVsyTA gt6iV6mhL2G957w7IbJVzNFV8QMHOzP5uOkgSp5QgzM
-Vvz1jjLKA9qbqAE1g0UyHySrrnG16ENdz9TxwyoML+g
--> ssh-ed25519 +3V2lQ g453jshh1sgCdUyhg3jlU0A0X+byL5jobpu2toWTYRU
-S2k6Nk+UBv8gcJZoIdZUc2Kd+Rv4jzzcEyGm+eb+KUg
---- 8ahetWGfwjnJYRnkeSS15sLjDBBtN28biMlYCPSvObQ
-i’cü'ióë4Aî6$}ß!IÚ3ó¨ÍÄ™	Ù›3yŒ<79>ç¶;¶ƒ
-O<EFBFBD>.<2E>œ[„Íf%jTà4ŸG¶÷ãÙ¸W#iÐzuä`'Á*zmû‡òèE‡6ÓضÑúéª[ê€

secrets/delite-wg-private-key.age

@@ -0,0 +1,43 @@
+age-encryption.org/v1
+-> ssh-ed25519 9RQHxg dVdaiC3H/M+tA/xIW3NdwQax68lkydLDLm6OxTx1lSc
+HRLezYbdAPHNbQm/2WXT16wVX+ZC7GKlVp48aIECsdw
+-> ssh-ed25519 uYcDNw SqHkg361mGpjrcynYld45CU/jfnPp55bt75apCWlADE
+Z55QoOPVt2u1d5Q/96PHfA0MFAaO4y3CWuJNBnVy2IU
+-> ssh-rsa f5THog
+yKfwc4z4mIVfDJW5aEk7O9ddRL0YKOZFV+xKE6OyMGv6luJ+yM3QcPxu9yppCfMA
+AIreVH2/4Tn+WhYv0dQuVeTR7SLqfn+TynkfK6ZZ8NbjSZXnakPqNLtOWi6C8VIH
+MAfTTg1NGOG7E3TGmMqFrQnxYdpIvtfeGaBfiMhGLoWnCzJOzVkZSf1Qx+ibor1T
+ZtJGsAVoAqGlFoWNDNrlemx9/5qxARe/GSSUyKtEb8VtbvN2hgJeg3YAKBANYkUS
+C2Py72R662WBKPSd8vZRaQEJzP78b+LjBPmF98E8EZcHD/L6I6QRirpD8E89nN8b
+/vp/Ze2q5R+9ot2TIO4Q9TJKLs8uMYUNUImrWQM9eBG444dmuTNDDfBFeI0Xwwug
+FrHxS5pE8QzRdla/fIt/Py02F/734pb1LRCBPaLr5F7gFTBtIXe1TCvwfigiYrZ8
+3zD4mpasIz1a7MlOIBIUuNAvAvJIwemenzxoCPTJUIPR9Ja06JUJe0qe/qOOaUAE
+iio/a7XLYAUnTqHQ8efufS7RF8HB61Q5/CEtjE4NnpaiZkG+KQ57xjTwrfKSALxK
+FN5ydIMY2aNg6Hh+n0MlCfpV2N8XYqOF4YWWRKY39516i8QCvvL21k+lt3P0Lpad
+E5fn9FWoLqJw+evJsF/PyTrLT68tQbcY90mIGC6n/Ww
+-> ssh-rsa kFDS0A
+dTkAwE1+eD5KvpKvI0Wja9vV3wgKogv9ADKyf/u6meetXQ0isUnqxhqBbQZKhVqU
+StNE4dSwg9gBwsPmNXJ2Avc9LMIBuhDomPOp8bqdO/r0IyDvJ+TngnJsroK7dfL5
+foIDtakdAOVZM12ytm5hZlUNfXppAJUF1T9w5LIBWAMQQYJs39u+RrLfHoIvXTor
+OvTikrdYfGiWdX3XIGQAfoPct3ZEKmD76ypH8ESw8kKF7ZfRQNDDP3LMbakoFs8K
+Jrh5S9e3b5hdiDsv0DFtMISm8l6ec2mhhkVhSPgJ0mZYimMUg2QW1b61aTbE+LEI
+K9O27ci1UKhxvLnxA3UWrYcm9SX3dvFgH8c+hJDdydHRGtMA3OU6Ut2HuGKaLfys
+P7F7gffBTEeW2na2I00k3/oq9CCT4AP1d/CKlGd8ZtRvv5Axv6ThehCFZ/FLYGCg
+kLBTHCX9qfvltdkKHIZwgxQ7JSGvNf/4bBzUz47o4gBhtnADaPondeLkqRS8sMwF
+gq4UtG13Y2NfVYLtydrZCpUSMXekibZ8Rj2ER1uy1PD32clevgmgbTSyniSbJDa9
+NFAeQS8KiyfvnxtLNNoMflR5yL2O4bX8Eo5ooaxILxga4M7ckcuKz2SMjDjpSBbK
+ndwen9EsZuQwK9Av09h3gZeRy/t1ne6MtZrsTY5W7+Q
+-> piv-p256 vRzPNw AyZQpsc9MqXbooqG+eK5gQQbfe4ka6pG7uixb8ONVGQz
+FWuy/qAQidT6C8YMb3674epUzZw0Rb2NMCK5t9wdnT0
+-> piv-p256 zqq/iw ApTqG55jHkxwd3cT2Hvw84V2DcoHo1M+q9eP2eLxSE2t
++27Dzy6pzGpOwTqUG17QaDC93O3PSJIfy/d4eBnuLw0
+-> ssh-ed25519 YFSOsg mRmdt4AzDKbzKvMPOEHg+jQSRs2RF7f7ev/jzP7SuFE
+VmNGaudQF6R8xDWBz6bFfmk2J8twCUEzcXj2AG5teKI
+-> ssh-ed25519 iHV63A pXrKk8kpTBDxhiio5ZY7krRJIDkxYJZOMqCaW9Q7OGQ
+9/xgfjzsd2JT6FQ2YWELl9jqph3+HTF8jChvbiHceJo
+-> ssh-ed25519 BVsyTA z8nXuz2JOAn8t8OW+AzFRAXb5ulAuderatBFDrb6klY
+Z+7S5aGCCV7f9WwHWr5LrsKW7rnpidImwoiP2dXcxew
+-> ssh-ed25519 +3V2lQ p99nuu5l75p1y3Ea1yRdFBQSxvYRVRJzX1undANyFVk
+QHzKD4WvtnRI0wgiaIYKWwXrG5Qg0vQ+V6eTJUk+A8k
+--- od3JqYVYOFEDzaNHY5oDbfOjhUBsiQFd9pNGSkAw8Dk
+øFJÒMmá|šž—>¿|ÉÏü‹µ
ï¾ê0½µ:+‡¬¥U^ØÑÚYؼ÷æ/‰ŠwÊ X+8gt
RNPÍ^\€N}«ü£

secrets/forgejo-mailer-password.age

@@ -1,43 +1,43 @@
 age-encryption.org/v1
--> ssh-ed25519 iDKjwg t2wVQkkmL35JIXb3x9xxWmRECF7yet7c4+EdoFhCaAs
-GJxaYARfYJdNDq6Ay81r4+TsTAMmPd7AxZMTq/GzuIY
--> ssh-ed25519 uYcDNw LeXqaLu9EylzTOle7lma4la5S3NDPjALTybvV2ppd3c
-JCdLMyK1c+Sldqla8dVUDK28Zl+nLXnsh/MXQ4d/yJw
+-> ssh-ed25519 iDKjwg vmr542fc5ndYRiW5ukHanhTMBV4TgSduloYjuWwmpw8
+ACSBzerGx8Rd17zNfgO3qSqwBuQio9oyYaa5ypUbBxk
+-> ssh-ed25519 uYcDNw fjP84i8BsNOfBoVDpH25MBC6D3xPXwbbO9ghLKr+mE4
+TRKHWqVvzLP/AOJ5SJ5qeKj6UdYp2+tsxO/GVRy+qt0
 -> ssh-rsa f5THog
-ePz4dSiyFwtX5HmwF7/vyXKAwH52aVfjwQcyovl8r9srFt2o19MLkMvPM+CVXXo+
-v0Dc6dNdfp6Oyn4rNvGMNtsL7Zb4B32OVck/uGn4pI9aC3wsOvr1JvetolaepATG
-vnxB+TYjM/S/iYpxaKbrhKBOM466dUygI4bS/GPT749+R9610mZ+jle1yylyvRTM
-WiTJaNbOEKOnwPPgZdBbZEcisbz5BFJhVHBxxd/1DXZPRBeg9Xn7514IwbtRgRS2
-x6gzSDFZBcpvXQ/g86JveORI9eoS97w1Fgv1bJnwhoZ3yebDThPnP0xU6I64nCwJ
-H0W+Rrutq4UgMkYghElCoFyunPz73M7PArzW44PPCOeGUrVDsqbTgf9YD5s3HDsc
-TZqrcgzY6/cLzulgM2Is5b4ZEecWLNGoe84sjHR5OXneGf9BXcB5GpgWIC//ZFym
-Sr/UOlfI0YtZpY5ph/10uWqi838idwMTjOZGuope5T5m8+rsbxlwAKRf2E31nx5e
-rjW4hdV4tm2yZm+E75FEIo1yhu4JKm+MCv8ri8217c2Ex/qqDQ5pi/meOrjKGFgc
-7FKkg2h6ir1rh0bMYTpZAelpKVicTQpY4HyphiFuk5cCt6cXiEMiwRBm3QrwmSvl
-KcmXweuoYiunkb5q2GG6sE0d3yZluG7g5mPWE7WFngM
+IMiwRGW5B09oJDDJamyQBOwtFu+KhxUQQ1w0RJEnpITZZzyfHOuhAuwsjLy+us9v
+znzSA7bo1YusdFjzfW4RZeueuW8vvWdsv9Cjadlg6Bb2uP7sWeh7qH+IoyB2Z9Kt
+fZ0O6O+65pOybsCKZ9cUuZ+E9E5nG2KRjzdV+csEKAshGLqeo2RILKZB8oSx9tZr
+sjPXRFLchj/6zbY1yBs+q0+qC7ldUBdJLDtl5Yj6hzB4fk1JuPTSDWtKSHBYDz60
+Ri3CO1Z0z+8/IJWZqaa0a7jCYH6v4ZzboGrdE7MJq6Tyorl26bitSSevXHJmCrJq
+ZpcpZfs/kVFu05ftaVul6vyPM09QOIQx+ia9tIKmCgWNcrM5mSy63C9klutXOW6j
+GZ60YUnOvVDz+i0wmaZvMs3C4THhdtOsln/dVrqOKFKf704mAsO4cSEwqNyoXy7n
+/WbxDQcTU62pKEkwlU//CjuA0rHwmX8m/tweNuokh8fPJ6SLE9dLqxPzmeq9JX7q
+RNO+eZZ2NnjpkrI46stVS3p3FUHPbk9mhIiyfr4LgSBCoNWv1TkLS8DOqWOeNKfy
+8wLAjgvpNIk8Vgbhoe4/G8wB5Qvr0vfGmdjzA5miu28YZ8PXHN6DlkaT9T0iROuZ
+HM8j2FND2EYLB/Je19ctYNDjCmEYFhsvH/H448QMODw
 -> ssh-rsa kFDS0A
-jvdXdLAU/L/dD6bmuXIXEaZIGAEOFTujxuVy7oFHSeAZk4AyT2ZBLfuDa1KI3iSR
-19hImXIZyaQJh5RSTkEHpwdsxexbzcQV8y77XdF2neNj6OYjMfpP8a5foGcu9Rnd
-/DRSkRNumkTfpet/gSiRIPz0Sj++HA8/X3GwRlwqPKE12As06ebL76003NX/u7ep
-6jJR/sOlLJV2VtqU1tvgQ5VV7KIyMTlP2a38MDCtOHTVmK8inVNqL28H0ptYJ1IZ
-KId96pKqR0v+h/qhHHQoYBJxCBUHc3Rp8eZdssLpLfRHGLQ1ccpV7KUr4ihKFh+9
-wu5/YD9qrQls1EjXi1mj/kCvT2FplUJEZqgBBRY1/mHjpWmx999E5QijjhWchBL/
-PgF0oF25MABESrm/edvThwp7MVTA8ZSY8xj40D9bfsBGrpVBlf5hKsSulpiaMQGK
-6ScH4sRFXDhwC7j8OOMZWcqNvEc3KjgjJwl+qFTXDdBe+zd5sF+orjORjIge1z3c
-v2wngm6vTX4GobrtqiC2ZmolWbRxz7a4IwWNPrmEql8Bnmf1bfRy4X5/EOuIy1zN
-s/3Qu0KjjVBikU60RyXAdRI2M+aCKSKushp7bThT2j+VJTGIsmEbG190VjChTkmb
-meF98Mt2uJoJjzCYdyaGCQRWAr/f8ER0GZgLXS4Apt8
--> piv-p256 vRzPNw Am/95gSJXOtEBedzoMQBFwWC4E30vZcMINW5JjeM7ot4
-TzGZSlKt50OxQryrIQHSqMj+dnIDV5+adfohG9heDTs
--> piv-p256 zqq/iw A5R9tfUoQmodbn9rt0M0x3lnJUMz02u66HR4+RTIcU6/
-D9JRgjM280CDu/ds6j4lequ3VGEwz2aEYf6Pu4e8Hf8
--> ssh-ed25519 YFSOsg 8YXBlEpaC0Vkewf7awp/RuHxaV92jp8Lnf391jd8UDc
-9bzGios7kSFpAKJbmjupcpf7/BCn6XwQMDfh1uRb1qs
--> ssh-ed25519 iHV63A 2WrmaliPbgh0rJLJJkgy1sUI4S0H07PwZ9oJNk5R+z4
-SdjvYKOlZLClUKs+KUQ2ek4X7amZn56Fcfo0TnAeTO4
--> ssh-ed25519 BVsyTA +RlXlmh3g6GLA7W26NmDEd4fRSVrJ8lzY1ZZ8OuuT2U
-+quL6DjgTibaE878QG9w281ArQ/9LS5467Nfm5TiADg
--> ssh-ed25519 +3V2lQ 4B1XgSPLOVjzzNlxNYmLeIxABUnXcvETOOWlaXnmgRU
-bgdGjPihq6SnGVWsntPhajgy4sLD8c6VWSYeSlllUx4
---- LEtvqKl5+C4uRIigcNFI3hVXkxov9Wt1CasYuMvEpvU
-¦V †ãCC{ËË©u%ŒÒr¤37ÓSG6	¤(Eëê1¡ çì–àmP©Fwï¸
+nSuC6Kgnp3+r41SXWEqIkryUlhWQTl5Qr0muXhnsBmuN+2iVBGYFqxqJSDpdnvbM
+SIIy5c5rLfGSNRejRebgvyCntEvrOiGtmgubpURMEaXwJiEI+Hqfju1V8yi2RUto
+KTR/YhYv0i50pLMV4JD7MWPga/zeYUVhE8eISUNFbiJS+ve3BP/3Tg2E9tUvTsCG
+j7MxM4Td1jH2jCa7ueYOJ0R/qr0BKIEOGP4e+7YCByujYWKTV1JHOZehzIYA5wFs
+Bqq7GZnTTiP5n/kh7CgNyWORdBW4gaBslyhjAJI3hCbBvZer547K4moP6aPKog32
+yTZQVVu/kZzCIgA/4TzWr/g11fKD0dUnPCmXbctQVgeOtYAbnWJBFwMmXvQdRU2P
+jt5Ce0WqwhExaE8fAx7v3AYXhYgAOBem4uIe34PAuj672Tc1U61hztSCynE2cXI/
+2ZyCoWZZC/4fjUTFXC8urATfjgGV5PrhRcU4JtnQwytjd0Ru4pm56zZCpzmHxWMi
+bbyVSF5NisCvjF5rxyS4XXNfg52fkV01gJsJ48B9dnSs5HYTBaWQWJ00sO5ssGRm
+9H+T/Bsi42X6lnlt8ybzYD7aDIJbfz56vHf18/tfRhOzYbIBE7Rnkvs0vsNwT05K
+OGOb3iLFtxGGIpSC5Ba1T8h1TuEAawhmrHkF/Lfdr8g
+-> piv-p256 vRzPNw AquhJELiSYcW1KeKiied88TcDZYgtjdZm46FlV7CRsmt
+ikxq4EIkI3UXqjns1QJdAe0N3mEh7sbzYPu9H0IvmCA
+-> piv-p256 zqq/iw A94RUts75asVyQG7IZSfg7mDgcWI/hruOfRqD8Pdn4Ff
+17kI8IKl98f5lMqdcvqpTO8EN7pr9HP0SJJxFlIMcrw
+-> ssh-ed25519 YFSOsg gvcjFcWsCaBjYXvab/eyIhDhfw2bjc3u+nOenGiebTc
+hi30KYU9aYxWG/ZWFZQ4qW/P+hi+ms3140/9BTYNo7k
+-> ssh-ed25519 iHV63A RVCt2pceoQfdaBWd5qXcT0x/0dQgLP3jKpa6xeDdnyY
+Q1arCu2nmcl3Qt/uqV6UeXUf/OffvNl7ZWtqbWjrw4k
+-> ssh-ed25519 BVsyTA DX/5O7PCLEBoraM0qFy821i8GZysh2+XhR1fGV50SCw
+hN/a4NwiX9oySM7Uyt6vS9hjOTHNbN3tF9DAHF0vDiI
+-> ssh-ed25519 +3V2lQ uXmSWC+gV5JsARpa8cDv6faykEoYJHH0TqnWc8yzhQM
+/y/94BU9Mwqcez6y2tJoCEkg8c5x9p8FkUbbhUpKh9M
+--- SA5BB1izTywLQB/5ghs3XqlOzr0ag2gI2/09M3eVtjY
+±µ¦ýE¨Õ8SLJðZ½Ð°s\–4Â#ÏRB
rç|eÉY/ü|/.B¹
<0A>ÖÑ

secrets/garage-rpc-secret.age

@@ -0,0 +1,47 @@
+age-encryption.org/v1
+-> ssh-ed25519 NID4eA HQEbsh8zoRkgWQHuHLF/iu6GZhCXJ7h0bCbj2z4uyh0
+lhRAIYM2mWCmwT9WGESgQnf1EadGPMZARC+EnVMH4Vg
+-> ssh-ed25519 9RQHxg GIKGKtJTCWVTZdXn+GpWPDTqQjk44ptA7/D2LmREmzU
+WuKeF8IZG70jA9zHaULJ4uXp3sj5MSga21rLOI4st4s
+-> ssh-ed25519 eP5MMw 6rWxeM8wnvSOxIbvQidqr0MREqzEdsWMTibrWmy0hVA
+LkTgGZyJsl4vfq7jUbeFbrL/naaULV9E6yHyVG3hbso
+-> ssh-ed25519 uYcDNw EC2iOzQLOUnt8a+V/zal82jrOBtBQrIWcaDo4tq0cUo
+T3iWlaRzbctnTrBCQ5aOwGND4QxmDnhulq9PcvtoBqA
+-> ssh-rsa f5THog
+NCQjF/OsBHHODFWMhgz0pkyo9YJN4bUsocgEIx+Zag60wR13cmt4dLMU7k9U2LT1
+S7gs0ZVY/7RrA9GaxTyXT9c4UlehJX33+lfR/nNCbWguilcZ8yYV2pa8DQqBRPSm
+Ub6AlkgRos0R3nBfUDiUWuwZ3ZY2qWOp4WWzcrCFej65WBdrQABD07LRUghnEPCM
+eJEOJE5nnScvK8wVmw7OTVjA9ZTOS5/opF0v76Y3kU88/VuovJz7hb2AcANuB7W+
+jT5+hKhpTaBlL/A/AmgQmJJ9GiAvdn9KUNIeqt8Nfgpc4Vs3sHJZ3HZVwC6gL3vR
+dOE0D+eh0bmI1J7t8jpRa1r9tZyKxEwZqUh/O80/d+yZygw2x8ignLpcRhLmrT9d
+c7guE8Y71+mSxeITFN6QY/+5hY4EJ0XR1t4FZHgKgMjtPBPYyx1ogQNTQf4WaNld
+XWh7goRD+w3RhducqRdmhc0RhDpCazvbGXCyBxZOImgis69CQZQlFJhpCt+Z8B+Y
+QJxesMxpvGsV9c5r8SP7UF2juhzsG6cciq8NgF/7zIq7LEbVOgBVSCLCWx+DgaBa
+Fkdnrc8xw89CA6aIwlwptpWn8rOlSiTljiMkogzWwU1QcTcZuMF7/xbDrXaZ+X9C
+JBliKQ8sO/BVEOexJ+WuF9ORyLB9JjbdGdw7LRUhPMI
+-> ssh-rsa kFDS0A
+dp8Sgszmxvk2xJ13SOQQ/ArEiTHr20nbyptGBO8esZmSOWW/7CYz7GzMT1VBNXhC
+vWgK5zknuaWWbj5o8zO3LFLPct9NvuPTSTCWYbGOR5r+I5Py+SNWNGXyZ8MycPqM
+5yNQXdhfpgTGT2I/ZbQVk3QrualTKX/usgDsIkFvkwq2/ACGsVRnfZUenZwc5XKm
+SWAzjbN8OEwtFG33fprOjYkrtfJeyMVSfdPkgg8P4Io4n1DXySZUAzR8z9uD4H5p
+HfaXWGF0JfC5LnS+8Q2lUeoB1nfb/BDMybNhfz0U9S2WyLlnjSpMBugi5bzR3Y6n
+bAHYIPnFbjstGGzBq7u0QYI8nPNrCxYdp0bZ6cgtUQfrUei6SMZcudXJXIK+DRk5
+FoHaTHkJBUTQPeN821KCjlYgV01pg86NNBMxJV9L8zJqtYp7nk7CLrG4JiwgN/+4
+MmEsmkifjOmSHuLSkrcrZdum3nlUkiaoWcjBYFwzXN6kxffefkzcoQK+Uio1m+m6
+cY6MDo5c1JzVNGIaeNJycutb7pG1iYzHGiVh5Xg2piU9Q8a+b445bShx8m9lKrHh
+VD2TeiU/P47LLURNsUpA87B2H6kLlQgj8mT9oxNp/a6rhKjaTWrt8tbApNVuttHs
+C4/N6cWeP+JwYLUSdjehTkS/3m6Gs9wPgx9uiBwi/Fk
+-> piv-p256 vRzPNw A8BVgFdKagBtB82Fbylx71jPgc8f8hVF1N06/HkIZ4oH
+/gxYFwlY4Gx8HbtKjMf9Ba+gxkLTb2ZGcyKEbnke5Ew
+-> piv-p256 zqq/iw Auev9DCuQrWVMFh8ckLgfrN1aOAVrjSsaKl00GNaneg0
+ZdqFFOSTWf1dsEdoponH1j9hXYag2/l6+pJMedYuVvA
+-> ssh-ed25519 YFSOsg GuwGTH0D0YimZksECOmqpsB8S8I/my7fDRGlJyO5oGY
+uyX0FBN3hIF3az8OiP5L+DwgG69eDATVHPaNOsX35Ys
+-> ssh-ed25519 iHV63A pD7q8qdejDfhxTWCvrfmm2oTmSdg0bNKNwPhhXH/fBE
+fVhPi/U/9MFfj2vORYM3gqEZK3dXawIAappo8eo1U94
+-> ssh-ed25519 BVsyTA iwscr5+fZh4ErmdvKJ1ta5ZSEWVHtQEIXTfzYyvTEmo
+7n+41EGr1knWuHScqWF25xZijMIAjvk9iMGZzIr3v8w
+-> ssh-ed25519 +3V2lQ HjPClU7+EumicsRX8wUk6r/W0G5eJOJ01rrunUHpBxM
+fOS+2bO9IgRd7pjZ3gWaFeLb/xBYFcJY/anHXaX6wFg
+--- 7M/9M7Lz3skL2JNpQZtfMcsL/aKutELcIyQtDu8Kies
+¨_…"šBþ=ò&3_¶^ŸýÜkÕqwÀ"<22>{õ+ú³à<C2B3>Aݵe’«ÿJ‘ŒÌ:ï[×íÄõ’
,gý 2:–rÿ|çÂ*vµÎ&ö"^ùksÀÈK–_âFñK

secrets/grafana-keycloak-client-secret.age

@@ -1,43 +1,43 @@
 age-encryption.org/v1
--> ssh-ed25519 Y0ZZaw X+1NaQ7337zR0E6f7AsjP9koEpvVlXVg55kotyqHSgQ
-u2WmIK+g9D86OVI/U3ZcxVvqToG+HguqNyhv5ockPuA
--> ssh-ed25519 uYcDNw fK9YX8nD3hLhUMENqDRADBHK6nlWEU3pxhvptVKWHUI
-L7xvLbeYVy/xiivumpbEF36nU1JiWLMLpnTjl+6G7i4
+-> ssh-ed25519 NID4eA M8vj+GPvFph/o7wGQ3tR8HS4qkbNdlKEE5v5cOVE+x4
+gIpKa/ivPM+9JP0E6kUUOk1XXudNPXy1I8exD+SJPGk
+-> ssh-ed25519 uYcDNw evMlmz/v8VFHkiHQn4TKRmCS+KLV59xqrgRXsjzj9wc
+cSN40Oq6RtXaYkgT33oR0J92IczJYJYsKkIeGK2hmIA
 -> ssh-rsa f5THog
-BV+/u+xR/a9LGR6eKLsmBYhCT+USFrV/OWs3HTMPxUt38egZjMU8dyjTOUSywA11
-T8RLbIo4AB74YeNuBULUSup3868B26Gn3WcjcXdd7ZENC/bFTXsgS40p/0Qs9sj3
-BWWWCJp+Hi/3MfbXiQmpl7iVhVBe0k+gtpktXBTiJktIdKCGGuL9hrFPz9BbvZ+m
-AciTu6uEQ6h5KOrADJWT5AtU0jdqIqr/WWEkoWqnv9AV7Oa0gVw0RVjaay+mzy/1
-ck/syftO6kmeo8Q3NNXPW2cfYWx3DHLyzFsZrdV2tXyqlSvgumBBgXjqHH/vwIHR
-oZs6gaXFjTaebdccXnjZqx05JjJneMNBjqgJT2ZVTSJan8eAyX48AqOE3KvDrNwd
-mkZNM1xw8plm0uidGaxw5wnMdMrwtjx4qx1pKgkYzmKd2D8ro7F1DJ+wuGZcZN3g
-dluzVHNHlghbVJ9zkMumYQ8wb2bETvHsRuPocWc0ObsGArKjkzrdE9aT3mBNSure
-ZkpetWuFqPAH2GCM0bz8WQIEUeSpzNkUBIriAAJwWkx/V62lWCFj+FzQFL8FhSnY
-N+u2wXb2H601ARJDrbVFaFqbOYh/wWP6S9dft2WPxiCOXMlHXV1E/cCILiPYASir
-kReVqb8T8Q+ehVkufXOcAJjx+xAmeKQ6VKSlqT1F3UM
+qj0FFtZwAA/vYiFnw+TXK78fPOuhXCLWJSUpMLSmuRCrJK1UVZVPa/mbvWGQODxV
+7ZOI2JlmacnJLZ+OvFLwoAQvicqIOCA4Gi2vWGgWlgaAjF+1k3FOPi2pNugXa2uS
++ZsHtOZQD3mqREBmfLA0z3CjkuFlTGfHYItghCBh05L49OUyPai2pjtj6VYLKdYp
+swYk9l783bYxMtu+emcyHHw4k+YXv6QMFBJBOY5WsCJYA1Tu99uXj6/Jul3e2O4x
+ho8NK7DvpuOuPYG72pS7RVDXA28n8Kg8Mwl706aT4RTjI+KbXYfohR1EmYKrjxQD
+pzBWR8LZS29UKL18PFJQmwq6E/xrNRzOOsjSRjFvXOJUd/Xs9FM8rzE6IykA1Tkq
+6cQZ3szxKF+iK55axYzPUM2loD5McBZbd5LusXke1FtjJMz+xRhFJfcjJrA1AnRv
+3EoSHY/rK5SnSNUspg3CByUbxRNvhG6DI4XAqHNUyaUT/kGrQslcpd6QiP8zgRa4
+ZXpPxgSVxGgSUirDSkmlQxVog83Qwxm7wrO1VTYcWh5smC41blHGIbDxPbAGghHa
+PclirLGpNVH871tJevHz7ZqtHhRlsBz4TFXu89IP62G9s6h6CBc7AkrdaIy4enoP
+hX91hmnoFWCIcOfHJqySjKFv+DqH6JkBEhvv8y38Wrs
 -> ssh-rsa kFDS0A
-VxreNdSiWmAtlpUvAAfTfiBcn1CLK5i4+TYdjdjSfn0QPs6QSFRQat2Ui6WIdD/w
-TBT/5/QmNwelUTFSGU+VUrCANtZ/Jl0dnWmyMObpFzrhtJG8EqISWI+QlE6JMwF9
-4jYXITmBQDQ/Jv1lOYQE2cTD8JRs8UwpvogOt6uZun9pnyQFP8uiwicearZiARas
-zHNxprPnPE1lAwqb0YRW11qsI6NEFLIY19/eybclQeeWRaC56FFUVkIIWaOPJUKS
-HsomdcMzz04RpO1Iy5R8TojGi1lmIlWNnMaZ8fL29t+pb4bV6JqmCZdGXhJBkeSt
-hMAglxs24/kahRkHoo7mTigx9ZkW6ul9KmGE0WN79s4XHOXgh3dcFXwBNFshklh5
-g27tfKPcRW8MAzowYaqdM55zpSExnu7YSmV46XCpKK0dLmjRBu4L8ttb4ZRJVkpd
-324BDs71E8l9F7h9u9Rg9OrcCBBjJ/fLWEnYa+q2voWNjde5xybg8DhC5ecfhcKy
-fvKaDEICqBGSDSFUd5u958KCx4Ar6WbVGpS1vmDWzeUZnn6QJrwfnBGE1zh8Usz+
-qJysxDf6k9IAVn4eVUpvakayt6MIixOZj83flKr3O9JMD3Hq7OJGuLVHNiiB/+Hl
-TGgdBtNeJ39OxUNI5xH//4ncGYCrBtQFogZTy7AveUA
--> piv-p256 vRzPNw AlwY6q48DQJPgXgFtbUlxfcv/2dj0hLU79Gi9z1MlI/h
-rK1p/XmoP9HYfYIQwlTwmjCe04lHAxGDN4kMlmeLvYY
--> piv-p256 zqq/iw AnKdELa32ZKGid6JZP+qqsth1YCETO/4H5kWg5oboEJg
-iNeCsNlz+Rn8J73Y0JdK5gkUsVqugK2pi1waWPMM3/E
--> ssh-ed25519 YFSOsg arcUnqe7a9uUPpfTQVNj/JRQnlF8bRifI/WVu8hhzjM
-QQ62KL/yKowwO9nQI7XaeULusVkntta2EExkS7VcSeI
--> ssh-ed25519 iHV63A SeayDA/rZ6vlPWa2TcPk1zD8YLtJMHoZ6jF1RNvWYhQ
-nUIh13KsiHWtU/tDAkBL7UGZC5DRmqjeXOPKT9WmMCA
--> ssh-ed25519 BVsyTA NnVwcpyrWesyJFQzCvhWl511LWgt/cJR4g4w21GIRGE
-csco6z6z9r4v8CV2mj10Prz5EqwyUin5GFQaUnEK1D0
--> ssh-ed25519 +3V2lQ IxN+9rih5S6VPbHKNfG1kLpthIw1sNxoFLb/o7YEFU0
-t+wrJWTt5ysWhMQeswb14vkfHPQKWjAvoPEkvEPhV88
---- EG+fx7HIrd6f2S/q67m0SSq2raTQupOR6PsiYKm7wGk
-"Ut<,瞁u霴顚哫件.?榌憊f絺X娌%0缿^罾a$彟h濄*g扽螴c照噈<EFBFBD>
+mpXeF3O2oLTU7MAWUUctR3bAPGVIr2Ic/1o1WFhyajP19ppjlIv5lhDdBXSNMrqe
+uoNFfRuqhn7f+mtbj3FkbJN0QogNGMic9YzCfuLD1hIedxAut235majPbHNAjYg7
+a6k0WKtZInIYnMSt++qqLoQw7MIx7Tm7DjM+EG7pJKr6ZT2w4b5sthBAkBDwLSPk
+cwOGZQR5AbyJnZH1QDiXfOhywZMia/quEd7ugGZaLwSV+Iz6DwjMCPYVTP+d1jI3
+pAd3CkWIsCLMvlTvfNbLX7IhAqdg4bOciGJ8d9O/pjfj2/15WpcCBYgvaR/UYKmS
+RhPkWnPiQXVsybgw1+mH7A3wi7YdLvmQRxPZF3pc07Abs5fHMpqE3G6lDk6bq4a2
+UJg7AIqMqxmkIk4s8XIVuUVF6WmydHKIcRUx4fvnqMKyNmvB9ge/jWzy507km6om
+nVitgVmc7BoaiuaJiDB4jNO5gGsbSYm4fH82lSLJy7nhak+BRgg0oombzzh5A96N
+pYbnrZrEHFrlyjpY1MRmUJuubKC7GmljqLCWlxD4YpOg70Ne25Z44Igv+2QhSage
+ipyOwosh0uCgoi2zfOLW6mqFdsWNMDUTtpaWQmH7ICSsKorHM/ySsVw/VY9SxyxO
+J6fkWlCakn27Cd7H39SroN4YdE1JvVA+QO7+7XVha+Q
+-> piv-p256 vRzPNw A4W+Q6MpVn1ZoNHkGmIeTFrteV8IMBj0R90x3bSx6z0q
+F9+cMoBsfUeEN6bI9lPz9IKIaUNjLwJd0JOI6OYsXcs
+-> piv-p256 zqq/iw Are2msR8ZZRaNu5Ab+BTl+AMaj/jPGra5BRBajmczXC/
+DXufB6ivej3Z4YHAvt4AMGtQcbcM69kNnYed3jnfapw
+-> ssh-ed25519 YFSOsg butziA7Ccty3azzh5/LFbClCau2Go2krNjPbm2cGUxc
+x9JyLzPYyhTcvzjmb7w1fcJvcLEEDpSUUv7WCtk2Y9Q
+-> ssh-ed25519 iHV63A hSs50Whtn+NxDOq3G5HhRs9vCh/0HS67bkB4akb9gwo
+CX4pQgasUT7Z7DWjxDz5L03/1+kMCUTiU8HrS4lV7uk
+-> ssh-ed25519 BVsyTA 4Y2EBmTJc7rlFOgVdZrM9MFzes32FsRNlaMmZ1Zlxks
+BwkgIL+1gMpA55vz32iMLXgr+tK3VEe/7+JnKO7xV5Q
+-> ssh-ed25519 +3V2lQ KXNeYb2HqYI/yMSu+0eCs0f9xW4d/t21uB4DcrAAIE4
+VLasUSXqhEU2U4P3IBXhVZ9jE7/OlJ4n3qrfaVH5Gkk
+--- ZfxR44elx3WGfKSTK4+mlWuqLLSA4esg7Ml4/84MROQ
+œ“T¾'7Km¢6ûnÏKk*s™§ö!û™æS3è@$)‹ºc?á”…UUØŠc"ÈšõßÔ¢zÖWà)ºRªš<EFBFBD>

secrets/grafana-smtp-password.age

@@ -1,43 +1,43 @@
 age-encryption.org/v1
--> ssh-ed25519 Y0ZZaw YW/GflpScjXOohk7fuyf4La3SAMsqzYPXQYcRfIldw0
-9JIhny8OgLF3/M0S53TLXimIIAiYP8txWE1xB4zZtj4
--> ssh-ed25519 uYcDNw +p4m8Y9WMfc42acNfy42wxgTvKfv2Oucc3WQUL59RlE
-+Z6/heh7+KK2xe412D/udZRlpMmcDFKi0xyghqAg/44
+-> ssh-ed25519 NID4eA 070w2iKFOrbla7LvC4xgbzl26rWta4SQAdfApaD2v0k
+iBz3f5kGyPK18/Hdh6GcuTirbe5CE/GfF5ZdpU36MXk
+-> ssh-ed25519 uYcDNw 4keJjA2UPPTPOX3iHcxIYdw0FuYfTZ1j4YF7OQpUp3M
+vSuhNGXSs/kWQGzifULsS+bTddpuy3CcKCE9UmNZBfc
 -> ssh-rsa f5THog
-BCgZnv4YCvxXau6Wm+mMC03lZVRbWjvoWymKj79/JMK52TLx5BUOS7hGkitgS2mr
-A7CkPxucLuMExdefcRiDYp1jX2rWU24nUy2MxwOm4sohuivsTJd6cuNvx/wDfbmC
-5JOUHiRsW+igclfQTYhYhPRVkWw+L3IBr/lav/vD4Zwp4sA0jnKXw/VRkOa4dHk+
-VeKMmc27K6FXpUNHh6xpi1lpZqgKBrsOQH3AxDHioKtTQOjIcIkniXmtjt+n1pAU
-z/GdtlBpI3JSd4pGHnXveN2zOMJb6A0DvfpXaTRlQmxNO1cxFm7DXLYy8uFCBI+D
-KWKJzHuvzIO2dRAano366SE/5MHv+ADL5mA3qJ2iF7uRiFkIi1yOMiejGWMA79m0
-aarx1K/EqSvGDC75N690vg+ol0MlVDFKHN9UlPMyjg3QQwQGGgtI1mWdxZoo7cjZ
-dVti/OCxTvbKMApWEvLGATSyX2tEL/PhcJFGSPGdKJU9C2WZcKiTy5rxCH/ZSgR0
-Ju20leD9TkCf0oHDFHIWRzxeOzYI8tlpK1vy7a1GQJEyeZn5eHXtTEL4tH3or0+p
-e3qW5djQrSZ0sQdaPt9XzzmdmBBODWomSweK3mLNKdOhdAVBwrV3l8Vm+qcVhuW0
-ZWfl2RYgGsfD7edFXmGTal4O8UIVl7NYjrhT3Qkl8G4
+rcBf1LitYkIzigSq3weM+/GsavQc+ZF6cxLLKDgFdPRMGZlImt/4jKcuCGNTmF/8
+5Cfu+obojl8njdBQsWxJUuvGtbl6tOrWrAu7wlUH8GRcoy0A8umiWPcM7GiPvq0j
+nOQofSPVUD2vYRXikfGyxtlr5sjU7w9QpVWbrTsTatYGtj/A9onrgFOi0huoEHKm
+4nQufpe4VaAOeOmIUEWoJj9vOVovvoj7RqLn9k7nOPuabsHVxw6sePkJYllJjrtc
+Bg+fRduXkfqmu4hPSwJqVCG26VlM9UqiOVr0e73b+s6v20c45zr2rAT/xhhngzJl
+CB+13qf+XK3IjZaDsR6kDJKcR6+z2H37KpXbzqUUGCQ5lx5tP3imn7KuBoY96/Sz
+4fVgvcM39DuoibLh19LoSsGIBFOmbT4OuFqbaoxgRap+RGgh2uPY3CdfyB0chC/R
+t1qZla7Z2nSEqY7oBAd9NK6+EIQ1H1xd0WSokRVmlJdif3zdadyMJahHDbORq5zi
+RG1A8i3FHgr/q/cLcQQ/2bVE2pSuKzhlxydtVNZEDInJNPy0djKF2j4AHTyWQdCn
+200EvktqfLwL26nSKrZCrmTKUJEVmMTKM8SH/ZIpOLou2eIjQtCskqsXGdwfYmGD
+1fIBb2VLJT8G6Nj/ihqv/WJ8Mlpj5CAPWgMeeYHywX0
 -> ssh-rsa kFDS0A
-Av0X9EWC+JOG5/XhJx9JhYe2ew2IE814J6qOv6/5hJehPHRs+x7pkyv7Z1KLbFJZ
-nQBQuZkqLY0fed0LF/7fEnNF41+/5+HEoTY1r/Je8akc9+pUoYgbGzJN0ddbXlOK
-P3antVHR9jx/E9eY8fgkNMASnea7BUc/yqxUljRKF3N7JTxC1S2BhSd9HIdT9XlB
-YhA1H0nRJbfp7SQw+xrCoMzln4a5brE7ycN94EaHnrgwf/AzYUawZ9xghwtS+HOo
-8FLwkjRPoscOEJyhSS1ONO3CKOUHfa+0cRWMA8LHhkwAv68/07zktF2eJa6dX4UN
-hNdxhwA4aSfM2lkCTzzlRpl1AEPlhjq66F6M9qy2B1nmsR3zeL1YhphDlty3r/2e
-Iw6wDKdMZgCbyQODXL/9w61cxpO8fjus3u1AkmR70MNc0QPjlGKs1V9LrYPvRl3O
-Hl0XUxg/7RE75Nqxy8GqO3YkgzLaMINgB9+mIS8XgtM87AAA2xSxWXII8iEbY0B+
-RjfM9MkhId8LQVY+3kH8WFJTBcEVUFmr52mXU/DImbvQCYdS1YRlfYfusXbABGDG
-5aRwcu+bJ42lX+G5ZtWM0wOnCS70zknvW79YabSLAM2wnvqCo+RDJbicfYeRb3Zf
-oyGR4JBZ9sAmOP9NlPTcWLMzEZxXdhGTyl0Ir0QEkm4
--> piv-p256 vRzPNw AwRvkmNKHx4OKTdVEntbBvtJQ81iF5JtV6CywO3dD3O6
-wghX1uvRlC+4isf2e4H/ydPOLtmb9OsYeE/mePv9zqI
--> piv-p256 zqq/iw AtEZGBvcuKRfmed4akDqnPlEIKQq5KjR9GHoXTudkl7l
-z2fQAhhisxDnjZszDtDI6VBQFsHHbKUlAZDNHmIFKVQ
--> ssh-ed25519 YFSOsg ZppVdvUJDX9LJbOGKFBWgCNDQ5MjTuZGnI22uy4ftC4
-THFlybpO+tBCKgG0LZ7ZOMpw16q1sPvhVZ1CVe+MVfI
--> ssh-ed25519 iHV63A 01XIo9M8Et2DkWSfO5sJzVu7S71EmJcRDht/u5XtRQQ
-iZjE46hyys1RW/ltjCmnXKcWfmoiurPoA3s1Ka0eIU0
--> ssh-ed25519 BVsyTA B97LazX9/2obX9KM4yhAdbrjkDCMJ/Bxv57NnBAAeD0
-SSSEbPjl+SCl8T8X884aLmqJRgnT8kvI/hXC4NVMTp8
--> ssh-ed25519 +3V2lQ EYVbEhF1NIpw/f0VmLgBc1tM56B+buSMGe/fY1ty000
-ZW6rEd+rMCgqG+Qc0xgFcThQcmfunIwIj+x/iwkeQ6s
---- JAH/pCzrYMNHoLFH1FlXxFZVFRCMmrtZOS2lh/pvmMs
-°òòyƒÇî´Ž*!jÞµ)Ë4fô\ñ
•(Úæq³yö’À.ÂGÌÑ»c•ß3
+bNxi7xzpuF6XMua//fX+UCJ0eaTR3Z85YuXR6zfhNoh1kkueYjHXlQ1/85X3AuZ6
+6hhPR1z01jbYm9yrUAq3i8yJj9SaBXpuRVSCDL5xi9tByDmwhLFMo2g8CtNK2z+Y
+EOrqMmZ14Q3RG93CT/rDdemPYycNtkTqdNhNq0fIpHM29UgjMjKGrq+6L4dCnMHC
+dBQ7xpxCgni6cdx1H0KVxo43X1oVjcSy3D3XmOQox7Dysom8s8bS+GQGnXnxDBHC
+yYzfvEr2JA8+kb6dufraD6f5zHzwww2P8FRhWj+7GxYMPZ0WCDZjlxF0yxxPkzBG
+71CMRwXJ6SbsStDi8WlIFUdv7gQA+0KbjrKf/VGUPPAzQrqnb3Q5TcHSFKl+Zo9K
+W3dTuFQVMLbASFJTPPu676Ulz1L5Aa9ZFSRr0y+ri8D5ceip0u3rqEc0VFa9NRoA
+EloBano50AqBVi8rba/BLK1vXFBY/YU98nUPBTb/abTR3q2J/RhwKzQwdTk3uCqi
+hTNZ3qK5P9jagfT/rX2oeOBXsWxv5RxZw+tHw2EyBfQ6uOsK4tQQs5/lMhu9xOGV
+sLmHSDCNnBswI6WCvnZdgJPZKTbDeXRTK3RziJLr6Nh3QqYFAe6NZfD/xQ3ARyC7
+1uHoRk2aLEckAy5N/VS42xTPoK7mfDhao5hNONxh6EY
+-> piv-p256 vRzPNw At7C/F8fSGgxpMZokvAphi2wDEgJ073XUXOprtxYV4iR
+3a/oxlKWvjRd1uxEJipyQV+/8VM23wVWWNYBm1GVn3k
+-> piv-p256 zqq/iw A/ubZ0Jc9mAiZ4q1DHczZyxrRqpMJGLsp7yAqznaogli
+vyJUM2Hg/bUf06Gp1sT1FWYMovqaChsfLtlKStNjK6k
+-> ssh-ed25519 YFSOsg tKDlPNLGGxJ5vn2JjLFzY++ZZ+8I3gGmCLZBXEYP7Aw
+0LrDm/4jIbZRL3GHmrKZovrM6mjacJwAAvw+niQCcC0
+-> ssh-ed25519 iHV63A zKALwEkOjLDZxbYxCX447LUgDT8XKCN63TSR0bz5Th0
+1QaWaGAQxiUtF5rCSvvVK7wNtObUjBx2b+hJ6srzGVo
+-> ssh-ed25519 BVsyTA htddHKuwj0GHraYeaz/G1wgmK1cVXc2rxOVEHJm5qDA
+ezhynFcRE/nWnu4F7DNOKzt+twXD8ruvsBevEZRHEI0
+-> ssh-ed25519 +3V2lQ F/VW/8lQRXjo5wCJfpaDfy67nRwKk7s7OuWROvHgQhk
+8oU5fQEeQ8QJxO0tN0LJpAk+GsyjMg7S6wJSE59m4mI
+--- AIuo0wtWpfspkSedhiuBzSwn4eCNY4PFmdzAdBVWE3s
+fp§‡Y¼ÙpswŸÀ<EFBFBD>à’"§](g¬mzr”ˆã!0œÜ2½µºJ`“1¢†ö<E280A0>ž

secrets/mail/admins.age

@@ -1,43 +1,44 @@
 age-encryption.org/v1
--> ssh-ed25519 UE5Ceg 6rewUSyj9mZOZp1Oi+DvWxj7u6r7HWUAnp/zSDLmZyA
-OLBPwlUCqlVZqrZaqT/sfzslgcYRViuTt9yzJZRPIPI
--> ssh-ed25519 uYcDNw JNpKkljIQIPKR/KNG9AF/DxbJjYoMeQdhOjmpig2Q3c
-bxu5hEvJi0ip74WUJNJhm6pAfdvVlFBbyCwQKYPkUXo
+-> ssh-ed25519 UE5Ceg gD5QRHbcUWSile/Qrcd8JzEd07QTQsgxD4NkyNZYYVI
+l2b1/Vue2S2ymlzCuHyHBDhSr3HQg4UMK12UCjTF6hw
+-> ssh-ed25519 uYcDNw os3fheZXnX06seK0rrB0jvVFULf7H+sxrymQHtkM+D4
++xzkMSJxAh7OJDEvznwCZXPi2tUAD0ejBV7qLM2KlV0
 -> ssh-rsa f5THog
-0Im1QWg1IHp5nYfo0OK908ohS+Mo0Jyyyimq3sc6q5WoDUzufaMVYfgVpHJxasO/
-SrVAwE6QLcHuTBZPeyr1HZ7chyQiWT+Lepp/MXhgS8nDOkgJaSNxY35PO6W/qtpE
-rxkgdNZdB2Orqq0wHo0is5+pfZdcD7n6O4VoiayUh6kv5Brk98BUCHrydXMfJv26
-0Kzwg3s+/kDwOeVOt7uy6n5VPhcSLiJgQlK4t0HkPB2rUoD8dfyVqUZV3YmgCoJM
-Km1lCxaS96xKGnvt0HklYy0OX5S7ActBGpQJjcNLTl7sb2M/U0XAF7O8teSKzdq4
-ejKOnzMdxFB+qOSZ3fGzHbjxNDwxPqyps0yhm72rT5tww3wOzYZXUebn7LwNKVwU
-99mA0CR9W3wg3Thv4nwmsrycTMFHh9jvGRXOYgIqXNDoo2oqqkzLnS+N2fx6Wush
-SNziOeZkgb25h0wrehxmqsEOVjlSE6C59E40XlmSj+MJf6siDLQGpLShE4Fz1tyx
-GXASxlTNcJ8TY0N4UmozdWRW8pyTOtl1MhiuaHdYLQGvd3Zlwkr9C7pV6eVBxPyF
-agSqbSZXprY5owp17fUc7HQUu5AcNJyQtDstwqOTPbaJFNfPnyaHU61jt52sk468
-W2d1hZ9SYxiN32rjYV6py2SiuOvHIWMz3ODkvhxQdAM
+sMUYZP1yspVxsI4zWydUKPyATbc/dHED1slRmQMYBCgiJbpXxrcfyU7JvwEiW2Nf
+KXGU1HxBdstuC78+1jgVnSqRG2FUol+gAZPvqJCO30SznefSWciCvBfC71MKUjx5
+00WRZjQT/oSVAha7o/5nRqfsYy3mEkCvYOAX0X+ajpbIRJlywB29JNf19Smr4XNH
+cYtiOMQlqYvHj+YAHIZPqvZN5Pt75zNRdHqr0b0s6GI9/SuCxwkwU5eeO/QYM94d
+QckXRdg56gZKyJ87QFRFCMXi2KAAwGjLNVhws7wvnh3G03ZCQ2rWGR8YZxtDYyee
+TXPBo1JNDlIrKyGOz3AfXSNq8UwZDodpRy7Vu0LQBQNCOicGYrWl2lQ8Mo5zfNp4
+SbJZCxHztvHqeBWC3EQnQx95dUfUiui/zTp7HkQZ0bLVPS2qrSJCTlceT7JX7cIy
+krgTQ1/qcFqt8VWSUFz5sKXwE2FnvvJ5QPGsRPkBs2GAKpMQXpyLkv173iPksBMb
+NU1slvMlPzfeg3xHuGCJYofVlhhcUMi/RTQKWmhwwzfLBXZfocINZqOKn72TnAb5
+yfJ2t3BEZkaY5rkGrUhnPFIteUczaavu+P76cqp79recEmFwq+hrtOMSsUF/t6eM
+R3uPzN+6gLHvc3IFL2WkKYT50cb1rOAK9FdZ5ilsVFc
 -> ssh-rsa kFDS0A
-TRrrVhtSIhhR9OXVAEwfmVn44a/LIaYJZWndqPAcAEhQp1Z3kPpolkxtKskz982G
-wQgSbzU3py4VRpXdy/FBttoEdBrhRMKG0z9N0szKlagfLA+DHQjTlaMn/UkxmO2S
-4AdwO8jEJVe26h6Y/3ne7N+/Ji8QKO6tKeNVapBKHYsJ8qqscgYW1WgKOAfJ3M6c
-6lyavfn2prTkM0xz6hMrywm1Is9ahM4vh39iLRAaVonFHmNJE+dAse8ijvKzjcYM
-KAiZtabdJkWwjD/3x513fU/o9DQCnBTHfM8KLb7DTPC9Ro1K//O7LjcG+WiaERSh
-0+dBZstMD7fQWEyJ/CgnRf54juZs2A7yBdrT9TcQtcgPKYk9QjFqHCmKB0R+TUaX
-nNh4h33i5V/8JfPRQTLz/YYFdG+kG5Hvucs9I2HN1n/vaHL9UIH3zC8BmkUd5fnR
-cnKXPjFCfrPPKg4DMT4gT5lIVtIBRx/IKxvjgR/8c8M9M3jk4SZSYHUlKtnzFOLq
-ycGJopWX7kBWGliEQ8jC+nKYOXpSYH+mbHOV54zplmNOZKMdLJ9ek23WoX5/BD7i
-arp4EtwYiD2LN3M1TG24gFW9VCY3Ofil6HAn5ySM9AMtIHwy/8srUBSCtdpWWGx+
-0fk+wGVu/5lCn51RPXl1L2YRloyx3giKvappuUcpho4
--> piv-p256 vRzPNw AjkP6Dy1dEQ58LVB01S/1stB6JMpl+q3EuqHQp6RCfH9
-cePnQF/DS9AJx0MJArNi/5b6tncv46lKpu/1SIb5X7Y
--> piv-p256 zqq/iw A7cNqXWWA3Zd4vccwwW/Wgfq5cCOjnIPq/Et0qpeQUMw
-p/e2OBgHoHA06WR4h3k1GK65u3qYH2YGPYQ10jz+pvQ
--> ssh-ed25519 YFSOsg +Tl7z0DL81uPhdBuEJG+9qnZ6eoAzyZfvJ5FtrtyRUE
-nfVzlc5NoSxHv+2tM3D444kH9fCjUEYD+7wE2h83qYk
--> ssh-ed25519 iHV63A FgYN6w2aRUPpBBp6lV8pqSyopRaWwzhkGXxncU83HVc
-PcNQ0P2ZGCnumKWuHVo0wwF3KCz13JadNkAHWgqIfbc
--> ssh-ed25519 BVsyTA X/VL2A5AlbG1m6uTqbYDJTJj0wVrYGx5w/geJTpgQR4
-zwlsYTehOA3oK92zFN2J+HhgaX1zYd3MP0vQ3W751Co
--> ssh-ed25519 +3V2lQ Hk8tcLh85helo+DXrRDhCHkDja+sEkM1CTz01s0SXDQ
-ftNhb63/JDulFgTukVu76XG2Dfcorbdt47EV6kqXw9g
---- 37wAuChTQKbjj/RCIh7ZRB2GOf2kT1we3D4bQKevM3A
-ñ(=žÙ>¤jIM¡ÿú ”ÑyÂA¼|à“áʯNè<4E>„…‰f‰1Dgí5èËÛÜ	¬á߈ßg<C39F>6ì²#>ßÐ
%UjÃX›ŠÈ@ÌÑðG*ªNó™äÀå\çJE
+mNMbBAtl8DqLVje6lOnsZv6C9ytsl/dJBoJj+qfM5mTo5vkhyGPzo4NaaZIPnMVs
+SIckvd8gUhaPC4D98oPANl/GgBHEYSjVVwYS99THMYurW0E/brG+Awy+wYchO8yU
+rN14o4uk+LkAuRhSpDs7ExA91H/BIQFwNbkubY5Hp1dFvEjn99vhTi9ntHcsOLgR
+PE1QlfW2uzBvm+gwJ4zFPWDe3XqIUb+hU0N9hCdtRF6LxR5FaMzWuF2Y06O3qlLf
+oamlPpB4jQAq4LXgm/GuW1csOmc+PPBEEc/V8DnLNqy9PAScd5LS/u0zL/D+NHCs
+i3CRwDce5yCp3S+mrhIwoVZW4doK6fiKvkoYCC7lZYvA++eEUmZ4Xuj6eK9H14Yi
+g3lF3k0NLH7xw5dUGB7faUOAtpVLaGRy4nyVHp9qOWgTvdxMCGjt+GbfSTnDQcvz
+S9+FfDIPVOAh0jMv1J6w1aREPSeEOikC/TDBwB3f7UwKiFSS8aDwqRraUwmqiLaq
+L/lYo2DK5vBf20C5iM6SwowrvDV47ZBUbLO/ulvKDXydG1C9whMJGsBnEo06hASL
+oZK4oRpzJapil350Z94q/mShQUkzxYe6GS5h5eV+jly1mvL++7CJYywsWfe3Z6O5
+Oy31UxjhpRhbW5iqOcvjALbCYEPiNst0zLqeFywZ+GE
+-> piv-p256 vRzPNw A+EBgtoEKicMn8YJpH3ZGwV0PVt9l+YW6fTjs3deN70x
+RCx4lJavJo3pZRvj661M91wbZY6XTAMuqex95J6A22g
+-> piv-p256 zqq/iw AhMeJNZ/JlBnn7+3scs1hU3NgQBsKDC9L6sK5i1IBPIx
+QDZFPPU/Y5t9WQkP5CfZtPkl9CidOyebsubwbav4PCQ
+-> ssh-ed25519 YFSOsg kbmALRNCorj3qcJYyV5X2CGk0PBjOI6ay3INT15dFAA
+pfKCr+INssY1gccTCbFvnaaXLMoXr6DsCqgz4UD0mDE
+-> ssh-ed25519 iHV63A 8jd9mJGDe6yCjPGDIOusCzCR95Y7wRla5QjaBE1ESCU
+vmjDhBr+lYTCu765o1FupE0/RbOGaB3X0wEbDyOfh3o
+-> ssh-ed25519 BVsyTA e8lbhf+RO8CDMrR6MmBYgyeYJMXNMJ5cO5GsdYosbh4
+RKGw/EqxnQXeLrdPtCh84zZSN6lu0FyFfNSF7Rllbxk
+-> ssh-ed25519 +3V2lQ f4is4PhlyDSkRwbMIW6nehwLD6feea0so9AXWECX0ys
+0wUtiufdA5FzCIqZrcUE1XKSNOA7YK/PNAdRMnbXaNk
+--- rHG5GfdaRsmhAQyX4M6fDFTwCWKfI4MXTgyFL4py7RY
+|#÷üðÅd/Q‡Ä™‹”gÍGÅÄ…Äû—kýôjF@:}¶‹©
+ˇÄ8:0ž"®ó“ì°Žß‹4ÿ¨PŒ›ª¶äJÑ?MxYhb—”Ï­•ê˜«#ç'tš

secrets/mail/bot.age

@@ -1,43 +1,43 @@
 age-encryption.org/v1
--> ssh-ed25519 UE5Ceg iKhPkRjtE/7UadHCdLoQR0/fe1LhVF9wSp5DQUw0hV8
-o8BmKJxLYcxml+hq7l57nWQ8xAQFrROcX/BDCpZW7YE
--> ssh-ed25519 uYcDNw It3n9bvJCC+H+r5VRrtjrga1S1TkhiHUTGL/ltQbk0c
-h/98devoPCP18pYqK7KcXaDspMzQMtvs5YxsoyodDes
+-> ssh-ed25519 UE5Ceg B2ypwlCArbfQlB9f6wCaNfoyigxlZLYPN+G1A0RzAEY
+phfpR0H12yHwNnAFqBs5Al+E0rykpX0JQsOSi21+9dg
+-> ssh-ed25519 uYcDNw 0nP78x89Ug9p2kybjnkl1LdHS1sFHs47Cgx7Y29PvjU
+WDQ0hyddpyWHHUV+xuSees1FAUDE9aq9L9kpt38cuR4
 -> ssh-rsa f5THog
-xVi9l7vg34PJaGhjOzOtPtoRMePzlvdYKjNnzCXLd0g6Y4JXQZMoKCeeWrO++rtY
-7/PDxJ0kJjJAEY7q2BnfV+87nmrGxFFerldDcEO9pP8/sN/u393WQpngb0tMNx6M
-cjhwv0Y9ygAb858G1NzvnALVZGmbUxX1JIsq8QDcoP3kz5JmonIKLM3b4LrO735I
-bfu3T+wTRebOHdC9SOhz6iuhyTnu/RmU9w22AKK/IL19z+11NJB2Xoejkfw0c6ZU
-cW25i3TdwmiJAZ+lCDJQyBXtLctDes1/e6HtOkXoJSKQA5QLfEtPeCMyBmE4y0pR
-z1DPiP0wMd37YR8dMXoYDRfo3EvsDJkNR0SDTZj86kio9e2sXA3OtIx8BLM0y01F
-0Vnh0FwpY9kclflboeY9w3Uq33/TCvy9aZ29XD+X7HGdqqiqxeo5rcAMXO9xAx3h
-2fIwdVyWYTnLt8TDOH9ZKDw8vausEITQM/D73AbVlLRKDnXTd+YTkYBgzU1rJtR0
-4FQK4PL2qkWYKEK7qDTp+Hrhc4vOnxURaLsdexTub/A/TXHhGAKPxpGBOcBbCjc5
-4mHSRQsDTbTNNE7bcDbkBiUcXAdlPgvEhfLmmBw8sho45M+krSeSd7V5CJ1NENhJ
-3SO92RqIuyGR48lmvsuN5js4uLS4ntoyQvnmIQIVSQI
+pzLQg7g9WXNhO2uoujkqlOFNTHjWBrWi6DCNlsJslkq+muOl2Jrnh82+781sc2ZB
+EayFfD0GN6w4qQ2/zknW01EKNNOV70qmx2C9jbKLT6XG7lzJGozIJMD78h8w4JAQ
+u0NTO+zSIzsrvdM3AXlcXvN9clvV3pdQ0ogKlxxAKf8dr7p4qy9dD1r5hvideLjd
+liIitoSZDzuVvlfsP9IAHIFavcLB8sHY+dTSQ0dBMZJQIDDCUSgqsCM48Wto+ePC
+YC+HSLUASKW8SAvWH2eyrGXtnFH20fTOrFiF3WkLvm5nd+H8rBiGQVZWzbyRR0cU
+vjZN/Iu+XefwVoMLEKtHA6NYOL0xuPgsbysU8DP45VjI3ULHX/f6qUJQbUfO82IF
+qSoPhVaWrCOMyc6NjSb5NvQ1ETR9lsvENmXXhQLQwVNozUM6fmXJt9Qf6E6u5xJa
+TJgK+gktDW+CWRSm8wJ3b+Cukx7qMkV+NGKyK2iDRZSmyc7bYgpTxjjYpAVz21UQ
+Djbc7uqsNf4EiwEwkxKK5zAMKgbVQqriy6RdBqjcgVXsPDmv+rbuDtZn7jKw4Btk
+3VganG2/CDzR5wxDdentOd6utHMKlZuTkJtwVh+gL+NO7AmbtWTpJAGJ+2tkyTpg
+Zg6gKsAYhH1hgoNuyWd5807L2BGaLZIN0zOyjT4OHQQ
 -> ssh-rsa kFDS0A
-EsW7RlBeeV69UwczFANtxqmz2Et2jpUL378UuMydlzRznbp/TJjrzCStMTOBEDyC
-SuADuvcvLf1WsVbf+rxRuFgte0YMiqUNlijN7tsOFg92odk8tHVwXEA71SW8/ZWh
-zFqUJ8pPFXPA6DEYMGmdNLV+tEx3YsUFCrTvhRIBGPCFbuYJj9Ta2xg0KK3uR5/l
-xziM5xxc7NtJGpW3dA/qFyneuY6gPm17PWav2l7gjAge/6FvLFzfev9TuF82iPgc
-RkCNgHZqClWLRO9b0af8FMGWIak6kr/mqao40net2azrFqMxmeQFLIKJSxa6Agz+
-UtlOND1COQwHrogQkHVuanBRRdUZzGk4QdW8MN49JPkvwvVPGS2XZrkE5m4k66Nu
-rfMtlcoSGSA+GIZXTDiDPLpfpYV/XDe4IoPTpLcivRNb8i75GwCT/5vD39Qmlyyc
-GHOX+v5JXh8WYpgvTEPDYE/oeKnsq27QT1wt8q0hKuHcRO4BcdPuiaSMnn0kjvLd
-o473b6cHE96F3cTKhXerLqeMFs1+DsJhrxYCmRikZot6Iz8H5GnqT82Me1by6cYt
-+GDcuVLIB0OzWfI9ibZB0ueMM8UfrLeGDq8hSF5M0rDCbFc6ZzQw8PgI97PNaDGg
-FdIMho7IXEQKXMV7ueZ2/PiQEA8vfBWRnxGKFRQLOTY
--> piv-p256 vRzPNw AjWew9VSba/AQKQ69l/4OhvZUT/bawt7AOSe4/LjanOI
-wHkZs8QQAOE69dq0d/2PAMgsi3xDBqEEvEFB7WKMC1Q
--> piv-p256 zqq/iw AkKV76ktPNKCS/KidRxBHdRQmtH3BNO2kbBz408ZJ+wu
-S8KdsoVZUgvW7E4mlVFpp7/wxBarAPTEBqsYoBXar+M
--> ssh-ed25519 YFSOsg SQt87e1+Lza1kqQl+AyqOu47+en8H2AbjCasMjDLfRE
-vBO3eKJPzagd9NdPmVG1SvO3x9rnf4H/8oddfCwpjLY
--> ssh-ed25519 iHV63A a1iFLv3FlMcfq6p8+dKlFB9cDPC8RFVc9DxtpNIXU3c
-eQW7PJ+eGgp2loZTMUf40D8V3LNAinBSXgxdlHEQq34
--> ssh-ed25519 BVsyTA KNSZgJezH8bUbpFOWiyBN9kPL6EvG/L7Yh9ZRGUJkzg
-Fb4oMWqk3OfdKFkLd8qq2wGvq9Fz1D4A9HmA5a412r8
--> ssh-ed25519 +3V2lQ z3vxaJYUXcqI6f6U85Oj0u6cqyarKTLidDHsURqaTh0
-HNC+nhMbrJOUUS5SAcqJDDjwhjvRxOibo7Xx911cyOg
---- 6hftMRn4kD/f/ixMq2T+VnXZwyfpcV7zxZ7PBAAcsDM
-Êü÷å5lŠk—9Ë¡zÉRÏÓ©õרMáFM.º}ÊD§¨%ŒXŽºlu]7íÞç‹"\¼û<C2BC>êœ(}-¼â.åÐÿÛê25§­‹>06ã
h'±^jˆK‹/5<16>Í
+VcGjofFOsYaE94MqcJGW2UZ99Bf5PIcFbYoWuMoyMlaRnOi145IpIemxwBY79UYx
+bnpgnUq0WjNEMhqgTIG1Us2/7C4es/hF3CTNDYItLiKqy1i/EWUCIiSGPIzoLASq
+tNkFcQCCOd6OhydzhlKm4EINmFfY/fvx1XmHJc1npdMfJt82mTU2ftxelOluEbmz
+wPdNxQCflswFNyEDoCT8FIIRw7RJvCu/wEc+6IH5oDupuZCT9LvD4QEPPSjVOjJo
+1w0BKOHBTHIe6/+DjS6oqb+Va6sy5urDA79/FZDSyO/4LtxoiB//K8xd0EUWu/D5
+e8P3aQVpM7efEDFNq00rNKNnwcwQwY+WnYQP4EZQOcj8auYTxI8YDqF4Lwl2Rlc8
+8+2oJyV2gYtybc0TEyoJIRJJKP3JcTLatAdZgXO39QP3ZQO4Z0ifYAWrRkmOkKRX
+8yaZnCSmumKkPZqNbB//YJyRQS4l/e0UoB0CYdU5ljXUeXc23a/0SiptioWCj0Fc
+dDzpFTQTz9M2d38kPnvC1FOlNolivbgeYJQsL8436LPB2R9EsI8x6CBqhoCYQPnH
+4/Be9NmJhNSFKM8gavQBw6tAuO+o3KUGQoAXO/0reXa4JlO6MMN4+wJ1hroBk6Qz
+LFJtI4dTG0eRiIprIw97azIbt5HRe3OZ+x2tf20Wsws
+-> piv-p256 vRzPNw AwMtcJbTUbO1P+z0l9mHhJGgsA/FNVatPnEYWvHmWjUS
+e8U3q1ZX4uJhjvn5rnkQmulOccEJI1VK5cvS+tan4qA
+-> piv-p256 zqq/iw AsZeadyx4Wyk9OswxgYnfgSbeT9NBaYN4bqAS1jXVKs5
+ahO6BcM6qxHnDYktXnnCuZlZwozprIH/1f6oMF7CrHg
+-> ssh-ed25519 YFSOsg vyX/xUJ85G2lXyGWYl1AfDmuFxEp7HyUlltzYvLFgXY
+7GAV7wgvCPEC9RQkEVYn6t8Ou6STZpXAL9q+8Vm2BFM
+-> ssh-ed25519 iHV63A 0fCn2yOmCXjfRR193mpXR87r4mwy8QjeNfA2n0F3ERA
+dUdVw7q5GCYS0CTyNczOj67MRcWTQfRUwGtEMsAe7tw
+-> ssh-ed25519 BVsyTA fUsiobRR4HsqaYl0hfZfE+++swOV3XkAgSmMsvk5qHU
+txp2qnf62KbrYcV+Yp80WYZRJpj6MhjO0523m9nvbm4
+-> ssh-ed25519 +3V2lQ OwLStSeY1+pJBZ9fXqsZdwjPfYMApi8HXgQMIEAgIXs
+X4eezCXShnn54qHgRJM1mo9FBtVK07XVLSHBn32pY9c
+--- WX7/Z0eRS9Fkwi6s2jSr76HHmWx5oWfCQDqRJj9ovsA
+˜ýò“èt¿KiÂUl>î€@Rt<>?Æy	M
*’W¬Âböÿ»
V}«žµ¤š(¸/Bu0³¬K—pjà+Ý›&žRÄ
Í$öE&µÂT{SEH84+HÇ7’

secrets/mail/crew.age

@@ -1,43 +1,44 @@
 age-encryption.org/v1
--> ssh-ed25519 UE5Ceg qBHHVskxlk6AOCGIusKKItMQVrJpjpyWXBfcmpx6Bn8
-RDGWdLn/D8h+dKixRk39zrMFuoaqjdbnUX+CiRq+TSA
--> ssh-ed25519 uYcDNw K4nqUOfxtA3GDpg32ndobWATCQBN2ylzD3wyLlnT2nQ
-hRPPtWcxI/paVmOHT3J5SS7Ov8+gvXDAqtceJFn7o+s
+-> ssh-ed25519 UE5Ceg VVagFKN67R8OKJNOthRBDK86mwsNhQfuptHfXmBQ/gE
++nodn6RvyxNrMSA/aLAzmYTJVO/YSo6GlcBn0VhWJnI
+-> ssh-ed25519 uYcDNw bVnHlcPbjGibV73krlHfvl4SWe/FnLI63G5mlAdtzXg
+FuswKHdEbwUeugeuSjjE/nu66qMl6EDT8nUsdihmGzc
 -> ssh-rsa f5THog
-n+B7fmdbS+uwPFyHhBCNAAuCsGh6nzA3Q1ttF7vtadi2yw6P940XKB9hXnCe1btz
-NBRvKkVtIzRqc/5xDTqbDJivIYzFu8StofWv4xRBFzpA3P9r1qQV1lHwxOCfrsdd
-296KHvqWVo4rdhkbd9Cye7cxndr2AWs0Gwn1uNvM1WQjTzUWzuKy6UsVztEcsB0J
-4avT6+S+yxpKkMIyLqlbis/VYe/CDpPJGnxeG2GN8POVQpSdyBCEL32qkj07wR17
-9rZFWU5WKfIr0XXJkhq+ewNdJzQKfWDFEhHrZYrg8LxKYsOWhydRBVEHkWVXnLin
-CSD1Cv4VNHnqCycJ1Dv2Lq2n7SHoGMLPyC1UPJudmpY1Z5XIvWOu5uxvv0674mdN
-WxOXgZpitwpgcmMC6K4mBZtqI8yqMP1Gijupoj4hFK7YGqKdn6+Q6ZFsttL97I00
-lU22H1kf/Rxh0ZxMPiT1JcTwAZdOHIuRG6xPhVIx1hNUOmdUpg3YZa8dMKeA3Yjz
-7YL7ZaYkwsIhMh6w+3xWUiYNkWfmGffRq0DfXIzTkKzapQtQJGLOpeot4wPkW51q
-fHoJ2MNvlB3Yo5AveAkIaJpofjFFZgy9XVPGH2XSAFRez3hixXkV2rWiM+GJAAnQ
-z45H8qWfGnRKSjgqEKVPDlfFEiG78Dtzjtl4oW1gfbY
+BO6DdPxmOSg0eY0ML+rTWOQE+TqrFPFZbeETY75xmP7Cp6q2lWr2ZhOvZEhViMOC
+4U6giC7M428mSp9CPlXQnpk4fHA/4wJlDdWuGlzbMLHPm6hhCPcryfp+JuPjbq5i
+SFxOnvUdnpw12nxCCsLXMQktPMhNsK/A9BFlnjLohfzZpcVxKxA/2mr3Je+3srJR
+IWlOLP17HoDP8UGJDQm/pfa3YdarU8ah9j0AMWOWXMrqLsjsnxdBvqf4Ci72tp2O
+Cp83TkUmHkF8gTKCPSuYVP6aAYkzCb4r/5yqizOdF7YVkv6t3YBarvRi7N6X7EHj
+08/0JiN7pHciyIMs3K++yqYg0J74//I1/TqK7JJqJ145STXhaD3RCDNaC8TUt7E3
+WWa2PEg4ZYS0DXsJhis6iP1abYjiadl89nOp2v2oZOR2PFBLMeFdIZz1yYAxh2bN
+VfI8mSbIZFp3A3Z3xqmmrdPWRB7VVl9e5IpwvysEjbLZ9e4V8q0PXwux7kPI08oZ
+0WfbpHgJ4XL21cBYE2UvSieQ3Pd+M1sgO30tLv6BT3P1z+Y4G2KhpRTlJUE9SlzB
+uW0hiFOnvEPIcNoPUVlnPTiWtEHReHnlEaDOYd3ltyBhA5pxK0z9Bm1oY+jWalVZ
+a2F2OnCBtyYjE/FMJgNN70f1tjxsfAimm6kQ3ca4vmI
 -> ssh-rsa kFDS0A
-bZc7lDzI0kG/lY1reQtVjggoWfLj9/zz+BxmbZfisxsEE18AkYGsk/Ki9ddXFxDW
-5EIbCHheFBvkq7eb5OKcTUf3AFTch2/8dY1hnmR6uPq1Zwgl4ATCpcQPY85+7bPb
-GBl0msNpRHuo6um895rL4omdv+DItmMdp3Lyf+CcFRvaXOpRnFmOqgatZ1bMePx4
-qJajnToar4YIEJBzc53oGWdAHfcmVrvEdOIUNoS3QoyCmusCkMNrSfqmvPfwqsWt
-g+pTrI3NqmTt3+L0EawcRLjRYb/qM/L9/nSFOnYOv3hLzWOhwSQU/gr1ZKMxYnaI
-GxqWzWg2dvkuHlRKVwwf8mNBrZlqQDV/ydOeyjJUKe48jM/PsIj8NVsqRhkgHrkH
-/lvQClYEBhrgHc9Wdxzy4KM3DPyKCQSYxBPnZpFVzuFBKML/cnYU84i7r4Gkb/z4
-Jxwy6jxRzjt+Sou6gTP9dIASaYfMKYnf4ijB3IZLNApkNMBd0qt5qptTCG0LylDX
-eTGGWjKQrC11znI/PWkSJQsKuBDHesL+QmjgJBhPdpl7Tk9ZaI/rJk2KYAjF6J9V
-add0KsLxAZbqlFo1CJO8HHysCRljXob0jYefmnDXO2x8xZvt3eSzVa8JsNLcMv5w
-4/tAdHBfH4mifA5mVdVbeRUDby54TdfIWGAZtyhgvYg
--> piv-p256 vRzPNw A/0edIuqR6hf5WE2qoSGqX18sbslgSxxgmDOc6wNqfQD
-GT94xHQpPOdNorZOaSi7EPdaqSSVjJNB2qaSYA6qZhY
--> piv-p256 zqq/iw A5bQxOBbSgsr6+TL8bgNWl287IF8Zvec6k9oAZPgIRt2
-z0ygD5ZRl3WZjfVA3Aku70mKddTZZ/W9rX2XOBJ9cco
--> ssh-ed25519 YFSOsg R487ufjbfae0x3wSAYH9d4Yz0dW/ze3wXxQI/DCFuWw
-klWo+lmfAMaZVo/gDz07/ht+szuA7YSpvDc0yEe0bgo
--> ssh-ed25519 iHV63A Ond1kPLFFFIC/lSpv6K1uobvXYFmw+yVwNUTN1HIUVw
-ElzaC1ho8F2X2jRZtmAdY9FUMiCs5XAEcFqEPTy6Ilc
--> ssh-ed25519 BVsyTA F9U4uSI1sNELggtM7/VwlYOlg+ghBg0xAQLux5Fmvw8
-4PY2p7QneYIuumlciTmEbR/DwBKVMXxsfRoSuSgfmR4
--> ssh-ed25519 +3V2lQ 6i+WKf5wToBT5vne7ACy51BTAZrzMHCyiQ4D65m5Ol0
-/kt6I4forttfn8SbZ/9K2mvZRh4Cbj+JqmlZ746Pqqw
---- ufN6THtH8xQ83XVERTJFwO8Ti0AJyflJwZtA8V2mba4
-g[Ä&‡໹î|þjG#¡—ßúíJÎ<0F>bƤT<C2A4>c@EˆÝÉë}Œñ><3E>m¥÷÷ÜÅÿc™D*ÙMèÛ,(Ï”Ò6Cê‚¥´ÞÓkÇ’°é=¸â”f
Q‹
+cRu4Fw3m7KA7FGrGkC+D0C92Pw3BPcInBXjSlzuAk+/K1zcCTywRwA9RK7O9GRjw
+wDPNcrsu5pQe186PNij4FRldykFBrPwvhqDgfx2lr0E0jZdxMQ//mi5yoKHr6Sex
+95lvFiQpt2+5GJn6PyMebJf+GBVt3gToeCiwwBoUrCva1pibuNXIxEk0E6MCGQ/O
+yJeCeu9Sae0qxNy95XH06WX4CFOYmbxZSBQKfccVjFFwDmmVdpFP8NnuQCB6I+CK
+Ou1RFenqIfj9k/2D6gbis7CE+lK0Ke6SmszQljMvO7pk/EILe1NNwWPw0lMFlG5B
+gJIxjQQW+7aNP6J6jT7R2HEH6NjQu+Bsi4WO7a5b0oIMIXQdeRr00EJG1AynFgtW
+7tzqj+WSBHsvymQt9JyA/ZwlUTn7SfNHIa2Yx0e1Ze15/GNfINXlJgqPqod3aJM6
+9jrmKofvluIFUwA3EXcMzzXVHLDOi6/tp40rq/CIFcH/A9nV6MPrOzRx7Z/Mn8sp
+kmwLcxf/Ps4pEuateFU0ZvZWfxVb4abBDGkHMh53gKTohCP09yFYNaviXnwIRI2S
+OUl6MQxoOM+17ClkX11R7u+q3UxNN1rGKBjXvmLeCw0RS1nknm8lv9AGs/SgaQWz
+ZYQgSs0t8PmvHdZYJotdC6AykkCo6OhWmLd7nzRq0ko
+-> piv-p256 vRzPNw AkURiU3pt8ZyKGLAroxSDAxxabHMWrUzfeze2oc3Cy9T
+l05K5As8wku+X93LG7vTNfHcAK2YeC6nqThSMuV4OSE
+-> piv-p256 zqq/iw A7DcyvRjyVa6cjkEsW70dE8lha6BQAJF+NuN/wRD9/YH
+j8gqPHnpo2ZCyWbsssBZUki7WmGwXUJwFoE6rQB/9qE
+-> ssh-ed25519 YFSOsg 6jKKEapBICrxIIP02TXy7Fn9v1711gUhlNE0C3KSTEs
+p+sGc/8aqd0kYZtLjeOF6QeJDDenALP8h3Gm2UE2eM8
+-> ssh-ed25519 iHV63A FrWXvXyUrX5KHEtRmm6z/upIOO7eq0gruqHkz8uxwSk
+oSe+vJO7xbEb3CiADcnWDk0CSfdsjN44HwZMhwuxpWU
+-> ssh-ed25519 BVsyTA QrTESB1FBYj4t35jhGIv5WXqlawYqPxJCoLJUtDMRgc
+Ghn5PWjiDI7SdjlcjpAfmm2NGE6zXcg2WGwAF9jHYYo
+-> ssh-ed25519 +3V2lQ fsmAPXFR3An/2V0RZ4mCXiiMmKGiwyeTYlsZvVY1cQE
+snqgajEMGwSxghHUXizm1Q2Ifwtxv1/eMqTgNGmaUNE
+--- m3Rt960K96VKVv+7+5F89I3ONU4q0TYT9GKwTbd/PHI
+Ò’
+#ã—…o½ƒþ`S|Â&ˆIÐòúhçEãè	^̦ƒñ:PÙC‡ïmôöÙ˜YP9·¶]n5:íê`¯½€'V]ïŒ@ôñ{ãÈôîë~ôCc‹«i