Merge pull request 'mediawiki: 1.42.4 -> 1.43.0' (#301) from mediawiki-1.43.0 into main

Reviewed-on: #301
Reviewed-by: b12f <b12f@noreply.git.pub.solar>

.editorconfig

@@ -20,41 +20,8 @@ indent_style = unset
 indent_size = unset
 
 [{.*,secrets}/**]
-end_of_line = unset
+end_of_line = false
-insert_final_newline = unset
+insert_final_newline = false
-trim_trailing_whitespace = unset
-charset = unset
-indent_style = unset
-indent_size = unset
-
-[*.rom]
-end_of_line = unset
-insert_final_newline = unset
-trim_trailing_whitespace = unset
-charset = unset
-indent_style = unset
-indent_size = unset
-
-[*.py]
-indent_size = 4
-
-[*.md]
-max_line_length = off
-trim_trailing_whitespace = false
-
-# Ignore diffs/patches
-[*.{diff,patch}]
-end_of_line = unset
-insert_final_newline = unset
-trim_trailing_whitespace = unset
-indent_size = unset
-charset = unset
-indent_style = unset
-indent_size = unset
-
-[{.*,secrets}/**]
-end_of_line = unset
-insert_final_newline = unset
 trim_trailing_whitespace = unset
 charset = unset
 indent_style = unset

.forgejo/workflows/check.yml

@@ -10,7 +10,7 @@ jobs:
 
       - name: Check formatting
         run: |
-          nix --accept-flake-config --access-tokens '' develop --command treefmt --fail-on-change
+          nix --accept-flake-config --access-tokens '' develop --command treefmt --ci
 
       - name: Run flake checks
         run: |
@@ -18,14 +18,7 @@ jobs:
           # Prevent cache garbage collection by creating GC roots
           mkdir -p /var/lib/gitea-runner/tankstelle/.local/state/nix/results
 
-          for target in $(nix flake show --json --all-systems | jq '
+          sed -i 's/virtualisation.cores .*/virtualisation.cores = 16;/' tests/keycloak.nix
-            .["nixosConfigurations"] |
+          sed -i 's/virtualisation.memorySize .*/virtualisation.memorySize = 16384;/' tests/keycloak.nix
-            to_entries[] |
+          # 1 eval-worker needs about 13GB of memory
-            .key
+          nix --accept-flake-config --access-tokens '' develop --command nix-fast-build --no-nom --skip-cached --systems "x86_64-linux" --max-jobs 10 --eval-workers 2 --out-link /var/lib/gitea-runner/tankstelle/.local/state/nix/results/nix-fast-build
-            ' | tr -d '"'
-          ); do
-            nix --print-build-logs --verbose --accept-flake-config --access-tokens '' \
-              build --out-link /var/lib/gitea-runner/tankstelle/.local/state/nix/results/"$target" ".#nixosConfigurations.${target}.config.system.build.toplevel"
-          done
-
-          nix --print-build-logs --verbose --accept-flake-config --access-tokens '' flake check

docs/administrative-access.md

@@ -28,18 +28,18 @@ People with admin access to the infrastructure are added to [`logins/admins.nix`
 SSH is not reachable from the open internet. Instead, SSH Port 22 is protected by a wireguard VPN network. Thus, to get root access on the servers, at least two pieces of information have to be added to the admins config:
 
 1. **SSH Public key**: self-explanatory. Add your public key to your user attrset under `sshPubKeys`.
-2. **Wireguard device**: each wireguard device has two parts: the public key and the IP addresses it should have in the wireguard network. The pub.solar wireguard network is spaced under `10.7.6.0/24` and `fd00:fae:fae:fae:fae::/80`. To add your device, it's best to choose a free number between 200 and 255 and use that in both the ipv4 and ipv6 ranges: `10.7.6.<ip-address>/32` `fd00:fae:fae:fae:fae:<ip-address>::/96`. For more information on how to generate keypairs, see [the NixOS Wireguard docs](https://nixos.wiki/wiki/WireGuard#Generate_keypair).
+2. **Wireguard device**: each wireguard device has two parts: the public key and the IP addresses it should have in the wireguard network. The pub.solar wireguard network uses the subnets `10.7.6.0/24` and `fd00:fae:fae:fae:fae::/80`. To add your device, it's best to choose a free number between 200 and 255 and use that in both the ipv4 and ipv6 ranges: `10.7.6.<ip-address>/32` `fd00:fae:fae:fae:fae:<ip-address>::/96`. For more information on how to generate keypairs, see [the NixOS Wireguard docs](https://nixos.wiki/wiki/WireGuard#Generate_keypair).
 
 One can access our hosts using this domain scheme:
 
 ```
-ssh barkeeper@<hostname>.wg.pub.solar
+ssh <unix-username>@<hostname>.wg.pub.solar
 ```
 
 So, for example for `nachtigall`:
 
 ```
-ssh barkeeper@nachtigall.wg.pub.solar
+ssh teutat3s@nachtigall.wg.pub.solar
 ```
 
 Example NixOS snippet for WireGuard client config
@@ -63,12 +63,6 @@ Example NixOS snippet for WireGuard client config
             #endpoint = "138.201.80.102:51820";
             persistentKeepalive = 15;
           }
-          { # flora-6.pub.solar
-            publicKey = "jtSR5G2P/nm9s8WrVc26Xc/SQLupRxyXE+5eIeqlsTU=";
-            allowedIPs = [ "10.7.6.2/32" "fd00:fae:fae:fae:fae:2::/96" ];
-            endpoint = "80.71.153.210:51820";
-            persistentKeepalive = 15;
-          }
           { # metronom.pub.solar
             publicKey = "zOSYGO7MfnOOUnzaTcWiKRQM0qqxR3JQrwx/gtEtHmo=";
             allowedIPs = [ "10.7.6.3/32" "fd00:fae:fae:fae:fae:3::/96" ];
@@ -85,6 +79,39 @@ Example NixOS snippet for WireGuard client config
             #endpoint = "80.244.242.5:51820";
             persistentKeepalive = 15;
           }
+          {
+            # trinkgenossin.pub.solar
+            publicKey = "QWgHovHxtqiQhnHLouSWiT6GIoQDmuvnThYL5c/rvU4=";
+            allowedIPs = [
+              "10.7.6.5/32"
+              "fd00:fae:fae:fae:fae:5::/96"
+            ];
+            #endpoint = "85.215.152.22:51820";
+            endpoint = "[2a01:239:35d:f500::1]:51820";
+            persistentKeepalive = 15;
+          }
+          {
+            # delite.pub.solar
+            publicKey = "ZT2qGWgMPwHRUOZmTQHWCRX4m14YwOsiszjsA5bpc2k=";
+            allowedIPs = [
+              "10.7.6.6/32"
+              "fd00:fae:fae:fae:fae:6::/96"
+            ];
+            #endpoint = "5.255.119.132:51820";
+            endpoint = "[2a04:52c0:124:9d8c::2]:51820";
+            persistentKeepalive = 15;
+          }
+          {
+            # blue-shell.pub.solar
+            publicKey = "bcrIpWrKc1M+Hq4ds3aN1lTaKE26f2rvXhd+93QrzR8=";
+            allowedIPs = [
+              "10.7.6.7/32"
+              "fd00:fae:fae:fae:fae:7::/96"
+            ];
+            #endpoint = "194.13.83.205:51820";
+            endpoint = "[2a03:4000:43:24e::1]:51820";
+            persistentKeepalive = 15;
+          }
         ];
       };
     };

docs/backups.md

@@ -0,0 +1,36 @@
+# Backups
+
+We use [Restic](https://restic.readthedocs.io/en/stable/) to create backups and push them to two repositories.
+Check `./modules/backups.nix` and `./hosts/nachtigall/backups.nix` for working examples.
+
+### Hetzner Storagebox
+
+- Uses SFTP for transfer of backups
+
+Adding a new host SSH public key to the storagebox:
+
+First, [SSH to nachtigall](./administrative-access.md#ssh-access), then become root and add the new SSH public key
+
+```
+sudo -i
+echo '<ssh-public-key>' | ssh -p23 u377325@u377325.your-storagebox.de install-ssh-key
+```
+
+[Link to Hetzner storagebox docs](https://docs.hetzner.com/robot/storage-box/backup-space-ssh-keys).
+
+### Garage S3 buckets
+
+- Uses S3 for transfer of backups
+- One bucket per host, e.g. `nachtigall-backups`, `metronom-backups`
+
+To start transfering backups from a new hosts, this is how to create a new bucket:
+
+First, [SSH to trinkgenossin](./administrative-access.md#ssh-access), then use the `garage` CLI to create a new key and bucket:
+
+```
+export GARAGE_RPC_SECRET=<secret-in-keepass>
+
+garage bucket create <hostname>-backups
+garage key create <hostname>-backups-key
+garage bucket allow <hostname>-backups --read --write --key <hostname>-backups-key
+```

docs/cachix.md

@@ -0,0 +1,55 @@
+# Cachix usage
+
+URL: https://pub-solar.cachix.org
+
+Requirements:
+
+- [Install cachix](https://docs.cachix.org/installation)
+- Optional: To push to the cache, you need to set `CACHIX_AUTH_TOKEN` in your environment. To generate one for you, follow the [Getting Started](https://docs.cachix.org/getting-started#authenticating) docs and login with your GitHub account.
+- Add our binary cache [to your nix config](https://docs.cachix.org/faq#cachix-use-effects). To add the pub-solar cache, run:
+
+```
+cachix use pub-solar
+```
+
+Example to build and push a custom package of a host in this flake (e.g. after creating an overlay):
+
+```
+nix build --json -f . '.#nixosConfigurations.nachtigall.pkgs.keycloak^*' \
+  | jq -r '.[].outputs | to_entries[].value' \
+  | cachix push pub-solar
+```
+
+Example to build and push a package in the `nixpkgs` repo:
+
+```
+cd nixpkgs
+nix build --json -f . 'pkgs.lix^*' \
+  | jq -r '.[].outputs | to_entries[].value' \
+  | cachix push pub-solar
+```
+
+Checking if a package has been correctly pushed to the cache:
+
+```
+❯ nix build --json '/nix/store/f76xi83z4xk9sn6pbh38rh97yvqhb5m0-noto-fonts-color-emoji-png-2.042.drv^*' | jq -r '.[].outputs | to_entries[].value'  | cachix push pub-solar
+Pushing 1 paths (0 are already present) using zstd to cache pub-solar ⏳
+
+✓ /nix/store/xpgpi84765dxqja3gd5pldj49xx2v0xl-noto-fonts-color-emoji-png-2.042 (10.30 MiB)
+
+All done.
+
+❯ curl -I https://pub-solar.cachix.org/xpgpi84765dxqja3gd5pldj49xx2v0xl.narinfo
+HTTP/2 200
+date: Mon, 26 Aug 2024 09:31:10 GMT
+content-type: text/x-nix-narinfo
+traceparent: 00-b99db37cc9c2581b8d226cdf81e54507-794fc49193659c03-01
+tracestate:
+cache-control: public, max-age=14400
+last-modified: Mon, 26 Aug 2024 09:31:10 GMT
+cf-cache-status: EXPIRED
+report-to: {"endpoints":[{"url":"https:\/\/a.nel.cloudflare.com\/report\/v4?s=A67KGsCIsYjoFdvndxJ0rkmb7BZ5ztIpm8WUJKAiUPRVWvbYeXU9gU27P7zryiUtArbwrLzHhhMija0yyXk0kwNa3suz8gNzKK6z1CX1FWDZiiP07rnq7zAg8nZbSBiEU%2FZrU9nSrR6mhuL9ihbmW1Hf"}],"group":"cf-nel","max_age":604800}
+nel: {"success_fraction":0,"report_to":"cf-nel","max_age":604800}
+server: cloudflare
+cf-ray: 8b92ceab0d19c80e-DUS
+```

docs/deletion-request.md

@@ -34,7 +34,13 @@ Docs: https://docs.nextcloud.com/server/latest/admin_manual/configuration_server
 ### Mastodon
 
 ```
+mkdir /tmp/tootctl
+sudo chown mastodon /tmp/tootctl
+cd /tmp/tootctl
+
 sudo -u mastodon mastodon-tootctl accounts delete --email <mail-address>
+
+rm -r /tmp/tootctl
 ```
 
 Docs: https://docs.joinmastodon.org/admin/tootctl/#accounts-delete
@@ -50,7 +56,7 @@ Docs: https://forgejo.org/docs/latest/admin/command-line/#delete
 ### Matrix
 
 ```
-curl --header "Authorization: Bearer <admin-access-token>" --request POST http://172.18.0.3:8008/_synapse/admin/v1/deactivate/@<username>:pub.solar --data '{"erase": true}'
+curl --header "Authorization: Bearer <admin-access-token>" --request POST http://127.0.0.1:8008/_synapse/admin/v1/deactivate/@<username>:pub.solar --data '{"erase": true}'
 ```
 
 Docs: https://matrix-org.github.io/synapse/latest/admin_api/user_admin_api.html#deactivate-account

docs/deploying.md

@@ -7,22 +7,29 @@ be manually deployed.
 To deploy, make sure you have a [working development shell](./development-shell.md).
 Then, run `deploy-rs` with the hostname of the server you want to deploy:
 
+### Dry-run
+
+Use `--dry-activate` to show a diff of updated packages and all services that
+would be restarted by the update. This will also put all files in place without
+switching to the new generation, enabling a quick switch to the new config at a
+later moment.
+
 For nachtigall.pub.solar:
 
 ```
-deploy --targets '.#nachtigall' --magic-rollback false --auto-rollback false --keep-result --result-path ./results
+deploy --targets '.#nachtigall' --ssh-user <unix-username> --magic-rollback false --auto-rollback false --keep-result --result-path ./results --dry-activate
 ```
 
-For flora-6.pub.solar:
+After reviewing the changes, apply the update with:
 
 ```
-deploy --targets '.#flora-6' --magic-rollback false --auto-rollback false --keep-result --result-path ./results
+deploy --targets '.#nachtigall' --ssh-user <unix-username> --magic-rollback false --auto-rollback false --keep-result --result-path ./results
 ```
 
 For metronom.pub.solar (aarch64-linux):
 
 ```
-deploy --targets '.#metronom' --magic-rollback false --auto-rollback false --keep-result --result-path ./results --remote-build
+deploy --targets '.#metronom' --ssh-user <unix-username> --magic-rollback false --auto-rollback false --keep-result --result-path ./results --remote-build
 ```
 
 Usually we skip all rollback functionality, but if you want to deploy a change
@@ -31,9 +38,6 @@ that might lock you out, e.g. to SSH, it might make sense to set these to `true`
 To skip flake checks, e.g. because you already ran them manually before
 deployment, add the flag `--skip-checks` at the end of the command.
 
-`--dry-activate` can be used to only put all files in place without switching,
-to enable switching to the new config quickly at a later moment.
-
 We use `--keep-result --result-path ./results` to keep the last `result`
 symlink of each `deploy` from being garbage collected. That way, we keep builds
 cached in the Nix store. This is optional and both flags can be removed if disk

docs/dns.md

@@ -1,18 +1,10 @@
 # Changing DNS entries
 
 Our current DNS provider is [namecheap](https://www.namecheap.com/).
-We use [Terraform](https://www.terraform.io) to declaratively manage our pub.solar DNS records.
+We use [OpenTofu](https://opentofu.org) to declaratively manage our pub.solar DNS records.
 
 ### Initial setup
 
-Skip this step if you already have a `triton` profile setup.
-
-```
-triton profile create
-```
-
-Please follow https://docs.greenbaum.cloud/en/devops/triton-cli.html for the details.
-
 You will need to setup the following [namecheap API credentials](https://www.namecheap.com/support/api/intro),
 look for "namecheap API key" in the pub.solar Keepass database.
 
@@ -28,13 +20,15 @@ You will probably also need to add your external IP to the [API allow list](http
 dig -4 ip @dns.toys
 ```
 
-Now, change into the terraform directory and initialize the terraform providers.
+Now, change into the terraform directory and initialize the terraform providers. To decrypt existing state,
+search for "terraform state passphrase" in the pub.solar Keepass database.
 
 ```
 cd terraform
-export TRITON_KEY_ID=$(cat ~/.config/triton/profiles.d/lev-1-pub_solar.json | jq --raw-output .keyId)
+export TF_VAR_state_passphrase=$(secret-tool lookup pub.solar terraform-state-passphrase-dns)
 
-terraform init
+alias tofu="terraform-backend-git --access-logs --tf tofu git terraform"
+tofu init
 ```
 
 Make your changes, e.g. in `dns.tf`.
@@ -46,20 +40,21 @@ $EDITOR dns.tf
 Plan your changes using:
 
 ```
-terraform plan -out pub-solar-infra.plan
+tofu plan -out pub-solar-infra.plan
 ```
 
 After verification, apply your changes with:
 
 ```
-terraform apply "pub-solar-infra.plan"
+tofu apply "pub-solar-infra.plan"
 ```
 
 ### Useful links
 
-We use the Manta remote backend to save the terraform state for collaboration.
+We use terraform-backend-git remote backend with opentofu state encryption for collaboration.
 
-- https://www.terraform.io/language/v1.2.x/settings/backends/manta
+- https://github.com/plumber-cd/terraform-backend-git
+- https://opentofu.org/docs/language/state/encryption
 
 Namecheap Terraform provider docs:
 

docs/drone-ci.md

@@ -1,19 +0,0 @@
-# Drone CI
-
-We currently use two CI systems, [drone CI](https://drone.io), reachable via
-https://ci.pub.solar and [Forgejo Actions](https://forgejo.org/docs/latest/user/actions/),
-which UI is integrated into https://git.pub.solar, for example
-https://git.pub.solar/pub-solar/infra/actions.
-
-### Signing the `.drone.yml` file
-
-Login to https://ci.pub.solar by clicking on the user icon in the bottom left.
-After logging in, you can view your personal API token by clicking on the same
-icon. If you're using the nix [development-shell](./development-shell.md), the
-`drone` command will already be installed.
-
-```
-export DRONE_TOKEN=<your-drone-api-token>
-
-drone --token $DRONE_TOKEN sign --save pub-solar/os
-```

docs/garage.md

@@ -0,0 +1,84 @@
+# Garage
+
+### How-To create a new bucket + keys
+
+Requirements:
+
+- `garage` RPC credentials, in the shared keepass, search for 'garage rpc secret'.
+- [Setup WireGuard](./administrative-access.md#ssh-access) for hosts: `trinkgenossin`, optionally: `delite`, `blue-shell`
+
+```
+ssh <unix-username>@trinkgenossin.wg.pub.solar
+```
+
+```
+# Add a few spaces to avoid leaking the secret to the shell history
+   export GARAGE_RPC_SECRET=<secret-in-keepass>
+```
+
+Now, you can run the following command to check the cluster status:
+
+```
+garage status
+```
+
+Command to list all existing buckets:
+
+```
+garage bucket list
+```
+
+Creating a new bucket and access keys:
+
+```
+garage bucket create <bucket-name>
+garage key create <bucket-name>-key
+garage bucket allow <bucket-name> --read --write --key <bucket-name>-key
+```
+
+Full example for `mastodon` bucket:
+
+```
+garage bucket create mastodon
+
+garage key create mastodon-key
+
+garage bucket allow mastodon --read --write --key mastodon-key
+```
+
+Then [setup your favourite S3 client](https://garagehq.deuxfleurs.fr/documentation/connect/cli/)
+or use the bucket with any [S3 compatible software](https://garagehq.deuxfleurs.fr/documentation/connect/).
+
+Further reading:
+
+- https://garagehq.deuxfleurs.fr/documentation/quick-start/
+- https://garagehq.deuxfleurs.fr/documentation/connect/
+- https://garagehq.deuxfleurs.fr/documentation/connect/apps/#mastodon
+
+### Notes on manual setup steps
+
+```
+ssh <unix-username>@trinkgenossin.wg.pub.solar
+
+# Add a few spaces to avoid leaking the secret to the shell history
+    export GARAGE_RPC_SECRET=<secret-in-keepass>
+
+# Uses the default config /etc/garage.toml
+garage node id
+
+garage node connect <node-id2>
+garage node connect <node-id3>
+
+garage status
+
+#Zones
+#DE-1 DE-2 NL-1
+
+garage layout assign fdaa -z DE-1 -c 800G -t trinkgenossin
+garage layout assign 8835 -z DE-2 -c 800G -t blue-shell
+garage layout assign 73da -z NL-1 -c 800G -t delite
+garage layout show
+garage layout apply --version 1
+```
+
+Source: https://garagehq.deuxfleurs.fr/documentation/cookbook/real-world/#creating-a-cluster-layout

docs/keycloak/delete-unverified-accounts.md

@@ -12,7 +12,7 @@ Run following after SSH'ing to `nachtigall`.
 Credentials for the following command are in keepass. Create a keycloak
 config/credentials file at `/tmp/kcadm.config`:
 
-```
+```bash
 sudo --user keycloak kcadm.sh config credentials \
   --config /tmp/kcadm.config \
   --server https://auth.pub.solar \
@@ -22,7 +22,7 @@ sudo --user keycloak kcadm.sh config credentials \
 
 Get list of accounts without a verified email address:
 
-```
+```bash
 sudo --user keycloak kcadm.sh get \
   --config /tmp/kcadm.config \
   users \
@@ -35,7 +35,7 @@ Review list of accounts, especially check `createdTimestamp` if any accounts
 were created in the past 2 days. If so, delete those from the
 `/tmp/keycloak-unverified-accounts` file.
 
-```
+```bash
 createdTimestamps=( $( nix run nixpkgs#jq -- -r '.[].createdTimestamp' < /tmp/keycloak-unverified-accounts ) )
 
 # timestamps are in nanoseconds since epoch, so we need to strip the last three digits
@@ -46,17 +46,17 @@ vim /tmp/keycloak-unverified-accounts
 
 Check how many accounts are going to be deleted:
 
-```
+```bash
 jq -r '.[].id' < /tmp/keycloak-unverified-accounts | wc -l
 ```
 
-```
+```bash
 jq -r '.[].id' < /tmp/keycloak-unverified-accounts > /tmp/keycloak-unverified-account-ids
 ```
 
 Final check before deletion (dry-run):
 
-```
+```bash
 for id in $(cat /tmp/keycloak-unverified-account-ids)
   do
     echo sudo --user keycloak kcadm.sh delete \
@@ -68,7 +68,7 @@ for id in $(cat /tmp/keycloak-unverified-account-ids)
 
 THIS WILL DELETE ACCOUNTS:
 
-```
+```bash
 for id in $(cat /tmp/keycloak-unverified-account-ids)
   do
     sudo --user keycloak kcadm.sh delete \
@@ -77,3 +77,9 @@ for id in $(cat /tmp/keycloak-unverified-account-ids)
       --realm pub.solar
   done
 ```
+
+Delete the temp files:
+
+```bash
+sudo rm /tmp/kcadm.config /tmp/keycloak-unverified-accounts /tmp/keycloak-unverified-account-ids
+```

docs/matrix-suspend-account.md

@@ -0,0 +1,27 @@
+# Matrix account suspension
+
+> Unlike [account locking](https://spec.matrix.org/v1.12/client-server-api/#account-locking),
+> [suspension](https://github.com/matrix-org/matrix-spec-proposals/blob/main/proposals/3823-code-for-account-suspension.md)
+> allows the user to have a (largely) readonly view of their account.
+> Homeserver administrators and moderators may use this functionality to
+> temporarily deactivate an account, or place conditions on the account's
+> experience. Critically, like locking, account suspension is reversible, unlike
+> the deactivation mechanism currently available in Matrix - a destructive,
+> irreversible, action.
+
+Required:
+
+- `matrix-synapse admin token`
+- [SSH access to host `nachtigall`](./administrative-access.md#ssh-access)
+
+## Suspending an account
+
+```bash
+curl --header "Authorization: Bearer <admin-access-token>" --request PUT http://127.0.0.1:8008/_synapse/admin/v1/suspend/@<username>:pub.solar --data '{"suspend": true}'
+```
+
+## Unsuspending an account
+
+```bash
+curl --header "Authorization: Bearer <admin-access-token>" --request PUT http://127.0.0.1:8008/_synapse/admin/v1/suspend/@<username>:pub.solar --data '{"suspend": false}'
+```

docs/mediawiki-updates.md

@@ -4,7 +4,7 @@ See the [mediawiki-oidc-docker repository](https://git.pub.solar/pub-solar/media
 for instructions on updating our customized mediawiki docker image.
 
 To deploy a new docker image to `nachtigall`, first bump the mediawiki version
-of the docker image tag in `hosts/nachtigall/apps/mediawiki.nix` (search for
+of the docker image tag in `modules/mediawiki/default.nix` (search for
 `image`).
 
 Next, push your changes to https://git.pub.solar and get them reviewed and
@@ -19,7 +19,7 @@ exit
 ```
 
 ```
-deploy --targets '.#nachtigall'
+deploy --targets '.#nachtigall' --magic-rollback false --auto-rollback false --keep-result --result-path ./results
 ```
 
 Then, finalize the update by running the database migration script (in a [SSH](./administrative-access.md#ssh-access) shell on `nachtigall`):

docs/nextcloud.md

@@ -0,0 +1,19 @@
+# Nextcloud debugging
+
+Set loglevel to `0` for debug logs:
+
+```nix
+services.nextcloud.settings.loglevel = 0;
+```
+
+Then, logs appear in the `phpfpm-nextcloud.service` logs:
+
+```bash
+sudo journalctl -fu phpfpm-nextcloud
+```
+
+Make sure to set the loglevel back to the default `2` warning after debugging:
+
+```nix
+services.nextcloud.settings.loglevel = 2;
+```

docs/nix-flake-updates.md

@@ -41,3 +41,7 @@ wrapped-ruby-mastodon-gems: 4.2.1 → 4.2.3
 zfs-kernel: 2.2.1-6.1.64 → 2.2.2-6.1.66
 zfs-user: 2.2.1 → 2.2.2
 ```
+
+### Deploying updates
+
+See [deploying.md](./deploying.md).

docs/nixos-anywhere.md

@@ -0,0 +1,13 @@
+```
+curl -L https://github.com/nix-community/nixos-images/releases/download/nixos-unstable/nixos-kexec-installer-noninteractive-x86_64-linux.tar.gz | tar -xzf- -C /root
+/root/kexec/run
+```
+
+```
+mkdir -p /etc/secrets/initrd
+ssh-keygen -t ed25519 -f /etc/secrets/initrd/ssh_host_ed25519_key
+```
+
+```
+nix run github:nix-community/nixos-anywhere -- --flake .#blue-shell root@194.13.83.205
+```

flake.lock

@@ -14,11 +14,11 @@
         "systems": "systems"
       },
       "locked": {
-        "lastModified": 1718371084,
+        "lastModified": 1736955230,
-        "narHash": "sha256-abpBi61mg0g+lFFU0zY4C6oP6fBwPzbHPKBGw676xsA=",
+        "narHash": "sha256-uenf8fv2eG5bKM8C/UvFaiJMZ4IpUFaQxk9OH5t/1gA=",
         "owner": "ryantm",
         "repo": "agenix",
-        "rev": "3a56735779db467538fb2e577eda28a9daacaca6",
+        "rev": "e600439ec4c273cf11e06fe4d9d906fb98fa097c",
         "type": "github"
       },
       "original": {
@@ -52,11 +52,11 @@
         "utils": "utils"
       },
       "locked": {
-        "lastModified": 1718194053,
+        "lastModified": 1727447169,
-        "narHash": "sha256-FaGrf7qwZ99ehPJCAwgvNY5sLCqQ3GDiE/6uLhxxwSY=",
+        "narHash": "sha256-3KyjMPUKHkiWhwR91J1YchF6zb6gvckCAY1jOE+ne0U=",
         "owner": "serokell",
         "repo": "deploy-rs",
-        "rev": "3867348fa92bc892eba5d9ddb2d7a97b9e127a8a",
+        "rev": "aa07eb05537d4cd025e2310397a6adcedfe72c76",
         "type": "github"
       },
       "original": {
@@ -87,6 +87,26 @@
         "type": "github"
       }
     },
+    "disko": {
+      "inputs": {
+        "nixpkgs": [
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1738765162,
+        "narHash": "sha256-3Z40qHaFScWUCVQrGc4Y+RdoPsh1R/wIh+AN4cTXP0I=",
+        "owner": "nix-community",
+        "repo": "disko",
+        "rev": "ff3568858c54bd306e9e1f2886f0f781df307dff",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "repo": "disko",
+        "type": "github"
+      }
+    },
     "element-stickers": {
       "inputs": {
         "maunium-stickerpicker": [
@@ -165,11 +185,11 @@
         "nixpkgs-lib": "nixpkgs-lib"
       },
       "locked": {
-        "lastModified": 1717285511,
+        "lastModified": 1738453229,
-        "narHash": "sha256-iKzJcpdXih14qYVcZ9QC9XuZYnPc6T8YImb6dX166kw=",
+        "narHash": "sha256-7H9XgNiGLKN1G1CgRh0vUL4AheZSYzPm+zmZ7vxbJdo=",
         "owner": "hercules-ci",
         "repo": "flake-parts",
-        "rev": "2a55567fcf15b1b1c7ed712a2c6fadaec7412ea8",
+        "rev": "32ea77a06711b758da0ad9bd6a844c5740a87abd",
         "type": "github"
       },
       "original": {
@@ -214,18 +234,19 @@
         "type": "github"
       }
     },
-    "flake-utils_3": {
+    "fork": {
       "locked": {
-        "lastModified": 1653893745,
+        "lastModified": 1738846146,
-        "narHash": "sha256-0jntwV3Z8//YwuOjzhV2sgJJPt+HY6KhU7VZUL0fKZQ=",
+        "narHash": "sha256-cIPiBEspPXQxju2AUZK9kjh6oqea+HkPFqmGv7yUztM=",
-        "owner": "numtide",
+        "owner": "teutat3s",
-        "repo": "flake-utils",
+        "repo": "nixpkgs",
-        "rev": "1ed9fb1935d260de5fe1c2f7ee0ebaae17ed2fa1",
+        "rev": "e370f40b129e47b08562524ab4f053a172a94273",
         "type": "github"
       },
       "original": {
-        "owner": "numtide",
+        "owner": "teutat3s",
-        "repo": "flake-utils",
+        "ref": "init-matrix-authentication-service-module-0.13.0",
+        "repo": "nixpkgs",
         "type": "github"
       }
     },
@@ -236,16 +257,16 @@
         ]
       },
       "locked": {
-        "lastModified": 1718530513,
+        "lastModified": 1736373539,
-        "narHash": "sha256-BmO8d0r+BVlwWtMLQEYnwmngqdXIuyFzMwvmTcLMee8=",
+        "narHash": "sha256-dinzAqCjenWDxuy+MqUQq0I4zUSfaCvN9rzuCmgMZJY=",
         "owner": "nix-community",
         "repo": "home-manager",
-        "rev": "a1fddf0967c33754271761d91a3d921772b30d0e",
+        "rev": "bd65bc3cde04c16755955630b344bc9e35272c56",
         "type": "github"
       },
       "original": {
         "owner": "nix-community",
-        "ref": "release-24.05",
+        "ref": "release-24.11",
         "repo": "home-manager",
         "type": "github"
       }
@@ -259,11 +280,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1707424749,
+        "lastModified": 1738012343,
-        "narHash": "sha256-eTvts5E3zmD4/DoAI9KedQjRwica0cg36wwIVp1NWbM=",
+        "narHash": "sha256-agMgWwVxXII+RtCqok8ROjzpKJung/5N5f2BVDmMC5Q=",
         "ref": "main",
-        "rev": "1202a23c205b3c07a5feb5caf6813f21b3c69307",
+        "rev": "4ffd7bc8ea032991756c5e8e8a37b039789045bc",
-        "revCount": 30,
+        "revCount": 38,
         "type": "git",
         "url": "https://git.pub.solar/pub-solar/keycloak-theme"
       },
@@ -277,11 +298,11 @@
       "flake": false,
       "locked": {
         "dir": "web",
-        "lastModified": 1718796561,
+        "lastModified": 1733177811,
-        "narHash": "sha256-RKAAHve17lrJokgAPkM2k/E+f9djencwwg3Xcd70Yfw=",
+        "narHash": "sha256-1n7bPSCRw7keTCIu4tJGnUlkoId6H1+dPsTPzKo3Rrk=",
         "owner": "maunium",
         "repo": "stickerpicker",
-        "rev": "333567f481e60443360aa7199d481e1a45b3a523",
+        "rev": "89d3aece041c85ebe5a1ad4e620388af5227cbb0",
         "type": "github"
       },
       "original": {
@@ -299,11 +320,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1719128254,
+        "lastModified": 1739034224,
-        "narHash": "sha256-I7jMpq0CAOZA/i70+HDQO/ulLttyQu/K70cSESiMX7A=",
+        "narHash": "sha256-Mj/8jDzh1KNmUhWqEeVlW3hO9MZkxqioJGnmR7rivaE=",
         "owner": "lnl7",
         "repo": "nix-darwin",
-        "rev": "50581970f37f06a4719001735828519925ef8310",
+        "rev": "0b6f96a6b9efcfa8d3cc8023008bcbcd1b9bc1a4",
         "type": "github"
       },
       "original": {
@@ -313,81 +334,49 @@
         "type": "github"
       }
     },
-    "nixos-flake": {
-      "locked": {
-        "lastModified": 1719437091,
-        "narHash": "sha256-UIZasVC36DS5dli1VimK0VgL6JKuxDG9cMxKq1I6OQ0=",
-        "owner": "srid",
-        "repo": "nixos-flake",
-        "rev": "8cefa1e7af06d366f5d3fd7c97e9edbf4d38c476",
-        "type": "github"
-      },
-      "original": {
-        "owner": "srid",
-        "repo": "nixos-flake",
-        "type": "github"
-      }
-    },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1719426051,
+        "lastModified": 1739055578,
-        "narHash": "sha256-yJL9VYQhaRM7xs0M867ZFxwaONB9T2Q4LnGo1WovuR4=",
+        "narHash": "sha256-2MhC2Bgd06uI1A0vkdNUyDYsMD0SLNGKtD8600mZ69A=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "89c49874fb15f4124bf71ca5f42a04f2ee5825fd",
+        "rev": "a45fa362d887f4d4a7157d95c28ca9ce2899b70e",
         "type": "github"
       },
       "original": {
         "owner": "nixos",
-        "ref": "nixos-24.05",
+        "ref": "nixos-24.11",
-        "repo": "nixpkgs",
-        "type": "github"
-      }
-    },
-    "nixpkgs-2205": {
-      "locked": {
-        "lastModified": 1685573264,
-        "narHash": "sha256-Zffu01pONhs/pqH07cjlF10NnMDLok8ix5Uk4rhOnZQ=",
-        "owner": "nixos",
-        "repo": "nixpkgs",
-        "rev": "380be19fbd2d9079f677978361792cb25e8a3635",
-        "type": "github"
-      },
-      "original": {
-        "owner": "nixos",
-        "ref": "nixos-22.05",
         "repo": "nixpkgs",
         "type": "github"
       }
     },
     "nixpkgs-lib": {
       "locked": {
-        "lastModified": 1717284937,
+        "lastModified": 1738452942,
-        "narHash": "sha256-lIbdfCsf8LMFloheeE6N31+BMIeixqyQWbSr2vk79EQ=",
+        "narHash": "sha256-vJzFZGaCpnmo7I6i416HaBLpC+hvcURh/BQwROcGIp8=",
         "type": "tarball",
-        "url": "https://github.com/NixOS/nixpkgs/archive/eb9ceca17df2ea50a250b6b27f7bf6ab0186f198.tar.gz"
+        "url": "https://github.com/NixOS/nixpkgs/archive/072a6db25e947df2f31aab9eccd0ab75d5b2da11.tar.gz"
       },
       "original": {
         "type": "tarball",
-        "url": "https://github.com/NixOS/nixpkgs/archive/eb9ceca17df2ea50a250b6b27f7bf6ab0186f198.tar.gz"
+        "url": "https://github.com/NixOS/nixpkgs/archive/072a6db25e947df2f31aab9eccd0ab75d5b2da11.tar.gz"
       }
     },
     "root": {
       "inputs": {
         "agenix": "agenix",
         "deploy-rs": "deploy-rs",
+        "disko": "disko",
         "element-stickers": "element-stickers",
         "element-themes": "element-themes",
         "flake-parts": "flake-parts",
+        "fork": "fork",
         "home-manager": "home-manager",
         "keycloak-theme-pub-solar": "keycloak-theme-pub-solar",
         "maunium-stickerpicker": "maunium-stickerpicker",
         "nix-darwin": "nix-darwin",
-        "nixos-flake": "nixos-flake",
         "nixpkgs": "nixpkgs",
-        "nixpkgs-2205": "nixpkgs-2205",
         "simple-nixos-mailserver": "simple-nixos-mailserver",
-        "triton-vmtools": "triton-vmtools",
         "unstable": "unstable"
       }
     },
@@ -398,22 +387,21 @@
         "nixpkgs": [
           "unstable"
         ],
-        "nixpkgs-24_05": [
+        "nixpkgs-24_11": [
           "nixpkgs"
-        ],
+        ]
-        "utils": "utils_2"
       },
       "locked": {
-        "lastModified": 1718084203,
+        "lastModified": 1734884447,
-        "narHash": "sha256-Cx1xoVfSMv1XDLgKg08CUd1EoTYWB45VmB9XIQzhmzI=",
+        "narHash": "sha256-HA9fAmGNGf0cOYrhgoa+B6BxNVqGAYXfLyx8zIS0ZBY=",
         "owner": "simple-nixos-mailserver",
         "repo": "nixos-mailserver",
-        "rev": "29916981e7b3b5782dc5085ad18490113f8ff63b",
+        "rev": "63209b1def2c9fc891ad271f474a3464a5833294",
         "type": "gitlab"
       },
       "original": {
         "owner": "simple-nixos-mailserver",
-        "ref": "nixos-24.05",
+        "ref": "nixos-24.11",
         "repo": "nixos-mailserver",
         "type": "gitlab"
       }
@@ -478,52 +466,13 @@
         "type": "github"
       }
     },
-    "systems_5": {
-      "locked": {
-        "lastModified": 1681028828,
-        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
-        "owner": "nix-systems",
-        "repo": "default",
-        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
-        "type": "github"
-      },
-      "original": {
-        "owner": "nix-systems",
-        "repo": "default",
-        "type": "github"
-      }
-    },
-    "triton-vmtools": {
-      "inputs": {
-        "flake-utils": "flake-utils_3",
-        "nixpkgs": [
-          "nixpkgs"
-        ]
-      },
-      "locked": {
-        "dir": "vmtools",
-        "lastModified": 1698443513,
-        "narHash": "sha256-wX2JIJ3JmJn6MAurdyjwZU+FZjLCwBArMrVSeeCb/ZU=",
-        "ref": "main",
-        "rev": "0d039dcf06afb8cbddd7ac54bae4d0d185f3e88e",
-        "revCount": 85,
-        "type": "git",
-        "url": "https://git.pub.solar/pub-solar/infra-vintage?dir=vmtools"
-      },
-      "original": {
-        "dir": "vmtools",
-        "ref": "main",
-        "type": "git",
-        "url": "https://git.pub.solar/pub-solar/infra-vintage?dir=vmtools"
-      }
-    },
     "unstable": {
       "locked": {
-        "lastModified": 1719254875,
+        "lastModified": 1739020877,
-        "narHash": "sha256-ECni+IkwXjusHsm9Sexdtq8weAq/yUyt1TWIemXt3Ko=",
+        "narHash": "sha256-mIvECo/NNdJJ/bXjNqIh8yeoSjVLAuDuTUzAo7dzs8Y=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "2893f56de08021cffd9b6b6dfc70fd9ccd51eb60",
+        "rev": "a79cfe0ebd24952b580b1cf08cd906354996d547",
         "type": "github"
       },
       "original": {
@@ -550,24 +499,6 @@
         "repo": "flake-utils",
         "type": "github"
       }
-    },
-    "utils_2": {
-      "inputs": {
-        "systems": "systems_5"
-      },
-      "locked": {
-        "lastModified": 1709126324,
-        "narHash": "sha256-q6EQdSeUZOG26WelxqkmR7kArjgWCdw5sfJVHPH/7j8=",
-        "owner": "numtide",
-        "repo": "flake-utils",
-        "rev": "d465f4819400de7c8d874d50b982301f28a84605",
-        "type": "github"
-      },
-      "original": {
-        "owner": "numtide",
-        "repo": "flake-utils",
-        "type": "github"
-      }
     }
   },
   "root": "root",

flake.nix

@@ -1,23 +1,24 @@
 {
   inputs = {
     # Track channels with commits tested and built by hydra
-    nixpkgs.url = "github:nixos/nixpkgs/nixos-24.05";
+    nixpkgs.url = "github:nixos/nixpkgs/nixos-24.11";
     unstable.url = "github:nixos/nixpkgs/nixos-unstable";
-
+    fork.url = "github:teutat3s/nixpkgs/init-matrix-authentication-service-module-0.13.0";
-    nixpkgs-2205.url = "github:nixos/nixpkgs/nixos-22.05";
 
     nix-darwin.url = "github:lnl7/nix-darwin/master";
     nix-darwin.inputs.nixpkgs.follows = "nixpkgs";
 
-    home-manager.url = "github:nix-community/home-manager/release-24.05";
+    home-manager.url = "github:nix-community/home-manager/release-24.11";
     home-manager.inputs.nixpkgs.follows = "nixpkgs";
 
     flake-parts.url = "github:hercules-ci/flake-parts";
-    nixos-flake.url = "github:srid/nixos-flake";
 
     deploy-rs.url = "github:serokell/deploy-rs";
     deploy-rs.inputs.nixpkgs.follows = "nixpkgs";
 
+    disko.url = "github:nix-community/disko";
+    disko.inputs.nixpkgs.follows = "nixpkgs";
+
     agenix.url = "github:ryantm/agenix";
     agenix.inputs.nixpkgs.follows = "nixpkgs";
     agenix.inputs.darwin.follows = "nix-darwin";
@@ -26,9 +27,6 @@
     keycloak-theme-pub-solar.url = "git+https://git.pub.solar/pub-solar/keycloak-theme?ref=main";
     keycloak-theme-pub-solar.inputs.nixpkgs.follows = "nixpkgs";
 
-    triton-vmtools.url = "git+https://git.pub.solar/pub-solar/infra-vintage?ref=main&dir=vmtools";
-    triton-vmtools.inputs.nixpkgs.follows = "nixpkgs";
-
     element-themes.url = "github:aaronraimist/element-themes/master";
     element-themes.flake = false;
 
@@ -39,8 +37,8 @@
     element-stickers.inputs.maunium-stickerpicker.follows = "maunium-stickerpicker";
     element-stickers.inputs.nixpkgs.follows = "nixpkgs";
 
-    simple-nixos-mailserver.url = "gitlab:simple-nixos-mailserver/nixos-mailserver/nixos-24.05";
+    simple-nixos-mailserver.url = "gitlab:simple-nixos-mailserver/nixos-mailserver/nixos-24.11";
-    simple-nixos-mailserver.inputs.nixpkgs-24_05.follows = "nixpkgs";
+    simple-nixos-mailserver.inputs.nixpkgs-24_11.follows = "nixpkgs";
     simple-nixos-mailserver.inputs.nixpkgs.follows = "unstable";
   };
 
@@ -53,7 +51,6 @@
       ];
 
       imports = [
-        inputs.nixos-flake.flakeModule
         ./logins
         ./lib
         ./overlays
@@ -65,6 +62,7 @@
           system,
           pkgs,
           config,
+          lib,
           ...
         }:
         {
@@ -75,12 +73,51 @@
               overlays = [ inputs.agenix.overlays.default ];
             };
             unstable = import inputs.unstable { inherit system; };
-            master = import inputs.master { inherit system; };
           };
+
+          checks =
+            let
+              machinesPerSystem = {
+                aarch64-linux = [
+                  "metronom"
+                ];
+                x86_64-linux = [
+                  "blue-shell"
+                  "delite"
+                  "nachtigall"
+                  "tankstelle"
+                  "trinkgenossin"
+                  "underground"
+                ];
+              };
+              nixosMachines = inputs.nixpkgs.lib.mapAttrs' (n: inputs.nixpkgs.lib.nameValuePair "nixos-${n}") (
+                inputs.nixpkgs.lib.genAttrs (machinesPerSystem.${system} or [ ]) (
+                  name: self.nixosConfigurations.${name}.config.system.build.toplevel
+                )
+              );
+              nixos-lib = import (inputs.nixpkgs + "/nixos/lib") { };
+              testDir = builtins.attrNames (builtins.readDir ./tests);
+              testFiles = builtins.filter (n: builtins.match "^.*.nix$" n != null) testDir;
+            in
+            builtins.listToAttrs (
+              map (x: {
+                name = "test-${lib.strings.removeSuffix ".nix" x}";
+                value = nixos-lib.runTest (
+                  import (./tests + "/${x}") {
+                    inherit self;
+                    inherit pkgs;
+                    inherit lib;
+                    inherit config;
+                  }
+                );
+              }) testFiles
+            )
+            // nixosMachines;
+
           devShells.default = pkgs.mkShell {
             buildInputs = with pkgs; [
               deploy-rs
-              nixpkgs-fmt
+              nix-fast-build
               agenix
               age-plugin-yubikey
               cachix
@@ -89,53 +126,55 @@
               nvfetcher
               shellcheck
               shfmt
-              treefmt
+              treefmt2
               nixos-generators
-              inputs.nixpkgs-2205.legacyPackages.${system}.terraform
+              opentofu
+              terraform-backend-git
+              terraform-ls
               jq
             ];
           };
+
           devShells.ci = pkgs.mkShell { buildInputs = with pkgs; [ nodejs ]; };
         };
 
-      flake =
+      flake = {
-        let
+        nixosModules = builtins.listToAttrs (
-          username = "barkeeper";
+          map (x: {
-        in
+            name = x;
-        {
+            value = import (./modules + "/${x}");
-          inherit username;
+          }) (builtins.attrNames (builtins.readDir ./modules))
+        );
 
-          nixosModules = builtins.listToAttrs (
+        checks = builtins.mapAttrs (
-            map (x: {
+          system: deployLib: deployLib.deployChecks self.deploy
-              name = x;
+        ) inputs.deploy-rs.lib;
-              value = import (./modules + "/${x}");
-            }) (builtins.attrNames (builtins.readDir ./modules))
-          );
 
-          checks = builtins.mapAttrs (
+        formatter."x86_64-linux" = inputs.nixpkgs.legacyPackages."x86_64-linux".nixfmt-rfc-style;
-            system: deployLib: deployLib.deployChecks self.deploy
-          ) inputs.deploy-rs.lib;
 
-          formatter."x86_64-linux" = inputs.unstable.legacyPackages."x86_64-linux".nixfmt-rfc-style;
+        deploy.nodes = self.lib.deploy.mkDeployNodes self.nixosConfigurations {
-
+          nachtigall = {
-          deploy.nodes = self.lib.deploy.mkDeployNodes self.nixosConfigurations {
+            hostname = "nachtigall.wg.pub.solar";
-            nachtigall = {
+          };
-              hostname = "nachtigall.wg.pub.solar";
+          metronom = {
-              sshUser = username;
+            hostname = "metronom.wg.pub.solar";
-            };
+          };
-            flora-6 = {
+          tankstelle = {
-              hostname = "flora-6.wg.pub.solar";
+            hostname = "tankstelle.wg.pub.solar";
-              sshUser = username;
+          };
-            };
+          underground = {
-            metronom = {
+            hostname = "80.244.242.3";
-              hostname = "metronom.wg.pub.solar";
+          };
-              sshUser = username;
+          trinkgenossin = {
-            };
+            hostname = "trinkgenossin.wg.pub.solar";
-            tankstelle = {
+          };
-              hostname = "tankstelle.wg.pub.solar";
+          delite = {
-              sshUser = username;
+            hostname = "delite.wg.pub.solar";
-            };
+          };
+          blue-shell = {
+            hostname = "blue-shell.wg.pub.solar";
           };
         };
+      };
     };
 }

hosts/blue-shell/configuration.nix

@@ -0,0 +1,33 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+{
+  boot.loader.grub.enable = true;
+
+  boot.kernelParams = [
+    "boot.shell_on_fail=1"
+    "ip=dhcp"
+  ];
+
+  # This option defines the first version of NixOS you have installed on this particular machine,
+  # and is used to maintain compatibility with application data (e.g. databases) created on older NixOS versions.
+  #
+  # Most users should NEVER change this value after the initial install, for any reason,
+  # even if you've upgraded your system to a new NixOS release.
+  #
+  # This value does NOT affect the Nixpkgs version your packages and OS are pulled from,
+  # so changing it will NOT upgrade your system - see https://nixos.org/manual/nixos/stable/#sec-upgrading for how
+  # to actually do that.
+  #
+  # This value being lower than the current NixOS release does NOT mean your system is
+  # out of date, out of support, or vulnerable.
+  #
+  # Do NOT change this value unless you have manually inspected all the changes it would make to your configuration,
+  # and migrated your data accordingly.
+  #
+  # For more information, see `man configuration.nix` or https://nixos.org/manual/nixos/stable/options#opt-system.stateVersion .
+  system.stateVersion = "24.05"; # Did you read the comment?
+}

hosts/blue-shell/default.nix

@@ -1,11 +1,13 @@
-{ ... }:
+{ flake, ... }:
 
 {
   imports = [
-    # Include the results of the hardware scan.
     ./hardware-configuration.nix
     ./configuration.nix
-    ./triton-vmtools.nix
+    ./disk-config.nix
+
+    ./networking.nix
     ./wireguard.nix
+    #./backups.nix
   ];
 }

hosts/blue-shell/disk-config.nix

@@ -0,0 +1,101 @@
+{
+  disko.devices = {
+    disk = {
+      main = {
+        type = "disk";
+        device = "/dev/vdb";
+        content = {
+          type = "gpt";
+          partitions = {
+            bios = {
+              size = "1M";
+              type = "EF02"; # for grub MBR
+            };
+            boot = {
+              size = "1G";
+              type = "8300";
+              content = {
+                type = "filesystem";
+                format = "ext4";
+                mountpoint = "/boot";
+                mountOptions = [ "defaults" ];
+              };
+            };
+            luks = {
+              size = "100%";
+              content = {
+                type = "luks";
+                name = "cryptroot";
+                extraOpenArgs = [ ];
+                # if you want to use the key for interactive login be sure there is no trailing newline
+                # for example use `echo -n "password" > /tmp/secret.key`
+                passwordFile = "/tmp/luks-password";
+                content = {
+                  type = "lvm_pv";
+                  vg = "vg0";
+                };
+              };
+            };
+          };
+        };
+      };
+      data = {
+        type = "disk";
+        device = "/dev/vdc";
+        content = {
+          type = "gpt";
+          partitions = {
+            luks = {
+              size = "100%";
+              content = {
+                type = "luks";
+                name = "cryptdata";
+                extraOpenArgs = [ ];
+                # if you want to use the key for interactive login be sure there is no trailing newline
+                # for example use `echo -n "password" > /tmp/secret.key`
+                passwordFile = "/tmp/luks-password";
+                content = {
+                  type = "filesystem";
+                  format = "xfs";
+                  mountpoint = "/var/lib/garage/data";
+                  mountOptions = [ "defaults" ];
+                };
+              };
+            };
+          };
+        };
+      };
+    };
+    lvm_vg = {
+      vg0 = {
+        type = "lvm_vg";
+        lvs = {
+          root = {
+            size = "100G";
+            content = {
+              type = "filesystem";
+              format = "ext4";
+              mountpoint = "/";
+              mountOptions = [ "defaults" ];
+            };
+          };
+          swap = {
+            size = "16G";
+            content = {
+              type = "swap";
+            };
+          };
+          metadata = {
+            size = "50G";
+            content = {
+              type = "filesystem";
+              format = "btrfs";
+              mountpoint = "/var/lib/garage/meta";
+              mountOptions = [ "defaults" ];
+            };
+          };
+        };
+      };
+    };
+  };
+}

hosts/blue-shell/hardware-configuration.nix

@@ -0,0 +1,27 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{
+  config,
+  lib,
+  pkgs,
+  modulesPath,
+  ...
+}:
+
+{
+  imports = [ (modulesPath + "/profiles/qemu-guest.nix") ];
+
+  boot.initrd.availableKernelModules = [
+    "ata_piix"
+    "uhci_hcd"
+    "virtio_pci"
+    "sr_mod"
+    "virtio_blk"
+  ];
+  boot.initrd.kernelModules = [ ];
+  boot.kernelModules = [ ];
+  boot.extraModulePackages = [ ];
+
+  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
+}

hosts/blue-shell/networking.nix

@@ -0,0 +1,26 @@
+{
+  config,
+  pkgs,
+  flake,
+  ...
+}:
+{
+  services.garage.settings.rpc_public_addr = "[2a03:4000:43:24e::1]:3901";
+
+  networking.hostName = "blue-shell";
+  networking.hostId = "00000005";
+
+  networking.useDHCP = false;
+  systemd.network.enable = true;
+  systemd.network.networks."10-wan" = {
+    matchConfig.Name = "ens3";
+    address = [
+      "194.13.83.205/22"
+      "2a03:4000:43:24e::1/64"
+    ];
+    gateway = [
+      "194.13.80.1"
+      "fe80::1"
+    ];
+  };
+}

hosts/blue-shell/wireguard.nix

@@ -0,0 +1,51 @@
+{
+  config,
+  pkgs,
+  flake,
+  ...
+}:
+let
+  wireguardIPv4 = "10.7.6.7";
+  wireguardIPv6 = "fd00:fae:fae:fae:fae:7::";
+in
+{
+  networking.firewall.allowedUDPPorts = [ 51820 ];
+
+  age.secrets.wg-private-key.file = "${flake.self}/secrets/blue-shell-wg-private-key.age";
+
+  networking.wireguard.interfaces = {
+    wg-ssh = {
+      listenPort = 51820;
+      mtu = 1300;
+      ips = [
+        "${wireguardIPv4}/32"
+        "${wireguardIPv6}/96"
+      ];
+      privateKeyFile = config.age.secrets.wg-private-key.path;
+      peers = flake.self.logins.wireguardDevices ++ [
+        {
+          # trinkgenossin.pub.solar
+          publicKey = "QWgHovHxtqiQhnHLouSWiT6GIoQDmuvnThYL5c/rvU4=";
+          allowedIPs = [
+            "10.7.6.5/32"
+            "fd00:fae:fae:fae:fae:5::/96"
+          ];
+          #endpoint = "85.215.152.22:51820";
+          endpoint = "[2a01:239:35d:f500::1]:51820";
+          persistentKeepalive = 15;
+        }
+      ];
+    };
+  };
+
+  services.openssh.listenAddresses = [
+    {
+      addr = wireguardIPv4;
+      port = 22;
+    }
+    {
+      addr = "[${wireguardIPv6}]";
+      port = 22;
+    }
+  ];
+}

hosts/default.nix

@@ -1,9 +1,35 @@
-{ self, ... }:
+{
+  self,
+  inputs,
+  config,
+  ...
+}:
 {
   flake = {
-    nixosConfigurations = {
+    nixosModules = {
-      nachtigall = self.nixos-flake.lib.mkLinuxSystem {
+      home-manager = {
         imports = [
+          inputs.home-manager.nixosModules.home-manager
+          {
+            home-manager.useGlobalPkgs = true;
+            home-manager.useUserPackages = true;
+            home-manager.extraSpecialArgs = {
+              flake = {
+                inherit self inputs config;
+              };
+            };
+          }
+        ];
+      };
+    };
+    nixosConfigurations = {
+      nachtigall = self.inputs.nixpkgs.lib.nixosSystem {
+        specialArgs = {
+          flake = {
+            inherit self inputs config;
+          };
+        };
+        modules = [
           self.inputs.agenix.nixosModules.default
           self.nixosModules.home-manager
           ./nachtigall
@@ -11,6 +37,7 @@
           self.nixosModules.unlock-zfs-on-boot
           self.nixosModules.core
           self.nixosModules.docker
+          self.nixosModules.backups
 
           self.nixosModules.nginx
           self.nixosModules.collabora
@@ -33,6 +60,7 @@
           self.nixosModules.promtail
           self.nixosModules.searx
           self.nixosModules.tmate
+          self.nixosModules.tt-rss
           self.nixosModules.obs-portal
           self.nixosModules.matrix
           self.nixosModules.matrix-irc
@@ -41,32 +69,20 @@
         ];
       };
 
-      flora-6 = self.nixos-flake.lib.mkLinuxSystem {
+      metronom = self.inputs.nixpkgs.lib.nixosSystem {
-        imports = [
+        specialArgs = {
-          self.inputs.agenix.nixosModules.default
+          flake = {
-          self.nixosModules.home-manager
+            inherit self inputs config;
-          ./flora-6
+          };
-          self.nixosModules.overlays
+        };
-          self.nixosModules.core
+        modules = [
-
-          self.nixosModules.keycloak
-          self.nixosModules.caddy
-          self.nixosModules.drone
-          self.nixosModules.forgejo-actions-runner
-          self.nixosModules.grafana
-          self.nixosModules.prometheus
-          self.nixosModules.loki
-        ];
-      };
-
-      metronom = self.nixos-flake.lib.mkLinuxSystem {
-        imports = [
           self.inputs.agenix.nixosModules.default
           self.nixosModules.home-manager
           ./metronom
           self.nixosModules.overlays
           self.nixosModules.unlock-zfs-on-boot
           self.nixosModules.core
+          self.nixosModules.backups
           self.nixosModules.mail
           self.nixosModules.prometheus-exporters
           self.nixosModules.promtail
@@ -75,17 +91,117 @@
         ];
       };
 
-      tankstelle = self.nixos-flake.lib.mkLinuxSystem {
+      tankstelle = self.inputs.nixpkgs.lib.nixosSystem {
-        imports = [
+        specialArgs = {
+          flake = {
+            inherit self inputs config;
+          };
+        };
+        modules = [
           self.inputs.agenix.nixosModules.default
           self.nixosModules.home-manager
           ./tankstelle
           self.nixosModules.overlays
           self.nixosModules.core
+          self.nixosModules.backups
           self.nixosModules.prometheus-exporters
           self.nixosModules.promtail
         ];
       };
+
+      trinkgenossin = self.inputs.nixpkgs.lib.nixosSystem {
+        specialArgs = {
+          flake = {
+            inherit self inputs config;
+          };
+        };
+        modules = [
+          self.inputs.agenix.nixosModules.default
+          self.nixosModules.home-manager
+          ./trinkgenossin
+          self.nixosModules.backups
+          self.nixosModules.overlays
+          self.nixosModules.unlock-luks-on-boot
+          self.nixosModules.core
+
+          self.nixosModules.garage
+          self.nixosModules.nginx
+
+          # This module is already using options, and those options are used by the grafana module
+          self.nixosModules.keycloak
+          self.nixosModules.grafana
+          self.nixosModules.prometheus
+          self.nixosModules.loki
+        ];
+      };
+
+      delite = self.inputs.nixpkgs.lib.nixosSystem {
+        specialArgs = {
+          flake = {
+            inherit self inputs config;
+          };
+        };
+        modules = [
+          self.inputs.agenix.nixosModules.default
+          self.inputs.disko.nixosModules.disko
+          self.nixosModules.home-manager
+          ./delite
+          self.nixosModules.overlays
+          self.nixosModules.unlock-luks-on-boot
+          self.nixosModules.core
+          self.nixosModules.prometheus-exporters
+          self.nixosModules.promtail
+
+          self.nixosModules.garage
+          self.nixosModules.nginx
+        ];
+      };
+
+      blue-shell = self.inputs.nixpkgs.lib.nixosSystem {
+        specialArgs = {
+          flake = {
+            inherit self inputs config;
+          };
+        };
+        modules = [
+          self.inputs.agenix.nixosModules.default
+          self.inputs.disko.nixosModules.disko
+          self.nixosModules.home-manager
+          ./blue-shell
+          self.nixosModules.overlays
+          self.nixosModules.unlock-luks-on-boot
+          self.nixosModules.core
+          self.nixosModules.prometheus-exporters
+          self.nixosModules.promtail
+
+          self.nixosModules.garage
+          self.nixosModules.nginx
+        ];
+      };
+
+      underground = self.inputs.nixpkgs.lib.nixosSystem {
+        specialArgs = {
+          flake = {
+            inherit self inputs config;
+          };
+        };
+        modules = [
+          self.inputs.agenix.nixosModules.default
+          self.nixosModules.home-manager
+          ./underground
+          self.nixosModules.overlays
+          self.nixosModules.unlock-luks-on-boot
+          self.nixosModules.core
+
+          self.nixosModules.backups
+          self.nixosModules.keycloak
+          self.nixosModules.postgresql
+          self.nixosModules.matrix
+          self.nixosModules.matrix-irc
+          self.nixosModules.nginx
+          self.nixosModules.nginx-matrix
+        ];
+      };
     };
   };
 }

hosts/delite/configuration.nix

@@ -0,0 +1,33 @@
+{
+  flake,
+  config,
+  pkgs,
+  ...
+}:
+{
+  boot.loader.grub.enable = true;
+
+  boot.kernelParams = [
+    "boot.shell_on_fail=1"
+    "ip=5.255.119.132::5.255.119.1:255.255.255.0:delite::off"
+  ];
+
+  # This option defines the first version of NixOS you have installed on this particular machine,
+  # and is used to maintain compatibility with application data (e.g. databases) created on older NixOS versions.
+  #
+  # Most users should NEVER change this value after the initial install, for any reason,
+  # even if you've upgraded your system to a new NixOS release.
+  #
+  # This value does NOT affect the Nixpkgs version your packages and OS are pulled from,
+  # so changing it will NOT upgrade your system - see https://nixos.org/manual/nixos/stable/#sec-upgrading for how
+  # to actually do that.
+  #
+  # This value being lower than the current NixOS release does NOT mean your system is
+  # out of date, out of support, or vulnerable.
+  #
+  # Do NOT change this value unless you have manually inspected all the changes it would make to your configuration,
+  # and migrated your data accordingly.
+  #
+  # For more information, see `man configuration.nix` or https://nixos.org/manual/nixos/stable/options#opt-system.stateVersion .
+  system.stateVersion = "24.05"; # Did you read the comment?
+}

hosts/delite/default.nix

@@ -0,0 +1,13 @@
+{ flake, ... }:
+
+{
+  imports = [
+    ./hardware-configuration.nix
+    ./configuration.nix
+    ./disk-config.nix
+
+    ./networking.nix
+    ./wireguard.nix
+    #./backups.nix
+  ];
+}

hosts/delite/disk-config.nix

@@ -0,0 +1,84 @@
+{
+  disko.devices = {
+    disk = {
+      main = {
+        type = "disk";
+        device = "/dev/vda";
+        content = {
+          type = "gpt";
+          partitions = {
+            bios = {
+              size = "1M";
+              type = "EF02"; # for grub MBR
+            };
+            boot = {
+              size = "1G";
+              type = "8300";
+              content = {
+                type = "filesystem";
+                format = "ext4";
+                mountpoint = "/boot";
+                mountOptions = [ "defaults" ];
+              };
+            };
+            luks = {
+              size = "100%";
+              content = {
+                type = "luks";
+                name = "cryptroot";
+                extraOpenArgs = [ ];
+                # if you want to use the key for interactive login be sure there is no trailing newline
+                # for example use `echo -n "password" > /tmp/secret.key`
+                passwordFile = "/tmp/luks-password";
+                content = {
+                  type = "lvm_pv";
+                  vg = "vg0";
+                };
+              };
+            };
+          };
+        };
+      };
+    };
+    lvm_vg = {
+      vg0 = {
+        type = "lvm_vg";
+        lvs = {
+          root = {
+            size = "40G";
+            content = {
+              type = "filesystem";
+              format = "ext4";
+              mountpoint = "/";
+              mountOptions = [ "defaults" ];
+            };
+          };
+          swap = {
+            size = "8G";
+            content = {
+              type = "swap";
+            };
+          };
+          data = {
+            size = "800G";
+            content = {
+              type = "filesystem";
+              format = "xfs";
+              mountpoint = "/var/lib/garage/data";
+              mountOptions = [ "defaults" ];
+            };
+          };
+          metadata = {
+            size = "50G";
+            content = {
+              type = "filesystem";
+              format = "btrfs";
+              mountpoint = "/var/lib/garage/meta";
+              mountOptions = [ "defaults" ];
+            };
+          };
+        };
+      };
+    };
+  };
+}

hosts/delite/hardware-configuration.nix

@@ -0,0 +1,26 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{
+  config,
+  lib,
+  pkgs,
+  modulesPath,
+  ...
+}:
+
+{
+  imports = [ (modulesPath + "/profiles/qemu-guest.nix") ];
+
+  boot.initrd.availableKernelModules = [
+    "ata_piix"
+    "uhci_hcd"
+    "virtio_pci"
+    "virtio_blk"
+  ];
+  boot.initrd.kernelModules = [ "dm-snapshot" ];
+  boot.kernelModules = [ ];
+  boot.extraModulePackages = [ ];
+
+  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
+}

hosts/delite/networking.nix

@@ -0,0 +1,26 @@
+{
+  config,
+  pkgs,
+  flake,
+  ...
+}:
+{
+  services.garage.settings.rpc_public_addr = "[2a04:52c0:124:9d8c::2]:3901";
+
+  networking.hostName = "delite";
+  networking.hostId = "00000004";
+
+  networking.useDHCP = false;
+  systemd.network.enable = true;
+  systemd.network.networks."10-wan" = {
+    matchConfig.Name = "ens3";
+    address = [
+      "5.255.119.132/24"
+      "2a04:52c0:124:9d8c::2/48"
+    ];
+    gateway = [
+      "5.255.119.1"
+      "2a04:52c0:124::1"
+    ];
+  };
+}

hosts/delite/wireguard.nix

@@ -0,0 +1,51 @@
+{
+  config,
+  pkgs,
+  flake,
+  ...
+}:
+let
+  wireguardIPv4 = "10.7.6.6";
+  wireguardIPv6 = "fd00:fae:fae:fae:fae:6::";
+in
+{
+  networking.firewall.allowedUDPPorts = [ 51820 ];
+
+  age.secrets.wg-private-key.file = "${flake.self}/secrets/delite-wg-private-key.age";
+
+  networking.wireguard.interfaces = {
+    wg-ssh = {
+      listenPort = 51820;
+      mtu = 1300;
+      ips = [
+        "${wireguardIPv4}/32"
+        "${wireguardIPv6}/96"
+      ];
+      privateKeyFile = config.age.secrets.wg-private-key.path;
+      peers = flake.self.logins.wireguardDevices ++ [
+        {
+          # trinkgenossin.pub.solar
+          publicKey = "QWgHovHxtqiQhnHLouSWiT6GIoQDmuvnThYL5c/rvU4=";
+          allowedIPs = [
+            "10.7.6.5/32"
+            "fd00:fae:fae:fae:fae:5::/96"
+          ];
+          #endpoint = "85.215.152.22:51820";
+          endpoint = "[2a01:239:35d:f500::1]:51820";
+          persistentKeepalive = 15;
+        }
+      ];
+    };
+  };
+
+  services.openssh.listenAddresses = [
+    {
+      addr = wireguardIPv4;
+      port = 22;
+    }
+    {
+      addr = "[${wireguardIPv6}]";
+      port = 22;
+    }
+  ];
+}

hosts/flora-6/configuration.nix

@@ -1,72 +0,0 @@
-{
-  config,
-  lib,
-  pkgs,
-  flake,
-  ...
-}:
-let
-  psCfg = config.pub-solar;
-in
-{
-  config = {
-    # Override nix.conf for more agressive garbage collection
-    nix.extraOptions = lib.mkForce ''
-      experimental-features = flakes nix-command
-      min-free = 536870912
-      keep-outputs = false
-      keep-derivations = false
-      fallback = true
-    '';
-
-    # # #
-    # # # Triton host specific options
-    # # # DO NOT ALTER below this line, changes might render system unbootable
-    # # #
-
-    # Use the systemd-boot EFI boot loader.
-    boot.loader.systemd-boot.enable = true;
-    boot.loader.efi.canTouchEfiVariables = true;
-
-    # Force getting the hostname from cloud-init
-    networking.hostName = lib.mkDefault "";
-
-    # We use cloud-init to configure networking, this option should fix
-    # systemd-networkd-wait-online timeouts
-    #systemd.services."systemd-networkd".environment.SYSTEMD_LOG_LEVEL = "debug";
-    systemd.network.wait-online.ignoredInterfaces = [
-      "docker0"
-      "wg-ssh"
-    ];
-
-    # List services that you want to enable:
-    services.cloud-init.enable = true;
-    services.cloud-init.ext4.enable = true;
-    services.cloud-init.network.enable = true;
-    # use the default NixOS cloud-init config, but add some SmartOS customization to it
-    environment.etc."cloud/cloud.cfg.d/90_smartos.cfg".text = ''
-      datasource_list: [ SmartOS ]
-
-      # Do not create the centos/ubuntu/debian user
-      users: [ ]
-
-      # mount second disk with label ephemeral0, gets formated by cloud-init
-      # this will fail to get added to /etc/fstab as it's read-only, but should
-      # mount at boot anyway
-      mounts:
-      - [ vdb, /data, auto, "defaults,nofail" ]
-    '';
-
-    # We manage the firewall with nix, too
-    # altough triton can also manage firewall rules via the triton fwrule subcommand
-    networking.firewall.enable = true;
-
-    # This value determines the NixOS release from which the default
-    # settings for stateful data, like file locations and database versions
-    # on your system were taken. It‘s perfectly fine and recommended to leave
-    # this value at the release version of the first install of this system.
-    # Before changing this value read the documentation for this option
-    # (e.g. man configuration.nix or on https://nixos.org/nixos/options.html).
-    system.stateVersion = "22.05"; # Did you read the comment?
-  };
-}

hosts/flora-6/triton-vmtools.nix

@@ -1,6 +0,0 @@
-{ pkgs, flake, ... }:
-{
-  environment.systemPackages = with pkgs; [
-    flake.inputs.triton-vmtools.packages.${pkgs.system}.default
-  ];
-}

hosts/metronom/backups.nix

@@ -1,13 +1,29 @@
-{ flake, ... }:
+{ config, flake, ... }:
 {
-  age.secrets."restic-repo-droppie" = {
+  age.secrets."restic-repo-storagebox-metronom" = {
-    file = "${flake.self}/secrets/restic-repo-droppie.age";
+    file = "${flake.self}/secrets/restic-repo-storagebox-metronom.age";
     mode = "400";
     owner = "root";
   };
-  age.secrets."restic-repo-storagebox" = {
+  age.secrets.restic-repo-garage-metronom = {
-    file = "${flake.self}/secrets/restic-repo-storagebox.age";
+    file = "${flake.self}/secrets/restic-repo-garage-metronom.age";
     mode = "400";
     owner = "root";
   };
+  age.secrets.restic-repo-garage-metronom-env = {
+    file = "${flake.self}/secrets/restic-repo-garage-metronom-env.age";
+    mode = "400";
+    owner = "root";
+  };
+
+  pub-solar-os.backups.repos.storagebox = {
+    passwordFile = config.age.secrets."restic-repo-storagebox-metronom".path;
+    repository = "sftp:u377325@u377325.your-storagebox.de:/metronom-backups";
+  };
+
+  pub-solar-os.backups.repos.garage = {
+    passwordFile = config.age.secrets."restic-repo-garage-metronom".path;
+    environmentFile = config.age.secrets."restic-repo-garage-metronom-env".path;
+    repository = "s3:https://buckets.pub.solar/metronom-backups";
+  };
 }

hosts/metronom/configuration.nix

@@ -23,6 +23,14 @@
     pools = [ "root_pool" ];
   };
 
+  # Declarative SSH private key
+  age.secrets."metronom-root-ssh-key" = {
+    file = "${flake.self}/secrets/metronom-root-ssh-key.age";
+    path = "/root/.ssh/id_ed25519";
+    mode = "400";
+    owner = "root";
+  };
+
   # Declarative SSH private key
   #age.secrets."metronom-root-ssh-key" = {
   #  file = "${flake.self}/secrets/metronom-root-ssh-key.age";

hosts/metronom/default.nix

@@ -7,6 +7,6 @@
 
     ./networking.nix
     ./wireguard.nix
-    #./backups.nix
+    ./backups.nix
   ];
 }

hosts/metronom/wireguard.nix

@@ -18,16 +18,7 @@
         "fd00:fae:fae:fae:fae:3::/96"
       ];
       privateKeyFile = config.age.secrets.wg-private-key.path;
-      peers = flake.self.logins.admins.wireguardDevices ++ [
+      peers = flake.self.logins.wireguardDevices ++ [
-        {
-          # flora-6.pub.solar
-          endpoint = "80.71.153.210:51820";
-          publicKey = "jtSR5G2P/nm9s8WrVc26Xc/SQLupRxyXE+5eIeqlsTU=";
-          allowedIPs = [
-            "10.7.6.2/32"
-            "fd00:fae:fae:fae:fae:2::/96"
-          ];
-        }
         {
           # nachtigall.pub.solar
           endpoint = "138.201.80.102:51820";
@@ -37,6 +28,17 @@
             "fd00:fae:fae:fae:fae:1::/96"
           ];
         }
+        {
+          # trinkgenossin.pub.solar
+          publicKey = "QWgHovHxtqiQhnHLouSWiT6GIoQDmuvnThYL5c/rvU4=";
+          allowedIPs = [
+            "10.7.6.5/32"
+            "fd00:fae:fae:fae:fae:5::/96"
+          ];
+          #endpoint = "85.215.152.22:51820";
+          endpoint = "[2a01:239:35d:f500::1]:51820";
+          persistentKeepalive = 15;
+        }
       ];
     };
   };

hosts/nachtigall/backups.nix

@@ -1,13 +1,34 @@
-{ flake, ... }:
+{ config, flake, ... }:
 {
   age.secrets."restic-repo-droppie" = {
     file = "${flake.self}/secrets/restic-repo-droppie.age";
     mode = "400";
     owner = "root";
   };
-  age.secrets."restic-repo-storagebox" = {
+  age.secrets."restic-repo-storagebox-nachtigall" = {
-    file = "${flake.self}/secrets/restic-repo-storagebox.age";
+    file = "${flake.self}/secrets/restic-repo-storagebox-nachtigall.age";
     mode = "400";
     owner = "root";
   };
+  age.secrets.restic-repo-garage-nachtigall = {
+    file = "${flake.self}/secrets/restic-repo-garage-nachtigall.age";
+    mode = "400";
+    owner = "root";
+  };
+  age.secrets.restic-repo-garage-nachtigall-env = {
+    file = "${flake.self}/secrets/restic-repo-garage-nachtigall-env.age";
+    mode = "400";
+    owner = "root";
+  };
+
+  pub-solar-os.backups.repos.storagebox = {
+    passwordFile = config.age.secrets."restic-repo-storagebox-nachtigall".path;
+    repository = "sftp:u377325@u377325.your-storagebox.de:/backups";
+  };
+
+  pub-solar-os.backups.repos.garage = {
+    passwordFile = config.age.secrets."restic-repo-garage-nachtigall".path;
+    environmentFile = config.age.secrets."restic-repo-garage-nachtigall-env".path;
+    repository = "s3:https://buckets.pub.solar/nachtigall-backups";
+  };
 }

hosts/nachtigall/configuration.nix

@@ -48,9 +48,79 @@
     owner = "root";
   };
 
-  pub-solar-os.auth.enable = true;
+  # keycloak
+  age.secrets.keycloak-database-password = {
+    file = "${flake.self}/secrets/keycloak-database-password.age";
+    mode = "600";
+    #owner = "keycloak";
+  };
 
-  nixpkgs.config.permittedInsecurePackages = [ "keycloak-23.0.6" ];
+  pub-solar-os.auth = {
+    enable = true;
+    database-password-file = config.age.secrets.keycloak-database-password.path;
+  };
+
+  # matrix-synapse
+  age.secrets."matrix-synapse-signing-key" = {
+    file = "${flake.self}/secrets/matrix-synapse-signing-key.age";
+    mode = "400";
+    owner = "matrix-synapse";
+  };
+
+  age.secrets."matrix-synapse-secret-config.yaml" = {
+    file = "${flake.self}/secrets/matrix-synapse-secret-config.yaml.age";
+    mode = "400";
+    owner = "matrix-synapse";
+  };
+
+  age.secrets."matrix-authentication-service-secret-config.yml" = {
+    file = "${flake.self}/secrets/matrix-authentication-service-secret-config.yml.age";
+    mode = "400";
+    owner = "matrix-authentication-service";
+  };
+
+  # matrix-appservice-irc
+  age.secrets."matrix-appservice-irc-mediaproxy-signing-key" = {
+    file = "${flake.self}/secrets/matrix-appservice-irc-mediaproxy-signing-key.jwk.age";
+    mode = "400";
+    owner = "matrix-appservice-irc";
+  };
+
+  pub-solar-os.matrix = {
+    enable = true;
+    appservice-irc.mediaproxy.signingKeyPath =
+      config.age.secrets."matrix-appservice-irc-mediaproxy-signing-key".path;
+    synapse = {
+      signing_key_path = config.age.secrets."matrix-synapse-signing-key".path;
+      extra-config-files = [
+        config.age.secrets."matrix-synapse-secret-config.yaml".path
+
+        # The registration file is automatically generated after starting the
+        # appservice for the first time.
+        # cp /var/lib/mautrix-telegram/telegram-registration.yaml \
+        #   /var/lib/matrix-synapse/
+        # chown matrix-synapse:matrix-synapse \
+        #   /var/lib/matrix-synapse/telegram-registration.yaml
+        "/var/lib/matrix-synapse/telegram-registration.yaml"
+      ];
+      app-service-config-files = [
+        "/var/lib/matrix-synapse/telegram-registration.yaml"
+        "/var/lib/matrix-appservice-irc/registration.yml"
+        # "/matrix-appservice-slack-registration.yaml"
+        # "/hookshot-registration.yml"
+        # "/matrix-mautrix-signal-registration.yaml"
+        # "/matrix-mautrix-telegram-registration.yaml"
+      ];
+    };
+    matrix-authentication-service.extra-config-files = [
+      config.age.secrets."matrix-authentication-service-secret-config.yml".path
+    ];
+  };
+
+  systemd.services.postgresql = {
+    after = [ "var-lib-postgresql.mount" ];
+    requisite = [ "var-lib-postgresql.mount" ];
+  };
 
   # This value determines the NixOS release with which your system is to be
   # compatible, in order to avoid breaking some software such as database

hosts/nachtigall/default.nix

@@ -9,5 +9,10 @@
     ./networking.nix
     ./wireguard.nix
     ./backups.nix
+    "${flake.inputs.fork}/nixos/modules/services/matrix/matrix-authentication-service.nix"
+  ];
+
+  disabledModules = [
+    "services/matrix/matrix-authentication-service.nix"
   ];
 }

hosts/nachtigall/wireguard.nix

@@ -18,16 +18,7 @@
         "fd00:fae:fae:fae:fae:1::/96"
       ];
       privateKeyFile = config.age.secrets.wg-private-key.path;
-      peers = flake.self.logins.admins.wireguardDevices ++ [
+      peers = flake.self.logins.wireguardDevices ++ [
-        {
-          # flora-6.pub.solar
-          endpoint = "80.71.153.210:51820";
-          publicKey = "jtSR5G2P/nm9s8WrVc26Xc/SQLupRxyXE+5eIeqlsTU=";
-          allowedIPs = [
-            "10.7.6.2/32"
-            "fd00:fae:fae:fae:fae:2::/96"
-          ];
-        }
         {
           # tankstelle.pub.solar
           endpoint = "80.244.242.5:51820";
@@ -37,6 +28,17 @@
             "fd00:fae:fae:fae:fae:4::/96"
           ];
         }
+        {
+          # trinkgenossin.pub.solar
+          publicKey = "QWgHovHxtqiQhnHLouSWiT6GIoQDmuvnThYL5c/rvU4=";
+          allowedIPs = [
+            "10.7.6.5/32"
+            "fd00:fae:fae:fae:fae:5::/96"
+          ];
+          #endpoint = "85.215.152.22:51820";
+          endpoint = "[2a01:239:35d:f500::1]:51820";
+          persistentKeepalive = 15;
+        }
       ];
     };
   };

hosts/tankstelle/backups.nix

@@ -5,8 +5,8 @@
     mode = "400";
     owner = "root";
   };
-  age.secrets."restic-repo-storagebox" = {
+  age.secrets."restic-repo-storagebox-tankstelle" = {
-    file = "${flake.self}/secrets/restic-repo-storagebox.age";
+    file = "${flake.self}/secrets/restic-repo-storagebox-tankstelle.age";
     mode = "400";
     owner = "root";
   };

hosts/tankstelle/configuration.nix

@@ -10,6 +10,9 @@
   boot.loader.systemd-boot.enable = true;
   boot.loader.efi.canTouchEfiVariables = true;
 
+  # kernel same-page merging
+  hardware.ksm.enable = true;
+
   boot.binfmt.emulatedSystems = [ "aarch64-linux" ];
 
   system.stateVersion = "23.11";

hosts/tankstelle/wireguard.nix

@@ -18,7 +18,7 @@
         "fd00:fae:fae:fae:fae:4::/96"
       ];
       privateKeyFile = config.age.secrets.wg-private-key.path;
-      peers = flake.self.logins.admins.wireguardDevices ++ [
+      peers = flake.self.logins.wireguardDevices ++ [
         {
           # nachtigall.pub.solar
           endpoint = "138.201.80.102:51820";
@@ -29,13 +29,15 @@
           ];
         }
         {
-          # flora-6.pub.solar
+          # trinkgenossin.pub.solar
-          endpoint = "80.71.153.210:51820";
+          publicKey = "QWgHovHxtqiQhnHLouSWiT6GIoQDmuvnThYL5c/rvU4=";
-          publicKey = "jtSR5G2P/nm9s8WrVc26Xc/SQLupRxyXE+5eIeqlsTU=";
           allowedIPs = [
-            "10.7.6.2/32"
+            "10.7.6.5/32"
-            "fd00:fae:fae:fae:fae:2::/96"
+            "fd00:fae:fae:fae:fae:5::/96"
           ];
+          #endpoint = "85.215.152.22:51820";
+          endpoint = "[2a01:239:35d:f500::1]:51820";
+          persistentKeepalive = 15;
         }
       ];
     };

hosts/trinkgenossin/configuration.nix

@@ -0,0 +1,35 @@
+{
+  flake,
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+{
+  boot.loader.grub.enable = true;
+  boot.loader.grub.devices = [ "/dev/vda" ];
+
+  boot.kernelParams = [
+    "boot.shell_on_fail=1"
+    "ip=dhcp"
+  ];
+
+  # This option defines the first version of NixOS you have installed on this particular machine,
+  # and is used to maintain compatibility with application data (e.g. databases) created on older NixOS versions.
+  #
+  # Most users should NEVER change this value after the initial install, for any reason,
+  # even if you've upgraded your system to a new NixOS release.
+  #
+  # This value does NOT affect the Nixpkgs version your packages and OS are pulled from,
+  # so changing it will NOT upgrade your system - see https://nixos.org/manual/nixos/stable/#sec-upgrading for how
+  # to actually do that.
+  #
+  # This value being lower than the current NixOS release does NOT mean your system is
+  # out of date, out of support, or vulnerable.
+  #
+  # Do NOT change this value unless you have manually inspected all the changes it would make to your configuration,
+  # and migrated your data accordingly.
+  #
+  # For more information, see `man configuration.nix` or https://nixos.org/manual/nixos/stable/options#opt-system.stateVersion .
+  system.stateVersion = "24.05"; # Did you read the comment?
+}

hosts/trinkgenossin/default.nix

@@ -0,0 +1,12 @@
+{ flake, ... }:
+
+{
+  imports = [
+    ./hardware-configuration.nix
+    ./configuration.nix
+
+    ./networking.nix
+    ./wireguard.nix
+    #./backups.nix
+  ];
+}

hosts/trinkgenossin/hardware-configuration.nix

@@ -8,45 +8,47 @@
   modulesPath,
   ...
 }:
+
 {
   imports = [ ];
 
   boot.initrd.availableKernelModules = [
-    "ahci"
+    "ata_piix"
+    "uhci_hcd"
     "virtio_pci"
-    "xhci_pci"
     "sr_mod"
     "virtio_blk"
+    "virtio_net"
   ];
-  boot.initrd.kernelModules = [ ];
+  boot.initrd.kernelModules = [ "dm-snapshot" ];
-  boot.kernelModules = [ ];
+  boot.kernelModules = [ "kvm-amd" ];
   boot.extraModulePackages = [ ];
 
+  boot.initrd.luks.devices."cryptroot" = {
+    device = "/dev/disk/by-uuid/52a1fd17-63d7-4d0a-b7ff-74aceaf6085a";
+  };
+
   fileSystems."/" = {
     device = "/dev/disk/by-label/nixos";
-    autoResize = true;
     fsType = "ext4";
   };
 
   fileSystems."/boot" = {
     device = "/dev/disk/by-label/boot";
-    fsType = "vfat";
-  };
-
-  fileSystems."/data" = {
-    device = "/dev/disk/by-label/ephemeral0";
     fsType = "ext4";
-    options = [
-      "defaults"
-      "nofail"
-    ];
   };
 
-  swapDevices = [ ];
+  fileSystems."/var/lib/garage/data" = {
+    device = "/dev/disk/by-label/data";
+    fsType = "xfs";
+  };
 
-  networking.useDHCP = lib.mkDefault false;
+  fileSystems."/var/lib/garage/meta" = {
-  networking.networkmanager.enable = lib.mkForce false;
+    device = "/dev/disk/by-label/metadata";
+    fsType = "btrfs";
+  };
+
+  swapDevices = [ { device = "/dev/disk/by-label/swap"; } ];
 
   nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
-  hardware.cpu.intel.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
 }

hosts/trinkgenossin/networking.nix

@@ -0,0 +1,15 @@
+{
+  config,
+  pkgs,
+  flake,
+  ...
+}:
+{
+  services.garage.settings.rpc_public_addr = "[2a01:239:35d:f500::1]:3901";
+
+  networking.hostName = "trinkgenossin";
+  networking.hostId = "00000003";
+
+  networking.enableIPv6 = true;
+  networking.useDHCP = true;
+}

hosts/trinkgenossin/wireguard.nix

@@ -4,21 +4,25 @@
   flake,
   ...
 }:
+let
+  wireguardIPv4 = "10.7.6.5";
+  wireguardIPv6 = "fd00:fae:fae:fae:fae:5::";
+in
 {
   networking.firewall.allowedUDPPorts = [ 51820 ];
 
-  age.secrets.wg-private-key.file = "${flake.self}/secrets/flora6-wg-private-key.age";
+  age.secrets.wg-private-key.file = "${flake.self}/secrets/trinkgenossin-wg-private-key.age";
 
   networking.wireguard.interfaces = {
     wg-ssh = {
       listenPort = 51820;
       mtu = 1300;
       ips = [
-        "10.7.6.2/32"
+        "${wireguardIPv4}/32"
-        "fd00:fae:fae:fae:fae:2::/96"
+        "${wireguardIPv6}/96"
       ];
       privateKeyFile = config.age.secrets.wg-private-key.path;
-      peers = flake.self.logins.admins.wireguardDevices ++ [
+      peers = flake.self.logins.wireguardDevices ++ [
         {
           # nachtigall.pub.solar
           endpoint = "138.201.80.102:51820";
@@ -47,17 +51,35 @@
             "fd00:fae:fae:fae:fae:4::/96"
           ];
         }
+        {
+          # delite.pub.solar
+          endpoint = "5.255.119.132:51820";
+          publicKey = "ZT2qGWgMPwHRUOZmTQHWCRX4m14YwOsiszjsA5bpc2k=";
+          allowedIPs = [
+            "10.7.6.6/32"
+            "fd00:fae:fae:fae:fae:6::/96"
+          ];
+        }
+        {
+          # blue-shell.pub.solar
+          endpoint = "194.13.83.205:51820";
+          publicKey = "bcrIpWrKc1M+Hq4ds3aN1lTaKE26f2rvXhd+93QrzR8=";
+          allowedIPs = [
+            "10.7.6.7/32"
+            "fd00:fae:fae:fae:fae:7::/96"
+          ];
+        }
       ];
     };
   };
 
   services.openssh.listenAddresses = [
     {
-      addr = "10.7.6.2";
+      addr = wireguardIPv4;
       port = 22;
     }
     {
-      addr = "[fd00:fae:fae:fae:fae:2::]";
+      addr = "[${wireguardIPv6}]";
       port = 22;
     }
   ];

hosts/underground/configuration.nix

@@ -0,0 +1,81 @@
+{
+  flake,
+  config,
+  pkgs,
+  ...
+}:
+{
+  # Use GRUB2 as the boot loader.
+  boot.loader.grub = {
+    enable = true;
+    devices = [ "/dev/vda" ];
+  };
+
+  pub-solar-os.networking.domain = "test.pub.solar";
+
+  systemd.tmpfiles.rules = [ "f /tmp/dbf 1777 root root 10d password" ];
+
+  # keycloak
+  pub-solar-os.auth = {
+    enable = true;
+    database-password-file = "/tmp/dbf";
+  };
+  services.keycloak.database.createLocally = true;
+
+  # matrix-synapse
+  # test.pub.solar /.well-known is required for federation
+  services.nginx.virtualHosts."${config.pub-solar-os.networking.domain}" = {
+    default = true;
+    enableACME = true;
+    forceSSL = true;
+  };
+
+  age.secrets."staging-matrix-synapse-secret-config.yaml" = {
+    file = "${flake.self}/secrets/staging-matrix-synapse-secret-config.yaml.age";
+    mode = "400";
+    owner = "matrix-synapse";
+  };
+
+  age.secrets."staging-matrix-authentication-service-secret-config.yml" = {
+    file = "${flake.self}/secrets/staging-matrix-authentication-service-secret-config.yml.age";
+    mode = "400";
+    owner = "matrix-authentication-service";
+  };
+
+  # matrix-appservice-irc
+  age.secrets."matrix-appservice-irc-mediaproxy-signing-key" = {
+    file = "${flake.self}/secrets/staging-matrix-appservice-irc-mediaproxy-signing-key.jwk.age";
+    mode = "400";
+    owner = "matrix-appservice-irc";
+  };
+
+  pub-solar-os.matrix = {
+    enable = true;
+    appservice-irc.mediaproxy.signingKeyPath =
+      config.age.secrets."matrix-appservice-irc-mediaproxy-signing-key".path;
+    synapse = {
+      extra-config-files = [
+        config.age.secrets."staging-matrix-synapse-secret-config.yaml".path
+
+        # The registration file is automatically generated after starting the
+        # appservice for the first time.
+        # cp /var/lib/mautrix-telegram/telegram-registration.yaml \
+        #   /var/lib/matrix-synapse/
+        # chown matrix-synapse:matrix-synapse \
+        #   /var/lib/matrix-synapse/telegram-registration.yaml
+        #"/var/lib/matrix-synapse/telegram-registration.yaml"
+      ];
+      app-service-config-files = [
+        "/var/lib/matrix-appservice-irc/registration.yml"
+        #"/var/lib/matrix-synapse/telegram-registration.yaml"
+      ];
+    };
+    matrix-authentication-service.extra-config-files = [
+      config.age.secrets."staging-matrix-authentication-service-secret-config.yml".path
+    ];
+  };
+
+  services.openssh.openFirewall = true;
+
+  system.stateVersion = "24.05";
+}

hosts/underground/default.nix

@@ -0,0 +1,16 @@
+{ flake, ... }:
+
+{
+  imports = [
+    # Include the results of the hardware scan.
+    ./hardware-configuration.nix
+    ./configuration.nix
+
+    ./networking.nix
+    "${flake.inputs.fork}/nixos/modules/services/matrix/matrix-authentication-service.nix"
+  ];
+
+  disabledModules = [
+    "services/matrix/matrix-authentication-service.nix"
+  ];
+}

hosts/underground/hardware-configuration.nix

@@ -0,0 +1,47 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{
+  config,
+  lib,
+  pkgs,
+  modulesPath,
+  ...
+}:
+
+{
+  imports = [
+    (modulesPath + "/profiles/qemu-guest.nix")
+  ];
+
+  boot.initrd.availableKernelModules = [
+    "ahci"
+    "xhci_pci"
+    "virtio_pci"
+    "virtio_scsi"
+    "sd_mod"
+    "sr_mod"
+  ];
+  boot.kernelModules = [ "kvm-intel" ];
+  boot.extraModulePackages = [ ];
+
+  boot.initrd.luks.devices."cryptroot" = {
+    device = "/dev/disk/by-label/cryptroot";
+  };
+
+  fileSystems."/" = {
+    device = "/dev/disk/by-label/root";
+    fsType = "ext4";
+  };
+
+  fileSystems."/boot" = {
+    device = "/dev/disk/by-label/boot";
+    fsType = "ext4";
+  };
+
+  swapDevices = [
+    { device = "/dev/disk/by-label/swap"; }
+  ];
+
+  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
+}

hosts/underground/networking.nix

@@ -0,0 +1,30 @@
+{
+  config,
+  pkgs,
+  flake,
+  ...
+}:
+{
+
+  networking.hostName = "underground";
+
+  networking = {
+    defaultGateway = {
+      address = "80.244.242.1";
+      interface = "enp1s0";
+    };
+    nameservers = [
+      "95.129.51.51"
+      "80.244.244.244"
+    ];
+    interfaces.enp1s0 = {
+      useDHCP = false;
+      ipv4.addresses = [
+        {
+          address = "80.244.242.3";
+          prefixLength = 29;
+        }
+      ];
+    };
+  };
+}

logins/admins.nix

@@ -38,6 +38,22 @@
           "fd00:fae:fae:fae:fae:200::/96"
         ];
       }
+      {
+        # chocolatebar
+        publicKey = "AS9w0zDUFLcH6IiF6T1vsyZPWPJ3p5fKsjIsM2AoZz8=";
+        allowedIPs = [
+          "10.7.6.205/32"
+          "fd00:fae:fae:fae:fae:205::/96"
+        ];
+      }
+      {
+        # biolimo
+        publicKey = "gnLq6KikFVVGxLxPW+3ZnreokEKLDoso+cUepPOZsBA=";
+        allowedIPs = [
+          "10.7.6.206/32"
+          "fd00:fae:fae:fae:fae:206::/96"
+        ];
+      }
     ];
   };
 
@@ -63,6 +79,7 @@
   teutat3s = {
     sshPubKeys = {
       teutat3s-1 = "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFro/k4Mgqyh8yV/7Zwjc0dv60ZM7bROBU9JNd99P/4co6fxPt1pJiU/pEz2Dax/HODxgcO+jFZfvPEuLMCeAl0= YubiKey #10593996 PIV Slot 9a";
+      teutat3s-2 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHcU6KPy4b1MQXd6EJhcYwbJu7E+0IrBZF/IP6T7gbMf teutat3s@dumpyourvms";
     };
 
     secretEncryptionKeys = {

logins/default.nix

@@ -6,19 +6,16 @@ in
 {
   flake = {
     logins = {
-      admins =
+      admins = admins;
-        lib.lists.foldl
+      wireguardDevices = lib.lists.foldl (
-          (logins: adminConfig: {
+        wireguardDevices: adminConfig:
-            sshPubKeys = logins.sshPubKeys ++ (lib.attrsets.attrValues adminConfig.sshPubKeys);
+        wireguardDevices ++ (if adminConfig ? "wireguardDevices" then adminConfig.wireguardDevices else [ ])
-            wireguardDevices =
+      ) [ ] (lib.attrsets.attrValues admins);
-              logins.wireguardDevices
+      sshPubKeys = lib.lists.foldl (
-              ++ (if adminConfig ? "wireguardDevices" then adminConfig.wireguardDevices else [ ]);
+        sshPubKeys: adminConfig:
-          })
+        sshPubKeys
-          {
+        ++ (if adminConfig ? "sshPubKeys" then lib.attrsets.attrValues adminConfig.sshPubKeys else [ ])
-            sshPubKeys = [ ];
+      ) [ ] (lib.attrsets.attrValues admins);
-            wireguardDevices = [ ];
-          }
-          (lib.attrsets.attrValues admins);
       robots.sshPubKeys = lib.attrsets.attrValues robots;
     };
   };

logins/robots.nix

@@ -1,7 +1,8 @@
 {
   # Used for restic backups to droppie, a server run by @b12f
-  "root@droppie" = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBZQSephFJU0NMbVbhwvVJ2/m6jcPYo1IsWCsoarqKin root@droppie";
+  "root@droppie" =
+    "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBZQSephFJU0NMbVbhwvVJ2/m6jcPYo1IsWCsoarqKin root@droppie";
 
-  # robot user on flora-6
+  "hakkonaut" =
-  "hakkonaut@flora-6" = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGP5MvCwNRtCcP1pSDrn0XZTNlpOqYnjHDm9/OI4hECW hakkonaut@flora-6";
+    "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGP5MvCwNRtCcP1pSDrn0XZTNlpOqYnjHDm9/OI4hECW hakkonaut";
 }

modules/backups/default.nix

@@ -0,0 +1,292 @@
+{
+  flake,
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  utils = import "${flake.inputs.nixpkgs}/nixos/lib/utils.nix" {
+    inherit lib;
+    inherit config;
+    inherit pkgs;
+  };
+  # Type for a valid systemd unit option. Needed for correctly passing "timerConfig" to "systemd.timers"
+  inherit (utils.systemdUtils.unitOptions) unitOption;
+  inherit (lib)
+    literalExpression
+    mkOption
+    mkPackageOption
+    types
+    ;
+in
+{
+  options.pub-solar-os.backups = {
+    repos = mkOption {
+      description = ''
+        Configuration of Restic repositories.
+      '';
+      type = types.attrsOf (
+        types.submodule (
+          { name, ... }:
+          {
+            options = {
+              passwordFile = mkOption {
+                type = types.str;
+                description = ''
+                  Read the repository password from a file.
+                '';
+                example = "/etc/nixos/restic-password";
+              };
+
+              environmentFile = mkOption {
+                type = with types; nullOr str;
+                default = null;
+                description = ''
+                  Read repository secrets as environment variables from a file.
+                '';
+                example = "/etc/nixos/restic-env";
+              };
+
+              repository = mkOption {
+                type = with types; nullOr str;
+                default = null;
+                description = ''
+                  repository to backup to.
+                '';
+                example = "sftp:backup@192.168.1.100:/backups/${name}";
+              };
+            };
+          }
+        )
+      );
+
+      default = { };
+      example = {
+        remotebackup = {
+          repository = "sftp:backup@host:/backups/home";
+          passwordFile = "/etc/nixos/secrets/restic-password";
+          environmentFile = "/etc/nixos/secrets/restic-env";
+        };
+      };
+    };
+
+    restic = mkOption {
+      description = ''
+        Periodic backups to create with Restic.
+      '';
+      type = types.attrsOf (
+        types.submodule (
+          { name, ... }:
+          {
+            options = {
+              paths = mkOption {
+                # This is nullable for legacy reasons only. We should consider making it a pure listOf
+                # after some time has passed since this comment was added.
+                type = types.nullOr (types.listOf types.str);
+                default = [ ];
+                description = ''
+                  Which paths to backup, in addition to ones specified via
+                  `dynamicFilesFrom`.  If null or an empty array and
+                  `dynamicFilesFrom` is also null, no backup command will be run.
+                   This can be used to create a prune-only job.
+                '';
+                example = [
+                  "/var/lib/postgresql"
+                  "/home/user/backup"
+                ];
+              };
+
+              exclude = mkOption {
+                type = types.listOf types.str;
+                default = [ ];
+                description = ''
+                  Patterns to exclude when backing up. See
+                  https://restic.readthedocs.io/en/latest/040_backup.html#excluding-files for
+                  details on syntax.
+                '';
+                example = [
+                  "/var/cache"
+                  "/home/*/.cache"
+                  ".git"
+                ];
+              };
+
+              timerConfig = mkOption {
+                type = types.nullOr (types.attrsOf unitOption);
+                default = {
+                  OnCalendar = "daily";
+                  Persistent = true;
+                };
+                description = ''
+                  When to run the backup. See {manpage}`systemd.timer(5)` for
+                  details. If null no timer is created and the backup will only
+                  run when explicitly started.
+                '';
+                example = {
+                  OnCalendar = "00:05";
+                  RandomizedDelaySec = "5h";
+                  Persistent = true;
+                };
+              };
+
+              user = mkOption {
+                type = types.str;
+                default = "root";
+                description = ''
+                  As which user the backup should run.
+                '';
+                example = "postgresql";
+              };
+
+              extraBackupArgs = mkOption {
+                type = types.listOf types.str;
+                default = [ ];
+                description = ''
+                  Extra arguments passed to restic backup.
+                '';
+                example = [ "--exclude-file=/etc/nixos/restic-ignore" ];
+              };
+
+              extraOptions = mkOption {
+                type = types.listOf types.str;
+                default = [ ];
+                description = ''
+                  Extra extended options to be passed to the restic --option flag.
+                '';
+                example = [ "sftp.command='ssh backup@192.168.1.100 -i /home/user/.ssh/id_rsa -s sftp'" ];
+              };
+
+              initialize = mkOption {
+                type = types.bool;
+                default = false;
+                description = ''
+                  Create the repository if it doesn't exist.
+                '';
+              };
+
+              pruneOpts = mkOption {
+                type = types.listOf types.str;
+                default = [ ];
+                description = ''
+                  A list of options (--keep-\* et al.) for 'restic forget
+                  --prune', to automatically prune old snapshots.  The
+                  'forget' command is run *after* the 'backup' command, so
+                  keep that in mind when constructing the --keep-\* options.
+                '';
+                example = [
+                  "--keep-daily 7"
+                  "--keep-weekly 5"
+                  "--keep-monthly 12"
+                  "--keep-yearly 75"
+                ];
+              };
+
+              runCheck = mkOption {
+                type = types.bool;
+                default = (builtins.length config.pub-solar-os.backups.restic.${name}.checkOpts > 0);
+                defaultText = literalExpression ''builtins.length config.services.backups.${name}.checkOpts > 0'';
+                description = "Whether to run the `check` command with the provided `checkOpts` options.";
+                example = true;
+              };
+
+              checkOpts = mkOption {
+                type = types.listOf types.str;
+                default = [ ];
+                description = ''
+                  A list of options for 'restic check'.
+                '';
+                example = [ "--with-cache" ];
+              };
+
+              dynamicFilesFrom = mkOption {
+                type = with types; nullOr str;
+                default = null;
+                description = ''
+                  A script that produces a list of files to back up.  The
+                  results of this command are given to the '--files-from'
+                  option. The result is merged with paths specified via `paths`.
+                '';
+                example = "find /home/matt/git -type d -name .git";
+              };
+
+              backupPrepareCommand = mkOption {
+                type = with types; nullOr str;
+                default = null;
+                description = ''
+                  A script that must run before starting the backup process.
+                '';
+              };
+
+              backupCleanupCommand = mkOption {
+                type = with types; nullOr str;
+                default = null;
+                description = ''
+                  A script that must run after finishing the backup process.
+                '';
+              };
+
+              package = mkPackageOption pkgs "restic" { };
+
+              createWrapper = lib.mkOption {
+                type = lib.types.bool;
+                default = true;
+                description = ''
+                  Whether to generate and add a script to the system path, that has the same environment variables set
+                  as the systemd service. This can be used to e.g. mount snapshots or perform other opterations, without
+                  having to manually specify most options.
+                '';
+              };
+            };
+          }
+        )
+      );
+      default = { };
+      example = {
+        localbackup = {
+          paths = [ "/home" ];
+          exclude = [ "/home/*/.cache" ];
+          initialize = true;
+        };
+        remotebackup = {
+          paths = [ "/home" ];
+          extraOptions = [
+            "sftp.command='ssh backup@host -i /etc/nixos/secrets/backup-private-key -s sftp'"
+          ];
+          timerConfig = {
+            OnCalendar = "00:05";
+            RandomizedDelaySec = "5h";
+          };
+        };
+      };
+    };
+  };
+
+  config = {
+    services.restic.backups =
+      let
+        repos = config.pub-solar-os.backups.repos;
+        restic = config.pub-solar-os.backups.restic;
+
+        repoNames = builtins.attrNames repos;
+        backupNames = builtins.attrNames restic;
+
+        createBackups =
+          backupName:
+          map (repoName: {
+            name = "${backupName}-${repoName}";
+            value = repos."${repoName}" // restic."${backupName}";
+          }) repoNames;
+
+      in
+      builtins.listToAttrs (lib.lists.flatten (map createBackups backupNames));
+
+    # Used for pub-solar-os.backups.repos.storagebox
+    programs.ssh.knownHosts = {
+      "u377325.your-storagebox.de".publicKey =
+        "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA5EB5p/5Hp3hGW1oHok+PIOH9Pbn7cnUiGmUEBrCVjnAw+HrKyN8bYVV0dIGllswYXwkG/+bgiBlE6IVIBAq+JwVWu1Sss3KarHY3OvFJUXZoZyRRg/Gc/+LRCE7lyKpwWQ70dbelGRyyJFH36eNv6ySXoUYtGkwlU5IVaHPApOxe4LHPZa/qhSRbPo2hwoh0orCtgejRebNtW5nlx00DNFgsvn8Svz2cIYLxsPVzKgUxs8Zxsxgn+Q/UvR7uq4AbAhyBMLxv7DjJ1pc7PJocuTno2Rw9uMZi1gkjbnmiOh6TTXIEWbnroyIhwc8555uto9melEUmWNQ+C+PwAK+MPw==";
+      "[u377325.your-storagebox.de]:23".publicKey =
+        "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIICf9svRenC/PLKIL9nk6K/pxQgoiFC41wTNvoIncOxs";
+    };
+  };
+}

modules/core/default.nix

@@ -54,9 +54,5 @@
     };
 
     time.timeZone = "Etc/UTC";
-
-    home-manager.users.${config.pub-solar-os.authentication.username} = {
-      home.stateVersion = "23.05";
-    };
   };
 }

modules/core/networking.nix

@@ -31,13 +31,17 @@
 
     networking.hosts = {
       "10.7.6.1" = [ "nachtigall.wg.${config.pub-solar-os.networking.domain}" ];
-      "10.7.6.2" = [ "flora-6.wg.${config.pub-solar-os.networking.domain}" ];
       "10.7.6.3" = [ "metronom.wg.${config.pub-solar-os.networking.domain}" ];
       "10.7.6.4" = [ "tankstelle.wg.${config.pub-solar-os.networking.domain}" ];
+      "10.7.6.5" = [ "trinkgenossin.wg.${config.pub-solar-os.networking.domain}" ];
+      "10.7.6.6" = [ "delite.wg.${config.pub-solar-os.networking.domain}" ];
+      "10.7.6.7" = [ "blue-shell.wg.${config.pub-solar-os.networking.domain}" ];
       "fd00:fae:fae:fae:fae:1::" = [ "nachtigall.wg.${config.pub-solar-os.networking.domain}" ];
-      "fd00:fae:fae:fae:fae:2::" = [ "flora-6.wg.${config.pub-solar-os.networking.domain}" ];
       "fd00:fae:fae:fae:fae:3::" = [ "metronom.wg.${config.pub-solar-os.networking.domain}" ];
       "fd00:fae:fae:fae:fae:4::" = [ "tankstelle.wg.${config.pub-solar-os.networking.domain}" ];
+      "fd00:fae:fae:fae:fae:5::" = [ "trinkgenossin.wg.${config.pub-solar-os.networking.domain}" ];
+      "fd00:fae:fae:fae:fae:6::" = [ "delite.wg.${config.pub-solar-os.networking.domain}" ];
+      "fd00:fae:fae:fae:fae:7::" = [ "blue-shell.wg.${config.pub-solar-os.networking.domain}" ];
     };
 
     services.openssh = {

modules/core/nix.nix

@@ -6,7 +6,21 @@
   ...
 }:
 {
-  nixpkgs.config.allowUnfreePredicate = pkg: builtins.elem (lib.getName pkg) [ ];
+  nixpkgs.config = lib.mkDefault {
+    allowUnfreePredicate = pkg: builtins.elem (lib.getName pkg) [ ];
+    permittedInsecurePackages = [ "olm-3.2.16" ];
+  };
+
+  system.activationScripts.diff-closures = {
+    text = ''
+      if [[ -e /run/current-system ]]; then
+        ${config.nix.package}/bin/nix store diff-closures \
+          /run/current-system "$systemConfig" \
+          --extra-experimental-features nix-command
+      fi
+    '';
+    supportsDryActivation = true;
+  };
 
   nix = {
     # Use default version alias for nix package

modules/core/terminal-tooling.nix

@@ -1,19 +1,33 @@
-{ flake, config, ... }:
+{ flake, lib, ... }:
 {
-  home-manager.users.${config.pub-solar-os.authentication.username} = {
+  home-manager.users = (
-    programs.git.enable = true;
+    lib.attrsets.foldlAttrs (
-    programs.starship.enable = true;
+      acc: name: value:
-    programs.bash.enable = true;
+      acc
-    programs.neovim = {
+      // {
-      enable = true;
+        ${name} = {
-      vimAlias = true;
+          programs.git.enable = true;
-      viAlias = true;
+          programs.starship.enable = true;
-      defaultEditor = true;
+          programs.bash = {
-      # configure = {
+            enable = true;
-      #   packages.myVimPackages = with pkgs.vimPlugins; {
+            historyControl = [
-      #     start = [vim-nix vim-surrund rainbow];
+              "ignoredups"
-      #   };
+              "ignorespace"
-      # };
+            ];
-    };
+          };
-  };
+          programs.neovim = {
+            enable = true;
+            vimAlias = true;
+            viAlias = true;
+            defaultEditor = true;
+            # configure = {
+            #   packages.myVimPackages = with pkgs.vimPlugins; {
+            #     start = [vim-nix vim-surrund rainbow];
+            #   };
+            # };
+          };
+        };
+      }
+    ) { } flake.self.logins.admins
+  );
 }

modules/core/users.nix

@@ -11,18 +11,6 @@
       inherit (lib) mkOption types;
     in
     {
-      username = mkOption {
-        description = "Username for the adminstrative user";
-        type = types.str;
-        default = flake.self.username;
-      };
-
-      sshPubKeys = mkOption {
-        description = "SSH Keys that should have administrative root access";
-        type = types.listOf types.str;
-        default = flake.self.logins.admins.sshPubKeys;
-      };
-
       root.initialHashedPassword = mkOption {
         description = "Hashed password of the root account";
         type = types.str;
@@ -43,36 +31,60 @@
     };
 
   config = {
-    users.users.${config.pub-solar-os.authentication.username} = {
+    users.users =
-      name = config.pub-solar-os.authentication.username;
+      (lib.attrsets.foldlAttrs (
-      group = config.pub-solar-os.authentication.username;
+        acc: name: value:
-      extraGroups = [
+        acc
-        "wheel"
+        // {
-        "docker"
+          ${name} = {
-      ];
+            name = name;
-      isNormalUser = true;
+            group = name;
-      openssh.authorizedKeys.keys = config.pub-solar-os.authentication.sshPubKeys;
+            extraGroups = [
-    };
+              "wheel"
-    users.groups.${config.pub-solar-os.authentication.username} = { };
+              "docker"
+            ];
+            isNormalUser = true;
+            openssh.authorizedKeys.keys = lib.attrsets.attrValues value.sshPubKeys;
+          };
+        }
+      ) { } flake.self.logins.admins)
+      // {
+        # TODO: Remove when we stop locking ourselves out.
+        root.openssh.authorizedKeys.keys = flake.self.logins.sshPubKeys;
+        root.initialHashedPassword = config.pub-solar-os.authentication.root.initialHashedPassword;
 
-    # TODO: Remove when we stop locking ourselves out.
+        ${config.pub-solar-os.authentication.robot.username} = {
-    users.users.root.openssh.authorizedKeys.keys = config.pub-solar-os.authentication.sshPubKeys;
+          description = "CI and automation user";
+          home = "/home/${config.pub-solar-os.authentication.robot.username}";
+          createHome = true;
+          useDefaultShell = true;
+          uid = 998;
+          group = "${config.pub-solar-os.authentication.robot.username}";
+          isSystemUser = true;
+          openssh.authorizedKeys.keys = config.pub-solar-os.authentication.robot.sshPubKeys;
+        };
+      };
 
-    users.users.${config.pub-solar-os.authentication.robot.username} = {
+    home-manager.users = (
-      description = "CI and automation user";
+      lib.attrsets.foldlAttrs (
-      home = "/home/${config.pub-solar-os.authentication.robot.username}";
+        acc: name: value:
-      createHome = true;
+        acc
-      useDefaultShell = true;
+        // {
-      uid = 998;
+          ${name} = {
-      group = "${config.pub-solar-os.authentication.robot.username}";
+            home.stateVersion = "23.05";
-      isSystemUser = true;
+          };
-      openssh.authorizedKeys.keys = config.pub-solar-os.authentication.robot.sshPubKeys;
+        }
-    };
+      ) { } flake.self.logins.admins
+    );
 
-    users.groups.${config.pub-solar-os.authentication.robot.username} = { };
+    users.groups =
-
+      (lib.attrsets.foldlAttrs (
-    users.users.root.initialHashedPassword =
+        acc: name: value:
-      config.pub-solar-os.authentication.root.initialHashedPassword;
+        acc // { "${name}" = { }; }
+      ) { } flake.self.logins.admins)
+      // {
+        ${config.pub-solar-os.authentication.robot.username} = { };
+      };
 
     security.sudo.wheelNeedsPassword = false;
   };

modules/coturn/default.nix

@@ -18,7 +18,7 @@
     min-port = 49000;
     max-port = 50000;
     use-auth-secret = true;
-    static-auth-secret-file = "/run/agenix/coturn-static-auth-secret";
+    static-auth-secret-file = config.age.secrets."coturn-static-auth-secret".path;
     realm = "turn.${config.pub-solar-os.networking.domain}";
     cert = "${config.security.acme.certs.${realm}.directory}/full.pem";
     pkey = "${config.security.acme.certs.${realm}.directory}/key.pem";

modules/drone/default.nix

@@ -1,114 +0,0 @@
-{
-  config,
-  lib,
-  pkgs,
-  flake,
-  ...
-}:
-{
-  age.secrets.drone-secrets = {
-    file = "${flake.self}/secrets/drone-secrets.age";
-    mode = "600";
-    owner = "drone";
-  };
-  age.secrets.drone-db-secrets = {
-    file = "${flake.self}/secrets/drone-db-secrets.age";
-    mode = "600";
-    owner = "drone";
-  };
-
-  users.users.drone = {
-    description = "Drone Service";
-    home = "/var/lib/drone";
-    useDefaultShell = true;
-    uid = 994;
-    group = "drone";
-    isSystemUser = true;
-  };
-
-  users.groups.drone = { };
-
-  systemd.tmpfiles.rules = [ "d '/var/lib/drone-db' 0750 drone drone - -" ];
-
-  services.caddy.virtualHosts."ci.${config.pub-solar-os.networking.domain}" = {
-    logFormat = lib.mkForce ''
-      output discard
-    '';
-    extraConfig = ''
-      reverse_proxy :4000
-    '';
-  };
-
-  systemd.services."docker-network-drone" =
-    let
-      docker = config.virtualisation.oci-containers.backend;
-      dockerBin = "${pkgs.${docker}}/bin/${docker}";
-    in
-    {
-      serviceConfig.Type = "oneshot";
-      before = [ "docker-drone-server.service" ];
-      script = ''
-        ${dockerBin} network inspect drone-net >/dev/null 2>&1 || ${dockerBin} network create drone-net --subnet 172.20.0.0/24
-      '';
-    };
-
-  virtualisation = {
-    docker = {
-      enable = true; # sadly podman is not supported rightnow
-      extraOptions = ''
-        --data-root /data/docker
-      '';
-    };
-
-    oci-containers = {
-      backend = "docker";
-      containers."drone-db" = {
-        image = "postgres:14";
-        autoStart = true;
-        user = "994";
-        volumes = [ "/var/lib/drone-db:/var/lib/postgresql/data" ];
-        extraOptions = [ "--network=drone-net" ];
-        environmentFiles = [ config.age.secrets.drone-db-secrets.path ];
-      };
-      containers."drone-server" = {
-        image = "drone/drone:2";
-        autoStart = true;
-        user = "994";
-        ports = [ "127.0.0.1:4000:80" ];
-        dependsOn = [ "drone-db" ];
-        extraOptions = [
-          "--network=drone-net"
-          "--pull=always"
-          "--add-host=nachtigall.${config.pub-solar-os.networking.domain}:10.7.6.1"
-        ];
-        environment = {
-          DRONE_GITEA_SERVER = "https://git.${config.pub-solar-os.networking.domain}";
-          DRONE_SERVER_HOST = "ci.${config.pub-solar-os.networking.domain}";
-          DRONE_SERVER_PROTO = "https";
-          DRONE_DATABASE_DRIVER = "postgres";
-        };
-        environmentFiles = [ config.age.secrets.drone-secrets.path ];
-      };
-      containers."drone-docker-runner" = {
-        image = "drone/drone-runner-docker:1";
-        autoStart = true;
-        # needs to run as root
-        #user = "994";
-        volumes = [ "/var/run/docker.sock:/var/run/docker.sock" ];
-        dependsOn = [ "drone-db" ];
-        extraOptions = [
-          "--network=drone-net"
-          "--pull=always"
-          "--add-host=nachtigall.${config.pub-solar-os.networking.domain}:10.7.6.1"
-        ];
-        environment = {
-          DRONE_RPC_HOST = "ci.${config.pub-solar-os.networking.domain}";
-          DRONE_RPC_PROTO = "https";
-          DRONE_RUNNER_CAPACITY = "2";
-          DRONE_RUNNER_NAME = "flora-6-docker-runner";
-        };
-        environmentFiles = [ config.age.secrets.drone-secrets.path ];
-      };
-    };
-  };
-}

modules/forgejo-actions-runner/default.nix

@@ -1,67 +0,0 @@
-{
-  config,
-  lib,
-  pkgs,
-  flake,
-  ...
-}:
-{
-  age.secrets.forgejo-actions-runner-token = {
-    file = "${flake.self}/secrets/forgejo-actions-runner-token.age";
-    mode = "440";
-  };
-
-  # Trust docker bridge interface traffic
-  # Needed for the docker runner to communicate with the act_runner cache
-  networking.firewall.trustedInterfaces = [ "br-+" ];
-
-  users.users.gitea-runner = {
-    home = "/var/lib/gitea-runner/flora-6";
-    useDefaultShell = true;
-    group = "gitea-runner";
-    isSystemUser = true;
-  };
-
-  users.groups.gitea-runner = { };
-
-  systemd.services."gitea-runner-flora\\x2d6".serviceConfig = {
-    DynamicUser = lib.mkForce false;
-  };
-
-  systemd.tmpfiles.rules = [
-    "d '/data/gitea-actions-runner' 0750 gitea-runner gitea-runner - -"
-    "d '/var/lib/gitea-runner' 0750 gitea-runner gitea-runner - -"
-  ];
-
-  # forgejo actions runner
-  # https://forgejo.org/docs/latest/admin/actions/
-  # https://docs.gitea.com/usage/actions/quickstart
-  services.gitea-actions-runner = {
-    package = pkgs.forgejo-runner;
-    instances."flora-6" = {
-      enable = true;
-      name = config.networking.hostName;
-      url = "https://git.pub.solar";
-      tokenFile = config.age.secrets.forgejo-actions-runner-token.path;
-      settings = {
-        cache = {
-          enabled = true;
-          dir = "/data/gitea-actions-runner/actcache";
-          host = "";
-          port = 0;
-          external_server = "";
-        };
-      };
-      labels = [
-        # provide a debian 12 bookworm base with Node.js for actions
-        "debian-latest:docker://git.pub.solar/pub-solar/actions-base-image:20-bookworm"
-        # fake the ubuntu name, commonly used in actions examples
-        "ubuntu-latest:docker://git.pub.solar/pub-solar/actions-base-image:20-bookworm"
-        # alpine with Node.js
-        "alpine-latest:docker://node:20-alpine"
-        # nix flakes enabled image with Node.js
-        "nix-flakes:docker://git.pub.solar/pub-solar/nix-flakes-node:latest"
-      ];
-    };
-  };
-}

modules/forgejo/default.nix

@@ -65,6 +65,7 @@
 
   services.forgejo = {
     enable = true;
+    package = pkgs.forgejo;
     user = "gitea";
     group = "gitea";
     database = {
@@ -75,7 +76,7 @@
     };
     stateDir = "/var/lib/forgejo";
     lfs.enable = true;
-    mailerPasswordFile = config.age.secrets.forgejo-mailer-password.path;
+    secrets.mailer.PASSWD = config.age.secrets.forgejo-mailer-password.path;
     settings = {
       DEFAULT.APP_NAME = "pub.solar git server";
 
@@ -141,6 +142,12 @@
         LOGIN_REMEMBER_DAYS = 365;
       };
 
+      # See https://docs.gitea.com/administration/config-cheat-sheet#migrations-migrations
+      migrations = {
+        # This allows migrations from the same forgejo instance
+        ALLOW_LOCALNETWORKS = true;
+      };
+
       # https://forgejo.org/docs/next/admin/config-cheat-sheet/#indexer-indexer
       indexer = {
         REPO_INDEXER_ENABLED = true;
@@ -182,7 +189,7 @@
       OnCalendar = "*-*-* 00:00:00 Etc/UTC";
     };
     initialize = true;
-    passwordFile = config.age.secrets."restic-repo-storagebox".path;
+    passwordFile = config.age.secrets."restic-repo-storagebox-nachtigall".path;
     repository = "sftp:u377325@u377325.your-storagebox.de:/backups";
     backupPrepareCommand = ''
       ${pkgs.sudo}/bin/sudo -u postgres ${pkgs.postgresql}/bin/pg_dump -d gitea > /tmp/forgejo-backup.sql

modules/garage/default.nix

@@ -0,0 +1,142 @@
+{
+  config,
+  lib,
+  pkgs,
+  flake,
+  ...
+}:
+{
+  age.secrets."garage-rpc-secret" = {
+    file = "${flake.self}/secrets/garage-rpc-secret.age";
+    mode = "400";
+  };
+
+  age.secrets."garage-admin-token" = {
+    file = "${flake.self}/secrets/garage-admin-token.age";
+    mode = "400";
+  };
+
+  age.secrets."acme-namecheap-env" = {
+    file = "${flake.self}/secrets/acme-namecheap-env.age";
+    mode = "400";
+  };
+
+  networking.firewall.allowedTCPPorts = [
+    3900
+    3901
+    3902
+  ];
+
+  networking.firewall.interfaces.wg-ssh.allowedTCPPorts = [ 3903 ];
+
+  security.acme = {
+    defaults = {
+      # LEGO_DISABLE_CNAME_SUPPORT=true set here to fix issues with CNAME
+      # detection, as we use wildcard DNS for garage
+      environmentFile = config.age.secrets.acme-namecheap-env.path;
+    };
+    certs = {
+      # Wildcard certificate gets created automatically
+      "buckets.${config.pub-solar-os.networking.domain}" = {
+        # disable http challenge
+        webroot = null;
+        # enable dns challenge
+        dnsProvider = "namecheap";
+      };
+      # Wildcard certificate gets created automatically
+      "web.${config.pub-solar-os.networking.domain}" = {
+        # disable http challenge
+        webroot = null;
+        # enable dns challenge
+        dnsProvider = "namecheap";
+      };
+    };
+  };
+
+  services.nginx = {
+    upstreams.s3_backend.servers = {
+      "[::1]:3900" = { };
+    };
+    upstreams.web_backend.servers = {
+      "[::1]:3902" = { };
+    };
+    virtualHosts."buckets.${config.pub-solar-os.networking.domain}" = {
+      serverAliases = [ "*.buckets.${config.pub-solar-os.networking.domain}" ];
+
+      enableACME = true;
+      forceSSL = true;
+
+      locations."/" = {
+        proxyPass = "http://s3_backend";
+        extraConfig = ''
+          client_max_body_size 64m;
+          proxy_max_temp_file_size 0;
+        '';
+      };
+    };
+    virtualHosts."web.${config.pub-solar-os.networking.domain}" = {
+      serverAliases = [ "*.web.${config.pub-solar-os.networking.domain}" ];
+
+      enableACME = true;
+      forceSSL = true;
+
+      locations."/" = {
+        proxyPass = "http://web_backend";
+      };
+    };
+  };
+
+  services.garage = {
+    enable = true;
+    package = pkgs.garage_1_0_1;
+    settings = {
+      data_dir = "/var/lib/garage/data";
+      metadata_dir = "/var/lib/garage/meta";
+      db_engine = "lmdb";
+      replication_factor = 3;
+      compression_level = 2;
+      rpc_bind_addr = "[::]:3901";
+      s3_api = {
+        s3_region = "eu-central";
+        api_bind_addr = "[::]:3900";
+        root_domain = ".buckets.${config.pub-solar-os.networking.domain}";
+      };
+      s3_web = {
+        bind_addr = "[::]:3902";
+        root_domain = ".web.${config.pub-solar-os.networking.domain}";
+        index = "index.html";
+      };
+      admin = {
+        api_bind_addr = "[::]:3903";
+      };
+    };
+  };
+
+  users.users.garage = {
+    isSystemUser = true;
+    home = "/var/lib/garage";
+    group = "garage";
+  };
+
+  users.groups.garage = { };
+
+  # Adapted from https://git.clan.lol/clan/clan-core/src/commit/23a9e35c665ff531fe1193dcc47056432fbbeacf/clanModules/garage/default.nix
+  # Disabled DynamicUser https://github.com/NixOS/nixpkgs/blob/nixos-24.05/nixos/modules/services/web-servers/garage.nix
+  # for mounts + permissions to work
+  systemd.services.garage = {
+    serviceConfig = {
+      user = "garage";
+      group = "garage";
+      DynamicUser = false;
+      LoadCredential = [
+        "rpc_secret_path:${config.age.secrets.garage-rpc-secret.path}"
+        "admin_token_path:${config.age.secrets.garage-admin-token.path}"
+      ];
+      Environment = [
+        "GARAGE_ALLOW_WORLD_READABLE_SECRETS=true"
+        "GARAGE_RPC_SECRET_FILE=%d/rpc_secret_path"
+        "GARAGE_ADMIN_TOKEN_FILE=%d/admin_token_path"
+      ];
+    };
+  };
+}

modules/grafana/default.nix

@@ -33,15 +33,18 @@
       group = "grafana";
       user = "grafana";
     };
+    "grafana-dashboards/grafana-garage-dashboard-prometheus.json" = {
+      source = ./grafana-dashboards/grafana-garage-dashboard-prometheus.json;
+      group = "grafana";
+      user = "grafana";
+    };
   };
 
-  services.caddy.virtualHosts."grafana.${config.pub-solar-os.networking.domain}" = {
+  services.nginx.virtualHosts."grafana.${config.pub-solar-os.networking.domain}" = {
-    logFormat = lib.mkForce ''
+    enableACME = true;
-      output discard
+    forceSSL = true;
-    '';
+    locations."/".proxyPass =
-    extraConfig = ''
+      "http://127.0.0.1:${toString config.services.grafana.settings.server.http_port}";
-      reverse_proxy :${toString config.services.grafana.settings.server.http_port}
-    '';
   };
 
   services.grafana = {
@@ -64,7 +67,7 @@
         password = "\$__file{${config.age.secrets.grafana-smtp-password.path}}";
         from_address = "no-reply@pub.solar";
         from_name = "grafana.pub.solar";
-        ehlo_identity = "flora-6.pub.solar";
+        ehlo_identity = "grafana.pub.solar";
       };
       security = {
         admin_email = "crew@pub.solar";

modules/keycloak/default.nix

@@ -6,23 +6,22 @@
   ...
 }:
 {
-  options.pub-solar-os.auth = {
+  options.pub-solar-os.auth = with lib; {
-    enable = lib.mkEnableOption "Enable keycloak to run on the node";
+    enable = mkEnableOption "Enable keycloak to run on the node";
 
-    realm = lib.mkOption {
+    realm = mkOption {
       description = "Name of the realm";
-      type = lib.types.str;
+      type = types.str;
       default = config.pub-solar-os.networking.domain;
     };
+
+    database-password-file = mkOption {
+      description = "Database password file path";
+      type = types.str;
+    };
   };
 
   config = lib.mkIf config.pub-solar-os.auth.enable {
-    age.secrets.keycloak-database-password = {
-      file = "${flake.self}/secrets/keycloak-database-password.age";
-      mode = "600";
-      #owner = "keycloak";
-    };
-
     services.nginx.virtualHosts."auth.${config.pub-solar-os.networking.domain}" = {
       enableACME = true;
       forceSSL = true;
@@ -46,12 +45,13 @@
     # keycloak
     services.keycloak = {
       enable = true;
-      database.passwordFile = config.age.secrets.keycloak-database-password.path;
+      database.passwordFile = config.pub-solar-os.auth.database-password-file;
       settings = {
         hostname = "auth.${config.pub-solar-os.networking.domain}";
         http-host = "127.0.0.1";
         http-port = 8080;
-        proxy = "edge";
+        proxy-headers = "xforwarded";
+        http-enabled = true;
       };
       themes = {
         "pub.solar" =
@@ -59,14 +59,12 @@
       };
     };
 
-    services.restic.backups.keycloak-storagebox = {
+    pub-solar-os.backups.restic.keycloak = {
       paths = [ "/tmp/keycloak-backup.sql" ];
       timerConfig = {
         OnCalendar = "*-*-* 03:00:00 Etc/UTC";
       };
       initialize = true;
-      passwordFile = config.age.secrets."restic-repo-storagebox".path;
-      repository = "sftp:u377325@u377325.your-storagebox.de:/backups";
       backupPrepareCommand = ''
         ${pkgs.sudo}/bin/sudo -u postgres ${pkgs.postgresql}/bin/pg_dump -d keycloak > /tmp/keycloak-backup.sql
       '';

modules/loki/default.nix

@@ -108,7 +108,7 @@
       };
       clients = [
         {
-          url = "http://flora-6.wg.pub.solar:${toString config.services.loki.configuration.server.http_listen_port}/loki/api/v1/push";
+          url = "http://trinkgenossin.wg.pub.solar:${toString config.services.loki.configuration.server.http_listen_port}/loki/api/v1/push";
         }
       ];
       scrape_configs = [
@@ -118,7 +118,7 @@
             max_age = "24h";
             labels = {
               job = "systemd-journal";
-              host = "flora-6";
+              host = "trinkgenossin";
             };
           };
           relabel_configs = [

modules/mail/default.nix

@@ -67,4 +67,20 @@
   };
   security.acme.acceptTerms = true;
   security.acme.defaults.email = "security@pub.solar";
+
+  pub-solar-os.backups.restic.mail = {
+    paths = [
+      "/var/vmail"
+      "/var/dkim"
+    ];
+    timerConfig = {
+      OnCalendar = "*-*-* 02:00:00 Etc/UTC";
+    };
+    initialize = true;
+    pruneOpts = [
+      "--keep-daily 7"
+      "--keep-weekly 4"
+      "--keep-monthly 3"
+    ];
+  };
 }

modules/mailman/default.nix

@@ -91,7 +91,7 @@
       OnCalendar = "*-*-* 02:00:00 Etc/UTC";
     };
     initialize = true;
-    passwordFile = config.age.secrets."restic-repo-storagebox".path;
+    passwordFile = config.age.secrets."restic-repo-storagebox-nachtigall".path;
     repository = "sftp:u377325@u377325.your-storagebox.de:/backups";
     pruneOpts = [
       "--keep-daily 7"

modules/mastodon/default.nix

@@ -7,6 +7,21 @@
 }:
 
 {
+  age.secrets."mastodon-active-record-encryption-deterministic-key" = {
+    file = "${flake.self}/secrets//mastodon-active-record-encryption-deterministic-key.age";
+    mode = "400";
+    owner = config.services.mastodon.user;
+  };
+  age.secrets."mastodon-active-record-encryption-key-derivation-salt" = {
+    file = "${flake.self}/secrets//mastodon-active-record-encryption-key-derivation-salt.age";
+    mode = "400";
+    owner = config.services.mastodon.user;
+  };
+  age.secrets."mastodon-active-record-encryption-primary-key" = {
+    file = "${flake.self}/secrets//mastodon-active-record-encryption-primary-key.age";
+    mode = "400";
+    owner = config.services.mastodon.user;
+  };
   age.secrets."mastodon-secret-key-base" = {
     file = "${flake.self}/secrets/mastodon-secret-key-base.age";
     mode = "400";
@@ -54,6 +69,9 @@
     webProcesses = 2;
     # Threads per process used by the mastodon-web service
     webThreads = 5;
+    activeRecordEncryptionDeterministicKeyFile = "/run/agenix/mastodon-active-record-encryption-deterministic-key";
+    activeRecordEncryptionKeyDerivationSaltFile = "/run/agenix/mastodon-active-record-encryption-key-derivation-salt";
+    activeRecordEncryptionPrimaryKeyFile = "/run/agenix/mastodon-active-record-encryption-primary-key";
     secretKeyBaseFile = "/run/agenix/mastodon-secret-key-base";
     otpSecretFile = "/run/agenix/mastodon-otp-secret";
     vapidPrivateKeyFile = "/run/agenix/mastodon-vapid-private-key";
@@ -67,20 +85,20 @@
       passwordFile = "/run/agenix/mastodon-smtp-password";
       fromAddress = "mastodon-notifications@pub.solar";
     };
+    # Defined in ./opensearch.nix
+    elasticsearch.host = "127.0.0.1";
     mediaAutoRemove = {
       olderThanDays = 7;
     };
     extraEnvFiles = [ "/run/agenix/mastodon-extra-env-secrets" ];
     extraConfig = {
       WEB_DOMAIN = "mastodon.${config.pub-solar-os.networking.domain}";
-      # Defined in ./opensearch.nix
-      ES_HOST = "127.0.0.1";
       # S3 File storage (optional)
       # -----------------------
       S3_ENABLED = "true";
-      S3_BUCKET = "pub-solar-mastodon";
+      S3_BUCKET = "mastodon";
-      S3_REGION = "europe-west-1";
+      S3_REGION = "eu-central";
-      S3_ENDPOINT = "https://gateway.tardigradeshare.io";
+      S3_ENDPOINT = "https://buckets.pub.solar";
       S3_ALIAS_HOST = "files.${config.pub-solar-os.networking.domain}";
       # Translation (optional)
       # -----------------------
@@ -106,7 +124,7 @@
       OnCalendar = "*-*-* 04:00:00 Etc/UTC";
     };
     initialize = true;
-    passwordFile = config.age.secrets."restic-repo-storagebox".path;
+    passwordFile = config.age.secrets."restic-repo-storagebox-nachtigall".path;
     repository = "sftp:u377325@u377325.your-storagebox.de:/backups";
     backupPrepareCommand = ''
       ${pkgs.sudo}/bin/sudo -u postgres ${pkgs.postgresql}/bin/pg_dump -d mastodon > /tmp/mastodon-backup.sql

modules/matrix-irc/default.nix

@@ -16,115 +16,128 @@ let
   synapseClientPort = "${toString listenerWithClient.port}";
 in
 {
-  systemd.services.matrix-appservice-irc.serviceConfig.SystemCallFilter = lib.mkForce [
+  options.pub-solar-os = {
-    "@system-service @pkey"
+    matrix.appservice-irc.mediaproxy = {
-    "~@privileged @resources"
+      signingKeyPath = lib.mkOption {
-    "@chown"
+        description = "Path to file containing the IRC appservice mediaproxy signing key";
-  ];
+        type = lib.types.str;
-  services.matrix-appservice-irc = {
+        default = "/var/lib/matrix-appservice-irc/media-signingkey.jwk";
-    enable = true;
-    localpart = "irc_bot";
-    port = 8010;
-    registrationUrl = "http://localhost:8010";
-    settings = {
-      homeserver = {
-        domain = "${config.pub-solar-os.networking.domain}";
-        url = "http://127.0.0.1:${synapseClientPort}";
-        media_url = "https://matrix.${config.pub-solar-os.networking.domain}";
-        enablePresence = false;
       };
-      ircService = {
+    };
-        ident = {
+  };
-          address = "::";
+  config = {
-          enabled = false;
+    services.matrix-appservice-irc = {
-          port = 1113;
+      enable = true;
+      localpart = "irc_bot";
+      port = 8010;
+      registrationUrl = "http://localhost:8010";
+      settings = {
+        homeserver = {
+          domain = "${config.pub-solar-os.networking.domain}";
+          url = "http://127.0.0.1:${synapseClientPort}";
+          enablePresence = false;
         };
-        logging = {
+        ircService = {
-          level = "debug";
+          ident = {
-          maxFiles = 5;
+            address = "::";
-          toCosole = true;
+            enabled = false;
-        };
+            port = 1113;
-        matrixHandler = {
+          };
-          eventCacheSize = 4096;
+          logging = {
-        };
+            # set to debug for debugging
-        metrics = {
+            level = "warn";
-          enabled = true;
+            maxFiles = 5;
-          remoteUserAgeBuckets = [
+            toCosole = true;
-            "1h"
+          };
-            "1d"
+          matrixHandler = {
-            "1w"
+            eventCacheSize = 4096;
-          ];
+          };
-        };
+          mediaProxy = {
-        provisioning = {
+            signingKeyPath = config.pub-solar-os.matrix.appservice-irc.mediaproxy.signingKeyPath;
-          enabled = false;
+            # keep media for 2 weeks
-          requestTimeoutSeconds = 300;
+            ttlSeconds = 1209600;
-        };
+            bindPort = 11111;
-        servers =
+            publicUrl = "https:///matrix.${config.pub-solar-os.networking.domain}/media";
-          let
+          };
-            commonConfig = {
+          metrics = {
-              allowExpiredCerts = false;
+            enabled = true;
-              botConfig = {
+            remoteUserAgeBuckets = [
-                enabled = false;
+              "1h"
-                joinChannelsIfNoUsers = false;
+              "1d"
-                nick = "MatrixBot";
+              "1w"
-              };
+            ];
-              dynamicChannels = {
+          };
-                createAlias = true;
+          provisioning = {
-                enabled = true;
+            enabled = false;
-                federate = true;
+            requestTimeoutSeconds = 300;
-                joinRule = "public";
+          };
-                published = true;
+          servers =
-              };
+            let
-              ircClients = {
+              commonConfig = {
-                allowNickChanges = true;
+                allowExpiredCerts = false;
-                concurrentReconnectLimit = 50;
+                botConfig = {
-                idleTimeout = 10800;
+                  enabled = false;
-                lineLimit = 3;
+                  joinChannelsIfNoUsers = false;
-                maxClients = 30;
+                  nick = "MatrixBot";
-                nickTemplate = "$DISPLAY[m]";
+                };
-                reconnectIntervalMs = 5000;
+                dynamicChannels = {
-              };
+                  createAlias = true;
-              matrixClients = {
+                  enabled = true;
-                joinAttempts = -1;
+                  federate = true;
-              };
+                  joinRule = "public";
-              membershipLists = {
+                  published = true;
-                enabled = true;
+                };
-                floodDelayMs = 10000;
+                ircClients = {
-                global = {
+                  allowNickChanges = true;
-                  ircToMatrix = {
+                  concurrentReconnectLimit = 50;
-                    incremental = true;
+                  idleTimeout = 10800;
-                    initial = true;
+                  lineLimit = 3;
-                  };
+                  maxClients = 30;
-                  matrixToIrc = {
+                  nickTemplate = "$DISPLAY[m]";
-                    incremental = true;
+                  reconnectIntervalMs = 5000;
-                    initial = true;
+                };
+                matrixClients = {
+                  joinAttempts = -1;
+                };
+                membershipLists = {
+                  enabled = true;
+                  floodDelayMs = 10000;
+                  global = {
+                    ircToMatrix = {
+                      incremental = true;
+                      initial = true;
+                    };
+                    matrixToIrc = {
+                      incremental = true;
+                      initial = true;
+                    };
                   };
                 };
+                port = 6697;
+                privateMessages = {
+                  enabled = true;
+                  federate = true;
+                };
+                sasl = false;
+                sendConnectionMessages = true;
+                ssl = true;
               };
-              port = 6697;
+            in
-              privateMessages = {
+            {
-                enabled = true;
+              "irc.libera.chat" = lib.attrsets.recursiveUpdate commonConfig {
-                federate = true;
+                name = "libera";
+                dynamicChannels.groupId = "+libera.chat:localhost";
+                dynamicChannels.aliasTemplate = "#_libera_$CHANNEL";
+                matrixClients.displayName = "$NICK (LIBERA-IRC)";
+              };
+              "irc.scratch-network.net" = lib.attrsets.recursiveUpdate commonConfig {
+                name = "scratch";
+                matrixClients.displayName = "$NICK (SCRATCH-IRC)";
+                dynamicChannels.aliasTemplate = "#_scratch_$CHANNEL";
+                dynamicChannels.groupId = "+scratch-network.net:localhost";
               };
-              sasl = false;
-              sendConnectionMessages = true;
-              ssl = true;
             };
-          in
+        };
-          {
-            "irc.libera.chat" = lib.attrsets.recursiveUpdate commonConfig {
-              name = "libera";
-              dynamicChannels.groupId = "+libera.chat:localhost";
-              dynamicChannels.aliasTemplate = "#_libera_$CHANNEL";
-              matrixClients.displayName = "$NICK (LIBERA-IRC)";
-            };
-            "irc.scratch-network.net" = lib.attrsets.recursiveUpdate commonConfig {
-              name = "scratch";
-              matrixClients.displayName = "$NICK (SCRATCH-IRC)";
-              dynamicChannels.aliasTemplate = "#_scratch_$CHANNEL";
-              dynamicChannels.groupId = "+scratch-network.net:localhost";
-            };
-          };
       };
     };
   };

modules/matrix/default.nix

@@ -1,6 +1,7 @@
 {
   flake,
   config,
+  lib,
   pkgs,
   ...
 }:
@@ -9,304 +10,355 @@ let
   serverDomain = "${config.pub-solar-os.networking.domain}";
 in
 {
-  age.secrets."matrix-synapse-signing-key" = {
+  options.pub-solar-os = {
-    file = "${flake.self}/secrets/matrix-synapse-signing-key.age";
+    matrix = {
-    mode = "400";
+      enable = lib.mkEnableOption "Enable matrix-synapse and matrix-authentication-service to run on the node";
-    owner = "matrix-synapse";
+
+      synapse = {
+        app-service-config-files = lib.mkOption {
+          description = "List of app service config files";
+          type = lib.types.listOf lib.types.str;
+          default = [ ];
+        };
+
+        extra-config-files = lib.mkOption {
+          description = "List of extra synapse config files";
+          type = lib.types.listOf lib.types.str;
+          default = [ ];
+        };
+
+        signing_key_path = lib.mkOption {
+          description = "Path to file containing the signing key";
+          type = lib.types.str;
+          default = "${config.services.matrix-synapse.dataDir}/homeserver.signing.key";
+        };
+      };
+
+      matrix-authentication-service = {
+        extra-config-files = lib.mkOption {
+          description = "List of extra mas config files";
+          type = lib.types.listOf lib.types.str;
+          default = [ ];
+        };
+      };
+    };
   };
 
-  age.secrets."matrix-synapse-secret-config.yaml" = {
+  config = lib.mkIf config.pub-solar-os.matrix.enable {
-    file = "${flake.self}/secrets/matrix-synapse-secret-config.yaml.age";
+    services.matrix-synapse = {
-    mode = "400";
+      enable = true;
-    owner = "matrix-synapse";
+      settings = {
-  };
+        server_name = serverDomain;
-
+        public_baseurl = "https://${publicDomain}/";
-  age.secrets."matrix-synapse-sliding-sync-secret" = {
+        database = {
-    file = "${flake.self}/secrets/matrix-synapse-sliding-sync-secret.age";
+          name = "psycopg2";
-    mode = "400";
+          args = {
-    owner = "matrix-synapse";
+            host = "/run/postgresql";
-  };
+            cp_max = 10;
-
+            cp_min = 5;
-  services.matrix-synapse = {
+            database = "matrix";
-    enable = true;
+          };
-    settings = {
+          allow_unsafe_locale = false;
-      server_name = serverDomain;
+          txn_limit = 0;
-      public_baseurl = "https://${publicDomain}/";
-      database = {
-        name = "psycopg2";
-        args = {
-          host = "/run/postgresql";
-          cp_max = 10;
-          cp_min = 5;
-          database = "matrix";
         };
-        allow_unsafe_locale = false;
+        listeners = [
-        txn_limit = 0;
+          {
-      };
+            bind_addresses = [ "127.0.0.1" ];
-      listeners = [
+            port = 8008;
-        {
+            resources = [
-          bind_addresses = [ "127.0.0.1" ];
+              {
-          port = 8008;
+                compress = true;
-          resources = [
+                names = [ "client" ];
-            {
+              }
-              compress = true;
+              {
-              names = [ "client" ];
+                compress = false;
-            }
+                names = [ "federation" ];
-            {
+              }
-              compress = false;
+            ];
-              names = [ "federation" ];
+            tls = false;
-            }
+            type = "http";
-          ];
+            x_forwarded = true;
-          tls = false;
+          }
-          type = "http";
+          {
-          x_forwarded = true;
+            bind_addresses = [ "127.0.0.1" ];
-        }
+            port = 8012;
-        {
+            resources = [ { names = [ "metrics" ]; } ];
-          bind_addresses = [ "127.0.0.1" ];
+            tls = false;
-          port = 8012;
+            type = "metrics";
-          resources = [ { names = [ "metrics" ]; } ];
+          }
-          tls = false;
+        ];
-          type = "metrics";
-        }
-      ];
 
-      account_threepid_delegates.msisdn = "";
+        account_threepid_delegates.msisdn = "";
-      alias_creation_rules = [
+        alias_creation_rules = [
-        {
+          {
-          action = "allow";
+            action = "allow";
-          alias = "*";
+            alias = "*";
-          room_id = "*";
+            room_id = "*";
-          user_id = "*";
+            user_id = "*";
-        }
+          }
-      ];
+        ];
-      allow_guest_access = false;
+        allow_guest_access = false;
-      allow_public_rooms_over_federation = true;
+        allow_public_rooms_over_federation = true;
-      allow_public_rooms_without_auth = false;
+        allow_public_rooms_without_auth = false;
-      auto_join_rooms = [
+        auto_join_rooms = [
-        "#community:${serverDomain}"
+          "#community:${serverDomain}"
-        "#general:${serverDomain}"
+          "#general:${serverDomain}"
-      ];
+        ];
 
-      autocreate_auto_join_rooms = true;
+        autocreate_auto_join_rooms = true;
-      caches.global_factor = 0.5;
+        caches.global_factor = 0.5;
 
-      default_room_version = "10";
+        default_room_version = "10";
-      disable_msisdn_registration = true;
+        disable_msisdn_registration = true;
-      enable_media_repo = true;
+        enable_media_repo = true;
-      enable_metrics = true;
+        enable_metrics = true;
-      mau_stats_only = true;
+        mau_stats_only = true;
-      enable_registration = false;
+        enable_registration = false;
-      enable_registration_captcha = false;
+        enable_registration_captcha = false;
-      enable_registration_without_verification = false;
+        enable_registration_without_verification = false;
-      enable_room_list_search = true;
+        enable_room_list_search = true;
-      encryption_enabled_by_default_for_room_type = "off";
+        encryption_enabled_by_default_for_room_type = "off";
-      event_cache_size = "100K";
+        event_cache_size = "100K";
-      federation_rr_transactions_per_room_per_second = 50;
-      federation_client_minimum_tls_version = "1.2";
-      forget_rooms_on_leave = true;
-      include_profile_data_on_invite = true;
-      instance_map = { };
-      limit_profile_requests_to_users_who_share_rooms = false;
 
-      max_spider_size = "10M";
+        # https://github.com/element-hq/synapse/issues/11203
-      max_upload_size = "50M";
+        # No YAML deep-merge, so this needs to be in secret extraConfigFiles
-      media_storage_providers = [ ];
+        # together with msc3861
+        #experimental_features = {
+        #  # Room summary API
+        #  msc3266_enabled = true;
+        #  # Rendezvous server for QR Code generation
+        #  msc4108_enabled = true;
+        #};
 
-      password_config = {
+        federation_rr_transactions_per_room_per_second = 50;
-        enabled = false;
+        federation_client_minimum_tls_version = "1.2";
-        localdb_enabled = false;
+        forget_rooms_on_leave = true;
-        pepper = "";
+        include_profile_data_on_invite = true;
-      };
+        instance_map = { };
+        limit_profile_requests_to_users_who_share_rooms = false;
 
-      presence.enabled = true;
+        max_spider_size = "10M";
-      push.include_content = false;
+        max_upload_size = "50M";
+        media_storage_providers = [ ];
 
-      rc_admin_redaction = {
+        password_config = {
-        burst_count = 50;
+          enabled = false;
-        per_second = 1;
+          localdb_enabled = false;
-      };
+          pepper = "";
-      rc_federation = {
+        };
-        concurrent = 3;
+
-        reject_limit = 50;
+        presence.enabled = true;
-        sleep_delay = 500;
+        push.include_content = false;
-        sleep_limit = 10;
+
-        window_size = 1000;
+        rc_admin_redaction = {
-      };
+          burst_count = 50;
-      rc_invites = {
+          per_second = 1;
-        per_issuer = {
+        };
+        rc_federation = {
+          concurrent = 3;
+          reject_limit = 50;
+          sleep_delay = 500;
+          sleep_limit = 10;
+          window_size = 1000;
+        };
+        rc_invites = {
+          per_issuer = {
+            burst_count = 10;
+            per_second = 0.3;
+          };
+          per_room = {
+            burst_count = 10;
+            per_second = 0.3;
+          };
+          per_user = {
+            burst_count = 5;
+            per_second = 3.0e-3;
+          };
+        };
+        rc_joins = {
+          local = {
+            burst_count = 10;
+            per_second = 0.1;
+          };
+          remote = {
+            burst_count = 10;
+            per_second = 1.0e-2;
+          };
+        };
+        rc_login = {
+          account = {
+            burst_count = 3;
+            per_second = 0.17;
+          };
+          address = {
+            burst_count = 3;
+            per_second = 0.17;
+          };
+          failed_attempts = {
+            burst_count = 3;
+            per_second = 0.17;
+          };
+        };
+        rc_message = {
           burst_count = 10;
-          per_second = 0.3;
+          per_second = 0.2;
         };
-        per_room = {
+        rc_registration = {
-          burst_count = 10;
-          per_second = 0.3;
-        };
-        per_user = {
-          burst_count = 5;
-          per_second = 3.0e-3;
-        };
-      };
-      rc_joins = {
-        local = {
-          burst_count = 10;
-          per_second = 0.1;
-        };
-        remote = {
-          burst_count = 10;
-          per_second = 1.0e-2;
-        };
-      };
-      rc_login = {
-        account = {
           burst_count = 3;
           per_second = 0.17;
         };
-        address = {
+        redaction_retention_period = "7d";
-          burst_count = 3;
+        forgotten_room_retention_period = "7d";
-          per_second = 0.17;
+        redis.enabled = false;
-        };
+        registration_requires_token = false;
-        failed_attempts = {
+        registrations_require_3pid = [ "email" ];
-          burst_count = 3;
+        report_stats = false;
-          per_second = 0.17;
+        require_auth_for_profile_requests = false;
+        room_list_publication_rules = [
+          {
+            action = "allow";
+            alias = "*";
+            room_id = "*";
+            user_id = "*";
+          }
+        ];
+
+        signing_key_path = config.pub-solar-os.matrix.synapse.signing_key_path;
+
+        stream_writers = { };
+        trusted_key_servers = [ { server_name = "matrix.org"; } ];
+        suppress_key_server_warning = true;
+
+        turn_allow_guests = false;
+        turn_uris = [
+          "turn:${config.services.coturn.realm}:3478?transport=udp"
+          "turn:${config.services.coturn.realm}:3478?transport=tcp"
+        ];
+        turn_user_lifetime = "1h";
+
+        url_preview_accept_language = [
+          "en-US"
+          "en"
+        ];
+        url_preview_enabled = true;
+        url_preview_ip_range_blacklist = [
+          "127.0.0.0/8"
+          "10.0.0.0/8"
+          "172.16.0.0/12"
+          "192.168.0.0/16"
+          "100.64.0.0/10"
+          "192.0.0.0/24"
+          "169.254.0.0/16"
+          "192.88.99.0/24"
+          "198.18.0.0/15"
+          "192.0.2.0/24"
+          "198.51.100.0/24"
+          "203.0.113.0/24"
+          "224.0.0.0/4"
+          "::1/128"
+          "fe80::/10"
+          "fc00::/7"
+          "2001:db8::/32"
+          "ff00::/8"
+          "fec0::/10"
+        ];
+
+        user_directory = {
+          prefer_local_users = false;
+          search_all_users = false;
         };
+        user_ips_max_age = "28d";
+
+        app_service_config_files = config.pub-solar-os.matrix.synapse.app-service-config-files;
       };
-      rc_message = {
+
-        burst_count = 10;
+      withJemalloc = true;
-        per_second = 0.2;
+
-      };
+      extraConfigFiles = config.pub-solar-os.matrix.synapse.extra-config-files;
-      rc_registration = {
+
-        burst_count = 3;
+      extras = [
-        per_second = 0.17;
+        "oidc"
-      };
+        "redis"
-      redaction_retention_period = "7d";
-      forgotten_room_retention_period = "7d";
-      redis.enabled = false;
-      registration_requires_token = false;
-      registrations_require_3pid = [ "email" ];
-      report_stats = false;
-      require_auth_for_profile_requests = false;
-      room_list_publication_rules = [
-        {
-          action = "allow";
-          alias = "*";
-          room_id = "*";
-          user_id = "*";
-        }
       ];
 
-      signing_key_path = "/run/agenix/matrix-synapse-signing-key";
+      plugins = [ config.services.matrix-synapse.package.plugins.matrix-synapse-shared-secret-auth ];
-
-      stream_writers = { };
-      trusted_key_servers = [ { server_name = "matrix.org"; } ];
-      suppress_key_server_warning = true;
-
-      turn_allow_guests = false;
-      turn_uris = [
-        "turn:${config.services.coturn.realm}:3478?transport=udp"
-        "turn:${config.services.coturn.realm}:3478?transport=tcp"
-      ];
-      turn_user_lifetime = "1h";
-
-      url_preview_accept_language = [
-        "en-US"
-        "en"
-      ];
-      url_preview_enabled = true;
-      url_preview_ip_range_blacklist = [
-        "127.0.0.0/8"
-        "10.0.0.0/8"
-        "172.16.0.0/12"
-        "192.168.0.0/16"
-        "100.64.0.0/10"
-        "192.0.0.0/24"
-        "169.254.0.0/16"
-        "192.88.99.0/24"
-        "198.18.0.0/15"
-        "192.0.2.0/24"
-        "198.51.100.0/24"
-        "203.0.113.0/24"
-        "224.0.0.0/4"
-        "::1/128"
-        "fe80::/10"
-        "fc00::/7"
-        "2001:db8::/32"
-        "ff00::/8"
-        "fec0::/10"
-      ];
-
-      user_directory = {
-        prefer_local_users = false;
-        search_all_users = false;
-      };
-      user_ips_max_age = "28d";
-
-      app_service_config_files = [
-        "/var/lib/matrix-synapse/telegram-registration.yaml"
-        "/var/lib/matrix-appservice-irc/registration.yml"
-        # "/matrix-appservice-slack-registration.yaml"
-        # "/hookshot-registration.yml"
-        # "/matrix-mautrix-signal-registration.yaml"
-        # "/matrix-mautrix-telegram-registration.yaml"
-      ];
     };
 
-    withJemalloc = true;
+    services.matrix-authentication-service = {
+      enable = true;
+      createDatabase = true;
+      extraConfigFiles = config.pub-solar-os.matrix.matrix-authentication-service.extra-config-files;
 
-    extraConfigFiles = [
+      # https://element-hq.github.io/matrix-authentication-service/reference/configuration.html
-      "/run/agenix/matrix-synapse-secret-config.yaml"
+      settings = {
-
+        account.email_change_allowed = false;
-      # The registration file is automatically generated after starting the
+        http.public_base = "https://mas.${config.pub-solar-os.networking.domain}";
-      # appservice for the first time.
+        http.issuer = "https://mas.${config.pub-solar-os.networking.domain}";
-      # cp /var/lib/mautrix-telegram/telegram-registration.yaml \
+        http.listeners = [
-      #   /var/lib/matrix-synapse/
+          {
-      # chown matrix-synapse:matrix-synapse \
+            name = "web";
-      #   /var/lib/matrix-synapse/telegram-registration.yaml
+            resources = [
-      "/var/lib/matrix-synapse/telegram-registration.yaml"
+              { name = "discovery"; }
-    ];
+              { name = "human"; }
-
+              { name = "oauth"; }
-    extras = [
+              { name = "compat"; }
-      "oidc"
+              { name = "graphql"; }
-      "redis"
+              {
-    ];
+                name = "assets";
-
+                path = "${config.services.matrix-authentication-service.package}/share/matrix-authentication-service/assets";
-    plugins = [ config.services.matrix-synapse.package.plugins.matrix-synapse-shared-secret-auth ];
+              }
-  };
+            ];
-
+            binds = [
-  services.matrix-sliding-sync = {
+              {
-    enable = true;
+                host = "0.0.0.0";
-    settings = {
+                port = 8090;
-      SYNCV3_SERVER = "https://${publicDomain}";
+              }
-      SYNCV3_BINDADDR = "127.0.0.1:8011";
+            ];
-      # The bind addr for Prometheus metrics, which will be accessible at
+            proxy_protocol = false;
-      # /metrics at this address
+          }
-      SYNCV3_PROM = "127.0.0.1:9100";
+          {
+            name = "internal";
+            resources = [
+              { name = "health"; }
+            ];
+            binds = [
+              {
+                host = "0.0.0.0";
+                port = 8081;
+              }
+            ];
+            proxy_protocol = false;
+          }
+        ];
+        passwords.enabled = false;
+      };
     };
-    environmentFile = config.age.secrets."matrix-synapse-sliding-sync-secret".path;
-  };
 
-  services.restic.backups.matrix-synapse-storagebox = {
+    pub-solar-os.backups.restic.matrix-synapse = {
-    paths = [
+      paths = [
-      "/var/lib/matrix-synapse"
+        "/var/lib/matrix-synapse"
-      "/var/lib/matrix-appservice-irc"
+        "/var/lib/matrix-appservice-irc"
-      "/var/lib/mautrix-telegram"
+        "/var/lib/mautrix-telegram"
-      "/tmp/matrix-synapse-backup.sql"
+        "/tmp/matrix-synapse-backup.sql"
-    ];
+        "/tmp/matrix-authentication-service-backup.sql"
-    timerConfig = {
+      ];
-      OnCalendar = "*-*-* 05:00:00 Etc/UTC";
+      timerConfig = {
+        OnCalendar = "*-*-* 05:00:00 Etc/UTC";
+      };
+      initialize = true;
+      backupPrepareCommand = ''
+        ${pkgs.sudo}/bin/sudo -u postgres ${pkgs.postgresql}/bin/pg_dump -d matrix > /tmp/matrix-synapse-backup.sql
+        ${pkgs.sudo}/bin/sudo -u postgres ${pkgs.postgresql}/bin/pg_dump -d matrix-authentication-service > /tmp/matrix-authentication-service-backup.sql
+      '';
+      backupCleanupCommand = ''
+        rm /tmp/matrix-synapse-backup.sql
+        rm /tmp/matrix-authentication-service-backup.sql
+      '';
+      pruneOpts = [
+        "--keep-daily 7"
+        "--keep-weekly 4"
+        "--keep-monthly 3"
+      ];
     };
-    initialize = true;
-    passwordFile = config.age.secrets."restic-repo-storagebox".path;
-    repository = "sftp:u377325@u377325.your-storagebox.de:/backups";
-    backupPrepareCommand = ''
-      ${pkgs.sudo}/bin/sudo -u postgres ${pkgs.postgresql}/bin/pg_dump -d matrix > /tmp/matrix-synapse-backup.sql
-    '';
-    backupCleanupCommand = ''
-      rm /tmp/matrix-synapse-backup.sql
-    '';
-    pruneOpts = [
-      "--keep-daily 7"
-      "--keep-weekly 4"
-      "--keep-monthly 3"
-    ];
   };
 }

modules/mediawiki/default.nix

@@ -139,6 +139,10 @@ let
       // https://www.mediawiki.org/wiki/Extension:PluggableAuth#Configuration
       $wgPluggableAuth_EnableAutoLogin = false;
       $wgPluggableAuth_ButtonLabel = 'Login with pub.solar ID';
+      // Avoid getting logged out after 30 minutes
+      // https://www.mediawiki.org/wiki/Topic:W4be4h6t63vf3y8p
+      // https://www.mediawiki.org/wiki/Manual:$wgRememberMe
+      $wgRememberMe = 'always';
 
       // https://www.mediawiki.org/wiki/Extension:OpenID_Connect#Keycloak
       $wgPluggableAuth_Config[] = [
@@ -211,7 +215,7 @@ in
       backend = "docker";
 
       containers."mediawiki" = {
-        image = "git.pub.solar/pub-solar/mediawiki-oidc-docker:1.41.1";
+        image = "git.pub.solar/pub-solar/mediawiki-oidc-docker:1.43.0";
         user = "1000:${builtins.toString gid}";
         autoStart = true;
 
@@ -232,4 +236,27 @@ in
       };
     };
   };
+
+  pub-solar-os.backups.restic.mediawiki = {
+    paths = [
+      "/var/lib/mediawiki/images"
+      "/var/lib/mediawiki/uploads"
+      "/tmp/mediawiki-backup.sql"
+    ];
+    timerConfig = {
+      OnCalendar = "*-*-* 00:30:00 Etc/UTC";
+    };
+    initialize = true;
+    backupPrepareCommand = ''
+      ${pkgs.sudo}/bin/sudo -u postgres ${pkgs.postgresql}/bin/pg_dump -d mediawiki > /tmp/mediawiki-backup.sql
+    '';
+    backupCleanupCommand = ''
+      rm /tmp/mediawiki-backup.sql
+    '';
+    pruneOpts = [
+      "--keep-daily 7"
+      "--keep-weekly 4"
+      "--keep-monthly 3"
+    ];
+  };
 }

modules/nextcloud/default.nix

@@ -2,6 +2,7 @@
   config,
   pkgs,
   flake,
+  lib,
   ...
 }:
 {
@@ -22,118 +23,226 @@
     forceSSL = true;
   };
 
-  services.nextcloud = {
+  services.nextcloud =
-    hostName = "cloud.${config.pub-solar-os.networking.domain}";
+    let
-    home = "/var/lib/nextcloud";
+      exiftool_1270 = pkgs.perlPackages.buildPerlPackage rec {
+        # NOTE nextcloud-memories needs this specific version of exiftool
+        pname = "Image-ExifTool";
+        version = "12.70";
+        src = pkgs.fetchFromGitHub {
+          owner = "exiftool";
+          repo = "exiftool";
+          rev = version;
+          hash = "sha256-YMWYPI2SDi3s4KCpSNwovemS5MDj5W9ai0sOkvMa8Zg=";
+        };
+        nativeBuildInputs = lib.optional pkgs.stdenv.hostPlatform.isDarwin pkgs.shortenPerlShebang;
+        postInstall = lib.optionalString pkgs.stdenv.hostPlatform.isDarwin ''
+          shortenPerlShebang $out/bin/exiftool
+        '';
+      };
+    in
+    {
+      hostName = "cloud.${config.pub-solar-os.networking.domain}";
+      home = "/var/lib/nextcloud";
 
-    enable = true;
-    package = pkgs.nextcloud29;
-    https = true;
-    secretFile = config.age.secrets."nextcloud-secrets".path; # secret
-    maxUploadSize = "1G";
-
-    configureRedis = true;
-
-    notify_push = {
       enable = true;
-      bendDomainToLocalhost = true;
+      # When updating package, remember to update nextcloud30Packages in
+      # services.nextcloud.extraApps
+      package = pkgs.nextcloud30;
+      https = true;
+      secretFile = config.age.secrets."nextcloud-secrets".path; # secret
+      maxUploadSize = "1G";
+
+      configureRedis = true;
+
+      notify_push = {
+        enable = true;
+        bendDomainToLocalhost = true;
+      };
+
+      config = {
+        adminuser = "admin";
+        adminpassFile = config.age.secrets."nextcloud-admin-pass".path;
+        dbuser = "nextcloud";
+        dbtype = "pgsql";
+        dbname = "nextcloud";
+      };
+
+      settings = {
+        overwrite.cli.url = "http://cloud.${config.pub-solar-os.networking.domain}";
+        overwriteprotocol = "https";
+
+        installed = true;
+        default_phone_region = "+49";
+        mail_sendmailmode = "smtp";
+        mail_from_address = "nextcloud";
+        mail_smtpmode = "smtp";
+        mail_smtpauthtype = "PLAIN";
+        mail_domain = "pub.solar";
+        mail_smtpname = "admins@pub.solar";
+        mail_smtpsecure = "ssl";
+        mail_smtpauth = true;
+        mail_smtphost = "mail.pub.solar";
+        mail_smtpport = "465";
+
+        # This is to allow connections to collabora and keycloak, among other services
+        # running on the same host
+        #
+        # https://docs.nextcloud.com/server/stable/admin_manual/configuration_server/config_sample_php_parameters.html?highlight=allow_local_remote_servers%20true
+        # https://github.com/ONLYOFFICE/onlyoffice-nextcloud/issues/293
+        allow_local_remote_servers = true;
+
+        enable_previews = true;
+        jpeg_quality = 60;
+        enabledPreviewProviders = [
+          "OC\\Preview\\PNG"
+          "OC\\Preview\\JPEG"
+          "OC\\Preview\\GIF"
+          "OC\\Preview\\BMP"
+          "OC\\Preview\\HEIC"
+          "OC\\Preview\\TIFF"
+          "OC\\Preview\\XBitmap"
+          "OC\\Preview\\SVG"
+          "OC\\Preview\\WebP"
+          "OC\\Preview\\Font"
+          "OC\\Preview\\Movie"
+          "OC\\Preview\\ImaginaryPDF"
+          "OC\\Preview\\MP3"
+          "OC\\Preview\\OpenDocument"
+          "OC\\Preview\\Krita"
+          "OC\\Preview\\TXT"
+          "OC\\Preview\\MarkDown"
+          "OC\\Preview\\Imaginary"
+        ];
+        preview_imaginary_url = "http://127.0.0.1:${toString config.services.imaginary.port}/";
+        preview_max_filesize_image = 128; # MB
+        preview_max_memory = 512; # MB
+        preview_max_x = 2048; # px
+        preview_max_y = 2048; # px
+        preview_max_scale_factor = 1;
+        "preview_ffmpeg_path" = lib.getExe pkgs.ffmpeg-headless;
+
+        "memories.exiftool_no_local" = false;
+        "memories.exiftool" = "${exiftool_1270}/bin/exiftool";
+        "memories.vod.ffmpeg" = lib.getExe pkgs.ffmpeg;
+        "memories.vod.ffprobe" = lib.getExe' pkgs.ffmpeg-headless "ffprobe";
+
+        auth.bruteforce.protection.enabled = true;
+        trashbin_retention_obligation = "auto,7";
+        skeletondirectory = "./nextcloud-skeleton";
+        defaultapp = "file";
+        activity_expire_days = "14";
+        integrity.check.disabled = false;
+        updater.release.channel = "stable";
+        loglevel = 2;
+        debug = false;
+        maintenance_window_start = "1";
+        # maintenance = false;
+        app_install_overwrite = [
+          "pdfdraw"
+          "integration_whiteboard"
+        ];
+        htaccess.RewriteBase = "/";
+        theme = "";
+        simpleSignUpLink.shown = false;
+      };
+
+      phpOptions = {
+        "opcache.interned_strings_buffer" = "32";
+        "opcache.max_accelerated_files" = "16229";
+        "opcache.memory_consumption" = "256";
+        # https://docs.nextcloud.com/server/latest/admin_manual/installation/server_tuning.html#enable-php-opcache
+        "opcache.revalidate_freq" = "60";
+        # https://docs.nextcloud.com/server/latest/admin_manual/installation/server_tuning.html#:~:text=opcache.jit%20%3D%201255%20opcache.jit_buffer_size%20%3D%20128m
+        "opcache.jit" = "1255";
+        "opcache.jit_buffer_size" = "128M";
+      };
+
+      # Calculated with 4GiB RAM, 80MiB process size available on
+      # https://spot13.com/pmcalculator/
+      poolSettings = {
+        pm = "dynamic";
+        "pm.max_children" = "52";
+        "pm.max_requests" = "500";
+        "pm.max_spare_servers" = "39";
+        "pm.min_spare_servers" = "13";
+        "pm.start_servers" = "13";
+      };
+
+      caching.redis = true;
+      appstoreEnable = true;
+      autoUpdateApps.enable = true;
+      extraApps = {
+        inherit (pkgs.nextcloud30Packages.apps) memories previewgenerator recognize;
+      };
+      database.createLocally = true;
     };
 
-    config = {
+  # https://docs.nextcloud.com/server/30/admin_manual/installation/server_tuning.html#previews
-      adminuser = "admin";
+  services.imaginary = {
-      adminpassFile = config.age.secrets."nextcloud-admin-pass".path;
+    enable = true;
-      dbuser = "nextcloud";
+    address = "127.0.0.1";
-      dbtype = "pgsql";
+    settings.return-size = true;
-      dbname = "nextcloud";
+  };
-      dbtableprefix = "oc_";
+
+  systemd = {
+    services =
+      let
+        occ = "/run/current-system/sw/bin/nextcloud-occ";
+      in
+      {
+        nextcloud-cron-preview-generator = {
+          environment.NEXTCLOUD_CONFIG_DIR = "${config.services.nextcloud.home}/config";
+          serviceConfig = {
+            ExecStart = "${occ} preview:pre-generate";
+            Type = "oneshot";
+            User = "nextcloud";
+          };
+        };
+
+        nextcloud-preview-generator-setup = {
+          wantedBy = [ "multi-user.target" ];
+          requires = [ "phpfpm-nextcloud.service" ];
+          after = [ "phpfpm-nextcloud.service" ];
+          environment.NEXTCLOUD_CONFIG_DIR = "${config.services.nextcloud.home}/config";
+          script = # bash
+            ''
+              # check with:
+              # for size in squareSizes widthSizes heightSizes; do echo -n "$size: "; nextcloud-occ config:app:get previewgenerator $size; done
+
+              # extra commands run for preview generator:
+              # 32   icon file list
+              # 64   icon file list android app, photos app
+              # 96   nextcloud client VFS windows file preview
+              # 256  file app grid view, many requests
+              # 512  photos app tags
+              ${occ} config:app:set --value="32 64 96 256 512" previewgenerator squareSizes
+
+              # 341 hover in maps app
+              # 1920 files/photos app when viewing picture
+              ${occ} config:app:set --value="341 1920" previewgenerator widthSizes
+
+              # 256 hover in maps app
+              # 1080 files/photos app when viewing picture
+              ${occ} config:app:set --value="256 1080" previewgenerator heightSizes
+            '';
+          serviceConfig = {
+            Type = "oneshot";
+            User = "nextcloud";
+          };
+        };
+      };
+    timers.nextcloud-cron-preview-generator = {
+      after = [ "nextcloud-setup.service" ];
+      timerConfig = {
+        OnCalendar = "*:0/10";
+        OnUnitActiveSec = "9m";
+        Persistent = true;
+        RandomizedDelaySec = 60;
+        Unit = "nextcloud-cron-preview-generator.service";
+      };
+      wantedBy = [ "timers.target" ];
     };
-
-    settings = {
-      overwrite.cli.url = "http://cloud.${config.pub-solar-os.networking.domain}";
-      overwriteprotocol = "https";
-
-      installed = true;
-      default_phone_region = "+49";
-      mail_sendmailmode = "smtp";
-      mail_from_address = "nextcloud";
-      mail_smtpmode = "smtp";
-      mail_smtpauthtype = "PLAIN";
-      mail_domain = "pub.solar";
-      mail_smtpname = "admins@pub.solar";
-      mail_smtpsecure = "ssl";
-      mail_smtpauth = true;
-      mail_smtphost = "mail.pub.solar";
-      mail_smtpport = "465";
-
-      # This is to allow connections to collabora and keycloak, among other services
-      # running on the same host
-      #
-      # https://docs.nextcloud.com/server/stable/admin_manual/configuration_server/config_sample_php_parameters.html?highlight=allow_local_remote_servers%20true
-      # https://github.com/ONLYOFFICE/onlyoffice-nextcloud/issues/293
-      allow_local_remote_servers = true;
-
-      enable_previews = true;
-      enabledPreviewProviders = [
-        "OC\\Preview\\PNG"
-        "OC\\Preview\\JPEG"
-        "OC\\Preview\\GIF"
-        "OC\\Preview\\BMP"
-        "OC\\Preview\\XBitmap"
-        "OC\\Preview\\Movie"
-        "OC\\Preview\\PDF"
-        "OC\\Preview\\MP3"
-        "OC\\Preview\\TXT"
-        "OC\\Preview\\MarkDown"
-      ];
-      preview_max_x = "1024";
-      preview_max_y = "768";
-      preview_max_scale_factor = "1";
-
-      auth.bruteforce.protection.enabled = true;
-      trashbin_retention_obligation = "auto,7";
-      skeletondirectory = "./nextcloud-skeleton";
-      defaultapp = "file";
-      activity_expire_days = "14";
-      integrity.check.disabled = false;
-      updater.release.channel = "stable";
-      loglevel = 2;
-      debug = false;
-      maintenance_window_start = "1";
-      # maintenance = false;
-      app_install_overwrite = [
-        "pdfdraw"
-        "integration_whiteboard"
-      ];
-      htaccess.RewriteBase = "/";
-      theme = "";
-      simpleSignUpLink.shown = false;
-    };
-
-    phpOptions = {
-      "opcache.interned_strings_buffer" = "32";
-      "opcache.max_accelerated_files" = "16229";
-      "opcache.memory_consumption" = "256";
-      # https://docs.nextcloud.com/server/latest/admin_manual/installation/server_tuning.html#enable-php-opcache
-      "opcache.revalidate_freq" = "60";
-      # https://docs.nextcloud.com/server/latest/admin_manual/installation/server_tuning.html#:~:text=opcache.jit%20%3D%201255%20opcache.jit_buffer_size%20%3D%20128m
-      "opcache.jit" = "1255";
-      "opcache.jit_buffer_size" = "128M";
-    };
-
-    # Calculated with 4GiB RAM, 80MiB process size available on
-    # https://spot13.com/pmcalculator/
-    poolSettings = {
-      pm = "dynamic";
-      "pm.max_children" = "52";
-      "pm.max_requests" = "500";
-      "pm.max_spare_servers" = "39";
-      "pm.min_spare_servers" = "13";
-      "pm.start_servers" = "13";
-    };
-
-    caching.redis = true;
-    autoUpdateApps.enable = true;
-    database.createLocally = true;
   };
 
   services.restic.backups.nextcloud-storagebox = {
@@ -145,7 +254,7 @@
       OnCalendar = "*-*-* 01:00:00 Etc/UTC";
     };
     initialize = true;
-    passwordFile = config.age.secrets."restic-repo-storagebox".path;
+    passwordFile = config.age.secrets."restic-repo-storagebox-nachtigall".path;
     repository = "sftp:u377325@u377325.your-storagebox.de:/backups";
     backupPrepareCommand = ''
       ${pkgs.sudo}/bin/sudo -u postgres ${pkgs.postgresql}/bin/pg_dump -d nextcloud > /tmp/nextcloud-backup.sql

modules/nginx-mastodon-files/default.nix

@@ -1,8 +1,7 @@
 { config, ... }:
 
 let
-  objStorHost = "link.tardigradeshare.io";
+  objStorHost = "mastodon.web.pub.solar";
-  objStorBucket = "s/jw24ad6l4a6zxsnd32cmf5hp5nsq/pub-solar-mastodon";
 in
 {
   services.nginx.virtualHosts = {
@@ -10,6 +9,12 @@ in
       enableACME = true;
       forceSSL = true;
 
+      # Use variable to force nginx to perform a DNS resolution on its value,
+      # the IP of the object storage provider may not always remain the same.
+      extraConfig = ''
+        set $s3_backend 'https://${objStorHost}';
+      '';
+
       locations = {
         "= /" = {
           index = "index.html";
@@ -25,7 +30,6 @@ in
               deny all;
             }
 
-            resolver 8.8.8.8;
             proxy_set_header Host ${objStorHost};
             proxy_set_header Connection \'\';
             proxy_set_header Authorization \'\';
@@ -40,7 +44,7 @@ in
             proxy_hide_header x-amz-bucket-region;
             proxy_hide_header x-amzn-requestid;
             proxy_ignore_headers Set-Cookie;
-            proxy_pass https://${objStorHost}/${objStorBucket}$request_uri?download;
+            proxy_pass $s3_backend$request_uri;
             proxy_intercept_errors off;
             proxy_ssl_protocols TLSv1.2 TLSv1.3;
             proxy_ssl_server_name on;

modules/nginx-matrix/default.nix

@@ -10,25 +10,20 @@ let
     add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always;
     add_header X-XSS-Protection "1; mode=block";
   '';
-  clientConfig = import ./element-client-config.nix { inherit lib pkgs; };
+  clientConfig = import ./element-client-config.nix { inherit config lib pkgs; };
   wellKnownClient = domain: {
     "m.homeserver".base_url = "https://matrix.${domain}";
     "m.identity_server".base_url = "https://matrix.${domain}";
-    "org.matrix.msc3575.proxy".url = "https://matrix.${domain}";
+    "org.matrix.msc2965.authentication" = {
+      issuer = "https://mas.${domain}/";
+      account = "https://mas.${domain}/account";
+    };
     "im.vector.riot.e2ee".default = true;
     "io.element.e2ee" = {
       default = true;
       secure_backup_required = false;
       secure_backup_setup_methods = [ ];
     };
-    "m.integrations" = {
-      managers = [
-        {
-          api_url = "https://dimension.${domain}/api/v1/scalar";
-          ui_url = "https://dimension.${domain}/element";
-        }
-      ];
-    };
   };
   wellKnownServer = domain: { "m.server" = "matrix.${domain}:8448"; };
   wellKnownSupport = {
@@ -85,6 +80,27 @@ in
       root = pkgs.element-stickerpicker;
     };
 
+    "mas.${config.pub-solar-os.networking.domain}" = {
+      root = "/dev/null";
+
+      forceSSL = lib.mkDefault true;
+      enableACME = lib.mkDefault true;
+
+      locations = {
+        "/" = {
+          proxyPass = "http://127.0.0.1:8090";
+
+          extraConfig = ''
+            ${commonHeaders}
+            proxy_http_version 1.1;
+
+            # Forward the client IP address
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+          '';
+        };
+      };
+    };
+
     "matrix.${config.pub-solar-os.networking.domain}" = {
       root = "/dev/null";
 
@@ -99,28 +115,48 @@ in
       locations = {
         # For telegram
         "/c3c3f34b-29fb-5feb-86e5-98c75ec8214b" = {
+          priority = 100;
           proxyPass = "http://127.0.0.1:8009";
           extraConfig = commonHeaders;
         };
 
-        # sliding-sync
+        # For IRC appservice media proxy
-        "~ ^/(client/|_matrix/client/unstable/org.matrix.msc3575/sync)" = {
+        "/media" = {
-          proxyPass = "http://127.0.0.1:8011";
+          priority = 100;
+          proxyPass = "http://127.0.0.1:${toString (config.services.matrix-appservice-irc.settings.ircService.mediaProxy.bindPort)}";
           extraConfig = commonHeaders;
         };
 
-        "~* ^(/_matrix|/_synapse/client|/_synapse/oidc)" = {
+        # Forward to the auth service
+        "~ ^/_matrix/client/(.*)/(login|logout|refresh)" = {
+          priority = 100;
+          proxyPass = "http://127.0.0.1:8090";
+
+          extraConfig = ''
+            ${commonHeaders}
+            proxy_http_version 1.1;
+
+            # Forward the client IP address
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+          '';
+        };
+
+        # Forward to Synapse
+        # as per https://element-hq.github.io/synapse/latest/reverse_proxy.html#nginx
+        "~ ^(/_matrix|/_synapse/client)" = {
+          priority = 200;
           proxyPass = "http://127.0.0.1:8008";
 
           extraConfig = ''
             ${commonHeaders}
+            proxy_set_header X-Forwarded-For $remote_addr;
+            proxy_set_header X-Forwarded-Proto $scheme;
             proxy_set_header Host $host;
-            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
-            proxy_set_header X-Forwarded-Proto $http_x_forwarded_proto;
 
             client_body_buffer_size 25M;
             client_max_body_size 50M;
             proxy_max_temp_file_size 0;
+            proxy_http_version 1.1;
           '';
         };
       };

modules/nginx-matrix/element-client-config.nix

@@ -1,9 +1,14 @@
-{ pkgs, lib, ... }:
+{
+  config,
+  pkgs,
+  lib,
+  ...
+}:
 {
   default_server_config = {
     "m.homeserver" = {
-      base_url = "https://matrix.pub.solar";
+      base_url = "https://matrix.${config.pub-solar-os.networking.domain}";
-      server_name = "pub.solar";
+      server_name = "${config.pub-solar-os.networking.domain}";
     };
     "m.identity_server" = {
       base_url = "";
@@ -45,4 +50,15 @@
     # FUTUREWORK: Replace with pub.solar logo
     auth_header_logo_url = "themes/element/img/logos/element-logo.svg";
   };
+  # Enable Element Call Beta
+  features = {
+    feature_video_rooms = true;
+    feature_group_calls = true;
+    feature_element_call_video_rooms = true;
+  };
+  element_call = {
+    url = "https://call.element.io";
+    participant_limit = 50;
+    brand = "Element Call";
+  };
 }

modules/nginx-website/default.nix

@@ -7,7 +7,7 @@
   services.nginx.virtualHosts = {
     "www.${config.pub-solar-os.networking.domain}" = {
       enableACME = true;
-      addSSL = true;
+      forceSSL = true;
 
       extraConfig = ''
         error_log /dev/null;

modules/nginx/default.nix

@@ -10,9 +10,10 @@ let
   webserverGroup = "hakkonaut";
 in
 {
+  users.users.nginx.extraGroups = [ webserverGroup ];
+
   services.nginx = {
     enable = true;
-    group = webserverGroup;
     enableReload = true;
     proxyCachePath.cache = {
       enable = true;
@@ -21,6 +22,13 @@ in
     recommendedOptimisation = true;
     recommendedProxySettings = true;
     recommendedTlsSettings = true;
+    resolver.addresses = [
+      # quad9.net
+      "9.9.9.9"
+      "149.112.112.112"
+      "[2620:fe::fe]"
+      "[2620:fe::9]"
+    ];
     appendHttpConfig = ''
       # https://my.f5.com/manage/s/article/K51798430
       proxy_headers_hash_bucket_size 128;

modules/obs-portal/default.nix

@@ -147,4 +147,26 @@ in
       };
     };
   };
+
+  pub-solar-os.backups.restic.obs-portal = {
+    paths = [
+      "/var/lib/obs-portal/data"
+      "/tmp/obs-portal-backup.sql"
+    ];
+    timerConfig = {
+      OnCalendar = "*-*-* 01:30:00 Etc/UTC";
+    };
+    initialize = true;
+    backupPrepareCommand = ''
+      ${pkgs.docker}/bin/docker exec -i --user postgres obs-portal-db pg_dump obs > /tmp/obs-portal-backup.sql
+    '';
+    backupCleanupCommand = ''
+      rm /tmp/obs-portal-backup.sql
+    '';
+    pruneOpts = [
+      "--keep-daily 7"
+      "--keep-weekly 4"
+      "--keep-monthly 3"
+    ];
+  };
 }

modules/postgresql/default.nix

@@ -25,9 +25,4 @@
       full_page_writes = false;
     };
   };
-
-  systemd.services.postgresql = {
-    after = [ "var-lib-postgresql.mount" ];
-    requisite = [ "var-lib-postgresql.mount" ];
-  };
 }

modules/prometheus/alert-rules.nix

@@ -142,8 +142,8 @@ lib.mapAttrsToList
 
     cpu_using_90percent = {
       condition = ''100 - (avg by (instance) (irate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) >= 90'';
-      time = "10m";
+      time = "20m";
-      description = "{{$labels.instance}} is running with cpu usage > 90% for at least 10 minutes: {{$value}}";
+      description = "{{$labels.instance}} is running with cpu usage > 90% for at least 20 minutes: {{$value}}";
     };
 
     reboot = {
@@ -234,10 +234,10 @@ lib.mapAttrsToList
       };
     */
 
-    host_memory_under_memory_pressure = {
+    #host_memory_under_memory_pressure = {
-      condition = "rate(node_vmstat_pgmajfault[1m]) > 1000";
+    #  condition = "rate(node_vmstat_pgmajfault[1m]) > 1000";
-      description = "{{$labels.instance}}: The node is under heavy memory pressure. High rate of major page faults: {{$value}}";
+    #  description = "{{$labels.instance}}: The node is under heavy memory pressure. High rate of major page faults: {{$value}}";
-    };
+    #};
 
     # ext4_errors = {
     #   condition = "ext4_errors_value > 0";
@@ -250,4 +250,10 @@ lib.mapAttrsToList
     #   description =
     #     "alertmanager: number of active silences has changed: {{$value}}";
     # };
+
+    garage_cluster_healthy = {
+      condition = "cluster_healthy == 0";
+      time = "15m";
+      description = "garage cluster on {{$labels.instance}} is not healthy: {{$labels.result}}!";
+    };
   })

modules/prometheus/default.nix

@@ -12,15 +12,27 @@
     owner = "alertmanager";
   };
 
-  services.caddy.virtualHosts."alerts.${config.pub-solar-os.networking.domain}" = {
+  security.acme.certs = {
-    logFormat = lib.mkForce ''
+    "alerts.${config.pub-solar-os.networking.domain}" = {
-      output discard
+      # disable http challenge
-    '';
+      webroot = null;
-    extraConfig = ''
+      # enable dns challenge
-      bind 10.7.6.2 fd00:fae:fae:fae:fae:2::
+      dnsProvider = "namecheap";
-      tls internal
+    };
-      reverse_proxy :${toString config.services.prometheus.alertmanager.port}
+  };
-    '';
+
+  services.nginx.virtualHosts."alerts.${config.pub-solar-os.networking.domain}" = {
+    enableACME = true;
+    forceSSL = true;
+
+    listenAddresses = [
+      "10.7.6.5"
+      "[fd00:fae:fae:fae:fae:5::]"
+    ];
+
+    locations."/" = {
+      proxyPass = "http://127.0.0.1:${toString config.services.prometheus.alertmanager.port}";
+    };
   };
 
   services.prometheus = {
@@ -41,12 +53,6 @@
       {
         job_name = "node-exporter";
         static_configs = [
-          {
-            targets = [ "127.0.0.1:${toString config.services.prometheus.exporters.node.port}" ];
-            labels = {
-              instance = "flora-6";
-            };
-          }
           {
             targets = [ "nachtigall.wg.${config.pub-solar-os.networking.domain}" ];
             labels = {
@@ -69,6 +75,30 @@
               instance = "tankstelle";
             };
           }
+          {
+            targets = [
+              "trinkgenossin.wg.${config.pub-solar-os.networking.domain}:${toString config.services.prometheus.exporters.node.port}"
+            ];
+            labels = {
+              instance = "trinkgenossin";
+            };
+          }
+          {
+            targets = [
+              "delite.wg.${config.pub-solar-os.networking.domain}:${toString config.services.prometheus.exporters.node.port}"
+            ];
+            labels = {
+              instance = "delite";
+            };
+          }
+          {
+            targets = [
+              "blue-shell.wg.${config.pub-solar-os.networking.domain}:${toString config.services.prometheus.exporters.node.port}"
+            ];
+            labels = {
+              instance = "blue-shell";
+            };
+          }
         ];
       }
       {
@@ -83,6 +113,29 @@
           }
         ];
       }
+      {
+        job_name = "garage";
+        static_configs = [
+          {
+            targets = [ "trinkgenossin.wg.${config.pub-solar-os.networking.domain}:3903" ];
+            labels = {
+              instance = "trinkgenossin";
+            };
+          }
+          {
+            targets = [ "delite.wg.${config.pub-solar-os.networking.domain}:3903" ];
+            labels = {
+              instance = "delite";
+            };
+          }
+          {
+            targets = [ "blue-shell.wg.${config.pub-solar-os.networking.domain}:3903" ];
+            labels = {
+              instance = "blue-shell";
+            };
+          }
+        ];
+      }
     ];
 
     ruleFiles = [

modules/promtail/default.nix

@@ -18,7 +18,7 @@
       };
       clients = [
         {
-          url = "http://flora-6.wg.pub.solar:${toString flake.self.nixosConfigurations.flora-6.config.services.loki.configuration.server.http_listen_port}/loki/api/v1/push";
+          url = "http://trinkgenossin.wg.pub.solar:${toString flake.self.nixosConfigurations.trinkgenossin.config.services.loki.configuration.server.http_listen_port}/loki/api/v1/push";
         }
       ];
       scrape_configs = [

modules/tt-rss/default.nix

@@ -0,0 +1,82 @@
+{
+  flake,
+  config,
+  pkgs,
+  ...
+}:
+let
+  ttrss-auth-oidc = pkgs.stdenv.mkDerivation {
+    name = "ttrss-auth-oidc";
+    version = "7ebfbc91e92bb133beb907c6bde79279ee5156df";
+    src = pkgs.fetchgit {
+      url = "https://git.tt-rss.org/fox/ttrss-auth-oidc.git";
+      rev = "7ebfbc91e92bb133beb907c6bde79279ee5156df";
+      hash = "sha256-G6vZBvSWms6s6nHZWsxJjMGuubt/imiBvbp6ykwrZbg=";
+    };
+    installPhase = ''
+      mkdir -p $out/auth_oidc
+      cp -r * $out/auth_oidc
+    '';
+  };
+in
+{
+  age.secrets.tt-rss-database-password = {
+    file = "${flake.self}/secrets/tt-rss-database-password.age";
+    owner = "tt_rss";
+    mode = "600";
+  };
+  age.secrets.tt-rss-keycloak-client-secret = {
+    file = "${flake.self}/secrets/tt-rss-keycloak-client-secret.age";
+    owner = "tt_rss";
+    mode = "600";
+  };
+  age.secrets.tt-rss-smtp-password = {
+    file = "${flake.self}/secrets/tt-rss-smtp-password.age";
+    owner = "tt_rss";
+    mode = "600";
+  };
+  age.secrets.tt-rss-feed-crypt-key = {
+    file = "${flake.self}/secrets/tt-rss-feed-crypt-key.age";
+    owner = "tt_rss";
+    mode = "600";
+  };
+
+  services.nginx.virtualHosts."rss.${config.pub-solar-os.networking.domain}" = {
+    enableACME = true;
+    forceSSL = true;
+  };
+
+  services.tt-rss = {
+    enable = true;
+    virtualHost = "rss.${config.pub-solar-os.networking.domain}";
+    selfUrlPath = "https://rss.${config.pub-solar-os.networking.domain}";
+    root = "/var/lib/tt-rss";
+    logDestination = "";
+    plugins = [
+      "auth_internal"
+      "note"
+      "auth_oidc"
+    ];
+    pluginPackages = [ ttrss-auth-oidc ];
+    email = {
+      server = "mail.pub.solar";
+      security = "tls";
+      login = "admins@pub.solar";
+      fromName = "pub.solar RSS server";
+      fromAddress = "rss@pub.solar";
+      digestSubject = "[RSS] New headlines for last 24 hours";
+    };
+    database = {
+      passwordFile = config.age.secrets.tt-rss-database-password.path;
+      createLocally = true;
+    };
+    extraConfig = ''
+      putenv('TTRSS_SMTP_PASSWORD=' . file_get_contents('${config.age.secrets.tt-rss-smtp-password.path}'));
+      putenv('TTRSS_AUTH_OIDC_NAME=pub.solar ID');
+      putenv('TTRSS_AUTH_OIDC_URL=https://auth.${config.pub-solar-os.networking.domain}/realms/${config.pub-solar-os.auth.realm}/');
+      putenv('TTRSS_AUTH_OIDC_CLIENT_ID=tt-rss');
+      putenv('TTRSS_AUTH_OIDC_CLIENT_SECRET=' . file_get_contents('${config.age.secrets.tt-rss-keycloak-client-secret.path}'));
+      putenv('TTRSS_FEED_CRYPT_KEY=' . file_get_contents('${config.age.secrets.tt-rss-feed-crypt-key.path}'));
+    '';
+  };
+}

modules/unlock-luks-on-boot/default.nix

@@ -0,0 +1,20 @@
+{ flake, config, ... }:
+{
+  boot.initrd.network = {
+    enable = true;
+    ssh = {
+      enable = true;
+      # To prevent ssh clients from freaking out because a different host key is used,
+      # a different port for ssh is useful (assuming the same host has also a regular sshd running)
+      port = 2222;
+
+      # Please create this manually the first time.
+      hostKeys = [ "/etc/secrets/initrd/ssh_host_ed25519_key" ];
+      authorizedKeys = flake.self.logins.sshPubKeys;
+    };
+    postCommands = ''
+      # Automatically ask for the password on SSH login
+      echo 'cryptsetup-askpass || echo "Unlock was successful; exiting SSH session" && exit 1' >> /root/.profile
+    '';
+  };
+}

modules/unlock-zfs-on-boot/default.nix

@@ -11,7 +11,7 @@
 
       # Please create this manually the first time.
       hostKeys = [ "/etc/secrets/initrd/ssh_host_ed25519_key" ];
-      authorizedKeys = config.pub-solar-os.authentication.sshPubKeys;
+      authorizedKeys = flake.self.logins.sshPubKeys;
     };
     # this will automatically load the zfs password prompt on login
     # and kill the other prompt so boot can continue

overlays/default.nix

@@ -1,7 +1,7 @@
-{ self, inputs, ... }:
+{ inputs, ... }:
 {
   flake = {
-    nixosModules = rec {
+    nixosModules = {
       overlays = (
         { ... }:
         {
@@ -12,6 +12,7 @@
                 unstable = import inputs.unstable { system = prev.system; };
               in
               {
+                matrix-authentication-service = unstable.matrix-authentication-service;
                 element-themes = prev.callPackage ./pkgs/element-themes { inherit (inputs) element-themes; };
                 element-stickerpicker = prev.callPackage ./pkgs/element-stickerpicker {
                   inherit (inputs) element-stickers maunium-stickerpicker;

secrets/acme-namecheap-env.age

@@ -0,0 +1,48 @@
+age-encryption.org/v1
+-> ssh-ed25519 NID4eA WtfgDmnK5l9s9DMhWgmk+tel+/uqPx8SHBd0qfWY3jk
+ZS3Qu4v3pnA+lYzJ3kad7T3LhcY7oE8fPsGQ1uQH1AA
+-> ssh-ed25519 9RQHxg SpHG3ijNizTi1YXvZCJS79Uwt4oGkYzqIme+eqQi9AQ
+GqVhyfaTF6tLwuo0vIby0vBv3JufHz59IdNX9ifWtSA
+-> ssh-ed25519 eP5MMw 9uU7tlyOzOxlsW/bfUmzjgicU3i2J5uCGWEVIljnHiM
+tDJdTB1rBJTXVaGFOOmtG5n2Ae0XOCsi41S0EagRmeM
+-> ssh-ed25519 uYcDNw ge+lEVE8+pS/S+eO+6sPqo/czym30CJbQnhTp11NsW4
+jxL7Xhn/7JRylJ/JbeGkmhMMeJ8G2KPEKVVq1icQXKU
+-> ssh-rsa f5THog
+Ybod3f7gvCiBUcNyLV6AXoBchtRGspQah9JwygSGCtBKmWPOUSw3/DVva9nPVwHB
+q4t05bEHINMZIoWy4l3VQ1jw+GTxW+6OeWDHrxHOG2hlu1/OT0tZnsQIjWwT/6Sg
+fzy6X04yD2ADkwHH6VJYjC2Lxa7kEOeCeKOACyyab7rlXk+HauytUDlcF3Nl3nOc
+JQZzfwIORU0XWVy+gDocwVqDaRJXZxhMW8oDjlU8BKgf/DpvExLfuZ9AHHJBU0Y9
+HefbTbGO1s5J0T+HEkuIDce9iPQEe8ufaSVO6tKyHpgguIAiLIkjqrdLNRmXv/y8
+9W653Xqar7fimd/sykb4K/PpdwvQcB9Ogy23t6s3Qxz5yPtC2m8IC3lgR+N+/nJO
+n29QuXFBNUZu/QBXnWMS2QF09MGE2aav/CiwFuNiTf5D4UGGN3Y7XhX/KVOFJTZX
+r1GLtch6rvD9RtfyKxAdbtCqbBEQJmoiut9ia5EzG4TvdPAE4XK3QNTn2BSmfjvI
+3aXiXOFSbdJqkxyI6ZU2mUMMor3OWrXxWizDDYef6iHZxGlWFqA/kVXyZgdwTK9n
+8Re6SYR8roH7T35eILzP4sskElN32UO/A+JyGfP1lOclGTlOrtp4HYTfY0NhhRJT
+L7YIB0pNbaRxMBsxsxwU47j3qMkaO1uzP+DgpUacWJY
+-> ssh-rsa kFDS0A
+GJjiIApapBS6F8pmh6lblCHG3FlVWL+WKN1Gi2u/6Pa1YbkiBCgYFTQBwm5GsBMR
+4tQwRJcQQDGgGddIH4/QcMAl1fTYLm3N1w8rueywgAbOwaWktKnJFYTj7lS6PSNr
+bZyqyiGvgi0oYYSVjRnm7MmCrycuKmhcGHv1ijj5J8yOxe6qFsomsn9QZm1DmR/m
+EZmc5DIYXjhuauzGgqtPVmjHi6hXTN8NX7Fg81aegko79yA12hmyHmaBj4P96Kqv
+RyWZ9Moc3ccyxq74jNzp0eFuPNhUJuNBqrKozCc2Lo3KQAmoqI27THkF/HA8ECGP
+BJDK7JdHBXyHhf/Fc5O5xOxHieIU8tHR0LLJn7VEvQyqTlKmWkZ5J53AqE8UDmm9
+0gY6zFh7h3SjyBwqktzGJ9zXn3bp4fpg0M1+SaYp9Qf6hkJ9k79Zth4s4ggxgvOl
+veib2sg3PCmL1OCMPMtyW3JkKsq0J+PtJdlAC9cmVvfvAMHKy2+aADsLt0H8Cpt2
+cNOxbnU29eLWgG9uzcCXfqqNtmSia6LUMu71GahAuteZUV8RnDOZdCNW4U2Ohnq/
+9znMqERVo0d3LgjaB0P3HXCCqhVFYTTDWg31R6N2RzSh7mb02CFgt7N+vHleQqAo
+G/6Pb+kKYSEbU884z95+o56eQrvPunCN9Vu1CjEBfG4
+-> piv-p256 vRzPNw A2dcPImS0ih5CjePQP5oPrPfwns6zAMP0J72P7fyzD/A
+p46umKyZjbc1MjOQGnJIRu6V99O+/PmVXQvryX/9XW4
+-> piv-p256 zqq/iw A5nBHU2O+bxsFqplf2GV6pK5wQ+hJ9l7tyFIe57QVKzw
+Ik6aUY3t4geZ3yiWPqBGlBem9xNU83x7t3UA7pYB55I
+-> ssh-ed25519 YFSOsg OhynWXlurzqU3ohq1ecH018Ja4wyWazDLv6isajeBUE
+Xnjo8yS9IkMwCGNeLi6BABYxjXDLbpuTrVfwAxjDWdQ
+-> ssh-ed25519 iHV63A 5CVIOtSwima5gIvwoAYExcy1tfOo8942RQ+SsflPbAM
+4HV21GcuyddIjonOZZFgjgpR5smjce7OlMN3DCy0/sU
+-> ssh-ed25519 BVsyTA mkLu2Vpr16bAZWimh6sViq5HlB1+lNOc2WPCxzgfqAg
+cIDgWit139jipd7XmZcT8mTRDKK8rJV9xIxIaPVL9pM
+-> ssh-ed25519 +3V2lQ eqfktAyV2Pia7T7XEfcYiHN9Jd4zivMzJk3in4XOTx0
+gZzO+MTyBOJR1EgGn4Mhh4rnIyr3N9gmlFty83ou+GU
+--- yJrzTzStOkRCNRu3Y+knfqTqHrwW0S0Bsko7oG/s86o
+®,Bgm°þ÷€få‚T¾èä`1†&1³%7Q˜(¯•¸Ÿ:?ßÝ
+êÎø—æ‡ðj£ùÄO_rqwÃÏi£O®´D›·)@0•ZK'óô+apU§<Ö`ºõµœctª. þ¡<C3BE>–ÌXÇNæ+íŒÂh†Ù=‰'‡VÑn^HHöv±5aa²nKÝþD¦×™

secrets/alertmanager-envfile.age

@@ -1,43 +1,43 @@
 age-encryption.org/v1
--> ssh-ed25519 Y0ZZaw TsTaRLA+9WtN9+FJWpXeP12Af5EXMbo+ANTaLC9YlC8
+-> ssh-ed25519 NID4eA jIwfpP0rFLANj63MsJAse0R+TQbGf7mUStdusSLkkCg
-Yols084RY1C9gfOrDMwJcFRuGZ/5dgGuJey7RXqm7g0
+RHyxZqWGYMvhQYfZUc89GPly42u7MR9gSpR8aFWH6LI
--> ssh-ed25519 uYcDNw ZLAINtv10PGMtK5TL5Tf0NyK/r1iww+vTC09ElMGoX0
+-> ssh-ed25519 uYcDNw JGsVrWwxwA8ftUM+Fo1jFigWfpvNUwoNkK5zKIu582Y
-EgBB3aiHHdaDue9+Zdxg6mTV2VHeLoDN9wT+hlAzVMk
+BzM82Iqmta2Dtb8xey2nkoil7mDipn1iZtGMPKwPcPI
 -> ssh-rsa f5THog
-aiJqMs3/u06tzs8lx2ISlQm87TDatqEn47v3LB3HehPanRpZx9O1HUIRTeiWkMU9
+LkPMatwkNWAElm+RQiCHtHH2QPgVsAAd5b4qF0R0O6r+0CYzEF2OAOZ0LDsytTB8
-XroGe27HQCCPd63QunBHUH7WStA10IS4rHVpMcULB5IM4jwcbOhSYSiGyY2sbv8+
+7oHAHxA6kAga/pqKUaJl28xw7ujVIb1CunZFvVSxtOTYRrEy1Rxe3AKUOm+ZmfPL
-Nn/04ZOwrfzTabC7moV1DqAw6hnlDqKWp/q5N6xMb780w5vn6Poni3OJfuLaBWaT
+66Ef58HWMCHzK9sc/ojo7Us4okfRhJBklB9lnORkSfdkvEHLeq0R0FfDtDCnynRD
-r6WhE5evVt3F4jyYI64fB2hFw4AR2N/zIMOMvBncLFwJf9lbIFdbsENZf94cYceF
+SKqlx3VbdWe9k6UOJidA+dY8Wx0w2TQM1c21nDr4vXsXpZf6ttT4HvrqbSrS9V1c
-Tj150xdMPuErBsSJQOlfDYSmyioNN3UJUWiYsDeM3nbPEVPHhfTk6b2/lMhSQkcY
+nGofWP+72WinOpFRDQdLvdKvaNbLPwhigqL1VqaIcsnye5zZjQDNn+SYf55byBkS
-KcuMj/mN/7w7i4HSxW6mUcK2sUMV1BcSSGYRH9ZFf7kq++KpyiP7vB8vaZkcKbfJ
+CMXj238UqvdDxB4E3mBEgpFxOnyi6kLQXcPEBF/xQ5fER0RS5MkWkuH4Up+BCQ+/
-qqrIcXTuXhR+/bWZWqf/GQOVwRwe1TnqN5MoZHipg3a/UCe0gMM617VwZcfhBzjA
+CppqZrw85OOa9jAyWxil3yLQNAnLGi/P+mesPxSI+i2Not9wbUTALr4COG+1qvfF
-eW6VUdjSewwA8YHEuDrAeoQ4CMs7y56EaIlr2IlQy6uzJPX9eeO0auO9RZ5AR40a
+2MbHiqREoajnQUJjhGhXaAA332X7hNuOF/DjmBr7i81oWVmKs6TjCDVL7Yo9xu3j
-7un0FrlTJX9uorpCD/zi3tvd22W5qVoMGZ8vXJShZmT9he9K3Bv6XbzG4DJQ9/nv
+BcFqMlaOgr8gObwnyJ9BbtW4sBtnOeD5onPxWluV1+Ql8Idjmu/BKeuqIyGX6wFl
-xZ676HUYhWeyYZFBvt6DnEBneiDJFeaV2AeuQY+juHBOfBrbYmlE0S4Pd8uRSJ7w
+606lbprSTRVjLZWvg3gWaIMlXdcnat9PmHeRk/yzTrHke2aFSkvUKLymnRCHETae
-u5UJTT+RV5TkZhpCqqYm7DphYocnrv7Ic+QKmvKE4ls
+Rh8ILeQTq36Ul9r7qklBNu4M7/f+jeX7gYPH/yDUCXc
 -> ssh-rsa kFDS0A
-HhilpvIiUps80SXYUXg5vqNmcy8SACvxpC5dTVBU2n+4OVXQY/35Il5ZOrUX3U7a
+X1vrBlpHkWOVyhBokgO0yNDQk57S92xADIi88w2UU+nTYFgo/RsyTCCFAFMaDvR3
-arfVp/KaQF7Oncu3x8F6Tp1ibUwmoyAV6OYqqs128nEPwkNbJvwrLY3aEBm+NIzm
+kQdtorCowxQpKLnTzER8i2ABZAgAmUzGQuRPDKcqZuZH9oypNkBs6qeVI3TA5GKu
-gMlLRjj6EP84TVWgOsenQCS4l957f0QoNVxQ3f+GWdOiZZJFsv//ndsflng8zPlF
+V/IfKLeR57K3cpZT+TcOzKUqm/AAZO+rwdnrfW9qVAb7vlo3TWawfBHb+Fl7y9JL
-bGZy8c1TxDZfOD0/kW3Nx05c9X0EHKOEoDUc0p4qntrWlflxcvLONCgv1gZuPMF+
+pEjhDMhnA7na67Ktz1MFm80XRneMTW+0NGtcTd1iQfjfHe0WfFuYU4H6aZ8ZpZYw
-jMsPFP81eu3rkEUxefJ1qbvvGuW0cbzfwiStv7iGQ+Skh/vcoM0qw6p+csNKyHVO
+2rLa+EmFqUpv0ELwdGViqmjUNwJunsJ6rhJZlMn43v5/XPLpapQr0zwtXzzfzZHd
-8nYFcs9kD8067zMnyuqiUHASfZ4rPqTji0iiPC5kZn6N0YSgz2bybkXcoqmy3m6y
+HnI6/X97zPYUFDsUeI6x2CiVKHVWMGjJ9VPAexpJepZSkgI6On2/mfs4++XnDWLv
-qs0S+RD99o2vCLhW46hZyKAgUyTU1DW42EmnZkPrLoqV7uin8fAwPO/98Q/b3Rkr
+qsvsJqVzM075eH0LUyXq4WGu7oJc4OdfMm1CrEBKAaqdKRl0HnBZGSER3C/qAhLK
-zBRtyTEbooHvOCL8limiRtDl+5LMcjRFNWk8AN+9vHMsYurXPNOCnd8n2Z4MbT2U
+Ihbk+kti5C7GTzRyUlzkwINVFV0pePClLP7AC5vdKMhXysGQlxNJsTeUTdAOhrAm
-AhpoAD/+8HXp0InBJ/sclITVAc6tPb2CbJW6mrFezH8Ri+/6u+zSF84JDd9ZrCOz
+UeWnd0Xp+K8OBsUgyGktKBMofNAJ+MilSKt3x7tJk3QuQIGjqHCshpkMf0ckixrh
-oIshiGZmhP5mIuspVrxgKlm78a56vQrygpqzvuSSYk3zIJxmhEkZhw09/ga+rhyB
+aDN9Rj+s0A8C3hrVv8z602jBrM5tfYOZv+q1/yFQo+ieic6Y9WEzzrTMJEHxFSzX
-pkKn7GRyZTfKjwt5nnvW5/bmQndTa13j+7RhkRgBSvU
+KadqfZii8HCGQODcXh8VRpWDVjzt/pDVR/zu/0YCP+0
--> piv-p256 vRzPNw Awpc8paUfKnP6r0bYsaoeDE9GVSnads4/a3jCVScgS4V
+-> piv-p256 vRzPNw A4KCXAKoTYy8euaKXot9+c5N21WG9/9uLPomiiI6rZ5W
-YydKOS09kyZDYN843SHIsYUimtSQKvGhIuycPWOFojc
+vbTcLpDNM1qVdTBCUPMrlX2GpyeMUsKaKLFl5GVVdYY
--> piv-p256 zqq/iw A54xbcufPkLpTD+N47AiIe/xZ/0vA5kDJ4p3rIZw0a4A
+-> piv-p256 zqq/iw ArX1s306JaaWVPiTA7XyzyTKcsBDHjeIiSoOg+5PhsBj
-1WFP2K3tfUxtdKDBEmT3cx/u1i5nCzFR7cK4kN3WjC4
+zzmU1T5q5ff6TzIXhWqiVM0Oxxo/ln3uExBXBlLFcME
--> ssh-ed25519 YFSOsg L0lPSkoPVRKGlJ9MzkJx+cQvnZw/5m/j/JO4aRzd52Q
+-> ssh-ed25519 YFSOsg tgHAr/emB3i+9Hd+q9oYCjkPO+RuXv22kimdXz06Zys
-o/N7zQkvbGGoadiJSvL6lfuP63uqzxEIxDtIg4tgKIo
+p8sYz3j5I95ZBJroWxUSzWljcj8E3Ic9uwwyrUWm1+E
--> ssh-ed25519 iHV63A qfLWZhbDisCSJ4vFFTR+XpRUR0WViuAqarf56M0ekT4
+-> ssh-ed25519 iHV63A 9UXBAcuwIfuoTHcWYhLVa9qtJ7UsLsIQsH2Bn0T/Wy8
-ZSWW34pFRr0M2jFhnphIPJ5ch37ASM6OgTzyHSo0KAs
+OvfX4cOKJYv9pwaQp3yD/QPZdDnGSC6f1qemtKENtpE
--> ssh-ed25519 BVsyTA JcFezSIfTF+AP8LYfFqz+wIpUrE0aoc1usiLtWxAPQE
+-> ssh-ed25519 BVsyTA nC+YMVK5YyCM79iNijTaBgIZDPi7Bvlunuzl2s9SrRw
-F9uhFyCPK46kIy+ud4V5/ESacQgc9R0JV+JTEZO6nBI
+xVUpZwdIcszqsRdZw74fJrSduzxqrO25EMfuypipys0
--> ssh-ed25519 +3V2lQ G4yT1e7B5O2Gy6tusRMxuWOFScynWfFY5AjrJvxMK1o
+-> ssh-ed25519 +3V2lQ 4n/lkQ1nwcXD7mNc3DzIfC5xGF2mn27AoO36Chei8AA
-n1OVFRqzijWlc+B93cBNdFPz+8CBYOsI5hpF1wz7xr0
+vDe0RU8Xm3L+/nFM0lKK3jv6hqiUE/YxZUFyHUsqAfI
---- 61u55uUc7z59iHF1IeyBLmcR6u7STUhpOPb/ODf75Vc
+--- D4n9aVPWABXpzO9DI20yHf11MRJ5ACWVhT16bDls5pA
-<$kxpû´ÚH:}ò*ä/Tâ®Ñ$‹ÕbÀJ\F*ðòWîzÉ6 Ý	±Âì<î̹>e?ñ¼<C3B1>Ÿ6ÚµÌ~Ô!
+iR	ÅÏÿ%µÙØY^Ï	Ýñý’µ¶{“²°Ý”#®Z0´P6šÿ+ÛÄR!i›J-\ul°9!å¬`Z÷¿Nh²

secrets/blue-shell-wg-private-key.age

@@ -0,0 +1,43 @@
+age-encryption.org/v1
+-> ssh-ed25519 eP5MMw 3H1XEr/Vt2TOQUvGu3K54CxbigkVpaC6hofMOfFR60U
+hqFTOoMhyhb/Fsywzu4nYXmEACOunenO/4NwPaVdrZs
+-> ssh-ed25519 uYcDNw raghSMdCaiQrfGviMlc9Pwh8cx33IWh+mnsxL8jgTQM
+aOMrh/746UulH7hkOV6XRiwEszgJtrI33mmzY5S2Ipg
+-> ssh-rsa f5THog
+Hm/2fpGDwqKG9K6zLYXuSDwOppDtDfw665ppaVzRvnNppizkilCohBzCrwXTMyeH
+KKZKqaEt+n35wrurfMfqQf1AkamEimjlXCMmr9IwrBHbJeIJuHn6vGSOssQ0Sq4R
+dr002QrKsGDzlL8dCLmaKU6VPTXcSbCOgDnBW/AyU7bvN51jtgm+jOIey7jH9Bm4
+0nU0UNcPiShgSbXLPp0V6O/1zQBOVeFGyGenblAKlqLILPvc04f4703alqSbKwXF
+q6GoS0Dipzr4N8X4Thzgc8H/YQ6cBVGQTebVGHqFEngLQ2C0yZWlDfJsKnKOGUce
+xSxSskfzEv2s9VHDrXeiTAzSYaLoZI0JEDHOSICmZT2wqT1voFokIQV8twORGpOo
+RAlNX0BULPEg5Mi2k3V4ZBcG5EwUjEcHAg+0cQ82D0f4OJEqMVIa3dse/lBbrFzC
+/Gkr8+LPWVv7f+mRr3sdKtZ1nBwz1uTP5qIA9m92UeLVdVjmp20hixmlgftAbTBm
+MtIH2xqkitpb6bhImGIbnxpg7U8IqQZNfavvOM9yqj0uud/Nh8ruUuwxci4RS7yP
+YlIcompHudeirCLPvYx1T6nYRB2GB1tpTcyQN60pb6YC7lH9w/tLhZf8GFcMdIY8
+eLx7FoWfNj2dIp8EpBrRUEaQlea6Eb1r2DYTjmNunnI
+-> ssh-rsa kFDS0A
+SPaIIABa0Wja26CIyeUvrlt+LWJY+Iw2+OouRZkMfi5YxDflb6SATesWh3zMD5+A
+SOiA/oUiRfGRVMmqiEznd2BLyUO7FQDv8tIT38R3m8Nj2Lb/kWtuhPKqrk0PPvr+
+kYR3iyM4klqccCzwOYwWfpO4ZkFACBApTOFYbU4WX50WDRQRco7M8funj10BAIsL
+H7hOKtigx2RJaJDcU3MPHV4beECpKbeH9scICEDmi098UP3850g1lN5AR7NjWvLi
+zr3Z8Up4UjEwH2mvuNnAbvA+0HOu5geIae+VUqy3EN7XFC5HnMHB6eK12Q1VAmF5
+MbgkppMeoqLnxIdLT9xxRNhD50B3QR8MNHgaNWuBqfl86XPfXjxYP5kRaCAraego
+nbE1gItlgIPESpf0ocfIEyXb0tSup8c+99ezgxBMrzNP6UtVGRhZvwLTpBZmrLoc
+5P/hTr0WQE7d/6lmY9VyDsXasPU758tUhM/cJxmmUyZcCOhjAiJdWSYHe7AUjSQy
+rUSKUSYwqM/epFNqdnanj4pmmRkSk/0AiEKikvb5DvWpHrnsL/EuQqtT382IGv6P
+VA6ORv3BmL8IjNnTppZsG8k91WWHLEqxoDKUHg6WHfJ5XqnEM4maFsRZs53QLqLB
+vjAxAeJHjFg13wkfJmo9mZNbw/0WXS/K+xmeOTYuH/8
+-> piv-p256 vRzPNw AwPYD0NiFDZ3/0L0+BEUS0hm7RddL3sPXUshz7XtIQVi
+7rzoQuAQQHxkuYFx5TrLEXZbGsERg78mAXcgQySwHGw
+-> piv-p256 zqq/iw A0Ec624/7FOTPVAbZDjhsBy0i5L1Tw9LwYfH/7DeKHi9
+djfKQINL2LVAAueovp/V1IGyhuy5LGQtOws5Dtih9sw
+-> ssh-ed25519 YFSOsg 6EeEfNtlQ7/a5Rc5iShfSa2ZjIoN6QcLDI0hJgpF8AY
+Tcp4iqFjBTTzSUAZrxRWe8QkvuEoPWVagNL4EiZLMIA
+-> ssh-ed25519 iHV63A P8IDXAspyflmLqtPOqPWE+J9s9e3OccKc5+8s/Wi9H8
+iRZba5723Ux5oo8YA2TDyiaWyGzHlAcvEiD7I99vq4o
+-> ssh-ed25519 BVsyTA LB7gg2/eozH+f9BNC4Q1m6Pl7b6znkO5rPVgvKSjen4
+AjNzM/44dMy7JyUcAT7c4pAFTtOuapiGtiqLdBPGrKA
+-> ssh-ed25519 +3V2lQ NHbovTrC4cTSsqb3AfmVOJ/pL0QQbK9GpMUpQMAW7w8
+iwAoDSQnucAzQPOgZZtl2bnJQ1mU19aoruItkQqJuZ8
+--- itqKtiBSCvkVJ5boq7PeY3uRMemElImzWvSeTwbz3y4
+×ZP38†¶0¿Òe¯8W’j–Ž÷[ªø#;ñHjÀëÏwïYÂœp¨µ“6W`ôhŒ²ªs§õvbÈ·èÓWu·ÔœxZ5f5½ 

secrets/coturn-static-auth-secret.age

@@ -1,44 +1,43 @@
 age-encryption.org/v1
--> ssh-ed25519 iDKjwg ZUEOvf7JnWeFNohEAhloJ0+YL2SwHujjm2YG85NLHyU
+-> ssh-ed25519 iDKjwg vLO2012STCeqJACpBNg5uKyWx/u0Yfvwxek3S+0Q1C4
-HwrrqLMlNmfSlZVt/lCkIwqmCYLARbDOBhIm+AYmDEM
+6vPjunf0CQeWTwznZXPc5iVL/eiF7SrPqGeuvgcfizM
--> ssh-ed25519 uYcDNw Lrek6ru/vb2JIZyALem40oNZCf3ia/U6sb5hRyDaakA
+-> ssh-ed25519 uYcDNw QwGWHxl6dTO1HEfw7pEtdvb2ne0RiNMb8SkWRIrRJQg
-N34LLq2+qJOlbyaYXUtNP17fDPjF+evgZ6kOs7mVhYI
+ffdyTEltr6wlrnA9isU17orFvSRmicPvX+w2t0QBJIY
 -> ssh-rsa f5THog
-jmwJ+hV1/50cWemVUhPTkTFgnd7iJ0YLtjU4fEKXghWIlie/OR3AK++1f2UJxKT4
+XjKRMzLPZlrTQEDJzgCwBbjZwIy6fMYGLuBR8TS15SAIbttLoikF/AV5zqJDaE9j
-Z/32ALRBnmb7FlAPyYbxIns3IUJP+Z/Il5SCeDrtwaUxmtluwXwwO07WlztqZJlO
+RaRJIIQV6LYCm1fHOsoua0XGxicvJkdithrDC4zsEEJ5n2luNj+sQd7h9ruOdkO5
-bvZ0ifDazxOFZO6QfXQE2SaPDOqcH2AAiiL50eXMgbdY5lARYW2Qbai/2a4t/PuT
+l+Og+MphPM9naul+MJ/DluzS863DewkCNENEWe5H5MkHujOoqEsRsGdrRPXUbAPW
-Qn8WAwjyXOIdOnaYb/MZWyp0GQYsa3nEhYyOWvTSjSROEfR5qBtGNYkUBBTYF+YO
+oNIr6h7FHCTFTkxfj1aAqDHdK8R4Yqo2K0vnHHfNS3PeN+CQJOrGNzokYybMRxqK
-DGOYStbPSkIhnYYQmNajlcy9wMW5fH36ujGdnMH7C6DgcSCY2iTDTdE3cyCxAuaJ
+RXoJD0QKAUCV8cdXGr0XuS/ljv0lFODKhupy5ObYU71052nxo8j7KTq2NpZXqjul
-bRThKyXYsvhMKgrFzbhlgt68taESb4KcKcNO5r7lqqID/I0b/fVltsKpkXrSCB53
+PWyetcPtH5nLCs2L31XoBk7cEE8g/eSjPky3gSMdjGjdB7qskmuPcAHlHEwQzecG
-Th/aXLXPUrYEkbdP6nqDBbUjeA8RDid1raIF1O29Ok72oU48q50QXqP8GF+honkg
+D6J1LjrPa8OMVD4AdR0KAXSnSvzt/RhyymiZWtBeKg52rm4KK3PVbJqq05m+PEYA
-HSdXmhPtlZyArlJNWogDaU9FkWp81E0JS8G0OnoNilCmiu7sF717GG4pkA6GTnaB
+a8wFT6fJOqmNr4gj4peIUHca0gYWhfzhLpXsj4/MKTPxzdUem/wbbMJrM7oQpwma
-hlJSiVWBPhmhVURIOKkRl5bIWLUvJESPLVVog/vsW7OJETOb2u+AlwvaBNY8w1wE
+svN0vvqCUc2wfw2Apr3WwoAnNTIohZOngNkKNWNweXtPOee2qgZO3ko2wpFa+Yh+
-An+m/qNO/H9Nksw0B4C9nLfasE/nDvbOT/Igc7k6jP0sw6/PAWnosJY5vDyIpR8k
+IMonHHVhtdchTidx4RttgDIaW/+i//XGfqPdmanO5wUmm+SwgqkkQkRHzmtbmgsq
-7q3rBPnsZRXUr213ue8xs0G7SsbLheYNu3/D4YdB1tg
+KlrAmjL5biH9f9sBItYMdKafgyQppMAQ3hXt5wgAgj8
 -> ssh-rsa kFDS0A
-BwaozSAR0Lcn3ZOHhC/OuOYRZqW0ayV4kL7CSLgaw6x9WqA7NLcsE+HDr7aDx/lP
+XAY8GSsx8B5q039L14C/t4cGK2sAm9eqO33r//YgpI5nkvw+pZrbJegdCItfHXHd
-K7TmFGYMrOiIk3siZ4Qc/JwZXPiayxGITcwoY82L+FrJKJmQd6c/3exggsHlc7B9
+9BwBGOowTe5Qmj6RVfz4rwsj57HJbt6ivoIrU3vH+GLsNs4JIg5lwz1/WCsotw6W
-1ijXoQgjnorlopI70Cyt3QLQyMCPFb7tuZFEKR0NqBzcFTi5fKVYcMrfa1WVxzMO
+8jQXiiZbA4nvzQzyZjJVKavTCfvbRXdzc+CUZiWgQDXsSFejp3ODeOvUds8YKWiz
-0Ic+mhwMIAst6SQqOkqaVbtUYxATupQx+9FwThk+9NDety1vacb+lQ7hvCnImpTd
+jYILyzUzyAf05HDC2SIUhfA/UoXokfpo6uZuryWXjRBgaRENa9csDnktc8V+61W7
-uENry/G68I7zWhNuCeE6wj8lCplFkW7dvrJyoxUVokWheFnUKjziA3ZybfMyAmI9
+gUnu6yt/rN6oiBesnUZQK4sPd5YE6EcOT2gtLp1qKxtRuF9TEX25oLHi52kPBu8j
-vJZnTvTc/7UxJCnuk/pB89q3ttm8LFT6AFAwZ1PY2ndWBMRlnOaB0JXSBKXZCYYV
+TNGbCU2ImGW3Z6TkAj+/XQzwEIrbLgb7APMkI3DtWyIIxZn5QJdDOOCseKMKt2Lu
-bmJ/NSNdzyO9Q4MrKwYO+O8SOkVWM9EqKYv+FMO5CksU/N9EOUkpZeLpMYh1WXPX
+VH1RF9C26mqcx7+WGCJKylARX8sbT1/ZsCWSUnmenYuGNQQppMwcQGSICs0YmFkH
-BMKmXzRWp3YEsFH0g74ZBjFpTo+FK0bbRfYfTj7wtS9LpOFPr51qRDwv0zocM9cQ
+XnV19+pt93i5rVFs9IUxFCqFKKjElCiPgIHe0QlGuxifeiMXYuNi8g4ObN5X6GEL
-MkpNtuSqpXboCLGytJE34pAsDY1BHJpdAOwlwavwK8N/yxlF89ktIAtHpOaV5QNF
+MPm0+sr19dheOZicyxqJ/jSlEOP8bHgN/VDHjKtsMWQD9r3NfLH7btNjA8HTITDI
-r8oW2DLERj/s2yunrjZ5kQXaxbn2GBeml5gFyYWPnKVIa5x0PA6LgT2OMYd2x4vA
+YvLLVCP6OR3ZlMz2HUXDpbaPYSSZvrEtwkqCIe3ij6066Y5cTsYHWEwOvXaKYh9P
-r7UGlMktJLosJGjJEUVLUHXarKkTz8Xwrw4vtaaLIyc
+OJtPgLQDV9VfU9hK60E+C5qGQAvHhBgPUfXS8JMJkyw
--> piv-p256 vRzPNw ApWXG3ayudUSrW8zw38cU6hYVeCVZhIQm/ZbjKpZqgnb
+-> piv-p256 vRzPNw A2vUnNzWtQNNOU//b3muMZeM1qdO3GyREn73VgdxMX4Q
-NqaQ7bjTAuMei08uNpVaK23uVmspjlkGyleF8phudVM
+6AzSUdoPB4zMbFsf0fr6sxbCsg+5/qmBtkCo3ry88Gc
--> piv-p256 zqq/iw AxdOZ9zfYgKZJY9HhQokUHwSKbfKl7i7X+FPO30EADcr
+-> piv-p256 zqq/iw AwVuYkScYFB1OzvBz9255ebDwPO4o8szD79gPnzgK/t9
-qsniaELyEVrTeSaJG/lp3sCPCmbTUA7CWdMxA9tsBXc
+UCm3jzlAPdfGvxO2VrE2DBvcGlaJpMTINJl2qcq+4oA
--> ssh-ed25519 YFSOsg 64fhQVd3dvwHCBXa0QiK6E8rYA1jScm0UiBvJVuL6Eo
+-> ssh-ed25519 YFSOsg wUzSRyoZOde45Uv+KaN/ARAxIRt1bPAqN30P6nM9b1o
-YAvXqNw6kQkTzBpDIboqa9gOoTgHE8hcaIMTg6UkODs
+pmufkyRBD4BoL4a+dbS321KSdjPRrB09MssNU6N0dtE
--> ssh-ed25519 iHV63A BlO/mSeyxTFBIa77g0Ce2CcaVf9SAiw9/OzkgnaHEV0
+-> ssh-ed25519 iHV63A qyqt+LHR4YGE+P2D2mq7qOS959vLZ9K2yalLvGg3riw
-sjmnXCpwe5KTgIJ1ZaM8j1U4fYi2Y5/WpwpUfAe8Dbk
+1oDuGVg7Jn+8MIlsHb8KCDImManVGnlIMoqFt9w9Wjg
--> ssh-ed25519 BVsyTA gt6iV6mhL2G957w7IbJVzNFV8QMHOzP5uOkgSp5QgzM
+-> ssh-ed25519 BVsyTA skF/Np1FrFUSWJgCw5PN9uSy+bMezPHV7lH4jm67TCc
-Vvz1jjLKA9qbqAE1g0UyHySrrnG16ENdz9TxwyoML+g
+QrtBW86S8cB6GLsw6LVGK5jhFQS56MvATcPspGJwmAE
--> ssh-ed25519 +3V2lQ g453jshh1sgCdUyhg3jlU0A0X+byL5jobpu2toWTYRU
+-> ssh-ed25519 +3V2lQ DPCBFzgin6QTJx0QZ0+52qW+6xXmGA4M+hFEIFAvpC4
-S2k6Nk+UBv8gcJZoIdZUc2Kd+Rv4jzzcEyGm+eb+KUg
+QuuoukU5PC4BW2ieS52rkGcPRPuvrROE37gZpd7cudw
---- 8ahetWGfwjnJYRnkeSS15sLjDBBtN28biMlYCPSvObQ
+--- fVPm/8JI93qQmr6bEdb8JEtRpKtsBHnK88A1tptYLIs
-i’cü'ióë4Aî6$}ß!IÚ3ó¨ÍÄ™	Ù›3yŒ<79>ç¶;¶ƒ
+|9Á:\Ù›ŒèHÐ(„–•a-[çf„-Bpýu[€,¤bz¿ö'jA¸áyp`4üð“ï<E2809C>lÆ•|—Nj‹3ç
;”˜¾¾)“±ëGȾb÷ÿ¦&ÓWãF/ý‹‚±yõ¹
-O<EFBFBD>.<2E>œ[„Íf%jTà4ŸG¶÷ãÙ¸W#iÐzuä`'Á*zmû‡òèE‡6ÓضÑúéª[ê€

secrets/delite-wg-private-key.age

@@ -0,0 +1,43 @@
+age-encryption.org/v1
+-> ssh-ed25519 9RQHxg dVdaiC3H/M+tA/xIW3NdwQax68lkydLDLm6OxTx1lSc
+HRLezYbdAPHNbQm/2WXT16wVX+ZC7GKlVp48aIECsdw
+-> ssh-ed25519 uYcDNw SqHkg361mGpjrcynYld45CU/jfnPp55bt75apCWlADE
+Z55QoOPVt2u1d5Q/96PHfA0MFAaO4y3CWuJNBnVy2IU
+-> ssh-rsa f5THog
+yKfwc4z4mIVfDJW5aEk7O9ddRL0YKOZFV+xKE6OyMGv6luJ+yM3QcPxu9yppCfMA
+AIreVH2/4Tn+WhYv0dQuVeTR7SLqfn+TynkfK6ZZ8NbjSZXnakPqNLtOWi6C8VIH
+MAfTTg1NGOG7E3TGmMqFrQnxYdpIvtfeGaBfiMhGLoWnCzJOzVkZSf1Qx+ibor1T
+ZtJGsAVoAqGlFoWNDNrlemx9/5qxARe/GSSUyKtEb8VtbvN2hgJeg3YAKBANYkUS
+C2Py72R662WBKPSd8vZRaQEJzP78b+LjBPmF98E8EZcHD/L6I6QRirpD8E89nN8b
+/vp/Ze2q5R+9ot2TIO4Q9TJKLs8uMYUNUImrWQM9eBG444dmuTNDDfBFeI0Xwwug
+FrHxS5pE8QzRdla/fIt/Py02F/734pb1LRCBPaLr5F7gFTBtIXe1TCvwfigiYrZ8
+3zD4mpasIz1a7MlOIBIUuNAvAvJIwemenzxoCPTJUIPR9Ja06JUJe0qe/qOOaUAE
+iio/a7XLYAUnTqHQ8efufS7RF8HB61Q5/CEtjE4NnpaiZkG+KQ57xjTwrfKSALxK
+FN5ydIMY2aNg6Hh+n0MlCfpV2N8XYqOF4YWWRKY39516i8QCvvL21k+lt3P0Lpad
+E5fn9FWoLqJw+evJsF/PyTrLT68tQbcY90mIGC6n/Ww
+-> ssh-rsa kFDS0A
+dTkAwE1+eD5KvpKvI0Wja9vV3wgKogv9ADKyf/u6meetXQ0isUnqxhqBbQZKhVqU
+StNE4dSwg9gBwsPmNXJ2Avc9LMIBuhDomPOp8bqdO/r0IyDvJ+TngnJsroK7dfL5
+foIDtakdAOVZM12ytm5hZlUNfXppAJUF1T9w5LIBWAMQQYJs39u+RrLfHoIvXTor
+OvTikrdYfGiWdX3XIGQAfoPct3ZEKmD76ypH8ESw8kKF7ZfRQNDDP3LMbakoFs8K
+Jrh5S9e3b5hdiDsv0DFtMISm8l6ec2mhhkVhSPgJ0mZYimMUg2QW1b61aTbE+LEI
+K9O27ci1UKhxvLnxA3UWrYcm9SX3dvFgH8c+hJDdydHRGtMA3OU6Ut2HuGKaLfys
+P7F7gffBTEeW2na2I00k3/oq9CCT4AP1d/CKlGd8ZtRvv5Axv6ThehCFZ/FLYGCg
+kLBTHCX9qfvltdkKHIZwgxQ7JSGvNf/4bBzUz47o4gBhtnADaPondeLkqRS8sMwF
+gq4UtG13Y2NfVYLtydrZCpUSMXekibZ8Rj2ER1uy1PD32clevgmgbTSyniSbJDa9
+NFAeQS8KiyfvnxtLNNoMflR5yL2O4bX8Eo5ooaxILxga4M7ckcuKz2SMjDjpSBbK
+ndwen9EsZuQwK9Av09h3gZeRy/t1ne6MtZrsTY5W7+Q
+-> piv-p256 vRzPNw AyZQpsc9MqXbooqG+eK5gQQbfe4ka6pG7uixb8ONVGQz
+FWuy/qAQidT6C8YMb3674epUzZw0Rb2NMCK5t9wdnT0
+-> piv-p256 zqq/iw ApTqG55jHkxwd3cT2Hvw84V2DcoHo1M+q9eP2eLxSE2t
++27Dzy6pzGpOwTqUG17QaDC93O3PSJIfy/d4eBnuLw0
+-> ssh-ed25519 YFSOsg mRmdt4AzDKbzKvMPOEHg+jQSRs2RF7f7ev/jzP7SuFE
+VmNGaudQF6R8xDWBz6bFfmk2J8twCUEzcXj2AG5teKI
+-> ssh-ed25519 iHV63A pXrKk8kpTBDxhiio5ZY7krRJIDkxYJZOMqCaW9Q7OGQ
+9/xgfjzsd2JT6FQ2YWELl9jqph3+HTF8jChvbiHceJo
+-> ssh-ed25519 BVsyTA z8nXuz2JOAn8t8OW+AzFRAXb5ulAuderatBFDrb6klY
+Z+7S5aGCCV7f9WwHWr5LrsKW7rnpidImwoiP2dXcxew
+-> ssh-ed25519 +3V2lQ p99nuu5l75p1y3Ea1yRdFBQSxvYRVRJzX1undANyFVk
+QHzKD4WvtnRI0wgiaIYKWwXrG5Qg0vQ+V6eTJUk+A8k
+--- od3JqYVYOFEDzaNHY5oDbfOjhUBsiQFd9pNGSkAw8Dk
+øFJÒMmá|šž—>¿|ÉÏü‹µ
ï¾ê0½µ:+‡¬¥U^ØÑÚYؼ÷æ/‰ŠwÊ X+8gt
RNPÍ^\€N}«ü£

secrets/drone-db-secrets.age

@@ -1,43 +0,0 @@
-age-encryption.org/v1
--> ssh-ed25519 Y0ZZaw 5slOxDM4xGALMpYxFm1WBX4Sds7itgPBMIiY97d7Egk
-mZNzn4I6obUHAdox6eVR4H82EZagZ1IrCcq0CDtK44Y
--> ssh-ed25519 uYcDNw w5lzhmA8wIMXihKF25d5jx4/Cc5BFE3Lw6ad60b0wBg
-v9z03cpts6oVlcTQ48hMw8rjWHp1JUOov2qCUjFN4bs
--> ssh-rsa f5THog
-A93Usdjf6yJFLFqDiy6UUIJ4faBgQXIvk0pZlABlj9M5n7fSf9uzR6sSih4HNCvm
-sMkZ4wKyQHJnUB4Uc2jGrdcWqpmP5MLYHhj74Rxsi6heZuCRf94KH7sE/03A958w
-jAV4v9z4EqmkvWLNQi/hxMVMs5A61Vs63WIX/TA6vhL8Yrn0FeIKlRZYUVIeMu42
-pbEuLWeIzbUioAuEA1ZKV5VDx+6ack8TS/Dj5bTNEnzFWpjnHjO0/GeQU+aaQZTe
-Zy325TcRosT0V7PIh0tDQZKKRpOH/e9LnDkd8NIjyfEsGdDYaP1EVOYVxPCqUDAh
-A0kV1kkTiBzaXDkuakc+HDCIxtYXLWthsmbD+vI3D7FlTl0CY4fOP0wwO/0rS5Yp
-KDuxjz89II1H4+ZvlcPUihyW7OEj4d+NwFQy+7Qq0Y9Ii0NONXNsnx17FKXJwOMo
-NKyLo097FvHV7k8F9wv9mmZboRulDAoRyDngeO0+SJA90uJass04DuiZvK+g3Hry
-xVzbkk59j9EQqUogopW/oSeSbUP0pvcKOahGcSIW8vmadDTgnN7zzqf3fq+dJ2TM
-QD2IXAwvoTFBE+9DnPOtptk1X1D2umZuAWTzGAseXOImrPFZ+bEr5MV3qLGlg7sL
-yA7Mvbp4diVdH5aePzeBefhxrYphz+yfCbELFTYam9g
--> ssh-rsa kFDS0A
-Ng0FhTDjASWJkrlNh+UZxU/dU/wfmoV1/fwTv6Xg69k/2qU9lk0oR6e5xAimvX6u
-h7rKAYt3zSRIFveGczPCflC1nycG9wLSpaoJghav+q+muoDQ/fbSKSgHFXITC7Me
-f/wblyWvJsUQbjxSW3g6/8EGz6FvpTnycPtD2vbRj+Ctq72GPA2ZWg/OC4jAUlDs
-r6X0Ql2jwWzy3Y12v0mPknlBezN8cIfjBmoNOWokUeGJIBjujlS7loA1yif09BLg
-PTSLCY1YH3QYcm6lCXK0HaNcMjSSk/ZK9D0wROriF9PBbkpWgg5NlIrqGaeqPN9z
-QwRR2DvhuCa1br57F36Y2LKGphYjmhWAtzCyQ0h9YQ+AzEy9uFCbK0IFyyeVl/fN
-+HBGgxacJBcEGsNV3mbJvh6dn1348eex0GgaQEf1B/lu/y66WHbmSqVyUDfWkqEz
-IytAC43VT2rKgg+B5u0d/JhLDLwXTp7iVDy52ul1n7keJHk8t1GDaufAXbWqalQ4
-vuyxs6ghSIXUi27IZrYblg/OEPFTBfcoMXkmCgyx5a+eK+DhnBazWjy5j+vgp2so
-ZQRQurbG02qpZasTwBM3iy4ZklX/uFjsKnk0c/YcmK4YcMviHcQQjdjKruEE93u+
-Za1KE+qZGLkhFCd9O3ZPMtEjRjpN10XIs5ylKQ9MKU4
--> piv-p256 vRzPNw AiNjNIR0OGHBu5Qn+bvn+Lk5VnpI2BQ3eJ3+2/FTJfZC
-elT3acRVdmtBl0qC5YbvfntxkJrsZwEJqlF6aN5hhWw
--> piv-p256 zqq/iw AjIzSibkqG+YcP894QekM61Wsty6MaKBghlWapHfU0Jn
-HyXBp8DxtnNsfuzZq13bwgma5CzLTf3UB5Eht6XUwe8
--> ssh-ed25519 YFSOsg WRBQZZYM+X26hfoH4zvNWQulZvVWP/Ha5OgkUmGK/Q4
-5Hw4ZDNawn5YRC673Op/sbpexOKeL3gez2B7oZxUKhA
--> ssh-ed25519 iHV63A wyr8R4DlqLAu0XypddVoFimK2ZMncWaa+KWV7vMEQm8
-puV3g1t5AbnEgC0S1U4ft1evB7KuNppEi1g/AtxHgWE
--> ssh-ed25519 BVsyTA 0N3iyyGqTCRAHHcK7QfN5xRttorc2E2GL0RDTIVIBU4
-Bph0OujqmXzi9IswduX9Mbh+yRdPKOwCf3fBv2zUzqI
--> ssh-ed25519 +3V2lQ 0p90VtsxWyGFaeeoTISIxQRyeKVk0HoGGq71tjpIPjg
-sRf73Tp3BJ0DsTnJO2xVGyCKjaX7C7oydXj+39dKMUg
---- +/HCG0s/x+c03NG5qrgliJ+5EXXI6UnuJz5XDv2aphY
-ÞšÂ<>™Ý@»=£L¬“7*®„ÐFq<46>UÒ*ûU꿉»È$e=þLgJ|*1Ͻ
BÚE ZG—_Ü5ê²ð—²ŽíÂ,òöÛi<C39B>_'¸d7
Ý3Ú“Nä3ãç¡*»ðªê<C2AA>£ŽáŽòqýŸ‰Oy#¶([l³†pÄf¼õ¾¥ö