Merge pull request 'prometheus: scrape more metrics and add alert rules' (#363) from prometheus-more-metrics into main

Reviewed-on: pub-solar/infra#363
Reviewed-by: Akshay Mankar <axeman@noreply.git.pub.solar>

.forgejo/workflows/check.yml

@@ -10,7 +10,7 @@ jobs:
 
       - name: Check formatting
         run: |
-          nix --accept-flake-config --access-tokens '' develop --command treefmt --fail-on-change
+          nix --accept-flake-config --access-tokens '' develop --command treefmt --ci
 
       - name: Run flake checks
         run: |
@@ -18,14 +18,7 @@ jobs:
           # Prevent cache garbage collection by creating GC roots
           mkdir -p /var/lib/gitea-runner/tankstelle/.local/state/nix/results
 
-          for target in $(nix flake show --json --all-systems | jq '
-            .["nixosConfigurations"] |
-            to_entries[] |
-            .key
-            ' | tr -d '"'
-          ); do
-            nix --print-build-logs --verbose --accept-flake-config --access-tokens '' \
-              build --out-link /var/lib/gitea-runner/tankstelle/.local/state/nix/results/"$target" ".#nixosConfigurations.${target}.config.system.build.toplevel"
-          done
-
-          nix --print-build-logs --verbose --accept-flake-config --access-tokens '' flake check
+          sed -i 's/virtualisation.cores .*/virtualisation.cores = 16;/' tests/keycloak.nix
+          sed -i 's/virtualisation.memorySize .*/virtualisation.memorySize = 16384;/' tests/keycloak.nix
+          # 1 eval-worker needs about 13GB of memory
+          nix --accept-flake-config --access-tokens '' develop --command nix-fast-build --no-nom --skip-cached --systems "x86_64-linux" --max-jobs 10 --eval-workers 2 --out-link /var/lib/gitea-runner/tankstelle/.local/state/nix/results/nix-fast-build

.forgejo/workflows/update-flake-lock.yml

@@ -0,0 +1,19 @@
+name: Update flake lock
+on: 
+  workflow_dispatch:
+  schedule: 
+    - cron: '33 3 * * 6' # “At 03:33 on Saturday.” 
+
+jobs:
+  update_lockfile:
+    runs-on: self-hosted
+    steps:
+      - name: nix flake update
+        uses: https://git.pub.solar/momo/forgejo-action-update-flake@825bba1d03c853ea9aeded656d230db82f9f1216
+        with:
+          forgejo_access_token: ${{ secrets.HAKKONAUT_ACCESS_TOKEN }}
+          gpg_private_key: ${{ secrets.GPG_SIGN_SUBKEY }}
+          gpg_fingerprint: F39BE7813956AB92C3821DFBC02540B6C5BD80EE
+          gpg_passphrase: ${{ secrets.GPG_PASSPHRASE }}
+          git_author: '"hakkonaut" <no-reply@pub.solar>'
+          closure-diff-nixos-configuration: nachtigall

docs/README.md

@@ -0,0 +1,7 @@
+# pub.solar documentation
+
+This directory holds a collection of notes and documentation for pub.solar admins.
+
+### Systems Overview
+
+To get a first overview of existing pub.solar systems, please see the [pub.solar systems overview](./systems-overview.md).

docs/administrative-access.md

@@ -28,18 +28,18 @@ People with admin access to the infrastructure are added to [`logins/admins.nix`
 SSH is not reachable from the open internet. Instead, SSH Port 22 is protected by a wireguard VPN network. Thus, to get root access on the servers, at least two pieces of information have to be added to the admins config:
 
 1. **SSH Public key**: self-explanatory. Add your public key to your user attrset under `sshPubKeys`.
-2. **Wireguard device**: each wireguard device has two parts: the public key and the IP addresses it should have in the wireguard network. The pub.solar wireguard network is spaced under `10.7.6.0/24` and `fd00:fae:fae:fae:fae::/80`. To add your device, it's best to choose a free number between 200 and 255 and use that in both the ipv4 and ipv6 ranges: `10.7.6.<ip-address>/32` `fd00:fae:fae:fae:fae:<ip-address>::/96`. For more information on how to generate keypairs, see [the NixOS Wireguard docs](https://nixos.wiki/wiki/WireGuard#Generate_keypair).
+2. **Wireguard device**: each wireguard device has two parts: the public key and the IP addresses it should have in the wireguard network. The pub.solar wireguard network uses the subnets `10.7.6.0/24` and `fd00:fae:fae:fae:fae::/80`. To add your device, it's best to choose a free number between 200 and 255 and use that in both the ipv4 and ipv6 ranges: `10.7.6.<ip-address>/32` `fd00:fae:fae:fae:fae:<ip-address>::/96`. For more information on how to generate keypairs, see [the NixOS Wireguard docs](https://nixos.wiki/wiki/WireGuard#Generate_keypair).
 
 One can access our hosts using this domain scheme:
 
 ```
-ssh barkeeper@<hostname>.wg.pub.solar
+ssh <unix-username>@<hostname>.wg.pub.solar
 ```
 
 So, for example for `nachtigall`:
 
 ```
-ssh barkeeper@nachtigall.wg.pub.solar
+ssh teutat3s@nachtigall.wg.pub.solar
 ```
 
 Example NixOS snippet for WireGuard client config
@@ -63,12 +63,6 @@ Example NixOS snippet for WireGuard client config
             #endpoint = "138.201.80.102:51820";
             persistentKeepalive = 15;
           }
-          { # flora-6.pub.solar
-            publicKey = "jtSR5G2P/nm9s8WrVc26Xc/SQLupRxyXE+5eIeqlsTU=";
-            allowedIPs = [ "10.7.6.2/32" "fd00:fae:fae:fae:fae:2::/96" ];
-            endpoint = "80.71.153.210:51820";
-            persistentKeepalive = 15;
-          }
           { # metronom.pub.solar
             publicKey = "zOSYGO7MfnOOUnzaTcWiKRQM0qqxR3JQrwx/gtEtHmo=";
             allowedIPs = [ "10.7.6.3/32" "fd00:fae:fae:fae:fae:3::/96" ];
@@ -85,6 +79,39 @@ Example NixOS snippet for WireGuard client config
             #endpoint = "80.244.242.5:51820";
             persistentKeepalive = 15;
           }
+          {
+            # trinkgenossin.pub.solar
+            publicKey = "QWgHovHxtqiQhnHLouSWiT6GIoQDmuvnThYL5c/rvU4=";
+            allowedIPs = [
+              "10.7.6.5/32"
+              "fd00:fae:fae:fae:fae:5::/96"
+            ];
+            #endpoint = "85.215.152.22:51820";
+            endpoint = "[2a01:239:35d:f500::1]:51820";
+            persistentKeepalive = 15;
+          }
+          {
+            # delite.pub.solar
+            publicKey = "ZT2qGWgMPwHRUOZmTQHWCRX4m14YwOsiszjsA5bpc2k=";
+            allowedIPs = [
+              "10.7.6.6/32"
+              "fd00:fae:fae:fae:fae:6::/96"
+            ];
+            #endpoint = "5.255.119.132:51820";
+            endpoint = "[2a04:52c0:124:9d8c::2]:51820";
+            persistentKeepalive = 15;
+          }
+          {
+            # blue-shell.pub.solar
+            publicKey = "bcrIpWrKc1M+Hq4ds3aN1lTaKE26f2rvXhd+93QrzR8=";
+            allowedIPs = [
+              "10.7.6.7/32"
+              "fd00:fae:fae:fae:fae:7::/96"
+            ];
+            #endpoint = "194.13.83.205:51820";
+            endpoint = "[2a03:4000:43:24e::1]:51820";
+            persistentKeepalive = 15;
+          }
         ];
       };
     };

docs/backups.md

@@ -0,0 +1,36 @@
+# Backups
+
+We use [Restic](https://restic.readthedocs.io/en/stable/) to create backups and push them to two repositories.
+Check `./modules/backups.nix` and `./hosts/nachtigall/backups.nix` for working examples.
+
+### Hetzner Storagebox
+
+- Uses SFTP for transfer of backups
+
+Adding a new host SSH public key to the storagebox:
+
+First, [SSH to nachtigall](./administrative-access.md#ssh-access), then become root and add the new SSH public key
+
+```
+sudo -i
+echo '<ssh-public-key>' | ssh -p23 u377325@u377325.your-storagebox.de install-ssh-key
+```
+
+[Link to Hetzner storagebox docs](https://docs.hetzner.com/robot/storage-box/backup-space-ssh-keys).
+
+### Garage S3 buckets
+
+- Uses S3 for transfer of backups
+- One bucket per host, e.g. `nachtigall-backups`, `metronom-backups`
+
+To start transfering backups from a new hosts, this is how to create a new bucket:
+
+First, [SSH to trinkgenossin](./administrative-access.md#ssh-access), then use the `garage` CLI to create a new key and bucket:
+
+```
+export GARAGE_RPC_SECRET=<secret-in-keepass>
+
+garage bucket create <hostname>-backups
+garage key create <hostname>-backups-key
+garage bucket allow <hostname>-backups --read --write --key <hostname>-backups-key
+```

docs/cachix.md

@@ -0,0 +1,55 @@
+# Cachix usage
+
+URL: https://pub-solar.cachix.org
+
+Requirements:
+
+- [Install cachix](https://docs.cachix.org/installation)
+- Optional: To push to the cache, you need to set `CACHIX_AUTH_TOKEN` in your environment. To generate one for you, follow the [Getting Started](https://docs.cachix.org/getting-started#authenticating) docs and login with your GitHub account.
+- Add our binary cache [to your nix config](https://docs.cachix.org/faq#cachix-use-effects). To add the pub-solar cache, run:
+
+```
+cachix use pub-solar
+```
+
+Example to build and push a custom package of a host in this flake (e.g. after creating an overlay):
+
+```
+nix build --json -f . '.#nixosConfigurations.nachtigall.pkgs.keycloak^*' \
+  | jq -r '.[].outputs | to_entries[].value' \
+  | cachix push pub-solar
+```
+
+Example to build and push a package in the `nixpkgs` repo:
+
+```
+cd nixpkgs
+nix build --json -f . 'pkgs.lix^*' \
+  | jq -r '.[].outputs | to_entries[].value' \
+  | cachix push pub-solar
+```
+
+Checking if a package has been correctly pushed to the cache:
+
+```
+❯ nix build --json '/nix/store/f76xi83z4xk9sn6pbh38rh97yvqhb5m0-noto-fonts-color-emoji-png-2.042.drv^*' | jq -r '.[].outputs | to_entries[].value'  | cachix push pub-solar
+Pushing 1 paths (0 are already present) using zstd to cache pub-solar ⏳
+
+✓ /nix/store/xpgpi84765dxqja3gd5pldj49xx2v0xl-noto-fonts-color-emoji-png-2.042 (10.30 MiB)
+
+All done.
+
+❯ curl -I https://pub-solar.cachix.org/xpgpi84765dxqja3gd5pldj49xx2v0xl.narinfo
+HTTP/2 200
+date: Mon, 26 Aug 2024 09:31:10 GMT
+content-type: text/x-nix-narinfo
+traceparent: 00-b99db37cc9c2581b8d226cdf81e54507-794fc49193659c03-01
+tracestate:
+cache-control: public, max-age=14400
+last-modified: Mon, 26 Aug 2024 09:31:10 GMT
+cf-cache-status: EXPIRED
+report-to: {"endpoints":[{"url":"https:\/\/a.nel.cloudflare.com\/report\/v4?s=A67KGsCIsYjoFdvndxJ0rkmb7BZ5ztIpm8WUJKAiUPRVWvbYeXU9gU27P7zryiUtArbwrLzHhhMija0yyXk0kwNa3suz8gNzKK6z1CX1FWDZiiP07rnq7zAg8nZbSBiEU%2FZrU9nSrR6mhuL9ihbmW1Hf"}],"group":"cf-nel","max_age":604800}
+nel: {"success_fraction":0,"report_to":"cf-nel","max_age":604800}
+server: cloudflare
+cf-ray: 8b92ceab0d19c80e-DUS
+```

docs/deletion-request.md

@@ -1,5 +1,17 @@
 # Process for handling a deletion request
 
+## Automated script
+
+Required:
+
+- [SSH access to host `nachtigall`](./administrative-access.md#ssh-access)
+
+SSH into nachtigall, and run the following script:
+
+```
+delete-pubsolar-id $(sudo cat /run/agenix/keycloak-admin-cli-client-secret) $(sudo cat /run/agenix/matrix-admin-access-token) $USERNAME
+```
+
 ### Keycloak
 
 Required:
@@ -34,13 +46,27 @@ Docs: https://docs.nextcloud.com/server/latest/admin_manual/configuration_server
 ### Mastodon
 
 ```
+mkdir /tmp/tootctl
+sudo chown mastodon /tmp/tootctl
+cd /tmp/tootctl
+
 sudo -u mastodon mastodon-tootctl accounts delete --email <mail-address>
+
+rm -r /tmp/tootctl
 ```
 
 Docs: https://docs.joinmastodon.org/admin/tootctl/#accounts-delete
 
 ### Forgejo
 
+Make sure you have access to the gitea/forgejo command:
+
+```
+nix shell nixpkgs#forgejo
+```
+
+Then, delete the user:
+
 ```
 sudo -u gitea gitea admin user delete --config /var/lib/forgejo/custom/conf/app.ini --purge --email <mail-address>
 ```
@@ -50,11 +76,37 @@ Docs: https://forgejo.org/docs/latest/admin/command-line/#delete
 ### Matrix
 
 ```
-curl --header "Authorization: Bearer <admin-access-token>" --request POST http://172.18.0.3:8008/_synapse/admin/v1/deactivate/@<username>:pub.solar --data '{"erase": true}'
+curl --header "Authorization: Bearer <admin-access-token>" --request POST http://127.0.0.1:8008/_synapse/admin/v1/deactivate/@<username>:pub.solar --data '{"erase": true}'
 ```
 
-Docs: https://matrix-org.github.io/synapse/latest/admin_api/user_admin_api.html#deactivate-account
+Docs: https://element-hq.github.io/synapse/latest/admin_api/user_admin_api.html#deactivate-account
+
+The authentication token should be in the keepass. If it is expired, you can get a new one by running the following:
+
+```
+# get full path to mas-cli command with current --config flags from
+# sudo systemctl cat matrix-authentication-service
+sudo -u matrix-authentication-service mas-cli --config nix-store-config --config /run/agenix/matrix-authentication-service-secret-config.yml manage issue-compatibility-token --yes-i-want-to-grant-synapse-admin-privileges crew
+```
 
 ### OpenBikeSensor
 
 Not implemented, see: https://github.com/openbikesensor/portal/issues/95
+
+## Notifying the user
+
+Make sure to send an e-mail to the specified address notifying the user of the accounts deletion.
+
+You can use this template:
+
+```
+Hello,
+
+Your pub.solar ID has been deactivated. Associated data in pub.solar services has been deleted.
+
+Please note that the username is now blocked to prevent impersonation attempts.
+
+Best,
+
+@<name> for the pub.solar crew
+```

docs/deploying.md

@@ -7,22 +7,29 @@ be manually deployed.
 To deploy, make sure you have a [working development shell](./development-shell.md).
 Then, run `deploy-rs` with the hostname of the server you want to deploy:
 
+### Dry-run
+
+Use `--dry-activate` to show a diff of updated packages and all services that
+would be restarted by the update. This will also put all files in place without
+switching to the new generation, enabling a quick switch to the new config at a
+later moment.
+
 For nachtigall.pub.solar:
 
 ```
-deploy --targets '.#nachtigall' --magic-rollback false --auto-rollback false --keep-result --result-path ./results
+deploy --targets '.#nachtigall' --ssh-user <unix-username> --magic-rollback false --auto-rollback false --keep-result --result-path ./results --dry-activate
 ```
 
-For flora-6.pub.solar:
+After reviewing the changes, apply the update with:
 
 ```
-deploy --targets '.#flora-6' --magic-rollback false --auto-rollback false --keep-result --result-path ./results
+deploy --targets '.#nachtigall' --ssh-user <unix-username> --magic-rollback false --auto-rollback false --keep-result --result-path ./results
 ```
 
 For metronom.pub.solar (aarch64-linux):
 
 ```
-deploy --targets '.#metronom' --magic-rollback false --auto-rollback false --keep-result --result-path ./results --remote-build
+deploy --targets '.#metronom' --ssh-user <unix-username> --magic-rollback false --auto-rollback false --keep-result --result-path ./results --remote-build
 ```
 
 Usually we skip all rollback functionality, but if you want to deploy a change
@@ -31,9 +38,6 @@ that might lock you out, e.g. to SSH, it might make sense to set these to `true`
 To skip flake checks, e.g. because you already ran them manually before
 deployment, add the flag `--skip-checks` at the end of the command.
 
-`--dry-activate` can be used to only put all files in place without switching,
-to enable switching to the new config quickly at a later moment.
-
 We use `--keep-result --result-path ./results` to keep the last `result`
 symlink of each `deploy` from being garbage collected. That way, we keep builds
 cached in the Nix store. This is optional and both flags can be removed if disk

docs/dns.md

@@ -1,18 +1,10 @@
 # Changing DNS entries
 
 Our current DNS provider is [namecheap](https://www.namecheap.com/).
-We use [Terraform](https://www.terraform.io) to declaratively manage our pub.solar DNS records.
+We use [OpenTofu](https://opentofu.org) to declaratively manage our pub.solar DNS records.
 
 ### Initial setup
 
-Skip this step if you already have a `triton` profile setup.
-
-```
-triton profile create
-```
-
-Please follow https://docs.greenbaum.cloud/en/devops/triton-cli.html for the details.
-
 You will need to setup the following [namecheap API credentials](https://www.namecheap.com/support/api/intro),
 look for "namecheap API key" in the pub.solar Keepass database.
 
@@ -28,13 +20,15 @@ You will probably also need to add your external IP to the [API allow list](http
 dig -4 ip @dns.toys
 ```
 
-Now, change into the terraform directory and initialize the terraform providers.
+Now, change into the terraform directory and initialize the terraform providers. To decrypt existing state,
+search for "terraform state passphrase" in the pub.solar Keepass database.
 
 ```
 cd terraform
-export TRITON_KEY_ID=$(cat ~/.config/triton/profiles.d/lev-1-pub_solar.json | jq --raw-output .keyId)
+export TF_VAR_state_passphrase=$(secret-tool lookup pub.solar terraform-state-passphrase-dns)
 
-terraform init
+alias tofu="terraform-backend-git --access-logs --tf tofu git terraform"
+tofu init
 ```
 
 Make your changes, e.g. in `dns.tf`.
@@ -46,20 +40,21 @@ $EDITOR dns.tf
 Plan your changes using:
 
 ```
-terraform plan -out pub-solar-infra.plan
+tofu plan -out pub-solar-infra.plan
 ```
 
 After verification, apply your changes with:
 
 ```
-terraform apply "pub-solar-infra.plan"
+tofu apply "pub-solar-infra.plan"
 ```
 
 ### Useful links
 
-We use the Manta remote backend to save the terraform state for collaboration.
+We use terraform-backend-git remote backend with opentofu state encryption for collaboration.
 
-- https://www.terraform.io/language/v1.2.x/settings/backends/manta
+- https://github.com/plumber-cd/terraform-backend-git
+- https://opentofu.org/docs/language/state/encryption
 
 Namecheap Terraform provider docs:
 

docs/drone-ci.md

@@ -1,19 +0,0 @@
-# Drone CI
-
-We currently use two CI systems, [drone CI](https://drone.io), reachable via
-https://ci.pub.solar and [Forgejo Actions](https://forgejo.org/docs/latest/user/actions/),
-which UI is integrated into https://git.pub.solar, for example
-https://git.pub.solar/pub-solar/infra/actions.
-
-### Signing the `.drone.yml` file
-
-Login to https://ci.pub.solar by clicking on the user icon in the bottom left.
-After logging in, you can view your personal API token by clicking on the same
-icon. If you're using the nix [development-shell](./development-shell.md), the
-`drone` command will already be installed.
-
-```
-export DRONE_TOKEN=<your-drone-api-token>
-
-drone --token $DRONE_TOKEN sign --save pub-solar/os
-```

docs/garage.md

@@ -0,0 +1,84 @@
+# Garage
+
+### How-To create a new bucket + keys
+
+Requirements:
+
+- `garage` RPC credentials, in the shared keepass, search for 'garage rpc secret'.
+- [Setup WireGuard](./administrative-access.md#ssh-access) for hosts: `trinkgenossin`, optionally: `delite`, `blue-shell`
+
+```
+ssh <unix-username>@trinkgenossin.wg.pub.solar
+```
+
+```
+# Add a few spaces to avoid leaking the secret to the shell history
+   export GARAGE_RPC_SECRET=<secret-in-keepass>
+```
+
+Now, you can run the following command to check the cluster status:
+
+```
+garage status
+```
+
+Command to list all existing buckets:
+
+```
+garage bucket list
+```
+
+Creating a new bucket and access keys:
+
+```
+garage bucket create <bucket-name>
+garage key create <bucket-name>-key
+garage bucket allow <bucket-name> --read --write --key <bucket-name>-key
+```
+
+Full example for `mastodon` bucket:
+
+```
+garage bucket create mastodon
+
+garage key create mastodon-key
+
+garage bucket allow mastodon --read --write --key mastodon-key
+```
+
+Then [setup your favourite S3 client](https://garagehq.deuxfleurs.fr/documentation/connect/cli/)
+or use the bucket with any [S3 compatible software](https://garagehq.deuxfleurs.fr/documentation/connect/).
+
+Further reading:
+
+- https://garagehq.deuxfleurs.fr/documentation/quick-start/
+- https://garagehq.deuxfleurs.fr/documentation/connect/
+- https://garagehq.deuxfleurs.fr/documentation/connect/apps/#mastodon
+
+### Notes on manual setup steps
+
+```
+ssh <unix-username>@trinkgenossin.wg.pub.solar
+
+# Add a few spaces to avoid leaking the secret to the shell history
+    export GARAGE_RPC_SECRET=<secret-in-keepass>
+
+# Uses the default config /etc/garage.toml
+garage node id
+
+garage node connect <node-id2>
+garage node connect <node-id3>
+
+garage status
+
+#Zones
+#DE-1 DE-2 NL-1
+
+garage layout assign fdaa -z DE-1 -c 800G -t trinkgenossin
+garage layout assign 8835 -z DE-2 -c 800G -t blue-shell
+garage layout assign 73da -z NL-1 -c 800G -t delite
+garage layout show
+garage layout apply --version 1
+```
+
+Source: https://garagehq.deuxfleurs.fr/documentation/cookbook/real-world/#creating-a-cluster-layout

docs/keycloak/delete-unverified-accounts.md

@@ -12,7 +12,7 @@ Run following after SSH'ing to `nachtigall`.
 Credentials for the following command are in keepass. Create a keycloak
 config/credentials file at `/tmp/kcadm.config`:
 
-```
+```bash
 sudo --user keycloak kcadm.sh config credentials \
   --config /tmp/kcadm.config \
   --server https://auth.pub.solar \
@@ -22,7 +22,7 @@ sudo --user keycloak kcadm.sh config credentials \
 
 Get list of accounts without a verified email address:
 
-```
+```bash
 sudo --user keycloak kcadm.sh get \
   --config /tmp/kcadm.config \
   users \
@@ -35,7 +35,7 @@ Review list of accounts, especially check `createdTimestamp` if any accounts
 were created in the past 2 days. If so, delete those from the
 `/tmp/keycloak-unverified-accounts` file.
 
-```
+```bash
 createdTimestamps=( $( nix run nixpkgs#jq -- -r '.[].createdTimestamp' < /tmp/keycloak-unverified-accounts ) )
 
 # timestamps are in nanoseconds since epoch, so we need to strip the last three digits
@@ -46,17 +46,17 @@ vim /tmp/keycloak-unverified-accounts
 
 Check how many accounts are going to be deleted:
 
-```
+```bash
 jq -r '.[].id' < /tmp/keycloak-unverified-accounts | wc -l
 ```
 
-```
+```bash
 jq -r '.[].id' < /tmp/keycloak-unverified-accounts > /tmp/keycloak-unverified-account-ids
 ```
 
 Final check before deletion (dry-run):
 
-```
+```bash
 for id in $(cat /tmp/keycloak-unverified-account-ids)
   do
     echo sudo --user keycloak kcadm.sh delete \
@@ -68,7 +68,7 @@ for id in $(cat /tmp/keycloak-unverified-account-ids)
 
 THIS WILL DELETE ACCOUNTS:
 
-```
+```bash
 for id in $(cat /tmp/keycloak-unverified-account-ids)
   do
     sudo --user keycloak kcadm.sh delete \
@@ -77,3 +77,9 @@ for id in $(cat /tmp/keycloak-unverified-account-ids)
       --realm pub.solar
   done
 ```
+
+Delete the temp files:
+
+```bash
+sudo rm /tmp/kcadm.config /tmp/keycloak-unverified-accounts /tmp/keycloak-unverified-account-ids
+```

docs/matrix-draupnir.md

@@ -0,0 +1,52 @@
+# Notes for setting up draupnir moderation bot
+
+From: https://the-draupnir-project.github.io/draupnir-documentation/bot/setup
+
+### Overview
+
+There are a number of steps to complete to get Draupnir running:
+
+1. [Create an account](https://the-draupnir-project.github.io/draupnir-documentation/bot/setup_draupnir_account) for Draupnir to use.
+   1. Optionally [disabling rate limits](https://matrix-org.github.io/synapse/latest/admin_api/user_admin_api.html#set-ratelimit) for this account.
+   1. Make user an [admin account in synapse](https://element-hq.github.io/synapse/latest/usage/administration/admin_api/index.html)
+1. Review our notes on [encryption](https://the-draupnir-project.github.io/draupnir-documentation/bot/encryption).
+1. [Create a management room](https://the-draupnir-project.github.io/draupnir-documentation/bot/setup_management_room) for Draupnir to use.
+1. Install Draupnir on your system (we use NixOS for this)
+
+### After creating the draupnir account
+
+Disable rate limit via synapse admin API:
+
+```
+curl --header "Authorization: Bearer $TOKEN" 'http://127.0.0.1:8008/_synapse/admin/v1/users/@draupnir:pub.solar/override_ratelimit' -X POST -d '{"messages_per_second": 0,  "burst_count": 0}'
+```
+
+Make draupnir admin
+
+```
+sudo -u postgres psql -d matrix
+matrix=# UPDATE users SET admin = 1 WHERE name = '@draupnir:pub.solar';
+```
+
+With matrix-authentication-service (MAS), getting an admin token for the
+draupnir user is different. Run the following commands on the host running
+matrix-authentication-service.
+
+Get the required nix store paths for `mas-cli`, copy it from the output of the
+following command:
+
+```
+sudo systemctl cat matrix-authentication-service
+```
+
+Get a admin token for the `draupnir` matrix account:
+
+```
+sudo -u matrix-authentication-service <nix-store-path>/bin/mas-cli --config <nix-store-path>/-config.yaml --config /run/agenix/staging-matrix-authentication-service-secret-config.yml manage issue-compatibility-token --yes-i-want-to-grant-synapse-admin-privileges draupnir
+```
+
+This ensures the `draupnir` user has devices configured, which is required by synapse:
+
+```
+sudo -u matrix-authentication-service <nix-store-path>/bin/mas-cli --config <nix-store-path>/-config.yaml --config /run/agenix/staging-matrix-authentication-service-secret-config.yml manage provision-all-users
+```

docs/matrix-suspend-account.md

@@ -0,0 +1,31 @@
+# Matrix account suspension
+
+> Unlike [account locking](https://spec.matrix.org/v1.12/client-server-api/#account-locking),
+> [suspension](https://github.com/matrix-org/matrix-spec-proposals/blob/main/proposals/3823-code-for-account-suspension.md)
+> allows the user to have a (largely) readonly view of their account.
+> Homeserver administrators and moderators may use this functionality to
+> temporarily deactivate an account, or place conditions on the account's
+> experience. Critically, like locking, account suspension is reversible, unlike
+> the deactivation mechanism currently available in Matrix - a destructive,
+> irreversible, action.
+
+Required:
+
+- `matrix-synapse admin token`
+- [SSH access to host `nachtigall`](./administrative-access.md#ssh-access)
+
+## Suspending an account
+
+```bash
+curl --header "Authorization: Bearer <admin-access-token>" --request PUT http://127.0.0.1:8008/_synapse/admin/v1/suspend/@<username>:pub.solar --data '{"suspend": true}'
+```
+
+## Unsuspending an account
+
+```bash
+curl --header "Authorization: Bearer <admin-access-token>" --request PUT http://127.0.0.1:8008/_synapse/admin/v1/suspend/@<username>:pub.solar --data '{"suspend": false}'
+```
+
+Links:
+
+- [synapse docs Suspend/Unsuspend Account](https://element-hq.github.io/synapse/latest/admin_api/user_admin_api.html#suspendunsuspend-account)

docs/nachtigall/notes-disk-extension.md

@@ -0,0 +1,348 @@
+# Notes on adding two disks to server nachtigall
+
+Status after Hetzner support added two additional 1TB NVMe disks:
+
+```
+teutat3s in 🌐 nachtigall in ~
+❯ lsblk -f
+NAME        FSTYPE     FSVER LABEL     UUID                                 FSAVAIL FSUSE% MOUNTPOINTS
+nvme0n1
+├─nvme0n1p1
+├─nvme0n1p2 vfat       FAT16           5494-BA1E                               385M    21% /boot2
+└─nvme0n1p3 zfs_member 5000  root_pool 8287701206764130981
+nvme1n1
+├─nvme1n1p1
+├─nvme1n1p2 vfat       FAT32           5493-EFF5                               1.8G     5% /boot1
+└─nvme1n1p3 zfs_member 5000  root_pool 8287701206764130981
+nvme2n1
+nvme3n1
+
+teutat3s in 🌐 nachtigall in ~
+❯ sudo fdisk -l /dev/nvme0n1
+Disk /dev/nvme0n1: 953.87 GiB, 1024209543168 bytes, 2000409264 sectors
+Disk model: KXG60ZNV1T02 TOSHIBA
+Units: sectors of 1 * 512 = 512 bytes
+Sector size (logical/physical): 512 bytes / 512 bytes
+I/O size (minimum/optimal): 512 bytes / 512 bytes
+Disklabel type: gpt
+Disk identifier: 28F8681A-4559-4801-BF3F-BFEC8058236B
+
+Device          Start        End    Sectors   Size Type
+/dev/nvme0n1p1   2048       4095       2048     1M BIOS boot
+/dev/nvme0n1p2   4096     999423     995328   486M EFI System
+/dev/nvme0n1p3 999424 2000408575 1999409152 953.4G Linux filesystem
+
+teutat3s in 🌐 nachtigall in ~
+❯ sudo fdisk -l /dev/nvme1n1
+Disk /dev/nvme1n1: 953.87 GiB, 1024209543168 bytes, 2000409264 sectors
+Disk model: SAMSUNG MZVL21T0HCLR-00B00
+Units: sectors of 1 * 512 = 512 bytes
+Sector size (logical/physical): 512 bytes / 512 bytes
+I/O size (minimum/optimal): 512 bytes / 512 bytes
+Disklabel type: gpt
+Disk identifier: A143A806-69C5-4EFC-8E34-20C35574D990
+
+Device           Start        End    Sectors  Size Type
+/dev/nvme1n1p1    2048       4095       2048    1M BIOS boot
+/dev/nvme1n1p2    4096    3905535    3901440  1.9G EFI System
+/dev/nvme1n1p3 3905536 2000408575 1996503040  952G Linux filesystem
+
+teutat3s in 🌐 nachtigall in ~
+❯ sudo fdisk -l /dev/nvme2n1
+Disk /dev/nvme2n1: 953.87 GiB, 1024209543168 bytes, 2000409264 sectors
+Disk model: SAMSUNG MZVL21T0HDLU-00B07
+Units: sectors of 1 * 512 = 512 bytes
+Sector size (logical/physical): 512 bytes / 512 bytes
+I/O size (minimum/optimal): 512 bytes / 512 bytes
+
+teutat3s in 🌐 nachtigall in ~
+❯ sudo fdisk -l /dev/nvme3n1
+Disk /dev/nvme3n1: 953.87 GiB, 1024209543168 bytes, 2000409264 sectors
+Disk model: SAMSUNG MZVL21T0HCLR-00B00
+Units: sectors of 1 * 512 = 512 bytes
+Sector size (logical/physical): 512 bytes / 512 bytes
+I/O size (minimum/optimal): 512 bytes / 512 bytes
+```
+
+Partitioning and formatting the new disks `/dev/nvme2n1` and `/dev/nvme3n1`:
+
+```
+teutat3s in 🌐 nachtigall in ~
+❯ sudo fdisk /dev/nvme2n1
+
+Welcome to fdisk (util-linux 2.39.4).
+Changes will remain in memory only, until you decide to write them.
+Be careful before using the write command.
+
+Device does not contain a recognized partition table.
+Created a new DOS (MBR) disklabel with disk identifier 0x0852470c.
+
+Command (m for help): p
+Disk /dev/nvme2n1: 953.87 GiB, 1024209543168 bytes, 2000409264 sectors
+Disk model: SAMSUNG MZVL21T0HDLU-00B07
+Units: sectors of 1 * 512 = 512 bytes
+Sector size (logical/physical): 512 bytes / 512 bytes
+I/O size (minimum/optimal): 512 bytes / 512 bytes
+Disklabel type: dos
+Disk identifier: 0x0852470c
+
+Command (m for help): m
+
+Help:
+
+  DOS (MBR)
+   a   toggle a bootable flag
+   b   edit nested BSD disklabel
+   c   toggle the dos compatibility flag
+
+  Generic
+   d   delete a partition
+   F   list free unpartitioned space
+   l   list known partition types
+   n   add a new partition
+   p   print the partition table
+   t   change a partition type
+   v   verify the partition table
+   i   print information about a partition
+
+  Misc
+   m   print this menu
+   u   change display/entry units
+   x   extra functionality (experts only)
+
+  Script
+   I   load disk layout from sfdisk script file
+   O   dump disk layout to sfdisk script file
+
+  Save & Exit
+   w   write table to disk and exit
+   q   quit without saving changes
+
+  Create a new label
+   g   create a new empty GPT partition table
+   G   create a new empty SGI (IRIX) partition table
+   o   create a new empty MBR (DOS) partition table
+   s   create a new empty Sun partition table
+
+
+Command (m for help): g
+Created a new GPT disklabel (GUID: 8CC98E3F-20A8-4A2D-8D50-9CD769EE4C65).
+
+Command (m for help): p
+Disk /dev/nvme2n1: 953.87 GiB, 1024209543168 bytes, 2000409264 sectors
+Disk model: SAMSUNG MZVL21T0HDLU-00B07
+Units: sectors of 1 * 512 = 512 bytes
+Sector size (logical/physical): 512 bytes / 512 bytes
+I/O size (minimum/optimal): 512 bytes / 512 bytes
+Disklabel type: gpt
+Disk identifier: 8CC98E3F-20A8-4A2D-8D50-9CD769EE4C65
+
+Command (m for help): n
+Partition number (1-128, default 1):
+First sector (2048-2000409230, default 2048):
+Last sector, +/-sectors or +/-size{K,M,G,T,P} (2048-2000409230, default 2000408575): 4095
+
+Created a new partition 1 of type 'Linux filesystem' and of size 1 MiB.
+
+Command (m for help): t
+Selected partition 1
+Partition type or alias (type L to list all): L
+  1 EFI System                     C12A7328-F81F-11D2-BA4B-00A0C93EC93B
+  2 MBR partition scheme           024DEE41-33E7-11D3-9D69-0008C781F39F
+  3 Intel Fast Flash               D3BFE2DE-3DAF-11DF-BA40-E3A556D89593
+  4 BIOS boot                      21686148-6449-6E6F-744E-656564454649
+  5 Sony boot partition            F4019732-066E-4E12-8273-346C5641494F
+  6 Lenovo boot partition          BFBFAFE7-A34F-448A-9A5B-6213EB736C22
+  7 PowerPC PReP boot              9E1A2D38-C612-4316-AA26-8B49521E5A8B
+  8 ONIE boot                      7412F7D5-A156-4B13-81DC-867174929325
+  9 ONIE config                    D4E6E2CD-4469-46F3-B5CB-1BFF57AFC149
+ 10 Microsoft reserved             E3C9E316-0B5C-4DB8-817D-F92DF00215AE
+ 11 Microsoft basic data           EBD0A0A2-B9E5-4433-87C0-68B6B72699C7
+ 12 Microsoft LDM metadata         5808C8AA-7E8F-42E0-85D2-E1E90434CFB3
+ 13 Microsoft LDM data             AF9B60A0-1431-4F62-BC68-3311714A69AD
+ 14 Windows recovery environment   DE94BBA4-06D1-4D40-A16A-BFD50179D6AC
+ 15 IBM General Parallel Fs        37AFFC90-EF7D-4E96-91C3-2D7AE055B174
+ 16 Microsoft Storage Spaces       E75CAF8F-F680-4CEE-AFA3-B001E56EFC2D
+ 17 HP-UX data                     75894C1E-3AEB-11D3-B7C1-7B03A0000000
+ 18 HP-UX service                  E2A1E728-32E3-11D6-A682-7B03A0000000
+ 19 Linux swap                     0657FD6D-A4AB-43C4-84E5-0933C84B4F4F
+ 20 Linux filesystem               0FC63DAF-8483-4772-8E79-3D69D8477DE4
+...
+Partition type or alias (type L to list all): 4
+Changed type of partition 'Linux filesystem' to 'BIOS boot'.
+
+Command (m for help): n
+Partition number (2-128, default 2):
+First sector (4096-2000409230, default 4096):
+Last sector, +/-sectors or +/-size{K,M,G,T,P} (4096-2000409230, default 2000408575): 3901440
+
+Created a new partition 2 of type 'Linux filesystem' and of size 1.9 GiB.
+
+Command (m for help): t
+Partition number (1,2, default 2): 2
+Partition type or alias (type L to list all): 1
+
+Changed type of partition 'Linux filesystem' to 'EFI System'.
+
+Command (m for help): n
+Partition number (3-128, default 3):
+First sector (3901441-2000409230, default 3903488):
+Last sector, +/-sectors or +/-size{K,M,G,T,P} (3903488-2000409230, default 2000408575):
+
+Created a new partition 3 of type 'Linux filesystem' and of size 952 GiB.
+
+Command (m for help): p
+Disk /dev/nvme2n1: 953.87 GiB, 1024209543168 bytes, 2000409264 sectors
+Disk model: SAMSUNG MZVL21T0HDLU-00B07
+Units: sectors of 1 * 512 = 512 bytes
+Sector size (logical/physical): 512 bytes / 512 bytes
+I/O size (minimum/optimal): 512 bytes / 512 bytes
+Disklabel type: gpt
+Disk identifier: 8CC98E3F-20A8-4A2D-8D50-9CD769EE4C65
+
+Device           Start        End    Sectors  Size Type
+/dev/nvme2n1p1    2048       4095       2048    1M BIOS boot
+/dev/nvme2n1p2    4096    3901440    3897345  1.9G EFI System
+/dev/nvme2n1p3 3903488 2000408575 1996505088  952G Linux filesystem
+
+Command (m for help): w
+The partition table has been altered.
+Calling ioctl() to re-read partition table.
+Syncing disks.
+
+
+teutat3s in 🌐 nachtigall in ~ took 5m41s
+❯ sudo fdisk /dev/nvme3n1
+
+Welcome to fdisk (util-linux 2.39.4).
+Changes will remain in memory only, until you decide to write them.
+Be careful before using the write command.
+
+Device does not contain a recognized partition table.
+Created a new DOS (MBR) disklabel with disk identifier 0xa77eb504.
+
+Command (m for help): g
+Created a new GPT disklabel (GUID: 56B64CEE-5E0C-4EAA-AE2F-5BF4356183A5).
+
+Command (m for help): n
+Partition number (1-128, default 1):
+First sector (2048-2000409230, default 2048):
+Last sector, +/-sectors or +/-size{K,M,G,T,P} (2048-2000409230, default 2000408575): 4095
+
+Created a new partition 1 of type 'Linux filesystem' and of size 1 MiB.
+
+Command (m for help): t
+Selected partition 1
+Partition type or alias (type L to list all): 4
+Changed type of partition 'Linux filesystem' to 'BIOS boot'.
+
+Command (m for help): n
+Partition number (2-128, default 2):
+First sector (4096-2000409230, default 4096):
+Last sector, +/-sectors or +/-size{K,M,G,T,P} (4096-2000409230, default 2000408575): 3901440
+
+Created a new partition 2 of type 'Linux filesystem' and of size 1.9 GiB.
+
+Command (m for help): t
+Partition number (1,2, default 2): 2
+Partition type or alias (type L to list all): 1
+
+Changed type of partition 'Linux filesystem' to 'EFI System'.
+
+Command (m for help): n
+Partition number (3-128, default 3):
+First sector (3901441-2000409230, default 3903488):
+Last sector, +/-sectors or +/-size{K,M,G,T,P} (3903488-2000409230, default 2000408575):
+
+Created a new partition 3 of type 'Linux filesystem' and of size 952 GiB.
+
+Command (m for help): p
+Disk /dev/nvme3n1: 953.87 GiB, 1024209543168 bytes, 2000409264 sectors
+Disk model: SAMSUNG MZVL21T0HCLR-00B00
+Units: sectors of 1 * 512 = 512 bytes
+Sector size (logical/physical): 512 bytes / 512 bytes
+I/O size (minimum/optimal): 512 bytes / 512 bytes
+Disklabel type: gpt
+Disk identifier: 56B64CEE-5E0C-4EAA-AE2F-5BF4356183A5
+
+Device           Start        End    Sectors  Size Type
+/dev/nvme3n1p1    2048       4095       2048    1M BIOS boot
+/dev/nvme3n1p2    4096    3901440    3897345  1.9G EFI System
+/dev/nvme3n1p3 3903488 2000408575 1996505088  952G Linux filesystem
+
+Command (m for help): w
+The partition table has been altered.
+Calling ioctl() to re-read partition table.
+Syncing disks.
+
+teutat3s in 🌐 nachtigall in ~
+❯ sudo mkfs.vfat /dev/nvme2n1p2
+mkfs.fat 4.2 (2021-01-31)
+
+teutat3s in 🌐 nachtigall in ~
+❯ sudo mkfs.vfat /dev/nvme3n1p2
+mkfs.fat 4.2 (2021-01-31)
+
+teutat3s in 🌐 nachtigall in ~
+❯ lsblk -f
+NAME        FSTYPE     FSVER LABEL     UUID                                 FSAVAIL FSUSE% MOUNTPOINTS
+nvme0n1
+├─nvme0n1p1
+├─nvme0n1p2 vfat       FAT16           5494-BA1E                               385M    21% /boot2
+└─nvme0n1p3 zfs_member 5000  root_pool 8287701206764130981
+nvme1n1
+├─nvme1n1p1
+├─nvme1n1p2 vfat       FAT32           5493-EFF5                               1.8G     5% /boot1
+└─nvme1n1p3 zfs_member 5000  root_pool 8287701206764130981
+nvme2n1
+├─nvme2n1p1
+├─nvme2n1p2 vfat       FAT32           E4E4-88C7
+└─nvme2n1p3
+nvme3n1
+├─nvme3n1p1
+├─nvme3n1p2 vfat       FAT32           E76C-A8A0
+└─nvme3n1p3
+```
+
+Finally, adding the new drives to the ZFS zpool `root_pool` to extend available disk space:
+
+```
+teutat3s in 🌐 nachtigall in ~
+❯ sudo zpool status
+  pool: root_pool
+ state: ONLINE
+  scan: scrub repaired 0B in 00:17:47 with 0 errors on Sat Mar  1 03:35:20 2025
+config:
+
+	NAME                                                      STATE     READ WRITE CKSUM
+	root_pool                                                 ONLINE       0     0     0
+	  mirror-0                                                ONLINE       0     0     0
+	    nvme-SAMSUNG_MZVL21T0HCLR-00B00_S676NF0R517371-part3  ONLINE       0     0     0
+	    nvme-KXG60ZNV1T02_TOSHIBA_Z9NF704ZF9ZL-part3          ONLINE       0     0     0
+
+errors: No known data errors
+
+teutat3s in 🌐 nachtigall in ~
+❯ sudo zpool add root_pool mirror nvme-SAMSUNG_MZVL21T0HDLU-00B07_S77WNF0XA01902-part3 nvme-SAMSUNG_MZVL21T0HCLR-00B00_S676NU0W623944-part3
+
+teutat3s in 🌐 nachtigall in ~
+❯ sudo zpool status
+  pool: root_pool
+ state: ONLINE
+  scan: scrub repaired 0B in 00:17:47 with 0 errors on Sat Mar  1 03:35:20 2025
+config:
+
+	NAME                                                      STATE     READ WRITE CKSUM
+	root_pool                                                 ONLINE       0     0     0
+	  mirror-0                                                ONLINE       0     0     0
+	    nvme-SAMSUNG_MZVL21T0HCLR-00B00_S676NF0R517371-part3  ONLINE       0     0     0
+	    nvme-KXG60ZNV1T02_TOSHIBA_Z9NF704ZF9ZL-part3          ONLINE       0     0     0
+	  mirror-1                                                ONLINE       0     0     0
+	    nvme-SAMSUNG_MZVL21T0HDLU-00B07_S77WNF0XA01902-part3  ONLINE       0     0     0
+	    nvme-SAMSUNG_MZVL21T0HCLR-00B00_S676NU0W623944-part3  ONLINE       0     0     0
+
+teutat3s in 🌐 nachtigall in ~
+❯ sudo zfs list root_pool
+NAME        USED  AVAIL  REFER  MOUNTPOINT
+root_pool   782G  1.04T   192K  none
+```

docs/nextcloud.md

@@ -0,0 +1,19 @@
+# Nextcloud debugging
+
+Set loglevel to `0` for debug logs:
+
+```nix
+services.nextcloud.settings.loglevel = 0;
+```
+
+Then, logs appear in the `phpfpm-nextcloud.service` and `nextcloud-*.service` logs:
+
+```bash
+sudo journalctl --follow --unit phpfpm-nextcloud --unit nextcloud-*.service
+```
+
+Make sure to set the loglevel back to the default `2` warning after debugging:
+
+```nix
+services.nextcloud.settings.loglevel = 2;
+```

docs/nix-flake-updates.md

@@ -41,3 +41,7 @@ wrapped-ruby-mastodon-gems: 4.2.1 → 4.2.3
 zfs-kernel: 2.2.1-6.1.64 → 2.2.2-6.1.66
 zfs-user: 2.2.1 → 2.2.2
 ```
+
+### Deploying updates
+
+See [deploying.md](./deploying.md).

docs/nixos-anywhere.md

@@ -0,0 +1,29 @@
+# Deploying with nixos-anywhere
+
+## On Target: Enter NixOS from non-NixOS host
+
+In case you cannot boot easily into a nixos-installer image you can download the kexec installer image of NixOS and kexec into it:
+
+```
+curl -L https://github.com/nix-community/nixos-images/releases/download/nixos-unstable/nixos-kexec-installer-noninteractive-x86_64-linux.tar.gz | tar -xzf- -C /root
+/root/kexec/run
+```
+
+## Run Disko
+
+```
+nix run github:nix-community/nixos-anywhere -- --flake .#<hostname> --target-host root@<host> --phases disko
+```
+
+## On Target: Create inital ssh host key used in initrd
+
+```
+mkdir -p /mnt/etc/secrets/initrd
+ssh-keygen -t ed25519 -f /mnt/etc/secrets/initrd/ssh_host_ed25519_key
+```
+
+## Run NixOS Anywhere
+
+```
+nix run github:nix-community/nixos-anywhere -- --flake .#<hostname> --target-host root@<host> --phases install,reboot
+```

docs/systems-overview.md

@@ -0,0 +1,187 @@
+# pub.solar Systems Overview
+
+Last updated: 2025-03-11
+
+Jump to:
+
+1. [Server nachtigall.pub.solar](#server-nachtigall-pub-solar)
+2. [Server metronom.pub.solar](#server-metronom-pub-solar)
+3. [Server trinkgenossin.pub.solar](#server-trinkgenossin-pub-solar)
+4. [Server blue-shell.pub.solar](#server-blue-shell-pub-solar)
+5. [Server delite.pub.solar](#server-delite-pub-solar)
+6. [Server tankstelle.pub.solar](#server-tankstelle-pub-solar)
+7. [Server underground.pub.solar](#server-underground-pub-solar)
+8. [Hetzner 1TB storagebox](#hetzner-1tb-storagebox)
+
+### Server nachtigall.pub.solar
+
+**Specs:**
+
+- AMD Ryzen 7 3700X 8-Core Processor
+- 64 GB RAM
+- 4x 1TB NVMe disks
+
+**Disk layout:**
+
+- Encrypted ZFS mirror vdevs
+
+**Operating System:**
+
+- NixOS 24.11 `x86_64-linux`
+
+**Usage:**
+Main pub.solar server. Hosts the majority of pub.solar services. Non-exhaustive list:
+
+- collabora
+- coturn
+- forgejo
+- keycloak
+- mailman
+- mastodon
+- matrix-synapse (homeserver)
+- mediawiki
+- nextcloud
+- owncast
+- searx
+- tmate
+- tt-rss
+- obs-portal
+
+### Server metronom.pub.solar
+
+**Specs:**
+
+- Hetzner VPS type: CAX11
+- 2 vCPU
+- 4 GB RAM
+- 40GB disk
+
+**Disk layout:**
+
+- Encrypted ZFS single disk (stripe)
+
+**Operating System:**
+
+- NixOS 24.11 `aarch64-linux`
+
+**Usage:**
+pub.solar mail server. Note this is an ARM server.
+
+### Server trinkgenossin.pub.solar
+
+**Specs:**
+
+- Strato VPS type: VPS Linux VC8-32
+- 8 core AMD EPYC-Milan Processor
+- 32 GB RAM
+- 1TB NVMe disk
+
+**Disk layout:**
+
+- Encrypted LUKS single disk
+
+**Operating System:**
+
+- NixOS 24.11 `x86_64-linux`
+
+**Usage:**
+Monitor, garage cluster node. Services:
+
+- grafana
+- loki
+- prometheus
+- garage
+- forgejo-actions-runner (docker)
+
+### Server blue-shell.pub.solar
+
+**Specs:**
+
+- netcup VPS type: VPS 1000 G11
+- 4 core AMD EPYC-Rome Processor
+- 8 GB RAM
+- 256 GB SSD disk
+- 850GB mechanical disk
+
+**Disk layout:**
+
+- Encrypted LVM on LUKS single disk and encrypted LUKS garage data disk
+
+**Operating System:**
+
+- NixOS 24.11 `x86_64-linux`
+
+**Usage:**
+Garage cluster node.
+
+### Server delite.pub.solar
+
+**Specs:**
+
+- liteserver VPS type: HDD Storage VPS - HDD-2G
+- 1 core AMD EPYC 7452
+- 2 GB RAM
+- 1TB mechanical disk
+
+**Disk layout:**
+
+- Encrypted LVM on LUKS single disk
+
+**Operating System:**
+
+- NixOS 24.11 `x86_64-linux`
+
+**Usage:**
+Garage cluster node.
+
+### Server tankstelle.pub.solar
+
+**Specs:**
+
+- 24 core Intel Xeon E5-2670 v2 @ 2.50GHz
+- 40 GB RAM
+- 80GB SSD disk
+
+**Disk layout:**
+
+- LVM
+
+**Operating System:**
+
+- NixOS 24.11 `x86_64-linux`
+
+**Usage:**
+
+- forgejo-actions-runner (selfhosted, NixOS)
+
+### Server underground.pub.solar
+
+**Specs:**
+
+- 8 core Intel Xeon E5-2670 v2 @ 2.50GHz
+- 16 GB RAM
+- 40 GB SSD disk
+
+**Disk layout:**
+
+- LVM on LUKS, single disk
+
+**Operating System:**
+
+- NixOS 24.11 `x86_64-linux`
+
+**Usage:**
+Testing server.
+
+### Hetzner 1TB storagebox
+
+Hostname:
+
+```
+u377325@u377325.your-storagebox.de
+```
+
+**Usage:**
+Backups get pushed to a Hetzner storagebox every night.
+
+### Garage cluster

docs/zfs-quickstart.md

@@ -0,0 +1,19 @@
+# ZFS Quick Start
+
+View current status of the ZFS pool (zpool):
+
+```
+sudo zpool status
+```
+
+View available disk space of the pool, replace `<pool-name>` with the pool name from the output above:
+
+```
+sudo zfs list <pool-name>
+```
+
+List all snapshots:
+
+```
+sudo zfs list -t snapshot
+```

flake.lock

@@ -14,11 +14,11 @@
         "systems": "systems"
       },
       "locked": {
-        "lastModified": 1723293904,
-        "narHash": "sha256-b+uqzj+Wa6xgMS9aNbX4I+sXeb5biPDi39VgvSFqFvU=",
+        "lastModified": 1736955230,
+        "narHash": "sha256-uenf8fv2eG5bKM8C/UvFaiJMZ4IpUFaQxk9OH5t/1gA=",
         "owner": "ryantm",
         "repo": "agenix",
-        "rev": "f6291c5935fdc4e0bef208cfc0dcab7e3f7a1c41",
+        "rev": "e600439ec4c273cf11e06fe4d9d906fb98fa097c",
         "type": "github"
       },
       "original": {
@@ -52,11 +52,11 @@
         "utils": "utils"
       },
       "locked": {
-        "lastModified": 1718194053,
-        "narHash": "sha256-FaGrf7qwZ99ehPJCAwgvNY5sLCqQ3GDiE/6uLhxxwSY=",
+        "lastModified": 1727447169,
+        "narHash": "sha256-3KyjMPUKHkiWhwR91J1YchF6zb6gvckCAY1jOE+ne0U=",
         "owner": "serokell",
         "repo": "deploy-rs",
-        "rev": "3867348fa92bc892eba5d9ddb2d7a97b9e127a8a",
+        "rev": "aa07eb05537d4cd025e2310397a6adcedfe72c76",
         "type": "github"
       },
       "original": {
@@ -87,6 +87,26 @@
         "type": "github"
       }
     },
+    "disko": {
+      "inputs": {
+        "nixpkgs": [
+          "nixpkgs"
+        ]
+      },
+      "locked": {
+        "lastModified": 1745502102,
+        "narHash": "sha256-LqhRwzvIVPEjH0TaPgwzqpyhW6DtCrvz7FnUJDoUZh8=",
+        "owner": "nix-community",
+        "repo": "disko",
+        "rev": "ca27b88c88948d96feeee9ed814cbd34f53d0d70",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-community",
+        "repo": "disko",
+        "type": "github"
+      }
+    },
     "element-stickers": {
       "inputs": {
         "maunium-stickerpicker": [
@@ -97,11 +117,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1714430716,
-        "narHash": "sha256-+WmgGbONe5u2wpvkUC5wNhYnrE6eziIv3Liq4pv7jro=",
+        "lastModified": 1745345582,
+        "narHash": "sha256-T4JHS/iZPMFFbSQJAYKZxRdvUk0y/r0GuaN/b0QD7s8=",
         "ref": "main",
-        "rev": "85716a9e30fb6fae47336ed0cae3d1425c80f17c",
-        "revCount": 6,
+        "rev": "5c65f0fef48ce8193767a5d0453e7cf6ad046de4",
+        "revCount": 9,
         "type": "git",
         "url": "https://git.pub.solar/pub-solar/maunium-stickerpicker-nix"
       },
@@ -165,11 +185,11 @@
         "nixpkgs-lib": "nixpkgs-lib"
       },
       "locked": {
-        "lastModified": 1722555600,
-        "narHash": "sha256-XOQkdLafnb/p9ij77byFQjDf5m5QYl9b2REiVClC+x4=",
+        "lastModified": 1743550720,
+        "narHash": "sha256-hIshGgKZCgWh6AYJpJmRgFdR3WUbkY04o82X05xqQiY=",
         "owner": "hercules-ci",
         "repo": "flake-parts",
-        "rev": "8471fe90ad337a8074e957b69ca4d0089218391d",
+        "rev": "c621e8422220273271f52058f618c94e405bb0f5",
         "type": "github"
       },
       "original": {
@@ -214,18 +234,19 @@
         "type": "github"
       }
     },
-    "flake-utils_3": {
+    "fork": {
       "locked": {
-        "lastModified": 1653893745,
-        "narHash": "sha256-0jntwV3Z8//YwuOjzhV2sgJJPt+HY6KhU7VZUL0fKZQ=",
-        "owner": "numtide",
-        "repo": "flake-utils",
-        "rev": "1ed9fb1935d260de5fe1c2f7ee0ebaae17ed2fa1",
+        "lastModified": 1741180627,
+        "narHash": "sha256-NM2X42gtXuYT7EG+zYcvNTZ/6+CTc3fSiIwdcQcp0dk=",
+        "owner": "teutat3s",
+        "repo": "nixpkgs",
+        "rev": "8a43eb74ac149c080d57d8c80d647fef74df84d8",
         "type": "github"
       },
       "original": {
-        "owner": "numtide",
-        "repo": "flake-utils",
+        "owner": "teutat3s",
+        "ref": "init-matrix-authentication-service-module-0.13.0",
+        "repo": "nixpkgs",
         "type": "github"
       }
     },
@@ -236,16 +257,16 @@
         ]
       },
       "locked": {
-        "lastModified": 1720042825,
-        "narHash": "sha256-A0vrUB6x82/jvf17qPCpxaM+ulJnD8YZwH9Ci0BsAzE=",
+        "lastModified": 1745557122,
+        "narHash": "sha256-eqSo9ugzsqhFgaDFYUZj943nurlX4L6f+AW0skJ4W+M=",
         "owner": "nix-community",
         "repo": "home-manager",
-        "rev": "e1391fb22e18a36f57e6999c7a9f966dc80ac073",
+        "rev": "dd26f75fb4ec1c731d4b1396eaf4439ce40a91c1",
         "type": "github"
       },
       "original": {
         "owner": "nix-community",
-        "ref": "release-24.05",
+        "ref": "release-24.11",
         "repo": "home-manager",
         "type": "github"
       }
@@ -259,11 +280,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1707424749,
-        "narHash": "sha256-eTvts5E3zmD4/DoAI9KedQjRwica0cg36wwIVp1NWbM=",
+        "lastModified": 1738012343,
+        "narHash": "sha256-agMgWwVxXII+RtCqok8ROjzpKJung/5N5f2BVDmMC5Q=",
         "ref": "main",
-        "rev": "1202a23c205b3c07a5feb5caf6813f21b3c69307",
-        "revCount": 30,
+        "rev": "4ffd7bc8ea032991756c5e8e8a37b039789045bc",
+        "revCount": 38,
         "type": "git",
         "url": "https://git.pub.solar/pub-solar/keycloak-theme"
       },
@@ -277,11 +298,11 @@
       "flake": false,
       "locked": {
         "dir": "web",
-        "lastModified": 1718796561,
-        "narHash": "sha256-RKAAHve17lrJokgAPkM2k/E+f9djencwwg3Xcd70Yfw=",
+        "lastModified": 1742926008,
+        "narHash": "sha256-PQ6Qv7VSumLa05Mrnylh1i8maWAHptd4vKwdElE4Tns=",
         "owner": "maunium",
         "repo": "stickerpicker",
-        "rev": "333567f481e60443360aa7199d481e1a45b3a523",
+        "rev": "4b96d236212b1212976f4c3c60479e7aaed866cb",
         "type": "github"
       },
       "original": {
@@ -299,11 +320,11 @@
         ]
       },
       "locked": {
-        "lastModified": 1724299755,
-        "narHash": "sha256-P5zMA17kD9tqiqMuNXwupkM7buM3gMNtoZ1VuJTRDE4=",
+        "lastModified": 1744478979,
+        "narHash": "sha256-dyN+teG9G82G+m+PX/aSAagkC+vUv0SgUw3XkPhQodQ=",
         "owner": "lnl7",
         "repo": "nix-darwin",
-        "rev": "a8968d88e5a537b0491f68ce910749cd870bdbef",
+        "rev": "43975d782b418ebf4969e9ccba82466728c2851b",
         "type": "github"
       },
       "original": {
@@ -313,81 +334,69 @@
         "type": "github"
       }
     },
-    "nixos-flake": {
-      "locked": {
-        "lastModified": 1721140942,
-        "narHash": "sha256-iEqZGdnkG+Hm0jZhS59NJwEyB6z9caVnudWPGHZ/FAE=",
-        "owner": "srid",
-        "repo": "nixos-flake",
-        "rev": "5734c1d9a5fe0bc8e8beaf389ad6227392ca0108",
-        "type": "github"
-      },
-      "original": {
-        "owner": "srid",
-        "repo": "nixos-flake",
-        "type": "github"
-      }
-    },
     "nixpkgs": {
       "locked": {
-        "lastModified": 1724242322,
-        "narHash": "sha256-HMpK7hNjhEk4z5SFg5UtxEio9OWFocHdaQzCfW1pE7w=",
+        "lastModified": 1745487689,
+        "narHash": "sha256-FQoi3R0NjQeBAsEOo49b5tbDPcJSMWc3QhhaIi9eddw=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "224042e9a3039291f22f4f2ded12af95a616cca0",
+        "rev": "5630cf13cceac06cefe9fc607e8dfa8fb342dde3",
         "type": "github"
       },
       "original": {
         "owner": "nixos",
-        "ref": "nixos-24.05",
+        "ref": "nixos-24.11",
         "repo": "nixpkgs",
         "type": "github"
       }
     },
-    "nixpkgs-2205": {
+    "nixpkgs-draupnir": {
       "locked": {
-        "lastModified": 1685573264,
-        "narHash": "sha256-Zffu01pONhs/pqH07cjlF10NnMDLok8ix5Uk4rhOnZQ=",
-        "owner": "nixos",
+        "lastModified": 1746282801,
+        "narHash": "sha256-lrPWzSULWzi6YyRjRA3nwQxRUO3z+dbKfKCzMBs4ac8=",
+        "owner": "teutat3s",
         "repo": "nixpkgs",
-        "rev": "380be19fbd2d9079f677978361792cb25e8a3635",
+        "rev": "f5dd6147d90a94dac04f82fb4814c7c72fcfd177",
         "type": "github"
       },
       "original": {
-        "owner": "nixos",
-        "ref": "nixos-22.05",
+        "owner": "teutat3s",
+        "ref": "draupnir-2025",
         "repo": "nixpkgs",
         "type": "github"
       }
     },
     "nixpkgs-lib": {
       "locked": {
-        "lastModified": 1722555339,
-        "narHash": "sha256-uFf2QeW7eAHlYXuDktm9c25OxOyCoUOQmh5SZ9amE5Q=",
-        "type": "tarball",
-        "url": "https://github.com/NixOS/nixpkgs/archive/a5d394176e64ab29c852d03346c1fc9b0b7d33eb.tar.gz"
+        "lastModified": 1743296961,
+        "narHash": "sha256-b1EdN3cULCqtorQ4QeWgLMrd5ZGOjLSLemfa00heasc=",
+        "owner": "nix-community",
+        "repo": "nixpkgs.lib",
+        "rev": "e4822aea2a6d1cdd36653c134cacfd64c97ff4fa",
+        "type": "github"
       },
       "original": {
-        "type": "tarball",
-        "url": "https://github.com/NixOS/nixpkgs/archive/a5d394176e64ab29c852d03346c1fc9b0b7d33eb.tar.gz"
+        "owner": "nix-community",
+        "repo": "nixpkgs.lib",
+        "type": "github"
       }
     },
     "root": {
       "inputs": {
         "agenix": "agenix",
         "deploy-rs": "deploy-rs",
+        "disko": "disko",
         "element-stickers": "element-stickers",
         "element-themes": "element-themes",
         "flake-parts": "flake-parts",
+        "fork": "fork",
         "home-manager": "home-manager",
         "keycloak-theme-pub-solar": "keycloak-theme-pub-solar",
         "maunium-stickerpicker": "maunium-stickerpicker",
         "nix-darwin": "nix-darwin",
-        "nixos-flake": "nixos-flake",
         "nixpkgs": "nixpkgs",
-        "nixpkgs-2205": "nixpkgs-2205",
+        "nixpkgs-draupnir": "nixpkgs-draupnir",
         "simple-nixos-mailserver": "simple-nixos-mailserver",
-        "triton-vmtools": "triton-vmtools",
         "unstable": "unstable"
       }
     },
@@ -398,22 +407,21 @@
         "nixpkgs": [
           "unstable"
         ],
-        "nixpkgs-24_05": [
+        "nixpkgs-24_11": [
           "nixpkgs"
-        ],
-        "utils": "utils_2"
+        ]
       },
       "locked": {
-        "lastModified": 1718084203,
-        "narHash": "sha256-Cx1xoVfSMv1XDLgKg08CUd1EoTYWB45VmB9XIQzhmzI=",
+        "lastModified": 1734884447,
+        "narHash": "sha256-HA9fAmGNGf0cOYrhgoa+B6BxNVqGAYXfLyx8zIS0ZBY=",
         "owner": "simple-nixos-mailserver",
         "repo": "nixos-mailserver",
-        "rev": "29916981e7b3b5782dc5085ad18490113f8ff63b",
+        "rev": "63209b1def2c9fc891ad271f474a3464a5833294",
         "type": "gitlab"
       },
       "original": {
         "owner": "simple-nixos-mailserver",
-        "ref": "nixos-24.05",
+        "ref": "nixos-24.11",
         "repo": "nixos-mailserver",
         "type": "gitlab"
       }
@@ -478,52 +486,13 @@
         "type": "github"
       }
     },
-    "systems_5": {
-      "locked": {
-        "lastModified": 1681028828,
-        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
-        "owner": "nix-systems",
-        "repo": "default",
-        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
-        "type": "github"
-      },
-      "original": {
-        "owner": "nix-systems",
-        "repo": "default",
-        "type": "github"
-      }
-    },
-    "triton-vmtools": {
-      "inputs": {
-        "flake-utils": "flake-utils_3",
-        "nixpkgs": [
-          "nixpkgs"
-        ]
-      },
-      "locked": {
-        "dir": "vmtools",
-        "lastModified": 1698443513,
-        "narHash": "sha256-wX2JIJ3JmJn6MAurdyjwZU+FZjLCwBArMrVSeeCb/ZU=",
-        "ref": "main",
-        "rev": "0d039dcf06afb8cbddd7ac54bae4d0d185f3e88e",
-        "revCount": 85,
-        "type": "git",
-        "url": "https://git.pub.solar/pub-solar/infra-vintage?dir=vmtools"
-      },
-      "original": {
-        "dir": "vmtools",
-        "ref": "main",
-        "type": "git",
-        "url": "https://git.pub.solar/pub-solar/infra-vintage?dir=vmtools"
-      }
-    },
     "unstable": {
       "locked": {
-        "lastModified": 1724224976,
-        "narHash": "sha256-Z/ELQhrSd7bMzTO8r7NZgi9g5emh+aRKoCdaAv5fiO0=",
+        "lastModified": 1745391562,
+        "narHash": "sha256-sPwcCYuiEopaafePqlG826tBhctuJsLx/mhKKM5Fmjo=",
         "owner": "nixos",
         "repo": "nixpkgs",
-        "rev": "c374d94f1536013ca8e92341b540eba4c22f9c62",
+        "rev": "8a2f738d9d1f1d986b5a4cd2fd2061a7127237d7",
         "type": "github"
       },
       "original": {
@@ -550,24 +519,6 @@
         "repo": "flake-utils",
         "type": "github"
       }
-    },
-    "utils_2": {
-      "inputs": {
-        "systems": "systems_5"
-      },
-      "locked": {
-        "lastModified": 1709126324,
-        "narHash": "sha256-q6EQdSeUZOG26WelxqkmR7kArjgWCdw5sfJVHPH/7j8=",
-        "owner": "numtide",
-        "repo": "flake-utils",
-        "rev": "d465f4819400de7c8d874d50b982301f28a84605",
-        "type": "github"
-      },
-      "original": {
-        "owner": "numtide",
-        "repo": "flake-utils",
-        "type": "github"
-      }
     }
   },
   "root": "root",

flake.nix

@@ -1,23 +1,25 @@
 {
   inputs = {
     # Track channels with commits tested and built by hydra
-    nixpkgs.url = "github:nixos/nixpkgs/nixos-24.05";
+    nixpkgs.url = "github:nixos/nixpkgs/nixos-24.11";
     unstable.url = "github:nixos/nixpkgs/nixos-unstable";
-
-    nixpkgs-2205.url = "github:nixos/nixpkgs/nixos-22.05";
+    fork.url = "github:teutat3s/nixpkgs/init-matrix-authentication-service-module-0.13.0";
+    nixpkgs-draupnir.url = "github:teutat3s/nixpkgs/draupnir-2025";
 
     nix-darwin.url = "github:lnl7/nix-darwin/master";
     nix-darwin.inputs.nixpkgs.follows = "nixpkgs";
 
-    home-manager.url = "github:nix-community/home-manager/release-24.05";
+    home-manager.url = "github:nix-community/home-manager/release-24.11";
     home-manager.inputs.nixpkgs.follows = "nixpkgs";
 
     flake-parts.url = "github:hercules-ci/flake-parts";
-    nixos-flake.url = "github:srid/nixos-flake";
 
     deploy-rs.url = "github:serokell/deploy-rs";
     deploy-rs.inputs.nixpkgs.follows = "nixpkgs";
 
+    disko.url = "github:nix-community/disko";
+    disko.inputs.nixpkgs.follows = "nixpkgs";
+
     agenix.url = "github:ryantm/agenix";
     agenix.inputs.nixpkgs.follows = "nixpkgs";
     agenix.inputs.darwin.follows = "nix-darwin";
@@ -26,9 +28,6 @@
     keycloak-theme-pub-solar.url = "git+https://git.pub.solar/pub-solar/keycloak-theme?ref=main";
     keycloak-theme-pub-solar.inputs.nixpkgs.follows = "nixpkgs";
 
-    triton-vmtools.url = "git+https://git.pub.solar/pub-solar/infra-vintage?ref=main&dir=vmtools";
-    triton-vmtools.inputs.nixpkgs.follows = "nixpkgs";
-
     element-themes.url = "github:aaronraimist/element-themes/master";
     element-themes.flake = false;
 
@@ -39,8 +38,8 @@
     element-stickers.inputs.maunium-stickerpicker.follows = "maunium-stickerpicker";
     element-stickers.inputs.nixpkgs.follows = "nixpkgs";
 
-    simple-nixos-mailserver.url = "gitlab:simple-nixos-mailserver/nixos-mailserver/nixos-24.05";
-    simple-nixos-mailserver.inputs.nixpkgs-24_05.follows = "nixpkgs";
+    simple-nixos-mailserver.url = "gitlab:simple-nixos-mailserver/nixos-mailserver/nixos-24.11";
+    simple-nixos-mailserver.inputs.nixpkgs-24_11.follows = "nixpkgs";
     simple-nixos-mailserver.inputs.nixpkgs.follows = "unstable";
   };
 
@@ -53,7 +52,6 @@
       ];
 
       imports = [
-        inputs.nixos-flake.flakeModule
         ./logins
         ./lib
         ./overlays
@@ -65,6 +63,7 @@
           system,
           pkgs,
           config,
+          lib,
           ...
         }:
         {
@@ -75,12 +74,51 @@
               overlays = [ inputs.agenix.overlays.default ];
             };
             unstable = import inputs.unstable { inherit system; };
-            master = import inputs.master { inherit system; };
           };
+
+          checks =
+            let
+              machinesPerSystem = {
+                aarch64-linux = [
+                  "metronom"
+                ];
+                x86_64-linux = [
+                  "blue-shell"
+                  "delite"
+                  "nachtigall"
+                  "tankstelle"
+                  "trinkgenossin"
+                  "underground"
+                ];
+              };
+              nixosMachines = inputs.nixpkgs.lib.mapAttrs' (n: inputs.nixpkgs.lib.nameValuePair "nixos-${n}") (
+                inputs.nixpkgs.lib.genAttrs (machinesPerSystem.${system} or [ ]) (
+                  name: self.nixosConfigurations.${name}.config.system.build.toplevel
+                )
+              );
+              nixos-lib = import (inputs.nixpkgs + "/nixos/lib") { };
+              testDir = builtins.attrNames (builtins.readDir ./tests);
+              testFiles = builtins.filter (n: builtins.match "^.*.nix$" n != null) testDir;
+            in
+            builtins.listToAttrs (
+              map (x: {
+                name = "test-${lib.strings.removeSuffix ".nix" x}";
+                value = nixos-lib.runTest (
+                  import (./tests + "/${x}") {
+                    inherit self;
+                    inherit pkgs;
+                    inherit lib;
+                    inherit config;
+                  }
+                );
+              }) testFiles
+            )
+            // nixosMachines;
+
           devShells.default = pkgs.mkShell {
             buildInputs = with pkgs; [
               deploy-rs
-              nixpkgs-fmt
+              nix-fast-build
               agenix
               age-plugin-yubikey
               cachix
@@ -89,53 +127,55 @@
               nvfetcher
               shellcheck
               shfmt
-              treefmt
+              treefmt2
               nixos-generators
-              inputs.nixpkgs-2205.legacyPackages.${system}.terraform
+              opentofu
+              terraform-backend-git
+              terraform-ls
               jq
             ];
           };
+
           devShells.ci = pkgs.mkShell { buildInputs = with pkgs; [ nodejs ]; };
         };
 
-      flake =
-        let
-          username = "barkeeper";
-        in
-        {
-          inherit username;
+      flake = {
+        nixosModules = builtins.listToAttrs (
+          map (x: {
+            name = x;
+            value = import (./modules + "/${x}");
+          }) (builtins.attrNames (builtins.readDir ./modules))
+        );
 
-          nixosModules = builtins.listToAttrs (
-            map (x: {
-              name = x;
-              value = import (./modules + "/${x}");
-            }) (builtins.attrNames (builtins.readDir ./modules))
-          );
+        checks = builtins.mapAttrs (
+          system: deployLib: deployLib.deployChecks self.deploy
+        ) inputs.deploy-rs.lib;
 
-          checks = builtins.mapAttrs (
-            system: deployLib: deployLib.deployChecks self.deploy
-          ) inputs.deploy-rs.lib;
+        formatter."x86_64-linux" = inputs.nixpkgs.legacyPackages."x86_64-linux".nixfmt-rfc-style;
 
-          formatter."x86_64-linux" = inputs.unstable.legacyPackages."x86_64-linux".nixfmt-rfc-style;
-
-          deploy.nodes = self.lib.deploy.mkDeployNodes self.nixosConfigurations {
-            nachtigall = {
-              hostname = "nachtigall.wg.pub.solar";
-              sshUser = username;
-            };
-            flora-6 = {
-              hostname = "flora-6.wg.pub.solar";
-              sshUser = username;
-            };
-            metronom = {
-              hostname = "metronom.wg.pub.solar";
-              sshUser = username;
-            };
-            tankstelle = {
-              hostname = "tankstelle.wg.pub.solar";
-              sshUser = username;
-            };
+        deploy.nodes = self.lib.deploy.mkDeployNodes self.nixosConfigurations {
+          nachtigall = {
+            hostname = "nachtigall.wg.pub.solar";
+          };
+          metronom = {
+            hostname = "metronom.wg.pub.solar";
+          };
+          tankstelle = {
+            hostname = "tankstelle.wg.pub.solar";
+          };
+          underground = {
+            hostname = "80.244.242.3";
+          };
+          trinkgenossin = {
+            hostname = "trinkgenossin.wg.pub.solar";
+          };
+          delite = {
+            hostname = "delite.wg.pub.solar";
+          };
+          blue-shell = {
+            hostname = "blue-shell.wg.pub.solar";
           };
         };
+      };
     };
 }

hosts/blue-shell/configuration.nix

@@ -0,0 +1,33 @@
+{
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+{
+  boot.loader.grub.enable = true;
+
+  boot.kernelParams = [
+    "boot.shell_on_fail=1"
+    "ip=dhcp"
+  ];
+
+  # This option defines the first version of NixOS you have installed on this particular machine,
+  # and is used to maintain compatibility with application data (e.g. databases) created on older NixOS versions.
+  #
+  # Most users should NEVER change this value after the initial install, for any reason,
+  # even if you've upgraded your system to a new NixOS release.
+  #
+  # This value does NOT affect the Nixpkgs version your packages and OS are pulled from,
+  # so changing it will NOT upgrade your system - see https://nixos.org/manual/nixos/stable/#sec-upgrading for how
+  # to actually do that.
+  #
+  # This value being lower than the current NixOS release does NOT mean your system is
+  # out of date, out of support, or vulnerable.
+  #
+  # Do NOT change this value unless you have manually inspected all the changes it would make to your configuration,
+  # and migrated your data accordingly.
+  #
+  # For more information, see `man configuration.nix` or https://nixos.org/manual/nixos/stable/options#opt-system.stateVersion .
+  system.stateVersion = "24.05"; # Did you read the comment?
+}

hosts/blue-shell/default.nix

@@ -1,11 +1,13 @@
-{ ... }:
+{ flake, ... }:
 
 {
   imports = [
-    # Include the results of the hardware scan.
     ./hardware-configuration.nix
     ./configuration.nix
-    ./triton-vmtools.nix
+    ./disk-config.nix
+
+    ./networking.nix
     ./wireguard.nix
+    #./backups.nix
   ];
 }

hosts/blue-shell/disk-config.nix

@@ -0,0 +1,101 @@
+{
+  disko.devices = {
+    disk = {
+      main = {
+        type = "disk";
+        device = "/dev/vdb";
+        content = {
+          type = "gpt";
+          partitions = {
+            bios = {
+              size = "1M";
+              type = "EF02"; # for grub MBR
+            };
+            boot = {
+              size = "1G";
+              type = "8300";
+              content = {
+                type = "filesystem";
+                format = "ext4";
+                mountpoint = "/boot";
+                mountOptions = [ "defaults" ];
+              };
+            };
+            luks = {
+              size = "100%";
+              content = {
+                type = "luks";
+                name = "cryptroot";
+                extraOpenArgs = [ ];
+                # if you want to use the key for interactive login be sure there is no trailing newline
+                # for example use `echo -n "password" > /tmp/secret.key`
+                passwordFile = "/tmp/luks-password";
+                content = {
+                  type = "lvm_pv";
+                  vg = "vg0";
+                };
+              };
+            };
+          };
+        };
+      };
+      data = {
+        type = "disk";
+        device = "/dev/vdc";
+        content = {
+          type = "gpt";
+          partitions = {
+            luks = {
+              size = "100%";
+              content = {
+                type = "luks";
+                name = "cryptdata";
+                extraOpenArgs = [ ];
+                # if you want to use the key for interactive login be sure there is no trailing newline
+                # for example use `echo -n "password" > /tmp/secret.key`
+                passwordFile = "/tmp/luks-password";
+                content = {
+                  type = "filesystem";
+                  format = "xfs";
+                  mountpoint = "/var/lib/garage/data";
+                  mountOptions = [ "defaults" ];
+                };
+              };
+            };
+          };
+        };
+      };
+    };
+    lvm_vg = {
+      vg0 = {
+        type = "lvm_vg";
+        lvs = {
+          root = {
+            size = "100G";
+            content = {
+              type = "filesystem";
+              format = "ext4";
+              mountpoint = "/";
+              mountOptions = [ "defaults" ];
+            };
+          };
+          swap = {
+            size = "16G";
+            content = {
+              type = "swap";
+            };
+          };
+          metadata = {
+            size = "50G";
+            content = {
+              type = "filesystem";
+              format = "btrfs";
+              mountpoint = "/var/lib/garage/meta";
+              mountOptions = [ "defaults" ];
+            };
+          };
+        };
+      };
+    };
+  };
+}

hosts/blue-shell/hardware-configuration.nix

@@ -0,0 +1,27 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{
+  config,
+  lib,
+  pkgs,
+  modulesPath,
+  ...
+}:
+
+{
+  imports = [ (modulesPath + "/profiles/qemu-guest.nix") ];
+
+  boot.initrd.availableKernelModules = [
+    "ata_piix"
+    "uhci_hcd"
+    "virtio_pci"
+    "sr_mod"
+    "virtio_blk"
+  ];
+  boot.initrd.kernelModules = [ ];
+  boot.kernelModules = [ ];
+  boot.extraModulePackages = [ ];
+
+  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
+}

hosts/blue-shell/networking.nix

@@ -0,0 +1,26 @@
+{
+  config,
+  pkgs,
+  flake,
+  ...
+}:
+{
+  services.garage.settings.rpc_public_addr = "[2a03:4000:43:24e::1]:3901";
+
+  networking.hostName = "blue-shell";
+  networking.hostId = "00000005";
+
+  networking.useDHCP = false;
+  systemd.network.enable = true;
+  systemd.network.networks."10-wan" = {
+    matchConfig.Name = "ens3";
+    address = [
+      "194.13.83.205/22"
+      "2a03:4000:43:24e::1/64"
+    ];
+    gateway = [
+      "194.13.80.1"
+      "fe80::1"
+    ];
+  };
+}

hosts/blue-shell/wireguard.nix

@@ -0,0 +1,51 @@
+{
+  config,
+  pkgs,
+  flake,
+  ...
+}:
+let
+  wireguardIPv4 = "10.7.6.7";
+  wireguardIPv6 = "fd00:fae:fae:fae:fae:7::";
+in
+{
+  networking.firewall.allowedUDPPorts = [ 51820 ];
+
+  age.secrets.wg-private-key.file = "${flake.self}/secrets/blue-shell-wg-private-key.age";
+
+  networking.wireguard.interfaces = {
+    wg-ssh = {
+      listenPort = 51820;
+      mtu = 1300;
+      ips = [
+        "${wireguardIPv4}/32"
+        "${wireguardIPv6}/96"
+      ];
+      privateKeyFile = config.age.secrets.wg-private-key.path;
+      peers = flake.self.logins.wireguardDevices ++ [
+        {
+          # trinkgenossin.pub.solar
+          publicKey = "QWgHovHxtqiQhnHLouSWiT6GIoQDmuvnThYL5c/rvU4=";
+          allowedIPs = [
+            "10.7.6.5/32"
+            "fd00:fae:fae:fae:fae:5::/96"
+          ];
+          #endpoint = "85.215.152.22:51820";
+          endpoint = "[2a01:239:35d:f500::1]:51820";
+          persistentKeepalive = 15;
+        }
+      ];
+    };
+  };
+
+  services.openssh.listenAddresses = [
+    {
+      addr = wireguardIPv4;
+      port = 22;
+    }
+    {
+      addr = "[${wireguardIPv6}]";
+      port = 22;
+    }
+  ];
+}

hosts/default.nix

@@ -1,9 +1,35 @@
-{ self, ... }:
+{
+  self,
+  inputs,
+  config,
+  ...
+}:
 {
   flake = {
-    nixosConfigurations = {
-      nachtigall = self.nixos-flake.lib.mkLinuxSystem {
+    nixosModules = {
+      home-manager = {
         imports = [
+          inputs.home-manager.nixosModules.home-manager
+          {
+            home-manager.useGlobalPkgs = true;
+            home-manager.useUserPackages = true;
+            home-manager.extraSpecialArgs = {
+              flake = {
+                inherit self inputs config;
+              };
+            };
+          }
+        ];
+      };
+    };
+    nixosConfigurations = {
+      nachtigall = self.inputs.nixpkgs.lib.nixosSystem {
+        specialArgs = {
+          flake = {
+            inherit self inputs config;
+          };
+        };
+        modules = [
           self.inputs.agenix.nixosModules.default
           self.nixosModules.home-manager
           ./nachtigall
@@ -11,6 +37,7 @@
           self.nixosModules.unlock-zfs-on-boot
           self.nixosModules.core
           self.nixosModules.docker
+          self.nixosModules.backups
 
           self.nixosModules.nginx
           self.nixosModules.collabora
@@ -22,8 +49,8 @@
           self.nixosModules.nginx-mastodon
           self.nixosModules.nginx-mastodon-files
           self.nixosModules.mediawiki
+          self.nixosModules.mollysocket
           self.nixosModules.nextcloud
-          self.nixosModules.nginx-prometheus-exporters
           self.nixosModules.nginx-website
           self.nixosModules.nginx-website-miom
           self.nixosModules.opensearch
@@ -36,38 +63,27 @@
           self.nixosModules.tt-rss
           self.nixosModules.obs-portal
           self.nixosModules.matrix
+          self.nixosModules.matrix-draupnir
           self.nixosModules.matrix-irc
           self.nixosModules.matrix-telegram
           self.nixosModules.nginx-matrix
         ];
       };
 
-      flora-6 = self.nixos-flake.lib.mkLinuxSystem {
-        imports = [
-          self.inputs.agenix.nixosModules.default
-          self.nixosModules.home-manager
-          ./flora-6
-          self.nixosModules.overlays
-          self.nixosModules.core
-
-          self.nixosModules.keycloak
-          self.nixosModules.caddy
-          self.nixosModules.drone
-          self.nixosModules.forgejo-actions-runner
-          self.nixosModules.grafana
-          self.nixosModules.prometheus
-          self.nixosModules.loki
-        ];
-      };
-
-      metronom = self.nixos-flake.lib.mkLinuxSystem {
-        imports = [
+      metronom = self.inputs.nixpkgs.lib.nixosSystem {
+        specialArgs = {
+          flake = {
+            inherit self inputs config;
+          };
+        };
+        modules = [
           self.inputs.agenix.nixosModules.default
           self.nixosModules.home-manager
           ./metronom
           self.nixosModules.overlays
           self.nixosModules.unlock-zfs-on-boot
           self.nixosModules.core
+          self.nixosModules.backups
           self.nixosModules.mail
           self.nixosModules.prometheus-exporters
           self.nixosModules.promtail
@@ -76,15 +92,120 @@
         ];
       };
 
-      tankstelle = self.nixos-flake.lib.mkLinuxSystem {
-        imports = [
+      tankstelle = self.inputs.nixpkgs.lib.nixosSystem {
+        specialArgs = {
+          flake = {
+            inherit self inputs config;
+          };
+        };
+        modules = [
           self.inputs.agenix.nixosModules.default
           self.nixosModules.home-manager
           ./tankstelle
           self.nixosModules.overlays
           self.nixosModules.core
+          self.nixosModules.backups
           self.nixosModules.prometheus-exporters
           self.nixosModules.promtail
+          self.nixosModules.forgejo-actions-runner
+        ];
+      };
+
+      trinkgenossin = self.inputs.nixpkgs.lib.nixosSystem {
+        specialArgs = {
+          flake = {
+            inherit self inputs config;
+          };
+        };
+        modules = [
+          self.inputs.agenix.nixosModules.default
+          self.nixosModules.home-manager
+          ./trinkgenossin
+          self.nixosModules.backups
+          self.nixosModules.overlays
+          self.nixosModules.unlock-luks-on-boot
+          self.nixosModules.core
+
+          self.nixosModules.garage
+          self.nixosModules.nginx
+
+          # This module is already using options, and those options are used by the grafana module
+          self.nixosModules.keycloak
+          self.nixosModules.grafana
+          self.nixosModules.prometheus
+          self.nixosModules.prometheus-exporters
+          self.nixosModules.loki
+          self.nixosModules.promtail
+          self.nixosModules.forgejo-actions-runner
+        ];
+      };
+
+      delite = self.inputs.nixpkgs.lib.nixosSystem {
+        specialArgs = {
+          flake = {
+            inherit self inputs config;
+          };
+        };
+        modules = [
+          self.inputs.agenix.nixosModules.default
+          self.inputs.disko.nixosModules.disko
+          self.nixosModules.home-manager
+          ./delite
+          self.nixosModules.overlays
+          self.nixosModules.unlock-luks-on-boot
+          self.nixosModules.core
+          self.nixosModules.prometheus-exporters
+          self.nixosModules.promtail
+
+          self.nixosModules.garage
+          self.nixosModules.nginx
+        ];
+      };
+
+      blue-shell = self.inputs.nixpkgs.lib.nixosSystem {
+        specialArgs = {
+          flake = {
+            inherit self inputs config;
+          };
+        };
+        modules = [
+          self.inputs.agenix.nixosModules.default
+          self.inputs.disko.nixosModules.disko
+          self.nixosModules.home-manager
+          ./blue-shell
+          self.nixosModules.overlays
+          self.nixosModules.unlock-luks-on-boot
+          self.nixosModules.core
+          self.nixosModules.prometheus-exporters
+          self.nixosModules.promtail
+
+          self.nixosModules.garage
+          self.nixosModules.nginx
+        ];
+      };
+
+      underground = self.inputs.nixpkgs.lib.nixosSystem {
+        specialArgs = {
+          flake = {
+            inherit self inputs config;
+          };
+        };
+        modules = [
+          self.inputs.agenix.nixosModules.default
+          self.nixosModules.home-manager
+          ./underground
+          self.nixosModules.overlays
+          self.nixosModules.unlock-luks-on-boot
+          self.nixosModules.core
+
+          self.nixosModules.backups
+          self.nixosModules.keycloak
+          self.nixosModules.postgresql
+          self.nixosModules.matrix
+          self.nixosModules.matrix-draupnir
+          self.nixosModules.matrix-irc
+          self.nixosModules.nginx
+          self.nixosModules.nginx-matrix
         ];
       };
     };

hosts/delite/configuration.nix

@@ -0,0 +1,33 @@
+{
+  flake,
+  config,
+  pkgs,
+  ...
+}:
+{
+  boot.loader.grub.enable = true;
+
+  boot.kernelParams = [
+    "boot.shell_on_fail=1"
+    "ip=5.255.119.132::5.255.119.1:255.255.255.0:delite::off"
+  ];
+
+  # This option defines the first version of NixOS you have installed on this particular machine,
+  # and is used to maintain compatibility with application data (e.g. databases) created on older NixOS versions.
+  #
+  # Most users should NEVER change this value after the initial install, for any reason,
+  # even if you've upgraded your system to a new NixOS release.
+  #
+  # This value does NOT affect the Nixpkgs version your packages and OS are pulled from,
+  # so changing it will NOT upgrade your system - see https://nixos.org/manual/nixos/stable/#sec-upgrading for how
+  # to actually do that.
+  #
+  # This value being lower than the current NixOS release does NOT mean your system is
+  # out of date, out of support, or vulnerable.
+  #
+  # Do NOT change this value unless you have manually inspected all the changes it would make to your configuration,
+  # and migrated your data accordingly.
+  #
+  # For more information, see `man configuration.nix` or https://nixos.org/manual/nixos/stable/options#opt-system.stateVersion .
+  system.stateVersion = "24.05"; # Did you read the comment?
+}

hosts/delite/default.nix

@@ -0,0 +1,13 @@
+{ flake, ... }:
+
+{
+  imports = [
+    ./hardware-configuration.nix
+    ./configuration.nix
+    ./disk-config.nix
+
+    ./networking.nix
+    ./wireguard.nix
+    #./backups.nix
+  ];
+}

hosts/delite/disk-config.nix

@@ -0,0 +1,84 @@
+{
+  disko.devices = {
+    disk = {
+      main = {
+        type = "disk";
+        device = "/dev/vda";
+        content = {
+          type = "gpt";
+          partitions = {
+            bios = {
+              size = "1M";
+              type = "EF02"; # for grub MBR
+            };
+            boot = {
+              size = "1G";
+              type = "8300";
+              content = {
+                type = "filesystem";
+                format = "ext4";
+                mountpoint = "/boot";
+                mountOptions = [ "defaults" ];
+              };
+            };
+            luks = {
+              size = "100%";
+              content = {
+                type = "luks";
+                name = "cryptroot";
+                extraOpenArgs = [ ];
+                # if you want to use the key for interactive login be sure there is no trailing newline
+                # for example use `echo -n "password" > /tmp/secret.key`
+                passwordFile = "/tmp/luks-password";
+                content = {
+                  type = "lvm_pv";
+                  vg = "vg0";
+                };
+              };
+            };
+          };
+        };
+      };
+    };
+    lvm_vg = {
+      vg0 = {
+        type = "lvm_vg";
+        lvs = {
+          root = {
+            size = "40G";
+            content = {
+              type = "filesystem";
+              format = "ext4";
+              mountpoint = "/";
+              mountOptions = [ "defaults" ];
+            };
+          };
+          swap = {
+            size = "8G";
+            content = {
+              type = "swap";
+            };
+          };
+          data = {
+            size = "800G";
+            content = {
+              type = "filesystem";
+              format = "xfs";
+              mountpoint = "/var/lib/garage/data";
+              mountOptions = [ "defaults" ];
+            };
+          };
+          metadata = {
+            size = "50G";
+            content = {
+              type = "filesystem";
+              format = "btrfs";
+              mountpoint = "/var/lib/garage/meta";
+              mountOptions = [ "defaults" ];
+            };
+          };
+        };
+      };
+    };
+  };
+}

hosts/delite/hardware-configuration.nix

@@ -0,0 +1,26 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{
+  config,
+  lib,
+  pkgs,
+  modulesPath,
+  ...
+}:
+
+{
+  imports = [ (modulesPath + "/profiles/qemu-guest.nix") ];
+
+  boot.initrd.availableKernelModules = [
+    "ata_piix"
+    "uhci_hcd"
+    "virtio_pci"
+    "virtio_blk"
+  ];
+  boot.initrd.kernelModules = [ "dm-snapshot" ];
+  boot.kernelModules = [ ];
+  boot.extraModulePackages = [ ];
+
+  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
+}

hosts/delite/networking.nix

@@ -0,0 +1,26 @@
+{
+  config,
+  pkgs,
+  flake,
+  ...
+}:
+{
+  services.garage.settings.rpc_public_addr = "[2a04:52c0:124:9d8c::2]:3901";
+
+  networking.hostName = "delite";
+  networking.hostId = "00000004";
+
+  networking.useDHCP = false;
+  systemd.network.enable = true;
+  systemd.network.networks."10-wan" = {
+    matchConfig.Name = "ens3";
+    address = [
+      "5.255.119.132/24"
+      "2a04:52c0:124:9d8c::2/48"
+    ];
+    gateway = [
+      "5.255.119.1"
+      "2a04:52c0:124::1"
+    ];
+  };
+}

hosts/delite/wireguard.nix

@@ -0,0 +1,51 @@
+{
+  config,
+  pkgs,
+  flake,
+  ...
+}:
+let
+  wireguardIPv4 = "10.7.6.6";
+  wireguardIPv6 = "fd00:fae:fae:fae:fae:6::";
+in
+{
+  networking.firewall.allowedUDPPorts = [ 51820 ];
+
+  age.secrets.wg-private-key.file = "${flake.self}/secrets/delite-wg-private-key.age";
+
+  networking.wireguard.interfaces = {
+    wg-ssh = {
+      listenPort = 51820;
+      mtu = 1300;
+      ips = [
+        "${wireguardIPv4}/32"
+        "${wireguardIPv6}/96"
+      ];
+      privateKeyFile = config.age.secrets.wg-private-key.path;
+      peers = flake.self.logins.wireguardDevices ++ [
+        {
+          # trinkgenossin.pub.solar
+          publicKey = "QWgHovHxtqiQhnHLouSWiT6GIoQDmuvnThYL5c/rvU4=";
+          allowedIPs = [
+            "10.7.6.5/32"
+            "fd00:fae:fae:fae:fae:5::/96"
+          ];
+          #endpoint = "85.215.152.22:51820";
+          endpoint = "[2a01:239:35d:f500::1]:51820";
+          persistentKeepalive = 15;
+        }
+      ];
+    };
+  };
+
+  services.openssh.listenAddresses = [
+    {
+      addr = wireguardIPv4;
+      port = 22;
+    }
+    {
+      addr = "[${wireguardIPv6}]";
+      port = 22;
+    }
+  ];
+}

hosts/flora-6/configuration.nix

@@ -1,72 +0,0 @@
-{
-  config,
-  lib,
-  pkgs,
-  flake,
-  ...
-}:
-let
-  psCfg = config.pub-solar;
-in
-{
-  config = {
-    # Override nix.conf for more agressive garbage collection
-    nix.extraOptions = lib.mkForce ''
-      experimental-features = flakes nix-command
-      min-free = 536870912
-      keep-outputs = false
-      keep-derivations = false
-      fallback = true
-    '';
-
-    # # #
-    # # # Triton host specific options
-    # # # DO NOT ALTER below this line, changes might render system unbootable
-    # # #
-
-    # Use the systemd-boot EFI boot loader.
-    boot.loader.systemd-boot.enable = true;
-    boot.loader.efi.canTouchEfiVariables = true;
-
-    # Force getting the hostname from cloud-init
-    networking.hostName = lib.mkDefault "";
-
-    # We use cloud-init to configure networking, this option should fix
-    # systemd-networkd-wait-online timeouts
-    #systemd.services."systemd-networkd".environment.SYSTEMD_LOG_LEVEL = "debug";
-    systemd.network.wait-online.ignoredInterfaces = [
-      "docker0"
-      "wg-ssh"
-    ];
-
-    # List services that you want to enable:
-    services.cloud-init.enable = true;
-    services.cloud-init.ext4.enable = true;
-    services.cloud-init.network.enable = true;
-    # use the default NixOS cloud-init config, but add some SmartOS customization to it
-    environment.etc."cloud/cloud.cfg.d/90_smartos.cfg".text = ''
-      datasource_list: [ SmartOS ]
-
-      # Do not create the centos/ubuntu/debian user
-      users: [ ]
-
-      # mount second disk with label ephemeral0, gets formated by cloud-init
-      # this will fail to get added to /etc/fstab as it's read-only, but should
-      # mount at boot anyway
-      mounts:
-      - [ vdb, /data, auto, "defaults,nofail" ]
-    '';
-
-    # We manage the firewall with nix, too
-    # altough triton can also manage firewall rules via the triton fwrule subcommand
-    networking.firewall.enable = true;
-
-    # This value determines the NixOS release from which the default
-    # settings for stateful data, like file locations and database versions
-    # on your system were taken. It‘s perfectly fine and recommended to leave
-    # this value at the release version of the first install of this system.
-    # Before changing this value read the documentation for this option
-    # (e.g. man configuration.nix or on https://nixos.org/nixos/options.html).
-    system.stateVersion = "22.05"; # Did you read the comment?
-  };
-}

hosts/flora-6/triton-vmtools.nix

@@ -1,6 +0,0 @@
-{ pkgs, flake, ... }:
-{
-  environment.systemPackages = with pkgs; [
-    flake.inputs.triton-vmtools.packages.${pkgs.system}.default
-  ];
-}

hosts/metronom/backups.nix

@@ -1,13 +1,29 @@
-{ flake, ... }:
+{ config, flake, ... }:
 {
-  age.secrets."restic-repo-droppie" = {
-    file = "${flake.self}/secrets/restic-repo-droppie.age";
+  age.secrets."restic-repo-storagebox-metronom" = {
+    file = "${flake.self}/secrets/restic-repo-storagebox-metronom.age";
     mode = "400";
     owner = "root";
   };
-  age.secrets."restic-repo-storagebox" = {
-    file = "${flake.self}/secrets/restic-repo-storagebox.age";
+  age.secrets.restic-repo-garage-metronom = {
+    file = "${flake.self}/secrets/restic-repo-garage-metronom.age";
     mode = "400";
     owner = "root";
   };
+  age.secrets.restic-repo-garage-metronom-env = {
+    file = "${flake.self}/secrets/restic-repo-garage-metronom-env.age";
+    mode = "400";
+    owner = "root";
+  };
+
+  pub-solar-os.backups.repos.storagebox = {
+    passwordFile = config.age.secrets."restic-repo-storagebox-metronom".path;
+    repository = "sftp:u377325@u377325.your-storagebox.de:/metronom-backups";
+  };
+
+  pub-solar-os.backups.repos.garage = {
+    passwordFile = config.age.secrets."restic-repo-garage-metronom".path;
+    environmentFile = config.age.secrets."restic-repo-garage-metronom-env".path;
+    repository = "s3:https://buckets.pub.solar/metronom-backups";
+  };
 }

hosts/metronom/configuration.nix

@@ -23,6 +23,14 @@
     pools = [ "root_pool" ];
   };
 
+  # Declarative SSH private key
+  age.secrets."metronom-root-ssh-key" = {
+    file = "${flake.self}/secrets/metronom-root-ssh-key.age";
+    path = "/root/.ssh/id_ed25519";
+    mode = "400";
+    owner = "root";
+  };
+
   # Declarative SSH private key
   #age.secrets."metronom-root-ssh-key" = {
   #  file = "${flake.self}/secrets/metronom-root-ssh-key.age";

hosts/metronom/default.nix

@@ -7,6 +7,6 @@
 
     ./networking.nix
     ./wireguard.nix
-    #./backups.nix
+    ./backups.nix
   ];
 }

hosts/metronom/wireguard.nix

@@ -18,16 +18,7 @@
         "fd00:fae:fae:fae:fae:3::/96"
       ];
       privateKeyFile = config.age.secrets.wg-private-key.path;
-      peers = flake.self.logins.admins.wireguardDevices ++ [
-        {
-          # flora-6.pub.solar
-          endpoint = "80.71.153.210:51820";
-          publicKey = "jtSR5G2P/nm9s8WrVc26Xc/SQLupRxyXE+5eIeqlsTU=";
-          allowedIPs = [
-            "10.7.6.2/32"
-            "fd00:fae:fae:fae:fae:2::/96"
-          ];
-        }
+      peers = flake.self.logins.wireguardDevices ++ [
         {
           # nachtigall.pub.solar
           endpoint = "138.201.80.102:51820";
@@ -37,6 +28,17 @@
             "fd00:fae:fae:fae:fae:1::/96"
           ];
         }
+        {
+          # trinkgenossin.pub.solar
+          publicKey = "QWgHovHxtqiQhnHLouSWiT6GIoQDmuvnThYL5c/rvU4=";
+          allowedIPs = [
+            "10.7.6.5/32"
+            "fd00:fae:fae:fae:fae:5::/96"
+          ];
+          #endpoint = "85.215.152.22:51820";
+          endpoint = "[2a01:239:35d:f500::1]:51820";
+          persistentKeepalive = 15;
+        }
       ];
     };
   };

hosts/nachtigall/backups.nix

@@ -1,13 +1,34 @@
-{ flake, ... }:
+{ config, flake, ... }:
 {
   age.secrets."restic-repo-droppie" = {
     file = "${flake.self}/secrets/restic-repo-droppie.age";
     mode = "400";
     owner = "root";
   };
-  age.secrets."restic-repo-storagebox" = {
-    file = "${flake.self}/secrets/restic-repo-storagebox.age";
+  age.secrets."restic-repo-storagebox-nachtigall" = {
+    file = "${flake.self}/secrets/restic-repo-storagebox-nachtigall.age";
     mode = "400";
     owner = "root";
   };
+  age.secrets.restic-repo-garage-nachtigall = {
+    file = "${flake.self}/secrets/restic-repo-garage-nachtigall.age";
+    mode = "400";
+    owner = "root";
+  };
+  age.secrets.restic-repo-garage-nachtigall-env = {
+    file = "${flake.self}/secrets/restic-repo-garage-nachtigall-env.age";
+    mode = "400";
+    owner = "root";
+  };
+
+  pub-solar-os.backups.repos.storagebox = {
+    passwordFile = config.age.secrets."restic-repo-storagebox-nachtigall".path;
+    repository = "sftp:u377325@u377325.your-storagebox.de:/backups";
+  };
+
+  pub-solar-os.backups.repos.garage = {
+    passwordFile = config.age.secrets."restic-repo-garage-nachtigall".path;
+    environmentFile = config.age.secrets."restic-repo-garage-nachtigall-env".path;
+    repository = "s3:https://buckets.pub.solar/nachtigall-backups";
+  };
 }

hosts/nachtigall/configuration.nix

@@ -20,6 +20,14 @@
         devices = [ "/dev/disk/by-id/nvme-KXG60ZNV1T02_TOSHIBA_Z9NF704ZF9ZL" ];
         path = "/boot2";
       }
+      {
+        devices = [ "/dev/disk/by-id/nvme-SAMSUNG_MZVL21T0HDLU-00B07_S77WNF0XA01902" ];
+        path = "/boot3";
+      }
+      {
+        devices = [ "/dev/disk/by-id/nvme-SAMSUNG_MZVL21T0HCLR-00B00_S676NU0W623944" ];
+        path = "/boot4";
+      }
     ];
     copyKernels = true;
   };
@@ -48,9 +56,102 @@
     owner = "root";
   };
 
-  pub-solar-os.auth.enable = true;
+  # keycloak
+  age.secrets.keycloak-database-password = {
+    file = "${flake.self}/secrets/keycloak-database-password.age";
+    mode = "600";
+    #owner = "keycloak";
+  };
 
-  nixpkgs.config.permittedInsecurePackages = [ "keycloak-23.0.6" ];
+  pub-solar-os.auth = {
+    enable = true;
+    database-password-file = config.age.secrets.keycloak-database-password.path;
+  };
+
+  # matrix-synapse
+  age.secrets."matrix-synapse-signing-key" = {
+    file = "${flake.self}/secrets/matrix-synapse-signing-key.age";
+    mode = "400";
+    owner = "matrix-synapse";
+  };
+
+  age.secrets."matrix-synapse-secret-config.yaml" = {
+    file = "${flake.self}/secrets/matrix-synapse-secret-config.yaml.age";
+    mode = "400";
+    owner = "matrix-synapse";
+  };
+
+  age.secrets."matrix-authentication-service-secret-config.yml" = {
+    file = "${flake.self}/secrets/matrix-authentication-service-secret-config.yml.age";
+    mode = "400";
+    owner = "matrix-authentication-service";
+  };
+
+  # matrix-appservice-irc
+  age.secrets."matrix-appservice-irc-mediaproxy-signing-key" = {
+    file = "${flake.self}/secrets/matrix-appservice-irc-mediaproxy-signing-key.jwk.age";
+    mode = "400";
+    owner = "matrix-appservice-irc";
+  };
+
+  age.secrets."matrix-draupnir-access-token" = {
+    file = "${flake.self}/secrets/matrix-draupnir-access-token.age";
+    mode = "400";
+    owner = "root";
+  };
+
+  pub-solar-os.matrix = {
+    enable = true;
+    appservice-irc.mediaproxy.signingKeyPath =
+      config.age.secrets."matrix-appservice-irc-mediaproxy-signing-key".path;
+    synapse = {
+      signing_key_path = config.age.secrets."matrix-synapse-signing-key".path;
+      extra-config-files = [
+        config.age.secrets."matrix-synapse-secret-config.yaml".path
+
+        # The registration file is automatically generated after starting the
+        # appservice for the first time.
+        # cp /var/lib/mautrix-telegram/telegram-registration.yaml \
+        #   /var/lib/matrix-synapse/
+        # chown matrix-synapse:matrix-synapse \
+        #   /var/lib/matrix-synapse/telegram-registration.yaml
+        "/var/lib/matrix-synapse/telegram-registration.yaml"
+      ];
+      app-service-config-files = [
+        "/var/lib/matrix-synapse/telegram-registration.yaml"
+        "/var/lib/matrix-appservice-irc/registration.yml"
+        # "/matrix-appservice-slack-registration.yaml"
+        # "/hookshot-registration.yml"
+        # "/matrix-mautrix-signal-registration.yaml"
+        # "/matrix-mautrix-telegram-registration.yaml"
+      ];
+    };
+    matrix-authentication-service.extra-config-files = [
+      config.age.secrets."matrix-authentication-service-secret-config.yml".path
+    ];
+    draupnir = {
+      enable = true;
+      homeserver-url = "http://127.0.200.10:8008";
+      access-token-file = config.age.secrets."matrix-draupnir-access-token".path;
+    };
+  };
+
+  systemd.services.postgresql = {
+    after = [ "var-lib-postgresql.mount" ];
+    requisite = [ "var-lib-postgresql.mount" ];
+  };
+
+  age.secrets."keycloak-admin-cli-client-secret" = {
+    file = "${flake.self}/secrets/keycloak-admin-cli-client-secret.age";
+  };
+
+  age.secrets."matrix-admin-access-token" = {
+    file = "${flake.self}/secrets/matrix-admin-access-token.age";
+  };
+
+  environment.systemPackages = [
+    pkgs.delete-pubsolar-id
+  ];
 
   # This value determines the NixOS release with which your system is to be
   # compatible, in order to avoid breaking some software such as database

hosts/nachtigall/default.nix

@@ -7,7 +7,13 @@
     ./configuration.nix
 
     ./networking.nix
+    ./prometheus-exporters.nix
     ./wireguard.nix
     ./backups.nix
+    "${flake.inputs.fork}/nixos/modules/services/matrix/matrix-authentication-service.nix"
+  ];
+
+  disabledModules = [
+    "services/matrix/matrix-authentication-service.nix"
   ];
 }

hosts/nachtigall/hardware-configuration.nix

@@ -50,6 +50,16 @@
     fsType = "vfat";
   };
 
+  fileSystems."/boot3" = {
+    device = "/dev/disk/by-uuid/E4E4-88C7";
+    fsType = "vfat";
+  };
+
+  fileSystems."/boot4" = {
+    device = "/dev/disk/by-uuid/E76C-A8A0";
+    fsType = "vfat";
+  };
+
   swapDevices = [ ];
 
   # Enables DHCP on each ethernet and wireless interface. In case of scripted networking

hosts/nachtigall/networking.nix

@@ -5,6 +5,15 @@
   ...
 }:
 {
+  systemd.services.nscd.environment = {
+    NSNCD_WORKER_COUNT = "16";
+    NSNCD_HANDOFF_TIMEOUT = "6";
+  };
+
+  networking.hosts = {
+    "138.201.80.102" = [ "nachtigall.${config.pub-solar-os.networking.domain}" ];
+    "2a01:4f8:172:1c25::1" = [ "nachtigall.${config.pub-solar-os.networking.domain}" ];
+  };
 
   networking.hostName = "nachtigall";
   networking.hostId = "00000001";

hosts/nachtigall/prometheus-exporters.nix

@@ -0,0 +1,53 @@
+{
+  config,
+  lib,
+  flake,
+  ...
+}:
+{
+  age.secrets."nextcloud-serverinfo-token" = {
+    file = "${flake.self}/secrets/nextcloud-serverinfo-token.age";
+    mode = "400";
+    owner = "nextcloud-exporter";
+  };
+
+  services.prometheus = {
+    exporters = {
+      # https://github.com/xperimental/nextcloud-exporter
+      nextcloud = {
+        enable = true;
+        openFirewall = true;
+        firewallFilter = "--in-interface wg-ssh --protocol tcp --match tcp --dport ${toString config.services.prometheus.exporters.nextcloud.port}";
+        url = "https://cloud.pub.solar";
+        tokenFile = config.age.secrets."nextcloud-serverinfo-token".path;
+        port = 9205;
+      };
+      # https://github.com/nginx/nginx-prometheus-exporter
+      nginx = {
+        enable = true;
+        openFirewall = true;
+        firewallFilter = "--in-interface wg-ssh --protocol tcp --match tcp --dport ${toString config.services.prometheus.exporters.nginx.port}";
+        port = 9113;
+      };
+      # https://github.com/hipages/php-fpm_exporter
+      #php-fpm = {
+      #  enable = true;
+      #  extraFlags = [
+      #    "--phpfpm.scrape-uri unix://${config.services.phpfpm.pools.nextcloud.socket};/status"
+      #  ];
+      #  group = "nginx";
+      #  openFirewall = true;
+      #  firewallFilter = "--in-interface wg-ssh --protocol tcp --match tcp --dport ${toString config.services.prometheus.exporters.php-fpm.port}";
+      #  port = 9253;
+      #};
+      # https://github.com/prometheus-community/postgres_exporter
+      postgres = {
+        enable = true;
+        dataSourceName = "user=postgres-exporter database=postgres host=/run/postgresql sslmode=disable";
+        openFirewall = true;
+        firewallFilter = "--in-interface wg-ssh --protocol tcp --match tcp --dport ${toString config.services.prometheus.exporters.postgres.port}";
+        port = 9187;
+      };
+    };
+  };
+}

hosts/nachtigall/wireguard.nix

@@ -18,16 +18,7 @@
         "fd00:fae:fae:fae:fae:1::/96"
       ];
       privateKeyFile = config.age.secrets.wg-private-key.path;
-      peers = flake.self.logins.admins.wireguardDevices ++ [
-        {
-          # flora-6.pub.solar
-          endpoint = "80.71.153.210:51820";
-          publicKey = "jtSR5G2P/nm9s8WrVc26Xc/SQLupRxyXE+5eIeqlsTU=";
-          allowedIPs = [
-            "10.7.6.2/32"
-            "fd00:fae:fae:fae:fae:2::/96"
-          ];
-        }
+      peers = flake.self.logins.wireguardDevices ++ [
         {
           # tankstelle.pub.solar
           endpoint = "80.244.242.5:51820";
@@ -37,6 +28,17 @@
             "fd00:fae:fae:fae:fae:4::/96"
           ];
         }
+        {
+          # trinkgenossin.pub.solar
+          publicKey = "QWgHovHxtqiQhnHLouSWiT6GIoQDmuvnThYL5c/rvU4=";
+          allowedIPs = [
+            "10.7.6.5/32"
+            "fd00:fae:fae:fae:fae:5::/96"
+          ];
+          #endpoint = "85.215.152.22:51820";
+          endpoint = "[2a01:239:35d:f500::1]:51820";
+          persistentKeepalive = 15;
+        }
       ];
     };
   };

hosts/tankstelle/backups.nix

@@ -5,8 +5,8 @@
     mode = "400";
     owner = "root";
   };
-  age.secrets."restic-repo-storagebox" = {
-    file = "${flake.self}/secrets/restic-repo-storagebox.age";
+  age.secrets."restic-repo-storagebox-tankstelle" = {
+    file = "${flake.self}/secrets/restic-repo-storagebox-tankstelle.age";
     mode = "400";
     owner = "root";
   };

hosts/tankstelle/configuration.nix

@@ -10,6 +10,9 @@
   boot.loader.systemd-boot.enable = true;
   boot.loader.efi.canTouchEfiVariables = true;
 
+  # kernel same-page merging
+  hardware.ksm.enable = true;
+
   boot.binfmt.emulatedSystems = [ "aarch64-linux" ];
 
   system.stateVersion = "23.11";

hosts/tankstelle/forgejo-actions-runner.nix

@@ -1,7 +1,5 @@
 {
   config,
-  lib,
-  pkgs,
   flake,
   ...
 }:
@@ -11,52 +9,18 @@
     mode = "440";
   };
 
-  # Trust docker bridge interface traffic
-  # Needed for the docker runner to communicate with the act_runner cache
-  networking.firewall.trustedInterfaces = [ "br-+" ];
-
-  users.users.gitea-runner = {
-    home = "/var/lib/gitea-runner/tankstelle";
-    useDefaultShell = true;
-    group = "gitea-runner";
-    # Required to interact with nix daemon
-    extraGroups = [ "wheel" ];
-    isSystemUser = true;
-  };
-
-  users.groups.gitea-runner = { };
-
-  systemd.tmpfiles.rules = [ "d '/var/lib/gitea-runner' 0750 gitea-runner gitea-runner - -" ];
-
-  systemd.services."gitea-runner-tankstelle" = {
-    serviceConfig.DynamicUser = lib.mkForce false;
-    path = with pkgs; [
-      coreutils
-      bash
-      coreutils
-      curl
-      gawk
-      gitMinimal
-      gnused
-      nodejs
-      wget
-      cachix
-      jq
-      nix
-    ];
-  };
-
   # forgejo actions runner
   # https://forgejo.org/docs/latest/admin/actions/
   # https://docs.gitea.com/usage/actions/quickstart
-  services.gitea-actions-runner = {
-    package = pkgs.forgejo-runner;
-    instances."tankstelle" = {
+  services.gitea-actions-runner.instances = {
+    tankstelle = {
       enable = true;
-      name = config.networking.hostName;
       url = "https://git.pub.solar";
+      name = config.networking.hostName;
       tokenFile = config.age.secrets.tankstelle-forgejo-actions-runner-token.path;
-      labels = [ "self-hosted:host://-self-hosted" ];
+      labels = [
+        "self-hosted:host://-self-hosted"
+      ];
     };
   };
 }

hosts/tankstelle/wireguard.nix

@@ -18,7 +18,7 @@
         "fd00:fae:fae:fae:fae:4::/96"
       ];
       privateKeyFile = config.age.secrets.wg-private-key.path;
-      peers = flake.self.logins.admins.wireguardDevices ++ [
+      peers = flake.self.logins.wireguardDevices ++ [
         {
           # nachtigall.pub.solar
           endpoint = "138.201.80.102:51820";
@@ -29,13 +29,15 @@
           ];
         }
         {
-          # flora-6.pub.solar
-          endpoint = "80.71.153.210:51820";
-          publicKey = "jtSR5G2P/nm9s8WrVc26Xc/SQLupRxyXE+5eIeqlsTU=";
+          # trinkgenossin.pub.solar
+          publicKey = "QWgHovHxtqiQhnHLouSWiT6GIoQDmuvnThYL5c/rvU4=";
           allowedIPs = [
-            "10.7.6.2/32"
-            "fd00:fae:fae:fae:fae:2::/96"
+            "10.7.6.5/32"
+            "fd00:fae:fae:fae:fae:5::/96"
           ];
+          #endpoint = "85.215.152.22:51820";
+          endpoint = "[2a01:239:35d:f500::1]:51820";
+          persistentKeepalive = 15;
         }
       ];
     };

hosts/trinkgenossin/configuration.nix

@@ -0,0 +1,35 @@
+{
+  flake,
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+{
+  boot.loader.grub.enable = true;
+  boot.loader.grub.devices = [ "/dev/vda" ];
+
+  boot.kernelParams = [
+    "boot.shell_on_fail=1"
+    "ip=dhcp"
+  ];
+
+  # This option defines the first version of NixOS you have installed on this particular machine,
+  # and is used to maintain compatibility with application data (e.g. databases) created on older NixOS versions.
+  #
+  # Most users should NEVER change this value after the initial install, for any reason,
+  # even if you've upgraded your system to a new NixOS release.
+  #
+  # This value does NOT affect the Nixpkgs version your packages and OS are pulled from,
+  # so changing it will NOT upgrade your system - see https://nixos.org/manual/nixos/stable/#sec-upgrading for how
+  # to actually do that.
+  #
+  # This value being lower than the current NixOS release does NOT mean your system is
+  # out of date, out of support, or vulnerable.
+  #
+  # Do NOT change this value unless you have manually inspected all the changes it would make to your configuration,
+  # and migrated your data accordingly.
+  #
+  # For more information, see `man configuration.nix` or https://nixos.org/manual/nixos/stable/options#opt-system.stateVersion .
+  system.stateVersion = "24.05"; # Did you read the comment?
+}

hosts/trinkgenossin/default.nix

@@ -0,0 +1,13 @@
+{ flake, ... }:
+
+{
+  imports = [
+    ./hardware-configuration.nix
+    ./configuration.nix
+
+    ./networking.nix
+    ./wireguard.nix
+    ./forgejo-actions-runner.nix
+    #./backups.nix
+  ];
+}

hosts/trinkgenossin/forgejo-actions-runner.nix

@@ -0,0 +1,73 @@
+{
+  config,
+  flake,
+  ...
+}:
+{
+  age.secrets."forgejo-actions-runner-token-miom" = {
+    file = "${flake.self}/secrets/trinkgenossin-forgejo-actions-runner-token-miom.age";
+    owner = "gitea-runner";
+    mode = "440";
+  };
+  age.secrets."forgejo-actions-runner-token-momo" = {
+    file = "${flake.self}/secrets/trinkgenossin-forgejo-actions-runner-token-momo.age";
+    owner = "gitea-runner";
+    mode = "440";
+  };
+  age.secrets."forgejo-actions-runner-token-pub-solar" = {
+    file = "${flake.self}/secrets/trinkgenossin-forgejo-actions-runner-token.age";
+    owner = "gitea-runner";
+    mode = "440";
+  };
+
+  # forgejo actions runner
+  # https://forgejo.org/docs/latest/admin/actions/
+  # https://docs.gitea.com/usage/actions/quickstart
+  services.gitea-actions-runner.instances = {
+    miom = {
+      enable = true;
+      url = "https://git.pub.solar";
+      name = config.networking.hostName;
+      tokenFile = config.age.secrets.forgejo-actions-runner-token-miom.path;
+      labels = [
+        # provide a debian 12 bookworm base with Node.js for actions
+        "debian-latest:docker://git.pub.solar/pub-solar/actions-base-image:20-bookworm"
+        # fake the ubuntu name, commonly used in actions examples
+        "ubuntu-latest:docker://git.pub.solar/pub-solar/actions-base-image:20-bookworm"
+        # alpine with Node.js
+        "alpine-latest:docker://node:20-alpine"
+      ];
+    };
+    momo = {
+      enable = true;
+      url = "https://git.pub.solar";
+      name = config.networking.hostName;
+      tokenFile = config.age.secrets.forgejo-actions-runner-token-momo.path;
+      labels = [
+        "self-hosted:host://-self-hosted"
+
+        # provide a debian 12 bookworm base with Node.js for actions
+        "debian-latest:docker://git.pub.solar/pub-solar/actions-base-image:20-bookworm"
+        # fake the ubuntu name, commonly used in actions examples
+        "ubuntu-latest:docker://git.pub.solar/pub-solar/actions-base-image:20-bookworm"
+        # alpine with Node.js
+        "alpine-latest:docker://node:20-alpine"
+      ];
+    };
+    # systemd does not like dashes in service unit names
+    pubsolar = {
+      enable = true;
+      url = "https://git.pub.solar";
+      name = config.networking.hostName;
+      tokenFile = config.age.secrets.forgejo-actions-runner-token-pub-solar.path;
+      labels = [
+        # provide a debian 12 bookworm base with Node.js for actions
+        "debian-latest:docker://git.pub.solar/pub-solar/actions-base-image:20-bookworm"
+        # fake the ubuntu name, commonly used in actions examples
+        "ubuntu-latest:docker://git.pub.solar/pub-solar/actions-base-image:20-bookworm"
+        # alpine with Node.js
+        "alpine-latest:docker://node:20-alpine"
+      ];
+    };
+  };
+}

hosts/trinkgenossin/hardware-configuration.nix

@@ -8,45 +8,47 @@
   modulesPath,
   ...
 }:
+
 {
   imports = [ ];
 
   boot.initrd.availableKernelModules = [
-    "ahci"
+    "ata_piix"
+    "uhci_hcd"
     "virtio_pci"
-    "xhci_pci"
     "sr_mod"
     "virtio_blk"
+    "virtio_net"
   ];
-  boot.initrd.kernelModules = [ ];
-  boot.kernelModules = [ ];
+  boot.initrd.kernelModules = [ "dm-snapshot" ];
+  boot.kernelModules = [ "kvm-amd" ];
   boot.extraModulePackages = [ ];
 
+  boot.initrd.luks.devices."cryptroot" = {
+    device = "/dev/disk/by-uuid/52a1fd17-63d7-4d0a-b7ff-74aceaf6085a";
+  };
+
   fileSystems."/" = {
     device = "/dev/disk/by-label/nixos";
-    autoResize = true;
     fsType = "ext4";
   };
 
   fileSystems."/boot" = {
     device = "/dev/disk/by-label/boot";
-    fsType = "vfat";
-  };
-
-  fileSystems."/data" = {
-    device = "/dev/disk/by-label/ephemeral0";
     fsType = "ext4";
-    options = [
-      "defaults"
-      "nofail"
-    ];
   };
 
-  swapDevices = [ ];
+  fileSystems."/var/lib/garage/data" = {
+    device = "/dev/disk/by-label/data";
+    fsType = "xfs";
+  };
 
-  networking.useDHCP = lib.mkDefault false;
-  networking.networkmanager.enable = lib.mkForce false;
+  fileSystems."/var/lib/garage/meta" = {
+    device = "/dev/disk/by-label/metadata";
+    fsType = "btrfs";
+  };
+
+  swapDevices = [ { device = "/dev/disk/by-label/swap"; } ];
 
   nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
-  hardware.cpu.intel.updateMicrocode = lib.mkDefault config.hardware.enableRedistributableFirmware;
 }

hosts/trinkgenossin/networking.nix

@@ -0,0 +1,15 @@
+{
+  config,
+  pkgs,
+  flake,
+  ...
+}:
+{
+  services.garage.settings.rpc_public_addr = "[2a01:239:35d:f500::1]:3901";
+
+  networking.hostName = "trinkgenossin";
+  networking.hostId = "00000003";
+
+  networking.enableIPv6 = true;
+  networking.useDHCP = true;
+}

hosts/trinkgenossin/wireguard.nix

@@ -4,21 +4,25 @@
   flake,
   ...
 }:
+let
+  wireguardIPv4 = "10.7.6.5";
+  wireguardIPv6 = "fd00:fae:fae:fae:fae:5::";
+in
 {
   networking.firewall.allowedUDPPorts = [ 51820 ];
 
-  age.secrets.wg-private-key.file = "${flake.self}/secrets/flora6-wg-private-key.age";
+  age.secrets.wg-private-key.file = "${flake.self}/secrets/trinkgenossin-wg-private-key.age";
 
   networking.wireguard.interfaces = {
     wg-ssh = {
       listenPort = 51820;
       mtu = 1300;
       ips = [
-        "10.7.6.2/32"
-        "fd00:fae:fae:fae:fae:2::/96"
+        "${wireguardIPv4}/32"
+        "${wireguardIPv6}/96"
       ];
       privateKeyFile = config.age.secrets.wg-private-key.path;
-      peers = flake.self.logins.admins.wireguardDevices ++ [
+      peers = flake.self.logins.wireguardDevices ++ [
         {
           # nachtigall.pub.solar
           endpoint = "138.201.80.102:51820";
@@ -47,17 +51,35 @@
             "fd00:fae:fae:fae:fae:4::/96"
           ];
         }
+        {
+          # delite.pub.solar
+          endpoint = "5.255.119.132:51820";
+          publicKey = "ZT2qGWgMPwHRUOZmTQHWCRX4m14YwOsiszjsA5bpc2k=";
+          allowedIPs = [
+            "10.7.6.6/32"
+            "fd00:fae:fae:fae:fae:6::/96"
+          ];
+        }
+        {
+          # blue-shell.pub.solar
+          endpoint = "194.13.83.205:51820";
+          publicKey = "bcrIpWrKc1M+Hq4ds3aN1lTaKE26f2rvXhd+93QrzR8=";
+          allowedIPs = [
+            "10.7.6.7/32"
+            "fd00:fae:fae:fae:fae:7::/96"
+          ];
+        }
       ];
     };
   };
 
   services.openssh.listenAddresses = [
     {
-      addr = "10.7.6.2";
+      addr = wireguardIPv4;
       port = 22;
     }
     {
-      addr = "[fd00:fae:fae:fae:fae:2::]";
+      addr = "[${wireguardIPv6}]";
       port = 22;
     }
   ];

hosts/underground/configuration.nix

@@ -0,0 +1,96 @@
+{
+  flake,
+  config,
+  pkgs,
+  ...
+}:
+{
+  # Use GRUB2 as the boot loader.
+  boot.loader.grub = {
+    enable = true;
+    devices = [ "/dev/vda" ];
+  };
+
+  pub-solar-os.networking.domain = "test.pub.solar";
+
+  systemd.tmpfiles.rules = [ "f /tmp/dbf 1777 root root 10d password" ];
+
+  # keycloak
+  pub-solar-os.auth = {
+    enable = true;
+    database-password-file = "/tmp/dbf";
+  };
+  services.keycloak.database.createLocally = true;
+
+  # matrix-synapse
+  # test.pub.solar /.well-known is required for federation
+  services.nginx.virtualHosts."${config.pub-solar-os.networking.domain}" = {
+    default = true;
+    enableACME = true;
+    forceSSL = true;
+  };
+
+  age.secrets."staging-matrix-synapse-secret-config.yaml" = {
+    file = "${flake.self}/secrets/staging-matrix-synapse-secret-config.yaml.age";
+    mode = "400";
+    owner = "matrix-synapse";
+  };
+
+  age.secrets."staging-matrix-authentication-service-secret-config.yml" = {
+    file = "${flake.self}/secrets/staging-matrix-authentication-service-secret-config.yml.age";
+    mode = "400";
+    owner = "matrix-authentication-service";
+  };
+
+  # matrix-appservice-irc
+  age.secrets."matrix-appservice-irc-mediaproxy-signing-key" = {
+    file = "${flake.self}/secrets/staging-matrix-appservice-irc-mediaproxy-signing-key.jwk.age";
+    mode = "400";
+    owner = "matrix-appservice-irc";
+  };
+
+  age.secrets."matrix-draupnir-access-token" = {
+    file = "${flake.self}/secrets/staging-matrix-draupnir-access-token.age";
+    mode = "400";
+    owner = "root";
+  };
+
+  pub-solar-os.matrix = {
+    enable = true;
+
+    appservice-irc.mediaproxy.signingKeyPath =
+      config.age.secrets."matrix-appservice-irc-mediaproxy-signing-key".path;
+
+    synapse = {
+      extra-config-files = [
+        config.age.secrets."staging-matrix-synapse-secret-config.yaml".path
+
+        # The registration file is automatically generated after starting the
+        # appservice for the first time.
+        # cp /var/lib/mautrix-telegram/telegram-registration.yaml \
+        #   /var/lib/matrix-synapse/
+        # chown matrix-synapse:matrix-synapse \
+        #   /var/lib/matrix-synapse/telegram-registration.yaml
+        #"/var/lib/matrix-synapse/telegram-registration.yaml"
+      ];
+      app-service-config-files = [
+        "/var/lib/matrix-appservice-irc/registration.yml"
+        #"/var/lib/matrix-synapse/telegram-registration.yaml"
+      ];
+    };
+
+    matrix-authentication-service.extra-config-files = [
+      config.age.secrets."staging-matrix-authentication-service-secret-config.yml".path
+    ];
+
+    draupnir = {
+      enable = true;
+      homeserver-url = "http://127.0.200.10:8008";
+      access-token-file = config.age.secrets."matrix-draupnir-access-token".path;
+    };
+  };
+
+  services.openssh.openFirewall = true;
+
+  system.stateVersion = "24.05";
+}

hosts/underground/default.nix

@@ -0,0 +1,16 @@
+{ flake, ... }:
+
+{
+  imports = [
+    # Include the results of the hardware scan.
+    ./hardware-configuration.nix
+    ./configuration.nix
+
+    ./networking.nix
+    "${flake.inputs.fork}/nixos/modules/services/matrix/matrix-authentication-service.nix"
+  ];
+
+  disabledModules = [
+    "services/matrix/matrix-authentication-service.nix"
+  ];
+}

hosts/underground/hardware-configuration.nix

@@ -0,0 +1,47 @@
+# Do not modify this file!  It was generated by ‘nixos-generate-config’
+# and may be overwritten by future invocations.  Please make changes
+# to /etc/nixos/configuration.nix instead.
+{
+  config,
+  lib,
+  pkgs,
+  modulesPath,
+  ...
+}:
+
+{
+  imports = [
+    (modulesPath + "/profiles/qemu-guest.nix")
+  ];
+
+  boot.initrd.availableKernelModules = [
+    "ahci"
+    "xhci_pci"
+    "virtio_pci"
+    "virtio_scsi"
+    "sd_mod"
+    "sr_mod"
+  ];
+  boot.kernelModules = [ "kvm-intel" ];
+  boot.extraModulePackages = [ ];
+
+  boot.initrd.luks.devices."cryptroot" = {
+    device = "/dev/disk/by-label/cryptroot";
+  };
+
+  fileSystems."/" = {
+    device = "/dev/disk/by-label/root";
+    fsType = "ext4";
+  };
+
+  fileSystems."/boot" = {
+    device = "/dev/disk/by-label/boot";
+    fsType = "ext4";
+  };
+
+  swapDevices = [
+    { device = "/dev/disk/by-label/swap"; }
+  ];
+
+  nixpkgs.hostPlatform = lib.mkDefault "x86_64-linux";
+}

hosts/underground/networking.nix

@@ -0,0 +1,30 @@
+{
+  config,
+  pkgs,
+  flake,
+  ...
+}:
+{
+
+  networking.hostName = "underground";
+
+  networking = {
+    defaultGateway = {
+      address = "80.244.242.1";
+      interface = "enp1s0";
+    };
+    nameservers = [
+      "95.129.51.51"
+      "80.244.244.244"
+    ];
+    interfaces.enp1s0 = {
+      useDHCP = false;
+      ipv4.addresses = [
+        {
+          address = "80.244.242.3";
+          prefixLength = 29;
+        }
+      ];
+    };
+  };
+}

logins/admins.nix

@@ -38,6 +38,22 @@
           "fd00:fae:fae:fae:fae:200::/96"
         ];
       }
+      {
+        # chocolatebar
+        publicKey = "AS9w0zDUFLcH6IiF6T1vsyZPWPJ3p5fKsjIsM2AoZz8=";
+        allowedIPs = [
+          "10.7.6.205/32"
+          "fd00:fae:fae:fae:fae:205::/96"
+        ];
+      }
+      {
+        # biolimo
+        publicKey = "gnLq6KikFVVGxLxPW+3ZnreokEKLDoso+cUepPOZsBA=";
+        allowedIPs = [
+          "10.7.6.206/32"
+          "fd00:fae:fae:fae:fae:206::/96"
+        ];
+      }
     ];
   };
 
@@ -63,6 +79,7 @@
   teutat3s = {
     sshPubKeys = {
       teutat3s-1 = "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFro/k4Mgqyh8yV/7Zwjc0dv60ZM7bROBU9JNd99P/4co6fxPt1pJiU/pEz2Dax/HODxgcO+jFZfvPEuLMCeAl0= YubiKey #10593996 PIV Slot 9a";
+      teutat3s-2 = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHcU6KPy4b1MQXd6EJhcYwbJu7E+0IrBZF/IP6T7gbMf teutat3s@dumpyourvms";
     };
 
     secretEncryptionKeys = {

logins/default.nix

@@ -6,19 +6,16 @@ in
 {
   flake = {
     logins = {
-      admins =
-        lib.lists.foldl
-          (logins: adminConfig: {
-            sshPubKeys = logins.sshPubKeys ++ (lib.attrsets.attrValues adminConfig.sshPubKeys);
-            wireguardDevices =
-              logins.wireguardDevices
-              ++ (if adminConfig ? "wireguardDevices" then adminConfig.wireguardDevices else [ ]);
-          })
-          {
-            sshPubKeys = [ ];
-            wireguardDevices = [ ];
-          }
-          (lib.attrsets.attrValues admins);
+      admins = admins;
+      wireguardDevices = lib.lists.foldl (
+        wireguardDevices: adminConfig:
+        wireguardDevices ++ (if adminConfig ? "wireguardDevices" then adminConfig.wireguardDevices else [ ])
+      ) [ ] (lib.attrsets.attrValues admins);
+      sshPubKeys = lib.lists.foldl (
+        sshPubKeys: adminConfig:
+        sshPubKeys
+        ++ (if adminConfig ? "sshPubKeys" then lib.attrsets.attrValues adminConfig.sshPubKeys else [ ])
+      ) [ ] (lib.attrsets.attrValues admins);
       robots.sshPubKeys = lib.attrsets.attrValues robots;
     };
   };

logins/robots.nix

@@ -1,7 +1,8 @@
 {
   # Used for restic backups to droppie, a server run by @b12f
-  "root@droppie" = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBZQSephFJU0NMbVbhwvVJ2/m6jcPYo1IsWCsoarqKin root@droppie";
+  "root@droppie" =
+    "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIBZQSephFJU0NMbVbhwvVJ2/m6jcPYo1IsWCsoarqKin root@droppie";
 
-  # robot user on flora-6
-  "hakkonaut@flora-6" = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGP5MvCwNRtCcP1pSDrn0XZTNlpOqYnjHDm9/OI4hECW hakkonaut@flora-6";
+  "hakkonaut" =
+    "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGP5MvCwNRtCcP1pSDrn0XZTNlpOqYnjHDm9/OI4hECW hakkonaut";
 }

modules/backups/default.nix

@@ -0,0 +1,383 @@
+{
+  flake,
+  config,
+  lib,
+  pkgs,
+  ...
+}:
+let
+  utils = import "${flake.inputs.nixpkgs}/nixos/lib/utils.nix" {
+    inherit lib;
+    inherit config;
+    inherit pkgs;
+  };
+  # Type for a valid systemd unit option. Needed for correctly passing "timerConfig" to "systemd.timers"
+  inherit (utils.systemdUtils.unitOptions) unitOption;
+  inherit (lib)
+    literalExpression
+    mkOption
+    mkPackageOption
+    types
+    ;
+in
+{
+  options.pub-solar-os.backups = {
+    repos = mkOption {
+      description = ''
+        Configuration of Restic repositories.
+      '';
+      type = types.attrsOf (
+        types.submodule (
+          { name, ... }:
+          {
+            options = {
+              passwordFile = mkOption {
+                type = types.str;
+                description = ''
+                  Read the repository password from a file.
+                '';
+                example = "/etc/nixos/restic-password";
+              };
+
+              environmentFile = mkOption {
+                type = with types; nullOr str;
+                default = null;
+                description = ''
+                  Read repository secrets as environment variables from a file.
+                '';
+                example = "/etc/nixos/restic-env";
+              };
+
+              repository = mkOption {
+                type = with types; nullOr str;
+                default = null;
+                description = ''
+                  repository to backup to.
+                '';
+                example = "sftp:backup@192.168.1.100:/backups/${name}";
+              };
+            };
+          }
+        )
+      );
+
+      default = { };
+      example = {
+        remotebackup = {
+          repository = "sftp:backup@host:/backups/home";
+          passwordFile = "/etc/nixos/secrets/restic-password";
+          environmentFile = "/etc/nixos/secrets/restic-env";
+        };
+      };
+    };
+
+    resources = mkOption {
+      description = "resources required to exist before starting restic backup archive process";
+      default = { };
+      type = types.attrsOf (
+        types.submodule (
+          { ... }:
+          {
+            options = {
+              resourceCreateCommand = mkOption {
+                type = with types; nullOr str;
+                default = null;
+                description = ''
+                  A script that must run successfully to create the resource. Optional.
+                '';
+              };
+
+              resourceDestroyCommand = mkOption {
+                type = with types; nullOr str;
+                default = null;
+                description = ''
+                  A script that runs when the resource is destroyed. Optional.
+                '';
+              };
+            };
+          }
+        )
+      );
+    };
+
+    restic = mkOption {
+      description = ''
+        Periodic backups to create with Restic.
+      '';
+      type = types.attrsOf (
+        types.submodule (
+          { name, ... }:
+          {
+            options = {
+              resources = mkOption {
+                type = types.listOf types.str;
+                default = [ ];
+              };
+
+              paths = mkOption {
+                # This is nullable for legacy reasons only. We should consider making it a pure listOf
+                # after some time has passed since this comment was added.
+                type = types.nullOr (types.listOf types.str);
+                default = [ ];
+                description = ''
+                  Which paths to backup, in addition to ones specified via
+                  `dynamicFilesFrom`.  If null or an empty array and
+                  `dynamicFilesFrom` is also null, no backup command will be run.
+                   This can be used to create a prune-only job.
+                '';
+                example = [
+                  "/var/lib/postgresql"
+                  "/home/user/backup"
+                ];
+              };
+
+              exclude = mkOption {
+                type = types.listOf types.str;
+                default = [ ];
+                description = ''
+                  Patterns to exclude when backing up. See
+                  https://restic.readthedocs.io/en/latest/040_backup.html#excluding-files for
+                  details on syntax.
+                '';
+                example = [
+                  "/var/cache"
+                  "/home/*/.cache"
+                  ".git"
+                ];
+              };
+
+              timerConfig = mkOption {
+                type = types.nullOr (types.attrsOf unitOption);
+                default = {
+                  OnCalendar = "daily";
+                  Persistent = true;
+                };
+                description = ''
+                  When to run the backup. See {manpage}`systemd.timer(5)` for
+                  details. If null no timer is created and the backup will only
+                  run when explicitly started.
+                '';
+                example = {
+                  OnCalendar = "00:05";
+                  RandomizedDelaySec = "5h";
+                  Persistent = true;
+                };
+              };
+
+              user = mkOption {
+                type = types.str;
+                default = "root";
+                description = ''
+                  As which user the backup should run.
+                '';
+                example = "postgresql";
+              };
+
+              extraBackupArgs = mkOption {
+                type = types.listOf types.str;
+                default = [ ];
+                description = ''
+                  Extra arguments passed to restic backup.
+                '';
+                example = [ "--exclude-file=/etc/nixos/restic-ignore" ];
+              };
+
+              extraOptions = mkOption {
+                type = types.listOf types.str;
+                default = [ ];
+                description = ''
+                  Extra extended options to be passed to the restic --option flag.
+                '';
+                example = [ "sftp.command='ssh backup@192.168.1.100 -i /home/user/.ssh/id_rsa -s sftp'" ];
+              };
+
+              initialize = mkOption {
+                type = types.bool;
+                default = false;
+                description = ''
+                  Create the repository if it doesn't exist.
+                '';
+              };
+
+              pruneOpts = mkOption {
+                type = types.listOf types.str;
+                default = [ ];
+                description = ''
+                  A list of options (--keep-\* et al.) for 'restic forget
+                  --prune', to automatically prune old snapshots.  The
+                  'forget' command is run *after* the 'backup' command, so
+                  keep that in mind when constructing the --keep-\* options.
+                '';
+                example = [
+                  "--keep-daily 7"
+                  "--keep-weekly 5"
+                  "--keep-monthly 12"
+                  "--keep-yearly 75"
+                ];
+              };
+
+              runCheck = mkOption {
+                type = types.bool;
+                default = (builtins.length config.pub-solar-os.backups.restic.${name}.checkOpts > 0);
+                defaultText = literalExpression ''builtins.length config.services.backups.${name}.checkOpts > 0'';
+                description = "Whether to run the `check` command with the provided `checkOpts` options.";
+                example = true;
+              };
+
+              checkOpts = mkOption {
+                type = types.listOf types.str;
+                default = [ ];
+                description = ''
+                  A list of options for 'restic check'.
+                '';
+                example = [ "--with-cache" ];
+              };
+
+              dynamicFilesFrom = mkOption {
+                type = with types; nullOr str;
+                default = null;
+                description = ''
+                  A script that produces a list of files to back up.  The
+                  results of this command are given to the '--files-from'
+                  option. The result is merged with paths specified via `paths`.
+                '';
+                example = "find /home/matt/git -type d -name .git";
+              };
+
+              backupPrepareCommand = mkOption {
+                type = with types; nullOr str;
+                default = null;
+                description = ''
+                  A script that must run before starting the backup process.
+                '';
+              };
+
+              backupCleanupCommand = mkOption {
+                type = with types; nullOr str;
+                default = null;
+                description = ''
+                  A script that must run after finishing the backup process.
+                '';
+              };
+
+              package = mkPackageOption pkgs "restic" { };
+
+              createWrapper = lib.mkOption {
+                type = lib.types.bool;
+                default = true;
+                description = ''
+                  Whether to generate and add a script to the system path, that has the same environment variables set
+                  as the systemd service. This can be used to e.g. mount snapshots or perform other opterations, without
+                  having to manually specify most options.
+                '';
+              };
+            };
+          }
+        )
+      );
+      default = { };
+      example = {
+        localbackup = {
+          paths = [ "/home" ];
+          exclude = [ "/home/*/.cache" ];
+          initialize = true;
+        };
+        remotebackup = {
+          paths = [ "/home" ];
+          extraOptions = [
+            "sftp.command='ssh backup@host -i /etc/nixos/secrets/backup-private-key -s sftp'"
+          ];
+          timerConfig = {
+            OnCalendar = "00:05";
+            RandomizedDelaySec = "5h";
+          };
+        };
+      };
+    };
+  };
+
+  config =
+    let
+      repos = config.pub-solar-os.backups.repos;
+      restic = config.pub-solar-os.backups.restic;
+      resources = config.pub-solar-os.backups.resources;
+
+      repoNames = builtins.attrNames repos;
+      resourceNames = builtins.attrNames resources;
+      backupNames = builtins.attrNames restic;
+
+      createResourceService = resourceName: {
+        name = "restic-backups-resource-${resourceName}";
+        value = {
+          serviceConfig =
+            let
+              createResourceApp = pkgs.writeShellApplication {
+                name = "create-resource-${resourceName}";
+                text = resources."${resourceName}".resourceCreateCommand;
+              };
+              destroyResourceApp = pkgs.writeShellApplication {
+                name = "destroy-resource-${resourceName}";
+                text = resources."${resourceName}".resourceDestroyCommand;
+              };
+
+            in
+            {
+              Type = "oneshot";
+              ExecStart = lib.mkIf (
+                resources."${resourceName}".resourceCreateCommand != null
+              ) "${createResourceApp}/bin/create-resource-${resourceName}";
+              ExecStop = lib.mkIf (
+                resources."${resourceName}".resourceDestroyCommand != null
+              ) "${destroyResourceApp}/bin/destroy-resource-${resourceName}";
+              RemainAfterExit = true;
+            };
+          unitConfig.StopWhenUnneeded = true;
+        };
+      };
+
+      createResourceDependency = resourceName: backupName: repoName: {
+        name = "restic-backups-${backupName}-${repoName}";
+        value = {
+          after = [ "restic-backups-resource-${resourceName}.service" ];
+          requires = [ "restic-backups-resource-${resourceName}.service" ];
+
+          serviceConfig.PrivateTmp = lib.mkForce false;
+          unitConfig = {
+            JoinsNamespaceOf = [ "restic-backups-resource-${resourceName}.service" ];
+          };
+        };
+      };
+
+      createResourceDependencies =
+        backupName:
+        map (
+          repoName:
+          map (resourceName: createResourceDependency resourceName backupName repoName)
+            restic."${backupName}".resources
+        ) repoNames;
+
+      createBackups =
+        backupName:
+        map (repoName: {
+          name = "${backupName}-${repoName}";
+          value = lib.attrsets.filterAttrs (key: val: (key != "resources")) (
+            repos."${repoName}" // restic."${backupName}"
+          );
+        }) repoNames;
+    in
+    {
+      systemd.services =
+        (builtins.listToAttrs (map createResourceService resourceNames))
+        // (builtins.listToAttrs (lib.lists.flatten (map createResourceDependencies backupNames)));
+
+      services.restic.backups = builtins.listToAttrs (lib.lists.flatten (map createBackups backupNames));
+
+      # Used for pub-solar-os.backups.repos.storagebox
+      programs.ssh.knownHosts = {
+        "u377325.your-storagebox.de".publicKey =
+          "ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA5EB5p/5Hp3hGW1oHok+PIOH9Pbn7cnUiGmUEBrCVjnAw+HrKyN8bYVV0dIGllswYXwkG/+bgiBlE6IVIBAq+JwVWu1Sss3KarHY3OvFJUXZoZyRRg/Gc/+LRCE7lyKpwWQ70dbelGRyyJFH36eNv6ySXoUYtGkwlU5IVaHPApOxe4LHPZa/qhSRbPo2hwoh0orCtgejRebNtW5nlx00DNFgsvn8Svz2cIYLxsPVzKgUxs8Zxsxgn+Q/UvR7uq4AbAhyBMLxv7DjJ1pc7PJocuTno2Rw9uMZi1gkjbnmiOh6TTXIEWbnroyIhwc8555uto9melEUmWNQ+C+PwAK+MPw==";
+        "[u377325.your-storagebox.de]:23".publicKey =
+          "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIICf9svRenC/PLKIL9nk6K/pxQgoiFC41wTNvoIncOxs";
+      };
+    };
+}

modules/core/default.nix

@@ -7,6 +7,7 @@
 }:
 {
   imports = [
+    ./logging.nix
     ./nix.nix
     ./networking.nix
     ./terminal-tooling.nix
@@ -41,7 +42,9 @@
     environment = {
       # Just a couple of global packages to make our lives easier
       systemPackages = with pkgs; [
+        btop
         git
+        jq
         vim
         wget
       ];
@@ -54,9 +57,5 @@
     };
 
     time.timeZone = "Etc/UTC";
-
-    home-manager.users.${config.pub-solar-os.authentication.username} = {
-      home.stateVersion = "23.05";
-    };
   };
 }

modules/core/logging.nix

@@ -0,0 +1,12 @@
+{
+  config,
+  pkgs,
+  lib,
+  flake,
+  ...
+}:
+{
+  services.journald.extraConfig = ''
+    MaxRetentionSec=3day
+  '';
+}

modules/core/networking.nix

@@ -27,17 +27,23 @@
     # Don't expose SSH via public interfaces
     networking.firewall.interfaces.wg-ssh.allowedTCPPorts = [ 22 ];
 
-    networking.domain = config.pub-solar-os.networking.domain;
+    # Setting this value breaks Matrix -> NextPush integration because
+    # matrix-synapse doesn't like it if nachtigall.pub.solar resolves to localhost.
+    #networking.domain = config.pub-solar-os.networking.domain;
 
     networking.hosts = {
       "10.7.6.1" = [ "nachtigall.wg.${config.pub-solar-os.networking.domain}" ];
-      "10.7.6.2" = [ "flora-6.wg.${config.pub-solar-os.networking.domain}" ];
       "10.7.6.3" = [ "metronom.wg.${config.pub-solar-os.networking.domain}" ];
       "10.7.6.4" = [ "tankstelle.wg.${config.pub-solar-os.networking.domain}" ];
+      "10.7.6.5" = [ "trinkgenossin.wg.${config.pub-solar-os.networking.domain}" ];
+      "10.7.6.6" = [ "delite.wg.${config.pub-solar-os.networking.domain}" ];
+      "10.7.6.7" = [ "blue-shell.wg.${config.pub-solar-os.networking.domain}" ];
       "fd00:fae:fae:fae:fae:1::" = [ "nachtigall.wg.${config.pub-solar-os.networking.domain}" ];
-      "fd00:fae:fae:fae:fae:2::" = [ "flora-6.wg.${config.pub-solar-os.networking.domain}" ];
       "fd00:fae:fae:fae:fae:3::" = [ "metronom.wg.${config.pub-solar-os.networking.domain}" ];
       "fd00:fae:fae:fae:fae:4::" = [ "tankstelle.wg.${config.pub-solar-os.networking.domain}" ];
+      "fd00:fae:fae:fae:fae:5::" = [ "trinkgenossin.wg.${config.pub-solar-os.networking.domain}" ];
+      "fd00:fae:fae:fae:fae:6::" = [ "delite.wg.${config.pub-solar-os.networking.domain}" ];
+      "fd00:fae:fae:fae:fae:7::" = [ "blue-shell.wg.${config.pub-solar-os.networking.domain}" ];
     };
 
     services.openssh = {
@@ -67,14 +73,31 @@
       };
     };
 
+    # These nameservers land in resolved.conf as 'DNS=<list>'
+    networking.nameservers = [
+      "193.110.81.0#dns0.eu"
+      "185.253.5.0#dns0.eu"
+      "9.9.9.9#dns.quad9.net"
+      "149.112.112.112#dns.quad9.net"
+      "2a0f:fc80::#dns0.eu"
+      "2a0f:fc81::#dns0.eu"
+      "2620:fe::fe#dns.quad9.net"
+      "2620:fe::9#dns.quad9.net"
+    ];
     services.resolved = {
       enable = true;
-      extraConfig = ''
-        DNS=193.110.81.0#dns0.eu 185.253.5.0#dns0.eu 2a0f:fc80::#dns0.eu 2a0f:fc81::#dns0.eu 9.9.9.9#dns.quad9.net 149.112.112.112#dns.quad9.net 2620:fe::fe#dns.quad9.net 2620:fe::9#dns.quad9.net
-        FallbackDNS=5.1.66.255#dot.ffmuc.net 185.150.99.255#dot.ffmuc.net 2001:678:e68:f000::#dot.ffmuc.net 2001:678:ed0:f000::#dot.ffmuc.net
-        Domains=~.
-        DNSOverTLS=yes
-      '';
+      dnsovertls = "true";
+      # default value in nixos module
+      dnssec = "false";
+      domains = [
+        "~."
+      ];
+      fallbackDns = [
+        "5.1.66.255#dot.ffmuc.net"
+        "185.150.99.255#dot.ffmuc.net"
+        "2001:678:e68:f000::#dot.ffmuc.net"
+        "2001:678:ed0:f000::#dot.ffmuc.net"
+      ];
     };
   };
 }

modules/core/nix.nix

@@ -6,7 +6,21 @@
   ...
 }:
 {
-  nixpkgs.config.allowUnfreePredicate = pkg: builtins.elem (lib.getName pkg) [ ];
+  nixpkgs.config = lib.mkDefault {
+    allowUnfreePredicate = pkg: builtins.elem (lib.getName pkg) [ ];
+    permittedInsecurePackages = [ "olm-3.2.16" ];
+  };
+
+  system.activationScripts.diff-closures = {
+    text = ''
+      if [[ -e /run/current-system ]]; then
+        ${config.nix.package}/bin/nix store diff-closures \
+          /run/current-system "$systemConfig" \
+          --extra-experimental-features nix-command
+      fi
+    '';
+    supportsDryActivation = true;
+  };
 
   nix = {
     # Use default version alias for nix package

modules/core/terminal-tooling.nix

@@ -1,19 +1,33 @@
-{ flake, config, ... }:
+{ flake, lib, ... }:
 {
-  home-manager.users.${config.pub-solar-os.authentication.username} = {
-    programs.git.enable = true;
-    programs.starship.enable = true;
-    programs.bash.enable = true;
-    programs.neovim = {
-      enable = true;
-      vimAlias = true;
-      viAlias = true;
-      defaultEditor = true;
-      # configure = {
-      #   packages.myVimPackages = with pkgs.vimPlugins; {
-      #     start = [vim-nix vim-surrund rainbow];
-      #   };
-      # };
-    };
-  };
+  home-manager.users = (
+    lib.attrsets.foldlAttrs (
+      acc: name: value:
+      acc
+      // {
+        ${name} = {
+          programs.git.enable = true;
+          programs.starship.enable = true;
+          programs.bash = {
+            enable = true;
+            historyControl = [
+              "ignoredups"
+              "ignorespace"
+            ];
+          };
+          programs.neovim = {
+            enable = true;
+            vimAlias = true;
+            viAlias = true;
+            defaultEditor = true;
+            # configure = {
+            #   packages.myVimPackages = with pkgs.vimPlugins; {
+            #     start = [vim-nix vim-surrund rainbow];
+            #   };
+            # };
+          };
+        };
+      }
+    ) { } flake.self.logins.admins
+  );
 }

modules/core/users.nix

@@ -11,18 +11,6 @@
       inherit (lib) mkOption types;
     in
     {
-      username = mkOption {
-        description = "Username for the adminstrative user";
-        type = types.str;
-        default = flake.self.username;
-      };
-
-      sshPubKeys = mkOption {
-        description = "SSH Keys that should have administrative root access";
-        type = types.listOf types.str;
-        default = flake.self.logins.admins.sshPubKeys;
-      };
-
       root.initialHashedPassword = mkOption {
         description = "Hashed password of the root account";
         type = types.str;
@@ -43,36 +31,62 @@
     };
 
   config = {
-    users.users.${config.pub-solar-os.authentication.username} = {
-      name = config.pub-solar-os.authentication.username;
-      group = config.pub-solar-os.authentication.username;
-      extraGroups = [
-        "wheel"
-        "docker"
-      ];
-      isNormalUser = true;
-      openssh.authorizedKeys.keys = config.pub-solar-os.authentication.sshPubKeys;
-    };
-    users.groups.${config.pub-solar-os.authentication.username} = { };
+    users.users =
+      (lib.attrsets.foldlAttrs (
+        acc: name: value:
+        acc
+        // {
+          ${name} = {
+            name = name;
+            group = name;
+            extraGroups = [
+              "wheel"
+              "docker"
+            ];
+            isNormalUser = true;
+            openssh.authorizedKeys.keys = lib.attrsets.attrValues value.sshPubKeys;
+          };
+        }
+      ) { } flake.self.logins.admins)
+      // {
+        # TODO: Remove when we stop locking ourselves out.
+        root.openssh.authorizedKeys.keys = flake.self.logins.sshPubKeys;
+        root.initialHashedPassword = config.pub-solar-os.authentication.root.initialHashedPassword;
 
-    # TODO: Remove when we stop locking ourselves out.
-    users.users.root.openssh.authorizedKeys.keys = config.pub-solar-os.authentication.sshPubKeys;
+        ${config.pub-solar-os.authentication.robot.username} = {
+          description = "CI and automation user";
+          home = "/home/${config.pub-solar-os.authentication.robot.username}";
+          createHome = true;
+          useDefaultShell = true;
+          uid = 1100;
+          group = "${config.pub-solar-os.authentication.robot.username}";
+          isSystemUser = true;
+          openssh.authorizedKeys.keys = config.pub-solar-os.authentication.robot.sshPubKeys;
+        };
+      };
 
-    users.users.${config.pub-solar-os.authentication.robot.username} = {
-      description = "CI and automation user";
-      home = "/home/${config.pub-solar-os.authentication.robot.username}";
-      createHome = true;
-      useDefaultShell = true;
-      uid = 998;
-      group = "${config.pub-solar-os.authentication.robot.username}";
-      isSystemUser = true;
-      openssh.authorizedKeys.keys = config.pub-solar-os.authentication.robot.sshPubKeys;
-    };
+    home-manager.users = (
+      lib.attrsets.foldlAttrs (
+        acc: name: value:
+        acc
+        // {
+          ${name} = {
+            home.stateVersion = "23.05";
+          };
+        }
+      ) { } flake.self.logins.admins
+    );
 
-    users.groups.${config.pub-solar-os.authentication.robot.username} = { };
-
-    users.users.root.initialHashedPassword =
-      config.pub-solar-os.authentication.root.initialHashedPassword;
+    users.groups =
+      (lib.attrsets.foldlAttrs (
+        acc: name: value:
+        acc // { "${name}" = { }; }
+      ) { } flake.self.logins.admins)
+      // {
+        ${config.pub-solar-os.authentication.robot.username} = {
+          gid = 1100;
+        };
+      };
 
     security.sudo.wheelNeedsPassword = false;
   };

modules/coturn/default.nix

@@ -18,7 +18,7 @@
     min-port = 49000;
     max-port = 50000;
     use-auth-secret = true;
-    static-auth-secret-file = "/run/agenix/coturn-static-auth-secret";
+    static-auth-secret-file = config.age.secrets."coturn-static-auth-secret".path;
     realm = "turn.${config.pub-solar-os.networking.domain}";
     cert = "${config.security.acme.certs.${realm}.directory}/full.pem";
     pkey = "${config.security.acme.certs.${realm}.directory}/key.pem";

modules/drone/default.nix

@@ -1,114 +0,0 @@
-{
-  config,
-  lib,
-  pkgs,
-  flake,
-  ...
-}:
-{
-  age.secrets.drone-secrets = {
-    file = "${flake.self}/secrets/drone-secrets.age";
-    mode = "600";
-    owner = "drone";
-  };
-  age.secrets.drone-db-secrets = {
-    file = "${flake.self}/secrets/drone-db-secrets.age";
-    mode = "600";
-    owner = "drone";
-  };
-
-  users.users.drone = {
-    description = "Drone Service";
-    home = "/var/lib/drone";
-    useDefaultShell = true;
-    uid = 994;
-    group = "drone";
-    isSystemUser = true;
-  };
-
-  users.groups.drone = { };
-
-  systemd.tmpfiles.rules = [ "d '/var/lib/drone-db' 0750 drone drone - -" ];
-
-  services.caddy.virtualHosts."ci.${config.pub-solar-os.networking.domain}" = {
-    logFormat = lib.mkForce ''
-      output discard
-    '';
-    extraConfig = ''
-      reverse_proxy :4000
-    '';
-  };
-
-  systemd.services."docker-network-drone" =
-    let
-      docker = config.virtualisation.oci-containers.backend;
-      dockerBin = "${pkgs.${docker}}/bin/${docker}";
-    in
-    {
-      serviceConfig.Type = "oneshot";
-      before = [ "docker-drone-server.service" ];
-      script = ''
-        ${dockerBin} network inspect drone-net >/dev/null 2>&1 || ${dockerBin} network create drone-net --subnet 172.20.0.0/24
-      '';
-    };
-
-  virtualisation = {
-    docker = {
-      enable = true; # sadly podman is not supported rightnow
-      extraOptions = ''
-        --data-root /data/docker
-      '';
-    };
-
-    oci-containers = {
-      backend = "docker";
-      containers."drone-db" = {
-        image = "postgres:14";
-        autoStart = true;
-        user = "994";
-        volumes = [ "/var/lib/drone-db:/var/lib/postgresql/data" ];
-        extraOptions = [ "--network=drone-net" ];
-        environmentFiles = [ config.age.secrets.drone-db-secrets.path ];
-      };
-      containers."drone-server" = {
-        image = "drone/drone:2";
-        autoStart = true;
-        user = "994";
-        ports = [ "127.0.0.1:4000:80" ];
-        dependsOn = [ "drone-db" ];
-        extraOptions = [
-          "--network=drone-net"
-          "--pull=always"
-          "--add-host=nachtigall.${config.pub-solar-os.networking.domain}:10.7.6.1"
-        ];
-        environment = {
-          DRONE_GITEA_SERVER = "https://git.${config.pub-solar-os.networking.domain}";
-          DRONE_SERVER_HOST = "ci.${config.pub-solar-os.networking.domain}";
-          DRONE_SERVER_PROTO = "https";
-          DRONE_DATABASE_DRIVER = "postgres";
-        };
-        environmentFiles = [ config.age.secrets.drone-secrets.path ];
-      };
-      containers."drone-docker-runner" = {
-        image = "drone/drone-runner-docker:1";
-        autoStart = true;
-        # needs to run as root
-        #user = "994";
-        volumes = [ "/var/run/docker.sock:/var/run/docker.sock" ];
-        dependsOn = [ "drone-db" ];
-        extraOptions = [
-          "--network=drone-net"
-          "--pull=always"
-          "--add-host=nachtigall.${config.pub-solar-os.networking.domain}:10.7.6.1"
-        ];
-        environment = {
-          DRONE_RPC_HOST = "ci.${config.pub-solar-os.networking.domain}";
-          DRONE_RPC_PROTO = "https";
-          DRONE_RUNNER_CAPACITY = "2";
-          DRONE_RUNNER_NAME = "flora-6-docker-runner";
-        };
-        environmentFiles = [ config.age.secrets.drone-secrets.path ];
-      };
-    };
-  };
-}

modules/forgejo-actions-runner/default.nix

@@ -2,66 +2,67 @@
   config,
   lib,
   pkgs,
-  flake,
   ...
 }:
+let
+  cfg = config.services.gitea-actions-runner;
+in
 {
-  age.secrets.forgejo-actions-runner-token = {
-    file = "${flake.self}/secrets/forgejo-actions-runner-token.age";
-    mode = "440";
-  };
 
-  # Trust docker bridge interface traffic
-  # Needed for the docker runner to communicate with the act_runner cache
-  networking.firewall.trustedInterfaces = [ "br-+" ];
+  config = lib.mkIf (cfg.instances != { }) {
+    # Label configuration on gitea-actions-runner instance requires either docker or podman
+    virtualisation.docker.enable = true;
 
-  users.users.gitea-runner = {
-    home = "/var/lib/gitea-runner/flora-6";
-    useDefaultShell = true;
-    group = "gitea-runner";
-    isSystemUser = true;
-  };
+    # Trust docker bridge interface traffic
+    # Needed for the docker runner to communicate with the act_runner cache
+    networking.firewall.trustedInterfaces = [ "br-+" ];
 
-  users.groups.gitea-runner = { };
-
-  systemd.services."gitea-runner-flora\\x2d6".serviceConfig = {
-    DynamicUser = lib.mkForce false;
-  };
-
-  systemd.tmpfiles.rules = [
-    "d '/data/gitea-actions-runner' 0750 gitea-runner gitea-runner - -"
-    "d '/var/lib/gitea-runner' 0750 gitea-runner gitea-runner - -"
-  ];
-
-  # forgejo actions runner
-  # https://forgejo.org/docs/latest/admin/actions/
-  # https://docs.gitea.com/usage/actions/quickstart
-  services.gitea-actions-runner = {
-    package = pkgs.forgejo-runner;
-    instances."flora-6" = {
-      enable = true;
-      name = config.networking.hostName;
-      url = "https://git.pub.solar";
-      tokenFile = config.age.secrets.forgejo-actions-runner-token.path;
-      settings = {
-        cache = {
-          enabled = true;
-          dir = "/data/gitea-actions-runner/actcache";
-          host = "";
-          port = 0;
-          external_server = "";
-        };
-      };
-      labels = [
-        # provide a debian 12 bookworm base with Node.js for actions
-        "debian-latest:docker://git.pub.solar/pub-solar/actions-base-image:20-bookworm"
-        # fake the ubuntu name, commonly used in actions examples
-        "ubuntu-latest:docker://git.pub.solar/pub-solar/actions-base-image:20-bookworm"
-        # alpine with Node.js
-        "alpine-latest:docker://node:20-alpine"
-        # nix flakes enabled image with Node.js
-        "nix-flakes:docker://git.pub.solar/pub-solar/nix-flakes-node:latest"
+    users.users.gitea-runner = {
+      home = "/var/lib/gitea-runner";
+      useDefaultShell = true;
+      group = "gitea-runner";
+      # Required to interact with nix daemon
+      extraGroups = [
+        "wheel"
+        "docker"
       ];
+      isSystemUser = true;
+    };
+
+    users.groups.gitea-runner = { };
+
+    systemd.tmpfiles.rules =
+      [
+        "d '/var/lib/gitea-runner' 0770 gitea-runner gitea-runner - -"
+      ]
+      ++ lib.mapAttrsToList (
+        name: instance: "d '/var/lib/gitea-runner/${name}' 0770 gitea-runner gitea-runner - -"
+      ) cfg.instances;
+
+    systemd.services = lib.concatMapAttrs (name: instance: {
+      "gitea-runner-${name}" = {
+        serviceConfig.DynamicUser = lib.mkForce false;
+        path = with pkgs; [
+          coreutils
+          openssh
+          gnupg
+          bash
+          curl
+          gawk
+          gitMinimal
+          gnused
+          nodejs_22
+          wget
+          cachix
+          jq
+          nix
+          docker
+        ];
+      };
+    }) cfg.instances;
+
+    services.gitea-actions-runner = {
+      package = pkgs.forgejo-runner;
     };
   };
 }

modules/forgejo/default.nix

@@ -65,6 +65,7 @@
 
   services.forgejo = {
     enable = true;
+    package = pkgs.forgejo;
     user = "gitea";
     group = "gitea";
     database = {
@@ -75,7 +76,7 @@
     };
     stateDir = "/var/lib/forgejo";
     lfs.enable = true;
-    mailerPasswordFile = config.age.secrets.forgejo-mailer-password.path;
+    secrets.mailer.PASSWD = config.age.secrets.forgejo-mailer-password.path;
     settings = {
       DEFAULT.APP_NAME = "pub.solar git server";
 
@@ -141,6 +142,12 @@
         LOGIN_REMEMBER_DAYS = 365;
       };
 
+      # See https://docs.gitea.com/administration/config-cheat-sheet#migrations-migrations
+      migrations = {
+        # This allows migrations from the same forgejo instance
+        ALLOW_LOCALNETWORKS = true;
+      };
+
       # https://forgejo.org/docs/next/admin/config-cheat-sheet/#indexer-indexer
       indexer = {
         REPO_INDEXER_ENABLED = true;
@@ -179,10 +186,10 @@
       "/tmp/forgejo-backup.sql"
     ];
     timerConfig = {
-      OnCalendar = "*-*-* 00:00:00 Etc/UTC";
+      OnCalendar = "*-*-* 23:00:00 Etc/UTC";
     };
     initialize = true;
-    passwordFile = config.age.secrets."restic-repo-storagebox".path;
+    passwordFile = config.age.secrets."restic-repo-storagebox-nachtigall".path;
     repository = "sftp:u377325@u377325.your-storagebox.de:/backups";
     backupPrepareCommand = ''
       ${pkgs.sudo}/bin/sudo -u postgres ${pkgs.postgresql}/bin/pg_dump -d gitea > /tmp/forgejo-backup.sql
@@ -190,10 +197,5 @@
     backupCleanupCommand = ''
       rm /tmp/forgejo-backup.sql
     '';
-    pruneOpts = [
-      "--keep-daily 7"
-      "--keep-weekly 4"
-      "--keep-monthly 3"
-    ];
   };
 }

modules/garage/default.nix

@@ -0,0 +1,142 @@
+{
+  config,
+  lib,
+  pkgs,
+  flake,
+  ...
+}:
+{
+  age.secrets."garage-rpc-secret" = {
+    file = "${flake.self}/secrets/garage-rpc-secret.age";
+    mode = "400";
+  };
+
+  age.secrets."garage-admin-token" = {
+    file = "${flake.self}/secrets/garage-admin-token.age";
+    mode = "400";
+  };
+
+  age.secrets."acme-namecheap-env" = {
+    file = "${flake.self}/secrets/acme-namecheap-env.age";
+    mode = "400";
+  };
+
+  networking.firewall.allowedTCPPorts = [
+    3900
+    3901
+    3902
+  ];
+
+  networking.firewall.interfaces.wg-ssh.allowedTCPPorts = [ 3903 ];
+
+  security.acme = {
+    defaults = {
+      # LEGO_DISABLE_CNAME_SUPPORT=true set here to fix issues with CNAME
+      # detection, as we use wildcard DNS for garage
+      environmentFile = config.age.secrets.acme-namecheap-env.path;
+    };
+    certs = {
+      # Wildcard certificate gets created automatically
+      "buckets.${config.pub-solar-os.networking.domain}" = {
+        # disable http challenge
+        webroot = null;
+        # enable dns challenge
+        dnsProvider = "namecheap";
+      };
+      # Wildcard certificate gets created automatically
+      "web.${config.pub-solar-os.networking.domain}" = {
+        # disable http challenge
+        webroot = null;
+        # enable dns challenge
+        dnsProvider = "namecheap";
+      };
+    };
+  };
+
+  services.nginx = {
+    upstreams.s3_backend.servers = {
+      "[::1]:3900" = { };
+    };
+    upstreams.web_backend.servers = {
+      "[::1]:3902" = { };
+    };
+    virtualHosts."buckets.${config.pub-solar-os.networking.domain}" = {
+      serverAliases = [ "*.buckets.${config.pub-solar-os.networking.domain}" ];
+
+      enableACME = true;
+      forceSSL = true;
+
+      locations."/" = {
+        proxyPass = "http://s3_backend";
+        extraConfig = ''
+          client_max_body_size 64m;
+          proxy_max_temp_file_size 0;
+        '';
+      };
+    };
+    virtualHosts."web.${config.pub-solar-os.networking.domain}" = {
+      serverAliases = [ "*.web.${config.pub-solar-os.networking.domain}" ];
+
+      enableACME = true;
+      forceSSL = true;
+
+      locations."/" = {
+        proxyPass = "http://web_backend";
+      };
+    };
+  };
+
+  services.garage = {
+    enable = true;
+    package = pkgs.garage_1_1_0;
+    settings = {
+      data_dir = "/var/lib/garage/data";
+      metadata_dir = "/var/lib/garage/meta";
+      db_engine = "lmdb";
+      replication_factor = 3;
+      compression_level = 2;
+      rpc_bind_addr = "[::]:3901";
+      s3_api = {
+        s3_region = "eu-central";
+        api_bind_addr = "[::]:3900";
+        root_domain = ".buckets.${config.pub-solar-os.networking.domain}";
+      };
+      s3_web = {
+        bind_addr = "[::]:3902";
+        root_domain = ".web.${config.pub-solar-os.networking.domain}";
+        index = "index.html";
+      };
+      admin = {
+        api_bind_addr = "[::]:3903";
+      };
+    };
+  };
+
+  users.users.garage = {
+    isSystemUser = true;
+    home = "/var/lib/garage";
+    group = "garage";
+  };
+
+  users.groups.garage = { };
+
+  # Adapted from https://git.clan.lol/clan/clan-core/src/commit/23a9e35c665ff531fe1193dcc47056432fbbeacf/clanModules/garage/default.nix
+  # Disabled DynamicUser https://github.com/NixOS/nixpkgs/blob/nixos-24.05/nixos/modules/services/web-servers/garage.nix
+  # for mounts + permissions to work
+  systemd.services.garage = {
+    serviceConfig = {
+      user = "garage";
+      group = "garage";
+      DynamicUser = false;
+      LoadCredential = [
+        "rpc_secret_path:${config.age.secrets.garage-rpc-secret.path}"
+        "admin_token_path:${config.age.secrets.garage-admin-token.path}"
+      ];
+      Environment = [
+        "GARAGE_ALLOW_WORLD_READABLE_SECRETS=true"
+        "GARAGE_RPC_SECRET_FILE=%d/rpc_secret_path"
+        "GARAGE_ADMIN_TOKEN_FILE=%d/admin_token_path"
+      ];
+    };
+  };
+}

modules/grafana/default.nix

@@ -23,29 +23,80 @@
   };
 
   environment.etc = {
+    "grafana-dashboards/nextcloud.json" = {
+      source = ./grafana-dashboards/nextcloud.json;
+      group = "grafana";
+      user = "grafana";
+    };
     "grafana-dashboards/node-exporter-full_rev33.json" = {
       source = ./grafana-dashboards/node-exporter-full_rev33.json;
       group = "grafana";
       user = "grafana";
     };
+    "grafana-dashboards/nginx.json" = {
+      source = ./grafana-dashboards/nginx.json;
+      group = "grafana";
+      user = "grafana";
+    };
+    "grafana-dashboards/postgres-overview.json" = {
+      source = ./grafana-dashboards/postgres-overview.json;
+      group = "grafana";
+      user = "grafana";
+    };
     "grafana-dashboards/synapse.json" = {
       source = ./grafana-dashboards/synapse.json;
       group = "grafana";
       user = "grafana";
     };
+    "grafana-dashboards/grafana-garage-dashboard-prometheus.json" = {
+      source = ./grafana-dashboards/grafana-garage-dashboard-prometheus.json;
+      group = "grafana";
+      user = "grafana";
+    };
+    "grafana-dashboards/loki/loki-chunks.json" = {
+      source = ./grafana-dashboards/loki/loki-chunks.json;
+      group = "grafana";
+      user = "grafana";
+    };
+    "grafana-dashboards/loki/loki-deletion.json" = {
+      source = ./grafana-dashboards/loki/loki-deletion.json;
+      group = "grafana";
+      user = "grafana";
+    };
+    "grafana-dashboards/loki/loki-operational.json" = {
+      source = ./grafana-dashboards/loki/loki-operational.json;
+      group = "grafana";
+      user = "grafana";
+    };
+    "grafana-dashboards/loki/loki-reads.json" = {
+      source = ./grafana-dashboards/loki/loki-reads.json;
+      group = "grafana";
+      user = "grafana";
+    };
+    "grafana-dashboards/loki/loki-retention.json" = {
+      source = ./grafana-dashboards/loki/loki-retention.json;
+      group = "grafana";
+      user = "grafana";
+    };
+    "grafana-dashboards/loki/loki-writes.json" = {
+      source = ./grafana-dashboards/loki/loki-writes.json;
+      group = "grafana";
+      user = "grafana";
+    };
   };
 
-  services.caddy.virtualHosts."grafana.${config.pub-solar-os.networking.domain}" = {
-    logFormat = lib.mkForce ''
-      output discard
-    '';
-    extraConfig = ''
-      reverse_proxy :${toString config.services.grafana.settings.server.http_port}
-    '';
+  services.nginx.virtualHosts."grafana.${config.pub-solar-os.networking.domain}" = {
+    enableACME = true;
+    forceSSL = true;
+    locations."/".proxyPass =
+      "http://127.0.0.1:${toString config.services.grafana.settings.server.http_port}";
   };
 
   services.grafana = {
     enable = true;
+    declarativePlugins = with pkgs.grafanaPlugins; [
+      grafana-lokiexplore-app
+    ];
     settings = {
       server = {
         # Listening Address
@@ -64,7 +115,7 @@
         password = "\$__file{${config.age.secrets.grafana-smtp-password.path}}";
         from_address = "no-reply@pub.solar";
         from_name = "grafana.pub.solar";
-        ehlo_identity = "flora-6.pub.solar";
+        ehlo_identity = "grafana.pub.solar";
       };
       security = {
         admin_email = "crew@pub.solar";
@@ -114,6 +165,7 @@
             {
               name = "pub.solar Dashboards";
               options.path = "/etc/grafana-dashboards";
+              options.foldersFromFilesStructure = true;
             }
           ];
         };

modules/grafana/grafana-dashboards/loki/loki-deletion.json

@@ -0,0 +1,750 @@
+{
+      "annotations": {
+         "list": [ ]
+      },
+      "editable": true,
+      "gnetId": null,
+      "graphTooltip": 0,
+      "hideControls": false,
+      "links": [
+         {
+            "asDropdown": true,
+            "icon": "external link",
+            "includeVars": true,
+            "keepTime": true,
+            "tags": [
+               "loki"
+            ],
+            "targetBlank": false,
+            "title": "Loki Dashboards",
+            "type": "dashboards"
+         }
+      ],
+      "refresh": "10s",
+      "rows": [
+         {
+            "collapse": false,
+            "height": "100px",
+            "panels": [
+               {
+                  "aliasColors": { },
+                  "bars": false,
+                  "dashLength": 10,
+                  "dashes": false,
+                  "datasource": "$datasource",
+                  "fill": 1,
+                  "format": "none",
+                  "id": 1,
+                  "legend": {
+                     "avg": false,
+                     "current": false,
+                     "max": false,
+                     "min": false,
+                     "show": true,
+                     "total": false,
+                     "values": false
+                  },
+                  "lines": true,
+                  "linewidth": 1,
+                  "links": [ ],
+                  "nullPointMode": "null as zero",
+                  "percentage": false,
+                  "pointradius": 5,
+                  "points": false,
+                  "renderer": "flot",
+                  "seriesOverrides": [ ],
+                  "spaceLength": 10,
+                  "span": 6,
+                  "stack": false,
+                  "steppedLine": false,
+                  "targets": [
+                     {
+                        "expr": "sum(loki_compactor_pending_delete_requests_count{cluster=~\"$cluster\", namespace=~\"$namespace\"})",
+                        "format": "time_series",
+                        "instant": true,
+                        "refId": "A"
+                     }
+                  ],
+                  "thresholds": "70,80",
+                  "timeFrom": null,
+                  "timeShift": null,
+                  "title": "Number of Pending Requests",
+                  "tooltip": {
+                     "shared": true,
+                     "sort": 2,
+                     "value_type": "individual"
+                  },
+                  "type": "singlestat",
+                  "xaxis": {
+                     "buckets": null,
+                     "mode": "time",
+                     "name": null,
+                     "show": true,
+                     "values": [ ]
+                  },
+                  "yaxes": [
+                     {
+                        "format": "short",
+                        "label": null,
+                        "logBase": 1,
+                        "max": null,
+                        "min": 0,
+                        "show": true
+                     },
+                     {
+                        "format": "short",
+                        "label": null,
+                        "logBase": 1,
+                        "max": null,
+                        "min": null,
+                        "show": false
+                     }
+                  ]
+               },
+               {
+                  "aliasColors": { },
+                  "bars": false,
+                  "dashLength": 10,
+                  "dashes": false,
+                  "datasource": "$datasource",
+                  "fill": 1,
+                  "format": "dtdurations",
+                  "id": 2,
+                  "legend": {
+                     "avg": false,
+                     "current": false,
+                     "max": false,
+                     "min": false,
+                     "show": true,
+                     "total": false,
+                     "values": false
+                  },
+                  "lines": true,
+                  "linewidth": 1,
+                  "links": [ ],
+                  "nullPointMode": "null as zero",
+                  "percentage": false,
+                  "pointradius": 5,
+                  "points": false,
+                  "renderer": "flot",
+                  "seriesOverrides": [ ],
+                  "spaceLength": 10,
+                  "span": 6,
+                  "stack": false,
+                  "steppedLine": false,
+                  "targets": [
+                     {
+                        "expr": "max(loki_compactor_oldest_pending_delete_request_age_seconds{cluster=~\"$cluster\", namespace=~\"$namespace\"})",
+                        "format": "time_series",
+                        "instant": true,
+                        "refId": "A"
+                     }
+                  ],
+                  "thresholds": "70,80",
+                  "timeFrom": null,
+                  "timeShift": null,
+                  "title": "Oldest Pending Request Age",
+                  "tooltip": {
+                     "shared": true,
+                     "sort": 2,
+                     "value_type": "individual"
+                  },
+                  "type": "singlestat",
+                  "xaxis": {
+                     "buckets": null,
+                     "mode": "time",
+                     "name": null,
+                     "show": true,
+                     "values": [ ]
+                  },
+                  "yaxes": [
+                     {
+                        "format": "short",
+                        "label": null,
+                        "logBase": 1,
+                        "max": null,
+                        "min": 0,
+                        "show": true
+                     },
+                     {
+                        "format": "short",
+                        "label": null,
+                        "logBase": 1,
+                        "max": null,
+                        "min": null,
+                        "show": false
+                     }
+                  ]
+               }
+            ],
+            "repeat": null,
+            "repeatIteration": null,
+            "repeatRowId": null,
+            "showTitle": false,
+            "title": "Headlines",
+            "titleSize": "h6"
+         },
+         {
+            "collapse": false,
+            "height": "250px",
+            "panels": [
+               {
+                  "datasource": "$datasource",
+                  "fieldConfig": {
+                     "defaults": {
+                        "custom": {
+                           "drawStyle": "line",
+                           "fillOpacity": 10,
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "showPoints": "never",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           }
+                        },
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [ ]
+                        },
+                        "unit": "short"
+                     },
+                     "overrides": [ ]
+                  },
+                  "id": 3,
+                  "links": [ ],
+                  "options": {
+                     "legend": {
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "span": 4,
+                  "targets": [
+                     {
+                        "expr": "(loki_compactor_delete_requests_received_total{cluster=~\"$cluster\", namespace=~\"$namespace\"} or on() vector(0)) - on () (loki_compactor_delete_requests_processed_total{cluster=~\"$cluster\", namespace=~\"$namespace\"} or on () vector(0))",
+                        "format": "time_series",
+                        "legendFormat": "in progress",
+                        "legendLink": null
+                     }
+                  ],
+                  "title": "# of Delete Requests (received - processed) ",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": "$datasource",
+                  "fieldConfig": {
+                     "defaults": {
+                        "custom": {
+                           "drawStyle": "line",
+                           "fillOpacity": 10,
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "showPoints": "never",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           }
+                        },
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [ ]
+                        },
+                        "unit": "short"
+                     },
+                     "overrides": [ ]
+                  },
+                  "id": 4,
+                  "links": [ ],
+                  "options": {
+                     "legend": {
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "span": 4,
+                  "targets": [
+                     {
+                        "expr": "sum(increase(loki_compactor_delete_requests_received_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[1d]))",
+                        "format": "time_series",
+                        "legendFormat": "received",
+                        "legendLink": null
+                     }
+                  ],
+                  "title": "Delete Requests Received / Day",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": "$datasource",
+                  "fieldConfig": {
+                     "defaults": {
+                        "custom": {
+                           "drawStyle": "line",
+                           "fillOpacity": 10,
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "showPoints": "never",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           }
+                        },
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [ ]
+                        },
+                        "unit": "short"
+                     },
+                     "overrides": [ ]
+                  },
+                  "id": 5,
+                  "links": [ ],
+                  "options": {
+                     "legend": {
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "span": 4,
+                  "targets": [
+                     {
+                        "expr": "sum(increase(loki_compactor_delete_requests_processed_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[1d]))",
+                        "format": "time_series",
+                        "legendFormat": "processed",
+                        "legendLink": null
+                     }
+                  ],
+                  "title": "Delete Requests Processed / Day",
+                  "type": "timeseries"
+               }
+            ],
+            "repeat": null,
+            "repeatIteration": null,
+            "repeatRowId": null,
+            "showTitle": true,
+            "title": "Churn",
+            "titleSize": "h6"
+         },
+         {
+            "collapse": false,
+            "height": "250px",
+            "panels": [
+               {
+                  "datasource": "$datasource",
+                  "fieldConfig": {
+                     "defaults": {
+                        "custom": {
+                           "drawStyle": "line",
+                           "fillOpacity": 10,
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "showPoints": "never",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           }
+                        },
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [ ]
+                        },
+                        "unit": "short"
+                     },
+                     "overrides": [ ]
+                  },
+                  "id": 6,
+                  "links": [ ],
+                  "options": {
+                     "legend": {
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "span": 4,
+                  "targets": [
+                     {
+                        "expr": "node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=~\"$cluster\", namespace=~\"$namespace\", pod=~\"loki.*\"}",
+                        "format": "time_series",
+                        "legendFormat": "{{pod}}",
+                        "legendLink": null
+                     }
+                  ],
+                  "title": "Compactor CPU usage",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": "$datasource",
+                  "fieldConfig": {
+                     "defaults": {
+                        "custom": {
+                           "drawStyle": "line",
+                           "fillOpacity": 10,
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "showPoints": "never",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           }
+                        },
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [ ]
+                        },
+                        "unit": "short"
+                     },
+                     "overrides": [ ]
+                  },
+                  "id": 7,
+                  "links": [ ],
+                  "options": {
+                     "legend": {
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "span": 4,
+                  "targets": [
+                     {
+                        "expr": "go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", pod=~\"loki.*\"} / 1024 / 1024 ",
+                        "format": "time_series",
+                        "legendFormat": " {{pod}} ",
+                        "legendLink": null
+                     }
+                  ],
+                  "title": "Compactor memory usage (MiB)",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": "$datasource",
+                  "fieldConfig": {
+                     "defaults": {
+                        "custom": {
+                           "drawStyle": "line",
+                           "fillOpacity": 10,
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "showPoints": "never",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           }
+                        },
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [ ]
+                        },
+                        "unit": "short"
+                     },
+                     "overrides": [ ]
+                  },
+                  "id": 8,
+                  "links": [ ],
+                  "options": {
+                     "legend": {
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "span": 4,
+                  "targets": [
+                     {
+                        "expr": "loki_boltdb_shipper_compact_tables_operation_duration_seconds{cluster=~\"$cluster\", namespace=~\"$namespace\"}",
+                        "format": "time_series",
+                        "legendFormat": "{{pod}}",
+                        "legendLink": null
+                     }
+                  ],
+                  "title": "Compaction run duration (seconds)",
+                  "type": "timeseries"
+               }
+            ],
+            "repeat": null,
+            "repeatIteration": null,
+            "repeatRowId": null,
+            "showTitle": true,
+            "title": "Compactor",
+            "titleSize": "h6"
+         },
+         {
+            "collapse": false,
+            "height": "250px",
+            "panels": [
+               {
+                  "datasource": "$datasource",
+                  "fieldConfig": {
+                     "defaults": {
+                        "custom": {
+                           "drawStyle": "line",
+                           "fillOpacity": 10,
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "showPoints": "never",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           }
+                        },
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [ ]
+                        },
+                        "unit": "short"
+                     },
+                     "overrides": [ ]
+                  },
+                  "id": 9,
+                  "links": [ ],
+                  "options": {
+                     "legend": {
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "span": 6,
+                  "targets": [
+                     {
+                        "expr": "sum(increase(loki_compactor_load_pending_requests_attempts_total{status=\"fail\", cluster=~\"$cluster\", namespace=~\"$namespace\"}[1h]))",
+                        "format": "time_series",
+                        "legendFormat": "failures",
+                        "legendLink": null
+                     }
+                  ],
+                  "title": "Failures in Loading Delete Requests / Hour",
+                  "type": "timeseries"
+               },
+               {
+                  "datasource": "$datasource",
+                  "fieldConfig": {
+                     "defaults": {
+                        "custom": {
+                           "drawStyle": "line",
+                           "fillOpacity": 10,
+                           "lineWidth": 1,
+                           "pointSize": 5,
+                           "showPoints": "never",
+                           "spanNulls": false,
+                           "stacking": {
+                              "group": "A",
+                              "mode": "none"
+                           }
+                        },
+                        "thresholds": {
+                           "mode": "absolute",
+                           "steps": [ ]
+                        },
+                        "unit": "short"
+                     },
+                     "overrides": [ ]
+                  },
+                  "id": 10,
+                  "links": [ ],
+                  "options": {
+                     "legend": {
+                        "showLegend": true
+                     },
+                     "tooltip": {
+                        "mode": "single",
+                        "sort": "none"
+                     }
+                  },
+                  "span": 6,
+                  "targets": [
+                     {
+                        "expr": "sum(rate(loki_compactor_deleted_lines{cluster=~\"$cluster\", namespace=~\"$namespace\", pod=~\"loki.*\"}[$__rate_interval])) by (user)",
+                        "format": "time_series",
+                        "legendFormat": "{{user}}",
+                        "legendLink": null
+                     }
+                  ],
+                  "title": "Lines Deleted / Sec",
+                  "type": "timeseries"
+               }
+            ],
+            "repeat": null,
+            "repeatIteration": null,
+            "repeatRowId": null,
+            "showTitle": true,
+            "title": "Deletion metrics",
+            "titleSize": "h6"
+         },
+         {
+            "collapse": false,
+            "height": "250px",
+            "panels": [
+               {
+                  "datasource": "$loki_datasource",
+                  "id": 11,
+                  "span": 6,
+                  "targets": [
+                     {
+                        "expr": "{cluster=~\"$cluster\", namespace=~\"$namespace\", pod=~\"loki.*\"} |~ \"Started processing delete request|delete request for user marked as processed\" | logfmt | line_format \"{{.ts}} user={{.user}} delete_request_id={{.delete_request_id}} msg={{.msg}}\" ",
+                        "refId": "A"
+                     }
+                  ],
+                  "title": "In progress/finished",
+                  "type": "logs"
+               },
+               {
+                  "datasource": "$loki_datasource",
+                  "id": 12,
+                  "span": 6,
+                  "targets": [
+                     {
+                        "expr": "{cluster=~\"$cluster\", namespace=~\"$namespace\", pod=~\"loki.*\"} |~ \"delete request for user added\" | logfmt | line_format \"{{.ts}} user={{.user}} query='{{.query}}'\"",
+                        "refId": "A"
+                     }
+                  ],
+                  "title": "Requests",
+                  "type": "logs"
+               }
+            ],
+            "repeat": null,
+            "repeatIteration": null,
+            "repeatRowId": null,
+            "showTitle": true,
+            "title": "List of deletion requests",
+            "titleSize": "h6"
+         }
+      ],
+      "schemaVersion": 14,
+      "style": "dark",
+      "tags": [
+         "loki"
+      ],
+      "templating": {
+         "list": [
+            {
+               "current": {
+                  "text": "default",
+                  "value": "default"
+               },
+               "hide": 0,
+               "label": "Data source",
+               "name": "datasource",
+               "options": [ ],
+               "query": "prometheus",
+               "refresh": 1,
+               "regex": "",
+               "type": "datasource"
+            },
+            {
+               "allValue": null,
+               "current": {
+                  "text": "prod",
+                  "value": "prod"
+               },
+               "datasource": "$datasource",
+               "hide": 0,
+               "includeAll": false,
+               "label": "cluster",
+               "multi": false,
+               "name": "cluster",
+               "options": [ ],
+               "query": "label_values(loki_build_info, cluster)",
+               "refresh": 1,
+               "regex": "",
+               "sort": 2,
+               "tagValuesQuery": "",
+               "tags": [ ],
+               "tagsQuery": "",
+               "type": "query",
+               "useTags": false
+            },
+            {
+               "allValue": null,
+               "current": {
+                  "text": "prod",
+                  "value": "prod"
+               },
+               "datasource": "$datasource",
+               "hide": 0,
+               "includeAll": false,
+               "label": "namespace",
+               "multi": false,
+               "name": "namespace",
+               "options": [ ],
+               "query": "label_values(loki_build_info{cluster=~\"$cluster\"}, namespace)",
+               "refresh": 1,
+               "regex": "",
+               "sort": 2,
+               "tagValuesQuery": "",
+               "tags": [ ],
+               "tagsQuery": "",
+               "type": "query",
+               "useTags": false
+            },
+            {
+               "hide": 0,
+               "label": null,
+               "name": "loki_datasource",
+               "options": [ ],
+               "query": "loki",
+               "refresh": 1,
+               "regex": "",
+               "type": "datasource"
+            }
+         ]
+      },
+      "time": {
+         "from": "now-1h",
+         "to": "now"
+      },
+      "timepicker": {
+         "refresh_intervals": [
+            "5s",
+            "10s",
+            "30s",
+            "1m",
+            "5m",
+            "15m",
+            "30m",
+            "1h",
+            "2h",
+            "1d"
+         ],
+         "time_options": [
+            "5m",
+            "15m",
+            "1h",
+            "6h",
+            "12h",
+            "24h",
+            "2d",
+            "7d",
+            "30d"
+         ]
+      },
+      "timezone": "utc",
+      "title": "Loki / Deletion",
+      "uid": "deletion",
+      "version": 0
+   }

modules/grafana/grafana-dashboards/loki/loki-mixin-recording-rules.json

@@ -0,0 +1,724 @@
+{
+      "annotations": {
+         "list": [ ]
+      },
+      "editable": true,
+      "fiscalYearStartMonth": 0,
+      "gnetId": null,
+      "graphTooltip": 0,
+      "hideControls": false,
+      "iteration": 1635347545534,
+      "links": [
+         {
+            "asDropdown": true,
+            "icon": "external link",
+            "includeVars": true,
+            "keepTime": true,
+            "tags": [
+               "loki"
+            ],
+            "targetBlank": false,
+            "title": "Loki Dashboards",
+            "type": "dashboards"
+         }
+      ],
+      "liveNow": false,
+      "panels": [
+         {
+            "datasource": "${datasource}",
+            "fieldConfig": {
+               "defaults": {
+                  "color": {
+                     "mode": "thresholds"
+                  },
+                  "mappings": [ ],
+                  "noValue": "0",
+                  "thresholds": {
+                     "mode": "absolute",
+                     "steps": [
+                        {
+                           "color": "green",
+                           "value": null
+                        },
+                        {
+                           "color": "red",
+                           "value": 1
+                        }
+                     ]
+                  }
+               },
+               "overrides": [ ]
+            },
+            "gridPos": {
+               "h": 10,
+               "w": 2,
+               "x": 0,
+               "y": 0
+            },
+            "id": 2,
+            "options": {
+               "colorMode": "value",
+               "graphMode": "area",
+               "justifyMode": "auto",
+               "orientation": "auto",
+               "reduceOptions": {
+                  "calcs": [
+                     "lastNotNull"
+                  ],
+                  "fields": "",
+                  "values": false
+               },
+               "textMode": "auto"
+            },
+            "pluginVersion": "8.3.0-38205pre",
+            "targets": [
+               {
+                  "datasource": "${datasource}",
+                  "exemplar": false,
+                  "expr": "sum(loki_ruler_wal_appender_ready) by (pod, tenant) == 0",
+                  "instant": true,
+                  "interval": "",
+                  "legendFormat": "",
+                  "refId": "A"
+               }
+            ],
+            "title": "Appenders Not Ready",
+            "type": "stat"
+         },
+         {
+            "datasource": "${datasource}",
+            "description": "",
+            "fieldConfig": {
+               "defaults": {
+                  "color": {
+                     "mode": "palette-classic"
+                  },
+                  "custom": {
+                     "axisLabel": "",
+                     "axisPlacement": "auto",
+                     "barAlignment": 0,
+                     "drawStyle": "line",
+                     "fillOpacity": 0,
+                     "gradientMode": "none",
+                     "hideFrom": {
+                        "legend": false,
+                        "tooltip": false,
+                        "viz": false
+                     },
+                     "lineInterpolation": "linear",
+                     "lineWidth": 1,
+                     "pointSize": 5,
+                     "scaleDistribution": {
+                        "type": "linear"
+                     },
+                     "showPoints": "auto",
+                     "spanNulls": false,
+                     "stacking": {
+                        "group": "A",
+                        "mode": "none"
+                     },
+                     "thresholdsStyle": {
+                        "mode": "off"
+                     }
+                  },
+                  "mappings": [ ],
+                  "thresholds": {
+                     "mode": "absolute",
+                     "steps": [
+                        {
+                           "color": "green",
+                           "value": null
+                        },
+                        {
+                           "color": "red",
+                           "value": 80
+                        }
+                     ]
+                  }
+               },
+               "overrides": [ ]
+            },
+            "gridPos": {
+               "h": 10,
+               "w": 11,
+               "x": 2,
+               "y": 0
+            },
+            "id": 4,
+            "options": {
+               "legend": {
+                  "calcs": [ ],
+                  "displayMode": "list",
+                  "placement": "bottom"
+               },
+               "tooltip": {
+                  "mode": "single"
+               }
+            },
+            "targets": [
+               {
+                  "datasource": "${datasource}",
+                  "exemplar": true,
+                  "expr": "sum(rate(loki_ruler_wal_samples_appended_total{tenant=~\"${tenant}\"}[$__rate_interval])) by (tenant) > 0",
+                  "interval": "",
+                  "legendFormat": "{{tenant}}",
+                  "refId": "A"
+               }
+            ],
+            "title": "Samples Appended to WAL per Second",
+            "type": "timeseries"
+         },
+         {
+            "datasource": "${datasource}",
+            "description": "Series are unique combinations of labels",
+            "fieldConfig": {
+               "defaults": {
+                  "color": {
+                     "mode": "palette-classic"
+                  },
+                  "custom": {
+                     "axisLabel": "",
+                     "axisPlacement": "auto",
+                     "barAlignment": 0,
+                     "drawStyle": "line",
+                     "fillOpacity": 0,
+                     "gradientMode": "none",
+                     "hideFrom": {
+                        "legend": false,
+                        "tooltip": false,
+                        "viz": false
+                     },
+                     "lineInterpolation": "linear",
+                     "lineWidth": 1,
+                     "pointSize": 5,
+                     "scaleDistribution": {
+                        "type": "linear"
+                     },
+                     "showPoints": "auto",
+                     "spanNulls": false,
+                     "stacking": {
+                        "group": "A",
+                        "mode": "none"
+                     },
+                     "thresholdsStyle": {
+                        "mode": "off"
+                     }
+                  },
+                  "mappings": [ ],
+                  "thresholds": {
+                     "mode": "absolute",
+                     "steps": [
+                        {
+                           "color": "green",
+                           "value": null
+                        },
+                        {
+                           "color": "red",
+                           "value": 80
+                        }
+                     ]
+                  }
+               },
+               "overrides": [ ]
+            },
+            "gridPos": {
+               "h": 10,
+               "w": 11,
+               "x": 13,
+               "y": 0
+            },
+            "id": 5,
+            "options": {
+               "legend": {
+                  "calcs": [ ],
+                  "displayMode": "list",
+                  "placement": "bottom"
+               },
+               "tooltip": {
+                  "mode": "single"
+               }
+            },
+            "targets": [
+               {
+                  "datasource": "${datasource}",
+                  "exemplar": true,
+                  "expr": "sum(rate(loki_ruler_wal_storage_created_series_total{tenant=~\"${tenant}\"}[$__rate_interval])) by (tenant) > 0",
+                  "interval": "",
+                  "legendFormat": "{{tenant}}",
+                  "refId": "A"
+               }
+            ],
+            "title": "Series Created per Second",
+            "type": "timeseries"
+         },
+         {
+            "datasource": "${datasource}",
+            "description": "Difference between highest timestamp appended to WAL and highest timestamp successfully written to remote storage",
+            "fieldConfig": {
+               "defaults": {
+                  "color": {
+                     "mode": "palette-classic"
+                  },
+                  "custom": {
+                     "axisLabel": "",
+                     "axisPlacement": "auto",
+                     "barAlignment": 0,
+                     "drawStyle": "line",
+                     "fillOpacity": 0,
+                     "gradientMode": "none",
+                     "hideFrom": {
+                        "legend": false,
+                        "tooltip": false,
+                        "viz": false
+                     },
+                     "lineInterpolation": "linear",
+                     "lineWidth": 1,
+                     "pointSize": 5,
+                     "scaleDistribution": {
+                        "type": "linear"
+                     },
+                     "showPoints": "auto",
+                     "spanNulls": false,
+                     "stacking": {
+                        "group": "A",
+                        "mode": "none"
+                     },
+                     "thresholdsStyle": {
+                        "mode": "off"
+                     }
+                  },
+                  "mappings": [ ],
+                  "thresholds": {
+                     "mode": "absolute",
+                     "steps": [
+                        {
+                           "color": "green",
+                           "value": null
+                        },
+                        {
+                           "color": "red",
+                           "value": 80
+                        }
+                     ]
+                  },
+                  "unit": "s"
+               },
+               "overrides": [ ]
+            },
+            "gridPos": {
+               "h": 10,
+               "w": 12,
+               "x": 0,
+               "y": 10
+            },
+            "id": 6,
+            "options": {
+               "legend": {
+                  "calcs": [ ],
+                  "displayMode": "list",
+                  "placement": "bottom"
+               },
+               "tooltip": {
+                  "mode": "single"
+               }
+            },
+            "targets": [
+               {
+                  "datasource": "${datasource}",
+                  "exemplar": true,
+                  "expr": "loki_ruler_wal_prometheus_remote_storage_highest_timestamp_in_seconds{tenant=~\"${tenant}\"}\n- on (tenant)\n  (\n    loki_ruler_wal_prometheus_remote_storage_queue_highest_sent_timestamp_seconds{tenant=~\"${tenant}\"}\n    or vector(0)\n  )",
+                  "interval": "",
+                  "legendFormat": "{{tenant}}",
+                  "refId": "A"
+               }
+            ],
+            "title": "Write Behind",
+            "type": "timeseries"
+         },
+         {
+            "datasource": "${datasource}",
+            "description": "",
+            "fieldConfig": {
+               "defaults": {
+                  "color": {
+                     "mode": "palette-classic"
+                  },
+                  "custom": {
+                     "axisLabel": "",
+                     "axisPlacement": "auto",
+                     "barAlignment": 0,
+                     "drawStyle": "line",
+                     "fillOpacity": 0,
+                     "gradientMode": "none",
+                     "hideFrom": {
+                        "legend": false,
+                        "tooltip": false,
+                        "viz": false
+                     },
+                     "lineInterpolation": "linear",
+                     "lineWidth": 1,
+                     "pointSize": 5,
+                     "scaleDistribution": {
+                        "type": "linear"
+                     },
+                     "showPoints": "auto",
+                     "spanNulls": false,
+                     "stacking": {
+                        "group": "A",
+                        "mode": "none"
+                     },
+                     "thresholdsStyle": {
+                        "mode": "off"
+                     }
+                  },
+                  "mappings": [ ],
+                  "thresholds": {
+                     "mode": "absolute",
+                     "steps": [
+                        {
+                           "color": "green",
+                           "value": null
+                        },
+                        {
+                           "color": "red",
+                           "value": 80
+                        }
+                     ]
+                  }
+               },
+               "overrides": [ ]
+            },
+            "gridPos": {
+               "h": 10,
+               "w": 12,
+               "x": 12,
+               "y": 10
+            },
+            "id": 7,
+            "options": {
+               "legend": {
+                  "calcs": [ ],
+                  "displayMode": "list",
+                  "placement": "bottom"
+               },
+               "tooltip": {
+                  "mode": "single"
+               }
+            },
+            "targets": [
+               {
+                  "datasource": "${datasource}",
+                  "exemplar": true,
+                  "expr": "sum(rate(loki_ruler_wal_prometheus_remote_storage_samples_total{tenant=~\"${tenant}\"}[$__rate_interval])) by (tenant) > 0",
+                  "interval": "",
+                  "legendFormat": "{{tenant}}",
+                  "refId": "A"
+               }
+            ],
+            "title": "Samples Sent per Second",
+            "type": "timeseries"
+         },
+         {
+            "datasource": "${datasource}",
+            "description": "\n",
+            "fieldConfig": {
+               "defaults": {
+                  "color": {
+                     "mode": "palette-classic"
+                  },
+                  "custom": {
+                     "axisLabel": "",
+                     "axisPlacement": "auto",
+                     "barAlignment": 0,
+                     "drawStyle": "line",
+                     "fillOpacity": 0,
+                     "gradientMode": "none",
+                     "hideFrom": {
+                        "legend": false,
+                        "tooltip": false,
+                        "viz": false
+                     },
+                     "lineInterpolation": "linear",
+                     "lineWidth": 1,
+                     "pointSize": 5,
+                     "scaleDistribution": {
+                        "type": "linear"
+                     },
+                     "showPoints": "auto",
+                     "spanNulls": false,
+                     "stacking": {
+                        "group": "A",
+                        "mode": "none"
+                     },
+                     "thresholdsStyle": {
+                        "mode": "off"
+                     }
+                  },
+                  "mappings": [ ],
+                  "thresholds": {
+                     "mode": "absolute",
+                     "steps": [
+                        {
+                           "color": "green",
+                           "value": null
+                        },
+                        {
+                           "color": "red",
+                           "value": 80
+                        }
+                     ]
+                  },
+                  "unit": "bytes"
+               },
+               "overrides": [ ]
+            },
+            "gridPos": {
+               "h": 10,
+               "w": 12,
+               "x": 0,
+               "y": 20
+            },
+            "id": 8,
+            "options": {
+               "legend": {
+                  "calcs": [ ],
+                  "displayMode": "list",
+                  "placement": "bottom"
+               },
+               "tooltip": {
+                  "mode": "single"
+               }
+            },
+            "targets": [
+               {
+                  "datasource": "${datasource}",
+                  "exemplar": true,
+                  "expr": "sum by (tenant) (loki_ruler_wal_disk_size{tenant=~\"${tenant}\"})",
+                  "interval": "",
+                  "legendFormat": "{{tenant}}",
+                  "refId": "A"
+               }
+            ],
+            "title": "WAL Disk Size",
+            "type": "timeseries"
+         },
+         {
+            "datasource": "${datasource}",
+            "description": "Some number of pending samples is expected, but if remote-write is failing this value will remain high",
+            "fieldConfig": {
+               "defaults": {
+                  "color": {
+                     "mode": "palette-classic"
+                  },
+                  "custom": {
+                     "axisLabel": "",
+                     "axisPlacement": "auto",
+                     "barAlignment": 0,
+                     "drawStyle": "line",
+                     "fillOpacity": 0,
+                     "gradientMode": "none",
+                     "hideFrom": {
+                        "legend": false,
+                        "tooltip": false,
+                        "viz": false
+                     },
+                     "lineInterpolation": "linear",
+                     "lineWidth": 1,
+                     "pointSize": 5,
+                     "scaleDistribution": {
+                        "type": "linear"
+                     },
+                     "showPoints": "auto",
+                     "spanNulls": false,
+                     "stacking": {
+                        "group": "A",
+                        "mode": "none"
+                     },
+                     "thresholdsStyle": {
+                        "mode": "off"
+                     }
+                  },
+                  "mappings": [ ],
+                  "thresholds": {
+                     "mode": "absolute",
+                     "steps": [
+                        {
+                           "color": "green",
+                           "value": null
+                        },
+                        {
+                           "color": "red",
+                           "value": 80
+                        }
+                     ]
+                  }
+               },
+               "overrides": [ ]
+            },
+            "gridPos": {
+               "h": 10,
+               "w": 12,
+               "x": 12,
+               "y": 20
+            },
+            "id": 9,
+            "options": {
+               "legend": {
+                  "calcs": [ ],
+                  "displayMode": "list",
+                  "placement": "bottom"
+               },
+               "tooltip": {
+                  "mode": "single"
+               }
+            },
+            "targets": [
+               {
+                  "datasource": "${datasource}",
+                  "exemplar": true,
+                  "expr": "max(loki_ruler_wal_prometheus_remote_storage_samples_pending{tenant=~\"${tenant}\"}) by (tenant,pod) > 0",
+                  "interval": "",
+                  "legendFormat": "{{tenant}}",
+                  "refId": "A"
+               }
+            ],
+            "title": "Pending Samples",
+            "type": "timeseries"
+         }
+      ],
+      "refresh": "10s",
+      "rows": [ ],
+      "schemaVersion": 14,
+      "style": "dark",
+      "tags": [
+         "loki"
+      ],
+      "templating": {
+         "list": [
+            {
+               "current": {
+                  "text": "default",
+                  "value": "default"
+               },
+               "hide": 0,
+               "label": "Data source",
+               "name": "datasource",
+               "options": [ ],
+               "query": "prometheus",
+               "refresh": 1,
+               "regex": "",
+               "type": "datasource"
+            },
+            {
+               "allValue": null,
+               "current": {
+                  "text": "prod",
+                  "value": "prod"
+               },
+               "datasource": "$datasource",
+               "hide": 0,
+               "includeAll": false,
+               "label": "cluster",
+               "multi": false,
+               "name": "cluster",
+               "options": [ ],
+               "query": "label_values(loki_build_info, cluster)",
+               "refresh": 1,
+               "regex": "",
+               "sort": 2,
+               "tagValuesQuery": "",
+               "tags": [ ],
+               "tagsQuery": "",
+               "type": "query",
+               "useTags": false
+            },
+            {
+               "allValue": null,
+               "current": {
+                  "text": "prod",
+                  "value": "prod"
+               },
+               "datasource": "$datasource",
+               "hide": 0,
+               "includeAll": false,
+               "label": "namespace",
+               "multi": false,
+               "name": "namespace",
+               "options": [ ],
+               "query": "label_values(loki_build_info{cluster=~\"$cluster\"}, namespace)",
+               "refresh": 1,
+               "regex": "",
+               "sort": 2,
+               "tagValuesQuery": "",
+               "tags": [ ],
+               "tagsQuery": "",
+               "type": "query",
+               "useTags": false
+            },
+            {
+               "hide": 0,
+               "label": null,
+               "name": "loki_datasource",
+               "options": [ ],
+               "query": "loki",
+               "refresh": 1,
+               "regex": "",
+               "type": "datasource"
+            },
+            {
+               "allValue": ".+",
+               "current": { },
+               "datasource": "$datasource",
+               "hide": 0,
+               "includeAll": true,
+               "label": null,
+               "multi": false,
+               "name": "tenant",
+               "options": [ ],
+               "query": "query_result(sum by (id) (grafanacloud_logs_instance_info) and sum(label_replace(loki_tenant:active_streams{cluster=\"$cluster\",namespace=\"$namespace\"},\"id\",\"$1\",\"tenant\",\"(.*)\")) by(id))",
+               "refresh": 0,
+               "regex": "/\"([^\"]+)\"/",
+               "sort": 1,
+               "tagValuesQuery": "",
+               "tags": [ ],
+               "tagsQuery": "",
+               "type": "query",
+               "useTags": false
+            }
+         ]
+      },
+      "time": {
+         "from": "now-1h",
+         "to": "now"
+      },
+      "timepicker": {
+         "refresh_intervals": [
+            "5s",
+            "10s",
+            "30s",
+            "1m",
+            "5m",
+            "15m",
+            "30m",
+            "1h",
+            "2h",
+            "1d"
+         ],
+         "time_options": [
+            "5m",
+            "15m",
+            "1h",
+            "6h",
+            "12h",
+            "24h",
+            "2d",
+            "7d",
+            "30d"
+         ]
+      },
+      "timezone": "utc",
+      "title": "Loki / Recording Rules",
+      "uid": "recording-rules",
+      "version": 0,
+      "weekStart": ""
+   }

modules/grafana/grafana-dashboards/nginx.json

@@ -0,0 +1,567 @@
+{
+  "__inputs": [
+    {
+      "description": "",
+      "label": "Prometheus",
+      "name": "DS_PROMETHEUS",
+      "pluginId": "prometheus",
+      "pluginName": "Prometheus",
+      "type": "datasource"
+    }
+  ],
+  "__requires": [
+    {
+      "id": "grafana",
+      "name": "Grafana",
+      "type": "grafana",
+      "version": "5.0.0"
+    },
+    {
+      "id": "graph",
+      "name": "Graph",
+      "type": "panel",
+      "version": ""
+    },
+    {
+      "id": "prometheus",
+      "name": "Prometheus",
+      "type": "datasource",
+      "version": "1.0.0"
+    },
+    {
+      "id": "singlestat",
+      "name": "Singlestat",
+      "type": "panel",
+      "version": ""
+    }
+  ],
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": "-- Grafana --",
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "description": "Official dashboard for NGINX Prometheus exporter",
+  "editable": true,
+  "gnetId": null,
+  "graphTooltip": 0,
+  "id": null,
+  "iteration": 1562682051068,
+  "links": [],
+  "panels": [
+    {
+      "collapsed": false,
+      "datasource": "${DS_PROMETHEUS}",
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 0
+      },
+      "id": 4,
+      "panels": [],
+      "title": "Status",
+      "type": "row"
+    },
+    {
+      "cacheTimeout": null,
+      "colorBackground": true,
+      "colorPostfix": false,
+      "colorPrefix": false,
+      "colorValue": false,
+      "colors": [
+        "#E02F44",
+        "#FF9830",
+        "#299c46"
+      ],
+      "datasource": "${DS_PROMETHEUS}",
+      "decimals": null,
+      "description": "",
+      "format": "none",
+      "gauge": {
+        "maxValue": 100,
+        "minValue": 0,
+        "show": false,
+        "thresholdLabels": false,
+        "thresholdMarkers": true
+      },
+      "gridPos": {
+        "h": 3,
+        "w": 12,
+        "x": 0,
+        "y": 1
+      },
+      "id": 8,
+      "interval": null,
+      "links": [],
+      "mappingType": 1,
+      "mappingTypes": [
+        {
+          "name": "value to text",
+          "value": 1
+        },
+        {
+          "name": "range to text",
+          "value": 2
+        }
+      ],
+      "maxDataPoints": 100,
+      "nullPointMode": "connected",
+      "nullText": null,
+      "options": {},
+      "postfix": "",
+      "postfixFontSize": "50%",
+      "prefix": "",
+      "prefixFontSize": "50%",
+      "rangeMaps": [
+        {
+          "from": "null",
+          "text": "N/A",
+          "to": "null"
+        }
+      ],
+      "repeat": "instance",
+      "repeatDirection": "h",
+      "sparkline": {
+        "fillColor": "rgba(31, 118, 189, 0.18)",
+        "full": false,
+        "lineColor": "rgb(31, 120, 193)",
+        "show": false
+      },
+      "tableColumn": "",
+      "targets": [
+        {
+          "expr": "nginx_up{instance=~\"$instance\"}",
+          "format": "time_series",
+          "instant": false,
+          "intervalFactor": 1,
+          "refId": "A"
+        }
+      ],
+      "thresholds": "1,1",
+      "timeFrom": null,
+      "timeShift": null,
+      "title": "NGINX Status for $instance",
+      "type": "singlestat",
+      "valueFontSize": "100%",
+      "valueMaps": [
+        {
+          "op": "=",
+          "text": "Down",
+          "value": "0"
+        },
+        {
+          "op": "=",
+          "text": "Up",
+          "value": "1"
+        }
+      ],
+      "valueName": "current"
+    },
+    {
+      "collapsed": false,
+      "datasource": "${DS_PROMETHEUS}",
+      "gridPos": {
+        "h": 1,
+        "w": 24,
+        "x": 0,
+        "y": 4
+      },
+      "id": 6,
+      "panels": [],
+      "title": "Metrics",
+      "type": "row"
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "${DS_PROMETHEUS}",
+      "decimals": null,
+      "description": "",
+      "fill": 1,
+      "gridPos": {
+        "h": 10,
+        "w": 12,
+        "x": 0,
+        "y": 5
+      },
+      "id": 10,
+      "legend": {
+        "alignAsTable": false,
+        "avg": false,
+        "current": false,
+        "hideEmpty": false,
+        "max": false,
+        "min": false,
+        "rightSide": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {},
+      "percentage": false,
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "irate(nginx_connections_accepted{instance=~\"$instance\"}[5m])",
+          "format": "time_series",
+          "instant": false,
+          "intervalFactor": 1,
+          "legendFormat": "{{instance}} accepted",
+          "refId": "A"
+        },
+        {
+          "expr": "irate(nginx_connections_handled{instance=~\"$instance\"}[5m])",
+          "format": "time_series",
+          "instant": false,
+          "intervalFactor": 1,
+          "legendFormat": "{{instance}} handled",
+          "refId": "B"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "Processed connections",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "decimals": 1,
+          "format": "short",
+          "label": "Connections (rate)",
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": "",
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "${DS_PROMETHEUS}",
+      "decimals": 0,
+      "fill": 1,
+      "gridPos": {
+        "h": 10,
+        "w": 12,
+        "x": 12,
+        "y": 5
+      },
+      "id": 12,
+      "legend": {
+        "alignAsTable": false,
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "rightSide": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {},
+      "percentage": false,
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "nginx_connections_active{instance=~\"$instance\"}",
+          "format": "time_series",
+          "intervalFactor": 1,
+          "legendFormat": "{{instance}} active",
+          "refId": "A"
+        },
+        {
+          "expr": "nginx_connections_reading{instance=~\"$instance\"}",
+          "format": "time_series",
+          "intervalFactor": 1,
+          "legendFormat": "{{instance}} reading",
+          "refId": "B"
+        },
+        {
+          "expr": "nginx_connections_waiting{instance=~\"$instance\"}",
+          "format": "time_series",
+          "intervalFactor": 1,
+          "legendFormat": "{{instance}} waiting",
+          "refId": "C"
+        },
+        {
+          "expr": "nginx_connections_writing{instance=~\"$instance\"}",
+          "format": "time_series",
+          "intervalFactor": 1,
+          "legendFormat": "{{instance}} writing",
+          "refId": "D"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "Active Connections",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "decimals": 0,
+          "format": "short",
+          "label": "Connections",
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    },
+    {
+      "aliasColors": {},
+      "bars": false,
+      "dashLength": 10,
+      "dashes": false,
+      "datasource": "${DS_PROMETHEUS}",
+      "fill": 1,
+      "gridPos": {
+        "h": 8,
+        "w": 24,
+        "x": 0,
+        "y": 15
+      },
+      "id": 15,
+      "legend": {
+        "avg": false,
+        "current": false,
+        "max": false,
+        "min": false,
+        "show": true,
+        "total": false,
+        "values": false
+      },
+      "lines": true,
+      "linewidth": 1,
+      "links": [],
+      "nullPointMode": "null",
+      "options": {},
+      "percentage": false,
+      "pointradius": 2,
+      "points": false,
+      "renderer": "flot",
+      "seriesOverrides": [],
+      "spaceLength": 10,
+      "stack": false,
+      "steppedLine": false,
+      "targets": [
+        {
+          "expr": "irate(nginx_http_requests_total{instance=~\"$instance\"}[5m])",
+          "format": "time_series",
+          "intervalFactor": 1,
+          "legendFormat": "{{instance}} total requests",
+          "refId": "A"
+        }
+      ],
+      "thresholds": [],
+      "timeFrom": null,
+      "timeRegions": [],
+      "timeShift": null,
+      "title": "Total requests",
+      "tooltip": {
+        "shared": true,
+        "sort": 0,
+        "value_type": "individual"
+      },
+      "type": "graph",
+      "xaxis": {
+        "buckets": null,
+        "mode": "time",
+        "name": null,
+        "show": true,
+        "values": []
+      },
+      "yaxes": [
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        },
+        {
+          "format": "short",
+          "label": null,
+          "logBase": 1,
+          "max": null,
+          "min": null,
+          "show": true
+        }
+      ],
+      "yaxis": {
+        "align": false,
+        "alignLevel": null
+      }
+    }
+  ],
+  "refresh": "5s",
+  "schemaVersion": 18,
+  "style": "dark",
+  "tags": [
+    "nginx",
+    "prometheus",
+    "nginx prometheus exporter"
+  ],
+  "templating": {
+    "list": [
+      {
+        "current": {
+          "selected": false,
+          "tags": [],
+          "text": "default",
+          "value": "default"
+        },
+        "hide": 0,
+        "includeAll": false,
+        "label": "datasource",
+        "multi": false,
+        "name": "DS_PROMETHEUS",
+        "options": [],
+        "query": "prometheus",
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "type": "datasource"
+      },
+      {
+        "allValue": null,
+        "current": {},
+        "datasource": "${DS_PROMETHEUS}",
+        "definition": "label_values(nginx_up, instance)",
+        "hide": 0,
+        "includeAll": true,
+        "label": "",
+        "multi": true,
+        "name": "instance",
+        "options": [],
+        "query": "label_values(nginx_up, instance)",
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 0,
+        "tagValuesQuery": "",
+        "tags": [],
+        "tagsQuery": "",
+        "type": "query",
+        "useTags": false
+      }
+    ]
+  },
+  "time": {
+    "from": "now-15m",
+    "to": "now"
+  },
+  "timepicker": {
+    "refresh_intervals": [
+      "5s",
+      "10s",
+      "30s",
+      "1m",
+      "5m",
+      "15m",
+      "30m",
+      "1h",
+      "2h",
+      "1d"
+    ],
+    "time_options": [
+      "5m",
+      "15m",
+      "1h",
+      "6h",
+      "12h",
+      "24h",
+      "2d",
+      "7d",
+      "30d"
+    ]
+  },
+  "timezone": "",
+  "title": "NGINX",
+  "uid": "MsjffzSZz",
+  "version": 1
+}

modules/grafana/grafana-dashboards/synapse.json

@@ -53,7 +53,7 @@
           "uid": "${DS_PROMETHEUS}"
         },
         "enable": true,
-        "expr": "changes(process_start_time_seconds{instance=\"$instance\",job=~\"synapse\"}[$bucket_size]) * on (instance, job) group_left(version) synapse_build_info{instance=\"$instance\",job=\"synapse\"}",
+        "expr": "changes(process_start_time_seconds{instance=\"$instance\",job=~\"$job\"}[$bucket_size]) * on (instance, job) group_left(version) synapse_build_info{instance=\"$instance\",job=\"$job\"}",
         "iconColor": "purple",
         "name": "deploys",
         "titleFormat": "Deployed {{version}}"
@@ -749,7 +749,7 @@
             "uid": "${DS_PROMETHEUS}"
           },
           "editorMode": "code",
-          "expr": "synapse_build_info{instance=\"$instance\", job=\"synapse\"} - 1",
+          "expr": "synapse_build_info{instance=\"$instance\", job=\"$job\"} - 1",
           "legendFormat": "version {{version}}",
           "range": true,
           "refId": "deployed_synapse_versions"
@@ -1491,7 +1491,7 @@
                 "uid": "${DS_PROMETHEUS}"
               },
               "editorMode": "code",
-              "expr": "synapse_build_info{instance=\"$instance\", job=\"synapse\"} - 1",
+              "expr": "synapse_build_info{instance=\"$instance\", job=\"$job\"} - 1",
               "hide": false,
               "legendFormat": "version {{version}}",
               "range": true,

modules/keycloak/default.nix

@@ -6,23 +6,22 @@
   ...
 }:
 {
-  options.pub-solar-os.auth = {
-    enable = lib.mkEnableOption "Enable keycloak to run on the node";
+  options.pub-solar-os.auth = with lib; {
+    enable = mkEnableOption "Enable keycloak to run on the node";
 
-    realm = lib.mkOption {
+    realm = mkOption {
       description = "Name of the realm";
-      type = lib.types.str;
+      type = types.str;
       default = config.pub-solar-os.networking.domain;
     };
+
+    database-password-file = mkOption {
+      description = "Database password file path";
+      type = types.str;
+    };
   };
 
   config = lib.mkIf config.pub-solar-os.auth.enable {
-    age.secrets.keycloak-database-password = {
-      file = "${flake.self}/secrets/keycloak-database-password.age";
-      mode = "600";
-      #owner = "keycloak";
-    };
-
     services.nginx.virtualHosts."auth.${config.pub-solar-os.networking.domain}" = {
       enableACME = true;
       forceSSL = true;
@@ -46,12 +45,13 @@
     # keycloak
     services.keycloak = {
       enable = true;
-      database.passwordFile = config.age.secrets.keycloak-database-password.path;
+      database.passwordFile = config.pub-solar-os.auth.database-password-file;
       settings = {
         hostname = "auth.${config.pub-solar-os.networking.domain}";
         http-host = "127.0.0.1";
         http-port = 8080;
-        proxy = "edge";
+        proxy-headers = "xforwarded";
+        http-enabled = true;
       };
       themes = {
         "pub.solar" =
@@ -59,25 +59,18 @@
       };
     };
 
-    services.restic.backups.keycloak-storagebox = {
-      paths = [ "/tmp/keycloak-backup.sql" ];
-      timerConfig = {
-        OnCalendar = "*-*-* 03:00:00 Etc/UTC";
+    pub-solar-os.backups = {
+      resources.keycloak-db.resourceCreateCommand = ''
+        ${pkgs.sudo}/bin/sudo -u postgres ${pkgs.postgresql}/bin/pg_dump -d keycloak -f /tmp/keycloak-backup.sql
+      '';
+      restic.keycloak = {
+        resources = [ "keycloak-db" ];
+        paths = [ "/tmp/keycloak-backup.sql" ];
+        timerConfig = {
+          OnCalendar = "*-*-* 03:00:00 Etc/UTC";
+        };
+        initialize = true;
       };
-      initialize = true;
-      passwordFile = config.age.secrets."restic-repo-storagebox".path;
-      repository = "sftp:u377325@u377325.your-storagebox.de:/backups";
-      backupPrepareCommand = ''
-        ${pkgs.sudo}/bin/sudo -u postgres ${pkgs.postgresql}/bin/pg_dump -d keycloak > /tmp/keycloak-backup.sql
-      '';
-      backupCleanupCommand = ''
-        rm /tmp/keycloak-backup.sql
-      '';
-      pruneOpts = [
-        "--keep-daily 7"
-        "--keep-weekly 4"
-        "--keep-monthly 3"
-      ];
     };
   };
 }

modules/loki/default.nix

@@ -25,7 +25,7 @@
           };
         };
         replication_factor = 1;
-        path_prefix = "/data/loki";
+        path_prefix = "/var/lib/loki";
         storage = {
           filesystem = {
             chunks_directory = "chunks/";
@@ -35,8 +35,12 @@
       };
       ingester = {
         chunk_encoding = "snappy";
-        chunk_idle_period = "1h";
+        chunk_idle_period = "8h";
+        max_chunk_age = "8h";
       };
+      pattern_ingester.enabled = true;
+      # 2x CPU cores
+      querier.max_concurrent = 16;
       query_range = {
         results_cache = {
           cache = {
@@ -56,11 +60,16 @@
           };
         };
       };
-      # Keep logs for 4 weeks
+      # Keep logs for 1 week
       # https://grafana.com/docs/loki/latest/operations/storage/retention/
       limits_config = {
-        retention_period = "4w";
-        split_queries_by_interval = "0";
+        allow_structured_metadata = true;
+        ingestion_rate_mb = 8;
+        ingestion_burst_size_mb = 12;
+        retention_period = "1w";
+        split_queries_by_interval = "1h";
+        tsdb_max_query_parallelism = 32;
+        volume_enabled = true;
       };
       compactor = {
         compaction_interval = "10m";
@@ -95,40 +104,4 @@
       };
     };
   };
-
-  services.promtail = {
-    enable = true;
-    configuration = {
-      server = {
-        http_listen_port = 9080;
-        grpc_listen_port = 0;
-      };
-      positions = {
-        filename = "/tmp/positions.yaml";
-      };
-      clients = [
-        {
-          url = "http://flora-6.wg.pub.solar:${toString config.services.loki.configuration.server.http_listen_port}/loki/api/v1/push";
-        }
-      ];
-      scrape_configs = [
-        {
-          job_name = "journal";
-          journal = {
-            max_age = "24h";
-            labels = {
-              job = "systemd-journal";
-              host = "flora-6";
-            };
-          };
-          relabel_configs = [
-            {
-              source_labels = [ "__journal__systemd_unit" ];
-              target_label = "unit";
-            }
-          ];
-        }
-      ];
-    };
-  };
 }

modules/mail/default.nix

@@ -67,4 +67,20 @@
   };
   security.acme.acceptTerms = true;
   security.acme.defaults.email = "security@pub.solar";
+
+  pub-solar-os.backups.restic.mail = {
+    paths = [
+      "/var/vmail"
+      "/var/dkim"
+    ];
+    timerConfig = {
+      OnCalendar = "*-*-* 02:00:00 Etc/UTC";
+    };
+    initialize = true;
+    pruneOpts = [
+      "--keep-daily 7"
+      "--keep-weekly 4"
+      "--keep-monthly 3"
+    ];
+  };
 }

modules/mailman/default.nix

@@ -91,12 +91,7 @@
       OnCalendar = "*-*-* 02:00:00 Etc/UTC";
     };
     initialize = true;
-    passwordFile = config.age.secrets."restic-repo-storagebox".path;
+    passwordFile = config.age.secrets."restic-repo-storagebox-nachtigall".path;
     repository = "sftp:u377325@u377325.your-storagebox.de:/backups";
-    pruneOpts = [
-      "--keep-daily 7"
-      "--keep-weekly 4"
-      "--keep-monthly 3"
-    ];
   };
 }

modules/mastodon/default.nix

@@ -7,6 +7,21 @@
 }:
 
 {
+  age.secrets."mastodon-active-record-encryption-deterministic-key" = {
+    file = "${flake.self}/secrets//mastodon-active-record-encryption-deterministic-key.age";
+    mode = "400";
+    owner = config.services.mastodon.user;
+  };
+  age.secrets."mastodon-active-record-encryption-key-derivation-salt" = {
+    file = "${flake.self}/secrets//mastodon-active-record-encryption-key-derivation-salt.age";
+    mode = "400";
+    owner = config.services.mastodon.user;
+  };
+  age.secrets."mastodon-active-record-encryption-primary-key" = {
+    file = "${flake.self}/secrets//mastodon-active-record-encryption-primary-key.age";
+    mode = "400";
+    owner = config.services.mastodon.user;
+  };
   age.secrets."mastodon-secret-key-base" = {
     file = "${flake.self}/secrets/mastodon-secret-key-base.age";
     mode = "400";
@@ -54,6 +69,9 @@
     webProcesses = 2;
     # Threads per process used by the mastodon-web service
     webThreads = 5;
+    activeRecordEncryptionDeterministicKeyFile = "/run/agenix/mastodon-active-record-encryption-deterministic-key";
+    activeRecordEncryptionKeyDerivationSaltFile = "/run/agenix/mastodon-active-record-encryption-key-derivation-salt";
+    activeRecordEncryptionPrimaryKeyFile = "/run/agenix/mastodon-active-record-encryption-primary-key";
     secretKeyBaseFile = "/run/agenix/mastodon-secret-key-base";
     otpSecretFile = "/run/agenix/mastodon-otp-secret";
     vapidPrivateKeyFile = "/run/agenix/mastodon-vapid-private-key";
@@ -67,20 +85,20 @@
       passwordFile = "/run/agenix/mastodon-smtp-password";
       fromAddress = "mastodon-notifications@pub.solar";
     };
+    # Defined in ./opensearch.nix
+    elasticsearch.host = "127.0.0.1";
     mediaAutoRemove = {
       olderThanDays = 7;
     };
     extraEnvFiles = [ "/run/agenix/mastodon-extra-env-secrets" ];
     extraConfig = {
       WEB_DOMAIN = "mastodon.${config.pub-solar-os.networking.domain}";
-      # Defined in ./opensearch.nix
-      ES_HOST = "127.0.0.1";
       # S3 File storage (optional)
       # -----------------------
       S3_ENABLED = "true";
-      S3_BUCKET = "pub-solar-mastodon";
-      S3_REGION = "europe-west-1";
-      S3_ENDPOINT = "https://gateway.tardigradeshare.io";
+      S3_BUCKET = "mastodon";
+      S3_REGION = "eu-central";
+      S3_ENDPOINT = "https://buckets.pub.solar";
       S3_ALIAS_HOST = "files.${config.pub-solar-os.networking.domain}";
       # Translation (optional)
       # -----------------------
@@ -106,7 +124,7 @@
       OnCalendar = "*-*-* 04:00:00 Etc/UTC";
     };
     initialize = true;
-    passwordFile = config.age.secrets."restic-repo-storagebox".path;
+    passwordFile = config.age.secrets."restic-repo-storagebox-nachtigall".path;
     repository = "sftp:u377325@u377325.your-storagebox.de:/backups";
     backupPrepareCommand = ''
       ${pkgs.sudo}/bin/sudo -u postgres ${pkgs.postgresql}/bin/pg_dump -d mastodon > /tmp/mastodon-backup.sql
@@ -114,10 +132,5 @@
     backupCleanupCommand = ''
       rm /tmp/mastodon-backup.sql
     '';
-    pruneOpts = [
-      "--keep-daily 7"
-      "--keep-weekly 4"
-      "--keep-monthly 3"
-    ];
   };
 }

modules/matrix-draupnir/default.nix

@@ -0,0 +1,51 @@
+{
+  config,
+  flake,
+  lib,
+  ...
+}:
+{
+  imports = [ "${flake.inputs.nixpkgs-draupnir}/nixos/modules/services/matrix/draupnir.nix" ];
+
+  disabledModules = [ "services/matrix/draupnir.nix" ];
+
+  options.pub-solar-os.matrix.draupnir = with lib; {
+    enable = mkEnableOption "Enable Matrix draupnir moderation bot";
+
+    homeserver-url = mkOption {
+      description = "Matrix homeserver URL";
+      type = types.str;
+      example = "http://127.0.0.1:8008";
+    };
+
+    access-token-file = mkOption {
+      description = "Path to access token file";
+      type = types.str;
+    };
+  };
+
+  config = lib.mkIf config.pub-solar-os.matrix.draupnir.enable {
+
+    services.draupnir = {
+      enable = true;
+      accessTokenFile = config.pub-solar-os.matrix.draupnir.access-token-file;
+      # https://github.com/the-draupnir-project/Draupnir/blob/main/config/default.yaml
+      homeserverUrl = config.pub-solar-os.matrix.draupnir.homeserver-url;
+      settings = {
+        managementRoom = "#matrix-moderators:${config.pub-solar-os.networking.domain}";
+        protectAllJoinedRooms = true;
+        recordIgnoredInvites = true;
+        automaticallyRedactForReasons = [
+          "*spam"
+          "advertising"
+        ];
+        web = {
+          enabled = true;
+          port = 8080;
+          address = "127.0.200.101";
+          abuseReporting.enabled = true;
+        };
+      };
+    };
+  };
+}

modules/matrix-irc/default.nix

@@ -13,118 +13,132 @@ let
     lib.findFirst (listener: builtins.any resourceHasClient listener.resources)
       (throw "Found no matrix-synapse.settings.listeners.*.resources.*.names containing string client")
       config.services.matrix-synapse.settings.listeners;
+  synapseIp = builtins.elemAt listenerWithClient.bind_addresses 0;
   synapseClientPort = "${toString listenerWithClient.port}";
 in
 {
-  systemd.services.matrix-appservice-irc.serviceConfig.SystemCallFilter = lib.mkForce [
-    "@system-service @pkey"
-    "~@privileged @resources"
-    "@chown"
-  ];
-  services.matrix-appservice-irc = {
-    enable = true;
-    localpart = "irc_bot";
-    port = 8010;
-    registrationUrl = "http://localhost:8010";
-    settings = {
-      homeserver = {
-        domain = "${config.pub-solar-os.networking.domain}";
-        url = "http://127.0.0.1:${synapseClientPort}";
-        media_url = "https://matrix.${config.pub-solar-os.networking.domain}";
-        enablePresence = false;
+  options.pub-solar-os = {
+    matrix.appservice-irc.mediaproxy = {
+      signingKeyPath = lib.mkOption {
+        description = "Path to file containing the IRC appservice mediaproxy signing key";
+        type = lib.types.str;
+        default = "/var/lib/matrix-appservice-irc/media-signingkey.jwk";
       };
-      ircService = {
-        ident = {
-          address = "::";
-          enabled = false;
-          port = 1113;
+    };
+  };
+  config = {
+    services.matrix-appservice-irc = {
+      enable = true;
+      localpart = "irc_bot";
+      port = 8010;
+      registrationUrl = "http://localhost:8010";
+      settings = {
+        homeserver = {
+          domain = "${config.pub-solar-os.networking.domain}";
+          url = "http://${synapseIp}:${synapseClientPort}";
+          enablePresence = false;
         };
-        logging = {
-          level = "debug";
-          maxFiles = 5;
-          toCosole = true;
-        };
-        matrixHandler = {
-          eventCacheSize = 4096;
-        };
-        metrics = {
-          enabled = true;
-          remoteUserAgeBuckets = [
-            "1h"
-            "1d"
-            "1w"
-          ];
-        };
-        provisioning = {
-          enabled = false;
-          requestTimeoutSeconds = 300;
-        };
-        servers =
-          let
-            commonConfig = {
-              allowExpiredCerts = false;
-              botConfig = {
-                enabled = false;
-                joinChannelsIfNoUsers = false;
-                nick = "MatrixBot";
-              };
-              dynamicChannels = {
-                createAlias = true;
-                enabled = true;
-                federate = true;
-                joinRule = "public";
-                published = true;
-              };
-              ircClients = {
-                allowNickChanges = true;
-                concurrentReconnectLimit = 50;
-                idleTimeout = 10800;
-                lineLimit = 3;
-                maxClients = 30;
-                nickTemplate = "$DISPLAY[m]";
-                reconnectIntervalMs = 5000;
-              };
-              matrixClients = {
-                joinAttempts = -1;
-              };
-              membershipLists = {
-                enabled = true;
-                floodDelayMs = 10000;
-                global = {
-                  ircToMatrix = {
-                    incremental = true;
-                    initial = true;
-                  };
-                  matrixToIrc = {
-                    incremental = true;
-                    initial = true;
+        ircService = {
+          ident = {
+            address = "::";
+            enabled = false;
+            port = 1113;
+          };
+          logging = {
+            # set to debug for debugging
+            level = "warn";
+            maxFiles = 5;
+            toCosole = true;
+          };
+          matrixHandler = {
+            eventCacheSize = 4096;
+          };
+          mediaProxy = {
+            signingKeyPath = config.pub-solar-os.matrix.appservice-irc.mediaproxy.signingKeyPath;
+            # keep media for 2 weeks
+            ttlSeconds = 1209600;
+            bindPort = 11111;
+            publicUrl = "https:///matrix.${config.pub-solar-os.networking.domain}/media";
+          };
+          metrics = {
+            enabled = true;
+            remoteUserAgeBuckets = [
+              "1h"
+              "1d"
+              "1w"
+            ];
+          };
+          provisioning = {
+            enabled = false;
+            requestTimeoutSeconds = 300;
+          };
+          servers =
+            let
+              commonConfig = {
+                allowExpiredCerts = false;
+                botConfig = {
+                  enabled = false;
+                  joinChannelsIfNoUsers = false;
+                  nick = "MatrixBot";
+                };
+                dynamicChannels = {
+                  createAlias = true;
+                  enabled = true;
+                  federate = true;
+                  joinRule = "public";
+                  published = true;
+                };
+                ircClients = {
+                  allowNickChanges = true;
+                  concurrentReconnectLimit = 50;
+                  idleTimeout = 10800;
+                  lineLimit = 3;
+                  maxClients = 30;
+                  nickTemplate = "$DISPLAY[m]";
+                  reconnectIntervalMs = 5000;
+                };
+                matrixClients = {
+                  joinAttempts = -1;
+                };
+                membershipLists = {
+                  enabled = true;
+                  floodDelayMs = 10000;
+                  global = {
+                    ircToMatrix = {
+                      incremental = true;
+                      initial = true;
+                    };
+                    matrixToIrc = {
+                      incremental = true;
+                      initial = true;
+                    };
                   };
                 };
+                port = 6697;
+                privateMessages = {
+                  enabled = true;
+                  federate = true;
+                };
+                sasl = false;
+                sendConnectionMessages = true;
+                ssl = true;
               };
-              port = 6697;
-              privateMessages = {
-                enabled = true;
-                federate = true;
+            in
+            {
+              "irc.libera.chat" = lib.attrsets.recursiveUpdate commonConfig {
+                name = "libera";
+                dynamicChannels.groupId = "+libera.chat:localhost";
+                dynamicChannels.aliasTemplate = "#_libera_$CHANNEL";
+                matrixClients.displayName = "$NICK (LIBERA-IRC)";
+              };
+              "irc.scratch-network.net" = lib.attrsets.recursiveUpdate commonConfig {
+                name = "scratch";
+                matrixClients.displayName = "$NICK (SCRATCH-IRC)";
+                dynamicChannels.aliasTemplate = "#_scratch_$CHANNEL";
+                dynamicChannels.groupId = "+scratch-network.net:localhost";
               };
-              sasl = false;
-              sendConnectionMessages = true;
-              ssl = true;
             };
-          in
-          {
-            "irc.libera.chat" = lib.attrsets.recursiveUpdate commonConfig {
-              name = "libera";
-              dynamicChannels.groupId = "+libera.chat:localhost";
-              dynamicChannels.aliasTemplate = "#_libera_$CHANNEL";
-              matrixClients.displayName = "$NICK (LIBERA-IRC)";
-            };
-            "irc.scratch-network.net" = lib.attrsets.recursiveUpdate commonConfig {
-              name = "scratch";
-              matrixClients.displayName = "$NICK (SCRATCH-IRC)";
-              dynamicChannels.aliasTemplate = "#_scratch_$CHANNEL";
-              dynamicChannels.groupId = "+scratch-network.net:localhost";
-            };
-          };
+        };
       };
     };
   };

modules/matrix-telegram/default.nix

@@ -5,6 +5,16 @@
   pkgs,
   ...
 }:
+let
+  nameHasClient = name: name == "client";
+  resourceHasClient = resource: builtins.any nameHasClient resource.names;
+  listenerWithClient =
+    lib.findFirst (listener: builtins.any resourceHasClient listener.resources)
+      (throw "Found no matrix-synapse.settings.listeners.*.resources.*.names containing string client")
+      config.services.matrix-synapse.settings.listeners;
+  synapseIp = builtins.elemAt listenerWithClient.bind_addresses 0;
+  synapseClientPort = "${toString listenerWithClient.port}";
+in
 {
   age.secrets."matrix-mautrix-telegram-env-file" = {
     file = "${flake.self}/secrets/matrix-mautrix-telegram-env-file.age";
@@ -18,7 +28,7 @@
     settings = {
       homeserver = {
         # TODO: Use the port from synapse config
-        address = "http://127.0.0.1:8008";
+        address = "http://${synapseIp}:${synapseClientPort}";
         domain = "${config.pub-solar-os.networking.domain}";
         verify_ssl = true;
       };

modules/matrix/default.nix

@@ -1,312 +1,541 @@
 {
   flake,
   config,
+  lib,
   pkgs,
   ...
 }:
 let
   publicDomain = "matrix.${config.pub-solar-os.networking.domain}";
   serverDomain = "${config.pub-solar-os.networking.domain}";
+  # Find element in list config.services.matrix-synapse.settings.listeners
+  # that sets type = "metrics"
+  listenerWithMetrics =
+    lib.findFirst (listener: listener.type == "metrics")
+      (throw "Found no matrix-synapse.settings.listeners.*.type containing string metrics")
+      config.services.matrix-synapse.settings.listeners;
+  synapseMetricsPort = listenerWithMetrics.port;
+  workerDefs = [
+    {
+      name = "client-1";
+      resources = [ "client" ];
+    }
+    {
+      name = "federation-sender-1";
+      resources = [ ];
+    }
+    {
+      name = "federation-receiver-1";
+      resources = [ "federation" ];
+    }
+    {
+      name = "federation-receiver-2";
+      resources = [ "federation" ];
+    }
+    {
+      name = "federation-receiver-3";
+      resources = [ "federation" ];
+    }
+    {
+      name = "federation-receiver-4";
+      resources = [ "federation" ];
+    }
+  ];
+
+  subnet = "127.0.200";
+  synapse_ip = "${subnet}.10";
+  worker_ip_start = 11;
+  metrics_port_start = 9101;
+  workers = lib.imap0 (
+    i: def:
+    let
+      ip = "${subnet}.${toString (worker_ip_start + i)}";
+      metrics_port = metrics_port_start + i;
+    in
+    {
+      inherit (def) name;
+      value = {
+        worker_app = "synapse.app.generic_worker";
+        worker_listeners = [
+          {
+            type = "http";
+            port = 8008;
+            bind_addresses = [ ip ];
+            tls = false;
+            x_forwarded = true;
+            resources = [ { names = def.resources ++ [ "health" ]; } ];
+          }
+          # add a metrics listener to all workers
+          # ports will be exposed only with wireguard via firewall rule
+          {
+            type = "metrics";
+            port = metrics_port;
+            bind_addresses = [ "0.0.0.0" ];
+            tls = false;
+            resources = [ { names = [ "metrics" ]; } ];
+          }
+        ];
+      };
+    }
+  ) workerDefs;
+
+  getWorkerHostsForResource =
+    resource:
+    lib.flatten (
+      builtins.map (
+        worker:
+        let
+          listener = lib.findFirst (
+            listener: lib.any (res: lib.any (name: name == resource) res.names) (listener.resources or [ ])
+          ) null worker.value.worker_listeners;
+        in
+        if listener != null then
+          [ "${builtins.head listener.bind_addresses}:${toString listener.port}" ]
+        else
+          [ ]
+      ) workers
+    );
+  getWorkerPortForResource =
+    resource:
+    lib.flatten (
+      builtins.map (
+        worker:
+        let
+          listener = lib.findFirst (
+            listener: lib.any (res: lib.any (name: name == resource) res.names) (listener.resources or [ ])
+          ) null worker.value.worker_listeners;
+        in
+        if listener != null then [ listener.port ] else [ ]
+      ) workers
+    );
+
+  federationReceivers = getWorkerHostsForResource "federation";
+  clientReceivers = getWorkerHostsForResource "client";
+  metricsPorts = getWorkerPortForResource "metrics";
 in
 {
-  age.secrets."matrix-synapse-signing-key" = {
-    file = "${flake.self}/secrets/matrix-synapse-signing-key.age";
-    mode = "400";
-    owner = "matrix-synapse";
+  options.pub-solar-os = {
+    matrix = {
+      enable = lib.mkEnableOption "Enable matrix-synapse and matrix-authentication-service to run on the node";
+
+      synapse = {
+        app-service-config-files = lib.mkOption {
+          description = "List of app service config files";
+          type = lib.types.listOf lib.types.str;
+          default = [ ];
+        };
+
+        extra-config-files = lib.mkOption {
+          description = "List of extra synapse config files";
+          type = lib.types.listOf lib.types.str;
+          default = [ ];
+        };
+
+        signing_key_path = lib.mkOption {
+          description = "Path to file containing the signing key";
+          type = lib.types.str;
+          default = "${config.services.matrix-synapse.dataDir}/homeserver.signing.key";
+        };
+      };
+
+      matrix-authentication-service = {
+        extra-config-files = lib.mkOption {
+          description = "List of extra mas config files";
+          type = lib.types.listOf lib.types.str;
+          default = [ ];
+        };
+      };
+    };
   };
 
-  age.secrets."matrix-synapse-secret-config.yaml" = {
-    file = "${flake.self}/secrets/matrix-synapse-secret-config.yaml.age";
-    mode = "400";
-    owner = "matrix-synapse";
-  };
+  config = lib.mkIf config.pub-solar-os.matrix.enable {
+    # Only expose matrix-synapse metrics ports via wireguard interface
+    networking.firewall.interfaces.wg-ssh.allowedTCPPorts = [ synapseMetricsPort ] ++ metricsPorts;
 
-  age.secrets."matrix-synapse-sliding-sync-secret" = {
-    file = "${flake.self}/secrets/matrix-synapse-sliding-sync-secret.age";
-    mode = "400";
-    owner = "matrix-synapse";
-  };
-
-  services.matrix-synapse = {
-    enable = true;
-    settings = {
-      server_name = serverDomain;
-      public_baseurl = "https://${publicDomain}/";
-      database = {
-        name = "psycopg2";
-        args = {
-          host = "/run/postgresql";
-          cp_max = 10;
-          cp_min = 5;
-          database = "matrix";
-        };
-        allow_unsafe_locale = false;
-        txn_limit = 0;
+    # generate nginx upstreams for configured workers
+    services.nginx.upstreams = {
+      "matrix-synapse".servers = {
+        "${synapse_ip}:8008" = { };
       };
-      listeners = [
-        {
-          bind_addresses = [ "127.0.0.1" ];
-          port = 8008;
-          resources = [
-            {
-              compress = true;
-              names = [ "client" ];
-            }
-            {
-              compress = false;
-              names = [ "federation" ];
-            }
-          ];
-          tls = false;
-          type = "http";
-          x_forwarded = true;
-        }
-        {
-          bind_addresses = [ "127.0.0.1" ];
-          port = 8012;
-          resources = [ { names = [ "metrics" ]; } ];
-          tls = false;
-          type = "metrics";
-        }
-      ];
+      "matrix-federation-receiver".servers = lib.genAttrs federationReceivers (host: { });
+      "matrix-client-receiver".servers = lib.genAttrs clientReceivers (host: { });
+    };
 
-      account_threepid_delegates.msisdn = "";
-      alias_creation_rules = [
-        {
-          action = "allow";
-          alias = "*";
-          room_id = "*";
-          user_id = "*";
-        }
-      ];
-      allow_guest_access = false;
-      allow_public_rooms_over_federation = true;
-      allow_public_rooms_without_auth = false;
-      auto_join_rooms = [
-        "#community:${serverDomain}"
-        "#general:${serverDomain}"
-      ];
-
-      autocreate_auto_join_rooms = true;
-      caches.global_factor = 0.5;
-
-      default_room_version = "10";
-      disable_msisdn_registration = true;
-      enable_media_repo = true;
-      enable_metrics = true;
-      mau_stats_only = true;
-      enable_registration = false;
-      enable_registration_captcha = false;
-      enable_registration_without_verification = false;
-      enable_room_list_search = true;
-      encryption_enabled_by_default_for_room_type = "off";
-      event_cache_size = "100K";
-      federation_rr_transactions_per_room_per_second = 50;
-      federation_client_minimum_tls_version = "1.2";
-      forget_rooms_on_leave = true;
-      include_profile_data_on_invite = true;
-      instance_map = { };
-      limit_profile_requests_to_users_who_share_rooms = false;
-
-      max_spider_size = "10M";
-      max_upload_size = "50M";
-      media_storage_providers = [ ];
-
-      password_config = {
-        enabled = false;
-        localdb_enabled = false;
-        pepper = "";
-      };
-
-      presence.enabled = true;
-      push.include_content = false;
-
-      rc_admin_redaction = {
-        burst_count = 50;
-        per_second = 1;
-      };
-      rc_federation = {
-        concurrent = 3;
-        reject_limit = 50;
-        sleep_delay = 500;
-        sleep_limit = 10;
-        window_size = 1000;
-      };
-      rc_invites = {
-        per_issuer = {
-          burst_count = 10;
-          per_second = 0.3;
+    services.matrix-synapse = {
+      enable = true;
+      log.root.level = "WARNING";
+      configureRedisLocally = true;
+      workers = builtins.listToAttrs workers;
+      settings = {
+        server_name = serverDomain;
+        public_baseurl = "https://${publicDomain}/";
+        database = {
+          name = "psycopg2";
+          args = {
+            host = "/run/postgresql";
+            cp_max = 10;
+            cp_min = 5;
+            database = "matrix";
+          };
+          allow_unsafe_locale = false;
+          txn_limit = 0;
         };
-        per_room = {
-          burst_count = 10;
-          per_second = 0.3;
+
+        listeners = [
+          {
+            bind_addresses = [ "${synapse_ip}" ];
+            port = 8008;
+            tls = false;
+            type = "http";
+            x_forwarded = true;
+            resources = [
+              {
+                names = [
+                  "client"
+                  "federation"
+                ];
+              }
+            ];
+          }
+          {
+            bind_addresses = [ "0.0.0.0" ];
+            port = 9000;
+            tls = false;
+            type = "metrics";
+            resources = [ { names = [ "metrics" ]; } ];
+          }
+          {
+            path = "/run/matrix-synapse/main_replication.sock";
+            mode = "660";
+            type = "http";
+            resources = [ { names = [ "replication" ]; } ];
+          }
+        ];
+        federation_sender_instances = [ "federation-sender-1" ];
+        instance_map = {
+          main = {
+            path = "/run/matrix-synapse/main_replication.sock";
+          };
         };
-        per_user = {
-          burst_count = 5;
-          per_second = 3.0e-3;
+
+        account_threepid_delegates.msisdn = "";
+        alias_creation_rules = [
+          {
+            action = "allow";
+            alias = "*";
+            room_id = "*";
+            user_id = "*";
+          }
+        ];
+        allow_guest_access = false;
+        allow_public_rooms_over_federation = true;
+        allow_public_rooms_without_auth = false;
+        auto_join_rooms = [
+          "#community:${serverDomain}"
+          "#general:${serverDomain}"
+        ];
+
+        autocreate_auto_join_rooms = true;
+
+        default_room_version = "10";
+        disable_msisdn_registration = true;
+        enable_media_repo = true;
+        enable_metrics = true;
+        federation_metrics_domains = [
+          "matrix.org"
+          "mozilla.org"
+          "systemli.org"
+          "tchncs.de"
+          "ccc.ac"
+          "fairydust.space"
+        ];
+        mau_stats_only = true;
+        enable_registration = false;
+        enable_registration_captcha = false;
+        enable_registration_without_verification = false;
+        enable_room_list_search = true;
+        encryption_enabled_by_default_for_room_type = "off";
+        event_cache_size = "100K";
+        caches.global_factor = 10;
+        # Based on https://github.com/spantaleev/matrix-docker-ansible-deploy/blob/37a7af52ab6a803e5fec72d37b0411a6c1a3ddb7/docs/maintenance-synapse.md#tuning-caches-and-cache-autotuning
+        # https://element-hq.github.io/synapse/latest/usage/configuration/config_documentation.html#caches-and-associated-values
+        cache_autotuning = {
+          max_cache_memory_usage = "4096M";
+          target_cache_memory_usage = "2048M";
+          min_cache_ttl = "5m";
         };
-      };
-      rc_joins = {
-        local = {
-          burst_count = 10;
-          per_second = 0.1;
+
+        # https://github.com/element-hq/synapse/issues/11203
+        # No YAML deep-merge, so this needs to be in secret extraConfigFiles
+        # together with msc3861
+        #experimental_features = {
+        #  # MSC3266: Room summary API. Used for knocking over federation
+        #  msc3266_enabled: true
+        #  # MSC4222 needed for syncv2 state_after. This allow clients to
+        #  # correctly track the state of the room.
+        #  msc4222_enabled: true
+        #  # Rendezvous server for QR Code generation
+        #  msc4108_enabled = true;
+        #};
+
+        # The maximum allowed duration by which sent events can be delayed, as
+        # per MSC4140.
+        max_event_delay_duration = "24h";
+
+        federation_rr_transactions_per_room_per_second = 50;
+        federation_client_minimum_tls_version = "1.2";
+        forget_rooms_on_leave = true;
+        include_profile_data_on_invite = true;
+        limit_profile_requests_to_users_who_share_rooms = false;
+
+        max_spider_size = "10M";
+        max_upload_size = "50M";
+        media_storage_providers = [ ];
+
+        password_config = {
+          enabled = false;
+          localdb_enabled = false;
+          pepper = "";
         };
-        remote = {
-          burst_count = 10;
-          per_second = 1.0e-2;
+
+        presence.enabled = false;
+        push.include_content = false;
+
+        rc_admin_redaction = {
+          burst_count = 50;
+          per_second = 1;
         };
-      };
-      rc_login = {
-        account = {
+        rc_federation = {
+          concurrent = 3;
+          reject_limit = 50;
+          sleep_delay = 500;
+          sleep_limit = 10;
+          window_size = 1000;
+        };
+        rc_invites = {
+          per_issuer = {
+            burst_count = 10;
+            per_second = 0.3;
+          };
+          per_room = {
+            burst_count = 10;
+            per_second = 0.3;
+          };
+          per_user = {
+            burst_count = 5;
+            per_second = 3.0e-3;
+          };
+        };
+        rc_joins = {
+          local = {
+            burst_count = 10;
+            per_second = 0.1;
+          };
+          remote = {
+            burst_count = 10;
+            per_second = 1.0e-2;
+          };
+        };
+        rc_login = {
+          account = {
+            burst_count = 3;
+            per_second = 0.17;
+          };
+          address = {
+            burst_count = 3;
+            per_second = 0.17;
+          };
+          failed_attempts = {
+            burst_count = 3;
+            per_second = 0.17;
+          };
+        };
+        rc_message = {
+          # This needs to match at least e2ee key sharing frequency plus a bit of headroom
+          # Note key sharing events are bursty
+          burst_count = 30;
+          per_second = 0.5;
+        };
+        rc_delayed_event_mgmt = {
+          # This needs to match at least the heart-beat frequency plus a bit of headroom
+          # Currently the heart-beat is every 5 seconds which translates into a rate of 0.2s
+          per_second = 1;
+          burst_count = 20;
+        };
+        rc_registration = {
           burst_count = 3;
           per_second = 0.17;
         };
-        address = {
-          burst_count = 3;
-          per_second = 0.17;
+        redaction_retention_period = "7d";
+        forgotten_room_retention_period = "7d";
+        registration_requires_token = false;
+        registrations_require_3pid = [ "email" ];
+        report_stats = false;
+        require_auth_for_profile_requests = false;
+        room_list_publication_rules = [
+          {
+            action = "allow";
+            alias = "*";
+            room_id = "*";
+            user_id = "*";
+          }
+        ];
+
+        signing_key_path = config.pub-solar-os.matrix.synapse.signing_key_path;
+
+        stream_writers = { };
+        trusted_key_servers = [ { server_name = "matrix.org"; } ];
+        suppress_key_server_warning = true;
+
+        turn_allow_guests = false;
+        turn_uris = [
+          "turn:${config.services.coturn.realm}:3478?transport=udp"
+          "turn:${config.services.coturn.realm}:3478?transport=tcp"
+        ];
+        turn_user_lifetime = "1h";
+
+        url_preview_accept_language = [
+          "en-US"
+          "en"
+        ];
+        url_preview_enabled = true;
+        url_preview_ip_range_blacklist = [
+          "127.0.0.0/8"
+          "10.0.0.0/8"
+          "172.16.0.0/12"
+          "192.168.0.0/16"
+          "100.64.0.0/10"
+          "192.0.0.0/24"
+          "169.254.0.0/16"
+          "192.88.99.0/24"
+          "198.18.0.0/15"
+          "192.0.2.0/24"
+          "198.51.100.0/24"
+          "203.0.113.0/24"
+          "224.0.0.0/4"
+          "::1/128"
+          "fe80::/10"
+          "fc00::/7"
+          "2001:db8::/32"
+          "ff00::/8"
+          "fec0::/10"
+        ];
+
+        user_directory = {
+          prefer_local_users = false;
+          search_all_users = false;
         };
-        failed_attempts = {
-          burst_count = 3;
-          per_second = 0.17;
+        user_ips_max_age = "28d";
+
+        app_service_config_files = config.pub-solar-os.matrix.synapse.app-service-config-files;
+
+        modules = [
+          {
+            module = "mjolnir.Module";
+            config = {
+              # Prevent servers/users in the ban lists from inviting users on this
+              # server to rooms. Default true.
+              block_invites = true;
+              # Flag messages sent by servers/users in the ban lists as spam. Currently
+              # this means that spammy messages will appear as empty to users. Default
+              # false.
+              block_messages = false;
+              # Remove users from the user directory search by filtering matrix IDs and
+              # display names by the entries in the user ban list. Default false.
+              block_usernames = false;
+              # The room IDs of the ban lists to honour. Unlike other parts of Mjolnir,
+              # this list cannot be room aliases or permalinks. This server is expected
+              # to already be joined to the room - Mjolnir will not automatically join
+              # these rooms.
+              ban_lists = [ "!roomid:example.org" ];
+            };
+          }
+        ];
+      };
+
+      withJemalloc = true;
+
+      extraConfigFiles = config.pub-solar-os.matrix.synapse.extra-config-files;
+
+      extras = [
+        "oidc"
+      ];
+
+      plugins = with config.services.matrix-synapse.package.plugins; [
+        matrix-synapse-shared-secret-auth
+        matrix-synapse-mjolnir-antispam
+      ];
+    };
+
+    services.matrix-authentication-service = {
+      enable = true;
+      createDatabase = true;
+      extraConfigFiles = config.pub-solar-os.matrix.matrix-authentication-service.extra-config-files;
+
+      # https://element-hq.github.io/matrix-authentication-service/reference/configuration.html
+      settings = {
+        account.email_change_allowed = false;
+        http.public_base = "https://mas.${config.pub-solar-os.networking.domain}";
+        http.issuer = "https://mas.${config.pub-solar-os.networking.domain}";
+        http.listeners = [
+          {
+            name = "web";
+            resources = [
+              { name = "discovery"; }
+              { name = "human"; }
+              { name = "oauth"; }
+              { name = "compat"; }
+              { name = "graphql"; }
+              {
+                name = "assets";
+                path = "${config.services.matrix-authentication-service.package}/share/matrix-authentication-service/assets";
+              }
+            ];
+            binds = [
+              {
+                host = "0.0.0.0";
+                port = 8090;
+              }
+            ];
+            proxy_protocol = false;
+          }
+          {
+            name = "internal";
+            resources = [
+              { name = "health"; }
+            ];
+            binds = [
+              {
+                host = "0.0.0.0";
+                port = 8081;
+              }
+            ];
+            proxy_protocol = false;
+          }
+        ];
+        passwords.enabled = false;
+      };
+    };
+
+    pub-solar-os.backups = {
+      resources.matrix-db.resourceCreateCommand = ''
+        ${pkgs.sudo}/bin/sudo -u postgres ${pkgs.postgresql}/bin/pg_dump -d matrix -f /tmp/matrix-synapse-backup.sql
+        ${pkgs.sudo}/bin/sudo -u postgres ${pkgs.postgresql}/bin/pg_dump -d matrix-authentication-service -f /tmp/matrix-authentication-service-backup.sql
+      '';
+      restic.matrix-synapse = {
+        resources = [ "matrix-db" ];
+        paths = [
+          "/var/lib/matrix-synapse"
+          "/var/lib/matrix-appservice-irc"
+          "/var/lib/mautrix-telegram"
+          "/tmp/matrix-synapse-backup.sql"
+          "/tmp/matrix-authentication-service-backup.sql"
+        ];
+        timerConfig = {
+          OnCalendar = "*-*-* 05:00:00 Etc/UTC";
         };
+        initialize = true;
       };
-      rc_message = {
-        burst_count = 10;
-        per_second = 0.2;
-      };
-      rc_registration = {
-        burst_count = 3;
-        per_second = 0.17;
-      };
-      redaction_retention_period = "7d";
-      forgotten_room_retention_period = "7d";
-      redis.enabled = false;
-      registration_requires_token = false;
-      registrations_require_3pid = [ "email" ];
-      report_stats = false;
-      require_auth_for_profile_requests = false;
-      room_list_publication_rules = [
-        {
-          action = "allow";
-          alias = "*";
-          room_id = "*";
-          user_id = "*";
-        }
-      ];
-
-      signing_key_path = "/run/agenix/matrix-synapse-signing-key";
-
-      stream_writers = { };
-      trusted_key_servers = [ { server_name = "matrix.org"; } ];
-      suppress_key_server_warning = true;
-
-      turn_allow_guests = false;
-      turn_uris = [
-        "turn:${config.services.coturn.realm}:3478?transport=udp"
-        "turn:${config.services.coturn.realm}:3478?transport=tcp"
-      ];
-      turn_user_lifetime = "1h";
-
-      url_preview_accept_language = [
-        "en-US"
-        "en"
-      ];
-      url_preview_enabled = true;
-      url_preview_ip_range_blacklist = [
-        "127.0.0.0/8"
-        "10.0.0.0/8"
-        "172.16.0.0/12"
-        "192.168.0.0/16"
-        "100.64.0.0/10"
-        "192.0.0.0/24"
-        "169.254.0.0/16"
-        "192.88.99.0/24"
-        "198.18.0.0/15"
-        "192.0.2.0/24"
-        "198.51.100.0/24"
-        "203.0.113.0/24"
-        "224.0.0.0/4"
-        "::1/128"
-        "fe80::/10"
-        "fc00::/7"
-        "2001:db8::/32"
-        "ff00::/8"
-        "fec0::/10"
-      ];
-
-      user_directory = {
-        prefer_local_users = false;
-        search_all_users = false;
-      };
-      user_ips_max_age = "28d";
-
-      app_service_config_files = [
-        "/var/lib/matrix-synapse/telegram-registration.yaml"
-        "/var/lib/matrix-appservice-irc/registration.yml"
-        # "/matrix-appservice-slack-registration.yaml"
-        # "/hookshot-registration.yml"
-        # "/matrix-mautrix-signal-registration.yaml"
-        # "/matrix-mautrix-telegram-registration.yaml"
-      ];
     };
-
-    withJemalloc = true;
-
-    extraConfigFiles = [
-      "/run/agenix/matrix-synapse-secret-config.yaml"
-
-      # The registration file is automatically generated after starting the
-      # appservice for the first time.
-      # cp /var/lib/mautrix-telegram/telegram-registration.yaml \
-      #   /var/lib/matrix-synapse/
-      # chown matrix-synapse:matrix-synapse \
-      #   /var/lib/matrix-synapse/telegram-registration.yaml
-      "/var/lib/matrix-synapse/telegram-registration.yaml"
-    ];
-
-    extras = [
-      "oidc"
-      "redis"
-    ];
-
-    plugins = [ config.services.matrix-synapse.package.plugins.matrix-synapse-shared-secret-auth ];
-  };
-
-  services.matrix-sliding-sync = {
-    enable = true;
-    settings = {
-      SYNCV3_SERVER = "https://${publicDomain}";
-      SYNCV3_BINDADDR = "127.0.0.1:8011";
-      # The bind addr for Prometheus metrics, which will be accessible at
-      # /metrics at this address
-      SYNCV3_PROM = "127.0.0.1:9100";
-    };
-    environmentFile = config.age.secrets."matrix-synapse-sliding-sync-secret".path;
-  };
-
-  services.restic.backups.matrix-synapse-storagebox = {
-    paths = [
-      "/var/lib/matrix-synapse"
-      "/var/lib/matrix-appservice-irc"
-      "/var/lib/mautrix-telegram"
-      "/tmp/matrix-synapse-backup.sql"
-    ];
-    timerConfig = {
-      OnCalendar = "*-*-* 05:00:00 Etc/UTC";
-    };
-    initialize = true;
-    passwordFile = config.age.secrets."restic-repo-storagebox".path;
-    repository = "sftp:u377325@u377325.your-storagebox.de:/backups";
-    backupPrepareCommand = ''
-      ${pkgs.sudo}/bin/sudo -u postgres ${pkgs.postgresql}/bin/pg_dump -d matrix > /tmp/matrix-synapse-backup.sql
-    '';
-    backupCleanupCommand = ''
-      rm /tmp/matrix-synapse-backup.sql
-    '';
-    pruneOpts = [
-      "--keep-daily 7"
-      "--keep-weekly 4"
-      "--keep-monthly 3"
-    ];
   };
 }