Merge pull request 'Mastodon updates, more docs' (#10) from mastodon-updates into main

Reviewed-on: pub-solar/infra#10 Reviewed-by: Akshay Mankar <axeman@noreply.git.pub.solar>
2023-07-21 12:32:21 +02:00 · 2023-07-21 12:32:21 +02:00 · 107809454b
parent c9863a68b2 a6d3dbb76d
commit 107809454b
10 changed files with 181 additions and 41 deletions
--- a/.gitignore
+++ b/.gitignore
@ -3,3 +3,4 @@
 *.plan
 result
 .env
+backups
--- a/docs/deletion-request.md
+++ b/docs/deletion-request.md
@ -0,0 +1,17 @@
+# Process for handling a deletion request
+
+### Keycloak
+Required:
+- auth.pub.solar ops user credentials
+- SSH access to host flora-6
+```
+ssh barkeeper@flora-6.pub.solar
+
+sudo --user keycloak kcadm.sh config credentials --config /tmp/kcadm.config --server http://localhost:8080 --realm pub.solar --user ops
+
+# Take note of user id in response from following command
+sudo --user keycloak kcadm.sh get --config /tmp/kcadm.config users --realm pub.solar --query email=<email-address>
+
+# Use user id from previous command, for example
+sudo --user keycloak kcadm.sh delete --config /tmp/kcadm.config users/2ec6f173-3c10-4b82-9808-e2f2d393ff11 --realm pub.solar
+```
--- a/docs/keycloak-reset-user-password.md
+++ b/docs/keycloak-reset-user-password.md
@ -0,0 +1,33 @@
+# Process for resetting keycloak user passwords
+
+### Keycloak
+Required:
+- auth.pub.solar ops user credentials
+- SSH access to host flora-6
+```
+ssh barkeeper@flora-6.pub.solar
+
+mkdir /tmp/keycloak-credential-reset
+
+sudo --user keycloak kcadm.sh config credentials --config /tmp/kcadm.config --server http://localhost:8080 --realm pub.solar --user ops
+
+sudo --user keycloak kcadm.sh get --config /tmp/kcadm.config users --realm pub.solar | jq --raw-output '.[] | .id' > /tmp/keycloak-credential-reset/all-uuids
+
+for UUID in $(cat /tmp/keycloak-credential-reset/all-uuids); do
+  sudo --user keycloak kcadm.sh get --config /tmp/kcadm.config users/$UUID/credentials --realm pub.solar > /tmp/keycloak-credential-reset/$UUID
+done
+
+mkdir /tmp/keycloak-credential-reset/accounts-with-creds
+
+find /tmp/keycloak-credential-reset -type f -size +3c -exec mv '{}' /tmp/keycloak-credential-reset/accounts-with-creds/ \;
+
+rm -r /tmp/keycloak-credential-reset/accounts-with-creds/
+
+find /tmp/keycloak-credential-reset/ -type f -exec basename '{}' \; > /tmp/keycloak-credential-reset/accounts-without-credentials
+
+vim /tmp/keycloak-credential-reset/accounts-without-credentials
+
+for UUID in $(cat /tmp/keycloak-credential-reset/accounts-without-credentials); do
+  sudo --user keycloak kcadm.sh update --config /tmp/kcadm.config users/$UUID/reset-password --target-realm pub.solar --set type=password --set value=$(< /dev/urandom tr -dc A-Z-a-z-0-9 | head -c${1:-32};echo;) --set temporary=true --no-merge
+done
+```
--- a/docs/keycloak-update-realm.md
+++ b/docs/keycloak-update-realm.md
@ -0,0 +1,19 @@
+# Process for updating a keycloak realm via CLI
+
+### Keycloak
+Required:
+- auth.pub.solar ops user credentials
+- SSH access to host flora-6
+```
+ssh barkeeper@flora-6.pub.solar
+
+sudo -u keycloak kcadm.sh config credentials --config /tmp/kcadm.config --server http://localhost:8080 --realm master --user admin
+
+sudo -u keycloak kcadm.sh get --config /tmp/kcadm.config realms/pub.solar
+
+sudo -u keycloak kcadm.sh update --config /tmp/kcadm.config realms/pub.solar -s browserFlow='Webauthn Browser'
+
+sudo -u keycloak kcadm.sh get --config /tmp/kcadm.config realms/pub.solar
+```
+
+Source: https://keycloak.ch/keycloak-tutorials/tutorial-webauthn/
--- a/flake.lock
+++ b/flake.lock
@ -2,19 +2,17 @@
  "nodes": {
    "devshell": {
      "inputs": {
-        "flake-utils": [
-          "flake-utils"
-        ],
        "nixpkgs": [
          "nixpkgs"
-        ]
+        ],
+        "systems": "systems"
      },
      "locked": {
-        "lastModified": 1667210711,
-        "narHash": "sha256-IoErjXZAkzYWHEpQqwu/DeRNJGFdR7X2OGbkhMqMrpw=",
+        "lastModified": 1683635384,
+        "narHash": "sha256-9goJTd05yOyD/McaMqZ4BUB8JW+mZMnZQJZ7VQ6C/Lw=",
        "owner": "numtide",
        "repo": "devshell",
-        "rev": "96a9dd12b8a447840cc246e17a47b81a4268bba7",
+        "rev": "5143ea68647c4cf5227e4ad2100db6671fc4c369",
        "type": "github"
      },
      "original": {
@ -24,12 +22,15 @@
      }
    },
    "flake-utils": {
+      "inputs": {
+        "systems": "systems_2"
+      },
      "locked": {
-        "lastModified": 1667395993,
-        "narHash": "sha256-nuEHfE/LcWyuSWnS8t12N1wc105Qtau+/OdUAjtQ0rA=",
+        "lastModified": 1681202837,
+        "narHash": "sha256-H+Rh19JDwRtpVPAWp64F+rlEtxUWBAQW28eAi3SRSzg=",
        "owner": "numtide",
        "repo": "flake-utils",
-        "rev": "5aed5285a952e0b949eb3ba02c12fa4fcfef535f",
+        "rev": "cfacdce06f30d2b68473a46042957675eebb3401",
        "type": "github"
      },
      "original": {
@ -40,11 +41,11 @@
    },
    "nixpkgs": {
      "locked": {
-        "lastModified": 1669542132,
-        "narHash": "sha256-DRlg++NJAwPh8io3ExBJdNW7Djs3plVI5jgYQ+iXAZQ=",
+        "lastModified": 1684935479,
+        "narHash": "sha256-6QMMsXMr2nhmOPHdti2j3KRHt+bai2zw+LJfdCl97Mk=",
        "owner": "nixos",
        "repo": "nixpkgs",
-        "rev": "a115bb9bd56831941be3776c8a94005867f316a7",
+        "rev": "f91ee3065de91a3531329a674a45ddcb3467a650",
        "type": "github"
      },
      "original": {
@ -56,11 +57,11 @@
    },
    "nixpkgs-2205": {
      "locked": {
-        "lastModified": 1672580127,
-        "narHash": "sha256-3lW3xZslREhJogoOkjeZtlBtvFMyxHku7I/9IVehhT8=",
+        "lastModified": 1682600000,
+        "narHash": "sha256-ha4BehR1dh8EnXSoE1m/wyyYVvHI9txjW4w5/oxsW5Y=",
        "owner": "nixos",
        "repo": "nixpkgs",
-        "rev": "0874168639713f547c05947c76124f78441ea46c",
+        "rev": "50fc86b75d2744e1ab3837ef74b53f103a9b55a0",
        "type": "github"
      },
      "original": {
@ -79,6 +80,36 @@
        "tritonshell-module": "tritonshell-module"
      }
    },
+    "systems": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    },
+    "systems_2": {
+      "locked": {
+        "lastModified": 1681028828,
+        "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
+        "owner": "nix-systems",
+        "repo": "default",
+        "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
+        "type": "github"
+      },
+      "original": {
+        "owner": "nix-systems",
+        "repo": "default",
+        "type": "github"
+      }
+    },
    "tritonshell-module": {
      "inputs": {
        "devshell": [
@ -92,11 +123,11 @@
        ]
      },
      "locked": {
-        "lastModified": 1669581047,
-        "narHash": "sha256-qs2VUUCCkWlc+5KvP/Vh2ToLKMkCjAws47bVT6rilG8=",
+        "lastModified": 1684242426,
+        "narHash": "sha256-kvFD6WP6I1fK9DMCPpuRDZxsAGKpzXMMd2G5MYP42kU=",
        "ref": "main",
-        "rev": "341aa68b667a8fb9b77f8af319b7439e82c78793",
-        "revCount": 53,
+        "rev": "d227038987158fa894872868f25bbf911c9cb8d1",
+        "revCount": 61,
        "type": "git",
        "url": "https://git.greenbaum.cloud/dev/tritonshell"
      },
--- a/flake.nix
+++ b/flake.nix
@ -8,7 +8,6 @@
    flake-utils.url = "github:numtide/flake-utils";

    devshell.url = "github:numtide/devshell";
-    devshell.inputs.flake-utils.follows = "flake-utils";
    devshell.inputs.nixpkgs.follows = "nixpkgs";

    tritonshell-module.url = "git+https://git.greenbaum.cloud/dev/tritonshell?ref=main";
@ -21,7 +20,7 @@
    flake-utils.lib.simpleFlake {
      inherit self nixpkgs;
      name = "infra-project";
-      preOverlays = [ devshell.overlay ];
+      preOverlays = [ devshell.overlays.default ];
      shell = { pkgs }:
        pkgs.devshell.mkShell {
          imports = [ tritonshell-module.devshellModules.x86_64-linux.tritonshell ];
--- a/mastodon/README.md
+++ b/mastodon/README.md
@ -1,7 +1,7 @@
 # pub.solar mastodon
 https://mastodon.pub.solar

-### Upgrading
+### Upgrading Mastodon
 This section assumes you edited `docker-compose.yml` and bumped the mastodon docker
 image version tag
 ```
@ -53,6 +53,45 @@ docker rm \
  blue-mastodon_sidekiq_($current_container_index - 1)
 ```

+### Upgrading Caddy
+```
+mkdir -p certificates/acme-v02.api.letsencrypt.org-directory
+docker cp --archive blue-mastodon_caddy_2:/data/caddy/certificates/acme-v02.api.letsencrypt.org-directory/files.pub.solar ./certificates/acme-v02.api.letsencrypt.org-directory/
+docker cp --archive blue-mastodon_caddy_2:/data/caddy/certificates/acme-v02.api.letsencrypt.org-directory/mastodon.pub.solar ./certificates/acme-v02.api.letsencrypt.org-directory/
+
+docker-compose --project-name blue-mastodon up \
+  --scale caddy=2 \
+  --no-recreate \
+  --no-start
+
+docker cp --archive ./backups/certificates blue-mastodon_caddy_3:/data/caddy/certificates
+docker start blue-mastodon_caddy_3
+
+# Stop old caddy container
+docker stop blue-mastodon_caddy_2
+
+# Verify everything works fine, then remove the old caddy container
+docker rm blue-mastodon_caddy_2
+```
+
+### Upgrading Elasticsearch
+Look for new releases on https://www.elastic.co/guide/en/elasticsearch/reference/7.17/es-release-notes.html
+and edit the docker image tag accordingly.
+```
+docker-compose --project-name blue-mastodon up \
+  --scale elasticsearch=2 \
+  --no-recreate \
+
+# Stop old elasticsearch container
+docker stop blue-mastodon_elasticsearch_2
+
+docker exec -it blue-mastodon_web_15 bash
+tootctl search deploy
+
+# Verify everything works fine, then remove the old caddy container
+docker rm blue-mastodon_elasticsearch_2
+```
+
 Todos:
 - implement automatic backups, they are only done manually during upgrades at the moment
 - switch proxy from nginx-dehydrated to caddy - done
--- a/mastodon/docker-compose.yml
+++ b/mastodon/docker-compose.yml
@ -28,7 +28,7 @@ services:
  #    - triton.cns.services=mastodon-proxy

  caddy:
-    image: caddy:2.5.1
+    image: caddy:2.6.4
    mem_limit: 256m
    restart: always
    environment:
@ -44,12 +44,16 @@ services:
    labels:
      - triton.cns.services=mastodon-proxy
    entrypoint: /bin/sh
-    command: >-
-      -c 'echo "
+    command:
+      - -c
+      - >-
+        echo "
        {
          email admins@pub.solar
+          servers {
+            protocols h1 h2
+          }
        }
-
        $$SITE_DOMAIN {
          @streaming {
            path /api/v1/streaming/*
@ -77,23 +81,21 @@ services:
          handle_errors {
            rewrite 500.html
          }
-
          encode zstd gzip
-
          header {
            Strict-Transport-Security "max-age=31536000"
+            # clickjacking protection
+            X-Frame-Options DENY
          }
          header /sw.js Cache-Control "public, max-age=0"
          header @cache_control Cache-Control "public, max-age=31536000, immutable"
        }
-
        files.pub.solar {
          handle {
            rewrite * /s/jw24ad6l4a6zxsnd32cmf5hp5nsq/pub-solar-mastodon{uri}?download
            reverse_proxy {
              # backends / upstreams
              to https://link.tardigradeshare.io
-
              # header manipulation
              # proxy to an HTTPS endpoint
              header_up Host {upstream_hostport}
@ -101,7 +103,6 @@ services:
              header_up Connection ""
              header_up Authorization ""
              # remove these header from the backends response
-              header_down -content-disposition
              header_down -Set-Cookie
              header_down -Access-Control-Allow-Origin
              header_down -Access-Control-Allow-Methods
@ -115,14 +116,14 @@ services:
              # add these header to the backends response
              # cache client side for 7 days
              header_down Cache-Control "public, max-age=604800"
+              header_down Access-Control-Allow-Origin "*"
+              header_down X-Content-Type-Options "nosniff"
            }
          }
          handle_errors {
            rewrite 500.html
          }
-        }
-        " | caddy run --adapter caddyfile --config -'
-
+        }" | caddy run --adapter caddyfile --config -

 # using SmartOS native zone mastodon-redis, lx-brand redis crashes regularly,
 # upstream bug: https://github.com/redis/redis/issues/8861
@ -135,7 +136,7 @@ services:
 #      - triton.cns.services=mastodon-redis

  web:
-    image: tootsuite/mastodon:v4.1.2
+    image: tootsuite/mastodon:v4.1.4
    mem_limit: 1g
    restart: always
    env_file: .env.production
@ -148,7 +149,7 @@ services:
      - triton.cns.services=mastodon-web

  streaming:
-    image: tootsuite/mastodon:v4.1.2
+    image: tootsuite/mastodon:v4.1.4
    mem_limit: 1g
    restart: always
    env_file: .env.production
@ -161,7 +162,7 @@ services:
      - triton.cns.services=mastodon-streaming

  sidekiq:
-    image: tootsuite/mastodon:v4.1.2
+    image: tootsuite/mastodon:v4.1.4
    mem_limit: 1g
    restart: always
    env_file: .env.production
@ -171,7 +172,7 @@ services:
      - triton.cns.services=mastodon-sidekiq

  elasticsearch:
-    image: docker.elastic.co/elasticsearch/elasticsearch:7.17.9
+    image: docker.elastic.co/elasticsearch/elasticsearch:7.17.11
    mem_limit: 512m
    restart: always
    environment:
--- a/nextcloud/README.md
+++ b/nextcloud/README.md
@ -3,7 +3,7 @@
 `nix flake --help` should give you some output, then we're good to go.

 ```
-git clone https://git.b12f.io/pub-solar/infra
+git clone https://git.pub.solar/pub-solar/infra
 cd infra
 nix develop --command zsh
 ```
--- a/terraform/README.md
+++ b/terraform/README.md
@ -3,7 +3,7 @@
 `nix flake --help` should give you some output, then we're good to go.

 ```
-git clone https://git.b12f.io/pub-solar/infra
+git clone https://git.pub.solar/pub-solar/infra
 cd infra
 nix develop --command zsh
 ```