Merge pull request 'k8s forgejo instance helpers' (#39) from earl-warren/documentation:wip-k8s-forgejo into main

Reviewed-on: https://code.forgejo.org/infrastructure/documentation/pulls/39
Reviewed-by: Michael Kriese <michael.kriese@gmx.de>

.gitignore

@@ -1,3 +1,4 @@
 *~
 k3s-host/secrets.sh
 k3s-host/variables.sh
+k8s-forgejo/*-secrets.yml

k3s-host/metallb.yml

@@ -4,5 +4,5 @@ metadata:
   name: first-pool
 spec:
   addresses:
-   - $failover_ipv4/$failover_ipv4_range
+   - $failover_ipv4/32
-   - $failover_ipv6/$failover_ipv6_range
+   - $failover_ipv6/128

k3s-host/setup.sh

@@ -163,6 +163,12 @@ EOF
 	if ! grep --quiet 'export KUBECONFIG' ~/.bashrc; then
 		echo "export KUBECONFIG=/etc/rancher/k3s/k3s.yaml" >>~/.bashrc
 	fi
+	#
+	# To upgrade, systemctl stop k3s before running this. A node
+	# that is already part of a cluster does not need the --token
+	# or --server so there is no need to provide the number of an
+	# existing node.
+	#
 	if ! sudo systemctl --quiet is-active k3s; then
 		args=""
 		if test "$existing"; then
@@ -175,7 +181,8 @@ EOF
 		if test "$self_node" = $node_k8s_etcd; then
 			args="$args --disable-apiserver --disable-controller-manager --disable-scheduler"
 		fi
-		curl -fL https://get.k3s.io | sh -s - server $args --cluster-init --disable=servicelb --write-kubeconfig-mode=644 --node-ip=$node_k8s_ipv4_prefix.$self_node,$node_k8s_ipv6_prefix::$self_node $node_k8s_cidr --flannel-ipv6-masq
+		export INSTALL_K3S_VERSION=$K3S_VERSION
+		curl -fL https://get.k3s.io | sh -s - server $args --cluster-init --disable=servicelb --disable=traefik --write-kubeconfig-mode=600 --node-ip=$node_k8s_ipv4_prefix.$self_node,$node_k8s_ipv6_prefix::$self_node $node_k8s_cidr --flannel-ipv6-masq
 		if test "$self_node" = $node_k8s_etcd; then
 			retry --times 20 -- kubectl taint nodes $(hostname) key1=value1:NoSchedule
 		fi
@@ -190,7 +197,9 @@ function setup_k8s_apply() {
 }
 
 function setup_k8s_traefik() {
-	setup_k8s_apply traefik.yml
+	# https://github.com/traefik/traefik-helm-chart?tab=readme-ov-file#deploying-traefik
+	$SELF_DIR/subst.sh traefik.yml | helm upgrade --install --namespace kube-system traefik -f - --set installCRDs=true --version $TRAEFIK_VERSION oci://ghcr.io/traefik/helm/traefik
+	setup_k8s_apply traefik-rate-limit.yml
 }
 
 function setup_k8s_nfs() {

k3s-host/traefik-rate-limit.yml

@@ -0,0 +1,9 @@
+apiVersion: traefik.io/v1alpha1
+kind: Middleware
+metadata:
+  name: forgejo-ratelimit
+spec:
+  # https://doc.traefik.io/traefik/v3.1/middlewares/http/ratelimit/
+  rateLimit:
+    average: 10
+    burst: 20

k3s-host/traefik.yml

@@ -1,19 +1,30 @@
-apiVersion: helm.cattle.io/v1
+deployment:
-kind: HelmChartConfig
+  replicas: 2
-metadata:
+ports:
-  name: traefik
+  web:
-  namespace: kube-system
+    port: 80
-spec:
+    redirectTo:
-  valuesContent: |-
+      port: websecure
-    ports:
+      priority: 1
-      web:
+  ssh-next:
-        port: 80
+    port: 2020
-        redirectTo:
+    exposedPort: 2020
-          port: websecure
+    # https://github.com/traefik/traefik-helm-chart/blob/v32.1.1/traefik/values.yaml#L611-L614
-          priority: 1
+    expose:
-        deployment:
+      default: true
-          replicas: 2
+service:
-    service:
+  annotations:
-      annotations:
+    metallb.universe.tf/loadBalancerIPs: $failover_ipv4,$failover_ipv6
-        metallb.universe.tf/allow-shared-ip: "key-to-share-failover"
+  spec:
-        metallb.universe.tf/loadBalancerIPs: $failover_ipv4,$failover_ipv6
+    externalTrafficPolicy: Local
+    ipFamilyPolicy: PreferDualStack
+logs:
+  general:
+    level: INFO
+  access:
+    enabled: true
+    fields:
+      headers:
+        # https://github.com/traefik/traefik-helm-chart/blob/v32.1.1/traefik/values.yaml#L365-L369
+        names:
+          User-Agent: keep

k3s-host/variables.sh.example

@@ -1,5 +1,8 @@
 #!/bin/bash
 
+K3S_VERSION=v1.30.5+k3s1
+TRAEFIK_VERSION=32.1.1
+
 nodes="5 6"
 
 node_interface=(

k8s-forgejo.md

@@ -1,48 +1,23 @@
-## Forgejo
+# Forgejo k8s instance
 
-[forgejo](https://code.forgejo.org/forgejo-helm/forgejo-helm) configuration in [ingress](https://code.forgejo.org/forgejo-helm/forgejo-helm#ingress) for the reverse proxy (`traefik`) to route the domain and for the ACME issuer (`cert-manager`) to obtain a certificate. And in [service](https://code.forgejo.org/forgejo-helm/forgejo-helm#service) for the `ssh` port to be bound to the desired IPs of the load balancer (`metallb`).
+[forgejo](https://code.forgejo.org/forgejo-helm/forgejo-helm) configuration in [ingress](https://code.forgejo.org/forgejo-helm/forgejo-helm#ingress) for the reverse proxy (`traefik`) to route the domain and for the ACME issuer (`cert-manager`) to obtain a certificate. And in [service](https://code.forgejo.org/forgejo-helm/forgejo-helm#service) for the `ssh` port to be bound to the desired IPs of the load balancer (`metallb`). A [PVC](https://code.forgejo.org/forgejo-helm/forgejo-helm#persistence) is created on the networked storage.
 
-```
+## Secrets
-ingress:
-  enabled: true
-  annotations:
-	# https://cert-manager.io/docs/usage/ingress/#supported-annotations
-	# https://github.com/cert-manager/cert-manager/issues/2239
-	cert-manager.io/cluster-issuer: letsencrypt-http
-	cert-manager.io/private-key-algorithm: ECDSA
-	cert-manager.io/private-key-size: 384
-	kubernetes.io/ingress.class: traefik
-	traefik.ingress.kubernetes.io/router.entrypoints: websecure
-  tls:
-	- hosts:
-		- t1.forgejo.org
-	  secretName: tls-forgejo-t1-ingress-http
-  hosts:
-	- host: t1.forgejo.org
-	  paths:
-		- path: /
-		  pathType: Prefix
 
-service:
+### New
-  http:
-	type: ClusterIP
-	ipFamilyPolicy: PreferDualStack
-	port: 3000
-  ssh:
-	type: LoadBalancer
-	annotations:
-	  metallb.universe.tf/loadBalancerIPs: 188.40.16.47,2a01:4f8:fff2:48::2
-	  metallb.universe.tf/allow-shared-ip: "key-to-share-failover"
-	ipFamilyPolicy: PreferDualStack
-	port: 2222
-```
 
-[Instruct the forgejo pod](https://code.forgejo.org/forgejo-helm/forgejo-helm#persistence) to use the `forgejo-data` pvc.
+- `cp forgejo-secrets.yml.example $name-secrets.yml`
+- edit
+- `kubectl create secret generic forgejo-$name-secrets --from-file=value=$name-secrets.yml`
 
-```yaml
+### Existing
-persistence:
-  enabled: true
-  create: false
-  claimName: forgejo-data
-```
 
+- `kubectl get secret forgejo-$name-secrets -o json | jq -r  '.data.value' | base64 -d > $name-secrets.yml`
+
+## Storage
+
+- `../k3s-host/setup.sh setup_k8s_pvc forgejo-$name 4Gi 1000`
+
+## Pod
+
+- `../k3s-host/subst.sh forgejo-values.yml | helm upgrade forgejo-$name -f - -f $name-values.yml -f crawler-block-values.yml -f $name-secrets.yml oci://code.forgejo.org/forgejo-helm/forgejo --atomic --wait --install`

k8s-forgejo/crawler-block-values.yml

@@ -0,0 +1,32 @@
+extraDeploy:
+  - apiVersion: traefik.io/v1alpha1
+    # https://doc.traefik.io/traefik/v3.1/routing/providers/kubernetes-crd/#kind-ingressroute
+    kind: IngressRoute
+    metadata:
+      name: forgejo-crawler
+      annotations:
+        kubernetes.io/ingress.class: traefik
+    spec:
+      entryPoints:
+        - web
+        - websecure
+      routes:
+        # https://doc.traefik.io/traefik/v3.1/routing/routers/#rule
+        - match: Host(`next.forgejo.org`) && HeaderRegexp(`user-agent`, `DataForSeoBot`)
+          kind: Rule
+          priority: 1000
+          services:
+            - name: noop@internal
+              kind: TraefikService
+          middlewares:
+            - name: forgejo-crawler-blocker
+      tls:
+        secretName: tls-forgejo-next-ingress-http
+  - apiVersion: traefik.io/v1alpha1
+    kind: Middleware
+    metadata:
+      name: forgejo-crawler-blocker
+    spec:
+      ipAllowList:
+        sourceRange:
+          - 127.0.0.1/32

k8s-forgejo/forgejo-secrets.yml.example

@@ -0,0 +1,6 @@
+gitea:
+  admin:
+    password: "***"
+  config:
+    mailer:
+      PASSWD: "***"

k8s-forgejo/forgejo-values.yml

@@ -0,0 +1,34 @@
+strategy:
+  type: 'Recreate'
+
+ingress:
+  enabled: true
+  annotations:
+    # https://cert-manager.io/docs/usage/ingress/#supported-annotations
+    # https://github.com/cert-manager/cert-manager/issues/2239
+    cert-manager.io/cluster-issuer: letsencrypt-http
+    cert-manager.io/private-key-algorithm: ECDSA
+    cert-manager.io/private-key-size: 384
+    kubernetes.io/ingress.class: traefik
+    traefik.ingress.kubernetes.io/router.entrypoints: websecure
+
+service:
+  http:
+    type: ClusterIP
+    ipFamilyPolicy: PreferDualStack
+    clusterIP: ~
+  ssh:
+    type: ClusterIP
+    clusterIP: ~
+    ipFamilyPolicy: PreferDualStack
+
+redis-cluster:
+  enabled: false
+postgresql:
+  enabled: false
+postgresql-ha:
+  enabled: false
+
+persistence:
+  enabled: true
+  create: false

k8s-forgejo/next-values.yml

@@ -0,0 +1,102 @@
+image:
+  registry: codeberg.org
+  repository: forgejo-experimental/forgejo
+  tag: '8.0-test'
+  rootless: false
+
+ingress:
+  annotations:
+    # https://doc.traefik.io/traefik/v3.1/routing/providers/kubernetes-ingress/#on-ingress
+    # reference middlewares via `<namspace>-<name>@kubernetescrd`
+    traefik.ingress.kubernetes.io/router.middlewares: default-forgejo-ratelimit@kubernetescrd
+  tls:
+    - hosts:
+        - next.forgejo.org
+      secretName: tls-forgejo-next-ingress-http
+  hosts:
+    - host: next.forgejo.org
+      paths:
+        - path: /
+          pathType: Prefix
+
+service:
+  ssh:
+    port: ssh
+
+extraDeploy:
+  # Route from traefik to forgejo
+  - apiVersion: traefik.io/v1alpha1
+    kind: IngressRouteTCP
+    metadata:
+      name: forgejo-next-ssh
+      annotations:
+        kubernetes.io/ingress.class: traefik
+    spec:
+      entryPoints:
+        - ssh-next # name from traefik port
+      routes:
+        - match: HostSNI(`*`)
+          services:
+            - name: forgejo-next-ssh
+              port: 2222 # forgejo ssh port on kubernetes service
+
+persistence:
+  claimName: forgejo-next
+
+gitea:
+  admin:
+    username: earl-warren
+    email: 'contact@earl-warren.org'
+  config:
+    APP_NAME: "Forgejo v8.0 demo"
+    APP_SLOGAN: "ARCHIVED USE v8.next.forgejo.org instead"
+    APP_DISPLAY_NAME_FORMAT: "{APP_NAME} [{APP_SLOGAN}]"
+    log:
+      LEVEL: "info"
+    server:
+      ROOT_URL: https://next.forgejo.org/
+      DOMAIN: next.forgejo.org
+      SSH_DOMAIN: next.forgejo.org
+      SSH_PORT: "2020"
+      LFS_START_SERVER: true
+      OFFLINE_MODE: true
+    repository:
+      ROOT: /data/git/repositories
+    service:
+      REGISTER_EMAIL_CONFIRM: true
+      DEFAULT_KEEP_EMAIL_PRIVATE: true
+      ENABLE_NOTIFY_MAIL: true
+      DISABLE_REGISTRATION: true
+    actions:
+      ENABLED: false
+    mirror:
+      ENABLED: false
+    federation:
+      ENABLED: true
+    admin:
+      SEND_NOTIFICATION_EMAIL_ON_NEW_USER: true
+    cors:
+      ENABLED: true
+      ALLOW_DOMAIN: "*"
+      HEADERS: "Access-Control-Allow-Origin"
+    mailer:
+      ENABLED: true
+      FROM: "noreply@forgejo.org"
+      PROTOCOL: "smtp+starttls"
+      SMTP_ADDR: "ssl0.ovh.net"
+      SMTP_PORT: "587"
+      USER: "next@forgejo.org"
+    database:
+      PATH: /data/gitea.db
+      DB_TYPE: sqlite3
+    session:
+      PROVIDER: db
+    cache:
+      ADAPTER: memory
+    queue:
+      TYPE: level
+    indexer:
+      REPO_INDEXER_ENABLED: true
+    cron.archive_cleanup:
+      SCHEDULE: "@hourly"
+      OLDER_THAN: "2h"

k8s.md

@@ -91,7 +91,7 @@ For the first node `./setup.sh setup_k8s`. For nodes joining the cluster `./setu
 
 - [metallb](https://metallb.universe.tf) instead of the default load balancer because it does not allow for a public IP different from the `k8s` node IP.
   `./setup.sh setup_k8s_metallb`
-- [traefik](https://traefik.io/) requests with [annotations](https://github.com/traefik/traefik-helm-chart/blob/7a13fc8a61a6ad30fcec32eec497dab9d8aea686/traefik/values.yaml#L736) specific IPs from `metalldb`.
+- [traefik](https://traefik.io/) [v2.10](https://doc.traefik.io/traefik/v3.1/) installed from the [v25.0](https://github.com/traefik/traefik-helm-chart/tree/v31.1.1) helm chart.
   `./setup.sh setup_k8s_traefik`
 - [cert-manager](https://cert-manager.io/).
   `./setup.sh setup_k8s_certmanager`