From 8f0057787a0f4fc06f8696b96731b43447143eb1 Mon Sep 17 00:00:00 2001 From: Earl Warren Date: Sun, 20 Oct 2024 18:05:04 +0200 Subject: [PATCH 01/14] k8s forgejo instance helpers --- .gitignore | 1 + k8s-forgejo.md | 59 ++++++------------ k8s-forgejo/forgejo-secrets.yml.example | 6 ++ k8s-forgejo/forgejo-values.yml | 36 +++++++++++ k8s-forgejo/next-values.yml | 81 +++++++++++++++++++++++++ 5 files changed, 141 insertions(+), 42 deletions(-) create mode 100644 k8s-forgejo/forgejo-secrets.yml.example create mode 100644 k8s-forgejo/forgejo-values.yml create mode 100644 k8s-forgejo/next-values.yml diff --git a/.gitignore b/.gitignore index 0178e91..a805a90 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ *~ k3s-host/secrets.sh k3s-host/variables.sh +k8s-forgejo/*-secrets.yml diff --git a/k8s-forgejo.md b/k8s-forgejo.md index 772e1f8..0f2742d 100644 --- a/k8s-forgejo.md +++ b/k8s-forgejo.md @@ -1,48 +1,23 @@ -## Forgejo +# Forgejo k8s instance -[forgejo](https://code.forgejo.org/forgejo-helm/forgejo-helm) configuration in [ingress](https://code.forgejo.org/forgejo-helm/forgejo-helm#ingress) for the reverse proxy (`traefik`) to route the domain and for the ACME issuer (`cert-manager`) to obtain a certificate. And in [service](https://code.forgejo.org/forgejo-helm/forgejo-helm#service) for the `ssh` port to be bound to the desired IPs of the load balancer (`metallb`). +[forgejo](https://code.forgejo.org/forgejo-helm/forgejo-helm) configuration in [ingress](https://code.forgejo.org/forgejo-helm/forgejo-helm#ingress) for the reverse proxy (`traefik`) to route the domain and for the ACME issuer (`cert-manager`) to obtain a certificate. And in [service](https://code.forgejo.org/forgejo-helm/forgejo-helm#service) for the `ssh` port to be bound to the desired IPs of the load balancer (`metallb`). A [PVC](https://code.forgejo.org/forgejo-helm/forgejo-helm#persistence) is created on the networked storage. -``` -ingress: - enabled: true - annotations: - # https://cert-manager.io/docs/usage/ingress/#supported-annotations - # https://github.com/cert-manager/cert-manager/issues/2239 - cert-manager.io/cluster-issuer: letsencrypt-http - cert-manager.io/private-key-algorithm: ECDSA - cert-manager.io/private-key-size: 384 - kubernetes.io/ingress.class: traefik - traefik.ingress.kubernetes.io/router.entrypoints: websecure - tls: - - hosts: - - t1.forgejo.org - secretName: tls-forgejo-t1-ingress-http - hosts: - - host: t1.forgejo.org - paths: - - path: / - pathType: Prefix +## Secrets -service: - http: - type: ClusterIP - ipFamilyPolicy: PreferDualStack - port: 3000 - ssh: - type: LoadBalancer - annotations: - metallb.universe.tf/loadBalancerIPs: 188.40.16.47,2a01:4f8:fff2:48::2 - metallb.universe.tf/allow-shared-ip: "key-to-share-failover" - ipFamilyPolicy: PreferDualStack - port: 2222 -``` +### New -[Instruct the forgejo pod](https://code.forgejo.org/forgejo-helm/forgejo-helm#persistence) to use the `forgejo-data` pvc. +- `cp forgejo-secrets.yml.example $name-secrets.yml` +- edit +- `kubectl create secret generic forgejo-$name-secrets --from-file=value=$name-secrets.yml` -```yaml -persistence: - enabled: true - create: false - claimName: forgejo-data -``` +### Existing +- `kubectl get secret forgejo-$name-secrets -o json | jq -r '.data.value' | base64 -d > $name-secrets.yml` + +## Storage + +- `../k3s-host/setup.sh setup_k8s_pvc forgejo-$name 4Gi 1000` + +## Pod + +- `../k3s-host/subst.sh forgejo-values.yml | helm upgrade forgejo-$name -f - -f $name-values.yml -f $name-secrets.yml oci://code.forgejo.org/forgejo-helm/forgejo --atomic --wait --install` diff --git a/k8s-forgejo/forgejo-secrets.yml.example b/k8s-forgejo/forgejo-secrets.yml.example new file mode 100644 index 0000000..a368c36 --- /dev/null +++ b/k8s-forgejo/forgejo-secrets.yml.example @@ -0,0 +1,6 @@ +gitea: + admin: + password: "***" + config: + mailer: + PASSWD: "***" diff --git a/k8s-forgejo/forgejo-values.yml b/k8s-forgejo/forgejo-values.yml new file mode 100644 index 0000000..c6cf783 --- /dev/null +++ b/k8s-forgejo/forgejo-values.yml @@ -0,0 +1,36 @@ +strategy: + type: 'Recreate' + +ingress: + enabled: true + annotations: + # https://cert-manager.io/docs/usage/ingress/#supported-annotations + # https://github.com/cert-manager/cert-manager/issues/2239 + cert-manager.io/cluster-issuer: letsencrypt-http + cert-manager.io/private-key-algorithm: ECDSA + cert-manager.io/private-key-size: 384 + kubernetes.io/ingress.class: traefik + traefik.ingress.kubernetes.io/router.entrypoints: websecure + +service: + http: + type: ClusterIP + ipFamilyPolicy: PreferDualStack + port: 3000 + ssh: + type: LoadBalancer + annotations: + metallb.universe.tf/loadBalancerIPs: $failover_ipv4,$failover_ipv6 + metallb.universe.tf/allow-shared-ip: "key-to-share-failover" + ipFamilyPolicy: PreferDualStack + +redis-cluster: + enabled: false +postgresql: + enabled: false +postgresql-ha: + enabled: false + +persistence: + enabled: true + create: false diff --git a/k8s-forgejo/next-values.yml b/k8s-forgejo/next-values.yml new file mode 100644 index 0000000..ca471b1 --- /dev/null +++ b/k8s-forgejo/next-values.yml @@ -0,0 +1,81 @@ +image: + registry: codeberg.org + repository: forgejo-experimental/forgejo + tag: '8.0-test' + rootless: false + +ingress: + tls: + - hosts: + - next.forgejo.org + secretName: tls-forgejo-next-ingress-http + hosts: + - host: next.forgejo.org + paths: + - path: / + pathType: Prefix + +service: + ssh: + port: 2020 + +persistence: + claimName: forgejo-next + +gitea: + admin: + username: earl-warren + email: 'contact@earl-warren.org' + config: + APP_NAME: "Forgejo v8.0 demo" + APP_SLOGAN: "ARCHIVED USE v8.next.forgejo.org instead" + APP_DISPLAY_NAME_FORMAT: "{APP_NAME} [{APP_SLOGAN}]" + log: + LEVEL: "info" + server: + ROOT_URL: https://next.forgejo.org/ + DOMAIN: next.forgejo.org + SSH_DOMAIN: next.forgejo.org + SSH_PORT: "2020" + LFS_START_SERVER: true + OFFLINE_MODE: true + repository: + ROOT: /data/git/repositories + service: + REGISTER_EMAIL_CONFIRM: true + DEFAULT_KEEP_EMAIL_PRIVATE: true + ENABLE_NOTIFY_MAIL: true + DISABLE_REGISTRATION: true + actions: + ENABLED: false + mirror: + ENABLED: false + federation: + ENABLED: true + admin: + SEND_NOTIFICATION_EMAIL_ON_NEW_USER: true + cors: + ENABLED: true + ALLOW_DOMAIN: "*" + HEADERS: "Access-Control-Allow-Origin" + mailer: + ENABLED: true + FROM: "noreply@forgejo.org" + PROTOCOL: "smtp+starttls" + SMTP_ADDR: "ssl0.ovh.net" + SMTP_PORT: "587" + USER: "next@forgejo.org" + database: + PATH: /data/gitea.db + DB_TYPE: sqlite3 + session: + PROVIDER: db + cache: + ADAPTER: memory + queue: + TYPE: level + indexer: + REPO_INDEXER_ENABLED: true + cron.archive_cleanup: + SCHEDULE: "@hourly" + OLDER_THAN: "2h" From e28e53589ad001fd480242b045b5ddf4936adb5d Mon Sep 17 00:00:00 2001 From: Earl Warren Date: Mon, 21 Oct 2024 14:35:08 +0200 Subject: [PATCH 02/14] enable traefik access logs --- k3s-host/traefik.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/k3s-host/traefik.yml b/k3s-host/traefik.yml index 5144452..fcf6ecb 100644 --- a/k3s-host/traefik.yml +++ b/k3s-host/traefik.yml @@ -17,3 +17,6 @@ spec: annotations: metallb.universe.tf/allow-shared-ip: "key-to-share-failover" metallb.universe.tf/loadBalancerIPs: $failover_ipv4,$failover_ipv6 + logs: + access: + enabled: true From de28b83d383f67f0ece64aee1c569bcba9fc4969 Mon Sep 17 00:00:00 2001 From: Earl Warren Date: Mon, 21 Oct 2024 20:18:46 +0200 Subject: [PATCH 03/14] pin k3s version & traefik version --- k3s-host/setup.sh | 2 ++ k8s.md | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/k3s-host/setup.sh b/k3s-host/setup.sh index 97ddf34..3ad54ff 100755 --- a/k3s-host/setup.sh +++ b/k3s-host/setup.sh @@ -9,6 +9,8 @@ else set -e fi +export INSTALL_K3S_VERSION=v1.30.5+k3s1 + source $SELF_DIR/variables.sh source $SELF_DIR/secrets.sh diff --git a/k8s.md b/k8s.md index 888017a..1e8af4c 100644 --- a/k8s.md +++ b/k8s.md @@ -91,7 +91,7 @@ For the first node `./setup.sh setup_k8s`. For nodes joining the cluster `./setu - [metallb](https://metallb.universe.tf) instead of the default load balancer because it does not allow for a public IP different from the `k8s` node IP. `./setup.sh setup_k8s_metallb` -- [traefik](https://traefik.io/) requests with [annotations](https://github.com/traefik/traefik-helm-chart/blob/7a13fc8a61a6ad30fcec32eec497dab9d8aea686/traefik/values.yaml#L736) specific IPs from `metalldb`. +- [traefik](https://traefik.io/) [v2.10](https://doc.traefik.io/traefik/v2.10/) installed from the [v25.0](https://github.com/traefik/traefik-helm-chart/tree/v25.0.0) helm chart. `./setup.sh setup_k8s_traefik` - [cert-manager](https://cert-manager.io/). `./setup.sh setup_k8s_certmanager` From 0ee041fb98096cf17fe497efaa9d44f26f61275c Mon Sep 17 00:00:00 2001 From: Earl Warren Date: Mon, 21 Oct 2024 20:22:45 +0200 Subject: [PATCH 04/14] use traefik as a reverse proxy for ssh too It is more uniform. It also allows to set externalTrafficPolicy: Local with the benefit of logging the ip of the incoming connection. --- k3s-host/traefik.yml | 11 ++++++++--- k8s-forgejo/forgejo-values.yml | 7 +++---- k8s-forgejo/next-values.yml | 19 ++++++++++++++++++- 3 files changed, 29 insertions(+), 8 deletions(-) diff --git a/k3s-host/traefik.yml b/k3s-host/traefik.yml index fcf6ecb..6fc7f96 100644 --- a/k3s-host/traefik.yml +++ b/k3s-host/traefik.yml @@ -5,18 +5,23 @@ metadata: namespace: kube-system spec: valuesContent: |- + deployment: + replicas: 2 ports: web: port: 80 redirectTo: port: websecure priority: 1 - deployment: - replicas: 2 + ssh-next: + port: 2020 + exposedPort: 2020 + expose: true service: annotations: - metallb.universe.tf/allow-shared-ip: "key-to-share-failover" metallb.universe.tf/loadBalancerIPs: $failover_ipv4,$failover_ipv6 + spec: + externalTrafficPolicy: Local logs: access: enabled: true diff --git a/k8s-forgejo/forgejo-values.yml b/k8s-forgejo/forgejo-values.yml index c6cf783..d509abc 100644 --- a/k8s-forgejo/forgejo-values.yml +++ b/k8s-forgejo/forgejo-values.yml @@ -16,12 +16,11 @@ service: http: type: ClusterIP ipFamilyPolicy: PreferDualStack + clusterIP: ~ port: 3000 ssh: - type: LoadBalancer - annotations: - metallb.universe.tf/loadBalancerIPs: $failover_ipv4,$failover_ipv6 - metallb.universe.tf/allow-shared-ip: "key-to-share-failover" + type: ClusterIP + clusterIP: ~ ipFamilyPolicy: PreferDualStack redis-cluster: diff --git a/k8s-forgejo/next-values.yml b/k8s-forgejo/next-values.yml index ca471b1..b7bcdc8 100644 --- a/k8s-forgejo/next-values.yml +++ b/k8s-forgejo/next-values.yml @@ -17,7 +17,24 @@ ingress: service: ssh: - port: 2020 + port: 2222 + +extraDeploy: + # Route from traefik to forgejo + - apiVersion: traefik.io/v1alpha1 + kind: IngressRouteTCP + metadata: + name: forgejo-next-ssh + annotations: + kubernetes.io/ingress.class: traefik + spec: + entryPoints: + - ssh-next # name from traefik port + routes: + - match: HostSNI(`*`) + services: + - name: forgejo-next-ssh + port: 2222 # forgejo ssh port on kubernetes service persistence: claimName: forgejo-next From 1f13f6699e07d3293112b47b845377e30a61fddb Mon Sep 17 00:00:00 2001 From: Earl Warren Date: Mon, 21 Oct 2024 20:55:10 +0200 Subject: [PATCH 05/14] metallb provides just one IP for v6 & v4, not a range --- k3s-host/metallb.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/k3s-host/metallb.yml b/k3s-host/metallb.yml index 1a1e77c..a4bd506 100644 --- a/k3s-host/metallb.yml +++ b/k3s-host/metallb.yml @@ -4,5 +4,5 @@ metadata: name: first-pool spec: addresses: - - $failover_ipv4/$failover_ipv4_range - - $failover_ipv6/$failover_ipv6_range + - $failover_ipv4/32 + - $failover_ipv6/128 From 804b76931dc1536c15daa3b53bd695ba97171fed Mon Sep 17 00:00:00 2001 From: Earl Warren Date: Tue, 22 Oct 2024 09:58:07 +0200 Subject: [PATCH 06/14] otherwise it will probably be single stack after a complete rebuild --- k3s-host/traefik.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/k3s-host/traefik.yml b/k3s-host/traefik.yml index 6fc7f96..77685d2 100644 --- a/k3s-host/traefik.yml +++ b/k3s-host/traefik.yml @@ -22,6 +22,7 @@ spec: metallb.universe.tf/loadBalancerIPs: $failover_ipv4,$failover_ipv6 spec: externalTrafficPolicy: Local + ipFamilyPolicy: PreferDualStack logs: access: enabled: true From 6ca6d676d8938f3caf599a0e088906e89efc86ab Mon Sep 17 00:00:00 2001 From: Earl Warren Date: Tue, 22 Oct 2024 10:16:48 +0200 Subject: [PATCH 07/14] traefik: display the user agent in the access logs --- k3s-host/traefik.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/k3s-host/traefik.yml b/k3s-host/traefik.yml index 77685d2..64119f0 100644 --- a/k3s-host/traefik.yml +++ b/k3s-host/traefik.yml @@ -26,3 +26,8 @@ spec: logs: access: enabled: true + fields: + headers: + # https://github.com/traefik/traefik-helm-chart/blob/v25.0.0/traefik/values.yaml#L304-L308 + names: + User-Agent: keep From b40fd5bd3c4a9e6429eca2ef5ee1704aee462b5d Mon Sep 17 00:00:00 2001 From: Earl Warren Date: Tue, 22 Oct 2024 10:57:31 +0200 Subject: [PATCH 08/14] traefik: bump log to INFO so that it shows which middleware are loaded and how they are interpreted --- k3s-host/traefik.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/k3s-host/traefik.yml b/k3s-host/traefik.yml index 64119f0..c3a214f 100644 --- a/k3s-host/traefik.yml +++ b/k3s-host/traefik.yml @@ -24,6 +24,8 @@ spec: externalTrafficPolicy: Local ipFamilyPolicy: PreferDualStack logs: + general: + level: INFO access: enabled: true fields: From 0f9b5ff8e2a81337aae57dcb7d703aed50e744e0 Mon Sep 17 00:00:00 2001 From: Earl Warren Date: Tue, 22 Oct 2024 10:21:54 +0200 Subject: [PATCH 09/14] next.forgejo.org: no more than 10 request per second --- k8s-forgejo/next-values.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/k8s-forgejo/next-values.yml b/k8s-forgejo/next-values.yml index b7bcdc8..5921a90 100644 --- a/k8s-forgejo/next-values.yml +++ b/k8s-forgejo/next-values.yml @@ -5,6 +5,10 @@ image: rootless: false ingress: + annotations: + # https://doc.traefik.io/traefik/v2.10/routing/providers/kubernetes-ingress/#on-ingress + # reference middlewares via `-@kubernetescrd` + traefik.ingress.kubernetes.io/router.middlewares: default-forgejo-ratelimit@kubernetescrd tls: - hosts: - next.forgejo.org @@ -20,6 +24,15 @@ service: port: 2222 extraDeploy: + - apiVersion: traefik.io/v1alpha1 + kind: Middleware + metadata: + name: forgejo-ratelimit + spec: + # https://doc.traefik.io/traefik/v2.10/middlewares/http/ratelimit/ + rateLimit: + average: 10 + burst: 20 # Route from traefik to forgejo - apiVersion: traefik.io/v1alpha1 kind: IngressRouteTCP From 60ddffb5148bd0c7b29e172fb3072d1fa7d7ce12 Mon Sep 17 00:00:00 2001 From: Earl Warren Date: Tue, 22 Oct 2024 13:02:27 +0200 Subject: [PATCH 10/14] traefik: switch to manual install and pinning of v3 --- k3s-host/setup.sh | 15 ++++++--- k3s-host/traefik.yml | 65 +++++++++++++++++-------------------- k8s-forgejo/next-values.yml | 4 +-- k8s.md | 2 +- 4 files changed, 44 insertions(+), 42 deletions(-) diff --git a/k3s-host/setup.sh b/k3s-host/setup.sh index 3ad54ff..51ec9cc 100755 --- a/k3s-host/setup.sh +++ b/k3s-host/setup.sh @@ -9,8 +9,6 @@ else set -e fi -export INSTALL_K3S_VERSION=v1.30.5+k3s1 - source $SELF_DIR/variables.sh source $SELF_DIR/secrets.sh @@ -165,6 +163,12 @@ EOF if ! grep --quiet 'export KUBECONFIG' ~/.bashrc; then echo "export KUBECONFIG=/etc/rancher/k3s/k3s.yaml" >>~/.bashrc fi + # + # To upgrade, systemctl stop k3s before running this. A node + # that is already part of a cluster does not need the --token + # or --server so there is no need to provide the number of an + # existing node. + # if ! sudo systemctl --quiet is-active k3s; then args="" if test "$existing"; then @@ -177,7 +181,8 @@ EOF if test "$self_node" = $node_k8s_etcd; then args="$args --disable-apiserver --disable-controller-manager --disable-scheduler" fi - curl -fL https://get.k3s.io | sh -s - server $args --cluster-init --disable=servicelb --write-kubeconfig-mode=644 --node-ip=$node_k8s_ipv4_prefix.$self_node,$node_k8s_ipv6_prefix::$self_node $node_k8s_cidr --flannel-ipv6-masq + export INSTALL_K3S_VERSION=v1.30.5+k3s1 + curl -fL https://get.k3s.io | sh -s - server $args --cluster-init --disable=servicelb --disable=traefik --write-kubeconfig-mode=644 --node-ip=$node_k8s_ipv4_prefix.$self_node,$node_k8s_ipv6_prefix::$self_node $node_k8s_cidr --flannel-ipv6-masq if test "$self_node" = $node_k8s_etcd; then retry --times 20 -- kubectl taint nodes $(hostname) key1=value1:NoSchedule fi @@ -192,7 +197,9 @@ function setup_k8s_apply() { } function setup_k8s_traefik() { - setup_k8s_apply traefik.yml + # https://github.com/traefik/traefik-helm-chart?tab=readme-ov-file#deploying-traefik + helm repo add traefik https://traefik.github.io/charts + $SELF_DIR/subst.sh traefik.yml | helm upgrade --install --namespace kube-system traefik -f - --set installCRDs=true --version 32.1.1 traefik/traefik } function setup_k8s_nfs() { diff --git a/k3s-host/traefik.yml b/k3s-host/traefik.yml index c3a214f..6ded3d0 100644 --- a/k3s-host/traefik.yml +++ b/k3s-host/traefik.yml @@ -1,35 +1,30 @@ -apiVersion: helm.cattle.io/v1 -kind: HelmChartConfig -metadata: - name: traefik - namespace: kube-system -spec: - valuesContent: |- - deployment: - replicas: 2 - ports: - web: - port: 80 - redirectTo: - port: websecure - priority: 1 - ssh-next: - port: 2020 - exposedPort: 2020 - expose: true - service: - annotations: - metallb.universe.tf/loadBalancerIPs: $failover_ipv4,$failover_ipv6 - spec: - externalTrafficPolicy: Local - ipFamilyPolicy: PreferDualStack - logs: - general: - level: INFO - access: - enabled: true - fields: - headers: - # https://github.com/traefik/traefik-helm-chart/blob/v25.0.0/traefik/values.yaml#L304-L308 - names: - User-Agent: keep +deployment: + replicas: 2 +ports: + web: + port: 80 + redirectTo: + port: websecure + priority: 1 + ssh-next: + port: 2020 + exposedPort: 2020 + # https://github.com/traefik/traefik-helm-chart/blob/v32.1.1/traefik/values.yaml#L611-L614 + expose: + default: true +service: + annotations: + metallb.universe.tf/loadBalancerIPs: $failover_ipv4,$failover_ipv6 + spec: + externalTrafficPolicy: Local + ipFamilyPolicy: PreferDualStack +logs: + general: + level: INFO + access: + enabled: true + fields: + headers: + # https://github.com/traefik/traefik-helm-chart/blob/v32.1.1/traefik/values.yaml#L365-L369 + names: + User-Agent: keep diff --git a/k8s-forgejo/next-values.yml b/k8s-forgejo/next-values.yml index 5921a90..a95b242 100644 --- a/k8s-forgejo/next-values.yml +++ b/k8s-forgejo/next-values.yml @@ -6,7 +6,7 @@ image: ingress: annotations: - # https://doc.traefik.io/traefik/v2.10/routing/providers/kubernetes-ingress/#on-ingress + # https://doc.traefik.io/traefik/v3.1/routing/providers/kubernetes-ingress/#on-ingress # reference middlewares via `-@kubernetescrd` traefik.ingress.kubernetes.io/router.middlewares: default-forgejo-ratelimit@kubernetescrd tls: @@ -29,7 +29,7 @@ extraDeploy: metadata: name: forgejo-ratelimit spec: - # https://doc.traefik.io/traefik/v2.10/middlewares/http/ratelimit/ + # https://doc.traefik.io/traefik/v3.1/middlewares/http/ratelimit/ rateLimit: average: 10 burst: 20 diff --git a/k8s.md b/k8s.md index 1e8af4c..3c1a63d 100644 --- a/k8s.md +++ b/k8s.md @@ -91,7 +91,7 @@ For the first node `./setup.sh setup_k8s`. For nodes joining the cluster `./setu - [metallb](https://metallb.universe.tf) instead of the default load balancer because it does not allow for a public IP different from the `k8s` node IP. `./setup.sh setup_k8s_metallb` -- [traefik](https://traefik.io/) [v2.10](https://doc.traefik.io/traefik/v2.10/) installed from the [v25.0](https://github.com/traefik/traefik-helm-chart/tree/v25.0.0) helm chart. +- [traefik](https://traefik.io/) [v2.10](https://doc.traefik.io/traefik/v3.1/) installed from the [v25.0](https://github.com/traefik/traefik-helm-chart/tree/v31.1.1) helm chart. `./setup.sh setup_k8s_traefik` - [cert-manager](https://cert-manager.io/). `./setup.sh setup_k8s_certmanager` From e0f3e624bccdc37ef00530c5eb9fd53ad47b24e0 Mon Sep 17 00:00:00 2001 From: Earl Warren Date: Tue, 22 Oct 2024 16:54:34 +0200 Subject: [PATCH 11/14] move versions to file variable for renovate convenience --- k3s-host/setup.sh | 5 ++--- k3s-host/variables.sh.example | 3 +++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/k3s-host/setup.sh b/k3s-host/setup.sh index 51ec9cc..9bc14d5 100755 --- a/k3s-host/setup.sh +++ b/k3s-host/setup.sh @@ -181,7 +181,7 @@ EOF if test "$self_node" = $node_k8s_etcd; then args="$args --disable-apiserver --disable-controller-manager --disable-scheduler" fi - export INSTALL_K3S_VERSION=v1.30.5+k3s1 + export INSTALL_K3S_VERSION=$K3S_VERSION curl -fL https://get.k3s.io | sh -s - server $args --cluster-init --disable=servicelb --disable=traefik --write-kubeconfig-mode=644 --node-ip=$node_k8s_ipv4_prefix.$self_node,$node_k8s_ipv6_prefix::$self_node $node_k8s_cidr --flannel-ipv6-masq if test "$self_node" = $node_k8s_etcd; then retry --times 20 -- kubectl taint nodes $(hostname) key1=value1:NoSchedule @@ -198,8 +198,7 @@ function setup_k8s_apply() { function setup_k8s_traefik() { # https://github.com/traefik/traefik-helm-chart?tab=readme-ov-file#deploying-traefik - helm repo add traefik https://traefik.github.io/charts - $SELF_DIR/subst.sh traefik.yml | helm upgrade --install --namespace kube-system traefik -f - --set installCRDs=true --version 32.1.1 traefik/traefik + $SELF_DIR/subst.sh traefik.yml | helm upgrade --install --namespace kube-system traefik -f - --set installCRDs=true --version $TRAEFIK_VERSION oci://ghcr.io/traefik/helm/traefik } function setup_k8s_nfs() { diff --git a/k3s-host/variables.sh.example b/k3s-host/variables.sh.example index 81b64b3..ec7c578 100755 --- a/k3s-host/variables.sh.example +++ b/k3s-host/variables.sh.example @@ -1,5 +1,8 @@ #!/bin/bash +K3S_VERSION=v1.30.5+k3s1 +TRAEFIK_VERSION=32.1.1 + nodes="5 6" node_interface=( From f5861bf00061226bb48dc809d29569b8f3c80f4b Mon Sep 17 00:00:00 2001 From: Earl Warren Date: Tue, 22 Oct 2024 17:06:39 +0200 Subject: [PATCH 12/14] traefik: apply forgejo-ratelimit once --- k3s-host/setup.sh | 1 + k3s-host/traefik-rate-limit.yml | 9 +++++++++ k8s-forgejo/next-values.yml | 9 --------- 3 files changed, 10 insertions(+), 9 deletions(-) create mode 100644 k3s-host/traefik-rate-limit.yml diff --git a/k3s-host/setup.sh b/k3s-host/setup.sh index 9bc14d5..8226d9f 100755 --- a/k3s-host/setup.sh +++ b/k3s-host/setup.sh @@ -199,6 +199,7 @@ function setup_k8s_apply() { function setup_k8s_traefik() { # https://github.com/traefik/traefik-helm-chart?tab=readme-ov-file#deploying-traefik $SELF_DIR/subst.sh traefik.yml | helm upgrade --install --namespace kube-system traefik -f - --set installCRDs=true --version $TRAEFIK_VERSION oci://ghcr.io/traefik/helm/traefik + setup_k8s_apply traefik-rate-limit.yml } function setup_k8s_nfs() { diff --git a/k3s-host/traefik-rate-limit.yml b/k3s-host/traefik-rate-limit.yml new file mode 100644 index 0000000..ef00a4d --- /dev/null +++ b/k3s-host/traefik-rate-limit.yml @@ -0,0 +1,9 @@ +apiVersion: traefik.io/v1alpha1 +kind: Middleware +metadata: + name: forgejo-ratelimit +spec: + # https://doc.traefik.io/traefik/v3.1/middlewares/http/ratelimit/ + rateLimit: + average: 10 + burst: 20 diff --git a/k8s-forgejo/next-values.yml b/k8s-forgejo/next-values.yml index a95b242..08fdd81 100644 --- a/k8s-forgejo/next-values.yml +++ b/k8s-forgejo/next-values.yml @@ -24,15 +24,6 @@ service: port: 2222 extraDeploy: - - apiVersion: traefik.io/v1alpha1 - kind: Middleware - metadata: - name: forgejo-ratelimit - spec: - # https://doc.traefik.io/traefik/v3.1/middlewares/http/ratelimit/ - rateLimit: - average: 10 - burst: 20 # Route from traefik to forgejo - apiVersion: traefik.io/v1alpha1 kind: IngressRouteTCP From 92f39f169dade799de04cda09a4d62eb712a80d4 Mon Sep 17 00:00:00 2001 From: Earl Warren Date: Tue, 22 Oct 2024 12:59:33 +0200 Subject: [PATCH 13/14] next.forgejo.org: block depending on user agent --- k8s-forgejo.md | 2 +- k8s-forgejo/crawler-block-values.yml | 32 ++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 k8s-forgejo/crawler-block-values.yml diff --git a/k8s-forgejo.md b/k8s-forgejo.md index 0f2742d..3ab2bfb 100644 --- a/k8s-forgejo.md +++ b/k8s-forgejo.md @@ -20,4 +20,4 @@ ## Pod -- `../k3s-host/subst.sh forgejo-values.yml | helm upgrade forgejo-$name -f - -f $name-values.yml -f $name-secrets.yml oci://code.forgejo.org/forgejo-helm/forgejo --atomic --wait --install` +- `../k3s-host/subst.sh forgejo-values.yml | helm upgrade forgejo-$name -f - -f $name-values.yml -f crawler-block-values.yml -f $name-secrets.yml oci://code.forgejo.org/forgejo-helm/forgejo --atomic --wait --install` diff --git a/k8s-forgejo/crawler-block-values.yml b/k8s-forgejo/crawler-block-values.yml new file mode 100644 index 0000000..9b6610d --- /dev/null +++ b/k8s-forgejo/crawler-block-values.yml @@ -0,0 +1,32 @@ +extraDeploy: + - apiVersion: traefik.io/v1alpha1 + # https://doc.traefik.io/traefik/v3.1/routing/providers/kubernetes-crd/#kind-ingressroute + kind: IngressRoute + metadata: + name: forgejo-crawler + annotations: + kubernetes.io/ingress.class: traefik + spec: + entryPoints: + - web + - websecure + routes: + # https://doc.traefik.io/traefik/v3.1/routing/routers/#rule + - match: Host(`next.forgejo.org`) && HeaderRegexp(`user-agent`, `DataForSeoBot`) + kind: Rule + priority: 1000 + services: + - name: noop@internal + kind: TraefikService + middlewares: + - name: forgejo-crawler-blocker + tls: + secretName: tls-forgejo-next-ingress-http + - apiVersion: traefik.io/v1alpha1 + kind: Middleware + metadata: + name: forgejo-crawler-blocker + spec: + ipAllowList: + sourceRange: + - 127.0.0.1/32 From 5f8969ee0ee0b985c5e27d970cdc458a25a807fa Mon Sep 17 00:00:00 2001 From: Earl Warren Date: Tue, 22 Oct 2024 18:54:06 +0200 Subject: [PATCH 14/14] @viceice review comments --- k3s-host/setup.sh | 2 +- k8s-forgejo/forgejo-values.yml | 1 - k8s-forgejo/next-values.yml | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/k3s-host/setup.sh b/k3s-host/setup.sh index 8226d9f..2064583 100755 --- a/k3s-host/setup.sh +++ b/k3s-host/setup.sh @@ -182,7 +182,7 @@ EOF args="$args --disable-apiserver --disable-controller-manager --disable-scheduler" fi export INSTALL_K3S_VERSION=$K3S_VERSION - curl -fL https://get.k3s.io | sh -s - server $args --cluster-init --disable=servicelb --disable=traefik --write-kubeconfig-mode=644 --node-ip=$node_k8s_ipv4_prefix.$self_node,$node_k8s_ipv6_prefix::$self_node $node_k8s_cidr --flannel-ipv6-masq + curl -fL https://get.k3s.io | sh -s - server $args --cluster-init --disable=servicelb --disable=traefik --write-kubeconfig-mode=600 --node-ip=$node_k8s_ipv4_prefix.$self_node,$node_k8s_ipv6_prefix::$self_node $node_k8s_cidr --flannel-ipv6-masq if test "$self_node" = $node_k8s_etcd; then retry --times 20 -- kubectl taint nodes $(hostname) key1=value1:NoSchedule fi diff --git a/k8s-forgejo/forgejo-values.yml b/k8s-forgejo/forgejo-values.yml index d509abc..b5b644c 100644 --- a/k8s-forgejo/forgejo-values.yml +++ b/k8s-forgejo/forgejo-values.yml @@ -17,7 +17,6 @@ service: type: ClusterIP ipFamilyPolicy: PreferDualStack clusterIP: ~ - port: 3000 ssh: type: ClusterIP clusterIP: ~ diff --git a/k8s-forgejo/next-values.yml b/k8s-forgejo/next-values.yml index 08fdd81..7cd9595 100644 --- a/k8s-forgejo/next-values.yml +++ b/k8s-forgejo/next-values.yml @@ -21,7 +21,7 @@ ingress: service: ssh: - port: 2222 + port: ssh extraDeploy: # Route from traefik to forgejo