diff --git a/.gitignore b/.gitignore index 0178e91..a805a90 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ *~ k3s-host/secrets.sh k3s-host/variables.sh +k8s-forgejo/*-secrets.yml diff --git a/k3s-host/metallb.yml b/k3s-host/metallb.yml index 1a1e77c..a4bd506 100644 --- a/k3s-host/metallb.yml +++ b/k3s-host/metallb.yml @@ -4,5 +4,5 @@ metadata: name: first-pool spec: addresses: - - $failover_ipv4/$failover_ipv4_range - - $failover_ipv6/$failover_ipv6_range + - $failover_ipv4/32 + - $failover_ipv6/128 diff --git a/k3s-host/setup.sh b/k3s-host/setup.sh index 97ddf34..2064583 100755 --- a/k3s-host/setup.sh +++ b/k3s-host/setup.sh @@ -163,6 +163,12 @@ EOF if ! grep --quiet 'export KUBECONFIG' ~/.bashrc; then echo "export KUBECONFIG=/etc/rancher/k3s/k3s.yaml" >>~/.bashrc fi + # + # To upgrade, systemctl stop k3s before running this. A node + # that is already part of a cluster does not need the --token + # or --server so there is no need to provide the number of an + # existing node. + # if ! sudo systemctl --quiet is-active k3s; then args="" if test "$existing"; then @@ -175,7 +181,8 @@ EOF if test "$self_node" = $node_k8s_etcd; then args="$args --disable-apiserver --disable-controller-manager --disable-scheduler" fi - curl -fL https://get.k3s.io | sh -s - server $args --cluster-init --disable=servicelb --write-kubeconfig-mode=644 --node-ip=$node_k8s_ipv4_prefix.$self_node,$node_k8s_ipv6_prefix::$self_node $node_k8s_cidr --flannel-ipv6-masq + export INSTALL_K3S_VERSION=$K3S_VERSION + curl -fL https://get.k3s.io | sh -s - server $args --cluster-init --disable=servicelb --disable=traefik --write-kubeconfig-mode=600 --node-ip=$node_k8s_ipv4_prefix.$self_node,$node_k8s_ipv6_prefix::$self_node $node_k8s_cidr --flannel-ipv6-masq if test "$self_node" = $node_k8s_etcd; then retry --times 20 -- kubectl taint nodes $(hostname) key1=value1:NoSchedule fi @@ -190,7 +197,9 @@ function setup_k8s_apply() { } function setup_k8s_traefik() { - setup_k8s_apply traefik.yml + # https://github.com/traefik/traefik-helm-chart?tab=readme-ov-file#deploying-traefik + $SELF_DIR/subst.sh traefik.yml | helm upgrade --install --namespace kube-system traefik -f - --set installCRDs=true --version $TRAEFIK_VERSION oci://ghcr.io/traefik/helm/traefik + setup_k8s_apply traefik-rate-limit.yml } function setup_k8s_nfs() { diff --git a/k3s-host/traefik-rate-limit.yml b/k3s-host/traefik-rate-limit.yml new file mode 100644 index 0000000..ef00a4d --- /dev/null +++ b/k3s-host/traefik-rate-limit.yml @@ -0,0 +1,9 @@ +apiVersion: traefik.io/v1alpha1 +kind: Middleware +metadata: + name: forgejo-ratelimit +spec: + # https://doc.traefik.io/traefik/v3.1/middlewares/http/ratelimit/ + rateLimit: + average: 10 + burst: 20 diff --git a/k3s-host/traefik.yml b/k3s-host/traefik.yml index 5144452..6ded3d0 100644 --- a/k3s-host/traefik.yml +++ b/k3s-host/traefik.yml @@ -1,19 +1,30 @@ -apiVersion: helm.cattle.io/v1 -kind: HelmChartConfig -metadata: - name: traefik - namespace: kube-system -spec: - valuesContent: |- - ports: - web: - port: 80 - redirectTo: - port: websecure - priority: 1 - deployment: - replicas: 2 - service: - annotations: - metallb.universe.tf/allow-shared-ip: "key-to-share-failover" - metallb.universe.tf/loadBalancerIPs: $failover_ipv4,$failover_ipv6 +deployment: + replicas: 2 +ports: + web: + port: 80 + redirectTo: + port: websecure + priority: 1 + ssh-next: + port: 2020 + exposedPort: 2020 + # https://github.com/traefik/traefik-helm-chart/blob/v32.1.1/traefik/values.yaml#L611-L614 + expose: + default: true +service: + annotations: + metallb.universe.tf/loadBalancerIPs: $failover_ipv4,$failover_ipv6 + spec: + externalTrafficPolicy: Local + ipFamilyPolicy: PreferDualStack +logs: + general: + level: INFO + access: + enabled: true + fields: + headers: + # https://github.com/traefik/traefik-helm-chart/blob/v32.1.1/traefik/values.yaml#L365-L369 + names: + User-Agent: keep diff --git a/k3s-host/variables.sh.example b/k3s-host/variables.sh.example index 81b64b3..ec7c578 100755 --- a/k3s-host/variables.sh.example +++ b/k3s-host/variables.sh.example @@ -1,5 +1,8 @@ #!/bin/bash +K3S_VERSION=v1.30.5+k3s1 +TRAEFIK_VERSION=32.1.1 + nodes="5 6" node_interface=( diff --git a/k8s-forgejo.md b/k8s-forgejo.md index 772e1f8..3ab2bfb 100644 --- a/k8s-forgejo.md +++ b/k8s-forgejo.md @@ -1,48 +1,23 @@ -## Forgejo +# Forgejo k8s instance -[forgejo](https://code.forgejo.org/forgejo-helm/forgejo-helm) configuration in [ingress](https://code.forgejo.org/forgejo-helm/forgejo-helm#ingress) for the reverse proxy (`traefik`) to route the domain and for the ACME issuer (`cert-manager`) to obtain a certificate. And in [service](https://code.forgejo.org/forgejo-helm/forgejo-helm#service) for the `ssh` port to be bound to the desired IPs of the load balancer (`metallb`). +[forgejo](https://code.forgejo.org/forgejo-helm/forgejo-helm) configuration in [ingress](https://code.forgejo.org/forgejo-helm/forgejo-helm#ingress) for the reverse proxy (`traefik`) to route the domain and for the ACME issuer (`cert-manager`) to obtain a certificate. And in [service](https://code.forgejo.org/forgejo-helm/forgejo-helm#service) for the `ssh` port to be bound to the desired IPs of the load balancer (`metallb`). A [PVC](https://code.forgejo.org/forgejo-helm/forgejo-helm#persistence) is created on the networked storage. -``` -ingress: - enabled: true - annotations: - # https://cert-manager.io/docs/usage/ingress/#supported-annotations - # https://github.com/cert-manager/cert-manager/issues/2239 - cert-manager.io/cluster-issuer: letsencrypt-http - cert-manager.io/private-key-algorithm: ECDSA - cert-manager.io/private-key-size: 384 - kubernetes.io/ingress.class: traefik - traefik.ingress.kubernetes.io/router.entrypoints: websecure - tls: - - hosts: - - t1.forgejo.org - secretName: tls-forgejo-t1-ingress-http - hosts: - - host: t1.forgejo.org - paths: - - path: / - pathType: Prefix +## Secrets -service: - http: - type: ClusterIP - ipFamilyPolicy: PreferDualStack - port: 3000 - ssh: - type: LoadBalancer - annotations: - metallb.universe.tf/loadBalancerIPs: 188.40.16.47,2a01:4f8:fff2:48::2 - metallb.universe.tf/allow-shared-ip: "key-to-share-failover" - ipFamilyPolicy: PreferDualStack - port: 2222 -``` +### New -[Instruct the forgejo pod](https://code.forgejo.org/forgejo-helm/forgejo-helm#persistence) to use the `forgejo-data` pvc. +- `cp forgejo-secrets.yml.example $name-secrets.yml` +- edit +- `kubectl create secret generic forgejo-$name-secrets --from-file=value=$name-secrets.yml` -```yaml -persistence: - enabled: true - create: false - claimName: forgejo-data -``` +### Existing +- `kubectl get secret forgejo-$name-secrets -o json | jq -r '.data.value' | base64 -d > $name-secrets.yml` + +## Storage + +- `../k3s-host/setup.sh setup_k8s_pvc forgejo-$name 4Gi 1000` + +## Pod + +- `../k3s-host/subst.sh forgejo-values.yml | helm upgrade forgejo-$name -f - -f $name-values.yml -f crawler-block-values.yml -f $name-secrets.yml oci://code.forgejo.org/forgejo-helm/forgejo --atomic --wait --install` diff --git a/k8s-forgejo/crawler-block-values.yml b/k8s-forgejo/crawler-block-values.yml new file mode 100644 index 0000000..9b6610d --- /dev/null +++ b/k8s-forgejo/crawler-block-values.yml @@ -0,0 +1,32 @@ +extraDeploy: + - apiVersion: traefik.io/v1alpha1 + # https://doc.traefik.io/traefik/v3.1/routing/providers/kubernetes-crd/#kind-ingressroute + kind: IngressRoute + metadata: + name: forgejo-crawler + annotations: + kubernetes.io/ingress.class: traefik + spec: + entryPoints: + - web + - websecure + routes: + # https://doc.traefik.io/traefik/v3.1/routing/routers/#rule + - match: Host(`next.forgejo.org`) && HeaderRegexp(`user-agent`, `DataForSeoBot`) + kind: Rule + priority: 1000 + services: + - name: noop@internal + kind: TraefikService + middlewares: + - name: forgejo-crawler-blocker + tls: + secretName: tls-forgejo-next-ingress-http + - apiVersion: traefik.io/v1alpha1 + kind: Middleware + metadata: + name: forgejo-crawler-blocker + spec: + ipAllowList: + sourceRange: + - 127.0.0.1/32 diff --git a/k8s-forgejo/forgejo-secrets.yml.example b/k8s-forgejo/forgejo-secrets.yml.example new file mode 100644 index 0000000..a368c36 --- /dev/null +++ b/k8s-forgejo/forgejo-secrets.yml.example @@ -0,0 +1,6 @@ +gitea: + admin: + password: "***" + config: + mailer: + PASSWD: "***" diff --git a/k8s-forgejo/forgejo-values.yml b/k8s-forgejo/forgejo-values.yml new file mode 100644 index 0000000..b5b644c --- /dev/null +++ b/k8s-forgejo/forgejo-values.yml @@ -0,0 +1,34 @@ +strategy: + type: 'Recreate' + +ingress: + enabled: true + annotations: + # https://cert-manager.io/docs/usage/ingress/#supported-annotations + # https://github.com/cert-manager/cert-manager/issues/2239 + cert-manager.io/cluster-issuer: letsencrypt-http + cert-manager.io/private-key-algorithm: ECDSA + cert-manager.io/private-key-size: 384 + kubernetes.io/ingress.class: traefik + traefik.ingress.kubernetes.io/router.entrypoints: websecure + +service: + http: + type: ClusterIP + ipFamilyPolicy: PreferDualStack + clusterIP: ~ + ssh: + type: ClusterIP + clusterIP: ~ + ipFamilyPolicy: PreferDualStack + +redis-cluster: + enabled: false +postgresql: + enabled: false +postgresql-ha: + enabled: false + +persistence: + enabled: true + create: false diff --git a/k8s-forgejo/next-values.yml b/k8s-forgejo/next-values.yml new file mode 100644 index 0000000..7cd9595 --- /dev/null +++ b/k8s-forgejo/next-values.yml @@ -0,0 +1,102 @@ +image: + registry: codeberg.org + repository: forgejo-experimental/forgejo + tag: '8.0-test' + rootless: false + +ingress: + annotations: + # https://doc.traefik.io/traefik/v3.1/routing/providers/kubernetes-ingress/#on-ingress + # reference middlewares via `-@kubernetescrd` + traefik.ingress.kubernetes.io/router.middlewares: default-forgejo-ratelimit@kubernetescrd + tls: + - hosts: + - next.forgejo.org + secretName: tls-forgejo-next-ingress-http + hosts: + - host: next.forgejo.org + paths: + - path: / + pathType: Prefix + +service: + ssh: + port: ssh + +extraDeploy: + # Route from traefik to forgejo + - apiVersion: traefik.io/v1alpha1 + kind: IngressRouteTCP + metadata: + name: forgejo-next-ssh + annotations: + kubernetes.io/ingress.class: traefik + spec: + entryPoints: + - ssh-next # name from traefik port + routes: + - match: HostSNI(`*`) + services: + - name: forgejo-next-ssh + port: 2222 # forgejo ssh port on kubernetes service + +persistence: + claimName: forgejo-next + +gitea: + admin: + username: earl-warren + email: 'contact@earl-warren.org' + config: + APP_NAME: "Forgejo v8.0 demo" + APP_SLOGAN: "ARCHIVED USE v8.next.forgejo.org instead" + APP_DISPLAY_NAME_FORMAT: "{APP_NAME} [{APP_SLOGAN}]" + log: + LEVEL: "info" + server: + ROOT_URL: https://next.forgejo.org/ + DOMAIN: next.forgejo.org + SSH_DOMAIN: next.forgejo.org + SSH_PORT: "2020" + LFS_START_SERVER: true + OFFLINE_MODE: true + repository: + ROOT: /data/git/repositories + service: + REGISTER_EMAIL_CONFIRM: true + DEFAULT_KEEP_EMAIL_PRIVATE: true + ENABLE_NOTIFY_MAIL: true + DISABLE_REGISTRATION: true + actions: + ENABLED: false + mirror: + ENABLED: false + federation: + ENABLED: true + admin: + SEND_NOTIFICATION_EMAIL_ON_NEW_USER: true + cors: + ENABLED: true + ALLOW_DOMAIN: "*" + HEADERS: "Access-Control-Allow-Origin" + mailer: + ENABLED: true + FROM: "noreply@forgejo.org" + PROTOCOL: "smtp+starttls" + SMTP_ADDR: "ssl0.ovh.net" + SMTP_PORT: "587" + USER: "next@forgejo.org" + database: + PATH: /data/gitea.db + DB_TYPE: sqlite3 + session: + PROVIDER: db + cache: + ADAPTER: memory + queue: + TYPE: level + indexer: + REPO_INDEXER_ENABLED: true + cron.archive_cleanup: + SCHEDULE: "@hourly" + OLDER_THAN: "2h" diff --git a/k8s.md b/k8s.md index 888017a..3c1a63d 100644 --- a/k8s.md +++ b/k8s.md @@ -91,7 +91,7 @@ For the first node `./setup.sh setup_k8s`. For nodes joining the cluster `./setu - [metallb](https://metallb.universe.tf) instead of the default load balancer because it does not allow for a public IP different from the `k8s` node IP. `./setup.sh setup_k8s_metallb` -- [traefik](https://traefik.io/) requests with [annotations](https://github.com/traefik/traefik-helm-chart/blob/7a13fc8a61a6ad30fcec32eec497dab9d8aea686/traefik/values.yaml#L736) specific IPs from `metalldb`. +- [traefik](https://traefik.io/) [v2.10](https://doc.traefik.io/traefik/v3.1/) installed from the [v25.0](https://github.com/traefik/traefik-helm-chart/tree/v31.1.1) helm chart. `./setup.sh setup_k8s_traefik` - [cert-manager](https://cert-manager.io/). `./setup.sh setup_k8s_certmanager`