From fff8a9c2d882e5275cd3047a6ea9769b184af609 Mon Sep 17 00:00:00 2001 From: Earl Warren Date: Tue, 26 Nov 2024 12:30:42 +0000 Subject: [PATCH] obsolete the LXC/DRBD/nginx stack It all migrated to k8s --- README.md | 198 ---------------------------------------------- drbd-nginx-lxc.md | 177 ----------------------------------------- 2 files changed, 375 deletions(-) delete mode 100644 drbd-nginx-lxc.md diff --git a/README.md b/README.md index c4de4bc..5c6ac69 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,6 @@ There is a [dedicated chatroom](https://matrix.to/#/#forgejo-ci:matrix.org). A m - K8S cluster [files and documentation](https://code.forgejo.org/infrastructure/k8s-cluster) - Setting up a new [LXC/DRBD Host](lxc.md) -- Managing services with a [LXC/DRBD/nginx stack](drbd-nginx-lxc.md) - Installing a [Forgejo runner in an LXC container](runner-lxc.md) - Managing the [Octopuce host](octopuce.md) @@ -62,203 +61,6 @@ They are dedicated to the long running K8S cluster [files and documentation](htt - Docker enabled -## hetzner{02,03} - -https://hetzner02.forgejo.org & https://hetzner03.forgejo.org run on [EX44](https://www.hetzner.com/dedicated-rootserver/ex44) Hetzner hardware. - -### LXC - -```sh -lxc-helpers.sh lxc_install_lxc_inside 10.6.83 fc16 -``` - -### Disk partitioning - -- First disk - - OS - - a partition configured with DRBD for precious data mounted on /var/lib/lxc -- Second disk - - non precious data such as the LXC containers with runners. - -### Root filesystem backups - -- `hetzner03:/etc/cron.daily/backup-hetzner02` - `rsync -aHS --delete-excluded --delete --numeric-ids --exclude /proc --exclude /dev --exclude /sys --exclude /srv --exclude /var/lib/lxc 10.53.100.2:/ /srv/backups/hetzner02/` -- `hetzner02:/etc/cron.daily/backup-hetzner03` - `rsync -aHS --delete-excluded --delete --numeric-ids --exclude /proc --exclude /dev --exclude /sys --exclude /srv --exclude /var/lib/lxc 10.53.100.3:/ /srv/backups/hetzner03/` - -### Public IP addresses - -The public IP addresses attached to the hosts are not failover IPs that can be moved from one host to the next. -The DNS entry needs to be updated if the primary hosts changes. - -When additional IP addresses are attached to the server, they are added to `/etc/network/interfaces` like -ipv4 65.21.67.71 and ipv6 2a01:4f9:3081:51ec::102 below. - -``` -auto enp5s0 -iface enp5s0 inet static - address 65.21.67.73 - netmask 255.255.255.192 - gateway 65.21.67.65 - # route 65.21.67.64/26 via 65.21.67.65 - up route add -net 65.21.67.64 netmask 255.255.255.192 gw 65.21.67.65 dev enp5s0 - # BEGIN code.forgejo.org - up ip addr add 65.21.67.71/32 dev enp5s0 - up nft -f /home/debian/code.nftables - down ip addr del 65.21.67.71/32 dev enp5s0 - # END code.forgejo.org - -iface enp5s0 inet6 static - address 2a01:4f9:3081:51ec::2 - netmask 64 - gateway fe80::1 - # BEGIN code.forgejo.org - up ip -6 addr add 2a01:4f9:3081:51ec::102/64 dev enp5s0 - down ip -6 addr del 2a01:4f9:3081:51ec::102/64 dev enp5s0 - # END code.forgejo.org -``` -For port forwarding to work, the LXC host must not bind them. For instance the ssh server configuration at `/etc/ssh/sshd_config` should not bind all IP but only a specific one. - -``` -Port 22 -AddressFamily inet -ListenAddress 65.21.67.73 -#ListenAddress :: -``` - -### Port forwarding - -Forwarding a port to an LXC container can be done with [nginx streeam](https://nginx.org/en/docs/stream/ngx_stream_core_module.html) for the public IP of code.forgejo.org (65.21.67.71 & 2a01:4f9:3081:51ec::102) to the private IP (10.6.83.195) of the `code` LXC container in `/etc/nginx/modules-enabled/ssh.conf`: - -``` -stream { - - # code.forgejo.org ip's - upstream codessh { - least_conn; - server 10.6.83.195:22; - } - - # code.forgejo.org definition - server { - listen 65.21.67.71:22; # the port to listen on this server - listen [2a01:4f9:3081:51ec::102]:22; - proxy_pass codessh; # forward traffic to this upstream group - proxy_timeout 3s; - proxy_connect_timeout 3s; - } -} -``` - -### 302 redirects - -- On hetzner02 - - try.next.forgejo.org redirects to v(latest stable).next.forgejo.org - - dev.next.forgejo.org redirects to v(latest dev).next.forgejo.org - -### Containers - -- `forgejo-code` on hetzner02 - - Dedicated to https://code.forgejo.org - - - Docker enabled - - upgrades checklist: - - `ssh -t debian@hetzner02.forgejo.org lxc-helpers.sh lxc_container_run forgejo-code -- sudo --user debian bash` - ```sh - emacs /home/debian/run-forgejo.sh # change the `image=` - docker stop forgejo - ``` - - `ssh -t debian@hetzner02.forgejo.org sudo /etc/cron.daily/backup-forgejo-code` - - `ssh -t debian@hetzner02.forgejo.org lxc-helpers.sh lxc_container_run forgejo-code -- sudo --user debian bash` - ```sh - docker rm forgejo - bash -x /home/debian/run-forgejo.sh - docker logs -n 200 -f forgejo - ``` - - Rotating 30 days backups happen daily `/etc/cron.daily/forgejo-code-backup.sh` - - Add code.forgejo.org to the forgejo.org SPF record - -- `forgejo-next` on hetzner02 - - Dedicated to https://next.forgejo.org - - - Docker enabled - - `/etc/cron.hourly/forgejo-upgrade` runs `/home/debian/run-forgejo.sh > /home/debian/run-forgejo-$(date +%d).log` - - When a new major version is published (8.0 for instance) `run-forgejo.sh` must be updated with it - - Reset everything - ```sh - docker stop forgejo - docker rm forgejo - sudo rm -fr /srv/forgejo.old - sudo mv /srv/forgejo /srv/forgejo.old - bash -x /home/debian/run-forgejo.sh - ``` - - `/home/debian/next.nftables` - ``` - add table ip next; - flush table ip next; - add chain ip next prerouting { - type nat hook prerouting priority 0; - policy accept; - ip daddr 65.21.67.65 tcp dport { 2020 } dnat to 10.6.83.213; - }; - ``` - - Add to `iface enp5s0 inet static` in `/etc/network/interfaces` - ``` - up nft -f /home/debian/next.nftables - ``` - - ``` - - `/etc/nginx/sites-available/next.forgejo.org` same as `/etc/nginx/sites-available/code.forgejo.org` - - ``` - -- `forgejo-v7` on hetzner02 - - Dedicated to https://v7.next.forgejo.org - - - Docker enabled - - `/etc/cron.hourly/forgejo-upgrade` runs `/home/debian/run-forgejo.sh > /home/debian/run-forgejo-$(date +%d).log` - - Reset everything - ```sh - docker stop forgejo - docker rm forgejo - sudo rm -fr /srv/forgejo.old - sudo mv /srv/forgejo /srv/forgejo.old - bash -x /home/debian/run-forgejo.sh - ``` - - `/home/debian/v7.nftables` - ``` - add table ip v7; - flush table ip v7; - add chain ip v7 prerouting { - type nat hook prerouting priority 0; - policy accept; - ip daddr 65.21.67.65 tcp dport { 2070 } dnat to 10.6.83.179; - }; - ``` - - Add to `iface enp5s0 inet static` in `/etc/network/interfaces` - ``` - up nft -f /home/debian/v7.nftables - ``` - - ``` - - `/etc/nginx/sites-available/v7.forgejo.org` same as `/etc/nginx/sites-available/code.forgejo.org` - - ``` - -- `static-pages` on hetzner02 - - See [the static pages documenation](../static-pages/) for more information. - - - Unprivileged - -- `runner-forgejo-helm` on hetzner03 - - Dedicated to https://codeberg.org/forgejo-contrib/forgejo-helm and running from an ephemeral disk - ## Uberspace The website https://forgejo.org is hosted at diff --git a/drbd-nginx-lxc.md b/drbd-nginx-lxc.md deleted file mode 100644 index 9194592..0000000 --- a/drbd-nginx-lxc.md +++ /dev/null @@ -1,177 +0,0 @@ -## nftables - -```sh -sudo nft list ruleset -``` - -## Host reverse proxy - -The reverse proxy on a host forwards to the designated LXC container with -something like the following examples in -`/etc/nginx/sites-available/example.com`, where A.B.C.D is the -IP allocated to the LXC container running the web service. - -And symlink: - -```sh -ln -s /etc/nginx/sites-available/example.com /etc/nginx/sites-enabled/example.com -``` - -The certificate is obtained once and automatically renewed with: - -``` -sudo apt-get install certbot python3-certbot-nginx -sudo certbot -n --agree-tos --email contact@forgejo.org -d example.com --nginx -``` - -When removing a configuration, the certificate can also be removed with: - -``` -sudo certbot delete --cert-name example.com -``` - -Forwarding TCP streams (useful for ssh) requires installing the module: - -```sh -sudo apt-get install libnginx-mod-stream -``` - -Rate limiting crawlers is done by adding the following to `/etc/nginx/conf.d/limit.conf`: - -``` -# http://nginx.org/en/docs/http/ngx_http_limit_req_module.html -# https://blog.nginx.org/blog/rate-limiting-nginx -map $http_user_agent $isbot_ua { - default 0; - ~*(GoogleBot|GoogleOther|bingbot|YandexBot) 1; -} -map $isbot_ua $limit_bot { - 0 ""; - 1 $binary_remote_addr; -} -limit_req_zone $limit_bot zone=bots:10m rate=1r/m; -limit_req_status 429; -``` - -and the following in the location to be rate limited: - -``` - location / { - limit_req zone=bots burst=2 nodelay; - ... -``` - -## Host wakeup-on-logs - -https://code.forgejo.org/infrastructure/wakeup-on-logs - -### K8S wakeup-on-logs script - -``` -$ cat /etc/wakeup-on-logs/forgejo-v8 -#!/bin/bash - -set -x - -self="${BASH_SOURCE[0]}" -name=$(basename $self) -# keep it lower than https://code.forgejo.org/infrastructure/wakeup-on-logs -# otherwise it will get killed by it -timeout=4m - -function lxc_run() { - lxc-attach $name -- sudo --user debian KUBECONFIG=/etc/rancher/k3s/k3s.yaml "$@" |& tee -a /var/log/$name.log -} - -image=codeberg.org/forgejo-experimental/forgejo -major=${name##*v} -digest=$(skopeo inspect --format "{{.Digest}}" docker://$image:$major-rootless) -values=https://code.forgejo.org/infrastructure/k8s/raw/branch/main/forgejo-v$major/values.yml -lxc_run helm upgrade forgejo -f $values -f /home/debian/secrets.yml oci://code.forgejo.org/forgejo-helm/forgejo --atomic --wait --timeout $timeout --install --set image.digest=$digest -``` - -### Forgejo example - -``` -server { - listen 80; - listen [::]:80; - - server_name example.com; - - location / { - proxy_pass http://A.B.C.D:8080; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto https; - client_max_body_size 2G; - } -} -``` - -### GitLab example - -```nginx -server { - listen 80; - listen [::]:80; - - server_name example.com; - - location / { - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection "upgrade"; - proxy_set_header Host $http_host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header X-Frame-Options SAMEORIGIN; - - client_body_timeout 60; - client_max_body_size 200M; - send_timeout 1200; - lingering_timeout 5; - - proxy_buffering off; - proxy_connect_timeout 90; - proxy_send_timeout 300; - proxy_read_timeout 600s; - - proxy_pass http://example.com; - proxy_http_version 1.1; - } -} -``` - -### Vanila example - -```nginx -server { - listen 80; - listen [::]:80; - - server_name example.com; - - location / { - proxy_pass http://A.B.C.D; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header X-Forwarded-Proto https; - } -} -``` - -### 302 redirection - -```nginx -server { - listen 80; - listen [::]:80; - - server_name example.com; - return 302 https://other.example.com$request_uri; -} -``` -