1
0
Fork 0
mirror of https://code.forgejo.org/infrastructure/documentation synced 2024-12-22 21:03:52 +00:00
infrastructure-documentation/k3s-host/setup.sh

227 lines
6.5 KiB
Bash
Executable file

#!/bin/bash
SELF_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
if ${VERBOSE:-false}; then
set -ex
PS4='${BASH_SOURCE[0]}:$LINENO: ${FUNCNAME[0]}: '
else
set -e
fi
export INSTALL_K3S_VERSION=v1.30.5+k3s1
source $SELF_DIR/variables.sh
source $SELF_DIR/secrets.sh
set -o pipefail
self_node=$(hostname | sed -e 's/hetzner0//')
interface=${node_interface[$self_node]}
dependencies="retry etckeeper"
if ! which $dependencies >&/dev/null; then
sudo apt-get -q install -qq -y $dependencies
fi
function setup_ufw() {
sudo apt-get -q install -qq -y ufw
sudo ufw --force reset
sudo ufw default allow incoming
sudo ufw default allow outgoing
sudo ufw default allow routed
for from in $nodes; do
for to in $nodes; do
if test $from != $to; then
for v in ipv4 ipv6; do
eval from_ip=\${node_$v[$from]}
eval to_ip=\${node_$v[$to]}
sudo ufw allow in on $interface from $from_ip to $to_ip
done
fi
done
done
for host_ip in ${node_ipv4[$self_node]} ${node_ipv6[$self_node]}; do
sudo ufw allow in on $interface to $host_ip port 22 proto tcp
sudo ufw deny in on $interface log-all to $host_ip
done
for public_ip in $failover_ipv4 $failover_ipv6; do
sudo ufw allow in on $interface to $public_ip port 22,80,443,2000:3000 proto tcp
sudo ufw deny in on $interface log-all to $public_ip
done
sudo ufw enable
sudo systemctl start ufw
sudo systemctl enable ufw
sudo ufw status verbose
}
function setup_drbd() {
if ! test -f /etc/network/interfaces.d/drbd; then
cat <<EOF | sudo tee /etc/network/interfaces.d/drbd
auto $interface.$node_drbd_vlan
iface $interface.$node_drbd_vlan inet static
address $node_drbd_ipv4_prefix.$self_node
netmask 255.255.255.0
vlan-raw-device $interface
mtu 1400
EOF
sudo ifup $interface.$node_drbd_vlan
fi
sudo apt-get install -y drbd-utils
res_file=/etc/drbd.d/$node_drbd_resource.res
if ! sudo test -f $res_file; then
(
cat <<EOF
resource $node_drbd_resource {
net {
# A : write completion is determined when data is written to the local disk and the local TCP transmission buffer
# B : write completion is determined when data is written to the local disk and remote buffer cache
# C : write completion is determined when data is written to both the local disk and the remote disk
protocol C;
cram-hmac-alg sha1;
# any secret key for authentication among nodes
shared-secret "$node_drbd_shared_secret";
}
disk {
resync-rate 100M;
}
EOF
for node in $nodes; do
cat <<EOF
on hetzner0$node {
address $node_drbd_ipv4_prefix.$node:7788;
volume 0 {
device /dev/drbd0;
disk ${node_drbd_device[$node]};
meta-disk internal;
}
}
EOF
done
cat <<EOF
}
EOF
) | sudo tee $res_file
fi
if ! sudo drbdadm status $node_drbd_resource >&/dev/null; then
sudo drbdadm create-md $node_drbd_resource
sudo systemctl enable drbd
sudo systemctl start drbd
fi
if ! grep --quiet '^/dev/drbd0 /precious' /etc/fstab; then
echo /dev/drbd0 /precious ext4 noauto,noatime,defaults 0 0 | sudo tee -a /etc/fstab
sudo mkdir -p /precious
fi
}
function setup_nfs() {
sudo apt-get install -y nfs-kernel-server nfs-common
if ! test -f /etc/network/interfaces.d/nfs; then
cat <<EOF | sudo tee /etc/network/interfaces.d/nfs
auto $interface.$node_nfs_vlan
iface $interface.$node_nfs_vlan inet static
address $node_nfs_ipv4_prefix.$self_node
netmask 255.255.255.0
vlan-raw-device $interface
mtu 1400
EOF
sudo ifup $interface.$node_nfs_vlan
fi
if ! grep --quiet '^/precious' /etc/exports; then
cat <<EOF | sudo tee -a /etc/exports
/precious $node_nfs_ipv4_prefix.0/24(rw,fsid=0,no_root_squash,no_subtree_check)
/precious/k8s $node_nfs_ipv4_prefix.0/24(rw,nohide,insecure,no_subtree_check)
EOF
sudo exportfs -va || true # it does not matter if the expored dirs do not exist, they will
sudo exportfs -s
fi
}
function setup_k8s() {
existing=$1
if ! test -f /etc/network/interfaces.d/k8s; then
cat <<EOF | sudo tee /etc/network/interfaces.d/k8s
auto $interface.$node_k8s_vlan
iface $interface.$node_k8s_vlan inet static
address $node_k8s_ipv4_prefix.$self_node
netmask 255.255.0.0
vlan-raw-device $interface
mtu 1400
up ip addr add $node_k8s_ipv6_prefix::$self_node/48 dev $interface.$node_k8s_vlan
up ip addr add $failover_ipv4/$failover_ipv4_range dev $interface
up ip addr add $failover_ipv6/$failover_ipv6_range dev $interface
EOF
sudo ifup $interface.$node_k8s_vlan
fi
if ! grep --quiet 'export KUBECONFIG' ~/.bashrc; then
echo "export KUBECONFIG=/etc/rancher/k3s/k3s.yaml" >>~/.bashrc
fi
if ! sudo systemctl --quiet is-active k3s; then
args=""
if test "$existing"; then
if ! test "$node_k8s_token"; then
echo "obtain the token from node $existing with sudo cat /var/lib/rancher/k3s/server/token and set node_k8s_token= in secrets.sh"
exit 1
fi
args="$args --token $node_k8s_token --server https://$node_k8s_ipv4_prefix.$existing:6443"
fi
if test "$self_node" = $node_k8s_etcd; then
args="$args --disable-apiserver --disable-controller-manager --disable-scheduler"
fi
curl -fL https://get.k3s.io | sh -s - server $args --cluster-init --disable=servicelb --write-kubeconfig-mode=644 --node-ip=$node_k8s_ipv4_prefix.$self_node,$node_k8s_ipv6_prefix::$self_node $node_k8s_cidr --flannel-ipv6-masq
if test "$self_node" = $node_k8s_etcd; then
retry --times 20 -- kubectl taint nodes $(hostname) key1=value1:NoSchedule
fi
if test "$self_node" != $node_k8s_etcd; then
curl -fsSL https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash -
fi
fi
}
function setup_k8s_apply() {
retry --delay 30 --times 10 -- bash -c "$SELF_DIR/subst.sh $1 | kubectl apply --server-side=true -f -"
}
function setup_k8s_traefik() {
setup_k8s_apply traefik.yml
}
function setup_k8s_nfs() {
setup_k8s_apply nfs.yml
}
function setup_k8s_metallb() {
helm repo add metallb https://metallb.github.io/metallb
helm upgrade --install metallb --set installCRDs=true metallb/metallb
setup_k8s_apply metallb.yml
}
function setup_k8s_certmanager() {
helm upgrade --install mycertmanager --set installCRDs=true oci://registry-1.docker.io/bitnamicharts/cert-manager
setup_k8s_apply certmanager.yml
}
function setup_k8s_pvc() {
export pvc_name=$1
export pvc_capacity=$2
export pvc_owner=$3
sudo mount -o nfsvers=4.2 $node_nfs_server:/k8s /opt
sudo mkdir -p /opt/$pvc_name
sudo chown $pvc_owner:$pvc_owner /opt/$pvc_name
sudo umount /opt
setup_k8s_apply pvc.yml
}
"$@"