hetzner-dedicated/zfs: Install right version of zfs

hosters/hetzner-dedicated/zfs-uefi-nvme-nixos.sh

@@ -6,7 +6,9 @@
 #
 #
 # Usage:
-#     ssh root@YOUR_SERVERS_IP bash -s < hetzner-dedicated-wipe-and-install-nixos.sh
+#     scp zfs-uefi-nvme-nixos.sh root@YOUR_SERVER_IP:
+#     ssh root@YOUR_SERVERS_IP zfs
+#     ssh root@YOUR_SERVERS_IP /root/zfs-uefi-nvme-nixos.sh
 #
 # When the script is done, make sure to boot the server from HD, not rescue mode again.
 
@@ -20,33 +22,32 @@
 #   inspired by https://nixos.wiki/wiki/Install_NixOS_on_Hetzner_Online
 # * This server has 2 SSDs.
 #   We put everything on mirror (RAID1 equivalent).
-# * A root user with empty password is created, so that you can just login
-#   as root and press enter when using the Hetzner spider KVM.
-#   Of course that empty-password login isn't exposed to the Internet.
-#   Change the password afterwards to avoid anyone with physical access
-#   being able to login without any authentication.
+# * A root user with a password is created, so that you can just login
+#   as root when using the Hetzner spider KVM.
 # * The script reboots at the end.
 # * exports of env vars are added throughout the script in case you want to run it manually
 export LC_ALL=C
 
-# WARNING: on 2023/07/16 the rescue system of hetzner boots with kernel 6.3.7 which
-# is by default not supported by the latest debian package. You need to update to debian
-# unstable to proceed with the zfs installation.
-
-cat > /etc/apt/preferences.d/90_zfs <<EOF
-Package: libnvpair1linux libnvpair3linux libuutil1linux libuutil3linux libzfs2linux libzfs4linux libzpool2linux libzpool4linux spl-dkms zfs-dkms zfs-test zfsutils-linux zfsutils-linux-dev zfs-zed
-Pin: release n=bullseye-backports
-Pin-Priority: 990
-EOF
-
-apt update -y
-apt install -y dpkg-dev linux-headers-$(uname -r) linux-image-amd64 sudo parted zfs-dkms zfsutils-linux
+apt update
+apt install -y dpkg-dev linux-headers-$(uname -r) linux-image-amd64 sudo parted libssl-dev uuid-dev zlib1g-dev libblkid-dev
 
 set -euox pipefail
 
-# hetzner has some weird symlinks to make you install zfs with their script
-rm /usr/local/sbin/zfs || true
-rm /usr/local/sbin/zpool || true
+rm -f /usr/local/sbin/zfs
+rm -f /usr/local/sbin/zpool
+
+cd "$(mktemp -d)"
+# Don't download latest, ZFS 2.2 creates zpool which cannot be loaded by zfs
+# 2.1.x
+#
+# wget "$(curl -Ls https://api.github.com/repos/openzfs/zfs/releases/107594126 | grep -E "browser_download_url.*\.tar.gz\"$"| cut -d '"' -f 4)"
+wget https://github.com/openzfs/zfs/releases/download/zfs-2.1.13/zfs-2.1.13.tar.gz
+tar xfv zfs*.tar.gz && rm zfs*.tar.gz
+cd zfs*
+./configure && make -j "$(nproc)"
+make install
+ldconfig
+modprobe zfs
 
 # Inspect existing disks
 # Should give you something like
@@ -89,12 +90,12 @@ ls /dev/disk/by-id
 # nvme-SAMSUNG_MZVLB512HBJQ-00000_S4GENA0NA00427
 
 # The following variables should be replaced
-export DISK1=/dev/disk/by-id/nvme-SAMSUNG_MZQLB3T8HALS-00007_S438NC0R804840
-export DISK2=/dev/disk/by-id/nvme-SAMSUNG_MZQLB3T8HALS-00007_S438NC0R811800
+export DISK1=/dev/disk/by-id/nvme-SAMSUNG_MZVL21T0HCLR-00B00_S676NF0R517371
+export DISK2=/dev/disk/by-id/nvme-KXG60ZNV1T02_TOSHIBA_Z9NF704ZF9ZL
 # Replace with your key
-export SSH_PUB_KEY="ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIGyQSeQ0CV/qhZPre37+Nd0E9eW+soGs+up6a/bwggoP raphael@RAPHAELs-MacBook-Pro.local"
+export SSH_PUB_KEY="ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMNeQYLFauAbzDyIbKC86NUh9yZfiyBm/BtIdkcpZnSU axeman@tuxnix"
 # choose whatever you want, it doesn't matter
-export MY_HOSTNAME=htz
+export MY_HOSTNAME=nachtigall
 # this has to be a number in this format exactly. You can replace the numbers though
 export MY_HOSTID=00000001
 
@@ -144,14 +145,14 @@ echo -e "#! /usr/bin/env bash\nset -e\n" 'parted $@ 2> parted-stderr.txt || grep
 # however if it's less the installation fails with
 # cannot copy /nix/store/d4xbrrailkn179cdp90v4m57mqd73hvh-linux-5.4.100/bzImage to /boot/kernels/d4xbrrailkn179cdp90v4m57mqd73hvh-linux-5.4.100-bzImage.tmp: No space left on device
 ./parted-ignoring-partprobe-error.sh --script --align optimal $DISK1 -- mklabel gpt \
-    mkpart 'BIOS-boot-partition' 1MB 2MB set 1 bios_grub on \
-    mkpart 'EFI-system-partition' 2MB 512MB set 2 esp on \
-    mkpart 'data-partition' 512MB '100%'
+    mkpart 'BIOS-boot' 1MB 2MB set 1 bios_grub on \
+    mkpart 'EFI-system' 2MB 2000MB set 2 esp on \
+    mkpart 'data' 2000MB '100%'
 
 ./parted-ignoring-partprobe-error.sh --script --align optimal $DISK2 -- mklabel gpt \
-    mkpart 'BIOS-boot-partition' 1MB 2MB set 1 bios_grub on \
-    mkpart 'EFI-system-partition' 2MB 512MB set 2 esp on \
-    mkpart 'data-partition' 512MB '100%'
+    mkpart 'BIOS-boot' 1MB 2MB set 1 bios_grub on \
+    mkpart 'EFI-system' 2MB 512MB set 2 esp on \
+    mkpart 'data' 512MB '100%'
 
 # Reload partitions
 partprobe
@@ -166,12 +167,12 @@ udevadm settle --timeout=5 --exit-if-exists=$DISK2-part3
 
 # Wipe any previous RAID signatures
 # sometimes they are not on a specific disk for some reason
-mdadm --zero-superblock --force $DISK1-part1 || true
-mdadm --zero-superblock --force $DISK1-part2 || true
-mdadm --zero-superblock --force $DISK1-part3 || true
-mdadm --zero-superblock --force $DISK2-part1 || true
-mdadm --zero-superblock --force $DISK2-part2 || true
-mdadm --zero-superblock --force $DISK2-part3 || true
+# mdadm --zero-superblock --force $DISK1-part1 || true
+# mdadm --zero-superblock --force $DISK1-part2 || true
+# mdadm --zero-superblock --force $DISK1-part3 || true
+# mdadm --zero-superblock --force $DISK2-part1 || true
+# mdadm --zero-superblock --force $DISK2-part2 || true
+# mdadm --zero-superblock --force $DISK2-part3 || true
 
 # Creating file systems changes their UUIDs.
 # Trigger udev so that the entries in /dev/disk/by-uuid get refreshed.
@@ -179,6 +180,9 @@ mdadm --zero-superblock --force $DISK2-part3 || true
 # See https://github.com/NixOS/nixpkgs/issues/62444
 udevadm trigger
 
+mkfs.vfat $DISK1-part2
+mkfs.vfat $DISK2-part2
+
 # taken from https://nixos.wiki/wiki/NixOS_on_ZFS
 # somehow there is a weird symlink in the default zfs
 zpool create -O mountpoint=none \
@@ -187,85 +191,48 @@ zpool create -O mountpoint=none \
     -O xattr=sa \
     -O acltype=posixacl \
     -o ashift=12 \
+    -O encryption=aes-256-gcm -O keyformat=passphrase \
     -f \
     root_pool mirror $DISK1-part3 $DISK2-part3
 
-# Create the filesystems. This layout is designed so that /home is separate from the root
-# filesystem, as you'll likely want to snapshot it differently for backup purposes. It also
-# makes a "nixos" filesystem underneath the root, to support installing multiple OSes if
-# that's something you choose to do in future.
 zfs create -o mountpoint=legacy root_pool/root
-zfs create -o mountpoint=legacy root_pool/root/nixos
-zfs create -o mountpoint=legacy root_pool/home
-# add 1G of reseved space in case the disk gets full
+# add 8G of reseved space in case the disk gets full
 # zfs needs space to delete files
-zfs create -o refreservation=1G -o mountpoint=none root_pool/reserved
+zfs create -o refreservation=8G -o mountpoint=none root_pool/reserved
+
+# all application data goes here. Only backups from this dataset and its
+# children are important
+zfs create -o mountpoint=legacy root_pool/data
+
 # this creates a special volume for db data see https://wiki.archlinux.org/index.php/ZFS#Databases
 zfs create -o mountpoint=legacy \
     -o recordsize=8K \
     -o primarycache=metadata \
     -o logbias=throughput \
-    root_pool/postgres
+    root_pool/data/postgresql
 
 # NixOS pre-installation mounts
 #
 # Mount the filesystems manually. The nixos installer will detect these mountpoints
 # and save them to /mnt/nixos/hardware-configuration.nix during the install process.
-mount -t zfs root_pool/root/nixos /mnt
-mkdir /mnt/home
-mount -t zfs root_pool/home /mnt/home
-mkdir -p /mnt/var/lib/postgres
-mount -t zfs root_pool/postgres /mnt/var/lib/postgres
+mount -t zfs root_pool/root /mnt
+mkdir -p /mnt/var/lib/postgresql
+mount -t zfs root_pool/data/postgresql /mnt/var/lib/postgresql
 
-# Create a raid mirror for the efi boot
-# see https://docs.hetzner.com/robot/dedicated-server/operating-systems/efi-system-partition/
-# TODO check this though the following article says it doesn't work properly
-# https://outflux.net/blog/archives/2018/04/19/uefi-booting-and-raid1/ 
-mdadm --create --run --verbose /dev/md127 \
-    --level 1 \
-    --raid-disks 2 \
-    --metadata 1.0 \
-    --homehost=$MY_HOSTNAME \
-    --name=boot_efi \
-    $DISK1-part2 $DISK2-part2
-
-# Assembling the RAID can result in auto-activation of previously-existing LVM
-# groups, preventing the RAID block device wiping below with
-# `Device or resource busy`. So disable all VGs first.
-vgchange -an
-
-# Wipe filesystem signatures that might be on the RAID from some
-# possibly existing older use of the disks (RAID creation does not do that).
-# See https://serverfault.com/questions/911370/why-does-mdadm-zero-superblock-preserve-file-system-information
-wipefs -a /dev/md127
-
-# Disable RAID recovery. We don't want this to slow down machine provisioning
-# in the rescue mode. It can run in normal operation after reboot.
-echo 0 > /proc/sys/dev/raid/speed_limit_max
-
-# Filesystems (-F to not ask on preexisting FS)
-mkfs.vfat -F 32 /dev/md127
-
-# Creating file systems changes their UUIDs.
-# Trigger udev so that the entries in /dev/disk/by-uuid get refreshed.
-# `nixos-generate-config` depends on those being up-to-date.
-# See https://github.com/NixOS/nixpkgs/issues/62444
-udevadm trigger
-
-mkdir -p /mnt/boot/efi
-mount /dev/md127 /mnt/boot/efi
+mkdir /mnt/boot1 /mnt/boot2
+mount $DISK1-part2 /mnt/boot1
+mount $DISK2-part2 /mnt/boot2
 
 # Installing nix
 
-# Allow installing nix as root, see
-#   https://github.com/NixOS/nix/issues/936#issuecomment-475795730
-mkdir -p /etc/nix
-echo "build-users-group =" > /etc/nix/nix.conf
-
 # using determinate systems installer, for more information
 # check https://github.com/DeterminateSystems/nix-installer
 curl --proto '=https' --tlsv1.2 -sSf -L https://install.determinate.systems/nix | sh -s -- install
 
+set +eu
+. /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh
+set -eu
+
 # Keep in sync with `system.stateVersion` set below!
 nix-channel --add https://nixos.org/channels/nixos-23.05 nixpkgs
 nix-channel --update
@@ -316,6 +283,9 @@ echo "Determined IP_V6 as $IP_V6"
 read _ _ DEFAULT_GATEWAY _ < <(ip route list match 0/0); echo "$DEFAULT_GATEWAY"
 echo "Determined DEFAULT_GATEWAY as $DEFAULT_GATEWAY"
 
+mkdir -p /mnt/etc/secrets/initrd
+ssh-keygen -t ed25519 -N "" -f /mnt/etc/secrets/initrd/ssh_host_ed25519_key
+
 # Generate `configuration.nix`. Note that we splice in shell variables.
 cat > /mnt/etc/nixos/configuration.nix <<EOF
 { config, pkgs, ... }:
@@ -332,11 +302,44 @@ cat > /mnt/etc/nixos/configuration.nix <<EOF
   boot.loader.grub = {
     enable = true;
     efiSupport = false;
-    devices = ["$DISK1" "$DISK2"];
-    copyKernels = true; 
+    mirroredBoots = [
+      {
+        devices = [
+          "$DISK1"
+        ];
+        path = "/boot1";
+      }
+      {
+        devices = [
+          "$DISK2"
+        ];
+        path = "/boot2";
+      }
+    ];
+    copyKernels = true;
   };
   boot.supportedFilesystems = [ "zfs" ];
 
+  # From https://nixos.wiki/wiki/ZFS#Unlock_encrypted_zfs_via_ssh_on_boot
+  boot.initrd.network = {
+    enable = true;
+    ssh = {
+      enable = true;
+      # To prevent ssh clients from freaking out because a different host key is used,
+      # a different port for ssh is useful (assuming the same host has also a regular sshd running)
+      port = 2222;
+      hostKeys = [ "/etc/secrets/initrd/ssh_host_ed25519_key" ];
+      authorizedKeys = [ "$SSH_PUB_KEY" ];
+    };
+  };
+
+  boot.kernelParams = [
+    "boot.shell_on_fail=1"
+	  "ip=${IP_V4}::${DEFAULT_GATEWAY}:255.255.255.192:${MY_HOSTNAME}::off"
+  ];
+
+  boot.initrd.availableKernelModules = [ "igb" ];
+
   networking.hostName = "$MY_HOSTNAME";
   networking.hostId = "$MY_HOSTID";
   
@@ -362,7 +365,7 @@ cat > /mnt/etc/nixos/configuration.nix <<EOF
   networking.interfaces."$NIXOS_INTERFACE".ipv4.addresses = [
     {
       address = "$IP_V4";
-      prefixLength = 24;
+      prefixLength = 26;
     }
   ];
   networking.interfaces."$NIXOS_INTERFACE".ipv6.addresses = [
@@ -384,8 +387,7 @@ cat > /mnt/etc/nixos/configuration.nix <<EOF
     "2001:4860:4860::8844"
   ];
 
-  # Initial empty root password for easy login:
-  users.users.root.initialHashedPassword = "";
+  users.users.root.initialHashedPassword = "\$y\$j9T\$bIN6GjQkmPMllOcQsq52K0\$q0Z5B5.KW/uxXK9fItB8H6HO79RYAcI/ZZdB0Djke32";
   services.openssh.permitRootLogin = "prohibit-password";
 
   users.users.root.openssh.authorizedKeys.keys = ["$SSH_PUB_KEY"];
@@ -404,7 +406,7 @@ EOF
 PATH="$PATH" $(which nixos-install) \
   --no-root-passwd --root /mnt --max-jobs 40
 
-umount /mnt
+umount -R /mnt
 
 reboot