hetzner-dedicated/zfs: Reorganize partitions, use mirroredBoots, set network on kernel params

hosters/hetzner-dedicated/zfs-uefi-nvme-nixos.sh

@@ -7,6 +7,7 @@
 #
 # Usage:
 #     scp zfs-uefi-nvme-nixos.sh root@YOUR_SERVER_IP:
+#     ssh root@YOUR_SERVERS_IP zfs
 #     ssh root@YOUR_SERVERS_IP /root/zfs-uefi-nvme-nixos.sh
 #
 # When the script is done, make sure to boot the server from HD, not rescue mode again.
@@ -128,14 +129,14 @@ echo -e "#! /usr/bin/env bash\nset -e\n" 'parted $@ 2> parted-stderr.txt || grep
 # however if it's less the installation fails with
 # cannot copy /nix/store/d4xbrrailkn179cdp90v4m57mqd73hvh-linux-5.4.100/bzImage to /boot/kernels/d4xbrrailkn179cdp90v4m57mqd73hvh-linux-5.4.100-bzImage.tmp: No space left on device
 ./parted-ignoring-partprobe-error.sh --script --align optimal $DISK1 -- mklabel gpt \
-    mkpart 'BIOS-boot-partition' 1MB 2MB set 1 bios_grub on \
+    mkpart 'BIOS-boot' 1MB 2MB set 1 bios_grub on \
-    mkpart 'EFI-system-partition' 2MB 512MB set 2 esp on \
+    mkpart 'EFI-system' 2MB 2000MB set 2 esp on \
-    mkpart 'data-partition' 512MB '100%'
+    mkpart 'data' 2000MB '100%'
 
 ./parted-ignoring-partprobe-error.sh --script --align optimal $DISK2 -- mklabel gpt \
-    mkpart 'BIOS-boot-partition' 1MB 2MB set 1 bios_grub on \
+    mkpart 'BIOS-boot' 1MB 2MB set 1 bios_grub on \
-    mkpart 'EFI-system-partition' 2MB 512MB set 2 esp on \
+    mkpart 'EFI-system' 2MB 512MB set 2 esp on \
-    mkpart 'data-partition' 512MB '100%'
+    mkpart 'data' 512MB '100%'
 
 # Reload partitions
 partprobe
@@ -150,12 +151,12 @@ udevadm settle --timeout=5 --exit-if-exists=$DISK2-part3
 
 # Wipe any previous RAID signatures
 # sometimes they are not on a specific disk for some reason
-mdadm --zero-superblock --force $DISK1-part1 || true
+# mdadm --zero-superblock --force $DISK1-part1 || true
-mdadm --zero-superblock --force $DISK1-part2 || true
+# mdadm --zero-superblock --force $DISK1-part2 || true
-mdadm --zero-superblock --force $DISK1-part3 || true
+# mdadm --zero-superblock --force $DISK1-part3 || true
-mdadm --zero-superblock --force $DISK2-part1 || true
+# mdadm --zero-superblock --force $DISK2-part1 || true
-mdadm --zero-superblock --force $DISK2-part2 || true
+# mdadm --zero-superblock --force $DISK2-part2 || true
-mdadm --zero-superblock --force $DISK2-part3 || true
+# mdadm --zero-superblock --force $DISK2-part3 || true
 
 # Creating file systems changes their UUIDs.
 # Trigger udev so that the entries in /dev/disk/by-uuid get refreshed.
@@ -163,6 +164,9 @@ mdadm --zero-superblock --force $DISK2-part3 || true
 # See https://github.com/NixOS/nixpkgs/issues/62444
 udevadm trigger
 
+mkfs.vfat $DISK1-part2
+mkfs.vfat $DISK2-part2
+
 # taken from https://nixos.wiki/wiki/NixOS_on_ZFS
 # somehow there is a weird symlink in the default zfs
 zpool create -O mountpoint=none \
@@ -171,74 +175,37 @@ zpool create -O mountpoint=none \
     -O xattr=sa \
     -O acltype=posixacl \
     -o ashift=12 \
+    -O encryption=aes-256-gcm -O keyformat=passphrase \
     -f \
     root_pool mirror $DISK1-part3 $DISK2-part3
 
-zfs create -o mountpoint=legacy \
+zfs create -o mountpoint=legacy root_pool/root
-    -o encryption=on \
+# add 8G of reseved space in case the disk gets full
-    -o keyformat=passphrase \
-    root_pool/encrypted
-zfs create -o mountpoint=legacy root_pool/encrypted/root
-# add 1G of reseved space in case the disk gets full
 # zfs needs space to delete files
-zfs create -o refreservation=1G -o mountpoint=none root_pool/reserved
+zfs create -o refreservation=8G -o mountpoint=none root_pool/reserved
 
 # all application data goes here. Only backups from this dataset and its
 # children are important
-zfs create -o mountpoint=legacy root_pool/encrypted/data
+zfs create -o mountpoint=legacy root_pool/data
 
 # this creates a special volume for db data see https://wiki.archlinux.org/index.php/ZFS#Databases
 zfs create -o mountpoint=legacy \
     -o recordsize=8K \
     -o primarycache=metadata \
     -o logbias=throughput \
-    root_pool/encrypted/data/postgresql
+    root_pool/data/postgresql
 
 # NixOS pre-installation mounts
 #
 # Mount the filesystems manually. The nixos installer will detect these mountpoints
 # and save them to /mnt/nixos/hardware-configuration.nix during the install process.
-mount -t zfs root_pool/encrypted/root /mnt
+mount -t zfs root_pool/root /mnt
 mkdir -p /mnt/var/lib/postgresql
-mount -t zfs root_pool/encrypted/data/postgresql /mnt/var/lib/postgresql
+mount -t zfs root_pool/data/postgresql /mnt/var/lib/postgresql
 
-# Create a raid mirror for the efi boot
+mkdir /mnt/boot1 /mnt/boot2
-# see https://docs.hetzner.com/robot/dedicated-server/operating-systems/efi-system-partition/
+mount $DISK1-part2 /mnt/boot1
-# TODO check this though the following article says it doesn't work properly
+mount $DISK2-part2 /mnt/boot2
-# https://outflux.net/blog/archives/2018/04/19/uefi-booting-and-raid1/ 
-mdadm --create --run --verbose /dev/md127 \
-    --level 1 \
-    --raid-disks 2 \
-    --metadata 1.0 \
-    --homehost=$MY_HOSTNAME \
-    --name=boot_efi \
-    $DISK1-part2 $DISK2-part2
-
-# Assembling the RAID can result in auto-activation of previously-existing LVM
-# groups, preventing the RAID block device wiping below with
-# `Device or resource busy`. So disable all VGs first.
-vgchange -an
-
-# Wipe filesystem signatures that might be on the RAID from some
-# possibly existing older use of the disks (RAID creation does not do that).
-# See https://serverfault.com/questions/911370/why-does-mdadm-zero-superblock-preserve-file-system-information
-wipefs -a /dev/md127
-
-# Disable RAID recovery. We don't want this to slow down machine provisioning
-# in the rescue mode. It can run in normal operation after reboot.
-echo 0 > /proc/sys/dev/raid/speed_limit_max
-
-# Filesystems (-F to not ask on preexisting FS)
-mkfs.vfat -F 32 /dev/md127
-
-# Creating file systems changes their UUIDs.
-# Trigger udev so that the entries in /dev/disk/by-uuid get refreshed.
-# `nixos-generate-config` depends on those being up-to-date.
-# See https://github.com/NixOS/nixpkgs/issues/62444
-udevadm trigger
-
-mkdir -p /mnt/boot/efi
-mount /dev/md127 /mnt/boot/efi
 
 # Installing nix
 
@@ -319,7 +286,20 @@ cat > /mnt/etc/nixos/configuration.nix <<EOF
   boot.loader.grub = {
     enable = true;
     efiSupport = false;
-    devices = ["$DISK1" "$DISK2"];
+    mirroredBoots = [
+      {
+        devices = [
+          "$DISK1"
+        ];
+        path = "/boot1";
+      }
+      {
+        devices = [
+          "$DISK2"
+        ];
+        path = "/boot2";
+      }
+    ];
     copyKernels = true;
   };
   boot.supportedFilesystems = [ "zfs" ];
@@ -337,6 +317,11 @@ cat > /mnt/etc/nixos/configuration.nix <<EOF
     };
   };
 
+  boot.kernelParams = [
+    "boot.shell_on_fail=1"
+	  "ip=${IP_V4}::${DEFAULT_GATEWAY}:255.255.255.192:${MY_HOSTNAME}::off"
+  ];
+
   boot.initrd.availableKernelModules = [ "igb" ];
 
   networking.hostName = "$MY_HOSTNAME";
@@ -405,7 +390,7 @@ EOF
 PATH="$PATH" $(which nixos-install) \
   --no-root-passwd --root /mnt --max-jobs 40
 
-umount /mnt
+umount -R /mnt
 
 reboot