From d33d5f5c6a3fcb7202d61f4a263561c9cb0614b5 Mon Sep 17 00:00:00 2001 From: Akshay Mankar Date: Sun, 15 Oct 2023 15:53:44 +0200 Subject: [PATCH] hetzner-dedicated/zfs: Reorganize partitions, use mirroredBoots, set network on kernel params --- .../hetzner-dedicated/zfs-uefi-nvme-nixos.sh | 111 ++++++++---------- 1 file changed, 48 insertions(+), 63 deletions(-) diff --git a/hosters/hetzner-dedicated/zfs-uefi-nvme-nixos.sh b/hosters/hetzner-dedicated/zfs-uefi-nvme-nixos.sh index 27ca0ee..3a76b95 100644 --- a/hosters/hetzner-dedicated/zfs-uefi-nvme-nixos.sh +++ b/hosters/hetzner-dedicated/zfs-uefi-nvme-nixos.sh @@ -7,6 +7,7 @@ # # Usage: # scp zfs-uefi-nvme-nixos.sh root@YOUR_SERVER_IP: +# ssh root@YOUR_SERVERS_IP zfs # ssh root@YOUR_SERVERS_IP /root/zfs-uefi-nvme-nixos.sh # # When the script is done, make sure to boot the server from HD, not rescue mode again. @@ -128,14 +129,14 @@ echo -e "#! /usr/bin/env bash\nset -e\n" 'parted $@ 2> parted-stderr.txt || grep # however if it's less the installation fails with # cannot copy /nix/store/d4xbrrailkn179cdp90v4m57mqd73hvh-linux-5.4.100/bzImage to /boot/kernels/d4xbrrailkn179cdp90v4m57mqd73hvh-linux-5.4.100-bzImage.tmp: No space left on device ./parted-ignoring-partprobe-error.sh --script --align optimal $DISK1 -- mklabel gpt \ - mkpart 'BIOS-boot-partition' 1MB 2MB set 1 bios_grub on \ - mkpart 'EFI-system-partition' 2MB 512MB set 2 esp on \ - mkpart 'data-partition' 512MB '100%' + mkpart 'BIOS-boot' 1MB 2MB set 1 bios_grub on \ + mkpart 'EFI-system' 2MB 2000MB set 2 esp on \ + mkpart 'data' 2000MB '100%' ./parted-ignoring-partprobe-error.sh --script --align optimal $DISK2 -- mklabel gpt \ - mkpart 'BIOS-boot-partition' 1MB 2MB set 1 bios_grub on \ - mkpart 'EFI-system-partition' 2MB 512MB set 2 esp on \ - mkpart 'data-partition' 512MB '100%' + mkpart 'BIOS-boot' 1MB 2MB set 1 bios_grub on \ + mkpart 'EFI-system' 2MB 512MB set 2 esp on \ + mkpart 'data' 512MB '100%' # Reload partitions partprobe @@ -150,12 +151,12 @@ udevadm settle --timeout=5 --exit-if-exists=$DISK2-part3 # Wipe any previous RAID signatures # sometimes they are not on a specific disk for some reason -mdadm --zero-superblock --force $DISK1-part1 || true -mdadm --zero-superblock --force $DISK1-part2 || true -mdadm --zero-superblock --force $DISK1-part3 || true -mdadm --zero-superblock --force $DISK2-part1 || true -mdadm --zero-superblock --force $DISK2-part2 || true -mdadm --zero-superblock --force $DISK2-part3 || true +# mdadm --zero-superblock --force $DISK1-part1 || true +# mdadm --zero-superblock --force $DISK1-part2 || true +# mdadm --zero-superblock --force $DISK1-part3 || true +# mdadm --zero-superblock --force $DISK2-part1 || true +# mdadm --zero-superblock --force $DISK2-part2 || true +# mdadm --zero-superblock --force $DISK2-part3 || true # Creating file systems changes their UUIDs. # Trigger udev so that the entries in /dev/disk/by-uuid get refreshed. @@ -163,6 +164,9 @@ mdadm --zero-superblock --force $DISK2-part3 || true # See https://github.com/NixOS/nixpkgs/issues/62444 udevadm trigger +mkfs.vfat $DISK1-part2 +mkfs.vfat $DISK2-part2 + # taken from https://nixos.wiki/wiki/NixOS_on_ZFS # somehow there is a weird symlink in the default zfs zpool create -O mountpoint=none \ @@ -171,74 +175,37 @@ zpool create -O mountpoint=none \ -O xattr=sa \ -O acltype=posixacl \ -o ashift=12 \ + -O encryption=aes-256-gcm -O keyformat=passphrase \ -f \ root_pool mirror $DISK1-part3 $DISK2-part3 -zfs create -o mountpoint=legacy \ - -o encryption=on \ - -o keyformat=passphrase \ - root_pool/encrypted -zfs create -o mountpoint=legacy root_pool/encrypted/root -# add 1G of reseved space in case the disk gets full +zfs create -o mountpoint=legacy root_pool/root +# add 8G of reseved space in case the disk gets full # zfs needs space to delete files -zfs create -o refreservation=1G -o mountpoint=none root_pool/reserved +zfs create -o refreservation=8G -o mountpoint=none root_pool/reserved # all application data goes here. Only backups from this dataset and its # children are important -zfs create -o mountpoint=legacy root_pool/encrypted/data +zfs create -o mountpoint=legacy root_pool/data # this creates a special volume for db data see https://wiki.archlinux.org/index.php/ZFS#Databases zfs create -o mountpoint=legacy \ -o recordsize=8K \ -o primarycache=metadata \ -o logbias=throughput \ - root_pool/encrypted/data/postgresql + root_pool/data/postgresql # NixOS pre-installation mounts # # Mount the filesystems manually. The nixos installer will detect these mountpoints # and save them to /mnt/nixos/hardware-configuration.nix during the install process. -mount -t zfs root_pool/encrypted/root /mnt +mount -t zfs root_pool/root /mnt mkdir -p /mnt/var/lib/postgresql -mount -t zfs root_pool/encrypted/data/postgresql /mnt/var/lib/postgresql +mount -t zfs root_pool/data/postgresql /mnt/var/lib/postgresql -# Create a raid mirror for the efi boot -# see https://docs.hetzner.com/robot/dedicated-server/operating-systems/efi-system-partition/ -# TODO check this though the following article says it doesn't work properly -# https://outflux.net/blog/archives/2018/04/19/uefi-booting-and-raid1/ -mdadm --create --run --verbose /dev/md127 \ - --level 1 \ - --raid-disks 2 \ - --metadata 1.0 \ - --homehost=$MY_HOSTNAME \ - --name=boot_efi \ - $DISK1-part2 $DISK2-part2 - -# Assembling the RAID can result in auto-activation of previously-existing LVM -# groups, preventing the RAID block device wiping below with -# `Device or resource busy`. So disable all VGs first. -vgchange -an - -# Wipe filesystem signatures that might be on the RAID from some -# possibly existing older use of the disks (RAID creation does not do that). -# See https://serverfault.com/questions/911370/why-does-mdadm-zero-superblock-preserve-file-system-information -wipefs -a /dev/md127 - -# Disable RAID recovery. We don't want this to slow down machine provisioning -# in the rescue mode. It can run in normal operation after reboot. -echo 0 > /proc/sys/dev/raid/speed_limit_max - -# Filesystems (-F to not ask on preexisting FS) -mkfs.vfat -F 32 /dev/md127 - -# Creating file systems changes their UUIDs. -# Trigger udev so that the entries in /dev/disk/by-uuid get refreshed. -# `nixos-generate-config` depends on those being up-to-date. -# See https://github.com/NixOS/nixpkgs/issues/62444 -udevadm trigger - -mkdir -p /mnt/boot/efi -mount /dev/md127 /mnt/boot/efi +mkdir /mnt/boot1 /mnt/boot2 +mount $DISK1-part2 /mnt/boot1 +mount $DISK2-part2 /mnt/boot2 # Installing nix @@ -319,8 +286,21 @@ cat > /mnt/etc/nixos/configuration.nix < /mnt/etc/nixos/configuration.nix <