From d5a3443410d6a442518c6016db92c27f6e9150fb Mon Sep 17 00:00:00 2001 From: Raphael Megzari Date: Fri, 9 Apr 2021 21:58:27 +0900 Subject: [PATCH] Add zfs uefi (#7) * initial try at adding zfs and uefi * update script to remove rescue mode * various fixes * remove uneeded grub efi * script improvements Co-authored-by: Sandro Co-authored-by: Sandro --- .../hetzner-dedicated/zfs-uefi-nvme-nixos.sh | 367 ++++++++++++++++++ 1 file changed, 367 insertions(+) create mode 100644 hosters/hetzner-dedicated/zfs-uefi-nvme-nixos.sh diff --git a/hosters/hetzner-dedicated/zfs-uefi-nvme-nixos.sh b/hosters/hetzner-dedicated/zfs-uefi-nvme-nixos.sh new file mode 100644 index 0000000..445369a --- /dev/null +++ b/hosters/hetzner-dedicated/zfs-uefi-nvme-nixos.sh @@ -0,0 +1,367 @@ +#!/usr/bin/env bash + +# Installs NixOS on a Hetzner server, wiping the server. +# +# This is for a specific server configuration; adjust where needed. +# +# Prerequisites: +# * this script requires ubuntu installed +# * Update the script to put in your SSH pubkey, adjust hostname, NixOS version etc. +# * have the following packages installed +# * - zfs-initramfs +# * - parted +# * - sudo +# * - grub-efi-amd64-bin +cat > /etc/apt/preferences.d/90_zfs <` because the file may already contain some detected RAID arrays, +# which would take precedence over our ``. +echo 'AUTO -all +ARRAY UUID=00000000:00000000:00000000:00000000' > /etc/mdadm/mdadm.conf + +# Create partition tables (--script to not ask) +parted --script $DISK1 mklabel gpt +parted --script $DISK2 mklabel gpt + +# Create partitions (--script to not ask) +# +# We create the 1MB BIOS boot partition at the front. +# +# Note we use "MB" instead of "MiB" because otherwise `--align optimal` has no effect; +# as per documentation https://www.gnu.org/software/parted/manual/html_node/unit.html#unit: +# > Note that as of parted-2.4, when you specify start and/or end values using IEC +# > binary units like "MiB", "GiB", "TiB", etc., parted treats those values as exact +# +# Note: When using `mkpart` on GPT, as per +# https://www.gnu.org/software/parted/manual/html_node/mkpart.html#mkpart +# the first argument to `mkpart` is not a `part-type`, but the GPT partition name: +# ... part-type is one of 'primary', 'extended' or 'logical', and may be specified only with 'msdos' or 'dvh' partition tables. +# A name must be specified for a 'gpt' partition table. +# GPT partition names are limited to 36 UTF-16 chars, see https://en.wikipedia.org/wiki/GUID_Partition_Table#Partition_entries_(LBA_2-33). +# TODO the bios partition should not be this big +# however if it's less the installation fails with +# cannot copy /nix/store/d4xbrrailkn179cdp90v4m57mqd73hvh-linux-5.4.100/bzImage to /boot/kernels/d4xbrrailkn179cdp90v4m57mqd73hvh-linux-5.4.100-bzImage.tmp: No space left on device +parted --script --align optimal $DISK1 -- mklabel gpt \ + mkpart 'BIOS-boot-partition' 1MB 2MB set 1 bios_grub on \ + mkpart 'EFI-system-partition' 2MB 512MB set 2 esp on \ + mkpart 'data-partition' 512MB '100%' + +parted --script --align optimal $DISK2 -- mklabel gpt \ + mkpart 'BIOS-boot-partition' 1MB 2MB set 1 bios_grub on \ + mkpart 'EFI-system-partition' 2MB 512MB set 2 esp on \ + mkpart 'data-partition' 512MB '100%' + +# Reload partitions +partprobe + +# Wait for all devices to exist +udevadm settle --timeout=5 --exit-if-exists=$DISK1-part1 +udevadm settle --timeout=5 --exit-if-exists=$DISK1-part2 +udevadm settle --timeout=5 --exit-if-exists=$DISK1-part3 +udevadm settle --timeout=5 --exit-if-exists=$DISK2-part1 +udevadm settle --timeout=5 --exit-if-exists=$DISK2-part2 +udevadm settle --timeout=5 --exit-if-exists=$DISK2-part3 + +# Wipe any previous RAID signatures +# somehow the previous RAID signature is only on part1 +mdadm --zero-superblock --force $DISK1-part1 +mdadm --zero-superblock --force $DISK2-part1 + +# Creating file systems changes their UUIDs. +# Trigger udev so that the entries in /dev/disk/by-uuid get refreshed. +# `nixos-generate-config` depends on those being up-to-date. +# See https://github.com/NixOS/nixpkgs/issues/62444 +udevadm trigger + +# taken from https://nixos.wiki/wiki/NixOS_on_ZFS +zpool create -O mountpoint=none \ + -O atime=off \ + -O compression=lz4 \ + -O xattr=sa \ + -O acltype=posixacl \ + -o ashift=12 \ + -f \ + rpool mirror $DISK1-part3 $DISK2-part3 + +# Create the filesystems. This layout is designed so that /home is separate from the root +# filesystem, as you'll likely want to snapshot it differently for backup purposes. It also +# makes a "nixos" filesystem underneath the root, to support installing multiple OSes if +# that's something you choose to do in future. +zfs create -o mountpoint=legacy rpool/root +zfs create -o mountpoint=legacy rpool/root/nixos +zfs create -o mountpoint=legacy rpool/home +# add 1G of reseved space in case the disk gets full +# zfs needs space to delete files +zfs create -o refreservation=1G -o mountpoint=none rpool/reserved +# this creates a special volume for db data see https://wiki.archlinux.org/index.php/ZFS#Databases +zfs create -o mountpoint=legacy \ + -o recordsize=8K \ + -o primarycache=metadata \ + -o logbias=throughput \ + rpool/postgres + +# NixOS pre-installation mounts +# +# Mount the filesystems manually. The nixos installer will detect these mountpoints +# and save them to /mnt/nixos/hardware-configuration.nix during the install process. +mount -t zfs rpool/root/nixos /mnt +mkdir /mnt/home +mount -t zfs rpool/home /mnt/home +mkdir -p /mnt/var/lib/postgres +mount -t zfs rpool/postgres /mnt/var/lib/postgres + +# Create a raid mirror for the efi boot +# see https://docs.hetzner.com/robot/dedicated-server/operating-systems/efi-system-partition/ +# TODO check this though the following article says it doesn't work properly +# https://outflux.net/blog/archives/2018/04/19/uefi-booting-and-raid1/ +mdadm --create --run --verbose /dev/md127 \ + --level 1 \ + --raid-disks 2 \ + --metadata 1.0 \ + --homehost=$MY_HOSTNAME \ + --name=boot_efi \ + $DISK1-part2 $DISK2-part2 + +# Assembling the RAID can result in auto-activation of previously-existing LVM +# groups, preventing the RAID block device wiping below with +# `Device or resource busy`. So disable all VGs first. +vgchange -an + +# Wipe filesystem signatures that might be on the RAID from some +# possibly existing older use of the disks (RAID creation does not do that). +# See https://serverfault.com/questions/911370/why-does-mdadm-zero-superblock-preserve-file-system-information +wipefs -a /dev/md127 + +# Disable RAID recovery. We don't want this to slow down machine provisioning +# in the rescue mode. It can run in normal operation after reboot. +echo 0 > /proc/sys/dev/raid/speed_limit_max + +# Filesystems (-F to not ask on preexisting FS) +mkfs.vfat -F 32 /dev/md127 + +# Creating file systems changes their UUIDs. +# Trigger udev so that the entries in /dev/disk/by-uuid get refreshed. +# `nixos-generate-config` depends on those being up-to-date. +# See https://github.com/NixOS/nixpkgs/issues/62444 +udevadm trigger + +mkdir -p /mnt/boot/efi +mount /dev/md127 /mnt/boot/efi + +# Installing nix + +# Allow installing nix as root, see +# https://github.com/NixOS/nix/issues/936#issuecomment-475795730 +mkdir -p /etc/nix +echo "build-users-group =" > /etc/nix/nix.conf + +# TODO +# warning: installing Nix as root is not supported by this script! +curl -L https://nixos.org/nix/install | sh +set +u +x # sourcing this may refer to unset variables that we have no control over +. $HOME/.nix-profile/etc/profile.d/nix.sh +set -u -x + +# Keep in sync with `system.stateVersion` set below! +nix-channel --add https://nixos.org/channels/nixos-20.09 nixpkgs +nix-channel --update + +# Getting NixOS installation tools +nix-env -iE "_: with import { configuration = {}; }; with config.system.build; [ nixos-generate-config nixos-install nixos-enter manual.manpages ]" + +# TODO +# perl: warning: Please check that your locale settings: +# LANGUAGE = (unset), +# LC_ALL = "en_US.UTF-8", +# LANG = "en_US.UTF-8" +# are supported and installed on your system. +nixos-generate-config --root /mnt + +# Find the name of the network interface that connects us to the Internet. +# Inspired by https://unix.stackexchange.com/questions/14961/how-to-find-out-which-interface-am-i-using-for-connecting-to-the-internet/302613#302613 +RESCUE_INTERFACE=$(ip route get 8.8.8.8 | grep -Po '(?<=dev )(\S+)') + +# Find what its name will be under NixOS, which uses stable interface names. +# See https://major.io/2015/08/21/understanding-systemds-predictable-network-device-names/#comment-545626 +# NICs for most Hetzner servers are not onboard, which is why we use +# `ID_NET_NAME_PATH`otherwise it would be `ID_NET_NAME_ONBOARD`. +INTERFACE_DEVICE_PATH=$(udevadm info -e | grep -Po "(?<=^P: )(.*${RESCUE_INTERFACE})") +UDEVADM_PROPERTIES_FOR_INTERFACE=$(udevadm info --query=property "--path=$INTERFACE_DEVICE_PATH") +NIXOS_INTERFACE=$(echo "$UDEVADM_PROPERTIES_FOR_INTERFACE" | grep -o -E 'ID_NET_NAME_PATH=\w+' | cut -d= -f2) +echo "Determined NIXOS_INTERFACE as '$NIXOS_INTERFACE'" + +IP_V4=$(ip route get 8.8.8.8 | grep -Po '(?<=src )(\S+)') +echo "Determined IP_V4 as $IP_V4" + +# Determine Internet IPv6 by checking route, and using ::1 +# (because Hetzner rescue mode uses ::2 by default). +# The `ip -6 route get` output on Hetzner looks like: +# # ip -6 route get 2001:4860:4860:0:0:0:0:8888 +# 2001:4860:4860::8888 via fe80::1 dev eth0 src 2a01:4f8:151:62aa::2 metric 1024 pref medium +IP_V6="$(ip route get 2001:4860:4860:0:0:0:0:8888 | head -1 | cut -d' ' -f7 | cut -d: -f1-4)::1" +echo "Determined IP_V6 as $IP_V6" + +# From https://stackoverflow.com/questions/1204629/how-do-i-get-the-default-gateway-in-linux-given-the-destination/15973156#15973156 +read _ _ DEFAULT_GATEWAY _ < <(ip route list match 0/0); echo "$DEFAULT_GATEWAY" +echo "Determined DEFAULT_GATEWAY as $DEFAULT_GATEWAY" + +# Generate `configuration.nix`. Note that we splice in shell variables. +cat > /mnt/etc/nixos/configuration.nix <