From 955310683206739931c2106ac661a670181be364 Mon Sep 17 00:00:00 2001 From: Klemens Nanni Date: Thu, 12 May 2022 15:24:52 +0200 Subject: [PATCH 1/5] nixos/stage-1: Ensure correct ZFS mount options Consider ZFS filesystems meant to be mounted with zfs.mount(8), e.g. ``` config.fileSystems."/media".options = [ "zfsutil" ]; config.fileSystems."/nix".options = [ "zfsutil" ]; ``` `zfsutil` uses dataset properties as mount options such that zfsprops(7) do not have to be duplicated in fstab(5) entries or manual mount(8) invocations. Given the example configuation above, /media is correctly mounted with `setuid=off` translated into `nosuid`: ``` $ zfs get -Ho value setuid /media off $ findmnt -t zfs -no options /media rw,nosuid,nodev,noexec,noatime,xattr,posixacl ``` /nix however was mounted with default mount(8) options: ``` $ zfs get -Ho value setuid /nix off $ findmnt -t zfs -no options /nix rw,relatime,xattr,noacl ``` This holds true for all other ZFS properties/mount options, including `exec/[no]exec`, `devices/[no]dev`, `atime/[no]atime`, etc. /nix is mounted using BusyBox's `mount` during stage 1 init while /media is mounted later using proper systemd and/or util-linux's `mount`. Tracing stage 1 init showed that BusyBox never tried to execute mount.zfs(8) as intended by `zfsutil`. Replacing it with util-linux's `mount` and adding the mount helper showed attempts to execute mount.zfs(8). Ensure ZFS filesystems are mounted with correct options iff `zfsutil` is used. --- nixos/modules/system/boot/stage-1.nix | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/nixos/modules/system/boot/stage-1.nix b/nixos/modules/system/boot/stage-1.nix index d10ebac5682..adb8eb7ccf7 100644 --- a/nixos/modules/system/boot/stage-1.nix +++ b/nixos/modules/system/boot/stage-1.nix @@ -31,6 +31,9 @@ let # mounting `/`, like `/` on a loopback). fileSystems = filter utils.fsNeededForBoot config.system.build.fileSystems; + # Determine whether zfs-mount(8) is needed. + zfsRequiresMountHelper = any (fs: lib.elem "zfsutil" fs.options) fileSystems; + # A utility for enumerating the shared-library dependencies of a program findLibs = pkgs.buildPackages.writeShellScriptBin "find-libs" '' set -euo pipefail @@ -107,6 +110,22 @@ let copy_bin_and_libs $BIN done + ${optionalString zfsRequiresMountHelper '' + # Filesystems using the "zfsutil" option are mounted regardless of the + # mount.zfs(8) helper, but it is required to ensure that ZFS properties + # are used as mount options. + # + # BusyBox does not use the ZFS helper in the first place. + # util-linux searches /sbin/ as last path for helpers (stage-1-init.sh + # must symlink it to the store PATH). + # Without helper program, both `mount`s silently fails back to internal + # code, using default options and effectively ignore security relevant + # ZFS properties such as `setuid=off` and `exec=off` (unless manually + # duplicated in `fileSystems.*.options`, defeating "zfsutil"'s purpose). + copy_bin_and_libs ${pkgs.util-linux}/bin/mount + copy_bin_and_libs ${pkgs.zfs}/bin/mount.zfs + ''} + # Copy some util-linux stuff. copy_bin_and_libs ${pkgs.util-linux}/sbin/blkid @@ -221,7 +240,12 @@ let echo "testing patched programs..." $out/bin/ash -c 'echo hello world' | grep "hello world" export LD_LIBRARY_PATH=$out/lib - $out/bin/mount --help 2>&1 | grep -q "BusyBox" + ${if zfsRequiresMountHelper then '' + $out/bin/mount -V 1>&1 | grep -q "mount from util-linux" + $out/bin/mount.zfs -h 2>&1 | grep -q "Usage: mount.zfs" + '' else '' + $out/bin/mount --help 2>&1 | grep -q "BusyBox" + ''} $out/bin/blkid -V 2>&1 | grep -q 'libblkid' $out/bin/udevadm --version $out/bin/dmsetup --version 2>&1 | tee -a log | grep -q "version:" From de77849ad689101c4a7af14a9d02c9906d2b0b13 Mon Sep 17 00:00:00 2001 From: Klemens Nanni Date: Thu, 12 May 2022 16:18:30 +0200 Subject: [PATCH 2/5] nixos/stage-1: Account for hardcoded executable paths At least pkgs/os-specific/linux/util-linux/default.nix uses ``` "--enable-fs-paths-default=/run/wrappers/bin:/run/current-system/sw/bin:/sbin" ``` which does not cover stage 1 init's PATH as all executables are put under /bin/. Fix util-linux's `mount` usage by symlinking /sbin to it. --- nixos/modules/system/boot/stage-1-init.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nixos/modules/system/boot/stage-1-init.sh b/nixos/modules/system/boot/stage-1-init.sh index 22d5ec76af7..337064034ef 100644 --- a/nixos/modules/system/boot/stage-1-init.sh +++ b/nixos/modules/system/boot/stage-1-init.sh @@ -14,6 +14,8 @@ extraUtils="@extraUtils@" export LD_LIBRARY_PATH=@extraUtils@/lib export PATH=@extraUtils@/bin ln -s @extraUtils@/bin /bin +# hardcoded in util-linux's mount helper search path `/run/wrappers/bin:/run/current-system/sw/bin:/sbin` +ln -s @extraUtils@/bin /sbin # Copy the secrets to their needed location if [ -d "@extraUtils@/secrets" ]; then From 4b045c70664617180c776d3d301fb7563572e66f Mon Sep 17 00:00:00 2001 From: Klemens Nanni Date: Thu, 12 May 2022 21:35:33 +0200 Subject: [PATCH 3/5] nixos/stage-1: Remove redundant symlink check find(1)'s test `-type f` already excludes symbolic links, so `test -L` will never return false for found files. --- nixos/modules/system/boot/stage-1.nix | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/nixos/modules/system/boot/stage-1.nix b/nixos/modules/system/boot/stage-1.nix index adb8eb7ccf7..ec2bd5ef352 100644 --- a/nixos/modules/system/boot/stage-1.nix +++ b/nixos/modules/system/boot/stage-1.nix @@ -223,16 +223,12 @@ let # Run patchelf to make the programs refer to the copied libraries. find $out/bin $out/lib -type f | while read i; do - if ! test -L $i; then - nuke-refs -e $out $i - fi + nuke-refs -e $out $i done find $out/bin -type f | while read i; do - if ! test -L $i; then - echo "patching $i..." - patchelf --set-interpreter $out/lib/ld*.so.? --set-rpath $out/lib $i || true - fi + echo "patching $i..." + patchelf --set-interpreter $out/lib/ld*.so.? --set-rpath $out/lib $i || true done if [ -z "${toString (pkgs.stdenv.hostPlatform != pkgs.stdenv.buildPlatform)}" ]; then From d33e52b2539c5b36a5a876df9006a7145efd42ea Mon Sep 17 00:00:00 2001 From: Klemens Nanni Date: Sun, 15 May 2022 01:00:37 +0200 Subject: [PATCH 4/5] nixos/stage-1: Fix library path in libraries also `extra-utils` composes the set of programs and libraries needed by 1. copying over all programs 2. copying over all libraries any program directly links against 3. set the runtime path for every program to the library directory It seems that this approach misses the case where a library itself links against another library. That is to say, `extra-utils` assumes that either only progams link against libraries or that every library linked to by a library is already linked to by a program. `mount.zfs` linking against `libcrypto`, in turn linking against `libdl` shows how the current approach falls short: ``` $ objdump -p $(which mount.zfs) | grep NEEDED | grep -e libdl -e libcrypto NEEDED libcrypto.so.1.1 $ ldd (which mount.zfs) | grep libdl libdl.so.2 => /nix/store/ybkkrhdwdj227kr20vk8qnzqnmj7a06x-glibc-2.34-115/lib/libdl.so.2 (0x00007f9967a9a000 ``` Using `mount.zfs` directly in stage 1 init still works since `LD_LIBRARY_PATH` overrides this (as intended). util-linux's `mount` however executes `mount.zfs` with LD_LIBRARY_PATH removed from its environment as can be seen with strace(1) in an interactive stage 1 init shell (`boot.shell_on_fail` kernel parameter): ``` # env -i LD_LIBRARY_PATH=$LD_LIBRARY_PATH $(which strace) -ff -e trace=/exec -v -qqq $(which mount) /mnt-root execve("/nix/store/3gqbb3swgiy749fxd5a4k6kirkr2jr9n-extra-utils/bin/mount", ["/nix/store/3gqbb3swgiy749fxd5a4k"..., "/mnt-root"], ["LD_LIBRARY_PATH=/nix/store/3gqbb"...]) = 0 [pid 1026] execve("/sbin/mount.zfs", ["/sbin/mount.zfs", "", "/mnt-root", "-o", "rw,zfsutil"], []) = 0 /sbin/mount.zfs: error while loading shared libraries: libdl.so.2: cannot open shared object file: No such file or directory --- SIGCHLD {si_signo=SIGCHLD, si_code=CLD_EXITED, si_pid=1026, si_uid=0, si_status=127, si_utime=0, si_stime=0} --- ``` env(1) is used for clarity (hence subshells for absoloute paths). While `mount` uses the right library path, `mount.zfs` is stripped of it, so ld.so(8) fails resolve `libdl` (as required by `libcrypto`). To fix this and not rely on `LD_LIBRARY_PATH` to be set, fix the library path inside libraries as well. This finally mounts all ZFS filesystems using `zfsutil` with correct and intended mount options. --- nixos/modules/system/boot/stage-1.nix | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/nixos/modules/system/boot/stage-1.nix b/nixos/modules/system/boot/stage-1.nix index ec2bd5ef352..2900919b4b6 100644 --- a/nixos/modules/system/boot/stage-1.nix +++ b/nixos/modules/system/boot/stage-1.nix @@ -231,6 +231,11 @@ let patchelf --set-interpreter $out/lib/ld*.so.? --set-rpath $out/lib $i || true done + find $out/lib -type f \! -name 'ld*.so.?' | while read i; do + echo "patching $i..." + patchelf --set-rpath $out/lib $i + done + if [ -z "${toString (pkgs.stdenv.hostPlatform != pkgs.stdenv.buildPlatform)}" ]; then # Make sure that the patchelf'ed binaries still work. echo "testing patched programs..." From 9eb704b65a43047d136218b98e6f01fc20a6e83c Mon Sep 17 00:00:00 2001 From: Klemens Nanni Date: Sun, 15 May 2022 01:35:15 +0200 Subject: [PATCH 5/5] nixos/stage-1: Zap no longer needed LD_LIBRARY_PATH The previous commit properly adjusts all library paths, thus no need to forcefully adjust the path at runtime any longer. --- nixos/modules/system/boot/stage-1.nix | 3 --- 1 file changed, 3 deletions(-) diff --git a/nixos/modules/system/boot/stage-1.nix b/nixos/modules/system/boot/stage-1.nix index 2900919b4b6..e35ccff2907 100644 --- a/nixos/modules/system/boot/stage-1.nix +++ b/nixos/modules/system/boot/stage-1.nix @@ -240,7 +240,6 @@ let # Make sure that the patchelf'ed binaries still work. echo "testing patched programs..." $out/bin/ash -c 'echo hello world' | grep "hello world" - export LD_LIBRARY_PATH=$out/lib ${if zfsRequiresMountHelper then '' $out/bin/mount -V 1>&1 | grep -q "mount from util-linux" $out/bin/mount.zfs -h 2>&1 | grep -q "Usage: mount.zfs" @@ -285,8 +284,6 @@ let } '' mkdir -p $out - echo 'ENV{LD_LIBRARY_PATH}="${extraUtils}/lib"' > $out/00-env.rules - cp -v ${udev}/lib/udev/rules.d/60-cdrom_id.rules $out/ cp -v ${udev}/lib/udev/rules.d/60-persistent-storage.rules $out/ cp -v ${udev}/lib/udev/rules.d/75-net-description.rules $out/