Merge pull request #245250 from nikstur/images

nixos/lib/systemd-lib.nix

@@ -443,4 +443,21 @@ in rec {
           ${attrsToSection def.sliceConfig}
         '';
     };
+
+  # Create a directory that contains systemd definition files from an attrset
+  # that contains the file names as keys and the content as values. The values
+  # in that attrset are determined by the supplied format.
+  definitions = directoryName: format: definitionAttrs:
+    let
+      listOfDefinitions = lib.mapAttrsToList
+        (name: format.generate "${name}.conf")
+        definitionAttrs;
+    in
+    pkgs.runCommand directoryName { } ''
+      mkdir -p $out
+      ${(lib.concatStringsSep "\n"
+        (map (pkg: "cp ${pkg} $out/${pkg.name}") listOfDefinitions)
+      )}
+    '';
+
 }

nixos/modules/image/amend-repart-definitions.py

@@ -0,0 +1,113 @@
+#!/usr/bin/env python
+
+"""Amend systemd-repart definiton files.
+
+In order to avoid Import-From-Derivation (IFD) when building images with
+systemd-repart, the definition files created by Nix need to be amended with the
+store paths from the closure.
+
+This is achieved by adding CopyFiles= instructions to the definition files.
+
+The arbitrary files configured via `contents` are also added to the definition
+files using the same mechanism.
+"""
+
+import json
+import sys
+import shutil
+import os
+import tempfile
+from pathlib import Path
+
+
+def add_contents_to_definition(
+    definition: Path, contents: dict[str, dict[str, str]] | None
+) -> None:
+    """Add CopyFiles= instructions to a definition for all files in contents."""
+    if not contents:
+        return
+
+    copy_files_lines: list[str] = []
+    for target, options in contents.items():
+        source = options["source"]
+
+        copy_files_lines.append(f"CopyFiles={source}:{target}\n")
+
+    with open(definition, "a") as f:
+        f.writelines(copy_files_lines)
+
+
+def add_closure_to_definition(
+    definition: Path, closure: Path | None, strip_nix_store_prefix: bool | None
+) -> None:
+    """Add CopyFiles= instructions to a definition for all paths in the closure.
+
+    If strip_nix_store_prefix is True, `/nix/store` is stripped from the target path.
+    """
+    if not closure:
+        return
+
+    copy_files_lines: list[str] = []
+    with open(closure, "r") as f:
+        for line in f:
+            if not isinstance(line, str):
+                continue
+
+            source = Path(line.strip())
+            target = str(source.relative_to("/nix/store/"))
+            target = f":{target}" if strip_nix_store_prefix else ""
+
+            copy_files_lines.append(f"CopyFiles={source}{target}\n")
+
+    with open(definition, "a") as f:
+        f.writelines(copy_files_lines)
+
+
+def main() -> None:
+    """Amend the provided repart definitions by adding CopyFiles= instructions.
+
+    For each file specified in the `contents` field of a partition in the
+    partiton config file, a `CopyFiles=` instruction is added to the
+    corresponding definition file.
+
+    The same is done for every store path of the `closure` field.
+
+    Print the path to a directory that contains the amended repart
+    definitions to stdout.
+    """
+    partition_config_file = sys.argv[1]
+    if not partition_config_file:
+        print("No partition config file was supplied.")
+        sys.exit(1)
+
+    repart_definitions = sys.argv[2]
+    if not repart_definitions:
+        print("No repart definitions were supplied.")
+        sys.exit(1)
+
+    with open(partition_config_file, "rb") as f:
+        partition_config = json.load(f)
+
+    if not partition_config:
+        print("Partition config is empty.")
+        sys.exit(1)
+
+    temp = tempfile.mkdtemp()
+    shutil.copytree(repart_definitions, temp, dirs_exist_ok=True)
+
+    for name, config in partition_config.items():
+        definition = Path(f"{temp}/{name}.conf")
+        os.chmod(definition, 0o644)
+
+        contents = config.get("contents")
+        add_contents_to_definition(definition, contents)
+
+        closure = config.get("closure")
+        strip_nix_store_prefix = config.get("stripStorePaths")
+        add_closure_to_definition(definition, closure, strip_nix_store_prefix)
+
+    print(temp)
+
+
+if __name__ == "__main__":
+    main()

nixos/modules/image/repart.md

@@ -0,0 +1,137 @@
+# Building Images via `systemd-repart` {#sec-image-repart}
+
+You can build disk images in NixOS with the `image.repart` option provided by
+the module [image/repart.nix][]. This module uses `systemd-repart` to build the
+images and exposes it's entire interface via the `repartConfig` option.
+
+[image/repart.nix]: https://github.com/NixOS/nixpkgs/blob/master/nixos/modules/image/repart.nix
+
+An example of how to build an image:
+
+```nix
+{ config, modulesPath, ... }: {
+
+  imports = [ "${modulesPath}/image/repart.nix" ];
+
+  image.repart = {
+    name = "image";
+    partitions = {
+      "esp" = {
+        contents = {
+          ...
+        };
+        repartConfig = {
+          Type = "esp";
+          ...
+        };
+      };
+      "root" = {
+        storePaths = [ config.system.build.toplevel ];
+        repartConfig = {
+          Type = "root";
+          Label = "nixos";
+          ...
+        };
+      };
+    };
+  };
+
+}
+```
+
+## Nix Store Partition {#sec-image-repart-store-partition}
+
+You can define a partition that only contains the Nix store and then mount it
+under `/nix/store`. Because the `/nix/store` part of the paths is already
+determined by the mount point, you have to set `stripNixStorePrefix = true;` so
+that the prefix is stripped from the paths before copying them into the image.
+
+```nix
+fileSystems."/nix/store".device = "/dev/disk/by-partlabel/nix-store"
+
+image.repart.partitions = {
+  "store" = {
+    storePaths = [ config.system.build.toplevel ];
+    stripNixStorePrefix = true;
+    repartConfig = {
+      Type = "linux-generic";
+      Label = "nix-store";
+      ...
+    };
+  };
+};
+```
+
+## Appliance Image {#sec-image-repart-appliance}
+
+The `image/repart.nix` module can also be used to build self-contained [software
+appliances][].
+
+[software appliances]: https://en.wikipedia.org/wiki/Software_appliance
+
+The generation based update mechanism of NixOS is not suited for appliances.
+Updates of appliances are usually either performed by replacing the entire
+image with a new one or by updating partitions via an A/B scheme. See the
+[Chrome OS update process][chrome-os-update] for an example of how to achieve
+this. The appliance image built in the following example does not contain a
+`configuration.nix` and thus you will not be able to call `nixos-rebuild` from
+this system.
+
+[chrome-os-update]: https://chromium.googlesource.com/aosp/platform/system/update_engine/+/HEAD/README.md
+
+```nix
+let
+  pkgs = import <nixpkgs> { };
+  efiArch = pkgs.stdenv.hostPlatform.efiArch;
+in
+(pkgs.nixos [
+  ({ config, lib, pkgs, modulesPath, ... }: {
+
+    imports = [ "${modulesPath}/image/repart.nix" ];
+
+    boot.loader.grub.enable = false;
+
+    fileSystems."/".device = "/dev/disk/by-label/nixos";
+
+    image.repart = {
+      name = "image";
+      partitions = {
+        "esp" = {
+          contents = {
+            "/EFI/BOOT/BOOT${lib.toUpper efiArch}.EFI".source =
+              "${pkgs.systemd}/lib/systemd/boot/efi/systemd-boot${efiArch}.efi";
+
+            "/loader/entries/nixos.conf".source = pkgs.writeText "nixos.conf" ''
+              title NixOS
+              linux /EFI/nixos/kernel.efi
+              initrd /EFI/nixos/initrd.efi
+              options init=${config.system.build.toplevel}/init ${toString config.boot.kernelParams}
+            '';
+
+            "/EFI/nixos/kernel.efi".source =
+              "${config.boot.kernelPackages.kernel}/${config.system.boot.loader.kernelFile}";
+
+            "/EFI/nixos/initrd.efi".source =
+              "${config.system.build.initialRamdisk}/${config.system.boot.loader.initrdFile}";
+          };
+          repartConfig = {
+            Type = "esp";
+            Format = "vfat";
+            SizeMinBytes = "96M";
+          };
+        };
+        "root" = {
+          storePaths = [ config.system.build.toplevel ];
+          repartConfig = {
+            Type = "root";
+            Format = "ext4";
+            Label = "nixos";
+            Minimize = "guess";
+          };
+        };
+      };
+    };
+
+  })
+]).image
+```

nixos/modules/image/repart.nix

@@ -0,0 +1,207 @@
+# This module exposes options to build a disk image with a GUID Partition Table
+# (GPT). It uses systemd-repart to build the image.
+
+{ config, pkgs, lib, utils, ... }:
+
+let
+  cfg = config.image.repart;
+
+  partitionOptions = {
+    options = {
+      storePaths = lib.mkOption {
+        type = with lib.types; listOf path;
+        default = [ ];
+        description = lib.mdDoc "The store paths to include in the partition.";
+      };
+
+      stripNixStorePrefix = lib.mkOption {
+        type = lib.types.bool;
+        default = false;
+        description = lib.mdDoc ''
+          Whether to strip `/nix/store/` from the store paths. This is useful
+          when you want to build a partition that only contains store paths and
+          is mounted under `/nix/store`.
+        '';
+      };
+
+      contents = lib.mkOption {
+        type = with lib.types; attrsOf (submodule {
+          options = {
+            source = lib.mkOption {
+              type = types.path;
+              description = lib.mdDoc "Path of the source file.";
+            };
+          };
+        });
+        default = { };
+        example = lib.literalExpression '' {
+          "/EFI/BOOT/BOOTX64.EFI".source =
+            "''${pkgs.systemd}/lib/systemd/boot/efi/systemd-bootx64.efi";
+
+          "/loader/entries/nixos.conf".source = systemdBootEntry;
+        }
+        '';
+        description = lib.mdDoc "The contents to end up in the filesystem image.";
+      };
+
+      repartConfig = lib.mkOption {
+        type = with lib.types; attrsOf (oneOf [ str int bool ]);
+        example = {
+          Type = "home";
+          SizeMinBytes = "512M";
+          SizeMaxBytes = "2G";
+        };
+        description = lib.mdDoc ''
+          Specify the repart options for a partiton as a structural setting.
+          See <https://www.freedesktop.org/software/systemd/man/repart.d.html>
+          for all available options.
+        '';
+      };
+    };
+  };
+in
+{
+  options.image.repart = {
+
+    name = lib.mkOption {
+      type = lib.types.str;
+      description = lib.mdDoc "The name of the image.";
+    };
+
+    seed = lib.mkOption {
+      type = with lib.types; nullOr str;
+      # Generated with `uuidgen`. Random but fixed to improve reproducibility.
+      default = "0867da16-f251-457d-a9e8-c31f9a3c220b";
+      description = lib.mdDoc ''
+        A UUID to use as a seed. You can set this to `null` to explicitly
+        randomize the partition UUIDs.
+      '';
+    };
+
+    split = lib.mkOption {
+      type = lib.types.bool;
+      default = false;
+      description = lib.mdDoc ''
+        Enables generation of split artifacts from partitions. If enabled, for
+        each partition with SplitName= set, a separate output file containing
+        just the contents of that partition is generated.
+      '';
+    };
+
+    partitions = lib.mkOption {
+      type = with lib.types; attrsOf (submodule partitionOptions);
+      default = { };
+      example = lib.literalExpression '' {
+        "10-esp" = {
+          contents = {
+            "/EFI/BOOT/BOOTX64.EFI".source =
+              "''${pkgs.systemd}/lib/systemd/boot/efi/systemd-bootx64.efi";
+          }
+          repartConfig = {
+            Type = "esp";
+            Format = "fat";
+          };
+        };
+        "20-root" = {
+          storePaths = [ config.system.build.toplevel ];
+          repartConfig = {
+            Type = "root";
+            Format = "ext4";
+            Minimize = "guess";
+          };
+        };
+      };
+      '';
+      description = lib.mdDoc ''
+        Specify partitions as a set of the names of the partitions with their
+        configuration as the key.
+      '';
+    };
+
+  };
+
+  config = {
+
+    system.build.image =
+      let
+        fileSystemToolMapping = with pkgs; {
+          "vfat" = [ dosfstools mtools ];
+          "ext4" = [ e2fsprogs.bin ];
+          "squashfs" = [ squashfsTools ];
+          "erofs" = [ erofs-utils ];
+          "btrfs" = [ btrfs-progs ];
+          "xfs" = [ xfsprogs ];
+        };
+
+        fileSystems = lib.filter
+          (f: f != null)
+          (lib.mapAttrsToList (_n: v: v.repartConfig.Format or null) cfg.partitions);
+
+        fileSystemTools = builtins.concatMap (f: fileSystemToolMapping."${f}") fileSystems;
+
+
+        makeClosure = paths: pkgs.closureInfo { rootPaths = paths; };
+
+        # Add the closure of the provided Nix store paths to cfg.partitions so
+        # that amend-repart-definitions.py can read it.
+        addClosure = _name: partitionConfig: partitionConfig // (
+          lib.optionalAttrs
+            (partitionConfig.storePaths or [ ] != [ ])
+            { closure = "${makeClosure partitionConfig.storePaths}/store-paths"; }
+        );
+
+
+        finalPartitions = lib.mapAttrs addClosure cfg.partitions;
+
+
+        amendRepartDefinitions = pkgs.runCommand "amend-repart-definitions.py"
+          {
+            nativeBuildInputs = with pkgs; [ black ruff mypy ];
+            buildInputs = [ pkgs.python3 ];
+          } ''
+          install ${./amend-repart-definitions.py} $out
+          patchShebangs --host $out
+
+          black --check --diff $out
+          ruff --line-length 88 $out
+          mypy --strict $out
+        '';
+
+        format = pkgs.formats.ini { };
+
+        definitionsDirectory = utils.systemdUtils.lib.definitions
+          "repart.d"
+          format
+          (lib.mapAttrs (_n: v: { Partition = v.repartConfig; }) finalPartitions);
+
+        partitions = pkgs.writeText "partitions.json" (builtins.toJSON finalPartitions);
+      in
+      pkgs.runCommand cfg.name
+        {
+          nativeBuildInputs = with pkgs; [
+            fakeroot
+            systemd
+          ] ++ fileSystemTools;
+        } ''
+        amendedRepartDefinitions=$(${amendRepartDefinitions} ${partitions} ${definitionsDirectory})
+
+        mkdir -p $out
+        cd $out
+
+        fakeroot systemd-repart \
+          --dry-run=no \
+          --empty=create \
+          --size=auto \
+          --seed="${cfg.seed}" \
+          --definitions="$amendedRepartDefinitions" \
+          --split="${lib.boolToString cfg.split}" \
+          image.raw
+      '';
+
+    meta = {
+      maintainers = with lib.maintainers; [ nikstur ];
+      doc = ./repart.md;
+    };
+
+  };
+}

nixos/modules/system/boot/systemd/repart.nix

@@ -1,28 +1,15 @@
-{ config, pkgs, lib, utils, ... }:
+{ config, lib, pkgs, utils, ... }:
 
 let
   cfg = config.systemd.repart;
   initrdCfg = config.boot.initrd.systemd.repart;
 
-  writeDefinition = name: partitionConfig: pkgs.writeText
-    "${name}.conf"
-    (lib.generators.toINI { } { Partition = partitionConfig; });
+  format = pkgs.formats.ini { };
 
-  listOfDefinitions = lib.mapAttrsToList
-    writeDefinition
-    (lib.filterAttrs (k: _: !(lib.hasPrefix "_" k)) cfg.partitions);
-
-  # Create a directory in the store that contains a copy of all definition
-  # files. This is then passed to systemd-repart in the initrd so it can access
-  # the definition files after the sysroot has been mounted but before
-  # activation. This needs a hard copy of the files and not just symlinks
-  # because otherwise the files do not show up in the sysroot.
-  definitionsDirectory = pkgs.runCommand "systemd-repart-definitions" { } ''
-    mkdir -p $out
-    ${(lib.concatStringsSep "\n"
-      (map (pkg: "cp ${pkg} $out/${pkg.name}") listOfDefinitions)
-    )}
-  '';
+  definitionsDirectory = utils.systemdUtils.lib.definitions
+    "repart.d"
+    format
+    (lib.mapAttrs (_n: v: { Partition = v; }) cfg.partitions);
 in
 {
   options = {

nixos/modules/system/boot/systemd/sysupdate.nix

@@ -5,16 +5,10 @@ let
 
   format = pkgs.formats.ini { };
 
-  listOfDefinitions = lib.mapAttrsToList
-    (name: format.generate "${name}.conf")
-    (lib.filterAttrs (k: _: !(lib.hasPrefix "_" k)) cfg.transfers);
-
-  definitionsDirectory = pkgs.runCommand "sysupdate.d" { } ''
-    mkdir -p $out
-    ${(lib.concatStringsSep "\n"
-      (map (pkg: "cp ${pkg} $out/${pkg.name}") listOfDefinitions)
-    )}
-  '';
+  definitionsDirectory = utils.systemdUtils.lib.definitions
+    "sysupdate.d"
+    format
+    cfg.transfers;
 in
 {
   options.systemd.sysupdate = {

nixos/tests/all-tests.nix

@@ -112,6 +112,7 @@ in {
   anuko-time-tracker = handleTest ./anuko-time-tracker.nix {};
   apcupsd = handleTest ./apcupsd.nix {};
   apfs = runTest ./apfs.nix;
+  appliance-repart-image = runTest ./appliance-repart-image.nix;
   apparmor = handleTest ./apparmor.nix {};
   atd = handleTest ./atd.nix {};
   atop = handleTest ./atop.nix {};

nixos/tests/appliance-repart-image.nix

@@ -0,0 +1,116 @@
+# Tests building and running a GUID Partition Table (GPT) appliance image.
+# "Appliance" here means that the image does not contain the normal NixOS
+# infrastructure of a system profile and cannot be re-built via
+# `nixos-rebuild`.
+
+{ lib, ... }:
+
+let
+  rootPartitionLabel = "root";
+
+  bootLoaderConfigPath = "/loader/entries/nixos.conf";
+  kernelPath = "/EFI/nixos/kernel.efi";
+  initrdPath = "/EFI/nixos/initrd.efi";
+in
+{
+  name = "appliance-gpt-image";
+
+  meta.maintainers = with lib.maintainers; [ nikstur ];
+
+  nodes.machine = { config, lib, pkgs, ... }: {
+
+    imports = [ ../modules/image/repart.nix ];
+
+    virtualisation.directBoot.enable = false;
+    virtualisation.mountHostNixStore = false;
+    virtualisation.useEFIBoot = true;
+
+    # Disable boot loaders because we install one "manually".
+    # TODO(raitobezarius): revisit this when #244907 lands
+    boot.loader.grub.enable = false;
+
+    virtualisation.fileSystems = lib.mkForce {
+      "/" = {
+        device = "/dev/disk/by-partlabel/${rootPartitionLabel}";
+        fsType = "ext4";
+      };
+    };
+
+    image.repart = {
+      name = "appliance-gpt-image";
+      partitions = {
+        "esp" = {
+          contents =
+            let
+              efiArch = config.nixpkgs.hostPlatform.efiArch;
+            in
+            {
+              "/EFI/BOOT/BOOT${lib.toUpper efiArch}.EFI".source =
+                "${pkgs.systemd}/lib/systemd/boot/efi/systemd-boot${efiArch}.efi";
+
+              # TODO: create an abstraction for Boot Loader Specification (BLS) entries.
+              "${bootLoaderConfigPath}".source = pkgs.writeText "nixos.conf" ''
+                title NixOS
+                linux ${kernelPath}
+                initrd ${initrdPath}
+                options init=${config.system.build.toplevel}/init ${toString config.boot.kernelParams}
+              '';
+
+              "${kernelPath}".source =
+                "${config.boot.kernelPackages.kernel}/${config.system.boot.loader.kernelFile}";
+
+              "${initrdPath}".source =
+                "${config.system.build.initialRamdisk}/${config.system.boot.loader.initrdFile}";
+            };
+          repartConfig = {
+            Type = "esp";
+            Format = "vfat";
+            # Minimize = "guess" seems to not work very vell for vfat
+            # partitons. It's better to set a sensible default instead. The
+            # aarch64 kernel seems to generally be a little bigger than the
+            # x86_64 kernel. To stay on the safe side, leave some more slack
+            # for every platform other than x86_64.
+            SizeMinBytes = if config.nixpkgs.hostPlatform.isx86_64 then "64M" else "96M";
+          };
+        };
+        "root" = {
+          storePaths = [ config.system.build.toplevel ];
+          repartConfig = {
+            Type = "root";
+            Format = config.fileSystems."/".fsType;
+            Label = rootPartitionLabel;
+            Minimize = "guess";
+          };
+        };
+      };
+    };
+  };
+
+  testScript = { nodes, ... }: ''
+    import os
+    import subprocess
+    import tempfile
+
+    tmp_disk_image = tempfile.NamedTemporaryFile()
+
+    subprocess.run([
+      "${nodes.machine.virtualisation.qemu.package}/bin/qemu-img",
+      "create",
+      "-f",
+      "qcow2",
+      "-b",
+      "${nodes.machine.system.build.image}/image.raw",
+      "-F",
+      "raw",
+      tmp_disk_image.name,
+    ])
+
+    # Set NIX_DISK_IMAGE so that the qemu script finds the right disk image.
+    os.environ['NIX_DISK_IMAGE'] = tmp_disk_image.name
+
+    bootctl_status = machine.succeed("bootctl status")
+    assert "${bootLoaderConfigPath}" in bootctl_status
+    assert "${kernelPath}" in bootctl_status
+    assert "${initrdPath}" in bootctl_status
+  '';
+}