From ededd308a83218db2e69bb3a94df79414c30eb6d Mon Sep 17 00:00:00 2001
From: V <v@anomalous.eu>
Date: Tue, 9 Feb 2021 18:06:58 +0100
Subject: [PATCH 01/23] nixos/boot: add /var/lib/nixos to pathsNeededForBoot

/var/lib/nixos is used by update-users-groups.pl in the activation
script for storing uid/gid mappings. If this has its own mountpoint
(as is the case in some setups with fine-grained bind mounts pointing
into persistent storage), the mappings are written to /var/lib, /var,
or /. These may be backed by a tmpfs or (otherwise ephemeral storage),
resulting in the mappings not persisting between reboots.
---
 nixos/lib/utils.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/nixos/lib/utils.nix b/nixos/lib/utils.nix
index c9dfdbed99a..9c8294b20e5 100644
--- a/nixos/lib/utils.nix
+++ b/nixos/lib/utils.nix
@@ -10,7 +10,7 @@ rec {
   # Check whenever fileSystem is needed for boot.  NOTE: Make sure
   # pathsNeededForBoot is closed under the parent relationship, i.e. if /a/b/c
   # is in the list, put /a and /a/b in as well.
-  pathsNeededForBoot = [ "/" "/nix" "/nix/store" "/var" "/var/log" "/var/lib" "/etc" ];
+  pathsNeededForBoot = [ "/" "/nix" "/nix/store" "/var" "/var/log" "/var/lib" "/var/lib/nixos" "/etc" ];
   fsNeededForBoot = fs: fs.neededForBoot || elem fs.mountPoint pathsNeededForBoot;
 
   # Check whenever `b` depends on `a` as a fileSystem

From cd42b9fff8581b36ddd843241d42e214715af851 Mon Sep 17 00:00:00 2001
From: Artturin <Artturin@artturin.com>
Date: Sun, 3 Oct 2021 03:43:41 +0300
Subject: [PATCH 02/23] fetchfirefoxaddon: Allow overriding the src and add a
 test for it

Co-authored-by: Thomas Sean Dominic Kelly <thomassdk@pm.me>
---
 .../fetchfirefoxaddon/default.nix             | 21 +++++++++++--------
 .../build-support/fetchfirefoxaddon/tests.nix | 13 +++++++++++-
 2 files changed, 24 insertions(+), 10 deletions(-)

diff --git a/pkgs/build-support/fetchfirefoxaddon/default.nix b/pkgs/build-support/fetchfirefoxaddon/default.nix
index 127f32dd61b..79014fd23c4 100644
--- a/pkgs/build-support/fetchfirefoxaddon/default.nix
+++ b/pkgs/build-support/fetchfirefoxaddon/default.nix
@@ -2,19 +2,26 @@
 
 {
   name
-, url
+, url ? null
 , md5 ? ""
 , sha1 ? ""
 , sha256 ? ""
 , sha512 ? ""
 , fixedExtid ? null
 , hash ? ""
+, src ? ""
 }:
 
-stdenv.mkDerivation rec {
-
-  inherit name;
+let
   extid = if fixedExtid == null then "nixos@${name}" else fixedExtid;
+  source = if url == null then src else fetchurl {
+    url = url;
+    inherit md5 sha1 sha256 sha512 hash;
+  };
+in
+stdenv.mkDerivation {
+  inherit name;
+
   passthru = {
     inherit extid;
   };
@@ -26,16 +33,12 @@ stdenv.mkDerivation rec {
 
     UUID="${extid}"
     mkdir -p "$out/$UUID"
-    unzip -q ${src} -d "$out/$UUID"
+    unzip -q ${source} -d "$out/$UUID"
     NEW_MANIFEST=$(jq '. + {"applications": { "gecko": { "id": "${extid}" }}, "browser_specific_settings":{"gecko":{"id": "${extid}"}}}' "$out/$UUID/manifest.json")
     echo "$NEW_MANIFEST" > "$out/$UUID/manifest.json"
     cd "$out/$UUID"
     zip -r -q -FS "$out/$UUID.xpi" *
     rm -r "$out/$UUID"
   '';
-  src = fetchurl {
-    url = url;
-    inherit md5 sha1 sha256 sha512 hash;
-  };
   nativeBuildInputs = [ coreutils unzip zip jq  ];
 }
diff --git a/pkgs/build-support/fetchfirefoxaddon/tests.nix b/pkgs/build-support/fetchfirefoxaddon/tests.nix
index d125be196af..c407d0e74b8 100644
--- a/pkgs/build-support/fetchfirefoxaddon/tests.nix
+++ b/pkgs/build-support/fetchfirefoxaddon/tests.nix
@@ -1,4 +1,4 @@
-{ invalidateFetcherByDrvHash, fetchFirefoxAddon, ... }:
+{ invalidateFetcherByDrvHash, fetchFirefoxAddon, fetchurl, ... }:
 
 {
   simple = invalidateFetcherByDrvHash fetchFirefoxAddon {
@@ -7,4 +7,15 @@
     url = "https://addons.mozilla.org/firefox/downloads/file/3059971/image_search_options-3.0.12-fx.xpi";
     sha256 = "sha256-H73YWX/DKxvhEwKpWOo7orAQ7c/rQywpljeyxYxv0Gg=";
   };
+  overidden-source =
+    let
+      image-search-options = fetchurl {
+        url = "https://addons.mozilla.org/firefox/downloads/file/3059971/image_search_options-3.0.12-fx.xpi";
+        sha256 = "sha256-H73YWX/DKxvhEwKpWOo7orAQ7c/rQywpljeyxYxv0Gg=";
+      };
+    in
+    invalidateFetcherByDrvHash fetchFirefoxAddon {
+      name = "image-search-options";
+      src = image-search-options;
+    };
 }

From 6ef3c96ddf7132e0dbb0d958b03fcc0ce327a0b0 Mon Sep 17 00:00:00 2001
From: laikq <55911173+laikq@users.noreply.github.com>
Date: Tue, 5 Oct 2021 08:47:08 +0200
Subject: [PATCH 03/23] doc: reference sourceRoot in description of srcs

---
 doc/stdenv/stdenv.chapter.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/doc/stdenv/stdenv.chapter.md b/doc/stdenv/stdenv.chapter.md
index 9befcaa51a9..02042407f6c 100644
--- a/doc/stdenv/stdenv.chapter.md
+++ b/doc/stdenv/stdenv.chapter.md
@@ -373,11 +373,11 @@ Additional file types can be supported by setting the `unpackCmd` variable (see
 
 ##### `srcs` / `src` {#var-stdenv-src}
 
-The list of source files or directories to be unpacked or copied. One of these must be set.
+The list of source files or directories to be unpacked or copied. One of these must be set. Note that if you use `srcs`, you should also set `sourceRoot` or `setSourceRoot`.
 
 ##### `sourceRoot` {#var-stdenv-sourceRoot}
 
-After running `unpackPhase`, the generic builder changes the current directory to the directory created by unpacking the sources. If there are multiple source directories, you should set `sourceRoot` to the name of the intended directory.
+After running `unpackPhase`, the generic builder changes the current directory to the directory created by unpacking the sources. If there are multiple source directories, you should set `sourceRoot` to the name of the intended directory. Set `sourceRoot = ".";` if you use `srcs` and control the unpack phase yourself.
 
 ##### `setSourceRoot` {#var-stdenv-setSourceRoot}
 

From b0fc9da879812e47c1ed3438fb0fd51db00a3494 Mon Sep 17 00:00:00 2001
From: David Arnold <dar@xoe.solutions>
Date: Sat, 12 Jun 2021 17:47:25 -0500
Subject: [PATCH 04/23] nixos/test/test-driver: Class-ify the test driver

This commit encapsulates the involved domain into classes and
defines explicit and typed arguments where untyped dicts where used.

It preserves backwards compatibility through legacy wrappers.
---
 nixos/lib/test-driver/test-driver.py          | 812 +++++++++++-------
 nixos/lib/testing-python.nix                  |  11 +-
 .../tools/nixos-build-vms/build-vms.nix       |  19 +-
 3 files changed, 531 insertions(+), 311 deletions(-)

diff --git a/nixos/lib/test-driver/test-driver.py b/nixos/lib/test-driver/test-driver.py
index f8502188bde..fdc440a896a 100755
--- a/nixos/lib/test-driver/test-driver.py
+++ b/nixos/lib/test-driver/test-driver.py
@@ -21,7 +21,6 @@ import shutil
 import socket
 import subprocess
 import sys
-import telnetlib
 import tempfile
 import time
 import unicodedata
@@ -89,55 +88,6 @@ CHAR_TO_KEY = {
     ")": "shift-0x0B",
 }
 
-global log, machines, test_script
-
-
-def eprint(*args: object, **kwargs: Any) -> None:
-    print(*args, file=sys.stderr, **kwargs)
-
-
-def make_command(args: list) -> str:
-    return " ".join(map(shlex.quote, (map(str, args))))
-
-
-def create_vlan(vlan_nr: str) -> Tuple[str, str, "subprocess.Popen[bytes]", Any]:
-    log.log("starting VDE switch for network {}".format(vlan_nr))
-    vde_socket = tempfile.mkdtemp(
-        prefix="nixos-test-vde-", suffix="-vde{}.ctl".format(vlan_nr)
-    )
-    pty_master, pty_slave = pty.openpty()
-    vde_process = subprocess.Popen(
-        ["vde_switch", "-s", vde_socket, "--dirmode", "0700"],
-        stdin=pty_slave,
-        stdout=subprocess.PIPE,
-        stderr=subprocess.PIPE,
-        shell=False,
-    )
-    fd = os.fdopen(pty_master, "w")
-    fd.write("version\n")
-    # TODO: perl version checks if this can be read from
-    # an if not, dies. we could hang here forever. Fix it.
-    assert vde_process.stdout is not None
-    vde_process.stdout.readline()
-    if not os.path.exists(os.path.join(vde_socket, "ctl")):
-        raise Exception("cannot start vde_switch")
-
-    return (vlan_nr, vde_socket, vde_process, fd)
-
-
-def retry(fn: Callable, timeout: int = 900) -> None:
-    """Call the given function repeatedly, with 1 second intervals,
-    until it returns True or a timeout is reached.
-    """
-
-    for _ in range(timeout):
-        if fn(False):
-            return
-        time.sleep(1)
-
-    if not fn(True):
-        raise Exception(f"action timed out after {timeout} seconds")
-
 
 class Logger:
     def __init__(self) -> None:
@@ -151,6 +101,10 @@ class Logger:
 
         self._print_serial_logs = True
 
+    @staticmethod
+    def _eprint(*args: object, **kwargs: Any) -> None:
+        print(*args, file=sys.stderr, **kwargs)
+
     def close(self) -> None:
         self.xml.endElement("logfile")
         self.xml.endDocument()
@@ -169,15 +123,27 @@ class Logger:
         self.xml.characters(message)
         self.xml.endElement("line")
 
+    def info(self, *args, **kwargs) -> None:  # type: ignore
+        self.log(*args, **kwargs)
+
+    def warning(self, *args, **kwargs) -> None:  # type: ignore
+        self.log(*args, **kwargs)
+
+    def error(self, *args, **kwargs) -> None:  # type: ignore
+        self.log(*args, **kwargs)
+        sys.exit(1)
+
     def log(self, message: str, attributes: Dict[str, str] = {}) -> None:
-        eprint(self.maybe_prefix(message, attributes))
+        self._eprint(self.maybe_prefix(message, attributes))
         self.drain_log_queue()
         self.log_line(message, attributes)
 
     def log_serial(self, message: str, machine: str) -> None:
         self.enqueue({"msg": message, "machine": machine, "type": "serial"})
         if self._print_serial_logs:
-            eprint(Style.DIM + "{} # {}".format(machine, message) + Style.RESET_ALL)
+            self._eprint(
+                Style.DIM + "{} # {}".format(machine, message) + Style.RESET_ALL
+            )
 
     def enqueue(self, item: Dict[str, str]) -> None:
         self.queue.put(item)
@@ -194,7 +160,7 @@ class Logger:
 
     @contextmanager
     def nested(self, message: str, attributes: Dict[str, str] = {}) -> Iterator[None]:
-        eprint(self.maybe_prefix(message, attributes))
+        self._eprint(self.maybe_prefix(message, attributes))
 
         self.xml.startElement("nest", attrs={})
         self.xml.startElement("head", attributes)
@@ -211,6 +177,27 @@ class Logger:
         self.xml.endElement("nest")
 
 
+rootlog = Logger()
+
+
+def make_command(args: list) -> str:
+    return " ".join(map(shlex.quote, (map(str, args))))
+
+
+def retry(fn: Callable, timeout: int = 900) -> None:
+    """Call the given function repeatedly, with 1 second intervals,
+    until it returns True or a timeout is reached.
+    """
+
+    for _ in range(timeout):
+        if fn(False):
+            return
+        time.sleep(1)
+
+    if not fn(True):
+        raise Exception(f"action timed out after {timeout} seconds")
+
+
 def _perform_ocr_on_screenshot(
     screenshot_path: str, model_ids: Iterable[int]
 ) -> List[str]:
@@ -242,113 +229,256 @@ def _perform_ocr_on_screenshot(
     return model_results
 
 
+class StartCommand:
+    """The Base Start Command knows how to append the necesary
+    runtime qemu options as determined by a particular test driver
+    run. Any such start command is expected to happily receive and
+    append additional qemu args.
+    """
+
+    _cmd: str
+
+    def cmd(
+        self,
+        monitor_socket_path: pathlib.Path,
+        shell_socket_path: pathlib.Path,
+        allow_reboot: bool = False,  # TODO: unused, legacy?
+    ) -> str:
+        display_opts = ""
+        display_available = any(x in os.environ for x in ["DISPLAY", "WAYLAND_DISPLAY"])
+        if display_available:
+            display_opts += " -nographic"
+
+        # qemu options
+        qemu_opts = ""
+        qemu_opts += (
+            ""
+            if allow_reboot
+            else " -no-reboot"
+            " -device virtio-serial"
+            " -device virtconsole,chardev=shell"
+            " -device virtio-rng-pci"
+            " -serial stdio"
+        )
+        # TODO: qemu script already catpures this env variable, legacy?
+        qemu_opts += " " + os.environ.get("QEMU_OPTS", "")
+
+        return (
+            f"{self._cmd}"
+            f" -monitor unix:{monitor_socket_path}"
+            f" -chardev socket,id=shell,path={shell_socket_path}"
+            f"{qemu_opts}"
+            f"{display_opts}"
+        )
+
+    @staticmethod
+    def build_environment(
+        state_dir: pathlib.Path,
+        shared_dir: pathlib.Path,
+    ) -> dict:
+        # We make a copy to not update the current environment
+        env = dict(os.environ)
+        env.update(
+            {
+                "TMPDIR": str(state_dir),
+                "SHARED_DIR": str(shared_dir),
+                "USE_TMPDIR": "1",
+            }
+        )
+        return env
+
+    def run(
+        self,
+        state_dir: pathlib.Path,
+        shared_dir: pathlib.Path,
+        monitor_socket_path: pathlib.Path,
+        shell_socket_path: pathlib.Path,
+    ) -> subprocess.Popen:
+        return subprocess.Popen(
+            self.cmd(monitor_socket_path, shell_socket_path),
+            stdin=subprocess.DEVNULL,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.STDOUT,
+            shell=True,
+            cwd=state_dir,
+            env=self.build_environment(state_dir, shared_dir),
+        )
+
+
+class NixStartScript(StartCommand):
+    """A start script from nixos/modules/virtualiation/qemu-vm.nix
+    that also satisfies the requirement of the BaseStartCommand.
+    These Nix commands have the particular charactersitic that the
+    machine name can be extracted out of them via a regex match.
+    (Admittedly a _very_ implicit contract, evtl. TODO fix)
+    """
+
+    def __init__(self, script: str):
+        self._cmd = script
+
+    @property
+    def machine_name(self) -> str:
+        match = re.search("run-(.+)-vm$", self._cmd)
+        name = "machine"
+        if match:
+            name = match.group(1)
+        return name
+
+
+class LegacyStartCommand(StartCommand):
+    """Used in some places to create an ad-hoc machine instead of
+    using nix test instrumentation + module system for that purpose.
+    Legacy.
+    """
+
+    def __init__(
+        self,
+        netBackendArgs: Optional[str] = None,
+        netFrontendArgs: Optional[str] = None,
+        hda: Optional[Tuple[pathlib.Path, str]] = None,
+        cdrom: Optional[str] = None,
+        usb: Optional[str] = None,
+        bios: Optional[str] = None,
+        qemuFlags: Optional[str] = None,
+    ):
+        self._cmd = "qemu-kvm -m 384"
+
+        # networking
+        net_backend = "-netdev user,id=net0"
+        net_frontend = "-device virtio-net-pci,netdev=net0"
+        if netBackendArgs is not None:
+            net_backend += "," + netBackendArgs
+        if netFrontendArgs is not None:
+            net_frontend += "," + netFrontendArgs
+        self._cmd += f" {net_backend} {net_frontend}"
+
+        # hda
+        hda_cmd = ""
+        if hda is not None:
+            hda_path = hda[0].resolve()
+            hda_interface = hda[1]
+            if hda_interface == "scsi":
+                hda_cmd += (
+                    f" -drive id=hda,file={hda_path},werror=report,if=none"
+                    " -device scsi-hd,drive=hda"
+                )
+            else:
+                hda_cmd += f" -drive file={hda_path},if={hda_interface},werror=report"
+        self._cmd += hda_cmd
+
+        # cdrom
+        if cdrom is not None:
+            self._cmd += f" -cdrom {cdrom}"
+
+        # usb
+        usb_cmd = ""
+        if usb is not None:
+            # https://github.com/qemu/qemu/blob/master/docs/usb2.txt
+            usb_cmd += (
+                " -device usb-ehci"
+                f" -drive id=usbdisk,file={usb},if=none,readonly"
+                " -device usb-storage,drive=usbdisk "
+            )
+        self._cmd += usb_cmd
+
+        # bios
+        if bios is not None:
+            self._cmd += f" -bios {bios}"
+
+        # qemu flags
+        if qemuFlags is not None:
+            self._cmd += f" {qemuFlags}"
+
+
 class Machine:
+    """A handle to the machine with this name, that also knows how to manage
+    the machine lifecycle with the help of a start script / command."""
+
+    name: str
+    tmp_dir: pathlib.Path
+    shared_dir: pathlib.Path
+    state_dir: pathlib.Path
+    monitor_path: pathlib.Path
+    shell_path: pathlib.Path
+
+    start_command: StartCommand
+    keep_vm_state: bool
+    allow_reboot: bool
+
+    process: Optional[subprocess.Popen] = None
+    pid: Optional[int] = None
+    monitor: Optional[socket.socket] = None
+    shell: Optional[socket.socket] = None
+
+    booted: bool = False
+    connected: bool = False
+    # Store last serial console lines for use
+    # of wait_for_console_text
+    last_lines: Queue = Queue()
+
     def __repr__(self) -> str:
         return f"<Machine '{self.name}'>"
 
-    def __init__(self, args: Dict[str, Any]) -> None:
-        if "name" in args:
-            self.name = args["name"]
-        else:
-            self.name = "machine"
-            cmd = args.get("startCommand", None)
-            if cmd:
-                match = re.search("run-(.+)-vm$", cmd)
-                if match:
-                    self.name = match.group(1)
-        self.logger = args["log"]
-        self.script = args.get("startCommand", self.create_startcommand(args))
+    def __init__(
+        self,
+        tmp_dir: pathlib.Path,
+        start_command: StartCommand,
+        name: str = "machine",
+        keep_vm_state: bool = False,
+        allow_reboot: bool = False,
+    ) -> None:
+        self.tmp_dir = tmp_dir
+        self.keep_vm_state = keep_vm_state
+        self.allow_reboot = allow_reboot
+        self.name = name
+        self.start_command = start_command
 
-        tmp_dir = os.environ.get("TMPDIR", tempfile.gettempdir())
+        # set up directories
+        self.shared_dir = self.tmp_dir / "shared-xchg"
+        self.shared_dir.mkdir(mode=0o700, exist_ok=True)
 
-        def create_dir(name: str) -> str:
-            path = os.path.join(tmp_dir, name)
-            os.makedirs(path, mode=0o700, exist_ok=True)
-            return path
-
-        self.state_dir = os.path.join(tmp_dir, f"vm-state-{self.name}")
-        if not args.get("keepVmState", False):
+        self.state_dir = self.tmp_dir / f"vm-state-{self.name}"
+        self.monitor_path = self.state_dir / "monitor"
+        self.shell_path = self.state_dir / "shell"
+        if (not self.keep_vm_state) and self.state_dir.exists():
             self.cleanup_statedir()
-        os.makedirs(self.state_dir, mode=0o700, exist_ok=True)
-        self.shared_dir = create_dir("shared-xchg")
-
-        self.booted = False
-        self.connected = False
-        self.pid: Optional[int] = None
-        self.socket = None
-        self.monitor: Optional[socket.socket] = None
-        self.allow_reboot = args.get("allowReboot", False)
+        self.state_dir.mkdir(mode=0o700, exist_ok=True)
 
     @staticmethod
-    def create_startcommand(args: Dict[str, str]) -> str:
-        net_backend = "-netdev user,id=net0"
-        net_frontend = "-device virtio-net-pci,netdev=net0"
-
-        if "netBackendArgs" in args:
-            net_backend += "," + args["netBackendArgs"]
-
-        if "netFrontendArgs" in args:
-            net_frontend += "," + args["netFrontendArgs"]
-
-        start_command = (
-            args.get("qemuBinary", "qemu-kvm")
-            + " -m 384 "
-            + net_backend
-            + " "
-            + net_frontend
-            + " $QEMU_OPTS "
+    def create_startcommand(args: Dict[str, str]) -> StartCommand:
+        rootlog.warning(
+            "Using legacy create_startcommand(),"
+            "please use proper nix test vm instrumentation, instead"
+            "to generate the appropriate nixos test vm qemu startup script"
+        )
+        hda = None
+        if args.get("hda"):
+            hda_arg: str = args.get("hda", "")
+            hda_arg_path: pathlib.Path = pathlib.Path(hda_arg)
+            hda = (hda_arg_path, args.get("hdaInterface", ""))
+        return LegacyStartCommand(
+            netBackendArgs=args.get("netBackendArgs"),
+            netFrontendArgs=args.get("netFrontendArgs"),
+            hda=hda,
+            cdrom=args.get("cdrom"),
+            usb=args.get("usb"),
+            bios=args.get("bios"),
+            qemuFlags=args.get("qemuFlags"),
         )
-
-        if "hda" in args:
-            hda_path = os.path.abspath(args["hda"])
-            if args.get("hdaInterface", "") == "scsi":
-                start_command += (
-                    "-drive id=hda,file="
-                    + hda_path
-                    + ",werror=report,if=none "
-                    + "-device scsi-hd,drive=hda "
-                )
-            else:
-                start_command += (
-                    "-drive file="
-                    + hda_path
-                    + ",if="
-                    + args["hdaInterface"]
-                    + ",werror=report "
-                )
-
-        if "cdrom" in args:
-            start_command += "-cdrom " + args["cdrom"] + " "
-
-        if "usb" in args:
-            # https://github.com/qemu/qemu/blob/master/docs/usb2.txt
-            start_command += (
-                "-device usb-ehci -drive "
-                + "id=usbdisk,file="
-                + args["usb"]
-                + ",if=none,readonly "
-                + "-device usb-storage,drive=usbdisk "
-            )
-        if "bios" in args:
-            start_command += "-bios " + args["bios"] + " "
-
-        start_command += args.get("qemuFlags", "")
-
-        return start_command
 
     def is_up(self) -> bool:
         return self.booted and self.connected
 
     def log(self, msg: str) -> None:
-        self.logger.log(msg, {"machine": self.name})
+        rootlog.log(msg, {"machine": self.name})
 
     def log_serial(self, msg: str) -> None:
-        self.logger.log_serial(msg, self.name)
+        rootlog.log_serial(msg, self.name)
 
     def nested(self, msg: str, attrs: Dict[str, str] = {}) -> _GeneratorContextManager:
         my_attrs = {"machine": self.name}
         my_attrs.update(attrs)
-        return self.logger.nested(msg, my_attrs)
+        return rootlog.nested(msg, my_attrs)
 
     def wait_for_monitor_prompt(self) -> str:
         assert self.monitor is not None
@@ -446,6 +576,7 @@ class Machine:
         self.connect()
 
         out_command = "( set -euo pipefail; {} ); echo '|!=EOF' $?\n".format(command)
+        assert self.shell
         self.shell.send(out_command.encode())
 
         output = ""
@@ -466,6 +597,8 @@ class Machine:
         Should only be used during test development, not in the production test."""
         self.connect()
         self.log("Terminal is ready (there is no prompt):")
+
+        assert self.shell
         subprocess.run(
             ["socat", "READLINE", f"FD:{self.shell.fileno()}"],
             pass_fds=[self.shell.fileno()],
@@ -534,6 +667,7 @@ class Machine:
 
         with self.nested("waiting for the VM to power off"):
             sys.stdout.flush()
+            assert self.process
             self.process.wait()
 
             self.pid = None
@@ -611,6 +745,8 @@ class Machine:
         with self.nested("waiting for the VM to finish booting"):
             self.start()
 
+            assert self.shell
+
             tic = time.time()
             self.shell.recv(1024)
             # TODO: Timeout
@@ -750,65 +886,35 @@ class Machine:
 
         self.log("starting vm")
 
-        def create_socket(path: str) -> socket.socket:
-            if os.path.exists(path):
-                os.unlink(path)
+        def clear(path: pathlib.Path) -> pathlib.Path:
+            if path.exists():
+                path.unlink()
+            return path
+
+        def create_socket(path: pathlib.Path) -> socket.socket:
             s = socket.socket(family=socket.AF_UNIX, type=socket.SOCK_STREAM)
-            s.bind(path)
+            s.bind(str(path))
             s.listen(1)
             return s
 
-        monitor_path = os.path.join(self.state_dir, "monitor")
-        self.monitor_socket = create_socket(monitor_path)
-
-        shell_path = os.path.join(self.state_dir, "shell")
-        self.shell_socket = create_socket(shell_path)
-
-        display_available = any(x in os.environ for x in ["DISPLAY", "WAYLAND_DISPLAY"])
-        qemu_options = (
-            " ".join(
-                [
-                    "" if self.allow_reboot else "-no-reboot",
-                    "-monitor unix:{}".format(monitor_path),
-                    "-chardev socket,id=shell,path={}".format(shell_path),
-                    "-device virtio-serial",
-                    "-device virtconsole,chardev=shell",
-                    "-device virtio-rng-pci",
-                    "-serial stdio" if display_available else "-nographic",
-                ]
-            )
-            + " "
-            + os.environ.get("QEMU_OPTS", "")
+        monitor_socket = create_socket(clear(self.monitor_path))
+        shell_socket = create_socket(clear(self.shell_path))
+        self.process = self.start_command.run(
+            self.state_dir,
+            self.shared_dir,
+            self.monitor_path,
+            self.shell_path,
         )
-
-        environment = dict(os.environ)
-        environment.update(
-            {
-                "TMPDIR": self.state_dir,
-                "SHARED_DIR": self.shared_dir,
-                "USE_TMPDIR": "1",
-                "QEMU_OPTS": qemu_options,
-            }
-        )
-
-        self.process = subprocess.Popen(
-            self.script,
-            stdin=subprocess.DEVNULL,
-            stdout=subprocess.PIPE,
-            stderr=subprocess.STDOUT,
-            shell=True,
-            cwd=self.state_dir,
-            env=environment,
-        )
-        self.monitor, _ = self.monitor_socket.accept()
-        self.shell, _ = self.shell_socket.accept()
+        self.monitor, _ = monitor_socket.accept()
+        self.shell, _ = shell_socket.accept()
 
         # Store last serial console lines for use
         # of wait_for_console_text
         self.last_lines: Queue = Queue()
 
         def process_serial_output() -> None:
-            assert self.process.stdout is not None
+            assert self.process
+            assert self.process.stdout
             for _line in self.process.stdout:
                 # Ignore undecodable bytes that may occur in boot menus
                 line = _line.decode(errors="ignore").replace("\r", "").rstrip()
@@ -825,15 +931,15 @@ class Machine:
         self.log("QEMU running (pid {})".format(self.pid))
 
     def cleanup_statedir(self) -> None:
-        if os.path.isdir(self.state_dir):
-            shutil.rmtree(self.state_dir)
-            self.logger.log(f"deleting VM state directory {self.state_dir}")
-            self.logger.log("if you want to keep the VM state, pass --keep-vm-state")
+        shutil.rmtree(self.state_dir)
+        rootlog.log(f"deleting VM state directory {self.state_dir}")
+        rootlog.log("if you want to keep the VM state, pass --keep-vm-state")
 
     def shutdown(self) -> None:
         if not self.booted:
             return
 
+        assert self.shell
         self.shell.send("poweroff\n".encode())
         self.wait_for_shutdown()
 
@@ -908,41 +1014,225 @@ class Machine:
         """Make the machine reachable."""
         self.send_monitor_command("set_link virtio-net-pci.1 on")
 
-
-def create_machine(args: Dict[str, Any]) -> Machine:
-    args["log"] = log
-    return Machine(args)
+    def release(self) -> None:
+        if self.pid is None:
+            return
+        rootlog.info(f"kill machine (pid {self.pid})")
+        assert self.process
+        assert self.shell
+        assert self.monitor
+        self.process.terminate()
+        self.shell.close()
+        self.monitor.close()
 
 
-def start_all() -> None:
-    with log.nested("starting all VMs"):
-        for machine in machines:
-            machine.start()
+class VLan:
+    """A handle to the vlan with this number, that also knows how to manage
+    it's lifecycle.
+    """
+
+    nr: int
+    socket_dir: pathlib.Path
+
+    process: Optional[subprocess.Popen]
+    pid: Optional[int]
+    fd: Optional[io.TextIOBase]
+
+    def __repr__(self) -> str:
+        return f"<Vlan Nr. {self.nr}>"
+
+    def __init__(self, nr: int, tmp_dir: pathlib.Path):
+        self.nr = nr
+        self.socket_dir = tmp_dir / f"vde{self.nr}.ctl"
+
+        # TODO: don't side-effect environment here
+        os.environ[f"QEMU_VDE_SOCKET_{self.nr}"] = str(self.socket_dir)
+
+    def start(self) -> None:
+
+        rootlog.info("start vlan")
+        pty_master, pty_slave = pty.openpty()
+
+        self.process = subprocess.Popen(
+            ["vde_switch", "-s", self.socket_dir, "--dirmode", "0700"],
+            stdin=pty_slave,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            shell=False,
+        )
+        self.pid = self.process.pid
+        self.fd = os.fdopen(pty_master, "w")
+        self.fd.write("version\n")
+
+        # TODO: perl version checks if this can be read from
+        # an if not, dies. we could hang here forever. Fix it.
+        assert self.process.stdout is not None
+        self.process.stdout.readline()
+        if not (self.socket_dir / "ctl").exists():
+            rootlog.error("cannot start vde_switch")
+
+        rootlog.info(f"running vlan (pid {self.pid})")
+
+    def release(self) -> None:
+        if self.pid is None:
+            return
+        rootlog.info(f"kill vlan (pid {self.pid})")
+        assert self.fd
+        assert self.process
+        self.fd.close()
+        self.process.terminate()
 
 
-def join_all() -> None:
-    with log.nested("waiting for all VMs to finish"):
-        for machine in machines:
-            machine.wait_for_shutdown()
+class Driver:
+    """A handle to the driver that sets up the environment
+    and runs the tests"""
 
+    tests: str
+    vlans: List[VLan]
+    machines: List[Machine]
 
-def run_tests(interactive: bool = False) -> None:
-    if interactive:
-        ptpython.repl.embed(test_symbols(), {})
-    else:
-        test_script()
+    def __init__(
+        self,
+        start_scripts: List[str],
+        vlans: List[int],
+        tests: str,
+        keep_vm_state: bool = False,
+    ):
+        self.tests = tests
+
+        tmp_dir = pathlib.Path(os.environ.get("TMPDIR", tempfile.gettempdir()))
+        tmp_dir.mkdir(mode=0o700, exist_ok=True)
+
+        self.vlans = [VLan(nr, tmp_dir) for nr in vlans]
+        with rootlog.nested("start all VLans"):
+            for vlan in self.vlans:
+                vlan.start()
+
+        def cmd(scripts: List[str]) -> Iterator[NixStartScript]:
+            for s in scripts:
+                yield NixStartScript(s)
+
+        self.machines = [
+            Machine(
+                start_command=cmd,
+                keep_vm_state=keep_vm_state,
+                name=cmd.machine_name,
+                tmp_dir=tmp_dir,
+            )
+            for cmd in cmd(start_scripts)
+        ]
+
+        @atexit.register
+        def clean_up() -> None:
+            with rootlog.nested("clean up"):
+                for machine in self.machines:
+                    machine.release()
+                for vlan in self.vlans:
+                    vlan.release()
+
+    def subtest(self, name: str) -> Iterator[None]:
+        """Group logs under a given test name"""
+        with rootlog.nested(name):
+            try:
+                yield
+                return True
+            except:
+                rootlog.error(f'Test "{name}" failed with error:')
+                raise
+
+    def test_symbols(self) -> Dict[str, Any]:
+        @contextmanager
+        def subtest(name: str) -> Iterator[None]:
+            return self.subtest(name)
+
+        general_symbols = dict(
+            start_all=self.start_all,
+            test_script=self.test_script,
+            machines=self.machines,
+            vlans=self.vlans,
+            driver=self,
+            log=rootlog,
+            os=os,
+            create_machine=self.create_machine,
+            subtest=subtest,
+            run_tests=self.run_tests,
+            join_all=self.join_all,
+            retry=retry,
+            serial_stdout_off=self.serial_stdout_off,
+            serial_stdout_on=self.serial_stdout_on,
+            Machine=Machine,  # for typing
+        )
+        machine_symbols = {
+            m.name: self.machines[idx] for idx, m in enumerate(self.machines)
+        }
+        vlan_symbols = {
+            f"vlan{v.nr}": self.vlans[idx] for idx, v in enumerate(self.vlans)
+        }
+        print(
+            "additionally exposed symbols:\n    "
+            + ", ".join(map(lambda m: m.name, self.machines))
+            + ",\n    "
+            + ", ".join(map(lambda v: f"vlan{v.nr}", self.vlans))
+            + ",\n    "
+            + ", ".join(list(general_symbols.keys()))
+        )
+        return {**general_symbols, **machine_symbols, **vlan_symbols}
+
+    def test_script(self) -> None:
+        """Run the test script"""
+        with rootlog.nested("run the VM test script"):
+            symbols = self.test_symbols()  # call eagerly
+            exec(self.tests, symbols, None)
+
+    def run_tests(self) -> None:
+        """Run the test script (for non-interactive test runs)"""
+        self.test_script()
         # TODO: Collect coverage data
-        for machine in machines:
+        for machine in self.machines:
             if machine.is_up():
                 machine.execute("sync")
 
+    def start_all(self) -> None:
+        """Start all machines"""
+        with rootlog.nested("start all VMs"):
+            for machine in self.machines:
+                machine.start()
 
-def serial_stdout_on() -> None:
-    log._print_serial_logs = True
+    def join_all(self) -> None:
+        """Wait for all machines to shut down"""
+        with rootlog.nested("wait for all VMs to finish"):
+            for machine in self.machines:
+                machine.wait_for_shutdown()
 
+    def create_machine(self, args: Dict[str, Any]) -> Machine:
+        rootlog.warning(
+            "Using legacy create_machine(), please instantiate the"
+            "Machine class directly, instead"
+        )
+        tmp_dir = pathlib.Path(os.environ.get("TMPDIR", tempfile.gettempdir()))
+        tmp_dir.mkdir(mode=0o700, exist_ok=True)
 
-def serial_stdout_off() -> None:
-    log._print_serial_logs = False
+        if args.get("startCommand"):
+            start_command: str = args.get("startCommand", "")
+            cmd = NixStartScript(start_command)
+            name = args.get("name", cmd.machine_name)
+        else:
+            cmd = Machine.create_startcommand(args)  # type: ignore
+            name = args.get("name", "machine")
+
+        return Machine(
+            tmp_dir=tmp_dir,
+            start_command=cmd,
+            name=name,
+            keep_vm_state=args.get("keep_vm_state", False),
+            allow_reboot=args.get("allow_reboot", False),
+        )
+
+    def serial_stdout_on(self) -> None:
+        rootlog._print_serial_logs = True
+
+    def serial_stdout_off(self) -> None:
+        rootlog._print_serial_logs = False
 
 
 class EnvDefault(argparse.Action):
@@ -970,52 +1260,6 @@ class EnvDefault(argparse.Action):
         setattr(namespace, self.dest, values)
 
 
-@contextmanager
-def subtest(name: str) -> Iterator[None]:
-    with log.nested(name):
-        try:
-            yield
-            return True
-        except Exception as e:
-            log.log(f'Test "{name}" failed with error: "{e}"')
-            raise e
-
-    return False
-
-
-def _test_symbols() -> Dict[str, Any]:
-    general_symbols = dict(
-        start_all=start_all,
-        test_script=globals().get("test_script"),  # same
-        machines=globals().get("machines"),  # without being initialized
-        log=globals().get("log"),  # extracting those symbol keys
-        os=os,
-        create_machine=create_machine,
-        subtest=subtest,
-        run_tests=run_tests,
-        join_all=join_all,
-        retry=retry,
-        serial_stdout_off=serial_stdout_off,
-        serial_stdout_on=serial_stdout_on,
-        Machine=Machine,  # for typing
-    )
-    return general_symbols
-
-
-def test_symbols() -> Dict[str, Any]:
-
-    general_symbols = _test_symbols()
-
-    machine_symbols = {m.name: machines[idx] for idx, m in enumerate(machines)}
-    print(
-        "additionally exposed symbols:\n    "
-        + ", ".join(map(lambda m: m.name, machines))
-        + ",\n    "
-        + ", ".join(list(general_symbols.keys()))
-    )
-    return {**general_symbols, **machine_symbols}
-
-
 if __name__ == "__main__":
     arg_parser = argparse.ArgumentParser(prog="nixos-test-driver")
     arg_parser.add_argument(
@@ -1055,44 +1299,18 @@ if __name__ == "__main__":
     )
 
     args = arg_parser.parse_args()
-    testscript = pathlib.Path(args.testscript).read_text()
 
-    global log, machines, test_script
+    if not args.keep_vm_state:
+        rootlog.info("Machine state will be reset. To keep it, pass --keep-vm-state")
 
-    log = Logger()
+    driver = Driver(
+        args.start_scripts, args.vlans, args.testscript.read_text(), args.keep_vm_state
+    )
 
-    vde_sockets = [create_vlan(v) for v in args.vlans]
-    for nr, vde_socket, _, _ in vde_sockets:
-        os.environ["QEMU_VDE_SOCKET_{}".format(nr)] = vde_socket
-
-    machines = [
-        create_machine({"startCommand": s, "keepVmState": args.keep_vm_state})
-        for s in args.start_scripts
-    ]
-    machine_eval = [
-        "{0} = machines[{1}]".format(m.name, idx) for idx, m in enumerate(machines)
-    ]
-    exec("\n".join(machine_eval))
-
-    @atexit.register
-    def clean_up() -> None:
-        with log.nested("cleaning up"):
-            for machine in machines:
-                if machine.pid is None:
-                    continue
-                log.log("killing {} (pid {})".format(machine.name, machine.pid))
-                machine.process.kill()
-            for _, _, process, _ in vde_sockets:
-                process.terminate()
-        log.close()
-
-    def test_script() -> None:
-        with log.nested("running the VM test script"):
-            symbols = test_symbols()  # call eagerly
-            exec(testscript, symbols, None)
-
-    interactive = args.interactive or (not bool(testscript))
-    tic = time.time()
-    run_tests(interactive)
-    toc = time.time()
-    print("test script finished in {:.2f}s".format(toc - tic))
+    if args.interactive:
+        ptpython.repl.embed(driver.test_symbols(), {})
+    else:
+        tic = time.time()
+        driver.run_tests()
+        toc = time.time()
+        rootlog.info(f"test script finished in {(toc-tic):.2f}s")
diff --git a/nixos/lib/testing-python.nix b/nixos/lib/testing-python.nix
index 43b4f9b159b..1969f40edb6 100644
--- a/nixos/lib/testing-python.nix
+++ b/nixos/lib/testing-python.nix
@@ -43,7 +43,8 @@ rec {
         from pydoc import importfile
         with open('driver-symbols', 'w') as fp:
           t = importfile('${testDriverScript}')
-          test_symbols = t._test_symbols()
+          d = t.Driver([],[],"")
+          test_symbols = d.test_symbols()
           fp.write(','.join(test_symbols.keys()))
         EOF
       '';
@@ -188,14 +189,6 @@ rec {
           --set startScripts "''${vmStartScripts[*]}" \
           --set testScript "$out/test-script" \
           --set vlans '${toString vlans}'
-
-        ${lib.optionalString (testScript == "") ''
-          ln -s ${testDriver}/bin/nixos-test-driver $out/bin/nixos-run-vms
-          wrapProgram $out/bin/nixos-run-vms \
-            --set startScripts "''${vmStartScripts[*]}" \
-            --set testScript "${pkgs.writeText "start-all" "start_all(); join_all();"}" \
-            --set vlans '${toString vlans}'
-        ''}
       '');
 
   # Make a full-blown test
diff --git a/nixos/modules/installer/tools/nixos-build-vms/build-vms.nix b/nixos/modules/installer/tools/nixos-build-vms/build-vms.nix
index e49ceba2424..ce69b16cffa 100644
--- a/nixos/modules/installer/tools/nixos-build-vms/build-vms.nix
+++ b/nixos/modules/installer/tools/nixos-build-vms/build-vms.nix
@@ -8,11 +8,20 @@ let
     _file = "${networkExpr}@node-${vm}";
     imports = [ module ];
   }) (import networkExpr);
+
+  testing = import ../../../../lib/testing-python.nix {
+    inherit system;
+    pkgs = import ../../../../.. { inherit system config; };
+  };
+
+  interactiveDriver = (testing.makeTest { inherit nodes; testScript = "start_all(); join_all();"; }).driverInteractive;
 in
 
-with import ../../../../lib/testing-python.nix {
-  inherit system;
-  pkgs = import ../../../../.. { inherit system config; };
-};
 
-(makeTest { inherit nodes; testScript = ""; }).driverInteractive
+pkgs.runCommand "nixos-build-vms" ''
+  mkdir -p $out/bin
+  ln -s ${interactiveDriver}/bin/nixos-test-driver $out/bin/nixos-test-driver
+  ln -s ${interactiveDriver}/bin/nixos-test-driver $out/bin/nixos-run-vms
+  wrapProgram $out/bin/nixos-test-driver \
+    --add-flags "--interactive"
+''

From b2e59bcf778db9be13c675dfc2524962acec2965 Mon Sep 17 00:00:00 2001
From: Maximilian Bosch <maximilian@mbosch.me>
Date: Fri, 20 Aug 2021 11:48:15 +0200
Subject: [PATCH 05/23] nixos/build-vms: fix eval

---
 .../modules/installer/tools/nixos-build-vms/build-vms.nix  | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/nixos/modules/installer/tools/nixos-build-vms/build-vms.nix b/nixos/modules/installer/tools/nixos-build-vms/build-vms.nix
index ce69b16cffa..8aedce2fb49 100644
--- a/nixos/modules/installer/tools/nixos-build-vms/build-vms.nix
+++ b/nixos/modules/installer/tools/nixos-build-vms/build-vms.nix
@@ -9,16 +9,17 @@ let
     imports = [ module ];
   }) (import networkExpr);
 
+  pkgs = import ../../../../.. { inherit system config; };
+
   testing = import ../../../../lib/testing-python.nix {
-    inherit system;
-    pkgs = import ../../../../.. { inherit system config; };
+    inherit system pkgs;
   };
 
   interactiveDriver = (testing.makeTest { inherit nodes; testScript = "start_all(); join_all();"; }).driverInteractive;
 in
 
 
-pkgs.runCommand "nixos-build-vms" ''
+pkgs.runCommand "nixos-build-vms" { nativeBuildInputs = [ pkgs.makeWrapper ]; } ''
   mkdir -p $out/bin
   ln -s ${interactiveDriver}/bin/nixos-test-driver $out/bin/nixos-test-driver
   ln -s ${interactiveDriver}/bin/nixos-test-driver $out/bin/nixos-run-vms

From 3f63e3ce653b476554c6c07dc66e9efae73ee9d0 Mon Sep 17 00:00:00 2001
From: Maximilian Bosch <maximilian@mbosch.me>
Date: Fri, 20 Aug 2021 12:00:40 +0200
Subject: [PATCH 06/23] nixos/test-driver: fix graphics for VM

`-nographic` should only be set if no display is available. Otherwise,
tools such as `nixos-build-vms(8)` will never open up a graphical
display.
---
 nixos/lib/test-driver/test-driver.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/nixos/lib/test-driver/test-driver.py b/nixos/lib/test-driver/test-driver.py
index fdc440a896a..7cea1d9ce69 100755
--- a/nixos/lib/test-driver/test-driver.py
+++ b/nixos/lib/test-driver/test-driver.py
@@ -246,7 +246,7 @@ class StartCommand:
     ) -> str:
         display_opts = ""
         display_available = any(x in os.environ for x in ["DISPLAY", "WAYLAND_DISPLAY"])
-        if display_available:
+        if not display_available:
             display_opts += " -nographic"
 
         # qemu options

From af859d1df18517a77440b1e117f670dc7c9f68f6 Mon Sep 17 00:00:00 2001
From: Antoine Eiche <lewo@abesis.fr>
Date: Tue, 21 Sep 2021 18:15:19 +0200
Subject: [PATCH 07/23] nixos.tests.usbguard: state_dir is now of type
 pathlib.Path

---
 nixos/tests/usbguard.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/nixos/tests/usbguard.nix b/nixos/tests/usbguard.nix
index cba905db44f..bb707bdbf70 100644
--- a/nixos/tests/usbguard.nix
+++ b/nixos/tests/usbguard.nix
@@ -22,7 +22,7 @@ import ./make-test-python.nix ({ pkgs, ... }: {
 
   testScript = ''
     # create a blank disk image for our fake USB stick
-    with open(machine.state_dir + "/usbstick.img", "wb") as stick:
+    with open(machine.state_dir / "usbstick.img", "wb") as stick:
         stick.write(b"\x00" * (1024 * 1024))
 
     # wait for machine to have started and the usbguard service to be up

From 32face8dea96371239cb4d48b34086940e43c3c6 Mon Sep 17 00:00:00 2001
From: Antoine Eiche <lewo@abesis.fr>
Date: Tue, 14 Sep 2021 23:17:15 +0200
Subject: [PATCH 08/23] nixos.tests.udisks2: state_dir is now of type
 pathlib.Path

---
 nixos/tests/udisks2.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/nixos/tests/udisks2.nix b/nixos/tests/udisks2.nix
index 1f01cc6de4d..6c4b71aaa2e 100644
--- a/nixos/tests/udisks2.nix
+++ b/nixos/tests/udisks2.nix
@@ -34,7 +34,7 @@ in
 
       with lzma.open(
           "${stick}"
-      ) as data, open(machine.state_dir + "/usbstick.img", "wb") as stick:
+      ) as data, open(machine.state_dir / "usbstick.img", "wb") as stick:
           stick.write(data.read())
 
       machine.succeed("udisksctl info -b /dev/vda >&2")

From 5c666cdf62615442cab413121384588c6ecebef5 Mon Sep 17 00:00:00 2001
From: Jacek Galowicz <jacek.galowicz@cyberus-technology.de>
Date: Mon, 27 Sep 2021 15:00:58 +0200
Subject: [PATCH 09/23] Re-RAII-ify the NixOS integration test driver's VLAN
 class.

We have no usecase for manually/selectively starting or stopping VLANs
in integration tests.
By starting and stopping the VLANs with the constructor and destructor
of VLAN objects, we remove the obligation and complexity to maintain
network lifetime separately.
---
 nixos/lib/test-driver/test-driver.py | 24 +++++++-----------------
 1 file changed, 7 insertions(+), 17 deletions(-)

diff --git a/nixos/lib/test-driver/test-driver.py b/nixos/lib/test-driver/test-driver.py
index 7cea1d9ce69..56c510c4e69 100755
--- a/nixos/lib/test-driver/test-driver.py
+++ b/nixos/lib/test-driver/test-driver.py
@@ -1027,16 +1027,16 @@ class Machine:
 
 
 class VLan:
-    """A handle to the vlan with this number, that also knows how to manage
-    it's lifecycle.
+    """This class handles a VLAN that the run-vm scripts identify via its
+    number handles. The network's lifetime equals the object's lifetime.
     """
 
     nr: int
     socket_dir: pathlib.Path
 
-    process: Optional[subprocess.Popen]
-    pid: Optional[int]
-    fd: Optional[io.TextIOBase]
+    process: subprocess.Popen
+    pid: int
+    fd: io.TextIOBase
 
     def __repr__(self) -> str:
         return f"<Vlan Nr. {self.nr}>"
@@ -1048,8 +1048,6 @@ class VLan:
         # TODO: don't side-effect environment here
         os.environ[f"QEMU_VDE_SOCKET_{self.nr}"] = str(self.socket_dir)
 
-    def start(self) -> None:
-
         rootlog.info("start vlan")
         pty_master, pty_slave = pty.openpty()
 
@@ -1073,12 +1071,8 @@ class VLan:
 
         rootlog.info(f"running vlan (pid {self.pid})")
 
-    def release(self) -> None:
-        if self.pid is None:
-            return
+    def __del__(self) -> None:
         rootlog.info(f"kill vlan (pid {self.pid})")
-        assert self.fd
-        assert self.process
         self.fd.close()
         self.process.terminate()
 
@@ -1103,10 +1097,8 @@ class Driver:
         tmp_dir = pathlib.Path(os.environ.get("TMPDIR", tempfile.gettempdir()))
         tmp_dir.mkdir(mode=0o700, exist_ok=True)
 
-        self.vlans = [VLan(nr, tmp_dir) for nr in vlans]
         with rootlog.nested("start all VLans"):
-            for vlan in self.vlans:
-                vlan.start()
+            self.vlans = [VLan(nr, tmp_dir) for nr in vlans]
 
         def cmd(scripts: List[str]) -> Iterator[NixStartScript]:
             for s in scripts:
@@ -1127,8 +1119,6 @@ class Driver:
             with rootlog.nested("clean up"):
                 for machine in self.machines:
                     machine.release()
-                for vlan in self.vlans:
-                    vlan.release()
 
     def subtest(self, name: str) -> Iterator[None]:
         """Group logs under a given test name"""

From 986159b82391e8744ee8925c4287d17341febb5d Mon Sep 17 00:00:00 2001
From: 0x4A6F <0x4A6F@users.noreply.github.com>
Date: Fri, 1 Oct 2021 23:49:43 +0200
Subject: [PATCH 10/23] libcdada: init at 0.3.5

---
 .../libraries/libcdada/default.nix            | 37 +++++++++++++++++++
 pkgs/top-level/all-packages.nix               |  2 +
 2 files changed, 39 insertions(+)
 create mode 100644 pkgs/development/libraries/libcdada/default.nix

diff --git a/pkgs/development/libraries/libcdada/default.nix b/pkgs/development/libraries/libcdada/default.nix
new file mode 100644
index 00000000000..94976c1be46
--- /dev/null
+++ b/pkgs/development/libraries/libcdada/default.nix
@@ -0,0 +1,37 @@
+{ lib
+, stdenv
+, fetchFromGitHub
+, autoreconfHook
+}:
+
+stdenv.mkDerivation rec {
+  pname = "libcdada";
+  version = "0.3.5";
+
+  src = fetchFromGitHub {
+    owner = "msune";
+    repo = "libcdada";
+    rev = "v${version}";
+    sha256 = "0vcsf3s4fbw2w33jjc8b509kc0xb6ld58l8wfxgqwjqx5icfg1ps";
+  };
+
+  nativeBuildInputs = [
+    autoreconfHook
+  ];
+
+  configureFlags = [
+    "--without-tests"
+    "--without-examples"
+  ];
+
+  meta = with lib; {
+    description = "Library for basic data structures in C";
+    longDescription = ''
+      Basic data structures in C: list, set, map/hashtable, queue... (libstdc++ wrapper)
+    '';
+    homepage = "https://github.com/msune/libcdada";
+    license = licenses.bsd2;
+    maintainers = with maintainers; [ _0x4A6F ];
+    platforms = platforms.unix;
+  };
+}
diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix
index c61bdbb4629..6234075b867 100644
--- a/pkgs/top-level/all-packages.nix
+++ b/pkgs/top-level/all-packages.nix
@@ -16852,6 +16852,8 @@ with pkgs;
 
   libcerf = callPackage ../development/libraries/libcerf {};
 
+  libcdada = callPackage ../development/libraries/libcdada { };
+
   libcdaudio = callPackage ../development/libraries/libcdaudio { };
 
   libcddb = callPackage ../development/libraries/libcddb { };

From fa3378e9c42cde077e2e9572df86d6a8de42e943 Mon Sep 17 00:00:00 2001
From: 0x4A6F <0x4A6F@users.noreply.github.com>
Date: Sat, 13 Feb 2021 14:17:33 +0000
Subject: [PATCH 11/23] pmacct: 1.7.5 -> 1.7.6

---
 pkgs/tools/networking/pmacct/default.nix | 67 ++++++++++++------------
 1 file changed, 34 insertions(+), 33 deletions(-)

diff --git a/pkgs/tools/networking/pmacct/default.nix b/pkgs/tools/networking/pmacct/default.nix
index 9cb8c0b88f7..86f1af029df 100644
--- a/pkgs/tools/networking/pmacct/default.nix
+++ b/pkgs/tools/networking/pmacct/default.nix
@@ -1,55 +1,56 @@
-{ lib, stdenv
+{ lib
+, stdenv
 , fetchFromGitHub
 , pkg-config
 , autoreconfHook
 , libtool
 , libpcap
-
+, libcdada
 # Optional Dependencies
-, zlib ? null
-, withJansson ? true, jansson ? null
-, withNflog ? true, libnetfilter_log ? null
-, withSQLite ? true, sqlite ? null
-, withPgSQL ? true, postgresql ? null
-, withMysql ? true, libmysqlclient ? null }:
-
-assert withJansson -> jansson != null;
-assert withNflog -> libnetfilter_log != null;
-assert withSQLite -> sqlite != null;
-assert withPgSQL -> postgresql != null;
-assert withMysql -> libmysqlclient != null;
-
-let inherit (lib) getDev optional optionalString; in
+, withJansson ? true, jansson
+, withNflog ? true, libnetfilter_log
+, withSQLite ? true, sqlite
+, withPgSQL ? true, postgresql
+, withMysql ? true, libmysqlclient, zlib
+, gnutlsSupport ? false, gnutls
+}:
 
 stdenv.mkDerivation rec {
-  version = "1.7.5";
+  version = "1.7.6";
   pname = "pmacct";
 
   src = fetchFromGitHub {
     owner = "pmacct";
-    repo = pname;
+    repo = "pmacct";
     rev = "v${version}";
-    sha256 = "17p5isrq5w58hvmzhc6akbd37ins3c95g0rvhhdm0v33khzxmran";
+    sha256 = "0x1i75hwz44siqvn4i58jgji0zwrqgn6ayv89s9m9nh3b423nsiv";
   };
 
-  nativeBuildInputs = [ autoreconfHook pkg-config libtool ];
-  buildInputs = [ libpcap ]
-    ++ optional withJansson jansson
-    ++ optional withNflog libnetfilter_log
-    ++ optional withSQLite sqlite
-    ++ optional withPgSQL postgresql
-    ++ optional withMysql [ libmysqlclient zlib ];
+  nativeBuildInputs = [
+    autoreconfHook
+    pkg-config
+    libtool
+  ];
+  buildInputs = [
+    libcdada
+    libpcap
+  ] ++ lib.optional withJansson jansson
+  ++ lib.optional withNflog libnetfilter_log
+  ++ lib.optional withSQLite sqlite
+  ++ lib.optional withPgSQL postgresql
+  ++ lib.optionals withMysql [ libmysqlclient zlib ]
+  ++ lib.optional gnutlsSupport gnutls;
 
-  MYSQL_CONFIG =
-    optionalString withMysql "${getDev libmysqlclient}/bin/mysql_config";
+  MYSQL_CONFIG = lib.optionalString withMysql "${lib.getDev libmysqlclient}/bin/mysql_config";
 
   configureFlags = [
     "--with-pcap-includes=${libpcap}/include"
-  ] ++ optional withJansson "--enable-jansson"
-    ++ optional withNflog "--enable-nflog"
-    ++ optional withSQLite "--enable-sqlite3"
-    ++ optional withPgSQL "--enable-pgsql"
-    ++ optional withMysql "--enable-mysql";
+  ] ++ lib.optional withJansson "--enable-jansson"
+  ++ lib.optional withNflog "--enable-nflog"
+  ++ lib.optional withSQLite "--enable-sqlite3"
+  ++ lib.optional withPgSQL "--enable-pgsql"
+  ++ lib.optional withMysql "--enable-mysql"
+  ++ lib.optional gnutlsSupport "--enable-gnutls";
 
   meta = with lib; {
     description = "A small set of multi-purpose passive network monitoring tools";

From 93eb7786c9365981854a98e508d5b1cae5ee2fcf Mon Sep 17 00:00:00 2001
From: "Neubauer, Sebastian" <Sebastian.Neubauer@amd.com>
Date: Wed, 6 Oct 2021 14:39:58 +0200
Subject: [PATCH 12/23] arcanist: Update certs to fix letsencrypt

Due to the old root certificate used by letsencrypt expiring, arcanist
could not connect anymore to servers using letsencrypt (like
reviews.llvm.org).

Fix it by using the default nix certificates.
---
 pkgs/development/tools/misc/arcanist/default.nix | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/pkgs/development/tools/misc/arcanist/default.nix b/pkgs/development/tools/misc/arcanist/default.nix
index 94e230e6a12..a2af61fb6a1 100644
--- a/pkgs/development/tools/misc/arcanist/default.nix
+++ b/pkgs/development/tools/misc/arcanist/default.nix
@@ -1,4 +1,5 @@
 { bison
+, cacert
 , fetchFromGitHub
 , flex
 , php
@@ -53,6 +54,7 @@ stdenv.mkDerivation {
     make install  -C support/xhpast $makeFlags "''${makeFlagsArray[@]}" -j $NIX_BUILD_CORES
     make cleanall -C support/xhpast $makeFlags "''${makeFlagsArray[@]}" -j $NIX_BUILD_CORES
     cp -R . $out/libexec/arcanist
+    ln -sf ${cacert}/etc/ssl/certs/ca-bundle.crt $out/libexec/arcanist/resources/ssl/default.pem
 
     ${makeArcWrapper "arc"}
     ${makeArcWrapper "phage"}

From eb70021803f16d1cf8c4d20aa565d660ee1580b5 Mon Sep 17 00:00:00 2001
From: figsoda <figsoda@pm.me>
Date: Wed, 6 Oct 2021 16:11:32 -0400
Subject: [PATCH 13/23] eureka-ideas: init at 1.8.1

---
 .../misc/eureka-ideas/default.nix             | 35 +++++++++++++++++++
 pkgs/top-level/all-packages.nix               |  4 +++
 2 files changed, 39 insertions(+)
 create mode 100644 pkgs/applications/misc/eureka-ideas/default.nix

diff --git a/pkgs/applications/misc/eureka-ideas/default.nix b/pkgs/applications/misc/eureka-ideas/default.nix
new file mode 100644
index 00000000000..5f5db97b332
--- /dev/null
+++ b/pkgs/applications/misc/eureka-ideas/default.nix
@@ -0,0 +1,35 @@
+{ lib
+, rustPlatform
+, fetchFromGitHub
+, pkg-config
+, openssl
+, stdenv
+, Security
+}:
+
+rustPlatform.buildRustPackage rec {
+  pname = "eureka-ideas";
+  version = "1.8.1";
+
+  src = fetchFromGitHub {
+    owner = "simeg";
+    repo = "eureka";
+    rev = "v${version}";
+    sha256 = "1qjf8nr7m9igy6h228gm9gnav6pi2rfarbd9bc5fchx4rqy59sp7";
+  };
+
+  cargoSha256 = "sha256-QujrFgliH8Mx1ES9KVl+O9UJP+7GDanQ7+z4QJuSOd0=";
+
+  nativeBuildInputs = [ pkg-config ];
+
+  buildInputs = [ openssl ] ++ lib.optionals stdenv.isDarwin [ Security ];
+
+  meta = with lib; {
+    description = "CLI tool to input and store your ideas without leaving the terminal";
+    homepage = "https://github.com/simeg/eureka";
+    changelog = "https://github.com/simeg/eureka/blob/v${version}/CHANGELOG.md";
+    license = licenses.mit;
+    maintainers = with maintainers; [ figsoda ];
+    mainProgram = "eureka";
+  };
+}
diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix
index 9f47f9875b4..d897c730c4a 100644
--- a/pkgs/top-level/all-packages.nix
+++ b/pkgs/top-level/all-packages.nix
@@ -29479,6 +29479,10 @@ with pkgs;
 
   eureka-editor = callPackage ../applications/misc/eureka-editor { };
 
+  eureka-ideas = callPackage ../applications/misc/eureka-ideas {
+    inherit (darwin.apple_sdk.frameworks) Security;
+  };
+
   extremetuxracer = callPackage ../games/extremetuxracer {
     libpng = libpng12;
   };

From fb559de7a7a6b54bd901b4f39e79c3c82d7fe5f0 Mon Sep 17 00:00:00 2001
From: Lin Yinfeng <lin.yinfeng@outlook.com>
Date: Thu, 7 Oct 2021 11:01:00 +0800
Subject: [PATCH 14/23] maintainers: add yinfeng

---
 maintainers/maintainer-list.nix | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/maintainers/maintainer-list.nix b/maintainers/maintainer-list.nix
index e34f9e823e1..2e4da00480e 100644
--- a/maintainers/maintainer-list.nix
+++ b/maintainers/maintainer-list.nix
@@ -12245,6 +12245,12 @@
     githubId = 4113027;
     name = "Jesper Geertsen Jonsson";
   };
+  yinfeng = {
+    email = "lin.yinfeng@outlook.com";
+    github = "linyinfeng";
+    githubId = 11229748;
+    name = "Lin Yinfeng";
+  };
   ylwghst = {
     email = "ylwghst@onionmail.info";
     github = "ylwghst";

From 7aa5a5eb8f52567efa0e91c7b0f7368327762a02 Mon Sep 17 00:00:00 2001
From: Harry Maclean <harry@hmac.dev>
Date: Fri, 3 Sep 2021 13:22:02 +0100
Subject: [PATCH 15/23] codeql: trim nativeBuildInputs

These dependencies are not needed for recent CodeQL versions. Removing
them also makes this package buildable on darwin.
---
 .../development/tools/analysis/codeql/default.nix | 15 +--------------
 1 file changed, 1 insertion(+), 14 deletions(-)

diff --git a/pkgs/development/tools/analysis/codeql/default.nix b/pkgs/development/tools/analysis/codeql/default.nix
index a4c51121cd5..66cd8a7217c 100644
--- a/pkgs/development/tools/analysis/codeql/default.nix
+++ b/pkgs/development/tools/analysis/codeql/default.nix
@@ -1,14 +1,4 @@
-{ lib, stdenv
-, fetchzip
-, zlib
-, xorg
-, freetype
-, alsa-lib
-, jdk11
-, curl
-, lttng-ust
-, autoPatchelfHook
-}:
+{ lib, stdenv, fetchzip, zlib, xorg, freetype, jdk11, curl, autoPatchelfHook }:
 
 stdenv.mkDerivation rec {
   pname = "codeql";
@@ -31,12 +21,9 @@ stdenv.mkDerivation rec {
     xorg.libXtst
     xorg.libXrender
     freetype
-    alsa-lib
     jdk11
     stdenv.cc.cc.lib
     curl
-    lttng-ust
-    autoPatchelfHook
   ];
 
   installPhase = ''

From f9ae94032f8b73191b06201e1d7ed3bcd606b7a7 Mon Sep 17 00:00:00 2001
From: Harry Maclean <harry@hmac.dev>
Date: Fri, 3 Sep 2021 13:22:39 +0100
Subject: [PATCH 16/23] codeql: update homepage

---
 pkgs/development/tools/analysis/codeql/default.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkgs/development/tools/analysis/codeql/default.nix b/pkgs/development/tools/analysis/codeql/default.nix
index 66cd8a7217c..9849616f042 100644
--- a/pkgs/development/tools/analysis/codeql/default.nix
+++ b/pkgs/development/tools/analysis/codeql/default.nix
@@ -41,7 +41,7 @@ stdenv.mkDerivation rec {
 
   meta = with lib; {
     description = "Semantic code analysis engine";
-    homepage = "https://semmle.com/codeql";
+    homepage = "https://codeql.github.com";
     maintainers = [ maintainers.dump_stack ];
     license = licenses.unfree;
   };

From 1d369f09634308b9a5a4603a88ea52014b537767 Mon Sep 17 00:00:00 2001
From: Harry Maclean <harry@hmac.dev>
Date: Fri, 3 Sep 2021 13:22:26 +0100
Subject: [PATCH 17/23] codeql: 2.5.9 -> 2.6.2

This requires a minor change to accommodate a different path to Java in
the codeql script.
---
 pkgs/development/tools/analysis/codeql/default.nix | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/pkgs/development/tools/analysis/codeql/default.nix b/pkgs/development/tools/analysis/codeql/default.nix
index 9849616f042..52169ccab11 100644
--- a/pkgs/development/tools/analysis/codeql/default.nix
+++ b/pkgs/development/tools/analysis/codeql/default.nix
@@ -2,7 +2,7 @@
 
 stdenv.mkDerivation rec {
   pname = "codeql";
-  version = "2.5.9";
+  version = "2.6.2";
 
   dontConfigure = true;
   dontBuild = true;
@@ -10,7 +10,7 @@ stdenv.mkDerivation rec {
 
   src = fetchzip {
     url = "https://github.com/github/codeql-cli-binaries/releases/download/v${version}/codeql.zip";
-    sha256 = "sha256-r3Jm+VYjn0Dz4BCSbADbgTWL1owbyIXlkoj6mOmZcZk=";
+    sha256 = "096w9w52rj854i7rmpgy99k9z9ja2dfvj2d02dnpagwd7pc6a6bl";
   };
 
   nativeBuildInputs = [
@@ -34,7 +34,7 @@ stdenv.mkDerivation rec {
 
     ln -sf $out/codeql/tools/linux64/lib64trace.so $out/codeql/tools/linux64/libtrace.so
 
-    sed -i 's;"$CODEQL_DIST/tools/$CODEQL_PLATFORM/java/bin/java";"${jdk11}/bin/java";' $out/codeql/codeql
+    sed -i 's%\$CODEQL_DIST/tools/\$CODEQL_PLATFORM/java%\${jdk11}%g' $out/codeql/codeql
 
     ln -s $out/codeql/codeql $out/bin/
   '';

From cdaf4b1290088c1aec6d8508c7c3020a0d5ef3f3 Mon Sep 17 00:00:00 2001
From: Julius de Bruijn <julius+github@nauk.io>
Date: Thu, 7 Oct 2021 14:22:16 +0200
Subject: [PATCH 18/23] prisma-engines: add support for aarch64-linux

---
 pkgs/development/tools/database/prisma-engines/default.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkgs/development/tools/database/prisma-engines/default.nix b/pkgs/development/tools/database/prisma-engines/default.nix
index eeba3f49fd1..c8fe44c5d04 100644
--- a/pkgs/development/tools/database/prisma-engines/default.nix
+++ b/pkgs/development/tools/database/prisma-engines/default.nix
@@ -52,7 +52,7 @@ in rustPlatform.buildRustPackage rec {
     description = "A collection of engines that power the core stack for Prisma";
     homepage = "https://www.prisma.io/";
     license = licenses.asl20;
-    platforms = [ "x86_64-linux" ];
+    platforms = [ "x86_64-linux" "aarch64-linux" ];
     maintainers = with maintainers; [ pamplemousse pimeys ];
   };
 }

From bd47be3c865302c04d20263bdda47e56879e099c Mon Sep 17 00:00:00 2001
From: Lin Yinfeng <lin.yinfeng@outlook.com>
Date: Thu, 7 Oct 2021 11:05:05 +0800
Subject: [PATCH 19/23] godns: init at 2.5

---
 pkgs/tools/networking/godns/default.nix | 27 +++++++++++++++++++++++++
 pkgs/top-level/all-packages.nix         |  2 ++
 2 files changed, 29 insertions(+)
 create mode 100644 pkgs/tools/networking/godns/default.nix

diff --git a/pkgs/tools/networking/godns/default.nix b/pkgs/tools/networking/godns/default.nix
new file mode 100644
index 00000000000..fb46a144ee7
--- /dev/null
+++ b/pkgs/tools/networking/godns/default.nix
@@ -0,0 +1,27 @@
+{ buildGoModule, fetchFromGitHub, lib }:
+
+buildGoModule rec {
+  pname = "godns";
+  version = "2.5";
+
+  src = fetchFromGitHub {
+    owner = "TimothyYe";
+    repo = "godns";
+    rev = "v${version}";
+    sha256 = "sha256-ia0FmV2KlFPh9gmKOqVxiStgmBbX9vUIc7KllpUt44Q=";
+  };
+
+  vendorSha256 = "sha256-FZLDaMrPEyoTGFmGBlpqPWsMuobqwkBaot5qjcRJe9w=";
+
+  # Some tests require internet access, broken in sandbox
+  doCheck = false;
+
+  ldflags = [ "-X main.Version=${version}" ];
+
+  meta = with lib; {
+    description = "A dynamic DNS client tool supports AliDNS, Cloudflare, Google Domains, DNSPod, HE.net & DuckDNS & DreamHost, etc";
+    homepage = "https://github.com/TimothyYe/godns";
+    license = licenses.asl20;
+    maintainers = with maintainers; [ yinfeng ];
+  };
+}
diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix
index 8c9fb71ab80..faa04f00cc1 100644
--- a/pkgs/top-level/all-packages.nix
+++ b/pkgs/top-level/all-packages.nix
@@ -933,6 +933,8 @@ with pkgs;
 
   gofu = callPackage ../applications/misc/gofu { };
 
+  godns = callPackage ../tools/networking/godns { };
+
   ksnip = libsForQt5.callPackage ../tools/misc/ksnip { };
 
   linux-router = callPackage ../tools/networking/linux-router { };

From 40fd8c9c24980dbb38769b2544ea318ab175761c Mon Sep 17 00:00:00 2001
From: "R. RyanTM" <ryantm-bot@ryantm.com>
Date: Thu, 7 Oct 2021 02:53:26 +0000
Subject: [PATCH 20/23] oh-my-zsh: 2021-10-05 -> 2021-10-06

---
 pkgs/shells/zsh/oh-my-zsh/default.nix | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/pkgs/shells/zsh/oh-my-zsh/default.nix b/pkgs/shells/zsh/oh-my-zsh/default.nix
index 1566974badd..41ff2b43542 100644
--- a/pkgs/shells/zsh/oh-my-zsh/default.nix
+++ b/pkgs/shells/zsh/oh-my-zsh/default.nix
@@ -5,15 +5,15 @@
 , git, nix, nixfmt, jq, coreutils, gnused, curl, cacert }:
 
 stdenv.mkDerivation rec {
-  version = "2021-10-05";
+  version = "2021-10-06";
   pname = "oh-my-zsh";
-  rev = "e5b9b80008a2fd71b441ef39fe620ed47dad82e5";
+  rev = "29b5c182bec4cec7704fb8bac9ee0ab971dfb89a";
 
   src = fetchFromGitHub {
     inherit rev;
     owner = "ohmyzsh";
     repo = "ohmyzsh";
-    sha256 = "09oTsUYLLZAoUwM63gAVYLWFvp0aKTM9K79alTISEJw=";
+    sha256 = "pPBeZj/QTQCIuBtE7+4CmuXacblU4RGXty+cdeQw+54=";
   };
 
   installPhase = ''

From 6bfc5e94f60751aaf5f83e3ffad2cbf51afc9a98 Mon Sep 17 00:00:00 2001
From: sternenseemann <sternenseemann@systemli.org>
Date: Wed, 6 Oct 2021 11:30:12 +0200
Subject: [PATCH 21/23] foot: 1.9.0 -> 1.9.2

foot's upstream has abandoned the approach of using a custom terminfo
install location and setting TERMINFO, but still supports it. For us
this has eliminated the need to build the foot terminfo files manually,
somehow which is appreciated.

We preserve our current approach: Install terminfo files to the terminfo
output and set TERMINFO to that store path instead of propagating the
output for user env insatllations.

Change logs:

* https://codeberg.org/dnkl/foot/releases/tag/1.9.1

* https://codeberg.org/dnkl/foot/releases/tag/1.9.2
---
 .../terminal-emulators/foot/default.nix       | 22 ++++++-------------
 1 file changed, 7 insertions(+), 15 deletions(-)

diff --git a/pkgs/applications/terminal-emulators/foot/default.nix b/pkgs/applications/terminal-emulators/foot/default.nix
index ec75454514c..f9fea74c113 100644
--- a/pkgs/applications/terminal-emulators/foot/default.nix
+++ b/pkgs/applications/terminal-emulators/foot/default.nix
@@ -27,7 +27,7 @@
 }:
 
 let
-  version = "1.9.0";
+  version = "1.9.2";
 
   # build stimuli file for PGO build and the script to generate it
   # independently of the foot's build, so we can cache the result
@@ -100,7 +100,7 @@ stdenv.mkDerivation rec {
     owner = "dnkl";
     repo = pname;
     rev = version;
-    sha256 = "0mkzq5lbgl5qp5nj8sk5gyg9hrrklmbjdqzlcr2a6rlmilkxlhwm";
+    sha256 = "15h01ijx87i60bdgjjap1ymwlxggsxc6iziykh3bahj8432s1836";
   };
 
   depsBuildBuild = [
@@ -144,16 +144,15 @@ stdenv.mkDerivation rec {
 
   mesonBuildType = "release";
 
+  # See https://codeberg.org/dnkl/foot/src/tag/1.9.2/INSTALL.md#options
   mesonFlags = [
+    # Use lto
     "-Db_lto=true"
-    # Prevent foot from installing its terminfo file into a custom location,
-    # we need to do this manually in postInstall.
-    # See https://codeberg.org/dnkl/foot/pulls/673,
-    # https://codeberg.org/dnkl/foot/src/tag/1.9.0/INSTALL.md#options
-    "-Dterminfo=disabled"
+    # “Build” and install terminfo db
+    "-Dterminfo=enabled"
     # Ensure TERM=foot is used
     "-Ddefault-terminfo=foot"
-    # Tell foot what to set TERMINFO to
+    # Tell foot to set TERMINFO and where to install the terminfo files
     "-Dcustom-terminfo-install-location=${terminfoDir}"
   ];
 
@@ -174,13 +173,6 @@ stdenv.mkDerivation rec {
 
   outputs = [ "out" "terminfo" ];
 
-  postInstall = ''
-    # build and install foot's terminfo to the standard location
-    # instead of its custom location
-    mkdir -p "${terminfoDir}"
-    tic -o "${terminfoDir}" -x -e foot,foot-direct "$NIX_BUILD_TOP/$sourceRoot/foot.info"
-  '';
-
   passthru.tests = {
     clang-default-compilation = foot.override {
       inherit (llvmPackages) stdenv;

From 9a76986ee0d46ea8074c9a8992df531dd90ac261 Mon Sep 17 00:00:00 2001
From: sternenseemann <sternenseemann@systemli.org>
Date: Wed, 6 Oct 2021 11:42:03 +0200
Subject: [PATCH 22/23] foot: cache stimuliFile more aggressively

stimuliFile would get rebuild with every change in version, since the
stimulusGenerator derivation would depend on the version string. Since
the script often doesn't change between releases, this causes
unnecessary rebuilds and cache duplication.

Additionally we add a test to passthru which checks if the hash of the
script is still up to date: By making its name depend on the version
string, it'll get rebuild on every version change, even if the hash
stays the same. This way we can detect new hash mismatches.
---
 pkgs/applications/terminal-emulators/foot/default.nix | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/pkgs/applications/terminal-emulators/foot/default.nix b/pkgs/applications/terminal-emulators/foot/default.nix
index f9fea74c113..837b2b45a39 100644
--- a/pkgs/applications/terminal-emulators/foot/default.nix
+++ b/pkgs/applications/terminal-emulators/foot/default.nix
@@ -36,8 +36,7 @@ let
   #
   # For every bump, make sure that the hash is still accurate.
   stimulusGenerator = stdenv.mkDerivation {
-    pname = "foot-generate-alt-random-writes";
-    inherit version;
+    name = "foot-generate-alt-random-writes";
 
     src = fetchurl {
       url = "https://codeberg.org/dnkl/foot/raw/tag/${version}/scripts/generate-alt-random-writes.py";
@@ -185,6 +184,13 @@ stdenv.mkDerivation rec {
     noPgo = foot.override {
       allowPgo = false;
     };
+
+    # By changing name, this will get rebuilt everytime we change version,
+    # even if the hash stays the same. Consequently it'll fail if we introduce
+    # a hash mismatch when updating.
+    stimulus-script-is-current = stimulusGenerator.src.overrideAttrs (_: {
+      name = "generate-alt-random-writes-${version}.py";
+    });
   };
 
   meta = with lib; {

From a47875938d82d8d6b0d6aae3d709bc29999c4873 Mon Sep 17 00:00:00 2001
From: Eelco Dolstra <edolstra@gmail.com>
Date: Thu, 7 Oct 2021 17:31:27 +0200
Subject: [PATCH 23/23] nix-fallback-paths.nix: Update to 2.3.16

---
 nixos/modules/installer/tools/nix-fallback-paths.nix | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/nixos/modules/installer/tools/nix-fallback-paths.nix b/nixos/modules/installer/tools/nix-fallback-paths.nix
index 15c76287e34..cb509b7340b 100644
--- a/nixos/modules/installer/tools/nix-fallback-paths.nix
+++ b/nixos/modules/installer/tools/nix-fallback-paths.nix
@@ -1,7 +1,7 @@
 {
-  x86_64-linux = "/nix/store/jhbxh1jwjc3hjhzs9y2hifdn0rmnfwaj-nix-2.3.15";
-  i686-linux = "/nix/store/9pspwnkdrgzma1l4xlv7arhwa56y16di-nix-2.3.15";
-  aarch64-linux = "/nix/store/72aqi5g7f4fhgvgafbcqwcpqjgnczj48-nix-2.3.15";
-  x86_64-darwin = "/nix/store/6p6qwp73dgfkqhynmxrzbx1lcfgfpqal-nix-2.3.15";
-  aarch64-darwin = "/nix/store/dmq2vksdhssgfl822shd0ky3x5x0klh4-nix-2.3.15";
+  x86_64-linux = "/nix/store/nzp4m3cmm7wawk031byh8jg4cdzjq212-nix-2.3.16";
+  i686-linux = "/nix/store/zsaza9pwim617ak15fsc31lv65b9w3in-nix-2.3.16";
+  aarch64-linux = "/nix/store/7f6z40gyd405yd50qkyzwilnqw106bx8-nix-2.3.16";
+  x86_64-darwin = "/nix/store/c43kyri67ia8mibs0id5ara7gqwlkybf-nix-2.3.16";
+  aarch64-darwin = "/nix/store/6jwhak3cvsgnbqs540n27g8pxnk427fr-nix-2.3.16";
 }