nixos/qemu-vm: Use disposable EROFS for store when writableStore = false

This avoids putting a large disk image in the store (and possibly
in a binary cache), while improving runtime performance.

Assuming you're running an SSD, and/or with plenty of cache (?)
it is feasible to preempt the virtualization overhead before
VM start, in single-digit seconds.

For some tests that perform many reads on the store, the improved
performance of EROFS is sufficient that not only the image creation
overhead is compensated for, but is actually faster.

Stats for nixosTests.gitlab:

Baseline without useNixStoreImage: >1000s

Baseline with useNixStoreImage without writableStore = false
ext4 image in store: 277 seconds
  + significant image build time and/or disk space

Disposable erofs image: 249 seconds _including_ image build time

Custom erofs overlay on 9p host store: 391 seconds; presumably
because the overlay still performs too many 9p accesses, or perhaps
some other overhead. This solution had no obvious performance
advantage, while requiring extra options to work, so it was
discarded.

nixos/modules/virtualisation/includes-to-excludes.py

@@ -0,0 +1,86 @@
+
+# Convert a list of strings to a regex that matches everything but those strings
+# ... and it had to be a POSIX regex; no negative lookahead :(
+# This is a workaround for erofs supporting only exclude regex, not an include list
+
+import sys
+import re
+from collections import defaultdict
+
+# We can configure this script to match in different ways if we need to.
+# The regex got too long for the argument list, so we had to truncate the
+# hashes and use MATCH_STRING_PREFIX. That's less accurate, and might pick up some
+# garbage like .lock files, but only if the sandbox doesn't hide those. Even
+# then it should be harmless.
+
+# Produce the negation of ^a$
+MATCH_EXACTLY = ".+"
+# Produce the negation of ^a
+MATCH_STRING_PREFIX = "//X" # //X should be epsilon regex instead. Not supported??
+# Produce the negation of ^a/?
+MATCH_SUBPATHS = "[^/].*$"
+
+# match_end = MATCH_SUBPATHS
+match_end = MATCH_STRING_PREFIX
+# match_end = MATCH_EXACTLY
+
+def chars_to_inverted_class(letters):
+    assert len(letters) > 0
+    letters = list(letters)
+
+    s = "[^"
+
+    if "]" in letters:
+        s += "]"
+        letters.remove("]")
+
+    final = ""
+    if "-" in letters:
+        final = "-"
+        letters.remove("-")
+
+    s += "".join(letters)
+
+    s += final
+
+    s += "]"
+
+    return s
+
+# There's probably at least one bug in here, but it seems to works well enough
+# for filtering store paths.
+def strings_to_inverted_regex(strings):
+    s = "("
+
+    # Match anything that starts with the wrong character
+
+    chars = defaultdict(list)
+
+    for item in strings:
+        if item != "":
+            chars[item[0]].append(item[1:])
+
+    if len(chars) == 0:
+        s += match_end
+    else:
+        s += chars_to_inverted_class(chars)
+
+    # Now match anything that starts with the right char, but then goes wrong
+
+    for char, sub in chars.items():
+        s += "|(" + re.escape(char) + strings_to_inverted_regex(sub) + ")"
+
+    s += ")"
+    return s
+
+if __name__ == "__main__":
+    stdin_lines = []
+    for line in sys.stdin:
+        if line.strip() != "":
+            stdin_lines.append(line.strip())
+
+    print("^" + strings_to_inverted_regex(stdin_lines))
+
+# Test:
+# (echo foo; echo fo/; echo foo/; echo foo/ba/r; echo b; echo az; echo az/; echo az/a; echo ab; echo ab/a; echo ab/; echo abc; echo abcde; echo abb; echo ac; echo b) | grep -vE "$((echo ab; echo az; echo foo;) | python includes-to-excludes.py | tee /dev/stderr )"
+# should print ab, az, foo and their subpaths

nixos/modules/virtualisation/qemu-vm.nix

@@ -122,11 +122,32 @@ let
           TMPDIR=$(mktemp -d nix-vm.XXXXXXXXXX --tmpdir)
       fi
 
-      ${lib.optionalString cfg.useNixStoreImage
-      ''
-        # Create a writable copy/snapshot of the store image.
-        ${qemu}/bin/qemu-img create -f qcow2 -F qcow2 -b ${storeImage}/nixos.qcow2 "$TMPDIR"/store.img
-      ''}
+      ${lib.optionalString (cfg.useNixStoreImage)
+        (if cfg.writableStore
+          then ''
+            # Create a writable copy/snapshot of the store image.
+            ${qemu}/bin/qemu-img create -f qcow2 -F qcow2 -b ${storeImage}/nixos.qcow2 "$TMPDIR"/store.img
+          ''
+          else ''
+            (
+              cd ${builtins.storeDir}
+              ${pkgs.erofs-utils}/bin/mkfs.erofs \
+                --force-uid=0 \
+                --force-gid=0 \
+                -U eb176051-bd15-49b7-9e6b-462e0b467019 \
+                -T 0 \
+                --exclude-regex="$(
+                  <${pkgs.closureInfo { rootPaths = [ config.system.build.toplevel regInfo ]; }}/store-paths \
+                    sed -e 's^.*/^^g' \
+                  | cut -c -10 \
+                  | ${pkgs.python3}/bin/python ${./includes-to-excludes.py} )" \
+                "$TMPDIR"/store.img \
+                . \
+                </dev/null >/dev/null
+            )
+          ''
+        )
+      }
 
       # Create a directory for exchanging data with the VM.
       mkdir -p "$TMPDIR/xchg"
@@ -769,6 +790,8 @@ in
     );
     boot.loader.grub.gfxmodeBios = with cfg.resolution; "${toString x}x${toString y}";
 
+    boot.initrd.kernelModules = optionals (cfg.useNixStoreImage && !cfg.writableStore) [ "erofs" ];
+
     boot.initrd.extraUtilsCommands = lib.mkIf (cfg.useDefaultFilesystems && !config.boot.initrd.systemd.enable)
       ''
         # We need mke2fs in the initrd.
@@ -905,6 +928,7 @@ in
         name = "nix-store";
         file = ''"$TMPDIR"/store.img'';
         deviceExtraOpts.bootindex = if cfg.useBootLoader then "3" else "2";
+        driveExtraOpts.format = if cfg.writableStore then "qcow2" else "raw";
       }])
       (mkIf cfg.useBootLoader [
         # The order of this list determines the device names, see