Merge pull request #194345 from codedownio/binary-cache

Introduce mkBinaryCache function

doc/builders/images.xml

@@ -11,4 +11,5 @@
  <xi:include href="images/snaptools.section.xml" />
  <xi:include href="images/portableservice.section.xml" />
  <xi:include href="images/makediskimage.section.xml" />
+ <xi:include href="images/binarycache.section.xml" />
 </chapter>

doc/builders/images/binarycache.section.md

@@ -0,0 +1,49 @@
+# pkgs.mkBinaryCache {#sec-pkgs-binary-cache}
+
+`pkgs.mkBinaryCache` is a function for creating Nix flat-file binary caches. Such a cache exists as a directory on disk, and can be used as a Nix substituter by passing `--substituter file:///path/to/cache` to Nix commands.
+
+Nix packages are most commonly shared between machines using [HTTP, SSH, or S3](https://nixos.org/manual/nix/stable/package-management/sharing-packages.html), but a flat-file binary cache can still be useful in some situations. For example, you can copy it directly to another machine, or make it available on a network file system. It can also be a convenient way to make some Nix packages available inside a container via bind-mounting.
+
+Note that this function is meant for advanced use-cases. The more idiomatic way to work with flat-file binary caches is via the [nix-copy-closure](https://nixos.org/manual/nix/stable/command-ref/nix-copy-closure.html) command. You may also want to consider [dockerTools](#sec-pkgs-dockerTools) for your containerization needs.
+
+## Example
+
+The following derivation will construct a flat-file binary cache containing the closure of `hello`.
+
+```nix
+mkBinaryCache {
+  rootPaths = [hello];
+}
+```
+
+- `rootPaths` specifies a list of root derivations. The transitive closure of these derivations' outputs will be copied into the cache.
+
+Here's an example of building and using the cache.
+
+Build the cache on one machine, `host1`:
+
+```shellSession
+nix-build -E 'with import <nixpkgs> {}; mkBinaryCache { rootPaths = [hello]; }'
+```
+
+```shellSession
+/nix/store/cc0562q828rnjqjyfj23d5q162gb424g-binary-cache
+```
+
+Copy the resulting directory to the other machine, `host2`:
+
+```shellSession
+scp result host2:/tmp/hello-cache
+```
+
+Substitute the derivation using the flat-file binary cache on the other machine, `host2`:
+```shellSession
+nix-build -A hello '<nixpkgs>' \
+  --option require-sigs false \
+  --option trusted-substituters file:///tmp/hello-cache \
+  --option substituters file:///tmp/hello-cache
+```
+
+```shellSession
+/nix/store/gl5a41azbpsadfkfmbilh9yk40dh5dl0-hello-2.12.1
+```

nixos/tests/all-tests.nix

@@ -92,6 +92,7 @@ in {
   bcachefs = handleTestOn ["x86_64-linux" "aarch64-linux"] ./bcachefs.nix {};
   beanstalkd = handleTest ./beanstalkd.nix {};
   bees = handleTest ./bees.nix {};
+  binary-cache = handleTest ./binary-cache.nix {};
   bind = handleTest ./bind.nix {};
   bird = handleTest ./bird.nix {};
   bitcoind = handleTest ./bitcoind.nix {};

nixos/tests/binary-cache.nix

@@ -0,0 +1,62 @@
+import ./make-test-python.nix ({ lib, ... }:
+
+with lib;
+
+{
+  name = "binary-cache";
+  meta.maintainers = with maintainers; [ thomasjm ];
+
+  nodes.machine =
+    { pkgs, ... }: {
+      imports = [ ../modules/installer/cd-dvd/channel.nix ];
+      environment.systemPackages = with pkgs; [python3];
+      system.extraDependencies = with pkgs; [hello.inputDerivation];
+      nix.extraOptions = ''
+        experimental-features = nix-command
+      '';
+    };
+
+  testScript = ''
+    # Build the cache, then remove it from the store
+    cachePath = machine.succeed("nix-build --no-out-link -E 'with import <nixpkgs> {}; mkBinaryCache { rootPaths = [hello]; }'").strip()
+    machine.succeed("cp -r %s/. /tmp/cache" % cachePath)
+    machine.succeed("nix-store --delete " + cachePath)
+
+    # Sanity test of cache structure
+    status, stdout = machine.execute("ls /tmp/cache")
+    cache_files = stdout.split()
+    assert ("nix-cache-info" in cache_files)
+    assert ("nar" in cache_files)
+
+    # Nix store ping should work
+    machine.succeed("nix store ping --store file:///tmp/cache")
+
+    # Cache should contain a .narinfo referring to "hello"
+    grepLogs = machine.succeed("grep -l 'StorePath: /nix/store/[[:alnum:]]*-hello-.*' /tmp/cache/*.narinfo")
+
+    # Get the store path referenced by the .narinfo
+    narInfoFile = grepLogs.strip()
+    narInfoContents = machine.succeed("cat " + narInfoFile)
+    import re
+    match = re.match(r"^StorePath: (/nix/store/[a-z0-9]*-hello-.*)$", narInfoContents, re.MULTILINE)
+    if not match: raise Exception("Couldn't find hello store path in cache")
+    storePath = match[1]
+
+    # Delete the store path
+    machine.succeed("nix-store --delete " + storePath)
+    machine.succeed("[ ! -d %s ] || exit 1" % storePath)
+
+    # Should be able to build hello using the cache
+    logs = machine.succeed("nix-build -A hello '<nixpkgs>' --option require-sigs false --option trusted-substituters file:///tmp/cache --option substituters file:///tmp/cache 2>&1")
+    logLines = logs.split("\n")
+    if not "this path will be fetched" in logLines[0]: raise Exception("Unexpected first log line")
+    def shouldBe(got, desired):
+      if got != desired: raise Exception("Expected '%s' but got '%s'" % (desired, got))
+    shouldBe(logLines[1], "  " + storePath)
+    shouldBe(logLines[2], "copying path '%s' from 'file:///tmp/cache'..." % storePath)
+    shouldBe(logLines[3], storePath)
+
+    # Store path should exist in the store now
+    machine.succeed("[ -d %s ] || exit 1" % storePath)
+  '';
+})

pkgs/build-support/binary-cache/default.nix

@@ -0,0 +1,40 @@
+{ stdenv, buildPackages }:
+
+# This function is for creating a flat-file binary cache, i.e. the kind created by
+# nix copy --to file:///some/path and usable as a substituter (with the file:// prefix).
+
+# For example, in the Nixpkgs repo:
+# nix-build -E 'with import ./. {}; mkBinaryCache { rootPaths = [hello]; }'
+
+{ name ? "binary-cache"
+, rootPaths
+}:
+
+stdenv.mkDerivation {
+  inherit name;
+
+  __structuredAttrs = true;
+
+  exportReferencesGraph.closure = rootPaths;
+
+  preferLocalBuild = true;
+
+  PATH = "${buildPackages.coreutils}/bin:${buildPackages.jq}/bin:${buildPackages.python3}/bin:${buildPackages.nix}/bin:${buildPackages.xz}/bin";
+
+  builder = builtins.toFile "builder" ''
+    . .attrs.sh
+
+    export out=''${outputs[out]}
+
+    mkdir $out
+    mkdir $out/nar
+
+    python ${./make-binary-cache.py}
+
+    # These directories must exist, or Nix might try to create them in LocalBinaryCacheStore::init(),
+    # which fails if mounted read-only
+    mkdir $out/realisations
+    mkdir $out/debuginfo
+    mkdir $out/log
+  '';
+}

pkgs/build-support/binary-cache/make-binary-cache.py

@@ -0,0 +1,43 @@
+
+import json
+import os
+import subprocess
+
+with open(".attrs.json", "r") as f:
+  closures = json.load(f)["closure"]
+
+os.chdir(os.environ["out"])
+
+nixPrefix = os.environ["NIX_STORE"] # Usually /nix/store
+
+with open("nix-cache-info", "w") as f:
+  f.write("StoreDir: " + nixPrefix + "\n")
+
+def dropPrefix(path):
+  return path[len(nixPrefix + "/"):]
+
+for item in closures:
+  narInfoHash = dropPrefix(item["path"]).split("-")[0]
+
+  xzFile = "nar/" + narInfoHash + ".nar.xz"
+  with open(xzFile, "w") as f:
+    subprocess.run("nix-store --dump %s | xz -c" % item["path"], stdout=f, shell=True)
+
+  fileHash = subprocess.run(["nix-hash", "--base32", "--type", "sha256", item["path"]], capture_output=True).stdout.decode().strip()
+  fileSize = os.path.getsize(xzFile)
+
+  # Rename the .nar.xz file to its own hash to match "nix copy" behavior
+  finalXzFile = "nar/" + fileHash + ".nar.xz"
+  os.rename(xzFile, finalXzFile)
+
+  with open(narInfoHash + ".narinfo", "w") as f:
+    f.writelines((x + "\n" for x in [
+      "StorePath: " + item["path"],
+      "URL: " + finalXzFile,
+      "Compression: xz",
+      "FileHash: sha256:" + fileHash,
+      "FileSize: " + str(fileSize),
+      "NarHash: " + item["narHash"],
+      "NarSize: " + str(item["narSize"]),
+      "References: " + " ".join(dropPrefix(ref) for ref in item["references"]),
+    ]))

pkgs/top-level/all-packages.nix

@@ -1033,6 +1033,8 @@ with pkgs;
       inherit kernel firmware rootModules allowMissing;
     };
 
+  mkBinaryCache = callPackage ../build-support/binary-cache { };
+
   mkShell = callPackage ../build-support/mkshell { };
   mkShellNoCC = mkShell.override { stdenv = stdenvNoCC; };