From d1a2a16a3a0220e8f571c07001a2ada4a664c492 Mon Sep 17 00:00:00 2001 From: Tom McLaughlin Date: Mon, 3 Oct 2022 22:06:28 -0600 Subject: [PATCH 1/2] Introduce mkBinaryCache function --- doc/builders/images.xml | 1 + doc/builders/images/binarycache.section.md | 49 +++++++++++++++ nixos/tests/all-tests.nix | 1 + nixos/tests/binary-cache.nix | 62 +++++++++++++++++++ pkgs/build-support/binary-cache/default.nix | 40 ++++++++++++ .../binary-cache/make-binary-cache.py | 43 +++++++++++++ pkgs/top-level/all-packages.nix | 2 + 7 files changed, 198 insertions(+) create mode 100644 doc/builders/images/binarycache.section.md create mode 100644 nixos/tests/binary-cache.nix create mode 100644 pkgs/build-support/binary-cache/default.nix create mode 100644 pkgs/build-support/binary-cache/make-binary-cache.py diff --git a/doc/builders/images.xml b/doc/builders/images.xml index 7d06130e3ec..a4661ab5a7a 100644 --- a/doc/builders/images.xml +++ b/doc/builders/images.xml @@ -11,4 +11,5 @@ + diff --git a/doc/builders/images/binarycache.section.md b/doc/builders/images/binarycache.section.md new file mode 100644 index 00000000000..71dc26311cf --- /dev/null +++ b/doc/builders/images/binarycache.section.md @@ -0,0 +1,49 @@ +# pkgs.mkBinaryCache {#sec-pkgs-binary-cache} + +`pkgs.mkBinaryCache` is a function for creating Nix flat-file binary caches. Such a cache exists as a directory on disk, and can be used as a Nix substituter by passing `--substituter file:///path/to/cache` to Nix commands. + +Nix packages are most commonly shared between machines using [HTTP, SSH, or S3](https://nixos.org/manual/nix/stable/package-management/sharing-packages.html), but a flat-file binary cache can still be useful in some situations. For example, you can copy it directly to another machine, or make it available on a network file system. It can also be a convenient way to make some Nix packages available inside a container via bind-mounting. + +Note that this function is meant for advanced use-cases. The more idiomatic way to work with flat-file binary caches is via the [nix-copy-closure](https://nixos.org/manual/nix/stable/command-ref/nix-copy-closure.html) command. You may also want to consider [dockerTools](#sec-pkgs-dockerTools) for your containerization needs. + +## Example + +The following derivation will construct a flat-file binary cache containing the closure of `hello`. + +```nix +mkBinaryCache { + rootPaths = [hello]; +} +``` + +- `rootPaths` specifies a list of root derivations. The transitive closure of these derivations' outputs will be copied into the cache. + +Here's an example of building and using the cache. + +Build the cache on one machine, `host1`: + +```shellSession +nix-build -E 'with import {}; mkBinaryCache { rootPaths = [hello]; }' +``` + +```shellSession +/nix/store/cc0562q828rnjqjyfj23d5q162gb424g-binary-cache +``` + +Copy the resulting directory to the other machine, `host2`: + +```shellSession +scp result host2:/tmp/hello-cache +``` + +Build the derivation using the flat-file binary cache on the other machine, `host2`: +```shellSession +nix-build -A hello '' \ + --option require-sigs false \ + --option trusted-substituters file:///tmp/hello-cache \ + --option substituters file:///tmp/hello-cache +``` + +```shellSession +/nix/store/gl5a41azbpsadfkfmbilh9yk40dh5dl0-hello-2.12.1 +``` diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix index d8eb00d5453..196c30b1387 100644 --- a/nixos/tests/all-tests.nix +++ b/nixos/tests/all-tests.nix @@ -92,6 +92,7 @@ in { bcachefs = handleTestOn ["x86_64-linux" "aarch64-linux"] ./bcachefs.nix {}; beanstalkd = handleTest ./beanstalkd.nix {}; bees = handleTest ./bees.nix {}; + binary-cache = handleTest ./binary-cache.nix {}; bind = handleTest ./bind.nix {}; bird = handleTest ./bird.nix {}; bitcoind = handleTest ./bitcoind.nix {}; diff --git a/nixos/tests/binary-cache.nix b/nixos/tests/binary-cache.nix new file mode 100644 index 00000000000..0809e59e5a1 --- /dev/null +++ b/nixos/tests/binary-cache.nix @@ -0,0 +1,62 @@ +import ./make-test-python.nix ({ lib, ... }: + +with lib; + +{ + name = "binary-cache"; + meta.maintainers = with maintainers; [ thomasjm ]; + + nodes.machine = + { pkgs, ... }: { + imports = [ ../modules/installer/cd-dvd/channel.nix ]; + environment.systemPackages = with pkgs; [python3]; + system.extraDependencies = with pkgs; [hello.inputDerivation]; + nix.extraOptions = '' + experimental-features = nix-command + ''; + }; + + testScript = '' + # Build the cache, then remove it from the store + cachePath = machine.succeed("nix-build --no-out-link -E 'with import {}; mkBinaryCache { rootPaths = [hello]; }'").strip() + machine.succeed("cp -r %s/. /tmp/cache" % cachePath) + machine.succeed("nix-store --delete " + cachePath) + + # Sanity test of cache structure + status, stdout = machine.execute("ls /tmp/cache") + cache_files = stdout.split() + assert ("nix-cache-info" in cache_files) + assert ("nar" in cache_files) + + # Nix store ping should work + machine.succeed("nix store ping --store file:///tmp/cache") + + # Cache should contain a .narinfo referring to "hello" + grepLogs = machine.succeed("grep -l 'StorePath: /nix/store/[[:alnum:]]*-hello-.*' /tmp/cache/*.narinfo") + + # Get the store path referenced by the .narinfo + narInfoFile = grepLogs.strip() + narInfoContents = machine.succeed("cat " + narInfoFile) + import re + match = re.match(r"^StorePath: (/nix/store/[a-z0-9]*-hello-.*)$", narInfoContents, re.MULTILINE) + if not match: raise Exception("Couldn't find hello store path in cache") + storePath = match[1] + + # Delete the store path + machine.succeed("nix-store --delete " + storePath) + machine.succeed("[ ! -d %s ] || exit 1" % storePath) + + # Should be able to build hello using the cache + logs = machine.succeed("nix-build -A hello '' --option require-sigs false --option trusted-substituters file:///tmp/cache --option substituters file:///tmp/cache 2>&1") + logLines = logs.split("\n") + if not "this path will be fetched" in logLines[0]: raise Exception("Unexpected first log line") + def shouldBe(got, desired): + if got != desired: raise Exception("Expected '%s' but got '%s'" % (desired, got)) + shouldBe(logLines[1], " " + storePath) + shouldBe(logLines[2], "copying path '%s' from 'file:///tmp/cache'..." % storePath) + shouldBe(logLines[3], storePath) + + # Store path should exist in the store now + machine.succeed("[ -d %s ] || exit 1" % storePath) + ''; +}) diff --git a/pkgs/build-support/binary-cache/default.nix b/pkgs/build-support/binary-cache/default.nix new file mode 100644 index 00000000000..577328cad92 --- /dev/null +++ b/pkgs/build-support/binary-cache/default.nix @@ -0,0 +1,40 @@ +{ stdenv, buildPackages }: + +# This function is for creating a flat-file binary cache, i.e. the kind created by +# nix copy --to file:///some/path and usable as a substituter (with the file:// prefix). + +# For example, in the Nixpkgs repo: +# nix-build -E 'with import ./. {}; mkBinaryCache { rootPaths = [hello]; }' + +{ name ? "binary-cache" +, rootPaths +}: + +stdenv.mkDerivation { + inherit name; + + __structuredAttrs = true; + + exportReferencesGraph.closure = rootPaths; + + preferLocalBuild = true; + + PATH = "${buildPackages.coreutils}/bin:${buildPackages.jq}/bin:${buildPackages.python3}/bin:${buildPackages.nix}/bin:${buildPackages.xz}/bin"; + + builder = builtins.toFile "builder" '' + . .attrs.sh + + export out=''${outputs[out]} + + mkdir $out + mkdir $out/nar + + python ${./make-binary-cache.py} + + # These directories must exist, or Nix might try to create them in LocalBinaryCacheStore::init(), + # which fails if mounted read-only + mkdir $out/realisations + mkdir $out/debuginfo + mkdir $out/log + ''; +} diff --git a/pkgs/build-support/binary-cache/make-binary-cache.py b/pkgs/build-support/binary-cache/make-binary-cache.py new file mode 100644 index 00000000000..16dd8a7e96b --- /dev/null +++ b/pkgs/build-support/binary-cache/make-binary-cache.py @@ -0,0 +1,43 @@ + +import json +import os +import subprocess + +with open(".attrs.json", "r") as f: + closures = json.load(f)["closure"] + +os.chdir(os.environ["out"]) + +nixPrefix = os.environ["NIX_STORE"] # Usually /nix/store + +with open("nix-cache-info", "w") as f: + f.write("StoreDir: " + nixPrefix + "\n") + +def dropPrefix(path): + return path[len(nixPrefix + "/"):] + +for item in closures: + narInfoHash = dropPrefix(item["path"]).split("-")[0] + + xzFile = "nar/" + narInfoHash + ".nar.xz" + with open(xzFile, "w") as f: + subprocess.run("nix-store --dump %s | xz -c" % item["path"], stdout=f, shell=True) + + fileHash = subprocess.run(["nix-hash", "--base32", "--type", "sha256", item["path"]], capture_output=True).stdout.decode().strip() + fileSize = os.path.getsize(xzFile) + + # Rename the .nar.xz file to its own hash to match "nix copy" behavior + finalXzFile = "nar/" + fileHash + ".nar.xz" + os.rename(xzFile, finalXzFile) + + with open(narInfoHash + ".narinfo", "w") as f: + f.writelines((x + "\n" for x in [ + "StorePath: " + item["path"], + "URL: " + finalXzFile, + "Compression: xz", + "FileHash: sha256:" + fileHash, + "FileSize: " + str(fileSize), + "NarHash: " + item["narHash"], + "NarSize: " + str(item["narSize"]), + "References: " + " ".join(dropPrefix(ref) for ref in item["references"]), + ])) diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 94664aeec73..cf549ca633e 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -1031,6 +1031,8 @@ with pkgs; inherit kernel firmware rootModules allowMissing; }; + mkBinaryCache = callPackage ../build-support/binary-cache { }; + mkShell = callPackage ../build-support/mkshell { }; mkShellNoCC = mkShell.override { stdenv = stdenvNoCC; }; From 700e6e588e942fb395d938068bbb8e9338a413ed Mon Sep 17 00:00:00 2001 From: Tom McLaughlin Date: Tue, 7 Feb 2023 16:17:52 -0800 Subject: [PATCH 2/2] Doc tweak: "Build" -> "Substitute" --- doc/builders/images/binarycache.section.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/builders/images/binarycache.section.md b/doc/builders/images/binarycache.section.md index 71dc26311cf..fe2772f33b4 100644 --- a/doc/builders/images/binarycache.section.md +++ b/doc/builders/images/binarycache.section.md @@ -36,7 +36,7 @@ Copy the resulting directory to the other machine, `host2`: scp result host2:/tmp/hello-cache ``` -Build the derivation using the flat-file binary cache on the other machine, `host2`: +Substitute the derivation using the flat-file binary cache on the other machine, `host2`: ```shellSession nix-build -A hello '' \ --option require-sigs false \