From 7d9a979b2e176190caadc9d73ccc07f36b6efa91 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Fri, 8 May 2020 14:40:00 +0100 Subject: [PATCH 1/6] nixos-container: Make configuration and state directories configurable /etc/containers is also used by Podman, Skopeo & other popular container tooling so we need to be able to move to another configuration directory. The state move is not strictly a requirement but is good for consistency. --- .../nixos-container/default.nix | 10 +++++++- .../nixos-container/nixos-container.pl | 23 +++++++++++-------- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/pkgs/tools/virtualization/nixos-container/default.nix b/pkgs/tools/virtualization/nixos-container/default.nix index 17065a2aa8b..4d46ec3299b 100644 --- a/pkgs/tools/virtualization/nixos-container/default.nix +++ b/pkgs/tools/virtualization/nixos-container/default.nix @@ -1,4 +1,10 @@ -{ substituteAll, perl, shadow, util-linux }: +{ substituteAll +, perl +, shadow +, util-linux +, configurationDirectory ? "/etc/nixos-containers" +, stateDirectory ? "/var/lib/nixos-containers" +}: substituteAll { name = "nixos-container"; @@ -9,6 +15,8 @@ substituteAll { su = "${shadow.su}/bin/su"; utillinux = util-linux; + inherit configurationDirectory stateDirectory; + postInstall = '' t=$out/share/bash-completion/completions mkdir -p $t diff --git a/pkgs/tools/virtualization/nixos-container/nixos-container.pl b/pkgs/tools/virtualization/nixos-container/nixos-container.pl index d99b4cfba4a..e945bc70b6b 100755 --- a/pkgs/tools/virtualization/nixos-container/nixos-container.pl +++ b/pkgs/tools/virtualization/nixos-container/nixos-container.pl @@ -12,6 +12,9 @@ use Time::HiRes; my $nsenter = "@utillinux@/bin/nsenter"; my $su = "@su@"; +my $configurationDirectory = "@configurationDirectory@"; +my $stateDirectory = "@stateDirectory@"; + # Ensure a consistent umask. umask 0022; @@ -132,11 +135,11 @@ if (defined $flake && $flake =~ /^(.*)#([^#"]+)$/) { # Execute the selected action. -mkpath("/etc/containers", 0, 0755); -mkpath("/var/lib/containers", 0, 0700); +mkpath("$configurationDirectory", 0, 0755); +mkpath("$stateDirectory", 0, 0700); if ($action eq "list") { - foreach my $confFile (glob "/etc/containers/*.conf") { + foreach my $confFile (glob "$configurationDirectory/*.conf") { $confFile =~ /\/([^\/]+).conf$/ or next; print "$1\n"; } @@ -198,15 +201,15 @@ if ($action eq "create") { open(my $lock, '>>', $lockFN) or die "$0: opening $lockFN: $!"; flock($lock, LOCK_EX) or die "$0: could not lock $lockFN: $!"; - my $confFile = "/etc/containers/$containerName.conf"; - my $root = "/var/lib/containers/$containerName"; + my $confFile = "$configurationDirectory/$containerName.conf"; + my $root = "$stateDirectory/$containerName"; # Maybe generate a unique name. if ($ensureUniqueName) { my $base = $containerName; for (my $nr = 0; ; $nr++) { - $confFile = "/etc/containers/$containerName.conf"; - $root = "/var/lib/containers/$containerName"; + $confFile = "$configurationDirectory/$containerName.conf"; + $root = "$stateDirectory/$containerName"; last unless -e $confFile || -e $root; $containerName = "$base-$nr"; } @@ -220,7 +223,7 @@ if ($action eq "create") { # Get an unused IP address. my %usedIPs; - foreach my $confFile2 (glob "/etc/containers/*.conf") { + foreach my $confFile2 (glob "$configurationDirectory/*.conf") { my $s = read_file($confFile2) or die; $usedIPs{$1} = 1 if $s =~ /^HOST_ADDRESS=([0-9\.]+)$/m; $usedIPs{$1} = 1 if $s =~ /^LOCAL_ADDRESS=([0-9\.]+)$/m; @@ -292,10 +295,10 @@ if ($action eq "create") { exit 0; } -my $root = "/var/lib/containers/$containerName"; +my $root = "$stateDirectory/$containerName"; my $profileDir = "/nix/var/nix/profiles/per-container/$containerName"; my $gcRootsDir = "/nix/var/nix/gcroots/per-container/$containerName"; -my $confFile = "/etc/containers/$containerName.conf"; +my $confFile = "$configurationDirectory/$containerName.conf"; if (!-e $confFile) { if ($action eq "destroy") { exit 0; From f535d6f45ef9556d392de76c2dd1c26b436c4ea8 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Fri, 8 May 2020 15:05:29 +0100 Subject: [PATCH 2/6] nixos-container: Use new configuration & state directories We need to move NixOS containers somewhere else so these don't clash with Podman, Skopeo & other container software in the libpod & cri-o/cri-u/libcontainer ecosystems. The state directory move is not strictly a requirement but is good for consistency. --- .../declarative-containers.section.md | 2 +- .../imperative-containers.section.md | 4 +-- .../declarative-containers.section.xml | 4 +-- .../imperative-containers.section.xml | 5 +-- .../virtualisation/nixos-containers.nix | 31 ++++++++++++------- nixos/tests/containers-ephemeral.nix | 6 ++-- nixos/tests/containers-imperative.nix | 12 +++---- nixos/tests/containers-tmpfs.nix | 10 +++--- 8 files changed, 42 insertions(+), 32 deletions(-) diff --git a/nixos/doc/manual/administration/declarative-containers.section.md b/nixos/doc/manual/administration/declarative-containers.section.md index 0d9d4017ed8..00fd244bb91 100644 --- a/nixos/doc/manual/administration/declarative-containers.section.md +++ b/nixos/doc/manual/administration/declarative-containers.section.md @@ -40,7 +40,7 @@ section for details on container networking.) To disable the container, just remove it from `configuration.nix` and run `nixos-rebuild switch`. Note that this will not delete the root directory of the -container in `/var/lib/containers`. Containers can be destroyed using +container in `/var/lib/nixos-containers`. Containers can be destroyed using the imperative method: `nixos-container destroy foo`. Declarative containers can be started and stopped using the diff --git a/nixos/doc/manual/administration/imperative-containers.section.md b/nixos/doc/manual/administration/imperative-containers.section.md index 05196bf5d81..f45991780c4 100644 --- a/nixos/doc/manual/administration/imperative-containers.section.md +++ b/nixos/doc/manual/administration/imperative-containers.section.md @@ -10,8 +10,8 @@ You create a container with identifier `foo` as follows: # nixos-container create foo ``` -This creates the container's root directory in `/var/lib/containers/foo` -and a small configuration file in `/etc/containers/foo.conf`. It also +This creates the container's root directory in `/var/lib/nixos-containers/foo` +and a small configuration file in `/etc/nixos-containers/foo.conf`. It also builds the container's initial system configuration and stores it in `/nix/var/nix/profiles/per-container/foo/system`. You can modify the initial configuration of the container on the command line. For diff --git a/nixos/doc/manual/from_md/administration/declarative-containers.section.xml b/nixos/doc/manual/from_md/administration/declarative-containers.section.xml index 7b35520d567..b8179dca1f8 100644 --- a/nixos/doc/manual/from_md/administration/declarative-containers.section.xml +++ b/nixos/doc/manual/from_md/administration/declarative-containers.section.xml @@ -48,8 +48,8 @@ containers.database = { configuration.nix and run nixos-rebuild switch. Note that this will not delete the root directory of the container in - /var/lib/containers. Containers can be destroyed - using the imperative method: + /var/lib/nixos-containers. Containers can be + destroyed using the imperative method: nixos-container destroy foo. diff --git a/nixos/doc/manual/from_md/administration/imperative-containers.section.xml b/nixos/doc/manual/from_md/administration/imperative-containers.section.xml index 59ecfdee5af..865fc468939 100644 --- a/nixos/doc/manual/from_md/administration/imperative-containers.section.xml +++ b/nixos/doc/manual/from_md/administration/imperative-containers.section.xml @@ -14,8 +14,9 @@ This creates the container’s root directory in - /var/lib/containers/foo and a small configuration - file in /etc/containers/foo.conf. It also builds + /var/lib/nixos-containers/foo and a small + configuration file in + /etc/nixos-containers/foo.conf. It also builds the container’s initial system configuration and stores it in /nix/var/nix/profiles/per-container/foo/system. You can modify the initial configuration of the container on the diff --git a/nixos/modules/virtualisation/nixos-containers.nix b/nixos/modules/virtualisation/nixos-containers.nix index 0838a57f0f3..ea9f9b02fad 100644 --- a/nixos/modules/virtualisation/nixos-containers.nix +++ b/nixos/modules/virtualisation/nixos-containers.nix @@ -4,6 +4,11 @@ with lib; let + configurationPrefix = optionalString (versionAtLeast config.system.stateVersion "22.05") "nixos-"; + configurationDirectoryName = "${configurationPrefix}containers"; + configurationDirectory = "/etc/${configurationDirectoryName}"; + stateDirectory = "/var/lib/${configurationPrefix}containers"; + # The container's init script, a small wrapper around the regular # NixOS stage-2 init script. containerInit = (cfg: @@ -77,7 +82,7 @@ let startScript = cfg: '' mkdir -p -m 0755 "$root/etc" "$root/var/lib" - mkdir -p -m 0700 "$root/var/lib/private" "$root/root" /run/containers + mkdir -p -m 0700 "$root/var/lib/private" "$root/root" /run/nixos-containers if ! [ -e "$root/etc/os-release" ]; then touch "$root/etc/os-release" fi @@ -249,11 +254,11 @@ let SyslogIdentifier = "container %i"; - EnvironmentFile = "-/etc/containers/%i.conf"; + EnvironmentFile = "-${configurationDirectory}/%i.conf"; Type = "notify"; - RuntimeDirectory = lib.optional cfg.ephemeral "containers/%i"; + RuntimeDirectory = lib.optional cfg.ephemeral "${configurationDirectoryName}/%i"; # Note that on reboot, systemd-nspawn returns 133, so this # unit will be restarted. On poweroff, it returns 0, so the @@ -740,12 +745,12 @@ in unit = { description = "Container '%i'"; - unitConfig.RequiresMountsFor = "/var/lib/containers/%i"; + unitConfig.RequiresMountsFor = "${stateDirectory}/%i"; path = [ pkgs.iproute2 ]; environment = { - root = "/var/lib/containers/%i"; + root = "${stateDirectory}/%i"; INSTANCE = "%i"; }; @@ -782,8 +787,8 @@ in script = startScript containerConfig; postStart = postStartScript containerConfig; serviceConfig = serviceDirectives containerConfig; - unitConfig.RequiresMountsFor = lib.optional (!containerConfig.ephemeral) "/var/lib/containers/%i"; - environment.root = if containerConfig.ephemeral then "/run/containers/%i" else "/var/lib/containers/%i"; + unitConfig.RequiresMountsFor = lib.optional (!containerConfig.ephemeral) "${stateDirectory}/%i"; + environment.root = if containerConfig.ephemeral then "/run/nixos-containers/%i" else "${stateDirectory}/%i"; } // ( if containerConfig.autoStart then { @@ -792,7 +797,7 @@ in after = [ "network.target" ]; restartTriggers = [ containerConfig.path - config.environment.etc."containers/${name}.conf".source + config.environment.etc."${configurationDirectoryName}/${name}.conf".source ]; restartIfChanged = true; } @@ -800,12 +805,12 @@ in )) config.containers) )); - # Generate a configuration file in /etc/containers for each + # Generate a configuration file in /etc/nixos-containers for each # container so that container@.target can get the container # configuration. environment.etc = let mkPortStr = p: p.protocol + ":" + (toString p.hostPort) + ":" + (if p.containerPort == null then toString p.hostPort else toString p.containerPort); - in mapAttrs' (name: cfg: nameValuePair "containers/${name}.conf" + in mapAttrs' (name: cfg: nameValuePair "${configurationDirectoryName}/${name}.conf" { text = '' SYSTEM_PATH=${cfg.path} @@ -854,7 +859,11 @@ in ENV{INTERFACE}=="v[eb]-*", ENV{NM_UNMANAGED}="1" ''; - environment.systemPackages = [ pkgs.nixos-container ]; + environment.systemPackages = [ + (pkgs.nixos-container.override { + inherit stateDirectory configurationDirectory; + }) + ]; boot.kernelModules = [ "bridge" diff --git a/nixos/tests/containers-ephemeral.nix b/nixos/tests/containers-ephemeral.nix index c9fe2778cac..cb4b7d4eba0 100644 --- a/nixos/tests/containers-ephemeral.nix +++ b/nixos/tests/containers-ephemeral.nix @@ -33,10 +33,10 @@ import ./make-test-python.nix ({ pkgs, lib, ... }: { machine.succeed("nixos-container start webserver") with subtest("Container got its own root folder"): - machine.succeed("ls /run/containers/webserver") + machine.succeed("ls /run/nixos-containers/webserver") with subtest("Container persistent directory is not created"): - machine.fail("ls /var/lib/containers/webserver") + machine.fail("ls /var/lib/nixos-containers/webserver") # Since "start" returns after the container has reached # multi-user.target, we should now be able to access it. @@ -49,6 +49,6 @@ import ./make-test-python.nix ({ pkgs, lib, ... }: { machine.fail(f"curl --fail --connect-timeout 2 http://{ip}/ > /dev/null") with subtest("Container's root folder was removed"): - machine.fail("ls /run/containers/webserver") + machine.fail("ls /run/nixos-containers/webserver") ''; }) diff --git a/nixos/tests/containers-imperative.nix b/nixos/tests/containers-imperative.nix index 44d2e50288b..a21ce97a23b 100644 --- a/nixos/tests/containers-imperative.nix +++ b/nixos/tests/containers-imperative.nix @@ -69,8 +69,8 @@ import ./make-test-python.nix ({ pkgs, lib, ... }: { with subtest(f"Put the root of {id2} into a bind mount"): machine.succeed( - f"mv /var/lib/containers/{id2} /id2-bindmount", - f"mount --bind /id2-bindmount /var/lib/containers/{id1}", + f"mv /var/lib/nixos-containers/{id2} /id2-bindmount", + f"mount --bind /id2-bindmount /var/lib/nixos-containers/{id1}", ) ip1 = machine.succeed(f"nixos-container show-ip {id1}").rstrip() @@ -88,7 +88,7 @@ import ./make-test-python.nix ({ pkgs, lib, ... }: { "Create a directory with a dummy file and bind-mount it into both containers." ): for id in id1, id2: - important_path = f"/var/lib/containers/{id}/very/important/data" + important_path = f"/var/lib/nixos-containers/{id}/very/important/data" machine.succeed( f"mkdir -p {important_path}", f"mount --bind /nested-bindmount {important_path}", @@ -154,13 +154,13 @@ import ./make-test-python.nix ({ pkgs, lib, ... }: { machine.succeed("grep -qF 'important data' /nested-bindmount/dummy") with subtest("Ensure that the container path is gone"): - print(machine.succeed("ls -lsa /var/lib/containers")) - machine.succeed(f"test ! -e /var/lib/containers/{id1}") + print(machine.succeed("ls -lsa /var/lib/nixos-containers")) + machine.succeed(f"test ! -e /var/lib/nixos-containers/{id1}") with subtest("Ensure that a failed container creation doesn'leave any state"): machine.fail( "nixos-container create b0rk --config-file ${brokenCfg}" ) - machine.succeed("test ! -e /var/lib/containers/b0rk") + machine.succeed("test ! -e /var/lib/nixos-containers/b0rk") ''; }) diff --git a/nixos/tests/containers-tmpfs.nix b/nixos/tests/containers-tmpfs.nix index 7a2c835b120..cf5b81656af 100644 --- a/nixos/tests/containers-tmpfs.nix +++ b/nixos/tests/containers-tmpfs.nix @@ -62,7 +62,7 @@ import ./make-test-python.nix ({ pkgs, lib, ... }: { machine.succeed( tmpfs_cmd("touch /root/test.file"), tmpfs_cmd("ls -l /root | grep -q test.file"), - "test -e /var/lib/containers/tmpfs/root/test.file", + "test -e /var/lib/nixos-containers/tmpfs/root/test.file", ) with subtest( @@ -73,7 +73,7 @@ import ./make-test-python.nix ({ pkgs, lib, ... }: { tmpfs_cmd("touch /some/random/path/test.file"), tmpfs_cmd("test -e /some/random/path/test.file"), ) - machine.fail("test -e /var/lib/containers/tmpfs/some/random/path/test.file") + machine.fail("test -e /var/lib/nixos-containers/tmpfs/some/random/path/test.file") with subtest( "files created in the hosts container dir in a path where a tmpfs " @@ -81,9 +81,9 @@ import ./make-test-python.nix ({ pkgs, lib, ... }: { + "the do not exist in the tmpfs" ): machine.succeed( - "touch /var/lib/containers/tmpfs/var/test.file", - "test -e /var/lib/containers/tmpfs/var/test.file", - "ls -l /var/lib/containers/tmpfs/var/ | grep -q test.file 2>/dev/null", + "touch /var/lib/nixos-containers/tmpfs/var/test.file", + "test -e /var/lib/nixos-containers/tmpfs/var/test.file", + "ls -l /var/lib/nixos-containers/tmpfs/var/ | grep -q test.file 2>/dev/null", ) machine.fail(tmpfs_cmd("ls -l /var | grep -q test.file")) ''; From 81d192cfa6d8f5ccbfe797c0006c533945327a03 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Fri, 8 May 2020 15:23:11 +0100 Subject: [PATCH 3/6] nixos-container: Add compatibility hack for pre-22.05 state/config directories --- .../virtualization/nixos-container/nixos-container.pl | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/pkgs/tools/virtualization/nixos-container/nixos-container.pl b/pkgs/tools/virtualization/nixos-container/nixos-container.pl index e945bc70b6b..38f4c8d3169 100755 --- a/pkgs/tools/virtualization/nixos-container/nixos-container.pl +++ b/pkgs/tools/virtualization/nixos-container/nixos-container.pl @@ -138,8 +138,14 @@ if (defined $flake && $flake =~ /^(.*)#([^#"]+)$/) { mkpath("$configurationDirectory", 0, 0755); mkpath("$stateDirectory", 0, 0700); + if ($action eq "list") { foreach my $confFile (glob "$configurationDirectory/*.conf") { + # Filter libpod configuration files + # From 22.05 and onwards this is not an issue any more as directories dont clash + if($confFile eq "/etc/containers/libpod.conf" || $confFile eq "/etc/containers/containers.conf" || $confFile eq "/etc/containers/registries.conf") { + next + } $confFile =~ /\/([^\/]+).conf$/ or next; print "$1\n"; } @@ -224,6 +230,11 @@ if ($action eq "create") { # Get an unused IP address. my %usedIPs; foreach my $confFile2 (glob "$configurationDirectory/*.conf") { + # Filter libpod configuration files + # From 22.05 and onwards this is not an issue any more as directories dont clash + if($confFile2 eq "/etc/containers/libpod.conf" || $confFile2 eq "/etc/containers/containers.conf" || $confFile2 eq "/etc/containers/registries.conf") { + next + } my $s = read_file($confFile2) or die; $usedIPs{$1} = 1 if $s =~ /^HOST_ADDRESS=([0-9\.]+)$/m; $usedIPs{$1} = 1 if $s =~ /^LOCAL_ADDRESS=([0-9\.]+)$/m; From 01e35db27895076fca4be0436ed584393cd0c8b0 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Fri, 8 May 2020 15:34:06 +0100 Subject: [PATCH 4/6] release-notes: Add release note about nixos containers directory moves --- .../from_md/release-notes/rl-2205.section.xml | 22 +++++++++++++++++++ .../manual/release-notes/rl-2205.section.md | 10 +++++++++ 2 files changed, 32 insertions(+) diff --git a/nixos/doc/manual/from_md/release-notes/rl-2205.section.xml b/nixos/doc/manual/from_md/release-notes/rl-2205.section.xml index 05b3822cab7..13089b1f866 100644 --- a/nixos/doc/manual/from_md/release-notes/rl-2205.section.xml +++ b/nixos/doc/manual/from_md/release-notes/rl-2205.section.xml @@ -471,6 +471,28 @@ new versions will release. + + + The configuration and state directories used by + nixos-containers have been moved from + /etc/containers and + /var/lib/containers to + /etc/nixos-containers and + /var/lib/nixos-containers. + + + If you are changing system.stateVersion to + "22.05" manually on an existing + system you are responsible for migrating these directories + yourself. + + + This is to improve compatibility with + libcontainer based software such as Podman + and Skopeo which assumes they have ownership over + /etc/containers. + + security.klogd was removed. Logging of diff --git a/nixos/doc/manual/release-notes/rl-2205.section.md b/nixos/doc/manual/release-notes/rl-2205.section.md index 16c59ce3ddd..e5960e0794e 100644 --- a/nixos/doc/manual/release-notes/rl-2205.section.md +++ b/nixos/doc/manual/release-notes/rl-2205.section.md @@ -151,6 +151,16 @@ In addition to numerous new and upgraded packages, this release has the followin org-contrib, refer to the ones in `pkgs.emacsPackages.elpaPackages` and `pkgs.emacsPackages.nongnuPackages` where the new versions will release. +- The configuration and state directories used by `nixos-containers` have been + moved from `/etc/containers` and `/var/lib/containers` to + `/etc/nixos-containers` and `/var/lib/nixos-containers`. + + If you are changing `system.stateVersion` to `"22.05"` manually on an existing + system you are responsible for migrating these directories yourself. + + This is to improve compatibility with `libcontainer` based software such as Podman and Skopeo + which assumes they have ownership over `/etc/containers`. + - `security.klogd` was removed. Logging of kernel messages is handled by systemd since Linux 3.5. From 3c49151f154a3872eb278c214863d926a4f8abf6 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Tue, 1 Sep 2020 14:58:47 +0200 Subject: [PATCH 5/6] nixos/nixos-containers: Add warning on unsupported state version combo --- nixos/modules/virtualisation/nixos-containers.nix | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/nixos/modules/virtualisation/nixos-containers.nix b/nixos/modules/virtualisation/nixos-containers.nix index ea9f9b02fad..23228a109bc 100644 --- a/nixos/modules/virtualisation/nixos-containers.nix +++ b/nixos/modules/virtualisation/nixos-containers.nix @@ -742,6 +742,12 @@ in config = mkIf (config.boot.enableContainers) (let + warnings = flatten [ + (optional (config.virtualisation.containers.enable && versionOlder config.system.stateVersion "22.05") '' + Enabling both boot.enableContainers & virtualisation.containers on system.stateVersion < 22.05 is unsupported. + '') + ]; + unit = { description = "Container '%i'"; From dc26602aedb9831093ff5c25673354751532a4d1 Mon Sep 17 00:00:00 2001 From: adisbladis Date: Wed, 27 Apr 2022 18:16:49 +1200 Subject: [PATCH 6/6] nixos-container: Add tests to passthru for CI --- .../tools/virtualization/nixos-container/default.nix | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/pkgs/tools/virtualization/nixos-container/default.nix b/pkgs/tools/virtualization/nixos-container/default.nix index 4d46ec3299b..be17753b343 100644 --- a/pkgs/tools/virtualization/nixos-container/default.nix +++ b/pkgs/tools/virtualization/nixos-container/default.nix @@ -4,6 +4,7 @@ , util-linux , configurationDirectory ? "/etc/nixos-containers" , stateDirectory ? "/var/lib/nixos-containers" +, nixosTests }: substituteAll { @@ -17,6 +18,17 @@ substituteAll { inherit configurationDirectory stateDirectory; + passthru = { + tests = { + inherit (nixosTests) + containers-imperative + containers-ip + containers-tmpfs + containers-ephemeral + ; + }; + }; + postInstall = '' t=$out/share/bash-completion/completions mkdir -p $t