From 04100cd281b774fbfa4954bec7b6ee9b3ed62acb Mon Sep 17 00:00:00 2001 From: Andreas Rammhold Date: Fri, 23 Oct 2020 23:09:18 +0200 Subject: [PATCH 1/5] nixos/tests: restructure test driver so that QEMU is actually overriden Previously you would be able to override only the QEMU package to be used in the test runner. Frankly that doesn't help a lot if you are trying to get a graphical session. The graphical session requires the option in the NixOS module system to bet set to the correct QEMU package. In this commit I moved most of the test node configuration and transformations into the `mkDriver` function (previously called `driver`). The motivation was to be able to create a `driver` instance with a given QEMU package that will be used consistently througout the test expression. --- nixos/lib/testing-python.nix | 79 ++++++++++++++++++++++-------------- 1 file changed, 48 insertions(+), 31 deletions(-) diff --git a/nixos/lib/testing-python.nix b/nixos/lib/testing-python.nix index 302c7f78bf8..b7d64d68877 100644 --- a/nixos/lib/testing-python.nix +++ b/nixos/lib/testing-python.nix @@ -9,7 +9,6 @@ # Modules to add to each VM , extraConfigurations ? [] }: -with import ./build-vms.nix { inherit system pkgs minimal specialArgs extraConfigurations; }; with pkgs; rec { @@ -51,9 +50,6 @@ rec { ''; }; - testDriver = mkTestDriver qemu_test; - testDriverInteractive = mkTestDriver qemu_kvm; - # Run an automated test suite in the given virtual network. # `driver' is the script that runs the network. runTests = driver: @@ -88,25 +84,7 @@ rec { maxTestNameLen = 50; testNameLen = builtins.stringLength name; - testDriverName = with builtins; - if testNameLen > maxTestNameLen then - abort ("The name of the test '${name}' must not be longer than ${toString maxTestNameLen} " + - "it's currently ${toString testNameLen} characters long.") - else - "nixos-test-driver-${name}"; - nodes = buildVirtualNetwork ( - t.nodes or (if t ? machine then { machine = t.machine; } else { })); - - testScript' = - # Call the test script with the computed nodes. - if lib.isFunction testScript - then testScript { inherit nodes; } - else testScript; - - vlans = map (m: m.config.virtualisation.vlans) (lib.attrValues nodes); - - vms = map (m: m.config.system.build.vm) (lib.attrValues nodes); ocrProg = tesseract4.override { enableLanguages = [ "eng" ]; }; @@ -115,8 +93,38 @@ rec { # Generate convenience wrappers for running the test driver # interactively with the specified network, and for starting the # VMs from the command line. - driver = testDriver: - let + mkDriver = qemu_pkg: + let + build-vms = import ./build-vms.nix { + inherit system pkgs minimal specialArgs; + extraConfigurations = extraConfigurations ++ (pkgs.lib.optional (qemu_pkg != null) + { + virtualisation.qemu.package = qemu_pkg; + } + ); + }; + + # FIXME: get this pkg from the module system + testDriver = mkTestDriver (if qemu_pkg == null then pkgs.qemu_test else qemu_pkg); + + nodes = build-vms.buildVirtualNetwork ( + t.nodes or (if t ? machine then { machine = t.machine; } else { })); + vlans = map (m: m.config.virtualisation.vlans) (lib.attrValues nodes); + vms = map (m: m.config.system.build.vm) (lib.attrValues nodes); + + testScript' = + # Call the test script with the computed nodes. + if lib.isFunction testScript + then testScript { inherit nodes; } + else testScript; + + testDriverName = with builtins; + if testNameLen > maxTestNameLen then + abort ("The name of the test '${name}' must not be longer than ${toString maxTestNameLen} " + + "it's currently ${toString testNameLen} characters long.") + else + "nixos-test-driver-${name}"; + warn = if skipLint then lib.warn "Linting is disabled!" else lib.id; in warn (runCommand testDriverName @@ -124,6 +132,9 @@ rec { testScript = testScript'; preferLocalBuild = true; testName = name; + passthru = { + inherit nodes; + }; } '' mkdir -p $out/bin @@ -154,9 +165,12 @@ rec { meta = (drv.meta or {}) // t.meta; }; - test = passMeta (runTests (driver testDriver)); + driver = mkDriver null; + driverInteractive = mkDriver pkgs.qemu; - nodeNames = builtins.attrNames nodes; + test = passMeta (runTests driver); + + nodeNames = builtins.attrNames driver.nodes; invalidNodeNames = lib.filter (node: builtins.match "^[A-z_]([A-z0-9_]+)?$" node == null) nodeNames; @@ -170,10 +184,9 @@ rec { Please stick to alphanumeric chars and underscores as separation. '' else - test // { - inherit nodes test; - driver = driver testDriver; - driverInteractive = driver testDriverInteractive; + test // { + inherit test driver driverInteractive; + inherit (test) nodes; }; runInMachine = @@ -184,7 +197,11 @@ rec { , ... # ??? }: let - vm = buildVM { } + build-vms = import ./build-vms.nix { + inherit system pkgs minimal specialArgs extraConfigurations; + }; + + vm = build-vms.buildVM { } [ machine { key = "run-in-machine"; networking.hostName = "client"; From 61b09f552c1b66befe83b114510c686ea98ffd0a Mon Sep 17 00:00:00 2001 From: Andreas Rammhold Date: Fri, 23 Oct 2020 23:13:38 +0200 Subject: [PATCH 2/5] nixos/tests: format the testing-python.nix file more consistenly --- nixos/lib/testing-python.nix | 202 ++++++++++++++++++----------------- 1 file changed, 105 insertions(+), 97 deletions(-) diff --git a/nixos/lib/testing-python.nix b/nixos/lib/testing-python.nix index b7d64d68877..2a2a14bab7a 100644 --- a/nixos/lib/testing-python.nix +++ b/nixos/lib/testing-python.nix @@ -3,11 +3,12 @@ # Use a minimal kernel? , minimal ? false # Ignored -, config ? {} +, config ? { } # !!! See comment about args in lib/modules.nix -, specialArgs ? {} +, specialArgs ? { } # Modules to add to each VM -, extraConfigurations ? [] }: +, extraConfigurations ? [ ] +}: with pkgs; @@ -16,39 +17,41 @@ rec { inherit pkgs; - mkTestDriver = let - testDriverScript = ./test-driver/test-driver.py; - in qemu_pkg: stdenv.mkDerivation { - name = "nixos-test-driver"; + mkTestDriver = + let + testDriverScript = ./test-driver/test-driver.py; + in + qemu_pkg: stdenv.mkDerivation { + name = "nixos-test-driver"; - nativeBuildInputs = [ makeWrapper ]; - buildInputs = [ (python3.withPackages (p: [ p.ptpython ])) ]; - checkInputs = with python3Packages; [ pylint black mypy ]; + nativeBuildInputs = [ makeWrapper ]; + buildInputs = [ (python3.withPackages (p: [ p.ptpython ])) ]; + checkInputs = with python3Packages; [ pylint black mypy ]; - dontUnpack = true; + dontUnpack = true; - preferLocalBuild = true; + preferLocalBuild = true; - doCheck = true; - checkPhase = '' - mypy --disallow-untyped-defs \ - --no-implicit-optional \ - --ignore-missing-imports ${testDriverScript} - pylint --errors-only ${testDriverScript} - black --check --diff ${testDriverScript} - ''; - - installPhase = - '' - mkdir -p $out/bin - cp ${testDriverScript} $out/bin/nixos-test-driver - chmod u+x $out/bin/nixos-test-driver - # TODO: copy user script part into this file (append) - - wrapProgram $out/bin/nixos-test-driver \ - --prefix PATH : "${lib.makeBinPath [ qemu_pkg vde2 netpbm coreutils ]}" \ + doCheck = true; + checkPhase = '' + mypy --disallow-untyped-defs \ + --no-implicit-optional \ + --ignore-missing-imports ${testDriverScript} + pylint --errors-only ${testDriverScript} + black --check --diff ${testDriverScript} ''; - }; + + installPhase = + '' + mkdir -p $out/bin + cp ${testDriverScript} $out/bin/nixos-test-driver + chmod u+x $out/bin/nixos-test-driver + # TODO: copy user script part into this file (append) + + wrapProgram $out/bin/nixos-test-driver \ + --prefix PATH : "${lib.makeBinPath [ qemu_pkg vde2 netpbm coreutils ]}" \ + ''; + }; # Run an automated test suite in the given virtual network. # `driver' is the script that runs the network. @@ -71,11 +74,10 @@ rec { { testScript , enableOCR ? false , name ? "unnamed" - # Skip linting (mainly intended for faster dev cycles) + # Skip linting (mainly intended for faster dev cycles) , skipLint ? false , ... } @ t: - let # A standard store path to the vm monitor is built like this: # /tmp/nix-build-vm-test-run-$name.drv-0/vm-state-machine/monitor @@ -94,7 +96,7 @@ rec { # interactively with the specified network, and for starting the # VMs from the command line. mkDriver = qemu_pkg: - let + let build-vms = import ./build-vms.nix { inherit system pkgs minimal specialArgs; extraConfigurations = extraConfigurations ++ (pkgs.lib.optional (qemu_pkg != null) @@ -108,7 +110,8 @@ rec { testDriver = mkTestDriver (if qemu_pkg == null then pkgs.qemu_test else qemu_pkg); nodes = build-vms.buildVirtualNetwork ( - t.nodes or (if t ? machine then { machine = t.machine; } else { })); + t.nodes or (if t ? machine then { machine = t.machine; } else { }) + ); vlans = map (m: m.config.virtualisation.vlans) (lib.attrValues nodes); vms = map (m: m.config.system.build.vm) (lib.attrValues nodes); @@ -120,49 +123,51 @@ rec { testDriverName = with builtins; if testNameLen > maxTestNameLen then - abort ("The name of the test '${name}' must not be longer than ${toString maxTestNameLen} " + - "it's currently ${toString testNameLen} characters long.") + abort + ("The name of the test '${name}' must not be longer than ${toString maxTestNameLen} " + + "it's currently ${toString testNameLen} characters long.") else "nixos-test-driver-${name}"; warn = if skipLint then lib.warn "Linting is disabled!" else lib.id; in warn (runCommand testDriverName - { buildInputs = [ makeWrapper]; - testScript = testScript'; - preferLocalBuild = true; - testName = name; - passthru = { - inherit nodes; - }; - } - '' - mkdir -p $out/bin + { + buildInputs = [ makeWrapper ]; + testScript = testScript'; + preferLocalBuild = true; + testName = name; + passthru = { + inherit nodes; + }; + } + '' + mkdir -p $out/bin - echo -n "$testScript" > $out/test-script - ${lib.optionalString (!skipLint) '' - ${python3Packages.black}/bin/black --check --diff $out/test-script - ''} + echo -n "$testScript" > $out/test-script + ${lib.optionalString (!skipLint) '' + ${python3Packages.black}/bin/black --check --diff $out/test-script + ''} - ln -s ${testDriver}/bin/nixos-test-driver $out/bin/ - vms=($(for i in ${toString vms}; do echo $i/bin/run-*-vm; done)) - wrapProgram $out/bin/nixos-test-driver \ - --add-flags "''${vms[*]}" \ - ${lib.optionalString enableOCR - "--prefix PATH : '${ocrProg}/bin:${imagemagick_tiff}/bin'"} \ - --run "export testScript=\"\$(${coreutils}/bin/cat $out/test-script)\"" \ - --set VLANS '${toString vlans}' - ln -s ${testDriver}/bin/nixos-test-driver $out/bin/nixos-run-vms - wrapProgram $out/bin/nixos-run-vms \ - --add-flags "''${vms[*]}" \ - ${lib.optionalString enableOCR "--prefix PATH : '${ocrProg}/bin'"} \ - --set tests 'start_all(); join_all();' \ - --set VLANS '${toString vlans}' \ - ${lib.optionalString (builtins.length vms == 1) "--set USE_SERIAL 1"} - ''); # " + ln -s ${testDriver}/bin/nixos-test-driver $out/bin/ + vms=($(for i in ${toString vms}; do echo $i/bin/run-*-vm; done)) + wrapProgram $out/bin/nixos-test-driver \ + --add-flags "''${vms[*]}" \ + ${lib.optionalString enableOCR + "--prefix PATH : '${ocrProg}/bin:${imagemagick_tiff}/bin'"} \ + --run "export testScript=\"\$(${coreutils}/bin/cat $out/test-script)\"" \ + --set VLANS '${toString vlans}' + ln -s ${testDriver}/bin/nixos-test-driver $out/bin/nixos-run-vms + wrapProgram $out/bin/nixos-run-vms \ + --add-flags "''${vms[*]}" \ + ${lib.optionalString enableOCR "--prefix PATH : '${ocrProg}/bin'"} \ + --set tests 'start_all(); join_all();' \ + --set VLANS '${toString vlans}' \ + ${lib.optionalString (builtins.length vms == 1) "--set USE_SERIAL 1"} + ''); # " passMeta = drv: drv // lib.optionalAttrs (t ? meta) { - meta = (drv.meta or {}) // t.meta; + meta = (drv.meta or { }) // t.meta; }; driver = mkDriver null; @@ -172,22 +177,23 @@ rec { nodeNames = builtins.attrNames driver.nodes; invalidNodeNames = lib.filter - (node: builtins.match "^[A-z_]([A-z0-9_]+)?$" node == null) nodeNames; + (node: builtins.match "^[A-z_]([A-z0-9_]+)?$" node == null) + nodeNames; in - if lib.length invalidNodeNames > 0 then - throw '' - Cannot create machines out of (${lib.concatStringsSep ", " invalidNodeNames})! - All machines are referenced as python variables in the testing framework which will break the - script when special characters are used. + if lib.length invalidNodeNames > 0 then + throw '' + Cannot create machines out of (${lib.concatStringsSep ", " invalidNodeNames})! + All machines are referenced as python variables in the testing framework which will break the + script when special characters are used. - Please stick to alphanumeric chars and underscores as separation. - '' - else + Please stick to alphanumeric chars and underscores as separation. + '' + else test // { - inherit test driver driverInteractive; - inherit (test) nodes; - }; + inherit test driver driverInteractive; + inherit (test) nodes; + }; runInMachine = { drv @@ -202,8 +208,10 @@ rec { }; vm = build-vms.buildVM { } - [ machine - { key = "run-in-machine"; + [ + machine + { + key = "run-in-machine"; networking.hostName = "client"; nix.readOnlyStore = false; virtualisation.writableStore = false; @@ -250,16 +258,16 @@ rec { ''; # */ in - lib.overrideDerivation drv (attrs: { - requiredSystemFeatures = [ "kvm" ]; - builder = "${bash}/bin/sh"; - args = ["-e" vmRunCommand]; - origArgs = attrs.args; - origBuilder = attrs.builder; - }); + lib.overrideDerivation drv (attrs: { + requiredSystemFeatures = [ "kvm" ]; + builder = "${bash}/bin/sh"; + args = [ "-e" vmRunCommand ]; + origArgs = attrs.args; + origBuilder = attrs.builder; + }); - runInMachineWithX = { require ? [], ... } @ args: + runInMachineWithX = { require ? [ ], ... } @ args: let client = { ... }: @@ -275,13 +283,13 @@ rec { services.xserver.windowManager.icewm.enable = true; }; in - runInMachine ({ - machine = client; - preBuild = - '' - client.wait_for_x() - ''; - } // args); + runInMachine ({ + machine = client; + preBuild = + '' + client.wait_for_x() + ''; + } // args); simpleTest = as: (makeTest as).test; From fa25d84d1330ee66e9df41d4e5dd8e85d8b506a0 Mon Sep 17 00:00:00 2001 From: Andreas Rammhold Date: Sat, 24 Oct 2020 21:28:38 +0200 Subject: [PATCH 3/5] nixos/tests: fix testDriver reference in runInMachine function In a previous commit I broke this as there is no longer one testDriver but only a function to generate one based on some QEMU inputs. --- nixos/lib/testing-python.nix | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nixos/lib/testing-python.nix b/nixos/lib/testing-python.nix index 2a2a14bab7a..d78193c2600 100644 --- a/nixos/lib/testing-python.nix +++ b/nixos/lib/testing-python.nix @@ -200,6 +200,7 @@ rec { , machine , preBuild ? "" , postBuild ? "" + , qemu ? pkgs.qemu_test , ... # ??? }: let @@ -254,7 +255,7 @@ rec { unset xchg export tests='${testScript}' - ${testDriver}/bin/nixos-test-driver ${vm.config.system.build.vm}/bin/run-*-vm + ${mkTestDriver qemu}/bin/nixos-test-driver ${vm.config.system.build.vm}/bin/run-*-vm ''; # */ in From 73635b859d75a0789623969adef9e46cc4b34e23 Mon Sep 17 00:00:00 2001 From: Andreas Rammhold Date: Sat, 24 Oct 2020 22:17:32 +0200 Subject: [PATCH 4/5] nixos/tests: fix runInMachine In 5500dc8 we introduced the --keep-vm-state flag and defaulted to that flag not being set. This lead to the `runInMachine` tests not longer working and that going unnoticed for quite some time now. --- nixos/lib/testing-python.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nixos/lib/testing-python.nix b/nixos/lib/testing-python.nix index d78193c2600..8dad4d53516 100644 --- a/nixos/lib/testing-python.nix +++ b/nixos/lib/testing-python.nix @@ -255,7 +255,7 @@ rec { unset xchg export tests='${testScript}' - ${mkTestDriver qemu}/bin/nixos-test-driver ${vm.config.system.build.vm}/bin/run-*-vm + ${mkTestDriver qemu}/bin/nixos-test-driver --keep-vm-state ${vm.config.system.build.vm}/bin/run-*-vm ''; # */ in From d4fb7daafdc694d01befebe7d941bfd729568ccb Mon Sep 17 00:00:00 2001 From: Andreas Rammhold Date: Sun, 25 Oct 2020 00:59:59 +0200 Subject: [PATCH 5/5] nixos-build-vms: use the driverInteractive attribute instead This reverts commit aab534b894c60e69199aba4cc57b8b3f8fd1a8c2 & uses the driverInteractive attribute for the test driver instead. This has the same effect but removes the extra module in the nixos-build-vms code. --- .../installer/tools/nixos-build-vms/build-vms.nix | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/nixos/modules/installer/tools/nixos-build-vms/build-vms.nix b/nixos/modules/installer/tools/nixos-build-vms/build-vms.nix index 199e5f9206b..e49ceba2424 100644 --- a/nixos/modules/installer/tools/nixos-build-vms/build-vms.nix +++ b/nixos/modules/installer/tools/nixos-build-vms/build-vms.nix @@ -6,12 +6,7 @@ let nodes = builtins.mapAttrs (vm: module: { _file = "${networkExpr}@node-${vm}"; - imports = [ - module - ({ pkgs, ... }: { - virtualisation.qemu.package = pkgs.qemu; - }) - ]; + imports = [ module ]; }) (import networkExpr); in @@ -20,4 +15,4 @@ with import ../../../../lib/testing-python.nix { pkgs = import ../../../../.. { inherit system config; }; }; -(makeTest { inherit nodes; testScript = ""; }).driver +(makeTest { inherit nodes; testScript = ""; }).driverInteractive