b12f/nixpkgs
1
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

Merge pull request #101598 from andir/nixos-build-vms-qemu

Browse source 
nixos/tests: follow-up to the closure reduction PR
This commit is contained in:
Andreas Rammhold 2020-10-26 14:19:45 +01:00 committed by GitHub
parent 4c07f77124 d4fb7daafd
commit 1088f05940
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 146 additions and 125 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

262
nixos/lib/testing-python.nix
View file

@ -3,13 +3,13 @@
# Use a minimal kernel? # Use a minimal kernel?
, minimal ? false , minimal ? false
# Ignored # Ignored
, config ? {} , config ? { }
# !!! See comment about args in lib/modules.nix # !!! See comment about args in lib/modules.nix
, specialArgs ? {} , specialArgs ? { }
# Modules to add to each VM # Modules to add to each VM
, extraConfigurations ? [] }: , extraConfigurations ? [ ]
}:
with import ./build-vms.nix { inherit system pkgs minimal specialArgs extraConfigurations; };
with pkgs; with pkgs;
rec { rec {
@ -17,42 +17,41 @@ rec {
inherit pkgs; inherit pkgs;
mkTestDriver = let mkTestDriver =
testDriverScript = ./test-driver/test-driver.py; let
in qemu_pkg: stdenv.mkDerivation { testDriverScript = ./test-driver/test-driver.py;
name = "nixos-test-driver"; in
qemu_pkg: stdenv.mkDerivation {
name = "nixos-test-driver";
nativeBuildInputs = [ makeWrapper ]; nativeBuildInputs = [ makeWrapper ];
buildInputs = [ (python3.withPackages (p: [ p.ptpython ])) ]; buildInputs = [ (python3.withPackages (p: [ p.ptpython ])) ];
checkInputs = with python3Packages; [ pylint black mypy ]; checkInputs = with python3Packages; [ pylint black mypy ];
dontUnpack = true; dontUnpack = true;
preferLocalBuild = true; preferLocalBuild = true;
doCheck = true; doCheck = true;
checkPhase = '' checkPhase = ''
mypy --disallow-untyped-defs \ mypy --disallow-untyped-defs \
--no-implicit-optional \ --no-implicit-optional \
--ignore-missing-imports ${testDriverScript} --ignore-missing-imports ${testDriverScript}
pylint --errors-only ${testDriverScript} pylint --errors-only ${testDriverScript}
black --check --diff ${testDriverScript} black --check --diff ${testDriverScript}
'';
installPhase =
''
mkdir -p $out/bin
cp ${testDriverScript} $out/bin/nixos-test-driver
chmod u+x $out/bin/nixos-test-driver
# TODO: copy user script part into this file (append)
wrapProgram $out/bin/nixos-test-driver \
--prefix PATH : "${lib.makeBinPath [ qemu_pkg vde2 netpbm coreutils ]}" \
''; '';
};
testDriver = mkTestDriver qemu_test; installPhase =
testDriverInteractive = mkTestDriver qemu_kvm; ''
mkdir -p $out/bin
cp ${testDriverScript} $out/bin/nixos-test-driver
chmod u+x $out/bin/nixos-test-driver
# TODO: copy user script part into this file (append)
wrapProgram $out/bin/nixos-test-driver \
--prefix PATH : "${lib.makeBinPath [ qemu_pkg vde2 netpbm coreutils ]}" \
'';
};
# Run an automated test suite in the given virtual network. # Run an automated test suite in the given virtual network.
# `driver' is the script that runs the network. # `driver' is the script that runs the network.
@ -75,11 +74,10 @@ rec {
{ testScript { testScript
, enableOCR ? false , enableOCR ? false
, name ? "unnamed" , name ? "unnamed"
# Skip linting (mainly intended for faster dev cycles) # Skip linting (mainly intended for faster dev cycles)
, skipLint ? false , skipLint ? false
, ... , ...
} @ t: } @ t:
let let
# A standard store path to the vm monitor is built like this: # A standard store path to the vm monitor is built like this:
# /tmp/nix-build-vm-test-run-$name.drv-0/vm-state-machine/monitor # /tmp/nix-build-vm-test-run-$name.drv-0/vm-state-machine/monitor
@ -88,25 +86,7 @@ rec {
maxTestNameLen = 50; maxTestNameLen = 50;
testNameLen = builtins.stringLength name; testNameLen = builtins.stringLength name;
testDriverName = with builtins;
if testNameLen > maxTestNameLen then
abort ("The name of the test '${name}' must not be longer than ${toString maxTestNameLen} " +
"it's currently ${toString testNameLen} characters long.")
else
"nixos-test-driver-${name}";
nodes = buildVirtualNetwork (
t.nodes or (if t ? machine then { machine = t.machine; } else { }));
testScript' =
# Call the test script with the computed nodes.
if lib.isFunction testScript
then testScript { inherit nodes; }
else testScript;
vlans = map (m: m.config.virtualisation.vlans) (lib.attrValues nodes);
vms = map (m: m.config.system.build.vm) (lib.attrValues nodes);
ocrProg = tesseract4.override { enableLanguages = [ "eng" ]; }; ocrProg = tesseract4.override { enableLanguages = [ "eng" ]; };
@ -115,78 +95,124 @@ rec {
# Generate convenience wrappers for running the test driver # Generate convenience wrappers for running the test driver
# interactively with the specified network, and for starting the # interactively with the specified network, and for starting the
# VMs from the command line. # VMs from the command line.
driver = testDriver: mkDriver = qemu_pkg:
let let
build-vms = import ./build-vms.nix {
inherit system pkgs minimal specialArgs;
extraConfigurations = extraConfigurations ++ (pkgs.lib.optional (qemu_pkg != null)
{
virtualisation.qemu.package = qemu_pkg;
}
);
};
# FIXME: get this pkg from the module system
testDriver = mkTestDriver (if qemu_pkg == null then pkgs.qemu_test else qemu_pkg);
nodes = build-vms.buildVirtualNetwork (
t.nodes or (if t ? machine then { machine = t.machine; } else { })
);
vlans = map (m: m.config.virtualisation.vlans) (lib.attrValues nodes);
vms = map (m: m.config.system.build.vm) (lib.attrValues nodes);
testScript' =
# Call the test script with the computed nodes.
if lib.isFunction testScript
then testScript { inherit nodes; }
else testScript;
testDriverName = with builtins;
if testNameLen > maxTestNameLen then
abort
("The name of the test '${name}' must not be longer than ${toString maxTestNameLen} " +
"it's currently ${toString testNameLen} characters long.")
else
"nixos-test-driver-${name}";
warn = if skipLint then lib.warn "Linting is disabled!" else lib.id; warn = if skipLint then lib.warn "Linting is disabled!" else lib.id;
in in
warn (runCommand testDriverName warn (runCommand testDriverName
{ buildInputs = [ makeWrapper]; {
testScript = testScript'; buildInputs = [ makeWrapper ];
preferLocalBuild = true; testScript = testScript';
testName = name; preferLocalBuild = true;
} testName = name;
'' passthru = {
mkdir -p $out/bin inherit nodes;
};
}
''
mkdir -p $out/bin
echo -n "$testScript" > $out/test-script echo -n "$testScript" > $out/test-script
${lib.optionalString (!skipLint) '' ${lib.optionalString (!skipLint) ''
${python3Packages.black}/bin/black --check --diff $out/test-script ${python3Packages.black}/bin/black --check --diff $out/test-script
''} ''}
ln -s ${testDriver}/bin/nixos-test-driver $out/bin/ ln -s ${testDriver}/bin/nixos-test-driver $out/bin/
vms=($(for i in ${toString vms}; do echo $i/bin/run-*-vm; done)) vms=($(for i in ${toString vms}; do echo $i/bin/run-*-vm; done))
wrapProgram $out/bin/nixos-test-driver \ wrapProgram $out/bin/nixos-test-driver \
--add-flags "''${vms[*]}" \ --add-flags "''${vms[*]}" \
${lib.optionalString enableOCR ${lib.optionalString enableOCR
"--prefix PATH : '${ocrProg}/bin:${imagemagick_tiff}/bin'"} \ "--prefix PATH : '${ocrProg}/bin:${imagemagick_tiff}/bin'"} \
--run "export testScript=\"\$(${coreutils}/bin/cat $out/test-script)\"" \ --run "export testScript=\"\$(${coreutils}/bin/cat $out/test-script)\"" \
--set VLANS '${toString vlans}' --set VLANS '${toString vlans}'
ln -s ${testDriver}/bin/nixos-test-driver $out/bin/nixos-run-vms ln -s ${testDriver}/bin/nixos-test-driver $out/bin/nixos-run-vms
wrapProgram $out/bin/nixos-run-vms \ wrapProgram $out/bin/nixos-run-vms \
--add-flags "''${vms[*]}" \ --add-flags "''${vms[*]}" \
${lib.optionalString enableOCR "--prefix PATH : '${ocrProg}/bin'"} \ ${lib.optionalString enableOCR "--prefix PATH : '${ocrProg}/bin'"} \
--set tests 'start_all(); join_all();' \ --set tests 'start_all(); join_all();' \
--set VLANS '${toString vlans}' \ --set VLANS '${toString vlans}' \
${lib.optionalString (builtins.length vms == 1) "--set USE_SERIAL 1"} ${lib.optionalString (builtins.length vms == 1) "--set USE_SERIAL 1"}
''); # " ''); # "
passMeta = drv: drv // lib.optionalAttrs (t ? meta) { passMeta = drv: drv // lib.optionalAttrs (t ? meta) {
meta = (drv.meta or {}) // t.meta; meta = (drv.meta or { }) // t.meta;
}; };
test = passMeta (runTests (driver testDriver)); driver = mkDriver null;
driverInteractive = mkDriver pkgs.qemu;
nodeNames = builtins.attrNames nodes; test = passMeta (runTests driver);
nodeNames = builtins.attrNames driver.nodes;
invalidNodeNames = lib.filter invalidNodeNames = lib.filter
(node: builtins.match "^[A-z_]([A-z0-9_]+)?$" node == null) nodeNames; (node: builtins.match "^[A-z_]([A-z0-9_]+)?$" node == null)
nodeNames;
in in
if lib.length invalidNodeNames > 0 then if lib.length invalidNodeNames > 0 then
throw '' throw ''
Cannot create machines out of (${lib.concatStringsSep ", " invalidNodeNames})! Cannot create machines out of (${lib.concatStringsSep ", " invalidNodeNames})!
All machines are referenced as python variables in the testing framework which will break the All machines are referenced as python variables in the testing framework which will break the
script when special characters are used. script when special characters are used.
Please stick to alphanumeric chars and underscores as separation. Please stick to alphanumeric chars and underscores as separation.
'' ''
else else
test // { test // {
inherit nodes test; inherit test driver driverInteractive;
driver = driver testDriver; inherit (test) nodes;
driverInteractive = driver testDriverInteractive; };
};
runInMachine = runInMachine =
{ drv { drv
, machine , machine
, preBuild ? "" , preBuild ? ""
, postBuild ? "" , postBuild ? ""
, qemu ? pkgs.qemu_test
, ... # ??? , ... # ???
}: }:
let let
vm = buildVM { } build-vms = import ./build-vms.nix {
[ machine inherit system pkgs minimal specialArgs extraConfigurations;
{ key = "run-in-machine"; };
vm = build-vms.buildVM { }
[
machine
{
key = "run-in-machine";
networking.hostName = "client"; networking.hostName = "client";
nix.readOnlyStore = false; nix.readOnlyStore = false;
virtualisation.writableStore = false; virtualisation.writableStore = false;
@ -229,20 +255,20 @@ rec {
unset xchg unset xchg
export tests='${testScript}' export tests='${testScript}'
${testDriver}/bin/nixos-test-driver ${vm.config.system.build.vm}/bin/run-*-vm ${mkTestDriver qemu}/bin/nixos-test-driver --keep-vm-state ${vm.config.system.build.vm}/bin/run-*-vm
''; # */ ''; # */
in in
lib.overrideDerivation drv (attrs: { lib.overrideDerivation drv (attrs: {
requiredSystemFeatures = [ "kvm" ]; requiredSystemFeatures = [ "kvm" ];
builder = "${bash}/bin/sh"; builder = "${bash}/bin/sh";
args = ["-e" vmRunCommand]; args = [ "-e" vmRunCommand ];
origArgs = attrs.args; origArgs = attrs.args;
origBuilder = attrs.builder; origBuilder = attrs.builder;
}); });
runInMachineWithX = { require ? [], ... } @ args: runInMachineWithX = { require ? [ ], ... } @ args:
let let
client = client =
{ ... }: { ... }:
@ -258,13 +284,13 @@ rec {
services.xserver.windowManager.icewm.enable = true; services.xserver.windowManager.icewm.enable = true;
}; };
in in
runInMachine ({ runInMachine ({
machine = client; machine = client;
preBuild = preBuild =
'' ''
client.wait_for_x() client.wait_for_x()
''; '';
} // args); } // args);
simpleTest = as: (makeTest as).test; simpleTest = as: (makeTest as).test;

9
nixos/modules/installer/tools/nixos-build-vms/build-vms.nix
View file

@ -6,12 +6,7 @@
let let
nodes = builtins.mapAttrs (vm: module: { nodes = builtins.mapAttrs (vm: module: {
_file = "${networkExpr}@node-${vm}"; _file = "${networkExpr}@node-${vm}";
imports = [ imports = [ module ];
module
({ pkgs, ... }: {
virtualisation.qemu.package = pkgs.qemu;
})
];
}) (import networkExpr); }) (import networkExpr);
in in
@ -20,4 +15,4 @@ with import ../../../../lib/testing-python.nix {
pkgs = import ../../../../.. { inherit system config; }; pkgs = import ../../../../.. { inherit system config; };
}; };
(makeTest { inherit nodes; testScript = ""; }).driver (makeTest { inherit nodes; testScript = ""; }).driverInteractive