2019-09-06 07:25:22 +00:00
|
|
|
{ system
|
|
|
|
, pkgs ? import ../.. { inherit system config; }
|
|
|
|
# Use a minimal kernel?
|
|
|
|
, minimal ? false
|
|
|
|
# Ignored
|
2020-10-23 21:13:38 +00:00
|
|
|
, config ? { }
|
2020-06-02 14:27:07 +00:00
|
|
|
# !!! See comment about args in lib/modules.nix
|
2020-10-23 21:13:38 +00:00
|
|
|
, specialArgs ? { }
|
2019-09-06 07:25:22 +00:00
|
|
|
# Modules to add to each VM
|
2020-10-23 21:13:38 +00:00
|
|
|
, extraConfigurations ? [ ]
|
|
|
|
}:
|
2019-09-06 07:25:22 +00:00
|
|
|
|
|
|
|
with pkgs;
|
|
|
|
|
2020-05-14 12:34:50 +00:00
|
|
|
rec {
|
2019-09-06 07:25:22 +00:00
|
|
|
|
|
|
|
inherit pkgs;
|
|
|
|
|
|
|
|
|
2020-10-23 21:13:38 +00:00
|
|
|
mkTestDriver =
|
|
|
|
let
|
|
|
|
testDriverScript = ./test-driver/test-driver.py;
|
|
|
|
in
|
|
|
|
qemu_pkg: stdenv.mkDerivation {
|
|
|
|
name = "nixos-test-driver";
|
2019-09-06 07:25:22 +00:00
|
|
|
|
2020-10-23 21:13:38 +00:00
|
|
|
nativeBuildInputs = [ makeWrapper ];
|
|
|
|
buildInputs = [ (python3.withPackages (p: [ p.ptpython ])) ];
|
|
|
|
checkInputs = with python3Packages; [ pylint black mypy ];
|
2019-09-06 07:25:22 +00:00
|
|
|
|
2020-10-23 21:13:38 +00:00
|
|
|
dontUnpack = true;
|
2019-09-06 07:25:22 +00:00
|
|
|
|
2020-10-23 21:13:38 +00:00
|
|
|
preferLocalBuild = true;
|
2019-09-06 07:25:22 +00:00
|
|
|
|
2020-10-23 21:13:38 +00:00
|
|
|
doCheck = true;
|
|
|
|
checkPhase = ''
|
|
|
|
mypy --disallow-untyped-defs \
|
|
|
|
--no-implicit-optional \
|
|
|
|
--ignore-missing-imports ${testDriverScript}
|
|
|
|
pylint --errors-only ${testDriverScript}
|
|
|
|
black --check --diff ${testDriverScript}
|
|
|
|
'';
|
2019-09-06 07:25:22 +00:00
|
|
|
|
2020-10-23 21:13:38 +00:00
|
|
|
installPhase =
|
|
|
|
''
|
|
|
|
mkdir -p $out/bin
|
|
|
|
cp ${testDriverScript} $out/bin/nixos-test-driver
|
|
|
|
chmod u+x $out/bin/nixos-test-driver
|
|
|
|
# TODO: copy user script part into this file (append)
|
2019-09-06 07:25:22 +00:00
|
|
|
|
2020-10-23 21:13:38 +00:00
|
|
|
wrapProgram $out/bin/nixos-test-driver \
|
|
|
|
--prefix PATH : "${lib.makeBinPath [ qemu_pkg vde2 netpbm coreutils ]}" \
|
|
|
|
'';
|
|
|
|
};
|
2019-09-06 07:25:22 +00:00
|
|
|
|
|
|
|
# Run an automated test suite in the given virtual network.
|
2021-05-03 11:33:41 +00:00
|
|
|
runTests = {
|
|
|
|
# the script that runs the network
|
|
|
|
driver,
|
|
|
|
# a source position in the format of builtins.unsafeGetAttrPos
|
|
|
|
# for meta.position
|
|
|
|
pos,
|
|
|
|
}:
|
2019-09-06 07:25:22 +00:00
|
|
|
stdenv.mkDerivation {
|
|
|
|
name = "vm-test-run-${driver.testName}";
|
|
|
|
|
|
|
|
requiredSystemFeatures = [ "kvm" "nixos-test" ];
|
|
|
|
|
|
|
|
buildCommand =
|
|
|
|
''
|
2020-05-07 13:56:30 +00:00
|
|
|
mkdir -p $out
|
2019-09-06 07:25:22 +00:00
|
|
|
|
2020-08-30 22:42:06 +00:00
|
|
|
LOGFILE=/dev/null tests='exec(os.environ["testScript"])' ${driver}/bin/nixos-test-driver
|
2019-09-06 07:25:22 +00:00
|
|
|
'';
|
2020-12-09 11:59:39 +00:00
|
|
|
|
|
|
|
passthru = driver.passthru;
|
2021-02-09 11:02:35 +00:00
|
|
|
|
|
|
|
inherit pos;
|
2019-09-06 07:25:22 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
|
|
|
|
makeTest =
|
|
|
|
{ testScript
|
|
|
|
, enableOCR ? false
|
|
|
|
, name ? "unnamed"
|
2020-10-23 21:13:38 +00:00
|
|
|
# Skip linting (mainly intended for faster dev cycles)
|
2019-12-22 13:50:08 +00:00
|
|
|
, skipLint ? false
|
2020-12-09 11:59:39 +00:00
|
|
|
, passthru ? {}
|
2021-02-09 11:02:35 +00:00
|
|
|
, # For meta.position
|
|
|
|
pos ? # position used in error messages and for meta.position
|
|
|
|
(if t.meta.description or null != null
|
|
|
|
then builtins.unsafeGetAttrPos "description" t.meta
|
|
|
|
else builtins.unsafeGetAttrPos "testScript" t)
|
2019-09-06 07:25:22 +00:00
|
|
|
, ...
|
|
|
|
} @ t:
|
|
|
|
let
|
|
|
|
# A standard store path to the vm monitor is built like this:
|
|
|
|
# /tmp/nix-build-vm-test-run-$name.drv-0/vm-state-machine/monitor
|
|
|
|
# The max filename length of a unix domain socket is 108 bytes.
|
|
|
|
# This means $name can at most be 50 bytes long.
|
|
|
|
maxTestNameLen = 50;
|
|
|
|
testNameLen = builtins.stringLength name;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
ocrProg = tesseract4.override { enableLanguages = [ "eng" ]; };
|
|
|
|
|
|
|
|
imagemagick_tiff = imagemagick_light.override { inherit libtiff; };
|
|
|
|
|
2020-07-01 03:31:32 +00:00
|
|
|
# Generate convenience wrappers for running the test driver
|
2019-09-06 07:25:22 +00:00
|
|
|
# interactively with the specified network, and for starting the
|
|
|
|
# VMs from the command line.
|
2020-10-23 21:09:18 +00:00
|
|
|
mkDriver = qemu_pkg:
|
2020-10-23 21:13:38 +00:00
|
|
|
let
|
2020-10-23 21:09:18 +00:00
|
|
|
build-vms = import ./build-vms.nix {
|
|
|
|
inherit system pkgs minimal specialArgs;
|
|
|
|
extraConfigurations = extraConfigurations ++ (pkgs.lib.optional (qemu_pkg != null)
|
|
|
|
{
|
|
|
|
virtualisation.qemu.package = qemu_pkg;
|
|
|
|
}
|
|
|
|
);
|
|
|
|
};
|
|
|
|
|
|
|
|
# FIXME: get this pkg from the module system
|
|
|
|
testDriver = mkTestDriver (if qemu_pkg == null then pkgs.qemu_test else qemu_pkg);
|
|
|
|
|
|
|
|
nodes = build-vms.buildVirtualNetwork (
|
2020-10-23 21:13:38 +00:00
|
|
|
t.nodes or (if t ? machine then { machine = t.machine; } else { })
|
|
|
|
);
|
2020-10-23 21:09:18 +00:00
|
|
|
vlans = map (m: m.config.virtualisation.vlans) (lib.attrValues nodes);
|
|
|
|
vms = map (m: m.config.system.build.vm) (lib.attrValues nodes);
|
|
|
|
|
|
|
|
testScript' =
|
|
|
|
# Call the test script with the computed nodes.
|
|
|
|
if lib.isFunction testScript
|
|
|
|
then testScript { inherit nodes; }
|
|
|
|
else testScript;
|
|
|
|
|
|
|
|
testDriverName = with builtins;
|
|
|
|
if testNameLen > maxTestNameLen then
|
2020-10-23 21:13:38 +00:00
|
|
|
abort
|
|
|
|
("The name of the test '${name}' must not be longer than ${toString maxTestNameLen} " +
|
|
|
|
"it's currently ${toString testNameLen} characters long.")
|
2020-10-23 21:09:18 +00:00
|
|
|
else
|
|
|
|
"nixos-test-driver-${name}";
|
2020-10-05 13:52:27 +00:00
|
|
|
in
|
2021-04-27 13:52:15 +00:00
|
|
|
lib.warnIf skipLint "Linting is disabled" (runCommand testDriverName
|
2020-10-23 21:13:38 +00:00
|
|
|
{
|
|
|
|
buildInputs = [ makeWrapper ];
|
|
|
|
testScript = testScript';
|
|
|
|
preferLocalBuild = true;
|
|
|
|
testName = name;
|
2020-12-09 11:59:39 +00:00
|
|
|
passthru = passthru // {
|
2020-10-23 21:13:38 +00:00
|
|
|
inherit nodes;
|
|
|
|
};
|
|
|
|
}
|
|
|
|
''
|
|
|
|
mkdir -p $out/bin
|
|
|
|
|
|
|
|
echo -n "$testScript" > $out/test-script
|
|
|
|
${lib.optionalString (!skipLint) ''
|
nixos/testing: Switch from black to pyflakes
So far, we have used "black" for formatting the test code, which is
rather strict and opinionated and when used inline in Nix expressions it
creates all sorts of trouble.
One of the main annoyances is that when using strings coming from Nix
expressions (eg. store paths or option definitions from NixOS modules),
completely unrelated changes could cause tests to fail, since eg. black
wants lines to be broken.
Another downside of enforcing a certain kind of formatting is that it
makes the Nix expression code inconsistent because we're mixing two
spaces of indentation (common in nixpkgs) with four spaces of
indentation as defined in PEP-8. While this is perfectly fine for
standalone Python files, it really looks ugly and inconsistent IMO when
used within Nix strings.
What we actually want though is a linter that catches problems early on
before actually running the test, because this is *actually* helping in
development because running the actual VM test takes much longer.
This is the reason why I switched from black to pyflakes, because the
latter actually has useful checks, eg. usage of undefined variables,
invalid format arguments, duplicate arguments, shadowed loop vars and
more.
Signed-off-by: aszlig <aszlig@nix.build>
Closes: https://github.com/NixOS/nixpkgs/issues/72964
2021-05-08 14:57:02 +00:00
|
|
|
${python3Packages.pyflakes}/bin/pyflakes $out/test-script
|
2020-10-23 21:13:38 +00:00
|
|
|
''}
|
|
|
|
|
|
|
|
ln -s ${testDriver}/bin/nixos-test-driver $out/bin/
|
|
|
|
vms=($(for i in ${toString vms}; do echo $i/bin/run-*-vm; done))
|
|
|
|
wrapProgram $out/bin/nixos-test-driver \
|
|
|
|
--add-flags "''${vms[*]}" \
|
|
|
|
${lib.optionalString enableOCR
|
|
|
|
"--prefix PATH : '${ocrProg}/bin:${imagemagick_tiff}/bin'"} \
|
|
|
|
--run "export testScript=\"\$(${coreutils}/bin/cat $out/test-script)\"" \
|
|
|
|
--set VLANS '${toString vlans}'
|
|
|
|
ln -s ${testDriver}/bin/nixos-test-driver $out/bin/nixos-run-vms
|
|
|
|
wrapProgram $out/bin/nixos-run-vms \
|
|
|
|
--add-flags "''${vms[*]}" \
|
|
|
|
${lib.optionalString enableOCR "--prefix PATH : '${ocrProg}/bin'"} \
|
|
|
|
--set tests 'start_all(); join_all();' \
|
|
|
|
--set VLANS '${toString vlans}' \
|
|
|
|
${lib.optionalString (builtins.length vms == 1) "--set USE_SERIAL 1"}
|
|
|
|
''); # "
|
2019-09-06 07:25:22 +00:00
|
|
|
|
|
|
|
passMeta = drv: drv // lib.optionalAttrs (t ? meta) {
|
2020-10-23 21:13:38 +00:00
|
|
|
meta = (drv.meta or { }) // t.meta;
|
2019-09-06 07:25:22 +00:00
|
|
|
};
|
|
|
|
|
2020-10-23 21:09:18 +00:00
|
|
|
driver = mkDriver null;
|
|
|
|
driverInteractive = mkDriver pkgs.qemu;
|
|
|
|
|
2021-02-09 11:02:35 +00:00
|
|
|
test = passMeta (runTests { inherit driver pos; });
|
2019-09-06 07:25:22 +00:00
|
|
|
|
2020-10-23 21:09:18 +00:00
|
|
|
nodeNames = builtins.attrNames driver.nodes;
|
2019-09-06 07:25:22 +00:00
|
|
|
invalidNodeNames = lib.filter
|
2020-10-23 21:13:38 +00:00
|
|
|
(node: builtins.match "^[A-z_]([A-z0-9_]+)?$" node == null)
|
|
|
|
nodeNames;
|
2019-09-06 07:25:22 +00:00
|
|
|
|
|
|
|
in
|
2020-10-23 21:13:38 +00:00
|
|
|
if lib.length invalidNodeNames > 0 then
|
|
|
|
throw ''
|
|
|
|
Cannot create machines out of (${lib.concatStringsSep ", " invalidNodeNames})!
|
|
|
|
All machines are referenced as python variables in the testing framework which will break the
|
|
|
|
script when special characters are used.
|
2019-09-06 07:25:22 +00:00
|
|
|
|
2020-10-23 21:13:38 +00:00
|
|
|
Please stick to alphanumeric chars and underscores as separation.
|
|
|
|
''
|
|
|
|
else
|
2020-10-23 21:09:18 +00:00
|
|
|
test // {
|
2020-10-23 21:13:38 +00:00
|
|
|
inherit test driver driverInteractive;
|
nixos/tests: fix wrong inherit that passes on the nodes attrs
The hydra tarball step would fail due to the nodes attribute not being
properly inherited. Since we can't execute all the tests and release
steps locally anymore (thanks to the JSONification and faster hydra
eval) these errors will probably keep in appearing.
This is hopefully the last of those introduced by me test runner
refactoring.
Error was seen on hydra (https://hydra.nixos.org/build/129282411):
> unpacking sources
> unpacking source archive /nix/store/bp95x52h6nv3j8apxrryyj2rviw682k1-source
> source root is source
> patching sources
> autoconfPhase
> No bootstrap, bootstrap.sh, configure.in or configure.ac. Assuming this is not an GNU Autotools package.
> configuring
> release name is nixpkgs-21.03pre249116.1088f059401
> git-revision is 1088f059401c43450ed85e4a4ec3f57dc8aa992f
> building
> no Makefile, doing nothing
> running tests
> warning: you did not specify '--add-root'; the result might be removed by the garbage collector
> warning: you did not specify '--add-root'; the result might be removed by the garbage collector
> checking Nixpkgs on i686-linux
> checking Nixpkgs on x86_64-linux
> checking Nixpkgs on x86_64-darwin
> checking eval-release.nix
> trace: `mkStrict' is obsolete; use `mkOverride 0' instead.
> trace: `lib.nixpkgsVersion` is deprecated, use `lib.version` instead!
> trace: warning: lib.readPathsFromFile is deprecated, use a list instead
> trace: Warning: `showVal` is deprecated and will be removed in the next release, please use `traceSeqN`
> trace: lib.zip is deprecated, use lib.zipAttrsWith instead
> checking find-tarballs.nix
> trace: `mkStrict' is obsolete; use `mkOverride 0' instead.
> trace: `lib.nixpkgsVersion` is deprecated, use `lib.version` instead!
> trace: warning: lib.readPathsFromFile is deprecated, use a list instead
> trace: Warning: `showVal` is deprecated and will be removed in the next release, please use `traceSeqN`
> trace: lib.zip is deprecated, use lib.zipAttrsWith instead
> error: while evaluating anonymous function at /build/source/maintainers/scripts/find-tarballs.nix:6:1, called from undefined position:
> while evaluating 'operator' at /build/source/maintainers/scripts/find-tarballs.nix:27:16, called from undefined position:
> while evaluating 'immediateDependenciesOf' at /build/source/maintainers/scripts/find-tarballs.nix:39:29, called from /build/source/maintainers/scripts/find-tarballs.nix:27:44:
> while evaluating anonymous function at /build/source/lib/attrsets.nix:234:10, called from undefined position:
> while evaluating anonymous function at /build/source/maintainers/scripts/find-tarballs.nix:40:37, called from /build/source/lib/attrsets.nix:234:16:
> while evaluating 'derivationsIn' at /build/source/maintainers/scripts/find-tarballs.nix:42:19, called from /build/source/maintainers/scripts/find-tarballs.nix:40:40:
> while evaluating 'canEval' at /build/source/maintainers/scripts/find-tarballs.nix:48:13, called from /build/source/maintainers/scripts/find-tarballs.nix:43:9:
> while evaluating the attribute 'nodes' at /build/source/nixos/lib/testing-python.nix:195:23:
> attribute 'nodes' missing, at /build/source/nixos/lib/testing-python.nix:193:16
> build time elapsed: 0m0.122s 0m0.043s 17m51.526s 0m56.668s
> builder for '/nix/store/96rk3c74vrk6m3snm7n6jhis3j640pn4-nixpkgs-tarball-21.03pre249116.1088f059401.drv' failed with exit code 1
2020-10-26 23:10:31 +00:00
|
|
|
inherit (driver) nodes;
|
2020-10-23 21:13:38 +00:00
|
|
|
};
|
2019-09-06 07:25:22 +00:00
|
|
|
|
|
|
|
runInMachine =
|
|
|
|
{ drv
|
|
|
|
, machine
|
|
|
|
, preBuild ? ""
|
|
|
|
, postBuild ? ""
|
2020-10-24 19:28:38 +00:00
|
|
|
, qemu ? pkgs.qemu_test
|
2019-09-06 07:25:22 +00:00
|
|
|
, ... # ???
|
|
|
|
}:
|
|
|
|
let
|
2020-10-23 21:09:18 +00:00
|
|
|
build-vms = import ./build-vms.nix {
|
|
|
|
inherit system pkgs minimal specialArgs extraConfigurations;
|
|
|
|
};
|
|
|
|
|
|
|
|
vm = build-vms.buildVM { }
|
2020-10-23 21:13:38 +00:00
|
|
|
[
|
|
|
|
machine
|
|
|
|
{
|
|
|
|
key = "run-in-machine";
|
2019-09-06 07:25:22 +00:00
|
|
|
networking.hostName = "client";
|
|
|
|
nix.readOnlyStore = false;
|
|
|
|
virtualisation.writableStore = false;
|
|
|
|
}
|
|
|
|
];
|
|
|
|
|
|
|
|
buildrunner = writeText "vm-build" ''
|
|
|
|
source $1
|
|
|
|
|
|
|
|
${coreutils}/bin/mkdir -p $TMPDIR
|
|
|
|
cd $TMPDIR
|
|
|
|
|
|
|
|
exec $origBuilder $origArgs
|
|
|
|
'';
|
|
|
|
|
|
|
|
testScript = ''
|
2020-02-09 21:21:52 +00:00
|
|
|
start_all()
|
|
|
|
client.wait_for_unit("multi-user.target")
|
2019-09-06 07:25:22 +00:00
|
|
|
${preBuild}
|
2020-02-09 21:21:52 +00:00
|
|
|
client.succeed("env -i ${bash}/bin/bash ${buildrunner} /tmp/xchg/saved-env >&2")
|
2019-09-06 07:25:22 +00:00
|
|
|
${postBuild}
|
2020-02-09 21:21:52 +00:00
|
|
|
client.succeed("sync") # flush all data before pulling the plug
|
2019-09-06 07:25:22 +00:00
|
|
|
'';
|
|
|
|
|
|
|
|
vmRunCommand = writeText "vm-run" ''
|
|
|
|
xchg=vm-state-client/xchg
|
|
|
|
${coreutils}/bin/mkdir $out
|
|
|
|
${coreutils}/bin/mkdir -p $xchg
|
|
|
|
|
|
|
|
for i in $passAsFile; do
|
|
|
|
i2=''${i}Path
|
|
|
|
_basename=$(${coreutils}/bin/basename ''${!i2})
|
|
|
|
${coreutils}/bin/cp ''${!i2} $xchg/$_basename
|
|
|
|
eval $i2=/tmp/xchg/$_basename
|
|
|
|
${coreutils}/bin/ls -la $xchg
|
|
|
|
done
|
|
|
|
|
|
|
|
unset i i2 _basename
|
|
|
|
export | ${gnugrep}/bin/grep -v '^xchg=' > $xchg/saved-env
|
|
|
|
unset xchg
|
|
|
|
|
|
|
|
export tests='${testScript}'
|
2020-10-24 20:17:32 +00:00
|
|
|
${mkTestDriver qemu}/bin/nixos-test-driver --keep-vm-state ${vm.config.system.build.vm}/bin/run-*-vm
|
2019-09-06 07:25:22 +00:00
|
|
|
''; # */
|
|
|
|
|
|
|
|
in
|
2020-10-23 21:13:38 +00:00
|
|
|
lib.overrideDerivation drv (attrs: {
|
|
|
|
requiredSystemFeatures = [ "kvm" ];
|
|
|
|
builder = "${bash}/bin/sh";
|
|
|
|
args = [ "-e" vmRunCommand ];
|
|
|
|
origArgs = attrs.args;
|
|
|
|
origBuilder = attrs.builder;
|
|
|
|
});
|
2019-09-06 07:25:22 +00:00
|
|
|
|
|
|
|
|
2020-10-23 21:13:38 +00:00
|
|
|
runInMachineWithX = { require ? [ ], ... } @ args:
|
2019-09-06 07:25:22 +00:00
|
|
|
let
|
|
|
|
client =
|
|
|
|
{ ... }:
|
|
|
|
{
|
|
|
|
inherit require;
|
2020-02-09 21:23:35 +00:00
|
|
|
imports = [
|
|
|
|
../tests/common/auto.nix
|
|
|
|
];
|
2019-09-06 07:25:22 +00:00
|
|
|
virtualisation.memorySize = 1024;
|
|
|
|
services.xserver.enable = true;
|
2020-02-09 21:23:35 +00:00
|
|
|
test-support.displayManager.auto.enable = true;
|
2019-12-10 14:10:30 +00:00
|
|
|
services.xserver.displayManager.defaultSession = "none+icewm";
|
2019-09-06 07:25:22 +00:00
|
|
|
services.xserver.windowManager.icewm.enable = true;
|
|
|
|
};
|
|
|
|
in
|
2020-10-23 21:13:38 +00:00
|
|
|
runInMachine ({
|
|
|
|
machine = client;
|
|
|
|
preBuild =
|
|
|
|
''
|
|
|
|
client.wait_for_x()
|
|
|
|
'';
|
|
|
|
} // args);
|
2019-09-06 07:25:22 +00:00
|
|
|
|
|
|
|
|
|
|
|
simpleTest = as: (makeTest as).test;
|
|
|
|
|
|
|
|
}
|