nixos/testing-python.nix: Add evalTest

This is a decomposition of the testing-python.nix and build-vms.nix
files into modules.

By refactoring the glue, we accomplish the following:

 - NixOS tests can now use `imports` and other module system features.
    - Network-wide test setup can now be reusable; example:
       - A setup with all VMs configured to use a DNS server
       - Split long, slow tests into multiple tests that import a
         common module that has most of the setup.
    - Type checking for the test arguments
    - (TBD) "generated" options reference docs
 - Aspects that had to be wired through all the glue are now in their
   own files.
    - Chief example: interactive.nix.
    - Also: network.nix

In rewriting this, I've generally stuck as close as possible to the
existing code; copying pieces of logic and rewiring them, without
changing the logic itself.

I've made two exceptions to this rule

 - Introduction of `extraDriverArgs` instead of hardcoded
   interactivity logic.

 - Incorporation of https://github.com/NixOS/nixpkgs/pull/144110
   in testScript.nix.

I might revert the latter and split it into a new commit.

nixos/lib/testing-python.nix

@@ -166,6 +166,26 @@ rec {
           ${lib.optionalString (interactive) "--add-flags --interactive"}
       '');
 
+  evalTest = module: lib.evalModules { modules = testModules ++ [ module ]; };
+  runTest = module: (evalTest module).config.run;
+
+  testModules = [
+    ./testing/driver.nix
+    ./testing/interactive.nix
+    ./testing/legacy.nix
+    ./testing/meta.nix
+    ./testing/name.nix
+    ./testing/network.nix
+    ./testing/nodes.nix
+    ./testing/run.nix
+    ./testing/testScript.nix
+    {
+      config = {
+        hostPkgs = pkgs;
+      };
+    }
+  ];
+
   # Make a full-blown test
   makeTest =
     { machine ? null

nixos/lib/testing/call-test.nix

@@ -0,0 +1,16 @@
+{ config, lib, ... }:
+let
+  inherit (lib) mkOption types;
+in
+{
+  options = {
+    callTest = mkOption {
+      internal = true;
+      type = types.functionTo types.raw;
+    };
+    result = mkOption {
+      internal = true;
+      default = config;
+    };
+  };
+}

nixos/lib/testing/driver.nix

@@ -0,0 +1,177 @@
+{ config, lib, hostPkgs, ... }:
+let
+  inherit (lib) mkOption types;
+
+  # Reifies and correctly wraps the python test driver for
+  # the respective qemu version and with or without ocr support
+  testDriver = hostPkgs.callPackage ../test-driver {
+    inherit (config) enableOCR extraPythonPackages;
+    qemu_pkg = config.qemu.package;
+    imagemagick_light = hostPkgs.imagemagick_light.override { inherit (hostPkgs) libtiff; };
+    tesseract4 = hostPkgs.tesseract4.override { enableLanguages = [ "eng" ]; };
+  };
+
+
+  vlans = map (m: m.virtualisation.vlans) (lib.attrValues config.nodes);
+  vms = map (m: m.system.build.vm) (lib.attrValues config.nodes);
+
+  nodeHostNames =
+    let
+      nodesList = map (c: c.system.name) (lib.attrValues config.nodes);
+    in
+    nodesList ++ lib.optional (lib.length nodesList == 1 && !lib.elem "machine" nodesList) "machine";
+
+  # TODO: This is an implementation error and needs fixing
+  # the testing famework cannot legitimately restrict hostnames further
+  # beyond RFC1035
+  invalidNodeNames = lib.filter
+    (node: builtins.match "^[A-z_]([A-z0-9_]+)?$" node == null)
+    nodeHostNames;
+
+  uniqueVlans = lib.unique (builtins.concatLists vlans);
+  vlanNames = map (i: "vlan${toString i}: VLan;") uniqueVlans;
+  machineNames = map (name: "${name}: Machine;") nodeHostNames;
+
+  withChecks =
+    if lib.length invalidNodeNames > 0 then
+      throw ''
+        Cannot create machines out of (${lib.concatStringsSep ", " invalidNodeNames})!
+        All machines are referenced as python variables in the testing framework which will break the
+        script when special characters are used.
+
+        This is an IMPLEMENTATION ERROR and needs to be fixed. Meanwhile,
+        please stick to alphanumeric chars and underscores as separation.
+      ''
+    else
+      lib.warnIf config.skipLint "Linting is disabled";
+
+  driver =
+    hostPkgs.runCommand "nixos-test-driver-${config.name}"
+      {
+        # inherit testName; TODO (roberth): need this?
+        nativeBuildInputs = [
+          hostPkgs.makeWrapper
+        ] ++ lib.optionals (!config.skipTypeCheck) [ hostPkgs.mypy ];
+        testScript = config.testScriptString;
+        preferLocalBuild = true;
+        passthru = config.passthru;
+        meta = config.meta // {
+          mainProgram = "nixos-test-driver";
+        };
+      }
+      ''
+        mkdir -p $out/bin
+
+        vmStartScripts=($(for i in ${toString vms}; do echo $i/bin/run-*-vm; done))
+
+        ${lib.optionalString (!config.skipTypeCheck) ''
+          # prepend type hints so the test script can be type checked with mypy
+          cat "${../test-script-prepend.py}" >> testScriptWithTypes
+          echo "${builtins.toString machineNames}" >> testScriptWithTypes
+          echo "${builtins.toString vlanNames}" >> testScriptWithTypes
+          echo -n "$testScript" >> testScriptWithTypes
+
+          cat -n testScriptWithTypes
+
+          # set pythonpath so mypy knows where to find the imports. this requires the py.typed file.
+          export PYTHONPATH='${../test-driver}'
+          mypy  --no-implicit-optional \
+                --pretty \
+                --no-color-output \
+                testScriptWithTypes
+          unset PYTHONPATH
+        ''}
+
+        echo -n "$testScript" >> $out/test-script
+
+        ln -s ${testDriver}/bin/nixos-test-driver $out/bin/nixos-test-driver
+
+        ${testDriver}/bin/generate-driver-symbols
+        ${lib.optionalString (!config.skipLint) ''
+          PYFLAKES_BUILTINS="$(
+            echo -n ${lib.escapeShellArg (lib.concatStringsSep "," nodeHostNames)},
+            < ${lib.escapeShellArg "driver-symbols"}
+          )" ${hostPkgs.python3Packages.pyflakes}/bin/pyflakes $out/test-script
+        ''}
+
+        # set defaults through environment
+        # see: ./test-driver/test-driver.py argparse implementation
+        wrapProgram $out/bin/nixos-test-driver \
+          --set startScripts "''${vmStartScripts[*]}" \
+          --set testScript "$out/test-script" \
+          --set vlans '${toString vlans}' \
+          ${lib.escapeShellArgs (lib.concatMap (arg: ["--add-flags" arg]) config.extraDriverArgs)}
+      '';
+
+in
+{
+  options = {
+
+    driver = mkOption {
+      description = "Script that runs the test.";
+      type = types.package;
+      defaultText = lib.literalDocBook "set by the test framework";
+    };
+
+    hostPkgs = mkOption {
+      description = "Nixpkgs attrset used outside the nodes.";
+      type = types.raw;
+      example = lib.literalExpression ''
+        import nixpkgs { inherit system config overlays; }
+      '';
+    };
+
+    qemu.package = mkOption {
+      description = "Which qemu package to use.";
+      type = types.package;
+      default = hostPkgs.qemu_test;
+      defaultText = "hostPkgs.qemu_test";
+    };
+
+    enableOCR = mkOption {
+      description = ''
+        Whether to enable Optical Character Recognition functionality for
+        testing graphical programs.
+      '';
+      type = types.bool;
+      default = false;
+    };
+
+    extraPythonPackages = mkOption {
+      description = ''
+        Python packages to add to the test driver.
+
+        The argument is a Python package set, similar to `pkgs.pythonPackages`.
+      '';
+      type = types.functionTo (types.listOf types.package);
+      default = ps: [ ];
+    };
+
+    extraDriverArgs = mkOption {
+      description = ''
+        Extra arguments to pass to the test driver.
+      '';
+      type = types.listOf types.str;
+      default = [];
+    };
+
+    skipLint = mkOption {
+      type = types.bool;
+      default = false;
+    };
+
+    skipTypeCheck = mkOption {
+      type = types.bool;
+      default = false;
+    };
+  };
+
+  config = {
+    _module.args.hostPkgs = config.hostPkgs;
+
+    driver = withChecks driver;
+
+    # make available on the test runner
+    passthru.driver = config.driver;
+  };
+}

nixos/lib/testing/interactive.nix

@@ -0,0 +1,18 @@
+{ config, lib, moduleType, hostPkgs, ... }:
+let
+  inherit (lib) mkOption types;
+in
+{
+  options = {
+    interactive = mkOption {
+      description = "All the same options, but configured for interactive use.";
+      type = moduleType;
+    };
+  };
+
+  config = {
+    interactive.qemu.package = hostPkgs.qemu;
+    interactive.extraDriverArgs = [ "--interactive" ];
+    passthru.driverInteractive = config.interactive.driver;
+  };
+}

nixos/lib/testing/legacy.nix

@@ -0,0 +1,25 @@
+{ config, options, lib, ... }:
+let
+  inherit (lib) mkIf mkOption types;
+in
+{
+  # This needs options.warnings, which we don't have (yet?).
+  # imports = [
+  #   (lib.mkRenamedOptionModule [ "machine" ] [ "nodes" "machine" ])
+  # ];
+
+  options = {
+    machine = mkOption {
+      internal = true;
+      type = types.raw;
+    };
+  };
+
+  config = {
+    nodes = mkIf options.machine.isDefined (
+      lib.warn
+        "In test `${config.name}': The `machine' attribute in NixOS tests (pkgs.nixosTest / make-test-python.nix / testing-python.nix / makeTest) is deprecated. Please set the equivalent `nodes.machine'."
+        { inherit (config) machine; }
+    );
+  };
+}

nixos/lib/testing/meta.nix

@@ -0,0 +1,12 @@
+{ lib, ... }:
+let
+  inherit (lib) types mkOption;
+in
+{
+  options = {
+    meta.maintainers = lib.mkOption {
+      type = types.listOf types.raw;
+      default = [];
+    };
+  };
+}

nixos/lib/testing/name.nix

@@ -0,0 +1,7 @@
+{ lib, ... }:
+{
+  options.name = lib.mkOption {
+    description = "The name of the test.";
+    type = lib.types.str;
+  };
+}

nixos/lib/testing/network.nix

@@ -0,0 +1,75 @@
+{ lib, nodes, ... }:
+
+with lib;
+
+
+let
+  machines = attrNames nodes;
+
+  machinesNumbered = zipLists machines (range 1 254);
+
+  nodes_ = forEach machinesNumbered (m: nameValuePair m.fst
+    [
+      ({ config, nodes, pkgs, ... }:
+        let
+          interfacesNumbered = zipLists config.virtualisation.vlans (range 1 255);
+          interfaces = forEach interfacesNumbered ({ fst, snd }:
+            nameValuePair "eth${toString snd}" {
+              ipv4.addresses =
+                [{
+                  address = "192.168.${toString fst}.${toString m.snd}";
+                  prefixLength = 24;
+                }];
+            });
+
+          networkConfig =
+            {
+              networking.hostName = mkDefault m.fst;
+
+              networking.interfaces = listToAttrs interfaces;
+
+              networking.primaryIPAddress =
+                optionalString (interfaces != [ ]) (head (head interfaces).value.ipv4.addresses).address;
+
+              # Put the IP addresses of all VMs in this machine's
+              # /etc/hosts file.  If a machine has multiple
+              # interfaces, use the IP address corresponding to
+              # the first interface (i.e. the first network in its
+              # virtualisation.vlans option).
+              networking.extraHosts = flip concatMapStrings machines
+                (m':
+                  let config = (getAttr m' nodes).config; in
+                  optionalString (config.networking.primaryIPAddress != "")
+                    ("${config.networking.primaryIPAddress} " +
+                      optionalString (config.networking.domain != null)
+                        "${config.networking.hostName}.${config.networking.domain} " +
+                      "${config.networking.hostName}\n"));
+
+              virtualisation.qemu.options =
+                let qemu-common = import ../qemu-common.nix { inherit lib pkgs; };
+                in
+                flip concatMap interfacesNumbered
+                  ({ fst, snd }: qemu-common.qemuNICFlags snd fst m.snd);
+            };
+
+        in
+        {
+          key = "ip-address";
+          config = networkConfig // {
+            # Expose the networkConfig items for tests like nixops
+            # that need to recreate the network config.
+            system.build.networkConfig = networkConfig;
+          };
+        }
+      )
+    ]);
+
+  extraNodeConfigs = lib.listToAttrs nodes_;
+in
+{
+  config = {
+    defaults = { config, name, ... }: {
+      imports = extraNodeConfigs.${name};
+    };
+  };
+}

nixos/lib/testing/nodes.nix

@@ -0,0 +1,91 @@
+testModuleArgs@{ config, lib, hostPkgs, nodes, ... }:
+
+let
+  inherit (lib) mkOption mkForce optional types mapAttrs mkDefault;
+
+  system = hostPkgs.stdenv.hostPlatform.system;
+
+  baseOS =
+    import ../eval-config.nix {
+      inherit system;
+      inherit (config.node) specialArgs;
+      modules = [ config.defaults ];
+      baseModules = (import ../../modules/module-list.nix) ++
+        [
+          ../../modules/virtualisation/qemu-vm.nix
+          ../../modules/testing/test-instrumentation.nix # !!! should only get added for automated test runs
+          { key = "no-manual"; documentation.nixos.enable = false; }
+          {
+            key = "no-revision";
+            # Make the revision metadata constant, in order to avoid needless retesting.
+            # The human version (e.g. 21.05-pre) is left as is, because it is useful
+            # for external modules that test with e.g. testers.nixosTest and rely on that
+            # version number.
+            config.system.nixos.revision = mkForce "constant-nixos-revision";
+          }
+          { key = "nodes"; _module.args.nodes = nodes; }
+
+          ({ config, ... }:
+            {
+              virtualisation.qemu.package = testModuleArgs.config.qemu.package;
+
+              # Ensure we do not use aliases. Ideally this is only set
+              # when the test framework is used by Nixpkgs NixOS tests.
+              nixpkgs.config.allowAliases = false;
+            })
+        ] ++ optional config.minimal ../../modules/testing/minimal-kernel.nix;
+    };
+
+
+in
+
+{
+
+  options = {
+    node.type = mkOption {
+      type = types.raw;
+      default = baseOS.type;
+      internal = true;
+    };
+
+    nodes = mkOption {
+      type = types.lazyAttrsOf config.node.type;
+    };
+
+    defaults = mkOption {
+      description = ''
+        NixOS configuration that is applied to all {option}`nodes`.
+      '';
+      type = types.deferredModule;
+      default = { };
+    };
+
+    node.specialArgs = mkOption {
+      type = types.lazyAttrsOf types.raw;
+      default = { };
+    };
+
+    minimal = mkOption {
+      type = types.bool;
+      default = false;
+    };
+
+    nodesCompat = mkOption {
+      internal = true;
+    };
+  };
+
+  config = {
+    _module.args.nodes = config.nodesCompat;
+    nodesCompat =
+      mapAttrs
+        (name: config: config // {
+          config = lib.warn
+            "Module argument `nodes.${name}.config` is deprecated. Use `nodes.${name}` instead."
+            config;
+        })
+        config.nodes;
+
+    passthru.nodes = config.nodesCompat;
+  };
+}

nixos/lib/testing/run.nix

@@ -0,0 +1,51 @@
+{ config, hostPkgs, lib, ... }:
+let
+  inherit (lib) types mkOption;
+in
+{
+  options = {
+    passthru = mkOption {
+      type = types.lazyAttrsOf types.raw;
+      description = ''
+        Attributes to add to the returned derivations,
+        which are not necessarily part of the build.
+
+        This is a bit like doing `drv // { myAttr = true; }` (which would be lost by `overrideAttrs`).
+        It does not change the actual derivation, but adds the attribute nonetheless, so that
+        consumers of what would be `drv` have more information.
+      '';
+    };
+
+    run = mkOption {
+      type = types.package;
+      description = ''
+        Derivation that runs the test.
+      '';
+    };
+  };
+
+  config = {
+    run = hostPkgs.stdenv.mkDerivation {
+      name = "vm-test-run-${config.name}";
+
+      requiredSystemFeatures = [ "kvm" "nixos-test" ];
+
+      buildCommand =
+        ''
+          mkdir -p $out
+
+          # effectively mute the XMLLogger
+          export LOGFILE=/dev/null
+
+          ${config.driver}/bin/nixos-test-driver -o $out
+        '';
+
+      passthru = config.passthru;
+
+      meta = config.meta;
+    };
+
+    # useful for inspection (debugging / exploration)
+    passthru.config = config;
+  };
+}

nixos/lib/testing/testScript.nix

@@ -0,0 +1,78 @@
+testModuleArgs@{ config, lib, hostPkgs, nodes, moduleType, ... }:
+let
+  inherit (lib) mkOption types;
+  inherit (types) either str functionTo;
+in
+{
+  options = {
+    testScript = mkOption {
+      type = either str (functionTo str);
+    };
+    testScriptString = mkOption {
+      type = str;
+      readOnly = true;
+      internal = true;
+    };
+
+    includeTestScriptReferences = mkOption {
+      type = types.bool;
+      default = true;
+      internal = true;
+    };
+    withoutTestScriptReferences = mkOption {
+      type = moduleType;
+      description = ''
+        A parallel universe where the testScript is invalid and has no references.
+      '';
+    };
+  };
+  config = {
+    withoutTestScriptReferences.includeTestScriptReferences = false;
+    withoutTestScriptReferences.testScript = lib.mkForce "testscript omitted";
+
+    testScriptString =
+      if lib.isFunction config.testScript
+      then
+        config.testScript
+          {
+            nodes =
+              lib.mapAttrs
+                (k: v:
+                  if v.virtualisation.useNixStoreImage
+                  then
+                  # prevent infinite recursion when testScript would
+                  # reference v's toplevel
+                    config.withoutTestScriptReferences.nodesCompat.${k}
+                  else
+                  # reuse memoized config
+                    v
+                )
+                config.nodesCompat;
+          }
+      else config.testScript;
+
+    defaults = { config, name, ... }: {
+      # Make sure all derivations referenced by the test
+      # script are available on the nodes. When the store is
+      # accessed through 9p, this isn't important, since
+      # everything in the store is available to the guest,
+      # but when building a root image it is, as all paths
+      # that should be available to the guest has to be
+      # copied to the image.
+      virtualisation.additionalPaths =
+        lib.optional
+          # A testScript may evaluate nodes, which has caused
+          # infinite recursions. The demand cycle involves:
+          #   testScript -->
+          #   nodes -->
+          #   toplevel -->
+          #   additionalPaths -->
+          #   hasContext testScript' -->
+          #   testScript (ad infinitum)
+          # If we don't need to build an image, we can break this
+          # cycle by short-circuiting when useNixStoreImage is false.
+          (config.virtualisation.useNixStoreImage && builtins.hasContext testModuleArgs.config.testScriptString && testModuleArgs.config.includeTestScriptReferences)
+          (hostPkgs.writeStringReferencesToFile testModuleArgs.config.testScriptString);
+    };
+  };
+}