nixos/tests/test-driver: normalise test driver entrypoint(s)
Previously the driver was configured exclusively through convoluted environment variables. Now the driver's defaults are configured through env variables. Some additional concerns are in the github comments of this PR.
This commit is contained in:
parent
077b2825cd
commit
926fb93968
|
@ -5,7 +5,7 @@ when developing or debugging a test:
|
||||||
|
|
||||||
```ShellSession
|
```ShellSession
|
||||||
$ nix-build nixos/tests/login.nix -A driverInteractive
|
$ nix-build nixos/tests/login.nix -A driverInteractive
|
||||||
$ ./result/bin/nixos-test-driver
|
$ ./result/bin/nixos-test-driver --interactive
|
||||||
starting VDE switch for network 1
|
starting VDE switch for network 1
|
||||||
>
|
>
|
||||||
```
|
```
|
||||||
|
@ -24,20 +24,11 @@ back into the test driver command line upon its completion. This allows
|
||||||
you to inspect the state of the VMs after the test (e.g. to debug the
|
you to inspect the state of the VMs after the test (e.g. to debug the
|
||||||
test script).
|
test script).
|
||||||
|
|
||||||
To just start and experiment with the VMs, run:
|
|
||||||
|
|
||||||
```ShellSession
|
|
||||||
$ nix-build nixos/tests/login.nix -A driverInteractive
|
|
||||||
$ ./result/bin/nixos-run-vms
|
|
||||||
```
|
|
||||||
|
|
||||||
The script `nixos-run-vms` starts the virtual machines defined by test.
|
|
||||||
|
|
||||||
You can re-use the VM states coming from a previous run by setting the
|
You can re-use the VM states coming from a previous run by setting the
|
||||||
`--keep-vm-state` flag.
|
`--keep-vm-state` flag.
|
||||||
|
|
||||||
```ShellSession
|
```ShellSession
|
||||||
$ ./result/bin/nixos-run-vms --keep-vm-state
|
$ ./result/bin/nixos-test-driver --interactive --keep-vm-state
|
||||||
```
|
```
|
||||||
|
|
||||||
The machine state is stored in the `$TMPDIR/vm-state-machinename`
|
The machine state is stored in the `$TMPDIR/vm-state-machinename`
|
||||||
|
|
|
@ -6,7 +6,7 @@
|
||||||
</para>
|
</para>
|
||||||
<programlisting>
|
<programlisting>
|
||||||
$ nix-build nixos/tests/login.nix -A driverInteractive
|
$ nix-build nixos/tests/login.nix -A driverInteractive
|
||||||
$ ./result/bin/nixos-test-driver
|
$ ./result/bin/nixos-test-driver --interactive
|
||||||
starting VDE switch for network 1
|
starting VDE switch for network 1
|
||||||
>
|
>
|
||||||
</programlisting>
|
</programlisting>
|
||||||
|
@ -25,23 +25,12 @@ starting VDE switch for network 1
|
||||||
completion. This allows you to inspect the state of the VMs after
|
completion. This allows you to inspect the state of the VMs after
|
||||||
the test (e.g. to debug the test script).
|
the test (e.g. to debug the test script).
|
||||||
</para>
|
</para>
|
||||||
<para>
|
|
||||||
To just start and experiment with the VMs, run:
|
|
||||||
</para>
|
|
||||||
<programlisting>
|
|
||||||
$ nix-build nixos/tests/login.nix -A driverInteractive
|
|
||||||
$ ./result/bin/nixos-run-vms
|
|
||||||
</programlisting>
|
|
||||||
<para>
|
|
||||||
The script <literal>nixos-run-vms</literal> starts the virtual
|
|
||||||
machines defined by test.
|
|
||||||
</para>
|
|
||||||
<para>
|
<para>
|
||||||
You can re-use the VM states coming from a previous run by setting
|
You can re-use the VM states coming from a previous run by setting
|
||||||
the <literal>--keep-vm-state</literal> flag.
|
the <literal>--keep-vm-state</literal> flag.
|
||||||
</para>
|
</para>
|
||||||
<programlisting>
|
<programlisting>
|
||||||
$ ./result/bin/nixos-run-vms --keep-vm-state
|
$ ./result/bin/nixos-test-driver --interactive --keep-vm-state
|
||||||
</programlisting>
|
</programlisting>
|
||||||
<para>
|
<para>
|
||||||
The machine state is stored in the
|
The machine state is stored in the
|
||||||
|
|
101
nixos/lib/test-driver/test-driver.py
Normal file → Executable file
101
nixos/lib/test-driver/test-driver.py
Normal file → Executable file
|
@ -24,7 +24,6 @@ import sys
|
||||||
import telnetlib
|
import telnetlib
|
||||||
import tempfile
|
import tempfile
|
||||||
import time
|
import time
|
||||||
import traceback
|
|
||||||
import unicodedata
|
import unicodedata
|
||||||
|
|
||||||
CHAR_TO_KEY = {
|
CHAR_TO_KEY = {
|
||||||
|
@ -930,29 +929,16 @@ def join_all() -> None:
|
||||||
machine.wait_for_shutdown()
|
machine.wait_for_shutdown()
|
||||||
|
|
||||||
|
|
||||||
def test_script() -> None:
|
def run_tests(interactive: bool = False) -> None:
|
||||||
exec(os.environ["testScript"])
|
|
||||||
|
|
||||||
|
|
||||||
def run_tests() -> None:
|
|
||||||
global machines
|
global machines
|
||||||
tests = os.environ.get("tests", None)
|
if interactive:
|
||||||
if tests is not None:
|
ptpython.repl.embed(globals(), locals())
|
||||||
with log.nested("running the VM test script"):
|
|
||||||
try:
|
|
||||||
exec(tests, globals())
|
|
||||||
except Exception as e:
|
|
||||||
eprint("error: ")
|
|
||||||
traceback.print_exc()
|
|
||||||
sys.exit(1)
|
|
||||||
else:
|
else:
|
||||||
ptpython.repl.embed(locals(), globals())
|
test_script()
|
||||||
|
# TODO: Collect coverage data
|
||||||
# TODO: Collect coverage data
|
for machine in machines:
|
||||||
|
if machine.is_up():
|
||||||
for machine in machines:
|
machine.execute("sync")
|
||||||
if machine.is_up():
|
|
||||||
machine.execute("sync")
|
|
||||||
|
|
||||||
|
|
||||||
def serial_stdout_on() -> None:
|
def serial_stdout_on() -> None:
|
||||||
|
@ -965,6 +951,31 @@ def serial_stdout_off() -> None:
|
||||||
log._print_serial_logs = False
|
log._print_serial_logs = False
|
||||||
|
|
||||||
|
|
||||||
|
class EnvDefault(argparse.Action):
|
||||||
|
"""An argpars Action that takes values from the specified
|
||||||
|
environment variable as the flags default value.
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, envvar, required=False, default=None, nargs=None, **kwargs): # type: ignore
|
||||||
|
if not default and envvar:
|
||||||
|
if envvar in os.environ:
|
||||||
|
if nargs is not None and (nargs.isdigit() or nargs in ["*", "+"]):
|
||||||
|
default = os.environ[envvar].split()
|
||||||
|
else:
|
||||||
|
default = os.environ[envvar]
|
||||||
|
kwargs["help"] = (
|
||||||
|
kwargs["help"] + f" (default from environment: {default})"
|
||||||
|
)
|
||||||
|
if required and default:
|
||||||
|
required = False
|
||||||
|
super(EnvDefault, self).__init__(
|
||||||
|
default=default, required=required, nargs=nargs, **kwargs
|
||||||
|
)
|
||||||
|
|
||||||
|
def __call__(self, parser, namespace, values, option_string=None): # type: ignore
|
||||||
|
setattr(namespace, self.dest, values)
|
||||||
|
|
||||||
|
|
||||||
@contextmanager
|
@contextmanager
|
||||||
def subtest(name: str) -> Iterator[None]:
|
def subtest(name: str) -> Iterator[None]:
|
||||||
with log.nested(name):
|
with log.nested(name):
|
||||||
|
@ -986,18 +997,52 @@ if __name__ == "__main__":
|
||||||
help="re-use a VM state coming from a previous run",
|
help="re-use a VM state coming from a previous run",
|
||||||
action="store_true",
|
action="store_true",
|
||||||
)
|
)
|
||||||
(cli_args, vm_scripts) = arg_parser.parse_known_args()
|
arg_parser.add_argument(
|
||||||
|
"-I",
|
||||||
|
"--interactive",
|
||||||
|
help="drop into a python repl and run the tests interactively",
|
||||||
|
action="store_true",
|
||||||
|
)
|
||||||
|
arg_parser.add_argument(
|
||||||
|
"--start-scripts",
|
||||||
|
metavar="START-SCRIPT",
|
||||||
|
action=EnvDefault,
|
||||||
|
envvar="startScripts",
|
||||||
|
nargs="*",
|
||||||
|
help="start scripts for participating virtual machines",
|
||||||
|
)
|
||||||
|
arg_parser.add_argument(
|
||||||
|
"--vlans",
|
||||||
|
metavar="VLAN",
|
||||||
|
action=EnvDefault,
|
||||||
|
envvar="vlans",
|
||||||
|
nargs="*",
|
||||||
|
help="vlans to span by the driver",
|
||||||
|
)
|
||||||
|
arg_parser.add_argument(
|
||||||
|
"testscript",
|
||||||
|
action=EnvDefault,
|
||||||
|
envvar="testScript",
|
||||||
|
help="the test script to run",
|
||||||
|
type=pathlib.Path,
|
||||||
|
)
|
||||||
|
|
||||||
|
args = arg_parser.parse_args()
|
||||||
|
global test_script
|
||||||
|
|
||||||
|
def test_script() -> None:
|
||||||
|
with log.nested("running the VM test script"):
|
||||||
|
exec(pathlib.Path(args.testscript).read_text(), globals())
|
||||||
|
|
||||||
log = Logger()
|
log = Logger()
|
||||||
|
|
||||||
vlan_nrs = list(dict.fromkeys(os.environ.get("VLANS", "").split()))
|
vde_sockets = [create_vlan(v) for v in args.vlans]
|
||||||
vde_sockets = [create_vlan(v) for v in vlan_nrs]
|
|
||||||
for nr, vde_socket, _, _ in vde_sockets:
|
for nr, vde_socket, _, _ in vde_sockets:
|
||||||
os.environ["QEMU_VDE_SOCKET_{}".format(nr)] = vde_socket
|
os.environ["QEMU_VDE_SOCKET_{}".format(nr)] = vde_socket
|
||||||
|
|
||||||
machines = [
|
machines = [
|
||||||
create_machine({"startCommand": s, "keepVmState": cli_args.keep_vm_state})
|
create_machine({"startCommand": s, "keepVmState": args.keep_vm_state})
|
||||||
for s in vm_scripts
|
for s in args.start_scripts
|
||||||
]
|
]
|
||||||
machine_eval = [
|
machine_eval = [
|
||||||
"{0} = machines[{1}]".format(m.name, idx) for idx, m in enumerate(machines)
|
"{0} = machines[{1}]".format(m.name, idx) for idx, m in enumerate(machines)
|
||||||
|
@ -1017,6 +1062,6 @@ if __name__ == "__main__":
|
||||||
log.close()
|
log.close()
|
||||||
|
|
||||||
tic = time.time()
|
tic = time.time()
|
||||||
run_tests()
|
run_tests(args.interactive)
|
||||||
toc = time.time()
|
toc = time.time()
|
||||||
print("test script finished in {:.2f}s".format(toc - tic))
|
print("test script finished in {:.2f}s".format(toc - tic))
|
||||||
|
|
|
@ -83,7 +83,10 @@ rec {
|
||||||
''
|
''
|
||||||
mkdir -p $out
|
mkdir -p $out
|
||||||
|
|
||||||
LOGFILE=/dev/null tests='exec(os.environ["testScript"])' ${driver}/bin/nixos-test-driver
|
# effectively mute the XMLLogger
|
||||||
|
export LOGFILE=/dev/null
|
||||||
|
|
||||||
|
${driver}/bin/nixos-test-driver
|
||||||
'';
|
'';
|
||||||
|
|
||||||
passthru = driver.passthru // {
|
passthru = driver.passthru // {
|
||||||
|
@ -161,7 +164,10 @@ rec {
|
||||||
''
|
''
|
||||||
mkdir -p $out/bin
|
mkdir -p $out/bin
|
||||||
|
|
||||||
|
vmStartScripts=($(for i in ${toString vms}; do echo $i/bin/run-*-vm; done))
|
||||||
echo -n "$testScript" > $out/test-script
|
echo -n "$testScript" > $out/test-script
|
||||||
|
ln -s ${testDriver}/bin/nixos-test-driver $out/bin/nixos-test-driver
|
||||||
|
|
||||||
${lib.optionalString (!skipLint) ''
|
${lib.optionalString (!skipLint) ''
|
||||||
PYFLAKES_BUILTINS="$(
|
PYFLAKES_BUILTINS="$(
|
||||||
echo -n ${lib.escapeShellArg (lib.concatStringsSep "," nodeHostNames)},
|
echo -n ${lib.escapeShellArg (lib.concatStringsSep "," nodeHostNames)},
|
||||||
|
@ -169,17 +175,12 @@ rec {
|
||||||
)" ${python3Packages.pyflakes}/bin/pyflakes $out/test-script
|
)" ${python3Packages.pyflakes}/bin/pyflakes $out/test-script
|
||||||
''}
|
''}
|
||||||
|
|
||||||
ln -s ${testDriver}/bin/nixos-test-driver $out/bin/
|
# set defaults through environment
|
||||||
vms=($(for i in ${toString vms}; do echo $i/bin/run-*-vm; done))
|
# see: ./test-driver/test-driver.py argparse implementation
|
||||||
wrapProgram $out/bin/nixos-test-driver \
|
wrapProgram $out/bin/nixos-test-driver \
|
||||||
--add-flags "''${vms[*]}" \
|
--set startScripts "''${vmStartScripts[*]}" \
|
||||||
--run "export testScript=\"\$(${coreutils}/bin/cat $out/test-script)\"" \
|
--set testScript "$out/test-script" \
|
||||||
--set VLANS '${toString vlans}'
|
--set vlans '${toString vlans}'
|
||||||
ln -s ${testDriver}/bin/nixos-test-driver $out/bin/nixos-run-vms
|
|
||||||
wrapProgram $out/bin/nixos-run-vms \
|
|
||||||
--add-flags "''${vms[*]}" \
|
|
||||||
--set tests 'start_all(); join_all();' \
|
|
||||||
--set VLANS '${toString vlans}'
|
|
||||||
'');
|
'');
|
||||||
|
|
||||||
# Make a full-blown test
|
# Make a full-blown test
|
||||||
|
|
Loading…
Reference in a new issue