Merge pull request #196251 from hercules-ci/testers-build-failure-and-equal-contents
`testers`: Add `testBuildFailure` and `testEqualContents`
This commit is contained in:
Robert Hensing
GitHub
commit
29352751cf
|
|
@ -35,6 +35,70 @@ passthru.tests.version = testers.testVersion {
|
|||
};
|
||||
```
|
||||
|
||||
## `testBuildFailure` {#tester-testBuildFailure}
|
||||
|
||||
Make sure that a build does not succeed. This is useful for testing testers.
|
||||
|
||||
This returns a derivation with an override on the builder, with the following effects:
|
||||
|
||||
- Fail the build when the original builder succeeds
|
||||
- Move `$out` to `$out/result`, if it exists (assuming `out` is the default output)
|
||||
- Save the build log to `$out/testBuildFailure.log` (same)
|
||||
|
||||
Example:
|
||||
|
||||
```nix
|
||||
runCommand "example" {
|
||||
failed = testers.testBuildFailure (runCommand "fail" {} ''
|
||||
echo ok-ish >$out
|
||||
echo failing though
|
||||
exit 3
|
||||
'');
|
||||
} ''
|
||||
grep -F 'ok-ish' $failed/result
|
||||
grep -F 'failing though' $failed/testBuildFailure.log
|
||||
[[ 3 = $(cat $failed/testBuildFailure.exit) ]]
|
||||
touch $out
|
||||
'';
|
||||
```
|
||||
|
||||
While `testBuildFailure` is designed to keep changes to the original builder's
|
||||
environment to a minimum, some small changes are inevitable.
|
||||
|
||||
- The file `$TMPDIR/testBuildFailure.log` is present. It should not be deleted.
|
||||
- `stdout` and `stderr` are a pipe instead of a tty. This could be improved.
|
||||
- One or two extra processes are present in the sandbox during the original
|
||||
builder's execution.
|
||||
- The derivation and output hashes are different, but not unusual.
|
||||
- The derivation includes a dependency on `buildPackages.bash` and
|
||||
`expect-failure.sh`, which is built to include a transitive dependency on
|
||||
`buildPackages.coreutils` and possibly more. These are not added to `PATH`
|
||||
or any other environment variable, so they should be hard to observe.
|
||||
|
||||
## `testEqualContents` {#tester-equalContents}
|
||||
|
||||
Check that two paths have the same contents.
|
||||
|
||||
Example:
|
||||
|
||||
```nix
|
||||
testers.testEqualContents {
|
||||
assertion = "sed -e performs replacement";
|
||||
expected = writeText "expected" ''
|
||||
foo baz baz
|
||||
'';
|
||||
actual = runCommand "actual" {
|
||||
# not really necessary for a package that's in stdenv
|
||||
nativeBuildInputs = [ gnused ];
|
||||
base = writeText "base" ''
|
||||
foo bar baz
|
||||
'';
|
||||
} ''
|
||||
sed -e 's/bar/baz/g' $base >$out
|
||||
'';
|
||||
}
|
||||
```
|
||||
|
||||
## `testEqualDerivation` {#tester-testEqualDerivation}
|
||||
|
||||
Checks that two packages produce the exact same build instructions.
|
||||
|
|
|
|||
|
|
@ -1,8 +1,60 @@
|
|||
{ pkgs, lib, callPackage, runCommand, stdenv }:
|
||||
{ pkgs, buildPackages, lib, callPackage, runCommand, stdenv, substituteAll, }:
|
||||
# Documentation is in doc/builders/testers.chapter.md
|
||||
{
|
||||
# See https://nixos.org/manual/nixpkgs/unstable/#tester-testBuildFailure
|
||||
# or doc/builders/testers.chapter.md
|
||||
testBuildFailure = drv: drv.overrideAttrs (orig: {
|
||||
builder = buildPackages.bash;
|
||||
args = [
|
||||
(substituteAll { coreutils = buildPackages.coreutils; src = ./expect-failure.sh; })
|
||||
orig.realBuilder or stdenv.shell
|
||||
] ++ orig.args or ["-e" (orig.builder or ../../stdenv/generic/default-builder.sh)];
|
||||
});
|
||||
|
||||
# See https://nixos.org/manual/nixpkgs/unstable/#tester-testEqualDerivation
|
||||
# or doc/builders/testers.chapter.md
|
||||
testEqualDerivation = callPackage ./test-equal-derivation.nix { };
|
||||
|
||||
# See https://nixos.org/manual/nixpkgs/unstable/#tester-testEqualContents
|
||||
# or doc/builders/testers.chapter.md
|
||||
testEqualContents = {
|
||||
assertion,
|
||||
actual,
|
||||
expected,
|
||||
}: runCommand "equal-contents-${lib.strings.toLower assertion}" {
|
||||
inherit assertion actual expected;
|
||||
} ''
|
||||
echo "Checking:"
|
||||
echo "$assertion"
|
||||
if ! diff -U5 -r "$actual" "$expected" --color=always
|
||||
then
|
||||
echo
|
||||
echo 'Contents must be equal, but were not!'
|
||||
echo
|
||||
echo "+: expected, at $expected"
|
||||
echo "-: unexpected, at $actual"
|
||||
exit 1
|
||||
else
|
||||
find "$expected" -type f -executable > expected-executables | sort
|
||||
find "$actual" -type f -executable > actual-executables | sort
|
||||
if ! diff -U0 actual-executables expected-executables --color=always
|
||||
then
|
||||
echo
|
||||
echo "Contents must be equal, but some files' executable bits don't match"
|
||||
echo
|
||||
echo "+: make this file executable in the actual contents"
|
||||
echo "-: make this file non-executable in the actual contents"
|
||||
exit 1
|
||||
else
|
||||
echo "expected $expected and actual $actual match."
|
||||
echo 'OK'
|
||||
touch $out
|
||||
fi
|
||||
fi
|
||||
'';
|
||||
|
||||
# See https://nixos.org/manual/nixpkgs/unstable/#tester-testVersion
|
||||
# or doc/builders/testers.chapter.md
|
||||
testVersion =
|
||||
{ package,
|
||||
command ? "${package.meta.mainProgram or package.pname or package.name} --version",
|
||||
|
|
|
|||
62
pkgs/build-support/testers/expect-failure.sh
Normal file
62
pkgs/build-support/testers/expect-failure.sh
Normal file
|
|
@ -0,0 +1,62 @@
|
|||
# Run a builder, flip exit code, save log and fix outputs
|
||||
#
|
||||
# Sub-goals:
|
||||
# - Delegate to another original builder passed via args
|
||||
# - Save the build log to output for further checks
|
||||
# - Make the derivation succeed if the original builder fails
|
||||
# - Make the derivation fail if the original builder returns exit code 0
|
||||
#
|
||||
# Requirements:
|
||||
# This runs before, without and after stdenv. Do not modify the environment;
|
||||
# especially not before invoking the original builder. For example, use
|
||||
# "@" substitutions instead of PATH.
|
||||
# Do not export any variables.
|
||||
|
||||
# Stricter bash
|
||||
set -eu
|
||||
|
||||
# ------------------------
|
||||
# Run the original builder
|
||||
|
||||
echo "testBuildFailure: Expecting non-zero exit from builder and args: ${*@Q}"
|
||||
|
||||
("$@" 2>&1) | @coreutils@/bin/tee $TMPDIR/testBuildFailure.log \
|
||||
| while read ln; do
|
||||
echo "original builder: $ln"
|
||||
done
|
||||
|
||||
r=${PIPESTATUS[0]}
|
||||
if [[ $r = 0 ]]; then
|
||||
echo "testBuildFailure: The builder did not fail, but a failure was expected!"
|
||||
exit 1
|
||||
fi
|
||||
echo "testBuildFailure: Original builder produced exit code: $r"
|
||||
|
||||
# -----------------------------------------
|
||||
# Write the build log to the default output
|
||||
|
||||
outs=( $outputs )
|
||||
defOut=${outs[0]}
|
||||
defOutPath=${!defOut}
|
||||
|
||||
if [[ ! -d $defOutPath ]]; then
|
||||
if [[ -e $defOutPath ]]; then
|
||||
@coreutils@/bin/mv $defOutPath $TMPDIR/out-node
|
||||
@coreutils@/bin/mkdir $defOutPath
|
||||
@coreutils@/bin/mv $TMPDIR/out-node $defOutPath/result
|
||||
fi
|
||||
fi
|
||||
|
||||
@coreutils@/bin/mkdir -p $defOutPath
|
||||
@coreutils@/bin/mv $TMPDIR/testBuildFailure.log $defOutPath/testBuildFailure.log
|
||||
echo $r >$defOutPath/testBuildFailure.exit
|
||||
|
||||
# ------------------------------------------------------
|
||||
# Put empty directories in place for any missing outputs
|
||||
|
||||
for outputName in ${outputs:-out}; do
|
||||
outputPath="${!outputName}"
|
||||
if [[ ! -e "${outputPath}" ]]; then
|
||||
@coreutils@/bin/mkdir "${outputPath}";
|
||||
fi
|
||||
done
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
{ testers, lib, pkgs, ... }:
|
||||
{ testers, lib, pkgs, hello, runCommand, ... }:
|
||||
let
|
||||
pkgs-with-overlay = pkgs.extend(final: prev: {
|
||||
proof-of-overlay-hello = prev.hello;
|
||||
|
|
@ -24,4 +24,167 @@ lib.recurseIntoAttrs {
|
|||
machine.succeed("hello | figlet >/dev/console")
|
||||
'';
|
||||
});
|
||||
|
||||
testBuildFailure = lib.recurseIntoAttrs {
|
||||
happy = runCommand "testBuildFailure-happy" {
|
||||
failed = testers.testBuildFailure (runCommand "fail" {} ''
|
||||
echo ok-ish >$out
|
||||
echo failing though
|
||||
echo also stderr 1>&2
|
||||
exit 3
|
||||
'');
|
||||
} ''
|
||||
grep -F 'failing though' $failed/testBuildFailure.log
|
||||
grep -F 'also stderr' $failed/testBuildFailure.log
|
||||
grep -F 'ok-ish' $failed/result
|
||||
[[ 3 = $(cat $failed/testBuildFailure.exit) ]]
|
||||
touch $out
|
||||
'';
|
||||
|
||||
helloDoesNotFail = runCommand "testBuildFailure-helloDoesNotFail" {
|
||||
failed = testers.testBuildFailure (testers.testBuildFailure hello);
|
||||
|
||||
# Add hello itself as a prerequisite, so we don't try to run this test if
|
||||
# there's an actual failure in hello.
|
||||
inherit hello;
|
||||
} ''
|
||||
echo "Checking $failed/testBuildFailure.log"
|
||||
grep -F 'testBuildFailure: The builder did not fail, but a failure was expected' $failed/testBuildFailure.log
|
||||
[[ 1 = $(cat $failed/testBuildFailure.exit) ]]
|
||||
touch $out
|
||||
'';
|
||||
|
||||
multiOutput = runCommand "testBuildFailure-multiOutput" {
|
||||
failed = testers.testBuildFailure (runCommand "fail" {
|
||||
# dev will be the default output
|
||||
outputs = ["dev" "doc" "out"];
|
||||
} ''
|
||||
echo i am failing
|
||||
exit 1
|
||||
'');
|
||||
} ''
|
||||
grep -F 'i am failing' $failed/testBuildFailure.log >/dev/null
|
||||
[[ 1 = $(cat $failed/testBuildFailure.exit) ]]
|
||||
|
||||
# Checking our note that dev is the default output
|
||||
echo $failed/_ | grep -- '-dev/_' >/dev/null
|
||||
echo 'All good.'
|
||||
touch $out
|
||||
'';
|
||||
};
|
||||
|
||||
testEqualContents = lib.recurseIntoAttrs {
|
||||
happy = testers.testEqualContents {
|
||||
assertion = "The same directory contents at different paths are recognized as equal";
|
||||
expected = runCommand "expected" {} ''
|
||||
mkdir -p $out/c
|
||||
echo a >$out/a
|
||||
echo b >$out/b
|
||||
echo d >$out/c/d
|
||||
'';
|
||||
actual = runCommand "actual" {} ''
|
||||
mkdir -p $out/c
|
||||
echo a >$out/a
|
||||
echo b >$out/b
|
||||
echo d >$out/c/d
|
||||
'';
|
||||
};
|
||||
|
||||
unequalExe =
|
||||
runCommand "testEqualContents-unequalExe" {
|
||||
log = testers.testBuildFailure (testers.testEqualContents {
|
||||
assertion = "The same directory contents at different paths are recognized as equal";
|
||||
expected = runCommand "expected" {} ''
|
||||
mkdir -p $out/c
|
||||
echo a >$out/a
|
||||
chmod a+x $out/a
|
||||
echo b >$out/b
|
||||
echo d >$out/c/d
|
||||
'';
|
||||
actual = runCommand "actual" {} ''
|
||||
mkdir -p $out/c
|
||||
echo a >$out/a
|
||||
echo b >$out/b
|
||||
chmod a+x $out/b
|
||||
echo d >$out/c/d
|
||||
'';
|
||||
});
|
||||
} ''
|
||||
(
|
||||
set -x
|
||||
grep -F -- "executable bits don't match" $log/testBuildFailure.log
|
||||
grep -E -- '+.*-actual/a' $log/testBuildFailure.log
|
||||
grep -E -- '-.*-actual/b' $log/testBuildFailure.log
|
||||
grep -F -- "--- actual-executables" $log/testBuildFailure.log
|
||||
grep -F -- "+++ expected-executables" $log/testBuildFailure.log
|
||||
) || {
|
||||
echo "Test failed: could not find pattern in build log $log"
|
||||
exit 1
|
||||
}
|
||||
echo 'All good.'
|
||||
touch $out
|
||||
'';
|
||||
|
||||
fileDiff =
|
||||
runCommand "testEqualContents-fileDiff" {
|
||||
log = testers.testBuildFailure (testers.testEqualContents {
|
||||
assertion = "The same directory contents at different paths are recognized as equal";
|
||||
expected = runCommand "expected" {} ''
|
||||
mkdir -p $out/c
|
||||
echo a >$out/a
|
||||
echo b >$out/b
|
||||
echo d >$out/c/d
|
||||
'';
|
||||
actual = runCommand "actual" {} ''
|
||||
mkdir -p $out/c
|
||||
echo a >$out/a
|
||||
echo B >$out/b
|
||||
echo d >$out/c/d
|
||||
'';
|
||||
});
|
||||
} ''
|
||||
(
|
||||
set -x
|
||||
grep -F -- "Contents must be equal but were not" $log/testBuildFailure.log
|
||||
grep -E -- '+++ .*-actual/b' $log/testBuildFailure.log
|
||||
grep -E -- '--- .*-actual/b' $log/testBuildFailure.log
|
||||
grep -F -- "-B" $log/testBuildFailure.log
|
||||
grep -F -- "+b" $log/testBuildFailure.log
|
||||
) || {
|
||||
echo "Test failed: could not find pattern in build log $log"
|
||||
exit 1
|
||||
}
|
||||
echo 'All good.'
|
||||
touch $out
|
||||
'';
|
||||
|
||||
fileMissing =
|
||||
runCommand "testEqualContents-fileMissing" {
|
||||
log = testers.testBuildFailure (testers.testEqualContents {
|
||||
assertion = "The same directory contents at different paths are recognized as equal";
|
||||
expected = runCommand "expected" {} ''
|
||||
mkdir -p $out/c
|
||||
echo a >$out/a
|
||||
echo b >$out/b
|
||||
echo d >$out/c/d
|
||||
'';
|
||||
actual = runCommand "actual" {} ''
|
||||
mkdir -p $out/c
|
||||
echo a >$out/a
|
||||
echo d >$out/c/d
|
||||
'';
|
||||
});
|
||||
} ''
|
||||
(
|
||||
set -x
|
||||
grep -F -- "Contents must be equal but were not" $log/testBuildFailure.log
|
||||
grep -E -- 'Only in .*-expected: b' $log/testBuildFailure.log
|
||||
) || {
|
||||
echo "Test failed: could not find pattern in build log $log"
|
||||
exit 1
|
||||
}
|
||||
echo 'All good.'
|
||||
touch $out
|
||||
'';
|
||||
};
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue