stdenv: Fix handling of dependencies and hooks

4 far-reaching changes: Smaller PATH, New vars, different propagation
logic, and different hook logic

Smaller PATH
------------

`buildInputs` no longer go on the PATH at build time, as they cannot be
run when cross compiling and we don't want to special case. Simply make
a `nativeBuildInput` too if one needs them on the PATH. Fixes #21191.

Many new depedendency variables
-------------------------------

See the stdenv chapter of the nixpkgs manual. I pulled out the existing
documentation of dependency specification into a new section, and added
language for these two (and their propagated equivalents) along side
the others'.

More complex propagation logic
------------------------------

Before a propagated*XXX*Input always acted as if it was specified
directly as a *XXX*Input downstream. That's simple enough, but violates
the intended roles of each sort of dep, which has functional and not
just stylistic consequences.

The new algorithm is detailed in the manual, and ensures everything
ends up in the right place. I tried to give both an informal and formal
description, but I suspect in practice it will not make much sense
until one tries cross compiling, after which it will immediately make
sense as the only sane option.

Simplified hook logic
---------------------

Rather than `envHook` and `crossEnvHook`, whose behavior differs
depending on whether we are cross compiling or not, there is now one
hook per sort (or rather non-propagated and propagated pair of sorts)
of dependency. These new hooks have the same meaning regardless of
cross compilation. See the setup hook section of stdenv chapter of the
Nixpkgs manual for more details.
This commit is contained in:
John Ericson 2017-09-11 12:20:20 -04:00
parent bb18a3b573
commit 7f3ca3e21a
2 changed files with 291 additions and 91 deletions

View file

@ -14,11 +14,27 @@ rec {
mkDerivation =
{ name ? ""
, nativeBuildInputs ? []
, buildInputs ? []
# These types of dependencies are all exhaustively documented in
# the "Specifying Dependencies" section of the "Standard
# Environment" chapter of the Nixpkgs manual.
, propagatedNativeBuildInputs ? []
, propagatedBuildInputs ? []
# TODO(@Ericson2314): Stop using legacy dep attribute names
# host offset -> target offset
, depsBuildBuild ? [] # -1 -> -1
, depsBuildBuildPropagated ? [] # -1 -> -1
, nativeBuildInputs ? [] # -1 -> 0 N.B. Legacy name
, propagatedNativeBuildInputs ? [] # -1 -> 0 N.B. Legacy name
, depsBuildTarget ? [] # -1 -> 1
, depsBuildTargetPropagated ? [] # -1 -> 1
, depsHostHost ? [] # 0 -> 0
, depsHostHostPropagated ? [] # 0 -> 0
, buildInputs ? [] # 0 -> 1 N.B. Legacy name
, propagatedBuildInputs ? [] # 0 -> 1 N.B. Legacy name
, depsTargetTarget ? [] # 1 -> 1
, depsTargetTargetPropagated ? [] # 1 -> 1
, configureFlags ? []
, # Target is not included by default because most programs don't care.
@ -56,15 +72,35 @@ rec {
inherit erroneousHardeningFlags hardeningDisable hardeningEnable supportedHardeningFlags;
})
else let
dependencies = map lib.chooseDevOutputs [
(map (drv: drv.nativeDrv or drv) nativeBuildInputs
++ lib.optional separateDebugInfo ../../build-support/setup-hooks/separate-debug-info.sh
++ lib.optional stdenv.hostPlatform.isWindows ../../build-support/setup-hooks/win-dll-link.sh)
(map (drv: drv.crossDrv or drv) buildInputs)
dependencies = map (map lib.chooseDevOutputs) [
[
(map (drv: drv.__spliced.buildBuild or drv) depsBuildBuild)
(map (drv: drv.nativeDrv or drv) nativeBuildInputs
++ lib.optional separateDebugInfo ../../build-support/setup-hooks/separate-debug-info.sh
++ lib.optional stdenv.hostPlatform.isWindows ../../build-support/setup-hooks/win-dll-link.sh)
(map (drv: drv.__spliced.buildTarget or drv) depsBuildTarget)
]
[
(map (drv: drv.__spliced.hostHost or drv) depsHostHost)
(map (drv: drv.crossDrv or drv) buildInputs)
]
[
(map (drv: drv.__spliced.targetTarget or drv) depsTargetTarget)
]
];
propagatedDependencies = map lib.chooseDevOutputs [
(map (drv: drv.nativeDrv or drv) propagatedNativeBuildInputs)
(map (drv: drv.crossDrv or drv) propagatedBuildInputs)
propagatedDependencies = map (map lib.chooseDevOutputs) [
[
(map (drv: drv.__spliced.buildBuild or drv) depsBuildBuildPropagated)
(map (drv: drv.nativeDrv or drv) propagatedNativeBuildInputs)
(map (drv: drv.__spliced.buildTarget or drv) depsBuildTargetPropagated)
]
[
(map (drv: drv.__spliced.hostHost or drv) depsHostHostPropagated)
(map (drv: drv.crossDrv or drv) propagatedBuildInputs)
]
[
(map (drv: drv.__spliced.targetTarget or drv) depsTargetTargetPropagated)
]
];
outputs' =
@ -105,11 +141,19 @@ rec {
userHook = config.stdenv.userHook or null;
__ignoreNulls = true;
nativeBuildInputs = lib.elemAt dependencies 0;
buildInputs = lib.elemAt dependencies 1;
depsBuildBuild = lib.elemAt (lib.elemAt dependencies 0) 0;
nativeBuildInputs = lib.elemAt (lib.elemAt dependencies 0) 1;
depsBuildTarget = lib.elemAt (lib.elemAt dependencies 0) 2;
depsHostBuild = lib.elemAt (lib.elemAt dependencies 1) 0;
buildInputs = lib.elemAt (lib.elemAt dependencies 1) 1;
depsTargetTarget = lib.elemAt (lib.elemAt dependencies 2) 0;
propagatedNativeBuildInputs = lib.elemAt propagatedDependencies 0;
propagatedBuildInputs = lib.elemAt propagatedDependencies 1;
depsBuildBuildPropagated = lib.elemAt (lib.elemAt propagatedDependencies 0) 0;
propagatedNativeBuildInputs = lib.elemAt (lib.elemAt propagatedDependencies 0) 1;
depsBuildTargetPropagated = lib.elemAt (lib.elemAt propagatedDependencies 0) 2;
depsHostBuildPropagated = lib.elemAt (lib.elemAt propagatedDependencies 1) 0;
propagatedBuildInputs = lib.elemAt (lib.elemAt propagatedDependencies 1) 1;
depsTargetTargetPropagated = lib.elemAt (lib.elemAt propagatedDependencies 2) 0;
# This parameter is sometimes a string, sometimes null, and sometimes a list, yuck
configureFlags = let inherit (lib) optional elem; in

View file

@ -300,11 +300,72 @@ runHook preHook
runHook addInputsHook
# Recursively find all build inputs.
# Package accumulators
# shellcheck disable=SC2034
declare -a pkgsBuildBuild pkgsBuildHost pkgsBuildTarget
declare -a pkgsHostHost pkgsHostTarget
declare -a pkgsTargetTarget
declare -ra pkgBuildAccumVars=(pkgsBuildBuild pkgsBuildHost pkgsBuildTarget)
declare -ra pkgHostAccumVars=(pkgsHostHost pkgsHostTarget)
declare -ra pkgTargetAccumVars=(pkgsTargetTarget)
declare -ra pkgAccumVarVars=(pkgBuildAccumVars pkgHostAccumVars pkgTargetAccumVars)
# Hooks
declare -a envBuildBuildHooks envBuildHostHooks envBuildTargetHooks
declare -a envHostHostHooks envHostTargetHooks
declare -a envTargetTargetHooks
declare -ra pkgBuildHookVars=(envBuildBuildHook envBuildHostHook envBuildTargetHook)
declare -ra pkgHostHookVars=(envHostHostHook envHostTargetHook)
declare -ra pkgTargetHookVars=(envTargetTargetHook)
declare -ra pkgHookVarVars=(pkgBuildHookVars pkgHostHookVars pkgTargetHookVars)
# Propagated dep files
declare -ra propagatedBuildDepFiles=(
propagated-build-build-deps
propagated-native-build-inputs # Legacy name for back-compat
propagated-build-target-deps
)
declare -ra propagatedHostDepFiles=(
propagated-host-host-deps
propagated-build-inputs # Legacy name for back-compat
)
declare -ra propagatedTargetDepFiles=(
propagated-target-target-deps
)
declare -ra propagatedDepFilesVars=(
propagatedBuildDepFiles
propagatedHostDepFiles
propagatedTargetDepFiles
)
# Platform offsets: build = -1, host = 0, target = 1
declare -ra allPlatOffsets=(-1 0 1)
# Mutually-recursively find all build inputs. See the dependency section of the
# stdenv chapter of the Nixpkgs manual for the specification this algorithm
# implements.
findInputs() {
local pkg="$1"; shift
local var="$1"; shift
local propagatedBuildInputsFiles=("$@")
local -r pkg="$1"
local -ri hostOffset="$2"
local -ri targetOffset="$3"
# Sanity check
(( "$hostOffset" <= "$targetOffset" )) || exit -1
local varVar="${pkgAccumVarVars[$hostOffset + 1]}"
local varRef="$varVar[\$targetOffset - \$hostOffset]"
local var="${!varRef}"
unset -v varVar varRef
# TODO(@Ericson2314): Restore using associative array once Darwin
# nix-shell doesn't use impure bash. This should replace the O(n)
@ -324,21 +385,106 @@ findInputs() {
exit 1
fi
local file
for file in "${propagatedBuildInputsFiles[@]}"; do
file="$pkg/nix-support/$file"
[[ -f "$file" ]] || continue
# The current package's host and target offset together
# provide a <=-preserving homomorphism from the relative
# offsets to current offset
function mapOffset() {
local -ri inputOffset="$1"
if (( "$inputOffset" <= 0 )); then
local -ri outputOffset="$inputOffset + $hostOffset"
else
local -ri outputOffset="$inputOffset - 1 + $targetOffset"
fi
echo "$outputOffset"
}
local pkgNext
for pkgNext in $(< "$file"); do
findInputs "$pkgNext" "$var" "${propagatedBuildInputsFiles[@]}"
# Host offset relative to that of the package whose immediate
# dependencies we are currently exploring.
local -i relHostOffset
for relHostOffset in "${allPlatOffsets[@]}"; do
# `+ 1` so we start at 0 for valid index
local files="${propagatedDepFilesVars[$relHostOffset + 1]}"
# Host offset relative to the package currently being
# built---as absolute an offset as will be used.
local -i hostOffsetNext
hostOffsetNext="$(mapOffset relHostOffset)"
# Ensure we're in bounds relative to the package currently
# being built.
[[ "${allPlatOffsets[*]}" = *"$hostOffsetNext"* ]] || continue
# Target offset relative to the *host* offset of the package
# whose immediate dependencies we are currently exploring.
local -i relTargetOffset
for relTargetOffset in "${allPlatOffsets[@]}"; do
(( "$relHostOffset" <= "$relTargetOffset" )) || continue
local fileRef="${files}[$relTargetOffset - $relHostOffset]"
local file="${!fileRef}"
unset -v fileRef
# Target offset relative to the package currently being
# built.
local -i targetOffsetNext
targetOffsetNext="$(mapOffset relTargetOffset)"
# Once again, ensure we're in bounds relative to the
# package currently being built.
[[ "${allPlatOffsets[*]}" = *"$targetOffsetNext"* ]] || continue
[[ -f "$pkg/nix-support/$file" ]] || continue
local pkgNext
for pkgNext in $(< "$pkg/nix-support/$file"); do
findInputs "$pkgNext" "$hostOffsetNext" "$targetOffsetNext"
done
done
done
}
# Make sure all are at least defined as empty
: ${depsBuildBuild=} ${depsBuildBuildPropagated=}
: ${nativeBuildInputs=} ${propagatedNativeBuildInputs=} ${defaultNativeBuildInputs=}
: ${depsBuildTarget=} ${depsBuildTargetPropagated=}
: ${depsHostHost=} ${depsHostHostPropagated=}
: ${buildInputs=} ${propagatedBuildInputs=} ${defaultBuildInputs=}
: ${depsTargetTarget=} ${depsTargetTargetPropagated=}
for pkg in $depsBuildBuild $depsBuildBuildPropagated; do
findInputs "$pkg" -1 -1
done
for pkg in $nativeBuildInputs $propagatedNativeBuildInputs; do
findInputs "$pkg" -1 0
done
for pkg in $depsBuildTarget $depsBuildTargetPropagated; do
findInputs "$pkg" -1 1
done
for pkg in $depsHostHost $depsHostHostPropagated; do
findInputs "$pkg" 0 0
done
for pkg in $buildInputs $propagatedBuildInputs ; do
findInputs "$pkg" 0 1
done
for pkg in $depsTargetTarget $depsTargetTargetPropagated; do
findInputs "$pkg" 1 1
done
# Default inputs must be processed last
for pkg in $defaultNativeBuildInputs; do
findInputs "$pkg" -1 0
done
for pkg in $defaultBuildInputs; do
findInputs "$pkg" 0 1
done
# Add package to the future PATH and run setup hooks
activatePackage() {
local pkg="$1"
local -ri hostOffset="$2"
local -ri targetOffset="$3"
# Sanity check
(( "$hostOffset" <= "$targetOffset" )) || exit -1
if [ -f "$pkg" ]; then
local oldOpts="$(shopt -po nounset)"
@ -347,11 +493,16 @@ activatePackage() {
eval "$oldOpts"
fi
if [ -d "$pkg/bin" ]; then
# Only dependencies whose host platform is guaranteed to match the
# build platform are included here. That would be `depsBuild*`,
# and legacy `nativeBuildInputs`. Other aren't because of cross
# compiling, and we want to have consistent rules whether or not
# we are cross compiling.
if [[ "$hostOffset" -le -1 && -d "$pkg/bin" ]]; then
addToSearchPath _PATH "$pkg/bin"
fi
if [ -f "$pkg/nix-support/setup-hook" ]; then
if [[ -f "$pkg/nix-support/setup-hook" ]]; then
local oldOpts="$(shopt -po nounset)"
set +u
source "$pkg/nix-support/setup-hook"
@ -359,67 +510,55 @@ activatePackage() {
fi
}
declare -a nativePkgs crossPkgs
if [ -z "${crossConfig:-}" ]; then
# Not cross-compiling - both buildInputs (and variants like propagatedBuildInputs)
# are handled identically to nativeBuildInputs
for i in ${nativeBuildInputs:-} ${buildInputs:-} \
${defaultNativeBuildInputs:-} ${defaultBuildInputs:-} \
${propagatedNativeBuildInputs:-} ${propagatedBuildInputs:-}; do
findInputs "$i" nativePkgs propagated-native-build-inputs propagated-build-inputs
done
else
for i in ${nativeBuildInputs:-} ${defaultNativeBuildInputs:-} ${propagatedNativeBuildInputs:-}; do
findInputs "$i" nativePkgs propagated-native-build-inputs
done
for i in ${buildInputs:-} ${defaultBuildInputs:-} ${propagatedBuildInputs:-}; do
findInputs "$i" crossPkgs propagated-build-inputs
done
fi
_activatePkgs() {
local -i hostOffset targetOffset
local pkg
for i in ${nativePkgs+"${nativePkgs[@]}"} ${crossPkgs+"${crossPkgs[@]}"}; do
activatePackage "$i"
done
for hostOffset in "${allPlatOffsets[@]}"; do
local pkgsVar="${pkgAccumVarVars[$hostOffset + 1]}"
for targetOffset in "${allPlatOffsets[@]}"; do
(( "$hostOffset" <= "$targetOffset" )) || continue
local pkgsRef="${pkgsVar}[$targetOffset - $hostOffset]"
local pkgsSlice="${!pkgsRef}[@]"
for pkg in ${!pkgsSlice+"${!pkgsSlice}"}; do
activatePackage "$pkg" "$hostOffset" "$targetOffset"
done
done
done
}
# Run the package setup hooks and build _PATH
_activatePkgs
# Set the relevant environment variables to point to the build inputs
# found above.
#
# These `depOffset`s tell the env hook what sort of dependency
# (ignoring propagatedness) is being passed to the env hook. In a real
# language, we'd append a closure with this information to the
# relevant env hook array, but bash doesn't have closures, so it's
# easier to just pass this in.
# These `depOffset`s, beyond indexing the arrays, also tell the env
# hook what sort of dependency (ignoring propagatedness) is being
# passed to the env hook. In a real language, we'd append a closure
# with this information to the relevant env hook array, but bash
# doesn't have closures, so it's easier to just pass this in.
_addToEnv() {
local -i depHostOffset depTargetOffset
local pkg
_addToNativeEnv() {
local pkg="$1"
if [[ -n "${crossConfig:-}" ]]; then
local -i depOffset=-1
else
local -i depOffset=0
fi
# Run the package-specific hooks set by the setup-hook scripts.
runHook envHook "$pkg"
for depHostOffset in "${allPlatOffsets[@]}"; do
local hookVar="${pkgHookVarVars[$depHostOffset + 1]}"
local pkgsVar="${pkgAccumVarVars[$depHostOffset + 1]}"
for depTargetOffset in "${allPlatOffsets[@]}"; do
(( "$depHostOffset" <= "$depTargetOffset" )) || continue
local hookRef="${hookVar}[$depTargetOffset - $depHostOffset]"
local pkgsRef="${pkgsVar}[$depTargetOffset - $depHostOffset]"
local pkgsSlice="${!pkgsRef}[@]"
for pkg in ${!pkgsSlice+"${!pkgsSlice}"}; do
runHook "${!hookRef}" "$pkg"
done
done
done
}
# Old bash empty array hack
for i in ${nativePkgs+"${nativePkgs[@]}"}; do
_addToNativeEnv "$i"
done
_addToCrossEnv() {
local pkg="$1"
local -i depOffset=0
# Run the package-specific hooks set by the setup-hook scripts.
runHook crossEnvHook "$pkg"
}
# Old bash empty array hack
for i in ${crossPkgs+"${crossPkgs[@]}"}; do
_addToCrossEnv "$i"
done
# Run the package-specific hooks set by the setup-hook scripts.
_addToEnv
_addRpathPrefix "$out"
@ -882,6 +1021,7 @@ installPhase() {
# propagated-build-inputs.
fixupPhase() {
# Make sure everything is writable so "strip" et al. work.
local output
for output in $outputs; do
if [ -e "${!output}" ]; then chmod -R u+w "${!output}"; fi
done
@ -895,19 +1035,35 @@ fixupPhase() {
done
# Propagate build inputs and setup hook into the development output.
# Propagate dependencies & setup hook into the development output.
declare -ra flatVars=(
# Build
depsBuildBuildPropagated
propagatedNativeBuildInputs
depsBuildTargetPropagated
# Host
depsHostHostPropagated
propagatedBuildInputs
# Target
depsTargetTargetPropagated
)
declare -ra flatFiles=(
"${propagatedBuildDepFiles[@]}"
"${propagatedHostDepFiles[@]}"
"${propagatedTargetDepFiles[@]}"
)
local propagatedInputsIndex
for propagatedInputsIndex in "${!flatVars[@]}"; do
local propagatedInputsSlice="${flatVars[$propagatedInputsIndex]}[@]"
local propagatedInputsFile="${flatFiles[$propagatedInputsIndex]}"
[[ "${!propagatedInputsSlice}" ]] || continue
if [ -n "${propagatedBuildInputs:-}" ]; then
mkdir -p "${!outputDev}/nix-support"
# shellcheck disable=SC2086
printWords $propagatedBuildInputs > "${!outputDev}/nix-support/propagated-build-inputs"
fi
if [ -n "${propagatedNativeBuildInputs:-}" ]; then
mkdir -p "${!outputDev}/nix-support"
# shellcheck disable=SC2086
printWords $propagatedNativeBuildInputs > "${!outputDev}/nix-support/propagated-native-build-inputs"
fi
printWords ${!propagatedInputsSlice} > "${!outputDev}/nix-support/$propagatedInputsFile"
done
if [ -n "${setupHook:-}" ]; then