diff --git a/pkgs/os-specific/gnu/default.nix b/pkgs/os-specific/gnu/default.nix index 457b670319e..c002447d64b 100644 --- a/pkgs/os-specific/gnu/default.nix +++ b/pkgs/os-specific/gnu/default.nix @@ -3,7 +3,7 @@ args@{ fetchgit, stdenv, autoconf, automake, automake111x, libtool , texinfo, glibcCross, hurdPartedCross, libuuid, samba , gccCrossStageStatic, gccCrossStageFinal -, forceNativeDrv, forceSystem, newScope, platform, config, crossSystem +, forcedNativePackages, forceSystem, newScope, platform, config, crossSystem , overrides ? {} }: with args; @@ -12,7 +12,7 @@ let callPackage = newScope gnu; gnu = { - hurdCross = forceNativeDrv (callPackage ./hurd { + hurdCross = forcedNativePackages.callPackage ./hurd { inherit fetchgit stdenv autoconf libtool texinfo glibcCross hurdPartedCross; inherit (gnu) machHeaders mig; @@ -21,9 +21,9 @@ let headersOnly = false; cross = assert crossSystem != null; crossSystem; gccCross = gccCrossStageFinal; - }); + }; - hurdCrossIntermediate = forceNativeDrv (callPackage ./hurd { + hurdCrossIntermediate = forcedNativePackages.callPackage ./hurd { inherit fetchgit stdenv autoconf libtool texinfo glibcCross; inherit (gnu) machHeaders mig; hurdPartedCross = null; @@ -42,7 +42,7 @@ let # libshouldbeinlibc. buildTarget = "libihash libstore libshouldbeinlibc"; installTarget = "libihash-install libstore-install libshouldbeinlibc-install"; - }); + }; hurdHeaders = callPackage ./hurd { automake = automake111x; @@ -58,13 +58,13 @@ let hurd = null; }; - libpthreadCross = forceNativeDrv (callPackage ./libpthread { + libpthreadCross = forcedNativePackages.callPackage ./libpthread { inherit fetchgit stdenv autoconf automake libtool glibcCross; inherit (gnu) machHeaders hurdHeaders; hurd = gnu.hurdCrossIntermediate; gccCross = gccCrossStageStatic; cross = assert crossSystem != null; crossSystem; - }); + }; # In theory GNU Mach doesn't have to be cross-compiled. However, since it # has to be built for i586 (it doesn't work on x86_64), one needs a cross diff --git a/pkgs/stdenv/adapters.nix b/pkgs/stdenv/adapters.nix index 11f9a43c035..daa180c644f 100644 --- a/pkgs/stdenv/adapters.nix +++ b/pkgs/stdenv/adapters.nix @@ -66,12 +66,10 @@ rec { # In nixpkgs, sometimes 'null' gets in as a buildInputs element, # and we handle that through isAttrs. - getNativeDrv = drv: drv.nativeDrv or drv; - getCrossDrv = drv: drv.crossDrv or drv; - nativeBuildInputsDrvs = map getNativeDrv nativeBuildInputs; - buildInputsDrvs = map getCrossDrv buildInputs; - propagatedBuildInputsDrvs = map getCrossDrv propagatedBuildInputs; - propagatedNativeBuildInputsDrvs = map getNativeDrv propagatedNativeBuildInputs; + nativeBuildInputsDrvs = nativeBuildInputs; + buildInputsDrvs = buildInputs; + propagatedBuildInputsDrvs = propagatedBuildInputs; + propagatedNativeBuildInputsDrvs = propagatedNativeBuildInputs; # The base stdenv already knows that nativeBuildInputs and # buildInputs should be built with the usual gcc-wrapper @@ -88,10 +86,7 @@ rec { (drv: builtins.isAttrs drv && drv ? nativeDrv) buildInputs; nativeInputsFromBuildInputs = stdenv.lib.filter hostAsNativeDrv buildInputsNotNull; - # We should overwrite the input attributes in crossDrv, to overwrite - # the defaults for only-native builds in the base stdenv - crossDrv = if cross == null then nativeDrv else - stdenv.mkDerivation (args // { + in stdenv.mkDerivation (args // { name = name + "-" + cross.config; nativeBuildInputs = nativeBuildInputsDrvs ++ nativeInputsFromBuildInputs @@ -112,9 +107,6 @@ rec { crossConfig = cross.config; } // args.crossAttrs or {}); - in nativeDrv // { - inherit crossDrv nativeDrv; - }; } // { inherit cross gccCross binutilsCross; ccCross = gccCross; diff --git a/pkgs/stdenv/booter.nix b/pkgs/stdenv/booter.nix index 11ca8e1440e..6e5d073e55a 100644 --- a/pkgs/stdenv/booter.nix +++ b/pkgs/stdenv/booter.nix @@ -57,12 +57,18 @@ stageFuns: let # debugging purposes. folder = stageFun: finalSoFar: let args = stageFun finalSoFar; - stdenv = args.stdenv // { - # For debugging - __bootPackages = finalSoFar; + args' = args // { + stdenv = args.stdenv // { + # For debugging + __bootPackages = finalSoFar; + }; }; - args' = args // { inherit stdenv; }; - in - (if args.__raw or false then lib.id else allPackages) args'; + self = + if args.__raw or false + then args' + else allPackages ((builtins.removeAttrs args' ["selfBuild"]) // { + buildPackages = if args.selfBuild or true then self else finalSoFar; + }); + in self; in lib.lists.fold folder {} withAllowCustomOverrides diff --git a/pkgs/stdenv/cross/default.nix b/pkgs/stdenv/cross/default.nix index 16f41671b76..e684c14da4a 100644 --- a/pkgs/stdenv/cross/default.nix +++ b/pkgs/stdenv/cross/default.nix @@ -12,13 +12,11 @@ let in bootStages ++ [ - # Build Packages. - # - # For now, this is just used to build the native stdenv. Eventually, it - # should be used to build compilers and other such tools targeting the cross - # platform. Then, `forceNativeDrv` can be removed. + # Build Packages (vanillaPackages: { inherit system platform crossSystem config overlays; + # Should be false, but we're trying to preserve hashes for now + selfBuild = true; # It's OK to change the built-time dependencies allowCustomOverrides = true; stdenv = vanillaPackages.stdenv // { @@ -28,9 +26,10 @@ in bootStages ++ [ }; }) - # Run packages + # Run Packages (buildPackages: { inherit system platform crossSystem config overlays; + selfBuild = false; stdenv = if crossSystem.useiOSCross or false then let inherit (buildPackages.darwin.ios-cross { diff --git a/pkgs/stdenv/generic/default.nix b/pkgs/stdenv/generic/default.nix index 32e0d894818..269d7ef893a 100644 --- a/pkgs/stdenv/generic/default.nix +++ b/pkgs/stdenv/generic/default.nix @@ -115,7 +115,19 @@ let , sandboxProfile ? "" , propagatedSandboxProfile ? "" , ... } @ attrs: - let + let # Rename argumemnts to avoid cycles + buildInputs__ = buildInputs; + nativeBuildInputs__ = nativeBuildInputs; + propagatedBuildInputs__ = propagatedBuildInputs; + propagatedNativeBuildInputs__ = propagatedNativeBuildInputs; + in let + getNativeDrv = drv: drv.nativeDrv or drv; + getCrossDrv = drv: drv.crossDrv or drv; + nativeBuildInputs = map getNativeDrv nativeBuildInputs__; + buildInputs = map getCrossDrv buildInputs__; + propagatedBuildInputs = map getCrossDrv propagatedBuildInputs__; + propagatedNativeBuildInputs = map getNativeDrv propagatedNativeBuildInputs__; + in let pos' = if pos != null then pos diff --git a/pkgs/stdenv/linux/make-bootstrap-tools-cross.nix b/pkgs/stdenv/linux/make-bootstrap-tools-cross.nix index 9f4a4517627..a1b1f02d83d 100644 --- a/pkgs/stdenv/linux/make-bootstrap-tools-cross.nix +++ b/pkgs/stdenv/linux/make-bootstrap-tools-cross.nix @@ -55,11 +55,12 @@ let if toolsArch == "armv6l" then raspberrypiCrossSystem else if toolsArch == "armv7l" then armv7l-hf-multiplatform-crossSystem else null; - pkgs = pkgsFun ({inherit system;} // selectedCrossSystem); + pkgsUnspliced = pkgsFun ({inherit system;} // selectedCrossSystem); + pkgs = pkgsUnspliced.splicedPackages; - inherit (pkgs) stdenv nukeReferences cpio binutilsCross; + inherit (pkgsUnspliced.buildPackages) stdenv nukeReferences cpio binutilsCross; - glibc = pkgs.libcCross; + glibc = pkgs.libcCross.nativeDrv; bash = pkgs.bash.crossDrv; findutils = pkgs.findutils.crossDrv; diffutils = pkgs.diffutils.crossDrv; @@ -71,7 +72,7 @@ let gnumake = pkgs.gnumake.crossDrv; patch = pkgs.patch.crossDrv; patchelf = pkgs.patchelf.crossDrv; - gcc = pkgs.gcc.cc.crossDrv; + gcc = pkgs.gcc.crossDrv.cc; gmpxx = pkgs.gmpxx.crossDrv; mpfr = pkgs.mpfr.crossDrv; zlib = pkgs.zlib.crossDrv; @@ -86,17 +87,17 @@ in rec { - coreutilsMinimal = (pkgs.coreutils.override (args: { + coreutilsMinimal = pkgs.coreutils.override (args: { # We want coreutils without ACL/attr support. aclSupport = false; attrSupport = false; # Our tooling currently can't handle scripts in bin/, only ELFs and symlinks. singleBinary = "symlinks"; - })).crossDrv; + }); - tarMinimal = (pkgs.gnutar.override { acl = null; }).crossDrv; + tarMinimal = pkgs.gnutar.override { acl = null; }; - busyboxMinimal = (pkgs.busybox.override { + busyboxMinimal = pkgs.busybox.override { useMusl = true; enableStatic = true; enableMinimal = true; @@ -109,13 +110,13 @@ rec { CONFIG_TAR y CONFIG_UNXZ y ''; - }).crossDrv; + }; build = stdenv.mkDerivation { name = "stdenv-bootstrap-tools-cross"; - crossConfig = stdenv.cross.config; + crossConfig = pkgsUnspliced.crossSystem.config; buildInputs = [nukeReferences cpio binutilsCross]; diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 8d2b4cf5201..bafe502da38 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -10,26 +10,12 @@ self: pkgs: with pkgs; -let - defaultScope = pkgs // pkgs.xorg; -in - { # Allow callPackage to fill in the pkgs argument inherit pkgs; - # We use `callPackage' to be able to omit function arguments that - # can be obtained from `pkgs' or `pkgs.xorg' (i.e. `defaultScope'). - # Use `newScope' for sets of packages in `pkgs' (see e.g. `gnome' - # below). - callPackage = newScope {}; - - callPackages = lib.callPackagesWith defaultScope; - - newScope = extra: lib.callPackageWith (defaultScope // extra); - # Override system. This is useful to build i686 packages on x86_64-linux. forceSystem = system: kernel: nixpkgsFun { inherit system; @@ -39,15 +25,9 @@ in # Used by wine, firefox with debugging version of Flash, ... pkgsi686Linux = forceSystem "i686-linux" "i386"; - callPackage_i686 = lib.callPackageWith (pkgsi686Linux // pkgsi686Linux.xorg); + callPackage_i686 = pkgsi686Linux.callPackage; - forceNativeDrv = drv: - # Even when cross compiling, some packages come from the stdenv's - # bootstrapping package set. Those packages are only built for the native - # platform. - if crossSystem != null && drv ? crossDrv - then drv // { crossDrv = drv.nativeDrv; } - else drv; + forcedNativePackages = if crossSystem == null then pkgs else buildPackages; # A stdenv capable of building 32-bit binaries. On x86_64-linux, # it uses GCC compiled with multilib support; on i686-linux, it's @@ -4771,41 +4751,45 @@ in gccApple = throw "gccApple is no longer supported"; - gccCrossStageStatic = let + gccCrossStageStatic = assert crossSystem != null; let libcCross1 = if stdenv.cross.libc == "msvcrt" then windows.mingw_w64_headers else if stdenv.cross.libc == "libSystem" then darwin.xcode else null; in wrapGCCCross { - gcc = forceNativeDrv (gcc.cc.override { + gcc = forcedNativePackages.gcc.cc.override { cross = crossSystem; crossStageStatic = true; langCC = false; libcCross = libcCross1; enableShared = false; - }); + # Why is this needed? + inherit (forcedNativePackages) binutilsCross; + }; libc = libcCross1; binutils = binutilsCross; cross = crossSystem; }; # Only needed for mingw builds - gccCrossMingw2 = wrapGCCCross { + gccCrossMingw2 = assert crossSystem != null; wrapGCCCross { gcc = gccCrossStageStatic.gcc; libc = windows.mingw_headers2; binutils = binutilsCross; cross = assert crossSystem != null; crossSystem; }; - gccCrossStageFinal = wrapGCCCross { - gcc = forceNativeDrv (gcc.cc.override { + gccCrossStageFinal = assert crossSystem != null; wrapGCCCross { + gcc = forcedNativePackages.gcc.cc.override { cross = crossSystem; crossStageStatic = false; # XXX: We have troubles cross-compiling libstdc++ on MinGW (see # ), so don't even try. langCC = crossSystem.config != "i686-pc-mingw32"; - }); + # Why is this needed? + inherit (forcedNativePackages) binutilsCross; + }; libc = libcCross; binutils = binutilsCross; cross = crossSystem; @@ -5499,12 +5483,12 @@ in wrapGCCCross = {gcc, libc, binutils, cross, shell ? "", name ? "gcc-cross-wrapper"}: - forceNativeDrv (callPackage ../build-support/gcc-cross-wrapper { + forcedNativePackages.callPackage ../build-support/gcc-cross-wrapper { nativeTools = false; nativeLibc = false; noLibc = (libc == null); inherit gcc binutils libc shell name cross; - }); + }; # prolog yap = callPackage ../development/compilers/yap { }; @@ -6084,12 +6068,12 @@ in gold = false; }); - binutilsCross = assert crossSystem != null; lowPrio (forceNativeDrv ( + binutilsCross = assert crossSystem != null; lowPrio ( if crossSystem.libc == "libSystem" then darwin.cctools_cross - else binutils.override { + else forcedNativePackages.binutils.override { noSysDirs = true; cross = crossSystem; - })); + }); bison2 = callPackage ../development/tools/parsing/bison/2.x.nix { }; bison3 = callPackage ../development/tools/parsing/bison/3.x.nix { }; @@ -6558,9 +6542,9 @@ in cross_renaming: we should make all programs use pkgconfig as nativeBuildInput after the renaming. */ - pkgconfig = forceNativeDrv (callPackage ../development/tools/misc/pkgconfig { + pkgconfig = forcedNativePackages.callPackage ../development/tools/misc/pkgconfig { fetchurl = fetchurlBoot; - }); + }; pkgconfigUpstream = lowPrio (pkgconfig.override { vanilla = true; }); postiats-utilities = callPackage ../development/tools/postiats-utilities {}; @@ -7336,10 +7320,10 @@ in withGd = true; }; - glibcCross = forceNativeDrv (glibc.override { + glibcCross = forcedNativePackages.glibc.override { gccCross = gccCrossStageStatic; linuxHeaders = linuxHeadersCross; - }); + }; # We can choose: libcCrossChooser = name: if name == "glibc" then glibcCross @@ -10910,7 +10894,7 @@ in cmdline = callPackage ../os-specific/darwin/command-line-tools {}; apple-source-releases = callPackage ../os-specific/darwin/apple-source-releases { }; in apple-source-releases // rec { - cctools_cross = callPackage (forceNativeDrv (callPackage ../os-specific/darwin/cctools/port.nix {}).cross) { + cctools_cross = callPackage (forcedNativePackages.callPackage ../os-specific/darwin/cctools/port.nix {}).cross { cross = assert crossSystem != null; crossSystem; inherit maloader; xctoolchain = xcode.toolchain; @@ -11148,13 +11132,13 @@ in linuxHeaders = linuxHeaders_4_4; - linuxHeaders24Cross = forceNativeDrv (callPackage ../os-specific/linux/kernel-headers/2.4.nix { + linuxHeaders24Cross = forcedNativePackages.callPackage ../os-specific/linux/kernel-headers/2.4.nix { cross = assert crossSystem != null; crossSystem; - }); + }; - linuxHeaders26Cross = forceNativeDrv (callPackage ../os-specific/linux/kernel-headers/4.4.nix { + linuxHeaders26Cross = forcedNativePackages.callPackage ../os-specific/linux/kernel-headers/4.4.nix { cross = assert crossSystem != null; crossSystem; - }); + }; linuxHeaders_3_18 = callPackage ../os-specific/linux/kernel-headers/3.18.nix { }; diff --git a/pkgs/top-level/release-cross.nix b/pkgs/top-level/release-cross.nix index f582bcf3b32..48f183162cf 100644 --- a/pkgs/top-level/release-cross.nix +++ b/pkgs/top-level/release-cross.nix @@ -32,8 +32,10 @@ let in { - # These `nativeDrv`s should be identical to their vanilla ones --- cross - # compiling should not affect the native derivation. + # These derivations from a cross package set's `buildPackages` should be + # identical to their vanilla equivalents --- none of these package should + # observe the target platform which is the only difference between those + # package sets. ensureUnaffected = let # Absurd values are fine here, as we are not building anything. In fact, # there probably a good idea to try to be "more parametric" --- i.e. avoid @@ -47,8 +49,12 @@ in # good idea lest there be some irrelevant pass-through debug attrs that # cause false negatives. testEqualOne = path: system: let - f = attrs: builtins.toString (lib.getAttrFromPath path (allPackages attrs)); - in assert f { inherit system; } == f { inherit system crossSystem; }; true; + f = path: attrs: builtins.toString (lib.getAttrFromPath path (allPackages attrs)); + in assert + f path { inherit system; } + == + f (["buildPackages"] ++ path) { inherit system crossSystem; }; + true; testEqual = path: systems: forAllSupportedSystems systems (testEqualOne path); diff --git a/pkgs/top-level/release-lib.nix b/pkgs/top-level/release-lib.nix index 5fe87711c01..04a57ef8063 100644 --- a/pkgs/top-level/release-lib.nix +++ b/pkgs/top-level/release-lib.nix @@ -76,8 +76,9 @@ rec { * parameter for allPackages, defining the target platform for cross builds, * and triggering the build of the host derivation (cross built - crossDrv). */ mapTestOnCross = crossSystem: mapAttrsRecursive - (path: systems: testOnCross crossSystem systems - (pkgs: addMetaAttrs { maintainers = crossMaintainers; } (getAttrFromPath path pkgs))); + (path: systems: testOnCross crossSystem systems (pkgs: addMetaAttrs + { maintainers = crossMaintainers; } + (getAttrFromPath path pkgs.splicedPackages))); /* Recursively map a (nested) set of derivations to an isomorphic diff --git a/pkgs/top-level/splice.nix b/pkgs/top-level/splice.nix new file mode 100644 index 00000000000..f57f42020c2 --- /dev/null +++ b/pkgs/top-level/splice.nix @@ -0,0 +1,76 @@ +# The `splicedPackages' package set, and its use by `callPackage` +# +# The `buildPackages` pkg set is a new concept, and the vast majority package +# expression (the other *.nix files) are not designed with it in mind. This +# presents us with a problem with how to get the right version (build-time vs +# run-time) of a package to a consumer that isn't used to thinking so cleverly. +# +# The solution is to splice the package sets together as we do below, so every +# `callPackage`d expression in fact gets both versions. Each# derivation (and +# each derivation's outputs) consists of the run-time version, augmented with a +# `nativeDrv` field for the build-time version, and `crossDrv` field for the +# run-time version. +# +# We could have used any names we want for the disambiguated versions, but +# `crossDrv` and `nativeDrv` were somewhat similarly used for the old +# cross-compiling infrastructure. The names are mostly invisible as +# `mkDerivation` knows how to pull out the right ones for `buildDepends` and +# friends, but a few packages use them directly, so it seemed efficient (to +# @Ericson2314) to reuse those names, at least initially, to minimize breakage. +lib: pkgs: + +let + defaultBuildScope = pkgs.buildPackages // pkgs.buildPackages.xorg; + # TODO(@Ericson2314): we shouldn't preclude run-time fetching by removing + # these attributes. We should have a more general solution for selecting + # whether `nativeDrv` or `crossDrv` is the default in `defaultScope`. + pkgsWithoutFetchers = lib.filterAttrs (n: _: !lib.hasPrefix "fetch" n) pkgs; + defaultRunScope = pkgsWithoutFetchers // pkgs.xorg; + + splicer = buildPkgs: runPkgs: let + mash = buildPkgs // runPkgs; + merge = name: { + inherit name; + value = let + defaultValue = mash.${name}; + buildValue = buildPkgs.${name} or {}; + runValue = runPkgs.${name} or {}; + augmentedValue = defaultValue + // (lib.optionalAttrs (buildPkgs ? ${name}) { nativeDrv = buildValue; }) + // (lib.optionalAttrs (runPkgs ? ${name}) { crossDrv = runValue; }); + # Get the set of outputs of a derivation + getOutputs = value: + lib.genAttrs (value.outputs or []) (output: value.${output}); + in + # Certain *Cross derivations will fail assertions, but we need their + # nativeDrv. We are assuming anything that fails to evaluate is an + # attrset (including derivation) and thus can be unioned. + if !(builtins.tryEval defaultValue).success then augmentedValue + # The derivation along with its outputs, which we recur + # on to splice them together. + else if lib.isDerivation defaultValue then augmentedValue + // splicer (getOutputs buildValue) (getOutputs runValue) + # Just recur on plain attrsets + else if lib.isAttrs defaultValue then splicer buildValue runValue + # Don't be fancy about non-derivations. But we could have used used + # `__functor__` for functions instead. + else defaultValue; + }; + in lib.listToAttrs (map merge (lib.attrNames mash)); + + splicedPackages = splicer defaultBuildScope defaultRunScope; + +in + +{ + splicedPackages = splicedPackages // { recurseForDerivations = false; }; + + # We use `callPackage' to be able to omit function arguments that can be + # obtained `pkgs` or `buildPackages` and their `xorg` package sets. Use + # `newScope' for sets of packages in `pkgs' (see e.g. `gnome' below). + callPackage = pkgs.newScope {}; + + callPackages = lib.callPackagesWith splicedPackages; + + newScope = extra: lib.callPackageWith (splicedPackages // extra); +} diff --git a/pkgs/top-level/stage.nix b/pkgs/top-level/stage.nix index cbf65870eb7..c100e21473c 100644 --- a/pkgs/top-level/stage.nix +++ b/pkgs/top-level/stage.nix @@ -12,6 +12,9 @@ { # The system (e.g., `i686-linux') for which to build the packages. system +, # the package set used at build-time + buildPackages + , # The standard environment to use for building packages. stdenv @@ -51,10 +54,13 @@ let stdenvBootstappingAndPlatforms = self: super: { stdenv = stdenv // { inherit platform; }; + buildPackages = buildPackages // { recurseForDerivations = false; }; inherit system platform crossSystem; }; + splice = self: super: import ./splice.nix lib self; + allPackages = self: super: let res = import ./all-packages.nix { inherit lib nixpkgsFun noSysDirs config; } @@ -85,6 +91,7 @@ let stdenvBootstappingAndPlatforms stdenvAdapters trivialBuilders + splice allPackages aliases stdenvOverrides