From 69cf5181c31c691e9fb225ada9a33f54208d914b Mon Sep 17 00:00:00 2001 From: Sergei Trofimovich Date: Tue, 21 Feb 2023 19:50:48 +0000 Subject: [PATCH] stdenv/generic/setup.sh: enable parallel installs by default The primary motivating example is openssl: Before the change full package build took 1m54s minutes. After the change full package build takes 59s. About a 2x speedup. The difference is visible because openssl builds hundreds of manpages spawning a perl process per manual in `install` phase. Such a workload is very easy to parallelize. Another example would be `autotools`+`libtool` based build system where install step requires relinking. The more binaries there are to relink the more gain it will be to do it in parallel. The change enables parallel installs by default only for buiilds that already have parallel builds enabled. There is a high chance those build systems already handle parallelism well but some packages will fail. Consistently propagated the enableParallelBuilding to: - cmake (enabled by default, similar to builds) - ninja (set parallelism explicitly, don't rely on default) - bmake (enable when requested) - scons (enable when requested) - meson (set parallelism explicitly, don't rely on default) - waf (set parallelism explicitly, don't rely on default) - qmake-4/5/6 (enable by default, similar to builds) - xorg (always enable, similar to builds) --- pkgs/development/libraries/qt-4.x/4.8/qmake-hook.sh | 5 +++++ pkgs/development/libraries/qt-5/hooks/qmake-hook.sh | 5 +++++ pkgs/development/libraries/qt-6/hooks/qmake-hook.sh | 5 +++++ pkgs/development/tools/build-managers/bmake/setup-hook.sh | 1 + pkgs/development/tools/build-managers/cmake/setup-hook.sh | 5 +++++ pkgs/development/tools/build-managers/meson/setup-hook.sh | 5 +++++ pkgs/development/tools/build-managers/ninja/setup-hook.sh | 8 ++++++++ pkgs/development/tools/build-managers/scons/setup-hook.sh | 1 + .../tools/build-managers/wafHook/setup-hook.sh | 6 ++++++ pkgs/servers/x11/xorg/builder.sh | 1 + pkgs/stdenv/generic/make-derivation.nix | 1 + pkgs/stdenv/generic/setup.sh | 1 + 12 files changed, 44 insertions(+) diff --git a/pkgs/development/libraries/qt-4.x/4.8/qmake-hook.sh b/pkgs/development/libraries/qt-4.x/4.8/qmake-hook.sh index f288e99dd12..b024f52f204 100644 --- a/pkgs/development/libraries/qt-4.x/4.8/qmake-hook.sh +++ b/pkgs/development/libraries/qt-4.x/4.8/qmake-hook.sh @@ -8,6 +8,11 @@ qmakeConfigurePhase() { echo "qmake4Hook: enabled parallel building" fi + if ! [[ -v enableParallelInstalling ]]; then + enableParallelInstalling=1 + echo "qmake: enabled parallel installing" + fi + runHook postConfigure } diff --git a/pkgs/development/libraries/qt-5/hooks/qmake-hook.sh b/pkgs/development/libraries/qt-5/hooks/qmake-hook.sh index 741225a5aa8..0d30f0e2665 100644 --- a/pkgs/development/libraries/qt-5/hooks/qmake-hook.sh +++ b/pkgs/development/libraries/qt-5/hooks/qmake-hook.sh @@ -38,6 +38,11 @@ qmakeConfigurePhase() { echo "qmake: enabled parallel building" fi + if ! [[ -v enableParallelInstalling ]]; then + enableParallelInstalling=1 + echo "qmake: enabled parallel installing" + fi + runHook postConfigure } diff --git a/pkgs/development/libraries/qt-6/hooks/qmake-hook.sh b/pkgs/development/libraries/qt-6/hooks/qmake-hook.sh index ec1d2ea6124..8c4ce096443 100644 --- a/pkgs/development/libraries/qt-6/hooks/qmake-hook.sh +++ b/pkgs/development/libraries/qt-6/hooks/qmake-hook.sh @@ -40,6 +40,11 @@ qmakeConfigurePhase() { echo "qmake: enabled parallel building" fi + if ! [[ -v enableParallelInstalling ]]; then + enableParallelInstalling=1 + echo "qmake: enabled parallel installing" + fi + runHook postConfigure } diff --git a/pkgs/development/tools/build-managers/bmake/setup-hook.sh b/pkgs/development/tools/build-managers/bmake/setup-hook.sh index c5ca27dd910..f477a96d57a 100644 --- a/pkgs/development/tools/build-managers/bmake/setup-hook.sh +++ b/pkgs/development/tools/build-managers/bmake/setup-hook.sh @@ -76,6 +76,7 @@ bmakeInstallPhase() { # shellcheck disable=SC2086 local flagsArray=( + ${enableParallelInstalling:+-j${NIX_BUILD_CORES}} SHELL=$SHELL # Old bash empty array hack $makeFlags ${makeFlagsArray+"${makeFlagsArray[@]}"} diff --git a/pkgs/development/tools/build-managers/cmake/setup-hook.sh b/pkgs/development/tools/build-managers/cmake/setup-hook.sh index 1d25887754a..b28ed42b689 100755 --- a/pkgs/development/tools/build-managers/cmake/setup-hook.sh +++ b/pkgs/development/tools/build-managers/cmake/setup-hook.sh @@ -130,6 +130,11 @@ cmakeConfigurePhase() { echo "cmake: enabled parallel building" fi + if ! [[ -v enableParallelInstalling ]]; then + enableParallelInstalling=1 + echo "cmake: enabled parallel installing" + fi + runHook postConfigure } diff --git a/pkgs/development/tools/build-managers/meson/setup-hook.sh b/pkgs/development/tools/build-managers/meson/setup-hook.sh index ded9c3c63b8..6305a405af2 100644 --- a/pkgs/development/tools/build-managers/meson/setup-hook.sh +++ b/pkgs/development/tools/build-managers/meson/setup-hook.sh @@ -28,6 +28,11 @@ mesonConfigurePhase() { echo "meson: enabled parallel building" fi + if ! [[ -v enableParallelInstalling ]]; then + enableParallelInstalling=1 + echo "meson: enabled parallel installing" + fi + runHook postConfigure } diff --git a/pkgs/development/tools/build-managers/ninja/setup-hook.sh b/pkgs/development/tools/build-managers/ninja/setup-hook.sh index ecaa8b88c1a..7fa5e4675f3 100644 --- a/pkgs/development/tools/build-managers/ninja/setup-hook.sh +++ b/pkgs/development/tools/build-managers/ninja/setup-hook.sh @@ -53,8 +53,16 @@ ninjaCheckPhase() { ninjaInstallPhase() { runHook preInstall + local buildCores=1 + + # Parallel building is enabled by default. + if [ "${enableParallelInstalling-1}" ]; then + buildCores="$NIX_BUILD_CORES" + fi + # shellcheck disable=SC2086 local flagsArray=( + -j$buildCores $ninjaFlags "${ninjaFlagsArray[@]}" ${installTargets:-install} ) diff --git a/pkgs/development/tools/build-managers/scons/setup-hook.sh b/pkgs/development/tools/build-managers/scons/setup-hook.sh index 0b908f68286..67358ee843d 100644 --- a/pkgs/development/tools/build-managers/scons/setup-hook.sh +++ b/pkgs/development/tools/build-managers/scons/setup-hook.sh @@ -33,6 +33,7 @@ sconsInstallPhase() { fi local flagsArray=( + ${enableParallelInstalling:+-j${NIX_BUILD_CORES}} $sconsFlags ${sconsFlagsArray[@]} $installFlags ${installFlagsArray[@]} ${installTargets:-install} diff --git a/pkgs/development/tools/build-managers/wafHook/setup-hook.sh b/pkgs/development/tools/build-managers/wafHook/setup-hook.sh index e2f6714a32d..4d95c28de1f 100644 --- a/pkgs/development/tools/build-managers/wafHook/setup-hook.sh +++ b/pkgs/development/tools/build-managers/wafHook/setup-hook.sh @@ -30,6 +30,11 @@ wafConfigurePhase() { echo "waf: enabled parallel building" fi + if ! [[ -v enableParallelInstalling ]]; then + enableParallelInstalling=1 + echo "waf: enabled parallel installing" + fi + runHook postConfigure } @@ -68,6 +73,7 @@ wafInstallPhase() { fi local flagsArray=( + ${enableParallelInstalling:+-j ${NIX_BUILD_CORES}} $wafFlags ${wafFlagsArray[@]} $installFlags ${installFlagsArray[@]} ${installTargets:-install} diff --git a/pkgs/servers/x11/xorg/builder.sh b/pkgs/servers/x11/xorg/builder.sh index 9ee81091584..a9c607ae35e 100644 --- a/pkgs/servers/x11/xorg/builder.sh +++ b/pkgs/servers/x11/xorg/builder.sh @@ -37,5 +37,6 @@ fi enableParallelBuilding=1 +enableParallelInstalling=1 genericBuild diff --git a/pkgs/stdenv/generic/make-derivation.nix b/pkgs/stdenv/generic/make-derivation.nix index 6bf319d0730..50487b9f2e2 100644 --- a/pkgs/stdenv/generic/make-derivation.nix +++ b/pkgs/stdenv/generic/make-derivation.nix @@ -433,6 +433,7 @@ else let } // lib.optionalAttrs (enableParallelBuilding) { inherit enableParallelBuilding; enableParallelChecking = attrs.enableParallelChecking or true; + enableParallelInstalling = attrs.enableParallelInstalling or true; } // lib.optionalAttrs (hardeningDisable != [] || hardeningEnable != [] || stdenv.hostPlatform.isMusl) { NIX_HARDENING_ENABLE = enabledHardeningOptions; } // lib.optionalAttrs (stdenv.hostPlatform.isx86_64 && stdenv.hostPlatform ? gcc.arch) { diff --git a/pkgs/stdenv/generic/setup.sh b/pkgs/stdenv/generic/setup.sh index 1a637bf13fd..734abb890c2 100644 --- a/pkgs/stdenv/generic/setup.sh +++ b/pkgs/stdenv/generic/setup.sh @@ -1376,6 +1376,7 @@ installPhase() { # shellcheck disable=SC2086 local flagsArray=( + ${enableParallelInstalling:+-j${NIX_BUILD_CORES}} SHELL=$SHELL ) _accumFlagsArray makeFlags makeFlagsArray installFlags installFlagsArray