From 9d03ff52229f36ff4b4e9d0ee2db4a46b90512bb Mon Sep 17 00:00:00 2001 From: Frederik Rietdijk Date: Wed, 30 Dec 2020 14:49:10 +0100 Subject: [PATCH] python: reproducible builds Achieve reproducible builds of the interpreter. Note this meant disabling optimizations again. --- doc/languages-frameworks/python.section.md | 17 +++++++++ nixos/doc/manual/release-notes/rl-2105.xml | 6 ++++ .../python/cpython/2.7/default.nix | 36 ++++++++++++++----- .../interpreters/python/cpython/default.nix | 33 ++++++++++------- 4 files changed, 71 insertions(+), 21 deletions(-) diff --git a/doc/languages-frameworks/python.section.md b/doc/languages-frameworks/python.section.md index e569cdaa935..ff039b5e638 100644 --- a/doc/languages-frameworks/python.section.md +++ b/doc/languages-frameworks/python.section.md @@ -788,6 +788,23 @@ Each interpreter has the following attributes: - `executable`. Name of the interpreter executable, e.g. `python3.8`. - `pkgs`. Set of Python packages for that specific interpreter. The package set can be modified by overriding the interpreter and passing `packageOverrides`. +### Optimizations + +The Python interpreters are by default not build with optimizations enabled, because +the builds are in that case not reproducible. To enable optimizations, override the +interpreter of interest, e.g using + +``` +let + pkgs = import ./. {}; + mypython = pkgs.python3.override { + enableOptimizations = true; + reproducibleBuild = false; + self = mypython; + }; +in mypython +``` + ### Building packages and applications Python libraries and applications that use `setuptools` or diff --git a/nixos/doc/manual/release-notes/rl-2105.xml b/nixos/doc/manual/release-notes/rl-2105.xml index b4cf1a20241..39b441a0049 100644 --- a/nixos/doc/manual/release-notes/rl-2105.xml +++ b/nixos/doc/manual/release-notes/rl-2105.xml @@ -57,6 +57,12 @@ for the motivation). + + + Python optimizations were disabled again. Builds with optimizations enabled + are not reproducible. Optimizations can now be enabled with an option. + + diff --git a/pkgs/development/interpreters/python/cpython/2.7/default.nix b/pkgs/development/interpreters/python/cpython/2.7/default.nix index adeafa80e6c..f905612d022 100644 --- a/pkgs/development/interpreters/python/cpython/2.7/default.nix +++ b/pkgs/development/interpreters/python/cpython/2.7/default.nix @@ -27,7 +27,10 @@ , sha256 , passthruFun , static ? false -, enableOptimizations ? (!stdenv.isDarwin) +, stripBytecode ? reproducibleBuild +, rebuildBytecode ? true +, reproducibleBuild ? true +, enableOptimizations ? false , pythonAttr ? "python${sourceVersion.major}${sourceVersion.minor}" }: @@ -36,12 +39,26 @@ assert x11Support -> tcl != null && xlibsWrapper != null && libX11 != null; +assert lib.assertMsg (enableOptimizations -> (!stdenv.cc.isClang)) + "Optimizations with clang are not supported. configure: error: llvm-profdata is required for a --enable-optimizations build but could not be found."; + +assert lib.assertMsg (reproducibleBuild -> stripBytecode) + "Deterministic builds require stripping bytecode."; + +assert lib.assertMsg (reproducibleBuild -> (!enableOptimizations)) + "Deterministic builds are not achieved when optimizations are enabled."; + + with lib; let buildPackages = pkgsBuildHost; inherit (passthru) pythonForBuild; + pythonForBuildInterpreter = if stdenv.hostPlatform == stdenv.buildPlatform then + "$out/bin/python" + else pythonForBuild.interpreter; + passthru = passthruFun rec { inherit self sourceVersion packageOverrides; implementation = "cpython"; @@ -272,14 +289,15 @@ in with passthru; stdenv.mkDerivation ({ # Determinism: Windows installers were not deterministic. # We're also not interested in building Windows installers. find "$out" -name 'wininst*.exe' | xargs -r rm -f - '' + optionalString (stdenv.hostPlatform == stdenv.buildPlatform) - '' - # Determinism: rebuild all bytecode - # We exclude lib2to3 because that's Python 2 code which fails - # We rebuild three times, once for each optimization level - find $out -name "*.py" | $out/bin/python -m compileall -q -f -x "lib2to3" -i - - find $out -name "*.py" | $out/bin/python -O -m compileall -q -f -x "lib2to3" -i - - find $out -name "*.py" | $out/bin/python -OO -m compileall -q -f -x "lib2to3" -i - + '' + optionalString stripBytecode '' + # Determinism: deterministic bytecode + # First we delete all old bytecode. + find $out -name "*.pyc" -delete + '' + optionalString rebuildBytecode '' + # Then, we build for the two optimization levels. + # We do not build unoptimized bytecode, because its not entirely deterministic yet. + find $out -name "*.py" | ${pythonForBuildInterpreter} -O -m compileall -q -f -x "lib2to3" -i - + find $out -name "*.py" | ${pythonForBuildInterpreter} -OO -m compileall -q -f -x "lib2to3" -i - '' + optionalString stdenv.hostPlatform.isCygwin '' cp libpython2.7.dll.a $out/lib ''; diff --git a/pkgs/development/interpreters/python/cpython/default.nix b/pkgs/development/interpreters/python/cpython/default.nix index fdf022213c5..6cfe2ad93b5 100644 --- a/pkgs/development/interpreters/python/cpython/default.nix +++ b/pkgs/development/interpreters/python/cpython/default.nix @@ -33,12 +33,11 @@ , stripTests ? false , stripTkinter ? false , rebuildBytecode ? true -, stripBytecode ? false +, stripBytecode ? reproducibleBuild , includeSiteCustomize ? true , static ? stdenv.hostPlatform.isStatic -# Not using optimizations on Darwin -# configure: error: llvm-profdata is required for a --enable-optimizations build but could not be found. -, enableOptimizations ? (!stdenv.isDarwin) +, enableOptimizations ? false +, reproducibleBuild ? true , pythonAttr ? "python${sourceVersion.major}${sourceVersion.minor}" }: @@ -54,6 +53,15 @@ assert x11Support -> tcl != null assert bluezSupport -> bluez != null; +assert lib.assertMsg (enableOptimizations -> (!stdenv.cc.isClang)) + "Optimizations with clang are not supported. configure: error: llvm-profdata is required for a --enable-optimizations build but could not be found."; + +assert lib.assertMsg (reproducibleBuild -> stripBytecode) + "Deterministic builds require stripping bytecode."; + +assert lib.assertMsg (reproducibleBuild -> (!enableOptimizations)) + "Deterministic builds are not achieved when optimizations are enabled."; + with lib; let @@ -360,18 +368,19 @@ in with passthru; stdenv.mkDerivation { '' + optionalString includeSiteCustomize '' # Include a sitecustomize.py file cp ${../sitecustomize.py} $out/${sitePackages}/sitecustomize.py - '' + optionalString rebuildBytecode '' - # Determinism: rebuild all bytecode - # We exclude lib2to3 because that's Python 2 code which fails - # We rebuild three times, once for each optimization level + '' + optionalString stripBytecode '' + # Determinism: deterministic bytecode + # First we delete all old bytecode. + find $out -type d -name __pycache__ -print0 | xargs -0 -I {} rm -rf "{}" + '' + optionalString rebuildBytecode '' + # Then, we build for the two optimization levels. + # We do not build unoptimized bytecode, because its not entirely deterministic yet. # Python 3.7 implements PEP 552, introducing support for deterministic bytecode. - # This is automatically used when `SOURCE_DATE_EPOCH` is set. - find $out -name "*.py" | ${pythonForBuildInterpreter} -m compileall -q -f -x "lib2to3" -i - + # compileall uses this checked-hash method by default when `SOURCE_DATE_EPOCH` is set. + # We exclude lib2to3 because that's Python 2 code which fails find $out -name "*.py" | ${pythonForBuildInterpreter} -O -m compileall -q -f -x "lib2to3" -i - find $out -name "*.py" | ${pythonForBuildInterpreter} -OO -m compileall -q -f -x "lib2to3" -i - - '' + optionalString stripBytecode '' - find $out -type d -name __pycache__ -print0 | xargs -0 -I {} rm -rf "{}" ''; preFixup = lib.optionalString (stdenv.hostPlatform != stdenv.buildPlatform) ''