python: reproducible builds

Achieve reproducible builds of the interpreter. Note this meant
disabling optimizations again.
This commit is contained in:
Frederik Rietdijk 2020-12-30 14:49:10 +01:00 committed by Frederik Rietdijk
parent 1adc69d4aa
commit 9d03ff5222
4 changed files with 71 additions and 21 deletions

View file

@ -788,6 +788,23 @@ Each interpreter has the following attributes:
- `executable`. Name of the interpreter executable, e.g. `python3.8`.
- `pkgs`. Set of Python packages for that specific interpreter. The package set can be modified by overriding the interpreter and passing `packageOverrides`.
### Optimizations
The Python interpreters are by default not build with optimizations enabled, because
the builds are in that case not reproducible. To enable optimizations, override the
interpreter of interest, e.g using
```
let
pkgs = import ./. {};
mypython = pkgs.python3.override {
enableOptimizations = true;
reproducibleBuild = false;
self = mypython;
};
in mypython
```
### Building packages and applications
Python libraries and applications that use `setuptools` or

View file

@ -57,6 +57,12 @@
for the motivation).
</para>
</listitem>
<listitem>
</para>
Python optimizations were disabled again. Builds with optimizations enabled
are not reproducible. Optimizations can now be enabled with an option.
<para>
</listitem>
</itemizedlist>
</section>

View file

@ -27,7 +27,10 @@
, sha256
, passthruFun
, static ? false
, enableOptimizations ? (!stdenv.isDarwin)
, stripBytecode ? reproducibleBuild
, rebuildBytecode ? true
, reproducibleBuild ? true
, enableOptimizations ? false
, pythonAttr ? "python${sourceVersion.major}${sourceVersion.minor}"
}:
@ -36,12 +39,26 @@ assert x11Support -> tcl != null
&& xlibsWrapper != null
&& libX11 != null;
assert lib.assertMsg (enableOptimizations -> (!stdenv.cc.isClang))
"Optimizations with clang are not supported. configure: error: llvm-profdata is required for a --enable-optimizations build but could not be found.";
assert lib.assertMsg (reproducibleBuild -> stripBytecode)
"Deterministic builds require stripping bytecode.";
assert lib.assertMsg (reproducibleBuild -> (!enableOptimizations))
"Deterministic builds are not achieved when optimizations are enabled.";
with lib;
let
buildPackages = pkgsBuildHost;
inherit (passthru) pythonForBuild;
pythonForBuildInterpreter = if stdenv.hostPlatform == stdenv.buildPlatform then
"$out/bin/python"
else pythonForBuild.interpreter;
passthru = passthruFun rec {
inherit self sourceVersion packageOverrides;
implementation = "cpython";
@ -272,14 +289,15 @@ in with passthru; stdenv.mkDerivation ({
# Determinism: Windows installers were not deterministic.
# We're also not interested in building Windows installers.
find "$out" -name 'wininst*.exe' | xargs -r rm -f
'' + optionalString (stdenv.hostPlatform == stdenv.buildPlatform)
''
# Determinism: rebuild all bytecode
# We exclude lib2to3 because that's Python 2 code which fails
# We rebuild three times, once for each optimization level
find $out -name "*.py" | $out/bin/python -m compileall -q -f -x "lib2to3" -i -
find $out -name "*.py" | $out/bin/python -O -m compileall -q -f -x "lib2to3" -i -
find $out -name "*.py" | $out/bin/python -OO -m compileall -q -f -x "lib2to3" -i -
'' + optionalString stripBytecode ''
# Determinism: deterministic bytecode
# First we delete all old bytecode.
find $out -name "*.pyc" -delete
'' + optionalString rebuildBytecode ''
# Then, we build for the two optimization levels.
# We do not build unoptimized bytecode, because its not entirely deterministic yet.
find $out -name "*.py" | ${pythonForBuildInterpreter} -O -m compileall -q -f -x "lib2to3" -i -
find $out -name "*.py" | ${pythonForBuildInterpreter} -OO -m compileall -q -f -x "lib2to3" -i -
'' + optionalString stdenv.hostPlatform.isCygwin ''
cp libpython2.7.dll.a $out/lib
'';

View file

@ -33,12 +33,11 @@
, stripTests ? false
, stripTkinter ? false
, rebuildBytecode ? true
, stripBytecode ? false
, stripBytecode ? reproducibleBuild
, includeSiteCustomize ? true
, static ? stdenv.hostPlatform.isStatic
# Not using optimizations on Darwin
# configure: error: llvm-profdata is required for a --enable-optimizations build but could not be found.
, enableOptimizations ? (!stdenv.isDarwin)
, enableOptimizations ? false
, reproducibleBuild ? true
, pythonAttr ? "python${sourceVersion.major}${sourceVersion.minor}"
}:
@ -54,6 +53,15 @@ assert x11Support -> tcl != null
assert bluezSupport -> bluez != null;
assert lib.assertMsg (enableOptimizations -> (!stdenv.cc.isClang))
"Optimizations with clang are not supported. configure: error: llvm-profdata is required for a --enable-optimizations build but could not be found.";
assert lib.assertMsg (reproducibleBuild -> stripBytecode)
"Deterministic builds require stripping bytecode.";
assert lib.assertMsg (reproducibleBuild -> (!enableOptimizations))
"Deterministic builds are not achieved when optimizations are enabled.";
with lib;
let
@ -360,18 +368,19 @@ in with passthru; stdenv.mkDerivation {
'' + optionalString includeSiteCustomize ''
# Include a sitecustomize.py file
cp ${../sitecustomize.py} $out/${sitePackages}/sitecustomize.py
'' + optionalString rebuildBytecode ''
# Determinism: rebuild all bytecode
# We exclude lib2to3 because that's Python 2 code which fails
# We rebuild three times, once for each optimization level
'' + optionalString stripBytecode ''
# Determinism: deterministic bytecode
# First we delete all old bytecode.
find $out -type d -name __pycache__ -print0 | xargs -0 -I {} rm -rf "{}"
'' + optionalString rebuildBytecode ''
# Then, we build for the two optimization levels.
# We do not build unoptimized bytecode, because its not entirely deterministic yet.
# Python 3.7 implements PEP 552, introducing support for deterministic bytecode.
# This is automatically used when `SOURCE_DATE_EPOCH` is set.
find $out -name "*.py" | ${pythonForBuildInterpreter} -m compileall -q -f -x "lib2to3" -i -
# compileall uses this checked-hash method by default when `SOURCE_DATE_EPOCH` is set.
# We exclude lib2to3 because that's Python 2 code which fails
find $out -name "*.py" | ${pythonForBuildInterpreter} -O -m compileall -q -f -x "lib2to3" -i -
find $out -name "*.py" | ${pythonForBuildInterpreter} -OO -m compileall -q -f -x "lib2to3" -i -
'' + optionalString stripBytecode ''
find $out -type d -name __pycache__ -print0 | xargs -0 -I {} rm -rf "{}"
'';
preFixup = lib.optionalString (stdenv.hostPlatform != stdenv.buildPlatform) ''