From e0961cb5f3616f3703a6c73d10b7ab49c325f5fc Mon Sep 17 00:00:00 2001 From: adisbladis Date: Tue, 2 Feb 2021 14:12:10 +0100 Subject: [PATCH] poetry2nix: 1.14.0 -> 1.15.2 --- .../tools/poetry2nix/poetry2nix/default.nix | 85 ++++++---- .../tools/poetry2nix/poetry2nix/lib.nix | 20 ++- .../poetry2nix/poetry2nix/mk-poetry-dep.nix | 4 +- .../tools/poetry2nix/poetry2nix/overrides.nix | 149 ++++++++++++++++-- .../tools/poetry2nix/poetry2nix/pep425.nix | 19 ++- .../tools/poetry2nix/poetry2nix/pep508.nix | 6 +- .../poetry2nix/poetry2nix/pkgs/poetry/update | 1 + pkgs/development/tools/poetry2nix/update | 2 +- 8 files changed, 233 insertions(+), 53 deletions(-) diff --git a/pkgs/development/tools/poetry2nix/poetry2nix/default.nix b/pkgs/development/tools/poetry2nix/poetry2nix/default.nix index b0e65ed85a0..dbec90d8bf5 100644 --- a/pkgs/development/tools/poetry2nix/poetry2nix/default.nix +++ b/pkgs/development/tools/poetry2nix/poetry2nix/default.nix @@ -1,7 +1,7 @@ { pkgs ? import { } , lib ? pkgs.lib , poetry ? null -, poetryLib ? import ./lib.nix { inherit lib pkgs; } +, poetryLib ? import ./lib.nix { inherit lib pkgs; stdenv = pkgs.stdenv; } }: let inherit (poetryLib) isCompatible readTOML moduleName; @@ -71,7 +71,40 @@ in lib.makeScope pkgs.newScope (self: { # Poetry2nix version - version = "1.14.0"; + version = "1.15.2"; + + /* Returns a package of editable sources whose changes will be available without needing to restart the + nix-shell. + In editablePackageSources you can pass a mapping from package name to source directory to have + those packages available in the resulting environment, whose source changes are immediately available. + + */ + mkPoetryEditablePackage = + { projectDir ? null + , pyproject ? projectDir + "/pyproject.toml" + , python ? pkgs.python3 + , pyProject ? readTOML pyproject + # Example: { my-app = ./src; } + , editablePackageSources + }: + assert editablePackageSources != { }; + import ./editable.nix { + inherit pyProject python pkgs lib poetryLib editablePackageSources; + }; + + /* Returns a package containing scripts defined in tool.poetry.scripts. + */ + mkPoetryScriptsPackage = + { projectDir ? null + , pyproject ? projectDir + "/pyproject.toml" + , python ? pkgs.python3 + , pyProject ? readTOML pyproject + , scripts ? pyProject.tool.poetry.scripts + }: + assert scripts != { }; + import ./shell-scripts.nix { + inherit lib python scripts; + }; /* Returns an attrset { python, poetryPackages, pyProject, poetryLock } for the given pyproject/lockfile. @@ -84,11 +117,25 @@ lib.makeScope pkgs.newScope (self: { , python ? pkgs.python3 , pwd ? projectDir , preferWheels ? false + # Example: { my-app = ./src; } + , editablePackageSources ? { } , __isBootstrap ? false # Hack: Always add Poetry as a build input unless bootstrapping }@attrs: let poetryPkg = poetry.override { inherit python; }; pyProject = readTOML pyproject; + + scripts = pyProject.tool.poetry.scripts or { }; + hasScripts = scripts != { }; + scriptsPackage = self.mkPoetryScriptsPackage { + inherit python scripts; + }; + + hasEditable = editablePackageSources != { }; + editablePackage = self.mkPoetryEditablePackage { + inherit pyProject python editablePackageSources; + }; + poetryLock = readTOML poetrylock; lockFiles = let @@ -180,10 +227,13 @@ lib.makeScope pkgs.newScope (self: { inputAttrs = mkInputAttrs { inherit py pyProject; attrs = { }; includeBuildSystem = false; }; + storePackages = builtins.foldl' (acc: v: acc ++ v) [ ] (lib.attrValues inputAttrs); in { python = py; - poetryPackages = builtins.foldl' (acc: v: acc ++ v) [ ] (lib.attrValues inputAttrs); + poetryPackages = storePackages + ++ lib.optional hasScripts scriptsPackage + ++ lib.optional hasEditable editablePackage; poetryLock = poetryLock; inherit pyProject; }; @@ -203,38 +253,17 @@ lib.makeScope pkgs.newScope (self: { , pwd ? projectDir , python ? pkgs.python3 , preferWheels ? false - # Example: { my-app = ./src; } , editablePackageSources ? { } }: let - py = self.mkPoetryPackages ( - { - inherit pyproject poetrylock overrides python pwd preferWheels; - } - ); - - inherit (py) pyProject; - - # Add executables from tool.poetry.scripts - scripts = pyProject.tool.poetry.scripts or { }; - hasScripts = scripts != { }; - scriptsPackage = import ./shell-scripts.nix { - inherit scripts lib; - inherit (py) python; + poetryPython = self.mkPoetryPackages { + inherit pyproject poetrylock overrides python pwd preferWheels editablePackageSources; }; - hasEditable = editablePackageSources != { }; - editablePackage = import ./editable.nix { - inherit pkgs lib poetryLib editablePackageSources; - inherit (py) pyProject python; - }; + inherit (poetryPython) poetryPackages; in - py.python.withPackages ( - _: py.poetryPackages - ++ lib.optional hasEditable editablePackage - ++ lib.optional hasScripts scriptsPackage - ); + poetryPython.python.withPackages (_: poetryPackages); /* Creates a Python application from pyproject.toml and poetry.lock diff --git a/pkgs/development/tools/poetry2nix/poetry2nix/lib.nix b/pkgs/development/tools/poetry2nix/poetry2nix/lib.nix index 6a0ee70f863..4626f7fec37 100644 --- a/pkgs/development/tools/poetry2nix/poetry2nix/lib.nix +++ b/pkgs/development/tools/poetry2nix/poetry2nix/lib.nix @@ -1,4 +1,4 @@ -{ lib, pkgs }: +{ lib, pkgs, stdenv }: let inherit (import ./semver.nix { inherit lib ireplace; }) satisfiesSemver; inherit (builtins) genList length; @@ -194,6 +194,23 @@ let inherit src; }; }; + + # Maps Nixpkgs CPU values to target machines known to be supported for manylinux* wheels. + # (a.k.a. `uname -m` output from CentOS 7) + # + # This is current as of manylinux2014 (PEP-0599), and is a superset of manylinux2010 / manylinux1. + # s390x is not supported in Nixpkgs, so we don't map it. + manyLinuxTargetMachines = { + x86_64 = "x86_64"; + i686 = "i686"; + aarch64 = "aarch64"; + armv7l = "armv7l"; + powerpc64 = "ppc64"; + powerpc64le = "ppc64le"; + }; + + # Machine tag for our target platform (if available) + targetMachine = manyLinuxTargetMachines.${stdenv.targetPlatform.parsed.cpu.name} or null; in { inherit @@ -207,5 +224,6 @@ in cleanPythonSources moduleName getPythonVersion + targetMachine ; } diff --git a/pkgs/development/tools/poetry2nix/poetry2nix/mk-poetry-dep.nix b/pkgs/development/tools/poetry2nix/poetry2nix/mk-poetry-dep.nix index c01f99c01d4..bb7b4e39b03 100644 --- a/pkgs/development/tools/poetry2nix/poetry2nix/mk-poetry-dep.nix +++ b/pkgs/development/tools/poetry2nix/poetry2nix/mk-poetry-dep.nix @@ -31,7 +31,7 @@ pythonPackages.callPackage inherit (poetryLib) isCompatible getManyLinuxDeps fetchFromPypi moduleName; inherit (import ./pep425.nix { - inherit lib python; + inherit lib poetryLib python; inherit (pkgs) stdenv; }) selectWheel ; @@ -161,7 +161,7 @@ pythonPackages.callPackage builtins.fetchGit { inherit (source) url; rev = source.resolved_reference or source.reference; - ref = sourceSpec.branch or sourceSpec.rev or sourceSpec.tag or "HEAD"; + ref = sourceSpec.branch or sourceSpec.rev or (if sourceSpec?tag then "refs/tags/${sourceSpec.tag}" else "HEAD"); } ) else if isUrl then diff --git a/pkgs/development/tools/poetry2nix/poetry2nix/overrides.nix b/pkgs/development/tools/poetry2nix/poetry2nix/overrides.nix index 5810d6d2a74..d400986d3d7 100644 --- a/pkgs/development/tools/poetry2nix/poetry2nix/overrides.nix +++ b/pkgs/development/tools/poetry2nix/poetry2nix/overrides.nix @@ -132,6 +132,18 @@ self: super: } ); + datadog-lambda = super.datadog-lambda.overridePythonAttrs (old: { + postPatch = '' + substituteInPlace setup.py --replace "setuptools==" "setuptools>=" + ''; + buildInputs = old.buildInputs ++ [ self.setuptools ]; + }); + + ddtrace = super.ddtrace.overridePythonAttrs (old: { + buildInputs = old.buildInputs ++ + (pkgs.lib.optionals pkgs.stdenv.isDarwin [ pkgs.darwin.IOKit ]) ++ [ self.cython ]; + }); + dictdiffer = super.dictdiffer.overridePythonAttrs ( old: { buildInputs = old.buildInputs ++ [ self.pytest-runner ]; @@ -235,7 +247,7 @@ self: super: old: if old.format != "wheel" then rec { nativeBuildInputs = old.nativeBuildInputs ++ [ pkgs.pkg-config ]; - buildInputs = old.buildInputs ++ [ pkgs.hdf5 self.pkg-config self.cython ]; + buildInputs = old.buildInputs ++ [ pkgs.hdf5 self.pkgconfig self.cython ]; configure_flags = "--hdf5=${pkgs.hdf5}"; postConfigure = '' ${self.python.executable} setup.py configure ${configure_flags} @@ -407,7 +419,7 @@ self: super: export LLVM_CONFIG=${pkgs.llvm}/bin/llvm-config ''; - __impureHostDeps = pkgs.lib.optionals pkgs.stdenv.isDarwin [ "/usr/lib/libm.dylib" ]; + __impureHostDeps = lib.optionals pkgs.stdenv.isDarwin [ "/usr/lib/libm.dylib" ]; passthru = old.passthru // { llvm = pkgs.llvm; }; } @@ -549,6 +561,12 @@ self: super: } ); + mysqlclient = super.mysqlclient.overridePythonAttrs ( + old: { + buildInputs = old.buildInputs ++ [ pkgs.libmysqlclient ]; + } + ); + netcdf4 = super.netcdf4.overridePythonAttrs ( old: { buildInputs = old.buildInputs ++ [ @@ -615,6 +633,13 @@ self: super: } ); + osqp = super.osqp.overridePythonAttrs ( + old: { + nativeBuildInputs = old.nativeBuildInputs ++ [ pkgs.cmake ]; + dontUseCmakeConfigure = true; + } + ); + parsel = super.parsel.overridePythonAttrs ( old: rec { nativeBuildInputs = old.nativeBuildInputs ++ [ self.pytest-runner ]; @@ -642,6 +667,28 @@ self: super: } ); + # Work around https://github.com/nix-community/poetry2nix/issues/244 + # where git deps are not picked up as they should + pip = + if lib.versionAtLeast super.pip.version "20.3" then + super.pip.overridePythonAttrs + (old: + let + pname = "pip"; + version = "20.2.4"; + in + { + name = pname + "-" + version; + inherit version; + src = pkgs.fetchFromGitHub { + owner = "pypa"; + repo = pname; + rev = version; + sha256 = "eMVV4ftgV71HLQsSeaOchYlfaJVgzNrwUynn3SA1/Do="; + name = "${pname}-${version}-source"; + }; + }) else super.pip; + poetry-core = super.poetry-core.overridePythonAttrs (old: { # "Vendor" dependencies (for build-system support) postPatch = '' @@ -972,6 +1019,10 @@ self: super: pytest = super.pytest.overridePythonAttrs ( old: { + # Fixes https://github.com/pytest-dev/pytest/issues/7891 + postPatch = old.postPatch or "" + '' + sed -i '/\[metadata\]/aversion = ${old.version}' setup.cfg + ''; doCheck = false; } ); @@ -996,6 +1047,28 @@ self: super: } ); + # pytest-splinter seems to put a .marker file in an empty directory + # presumably so it's tracked by and can be installed with MANIFEST.in, see + # https://github.com/pytest-dev/pytest-splinter/commit/a48eeef662f66ff9d3772af618748e73211a186b + # + # This directory then gets used as an empty initial profile directory and is + # zipped up. But if the .marker file is in the Nix store, it has the + # creation date of 1970, and Zip doesn't work with such old files, so it + # fails at runtime! + # + # We fix this here by just removing the file after the installation + # + # The error you get without this is: + # + # E ValueError: ZIP does not support timestamps before 1980 + # /nix/store/55b9ip7xkpimaccw9pa0vacy5q94f5xa-python3-3.7.6/lib/python3.7/zipfile.py:357: ValueError + pytest-splinter = super.pytest-splinter.overrideAttrs (old: { + postInstall = old.postInstall or "" + '' + rm $out/${super.python.sitePackages}/pytest_splinter/profiles/firefox/.marker + ''; + }); + + ffmpeg-python = super.ffmpeg-python.overridePythonAttrs ( old: { buildInputs = old.buildInputs ++ [ self.pytest-runner ]; @@ -1168,14 +1241,16 @@ self: super: # is explicitly disabled with USE_CUDA=0. find $out -name "*.so" -exec ${pkgs.patchelf}/bin/patchelf --remove-needed libcuda.so.1 {} \; ''; - buildInputs = old.buildInputs ++ lib.optionals enableCuda [ + buildInputs = (old.buildInputs or [ ]) + ++ [ self.typing-extensions ] + ++ lib.optionals enableCuda [ pkgs.linuxPackages.nvidia_x11 pkgs.nccl.dev pkgs.nccl.out ]; propagatedBuildInputs = [ - super.numpy - super.future + self.numpy + self.future ]; }) ) @@ -1257,15 +1332,15 @@ self: super: format = "wheel"; }; # If "wheel" is built from source - sourcePackage = ( + sourcePackage = (( pkgs.python3.pkgs.override { python = self.python; } - ).wheel.overridePythonAttrs ( - old: { - inherit (super.wheel) pname name version src; - } - ); + ).wheel.override { + inherit (self) buildPythonPackage bootstrapped-pip setuptools; + }).overrideAttrs (old: { + inherit (super.wheel) pname name version src; + }); in if isWheel then wheelPackage else sourcePackage; @@ -1303,6 +1378,15 @@ self: super: } ); + packaging = super.packaging.overridePythonAttrs ( + old: { + buildInputs = old.buildInputs ++ + # From 20.5 until 20.7, packaging used flit for packaging (heh) + # See https://github.com/pypa/packaging/pull/352 and https://github.com/pypa/packaging/pull/367 + lib.optional (lib.versionAtLeast old.version "20.5" && lib.versionOlder old.version "20.8") [ self.flit-core ]; + } + ); + supervisor = super.supervisor.overridePythonAttrs ( old: { propagatedBuildInputs = old.propagatedBuildInputs ++ [ @@ -1317,4 +1401,47 @@ self: super: propagatedBuildInputs = old.propagatedBuildInputs ++ [ self.toolz ]; } ); + + # For some reason the toml dependency of tqdm declared here: + # https://github.com/tqdm/tqdm/blob/67130a23646ae672836b971e1086b6ae4c77d930/pyproject.toml#L2 + # is not translated correctly to a nix dependency. + tqdm = super.tqdm.overrideAttrs ( + old: { + buildInputs = [ super.toml ] ++ old.buildInputs; + } + ); + + watchdog = super.watchdog.overrideAttrs ( + old: { + buildInputs = old.buildInputs or [ ] + ++ pkgs.lib.optional pkgs.stdenv.isDarwin pkgs.darwin.apple_sdk.frameworks.CoreServices; + } + ); + + # pyee cannot find `vcversioner` and other "setup requirements", so it tries to + # download them from the internet, which only works when nix sandboxing is disabled. + # Additionally, since pyee uses vcversioner to specify its version, we need to do this + # manually specify its version. + pyee = super.pyee.overrideAttrs ( + old: { + postPatch = old.postPatch or "" + '' + sed -i setup.py \ + -e '/setup_requires/,/],/d' \ + -e 's/vcversioner={},/version="${old.version}",/' + ''; + } + ); + + # nixpkgs has setuptools_scm 4.1.2 + # but newrelic has a seemingly unnecessary version constraint for <4 + # So we patch that out + newrelic = super.newrelic.overridePythonAttrs ( + old: { + postPatch = old.postPatch or "" + '' + substituteInPlace setup.py --replace '"setuptools_scm>=3.2,<4"' '"setuptools_scm"' + ''; + } + ); + + } diff --git a/pkgs/development/tools/poetry2nix/poetry2nix/pep425.nix b/pkgs/development/tools/poetry2nix/poetry2nix/pep425.nix index edf556377b3..9e29c5301ab 100644 --- a/pkgs/development/tools/poetry2nix/poetry2nix/pep425.nix +++ b/pkgs/development/tools/poetry2nix/poetry2nix/pep425.nix @@ -1,6 +1,7 @@ -{ lib, stdenv, python, isLinux ? stdenv.isLinux }: +{ lib, stdenv, poetryLib, python, isLinux ? stdenv.isLinux }: let inherit (lib.strings) hasSuffix hasInfix splitString removeSuffix; + inherit (poetryLib) targetMachine; # The 'cpxy" as determined by `python.version` # @@ -72,12 +73,16 @@ let withPlatform = if isLinux then - ( - x: x.platform == "manylinux1_${stdenv.hostPlatform.linuxArch}" - || x.platform == "manylinux2010_${stdenv.hostPlatform.linuxArch}" - || x.platform == "manylinux2014_${stdenv.hostPlatform.linuxArch}" - || x.platform == "any" - ) + if targetMachine != null + then + ( + x: x.platform == "manylinux1_${targetMachine}" + || x.platform == "manylinux2010_${targetMachine}" + || x.platform == "manylinux2014_${targetMachine}" + || x.platform == "any" + ) + else + (x: x.platform == "any") else (x: hasInfix "macosx" x.platform || x.platform == "any"); filterWheel = x: let diff --git a/pkgs/development/tools/poetry2nix/poetry2nix/pep508.nix b/pkgs/development/tools/poetry2nix/poetry2nix/pep508.nix index e8d4159deee..dc9cd2936d0 100644 --- a/pkgs/development/tools/poetry2nix/poetry2nix/pep508.nix +++ b/pkgs/development/tools/poetry2nix/poetry2nix/pep508.nix @@ -1,6 +1,6 @@ { lib, stdenv, poetryLib }: python: let - inherit (poetryLib) ireplace; + inherit (poetryLib) ireplace targetMachine; # Like builtins.substring but with stop being offset instead of length substr = start: stop: s: builtins.substring start (stop - start) s; @@ -95,7 +95,7 @@ let else if stdenv.isDarwin then "darwin" else throw "Unsupported platform" ); - platform_machine = stdenv.hostPlatform.linuxArch; + platform_machine = targetMachine; platform_python_implementation = let impl = python.passthru.implementation; @@ -132,7 +132,7 @@ let mVal = ''[a-zA-Z0-9\'"_\. ]+''; mOp = "in|[!=<>]+"; e = stripStr exprs.value; - m = builtins.map stripStr (builtins.match "^(${mVal}) *(${mOp}) *(${mVal})$" e); + m = builtins.map stripStr (builtins.match ''^(${mVal}) *(${mOp}) *(${mVal})$'' e); m0 = processVar (builtins.elemAt m 0); m2 = processVar (builtins.elemAt m 2); in diff --git a/pkgs/development/tools/poetry2nix/poetry2nix/pkgs/poetry/update b/pkgs/development/tools/poetry2nix/poetry2nix/pkgs/poetry/update index d69878ad084..33a2823f360 100755 --- a/pkgs/development/tools/poetry2nix/poetry2nix/pkgs/poetry/update +++ b/pkgs/development/tools/poetry2nix/poetry2nix/pkgs/poetry/update @@ -3,6 +3,7 @@ rev=$(curl -s https://api.github.com/repos/python-poetry/poetry/releases/latest | jq -r '.name') nix-prefetch-github --rev "$rev" python-poetry poetry > src.json +echo >> src.json src=$(nix-build --no-out-link --expr 'with import {}; fetchFromGitHub (lib.importJSON ./src.json)') cp $src/pyproject.toml $src/poetry.lock . diff --git a/pkgs/development/tools/poetry2nix/update b/pkgs/development/tools/poetry2nix/update index f96763dbee3..41866437aa4 100755 --- a/pkgs/development/tools/poetry2nix/update +++ b/pkgs/development/tools/poetry2nix/update @@ -16,7 +16,7 @@ mv poetry2nix-master/* . mkdir build cp *.* build/ cp -r pkgs hooks bin build/ -rm build/shell.nix build/generate.py build/overlay.nix build/flake.nix +rm build/shell.nix build/generate.py build/overlay.nix build/flake.* cat > build/README.md << EOF Dont change these files here, they are maintained at https://github.com/nix-community/poetry2nix