pythonPackagesExtensions: override all Python package sets at once

Python package sets can be overridden by overriding an interpreter
and passing in `packageOverrides = self: super: {...};`. This is fine
in case you need a single interpreter, however, it does not help you
when you want to override all sets.

With this change it is possible to override all sets at once by
appending a list of "extensions" to `pythonPackagesExtensions`.

From reading the implementation you might wonder why a list is used, and
not
`lib.composeExtensions`? The reason is the latter requires knowledge of
the library function. This approach should be easier for most users
as it is similar to how we append to lists of e.g. inputs or patches
when overriding a derivation.

doc/languages-frameworks/python.section.md

@@ -1664,6 +1664,26 @@ self: super: {
 }
 ```
 
+### How to override a Python package for all Python versions using extensions? {#how-to-override-a-python-package-for-all-python-versions-using-extensions}
+
+The following overlay overrides the call to `buildPythonPackage` for the
+`foo` package for all interpreters by appending a Python extension to the
+`pythonPackagesExtensions` list of extensions.
+
+```nix
+final: prev: {
+  pythonPackagesExtensions = prev.pythonPackagesExtensions ++ [
+    (
+      python-final: python-prev: {
+        foo = python-prev.foo.overridePythonAttrs (oldAttrs: {
+          ...
+        });
+      }
+    )
+  ];
+}
+```
+
 ### How to use Intel’s MKL with numpy and scipy? {#how-to-use-intels-mkl-with-numpy-and-scipy}
 
 MKL can be configured using an overlay. See the section "[Using overlays to

pkgs/development/interpreters/python/cpython/2.7/default.nix

@@ -68,7 +68,7 @@ let
     executable = libPrefix;
     pythonVersion = with sourceVersion; "${major}.${minor}";
     sitePackages = "lib/${libPrefix}/site-packages";
-    inherit hasDistutilsCxxPatch;
+    inherit hasDistutilsCxxPatch pythonAttr;
     pythonOnBuildForBuild = pkgsBuildBuild.${pythonAttr};
     pythonOnBuildForHost = pkgsBuildHost.${pythonAttr};
     pythonOnBuildForTarget = pkgsBuildTarget.${pythonAttr};

pkgs/development/interpreters/python/cpython/default.nix

@@ -91,7 +91,7 @@ let
     executable = libPrefix;
     pythonVersion = with sourceVersion; "${major}.${minor}";
     sitePackages = "lib/${libPrefix}/site-packages";
-    inherit hasDistutilsCxxPatch;
+    inherit hasDistutilsCxxPatch pythonAttr;
     pythonOnBuildForBuild = override pkgsBuildBuild.${pythonAttr};
     pythonOnBuildForHost = override pkgsBuildHost.${pythonAttr};
     pythonOnBuildForTarget = override pkgsBuildTarget.${pythonAttr};

pkgs/development/interpreters/python/default.nix

@@ -19,9 +19,14 @@ with pkgs;
     , pythonOnBuildForTarget
     , pythonOnHostForHost
     , pythonOnTargetForTarget
+    , pythonAttr ? null
     , self # is pythonOnHostForTarget
     }: let
       pythonPackages = callPackage
+        # Function that when called
+        # - imports python-packages.nix
+        # - adds spliced package sets to the package set
+        # - applies overrides from `packageOverrides` and `pythonPackagesOverlays`.
         ({ pkgs, stdenv, python, overrides }: let
           pythonPackagesFun = import ../../../top-level/python-packages.nix {
             inherit stdenv pkgs lib;
@@ -74,7 +79,7 @@ with pkgs;
           extra = _: {};
           optionalExtensions = cond: as: if cond then as else [];
           python2Extension = import ../../../top-level/python2-packages.nix;
-          extensions = lib.composeManyExtensions ((optionalExtensions (!self.isPy3k) [python2Extension]) ++ [ overrides ]);
+          extensions = lib.composeManyExtensions ((optionalExtensions (!self.isPy3k) [python2Extension]) ++ pkgs.pythonPackagesExtensions ++ [ overrides ]);
           aliases = self: super: lib.optionalAttrs config.allowAliases (import ../../../top-level/python-aliases.nix lib self super);
         in lib.makeScopeWithSplicing
           pkgs.splicePackages
@@ -117,6 +122,8 @@ with pkgs;
         tests = callPackage ./tests.nix {
           python = self;
         };
+
+        inherit pythonAttr;
   };
 
   sources = {

pkgs/development/interpreters/python/pypy/default.nix

@@ -30,6 +30,7 @@ let
     executable = "pypy${if isPy3k then "3" else ""}";
     sitePackages = "site-packages";
     hasDistutilsCxxPatch = false;
+    inherit pythonAttr;
 
     pythonOnBuildForBuild = pkgsBuildBuild.${pythonAttr};
     pythonOnBuildForHost = pkgsBuildHost.${pythonAttr};

pkgs/development/interpreters/python/tests.nix

@@ -11,6 +11,7 @@
 , substituteAll
 , lib
 , callPackage
+, pkgs
 }:
 
 let
@@ -133,6 +134,17 @@ let
     # test-overrideScope = let
     #  myPackages = python.pkgs.overrideScope extension;
     # in assert myPackages.foobar == myPackages.numpy; myPackages.python.withPackages(ps: with ps; [ foobar ]);
+  } // lib.optionalAttrs (python ? pythonAttr) {
+    # Test applying overrides using pythonPackagesOverlays.
+    test-pythonPackagesExtensions = let
+      pkgs_ = pkgs.extend(final: prev: {
+        pythonPackagesExtensions = prev.pythonPackagesExtensions ++ [
+          (python-final: python-prev: {
+            foo = python-prev.setuptools;
+          })
+        ];
+      });
+    in pkgs_.${python.pythonAttr}.pkgs.foo;
   };
 
   condaTests = let

pkgs/top-level/all-packages.nix

@@ -14924,6 +14924,8 @@ with pkgs;
   pythonInterpreters = callPackage ./../development/interpreters/python { };
   inherit (pythonInterpreters) python27 python37 python38 python39 python310 python311 python3Minimal pypy27 pypy38 pypy37 rustpython;
 
+  # List of extensions with overrides to apply to all Python package sets.
+  pythonPackagesExtensions = [ ];
   # Python package sets.
   python27Packages = python27.pkgs;
   python37Packages = python37.pkgs;