Switch to PyPA build, installer, and flit-core to build Python packages

pkgs/development/interpreters/python/hooks/default.nix

@@ -1,7 +1,8 @@
 self: dontUse: with self;
 
 let
-  pythonInterpreter = python.pythonForBuild.interpreter;
+  inherit (python) pythonForBuild;
+  pythonInterpreter = pythonForBuild.interpreter;
   pythonSitePackages = python.sitePackages;
   pythonCheckInterpreter = python.interpreter;
   setuppy = ../run_setup.py;
@@ -66,11 +67,9 @@ in {
     makePythonHook {
       name = "pypa-build-hook.sh";
       propagatedBuildInputs = [ build wheel ];
-      substitutions = {
-        inherit pythonInterpreter;
-      };
-    } ./pypa-build-hook.sh) {};
-
+    } ./pypa-build-hook.sh) {
+      inherit (pythonForBuild.pkgs) build;
+    };
 
   pipInstallHook = callPackage ({ makePythonHook, pip }:
     makePythonHook {
@@ -81,6 +80,17 @@ in {
       };
     } ./pip-install-hook.sh) {};
 
+  pypaInstallHook = callPackage ({ makePythonHook, installer }:
+    makePythonHook {
+      name = "pypa-install-hook";
+      propagatedBuildInputs = [ installer ];
+      substitutions = {
+        inherit pythonInterpreter pythonSitePackages;
+      };
+    } ./pypa-install-hook.sh) {
+      inherit (pythonForBuild.pkgs) installer;
+    };
+
   pytestCheckHook = callPackage ({ makePythonHook, pytest }:
     makePythonHook {
       name = "pytest-check-hook";
@@ -133,9 +143,8 @@ in {
   pythonRelaxDepsHook = callPackage ({ makePythonHook, wheel }:
     makePythonHook {
       name = "python-relax-deps-hook";
-      propagatedBuildInputs = [ wheel ];
       substitutions = {
-        inherit pythonInterpreter;
+        inherit pythonInterpreter pythonSitePackages wheel;
       };
     } ./python-relax-deps-hook.sh) {};
 

pkgs/development/interpreters/python/hooks/pypa-build-hook.sh

@@ -6,7 +6,7 @@ pypaBuildPhase() {
     runHook preBuild
 
     echo "Creating a wheel..."
-    @pythonInterpreter@ -m build --no-isolation --outdir dist/ --wheel $pypaBuildFlags
+    pyproject-build --no-isolation --outdir dist/ --wheel $pypaBuildFlags
     echo "Finished creating a wheel..."
 
     runHook postBuild

pkgs/development/interpreters/python/hooks/pypa-install-hook.sh

@@ -0,0 +1,21 @@
+# Setup hook for PyPA installer.
+echo "Sourcing pypa-install-hook"
+
+pypaInstallPhase() {
+    echo "Executing pypaInstallPhase"
+    runHook preInstall
+
+    for wheel in dist/*.whl; do
+        @pythonInterpreter@ -m installer --prefix "$out" "$wheel"
+    done
+
+    export PYTHONPATH="$out/@pythonSitePackages@:$PYTHONPATH"
+
+    runHook postInstall
+    echo "Finished executing pypaInstallPhase"
+}
+
+if [ -z "${dontUsePypaInstall-}" ] && [ -z "${installPhase-}" ]; then
+    echo "Using pypaInstallPhase"
+    installPhase=pypaInstallPhase
+fi

pkgs/development/interpreters/python/hooks/python-relax-deps-hook.sh

@@ -84,7 +84,8 @@ pythonRelaxDepsHook() {
 
     # We generally shouldn't have multiple wheel files, but let's be safer here
     for wheel in "$pkg_name"*".whl"; do
-        @pythonInterpreter@ -m wheel unpack --dest "$unpack_dir" "$wheel"
+        PYTHONPATH="@wheel@/@pythonSitePackages@:$PYTHONPATH" \
+            @pythonInterpreter@ -m wheel unpack --dest "$unpack_dir" "$wheel"
         rm -rf "$wheel"
 
         _pythonRelaxDeps "$metadata_file"
@@ -95,7 +96,8 @@ pythonRelaxDepsHook() {
             cat "$unpack_dir/$pkg_name/$pkg_name.dist-info/METADATA"
         fi
 
-        @pythonInterpreter@ -m wheel pack "$unpack_dir/$pkg_name"
+        PYTHONPATH="@wheel@/@pythonSitePackages@:$PYTHONPATH" \
+            @pythonInterpreter@ -m wheel pack "$unpack_dir/$pkg_name"
     done
 
     # Remove the folder since it will otherwise be in the dist output.

pkgs/development/interpreters/python/mk-python-derivation.nix

@@ -12,8 +12,8 @@
 , update-python-libraries
 , setuptools
 , flitBuildHook
-, pipBuildHook
-, pipInstallHook
+, pypaBuildHook
+, pypaInstallHook
 , pythonCatchConflictsHook
 , pythonImportsCheckHook
 , pythonNamespacesHook
@@ -161,6 +161,20 @@ let
 
     in inputs: builtins.map (checkDrv) inputs;
 
+  isBootstrapInstallPackage = builtins.elem (attrs.pname or null) [
+    "flit-core" "installer"
+  ];
+
+  isBootstrapPackage = isBootstrapInstallPackage || builtins.elem (attrs.pname or null) ([
+    "build" "packaging" "pyproject-hooks" "wheel"
+  ] ++ lib.optionals (python.pythonOlder "3.11") [
+    "tomli"
+  ]);
+
+  isSetuptoolsDependency = builtins.elem (attrs.pname or null) [
+    "setuptools" "wheel"
+  ];
+
   # Keep extra attributes from `attrs`, e.g., `patchPhase', etc.
   self = toPythonModule (stdenv.mkDerivation ((builtins.removeAttrs attrs [
     "disabled" "checkPhase" "checkInputs" "nativeCheckInputs" "doCheck" "doInstallCheck" "dontWrapPythonPrograms" "catchConflicts" "format"
@@ -174,7 +188,15 @@ let
       wrapPython
       ensureNewerSourcesForZipFilesHook  # move to wheel installer (pip) or builder (setuptools, flit, ...)?
       pythonRemoveTestsDirHook
-    ] ++ lib.optionals catchConflicts [
+    ] ++ lib.optionals (catchConflicts && !isBootstrapPackage && !isSetuptoolsDependency) [
+      #
+      # 1. When building a package that is also part of the bootstrap chain, we
+      #    must ignore conflicts after installation, because there will be one with
+      #    the package in the bootstrap.
+      #
+      # 2. When a package is a dependency of setuptools, we must ignore conflicts
+      #    because the hook that checks for conflicts uses setuptools.
+      #
       pythonCatchConflictsHook
     ] ++ lib.optionals removeBinBytecode [
       pythonRemoveBinBytecodeHook
@@ -184,15 +206,26 @@ let
       setuptoolsBuildHook
     ] ++ lib.optionals (format == "flit") [
       flitBuildHook
-    ] ++ lib.optionals (format == "pyproject") [
-      pipBuildHook
-    ] ++ lib.optionals (format == "wheel") [
+    ] ++ lib.optionals (format == "pyproject") [(
+      if isBootstrapPackage then
+        pypaBuildHook.override {
+          inherit (python.pythonForBuild.pkgs.bootstrap) build;
+          wheel = null;
+        }
+      else
+        pypaBuildHook
+    )] ++ lib.optionals (format == "wheel") [
       wheelUnpackHook
     ] ++ lib.optionals (format == "egg") [
       eggUnpackHook eggBuildHook eggInstallHook
-    ] ++ lib.optionals (!(format == "other") || dontUsePipInstall) [
-      pipInstallHook
-    ] ++ lib.optionals (stdenv.buildPlatform == stdenv.hostPlatform) [
+    ] ++ lib.optionals (format != "other") [(
+      if isBootstrapInstallPackage then
+        pypaInstallHook.override {
+          inherit (python.pythonForBuild.pkgs.bootstrap) installer;
+        }
+      else
+        pypaInstallHook
+    )] ++ lib.optionals (stdenv.buildPlatform == stdenv.hostPlatform) [
       # This is a test, however, it should be ran independent of the checkPhase and checkInputs
       pythonImportsCheckHook
     ] ++ lib.optionals (python.pythonAtLeast "3.3") [

pkgs/development/interpreters/python/python-packages-base.nix

@@ -32,12 +32,17 @@ let
       }
       else result;
 
-  buildPythonPackage = makeOverridablePythonPackage (lib.makeOverridable (callPackage ./mk-python-derivation.nix {
+  mkPythonDerivation = if python.isPy3k then
+    ./mk-python-derivation.nix
+  else
+    ./python2/mk-python-derivation.nix;
+
+  buildPythonPackage = makeOverridablePythonPackage (lib.makeOverridable (callPackage mkPythonDerivation {
     inherit namePrefix;     # We want Python libraries to be named like e.g. "python3.6-${name}"
     inherit toPythonModule; # Libraries provide modules
   }));
 
-  buildPythonApplication = makeOverridablePythonPackage (lib.makeOverridable (callPackage ./mk-python-derivation.nix {
+  buildPythonApplication = makeOverridablePythonPackage (lib.makeOverridable (callPackage mkPythonDerivation {
     namePrefix = "";        # Python applications should not have any prefix
     toPythonModule = x: x;  # Application does not provide modules.
   }));