diff --git a/pkgs/development/interpreters/python/hooks/default.nix b/pkgs/development/interpreters/python/hooks/default.nix index a653ada4837d4..391ccb463e707 100644 --- a/pkgs/development/interpreters/python/hooks/default.nix +++ b/pkgs/development/interpreters/python/hooks/default.nix @@ -1,7 +1,8 @@ self: dontUse: with self; let - pythonInterpreter = python.pythonForBuild.interpreter; + inherit (python) pythonForBuild; + pythonInterpreter = pythonForBuild.interpreter; pythonSitePackages = python.sitePackages; pythonCheckInterpreter = python.interpreter; setuppy = ../run_setup.py; @@ -66,11 +67,9 @@ in { makePythonHook { name = "pypa-build-hook.sh"; propagatedBuildInputs = [ build wheel ]; - substitutions = { - inherit pythonInterpreter; - }; - } ./pypa-build-hook.sh) {}; - + } ./pypa-build-hook.sh) { + inherit (pythonForBuild.pkgs) build; + }; pipInstallHook = callPackage ({ makePythonHook, pip }: makePythonHook { @@ -81,6 +80,17 @@ in { }; } ./pip-install-hook.sh) {}; + pypaInstallHook = callPackage ({ makePythonHook, installer }: + makePythonHook { + name = "pypa-install-hook"; + propagatedBuildInputs = [ installer ]; + substitutions = { + inherit pythonInterpreter pythonSitePackages; + }; + } ./pypa-install-hook.sh) { + inherit (pythonForBuild.pkgs) installer; + }; + pytestCheckHook = callPackage ({ makePythonHook, pytest }: makePythonHook { name = "pytest-check-hook"; @@ -133,9 +143,8 @@ in { pythonRelaxDepsHook = callPackage ({ makePythonHook, wheel }: makePythonHook { name = "python-relax-deps-hook"; - propagatedBuildInputs = [ wheel ]; substitutions = { - inherit pythonInterpreter; + inherit pythonInterpreter pythonSitePackages wheel; }; } ./python-relax-deps-hook.sh) {}; diff --git a/pkgs/development/interpreters/python/hooks/pypa-build-hook.sh b/pkgs/development/interpreters/python/hooks/pypa-build-hook.sh index 3b71300497691..5d77613bf565f 100644 --- a/pkgs/development/interpreters/python/hooks/pypa-build-hook.sh +++ b/pkgs/development/interpreters/python/hooks/pypa-build-hook.sh @@ -6,7 +6,7 @@ pypaBuildPhase() { runHook preBuild echo "Creating a wheel..." - @pythonInterpreter@ -m build --no-isolation --outdir dist/ --wheel $pypaBuildFlags + pyproject-build --no-isolation --outdir dist/ --wheel $pypaBuildFlags echo "Finished creating a wheel..." runHook postBuild diff --git a/pkgs/development/interpreters/python/hooks/pypa-install-hook.sh b/pkgs/development/interpreters/python/hooks/pypa-install-hook.sh new file mode 100644 index 0000000000000..1a7b30116904a --- /dev/null +++ b/pkgs/development/interpreters/python/hooks/pypa-install-hook.sh @@ -0,0 +1,21 @@ +# Setup hook for PyPA installer. +echo "Sourcing pypa-install-hook" + +pypaInstallPhase() { + echo "Executing pypaInstallPhase" + runHook preInstall + + for wheel in dist/*.whl; do + @pythonInterpreter@ -m installer --prefix "$out" "$wheel" + done + + export PYTHONPATH="$out/@pythonSitePackages@:$PYTHONPATH" + + runHook postInstall + echo "Finished executing pypaInstallPhase" +} + +if [ -z "${dontUsePypaInstall-}" ] && [ -z "${installPhase-}" ]; then + echo "Using pypaInstallPhase" + installPhase=pypaInstallPhase +fi diff --git a/pkgs/development/interpreters/python/hooks/python-relax-deps-hook.sh b/pkgs/development/interpreters/python/hooks/python-relax-deps-hook.sh index 31bdec914f690..2c036e90fe54f 100644 --- a/pkgs/development/interpreters/python/hooks/python-relax-deps-hook.sh +++ b/pkgs/development/interpreters/python/hooks/python-relax-deps-hook.sh @@ -84,7 +84,8 @@ pythonRelaxDepsHook() { # We generally shouldn't have multiple wheel files, but let's be safer here for wheel in "$pkg_name"*".whl"; do - @pythonInterpreter@ -m wheel unpack --dest "$unpack_dir" "$wheel" + PYTHONPATH="@wheel@/@pythonSitePackages@:$PYTHONPATH" \ + @pythonInterpreter@ -m wheel unpack --dest "$unpack_dir" "$wheel" rm -rf "$wheel" _pythonRelaxDeps "$metadata_file" @@ -95,7 +96,8 @@ pythonRelaxDepsHook() { cat "$unpack_dir/$pkg_name/$pkg_name.dist-info/METADATA" fi - @pythonInterpreter@ -m wheel pack "$unpack_dir/$pkg_name" + PYTHONPATH="@wheel@/@pythonSitePackages@:$PYTHONPATH" \ + @pythonInterpreter@ -m wheel pack "$unpack_dir/$pkg_name" done # Remove the folder since it will otherwise be in the dist output. diff --git a/pkgs/development/interpreters/python/mk-python-derivation.nix b/pkgs/development/interpreters/python/mk-python-derivation.nix index 79e45b8dc395b..8a75dbf43a742 100644 --- a/pkgs/development/interpreters/python/mk-python-derivation.nix +++ b/pkgs/development/interpreters/python/mk-python-derivation.nix @@ -12,8 +12,8 @@ , update-python-libraries , setuptools , flitBuildHook -, pipBuildHook -, pipInstallHook +, pypaBuildHook +, pypaInstallHook , pythonCatchConflictsHook , pythonImportsCheckHook , pythonNamespacesHook @@ -161,6 +161,20 @@ let in inputs: builtins.map (checkDrv) inputs; + isBootstrapInstallPackage = builtins.elem (attrs.pname or null) [ + "flit-core" "installer" + ]; + + isBootstrapPackage = isBootstrapInstallPackage || builtins.elem (attrs.pname or null) ([ + "build" "packaging" "pyproject-hooks" "wheel" + ] ++ lib.optionals (python.pythonOlder "3.11") [ + "tomli" + ]); + + isSetuptoolsDependency = builtins.elem (attrs.pname or null) [ + "setuptools" "wheel" + ]; + # Keep extra attributes from `attrs`, e.g., `patchPhase', etc. self = toPythonModule (stdenv.mkDerivation ((builtins.removeAttrs attrs [ "disabled" "checkPhase" "checkInputs" "nativeCheckInputs" "doCheck" "doInstallCheck" "dontWrapPythonPrograms" "catchConflicts" "format" @@ -174,7 +188,15 @@ let wrapPython ensureNewerSourcesForZipFilesHook # move to wheel installer (pip) or builder (setuptools, flit, ...)? pythonRemoveTestsDirHook - ] ++ lib.optionals catchConflicts [ + ] ++ lib.optionals (catchConflicts && !isBootstrapPackage && !isSetuptoolsDependency) [ + # + # 1. When building a package that is also part of the bootstrap chain, we + # must ignore conflicts after installation, because there will be one with + # the package in the bootstrap. + # + # 2. When a package is a dependency of setuptools, we must ignore conflicts + # because the hook that checks for conflicts uses setuptools. + # pythonCatchConflictsHook ] ++ lib.optionals removeBinBytecode [ pythonRemoveBinBytecodeHook @@ -184,15 +206,26 @@ let setuptoolsBuildHook ] ++ lib.optionals (format == "flit") [ flitBuildHook - ] ++ lib.optionals (format == "pyproject") [ - pipBuildHook - ] ++ lib.optionals (format == "wheel") [ + ] ++ lib.optionals (format == "pyproject") [( + if isBootstrapPackage then + pypaBuildHook.override { + inherit (python.pythonForBuild.pkgs.bootstrap) build; + wheel = null; + } + else + pypaBuildHook + )] ++ lib.optionals (format == "wheel") [ wheelUnpackHook ] ++ lib.optionals (format == "egg") [ eggUnpackHook eggBuildHook eggInstallHook - ] ++ lib.optionals (!(format == "other") || dontUsePipInstall) [ - pipInstallHook - ] ++ lib.optionals (stdenv.buildPlatform == stdenv.hostPlatform) [ + ] ++ lib.optionals (format != "other") [( + if isBootstrapInstallPackage then + pypaInstallHook.override { + inherit (python.pythonForBuild.pkgs.bootstrap) installer; + } + else + pypaInstallHook + )] ++ lib.optionals (stdenv.buildPlatform == stdenv.hostPlatform) [ # This is a test, however, it should be ran independent of the checkPhase and checkInputs pythonImportsCheckHook ] ++ lib.optionals (python.pythonAtLeast "3.3") [ diff --git a/pkgs/development/interpreters/python/python-packages-base.nix b/pkgs/development/interpreters/python/python-packages-base.nix index 6a723413c850c..91ca84b34b83d 100644 --- a/pkgs/development/interpreters/python/python-packages-base.nix +++ b/pkgs/development/interpreters/python/python-packages-base.nix @@ -32,12 +32,17 @@ let } else result; - buildPythonPackage = makeOverridablePythonPackage (lib.makeOverridable (callPackage ./mk-python-derivation.nix { + mkPythonDerivation = if python.isPy3k then + ./mk-python-derivation.nix + else + ./python2/mk-python-derivation.nix; + + buildPythonPackage = makeOverridablePythonPackage (lib.makeOverridable (callPackage mkPythonDerivation { inherit namePrefix; # We want Python libraries to be named like e.g. "python3.6-${name}" inherit toPythonModule; # Libraries provide modules })); - buildPythonApplication = makeOverridablePythonPackage (lib.makeOverridable (callPackage ./mk-python-derivation.nix { + buildPythonApplication = makeOverridablePythonPackage (lib.makeOverridable (callPackage mkPythonDerivation { namePrefix = ""; # Python applications should not have any prefix toPythonModule = x: x; # Application does not provide modules. })); diff --git a/pkgs/development/interpreters/python/python2/mk-python-derivation.nix b/pkgs/development/interpreters/python/python2/mk-python-derivation.nix new file mode 100644 index 0000000000000..e5f9c00b2fb22 --- /dev/null +++ b/pkgs/development/interpreters/python/python2/mk-python-derivation.nix @@ -0,0 +1,252 @@ +# Generic builder only used for EOL and deprecated Python 2. + +{ lib +, config +, python +, wrapPython +, unzip +, ensureNewerSourcesForZipFilesHook +# Whether the derivation provides a Python module or not. +, toPythonModule +, namePrefix +, update-python-libraries +, setuptools +, pipBuildHook +, pipInstallHook +, pythonCatchConflictsHook +, pythonImportsCheckHook +, pythonOutputDistHook +, pythonRemoveBinBytecodeHook +, pythonRemoveTestsDirHook +, setuptoolsBuildHook +, setuptoolsCheckHook +, wheelUnpackHook +, eggUnpackHook +, eggBuildHook +, eggInstallHook +}: + +{ name ? "${attrs.pname}-${attrs.version}" + +# Build-time dependencies for the package +, nativeBuildInputs ? [] + +# Run-time dependencies for the package +, buildInputs ? [] + +# Dependencies needed for running the checkPhase. +# These are added to buildInputs when doCheck = true. +, checkInputs ? [] +, nativeCheckInputs ? [] + +# propagate build dependencies so in case we have A -> B -> C, +# C can import package A propagated by B +, propagatedBuildInputs ? [] + +# DEPRECATED: use propagatedBuildInputs +, pythonPath ? [] + +# Enabled to detect some (native)BuildInputs mistakes +, strictDeps ? true + +, outputs ? [ "out" ] + +# used to disable derivation, useful for specific python versions +, disabled ? false + +# Raise an error if two packages are installed with the same name +# TODO: For cross we probably need a different PYTHONPATH, or not +# add the runtime deps until after buildPhase. +, catchConflicts ? (python.stdenv.hostPlatform == python.stdenv.buildPlatform) + +# Additional arguments to pass to the makeWrapper function, which wraps +# generated binaries. +, makeWrapperArgs ? [] + +# Skip wrapping of python programs altogether +, dontWrapPythonPrograms ? false + +# Don't use Pip to install a wheel +# Note this is actually a variable for the pipInstallPhase in pip's setupHook. +# It's included here to prevent an infinite recursion. +, dontUsePipInstall ? false + +# Skip setting the PYTHONNOUSERSITE environment variable in wrapped programs +, permitUserSite ? false + +# Remove bytecode from bin folder. +# When a Python script has the extension `.py`, bytecode is generated +# Typically, executables in bin have no extension, so no bytecode is generated. +# However, some packages do provide executables with extensions, and thus bytecode is generated. +, removeBinBytecode ? true + +# Several package formats are supported. +# "setuptools" : Install a common setuptools/distutils based package. This builds a wheel. +# "wheel" : Install from a pre-compiled wheel. +# "pyproject": Install a package using a ``pyproject.toml`` file (PEP517). This builds a wheel. +# "egg": Install a package from an egg. +# "other" : Provide your own buildPhase and installPhase. +, format ? "setuptools" + +, meta ? {} + +, passthru ? {} + +, doCheck ? config.doCheckByDefault or false + +, disabledTestPaths ? [] + +, ... } @ attrs: + +assert lib.assertMsg (format != "flit") "flit is not a supported Python 2 format"; + +let + inherit (python) stdenv; + + withDistOutput = lib.elem format ["pyproject" "setuptools" "flit" "wheel"]; + + name_ = name; + + validatePythonMatches = attrName: let + isPythonModule = drv: + # all pythonModules have the pythonModule attribute + (drv ? "pythonModule") + # Some pythonModules are turned in to a pythonApplication by setting the field to false + && (!builtins.isBool drv.pythonModule); + isMismatchedPython = drv: drv.pythonModule != python; + + optionalLocation = let + pos = builtins.unsafeGetAttrPos (if attrs ? "pname" then "pname" else "name") attrs; + in lib.optionalString (pos != null) " at ${pos.file}:${toString pos.line}:${toString pos.column}"; + + leftPadName = name: against: let + len = lib.max (lib.stringLength name) (lib.stringLength against); + in lib.strings.fixedWidthString len " " name; + + throwMismatch = drv: let + myName = "'${namePrefix}${name}'"; + theirName = "'${drv.name}'"; + in throw '' + Python version mismatch in ${myName}: + + The Python derivation ${myName} depends on a Python derivation + named ${theirName}, but the two derivations use different versions + of Python: + + ${leftPadName myName theirName} uses ${python} + ${leftPadName theirName myName} uses ${toString drv.pythonModule} + + Possible solutions: + + * If ${theirName} is a Python library, change the reference to ${theirName} + in the ${attrName} of ${myName} to use a ${theirName} built from the same + version of Python + + * If ${theirName} is used as a tool during the build, move the reference to + ${theirName} in ${myName} from ${attrName} to nativeBuildInputs + + * If ${theirName} provides executables that are called at run time, pass its + bin path to makeWrapperArgs: + + makeWrapperArgs = [ "--prefix PATH : ''${lib.makeBinPath [ ${lib.getName drv } ] }" ]; + + ${optionalLocation} + ''; + + checkDrv = drv: + if (isPythonModule drv) && (isMismatchedPython drv) + then throwMismatch drv + else drv; + + in inputs: builtins.map (checkDrv) inputs; + + # Keep extra attributes from `attrs`, e.g., `patchPhase', etc. + self = toPythonModule (stdenv.mkDerivation ((builtins.removeAttrs attrs [ + "disabled" "checkPhase" "checkInputs" "nativeCheckInputs" "doCheck" "doInstallCheck" "dontWrapPythonPrograms" "catchConflicts" "format" + "disabledTestPaths" "outputs" + ]) // { + + name = namePrefix + name_; + + nativeBuildInputs = [ + python + wrapPython + ensureNewerSourcesForZipFilesHook # move to wheel installer (pip) or builder (setuptools, flit, ...)? + pythonRemoveTestsDirHook + ] ++ lib.optionals catchConflicts [ + pythonCatchConflictsHook + ] ++ lib.optionals removeBinBytecode [ + pythonRemoveBinBytecodeHook + ] ++ lib.optionals (lib.hasSuffix "zip" (attrs.src.name or "")) [ + unzip + ] ++ lib.optionals (format == "setuptools") [ + setuptoolsBuildHook + ] ++ lib.optionals (format == "pyproject") [( + pipBuildHook + )] ++ lib.optionals (format == "wheel") [ + wheelUnpackHook + ] ++ lib.optionals (format == "egg") [ + eggUnpackHook eggBuildHook eggInstallHook + ] ++ lib.optionals (format != "other") [( + pipInstallHook + )] ++ lib.optionals (stdenv.buildPlatform == stdenv.hostPlatform) [ + # This is a test, however, it should be ran independent of the checkPhase and checkInputs + pythonImportsCheckHook + ] ++ lib.optionals withDistOutput [ + pythonOutputDistHook + ] ++ nativeBuildInputs; + + buildInputs = validatePythonMatches "buildInputs" (buildInputs ++ pythonPath); + + propagatedBuildInputs = validatePythonMatches "propagatedBuildInputs" (propagatedBuildInputs ++ [ + # we propagate python even for packages transformed with 'toPythonApplication' + # this pollutes the PATH but avoids rebuilds + # see https://github.com/NixOS/nixpkgs/issues/170887 for more context + python + ]); + + inherit strictDeps; + + LANG = "${if python.stdenv.isDarwin then "en_US" else "C"}.UTF-8"; + + # Python packages don't have a checkPhase, only an installCheckPhase + doCheck = false; + doInstallCheck = attrs.doCheck or true; + nativeInstallCheckInputs = [ + ] ++ lib.optionals (format == "setuptools") [ + # Longer-term we should get rid of this and require + # users of this function to set the `installCheckPhase` or + # pass in a hook that sets it. + setuptoolsCheckHook + ] ++ nativeCheckInputs; + installCheckInputs = checkInputs; + + postFixup = lib.optionalString (!dontWrapPythonPrograms) '' + wrapPythonPrograms + '' + attrs.postFixup or ""; + + # Python packages built through cross-compilation are always for the host platform. + disallowedReferences = lib.optionals (python.stdenv.hostPlatform != python.stdenv.buildPlatform) [ python.pythonForBuild ]; + + outputs = outputs ++ lib.optional withDistOutput "dist"; + + meta = { + # default to python's platforms + platforms = python.meta.platforms; + isBuildPythonPackage = python.meta.platforms; + } // meta; + } // lib.optionalAttrs (attrs?checkPhase) { + # If given use the specified checkPhase, otherwise use the setup hook. + # Longer-term we should get rid of `checkPhase` and use `installCheckPhase`. + installCheckPhase = attrs.checkPhase; + } // lib.optionalAttrs (disabledTestPaths != []) { + disabledTestPaths = lib.escapeShellArgs disabledTestPaths; + })); + + passthru.updateScript = let + filename = builtins.head (lib.splitString ":" self.meta.position); + in attrs.passthru.updateScript or [ update-python-libraries filename ]; +in lib.extendDerivation + (disabled -> throw "${name} not supported for interpreter ${python.executable}") + passthru + self diff --git a/pkgs/development/python-modules/bootstrap/build/default.nix b/pkgs/development/python-modules/bootstrap/build/default.nix new file mode 100644 index 0000000000000..639d2e3292cb0 --- /dev/null +++ b/pkgs/development/python-modules/bootstrap/build/default.nix @@ -0,0 +1,49 @@ +{ lib +, stdenv +, python +, build +, flit-core +, installer +, packaging +, pyproject-hooks +, tomli +}: +let + buildBootstrapPythonModule = basePackage: attrs: stdenv.mkDerivation ({ + pname = "${python.libPrefix}-bootstrap-${basePackage.pname}"; + inherit (basePackage) version src meta; + + buildPhase = '' + runHook preBuild + + PYTHONPATH="${flit-core}/${python.sitePackages}" \ + ${python.interpreter} -m flit_core.wheel + + runHook postBuild + ''; + + installPhase = '' + runHook preInstall + + PYTHONPATH="${installer}/${python.sitePackages}" \ + ${python.interpreter} -m installer \ + --destdir "$out" --prefix "" dist/*.whl + + runHook postInstall + ''; + } // attrs); + + bootstrap-packaging = buildBootstrapPythonModule packaging {}; + + bootstrap-pyproject-hooks = buildBootstrapPythonModule pyproject-hooks {}; + + bootstrap-tomli = buildBootstrapPythonModule tomli {}; +in +buildBootstrapPythonModule build { + propagatedBuildInputs = [ + bootstrap-packaging + bootstrap-pyproject-hooks + ] ++ lib.optionals (python.pythonOlder "3.11") [ + bootstrap-tomli + ]; +} diff --git a/pkgs/development/python-modules/bootstrap/flit-core/default.nix b/pkgs/development/python-modules/bootstrap/flit-core/default.nix new file mode 100644 index 0000000000000..ab9e52538d341 --- /dev/null +++ b/pkgs/development/python-modules/bootstrap/flit-core/default.nix @@ -0,0 +1,29 @@ +{ lib +, stdenv +, python +, flit-core +}: + +stdenv.mkDerivation { + pname = "${python.libPrefix}-bootstrap-${flit-core.pname}"; + inherit (flit-core) version src patches meta; + + sourceRoot = "source/flit_core"; + + buildPhase = '' + runHook preBuild + + ${python.interpreter} -m flit_core.wheel + + runHook postBuild + ''; + + installPhase = '' + runHook preInstall + + ${python.interpreter} bootstrap_install.py dist/flit_core-*.whl \ + --install-root "$out" --installdir "/${python.sitePackages}" + + runHook postInstall + ''; +} diff --git a/pkgs/development/python-modules/bootstrap/installer/default.nix b/pkgs/development/python-modules/bootstrap/installer/default.nix new file mode 100644 index 0000000000000..b569e7fa6dc80 --- /dev/null +++ b/pkgs/development/python-modules/bootstrap/installer/default.nix @@ -0,0 +1,29 @@ +{ lib +, stdenv +, python +, flit-core +, installer +}: + +stdenv.mkDerivation { + pname = "${python.libPrefix}-bootstrap-${installer.pname}"; + inherit (installer) version src meta; + + buildPhase = '' + runHook preBuild + + PYTHONPATH="${flit-core}/${python.sitePackages}" \ + ${python.interpreter} -m flit_core.wheel + + runHook postBuild + ''; + + installPhase = '' + runHook preInstall + + PYTHONPATH=src ${python.interpreter} -m installer \ + --destdir "$out" --prefix "" dist/installer-*.whl + + runHook postInstall + ''; +} diff --git a/pkgs/development/python-modules/build/default.nix b/pkgs/development/python-modules/build/default.nix index e7c45afdd317d..a61c6166632b9 100644 --- a/pkgs/development/python-modules/build/default.nix +++ b/pkgs/development/python-modules/build/default.nix @@ -1,5 +1,6 @@ { lib , stdenv +, build , buildPythonPackage , fetchFromGitHub , flit-core @@ -41,36 +42,52 @@ buildPythonPackage rec { tomli ]; - nativeCheckInputs = [ - filelock - pytest-mock - pytest-rerunfailures - pytest-xdist - pytestCheckHook - setuptools - toml - ]; + # We need to disable tests because this package is part of the bootstrap chain + # and its test dependencies cannot be built yet when this is being built. + doCheck = false; - pytestFlagsArray = [ - "-W" - "ignore::DeprecationWarning" - ]; + passthru.tests = { + pytest = buildPythonPackage { + pname = "${pname}-pytest"; + inherit version; + format = "other"; - __darwinAllowLocalNetworking = true; + dontBuild = true; + dontInstall = true; - disabledTests = [ - # Tests often fail with StopIteration - "test_isolat" - "test_default_pip_is_never_too_old" - "test_build" - "test_with_get_requires" - "test_init" - "test_output" - "test_wheel_metadata" - ] ++ lib.optionals stdenv.isDarwin [ - # Expects Apple's Python and its quirks - "test_can_get_venv_paths_with_conflicting_default_scheme" - ]; + nativeCheckInputs = [ + build + filelock + pytest-mock + pytest-rerunfailures + pytest-xdist + pytestCheckHook + setuptools + toml + ]; + + pytestFlagsArray = [ + "-W" + "ignore::DeprecationWarning" + ]; + + __darwinAllowLocalNetworking = true; + + disabledTests = [ + # Tests often fail with StopIteration + "test_isolat" + "test_default_pip_is_never_too_old" + "test_build" + "test_with_get_requires" + "test_init" + "test_output" + "test_wheel_metadata" + ] ++ lib.optionals stdenv.isDarwin [ + # Expects Apple's Python and its quirks + "test_can_get_venv_paths_with_conflicting_default_scheme" + ]; + }; + }; pythonImportsCheck = [ "build" diff --git a/pkgs/development/python-modules/cypari2/default.nix b/pkgs/development/python-modules/cypari2/default.nix index bc8a5df2ed177..70cd89afdcade 100644 --- a/pkgs/development/python-modules/cypari2/default.nix +++ b/pkgs/development/python-modules/cypari2/default.nix @@ -37,11 +37,12 @@ buildPythonPackage rec { export PYTHONPATH="$out/${python.sitePackages}:$PYTHONPATH" # install "." instead of "*.whl" - ${python.pythonForBuild.pkgs.bootstrapped-pip}/bin/pip install . --no-index --no-warn-script-location --prefix="$out" --no-cache + pip install . --no-index --no-warn-script-location --prefix="$out" --no-cache ''; nativeBuildInputs = [ pari + python.pythonForBuild.pkgs.pip ]; buildInputs = [ diff --git a/pkgs/development/python-modules/installer/default.nix b/pkgs/development/python-modules/installer/default.nix index 9d3c8185322a1..7bf1f7bff27ac 100644 --- a/pkgs/development/python-modules/installer/default.nix +++ b/pkgs/development/python-modules/installer/default.nix @@ -4,6 +4,7 @@ , fetchFromGitHub , pytestCheckHook , flit-core +, installer , mock }: @@ -21,10 +22,26 @@ buildPythonPackage rec { nativeBuildInputs = [ flit-core ]; - nativeCheckInputs = [ - pytestCheckHook - mock - ]; + # We need to disable tests because this package is part of the bootstrap chain + # and its test dependencies cannot be built yet when this is being built. + doCheck = false; + + passthru.tests = { + pytest = buildPythonPackage { + pname = "${pname}-pytest"; + inherit version; + format = "other"; + + dontBuild = true; + dontInstall = true; + + nativeCheckInputs = [ + installer + mock + pytestCheckHook + ]; + }; + }; meta = with lib; { changelog = "https://github.com/pypa/installer/blob/${src.rev}/docs/changelog.md"; diff --git a/pkgs/development/python-modules/pip-tools/default.nix b/pkgs/development/python-modules/pip-tools/default.nix index f1fcf59ce2fe1..d549822713556 100644 --- a/pkgs/development/python-modules/pip-tools/default.nix +++ b/pkgs/development/python-modules/pip-tools/default.nix @@ -11,19 +11,21 @@ , pythonOlder , setuptools , setuptools-scm +, tomli +, tomli-w , wheel }: buildPythonPackage rec { pname = "pip-tools"; - version = "6.13.0"; + version = "7.2.0"; format = "pyproject"; - disabled = pythonOlder "3.7"; + disabled = pythonOlder "3.8"; src = fetchPypi { inherit pname version; - hash = "sha256-YdRr0uuAFu1Kkk4Zbm5bCiaM07q9eeWTBIcg2yNSK7E="; + hash = "sha256-YWSItTnhS4qoVDbtWXozwpH0iFwdLgvsl0AKvlr/LA0="; }; patches = [ ./fix-setup-py-bad-syntax-detection.patch ]; @@ -39,11 +41,16 @@ buildPythonPackage rec { pip setuptools wheel + ] ++ lib.optionals (pythonOlder "3.11") [ + tomli ]; + __darwinAllowLocalNetworking = true; + nativeCheckInputs = [ pytest-xdist pytestCheckHook + tomli-w ]; preCheck = lib.optionalString (stdenv.isDarwin && stdenv.isAarch64) '' diff --git a/pkgs/development/python-modules/pip/default.nix b/pkgs/development/python-modules/pip/default.nix index ebac5724d66fa..bc06cb39e247b 100644 --- a/pkgs/development/python-modules/pip/default.nix +++ b/pkgs/development/python-modules/pip/default.nix @@ -1,10 +1,12 @@ { lib , buildPythonPackage -, bootstrapped-pip , fetchFromGitHub +, installShellFiles , mock , scripttest +, setuptools , virtualenv +, wheel , pretend , pytest @@ -14,33 +16,40 @@ buildPythonPackage rec { pname = "pip"; - version = "23.0.1"; - format = "other"; + version = "23.2.1"; + format = "pyproject"; src = fetchFromGitHub { owner = "pypa"; repo = pname; rev = "refs/tags/${version}"; - hash = "sha256-BSonlwKmegrlrQTTIL0avPi61/TY2M0f7kOZpSzPRQk="; - name = "${pname}-${version}-source"; + hash = "sha256-mUlzfYmq1FE3X1/2o7sYJzMgwHRI4ib4EMhpg83VvrI="; }; - nativeBuildInputs = [ bootstrapped-pip ]; - postPatch = '' # Remove vendored Windows PE binaries # Note: These are unused but make the package unreproducible. find -type f -name '*.exe' -delete ''; - # pip detects that we already have bootstrapped_pip "installed", so we need - # to force it a little. - pipInstallFlags = [ "--ignore-installed" ]; + nativeBuildInputs = [ + installShellFiles + setuptools + wheel + ]; nativeCheckInputs = [ mock scripttest virtualenv pretend pytest ]; + # Pip wants pytest, but tests are not distributed doCheck = false; + postInstall = '' + installShellCompletion --cmd pip \ + --bash <($out/bin/pip completion --bash) \ + --fish <($out/bin/pip completion --fish) \ + --zsh <($out/bin/pip completion --zsh) + ''; + passthru.tests = { inherit pip-tools; }; meta = { @@ -48,6 +57,5 @@ buildPythonPackage rec { license = with lib.licenses; [ mit ]; homepage = "https://pip.pypa.io/"; changelog = "https://pip.pypa.io/en/stable/news/#v${lib.replaceStrings [ "." ] [ "-" ] version}"; - priority = 10; }; } diff --git a/pkgs/development/python-modules/pyproject-api/default.nix b/pkgs/development/python-modules/pyproject-api/default.nix index 6ca6ac25a98d3..6cbef5b8a77f1 100644 --- a/pkgs/development/python-modules/pyproject-api/default.nix +++ b/pkgs/development/python-modules/pyproject-api/default.nix @@ -6,7 +6,6 @@ # build time , hatchling , hatch-vcs -, setuptools-scm # runtime , packaging @@ -21,13 +20,14 @@ # tests , pytest-mock , pytestCheckHook +, setuptools , virtualenv , wheel }: buildPythonPackage rec { pname = "pyproject-api"; - version = "1.5.0"; + version = "1.5.4"; format = "pyproject"; disabled = pythonOlder "3.8"; @@ -36,7 +36,7 @@ buildPythonPackage rec { owner = "tox-dev"; repo = "pyproject-api"; rev = "refs/tags/${version}"; - hash = "sha256-VO+huA9i7uMpCVaWHC29XlfestSu+N9vWWHteY21uqs="; + hash = "sha256-HX+5BypfEOfQ3vg3vha0QCVrEarjMu/Q8id+xgmWGfA="; }; outputs = [ @@ -44,12 +44,11 @@ buildPythonPackage rec { "doc" ]; - SETUPTOOLS_SCM_PRETEND_VERSION = version; + env.SETUPTOOLS_SCM_PRETEND_VERSION = version; nativeBuildInputs = [ hatchling hatch-vcs - setuptools-scm # docs sphinxHook @@ -66,6 +65,7 @@ buildPythonPackage rec { nativeCheckInputs = [ pytest-mock pytestCheckHook + setuptools virtualenv wheel ]; diff --git a/pkgs/development/python-modules/pyproject-hooks/default.nix b/pkgs/development/python-modules/pyproject-hooks/default.nix index 76bf3902c3d21..147e173db2daa 100644 --- a/pkgs/development/python-modules/pyproject-hooks/default.nix +++ b/pkgs/development/python-modules/pyproject-hooks/default.nix @@ -2,6 +2,7 @@ , buildPythonPackage , fetchPypi , flit-core +, pyproject-hooks , pytestCheckHook , pythonOlder , setuptools @@ -31,17 +32,33 @@ buildPythonPackage rec { tomli ]; - nativeCheckInputs = [ - pytestCheckHook - setuptools - testpath - ]; + # We need to disable tests because this package is part of the bootstrap chain + # and its test dependencies cannot be built yet when this is being built. + doCheck = false; - disabledTests = [ - # fail to import setuptools - "test_setup_py" - "test_issue_104" - ]; + passthru.tests = { + pytest = buildPythonPackage { + pname = "${pname}-pytest"; + inherit version; + format = "other"; + + dontBuild = true; + dontInstall = true; + + nativeCheckInputs = [ + pyproject-hooks + pytestCheckHook + setuptools + testpath + ]; + + disabledTests = [ + # fail to import setuptools + "test_setup_py" + "test_issue_104" + ]; + }; + }; pythonImportsCheck = [ "pyproject_hooks" diff --git a/pkgs/development/python-modules/setuptools/default.nix b/pkgs/development/python-modules/setuptools/default.nix index c851cef24b308..ad7638f3cb38b 100644 --- a/pkgs/development/python-modules/setuptools/default.nix +++ b/pkgs/development/python-modules/setuptools/default.nix @@ -1,73 +1,36 @@ { stdenv +, lib , buildPythonPackage , fetchFromGitHub , python -, bootstrapped-pip -, lib -, pipInstallHook -, setuptoolsBuildHook +, wheel }: -let +buildPythonPackage rec { pname = "setuptools"; - version = "67.4.0"; - - # Create an sdist of setuptools - sdist = stdenv.mkDerivation rec { - name = "${pname}-${version}-sdist.tar.gz"; - - src = fetchFromGitHub { - owner = "pypa"; - repo = pname; - rev = "refs/tags/v${version}"; - hash = "sha256-QDHycUFA2VRUE9alan8rF0efZTNV3Jt0CskjkCc+in0="; - name = "${pname}-${version}-source"; - }; - - patches = [ - ./tag-date.patch - ./setuptools-distutils-C++.patch - ]; - - buildPhase = '' - ${python.pythonForBuild.interpreter} setup.py egg_info - ${python.pythonForBuild.interpreter} setup.py sdist --formats=gztar - - # Here we untar the sdist and retar it in order to control the timestamps - # of all the files included - tar -xzf dist/${pname}-${version}.post0.tar.gz -C dist/ - tar -czf dist/${name} -C dist/ --mtime="@$SOURCE_DATE_EPOCH" --sort=name ${pname}-${version}.post0 - ''; - - installPhase = '' - echo "Moving sdist..." - mv dist/${name} $out - ''; + version = "68.0.0"; + format = "pyproject"; + + src = fetchFromGitHub { + owner = "pypa"; + repo = "setuptools"; + rev = "refs/tags/v${version}"; + hash = "sha256-Ew/DpI5XsYMB4kBfkFeeB5wv5cjx4oYTddZCWLgBHBs="; }; -in buildPythonPackage { - inherit pname version; - # Because of bootstrapping we don't use the setuptoolsBuildHook that comes with format="setuptools" directly. - # Instead, we override it to remove setuptools to avoid a circular dependency. - # The same is done for pip and the pipInstallHook. - format = "other"; - src = sdist; + patches = [ + ./tag-date.patch + ./setuptools-distutils-C++.patch + ]; nativeBuildInputs = [ - bootstrapped-pip - (pipInstallHook.override{pip=null;}) - (setuptoolsBuildHook.override{setuptools=null; wheel=null;}) + wheel ]; preBuild = lib.optionalString (!stdenv.hostPlatform.isWindows) '' export SETUPTOOLS_INSTALL_WINDOWS_SPECIFIC_FILES=0 ''; - pipInstallFlags = [ "--ignore-installed" ]; - - # Adds setuptools to nativeBuildInputs causing infinite recursion. - catchConflicts = false; - # Requires pytest, causing infinite recursion. doCheck = false; @@ -77,7 +40,6 @@ in buildPythonPackage { changelog = "https://setuptools.pypa.io/en/stable/history.html#v${replaceStrings [ "." ] [ "-" ] version}"; license = with licenses; [ mit ]; platforms = python.meta.platforms; - priority = 10; maintainers = teams.python.members; }; } diff --git a/pkgs/development/python-modules/wheel/default.nix b/pkgs/development/python-modules/wheel/default.nix index 3be508a0ec590..16c6c2dc087d1 100644 --- a/pkgs/development/python-modules/wheel/default.nix +++ b/pkgs/development/python-modules/wheel/default.nix @@ -1,21 +1,19 @@ { lib , buildPythonPackage , fetchFromGitHub -, bootstrapped-pip -, setuptools +, flit-core }: buildPythonPackage rec { pname = "wheel"; - version = "0.38.4"; - format = "other"; + version = "0.41.1"; + format = "pyproject"; src = fetchFromGitHub { owner = "pypa"; repo = pname; rev = version; - hash = "sha256-yZLU0t/nz6kfnnoLL15bybOxN4+SJUaTJsCpGffl1QU="; - name = "${pname}-${version}-source"; + hash = "sha256-/EaDJ2zI/ly2BrrGhiZGwiBYDVPYWTki+87UqtCS3bw="; postFetch = '' cd $out mv tests/testdata/unicode.dist/unicodedist/åäö_日本語.py \ @@ -25,16 +23,13 @@ buildPythonPackage rec { }; nativeBuildInputs = [ - bootstrapped-pip - setuptools + flit-core ]; # No tests in archive doCheck = false; - pythonImportsCheck = [ "wheel" ]; - # We add this flag to ignore the copy installed by bootstrapped-pip - pipInstallFlags = [ "--ignore-installed" ]; + pythonImportsCheck = [ "wheel" ]; meta = with lib; { homepage = "https://github.com/pypa/wheel"; diff --git a/pkgs/development/tools/poetry2nix/poetry2nix/overrides/build-systems.json b/pkgs/development/tools/poetry2nix/poetry2nix/overrides/build-systems.json index b212fffc1b1c6..176881caeed1f 100644 --- a/pkgs/development/tools/poetry2nix/poetry2nix/overrides/build-systems.json +++ b/pkgs/development/tools/poetry2nix/poetry2nix/overrides/build-systems.json @@ -2505,8 +2505,14 @@ "setuptools" ], "build": [ - "flit-core", - "setuptools" + { + "buildSystem": "setuptools", + "until": "0.10.0" + }, + { + "buildSystem": "flit-core", + "from": "0.10.0" + } ], "buildbot": [ "setuptools" @@ -5812,11 +5818,7 @@ "setuptools" ], "flit": [ - "flit-core", - "setuptools" - ], - "flit-core": [ - "setuptools" + "flit-core" ], "flit-scm": [ "flit-core", @@ -7540,8 +7542,7 @@ "setuptools" ], "installer": [ - "flit-core", - "setuptools" + "flit-core" ], "insteon-frontend-home-assistant": [ "setuptools" @@ -10863,8 +10864,24 @@ "setuptools" ], "packaging": [ - "flit-core", - "setuptools" + { + "buildSystem": "setuptools", + "until": "20.4" + }, + { + "buildSystem": "flit-core", + "from": "20.4", + "until": "20.8" + }, + { + "buildSystem": "setuptools", + "from": "20.8", + "until": "22.0" + }, + { + "buildSystem": "flit-core", + "from": "22.0" + } ], "packbits": [ "setuptools" @@ -18088,8 +18105,14 @@ "poetry" ], "tomli": [ - "flit-core", - "setuptools" + { + "buildSystem": "poetry-core", + "until": "1.0.2" + }, + { + "buildSystem": "flit-core", + "from": "1.0.2" + } ], "tomli-w": [ "flit-core", @@ -19331,7 +19354,14 @@ "setuptools" ], "wheel": [ - "setuptools" + { + "buildSystem": "setuptools", + "until": "0.40.0" + }, + { + "buildSystem": "flit-core", + "from": "0.40.0" + } ], "wheel-filename": [ "setuptools" diff --git a/pkgs/development/tools/poetry2nix/poetry2nix/overrides/default.nix b/pkgs/development/tools/poetry2nix/poetry2nix/overrides/default.nix index 14338423ec390..d96d920c70940 100644 --- a/pkgs/development/tools/poetry2nix/poetry2nix/overrides/default.nix +++ b/pkgs/development/tools/poetry2nix/poetry2nix/overrides/default.nix @@ -2738,15 +2738,12 @@ lib.composeManyExtensions [ } ); - wheel = (( - pkgs.python3.pkgs.override { - python = self.python; + wheel = if self.python.isPy2 then + super.wheel.override { + inherit (self) bootstrapped-pip; } - ).wheel.override { - inherit (self) buildPythonPackage bootstrapped-pip setuptools; - }).overrideAttrs (old: { - inherit (super.wheel) pname name version src; - }); + else + super.wheel; zipp = if super.zipp == null then null else super.zipp.overridePythonAttrs ( @@ -2757,20 +2754,6 @@ lib.composeManyExtensions [ } ); - packaging = - let - old = super.packaging; - in - # From 20.5 until 20.7, packaging used flit for packaging (heh) - # See https://github.com/pypa/packaging/pull/352 and https://github.com/pypa/packaging/pull/367 - if (lib.versionAtLeast old.version "20.5" && lib.versionOlder old.version "20.8") then - addBuildSystem - { - inherit self; - drv = old; - attr = "flit-core"; - } else old; - psutil = super.psutil.overridePythonAttrs ( old: { buildInputs = (old.buildInputs or [ ]) ++ diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index 4f52bf39033c8..aa05eb038f6c9 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -8,6 +8,16 @@ self: super: with self; { + bootstrap = lib.recurseIntoAttrs { + flit-core = toPythonModule (callPackage ../development/python-modules/bootstrap/flit-core { }); + installer = toPythonModule (callPackage ../development/python-modules/bootstrap/installer { + inherit (bootstrap) flit-core; + }); + build = toPythonModule (callPackage ../development/python-modules/bootstrap/build { + inherit (bootstrap) flit-core installer; + }); + }; + bootstrapped-pip = toPythonModule (callPackage ../development/python-modules/bootstrapped-pip { }); setuptools = callPackage ../development/python-modules/setuptools { };