From 2d23255ff51fd03c0280cdebcf2e87c76c71a99f Mon Sep 17 00:00:00 2001 From: FliegendeWurst Date: Mon, 7 Apr 2025 12:46:10 +0200 Subject: [PATCH 1/5] pypy{27,310}: 7.3.17 -> 7.3.19, pypy311: init at 7.3.19 --- .../interpreters/python/default.nix | 63 ++++++++--- .../interpreters/python/pypy/default.nix | 105 +++++++++++++++++- .../interpreters/python/pypy/prebuilt_2_7.nix | 1 - pkgs/top-level/all-packages.nix | 4 +- 4 files changed, 153 insertions(+), 20 deletions(-) diff --git a/pkgs/development/interpreters/python/default.nix b/pkgs/development/interpreters/python/default.nix index d738e391ea86b..c404bfe6d46b5 100644 --- a/pkgs/development/interpreters/python/default.nix +++ b/pkgs/development/interpreters/python/default.nix @@ -138,10 +138,10 @@ sourceVersion = { major = "7"; minor = "3"; - patch = "17"; + patch = "19"; }; - hash = "sha256-UOBoQPS73pFEgICkEYBoqJuPvK4l/42h4rsUAtyaA0Y="; + hash = "sha256-hwPNywH5+Clm3UO2pgGPFAOZ21HrtDwSXB+aIV57sAM="; pythonVersion = "2.7"; db = db.override { dbmSupport = !stdenv.hostPlatform.isDarwin; }; python = __splicedPackages.pythonInterpreters.pypy27_prebuilt; @@ -153,31 +153,46 @@ sourceVersion = { major = "7"; minor = "3"; - patch = "17"; + patch = "19"; }; - hash = "sha256-atdLxXjpxtOoocUVAzEwWOPFjDXfhvdIVFPEvmqyS/c="; + hash = "sha256-p8IpMLkY9Ahwhl7Yp0FH9ENO+E09bKKzweupNV1JKcg="; pythonVersion = "3.10"; db = db.override { dbmSupport = !stdenv.hostPlatform.isDarwin; }; python = __splicedPackages.pypy27; inherit passthruFun; }; + pypy311 = callPackage ./pypy { + self = __splicedPackages.pypy311; + sourceVersion = { + major = "7"; + minor = "3"; + patch = "19"; + }; + + hash = "sha256-SBfARLtGmjJ05gqjZFdw+B60+RZup/3E5sNRNFVUyNg="; + pythonVersion = "3.11"; + db = db.override { dbmSupport = !stdenv.hostPlatform.isDarwin; }; + python = __splicedPackages.pypy27; + inherit passthruFun; + }; + pypy27_prebuilt = callPackage ./pypy/prebuilt_2_7.nix { # Not included at top-level self = __splicedPackages.pythonInterpreters.pypy27_prebuilt; sourceVersion = { major = "7"; minor = "3"; - patch = "17"; + patch = "19"; }; hash = { - aarch64-linux = "sha256-qN9c4WUPR1aTP4eAhwyRoKQOfJhw10YpvyQTkry1wuM="; - x86_64-linux = "sha256-nzSX+HszctF+RHNp4AFqS+yZprTSpZq6d0olv+Q1NHQ="; - aarch64-darwin = "sha256-gCJIc5sqzIwb5tlH8Zsy/A44wI4xKzXAXMf7IvEHCeQ="; - x86_64-darwin = "sha256-gtRgQhRmyBraSh2Z3y3xuLNTQbOXyF///lGkwwItCDM="; + aarch64-linux = "sha256-/onU/UrxP3bf5zFZdQA1GM8XZSDjzOwVRKiNF09QkQ4="; + x86_64-linux = "sha256-04RFUIwurxTrs4DZwd7TIcXr6uMcfmaAAXPYPLjd9CM="; + aarch64-darwin = "sha256-KHgOC5CK1ttLTglvQjcSS+eezJcxlG2EDZyHSetnp1k="; + x86_64-darwin = "sha256-a+KNRI2OZP/8WG2bCuTQkGSoPMrrW4BgxlHFzZrgaHg="; } .${stdenv.system}; pythonVersion = "2.7"; @@ -190,19 +205,39 @@ sourceVersion = { major = "7"; minor = "3"; - patch = "17"; + patch = "19"; }; hash = { - aarch64-linux = "sha256-v79JVJirwv53G2C/ZOXDwHLgr7z8pprHKCxP9Dd/9BY="; - x86_64-linux = "sha256-NA2kGWYGsiRQmhuLMa/SAYE/CCYB3xicE46QXB1g4K8="; - aarch64-darwin = "sha256-KPKf/JxcyQbo6QgT/BRPA34js4TwUuGE4kIzL3tgqwY="; - x86_64-darwin = "sha256-I/8mS3PlvFt8OhufrHdosj35bH1mDLZBLxxSNSGjNL8="; + aarch64-linux = "sha256-ryeliRePERmOIkSrZcpRBjC6l8Ex18zEAh61vFjef1c="; + x86_64-linux = "sha256-xzrCzCOArJIn/Sl0gr8qPheoBhi6Rtt1RNU1UVMh7B4="; + aarch64-darwin = "sha256-PbigP8SWFkgBZGhE1/OxK6oK2zrZoLfLEkUhvC4WijY="; + x86_64-darwin = "sha256-LF5cKjOsiCVR1/KLmNGdSGuJlapQgkpztO3Mau7DXGM="; } .${stdenv.system}; pythonVersion = "3.10"; inherit passthruFun; }; + + pypy311_prebuilt = callPackage ./pypy/prebuilt.nix { + # Not included at top-level + self = __splicedPackages.pythonInterpreters.pypy311_prebuilt; + sourceVersion = { + major = "7"; + minor = "3"; + patch = "19"; + }; + hash = + { + aarch64-linux = "sha256-EyB9v4HOJOltp2CxuGNie3e7ILH7TJUZHgKgtyOD33Q="; + x86_64-linux = "sha256-kXfZ4LuRsF+SHGQssP9xoPNlO10ppC1A1qB4wVt1cg8="; + aarch64-darwin = "sha256-dwTg1TAuU5INMtz+mv7rEENtTJQjPogwz2A6qVWoYcE="; + x86_64-darwin = "sha256-okOfnTDf2ulqXpEBx9xUqKaLVsnXMU6jmbCiXT6H67I="; + } + .${stdenv.system}; + pythonVersion = "3.11"; + inherit passthruFun; + }; } // lib.optionalAttrs config.allowAliases { pypy39_prebuilt = throw "pypy 3.9 has been removed, use pypy 3.10 instead"; # Added 2025-01-03 diff --git a/pkgs/development/interpreters/python/pypy/default.nix b/pkgs/development/interpreters/python/pypy/default.nix index 148b2cb2ecf0e..a5fe47f7921e1 100644 --- a/pkgs/development/interpreters/python/pypy/default.nix +++ b/pkgs/development/interpreters/python/pypy/default.nix @@ -121,6 +121,9 @@ stdenv.mkDerivation rec { dontPatchShebangs = true; disallowedReferences = [ python ]; + # fix compiler error in curses cffi module, where char* != const char* + NIX_CFLAGS_COMPILE = + if stdenv.cc.isClang then "-Wno-error=incompatible-function-pointer-types" else null; C_INCLUDE_PATH = lib.makeSearchPathOutput "dev" "include" buildInputs; LIBRARY_PATH = lib.makeLibraryPath buildInputs; LD_LIBRARY_PATH = lib.makeLibraryPath ( @@ -192,13 +195,18 @@ stdenv.mkDerivation rec { mkdir -p $out/${executable}-c/pypy/bin mv $out/bin/${executable} $out/${executable}-c/pypy/bin/${executable} ln -s $out/${executable}-c/pypy/bin/${executable} $out/bin/${executable} + '' + # _testcapi is compiled dynamically, into the store. + # This would fail if we don't do it here. + + lib.optionalString (stdenv.buildPlatform.canExecute stdenv.hostPlatform) '' + pushd / + $out/bin/${executable} -c "from test import support" + popd ''; setupHook = python-setup-hook sitePackages; - # TODO: A bunch of tests are failing as of 7.1.1, please feel free to - # fix and re-enable if you have the patience and tenacity. - doCheck = false; + # TODO: Investigate why so many tests are failing. checkPhase = let disabledTests = @@ -213,6 +221,9 @@ stdenv.mkDerivation rec { "test_urllib2net" "test_urllibnet" "test_urllib2_localnet" + # test_subclass fails with "internal error" + # test_load_default_certs_env fails for unknown reason + "test_ssl" ] ++ lib.optionals isPy3k [ # disable asyncio due to https://github.com/NixOS/nix/issues/1238 @@ -226,6 +237,88 @@ stdenv.mkDerivation rec { # disable __all__ because of spurious imp/importlib warning and # warning-to-error test policy "test___all__" + # fail for multiple reasons, TODO: investigate + "test__opcode" + "test_ast" + "test_audit" + "test_builtin" + "test_c_locale_coercion" + "test_call" + "test_class" + "test_cmd_line" + "test_cmd_line_script" + "test_code" + "test_code_module" + "test_codeop" + "test_compile" + "test_coroutines" + "test_cprofile" + "test_ctypes" + "test_embed" + "test_exceptions" + "test_extcall" + "test_frame" + "test_generators" + "test_grammar" + "test_idle" + "test_iter" + "test_itertools" + "test_list" + "test_marshal" + "test_memoryio" + "test_memoryview" + "test_metaclass" + "test_mmap" + "test_multibytecodec" + "test_opcache" + "test_pdb" + "test_peepholer" + "test_positional_only_arg" + "test_print" + "test_property" + "test_pyclbr" + "test_range" + "test_re" + "test_readline" + "test_regrtest" + "test_repl" + "test_rlcompleter" + "test_signal" + "test_sort" + "test_source_encoding" + "test_ssl" + "test_string_literals" + "test_structseq" + "test_subprocess" + "test_super" + "test_support" + "test_syntax" + "test_sys" + "test_sys_settrace" + "test_tcl" + "test_termios" + "test_threading" + "test_trace" + "test_tty" + "test_unpack_ex" + "test_utf8_mode" + "test_weakref" + "test_capi" + "test_concurrent_futures" + "test_dataclasses" + "test_doctest" + "test_future_stmt" + "test_importlib" + "test_inspect" + "test_pydoc" + "test_warnings" + ] + ++ lib.optionals isPy310 [ + "test_contextlib_async" + "test_future" + "test_lzma" + "test_module" + "test_typing" ]; in '' @@ -264,6 +357,7 @@ stdenv.mkDerivation rec { meta = with lib; { homepage = "https://www.pypy.org/"; + changelog = "https://doc.pypy.org/en/stable/release-v${version}.html"; description = "Fast, compliant alternative implementation of the Python language (${pythonVersion})"; mainProgram = "pypy"; license = licenses.mit; @@ -274,6 +368,9 @@ stdenv.mkDerivation rec { "x86_64-darwin" ]; broken = optimizationLevel == "0"; # generates invalid code - maintainers = with maintainers; [ andersk ]; + maintainers = with maintainers; [ + andersk + fliegendewurst + ]; }; } diff --git a/pkgs/development/interpreters/python/pypy/prebuilt_2_7.nix b/pkgs/development/interpreters/python/pypy/prebuilt_2_7.nix index 82da8e953e669..95e89c54b68fe 100644 --- a/pkgs/development/interpreters/python/pypy/prebuilt_2_7.nix +++ b/pkgs/development/interpreters/python/pypy/prebuilt_2_7.nix @@ -100,7 +100,6 @@ stdenv.mkDerivation { mv -t $out bin include lib-python lib_pypy site-packages mv $out/bin/libpypy*-c${stdenv.hostPlatform.extensions.sharedLibrary} $out/lib/ ${lib.optionalString stdenv.hostPlatform.isLinux '' - mv lib/libffi.so.6* $out/lib/ rm $out/bin/*.debug ''} diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index e71c2e9367efa..058e7152a1fbf 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -6580,7 +6580,7 @@ with pkgs; pypy = pypy2; pypy2 = pypy27; - pypy3 = pypy310; + pypy3 = pypy311; # Python interpreter that is build with all modules, including tkinter. # These are for compatibility and should not be used inside Nixpkgs. @@ -6652,6 +6652,7 @@ with pkgs; python3Minimal pypy27 pypy310 + pypy311 ; # List of extensions with overrides to apply to all Python package sets. @@ -6669,6 +6670,7 @@ with pkgs; pypy27Packages = pypy27.pkgs; pypy3Packages = pypy3.pkgs; pypy310Packages = pypy310.pkgs; + pypy311Packages = pypy311.pkgs; pythonManylinuxPackages = callPackage ./../development/interpreters/python/manylinux { }; From 81b65170bb619c2af655c91acbf5e23dc9cc6e0b Mon Sep 17 00:00:00 2001 From: FliegendeWurst Date: Tue, 8 Apr 2025 09:28:01 +0200 Subject: [PATCH 2/5] pypy3Packages.mypy: disable --- pkgs/development/python-modules/mypy/default.nix | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkgs/development/python-modules/mypy/default.nix b/pkgs/development/python-modules/mypy/default.nix index 1ffadd2005f60..e7178dfb05f32 100644 --- a/pkgs/development/python-modules/mypy/default.nix +++ b/pkgs/development/python-modules/mypy/default.nix @@ -6,6 +6,7 @@ gitUpdater, pythonAtLeast, pythonOlder, + isPyPy, # build-system setuptools, @@ -35,7 +36,8 @@ buildPythonPackage rec { version = "1.15.0"; pyproject = true; - disabled = pythonOlder "3.8"; + # relies on several CPython internals + disabled = pythonOlder "3.8" || isPyPy; src = fetchFromGitHub { owner = "python"; From 233f899f64bb97a681d2983604c44e924c7f917f Mon Sep 17 00:00:00 2001 From: FliegendeWurst Date: Tue, 8 Apr 2025 09:28:01 +0200 Subject: [PATCH 3/5] pypy.tests: disable broken test --- pkgs/development/interpreters/python/tests.nix | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pkgs/development/interpreters/python/tests.nix b/pkgs/development/interpreters/python/tests.nix index e2dc7a69fbc69..f0cd64d23e66a 100644 --- a/pkgs/development/interpreters/python/tests.nix +++ b/pkgs/development/interpreters/python/tests.nix @@ -237,7 +237,8 @@ let } ); - condaTests = + # depends on mypy, which depends on CPython internals + condaTests = lib.optionalAttrs (!python.isPyPy) ( let requests = callPackage ( { @@ -276,7 +277,8 @@ let condaExamplePackage = runCommand "import-requests" { } '' ${pythonWithRequests.interpreter} -c "import requests" > $out ''; - }; + } + ); in lib.optionalAttrs (stdenv.hostPlatform == stdenv.buildPlatform) ( From 7ecebd4a14589e16f2800b63506fa919bf53f413 Mon Sep 17 00:00:00 2001 From: FliegendeWurst Date: Tue, 8 Apr 2025 09:28:01 +0200 Subject: [PATCH 4/5] meson: fix build when using pypy --- pkgs/by-name/me/meson/package.nix | 39 +++++++++++++++++++++++++++---- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/pkgs/by-name/me/meson/package.nix b/pkgs/by-name/me/meson/package.nix index ea9e4e002d69d..9ac84af31d3d7 100644 --- a/pkgs/by-name/me/meson/package.nix +++ b/pkgs/by-name/me/meson/package.nix @@ -10,6 +10,7 @@ pkg-config, python3, replaceVars, + writeShellScriptBin, zlib, }: @@ -66,12 +67,34 @@ python3.pkgs.buildPythonApplication rec { ./007-freebsd-pkgconfig-path.patch ]; + postPatch = + if python3.isPyPy then + '' + substituteInPlace mesonbuild/modules/python.py \ + --replace-fail "PythonExternalProgram('python3', mesonlib.python_command)" \ + "PythonExternalProgram('${python3.meta.mainProgram}', mesonlib.python_command)" + substituteInPlace mesonbuild/modules/python3.py \ + --replace-fail "state.environment.lookup_binary_entry(mesonlib.MachineChoice.HOST, 'python3')" \ + "state.environment.lookup_binary_entry(mesonlib.MachineChoice.HOST, '${python3.meta.mainProgram}')" + substituteInPlace "test cases"/*/*/*.py "test cases"/*/*/*/*.py \ + --replace-quiet '#!/usr/bin/env python3' '#!/usr/bin/env pypy3' \ + --replace-quiet '#! /usr/bin/env python3' '#!/usr/bin/env pypy3' + chmod +x "test cases"/*/*/*.py "test cases"/*/*/*/*.py + '' + else + null; + nativeBuildInputs = [ installShellFiles ]; - nativeCheckInputs = [ - ninja - pkg-config - ]; + nativeCheckInputs = + [ + ninja + pkg-config + ] + ++ lib.optionals python3.isPyPy [ + # Several tests hardcode python3. + (writeShellScriptBin "python3" ''exec pypy3 "$@"'') + ]; checkInputs = [ @@ -116,9 +139,15 @@ python3.pkgs.buildPythonApplication rec { # pch doesn't work quite right on FreeBSD, I think ''test cases/common/13 pch'' ] + ++ lib.optionals python3.isPyPy [ + # fails for unknown reason + ''test cases/python/4 custom target depends extmodule'' + ] )) ++ [ - ''HOME="$TMPDIR" python ./run_project_tests.py'' + ''HOME="$TMPDIR" ${ + if python3.isPyPy then python3.interpreter else "python" + } ./run_project_tests.py'' "runHook postCheck" ] ); From 214357a159992e44f8aef1f4898c8fa6dcdf3f4c Mon Sep 17 00:00:00 2001 From: FliegendeWurst Date: Tue, 8 Apr 2025 09:28:01 +0200 Subject: [PATCH 5/5] pypy3Packages.cython: fix --- pkgs/development/python-modules/cython/default.nix | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pkgs/development/python-modules/cython/default.nix b/pkgs/development/python-modules/cython/default.nix index 41c4230dc1948..9e3cb88b49249 100644 --- a/pkgs/development/python-modules/cython/default.nix +++ b/pkgs/development/python-modules/cython/default.nix @@ -3,6 +3,7 @@ buildPythonPackage, fetchFromGitHub, gdb, + isPyPy, ncurses, numpy, pkg-config, @@ -36,7 +37,9 @@ buildPythonPackage rec { ncurses ]; - env.LC_ALL = "en_US.UTF-8"; + env = lib.optionalAttrs (!isPyPy) { + LC_ALL = "en_US.UTF-8"; + }; # https://github.com/cython/cython/issues/2785 # Temporary solution