python.buildEnv: fix venv creation

pkgs/by-name/xo/xonsh/unwrapped.nix

@@ -104,9 +104,6 @@ buildPythonPackage rec {
     "tests/completers/test_bash_completer.py"
   ];
 
-  # https://github.com/NixOS/nixpkgs/issues/248978
-  dontWrapPythonPrograms = true;
-
   env.LC_ALL = "en_US.UTF-8";
 
   postPatch = ''

pkgs/development/interpreters/python/cpython/2.7/default.nix

@@ -370,24 +370,21 @@ stdenv.mkDerivation (
 
     inherit passthru;
 
-    postFixup = ''
-      # Include a sitecustomize.py file. Note it causes an error when it's in postInstall with 2.7.
-      cp ${../../sitecustomize.py} $out/${sitePackages}/sitecustomize.py
-    ''
-    + lib.optionalString strip2to3 ''
-      rm -R $out/bin/2to3 $out/lib/python*/lib2to3
-    ''
-    + lib.optionalString stripConfig ''
-      rm -R $out/bin/python*-config $out/lib/python*/config*
-    ''
-    + lib.optionalString stripIdlelib ''
-      # Strip IDLE
-      rm -R $out/bin/idle* $out/lib/python*/idlelib
-    ''
-    + lib.optionalString stripTests ''
-      # Strip tests
-      rm -R $out/lib/python*/test $out/lib/python*/**/test{,s}
-    '';
+    postFixup =
+      lib.optionalString strip2to3 ''
+        rm -R $out/bin/2to3 $out/lib/python*/lib2to3
+      ''
+      + lib.optionalString stripConfig ''
+        rm -R $out/bin/python*-config $out/lib/python*/config*
+      ''
+      + lib.optionalString stripIdlelib ''
+        # Strip IDLE
+        rm -R $out/bin/idle* $out/lib/python*/idlelib
+      ''
+      + lib.optionalString stripTests ''
+        # Strip tests
+        rm -R $out/lib/python*/test $out/lib/python*/**/test{,s}
+      '';
 
     enableParallelBuilding = true;
 

pkgs/development/interpreters/python/cpython/default.nix

@@ -75,7 +75,6 @@
   stripTkinter ? withMinimalDeps,
   rebuildBytecode ? !withMinimalDeps,
   stripBytecode ? true,
-  includeSiteCustomize ? !withMinimalDeps,
   static ? stdenv.hostPlatform.isStatic,
   enableFramework ? false,
   noldconfigPatch ? ./. + "/${sourceVersion.major}.${sourceVersion.minor}/no-ldconfig.patch",
@@ -691,10 +690,6 @@ stdenv.mkDerivation (finalAttrs: {
       # Strip tests
       rm -R $out/lib/python*/test $out/lib/python*/**/test{,s}
     ''
-    + optionalString includeSiteCustomize ''
-      # Include a sitecustomize.py file
-      cp ${../sitecustomize.py} $out/${sitePackages}/sitecustomize.py
-    ''
     + optionalString stripBytecode ''
       # Determinism: deterministic bytecode
       # First we delete all old bytecode.

pkgs/development/interpreters/python/pypy/default.nix

@@ -7,6 +7,7 @@
   zlib,
   bzip2,
   pkg-config,
+  lndir,
   libffi,
   sqlite,
   openssl,
@@ -80,7 +81,10 @@ stdenv.mkDerivation rec {
     inherit hash;
   };
 
-  nativeBuildInputs = [ pkg-config ];
+  nativeBuildInputs = [
+    pkg-config
+    lndir
+  ];
   buildInputs = [
     bzip2
     openssl
@@ -181,21 +185,17 @@ stdenv.mkDerivation rec {
   installPhase = ''
     runHook preInstall
 
-    mkdir -p $out/{bin,include,lib,${executable}-c}
+    mkdir -p $out/{bin,lib/${libPrefix}}
 
-    cp -R {include,lib_pypy,lib-python,${executable}-c} $out/${executable}-c
-    cp lib${executable}-c${stdenv.hostPlatform.extensions.sharedLibrary} $out/lib/
-    ln -s $out/${executable}-c/${executable}-c $out/bin/${executable}
+    cp -R {include,lib_pypy,lib-python} $out
+    install -Dm755 lib${executable}-c${stdenv.hostPlatform.extensions.sharedLibrary} $out/lib/
+    install -Dm755 ${executable}-c $out/bin/${executable}
     ${lib.optionalString isPy39OrNewer "ln -s $out/bin/${executable} $out/bin/pypy3"}
 
     # other packages expect to find stuff according to libPrefix
-    ln -s $out/${executable}-c/include $out/include/${libPrefix}
-    ln -s $out/${executable}-c/lib-python/${if isPy3k then "3" else pythonVersion} $out/lib/${libPrefix}
-
-    # Include a sitecustomize.py file
-    cp ${../sitecustomize.py} $out/${
-      if isPy38OrNewer then sitePackages else "lib/${libPrefix}/${sitePackages}"
-    }/sitecustomize.py
+    ln -s $out/include $out/include/${libPrefix}
+    lndir $out/lib-python/${if isPy3k then "3" else pythonVersion} $out/lib/${libPrefix}
+    lndir $out/lib_pypy $out/lib/${libPrefix}
 
     runHook postInstall
   '';
@@ -204,11 +204,6 @@ stdenv.mkDerivation rec {
     lib.optionalString (stdenv.hostPlatform.isDarwin) ''
       install_name_tool -change @rpath/lib${executable}-c.dylib $out/lib/lib${executable}-c.dylib $out/bin/${executable}
     ''
-    + lib.optionalString (stdenv.hostPlatform.isDarwin && stdenv.hostPlatform.isAarch64) ''
-      mkdir -p $out/${executable}-c/pypy/bin
-      mv $out/bin/${executable} $out/${executable}-c/pypy/bin/${executable}
-      ln -s $out/${executable}-c/pypy/bin/${executable} $out/bin/${executable}
-    ''
     # _testcapi is compiled dynamically, into the store.
     # This would fail if we don't do it here.
     + lib.optionalString (stdenv.buildPlatform.canExecute stdenv.hostPlatform) ''

pkgs/development/interpreters/python/pypy/prebuilt.nix

@@ -105,9 +105,6 @@ stdenv.mkDerivation {
     echo "Removing bytecode"
     find . -name "__pycache__" -type d -depth -delete
 
-    # Include a sitecustomize.py file
-    cp ${../sitecustomize.py} $out/${sitePackages}/sitecustomize.py
-
     runHook postInstall
   '';
 

pkgs/development/interpreters/python/pypy/prebuilt_2_7.nix

@@ -105,9 +105,6 @@ stdenv.mkDerivation {
     echo "Removing bytecode"
     find . -name "__pycache__" -type d -depth -delete
 
-    # Include a sitecustomize.py file
-    cp ${../sitecustomize.py} $out/${sitePackages}/sitecustomize.py
-
     runHook postInstall
   '';
 

pkgs/development/interpreters/python/wrap-python.nix

@@ -45,11 +45,14 @@ makePythonHook {
       # * Sets argv[0] to the original application's name; otherwise it would be .foo-wrapped.
       #   Python doesn't support `exec -a`.
       # * Adds all required libraries to sys.path via `site.addsitedir`. It also handles *.pth files.
+      # * Adds optional libraries to sys.path when calling from a python environment.
       preamble = ''
+        import os
         import sys
         import site
         import functools
-        sys.argv[0] = '"'$(readlink -f "$f")'"'
+        sys.argv[0] = os.environ.pop("NIX_PYTHONSCRIPT", sys.argv[0])
+        site.addsitedir(os.path.join(os.path.dirname(os.path.abspath(sys.argv[0])), "..", "${python.sitePackages}"),site._init_pathinfo())
         functools.reduce(lambda k, p: site.addsitedir(p, k), ['"$([ -n "$program_PYTHONPATH" ] && (echo "'$program_PYTHONPATH'" | sed "s|:|','|g") || true)"'], site._init_pathinfo())
       '';
 

pkgs/development/interpreters/python/wrap.sh

@@ -92,6 +92,10 @@ wrapPythonProgramsIn() {
 
                     local -a wrapProgramArgs=("${wrap_args[@]}" "${user_args[@]}")
                     wrapProgram "${wrapProgramArgs[@]}"
+
+                    # Python doesn't support `exec -a`. We have to use environment
+                    # to pass argv[0]
+                    sed -i '2iexport NIX_PYTHONSCRIPT="$0"' "$f"
                 fi
             fi
         done

pkgs/development/interpreters/python/wrapper.nix

@@ -1,6 +1,7 @@
 {
   lib,
   stdenv,
+  runCommand,
   buildEnv,
   makeBinaryWrapper,
 
@@ -20,62 +21,51 @@
 
 # Create a python executable that knows about additional packages.
 let
-  env =
-    let
-      paths = requiredPythonModules (extraLibs ++ [ python ]);
-      pythonPath = "${placeholder "out"}/${python.sitePackages}";
-      pythonExecutable = "${placeholder "out"}/bin/${python.executable}";
-    in
-    buildEnv {
-      name = "${python.name}-env";
+  env = buildEnv {
+    name = "${python.name}-env";
 
-      inherit paths;
-      inherit ignoreCollisions;
-      extraOutputsToInstall = [ "out" ] ++ extraOutputsToInstall;
+    paths = requiredPythonModules (extraLibs ++ [ python ]) ++ [
+      (runCommand "bin" { } ''
+        mkdir -p $out/bin
+      '')
+    ];
+    inherit ignoreCollisions;
+    extraOutputsToInstall = [ "out" ] ++ extraOutputsToInstall;
 
-      nativeBuildInputs = [ makeBinaryWrapper ];
+    nativeBuildInputs = [ makeBinaryWrapper ];
 
-      postBuild = ''
-        if [ -L "$out/bin" ]; then
-            unlink "$out/bin"
+    postBuild = ''
+      rm -f "$out/bin/${python.executable}"
+      makeWrapper "${python.interpreter}" "$out/bin/${python.executable}" --inherit-argv0 --resolve-argv0 ${
+        lib.optionalString (!permitUserSite) ''--set PYTHONNOUSERSITE "true"''
+      } ${lib.concatStringsSep " " makeWrapperArgs}
+      cd ${python}/bin
+      for prg in *; do
+        if [ "$(readlink "$prg")" = "${python.executable}" ]; then
+          rm -f "$out/bin/$prg"
+          ln -s "${python.executable}" "$out/bin/$prg"
         fi
-        mkdir -p "$out/bin"
+      done
+    ''
+    + postBuild;
 
-        for path in ${lib.concatStringsSep " " paths}; do
-          if [ -d "$path/bin" ]; then
-            cd "$path/bin"
-            for prg in *; do
-              if [ -f "$prg" ]; then
-                rm -f "$out/bin/$prg"
-                if [ -x "$prg" ]; then
-                  makeWrapper "$path/bin/$prg" "$out/bin/$prg" --set NIX_PYTHONPREFIX "$out" --set NIX_PYTHONEXECUTABLE ${pythonExecutable} --set NIX_PYTHONPATH ${pythonPath} ${
-                    lib.optionalString (!permitUserSite) ''--set PYTHONNOUSERSITE "true"''
-                  } ${lib.concatStringsSep " " makeWrapperArgs}
-                fi
-              fi
-            done
-          fi
-        done
-      ''
-      + postBuild;
+    inherit (python) meta;
 
-      inherit (python) meta;
+    passthru = python.passthru // {
+      interpreter = "${env}/bin/${python.executable}";
+      inherit python;
+      env = stdenv.mkDerivation {
+        name = "interactive-${python.name}-environment";
+        nativeBuildInputs = [ env ];
 
-      passthru = python.passthru // {
-        interpreter = "${env}/bin/${python.executable}";
-        inherit python;
-        env = stdenv.mkDerivation {
-          name = "interactive-${python.name}-environment";
-          nativeBuildInputs = [ env ];
-
-          buildCommand = ''
-            echo >&2 ""
-            echo >&2 "*** Python 'env' attributes are intended for interactive nix-shell sessions, not for building! ***"
-            echo >&2 ""
-            exit 1
-          '';
-        };
+        buildCommand = ''
+          echo >&2 ""
+          echo >&2 "*** Python 'env' attributes are intended for interactive nix-shell sessions, not for building! ***"
+          echo >&2 ""
+          exit 1
+        '';
       };
     };
+  };
 in
 env