Disable native builds within a PEP 517 environment that's building fo…

…r the target platform (#715)

product/gradle-plugin/src/main/python/chaquopy_monkey.py

@@ -0,0 +1,87 @@
+# We want to cause a quick and comprehensible failure when a package attempts to build
+# native code, while still allowing a pure-Python fallback if available. This is tricky,
+# because different packages have different approaches to pure-Python fallbacks:
+#
+# * Some packages simply catch any distutils exception thrown by setup(), and then run it again
+#   with the native components removed.
+#
+# * Some (e.g. sqlalchemy, wrapt) extend the distutils build_ext or build_clib command and
+#   override its run() method to wrap it with an exception handler. This means we can't simply
+#   block the commands by name, e.g. by overriding Distribution.run_command.
+#
+# * Some (e.g. msgpack) go lower-level and catch exceptions in build_ext.build_extension. In
+#   Python 3, there's an `optional` keyword to Extension which has the same effect (used e.g.
+#   by websockets). Blocking build_ext.run, or CCompiler.__init__, would cause these builds to
+#   fail before build_extension is called, and the pure-Python fallback wouldn't happen.
+#
+# Creating a new compiler class with a new name minimizes the chance of code trying to do
+# things which will only work on the standard classses. For example,
+# distutils.sysconfig.customize_compiler does things with a "unix" compiler which will crash on
+# Windows because get_config_vars won't have certain settings.
+#
+# This is simpler than determining the regular compiler class and extending it. It avoids
+# interference from NumPy's widespread monkey-patching (including new_compiler, CCompiler and
+# its subclasses), which takes place after this code is run. It also avoids the default
+# behaviour on Windows when no compiler is installed, which is either to give the "Unable to
+# find vcvarsall.bat" error, or advice on how to install Visual C++, both of which will waste
+# the user's time.
+#
+# This approach will block builds of packages which require the compiler name to be in a known
+# list (e.g. minorminer, lz4), but the error messages from these packages aren't too bad, and
+# I've never seen one which has a pure-Python fallback.
+def disable_native():
+    # Recent versions of setuptools redirect distutils to their own bundled copy, so try
+    # to import that first.
+    try:
+        import setuptools  # noqa: F401
+    except ImportError:
+        pass
+
+    from distutils import ccompiler
+    from distutils.unixccompiler import UnixCCompiler
+    import os
+    import sys
+    import types
+
+    ccompiler.get_default_compiler = lambda *args, **kwargs: "disabled"
+    ccompiler.compiler_class["disabled"] = (
+        "disabledcompiler", "DisabledCompiler",
+        "Compiler disabled ({})".format(CHAQUOPY_NATIVE_ERROR))
+
+    class DisabledCompiler(ccompiler.CCompiler):
+        compiler_type = "disabled"
+        def preprocess(*args, **kwargs):
+            chaquopy_block_native("CCompiler.preprocess")
+        def compile(*args, **kwargs):
+            chaquopy_block_native("CCompiler.compile")
+        def create_static_lib(*args, **kwargs):
+            chaquopy_block_native("CCompiler.create_static_lib")
+        def link(*args, **kwargs):
+            chaquopy_block_native("CCompiler.link")
+
+    # To maximize the chance of the build getting as far as actually calling compile(), make
+    # sure the class has all of the expected attributes.
+    for name in ["src_extensions", "obj_extension", "static_lib_extension",
+                 "shared_lib_extension", "static_lib_format", "shared_lib_format",
+                 "exe_extension"]:
+        setattr(DisabledCompiler, name, getattr(UnixCCompiler, name))
+    DisabledCompiler.executables = {name: [CHAQUOPY_NATIVE_ERROR.replace(" ", "_")]
+                                    for name in UnixCCompiler.executables}
+
+    disabled_mod_name = "distutils.disabledcompiler"
+    disabled_mod = types.ModuleType(disabled_mod_name)
+    disabled_mod.DisabledCompiler = DisabledCompiler
+    sys.modules[disabled_mod_name] = disabled_mod
+
+    # Try to disable native builds for packages which don't use the distutils native build
+    # system at all (e.g. uwsgi), or only use it to wrap an external build script (e.g. pynacl).
+    for tool in ["ar", "as", "cc", "cxx", "ld"]:
+        os.environ[tool.upper()] = CHAQUOPY_NATIVE_ERROR.replace(" ", "_")
+
+
+CHAQUOPY_NATIVE_ERROR = "Chaquopy cannot compile native code"
+
+def chaquopy_block_native(prefix):
+    # No need to give any more advice here: that will come from the higher-level code in pip.
+    from distutils.errors import DistutilsPlatformError
+    raise DistutilsPlatformError("{}: {}".format(prefix, CHAQUOPY_NATIVE_ERROR))

product/gradle-plugin/src/main/python/pip/_internal/build_env.py

@@ -104,6 +104,22 @@ def __init__(self):
                 '''
             ).format(system_sites=system_sites, lib_dirs=self._lib_dirs))
 
+            # Chaquopy
+            from os.path import dirname, join
+            import shutil
+            # Copy chaquopy_monkey unconditionally, so we can import it unconditionally in
+            # check_chaquopy_exception.
+            shutil.copy(join(dirname(pip_location), "../chaquopy_monkey.py"),
+                        self._site_dir)
+            from pip._vendor.packaging import markers
+            if markers.python_version_info:
+                fp.write(textwrap.dedent(
+                    '''
+                    import chaquopy_monkey
+                    chaquopy_monkey.disable_native()
+                    '''
+                ))
+
     def __enter__(self):
         self._save_env = {
             name: os.environ.get(name, None)

product/gradle-plugin/src/main/python/pip/_internal/cli/cmdoptions.py

@@ -111,7 +111,7 @@ def check_dist_restriction(options, check_target=False):
     # guaranteed to be locally compatible.
     #
     # Chaquopy: added False to disable this restriction. It's safe for us to run source
-    # distributions, because we monkey-patch setuptools to ensure they fail immediately if they
+    # distributions, because chaquopy_monkey ensures they fail immediately if they
     # try to build anything native.
     if False and dist_restriction_set and sdist_dependencies_allowed:
         raise CommandError(

product/gradle-plugin/src/main/python/pip/_internal/operations/prepare.py

@@ -211,9 +211,13 @@ def prepare_linked_requirement(
                 )
             abstract_dist = make_distribution_for_install_requirement(req)
             with self.req_tracker.track(req):
-                abstract_dist.prepare_distribution_metadata(
-                    finder, self.build_isolation,
-                )
+                try:
+                    abstract_dist.prepare_distribution_metadata(
+                        finder, self.build_isolation,
+                    )
+                except InstallationError:
+                    abstract_dist.req.chaquopy_setup_py_failed()
+
             if self._download_should_save:
                 # Make a .zip of the source_dir we already created.
                 if not req.link.is_artifact:

product/gradle-plugin/src/main/python/pip/_internal/req/req_install.py

@@ -623,14 +623,10 @@ def run_egg_info(self):
             ensure_dir(egg_info_dir)
             egg_base_option = ['--egg-base', 'pip-egg-info']
         with self.build_env:
-            try:
-                call_subprocess(
-                    egg_info_cmd + egg_base_option,
-                    cwd=self.setup_py_dir,
-                    command_desc='python setup.py egg_info')
-            except InstallationError as exc:
-                self.chaquopy_setup_py_failed(exc)
-
+            call_subprocess(
+                egg_info_cmd + egg_base_option,
+                cwd=self.setup_py_dir,
+                command_desc='python setup.py egg_info')
 
     @property
     def egg_info_path(self):
@@ -1006,7 +1002,7 @@ def prepend_root(path):
             with open(inst_files_path, 'w') as f:
                 f.write('\n'.join(new_lines) + '\n')
 
-    def chaquopy_setup_py_failed(self, exc):
+    def chaquopy_setup_py_failed(self):
         from pip._internal.exceptions import CommandError
         # {} may be a long URL, hence the newline.
         message = ("Failed to install {}.\nFor assistance, please raise an issue "
@@ -1019,7 +1015,6 @@ def chaquopy_setup_py_failed(self, exc):
             if wheel_versions:
                 message += ("\nOr try using one of the following versions, which are available "
                             "as pre-built wheels: {}.".format(wheel_versions))
-        logger.critical(str(exc))
         raise CommandError(message)
 
     def get_install_args(

product/gradle-plugin/src/main/python/pip/_internal/utils/setuptools_build.py

@@ -13,6 +13,7 @@
 # warning: "warning: manifest_maker: standard file '-c' not found".
 _SETUPTOOLS_SHIM = (
     "import sys, setuptools, tokenize; sys.argv[0] = {0!r}; __file__={0!r};"
+    "{chaquopy_monkey};"
     "f=getattr(tokenize, 'open', open)(__file__);"
     "code=f.read().replace('\\r\\n', '\\n');"
     "f.close();"
@@ -36,8 +37,15 @@ def make_setuptools_shim_args(setup_py_path, unbuffered_output=False):
     # Chaquopy: added '-S' to avoid interference from site-packages. This makes
     # non-installable packages fail more quickly and consistently. Also, some packages
     # (e.g. Cython) install distutils hooks which can interfere with our attempts to
-    # disable compilers in setuptools/monkey.py.
+    # disable compilers in chaquopy_monkey.
     args.append('-S')
 
-    args.extend(['-c', _SETUPTOOLS_SHIM.format(setup_py_path)])
+    from pip._vendor.packaging import markers
+    chaquopy_monkey = (
+        "import chaquopy_monkey; chaquopy_monkey.disable_native()"
+        if markers.python_version_info
+        else "pass"
+    )
+    args.extend(['-c', _SETUPTOOLS_SHIM.format(setup_py_path,
+                                               chaquopy_monkey=chaquopy_monkey)])
     return args

product/gradle-plugin/src/main/python/pip/_internal/wheel.py

@@ -921,7 +921,9 @@ def _build_one_inside_env(self, req, output_dir, python_tag=None):
                 except Exception:
                     pass
             # Ignore return, we can't do anything else useful.
-            self._clean_one(req)
+            # Chaquopy: backport from https://github.com/pypa/pip/pull/7477
+            if not req.use_pep517:
+                self._clean_one_legacy(req)
             return None
 
     def _base_setup_args(self, req):
@@ -963,6 +965,7 @@ def _build_one_pep517(self, req, tempd, python_tag=None):
                 # Reassign to simplify the return at the end of function
                 wheel_name = new_name
         except Exception:
+            self.check_chaquopy_exception(req)
             logger.error('Failed building wheel for %s', req.name)
             return None
         return os.path.join(tempd, wheel_name)
@@ -988,20 +991,7 @@ def _build_one_legacy(self, req, tempd, python_tag=None):
                                          spinner=spinner)
             except Exception:
                 spinner.finish("error")
-
-                # Chaquopy: if `bdist_wheel` failed because of native code, don't fall back on
-                # `install` because it'll definitely fail, wasting the user's time and making
-                # the failure message harder to read. But if `bdist_wheel` failed for some
-                # other reason, then it's still worth trying `install` (#5630).
-                #
-                # The error message may use spaces or underscores (see
-                # setuptools.monkey.disable_native).
-                from setuptools.monkey import CHAQUOPY_NATIVE_ERROR
-                exc = sys.exc_info()[1]
-                if isinstance(exc, InstallationError) and \
-                   re.search(CHAQUOPY_NATIVE_ERROR.replace(" ", "."), exc.output):
-                    req.chaquopy_setup_py_failed(exc)
-
+                self.check_chaquopy_exception(req)
                 logger.error('Failed building wheel for %s', req.name)
                 return None
             names = os.listdir(tempd)
@@ -1014,7 +1004,21 @@ def _build_one_legacy(self, req, tempd, python_tag=None):
             )
             return wheel_path
 
-    def _clean_one(self, req):
+    def check_chaquopy_exception(self, req):
+        # If `bdist_wheel` failed because of native code, don't fall back on `install`
+        # because it'll definitely fail, wasting the user's time and making the failure
+        # message harder to read. But if `bdist_wheel` failed for some other reason, then
+        # it's still worth trying `install` (#5630).
+        #
+        # The error message may use spaces or underscores (see
+        # chaquopy_monkey.disable_native).
+        from chaquopy_monkey import CHAQUOPY_NATIVE_ERROR
+        exc = sys.exc_info()[1]
+        if isinstance(exc, InstallationError) and \
+           re.search(CHAQUOPY_NATIVE_ERROR.replace(" ", "."), exc.output):
+            req.chaquopy_setup_py_failed()
+
+    def _clean_one_legacy(self, req):
         base_args = self._base_setup_args(req)
 
         logger.info('Running setup.py clean for %s', req.name)

product/gradle-plugin/src/main/python/setuptools/monkey.py

@@ -98,91 +98,7 @@ def patch_all():
             setuptools.extension.Extension
         )
 
-    # Chaquopy disabled: importing distutils.msvc9compiler causes exception "not supported by
-    # this module" on MSYS2 Python, because it isn't built with MSVC.
-    # patch_for_msvc_specialized_compiler()
-
-    disable_native()
-
-
-# Chaquopy: We want to cause a quick and comprehensible failure when a package attempts to
-# build native code, while still allowing a pure-Python fallback if available. This is tricky,
-# because different packages have different approaches to pure-Python fallbacks:
-#
-# * Some packages simply catch any distutils exception thrown by setup(), and then run it again
-#   with the native components removed.
-#
-# * Some (e.g. sqlalchemy, wrapt) extend the distutils build_ext or build_clib command and
-#   override its run() method to wrap it with an exception handler. This means we can't simply
-#   block the commands by name, e.g. by overriding Distribution.run_command.
-#
-# * Some (e.g. msgpack) go lower-level and catch exceptions in build_ext.build_extension. In
-#   Python 3, there's an `optional` keyword to Extension which has the same effect (used e.g.
-#   by websockets). Blocking build_ext.run, or CCompiler.__init__, would cause these builds to
-#   fail before build_extension is called, and the pure-Python fallback wouldn't happen.
-#
-# Creating a new compiler class with a new name minimizes the chance of code trying to do
-# things which will only work on the standard classses. For example,
-# distutils.sysconfig.customize_compiler does things with a "unix" compiler which will crash on
-# Windows because get_config_vars won't have certain settings.
-#
-# This is simpler than determining the regular compiler class and extending it. It avoids
-# interference from NumPy's widespread monkey-patching (including new_compiler, CCompiler and
-# its subclasses), which takes place after this code is run. It also avoids the default
-# behaviour on Windows when no compiler is installed, which is either to give the "Unable to
-# find vcvarsall.bat" error, or advice on how to install Visual C++, both of which will waste
-# the user's time.
-#
-# This approach will block builds of packages which require the compiler name to be in a known
-# list (e.g. minorminer, lz4), but the error messages from these packages aren't too bad, and
-# I've never seen one which has a pure-Python fallback.
-def disable_native():
-    from distutils import ccompiler
-    from distutils.unixccompiler import UnixCCompiler
-    import os
-    import types
-
-    ccompiler.get_default_compiler = lambda *args, **kwargs: "disabled"
-    ccompiler.compiler_class["disabled"] = (
-        "disabledcompiler", "DisabledCompiler",
-        "Compiler disabled ({})".format(CHAQUOPY_NATIVE_ERROR))
-
-    class DisabledCompiler(ccompiler.CCompiler):
-        compiler_type = "disabled"
-        def preprocess(*args, **kwargs):
-            chaquopy_block_native("CCompiler.preprocess")
-        def compile(*args, **kwargs):
-            chaquopy_block_native("CCompiler.compile")
-        def create_static_lib(*args, **kwargs):
-            chaquopy_block_native("CCompiler.create_static_lib")
-        def link(*args, **kwargs):
-            chaquopy_block_native("CCompiler.link")
-
-    # To maximize the chance of the build getting as far as actually calling compile(), make
-    # sure the class has all of the expected attributes.
-    for name in ["src_extensions", "obj_extension", "static_lib_extension",
-                 "shared_lib_extension", "static_lib_format", "shared_lib_format",
-                 "exe_extension"]:
-        setattr(DisabledCompiler, name, getattr(UnixCCompiler, name))
-    DisabledCompiler.executables = {name: [CHAQUOPY_NATIVE_ERROR.replace(" ", "_")]
-                                    for name in UnixCCompiler.executables}
-
-    disabled_mod = types.ModuleType("distutils.disabledcompiler")
-    disabled_mod.DisabledCompiler = DisabledCompiler
-    sys.modules["distutils.disabledcompiler"] = disabled_mod
-
-    # Try to disable native builds for packages which don't use the distutils native build
-    # system at all (e.g. uwsgi), or only use it to wrap an external build script (e.g. pynacl).
-    for tool in ["ar", "as", "cc", "cxx", "ld"]:
-        os.environ[tool.upper()] = CHAQUOPY_NATIVE_ERROR.replace(" ", "_")
-
-
-CHAQUOPY_NATIVE_ERROR = "Chaquopy cannot compile native code"
-
-def chaquopy_block_native(prefix):
-    # No need to give any more advice here: that will come from the higher-level code in pip.
-    from distutils.errors import DistutilsPlatformError
-    raise DistutilsPlatformError("{}: {}".format(prefix, CHAQUOPY_NATIVE_ERROR))
+    patch_for_msvc_specialized_compiler()
 
 
 def _patch_distribution_metadata():

product/gradle-plugin/src/test/integration/data/PythonReqs/pep517/app/build.gradle

@@ -11,7 +11,7 @@ android {
         versionName "0.0.1"
         python {
             pip {
-                install "./sdist_pep517"
+                install "./pep517"
             }
         }
         ndk {

product/gradle-plugin/src/test/integration/data/PythonReqs/pep517/app/pep517/setup.py

@@ -0,0 +1,12 @@
+import setuptools
+from setuptools import setup
+
+# See https://github.com/googleapis/python-crc32c/blob/main/tests/test___init__.py
+import google_crc32c
+assert google_crc32c.implementation == "c"
+version = google_crc32c.value(b"\x00" * 32)
+
+setup(
+    name="pep517",
+    version=version,
+)