diff --git a/product/gradle-plugin/src/main/python/chaquopy_monkey.py b/product/gradle-plugin/src/main/python/chaquopy_monkey.py new file mode 100644 index 00000000000..53e49c21d46 --- /dev/null +++ b/product/gradle-plugin/src/main/python/chaquopy_monkey.py @@ -0,0 +1,87 @@ +# We want to cause a quick and comprehensible failure when a package attempts to build +# native code, while still allowing a pure-Python fallback if available. This is tricky, +# because different packages have different approaches to pure-Python fallbacks: +# +# * Some packages simply catch any distutils exception thrown by setup(), and then run it again +# with the native components removed. +# +# * Some (e.g. sqlalchemy, wrapt) extend the distutils build_ext or build_clib command and +# override its run() method to wrap it with an exception handler. This means we can't simply +# block the commands by name, e.g. by overriding Distribution.run_command. +# +# * Some (e.g. msgpack) go lower-level and catch exceptions in build_ext.build_extension. In +# Python 3, there's an `optional` keyword to Extension which has the same effect (used e.g. +# by websockets). Blocking build_ext.run, or CCompiler.__init__, would cause these builds to +# fail before build_extension is called, and the pure-Python fallback wouldn't happen. +# +# Creating a new compiler class with a new name minimizes the chance of code trying to do +# things which will only work on the standard classses. For example, +# distutils.sysconfig.customize_compiler does things with a "unix" compiler which will crash on +# Windows because get_config_vars won't have certain settings. +# +# This is simpler than determining the regular compiler class and extending it. It avoids +# interference from NumPy's widespread monkey-patching (including new_compiler, CCompiler and +# its subclasses), which takes place after this code is run. It also avoids the default +# behaviour on Windows when no compiler is installed, which is either to give the "Unable to +# find vcvarsall.bat" error, or advice on how to install Visual C++, both of which will waste +# the user's time. +# +# This approach will block builds of packages which require the compiler name to be in a known +# list (e.g. minorminer, lz4), but the error messages from these packages aren't too bad, and +# I've never seen one which has a pure-Python fallback. +def disable_native(): + # Recent versions of setuptools redirect distutils to their own bundled copy, so try + # to import that first. + try: + import setuptools # noqa: F401 + except ImportError: + pass + + from distutils import ccompiler + from distutils.unixccompiler import UnixCCompiler + import os + import sys + import types + + ccompiler.get_default_compiler = lambda *args, **kwargs: "disabled" + ccompiler.compiler_class["disabled"] = ( + "disabledcompiler", "DisabledCompiler", + "Compiler disabled ({})".format(CHAQUOPY_NATIVE_ERROR)) + + class DisabledCompiler(ccompiler.CCompiler): + compiler_type = "disabled" + def preprocess(*args, **kwargs): + chaquopy_block_native("CCompiler.preprocess") + def compile(*args, **kwargs): + chaquopy_block_native("CCompiler.compile") + def create_static_lib(*args, **kwargs): + chaquopy_block_native("CCompiler.create_static_lib") + def link(*args, **kwargs): + chaquopy_block_native("CCompiler.link") + + # To maximize the chance of the build getting as far as actually calling compile(), make + # sure the class has all of the expected attributes. + for name in ["src_extensions", "obj_extension", "static_lib_extension", + "shared_lib_extension", "static_lib_format", "shared_lib_format", + "exe_extension"]: + setattr(DisabledCompiler, name, getattr(UnixCCompiler, name)) + DisabledCompiler.executables = {name: [CHAQUOPY_NATIVE_ERROR.replace(" ", "_")] + for name in UnixCCompiler.executables} + + disabled_mod_name = "distutils.disabledcompiler" + disabled_mod = types.ModuleType(disabled_mod_name) + disabled_mod.DisabledCompiler = DisabledCompiler + sys.modules[disabled_mod_name] = disabled_mod + + # Try to disable native builds for packages which don't use the distutils native build + # system at all (e.g. uwsgi), or only use it to wrap an external build script (e.g. pynacl). + for tool in ["ar", "as", "cc", "cxx", "ld"]: + os.environ[tool.upper()] = CHAQUOPY_NATIVE_ERROR.replace(" ", "_") + + +CHAQUOPY_NATIVE_ERROR = "Chaquopy cannot compile native code" + +def chaquopy_block_native(prefix): + # No need to give any more advice here: that will come from the higher-level code in pip. + from distutils.errors import DistutilsPlatformError + raise DistutilsPlatformError("{}: {}".format(prefix, CHAQUOPY_NATIVE_ERROR)) diff --git a/product/gradle-plugin/src/main/python/pip/_internal/build_env.py b/product/gradle-plugin/src/main/python/pip/_internal/build_env.py index a060ceea2ca..3601642e84f 100644 --- a/product/gradle-plugin/src/main/python/pip/_internal/build_env.py +++ b/product/gradle-plugin/src/main/python/pip/_internal/build_env.py @@ -104,6 +104,22 @@ def __init__(self): ''' ).format(system_sites=system_sites, lib_dirs=self._lib_dirs)) + # Chaquopy + from os.path import dirname, join + import shutil + # Copy chaquopy_monkey unconditionally, so we can import it unconditionally in + # check_chaquopy_exception. + shutil.copy(join(dirname(pip_location), "../chaquopy_monkey.py"), + self._site_dir) + from pip._vendor.packaging import markers + if markers.python_version_info: + fp.write(textwrap.dedent( + ''' + import chaquopy_monkey + chaquopy_monkey.disable_native() + ''' + )) + def __enter__(self): self._save_env = { name: os.environ.get(name, None) diff --git a/product/gradle-plugin/src/main/python/pip/_internal/cli/cmdoptions.py b/product/gradle-plugin/src/main/python/pip/_internal/cli/cmdoptions.py index 244c9f993b2..67995390776 100644 --- a/product/gradle-plugin/src/main/python/pip/_internal/cli/cmdoptions.py +++ b/product/gradle-plugin/src/main/python/pip/_internal/cli/cmdoptions.py @@ -111,7 +111,7 @@ def check_dist_restriction(options, check_target=False): # guaranteed to be locally compatible. # # Chaquopy: added False to disable this restriction. It's safe for us to run source - # distributions, because we monkey-patch setuptools to ensure they fail immediately if they + # distributions, because chaquopy_monkey ensures they fail immediately if they # try to build anything native. if False and dist_restriction_set and sdist_dependencies_allowed: raise CommandError( diff --git a/product/gradle-plugin/src/main/python/pip/_internal/operations/prepare.py b/product/gradle-plugin/src/main/python/pip/_internal/operations/prepare.py index 6cf5f0edd68..216053de4be 100644 --- a/product/gradle-plugin/src/main/python/pip/_internal/operations/prepare.py +++ b/product/gradle-plugin/src/main/python/pip/_internal/operations/prepare.py @@ -211,9 +211,13 @@ def prepare_linked_requirement( ) abstract_dist = make_distribution_for_install_requirement(req) with self.req_tracker.track(req): - abstract_dist.prepare_distribution_metadata( - finder, self.build_isolation, - ) + try: + abstract_dist.prepare_distribution_metadata( + finder, self.build_isolation, + ) + except InstallationError: + abstract_dist.req.chaquopy_setup_py_failed() + if self._download_should_save: # Make a .zip of the source_dir we already created. if not req.link.is_artifact: diff --git a/product/gradle-plugin/src/main/python/pip/_internal/req/req_install.py b/product/gradle-plugin/src/main/python/pip/_internal/req/req_install.py index 9810eb43508..ce5b2c3424b 100644 --- a/product/gradle-plugin/src/main/python/pip/_internal/req/req_install.py +++ b/product/gradle-plugin/src/main/python/pip/_internal/req/req_install.py @@ -623,14 +623,10 @@ def run_egg_info(self): ensure_dir(egg_info_dir) egg_base_option = ['--egg-base', 'pip-egg-info'] with self.build_env: - try: - call_subprocess( - egg_info_cmd + egg_base_option, - cwd=self.setup_py_dir, - command_desc='python setup.py egg_info') - except InstallationError as exc: - self.chaquopy_setup_py_failed(exc) - + call_subprocess( + egg_info_cmd + egg_base_option, + cwd=self.setup_py_dir, + command_desc='python setup.py egg_info') @property def egg_info_path(self): @@ -1006,7 +1002,7 @@ def prepend_root(path): with open(inst_files_path, 'w') as f: f.write('\n'.join(new_lines) + '\n') - def chaquopy_setup_py_failed(self, exc): + def chaquopy_setup_py_failed(self): from pip._internal.exceptions import CommandError # {} may be a long URL, hence the newline. message = ("Failed to install {}.\nFor assistance, please raise an issue " @@ -1019,7 +1015,6 @@ def chaquopy_setup_py_failed(self, exc): if wheel_versions: message += ("\nOr try using one of the following versions, which are available " "as pre-built wheels: {}.".format(wheel_versions)) - logger.critical(str(exc)) raise CommandError(message) def get_install_args( diff --git a/product/gradle-plugin/src/main/python/pip/_internal/utils/setuptools_build.py b/product/gradle-plugin/src/main/python/pip/_internal/utils/setuptools_build.py index 01aa6a57c20..1e002fb797c 100644 --- a/product/gradle-plugin/src/main/python/pip/_internal/utils/setuptools_build.py +++ b/product/gradle-plugin/src/main/python/pip/_internal/utils/setuptools_build.py @@ -13,6 +13,7 @@ # warning: "warning: manifest_maker: standard file '-c' not found". _SETUPTOOLS_SHIM = ( "import sys, setuptools, tokenize; sys.argv[0] = {0!r}; __file__={0!r};" + "{chaquopy_monkey};" "f=getattr(tokenize, 'open', open)(__file__);" "code=f.read().replace('\\r\\n', '\\n');" "f.close();" @@ -36,8 +37,15 @@ def make_setuptools_shim_args(setup_py_path, unbuffered_output=False): # Chaquopy: added '-S' to avoid interference from site-packages. This makes # non-installable packages fail more quickly and consistently. Also, some packages # (e.g. Cython) install distutils hooks which can interfere with our attempts to - # disable compilers in setuptools/monkey.py. + # disable compilers in chaquopy_monkey. args.append('-S') - args.extend(['-c', _SETUPTOOLS_SHIM.format(setup_py_path)]) + from pip._vendor.packaging import markers + chaquopy_monkey = ( + "import chaquopy_monkey; chaquopy_monkey.disable_native()" + if markers.python_version_info + else "pass" + ) + args.extend(['-c', _SETUPTOOLS_SHIM.format(setup_py_path, + chaquopy_monkey=chaquopy_monkey)]) return args diff --git a/product/gradle-plugin/src/main/python/pip/_internal/wheel.py b/product/gradle-plugin/src/main/python/pip/_internal/wheel.py index 5f9401bfd6e..488a907a69c 100644 --- a/product/gradle-plugin/src/main/python/pip/_internal/wheel.py +++ b/product/gradle-plugin/src/main/python/pip/_internal/wheel.py @@ -921,7 +921,9 @@ def _build_one_inside_env(self, req, output_dir, python_tag=None): except Exception: pass # Ignore return, we can't do anything else useful. - self._clean_one(req) + # Chaquopy: backport from https://github.com/pypa/pip/pull/7477 + if not req.use_pep517: + self._clean_one_legacy(req) return None def _base_setup_args(self, req): @@ -963,6 +965,7 @@ def _build_one_pep517(self, req, tempd, python_tag=None): # Reassign to simplify the return at the end of function wheel_name = new_name except Exception: + self.check_chaquopy_exception(req) logger.error('Failed building wheel for %s', req.name) return None return os.path.join(tempd, wheel_name) @@ -988,20 +991,7 @@ def _build_one_legacy(self, req, tempd, python_tag=None): spinner=spinner) except Exception: spinner.finish("error") - - # Chaquopy: if `bdist_wheel` failed because of native code, don't fall back on - # `install` because it'll definitely fail, wasting the user's time and making - # the failure message harder to read. But if `bdist_wheel` failed for some - # other reason, then it's still worth trying `install` (#5630). - # - # The error message may use spaces or underscores (see - # setuptools.monkey.disable_native). - from setuptools.monkey import CHAQUOPY_NATIVE_ERROR - exc = sys.exc_info()[1] - if isinstance(exc, InstallationError) and \ - re.search(CHAQUOPY_NATIVE_ERROR.replace(" ", "."), exc.output): - req.chaquopy_setup_py_failed(exc) - + self.check_chaquopy_exception(req) logger.error('Failed building wheel for %s', req.name) return None names = os.listdir(tempd) @@ -1014,7 +1004,21 @@ def _build_one_legacy(self, req, tempd, python_tag=None): ) return wheel_path - def _clean_one(self, req): + def check_chaquopy_exception(self, req): + # If `bdist_wheel` failed because of native code, don't fall back on `install` + # because it'll definitely fail, wasting the user's time and making the failure + # message harder to read. But if `bdist_wheel` failed for some other reason, then + # it's still worth trying `install` (#5630). + # + # The error message may use spaces or underscores (see + # chaquopy_monkey.disable_native). + from chaquopy_monkey import CHAQUOPY_NATIVE_ERROR + exc = sys.exc_info()[1] + if isinstance(exc, InstallationError) and \ + re.search(CHAQUOPY_NATIVE_ERROR.replace(" ", "."), exc.output): + req.chaquopy_setup_py_failed() + + def _clean_one_legacy(self, req): base_args = self._base_setup_args(req) logger.info('Running setup.py clean for %s', req.name) diff --git a/product/gradle-plugin/src/main/python/setuptools/monkey.py b/product/gradle-plugin/src/main/python/setuptools/monkey.py index f735535d75d..3c77f8cf27f 100644 --- a/product/gradle-plugin/src/main/python/setuptools/monkey.py +++ b/product/gradle-plugin/src/main/python/setuptools/monkey.py @@ -98,91 +98,7 @@ def patch_all(): setuptools.extension.Extension ) - # Chaquopy disabled: importing distutils.msvc9compiler causes exception "not supported by - # this module" on MSYS2 Python, because it isn't built with MSVC. - # patch_for_msvc_specialized_compiler() - - disable_native() - - -# Chaquopy: We want to cause a quick and comprehensible failure when a package attempts to -# build native code, while still allowing a pure-Python fallback if available. This is tricky, -# because different packages have different approaches to pure-Python fallbacks: -# -# * Some packages simply catch any distutils exception thrown by setup(), and then run it again -# with the native components removed. -# -# * Some (e.g. sqlalchemy, wrapt) extend the distutils build_ext or build_clib command and -# override its run() method to wrap it with an exception handler. This means we can't simply -# block the commands by name, e.g. by overriding Distribution.run_command. -# -# * Some (e.g. msgpack) go lower-level and catch exceptions in build_ext.build_extension. In -# Python 3, there's an `optional` keyword to Extension which has the same effect (used e.g. -# by websockets). Blocking build_ext.run, or CCompiler.__init__, would cause these builds to -# fail before build_extension is called, and the pure-Python fallback wouldn't happen. -# -# Creating a new compiler class with a new name minimizes the chance of code trying to do -# things which will only work on the standard classses. For example, -# distutils.sysconfig.customize_compiler does things with a "unix" compiler which will crash on -# Windows because get_config_vars won't have certain settings. -# -# This is simpler than determining the regular compiler class and extending it. It avoids -# interference from NumPy's widespread monkey-patching (including new_compiler, CCompiler and -# its subclasses), which takes place after this code is run. It also avoids the default -# behaviour on Windows when no compiler is installed, which is either to give the "Unable to -# find vcvarsall.bat" error, or advice on how to install Visual C++, both of which will waste -# the user's time. -# -# This approach will block builds of packages which require the compiler name to be in a known -# list (e.g. minorminer, lz4), but the error messages from these packages aren't too bad, and -# I've never seen one which has a pure-Python fallback. -def disable_native(): - from distutils import ccompiler - from distutils.unixccompiler import UnixCCompiler - import os - import types - - ccompiler.get_default_compiler = lambda *args, **kwargs: "disabled" - ccompiler.compiler_class["disabled"] = ( - "disabledcompiler", "DisabledCompiler", - "Compiler disabled ({})".format(CHAQUOPY_NATIVE_ERROR)) - - class DisabledCompiler(ccompiler.CCompiler): - compiler_type = "disabled" - def preprocess(*args, **kwargs): - chaquopy_block_native("CCompiler.preprocess") - def compile(*args, **kwargs): - chaquopy_block_native("CCompiler.compile") - def create_static_lib(*args, **kwargs): - chaquopy_block_native("CCompiler.create_static_lib") - def link(*args, **kwargs): - chaquopy_block_native("CCompiler.link") - - # To maximize the chance of the build getting as far as actually calling compile(), make - # sure the class has all of the expected attributes. - for name in ["src_extensions", "obj_extension", "static_lib_extension", - "shared_lib_extension", "static_lib_format", "shared_lib_format", - "exe_extension"]: - setattr(DisabledCompiler, name, getattr(UnixCCompiler, name)) - DisabledCompiler.executables = {name: [CHAQUOPY_NATIVE_ERROR.replace(" ", "_")] - for name in UnixCCompiler.executables} - - disabled_mod = types.ModuleType("distutils.disabledcompiler") - disabled_mod.DisabledCompiler = DisabledCompiler - sys.modules["distutils.disabledcompiler"] = disabled_mod - - # Try to disable native builds for packages which don't use the distutils native build - # system at all (e.g. uwsgi), or only use it to wrap an external build script (e.g. pynacl). - for tool in ["ar", "as", "cc", "cxx", "ld"]: - os.environ[tool.upper()] = CHAQUOPY_NATIVE_ERROR.replace(" ", "_") - - -CHAQUOPY_NATIVE_ERROR = "Chaquopy cannot compile native code" - -def chaquopy_block_native(prefix): - # No need to give any more advice here: that will come from the higher-level code in pip. - from distutils.errors import DistutilsPlatformError - raise DistutilsPlatformError("{}: {}".format(prefix, CHAQUOPY_NATIVE_ERROR)) + patch_for_msvc_specialized_compiler() def _patch_distribution_metadata(): diff --git a/product/gradle-plugin/src/test/integration/data/PythonReqs/pep517/app/build.gradle b/product/gradle-plugin/src/test/integration/data/PythonReqs/pep517/app/build.gradle index ad0436b8a80..3797573e361 100644 --- a/product/gradle-plugin/src/test/integration/data/PythonReqs/pep517/app/build.gradle +++ b/product/gradle-plugin/src/test/integration/data/PythonReqs/pep517/app/build.gradle @@ -11,7 +11,7 @@ android { versionName "0.0.1" python { pip { - install "./sdist_pep517" + install "./pep517" } } ndk { diff --git a/product/gradle-plugin/src/test/integration/data/PythonReqs/pep517/app/pep517/setup.py b/product/gradle-plugin/src/test/integration/data/PythonReqs/pep517/app/pep517/setup.py new file mode 100644 index 00000000000..60e24e50de0 --- /dev/null +++ b/product/gradle-plugin/src/test/integration/data/PythonReqs/pep517/app/pep517/setup.py @@ -0,0 +1,12 @@ +import setuptools +from setuptools import setup + +# See https://github.com/googleapis/python-crc32c/blob/main/tests/test___init__.py +import google_crc32c +assert google_crc32c.implementation == "c" +version = google_crc32c.value(b"\x00" * 32) + +setup( + name="pep517", + version=version, +) diff --git a/product/gradle-plugin/src/test/integration/data/PythonReqs/pep517/app/sdist_pep517/sdist_pep517.py b/product/gradle-plugin/src/test/integration/data/PythonReqs/pep517/app/sdist_pep517/sdist_pep517.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/product/gradle-plugin/src/test/integration/data/PythonReqs/pep517/app/sdist_pep517/setup.py b/product/gradle-plugin/src/test/integration/data/PythonReqs/pep517/app/sdist_pep517/setup.py deleted file mode 100644 index 805e26e290b..00000000000 --- a/product/gradle-plugin/src/test/integration/data/PythonReqs/pep517/app/sdist_pep517/setup.py +++ /dev/null @@ -1,9 +0,0 @@ -import setuptools -from setuptools import setup - - -setup( - name="sdist_pep517", - version=setuptools.__version__, - py_modules=["sdist_pep517"], -) diff --git a/product/gradle-plugin/src/test/integration/data/PythonReqs/pep517_default_backend/app/build.gradle b/product/gradle-plugin/src/test/integration/data/PythonReqs/pep517_default_backend/app/build.gradle deleted file mode 100644 index ad0436b8a80..00000000000 --- a/product/gradle-plugin/src/test/integration/data/PythonReqs/pep517_default_backend/app/build.gradle +++ /dev/null @@ -1,21 +0,0 @@ -apply plugin: 'com.android.application' -apply plugin: 'com.chaquo.python' - -android { - compileSdkVersion 23 - defaultConfig { - applicationId "com.chaquo.python.test" - minSdkVersion 21 - targetSdkVersion 23 - versionCode 1 - versionName "0.0.1" - python { - pip { - install "./sdist_pep517" - } - } - ndk { - abiFilters "x86" - } - } -} diff --git a/product/gradle-plugin/src/test/integration/data/PythonReqs/pep517_default_backend/app/pep517/pyproject.toml b/product/gradle-plugin/src/test/integration/data/PythonReqs/pep517_default_backend/app/pep517/pyproject.toml new file mode 100644 index 00000000000..f4a025bb27d --- /dev/null +++ b/product/gradle-plugin/src/test/integration/data/PythonReqs/pep517_default_backend/app/pep517/pyproject.toml @@ -0,0 +1,2 @@ +[build-system] +requires = ["setuptools", "wheel", "google-crc32c"] diff --git a/product/gradle-plugin/src/test/integration/data/PythonReqs/pep517_default_backend/app/sdist_pep517/pyproject.toml b/product/gradle-plugin/src/test/integration/data/PythonReqs/pep517_default_backend/app/sdist_pep517/pyproject.toml deleted file mode 100644 index 77e3cca19ea..00000000000 --- a/product/gradle-plugin/src/test/integration/data/PythonReqs/pep517_default_backend/app/sdist_pep517/pyproject.toml +++ /dev/null @@ -1,2 +0,0 @@ -[build-system] -requires = ["setuptools==40.8.0", "wheel"] diff --git a/product/gradle-plugin/src/test/integration/data/PythonReqs/pep517_default_backend/app/sdist_pep517/sdist_pep517.py b/product/gradle-plugin/src/test/integration/data/PythonReqs/pep517_default_backend/app/sdist_pep517/sdist_pep517.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/product/gradle-plugin/src/test/integration/data/PythonReqs/pep517_default_backend/app/sdist_pep517/setup.py b/product/gradle-plugin/src/test/integration/data/PythonReqs/pep517_default_backend/app/sdist_pep517/setup.py deleted file mode 100644 index 805e26e290b..00000000000 --- a/product/gradle-plugin/src/test/integration/data/PythonReqs/pep517_default_backend/app/sdist_pep517/setup.py +++ /dev/null @@ -1,9 +0,0 @@ -import setuptools -from setuptools import setup - - -setup( - name="sdist_pep517", - version=setuptools.__version__, - py_modules=["sdist_pep517"], -) diff --git a/product/gradle-plugin/src/test/integration/data/PythonReqs/pep517_explicit_backend/app/pep517/pyproject.toml b/product/gradle-plugin/src/test/integration/data/PythonReqs/pep517_explicit_backend/app/pep517/pyproject.toml new file mode 100644 index 00000000000..7f1bf86d801 --- /dev/null +++ b/product/gradle-plugin/src/test/integration/data/PythonReqs/pep517_explicit_backend/app/pep517/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["setuptools", "wheel", "google-crc32c"] +build-backend = "setuptools.build_meta" diff --git a/product/gradle-plugin/src/test/integration/data/PythonReqs/sdist_native_cc/app/build.gradle b/product/gradle-plugin/src/test/integration/data/PythonReqs/sdist_native_cc/app/build.gradle index a28cc6a57dc..7d570f71a7e 100644 --- a/product/gradle-plugin/src/test/integration/data/PythonReqs/sdist_native_cc/app/build.gradle +++ b/product/gradle-plugin/src/test/integration/data/PythonReqs/sdist_native_cc/app/build.gradle @@ -11,7 +11,7 @@ android { versionName "0.0.1" python { pip { - install "sdist_native_cc-1.0.tar.gz" + install "./sdist_native" } } ndk { diff --git a/product/gradle-plugin/src/test/integration/packages/src/sdist_native_cc/setup.py b/product/gradle-plugin/src/test/integration/data/PythonReqs/sdist_native_cc/app/sdist_native/setup.py similarity index 100% rename from product/gradle-plugin/src/test/integration/packages/src/sdist_native_cc/setup.py rename to product/gradle-plugin/src/test/integration/data/PythonReqs/sdist_native_cc/app/sdist_native/setup.py diff --git a/product/gradle-plugin/src/test/integration/data/PythonReqs/sdist_native_cc/app/sdist_native_cc-1.0.tar.gz b/product/gradle-plugin/src/test/integration/data/PythonReqs/sdist_native_cc/app/sdist_native_cc-1.0.tar.gz deleted file mode 100644 index aabf0ca0496..00000000000 Binary files a/product/gradle-plugin/src/test/integration/data/PythonReqs/sdist_native_cc/app/sdist_native_cc-1.0.tar.gz and /dev/null differ diff --git a/product/gradle-plugin/src/test/integration/data/PythonReqs/sdist_native_clib/app/build.gradle b/product/gradle-plugin/src/test/integration/data/PythonReqs/sdist_native_clib/app/build.gradle index 3aa7a96b1a3..7d570f71a7e 100644 --- a/product/gradle-plugin/src/test/integration/data/PythonReqs/sdist_native_clib/app/build.gradle +++ b/product/gradle-plugin/src/test/integration/data/PythonReqs/sdist_native_clib/app/build.gradle @@ -11,7 +11,7 @@ android { versionName "0.0.1" python { pip { - install "sdist_native_clib-1.0.tar.gz" + install "./sdist_native" } } ndk { diff --git a/product/gradle-plugin/src/test/integration/packages/src/sdist_native_clib/libtest.c b/product/gradle-plugin/src/test/integration/data/PythonReqs/sdist_native_clib/app/sdist_native/libtest.c similarity index 100% rename from product/gradle-plugin/src/test/integration/packages/src/sdist_native_clib/libtest.c rename to product/gradle-plugin/src/test/integration/data/PythonReqs/sdist_native_clib/app/sdist_native/libtest.c diff --git a/product/gradle-plugin/src/test/integration/packages/src/sdist_native_clib/setup.py b/product/gradle-plugin/src/test/integration/data/PythonReqs/sdist_native_clib/app/sdist_native/setup.py similarity index 100% rename from product/gradle-plugin/src/test/integration/packages/src/sdist_native_clib/setup.py rename to product/gradle-plugin/src/test/integration/data/PythonReqs/sdist_native_clib/app/sdist_native/setup.py diff --git a/product/gradle-plugin/src/test/integration/data/PythonReqs/sdist_native_clib/app/sdist_native_clib-1.0.tar.gz b/product/gradle-plugin/src/test/integration/data/PythonReqs/sdist_native_clib/app/sdist_native_clib-1.0.tar.gz deleted file mode 100644 index 5f0e5820979..00000000000 Binary files a/product/gradle-plugin/src/test/integration/data/PythonReqs/sdist_native_clib/app/sdist_native_clib-1.0.tar.gz and /dev/null differ diff --git a/product/gradle-plugin/src/test/integration/data/PythonReqs/sdist_native_compiler/app/build.gradle b/product/gradle-plugin/src/test/integration/data/PythonReqs/sdist_native_compiler/app/build.gradle index 500995e2414..7d570f71a7e 100644 --- a/product/gradle-plugin/src/test/integration/data/PythonReqs/sdist_native_compiler/app/build.gradle +++ b/product/gradle-plugin/src/test/integration/data/PythonReqs/sdist_native_compiler/app/build.gradle @@ -11,7 +11,7 @@ android { versionName "0.0.1" python { pip { - install "sdist_native_compiler-1.0.tar.gz" + install "./sdist_native" } } ndk { diff --git a/product/gradle-plugin/src/test/integration/packages/src/sdist_native_compiler/setup.py b/product/gradle-plugin/src/test/integration/data/PythonReqs/sdist_native_compiler/app/sdist_native/setup.py similarity index 100% rename from product/gradle-plugin/src/test/integration/packages/src/sdist_native_compiler/setup.py rename to product/gradle-plugin/src/test/integration/data/PythonReqs/sdist_native_compiler/app/sdist_native/setup.py diff --git a/product/gradle-plugin/src/test/integration/data/PythonReqs/sdist_native_compiler/app/sdist_native_compiler-1.0.tar.gz b/product/gradle-plugin/src/test/integration/data/PythonReqs/sdist_native_compiler/app/sdist_native_compiler-1.0.tar.gz deleted file mode 100644 index 4b227c88e05..00000000000 Binary files a/product/gradle-plugin/src/test/integration/data/PythonReqs/sdist_native_compiler/app/sdist_native_compiler-1.0.tar.gz and /dev/null differ diff --git a/product/gradle-plugin/src/test/integration/data/PythonReqs/sdist_native_ext/app/build.gradle b/product/gradle-plugin/src/test/integration/data/PythonReqs/sdist_native_ext/app/build.gradle index 3a553417d83..7d570f71a7e 100644 --- a/product/gradle-plugin/src/test/integration/data/PythonReqs/sdist_native_ext/app/build.gradle +++ b/product/gradle-plugin/src/test/integration/data/PythonReqs/sdist_native_ext/app/build.gradle @@ -11,7 +11,7 @@ android { versionName "0.0.1" python { pip { - install "sdist_native_ext-1.0.tar.gz" + install "./sdist_native" } } ndk { diff --git a/product/gradle-plugin/src/test/integration/packages/src/sdist_native_ext/ext_module.c b/product/gradle-plugin/src/test/integration/data/PythonReqs/sdist_native_ext/app/sdist_native/ext_module.c similarity index 100% rename from product/gradle-plugin/src/test/integration/packages/src/sdist_native_ext/ext_module.c rename to product/gradle-plugin/src/test/integration/data/PythonReqs/sdist_native_ext/app/sdist_native/ext_module.c diff --git a/product/gradle-plugin/src/test/integration/packages/src/sdist_native_ext/setup.py b/product/gradle-plugin/src/test/integration/data/PythonReqs/sdist_native_ext/app/sdist_native/setup.py similarity index 100% rename from product/gradle-plugin/src/test/integration/packages/src/sdist_native_ext/setup.py rename to product/gradle-plugin/src/test/integration/data/PythonReqs/sdist_native_ext/app/sdist_native/setup.py diff --git a/product/gradle-plugin/src/test/integration/data/PythonReqs/sdist_native_ext/app/sdist_native_ext-1.0.tar.gz b/product/gradle-plugin/src/test/integration/data/PythonReqs/sdist_native_ext/app/sdist_native_ext-1.0.tar.gz deleted file mode 100644 index 4800d06d7e5..00000000000 Binary files a/product/gradle-plugin/src/test/integration/data/PythonReqs/sdist_native_ext/app/sdist_native_ext-1.0.tar.gz and /dev/null differ diff --git a/product/gradle-plugin/src/test/integration/data/PythonReqs/sdist_native_optional_compiler/app/build.gradle b/product/gradle-plugin/src/test/integration/data/PythonReqs/sdist_native_optional_compiler/app/build.gradle index 76c133da074..7d570f71a7e 100644 --- a/product/gradle-plugin/src/test/integration/data/PythonReqs/sdist_native_optional_compiler/app/build.gradle +++ b/product/gradle-plugin/src/test/integration/data/PythonReqs/sdist_native_optional_compiler/app/build.gradle @@ -11,7 +11,7 @@ android { versionName "0.0.1" python { pip { - install "sdist_native_optional_compiler-1.0.zip" + install "./sdist_native" } } ndk { diff --git a/product/gradle-plugin/src/test/integration/packages/src/sdist_native_optional_compiler/sdist_native_optional_compiler.py b/product/gradle-plugin/src/test/integration/data/PythonReqs/sdist_native_optional_compiler/app/sdist_native/sdist_native_optional_compiler.py similarity index 100% rename from product/gradle-plugin/src/test/integration/packages/src/sdist_native_optional_compiler/sdist_native_optional_compiler.py rename to product/gradle-plugin/src/test/integration/data/PythonReqs/sdist_native_optional_compiler/app/sdist_native/sdist_native_optional_compiler.py diff --git a/product/gradle-plugin/src/test/integration/packages/src/sdist_native_optional_compiler/setup.py b/product/gradle-plugin/src/test/integration/data/PythonReqs/sdist_native_optional_compiler/app/sdist_native/setup.py similarity index 100% rename from product/gradle-plugin/src/test/integration/packages/src/sdist_native_optional_compiler/setup.py rename to product/gradle-plugin/src/test/integration/data/PythonReqs/sdist_native_optional_compiler/app/sdist_native/setup.py diff --git a/product/gradle-plugin/src/test/integration/data/PythonReqs/sdist_native_optional_compiler/app/sdist_native_optional_compiler-1.0.zip b/product/gradle-plugin/src/test/integration/data/PythonReqs/sdist_native_optional_compiler/app/sdist_native_optional_compiler-1.0.zip deleted file mode 100644 index c5a6a16d81a..00000000000 Binary files a/product/gradle-plugin/src/test/integration/data/PythonReqs/sdist_native_optional_compiler/app/sdist_native_optional_compiler-1.0.zip and /dev/null differ diff --git a/product/gradle-plugin/src/test/integration/data/PythonReqs/sdist_native_optional_ext/app/build.gradle b/product/gradle-plugin/src/test/integration/data/PythonReqs/sdist_native_optional_ext/app/build.gradle index 5b74019e7b0..7d570f71a7e 100644 --- a/product/gradle-plugin/src/test/integration/data/PythonReqs/sdist_native_optional_ext/app/build.gradle +++ b/product/gradle-plugin/src/test/integration/data/PythonReqs/sdist_native_optional_ext/app/build.gradle @@ -11,7 +11,7 @@ android { versionName "0.0.1" python { pip { - install "sdist_native_optional_ext-1.0.zip" + install "./sdist_native" } } ndk { diff --git a/product/gradle-plugin/src/test/integration/packages/src/sdist_native_optional_ext/ext_module.c b/product/gradle-plugin/src/test/integration/data/PythonReqs/sdist_native_optional_ext/app/sdist_native/ext_module.c similarity index 100% rename from product/gradle-plugin/src/test/integration/packages/src/sdist_native_optional_ext/ext_module.c rename to product/gradle-plugin/src/test/integration/data/PythonReqs/sdist_native_optional_ext/app/sdist_native/ext_module.c diff --git a/product/gradle-plugin/src/test/integration/packages/src/sdist_native_optional_ext/sdist_native_optional_ext.py b/product/gradle-plugin/src/test/integration/data/PythonReqs/sdist_native_optional_ext/app/sdist_native/sdist_native_optional_ext.py similarity index 100% rename from product/gradle-plugin/src/test/integration/packages/src/sdist_native_optional_ext/sdist_native_optional_ext.py rename to product/gradle-plugin/src/test/integration/data/PythonReqs/sdist_native_optional_ext/app/sdist_native/sdist_native_optional_ext.py diff --git a/product/gradle-plugin/src/test/integration/packages/src/sdist_native_optional_ext/setup.py b/product/gradle-plugin/src/test/integration/data/PythonReqs/sdist_native_optional_ext/app/sdist_native/setup.py similarity index 100% rename from product/gradle-plugin/src/test/integration/packages/src/sdist_native_optional_ext/setup.py rename to product/gradle-plugin/src/test/integration/data/PythonReqs/sdist_native_optional_ext/app/sdist_native/setup.py diff --git a/product/gradle-plugin/src/test/integration/data/PythonReqs/sdist_native_optional_ext/app/sdist_native_optional_ext-1.0.zip b/product/gradle-plugin/src/test/integration/data/PythonReqs/sdist_native_optional_ext/app/sdist_native_optional_ext-1.0.zip deleted file mode 100644 index a94dd4fbb82..00000000000 Binary files a/product/gradle-plugin/src/test/integration/data/PythonReqs/sdist_native_optional_ext/app/sdist_native_optional_ext-1.0.zip and /dev/null differ diff --git a/product/gradle-plugin/src/test/integration/data/PythonReqs/pep517/app/sdist_pep517/pyproject.toml b/product/gradle-plugin/src/test/integration/data/PythonReqs/sdist_native_pep517/app/sdist_native/pyproject.toml similarity index 56% rename from product/gradle-plugin/src/test/integration/data/PythonReqs/pep517/app/sdist_pep517/pyproject.toml rename to product/gradle-plugin/src/test/integration/data/PythonReqs/sdist_native_pep517/app/sdist_native/pyproject.toml index 746c0a69ff0..9787c3bdf00 100644 --- a/product/gradle-plugin/src/test/integration/data/PythonReqs/pep517/app/sdist_pep517/pyproject.toml +++ b/product/gradle-plugin/src/test/integration/data/PythonReqs/sdist_native_pep517/app/sdist_native/pyproject.toml @@ -1,3 +1,3 @@ [build-system] -requires = ["setuptools==40.8.0", "wheel"] +requires = ["setuptools", "wheel"] build-backend = "setuptools.build_meta" diff --git a/product/gradle-plugin/src/test/integration/packages/README.txt b/product/gradle-plugin/src/test/integration/packages/README.txt index 1043c449a7f..972526779c4 100644 --- a/product/gradle-plugin/src/test/integration/packages/README.txt +++ b/product/gradle-plugin/src/test/integration/packages/README.txt @@ -7,3 +7,10 @@ commands in the src/ subdirectories: Some of the scripts require extra arguments: see comments. Some of the packages were then renamed to meet the needs of the tests. + +However, unless a test actually requires a package file, it's better to put the Python +source tree inside the test's own data directory, and install it like this: + + pip { + install "./dir_name" + } diff --git a/product/gradle-plugin/src/test/integration/test_gradle_plugin.py b/product/gradle-plugin/src/test/integration/test_gradle_plugin.py index e75effb1c1a..2cef960e02b 100644 --- a/product/gradle-plugin/src/test/integration/test_gradle_plugin.py +++ b/product/gradle-plugin/src/test/integration/test_gradle_plugin.py @@ -888,16 +888,16 @@ def test_wheel_data(self): def test_sdist_file(self): self.RunGradle("base", "PythonReqs/sdist_file", requirements=["alpha_dep/__init__.py"]) - # This project uses pyproject.toml to require a specific version of setuptools, then - # gives itself the same version number. - def test_pep517(self): - self.RunGradle("base", "PythonReqs/pep517", requirements=["sdist_pep517.py"], - dist_versions=[("sdist_pep517", "40.8.0")]) - - # Same as test_pep517, but pyproject.toml does not contain a `build-backend` setting. + # These tests install a package with a native build requirement in its pyproject.toml, + # which is used to generate the package's version number. This verifies that the build + # environment is installed for the build platform, not the target platform. def test_pep517_default_backend(self): - self.RunGradle("base", "PythonReqs/pep517", requirements=["sdist_pep517.py"], - dist_versions=[("sdist_pep517", "40.8.0")]) + self.RunGradle("base", "PythonReqs/pep517", "PythonReqs/pep517_default_backend", + dist_versions=[("pep517", "2324772522")]) + + def test_pep517_explicit_backend(self): + self.RunGradle("base", "PythonReqs/pep517", "PythonReqs/pep517_explicit_backend", + dist_versions=[("pep517", "2324772522")]) # Make sure we're not affected by a setup.cfg file containing a `prefix` line. def test_cfg_wheel(self): @@ -915,13 +915,25 @@ def test_cfg_sdist(self): # can use the absence of the string in other tests to prove that no fallback occurred. RUNNING_INSTALL = "Running setup.py install" - def test_sdist_native(self): - run = self.RunGradle("base", run=False) - for name in ["sdist_native_ext", "sdist_native_clib", "sdist_native_compiler", - "sdist_native_cc"]: - with self.subTest(name=name): - run.apply_layers(f"PythonReqs/{name}") - run.rerun(succeed=False) + def test_sdist_native_ext(self): + self.sdist_native("sdist_native_ext") + + def test_sdist_native_clib(self): + self.sdist_native("sdist_native_clib") + + def test_sdist_native_compiler(self): + self.sdist_native("sdist_native_compiler") + + def test_sdist_native_cc(self): + self.sdist_native("sdist_native_cc") + + def sdist_native(self, name): + for pep517 in [True, False]: + with self.subTest(pep517=pep517): + layers = ["base", f"PythonReqs/{name}"] + if pep517: + layers.append("PythonReqs/sdist_native_pep517") + run = self.RunGradle(*layers, succeed=False) if name == "sdist_native_cc": setup_error = "Failed to run Chaquopy_cannot_compile_native_code" @@ -933,7 +945,7 @@ def test_sdist_native(self): # setup.py install. self.assertNotInLong(self.RUNNING_INSTALL, run.stdout) - url = fr"file:.*app/{name}-1.0.tar.gz" + url = r"file:.*app/sdist_native" if name in ["sdist_native_compiler", "sdist_native_cc"]: # These tests fail at the egg_info stage, so the name and version are # unavailable. @@ -945,12 +957,19 @@ def test_sdist_native(self): self.assertInLong(fr"Failed to install {req_str}." + self.tracker_advice() + r"$", run.stderr, re=True) - def test_sdist_native_optional(self): - run = self.RunGradle("base", run=False) - for name in ["sdist_native_optional_ext", "sdist_native_optional_compiler"]: - with self.subTest(name=name): - run.apply_layers(f"PythonReqs/{name}") - run.rerun(requirements=[f"{name}.py"]) + def test_sdist_native_optional_ext(self): + self.sdist_native_optional("sdist_native_optional_ext") + + def test_sdist_native_optional_compiler(self): + self.sdist_native_optional("sdist_native_optional_compiler") + + def sdist_native_optional(self, name): + for pep517 in [True, False]: + with self.subTest(pep517=pep517): + layers = ["base", f"PythonReqs/{name}"] + if pep517: + layers.append("PythonReqs/sdist_native_pep517") + self.RunGradle(*layers, requirements=[f"{name}.py"]) # If bdist_wheel fails without a "native code" message, we should fall back on setup.py # install. For example, see acoustics==0.2.4 (#5630).