"Find Wheel" now handles platform-specific artifacts during selection

eng/tox/create_package_and_install.py

@@ -18,10 +18,10 @@
 import shutil
 from pkg_resources import parse_version
 
-from tox_helper_tasks import find_whl, find_sdist, get_pip_list_output
+from tox_helper_tasks import get_pip_list_output
 from ci_tools.parsing import ParsedSetup, parse_require
 from ci_tools.build import create_package
-from ci_tools.functions import get_package_from_repo
+from ci_tools.functions import get_package_from_repo, find_whl, find_sdist, discover_prebuilt_package
 
 logging.getLogger().setLevel(logging.INFO)
 
@@ -46,6 +46,15 @@ def discover_packages(setuppy_path, args):
     packages = []
     if os.getenv("PREBUILT_WHEEL_DIR") is not None and not args.force_create:
         packages = discover_prebuilt_package(os.getenv("PREBUILT_WHEEL_DIR"), setuppy_path, args.package_type)
+        pkg = ParsedSetup.from_path(setuppy_path)
+
+        if not packages:
+            logging.error(
+                "Package is missing in prebuilt directory {0} for package {1} and version {2}".format(
+                    os.getenv("PREBUILT_WHEEL_DIR"), pkg.name, pkg.version
+                )
+            )
+            exit(1)
     else:
         packages = build_and_discover_package(
             setuppy_path,
@@ -56,25 +65,6 @@ def discover_packages(setuppy_path, args):
     return packages
 
 
-def discover_prebuilt_package(dist_directory, setuppy_path, package_type):
-    packages = []
-    pkg = ParsedSetup.from_path(setuppy_path)
-    if package_type == "wheel":
-        prebuilt_package = find_whl(dist_directory, pkg.name, pkg.version)
-    else:
-        prebuilt_package = find_sdist(dist_directory, pkg.name, pkg.version)
-
-    if prebuilt_package is None:
-        logging.error(
-            "Package is missing in prebuilt directory {0} for package {1} and version {2}".format(
-                dist_directory, pkg.name, pkg.version
-            )
-        )
-        exit(1)
-    packages.append(prebuilt_package)
-    return packages
-
-
 def in_ci():
     return os.getenv("TF_BUILD", False)
 

eng/tox/run_apistubgen.py

@@ -9,7 +9,7 @@
 import os
 import logging
 
-from tox_helper_tasks import find_whl
+from ci_tools.functions import find_whl
 from ci_tools.parsing import ParsedSetup
 
 logging.getLogger().setLevel(logging.INFO)

eng/tox/tox_helper_tasks.py

@@ -73,76 +73,3 @@ def move_and_rename(source_location):
 
     os.rename(source_location, new_location)
     return new_location
-
-
-def find_sdist(dist_dir, pkg_name, pkg_version):
-    # This function will find a sdist for given package name
-    if not os.path.exists(dist_dir):
-        logging.error("dist_dir is incorrect")
-        return
-
-    if pkg_name is None:
-        logging.error("Package name cannot be empty to find sdist")
-        return
-
-    pkg_name_format = "{0}-{1}.zip".format(pkg_name, pkg_version)
-    packages = []
-    for root, dirnames, filenames in os.walk(dist_dir):
-        for filename in fnmatch.filter(filenames, pkg_name_format):
-            packages.append(os.path.join(root, filename))
-
-    packages = [os.path.relpath(w, dist_dir) for w in packages]
-
-    if not packages:
-        logging.error("No sdist is found in directory %s with package name format %s", dist_dir, pkg_name_format)
-        return
-    return packages[0]
-
-
-def find_whl(whl_dir, pkg_name, pkg_version):
-    # This function will find a whl for given package name
-    if not os.path.exists(whl_dir):
-        logging.error("whl_dir is incorrect")
-        return
-
-    if pkg_name is None:
-        logging.error("Package name cannot be empty to find whl")
-        return
-
-    pkg_name_format = "{0}-{1}*.whl".format(pkg_name.replace("-", "_"), pkg_version)
-    whls = []
-    for root, dirnames, filenames in os.walk(whl_dir):
-        for filename in fnmatch.filter(filenames, pkg_name_format):
-            whls.append(os.path.join(root, filename))
-
-    whls = [os.path.relpath(w, whl_dir) for w in whls]
-
-    if not whls:
-        logging.error("No whl is found in directory %s with package name format %s", whl_dir, pkg_name_format)
-        logging.info("List of whls in directory: %s", glob.glob(os.path.join(whl_dir, "*.whl")))
-        return
-
-    if len(whls) > 1:
-        # So we have whls specific to py version or platform since we are here
-        py_version = "py{0}".format(sys.version_info.major)
-        # filter whl for py version and check if we found just one whl
-        whls = [w for w in whls if py_version in w]
-
-        # if whl is platform independent then there should only be one whl in filtered list
-        if len(whls) > 1:
-            # if we have reached here, that means we have whl specific to platform as well.
-            # for now we are failing the test if platform specific wheels are found. Todo: enhance to find platform specific whl
-            logging.error(
-                "More than one whl is found in wheel directory for package {}. Platform specific whl discovery is not supported now".format(
-                    pkg_name
-                )
-            )
-            sys.exit(1)
-
-    # Additional filtering based on arch type willbe required in future if that need arises.
-    # for now assumption is that no arch specific whl is generated
-    if len(whls) == 1:
-        logging.info("Found whl {}".format(whls[0]))
-        return whls[0]
-    else:
-        return None

scripts/devops_tasks/test_regression.py

@@ -32,7 +32,7 @@
     clone_repo,
 )
 
-from ci_tools.functions import discover_targeted_packages, str_to_bool, find_whl
+from ci_tools.functions import discover_targeted_packages, str_to_bool, build_temp_whl
 from ci_tools.parsing import ParsedSetup
 
 AZURE_GLOB_STRING = "azure*"
@@ -125,7 +125,7 @@ def run(self):
 
             self.whl_path = os.path.join(
                 self.context.whl_directory,
-                find_whl(pkg_name, self.context.pkg_version, self.context.whl_directory),
+                build_temp_whl(pkg_name, self.context.pkg_version, self.context.whl_directory),
             )
             if find_packages_missing_on_pypi(self.whl_path):
                 logging.error("Required packages are not available on PyPI. Skipping regression test")

tools/azure-sdk-tools/ci_tools/functions.py

@@ -59,6 +59,7 @@
     "Omit_management": omit_mgmt,
 }
 
+
 def apply_compatibility_filter(package_set: List[str]) -> List[str]:
     """
     This function takes in a set of paths to python packages. It returns the set filtered by compatibility with the currently running python executable.
@@ -381,7 +382,7 @@ def build_and_install_dev_reqs(file: str, pkg_root: str) -> None:
     shutil.rmtree(os.path.join(pkg_root, ".tmp_whl_dir"))
 
 
-def find_whl(package_name: str, version: str, whl_directory: str) -> str:
+def build_temp_whl(package_name: str, version: str, whl_directory: str) -> str:
     """Helper function to find where the built whl resides.
 
     :param str package_name: the name of the package, e.g. azure-core
@@ -414,6 +415,7 @@ def find_whl(package_name: str, version: str, whl_directory: str) -> str:
 
     return whls[0]
 
+
 def build_whl_for_req(req: str, package_path: str) -> str:
     """Builds a whl from the dev_requirements file.
 
@@ -422,6 +424,7 @@ def build_whl_for_req(req: str, package_path: str) -> str:
     :return: The absolute path to the whl built or the requirement if a third-party package
     """
     from ci_tools.build import create_package
+
     if ".." in req:
         # Create temp path if it doesn't exist
         temp_dir = os.path.join(package_path, ".tmp_whl_dir")
@@ -434,9 +437,131 @@ def build_whl_for_req(req: str, package_path: str) -> str:
         logging.info("Building wheel for package {}".format(parsed.name))
         create_package(req_pkg_path, temp_dir, enable_sdist=False)
 
-        whl_path = os.path.join(temp_dir, find_whl(parsed.name, parsed.version, temp_dir))
+        whl_path = os.path.join(temp_dir, build_temp_whl(parsed.name, parsed.version, temp_dir))
         logging.info("Wheel for package {0} is {1}".format(parsed.name, whl_path))
         logging.info("Replacing dev requirement. Old requirement:{0}, New requirement:{1}".format(req, whl_path))
         return whl_path
     else:
         return req
+
+
+def find_sdist(dist_dir: str, pkg_name: str, pkg_version: str) -> str:
+    """This function attempts to look within a directory (and all subdirs therein) and find a source distribution for the targeted package and version."""
+    # This function will find a sdist for given package name
+    if not os.path.exists(dist_dir):
+        logging.error("dist_dir is incorrect")
+        return
+
+    if pkg_name is None:
+        logging.error("Package name cannot be empty to find sdist")
+        return
+
+    pkg_name_format = "{0}-{1}.zip".format(pkg_name, pkg_version)
+    pkg_name_format_alt = "${0}-{1}.tar.gz"
+
+    # todo: replace with glob, we aren't using py2 anymore!
+    packages = []
+    for root, dirnames, filenames in os.walk(dist_dir):
+        for filename in fnmatch.filter(filenames, pkg_name_format):
+            packages.append(os.path.join(root, filename))
+
+    packages = [os.path.relpath(w, dist_dir) for w in packages]
+
+    if not packages:
+        logging.error("No sdist is found in directory %s with package name format %s", dist_dir, pkg_name_format)
+        return
+    return packages[0]
+
+
+def get_interpreter_compatible_tags() -> List[str]:
+    """
+    This function invokes pip from the invoking interpreter and discovers which tags the interpreter is compatible with.
+    """
+
+    commands = [sys.executable, "-m", "pip", "debug", "--verbose"]
+
+    output = subprocess.run(
+        commands,
+        check=True,
+        capture_output=True,
+    ).stdout.decode(encoding="utf-8")
+
+    tag_strings = output.split(os.linesep)
+
+    for index, value in enumerate(tag_strings):
+        if "Compatible tags" in value:
+            break
+
+    tags = tag_strings[index + 1 :]
+
+    return [tag.strip() for tag in tags if tag]
+
+
+def check_whl_against_tags(whl_name: str, tags: List[str]) -> bool:
+    for tag in tags:
+        if tag in whl_name:
+            return True
+    return False
+
+
+def find_whl(whl_dir: str, pkg_name: str, pkg_version: str) -> str:
+    """This function attempts to look within a directory (and all subdirs therein) and find a wheel that matches our targeted name and version AND
+    whose compilation is compatible with the invoking interpreter."""
+    if not os.path.exists(whl_dir):
+        logging.error("whl_dir is incorrect")
+        return
+
+    if pkg_name is None:
+        logging.error("Package name cannot be empty to find whl")
+        return
+
+    pkg_name_format = "{0}-{1}*.whl".format(pkg_name.replace("-", "_"), pkg_version)
+    whls = []
+
+    # todo: replace with glob, we aren't using py2 anymore!
+    for root, dirnames, filenames in os.walk(whl_dir):
+        for filename in fnmatch.filter(filenames, pkg_name_format):
+            whls.append(os.path.join(root, filename))
+
+    whls = [os.path.relpath(w, whl_dir) for w in whls]
+
+    if not whls:
+        logging.error("No whl is found in directory %s with package name format %s", whl_dir, pkg_name_format)
+        logging.info("List of whls in directory: %s", glob.glob(os.path.join(whl_dir, "*.whl")))
+        return
+
+    compatible_tags = get_interpreter_compatible_tags()
+
+    logging.info("Dumping visible tags and whls")
+    logging.info(compatible_tags)
+    logging.info(whls)
+
+    if whls:
+        # grab the first whl that matches a tag from our compatible_tags list
+        for whl in whls:
+            if check_whl_against_tags(whl, compatible_tags):
+                logging.info("Found whl {}".format(whl))
+                return whl
+
+        # if whl is platform independent then there should only be one whl in filtered list
+        if len(whls) > 1:
+            # if we have reached here, that means we have whl specific to platform as well.
+            # for now we are failing the test if platform specific wheels are found. Todo: enhance to find platform specific whl
+            logging.error(f"We were unable to locate a compatible wheel for {pkg_name}")
+            sys.exit(1)
+
+    return None
+
+
+def discover_prebuilt_package(dist_directory: str, setup_path: str, package_type: str) -> List[str]:
+    """Discovers a prebuild wheel or sdist for a given setup path."""
+    packages = []
+    pkg = ParsedSetup.from_path(setup_path)
+    if package_type == "wheel":
+        prebuilt_package = find_whl(dist_directory, pkg.name, pkg.version)
+    else:
+        prebuilt_package = find_sdist(dist_directory, pkg.name, pkg.version)
+
+    if prebuilt_package is not None:
+        packages.append(prebuilt_package)
+    return packages

tools/azure-sdk-tools/tests/integration/test_package_discovery.py

@@ -109,3 +109,4 @@ def test_discovery_honors_override():
         "azure-core-tracing-opentelemetry",
         "azure-mgmt-core",
     ]
+