"Find Wheel" now handles platform-specific artifacts during selection

eng/tox/create_package_and_install.py

@@ -18,10 +18,10 @@
 import shutil
 from pkg_resources import parse_version
 
-from tox_helper_tasks import find_whl, find_sdist, get_pip_list_output
+from tox_helper_tasks import get_pip_list_output
 from ci_tools.parsing import ParsedSetup, parse_require
 from ci_tools.build import create_package
-from ci_tools.functions import get_package_from_repo
+from ci_tools.functions import get_package_from_repo, find_whl, find_sdist, discover_prebuilt_package
 
 logging.getLogger().setLevel(logging.INFO)
 
@@ -46,6 +46,15 @@ def discover_packages(setuppy_path, args):
     packages = []
     if os.getenv("PREBUILT_WHEEL_DIR") is not None and not args.force_create:
         packages = discover_prebuilt_package(os.getenv("PREBUILT_WHEEL_DIR"), setuppy_path, args.package_type)
+        pkg = ParsedSetup.from_path(setuppy_path)
+
+        if not packages:
+            logging.error(
+                "Package is missing in prebuilt directory {0} for package {1} and version {2}".format(
+                    os.getenv("PREBUILT_WHEEL_DIR"), pkg.name, pkg.version
+                )
+            )
+            exit(1)
     else:
         packages = build_and_discover_package(
             setuppy_path,
@@ -56,25 +65,6 @@ def discover_packages(setuppy_path, args):
     return packages
 
 
-def discover_prebuilt_package(dist_directory, setuppy_path, package_type):
-    packages = []
-    pkg = ParsedSetup.from_path(setuppy_path)
-    if package_type == "wheel":
-        prebuilt_package = find_whl(dist_directory, pkg.name, pkg.version)
-    else:
-        prebuilt_package = find_sdist(dist_directory, pkg.name, pkg.version)
-
-    if prebuilt_package is None:
-        logging.error(
-            "Package is missing in prebuilt directory {0} for package {1} and version {2}".format(
-                dist_directory, pkg.name, pkg.version
-            )
-        )
-        exit(1)
-    packages.append(prebuilt_package)
-    return packages
-
-
 def in_ci():
     return os.getenv("TF_BUILD", False)
 

eng/tox/tox_helper_tasks.py

@@ -73,76 +73,3 @@ def move_and_rename(source_location):
 
     os.rename(source_location, new_location)
     return new_location
-
-
-def find_sdist(dist_dir, pkg_name, pkg_version):
-    # This function will find a sdist for given package name
-    if not os.path.exists(dist_dir):
-        logging.error("dist_dir is incorrect")
-        return
-
-    if pkg_name is None:
-        logging.error("Package name cannot be empty to find sdist")
-        return
-
-    pkg_name_format = "{0}-{1}.zip".format(pkg_name, pkg_version)
-    packages = []
-    for root, dirnames, filenames in os.walk(dist_dir):
-        for filename in fnmatch.filter(filenames, pkg_name_format):
-            packages.append(os.path.join(root, filename))
-
-    packages = [os.path.relpath(w, dist_dir) for w in packages]
-
-    if not packages:
-        logging.error("No sdist is found in directory %s with package name format %s", dist_dir, pkg_name_format)
-        return
-    return packages[0]
-
-
-def find_whl(whl_dir, pkg_name, pkg_version):
-    # This function will find a whl for given package name
-    if not os.path.exists(whl_dir):
-        logging.error("whl_dir is incorrect")
-        return
-
-    if pkg_name is None:
-        logging.error("Package name cannot be empty to find whl")
-        return
-
-    pkg_name_format = "{0}-{1}*.whl".format(pkg_name.replace("-", "_"), pkg_version)
-    whls = []
-    for root, dirnames, filenames in os.walk(whl_dir):
-        for filename in fnmatch.filter(filenames, pkg_name_format):
-            whls.append(os.path.join(root, filename))
-
-    whls = [os.path.relpath(w, whl_dir) for w in whls]
-
-    if not whls:
-        logging.error("No whl is found in directory %s with package name format %s", whl_dir, pkg_name_format)
-        logging.info("List of whls in directory: %s", glob.glob(os.path.join(whl_dir, "*.whl")))
-        return
-
-    if len(whls) > 1:
-        # So we have whls specific to py version or platform since we are here
-        py_version = "py{0}".format(sys.version_info.major)
-        # filter whl for py version and check if we found just one whl
-        whls = [w for w in whls if py_version in w]
-
-        # if whl is platform independent then there should only be one whl in filtered list
-        if len(whls) > 1:
-            # if we have reached here, that means we have whl specific to platform as well.
-            # for now we are failing the test if platform specific wheels are found. Todo: enhance to find platform specific whl
-            logging.error(
-                "More than one whl is found in wheel directory for package {}. Platform specific whl discovery is not supported now".format(
-                    pkg_name
-                )
-            )
-            sys.exit(1)
-
-    # Additional filtering based on arch type willbe required in future if that need arises.
-    # for now assumption is that no arch specific whl is generated
-    if len(whls) == 1:
-        logging.info("Found whl {}".format(whls[0]))
-        return whls[0]
-    else:
-        return None

tools/azure-sdk-tools/ci_tools/functions.py

@@ -59,6 +59,7 @@
     "Omit_management": omit_mgmt,
 }
 
+
 def apply_compatibility_filter(package_set: List[str]) -> List[str]:
     """
     This function takes in a set of paths to python packages. It returns the set filtered by compatibility with the currently running python executable.
@@ -414,6 +415,7 @@ def find_whl(package_name: str, version: str, whl_directory: str) -> str:
 
     return whls[0]
 
+
 def build_whl_for_req(req: str, package_path: str) -> str:
     """Builds a whl from the dev_requirements file.
 
@@ -422,6 +424,7 @@ def build_whl_for_req(req: str, package_path: str) -> str:
     :return: The absolute path to the whl built or the requirement if a third-party package
     """
     from ci_tools.build import create_package
+
     if ".." in req:
         # Create temp path if it doesn't exist
         temp_dir = os.path.join(package_path, ".tmp_whl_dir")
@@ -440,3 +443,121 @@ def build_whl_for_req(req: str, package_path: str) -> str:
         return whl_path
     else:
         return req
+
+
+def find_sdist(dist_dir: str, pkg_name: str, pkg_version: str) -> str:
+    """This function attempts to look within a directory (and all subdirs therein) and find a source distribution for the targeted package and version."""
+    # This function will find a sdist for given package name
+    if not os.path.exists(dist_dir):
+        logging.error("dist_dir is incorrect")
+        return
+
+    if pkg_name is None:
+        logging.error("Package name cannot be empty to find sdist")
+        return
+
+    pkg_name_format = "{0}-{1}.zip".format(pkg_name, pkg_version)
+    pkg_name_format_alt = "${0}-{1}.tar.gz"
+
+    # todo: replace with glob, we aren't using py2 anymore!
+    packages = []
+    for root, dirnames, filenames in os.walk(dist_dir):
+        for filename in fnmatch.filter(filenames, pkg_name_format):
+            packages.append(os.path.join(root, filename))
+
+    packages = [os.path.relpath(w, dist_dir) for w in packages]
+
+    if not packages:
+        logging.error("No sdist is found in directory %s with package name format %s", dist_dir, pkg_name_format)
+        return
+    return packages[0]
+
+
+def get_interpreter_compatible_tags() -> List[str]:
+    """
+    This function invokes pip from the invoking interpreter and discovers which tags the interpreter is compatible with.
+    """
+
+    commands = [sys.executable, "-m", "pip", "debug", "--verbose"]
+
+    output = subprocess.run(
+        commands,
+        check=True,
+        capture_output=True,
+    ).stdout.decode(encoding="utf-8")
+
+    tag_strings = output.split(os.linesep)
+
+    for index, value in enumerate(tag_strings):
+        if "Compatible tags" in value:
+            break
+
+    tags = tag_strings[index + 1 :]
+
+    return [tag.strip() for tag in tags if tag]
+
+
+def check_whl_against_tags(whl_name: str, tags: List[str]) -> bool:
+    for tag in tags:
+        if tag in whl_name:
+            return True
+    return False
+
+
+def find_whl(whl_dir: str, pkg_name: str, pkg_version: str) -> str:
+    """This function attempts to look within a directory (and all subdirs therein) and find a wheel that matches our targeted name and version AND
+    whose compilation is compatible with the invoking interpreter."""
+    if not os.path.exists(whl_dir):
+        logging.error("whl_dir is incorrect")
+        return
+
+    if pkg_name is None:
+        logging.error("Package name cannot be empty to find whl")
+        return
+
+    pkg_name_format = "{0}-{1}*.whl".format(pkg_name.replace("-", "_"), pkg_version)
+    whls = []
+
+    # todo: replace with glob, we aren't using py2 anymore!
+    for root, dirnames, filenames in os.walk(whl_dir):
+        for filename in fnmatch.filter(filenames, pkg_name_format):
+            whls.append(os.path.join(root, filename))
+
+    whls = [os.path.relpath(w, whl_dir) for w in whls]
+
+    if not whls:
+        logging.error("No whl is found in directory %s with package name format %s", whl_dir, pkg_name_format)
+        logging.info("List of whls in directory: %s", glob.glob(os.path.join(whl_dir, "*.whl")))
+        return
+
+    compatible_tags = get_interpreter_compatible_tags()
+
+    if whls:
+        # grab the first whl that matches a tag from our compatible_tags list
+        for whl in whls:
+            if check_whl_against_tags(whl, compatible_tags):
+                logging.info("Found whl {}".format(whl))
+                return whl
+
+        # if whl is platform independent then there should only be one whl in filtered list
+        if len(whls) > 1:
+            # if we have reached here, that means we have whl specific to platform as well.
+            # for now we are failing the test if platform specific wheels are found. Todo: enhance to find platform specific whl
+            logging.error(f"We were unable to locate a compatible wheel for {pkg_name}")
+            sys.exit(1)
+
+    return None
+
+
+def discover_prebuilt_package(dist_directory: str, setup_path: str, package_type: str) -> List[str]:
+    """Discovers a prebuild wheel or sdist for a given setup path."""
+    packages = []
+    pkg = ParsedSetup.from_path(setup_path)
+    if package_type == "wheel":
+        prebuilt_package = find_whl(dist_directory, pkg.name, pkg.version)
+    else:
+        prebuilt_package = find_sdist(dist_directory, pkg.name, pkg.version)
+
+    if prebuilt_package is not None:
+        packages.append(prebuilt_package)
+    return packages

tools/azure-sdk-tools/tests/integration/test_package_discovery.py

@@ -109,3 +109,4 @@ def test_discovery_honors_override():
         "azure-core-tracing-opentelemetry",
         "azure-mgmt-core",
     ]
+

tools/azure-sdk-tools/tests/test_whl_discovery.py

@@ -0,0 +1,109 @@
+import tempfile
+import os
+
+from tempfile import TemporaryDirectory
+from unittest.mock import patch
+
+from typing import List
+
+from ci_tools.functions import find_whl
+
+
+def create_temp_directory(fake_creation_paths: List[str]) -> TemporaryDirectory:
+    tmp_dir = TemporaryDirectory()
+
+    for file in fake_creation_paths:
+        target_path = os.path.join(tmp_dir.name, file)
+        dirname = os.path.join(tmp_dir.name, os.path.dirname(file))
+
+        if not os.path.exists(dirname):
+            os.mkdir(dirname)
+
+        with open(target_path, "w"):
+            pass
+
+    return tmp_dir
+
+
+def create_basic_temp_dir() -> TemporaryDirectory:
+    tmp_dir = create_temp_directory(
+        [
+            os.path.join("azure-common", "azure_common-1.1.29-py3-none-any.whl"),
+            os.path.join("azure-core", "azure_core-1.26.5-py3-none-any.whl"),
+            os.path.join("azure-core-experimental", "azure_core_experimental-1.0.0b3-py3-none-any.whl"),
+            os.path.join("azure-tracing-opencensus", "azure_core_tracing_opencensus-1.0.0b9-py3-none-any.whl"),
+            os.path.join(
+                "azure-core-tracing-opentelemetry", "azure_core_tracing_opentelemetry-1.0.0b10-py3-none-any.whl"
+            ),
+            os.path.join("azure-mgmt-core", "azure_mgmt_core-1.4.0-py3-none-any.whl"),
+            os.path.join(
+                "azure-servicemanagement-legacy", "azure_servicemanagement_legacy-0.20.7-py2.py3-none-any.whl"
+            ),
+        ]
+    )
+    return tmp_dir
+
+
+@patch("ci_tools.functions.get_interpreter_compatible_tags")
+def test_find_discovers_standard_whls(test_patch):
+    tmp_dir = create_basic_temp_dir()
+    test_patch.return_value = ["py3-none-any"]
+
+    # basic positive cases
+    found_core = find_whl(tmp_dir.name, "azure-core", "1.26.5")
+    found_legacy = find_whl(tmp_dir.name, "azure-servicemanagement-legacy", "0.20.7")
+    assert found_core is not None
+    assert found_legacy is not None
+
+    # basic negative cases
+    not_found_core = find_whl(tmp_dir.name, "azure-core", "1.26.4")
+    assert not_found_core is None
+
+    tmp_dir.cleanup()
+
+
+@patch("ci_tools.functions.get_interpreter_compatible_tags")
+def test_find_whl_fails_on_incompatible_interpreter(test_patch):
+    tmp_dir = create_basic_temp_dir()
+    test_patch.return_value = []
+
+    found = find_whl(tmp_dir.name, "azure-core", "1.26.5")
+    assert found is None
+
+    tmp_dir.cleanup()
+
+
+@patch("ci_tools.functions.get_interpreter_compatible_tags")
+def test_find_whl_discovers_specific_wheels(test_patch):
+    tmp_dir = create_temp_directory(
+        [
+            "azure_storage_extensions-1.0.0b1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl",
+            "azure_storage_extensions-1.0.0b1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
+            "azure_storage_extensions-1.0.0b1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl",
+            "azure_storage_extensions-1.0.0b1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
+            "azure_storage_extensions-1.0.0b1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl",
+            "azure_storage_extensions-1.0.0b1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
+            "azure_storage_extensions-1.0.0b1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl",
+            "azure_storage_extensions-1.0.0b1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
+            "azure_storage_extensions-1.0.0b1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl",
+            "azure_storage_extensions-1.0.0b1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
+            "azure_storage_extensions-1.0.0b1-cp310-cp310-macosx_10_9_x86_64.whl",
+            "azure_storage_extensions-1.0.0b1-cp311-cp311-macosx_10_9_x86_64.whl",
+            "azure_storage_extensions-1.0.0b1-cp37-cp37m-macosx_10_9_x86_64.whl",
+            "azure_storage_extensions-1.0.0b1-cp38-cp38-macosx_10_9_x86_64.whl",
+            "azure_storage_extensions-1.0.0b1-cp39-cp39-macosx_10_9_x86_64.whl",
+            "azure_storage_extensions-1.0.0b1-cp310-cp310-win_amd64.whl",
+            "azure_storage_extensions-1.0.0b1-cp310-cp310-win32.whl",
+            "azure_storage_extensions-1.0.0b1-cp311-cp311-win_amd64.whl",
+            "azure_storage_extensions-1.0.0b1-cp311-cp311-win32.whl",
+            "azure_storage_extensions-1.0.0b1-cp37-cp37m-win_amd64.whl",
+            "azure_storage_extensions-1.0.0b1-cp37-cp37m-win32.whl",
+            "azure_storage_extensions-1.0.0b1-cp38-cp38-win_amd64.whl",
+            "azure_storage_extensions-1.0.0b1-cp38-cp38-win32.whl",
+            "azure_storage_extensions-1.0.0b1-cp39-cp39-win_amd64.whl",
+            "azure_storage_extensions-1.0.0b1-cp39-cp39-win32.whl",
+        ]
+    )
+    test_patch.return_value = ["cp39-cp39-win_amd64"]
+    found = find_whl(tmp_dir.name, "azure-storage-extensions", "1.0.0b1")
+    tmp_dir.cleanup()