Update builder to consider package constraints when skipping binaries (grpc fix)

builder/__main__.py

@@ -22,6 +22,7 @@
     build_wheels_requirement,
     extract_packages,
     install_pips,
+    parse_requirements,
     write_requirement,
 )
 from builder.upload import run_upload
@@ -129,7 +130,13 @@ def builder(
         elif single:
             # Build every wheel like a single installation
             packages = extract_packages(requirement, requirement_diff)
-            skip_binary = check_available_binary(wheels_index, skip_binary, packages)
+            constraints = parse_requirements(constraint) if constraint else []
+            skip_binary = check_available_binary(
+                wheels_index,
+                skip_binary,
+                packages,
+                constraints,
+            )
             for package in packages:
                 print(f"Process package: {package}", flush=True)
                 try:
@@ -152,8 +159,13 @@ def builder(
             packages = extract_packages(requirement, requirement_diff)
             temp_requirement = Path("/tmp/wheels_requirement.txt")
             write_requirement(temp_requirement, packages)
-
-            skip_binary = check_available_binary(wheels_index, skip_binary, packages)
+            constraints = parse_requirements(constraint) if constraint else []
+            skip_binary = check_available_binary(
+                wheels_index,
+                skip_binary,
+                packages,
+                constraints,
+            )
             try:
                 build_wheels_requirement(
                     temp_requirement,

builder/infra.py

@@ -23,7 +23,9 @@ def create_wheels_index(base_index: str) -> str:
     return f"{base_index}/{alpine_version()}/{build_arch()}/"
 
 
-def check_available_binary(index: str, skip_binary: str, packages: List[str]) -> str:
+def check_available_binary(
+    index: str, skip_binary: str, packages: List[str], constraints: List[str]
+) -> str:
     """Check if binary exists and ignore this skip."""
     if skip_binary == ":none:":
         return skip_binary
@@ -33,15 +35,14 @@ def check_available_binary(index: str, skip_binary: str, packages: List[str]) ->
 
     list_needed: Set[str] = set()
     for binary in list_binary:
-        for package in packages.copy():
+        for package in packages + constraints:
             if not package.startswith(binary):
                 continue
 
             # Check more details
             find = _RE_REQUIREMENT.match(package)
             if not find:
-                packages.remove(package)
-                continue
+                raise ValueError(f"package requirement malformed: {package}")
 
             # Check full name
             if binary != find["package"]:

tests/test_infra.py

@@ -0,0 +1,149 @@
+"""Tests for infra module.
+
+pip module."""
+import pytest
+from builder.infra import check_available_binary
+from unittest.mock import patch
+
+
+# The test makes a fake index with an arbitrary set of wheels and versions based on
+# behavior the tests need to exercise. The test will adjust the input packages and
+# versions to exercise different corner cases.
+TEST_INDEX_URL = "http://example"
+TEST_INDEX_FILES = [
+    "aiohttp-3.6.1-cp38-none-any.whl",
+    "aiohttp-3.7.3-cp38-none-any.whl",
+    "aiohttp-3.7.4-cp38-none-any.whl",
+    "google_cloud_pubsub-2.1.0-py2.py3-none-any.whl",
+    "grpcio-1.31.0-cp39-none-any.whl",
+]
+
+
+@pytest.fixture(autouse=True)
+def mock_index_data():
+    """Prepares a fake existing wheel index for use in tests."""
+    # Mimc the HTML of a webserver autoindex.
+    content = "\n".join(
+        f'<a href="{wheel}">{wheel}</a>     28-May-2021 09:53  38181515'
+        for wheel in TEST_INDEX_FILES
+    )
+    with patch("builder.infra.requests.get") as mock_request_get:
+        mock_request_get.return_value.status_code = 200
+        mock_request_get.return_value.text = content
+        yield
+
+
+def test_check_available_binary_none() -> None:
+    """No-op when no binaries specified to skip."""
+    assert (
+        check_available_binary(
+            TEST_INDEX_URL,
+            ":none:",
+            packages=[
+                "aiohttp==3.7.4",
+                "google_cloud_pubsub==2.1.0",
+            ],
+            constraints=[],
+        )
+        == ":none:"
+    )
+
+
+def test_check_available_binary_all() -> None:
+    """This tool does not allow skipping all binaries."""
+    assert (
+        check_available_binary(
+            TEST_INDEX_URL,
+            ":all:",
+            packages=[
+                "aiohttp==3.7.4",
+                "google_cloud_pubsub==2.1.0",
+            ],
+            constraints=[],
+        )
+        == ":none:"
+    )
+
+
+def test_check_available_binary_version_present() -> None:
+    """Test to skip a binary where the package version is already in the index."""
+    assert (
+        check_available_binary(
+            TEST_INDEX_URL,
+            "aiohttp",
+            packages=[
+                "aiohttp==3.7.4",
+                "google_cloud_pubsub==2.1.0",
+            ],
+            constraints=[],
+        )
+        == ":none:"
+    )
+
+
+def test_check_available_binary_version_missing() -> None:
+    """Test to skip a binary where the package version is not in the index."""
+    assert (
+        check_available_binary(
+            TEST_INDEX_URL,
+            "aiohttp",
+            packages=[
+                "aiohttp==3.7.5",  # Not in the index
+                "google_cloud_pubsub==2.1.0",
+            ],
+            constraints=[],
+        )
+        == "aiohttp"
+    )
+
+
+def test_check_available_binary_implicit_dep_skipped() -> None:
+    """Test case where skip binary lists an implicit dep which is ignored."""
+    assert (
+        check_available_binary(
+            TEST_INDEX_URL,
+            "aiohttp,grpcio",
+            packages=[
+                "aiohttp==3.7.4",
+                "google_cloud_pubsub==2.1.0",
+            ],
+            constraints=[],
+        )
+        == ":none:"
+    )
+
+
+def test_check_available_binary_skip_constraint() -> None:
+    """Test case where skip binary is for constraint in the index."""
+    assert (
+        check_available_binary(
+            TEST_INDEX_URL,
+            "aiohttp,grpcio",
+            packages=[
+                "aiohttp==3.7.4",
+                "google_cloud_pubsub==2.1.0",
+            ],
+            constraints=[
+                "grpcio==1.31.0",  # Already exists in index
+            ],
+        )
+        == ":none:"
+    )
+
+
+def test_check_available_binary_for_missing_constraint() -> None:
+    """Test case where skip binary is for constraint notin the index."""
+    assert (
+        check_available_binary(
+            TEST_INDEX_URL,
+            "aiohttp,grpcio",
+            packages=[
+                "aiohttp==3.7.4",
+                "google_cloud_pubsub==2.1.0",
+            ],
+            constraints=[
+                "grpcio==1.43.0",  # Not in index
+            ],
+        )
+        == "grpcio"
+    )