diff --git a/eng/tools/azure-sdk-tools/ci_tools/parsing/parse_functions.py b/eng/tools/azure-sdk-tools/ci_tools/parsing/parse_functions.py index 93560d709afd..5c17aa4ea7b6 100644 --- a/eng/tools/azure-sdk-tools/ci_tools/parsing/parse_functions.py +++ b/eng/tools/azure-sdk-tools/ci_tools/parsing/parse_functions.py @@ -156,46 +156,31 @@ def _normalize_url_fields(pkg_info, metadata: Dict[str, Any]) -> None: # Homepage from PEP 566 style if pkg_info.home_page: metadata["homepage"] = pkg_info.home_page - - # Handle project URLs (can be in various formats) - if pkg_info.project_urls: - metadata["project_urls"] = pkg_info.project_urls - - # Try to extract homepage from project_urls if not already set - if "homepage" not in metadata: - homepage = _extract_homepage_from_project_urls(pkg_info.project_urls) - if homepage: - metadata["homepage"] = homepage + elif pkg_info.project_urls: # If homepage not found, try to extract from project_urls + if isinstance(pkg_info.project_urls, (list, tuple)): + for url_entry in pkg_info.project_urls: + if isinstance(url_entry, str) and "," in url_entry: + # Format: "Label, https://example.com" + label, url_value = url_entry.split(",", 1) + label_lower = label.strip().lower() + url_value = url_value.strip() + if label_lower in ["homepage", "home-page", "home"]: + metadata["homepage"] = url_value + elif label_lower == "repository": + metadata["repository"] = url_value + elif isinstance(pkg_info.project_urls, dict): + for key, value in pkg_info.project_urls.items(): + key_lower = key.lower() + if key_lower in ["homepage", "home-page", "home"]: + metadata["homepage"] = value + elif key_lower == "repository": + metadata["repository"] = value # Download URL if hasattr(pkg_info, "download_url") and getattr(pkg_info, "download_url", None): metadata["download_url"] = pkg_info.download_url -def _extract_homepage_from_project_urls(project_urls) -> Optional[str]: - """Extract homepage URL from project_urls in various formats.""" - if not project_urls: - return None - - # Handle different project_urls formats - if isinstance(project_urls, (list, tuple)): - for url_entry in project_urls: - if isinstance(url_entry, str) and "," in url_entry: - # Format: "Homepage, https://example.com" - url_type, url_value = url_entry.split(",", 1) - url_type = url_type.strip().lower() - url_value = url_value.strip() - if url_type in ["homepage", "home-page", "home", "website"]: - return url_value - elif isinstance(project_urls, dict): - # Handle dictionary format - for key, value in project_urls.items(): - if key.lower() in ["homepage", "home-page", "home", "website"]: - return value - - return None - - def _add_optional_fields(pkg_info, metadata: Dict[str, Any]) -> None: """Add optional metadata fields that may be present.""" optional_fields = ["obsoletes_dist", "provides_dist", "requires_external", "platform", "supported_platform"] diff --git a/eng/tools/azure-sdk-tools/devtools_testutils/helpers.py b/eng/tools/azure-sdk-tools/devtools_testutils/helpers.py index 7fac3eb38a52..93d97bd2805c 100644 --- a/eng/tools/azure-sdk-tools/devtools_testutils/helpers.py +++ b/eng/tools/azure-sdk-tools/devtools_testutils/helpers.py @@ -19,6 +19,7 @@ this = sys.modules[__name__] this.recording_ids = {} + def locate_assets(current_test_file: str) -> str: """Locate the test assets directory for the targeted testfile. @@ -75,6 +76,7 @@ def locate_assets(current_test_file: str) -> str: raise FileNotFoundError(f"No matching breadcrumb file found for asset path {relative_asset_path}") + def get_http_client(**kwargs): """Returns a `urllib3` client that provides the test proxy's self-signed certificate if it's available. diff --git a/eng/tools/azure-sdk-tools/tests/integration/scenarios/pyproject_invalid_metadata/pyproject.toml b/eng/tools/azure-sdk-tools/tests/integration/scenarios/pyproject_invalid_metadata/pyproject.toml index 4f2ea56ec614..68858dd3ae35 100644 --- a/eng/tools/azure-sdk-tools/tests/integration/scenarios/pyproject_invalid_metadata/pyproject.toml +++ b/eng/tools/azure-sdk-tools/tests/integration/scenarios/pyproject_invalid_metadata/pyproject.toml @@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta" [project] name = "azure-storage-blob" authors = [ - {email = "ascl@microsoft.com"}, + {name = "Microsoft Corporation", email = "ascl@microsoft.com"}, ] description = "Microsoft Azure Blob Storage Client Library for Python" keywords = ["azure", "azure sdk"] @@ -35,7 +35,7 @@ aio = [ ] [project.urls] -repository = "https://github.com/Azure/azure-sdk-for-python" +source = "https://github.com/Azure/azure-sdk-for-python" [tool.setuptools.dynamic] version = {attr = "azure.storage.blob._version.VERSION"} diff --git a/eng/tools/azure-sdk-tools/tests/integration/scenarios/pyproject_metadata/pyproject.toml b/eng/tools/azure-sdk-tools/tests/integration/scenarios/pyproject_metadata/pyproject.toml index 7905348af227..ceede0f08ac5 100644 --- a/eng/tools/azure-sdk-tools/tests/integration/scenarios/pyproject_metadata/pyproject.toml +++ b/eng/tools/azure-sdk-tools/tests/integration/scenarios/pyproject_metadata/pyproject.toml @@ -35,7 +35,6 @@ aio = [ ] [project.urls] -homepage = "https://github.com/Azure/azure-sdk-for-python/tree/main/sdk/storage/azure-storage-blob" repository = "https://github.com/Azure/azure-sdk-for-python" [tool.setuptools.dynamic] diff --git a/eng/tools/azure-sdk-tools/tests/test_metadata_verification.py b/eng/tools/azure-sdk-tools/tests/test_metadata_verification.py index cdd5085ecb99..793dd7712ff2 100644 --- a/eng/tools/azure-sdk-tools/tests/test_metadata_verification.py +++ b/eng/tools/azure-sdk-tools/tests/test_metadata_verification.py @@ -113,15 +113,15 @@ def test_verify_valid_metadata_passes(package_type, scenario_name, scenario_path @pytest.mark.parametrize( - "package_type,scenario_name,scenario_path", + "package_type,scenario_name,scenario_path,missing_keys", [ - ("wheel", "stable", "pyproject_invalid_metadata_scenario"), - ("sdist", "stable", "pyproject_invalid_metadata_scenario"), - ("wheel", "beta", "pyproject_beta_invalid_metadata_scenario"), - ("sdist", "beta", "pyproject_beta_invalid_metadata_scenario"), + ("wheel", "stable", "pyproject_invalid_metadata_scenario", ["homepage", "repository"]), + ("sdist", "stable", "pyproject_invalid_metadata_scenario", ["homepage", "repository"]), + ("wheel", "beta", "pyproject_beta_invalid_metadata_scenario", ["author_email", "summary"]), + ("sdist", "beta", "pyproject_beta_invalid_metadata_scenario", ["author_email", "summary"]), ], ) -def test_verify_invalid_metadata_fails_with_missing_keys(package_type, scenario_name, scenario_path, caplog): +def test_verify_invalid_metadata_fails(package_type, scenario_name, scenario_path, missing_keys, caplog): """Test that verify_whl/verify_sdist fails for scenarios with invalid metadata and reports missing author_name and homepage.""" # Get the actual scenario path from globals actual_scenario_path = globals()[scenario_path] @@ -154,22 +154,20 @@ def test_verify_invalid_metadata_fails_with_missing_keys(package_type, scenario_ # Check that the error log contains information about missing keys error_logs = [record.message for record in caplog.records if record.levelname == "ERROR"] - # Different scenarios have different missing keys - if scenario_name == "stable": - # Stable scenario is missing author name and homepage - expected_missing_keys = ["author", "homepage"] - else: # beta scenario - # Beta scenario is missing author email and description - expected_missing_keys = ["author_email", "summary"] - - # Check for either order of the missing keys - missing_keys_pattern1 = f"Missing keys: {{'{expected_missing_keys[0]}', '{expected_missing_keys[1]}'}}" - missing_keys_pattern2 = f"Missing keys: {{'{expected_missing_keys[1]}', '{expected_missing_keys[0]}'}}" - has_missing_keys_error = any(missing_keys_pattern1 in msg or missing_keys_pattern2 in msg for msg in error_logs) - - assert ( - has_missing_keys_error - ), f"Expected error log about missing keys '{expected_missing_keys[0]}' and '{expected_missing_keys[1]}' for {scenario_name} scenario, but got: {error_logs}" + # Raise error if homepage AND repository not found in current version + if "homepage" in missing_keys: + assert f"Current metadata must contain at least one of: {missing_keys}" in error_logs + # Otherwise, check for missing keys from prior version + else: + missing_keys_pattern1 = f"Missing keys: {{'{missing_keys[0]}', '{missing_keys[1]}'}}" + missing_keys_pattern2 = f"Missing keys: {{'{missing_keys[1]}', '{missing_keys[0]}'}}" + has_missing_keys_error = any( + missing_keys_pattern1 in msg or missing_keys_pattern2 in msg for msg in error_logs + ) + + assert ( + has_missing_keys_error + ), f"Expected error log about Missing keys: '{missing_keys[0]}' and '{missing_keys[1]}' for {scenario_name} scenario, but got: {error_logs}" finally: # Cleanup dist directory diff --git a/eng/tox/verify_whl.py b/eng/tox/verify_whl.py index 33ee4ca52a46..b34325164083 100644 --- a/eng/tox/verify_whl.py +++ b/eng/tox/verify_whl.py @@ -125,26 +125,47 @@ def verify_prior_version_metadata(package_name: str, prior_version: str, current f"{package_name}=={prior_version}", "--dest", tmp_dir ], check=True, capture_output=True) zip_files = glob.glob(os.path.join(tmp_dir, package_type)) - if not zip_files: + # If no match and we're not constrained to wheel-only, attempt legacy sdist (zip) once. + if not zip_files and package_type != "*.whl": + zip_files = glob.glob(os.path.join(tmp_dir, "*.zip")) + if not zip_files: # Still nothing -> treat as no prior artifact to compare. return True prior_metadata: Dict[str, Any] = extract_package_metadata(zip_files[0]) is_compatible = verify_metadata_compatibility(current_metadata, prior_metadata) - if not is_compatible: - missing_keys = set(prior_metadata.keys()) - set(current_metadata.keys()) - logging.error(f"Metadata compatibility failed for {package_name}. Missing keys: {missing_keys}") return is_compatible except Exception: return True def verify_metadata_compatibility(current_metadata: Dict[str, Any], prior_metadata: Dict[str, Any]) -> bool: - """Verify that all keys from prior version metadata are present in current version.""" - if not prior_metadata: - return True + """Verify that all keys from prior version metadata are present in current version. + + Special handling: homepage/repository keys are exempt from prior compatibility check, + but current version must have at least one of them. + """ if not current_metadata: return False - return set(prior_metadata.keys()).issubset(set(current_metadata.keys())) + # Check that current version has at least one homepage or repository URL + repo_urls = ['homepage', 'repository'] + current_keys_lower = {k.lower() for k in current_metadata.keys()} + if not any(key in current_keys_lower for key in repo_urls): + logging.error(f"Current metadata must contain at least one of: {repo_urls}") + return False + + if not prior_metadata: + return True + + # For backward compatibility check, exclude homepage/repository from prior requirements + prior_keys_filtered = {k for k in prior_metadata.keys() if k.lower() not in repo_urls} + current_keys = set(current_metadata.keys()) + + is_compatible = prior_keys_filtered.issubset(current_keys) + if not is_compatible: + missing_keys = prior_keys_filtered - current_keys + logging.error("Metadata compatibility failed. Missing keys: %s", missing_keys) + return is_compatible + def get_path_to_zip(dist_dir: str, version: str, package_type: str = "*.whl") -> str: return glob.glob(os.path.join(dist_dir, "**", "*{}{}".format(version, package_type)), recursive=True)[0]