diff --git a/.github/scripts/stage_crates.py b/.github/scripts/stage_crates.py index 572644b6bf63..5bf4e2e06723 100755 --- a/.github/scripts/stage_crates.py +++ b/.github/scripts/stage_crates.py @@ -10,7 +10,7 @@ The Artifactory reference token (ARTIFACTORY_TOKEN, the same one the wheel upload uses) is sent as a Bearer header for both the cargo registry and the idempotency -HEAD check. The registry location is read from ARTIFACTORY_CARGO_INDEX (a secret, +HEAD check. The registry location is read from ARTIFACTORY_CARGO_INDEX (configuration, e.g. sparse+https:///artifactory/api/cargo//index/) so it is not hardcoded in the workflow. Idempotent: crates already present are skipped. @@ -19,18 +19,58 @@ staging; the exit code is nonzero when anything failed so the workflow can report loudly without blocking the release. +External Git-patched dependencies can declare publication counterparts in the +source Cargo.toml, independently of the normal source-build requirement: + + [workspace.metadata.artifactory.dependencies.example-core] + version = "=1.2.0-rc.0" + registry = "example" + git-rev = "" + +ARTIFACTORY_CARGO_DEPENDENCY_REGISTRIES is non-secret JSON configuration mapping +these aliases to sparse Cargo index URLs on the same Artifactory server as the +publication registry. ARTIFACTORY_TOKEN must have read access to those indexes. + +To make the AISim registry available to release branches, create a repository +variable in ai-dynamo/dynamo: +Settings > Secrets and variables > Actions > Variables > New repository variable. +Name: ARTIFACTORY_CARGO_DEPENDENCY_REGISTRIES +Value (a JSON object, not a quoted JSON string): + + {"aisim":"sparse+https://artifactory.nvidia.com/artifactory/api/cargo/sw-dynamo-aisimulate-cargo-local/index/"} + +The key "aisim" matches registry = "aisim" in the source Cargo.toml's +workspace.metadata.artifactory.dependencies. Only a source revision containing +that metadata selects a publication version; the registry mapping does not pin +or override any dependency on main. Keep release-specific version and Git +revision selections on the release branch. Versions and crate download paths do +not belong in this variable. Keep the sparse+ prefix and trailing /index/. + +Store credentials only in the existing ARTIFACTORY_TOKEN secret, which needs +read access to the dependency repository as well as publication access to the +Dynamo repository. ARTIFACTORY_CARGO_INDEX still selects the Dynamo publication +destination; this mapping only adds dependency registries. The automated-release +environment must not override the repository variable with a different value. +An unset variable is allowed only when the source has no publication references. + +Only this disposable staging checkout gets rewritten; ordinary builds keep their +Git pins. The matching patch is removed so cargo check/package use the published +counterpart too. Start retries from the committed source checkout. + Non-publishable members (the python/c bindings, example binaries) are excluded by manifest-path; anything with `publish = false` is excluded too. """ from __future__ import annotations import argparse +import copy import json import os import re import subprocess import sys import time +import tomllib import urllib.error import urllib.parse import urllib.request @@ -109,7 +149,212 @@ def topo_order(pkgs: dict[str, dict]) -> list[str]: return order -def write_cargo_config(root: Path, alias: str, index: str) -> None: +def dependency_tables(document: dict) -> dict[str, dict]: + """Enumerate actual Cargo dependency tables, excluding package metadata.""" + kinds = ("dependencies", "build-dependencies", "dev-dependencies") + tables = {kind: document.get(kind, {}) for kind in kinds} + tables["workspace.dependencies"] = document.get("workspace", {}).get( + "dependencies", {} + ) + for target, fields in document.get("target", {}).items(): + for spelling in (target, json.dumps(target), f"'{target}'"): + for kind in kinds: + tables[f"target.{spelling}.{kind}"] = fields.get(kind, {}) + return tables + + +def configure_dependency_registries( + root: Path, manifests: list[Path], metadata: dict, alias: str, index: str, env: dict +) -> dict[str, str]: + """Apply reviewed publication references only to the staging checkout.""" + configuration = metadata.get("artifactory", {}) + if not isinstance(configuration, dict): + raise RuntimeError("workspace.metadata.artifactory must be a table") + references = configuration.get("dependencies", {}) + if not isinstance(references, dict): + raise RuntimeError( + "workspace.metadata.artifactory.dependencies must be a table" + ) + if not references: + return {} + try: + indexes = json.loads(env.get("ARTIFACTORY_CARGO_DEPENDENCY_REGISTRIES", "{}")) + except json.JSONDecodeError as error: + raise RuntimeError( + "ARTIFACTORY_CARGO_DEPENDENCY_REGISTRIES must be a JSON object" + ) from error + if not isinstance(indexes, dict): + raise RuntimeError( + "ARTIFACTORY_CARGO_DEPENDENCY_REGISTRIES must be a JSON object" + ) + + source = tomllib.loads((root / "Cargo.toml").read_text()) + patches = source.get("patch", {}).get("crates-io", {}) + approved_base = index.removeprefix("sparse+").split("/api/cargo/", 1)[0] + registries: dict[str, str] = {} + for package, reference in references.items(): + if not isinstance(reference, dict) or set(reference) != { + "version", + "registry", + "git-rev", + }: + raise RuntimeError( + f"{package}: expected version, registry, and git-rev publication fields" + ) + version, registry, revision = ( + reference[key] for key in ("version", "registry", "git-rev") + ) + if not isinstance(version, str) or not re.fullmatch( + r"=\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?", version + ): + raise RuntimeError( + f"{package}: publication version must be an exact Cargo requirement" + ) + if not isinstance(revision, str) or not re.fullmatch(r"[0-9a-f]{40}", revision): + raise RuntimeError(f"{package}: git-rev must be a full commit SHA") + if not isinstance(registry, str) or not re.fullmatch( + r"[a-z][a-z0-9_]*", registry + ): + raise RuntimeError(f"{package}: invalid dependency registry alias") + if registry in {alias, "crates_io"}: + raise RuntimeError( + f"{package}: dependency registry alias collides with a publish/default registry" + ) + url = indexes.get(registry) + if not isinstance(url, str): + raise RuntimeError( + f"{package}: registry {registry} is missing from ARTIFACTORY_CARGO_DEPENDENCY_REGISTRIES" + ) + parsed = urllib.parse.urlsplit(url.removeprefix("sparse+")) + if ( + not url.startswith(f"sparse+{approved_base}/api/cargo/") + or not url.endswith("/index/") + or parsed.username is not None + or parsed.password is not None + or parsed.query + or parsed.fragment + ): + raise RuntimeError( + f"{package}: dependency index must use the publication Artifactory origin and /api/cargo//index/ path" + ) + patch = patches.get(package) + if ( + not isinstance(patch, dict) + or not patch.get("git") + or patch.get("rev") != revision + or patch.get("package", package) != package + ): + raise RuntimeError( + f"{package}: publication reference does not match the current Git source pin" + ) + registries[registry] = url + + # Existing staging rewrites use inline dependency declarations. Fail closed + # on other layouts rather than silently leaving a crates.io dependency. + assignment = re.compile(r"^(\s*)([A-Za-z0-9_-]+)\s*=\s*(\{.*\})(\s*(?:#.*)?)$") + edits: dict[Path, str] = {} + expected_documents: dict[Path, dict] = {} + rewritten, removed = set(), set() + for manifest in dict.fromkeys(manifests): + original = manifest.read_text() + document = tomllib.loads(original) + sections = dependency_tables(document) + expected = copy.deepcopy(document) + tables = {id(table): table for table in dependency_tables(expected).values()} + for table in tables.values(): + for name, attributes in list(table.items()): + if isinstance(attributes, str): + attributes = {"version": attributes} + package = attributes.get("package", name) + if package in references and not attributes.get("workspace"): + if any( + key in attributes + for key in ("git", "rev", "branch", "tag", "path", "registry") + ): + raise RuntimeError( + f"{manifest}: {name} must use the reviewed crates.io Git patch" + ) + attributes.update( + { + key: references[package][key] + for key in ("version", "registry") + } + ) + table[name] = attributes + if manifest == root / "Cargo.toml": + for package in references: + del expected["patch"]["crates-io"][package] + expected_documents[manifest] = expected + section = "" + lines = [] + for line in original.splitlines(keepends=True): + header = re.match(r"^\s*\[([^\]]+)\]\s*$", line) + if header: + section = header.group(1) + match = assignment.match(line.rstrip("\n")) + if ( + match + and manifest == root / "Cargo.toml" + and section == "patch.crates-io" + and match[2] in references + ): + removed.add(match[2]) + continue + if match and section in sections: + attributes = tomllib.loads(f"dependency = {match[3]}")["dependency"] + package = attributes.get("package", match[2]) + if package in references and "workspace" not in attributes: + if any( + key in attributes + for key in ("git", "rev", "branch", "tag", "path", "registry") + ): + raise RuntimeError( + f"{manifest}: {match[2]} must use the reviewed crates.io Git patch" + ) + attributes.update( + { + key: references[package][key] + for key in ("version", "registry") + } + ) + fields = ", ".join( + f"{key} = {json.dumps(value)}" + for key, value in attributes.items() + ) + line = f"{match[1]}{match[2]} = {{ {fields} }}{match[4]}\n" + rewritten.add(package) + lines.append(line) + edits[manifest] = "".join(lines) + if rewritten != set(references) or removed != set(references): + raise RuntimeError( + "external publication references require matching inline dependencies and Git patches" + ) + for manifest, text in edits.items(): + # The line rewrite must change exactly the approved dependency fields, + # never metadata or strings that resemble Cargo declarations. + if tomllib.loads(text) != expected_documents[manifest]: + raise RuntimeError( + f"{manifest}: unsupported declaration layout for publication references" + ) + for manifest, text in edits.items(): + manifest.write_text(text) + for registry in registries: + env[ + f"CARGO_REGISTRIES_{registry.upper()}_TOKEN" + ] = f"Bearer {env['ARTIFACTORY_TOKEN']}" + for package, reference in references.items(): + print( + f"publication dependency: {package} {reference['version']} via {reference['registry']}" + ) + return registries + + +def write_cargo_config( + root: Path, + alias: str, + index: str, + dependency_registries: dict[str, str] | None = None, +) -> None: # Append to the workspace .cargo/config.toml -- it carries required build # settings (rustflags, PCRE2_SYS_STATIC) that an overwrite would drop. # The marker makes re-runs idempotent (no duplicated tables). @@ -128,7 +373,12 @@ def write_cargo_config(root: Path, alias: str, index: str) -> None: ) cfg.write_text( f"{text}\n{marker}\n" - f'[registries]\n{alias} = {{ index = "{index}" }}\n\n' + f'[registries]\n{alias} = {{ index = "{index}" }}\n' + + "".join( + f"{name} = {{ index = {json.dumps(url)} }}\n" + for name, url in sorted((dependency_registries or {}).items()) + ) + + "\n" f'[registry]\ndefault = "{alias}"\n' 'global-credential-providers = ["cargo:token"]\n' ) @@ -356,8 +606,7 @@ def main() -> int: ) return 1 - # Reuse the wheel-upload token; keep the registry location out of the workflow - # via the ARTIFACTORY_CARGO_INDEX secret. + # Reuse the wheel-upload token; registry locations are supplied by the workflow. token = os.environ.get("ARTIFACTORY_TOKEN", "") index = os.environ.get("ARTIFACTORY_CARGO_INDEX", "") if not token or not index: @@ -496,7 +745,20 @@ def main() -> int: ) return 1 - write_cargo_config(root, args.registry, index) + env = dict(os.environ) + try: + dependency_registries = configure_dependency_registries( + root, + [root / "Cargo.toml", *member_manifests], + meta.get("metadata") or {}, + args.registry, + index, + env, + ) + except RuntimeError as error: + print(f"::error::{error}", file=sys.stderr) + return 1 + write_cargo_config(root, args.registry, index, dependency_registries) inject_registry( [root / "Cargo.toml", *(Path(pkgs[n]["manifest_path"]) for n in order)], order, @@ -508,7 +770,6 @@ def main() -> int: # older release branches still carry e.g. aiconfigurator-core). strip_git_deps(root, [Path(pkgs[n]["manifest_path"]) for n in order]) - env = dict(os.environ) env[f"CARGO_REGISTRIES_{args.registry.upper()}_TOKEN"] = f"Bearer {token}" env["RUSTFLAGS"] = f"{env.get('RUSTFLAGS', '')} --cfg tokio_unstable".strip() # Only the upload PUT runs on PUBLISH_TOOLCHAIN (--no-verify skips compilation); diff --git a/.github/scripts/test_stage_crates.py b/.github/scripts/test_stage_crates.py new file mode 100644 index 000000000000..ac35ef328a7c --- /dev/null +++ b/.github/scripts/test_stage_crates.py @@ -0,0 +1,414 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Test external publication references without publishing any crates.""" + +import copy +import hashlib +import importlib.util +import io +import json +import os +import shutil +import subprocess +import tarfile +import tempfile +import threading +import tomllib +import unittest +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path + +SPEC = importlib.util.spec_from_file_location( + "stage_crates", Path(__file__).with_name("stage_crates.py") +) +stage = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(stage) +INDEX = "sparse+https://artifactory.example/artifactory/api/cargo/dynamo/index/" +DEPENDENCY_INDEX = ( + "sparse+https://artifactory.example/artifactory/api/cargo/aisim/index/" +) +REVISION = "a" * 40 +REFERENCES = { + "artifactory": { + "dependencies": { + "aisim-core": { + "version": "=0.14.0-rc.0", + "registry": "aisim", + "git-rev": REVISION, + } + } + } +} + + +class ExternalRegistryTests(unittest.TestCase): + def setUp(self): + self.directory = tempfile.TemporaryDirectory() + self.addCleanup(self.directory.cleanup) + self.root = Path(self.directory.name) + self.manifest = self.root / "Cargo.toml" + self.manifest.write_text( + '[workspace]\nmembers = ["lib/consumer"]\nresolver = "2"\n' + "[workspace.dependencies]\n" + 'aisimulate-core = { package = "aisim-core", version = "=0.14.0", default-features = false }\n' + "[patch.crates-io]\n" + f'aisim-core = {{ git = "https://example.invalid/source", rev = "{REVISION}" }}\n' + 'other-core = { git = "https://example.invalid/other", rev = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" }\n' + ) + member = self.root / "lib/consumer" + (member / "src").mkdir(parents=True) + self.member = member / "Cargo.toml" + self.member.write_text( + '[package]\nname = "consumer"\nversion = "1.0.0"\nedition = "2021"\n' + 'description = "Registry test fixture"\nlicense = "Apache-2.0"\n' + "[dependencies]\naisimulate-core = { workspace = true }\n" + ) + (member / "src/lib.rs").write_text( + "pub fn value() -> u8 { aisimulate_core::value() }\n" + ) + self.manifests = [self.manifest, self.member] + self.env = { + "ARTIFACTORY_TOKEN": "test-token", + "ARTIFACTORY_CARGO_DEPENDENCY_REGISTRIES": json.dumps( + {"aisim": DEPENDENCY_INDEX} + ), + } + + def configure(self, metadata=None): + return stage.configure_dependency_registries( + self.root, + self.manifests, + metadata if metadata is not None else REFERENCES, + "artifactory", + INDEX, + self.env, + ) + + def test_no_references_preserves_existing_behavior(self): + before = [p.read_bytes() for p in self.manifests] + self.env["ARTIFACTORY_CARGO_DEPENDENCY_REGISTRIES"] = "" + self.assertEqual(self.configure({}), {}) + self.assertEqual([p.read_bytes() for p in self.manifests], before) + self.assertNotIn("CARGO_REGISTRIES_AISIM_TOKEN", self.env) + + def test_alias_and_workspace_inheritance_survive(self): + self.member.write_text( + self.member.read_text() + + 'direct = { package = "aisim-core", version = "=0.14.0", optional = true, features = ["python"] }\n' + ) + self.member.write_text( + self.member.read_text() + "[target.'cfg(unix)'.build-dependencies]\n" + 'target_alias = { package = "aisim-core", version = "=0.14.0" }\n' + ) + registries = self.configure() + target = tomllib.loads(self.member.read_text())["target"]["cfg(unix)"][ + "build-dependencies" + ]["target_alias"] + self.assertEqual(target["version"], "=0.14.0-rc.0") + self.assertEqual(target["registry"], "aisim") + root = tomllib.loads(self.manifest.read_text()) + dependency = root["workspace"]["dependencies"]["aisimulate-core"] + self.assertEqual( + dependency, + { + "package": "aisim-core", + "version": "=0.14.0-rc.0", + "registry": "aisim", + "default-features": False, + }, + ) + member = tomllib.loads(self.member.read_text())["dependencies"] + self.assertEqual(member["aisimulate-core"], {"workspace": True}) + self.assertEqual( + member["direct"], + { + "package": "aisim-core", + "version": "=0.14.0-rc.0", + "registry": "aisim", + "optional": True, + "features": ["python"], + }, + ) + self.assertEqual(set(root["patch"]["crates-io"]), {"other-core"}) + stage.strip_git_deps(self.root, [self.member]) + self.assertEqual(tomllib.loads(self.member.read_text())["dependencies"], member) + (self.root / ".cargo").mkdir() + (self.root / ".cargo/config.toml").write_text( + '[build]\nrustflags = ["--cfg", "tokio_unstable"]\n' + ) + stage.write_cargo_config(self.root, "artifactory", INDEX, registries) + config = tomllib.loads((self.root / ".cargo/config.toml").read_text()) + self.assertEqual(config["build"]["rustflags"], ["--cfg", "tokio_unstable"]) + self.assertEqual(config["registries"]["aisim"]["index"], DEPENDENCY_INDEX) + self.assertEqual(config["registry"]["default"], "artifactory") + self.assertEqual(self.env["CARGO_REGISTRIES_AISIM_TOKEN"], "Bearer test-token") + self.assertNotIn("test-token", (self.root / ".cargo/config.toml").read_text()) + + def test_invalid_reference_cannot_mutate_manifests(self): + for field, value in [ + ("version", "^0.14"), + ("git-rev", "b" * 40), + ("registry", "artifactory"), + ("registry", "bad-name"), + ]: + with self.subTest(field=field, value=value): + metadata = copy.deepcopy(REFERENCES) + metadata["artifactory"]["dependencies"]["aisim-core"][field] = value + before = [p.read_bytes() for p in self.manifests] + with self.assertRaises(RuntimeError): + self.configure(metadata) + self.assertEqual([p.read_bytes() for p in self.manifests], before) + + def test_invalid_registry_configuration_cannot_receive_token(self): + cases = ["", "[]", "{}"] + [ + json.dumps({"aisim": url}) + for url in ( + "sparse+https://different.example/artifactory/api/cargo/aisim/index/", + "sparse+http://artifactory.example/artifactory/api/cargo/aisim/index/", + DEPENDENCY_INDEX + "?token=bad", + "sparse+https://user:password@artifactory.example/artifactory/api/cargo/aisim/index/", + ) + ] + for value in cases: + with self.subTest(value=value): + before = [p.read_bytes() for p in self.manifests] + self.env["ARTIFACTORY_CARGO_DEPENDENCY_REGISTRIES"] = value + with self.assertRaises(RuntimeError): + self.configure() + self.assertEqual([p.read_bytes() for p in self.manifests], before) + self.assertNotIn("CARGO_REGISTRIES_AISIM_TOKEN", self.env) + + def test_unsupported_multiline_declaration_fails_before_mutation(self): + self.manifest.write_text( + self.manifest.read_text().replace( + 'aisimulate-core = { package = "aisim-core", version = "=0.14.0", default-features = false }', + '[workspace.dependencies.aisimulate-core]\npackage = "aisim-core"\nversion = "=0.14.0"', + ) + ) + before = [p.read_bytes() for p in self.manifests] + with self.assertRaisesRegex(RuntimeError, "inline dependencies"): + self.configure() + self.assertEqual([p.read_bytes() for p in self.manifests], before) + + def test_mixed_declaration_layouts_fail_before_any_write(self): + original = self.member.read_text() + for section in ( + "dependencies.direct", + "build-dependencies.direct", + "target.'cfg(unix)'.dependencies.direct", + ): + with self.subTest(section=section): + self.member.write_text( + original + + f'[{section}]\npackage = "aisim-core"\nversion = "=0.14.0"\n' + ) + before = [p.read_bytes() for p in self.manifests] + with self.assertRaisesRegex(RuntimeError, "unsupported declaration"): + self.configure() + self.assertEqual([p.read_bytes() for p in self.manifests], before) + + def test_package_metadata_is_not_a_dependency(self): + self.member.write_text( + self.member.read_text() + + '[package.metadata.tool.dependencies]\nexample = { package = "aisim-core", version = "=0.14.0" }\n' + ) + before = tomllib.loads(self.member.read_text())["package"]["metadata"] + self.configure() + self.assertEqual( + tomllib.loads(self.member.read_text())["package"]["metadata"], before + ) + + def test_misleading_metadata_text_cannot_be_mutated(self): + original = self.member.read_text() + quote = "'" * 3 + variants = [ + "[package.metadata.tool] # trailing comment\n" + 'aisim-core = { package = "aisim-core", version = "=0.14.0" }\n', + f"[package.metadata.tool]\ntext = {quote}\n[dependencies]\n" + 'aisim-core = { version = "=0.14.0" }\n' + f"{quote}\n", + ] + for metadata in variants: + with self.subTest(metadata=metadata): + self.member.write_text(original + metadata) + before = [p.read_bytes() for p in self.manifests] + with self.assertRaisesRegex(RuntimeError, "unsupported declaration"): + self.configure() + self.assertEqual([p.read_bytes() for p in self.manifests], before) + + def test_expanded_reference_cannot_retain_another_source(self): + self.member.write_text( + self.member.read_text() + '[dependencies.direct]\npackage = "aisim-core"\n' + 'version = "=0.14.0-rc.0"\nregistry = "aisim"\npath = "../other-source"\n' + ) + before = [p.read_bytes() for p in self.manifests] + with self.assertRaisesRegex(RuntimeError, "reviewed crates.io Git patch"): + self.configure() + self.assertEqual([p.read_bytes() for p in self.manifests], before) + + def test_patch_package_identity_must_match(self): + self.manifest.write_text( + self.manifest.read_text().replace( + "aisim-core = { git =", + 'aisim-core = { package = "unrelated-core", git =', + ) + ) + before = [p.read_bytes() for p in self.manifests] + with self.assertRaisesRegex(RuntimeError, "current Git source pin"): + self.configure() + self.assertEqual([p.read_bytes() for p in self.manifests], before) + + @unittest.skipUnless( + shutil.which("cargo"), "Cargo is needed for the package contract" + ) + def test_cargo_package_records_the_external_registry(self): + # Serve a tiny real Cargo dependency so package normalization is tested, + # including authenticated sparse-index/download requests, without upload. + self.manifest.write_text( + "\n".join( + line + for line in self.manifest.read_text().splitlines() + if not line.startswith("other-core =") + ) + + "\n" + ) + self.manifest.write_text( + self.manifest.read_text() + + "[workspace.metadata.artifactory.dependencies.aisim-core]\n" + 'version = "=0.14.0-rc.0"\nregistry = "aisim"\n' + f'git-rev = "{REVISION}"\n' + ) + metadata = stage.cargo_metadata(self.root)["metadata"] + self.assertEqual(metadata, REFERENCES) + self.configure(metadata) + crate = io.BytesIO() + prefix = "aisim-core-0.14.0-rc.0" + files = { + "Cargo.toml": '[package]\nname = "aisim-core"\nversion = "0.14.0-rc.0"\nedition = "2021"\n[lib]\nname = "aisimulate_core"\n', + "src/lib.rs": "pub fn value() -> u8 { 1 }\n", + } + with tarfile.open(fileobj=crate, mode="w:gz") as archive: + for name, content in files.items(): + data = content.encode() + info = tarfile.TarInfo(f"{prefix}/{name}") + info.size = len(data) + archive.addfile(info, io.BytesIO(data)) + payload = crate.getvalue() + row = ( + json.dumps( + { + "name": "aisim-core", + "vers": "0.14.0-rc.0", + "deps": [], + "cksum": hashlib.sha256(payload).hexdigest(), + "features": {}, + "yanked": False, + } + ).encode() + + b"\n" + ) + requests = [] + + class Handler(BaseHTTPRequestHandler): + def do_GET(self): + requests.append((self.path, self.headers.get("Authorization"))) + if self.path.endswith("/config.json"): + repo = self.path.split("/api/cargo/", 1)[1].split("/", 1)[0] + body = json.dumps( + { + "dl": f"{base}/api/cargo/{repo}/v1/crates", + "api": f"{base}/api/cargo/{repo}", + "auth-required": True, + } + ).encode() + elif self.headers.get("Authorization") != "Bearer test-token": + self.send_error(401) + return + elif self.path == "/api/cargo/aisim/index/ai/si/aisim-core": + body = row + elif ( + self.path + == "/api/cargo/aisim/v1/crates/aisim-core/0.14.0-rc.0/download" + ): + body = payload + else: + self.send_error(404) + return + self.send_response(200) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *_args): + pass + + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + base = f"http://127.0.0.1:{server.server_port}" + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + dependency_index = f"sparse+{base}/api/cargo/aisim/index/" + stage.write_cargo_config( + self.root, + "artifactory", + f"sparse+{base}/api/cargo/dynamo/index/", + {"aisim": dependency_index}, + ) + env = { + key: value + for key, value in os.environ.items() + if not key.startswith(("CARGO_", "GIT_")) + and key not in {"RUSTC_WRAPPER", "RUSTC_WORKSPACE_WRAPPER"} + } + env.update( + { + "CARGO_HOME": str(self.root / "cargo-home"), + "CARGO_REGISTRIES_AISIM_TOKEN": "Bearer test-token", + "CARGO_REGISTRIES_ARTIFACTORY_TOKEN": "Bearer test-token", + "CARGO_REGISTRIES_CRATES_IO_INDEX": f"sparse+{base}/unused-cratesio/", + "NO_PROXY": "127.0.0.1,localhost", + } + ) + result = subprocess.run( + [ + "cargo", + "package", + "--manifest-path", + str(self.member), + "--allow-dirty", + "--registry", + "artifactory", + ], + cwd=self.root, + env=env, + capture_output=True, + text=True, + timeout=60, + ) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + with tarfile.open( + self.root / "target/package/consumer-1.0.0.crate" + ) as package: + normalized = tomllib.loads( + package.extractfile("consumer-1.0.0/Cargo.toml").read().decode() + ) + dependency = normalized["dependencies"]["aisimulate-core"] + self.assertEqual(dependency["package"], "aisim-core") + self.assertEqual(dependency["version"], "=0.14.0-rc.0") + self.assertEqual(dependency["registry-index"], dependency_index) + self.assertNotIn("git", dependency) + self.assertNotIn("path", dependency) + self.assertTrue( + any( + path.endswith("/download") and auth == "Bearer test-token" + for path, auth in requests + ) + ) + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b7e90533c766..54a737232b6c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1692,7 +1692,8 @@ jobs: id: publish_crates env: ARTIFACTORY_TOKEN: ${{ secrets.ARTIFACTORY_TOKEN }} - ARTIFACTORY_CARGO_INDEX: ${{ secrets.ARTIFACTORY_CARGO_INDEX }} + ARTIFACTORY_CARGO_INDEX: ${{ vars.ARTIFACTORY_CARGO_INDEX || secrets.ARTIFACTORY_CARGO_INDEX }} + ARTIFACTORY_CARGO_DEPENDENCY_REGISTRIES: ${{ vars.ARTIFACTORY_CARGO_DEPENDENCY_REGISTRIES }} EXPECT_VERSION: ${{ needs.prepare-release.outputs.crate_version }} # Same rc-suffixed value: stage_crates.py rewrites every publishable crate's # version to this before publishing, so re-stages don't collide on the @@ -1722,6 +1723,7 @@ jobs: -e CARGO_TARGET_DIR=/tmp/cargo-target \ -e ARTIFACTORY_TOKEN \ -e ARTIFACTORY_CARGO_INDEX \ + -e ARTIFACTORY_CARGO_DEPENDENCY_REGISTRIES \ -e EXPECT_VERSION \ -e STAGE_VERSION \ -e CRATE_SUBSET \ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 77adc8394ac7..2b7d2c26cf5d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -75,6 +75,12 @@ repos: # and give it visiblity into all the local and third_party packages expected. - repo: local hooks: + - id: test-stage-crates + name: Test external Cargo publication references + entry: python .github/scripts/test_stage_crates.py + language: python + files: ^(Cargo\.toml|\.github/scripts/(stage_crates|test_stage_crates)\.py|\.github/workflows/release\.yml)$ + pass_filenames: false - id: operator-upgrade-history name: Check operator upgrade history version and provenance entry: python3 deploy/operator/hack/generate_upgrade_fixtures.py --check