diff --git a/RELEASE.md b/RELEASE.md index 462585509a..221686dd7c 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -109,11 +109,19 @@ The chain currently publishes: 6. `model-artifact` 7. `model-hf` 8. `mesh-llm-client` -9. `mesh-llm-node` -10. `mesh-llm-api-server` +9. `mesh-llm-api-client` +10. `mesh-llm-node` +11. `mesh-llm-api-server` Run the dry-run before cutting a GA tag after changing SDK crate manifests or workspace-internal SDK dependencies. On the first release that introduces a new internal SDK crate, the dry-run validates packages whose registry dependencies already exist and reports downstream packages that will be fully verified during the real sequential publish after their upstream crates land. + +If crates.io rate-limits the non-prerelease publish chain after some crates +have already uploaded, rerun `scripts/publish-crates.sh` for the same checked +out release tag instead of recutting the GitHub release or moving the tag. The +script checks crates.io before each real publish, skips crate versions that are +already visible, and retries HTTP 429 new-crate rate-limit responses using the +retry time from crates.io when one is provided. diff --git a/scripts/publish-crates.sh b/scripts/publish-crates.sh index 9a1a72ea3f..73ab9d5c51 100755 --- a/scripts/publish-crates.sh +++ b/scripts/publish-crates.sh @@ -10,9 +10,40 @@ Publishes the crates.io package chain in dependency order. Use --dry-run for local and CI validation without uploading packages. --allow-dirty is accepted only with --dry-run so local pre-commit validation can include uncommitted manifest changes; real publishing always requires Cargo's clean-tree check. + +Environment: + CRATES_IO_PUBLISH_MAX_ATTEMPTS Real-publish retry attempts for crates.io 429s (default: 6) + CRATES_IO_PUBLISH_RETRY_BASE_SECONDS Fallback retry base when crates.io gives no timestamp (default: 60) + CRATES_IO_PUBLISH_RETRY_MAX_SECONDS Fallback retry cap when crates.io gives no timestamp (default: 900) USAGE } +log() { + echo "publish-crates: $*" +} + +warn() { + echo "publish-crates: $*" >&2 +} + +require_positive_int() { + local name="$1" + local value="$2" + if [[ ! "$value" =~ ^[1-9][0-9]*$ ]]; then + echo "${name} must be a positive integer" >&2 + exit 1 + fi +} + +require_nonnegative_int() { + local name="$1" + local value="$2" + if [[ ! "$value" =~ ^[0-9]+$ ]]; then + echo "${name} must be a non-negative integer" >&2 + exit 1 + fi +} + dry_run=0 allow_dirty=0 sleep_seconds="" @@ -59,6 +90,15 @@ if [[ -z "$sleep_seconds" ]]; then fi fi +max_attempts="${CRATES_IO_PUBLISH_MAX_ATTEMPTS:-6}" +retry_base_seconds="${CRATES_IO_PUBLISH_RETRY_BASE_SECONDS:-60}" +retry_max_seconds="${CRATES_IO_PUBLISH_RETRY_MAX_SECONDS:-900}" + +require_nonnegative_int CRATES_IO_PUBLISH_SETTLE_SECONDS "$sleep_seconds" +require_positive_int CRATES_IO_PUBLISH_MAX_ATTEMPTS "$max_attempts" +require_positive_int CRATES_IO_PUBLISH_RETRY_BASE_SECONDS "$retry_base_seconds" +require_positive_int CRATES_IO_PUBLISH_RETRY_MAX_SECONDS "$retry_max_seconds" + if [[ "$dry_run" -eq 0 && -z "${CARGO_REGISTRY_TOKEN:-}" ]]; then echo "CARGO_REGISTRY_TOKEN is required for real crates.io publishing" >&2 exit 1 @@ -80,15 +120,15 @@ if [[ -z "$workspace_version" ]]; then exit 1 fi -crate_version_published() { +registry_version_status() { local crate="$1" local status if ! command -v curl >/dev/null 2>&1; then - return 1 + echo "unknown" + return 0 fi status="$( curl \ - --fail \ --silent \ --show-error \ --output /dev/null \ @@ -96,7 +136,166 @@ crate_version_published() { "https://crates.io/api/v1/crates/${crate}/${workspace_version}" \ 2>/dev/null || true )" - [[ "$status" == "200" ]] + case "$status" in + 200) + echo "published" + ;; + 404) + echo "missing" + ;; + *) + echo "unknown" + ;; + esac +} + +crate_version_published() { + local crate="$1" + [[ "$(registry_version_status "$crate")" == "published" ]] +} + +publish_error_is_429() { + local output="$1" + [[ "$output" == *"status 429 Too Many Requests"* || "$output" == *"published too many new crates"* ]] +} + +print_publish_output() { + local output="$1" + if [[ -z "$output" ]]; then + return 0 + fi + if [[ -n "${CARGO_REGISTRY_TOKEN:-}" ]]; then + output="${output//${CARGO_REGISTRY_TOKEN}/}" + fi + printf '%s\n' "$output" +} + +retry_after_epoch() { + local output="$1" + local retry_after + retry_after="$( + printf '%s\n' "$output" \ + | sed -nE 's/.*Please try again after ([^"]+)$/\1/p' \ + | head -n 1 \ + || true + )" + retry_after="${retry_after%.}" + retry_after="${retry_after%\"}" + if [[ -z "$retry_after" ]]; then + return 1 + fi + if date -u -d "$retry_after" +%s 2>/dev/null; then + return 0 + fi + date -u -j -f "%a, %d %b %Y %H:%M:%S %Z" "$retry_after" +%s 2>/dev/null +} + +retry_delay_seconds() { + local output="$1" + local attempt="$2" + local target_epoch now_epoch delay + if target_epoch="$(retry_after_epoch "$output")" && now_epoch="$(date -u +%s 2>/dev/null)"; then + delay=$((target_epoch - now_epoch + 5)) + if [[ "$delay" -lt 1 ]]; then + delay=1 + fi + echo "$delay" + return 0 + fi + + delay="$retry_base_seconds" + for ((step = 1; step < attempt; step++)); do + delay=$((delay * 2)) + if [[ "$delay" -ge "$retry_max_seconds" ]]; then + delay="$retry_max_seconds" + break + fi + done + echo "$delay" +} + +last_publish_output="" + +run_cargo_publish_once() { + local crate="$1" + local output status + local args=(publish --locked -p "$crate") + if [[ "$dry_run" -eq 1 ]]; then + args+=(--dry-run) + fi + if [[ "$allow_dirty" -eq 1 ]]; then + args+=(--allow-dirty) + fi + + echo "cargo ${args[*]}" + if output="$(cargo "${args[@]}" 2>&1)"; then + last_publish_output="$output" + print_publish_output "$output" + return 0 + else + status=$? + fi + + last_publish_output="$output" + print_publish_output "$output" >&2 + return "$status" +} + +publish_crate_with_retry() { + local crate="$1" + local index="$2" + local total="$3" + local attempt status delay + + if [[ "$dry_run" -eq 0 ]]; then + status="$(registry_version_status "$crate")" + if [[ "$status" == "published" ]]; then + log "[${index}/${total}] ${crate}@${workspace_version} already published; skipping" + return 0 + fi + if [[ "$status" == "unknown" ]]; then + warn "[${index}/${total}] could not verify ${crate}@${workspace_version} on crates.io; aborting before publish" + return 1 + fi + fi + + attempt=1 + while [[ "$attempt" -le "$max_attempts" ]]; do + if [[ "$dry_run" -eq 1 ]]; then + log "[${index}/${total}] ${crate}@${workspace_version} dry-run" + elif [[ "$attempt" -eq 1 ]]; then + log "[${index}/${total}] ${crate}@${workspace_version} publish" + else + log "[${index}/${total}] ${crate}@${workspace_version} publish retry ${attempt}/${max_attempts}" + fi + + if run_cargo_publish_once "$crate"; then + return 0 + fi + + if [[ "$dry_run" -eq 0 && "$(registry_version_status "$crate")" == "published" ]]; then + log "[${index}/${total}] ${crate}@${workspace_version} is now visible on crates.io; continuing" + return 0 + fi + + if [[ "$dry_run" -eq 0 ]] && publish_error_is_429 "$last_publish_output"; then + warn "crates.io rate limit hit for ${crate}@${workspace_version} on attempt ${attempt}/${max_attempts}" + if [[ "$attempt" -ge "$max_attempts" ]]; then + warn "retry limit exceeded for ${crate}@${workspace_version} after ${max_attempts} attempts" + return 101 + fi + delay="$(retry_delay_seconds "$last_publish_output" "$attempt")" + warn "retrying ${crate}@${workspace_version} after ${delay}s" + sleep "$delay" + attempt=$((attempt + 1)) + continue + fi + + return 101 + done + + warn "retry limit exceeded for ${crate}@${workspace_version} after ${max_attempts} attempts" + return 101 } unpublished_registry_deps() { @@ -168,20 +367,10 @@ for index in "${!publish_crates[@]}"; do if [[ "$dry_run" -eq 1 ]] && should_skip_initial_dry_run "$crate"; then continue fi - - args=(publish --locked -p "$crate") - if [[ "$dry_run" -eq 1 ]]; then - args+=(--dry-run) - fi - if [[ "$allow_dirty" -eq 1 ]]; then - args+=(--allow-dirty) - fi - - echo "cargo ${args[*]}" - cargo "${args[@]}" + publish_crate_with_retry "$crate" "$((index + 1))" "${#publish_crates[@]}" if [[ "$index" -lt "$((${#publish_crates[@]} - 1))" && "$sleep_seconds" -gt 0 ]]; then - echo "waiting ${sleep_seconds}s for crates.io index propagation" + log "waiting ${sleep_seconds}s for crates.io index propagation" sleep "$sleep_seconds" fi done diff --git a/scripts/tests/test_publish_crates.py b/scripts/tests/test_publish_crates.py new file mode 100644 index 0000000000..f7e6f6fc8d --- /dev/null +++ b/scripts/tests/test_publish_crates.py @@ -0,0 +1,316 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import os +from pathlib import Path +import stat +import subprocess +import tempfile +import textwrap +import unittest + + +ROOT = Path(__file__).resolve().parents[2] +SCRIPT = ROOT / "scripts" / "publish-crates.sh" + + +class PublishCratesScriptTests(unittest.TestCase): + def test_retries_cargo_publish_429_then_continues_chain(self) -> None: + with PublishCratesFixture() as fixture: + fixture.write_curl_statuses({}) + fixture.write_fake_cargo( + fail_crates={"model-artifact": 1}, + failure_output=CRATES_IO_429, + ) + fixture.write_fake_sleep() + fixture.write_fake_date() + + result = fixture.run( + env={ + "CARGO_REGISTRY_TOKEN": "test-token", + "CRATES_IO_PUBLISH_MAX_ATTEMPTS": "3", + "CRATES_IO_PUBLISH_SETTLE_SECONDS": "0", + } + ) + + self.assertEqual(result.returncode, 0, result.stderr + result.stdout) + cargo_log = fixture.read_log("cargo.log") + self.assertEqual(cargo_log.count("-p model-artifact"), 2) + self.assertIn("-p model-hf", cargo_log) + self.assertIn("-p mesh-llm-client", cargo_log) + self.assertIn("-p mesh-llm-api-server", cargo_log) + self.assertRegex(fixture.read_log("sleep.log"), r"^[1-9][0-9]*$") + self.assertIn( + "crates.io rate limit hit for model-artifact@0.66.0", + result.stderr, + ) + + def test_exhausts_429_retries_and_fails_loudly_without_continuing(self) -> None: + with PublishCratesFixture() as fixture: + fixture.write_curl_statuses({}) + fixture.write_fake_cargo( + fail_crates={"model-artifact": 5}, + failure_output=CRATES_IO_429, + ) + fixture.write_fake_sleep() + fixture.write_fake_date() + + result = fixture.run( + env={ + "CARGO_REGISTRY_TOKEN": "test-token", + "CRATES_IO_PUBLISH_MAX_ATTEMPTS": "2", + "CRATES_IO_PUBLISH_SETTLE_SECONDS": "0", + } + ) + + self.assertNotEqual(result.returncode, 0) + cargo_log = fixture.read_log("cargo.log") + self.assertEqual(cargo_log.count("-p model-artifact"), 2) + self.assertNotIn("-p model-hf", cargo_log) + self.assertIn( + "retry limit exceeded for model-artifact@0.66.0 after 2 attempts", + result.stderr, + ) + + def test_dry_run_skips_crates_with_unpublished_registry_deps_without_sleeping(self) -> None: + with PublishCratesFixture() as fixture: + fixture.write_curl_statuses({"model-ref": 404}) + fixture.write_fake_cargo() + fixture.write_fake_sleep() + fixture.write_fake_date() + + result = fixture.run(["--dry-run", "--allow-dirty"]) + + self.assertEqual(result.returncode, 0, result.stderr + result.stdout) + self.assertIn( + "dry-run cannot verify model-artifact until model-ref@0.66.0 exists in crates.io", + result.stdout, + ) + self.assertIn( + "publish --locked -p model-ref --dry-run --allow-dirty", + fixture.read_log("cargo.log"), + ) + self.assertEqual(fixture.read_log("sleep.log"), "") + + def test_real_publish_requires_registry_token_before_any_cargo_call(self) -> None: + with PublishCratesFixture() as fixture: + fixture.write_curl_statuses({}) + fixture.write_fake_cargo() + fixture.write_fake_sleep() + fixture.write_fake_date() + + result = fixture.run() + + self.assertNotEqual(result.returncode, 0) + self.assertIn("CARGO_REGISTRY_TOKEN is required", result.stderr) + self.assertFalse((fixture.tmp_path / "cargo.log").exists()) + + def test_real_publish_skips_crate_version_that_is_already_published(self) -> None: + with PublishCratesFixture() as fixture: + fixture.write_curl_statuses({"model-ref": 200}) + fixture.write_fake_cargo() + fixture.write_fake_sleep() + fixture.write_fake_date() + + result = fixture.run( + env={ + "CARGO_REGISTRY_TOKEN": "test-token", + "CRATES_IO_PUBLISH_SETTLE_SECONDS": "0", + } + ) + + self.assertEqual(result.returncode, 0, result.stderr + result.stdout) + self.assertIn("model-ref@0.66.0 already published; skipping", result.stdout) + self.assertNotIn("-p model-ref", fixture.read_log("cargo.log")) + + def test_real_publish_fails_closed_when_registry_status_is_unknown(self) -> None: + with PublishCratesFixture() as fixture: + fixture.write_curl_statuses({"model-ref": 500}) + fixture.write_fake_cargo() + fixture.write_fake_sleep() + fixture.write_fake_date() + + result = fixture.run(env={"CARGO_REGISTRY_TOKEN": "test-token"}) + + self.assertNotEqual(result.returncode, 0) + self.assertIn( + "could not verify model-ref@0.66.0 on crates.io; aborting before publish", + result.stderr, + ) + self.assertFalse((fixture.tmp_path / "cargo.log").exists()) + + def test_cargo_failure_output_redacts_registry_token(self) -> None: + with PublishCratesFixture() as fixture: + fixture.write_curl_statuses({}) + fixture.write_fake_cargo( + fail_crates={"model-ref": 1}, + failure_output="fatal: registry token secret-token leaked in diagnostic\n", + ) + fixture.write_fake_sleep() + fixture.write_fake_date() + + result = fixture.run(env={"CARGO_REGISTRY_TOKEN": "secret-token"}) + + self.assertNotEqual(result.returncode, 0) + self.assertNotIn("secret-token", result.stderr) + self.assertIn("", result.stderr) + + +class PublishCratesFixture: + def __init__(self) -> None: + self.tmpdir = tempfile.TemporaryDirectory() + self.tmp_path = Path(self.tmpdir.name) + self.bin_dir = self.tmp_path / "bin" + self.bin_dir.mkdir() + (self.tmp_path / "Cargo.toml").write_text( + '[workspace.package]\nversion = "0.66.0"\n', + encoding="utf-8", + ) + + def __enter__(self) -> "PublishCratesFixture": + return self + + def __exit__(self, *args: object) -> None: + self.tmpdir.cleanup() + + def run( + self, + args: list[str] | None = None, + *, + env: dict[str, str] | None = None, + ) -> subprocess.CompletedProcess[str]: + merged_env = os.environ.copy() + merged_env.update(env or {}) + merged_env["PATH"] = f"{self.bin_dir}{os.pathsep}{merged_env['PATH']}" + return subprocess.run( + ["bash", str(SCRIPT), *(args or [])], + cwd=self.tmp_path, + env=merged_env, + text=True, + capture_output=True, + check=False, + ) + + def write_fake_cargo( + self, + *, + fail_crates: dict[str, int] | None = None, + failure_output: str = "", + ) -> None: + failure_path = self.tmp_path / "cargo-failure.txt" + failure_path.write_text(failure_output, encoding="utf-8") + fail_cases = "\n".join( + f"{crate}:{count}" for crate, count in (fail_crates or {}).items() + ) + self._write_executable( + "cargo", + f"""#!/usr/bin/env bash +set -euo pipefail +crate="" +prev="" +for arg in "$@"; do + if [[ "$prev" == "-p" ]]; then + crate="$arg" + break + fi + prev="$arg" +done +echo "$*" >> "{self.tmp_path}/cargo.log" +case "$crate" in +{self._cargo_case_arms(fail_cases, failure_path)} +esac +exit 0 +""", + ) + + def _cargo_case_arms(self, fail_cases: str, failure_path: Path) -> str: + arms: list[str] = [] + for line in fail_cases.splitlines(): + crate, count = line.split(":", 1) + state_file = self.tmp_path / f"cargo-{crate}.count" + arms.append( + textwrap.dedent( + f""" + {crate}) + current=0 + if [[ -f "{state_file}" ]]; then + current="$(cat "{state_file}")" + fi + current="$((current + 1))" + echo "$current" > "{state_file}" + if [[ "$current" -le "{count}" ]]; then + cat "{failure_path}" >&2 + exit 101 + fi + ;; + """ + ).strip() + ) + return "\n".join(arms) + + def write_curl_statuses(self, statuses: dict[str, int]) -> None: + cases = "\n".join( + f"*crates/{crate}/0.66.0*) status={status} ;;" + for crate, status in statuses.items() + ) + self._write_executable( + "curl", + f"""#!/usr/bin/env bash +set -euo pipefail +url="${{@: -1}}" +status=404 +case "$url" in +{cases} +esac +echo "$url" >> "{self.tmp_path}/curl.log" +printf '%s' "$status" +""", + ) + + def write_fake_sleep(self) -> None: + self._write_executable( + "sleep", + f"""#!/usr/bin/env bash +set -euo pipefail +echo "$1" >> "{self.tmp_path}/sleep.log" +""", + ) + + def write_fake_date(self) -> None: + self._write_executable( + "date", + """#!/usr/bin/env bash +set -euo pipefail +if [[ "$*" == *"Fri, 22 May 2026 09:58:23 GMT"* ]]; then + printf '%s\n' 1779443903 +elif [[ "$*" == *"+%s"* ]]; then + printf '%s\n' 1779443600 +else + /bin/date "$@" +fi +""", + ) + + def read_log(self, name: str) -> str: + path = self.tmp_path / name + if not path.exists(): + return "" + return path.read_text(encoding="utf-8") + + def _write_executable(self, name: str, content: str) -> None: + path = self.bin_dir / name + path.write_text(textwrap.dedent(content), encoding="utf-8") + path.chmod(path.stat().st_mode | stat.S_IXUSR) + + +CRATES_IO_429 = """error: failed to publish model-artifact v0.66.0 +status 429 Too Many Requests: +"You have published too many new crates in a short period of time. +Please try again after Fri, 22 May 2026 09:58:23 GMT +and see https://crates.io/docs/rate-limits for more details." +""" + + +if __name__ == "__main__": + unittest.main()