From 58d35e17d05eadd7ab74c06e890d175cbbf027a0 Mon Sep 17 00:00:00 2001 From: pvijayakrish Date: Fri, 14 Aug 2026 20:54:13 -0700 Subject: [PATCH 01/29] build: stamp workspace-version pins in both version modes Mirror main's WORKSPACE_PIN_CARGO_TARGETS (PR #13303 / 907976f) into the diverged apply_dev_version.py: backend-common's inline dynamo-llm pin is now rewritten by the dev-suffix stamp (rewrite_root_cargo) AND by --set-version (set_release_version), so neither nightly stamping nor a release cut leaves the pin stale. Verified on a release/1.4.0 scratch checkout: suffix -> 1.4.0-dev20260814, set-version 1.4.1 -> 1.4.1, 1.4.1.dev2 -> 1.4.1-dev2, idempotent, and a bare pin (old branches) is left untouched. Signed-off-by: pvijayakrish --- .github/scripts/apply_dev_version.py | 392 +++++++++- .github/scripts/stage_crates.py | 444 +++++++++++ .github/workflows/on-demand-release.yml | 527 +++++++++++++ .github/workflows/release.yml | 953 ++++++++++++++++++++---- 4 files changed, 2172 insertions(+), 144 deletions(-) create mode 100644 .github/scripts/stage_crates.py create mode 100644 .github/workflows/on-demand-release.yml diff --git a/.github/scripts/apply_dev_version.py b/.github/scripts/apply_dev_version.py index 68a225650e0c..c541fbff8f93 100755 --- a/.github/scripts/apply_dev_version.py +++ b/.github/scripts/apply_dev_version.py @@ -12,11 +12,20 @@ - The `version = "1.1.0"` pins on dynamo-*/kvbm-* path deps in root Cargo.toml Empty suffix is a no-op, so safe to run unconditionally in every workflow. + +With `--set-version X.Y.Z[.devN|.postN]` it instead SETS an absolute release +version: it replaces the current workspace version M wherever it appears in those +same files, plus the Helm Chart.yaml version/appVersion/dependency sites. Python +keeps PEP 440 form ('0.8.1.post1', '0.8.1.dev3'); for Cargo/Helm a .devN becomes a +SemVer pre-release ('0.8.1-dev3', sorts before 0.8.1) and a .postN becomes SemVer +build metadata ('0.8.1+post1'). Sites holding an independent version (not M) are +left alone. """ from __future__ import annotations import argparse import re +import subprocess import sys from pathlib import Path @@ -44,6 +53,79 @@ "lib/kvbm-physical/Cargo.toml", ] +# Member manifests that pin the workspace version inside a dependency +# inline table, e.g. backend-common's +# `dynamo-llm = { path = "../llm", version = "1.4.0", default-features = false }` +# (a direct path dep because cargo cannot express `workspace = true` + +# `default-features = false`). VERSION_LINE_RE is line-anchored and +# intentionally skips inline tables, so these files get the same exact-string +# rewrite as the root path-dep pins (the pinned value always equals +# [workspace.package].version). Covered by BOTH modes: the dev-suffix stamp +# (rewrite_root_cargo) and --set-version (set_release_version). +WORKSPACE_PIN_CARGO_TARGETS = [ + "lib/backend-common/Cargo.toml", +] + +# Helm charts carry the unified version in version / appVersion / dependency +# version. Each entry is (helm_subset_token, Chart.yaml path); a chart is bumped +# only when its token is in the --helm subset. operator is a subchart of platform, +# so it rides the "platform" token. Only touched in --set-version (release) mode; +# nightly never bumps charts. +HELM_CHART_TARGETS = [ + ("platform", "deploy/helm/charts/platform/Chart.yaml"), + ("platform", "deploy/helm/charts/platform/components/operator/Chart.yaml"), + ("snapshot", "deploy/helm/charts/snapshot/Chart.yaml"), +] + +# First-party image `tag:` sites in values.yaml. Each entry is +# (container_token, helm_token, values.yaml path, image repository). The tag is set +# to the release version only if the chart is published (helm_token in --helm) AND +# its image is published (container_token in --containers). If the chart is +# published but the image is excluded, the tag is PINNED to the last-published value +# so the chart never references a missing image; if the chart is not published the +# site is left untouched. The operator tag is written explicitly here, decoupling it +# from its `tag: "" -> .Chart.AppVersion` inheritance. 3rd-party tags (etcd/nats) are +# never matched (different repositories). +HELM_IMAGE_TAG_SITES = [ + ("operator", "platform", "deploy/helm/charts/platform/values.yaml", + "nvcr.io/nvidia/ai-dynamo/kubernetes-operator"), + ("operator", "platform", "deploy/helm/charts/platform/components/operator/values.yaml", + "nvcr.io/nvidia/ai-dynamo/kubernetes-operator"), + ("snapshot", "snapshot", "deploy/helm/charts/snapshot/values.yaml", + "nvcr.io/nvidia/ai-dynamo/snapshot-agent"), +] + +# Normalized subset universes for --containers / --helm token validation. +CONTAINER_TOKENS = { + "vllm-runtime", "vllm-efa", "sglang-runtime", "sglang-efa", + "trtllm-runtime", "trtllm-efa", "frontend", "operator", "planner", "snapshot", +} +HELM_TOKENS = {"platform", "snapshot"} + +# Container token -> the NGC repo release.yml actually publishes at :. +# Used by --image-refs to rewrite the `my-registry`/`my-tag` placeholders in docs, +# examples and deploy manifests ONLY for images this release publishes, so the tree +# never advertises a tag that will not exist. +# The `-efa` tokens are deliberately ABSENT: they publish :-efa, so an +# EFA-only selection must leave the plain : references alone. +# Images with no token (fastvideo-runtime, epp-image, nixlbench, tensorrt-llm, dynamo) +# are never published by release.yml, so their placeholders stay placeholders. +IMAGE_REF_TOKENS = { + "vllm-runtime": "vllm-runtime", + "sglang-runtime": "sglang-runtime", + "trtllm-runtime": "tensorrtllm-runtime", + "frontend": "dynamo-frontend", + "operator": "kubernetes-operator", + "planner": "dynamo-planner", + "snapshot": "snapshot-agent", +} +GA_REGISTRY = "nvcr.io/nvidia/ai-dynamo" +PLACEHOLDER_REGISTRY = "my-registry" + +# .devN is a PRE-release (sorts before X.Y.Z) -> SemVer '-devN'; .postN is a +# post-release -> SemVer build metadata '+postN'. Both keep PEP 440 form for Python. +SET_RE = re.compile(r"^(\d+)\.(\d+)\.(\d+)(?:\.(dev\d+|post\d+))?$") + # Line-anchored: matches `version = "X.Y.Z"` lines. Skips `version.workspace = true` # (no quotes) and `version = { ... }` (no string). Safe for sub-crate Cargo.tomls # whose only `version = "..."` line is the [package] one; external-crate deps use @@ -140,25 +222,327 @@ def rewrite_root_cargo(root: Path, suffix: str) -> None: return # already stamped -- idempotent no-op new = semver(suffix, base) + pin_re = re.compile(rf'(\bversion\s*=\s*"){re.escape(base)}(")') + text = pin_re.sub(lambda mm: f"{mm.group(1)}{new}{mm.group(2)}", text) + path.write_text(text) + + # Bump the same literal pin where it lives in member manifests (inline + # dep tables that VERSION_LINE_RE deliberately skips). The early + # "already stamped" return above keeps this idempotent. + for rel in WORKSPACE_PIN_CARGO_TARGETS: + p = root / rel + if not p.exists(): + continue + t = p.read_text() + t2 = pin_re.sub(lambda mm: f"{mm.group(1)}{new}{mm.group(2)}", t) + if t2 != t: + p.write_text(t2) + + +def _workspace_version(root: Path) -> str: + text = (root / "Cargo.toml").read_text() + m = re.search(r'\[workspace\.package\][^\[]*?\n\s*version\s*=\s*"([^"]+)"', text) + if not m: + raise RuntimeError("no [workspace.package].version in root Cargo.toml") + return m.group(1) + + +def _semver_form(new: str) -> str: + m = SET_RE.match(new) + if not m: + raise RuntimeError(f"--set-version must be X.Y.Z, X.Y.Z.devN, or X.Y.Z.postN (got '{new}')") + base = f"{m.group(1)}.{m.group(2)}.{m.group(3)}" + suffix = m.group(4) + if not suffix: + return base + # dev -> pre-release '-devN' (sorts before base); post -> build metadata '+postN'. + return f"{base}-{suffix}" if suffix.startswith("dev") else f"{base}+{suffix}" + + +def set_pyproject(path: Path, old: str, new: str, is_root: bool) -> None: + text = VERSION_LINE_RE.sub( + lambda m: f"{m.group(1)}{new}{m.group(3)}" if m.group(2) == old else m.group(0), + path.read_text(), + count=1, + ) + if is_root: + text = PY_RUNTIME_PIN_RE.sub( + lambda m: f"{m.group(1)}{new}{m.group(3)}" if m.group(2) == old else m.group(0), + text, + ) + path.write_text(text) + + +def set_cargo(path: Path, old: str, new: str) -> None: text = re.sub( - rf'(\bversion\s*=\s*"){re.escape(base)}(")', - lambda mm: f"{mm.group(1)}{new}{mm.group(2)}", + rf'(\bversion\s*=\s*"){re.escape(old)}(")', + lambda m: f"{m.group(1)}{new}{m.group(2)}", + path.read_text(), + ) + path.write_text(text) + + +def set_helm(path: Path, old: str, new: str) -> None: + text = path.read_text() + + # Top-level version/appVersion set unconditionally: a rewrite keyed on `old` + # leaves stale values when a reused branch widens the helm subset. + top = re.compile( + r'^(?P
(?:appVersion|version)\s*:\s*)(?P"?)[^"\n]*(?P=q)(?P\s*)$',
+        re.MULTILINE,
+    )
+    text, n_top = top.subn(
+        lambda m: f"{m.group('pre')}{m.group('q')}{new}{m.group('q')}{m.group('post')}", text
+    )
+    if n_top == 0:
+        raise RuntimeError(f"no top-level version/appVersion in {path}")
+
+    # dynamo-operator (file:// subchart) pin always rides the workspace version;
+    # the hop is bounded to the entry so it can't reach nats/etcd/....
+    text = re.sub(
+        r'(?m)^(\s*-\s+name:\s*dynamo-operator\s*\n'
+        r'(?:(?!\s*-\s)[^\n]*\n)*?'
+        r'\s*version\s*:\s*)("?)[^"\n]*\2(\s*)$',
+        lambda m: f"{m.group(1)}{m.group(2)}{new}{m.group(2)}{m.group(3)}",
         text,
     )
+
+    # Deliberately no generic indented-version rewrite: dynamo-operator is the
+    # only first-party dep pin, and a keyed catch-all would clobber a foreign
+    # pin that equals the workspace version (nats is pinned 1.3.2).
     path.write_text(text)
 
 
+# Bounds the repository->tag hop at the next `repository:` line, so a block
+# with no tag fails loudly instead of rewriting another image's tag.
+_TAG_HOP = r'(?:(?![^\n]*repository:)[^\n]*\n)*?'
+
+
+def set_helm_values_tag(path: Path, repo: str, new: str) -> None:
+    # Set the `tag:` that follows the image `repository: ` line to `new`,
+    # regardless of its current value (the published image tag is the release tag).
+    pat = re.compile(
+        r'(repository:\s*"?' + re.escape(repo) + r'"?\s*\n' + _TAG_HOP + r'\s*tag:\s*)"?[^"\n]*"?',
+        re.MULTILINE,
+    )
+    text, n = pat.subn(lambda m: f"{m.group(1)}{new}", path.read_text(), count=1)
+    if n != 1:
+        raise RuntimeError(f"could not find image tag for {repo} in {path}")
+    path.write_text(text)
+
+
+def _current_image_tag(path: Path, repo: str) -> str:
+    # The tag currently set for `repo` in values.yaml ('' if unset/missing —
+    # an empty tag inherits the chart appVersion at deploy time, so there is no
+    # recorded last-published tag to pin to).
+    m = re.search(
+        r'repository:\s*"?' + re.escape(repo) + r'"?\s*\n' + _TAG_HOP + r'\s*tag:\s*"?([^"\n]*)"?',
+        path.read_text(),
+    )
+    return m.group(1).strip() if m else ""
+
+
+def _tracked_files(root: Path) -> list[Path]:
+    out = subprocess.run(["git", "ls-files", "-z"], cwd=root, check=True,
+                         capture_output=True, text=True).stdout
+    files = []
+    for rel in out.split("\0"):
+        if not rel or rel.startswith(".github/"):
+            # .github holds the release tooling itself — rewriting it would corrupt
+            # the very placeholder patterns this step relies on.
+            continue
+        p = root / rel
+        if p.is_file() and not p.is_symlink():
+            files.append(p)
+    return files
+
+
+def rewrite_image_refs(root: Path, new_version: str, containers: set[str],
+                       old_version: str) -> tuple[int, int]:
+    """Point first-party image references at the GA registry + release version —
+    but ONLY for images this release actually publishes.
+
+    Rewrites, per selected image:
+        my-registry/:my-tag        -> /:
+        /:my-tag               -> /:   (tag-only placeholder)
+        /:                -> /:   (re-cut at a new version)
+
+    Anything not in the selection keeps its placeholder, so a container-only release
+    can never ship a doc telling users to pull an image that was never built.
+    Returns (files_changed, refs_rewritten)."""
+    images = sorted({IMAGE_REF_TOKENS[t] for t in containers if t in IMAGE_REF_TOKENS})
+    if not images:
+        print("rewrite_image_refs: no publishable image selected; placeholders left intact",
+              file=sys.stderr)
+        return (0, 0)
+
+    # old_version comes from Cargo.toml, i.e. the SemVer form (1.3.0+post1,
+    # 1.3.1-dev0), but image tags carry the PEP 440 form (1.3.0.post1, 1.3.1.dev0).
+    # Match BOTH or a re-cut off a .devN/.postN branch silently leaves every image
+    # reference pinned to the previous release.
+    old_literals = ["my-tag"]
+    if old_version and old_version != new_version:
+        old_pep = re.sub(r"[-+](dev|post)", r".\1", old_version)
+        for t in dict.fromkeys((old_version, old_pep)):
+            if t and t != new_version:
+                old_literals.append(t)
+    tag_alt = "|".join(re.escape(t) for t in old_literals)
+    reg_alt = f"(?:{re.escape(PLACEHOLDER_REGISTRY)}|{re.escape(GA_REGISTRY)})"
+    # The tag must END here: `(?![\w.-])` refuses to match a PREFIX of a longer tag.
+    # Without it `:1.3.0` would also match inside `:1.3.0-nemotron`, `:1.2.0-efa`,
+    # `:1.3.0-cuda13` and `:1.4.0.dev1`, rewriting them to a bare version and
+    # silently destroying the variant suffix.
+    pats = [
+        (img, re.compile(rf"{reg_alt}/{re.escape(img)}:(?:{tag_alt})(?![\w.-])"), f"{GA_REGISTRY}/{img}:{new_version}")
+        for img in images
+    ]
+    # Untagged prose references (`my-registry/vllm-runtime` with no `:tag`) still
+    # move to the GA registry — but only for selected images, so an unpublished
+    # image is never given a real registry path. Runs after the tagged patterns,
+    # which have already consumed the `/:` forms.
+    pats += [
+        (img, re.compile(rf"{re.escape(PLACEHOLDER_REGISTRY)}/{re.escape(img)}(?![\w.:-])"), f"{GA_REGISTRY}/{img}")
+        for img in images
+    ]
+
+    files_changed = refs = 0
+    for path in _tracked_files(root):
+        try:
+            text = original = path.read_text()
+        except (UnicodeDecodeError, OSError):
+            continue  # binary or unreadable — nothing to substitute
+        # Fast path — must test every literal the regex can match, including the
+        # PEP 440 spelling of the old version and the untagged placeholder registry.
+        if not any(t in text for t in old_literals) and PLACEHOLDER_REGISTRY not in text:
+            continue
+        for _img, pat, repl in pats:
+            text, n = pat.subn(repl, text)
+            refs += n
+        if text != original:
+            path.write_text(text)
+            files_changed += 1
+
+    print(f"rewrite_image_refs: {refs} reference(s) in {files_changed} file(s) -> "
+          f"{GA_REGISTRY}/:{new_version} for {images}", file=sys.stderr)
+
+    # Advisory only: a re-cut of a branch that previously shipped a wider selection
+    # legitimately still carries those older refs, so warn rather than fail.
+    unselected = sorted(set(IMAGE_REF_TOKENS.values()) - set(images))
+    stale = []
+    for path in _tracked_files(root):
+        try:
+            text = path.read_text()
+        except (UnicodeDecodeError, OSError):
+            continue
+        for img in unselected:
+            if f"{GA_REGISTRY}/{img}:{new_version}" in text:
+                stale.append(f"{path.relative_to(root)} -> {img}")
+    if stale:
+        print(f"::warning::{len(stale)} reference(s) point at unselected image(s) at "
+              f"{new_version}: {stale[:8]}{' …' if len(stale) > 8 else ''}", file=sys.stderr)
+    return (files_changed, refs)
+
+
+def _parse_subset(spec: str, universe: set[str]) -> set[str]:
+    spec = (spec or "all").strip()
+    if spec == "all":
+        return set(universe)
+    if spec in ("", "none"):
+        return set()
+    sel = {t.strip() for t in spec.split(",") if t.strip()}
+    unknown = sel - universe
+    if unknown:
+        raise RuntimeError(f"unknown subset token(s) {sorted(unknown)}; valid: {sorted(universe)}")
+    return sel
+
+
+def set_release_version(root: Path, new_version: str, containers: set[str], helm: set[str],
+                        image_refs: bool = False) -> None:
+    old = _workspace_version(root)
+    semver = _semver_form(new_version)
+
+    def _exists(rel: str) -> bool:
+        # source_ref may predate a target (older release branches / main SHAs);
+        # a path that doesn't exist there has nothing to stamp.
+        if (root / rel).exists():
+            return True
+        print(f"set_release_version: skip {rel} (absent at this source ref)", file=sys.stderr)
+        return False
+
+    # Package identity -- ALWAYS bumped, regardless of the wheels/crates selection:
+    # the containers embed wheels built from this tree, so a container-only release
+    # still needs the workspace/pyproject versions stamped or the shipped image would
+    # carry the previous version. (wheels/crates are intentionally not passed in.)
+    for rel in PYPROJECT_TARGETS:
+        if rel == "pyproject.toml" or _exists(rel):
+            set_pyproject(root / rel, old, new_version, is_root=(rel == "pyproject.toml"))
+    set_cargo(root / "Cargo.toml", old, semver)
+    for rel in SUBCRATE_CARGO_TARGETS:
+        if _exists(rel):
+            set_cargo(root / rel, old, semver)
+    # Workspace-version pins in member manifests (inline dep tables, e.g.
+    # backend-common's dynamo-llm pin) carry the same literal as the root
+    # pins; set_cargo's exact-string replace stamps them identically.
+    for rel in WORKSPACE_PIN_CARGO_TARGETS:
+        if _exists(rel):
+            set_cargo(root / rel, old, semver)
+    # Chart identity -- only for charts in the --helm subset.
+    for token, rel in HELM_CHART_TARGETS:
+        if token in helm and _exists(rel):
+            set_helm(root / rel, old, semver)
+    # First-party image tags: published image -> new_version (NGC tag form, not
+    # SemVer); published chart with excluded image -> pin to the recorded tag; no
+    # recorded tag (''/'my-tag') -> fail at cut time, the chart could never resolve.
+    for ctoken, htoken, rel, repo in HELM_IMAGE_TAG_SITES:
+        if htoken not in helm or not _exists(rel):
+            continue
+        path = root / rel
+        if ctoken in containers:
+            tag = new_version
+        else:
+            tag = _current_image_tag(path, repo)
+            if tag in ("", "my-tag"):
+                raise RuntimeError(
+                    f"chart '{htoken}' is selected but its image '{ctoken}' is excluded and "
+                    f"{rel} records no previously published tag for {repo}; either add "
+                    f"'{ctoken}' to the container selection or drop '{htoken}' from the helm selection")
+        set_helm_values_tag(path, repo, tag)
+    print(f"set_release_version: {old} -> py={new_version} semver={semver} "
+          f"containers={sorted(containers)} helm={sorted(helm)}", file=sys.stderr)
+    # Docs / examples / deploy manifests: selection-gated so the release branch never
+    # advertises an image tag this release does not publish.
+    if image_refs:
+        rewrite_image_refs(root, new_version, containers, old)
+
+
 def main() -> int:
     ap = argparse.ArgumentParser(description=__doc__)
-    ap.add_argument("suffix", help="e.g. .dev20260423 (empty = no-op)")
+    ap.add_argument("suffix", nargs="?", default="", help="e.g. .dev20260423 (empty = no-op)")
     ap.add_argument("root", nargs="?", default=".", help="repo root")
+    ap.add_argument("--set-version", dest="set_version", default="",
+                    help="set an absolute release version X.Y.Z[.devN|.postN] instead of appending a suffix")
+    ap.add_argument("--containers", default="all",
+                    help="normalized container subset (all|none|csv) gating image-tag bumps")
+    ap.add_argument("--helm", default="all",
+                    help="helm chart subset (all|none|csv of platform,snapshot) gating chart bumps")
+    ap.add_argument("--image-refs", action="store_true",
+                    help="also point the my-registry/my-tag placeholders in docs, examples and "
+                         "deploy manifests at the GA registry + release version — only for images "
+                         "in --containers; unselected images keep their placeholder")
     args = ap.parse_args()
 
+    root = Path(args.root).resolve()
+
+    if args.set_version:
+        containers = _parse_subset(args.containers, CONTAINER_TOKENS)
+        helm = _parse_subset(args.helm, HELM_TOKENS)
+        set_release_version(root, args.set_version, containers, helm, image_refs=args.image_refs)
+        return 0
+
     if not args.suffix:
         print("apply_dev_version: empty suffix, no-op", file=sys.stderr)
         return 0
 
-    root = Path(args.root).resolve()
     for rel in PYPROJECT_TARGETS:
         rewrite_pyproject(root / rel, args.suffix, is_root=(rel == "pyproject.toml"))
     rewrite_root_cargo(root, args.suffix)
diff --git a/.github/scripts/stage_crates.py b/.github/scripts/stage_crates.py
new file mode 100644
index 000000000000..d60cc86b29da
--- /dev/null
+++ b/.github/scripts/stage_crates.py
@@ -0,0 +1,444 @@
+#!/usr/bin/env python3
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+"""Publish Dynamo's publishable workspace crates to an Artifactory cargo registry.
+
+Mirrors GITLAB/runbooks/rcs/dynamo/crates_*.sh, but derives the crate set and
+dependency order from `cargo metadata` instead of a hardcoded list, so it stays
+correct as the workspace graph changes. Crates are published leaves-first so each
+crate's intra-workspace deps are indexed before its dependents.
+
+The Artifactory reference token (ARTIFACTORY_TOKEN, the same one the wheel upload
+uses) is sent as a Bearer header for both the cargo registry and the idempotency
+HEAD check. The registry location is read from ARTIFACTORY_CARGO_INDEX (a secret,
+e.g. sparse+https:///artifactory/api/cargo//index/) so it is not
+hardcoded in the workflow. Idempotent: crates already present are skipped.
+
+Fail-soft: a crate that fails to publish is recorded (FAILED_CRATE=...), its
+workspace dependents are skipped with a reason, and the rest of the graph keeps
+staging; the exit code is nonzero when anything failed so the workflow can
+report loudly without blocking the release.
+
+Non-publishable members (the python/c bindings, example binaries) are excluded by
+manifest-path; anything with `publish = false` is excluded too.
+"""
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import re
+import subprocess
+import sys
+import time
+import urllib.error
+import urllib.request
+from pathlib import Path
+
+# Manifest-path substrings whose crates are never published: python/C bindings,
+# example binaries, and deploy/ components — deployable binaries/services (e.g. the
+# inference-gateway ext-proc) that are not library crates consumers pull from the
+# registry, and whose build scripts (envoy/protoc codegen) aren't wired up here.
+EXCLUDE_PATH_SUBSTRINGS = ("/bindings/", "/examples/", "/deploy/")
+# Workspace members never published from the on-demand flow: internal crates that
+# carry an independent version (not the workspace version) and that no published
+# crate depends on. kvbm-consolidator pins its own 1.2.0, so the version gate would
+# (correctly) abort the release; it is a leaf, so excluding it is closure-safe.
+EXCLUDE_NAMES = frozenset({"kvbm-consolidator"})
+# Artifactory < 7.148 rejects the `Content-Type: application/octet-stream` header
+# cargo >= 1.96 sends on publish with a 415 (JFrog RTDEV-83141, rust-lang/cargo#17086),
+# so the upload runs on the last pre-header toolchain. Remove once
+# artifactory.nvidia.com runs Artifactory 7.148+.
+PUBLISH_TOOLCHAIN = "1.95.0"
+# No bare "HTTP/2" alternation here: cargo dumps the raw status line ("HTTP/2 415")
+# on every HTTP error, which made deterministic 4xx failures retry as transient.
+RETRYABLE = re.compile(r"stream error|INTERNAL_ERROR|connection error|timed out|\b(429|50[0-9])\b")
+ALREADY = re.compile(r"already exists|409 Conflict|already uploaded")
+# Sparse-index propagation lag after publishing a dep — retried before failing.
+DEP_NOT_INDEXED = re.compile(r"no matching package named")
+
+
+def cargo_metadata(root: Path) -> dict:
+    out = subprocess.run(
+        ["cargo", "metadata", "--format-version", "1", "--no-deps"],
+        cwd=root, check=True, capture_output=True, text=True,
+    ).stdout
+    return json.loads(out)
+
+
+def publishable(meta: dict) -> dict[str, dict]:
+    members = set(meta["workspace_members"])
+    pkgs: dict[str, dict] = {}
+    for p in meta["packages"]:
+        if p["id"] not in members:
+            continue
+        if any(s in p["manifest_path"] for s in EXCLUDE_PATH_SUBSTRINGS):
+            continue
+        if p["name"] in EXCLUDE_NAMES:
+            continue
+        if p.get("publish") == []:  # `publish = false` serializes to []
+            continue
+        pkgs[p["name"]] = p
+    return pkgs
+
+
+def topo_order(pkgs: dict[str, dict]) -> list[str]:
+    names = set(pkgs)
+    incoming = {
+        n: {d["name"] for d in pkgs[n]["dependencies"]
+            if d["name"] in names and d["name"] != n and d.get("kind") != "dev"}
+        for n in names
+    }
+    order: list[str] = []
+    ready = sorted(n for n in names if not incoming[n])
+    while ready:
+        n = ready.pop(0)
+        order.append(n)
+        for m in names:
+            if n in incoming[m]:
+                incoming[m].discard(n)
+                if not incoming[m] and m not in order and m not in ready:
+                    ready.append(m)
+        ready.sort()
+    if len(order) != len(names):
+        raise RuntimeError(f"dependency cycle among crates: {sorted(names - set(order))}")
+    return order
+
+
+def write_cargo_config(root: Path, alias: str, index: str) -> None:
+    cfg = root / ".cargo" / "config.toml"
+    cfg.parent.mkdir(exist_ok=True)
+    cfg.write_text(
+        f'[registries]\n{alias} = {{ index = "{index}" }}\n\n'
+        f'[registry]\ndefault = "{alias}"\n'
+        'global-credential-providers = ["cargo:token"]\n'
+    )
+
+
+def inject_registry(manifests: list[Path], names: list[str], alias: str, versions: dict[str, str]) -> None:
+    # Internal deps must carry a registry + a version when published, else cargo
+    # publish defaults them to crates.io / rejects them. `workspace = true` deps
+    # inherit both from the root table (which we also rewrite), so skip them.
+    # A direct path dep (e.g. `dynamo-vllm-sidecar = { path = "../../../sidecar/vllm" }`
+    # in the vllm mocker server) has neither — add the registry AND the depended-on
+    # crate's publish version so `cargo publish` accepts it (cargo drops the path on
+    # publish). Manifests come from cargo metadata, not a glob: a shallow lib/* glob
+    # missed nested members like lib/mocker/servers/vllm, whose un-rewritten path dep
+    # then failed `cargo publish` with "does not specify a version".
+    pat = re.compile(
+        r"^(" + "|".join(re.escape(n) for n in names) + r")(\s*=\s*)\{([^}]*)\}",
+        re.MULTILINE,
+    )
+
+    def repl(m: re.Match) -> str:
+        name, inner = m.group(1), m.group(3).strip()
+        if "registry" in inner or "workspace" in inner:
+            return m.group(0)
+        parts = inner.rstrip(",")
+        if "version" not in inner and name in versions:
+            parts = f'{parts}, version = "{versions[name]}"'
+        return f'{m.group(1)}{m.group(2)}{{ {parts}, registry = "{alias}" }}'
+
+    for manifest in manifests:
+        text = manifest.read_text()
+        new = pat.sub(repl, text)
+        if new != text:
+            manifest.write_text(new)
+
+
+def rewrite_versions(manifests: list[Path], old: str, new: str) -> int:
+    """Replace `version = ""` with `version = ""` across the given
+    manifests (root + every workspace member, from cargo metadata) — both
+    [workspace.package].version and the path-dep pins that share it. Used to stamp
+    the internal staging rc version (e.g. 0.1.0-rc0) without re-bumping the commit;
+    mirrors the bump's version replace. Returns the number of manifests changed."""
+    pat = re.compile(rf'(\bversion\s*=\s*"){re.escape(old)}(")')
+    changed = 0
+    for manifest in manifests:
+        text = manifest.read_text()
+        new_text = pat.sub(lambda m: f"{m.group(1)}{new}{m.group(2)}", text)
+        if new_text != text:
+            manifest.write_text(new_text)
+            changed += 1
+    return changed
+
+
+def strip_git_deps(root: Path, manifests: list[Path]) -> None:
+    """Remove git-sourced deps (and feature refs to them) before publish: cargo
+    publish rejects deps without a registry version, and a git dep — including a
+    versioned dep redirected by [patch.crates-io] — can't resolve faithfully for
+    a registry consumer. Only optional deps are dropped (non-optional is a hard
+    error); the feature KEY is kept (emptied) so `/` refs resolve."""
+    git_names = set(re.findall(
+        r'(?m)^\s*([A-Za-z0-9_-]+)\s*=\s*\{[^}\n]*\bgit\s*=',
+        (root / "Cargo.toml").read_text()))
+    if not git_names:
+        return
+    for mp in manifests:
+        orig = mp.read_text()
+        kept, removed = [], set()
+        for line in orig.splitlines(keepends=True):
+            m = re.match(r'^[ \t]*([A-Za-z0-9_-]+)[ \t]*=[ \t]*\{(.*)\}[ \t]*$', line)
+            if m and (re.search(r'\bgit\s*=', m.group(2))
+                      or (m.group(1) in git_names and re.search(r'\bworkspace\s*=\s*true', m.group(2)))):
+                removed.add(m.group(1))
+                continue
+            if m and m.group(1) in git_names and re.search(r'\bversion\s*=', m.group(2)):
+                # Versioned dep git-patched via [patch.crates-io]: droppable only if optional.
+                if not re.search(r'\boptional\s*=\s*true', m.group(2)):
+                    raise RuntimeError(
+                        f"{mp}: non-optional dependency '{m.group(1)}' is git-patched in the "
+                        "workspace; a registry consumer would resolve different code. "
+                        "Vendor/publish the patched crate or make the dependency optional.")
+                removed.add(m.group(1))
+                continue
+            kept.append(line)
+        if not removed:
+            continue
+        text = "".join(kept)
+        for nm in removed:
+            # scrub feature-array tokens: "dep:nm", "nm", "nm/feat", "nm?/feat"
+            text = re.sub(rf'"(?:dep:)?{re.escape(nm)}(?:\?)?(?:/[^"]*)?"', "", text)
+        text = re.sub(r'\[\s*,', "[", text)          # [ , ...   -> [ ...
+        text = re.sub(r',\s*,', ",", text)            # , ,       -> ,
+        text = re.sub(r',(\s*[\]\n])', r'\1', text)   # , ]  / ,\n -> ] / \n
+        mp.write_text(text)
+        print(f"strip-git-deps: removed {sorted(removed)} from {mp}")
+
+
+def crate_exists(raw_base: str, name: str, version: str, token: str) -> bool:
+    # 200 -> present, 404 -> absent; anything else (auth/outage) is a hard error.
+    url = f"{raw_base}/{name}/{name}-{version}.crate"
+    req = urllib.request.Request(url, method="HEAD", headers={"Authorization": f"Bearer {token}"})
+    try:
+        with urllib.request.urlopen(req, timeout=30) as r:
+            return r.status == 200
+    except urllib.error.HTTPError as e:
+        if e.code == 404:
+            return False
+        raise RuntimeError(f"registry HEAD {url} failed: HTTP {e.code} (bad ARTIFACTORY_TOKEN or registry outage?)") from e
+    except urllib.error.URLError as e:
+        raise RuntimeError(f"registry HEAD {url} unreachable: {e.reason}") from e
+
+
+def publish(manifest: str, alias: str, env: dict) -> str:
+    # Returns 'published' / 'already' (both staged); raises on definitive
+    # failure — the caller records it and continues (fail-soft).
+    for attempt in range(1, 4):
+        r = subprocess.run(
+            ["cargo", f"+{PUBLISH_TOOLCHAIN}", "publish", "--allow-dirty", "--no-verify",
+             "--manifest-path", manifest, "--registry", alias],
+            env=env, capture_output=True, text=True,
+        )
+        sys.stdout.write(r.stdout)
+        sys.stderr.write(r.stderr)
+        out = r.stdout + r.stderr
+        if r.returncode == 0:
+            return "published"
+        if ALREADY.search(out):
+            print(f"  -> {manifest}: skip (already present)")
+            return "already"
+        if attempt < 3 and DEP_NOT_INDEXED.search(out):
+            print(f"  -> dep not yet indexed (sparse-index lag?), retry in 15s ({attempt}/3)")
+            time.sleep(15)
+            continue
+        if attempt < 3 and RETRYABLE.search(out):
+            print(f"  -> transient error, retry in 10s ({attempt}/3)")
+            time.sleep(10)
+            continue
+        raise RuntimeError(f"cargo publish failed for {manifest}")
+    raise RuntimeError(f"exhausted retries for {manifest}")
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser(description=__doc__)
+    ap.add_argument("--registry", default="artifactory", help="cargo registry alias")
+    ap.add_argument("--root", default=".", help="repo root")
+    ap.add_argument("--expect-version", default=os.environ.get("EXPECT_VERSION", ""),
+                    help="fail fast if any publishable crate's version differs from this")
+    ap.add_argument("--only", default=os.environ.get("CRATE_SUBSET", ""),
+                    help="csv of crate names to publish (strict: must be dependency-closed); "
+                         "empty/'all' = every publishable crate")
+    ap.add_argument("--stage-version", default=os.environ.get("STAGE_VERSION", ""),
+                    help="rewrite every publishable crate's version to this exact value before "
+                         "publishing (internal staging rc disambiguator, e.g. 0.1.0-rc0), so a "
+                         "re-stage of the same release doesn't collide on the immutable registry. "
+                         "Empty = leave the committed versions as-is")
+    args = ap.parse_args()
+
+    # ARTIFACTORY ONLY — never crates.io. Refuse a crates.io alias outright; every
+    # `cargo publish` below passes --registry  and write_cargo_config
+    # sets it as the default registry, so crates.io is never a publish target.
+    if args.registry in ("crates-io", "crates.io", "crates_io"):
+        print("::error::refusing to publish to crates.io; this stages to Artifactory only", file=sys.stderr)
+        return 1
+
+    # Reuse the wheel-upload token; keep the registry location out of the workflow
+    # via the ARTIFACTORY_CARGO_INDEX secret.
+    token = os.environ.get("ARTIFACTORY_TOKEN", "")
+    index = os.environ.get("ARTIFACTORY_CARGO_INDEX", "")
+    if not token or not index:
+        print("::error::ARTIFACTORY_TOKEN and ARTIFACTORY_CARGO_INDEX must be set", file=sys.stderr)
+        return 1
+    m = re.match(r"sparse\+(https://.+?)/api/cargo/([^/]+)/index/?$", index)
+    if not m:
+        print("::error::ARTIFACTORY_CARGO_INDEX must be 'sparse+https:///.../api/cargo//index/'", file=sys.stderr)
+        return 1
+    raw_base = f"{m.group(1)}/{m.group(2)}/crates"
+
+    root = Path(args.root).resolve()
+    meta = cargo_metadata(root)
+    pkgs = publishable(meta)
+    # Every workspace member's manifest (nested ones included — lib/sidecar/*,
+    # lib/mocker/servers/*), for the version/registry rewrites below.
+    members = set(meta["workspace_members"])
+    member_manifests = sorted(
+        Path(p["manifest_path"]) for p in meta["packages"] if p["id"] in members
+    )
+    order = topo_order(pkgs)
+    print("Publish order:", " ".join(order))
+
+    # Narrow to an explicit subset (strict: no silent auto-expansion). Every
+    # intra-workspace dependency of a selected crate must also be selected, or a
+    # dependent would fail to resolve its dep from the registry.
+    only = args.only.strip()
+    if only and only != "all":
+        selected = {c.strip() for c in only.split(",") if c.strip()}
+        # The --only list is a curated wishlist that may name crates absent on the
+        # branch being staged (the workspace graph differs across release branches).
+        # Drop those with a warning rather than failing the whole run; closure is then
+        # checked on what actually exists here.
+        unknown = selected - set(pkgs)
+        if unknown:
+            print(f"::warning::skipping requested crate(s) not publishable on this branch: "
+                  f"{sorted(unknown)} (branch publishable set: {sorted(pkgs)})", file=sys.stderr)
+            selected -= unknown
+        if not selected:
+            print("::error::none of the requested crates are publishable on this branch", file=sys.stderr)
+            return 1
+        gaps = {n: sorted({d["name"] for d in pkgs[n]["dependencies"]
+                           if d["name"] in pkgs and d["name"] != n and d.get("kind") != "dev"} - selected)
+                for n in selected}
+        gaps = {n: g for n, g in gaps.items() if g}
+        if gaps:
+            for n, g in gaps.items():
+                print(f"::error::crate {n} depends on un-selected workspace crate(s) {g}", file=sys.stderr)
+            print("::error::crate subset is not dependency-closed; add the missing crate(s) or use 'all'", file=sys.stderr)
+            return 1
+        order = [n for n in order if n in selected]
+        print("Selected crates (dependency-closed):", " ".join(order))
+
+    # Internal staging: rewrite the bumped workspace version to the rc-suffixed staging
+    # version (e.g. 0.1.0 -> 0.1.0-rc0) so each re-stage publishes a distinct,
+    # non-colliding version on the immutable registry. The expect-version gate below
+    # then validates the new value, and cargo publish uploads it (--allow-dirty).
+    stage_version = args.stage_version.strip()
+    if stage_version:
+        wm = re.search(r'\[workspace\.package\][^\[]*?\n\s*version\s*=\s*"([^"]+)"',
+                       (root / "Cargo.toml").read_text())
+        if not wm:
+            print("::error::--stage-version: cannot read [workspace.package].version", file=sys.stderr)
+            return 1
+        cur = wm.group(1)
+        if cur != stage_version:
+            n = rewrite_versions([root / "Cargo.toml", *member_manifests], cur, stage_version)
+            print(f"stage-version: rewrote {cur} -> {stage_version} across {n} manifest(s)")
+            for nm in pkgs:
+                if pkgs[nm]["version"] == cur:
+                    pkgs[nm]["version"] = stage_version
+
+    # Fail fast BEFORE any build/publish if a crate carries an unexpected version
+    # (e.g. a hardcoded version the bump missed) — never silently publish wrong tags.
+    if args.expect_version:
+        mismatched = [(n, pkgs[n]["version"]) for n in order if pkgs[n]["version"] != args.expect_version]
+        if mismatched:
+            for n, v in mismatched:
+                print(f"::error::crate {n} is at version {v}, expected {args.expect_version} (hardcoded/un-bumped version)", file=sys.stderr)
+            print(f"::error::aborting before publish: {len(mismatched)} crate(s) carry an unexpected version; "
+                  "set 'version.workspace = true' (or fix Cargo.toml) and re-run", file=sys.stderr)
+            return 1
+
+    write_cargo_config(root, args.registry, index)
+    inject_registry(
+        [root / "Cargo.toml", *(Path(pkgs[n]["manifest_path"]) for n in order)],
+        order, args.registry, {n: pkgs[n]["version"] for n in order},
+    )
+    # cargo publish rejects deps without a registry version; drop the optional,
+    # private git deps (e.g. aiconfigurator-core) from the crates being published.
+    strip_git_deps(root, [Path(pkgs[n]["manifest_path"]) for n in order])
+
+    env = dict(os.environ)
+    env[f"CARGO_REGISTRIES_{args.registry.upper()}_TOKEN"] = f"Bearer {token}"
+    env["RUSTFLAGS"] = f"{env.get('RUSTFLAGS', '')} --cfg tokio_unstable".strip()
+    # Only the upload PUT runs on PUBLISH_TOOLCHAIN (--no-verify skips compilation);
+    # cargo check/update stay on the workspace's rust-toolchain.toml toolchain.
+    r = subprocess.run(["rustup", "toolchain", "install", "--profile", "minimal",
+                        "--no-self-update", PUBLISH_TOOLCHAIN], env=env)
+    if r.returncode != 0:
+        print(f"::error::rustup install of publish toolchain {PUBLISH_TOOLCHAIN} failed",
+              file=sys.stderr)
+        return 1
+    # Resync Cargo.lock after the stage-version rewrite; fail loudly here rather
+    # than as a confusing publish error later.
+    if stage_version:
+        r = subprocess.run(["cargo", "update", "--workspace"], cwd=root, env=env)
+        if r.returncode != 0:
+            print("::error::cargo update --workspace failed after the stage-version rewrite", file=sys.stderr)
+            return 1
+
+    # Fail-soft loop: a failed crate skips its dependents but not the rest.
+    # Machine-readable lines consumed by the workflow:
+    #   STAGED_CRATE=            on the registry (this run or earlier)
+    #   FAILED_CRATE=    failed, or skipped due to a failed dep
+    published = already = 0
+    staged: list[str] = []
+    failed: dict[str, str] = {}
+    deps_of = {
+        n: {d["name"] for d in pkgs[n]["dependencies"]
+            if d["name"] in pkgs and d["name"] != n and d.get("kind") != "dev"}
+        for n in order
+    }
+    for name in order:
+        p = pkgs[name]
+        manifest, version = p["manifest_path"], p["version"]
+        print(f"=== {name} {version} ===")
+        bad_deps = sorted(deps_of[name] & set(failed))
+        if bad_deps:
+            failed[name] = f"dependency {','.join(bad_deps)} failed"
+            print(f"::error::{name} {version} skipped: {failed[name]}", file=sys.stderr)
+            print(f"FAILED_CRATE={name} {failed[name]}", flush=True)
+            continue
+        try:
+            if crate_exists(raw_base, name, version, token):
+                print(f"  -> {name} {version} already on the registry (skip)")
+                already += 1
+                staged.append(name)
+                print(f"STAGED_CRATE={name}", flush=True)  # captured by the staging job
+                continue
+            if subprocess.run(["cargo", "check", "--manifest-path", manifest], cwd=root, env=env).returncode != 0:
+                raise RuntimeError(f"cargo check failed for {manifest}")
+            status = publish(manifest, args.registry, env)
+        except RuntimeError as e:
+            failed[name] = str(e)
+            print(f"::error::{name} {version} not staged: {e}", file=sys.stderr)
+            print(f"FAILED_CRATE={name} {e}", flush=True)
+            continue
+        if status == "published":
+            published += 1
+        else:
+            already += 1
+        staged.append(name)
+        print(f"STAGED_CRATE={name}", flush=True)  # captured by the staging job
+
+    print(f"Done: {len(order)} crates ({published} published, {already} already present, "
+          f"{len(staged)} staged, {len(failed)} failed).")
+    print(f"STAGED_CRATES={','.join(staged)}")
+    if failed:
+        print(f"FAILED_CRATES={','.join(failed)}")
+        print(f"::error::{len(failed)} crate(s) not staged: {', '.join(failed)}", file=sys.stderr)
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())
diff --git a/.github/workflows/on-demand-release.yml b/.github/workflows/on-demand-release.yml
new file mode 100644
index 000000000000..8802b3cd99d2
--- /dev/null
+++ b/.github/workflows/on-demand-release.yml
@@ -0,0 +1,527 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+# On-demand release: branch from main / a release branch / a main commit SHA,
+# set the release version, commit it to a new release/ branch (which
+# triggers post-merge CI to build the images), then stage all artifacts and
+# trigger the GitLab release-automation pipeline via release.yml.
+
+name: On-Demand Release
+
+# workflow_dispatch is capped at 10 inputs by GitHub — this block is at 10;
+# fold any new knob into an existing input.
+on:
+  # TEMPORARY(test): push events have no inputs, so the fallbacks below run the
+  # test flow as release_version 1.4.0 on the existing release/1.4.0 branch, rc10,
+  # all artifacts (containers/wheels/crates/helm=all). REVERT before merge.
+  push:
+    branches:
+      # copy-pr-bot's reviewed-copy branch. Velonix job admission only admits
+      # push on main, release/*, and pull-request/N, so the test loop runs off
+      # the PR copy: comment `/ok to test ` to refresh it. Same filter
+      # fern-docs.yml uses.
+      - pull-request/[0-9]+
+  workflow_dispatch:
+    inputs:
+      source_ref:
+        description: "Branch to fork from: 'main', a 'release/x.y.z' branch, or a 40-char main commit SHA."
+        required: false
+        type: string
+        default: main
+      release_version:
+        description: 'Version to STAMP into the tree, X.Y.Z[.devN|.postN] (PEP 440; .devN = pre-release, .postN = post-release; e.g. 0.0.0, 0.0.0.dev3). Not the branch name.'
+        required: true
+        type: string
+      release_branch:
+        description: 'Branch to create (default release/). For a throwaway run use e.g. release/0.0.0test — it still matches release/*.*.* so post-merge + release-branch checks run, but the publish jobs skip.'
+        required: false
+        type: string
+        default: ''
+      rc_round:
+        description: 'Internal staging RC:  or . (e.g. 2 or 2.1 -> staging versions X.Y.Z-rc.2 / X.Y.Z-rc.2.1, a numerically-ordered SemVer pre-release; also the wheel rc subpath). Re-stages of the same version need a distinct value (immutable registries). NO GitHub tag is created. Default 10.'
+        required: false
+        type: string
+        default: '10'
+      containers:
+        description: 'Containers to release: all | none | csv of vllm[:runtime|:efa],sglang[...],trtllm[...],frontend,operator,planner,snapshot.'
+        required: false
+        type: string
+        default: all
+      wheels:
+        description: 'Wheels to release: all | none | csv of ai-dynamo,ai-dynamo-runtime,kvbm.'
+        required: false
+        type: string
+        default: all
+      crates:
+        description: 'Crates to release: all | none | csv of crate names (must be dependency-closed).'
+        required: false
+        type: string
+        default: all
+      helm:
+        description: 'Helm charts to release: all | none | csv of platform,snapshot.'
+        required: false
+        type: string
+        default: all
+      dry_run:
+        description: 'Create the branch + version bump only; skip staging and the GitLab trigger.'
+        required: false
+        type: boolean
+        default: false
+      skip_gitlab_pipeline:
+        description: 'Skip the GitLab release automation pipeline trigger. Emergency use only.'
+        required: false
+        type: boolean
+        default: false
+
+permissions:
+  contents: read
+
+jobs:
+  # ============================================================================
+  # CREATE RELEASE BRANCH + VERSION BUMP
+  # Pushes with a dedicated token (not GITHUB_TOKEN) so the push triggers
+  # post-merge CI, which builds the release images for the bump commit.
+  # ============================================================================
+  prepare-branch:
+    name: Create release branch + version bump
+    runs-on: prod-default-small-v2
+    environment: automated-release
+    permissions:
+      contents: read
+    outputs:
+      branch: ${{ steps.bump.outputs.branch }}
+      bump_sha: ${{ steps.bump.outputs.bump_sha }}
+      is_test: ${{ steps.bump.outputs.is_test }}
+      rc_number: ${{ steps.rc.outputs.rc_number }}
+      containers: ${{ steps.resolve.outputs.containers }}
+      wheels: ${{ steps.resolve.outputs.wheels }}
+      crates: ${{ steps.resolve.outputs.crates }}
+      helm: ${{ steps.resolve.outputs.helm }}
+    steps:
+      - name: Validate inputs
+        env:
+          # TEMPORARY(test): the || fallbacks cover inputless push events. REVERT
+          # to bare inputs.* before merge.
+          RELEASE_VERSION: ${{ inputs.release_version || '1.4.0' }}
+          SOURCE_REF: ${{ inputs.source_ref || 'main' }}
+          RELEASE_BRANCH: ${{ inputs.release_branch || (github.event_name == 'push' && 'release/1.4.0' || '') }}
+          RC_ROUND: ${{ inputs.rc_round || '10' }}
+        run: |
+          set -euo pipefail
+          if [[ ! "${RELEASE_VERSION}" =~ ^[0-9]+\.[0-9]+\.[0-9]+(\.(dev|post)[0-9]+)?$ ]]; then
+            echo "::error::release_version must be X.Y.Z[.devN|.postN] (got '${RELEASE_VERSION}')"
+            exit 1
+          fi
+          # Validated before the branch cut — a typo caught after the push wastes hours of CI.
+          if ! [[ "${RC_ROUND}" =~ ^[0-9]+(\.[0-9]+)?$ ]]; then
+            echo "::error::rc_round must be '' or '.' (non-negative integers; got '${RC_ROUND}')"
+            exit 1
+          fi
+          if [[ -n "${RELEASE_BRANCH}" && ! "${RELEASE_BRANCH}" =~ ^release/[A-Za-z0-9._-]+$ ]]; then
+            echo "::error::release_branch must look like release/ (got '${RELEASE_BRANCH}')"
+            exit 1
+          fi
+          if [[ ! "${SOURCE_REF}" =~ ^(main|release/[0-9]+\.[0-9]+\.[0-9]+(\.(dev|post)[0-9]+)?|[0-9a-f]{40})$ ]]; then
+            echo "::error::source_ref must be 'main', a release/x.y.z branch, or a 40-char SHA (got '${SOURCE_REF}')"
+            exit 1
+          fi
+
+      # Normalize the per-type fields into canonical subset strings (all|none|csv
+      # of normalized tokens) consumed by the bump, release.yml, and the GitLab
+      # trigger. The container grammar expands [:runtime|:efa] (bare fw = both)
+      # and validates every token; unknown tokens fail fast.
+      - name: Resolve artifact selection
+        id: resolve
+        env:
+          RELEASE_VERSION: ${{ inputs.release_version }}
+          CONTAINERS: ${{ inputs.containers || 'all' }}
+          WHEELS: ${{ inputs.wheels || 'all' }}
+          # Stage ALL publishable workspace crates ('all' = the max). A hardcoded
+          # "public-only" list is infeasible: dynamo-llm depends on dynamo-bench
+          # (lib/llm/Cargo.toml), so excluding bench breaks the dependency closure, and
+          # the publishable graph differs across release branches (e.g. release/1.3.0 has
+          # no dynamo-parsers/protocols/renderer/tokenizers). stage_crates.py publishes
+          # the full publishable set for whichever branch it runs on, topo-ordered, with
+          # dynamo-mocker's aiconfigurator-core git dep stripped at publish (strip_git_deps);
+          # the expect-version gate guards against any crate not at the release version.
+          CRATES: ${{ inputs.crates || 'all' }}
+          HELM: ${{ inputs.helm || 'all' }}
+        run: |
+          set -euo pipefail
+          python3 <<'PY' > "${RUNNER_TEMP}/selection.env"
+          import os, re, sys
+
+          FRAMEWORKS = {"vllm", "sglang", "trtllm"}
+          VARIANTS = {"runtime", "efa"}
+          COMPONENTS = {"frontend", "operator", "planner", "snapshot"}
+          WHEEL_TOKENS = {"ai-dynamo", "ai-dynamo-runtime", "kvbm"}
+          HELM_TOKENS = {"platform", "snapshot"}
+
+          def fail(msg):
+              print(f"::error::{msg}", file=sys.stderr)
+              sys.exit(1)
+
+          def dedupe(xs):
+              seen = []
+              for x in xs:
+                  if x not in seen:
+                      seen.append(x)
+              return seen
+
+          def norm_containers(spec):
+              spec = (spec or "all").strip()
+              if spec == "all":
+                  return "all"
+              if spec in ("", "none"):
+                  return "none"
+              out = []
+              for tok in (t.strip() for t in spec.split(",") if t.strip()):
+                  if tok in COMPONENTS:
+                      out.append(tok)
+                      continue
+                  fw, _, var = tok.partition(":")
+                  if fw not in FRAMEWORKS:
+                      fail(f"unknown container token '{tok}'")
+                  if var == "":
+                      out += [f"{fw}-runtime", f"{fw}-efa"]
+                  elif var in VARIANTS:
+                      out.append(f"{fw}-{var}")
+                  else:
+                      fail(f"unknown variant in '{tok}' (use runtime|efa)")
+              return ",".join(dedupe(out))
+
+          def norm_simple(spec, universe, label):
+              spec = (spec or "all").strip()
+              if spec == "all":
+                  return "all"
+              if spec in ("", "none"):
+                  return "none"
+              sel = [t.strip() for t in spec.split(",") if t.strip()]
+              bad = [t for t in sel if t not in universe]
+              if bad:
+                  fail(f"unknown {label} token(s) {bad}; valid: {sorted(universe)}")
+              sel = dedupe(sel)
+              # ai-dynamo hard-pins ai-dynamo-runtime== in pyproject, so
+              # shipping it alone publishes a wheel whose only dependency does not
+              # exist at that version.
+              if label == "wheel" and "ai-dynamo" in sel and "ai-dynamo-runtime" not in sel:
+                  fail("wheels=ai-dynamo requires ai-dynamo-runtime (ai-dynamo pins "
+                       "ai-dynamo-runtime==); add it or use wheels=all")
+              return ",".join(sel)
+
+          def norm_crates(spec):
+              spec = (spec or "all").strip()
+              if spec == "all":
+                  return "all"
+              if spec in ("", "none"):
+                  return "none"
+              # Closure is checked by stage_crates.py; the charset gate keeps
+              # arbitrary text out of $GITHUB_OUTPUT and the GitLab variables.
+              sel = dedupe(t.strip() for t in spec.split(",") if t.strip())
+              bad = [t for t in sel if not re.fullmatch(r"[A-Za-z0-9_-]+", t)]
+              if bad:
+                  fail(f"invalid crate token(s) {bad}; crate names are [A-Za-z0-9_-]+")
+              return ",".join(sel)
+
+          c = norm_containers(os.environ.get("CONTAINERS", "all"))
+          w = norm_simple(os.environ.get("WHEELS", "all"), WHEEL_TOKENS, "wheel")
+          cr = norm_crates(os.environ.get("CRATES", "all"))
+          h = norm_simple(os.environ.get("HELM", "all"), HELM_TOKENS, "helm")
+
+          # SemVer has no post-release form: .postN would sort before X.Y.Z or
+          # collide on crates.io — post releases ship containers/wheels only.
+          if re.search(r"\.post[0-9]+$", os.environ.get("RELEASE_VERSION", "")):
+              if cr != "none" or h != "none":
+                  fail("post releases (X.Y.Z.postN) cannot ship crates or helm charts "
+                       "(no SemVer post-release form); re-run with crates=none and helm=none")
+
+          print(f"containers={c}")
+          print(f"wheels={w}")
+          print(f"crates={cr}")
+          print(f"helm={h}")
+          PY
+          echo "Resolved selection:"
+          cat "${RUNNER_TEMP}/selection.env"
+          cat "${RUNNER_TEMP}/selection.env" >> "$GITHUB_OUTPUT"
+
+      - name: Mint app token
+        id: app-token
+        uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1  # v3.2.0
+        with:
+          app-id: ${{ secrets.RELEASE_APP_ID }}
+          private-key: ${{ secrets.RELEASE_APP_PRIVATE_KEY }}
+
+      - name: Checkout source
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        with:
+          # TEMPORARY(test): || 'main' covers inputless push events. REVERT.
+          ref: ${{ inputs.source_ref || 'main' }}
+          fetch-depth: 0
+          token: ${{ steps.app-token.outputs.token }}
+
+      - name: Set version and create branch
+        id: bump
+        env:
+          # TEMPORARY(test): push fallbacks. REVERT to bare inputs.* before merge.
+          RELEASE_VERSION: ${{ inputs.release_version || '1.4.0' }}
+          RELEASE_BRANCH: ${{ inputs.release_branch || (github.event_name == 'push' && 'release/1.4.0' || '') }}
+          APP_SLUG: ${{ steps.app-token.outputs.app-slug }}
+          GH_TOKEN: ${{ steps.app-token.outputs.token }}
+          CONTAINERS: ${{ steps.resolve.outputs.containers }}
+          HELM: ${{ steps.resolve.outputs.helm }}
+          # Not passed to the bump (the version stamp is unified) — recorded in the
+          # release commit message so the branch states what it actually ships.
+          WHEELS: ${{ steps.resolve.outputs.wheels }}
+          CRATES: ${{ steps.resolve.outputs.crates }}
+        run: |
+          set -euo pipefail
+
+          BRANCH="${RELEASE_BRANCH:-release/${RELEASE_VERSION}}"
+          # A pure release/X.Y.Z[.devN|.postN] branch is a real release; anything
+          # else (e.g. release/0.0.0test) is a throwaway test branch.
+          if [[ "${BRANCH}" =~ ^release/[0-9]+\.[0-9]+\.[0-9]+(\.(dev|post)[0-9]+)?$ ]]; then
+            IS_TEST=false
+          else
+            IS_TEST=true
+          fi
+
+          # Fetch the version-bump script from THIS workflow's commit (GITHUB_SHA);
+          # source_ref may predate --set-version.
+          git show "${GITHUB_SHA}:.github/scripts/apply_dev_version.py" > "${RUNNER_TEMP}/apply_dev_version.py"
+
+          # Base the release branch on the existing remote branch when it exists, so a
+          # re-run only adds the diff (e.g. a newly-added script step lands on the branch
+          # that was cut before it existed); otherwise base it on the checked-out
+          # source_ref. We never delete/clobber — the diff (if any) is committed on top.
+          if git ls-remote --exit-code --heads origin "${BRANCH}" >/dev/null 2>&1; then
+            git fetch origin "${BRANCH}"
+            git checkout -B "${BRANCH}" FETCH_HEAD
+            EXISTED=1
+          else
+            git checkout -b "${BRANCH}"
+            EXISTED=0
+          fi
+
+          # Set the absolute version, then sync the lockfile (below).
+          # --image-refs points the my-registry/:my-tag placeholders in docs, examples
+          # and deploy manifests at the GA registry + release version, but ONLY for the
+          # images in --containers: a partial release must never publish a branch that
+          # tells users to pull an image it didn't build. wheels/crates are deliberately
+          # NOT passed — the package version stamp is unified (the containers embed
+          # wheels built from this tree, so it must move regardless of the selection).
+          python3 "${RUNNER_TEMP}/apply_dev_version.py" "" . --set-version "${RELEASE_VERSION}" \
+            --containers "${CONTAINERS}" --helm "${HELM}" --image-refs
+
+          if ! command -v cargo >/dev/null 2>&1; then
+            curl --proto '=https' --tlsv1.2 --retry 5 --retry-delay 2 -sSf https://sh.rustup.rs \
+              | sh -s -- -y --profile minimal --default-toolchain stable
+            source "${HOME}/.cargo/env"
+          fi
+          # Keep Cargo.lock in sync so post-merge `cargo test --locked` passes.
+          cargo update --workspace
+
+          APP_USER_ID=$(curl -fsSL --retry 3 \
+            -H "Authorization: Bearer ${GH_TOKEN}" \
+            -H "Accept: application/vnd.github+json" \
+            "https://api.github.com/users/${APP_SLUG}%5Bbot%5D" \
+            | python3 -c "import sys,json;print(json.load(sys.stdin)['id'])")
+          git config user.name "${APP_SLUG}[bot]"
+          git config user.email "${APP_USER_ID}+${APP_SLUG}[bot]@users.noreply.github.com"
+          # Commit + push only when the scripts produced a diff vs the branch's current
+          # state. No diff on an existing branch => reuse (no push); no diff on a brand-
+          # new branch => still record a release commit so the push fires post-merge CI.
+          # Record the artifact selection in the commit body: the version stamp is
+          # unified, so the diff alone cannot tell a reader which artifacts this
+          # release actually ships.
+          SELECTION_MSG="containers=${CONTAINERS} wheels=${WHEELS} crates=${CRATES} helm=${HELM}"
+          git add -A
+          if git diff --cached --quiet; then
+            if [ "${EXISTED}" = "1" ]; then
+              echo "${BRANCH} already matches the release scripts; nothing to commit (reuse, no push)."
+            else
+              git commit -s --allow-empty -m "release: set version ${RELEASE_VERSION}" -m "${SELECTION_MSG}"
+              git push origin "${BRANCH}"
+            fi
+          else
+            git commit -s -m "release: set version ${RELEASE_VERSION}" -m "${SELECTION_MSG}"
+            git push origin "${BRANCH}"
+          fi
+
+          echo "branch=${BRANCH}" >> "$GITHUB_OUTPUT"
+          echo "bump_sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
+          echo "is_test=${IS_TEST}" >> "$GITHUB_OUTPUT"
+          echo "Prepared ${BRANCH} @ $(git rev-parse HEAD) (is_test=${IS_TEST}, existed=${EXISTED})"
+
+      - name: Resolve internal staging rc
+        id: rc
+        # A dry run sets up the branch/bump only — don't consume an rc.
+        if: ${{ !inputs.dry_run }}
+        env:
+          RC_ROUND: ${{ inputs.rc_round || '10' }}
+        run: |
+          set -euo pipefail
+          # Internal staging registries (crates/helm/wheels) are immutable, so each
+          # (re-)stage of the same version needs a distinct rc. We take the rc as an
+          # INPUT (no GitHub tag is created): '' or '.'.
+          # Downstream builds X.Y.Z-rc.[.] — a SemVer pre-release whose
+          # dot-separated numeric identifiers order correctly (rc.2.9 < rc.2.10 <
+          # rc.10.0). GA promotion (GitLab) still uses the clean version.
+          if ! [[ "${RC_ROUND}" =~ ^[0-9]+(\.[0-9]+)?$ ]]; then
+            echo "::error::rc_round must be '' or '.' (non-negative integers; got '${RC_ROUND}')"
+            exit 1
+          fi
+          echo "rc_number=${RC_ROUND}" >> "$GITHUB_OUTPUT"
+          echo "Internal staging rc=${RC_ROUND} (no GitHub tag created)"
+
+  # ============================================================================
+  # WAIT FOR POST-MERGE BUILDS (not tests)
+  # The branch push above triggers post-merge-ci.yml, which builds + pushes the
+  # release images to ECR for the bump commit. We wait only for the ECR images the
+  # SELECTED artifacts consume, so we proceed as soon as those builds finish —
+  # WITHOUT waiting for post-merge tests/compliance/deploy. Tests don't push images,
+  # and the bump SHA is unique, so a present tag is unambiguously this run's build
+  # output. Runs on an ECR-capable runner (same IRSA path as stage-wheels).
+  # ============================================================================
+  await-post-merge:
+    name: Wait for post-merge builds
+    needs: [prepare-branch]
+    # Test branches (is_test) stop at the branch cut: post-merge + release-branch
+    # checks still run on them, but nothing is staged or published.
+    if: ${{ !inputs.dry_run && needs.prepare-branch.outputs.is_test != 'true' }}
+    runs-on: prod-tester-amd-v1
+    environment: automated-release
+    permissions:
+      contents: read
+      actions: read
+    steps:
+      - name: Wait for the selected artifacts' build images in ECR
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          REPO: ${{ github.repository }}
+          SHA: ${{ needs.prepare-branch.outputs.bump_sha }}
+          AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
+          CONTAINERS: ${{ needs.prepare-branch.outputs.containers }}
+          WHEELS: ${{ needs.prepare-branch.outputs.wheels }}
+          CRATES: ${{ needs.prepare-branch.outputs.crates }}
+        run: |
+          set -euo pipefail
+          aws sts get-caller-identity >/dev/null || { echo "::error::no AWS credentials on this runner"; exit 1; }
+
+          # ECR tags this release will consume, derived from the selection. The
+          # container token -> source tag map mirrors release.yml's copy step; wheels
+          # come from the dynamo-runtime image, crates from dynamo-runtime-test.
+          CSET="${CONTAINERS:-all}"
+          ctr_selected() {
+            [ "${CSET}" = "all" ] && return 0
+            [ "${CSET}" = "none" ] && return 1
+            case ",${CSET}," in *,"$1",*) return 0 ;; esac
+            return 1
+          }
+          declare -A CTR=(
+            [vllm-runtime]=vllm-runtime     [vllm-efa]=vllm-runtime-efa
+            [sglang-runtime]=sglang-runtime [sglang-efa]=sglang-runtime-efa
+            [trtllm-runtime]=trtllm-runtime [trtllm-efa]=trtllm-runtime-efa
+            [frontend]=dynamo-frontend [operator]=operator [planner]=dynamo-planner [snapshot]=snapshot-agent
+          )
+          declare -a TAGS=()
+          for tok in "${!CTR[@]}"; do
+            if ctr_selected "${tok}"; then TAGS+=("${SHA}-${CTR[$tok]}"); fi
+          done
+          if [ "${WHEELS:-all}" != "none" ]; then TAGS+=("${SHA}-dynamo-runtime"); fi
+          if [ "${CRATES:-all}" != "none" ]; then TAGS+=("${SHA}-dynamo-runtime-test"); fi
+
+          if [ ${#TAGS[@]} -eq 0 ]; then
+            echo "No ECR build artifacts selected (helm-only); nothing to wait for."
+            exit 0
+          fi
+
+          echo "Waiting for post-merge builds to publish ${#TAGS[@]} ECR tag(s): ${TAGS[*]}"
+          # 3h budget (runtime builds alone get 120 min); 15-min fast-fail if
+          # post-merge never started for the bump SHA.
+          start=$(date +%s)
+          deadline=$(( start + 10800 ))
+          start_deadline=$(( start + 900 ))
+          while :; do
+            missing=()
+            for TAG in "${TAGS[@]}"; do
+              if ! aws ecr describe-images --repository-name "ai-dynamo/dynamo" \
+                     --image-ids imageTag="${TAG}" --region "${AWS_DEFAULT_REGION}" >/dev/null 2>&1; then
+                missing+=("${TAG}")
+              fi
+            done
+            if [ ${#missing[@]} -eq 0 ]; then
+              echo "All ${#TAGS[@]} required build images are in ECR; proceeding (not waiting for post-merge tests)."
+              break
+            fi
+            echo "still missing ${#missing[@]}/${#TAGS[@]}: ${missing[*]}"
+
+            # Fast-fail if post-merge finished (or never started) — but only on a
+            # successful API response, so a transient curl/JSON failure can't
+            # masquerade as "post-merge never ran".
+            API_OK=true
+            RUNS=$(curl -fsSL --retry 5 --retry-delay 2 --retry-all-errors \
+              -H "Authorization: Bearer ${GH_TOKEN}" \
+              -H "Accept: application/vnd.github+json" \
+              "https://api.github.com/repos/${REPO}/actions/workflows/post-merge-ci.yml/runs?head_sha=${SHA}&per_page=1") || { API_OK=false; RUNS='{}'; }
+            COUNT=$(echo "${RUNS}" | jq -r '.workflow_runs | length' 2>/dev/null || { API_OK=false; echo 0; })
+            STATUS=$(echo "${RUNS}" | jq -r '.workflow_runs[0].status // "pending"' 2>/dev/null || { API_OK=false; echo pending; })
+            if [ "${API_OK}" = "true" ]; then
+              if [ "${COUNT}" = "0" ] && [ "$(date +%s)" -ge "${start_deadline}" ]; then
+                echo "::error::post-merge CI never started for ${SHA} (branch push may not have triggered it); required image(s) missing: ${missing[*]}"
+                exit 1
+              fi
+              if [ "${COUNT}" != "0" ] && [ "${STATUS}" = "completed" ]; then
+                echo "::error::post-merge CI completed but required image(s) still missing: ${missing[*]}"
+                exit 1
+              fi
+            else
+              echo "workflow-runs API probe failed (transient); skipping fast-fail checks this round"
+            fi
+
+            if [ "$(date +%s)" -ge "${deadline}" ]; then
+              echo "::error::timed out waiting for post-merge build images: ${missing[*]}"
+              exit 1
+            fi
+            sleep 30
+          done
+
+  # ============================================================================
+  # MANUAL APPROVAL GATE before publishing.
+  # ============================================================================
+  release-approval:
+    name: Release approval gate
+    needs: [prepare-branch, await-post-merge]
+    if: ${{ !inputs.dry_run && needs.prepare-branch.outputs.is_test != 'true' }}
+    runs-on: prod-default-small-v2
+    environment: manual-release-approval
+    permissions:
+      contents: read
+    steps:
+      - name: Record approval
+        env:
+          BRANCH: ${{ needs.prepare-branch.outputs.branch }}
+          ACTOR: ${{ github.actor }}
+        run: echo "On-demand release ${BRANCH} approved by ${ACTOR}"
+
+  # ============================================================================
+  # STAGE ARTIFACTS + TRIGGER GITLAB (release.yml in on-demand mode)
+  # ============================================================================
+  invoke-release:
+    name: Stage release
+    needs: [prepare-branch, await-post-merge, release-approval]
+    if: ${{ !inputs.dry_run && needs.prepare-branch.outputs.is_test != 'true' }}
+    uses: ./.github/workflows/release.yml
+    with:
+      commit_sha: ${{ needs.prepare-branch.outputs.bump_sha }}
+      on_demand: true
+      # TEMPORARY(test): || fallback for inputless push events. REVERT.
+      release_version: ${{ inputs.release_version || '1.4.0' }}
+      # Operator-chosen internal-staging rc (immutable registries can't be overwritten).
+      rc_number: ${{ needs.prepare-branch.outputs.rc_number }}
+      # Normalized subsets from resolve-selection.
+      containers: ${{ needs.prepare-branch.outputs.containers }}
+      wheels: ${{ needs.prepare-branch.outputs.wheels }}
+      crates: ${{ needs.prepare-branch.outputs.crates }}
+      helm: ${{ needs.prepare-branch.outputs.helm }}
+      skip_gitlab_pipeline: ${{ inputs.skip_gitlab_pipeline || false }}
+    secrets: inherit
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 676e28184662..05ad0f8ec63c 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -3,6 +3,9 @@
 
 name: Release Pipeline
 
+# workflow_dispatch is capped at 10 inputs by GitHub (this block has 9).
+# on_demand/release_version are workflow_call-only: the sole on-demand entry
+# point is on-demand-release.yml, which owns the manual approval gate.
 on:
   workflow_dispatch:
     inputs:
@@ -29,6 +32,26 @@ on:
         required: false
         type: string
         default: ''
+      containers:
+        description: 'Container subset to publish (all|none|csv of vllm-runtime,vllm-efa,sglang-runtime,sglang-efa,trtllm-runtime,trtllm-efa,frontend,operator,planner,snapshot). Partial rc re-publishes supported.'
+        required: false
+        type: string
+        default: all
+      wheels:
+        description: 'Wheel subset to stage (all|none|csv of ai-dynamo,ai-dynamo-runtime,kvbm).'
+        required: false
+        type: string
+        default: all
+      crates:
+        description: 'Crate subset (all|none|csv of crate names, dependency-closed). Only staged on on-demand runs.'
+        required: false
+        type: string
+        default: all
+      helm:
+        description: 'Helm chart subset to push (all|none|csv of platform,snapshot).'
+        required: false
+        type: string
+        default: all
   workflow_call:
     inputs:
       commit_sha:
@@ -49,6 +72,30 @@ on:
         required: false
         type: string
         default: ''
+      on_demand:
+        required: false
+        type: boolean
+        default: false
+      release_version:
+        required: false
+        type: string
+        default: ''
+      containers:
+        required: false
+        type: string
+        default: all
+      wheels:
+        required: false
+        type: string
+        default: all
+      crates:
+        required: false
+        type: string
+        default: all
+      helm:
+        required: false
+        type: string
+        default: all
       # Nightly-only: the result of THIS nightly run's operator-build /
       # planner-build (needs..result — success/failure/skipped/cancelled).
       # The copy step layers this on top of the crane probe:
@@ -88,6 +135,13 @@ on:
         required: false
         type: string
         default: ''
+    outputs:
+      # The exact NGC docker tag published for this run (nightly: -).
+      # Callers use it to reference what actually shipped instead of recomputing
+      # the date, which can drift across a UTC-midnight boundary.
+      ngc_version_tag:
+        description: 'NGC docker version tag published by this run.'
+        value: ${{ jobs.prepare-release.outputs.ngc_version_tag }}
 
 # Note: workflow_dispatch can only be triggered from release/* branches (or main when nightly=true)
 # This is enforced in the prepare-release job via branch validation. workflow_call paths
@@ -113,10 +167,12 @@ jobs:
       version: ${{ steps.extract.outputs.version }}
       commit_sha: ${{ steps.extract.outputs.commit_sha }}
       nightly: ${{ steps.extract.outputs.nightly }}
+      on_demand: ${{ steps.extract.outputs.on_demand }}
       rc_number: ${{ steps.rc_tag.outputs.rc_number }}
       ngc_version_tag: ${{ steps.rc_tag.outputs.ngc_version_tag }}
       helm_chart_version: ${{ steps.rc_tag.outputs.helm_chart_version }}
       wheel_version: ${{ steps.rc_tag.outputs.wheel_version }}
+      crate_version: ${{ steps.rc_tag.outputs.crate_version }}
     steps:
       - name: Extract version and validate inputs
         id: extract
@@ -124,6 +180,13 @@ jobs:
           COMMIT_SHA: ${{ inputs.commit_sha }}
           BRANCH_NAME: ${{ github.ref_name }}
           NIGHTLY: ${{ inputs.nightly }}
+          ON_DEMAND: ${{ inputs.on_demand }}
+          RELEASE_VERSION: ${{ inputs.release_version }}
+          CONTAINER_SUBSET: ${{ inputs.containers }}
+          WHEEL_SUBSET: ${{ inputs.wheels }}
+          CRATE_SUBSET: ${{ inputs.crates }}
+          HELM_SUBSET: ${{ inputs.helm }}
+          SLACK_THREAD_TS: ${{ inputs.slack_thread_ts }}
         run: |
           set -euo pipefail
 
@@ -132,15 +195,60 @@ jobs:
             exit 1
           fi
 
-          if [[ "${NIGHTLY}" == "true" ]]; then
+          # A typo'd subset token would be silently skipped by every want()
+          # filter and stage nothing — fail fast instead. Crate names are
+          # graph-checked by stage_crates.py; only the charset is gated here.
+          validate_subset() {
+            local LABEL="$1" SPEC="$2" UNIVERSE="$3" TOK
+            if [ "${SPEC}" = "all" ] || [ "${SPEC}" = "none" ] || [ -z "${SPEC}" ]; then
+              return 0
+            fi
+            IFS=',' read -ra TOKS <<<"${SPEC}"
+            for TOK in "${TOKS[@]}"; do
+              case ",${UNIVERSE}," in
+                *,"${TOK}",*) ;;
+                *) echo "Error: unknown ${LABEL} token '${TOK}' (valid: ${UNIVERSE})"; exit 1 ;;
+              esac
+            done
+          }
+          validate_subset containers "${CONTAINER_SUBSET:-all}" "vllm-runtime,vllm-efa,sglang-runtime,sglang-efa,trtllm-runtime,trtllm-efa,frontend,operator,planner,snapshot"
+          validate_subset wheels "${WHEEL_SUBSET:-all}" "ai-dynamo,ai-dynamo-runtime,kvbm"
+          validate_subset helm "${HELM_SUBSET:-all}" "platform,snapshot"
+          if [[ ! "${CRATE_SUBSET:-all}" =~ ^[A-Za-z0-9_,-]*$ ]]; then
+            echo "Error: crate subset may only contain [A-Za-z0-9_-] names separated by commas (got '${CRATE_SUBSET}')"
+            exit 1
+          fi
+          if [[ -n "${SLACK_THREAD_TS}" && ! "${SLACK_THREAD_TS}" =~ ^[0-9]+\.[0-9]+$ ]]; then
+            echo "Error: slack_thread_ts must look like a Slack ts '1234567890.123456' (got '${SLACK_THREAD_TS}')"
+            exit 1
+          fi
+
+          if [[ "${ON_DEMAND}" == "true" ]]; then
+            # Branch + version bump are created/validated by on-demand-release.yml;
+            # the version is passed explicitly rather than derived from the ref.
+            if [[ ! "${RELEASE_VERSION}" =~ ^[0-9]+\.[0-9]+\.[0-9]+(\.(dev|post)[0-9]+)?$ ]]; then
+              echo "Error: on_demand=true requires release_version X.Y.Z[.devN|.postN] (got: '${RELEASE_VERSION}')"
+              exit 1
+            fi
+            # SemVer has no post-release form: .postN crates/helm would sort
+            # BEFORE X.Y.Z or collide with it on crates.io. Containers/wheels are fine.
+            if [[ "${RELEASE_VERSION}" =~ \.post[0-9]+$ ]]; then
+              if [ "${CRATE_SUBSET:-all}" != "none" ] || [ "${HELM_SUBSET:-all}" != "none" ]; then
+                echo "Error: post releases (X.Y.Z.postN) cannot ship crates or helm charts; pass crates=none and helm=none"
+                exit 1
+              fi
+            fi
+            VERSION="${RELEASE_VERSION}"
+          elif [[ "${NIGHTLY}" == "true" ]]; then
             if [[ "${BRANCH_NAME}" != "main" ]]; then
               echo "Error: nightly=true requires branch=main (got: ${BRANCH_NAME})"
               exit 1
             fi
             VERSION="nightly"
           else
-            if [[ ! "$BRANCH_NAME" =~ ^release/[0-9]+\.[0-9]+\.[0-9]+ ]]; then
-              echo "Error: workflow_dispatch must be triggered from a release/* branch (or main with nightly=true)"
+            # Fully anchored — a prefix-only match would accept arbitrary trailing text.
+            if [[ ! "$BRANCH_NAME" =~ ^release/[0-9]+\.[0-9]+\.[0-9]+(\.(dev|post)[0-9]+)?$ ]]; then
+              echo "Error: workflow_dispatch must be triggered from a release/X.Y.Z[.devN|.postN] branch (or main with nightly=true)"
               echo "Current branch: $BRANCH_NAME"
               exit 1
             fi
@@ -150,6 +258,7 @@ jobs:
           echo "version=${VERSION}" >> $GITHUB_OUTPUT
           echo "commit_sha=${COMMIT_SHA}" >> $GITHUB_OUTPUT
           echo "nightly=${NIGHTLY}" >> $GITHUB_OUTPUT
+          echo "on_demand=${ON_DEMAND}" >> $GITHUB_OUTPUT
           echo "Detected version: ${VERSION}"
           echo "Source commit SHA: ${COMMIT_SHA}"
           echo "Nightly: ${NIGHTLY}"
@@ -173,13 +282,16 @@ jobs:
           VERSION: ${{ steps.extract.outputs.version }}
           COMMIT_SHA: ${{ steps.extract.outputs.commit_sha }}
           NIGHTLY: ${{ steps.extract.outputs.nightly }}
+          ON_DEMAND: ${{ steps.extract.outputs.on_demand }}
         run: |
           set -euo pipefail
 
           # Base version pinned in pyproject.toml at the source SHA. Same value
-          # cargo and maturin stamp into the wheel filename / METADATA.
+          # cargo and maturin stamp into the wheel filename / METADATA. The
+          # optional .devN (pre-release) / .postN (post-release) tail covers dev
+          # builds and post-release respins.
           BASE_VERSION=$(awk '/^version = / {gsub(/"/, "", $3); print $3; exit}' pyproject.toml)
-          if [[ ! "${BASE_VERSION}" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+          if [[ ! "${BASE_VERSION}" =~ ^[0-9]+\.[0-9]+\.[0-9]+(\.(dev|post)[0-9]+)?$ ]]; then
             echo "Error: cannot parse base version from pyproject.toml (got '${BASE_VERSION}')"
             exit 1
           fi
@@ -206,6 +318,44 @@ jobs:
             exit 0
           fi
 
+          if [[ "${ON_DEMAND}" == "true" ]]; then
+            # Internal staging carries an rc disambiguator: the crate/helm registries
+            # are immutable, so each re-stage of the same version needs a distinct
+            # internal version. GA stays clean — the GitLab trigger forwards
+            # RELEASE_VERSION=${VERSION} + RC_NUMBER and promotes the rc-staged
+            # artifacts to the clean version.
+            #   NGC tag (docker string): ${VERSION}-rcN     (e.g. 0.1.0-rc0, 0.1.0.dev3-rc0)
+            #   helm + crate (SemVer):   ${SEMVER_BASE}-rcN  where .devN/.postN become a
+            #                            -devN/-postN pre-release (e.g. 0.1.0-rc0, 0.1.0-post3-rc0)
+            #   wheel: clean version, disambiguated by the rcN upload subpath.
+            # Passed through from on-demand-release.yml's rc_number INPUT (default rc0;
+            # NO GitHub tag is created). Require it (don't default to 0) so a broken
+            # hand-off fails loud instead of silently colliding on rc0.
+            RC_NUM="${INPUT_RC_NUMBER}"
+            # RC_NUM is  or . (dotted sub-increment). Both form a
+            # valid, numerically-ordered SemVer pre-release when emitted as ...-rc.
+            # (e.g. 1.3.0-rc.2 or 1.3.0-rc.2.1). The wheel subpath uses the glued rc
+            # (a path segment, not a version) — no SemVer/PEP440 constraint there.
+            if ! [[ "${RC_NUM}" =~ ^[0-9]+(\.[0-9]+)?$ ]]; then
+              echo "Error: on_demand requires rc_number as  or . (got '${RC_NUM}')"
+              exit 1
+            fi
+            # Tree at COMMIT_SHA must be stamped with the requested version, or
+            # images/charts get tagged VERSION while their wheels carry BASE_VERSION.
+            if [ "${BASE_VERSION}" != "${VERSION}" ]; then
+              echo "Error: release_version '${VERSION}' does not match pyproject.toml version '${BASE_VERSION}' at ${COMMIT_SHA} (wrong commit_sha, or the version bump never landed)"
+              exit 1
+            fi
+            SEMVER_BASE=$(echo "${VERSION}" | sed -E 's/^([0-9]+\.[0-9]+\.[0-9]+)\.(.+)/\1-\2/')
+            echo "rc_number=${RC_NUM}" >> $GITHUB_OUTPUT
+            echo "ngc_version_tag=${VERSION}-rc.${RC_NUM}" >> $GITHUB_OUTPUT
+            echo "helm_chart_version=${SEMVER_BASE}-rc.${RC_NUM}" >> $GITHUB_OUTPUT
+            echo "wheel_version=${BASE_VERSION}" >> $GITHUB_OUTPUT
+            echo "crate_version=${SEMVER_BASE}-rc.${RC_NUM}" >> $GITHUB_OUTPUT
+            echo "On-demand internal staging (rc.${RC_NUM}): NGC ${VERSION}-rc.${RC_NUM}, helm ${SEMVER_BASE}-rc.${RC_NUM}, wheel ${BASE_VERSION} @ rc${RC_NUM}, crate ${SEMVER_BASE}-rc.${RC_NUM}"
+            exit 0
+          fi
+
           if [ -n "${INPUT_RC_NUMBER}" ]; then
             if ! [[ "${INPUT_RC_NUMBER}" =~ ^[0-9]+$ ]]; then
               echo "Error: rc_number must be a non-negative integer (got: ${INPUT_RC_NUMBER})"
@@ -253,18 +403,31 @@ jobs:
   release-publish:
     name: Publish to NGC
     needs: [prepare-release]
-    if: needs.prepare-release.result == 'success'
+    # Skip only when neither containers nor helm are selected; nightly/rc default
+    # to 'all' for both, so they always run unless explicitly narrowed.
+    if: ${{ needs.prepare-release.result == 'success' && !(inputs.containers == 'none' && inputs.helm == 'none') }}
     runs-on: prod-builder-amd-v1 # TODO: needs to identify the correct runner here, definitely should not be prod-builder-amd-v1
     environment: automated-release
     outputs:
+      # Comma-separated NGC image refs (repo:tag, no registry prefix) actually copied
+      # to NGC — primary version tags + -efa variants only (the -cuda13 digest aliases
+      # and floating :nightly tags are dropped: same image, not worth a second scan).
+      # Forwarded to GitLab as STAGED_CONTAINERS for nspect registration.
+      staged_containers: ${{ steps.copy_images.outputs.staged_containers }}
+      # Comma-separated chart:version pairs actually pushed to the NGC helm
+      # registry — helm_push covers rc/on-demand runs, nightly_helm_push covers
+      # nightly (the two steps are mutually exclusive on env.NIGHTLY, so exactly
+      # one contributes). Forwarded to GitLab for registration at the true
+      # chart version.
+      staged_helm: ${{ steps.helm_push.outputs.staged_helm || steps.nightly_helm_push.outputs.staged_helm }}
       failed_copy_count: ${{ steps.copy_images.outputs.failed_count }}
-      # Nightly only: chart:version pairs the helm step pushed (empty on rc
-      # runs); forwarded to GitLab for registration at the true chart version.
-      staged_helm: ${{ steps.nightly_helm_push.outputs.staged_helm }}
     env:
       VERSION: ${{ needs.prepare-release.outputs.version }}
       COMMIT_SHA: ${{ needs.prepare-release.outputs.commit_sha }}
       NIGHTLY: ${{ needs.prepare-release.outputs.nightly }}
+      ON_DEMAND: ${{ needs.prepare-release.outputs.on_demand }}
+      CONTAINER_SUBSET: ${{ inputs.containers }}
+      HELM_SUBSET: ${{ inputs.helm }}
       OPERATOR_BUILD_RESULT: ${{ inputs.operator_build_result }}
       PLANNER_BUILD_RESULT: ${{ inputs.planner_build_result }}
       FRONTEND_BUILD_RESULT: ${{ inputs.frontend_build_result }}
@@ -302,6 +465,76 @@ jobs:
           echo "${NGC_TOKEN}" | docker login nvcr.io -u "${NGC_USERNAME}" --password-stdin
           echo "${NGC_TOKEN}" | crane auth login nvcr.io -u "${NGC_USERNAME}" --password-stdin
 
+      - name: Inventory selected container images in ECR
+        run: |
+          set -uo pipefail
+          # Report-only inventory of which selected container images are in ECR. Never fails.
+          # Mirrors the copy step's subset gating, nightly rules, and source tags.
+          NIGHTLY_TAG_SUFFIX=""
+          [ "${NIGHTLY}" = "true" ] && NIGHTLY_TAG_SUFFIX="-nightly"
+
+          SUBSET="${CONTAINER_SUBSET:-all}"
+          want() {
+            [ "${SUBSET}" = "all" ] && return 0
+            case ",${SUBSET}," in *,"$1",*) return 0 ;; esac
+            return 1
+          }
+
+          # token|ECR source tag — must match the "Copy images to NGC" sources.
+          ENTRIES=(
+            "vllm-runtime|${COMMIT_SHA}-vllm-runtime${NIGHTLY_TAG_SUFFIX}"
+            "sglang-runtime|${COMMIT_SHA}-sglang-runtime${NIGHTLY_TAG_SUFFIX}"
+            "trtllm-runtime|${COMMIT_SHA}-trtllm-runtime${NIGHTLY_TAG_SUFFIX}"
+          )
+          if [ "${NIGHTLY}" != "true" ]; then
+            ENTRIES+=(
+              "vllm-efa|${COMMIT_SHA}-vllm-runtime-efa"
+              "sglang-efa|${COMMIT_SHA}-sglang-runtime-efa"
+              "trtllm-efa|${COMMIT_SHA}-trtllm-runtime-efa"
+              "frontend|${COMMIT_SHA}-dynamo-frontend"
+              "operator|${COMMIT_SHA}-operator"
+              "planner|${COMMIT_SHA}-dynamo-planner"
+              "snapshot|${COMMIT_SHA}-snapshot-agent"
+            )
+          fi
+
+          {
+            echo "### Container image availability (ECR)"
+            echo ""
+            echo "Selection: \`${SUBSET}\`"
+            echo ""
+            echo "| Token | ECR tag | Status |"
+            echo "|-------|---------|--------|"
+          } >> "${GITHUB_STEP_SUMMARY}"
+
+          avail=0; missing=0; selected=0
+          echo "=== Container image availability (selection: ${SUBSET}) ==="
+          for E in "${ENTRIES[@]}"; do
+            IFS='|' read -r TOKEN TAG <<<"${E}"
+            want "${TOKEN}" || continue
+            selected=$((selected+1))
+            REF="${ECR_REGISTRY}/${ECR_REPOSITORY}:${TAG}"
+            if crane manifest "${REF}" >/dev/null 2>&1; then
+              echo "  available: ${TOKEN} (${TAG})"
+              echo "| ${TOKEN} | \`${TAG}\` | available |" >> "${GITHUB_STEP_SUMMARY}"
+              avail=$((avail+1))
+            else
+              echo "  MISSING:   ${TOKEN} (${TAG})"
+              echo "| ${TOKEN} | \`${TAG}\` | **missing** |" >> "${GITHUB_STEP_SUMMARY}"
+              missing=$((missing+1))
+            fi
+          done
+
+          if [ "${selected}" -eq 0 ]; then
+            echo "No container images selected (subset='${SUBSET}')."
+            echo "_No container images selected._" >> "${GITHUB_STEP_SUMMARY}"
+          fi
+          echo "Summary: ${avail} available, ${missing} missing (of ${selected} selected)."
+          {
+            echo ""
+            echo "**${avail} available, ${missing} missing** (of ${selected} selected)."
+          } >> "${GITHUB_STEP_SUMMARY}"
+
       - name: Copy images to NGC
         id: copy_images
         env:
@@ -333,7 +566,12 @@ jobs:
             local SRC="$1" DST="$2" LABEL="$3"
             echo "----------------------------------------"
             echo "Copying: ${LABEL}"
-            # crane copy preserves multi-arch manifest lists by default (no --platform needed)
+            # crane copy preserves multi-arch manifest lists by default (no --platform needed).
+            # Returns the real copy status — stage_optional() gates its *_STAGED
+            # flags and floating-tag pushes on it. Under `set -e`, BARE call
+            # sites must therefore carry `|| true` (a nonzero return would abort
+            # the whole copy loop on the first failed image); condition-context
+            # calls (if/elif) are exempt by shell rules.
             if crane copy "${SRC}" "${DST}"; then
               echo "  Copied: ${LABEL}"
               SUCCESSFUL_COPIES+=("${LABEL}")
@@ -354,11 +592,22 @@ jobs:
             NGC_NAME_SUFFIX="-nightly"
           fi
 
+          # Copy only the selected container subset; nightly/rc default to the full
+          # set (SUBSET=all). Tokens: -runtime (bare + -cuda13 alias),
+          # -efa, frontend, operator, planner, snapshot.
+          SUBSET="${CONTAINER_SUBSET:-all}"
+          want() {
+            [ "${SUBSET}" = "all" ] && return 0
+            case ",${SUBSET}," in *,"$1",*) return 0 ;; esac
+            return 1
+          }
+
           # ---- CUDA 13 runtime images (vllm, sglang, trtllm) — DEFAULT ----
           echo ""
           echo "=== CUDA 13 Runtime Images (vllm, sglang, trtllm) — default ==="
           CUDA13_FRAMEWORKS=("vllm" "sglang" "trtllm")
           for FRAMEWORK in "${CUDA13_FRAMEWORKS[@]}"; do
+            want "${FRAMEWORK}-runtime" || { echo "skip ${FRAMEWORK}-runtime (not selected)"; continue; }
             if [ "${FRAMEWORK}" = "trtllm" ]; then
               NGC_NAME="tensorrtllm-runtime${NGC_NAME_SUFFIX}"
               STABLE_NAME="tensorrtllm-runtime"
@@ -369,23 +618,23 @@ jobs:
 
             SOURCE="${ECR_REGISTRY}/${ECR_REPOSITORY}:${COMMIT_SHA}-${FRAMEWORK}-runtime${NIGHTLY_TAG_SUFFIX}"
             TARGET="${NGC_REGISTRY}/${NGC_ORG}/ai-dynamo/${NGC_NAME}:${NGC_VERSION_TAG}"
-            copy_image "${SOURCE}" "${TARGET}" "${NGC_NAME}:${NGC_VERSION_TAG}"
+            copy_image "${SOURCE}" "${TARGET}" "${NGC_NAME}:${NGC_VERSION_TAG}" || true
 
             # Explicit `-cuda13` alias on the same digest so consumers can
             # pin to cuda13 specifically. The bare tag above stays as the
             # implicit default and will move to cuda14+ when that flips.
             EXPLICIT_TARGET="${NGC_REGISTRY}/${NGC_ORG}/ai-dynamo/${NGC_NAME}:${NGC_VERSION_TAG}-cuda13"
-            copy_image "${SOURCE}" "${EXPLICIT_TARGET}" "${NGC_NAME}:${NGC_VERSION_TAG}-cuda13"
+            copy_image "${SOURCE}" "${EXPLICIT_TARGET}" "${NGC_NAME}:${NGC_VERSION_TAG}-cuda13" || true
 
             # Mutable `:nightly` floating tag on the stable repo (e.g.
             # vllm-runtime:nightly). cuda13 is the default so no cuda suffix.
             if [[ "${NIGHTLY}" == "true" ]]; then
               MUTABLE_TARGET="${NGC_REGISTRY}/${NGC_ORG}/ai-dynamo/${STABLE_NAME}:nightly"
-              copy_image "${SOURCE}" "${MUTABLE_TARGET}" "${STABLE_NAME}:nightly"
+              copy_image "${SOURCE}" "${MUTABLE_TARGET}" "${STABLE_NAME}:nightly" || true
 
               # Explicit `:nightly-cuda13` floating alias.
               EXPLICIT_MUTABLE_TARGET="${NGC_REGISTRY}/${NGC_ORG}/ai-dynamo/${STABLE_NAME}:nightly-cuda13"
-              copy_image "${SOURCE}" "${EXPLICIT_MUTABLE_TARGET}" "${STABLE_NAME}:nightly-cuda13"
+              copy_image "${SOURCE}" "${EXPLICIT_MUTABLE_TARGET}" "${STABLE_NAME}:nightly-cuda13" || true
             fi
           done
 
@@ -406,47 +655,61 @@ jobs:
             echo "=== EFA Runtime Images ==="
 
             # vllm EFA (CUDA 13 — default; no cuda suffix on ECR or NGC)
-            SOURCE="${ECR_REGISTRY}/${ECR_REPOSITORY}:${COMMIT_SHA}-vllm-runtime-efa"
-            TARGET="${NGC_REGISTRY}/${NGC_ORG}/ai-dynamo/vllm-runtime:${NGC_VERSION_TAG}-efa"
-            copy_image "${SOURCE}" "${TARGET}" "vllm-runtime:${NGC_VERSION_TAG}-efa"
+            if want vllm-efa; then
+              SOURCE="${ECR_REGISTRY}/${ECR_REPOSITORY}:${COMMIT_SHA}-vllm-runtime-efa"
+              TARGET="${NGC_REGISTRY}/${NGC_ORG}/ai-dynamo/vllm-runtime:${NGC_VERSION_TAG}-efa"
+              copy_image "${SOURCE}" "${TARGET}" "vllm-runtime:${NGC_VERSION_TAG}-efa" || true
+            fi
 
             # sglang EFA (CUDA 13 — default; no cuda suffix on ECR or NGC)
-            SOURCE="${ECR_REGISTRY}/${ECR_REPOSITORY}:${COMMIT_SHA}-sglang-runtime-efa"
-            TARGET="${NGC_REGISTRY}/${NGC_ORG}/ai-dynamo/sglang-runtime:${NGC_VERSION_TAG}-efa"
-            copy_image "${SOURCE}" "${TARGET}" "sglang-runtime:${NGC_VERSION_TAG}-efa"
+            if want sglang-efa; then
+              SOURCE="${ECR_REGISTRY}/${ECR_REPOSITORY}:${COMMIT_SHA}-sglang-runtime-efa"
+              TARGET="${NGC_REGISTRY}/${NGC_ORG}/ai-dynamo/sglang-runtime:${NGC_VERSION_TAG}-efa"
+              copy_image "${SOURCE}" "${TARGET}" "sglang-runtime:${NGC_VERSION_TAG}-efa" || true
+            fi
 
             # trtllm EFA (CUDA 13 — default; no cuda suffix on ECR or NGC)
-            SOURCE="${ECR_REGISTRY}/${ECR_REPOSITORY}:${COMMIT_SHA}-trtllm-runtime-efa"
-            TARGET="${NGC_REGISTRY}/${NGC_ORG}/ai-dynamo/tensorrtllm-runtime:${NGC_VERSION_TAG}-efa"
-            copy_image "${SOURCE}" "${TARGET}" "tensorrtllm-runtime:${NGC_VERSION_TAG}-efa"
+            if want trtllm-efa; then
+              SOURCE="${ECR_REGISTRY}/${ECR_REPOSITORY}:${COMMIT_SHA}-trtllm-runtime-efa"
+              TARGET="${NGC_REGISTRY}/${NGC_ORG}/ai-dynamo/tensorrtllm-runtime:${NGC_VERSION_TAG}-efa"
+              copy_image "${SOURCE}" "${TARGET}" "tensorrtllm-runtime:${NGC_VERSION_TAG}-efa" || true
+            fi
 
             # ---- Frontend image (already multi-arch from build-frontend-image workflow) ----
-            echo ""
-            echo "=== Frontend Image ==="
-            SOURCE="${ECR_REGISTRY}/${ECR_REPOSITORY}:${COMMIT_SHA}-frontend"
-            TARGET="${NGC_REGISTRY}/${NGC_ORG}/ai-dynamo/dynamo-frontend:${NGC_VERSION_TAG}"
-            copy_image "${SOURCE}" "${TARGET}" "dynamo-frontend:${NGC_VERSION_TAG}"
+            if want frontend; then
+              echo ""
+              echo "=== Frontend Image ==="
+              SOURCE="${ECR_REGISTRY}/${ECR_REPOSITORY}:${COMMIT_SHA}-dynamo-frontend"
+              TARGET="${NGC_REGISTRY}/${NGC_ORG}/ai-dynamo/dynamo-frontend:${NGC_VERSION_TAG}"
+              copy_image "${SOURCE}" "${TARGET}" "dynamo-frontend:${NGC_VERSION_TAG}" || true
+            fi
 
             # ---- Operator image (multi-arch manifest already built by post-merge operator-build) ----
-            echo ""
-            echo "=== Operator Image ==="
-            OPERATOR_SOURCE="${ECR_REGISTRY}/${ECR_REPOSITORY}:${COMMIT_SHA}-operator"
-            OPERATOR_TARGET="${NGC_REGISTRY}/${NGC_ORG}/ai-dynamo/kubernetes-operator:${NGC_VERSION_TAG}"
-            copy_image "${OPERATOR_SOURCE}" "${OPERATOR_TARGET}" "kubernetes-operator:${NGC_VERSION_TAG}"
+            if want operator; then
+              echo ""
+              echo "=== Operator Image ==="
+              OPERATOR_SOURCE="${ECR_REGISTRY}/${ECR_REPOSITORY}:${COMMIT_SHA}-operator"
+              OPERATOR_TARGET="${NGC_REGISTRY}/${NGC_ORG}/ai-dynamo/kubernetes-operator:${NGC_VERSION_TAG}"
+              copy_image "${OPERATOR_SOURCE}" "${OPERATOR_TARGET}" "kubernetes-operator:${NGC_VERSION_TAG}" || true
+            fi
 
             # ---- Planner image (CPU-only, multi-arch from post-merge planner-pipeline) ----
-            echo ""
-            echo "=== Planner Image ==="
-            SOURCE="${ECR_REGISTRY}/${ECR_REPOSITORY}:${COMMIT_SHA}-dynamo-planner"
-            TARGET="${NGC_REGISTRY}/${NGC_ORG}/ai-dynamo/dynamo-planner:${NGC_VERSION_TAG}"
-            copy_image "${SOURCE}" "${TARGET}" "dynamo-planner:${NGC_VERSION_TAG}"
+            if want planner; then
+              echo ""
+              echo "=== Planner Image ==="
+              SOURCE="${ECR_REGISTRY}/${ECR_REPOSITORY}:${COMMIT_SHA}-dynamo-planner"
+              TARGET="${NGC_REGISTRY}/${NGC_ORG}/ai-dynamo/dynamo-planner:${NGC_VERSION_TAG}"
+              copy_image "${SOURCE}" "${TARGET}" "dynamo-planner:${NGC_VERSION_TAG}" || true
+            fi
 
             # ---- Snapshot image ----
-            echo ""
-            echo "=== Snapshot Image ==="
-            SNAPSHOT_SOURCE="${ECR_REGISTRY}/${ECR_REPOSITORY}:${COMMIT_SHA}-snapshot-agent"
-            SNAPSHOT_TARGET="${NGC_REGISTRY}/${NGC_ORG}/ai-dynamo/snapshot-agent:${NGC_VERSION_TAG}"
-            copy_image "${SNAPSHOT_SOURCE}" "${SNAPSHOT_TARGET}" "snapshot-agent:${NGC_VERSION_TAG}"
+            if want snapshot; then
+              echo ""
+              echo "=== Snapshot Image ==="
+              SNAPSHOT_SOURCE="${ECR_REGISTRY}/${ECR_REPOSITORY}:${COMMIT_SHA}-snapshot-agent"
+              SNAPSHOT_TARGET="${NGC_REGISTRY}/${NGC_ORG}/ai-dynamo/snapshot-agent:${NGC_VERSION_TAG}"
+              copy_image "${SNAPSHOT_SOURCE}" "${SNAPSHOT_TARGET}" "snapshot-agent:${NGC_VERSION_TAG}" || true
+            fi
           else
             # Optional nightly images — operator, planner, frontend, snapshot
             # agent and the three runtime EFA variants — are built by THIS
@@ -499,19 +762,19 @@ jobs:
             NGC_BASE="${NGC_REGISTRY}/${NGC_ORG}/ai-dynamo"
 
             OPERATOR_STAGED=false
-            if stage_optional "kubernetes-operator-nightly:${NGC_VERSION_TAG}" "${OPERATOR_BUILD_RESULT}" \
+            if want operator && stage_optional "kubernetes-operator-nightly:${NGC_VERSION_TAG}" "${OPERATOR_BUILD_RESULT}" \
                 "${ECR_BASE}:${COMMIT_SHA}-operator${NIGHTLY_TAG_SUFFIX}" \
                 "${NGC_BASE}/kubernetes-operator-nightly:${NGC_VERSION_TAG}" \
                 "${NGC_BASE}/kubernetes-operator:nightly" "kubernetes-operator:nightly"; then OPERATOR_STAGED=true; fi
 
             PLANNER_STAGED=false
-            if stage_optional "dynamo-planner-nightly:${NGC_VERSION_TAG}" "${PLANNER_BUILD_RESULT}" \
+            if want planner && stage_optional "dynamo-planner-nightly:${NGC_VERSION_TAG}" "${PLANNER_BUILD_RESULT}" \
                 "${ECR_BASE}:${COMMIT_SHA}-dynamo-planner${NIGHTLY_TAG_SUFFIX}" \
                 "${NGC_BASE}/dynamo-planner-nightly:${NGC_VERSION_TAG}" \
                 "${NGC_BASE}/dynamo-planner:nightly" "dynamo-planner:nightly"; then PLANNER_STAGED=true; fi
 
             FRONTEND_STAGED=false
-            if stage_optional "dynamo-frontend-nightly:${NGC_VERSION_TAG}" "${FRONTEND_BUILD_RESULT}" \
+            if want frontend && stage_optional "dynamo-frontend-nightly:${NGC_VERSION_TAG}" "${FRONTEND_BUILD_RESULT}" \
                 "${ECR_BASE}:${COMMIT_SHA}-dynamo-frontend${NIGHTLY_TAG_SUFFIX}" \
                 "${NGC_BASE}/dynamo-frontend-nightly:${NGC_VERSION_TAG}" \
                 "${NGC_BASE}/dynamo-frontend:nightly" "dynamo-frontend:nightly"; then FRONTEND_STAGED=true; fi
@@ -519,7 +782,7 @@ jobs:
             # Snapshot agent is amd64-only by design (cuda-checkpoint has no
             # arm64 binary); crane copies whatever manifest exists.
             SNAPSHOT_STAGED=false
-            if stage_optional "snapshot-agent-nightly:${NGC_VERSION_TAG}" "${SNAPSHOT_BUILD_RESULT}" \
+            if want snapshot && stage_optional "snapshot-agent-nightly:${NGC_VERSION_TAG}" "${SNAPSHOT_BUILD_RESULT}" \
                 "${ECR_BASE}:${COMMIT_SHA}-snapshot-agent${NIGHTLY_TAG_SUFFIX}" \
                 "${NGC_BASE}/snapshot-agent-nightly:${NGC_VERSION_TAG}" \
                 "${NGC_BASE}/snapshot-agent:nightly" "snapshot-agent:nightly"; then SNAPSHOT_STAGED=true; fi
@@ -527,19 +790,19 @@ jobs:
             # EFA variants live in the runtime -nightly repos under a -efa tag
             # suffix, mirroring the RC convention (vllm-runtime:-efa).
             VLLM_EFA_STAGED=false
-            if stage_optional "vllm-runtime-nightly:${NGC_VERSION_TAG}-efa" "${VLLM_EFA_BUILD_RESULT}" \
+            if want vllm-efa && stage_optional "vllm-runtime-nightly:${NGC_VERSION_TAG}-efa" "${VLLM_EFA_BUILD_RESULT}" \
                 "${ECR_BASE}:${COMMIT_SHA}-vllm-runtime-efa${NIGHTLY_TAG_SUFFIX}" \
                 "${NGC_BASE}/vllm-runtime-nightly:${NGC_VERSION_TAG}-efa" \
                 "${NGC_BASE}/vllm-runtime:nightly-efa" "vllm-runtime:nightly-efa"; then VLLM_EFA_STAGED=true; fi
 
             SGLANG_EFA_STAGED=false
-            if stage_optional "sglang-runtime-nightly:${NGC_VERSION_TAG}-efa" "${SGLANG_EFA_BUILD_RESULT}" \
+            if want sglang-efa && stage_optional "sglang-runtime-nightly:${NGC_VERSION_TAG}-efa" "${SGLANG_EFA_BUILD_RESULT}" \
                 "${ECR_BASE}:${COMMIT_SHA}-sglang-runtime-efa${NIGHTLY_TAG_SUFFIX}" \
                 "${NGC_BASE}/sglang-runtime-nightly:${NGC_VERSION_TAG}-efa" \
                 "${NGC_BASE}/sglang-runtime:nightly-efa" "sglang-runtime:nightly-efa"; then SGLANG_EFA_STAGED=true; fi
 
             TRTLLM_EFA_STAGED=false
-            if stage_optional "tensorrtllm-runtime-nightly:${NGC_VERSION_TAG}-efa" "${TRTLLM_EFA_BUILD_RESULT}" \
+            if want trtllm-efa && stage_optional "tensorrtllm-runtime-nightly:${NGC_VERSION_TAG}-efa" "${TRTLLM_EFA_BUILD_RESULT}" \
                 "${ECR_BASE}:${COMMIT_SHA}-trtllm-runtime-efa${NIGHTLY_TAG_SUFFIX}" \
                 "${NGC_BASE}/tensorrtllm-runtime-nightly:${NGC_VERSION_TAG}-efa" \
                 "${NGC_BASE}/tensorrtllm-runtime:nightly-efa" "tensorrtllm-runtime:nightly-efa"; then TRTLLM_EFA_STAGED=true; fi
@@ -556,6 +819,23 @@ jobs:
           echo "sglang_efa_staged=${SGLANG_EFA_STAGED:-}" >> $GITHUB_OUTPUT
           echo "trtllm_efa_staged=${TRTLLM_EFA_STAGED:-}" >> $GITHUB_OUTPUT
 
+          # Staged container refs for nspect: repo:tag labels only (NO registry/org —
+          # that's a secret; GitLab prefixes with its own NGC base). Keep primary version
+          # tags + -efa; drop -cuda13 digest aliases and floating :nightly* (same image).
+          STAGED_CTRS=""
+          for L in "${SUCCESSFUL_COPIES[@]:-}"; do
+            [ -z "${L}" ] && continue
+            case "${L}" in *-cuda13|*:nightly|*:nightly-*) continue ;; esac
+            [ -n "${STAGED_CTRS}" ] && STAGED_CTRS="${STAGED_CTRS},"
+            STAGED_CTRS="${STAGED_CTRS}${L}"
+          done
+          echo "staged_containers=${STAGED_CTRS}" >> $GITHUB_OUTPUT
+          {
+            echo "### Staged containers (repo:tag)"
+            echo ""
+            echo "\`${STAGED_CTRS:-}\`"
+          } >> "${GITHUB_STEP_SUMMARY}"
+
           printf '%s\n' "${SUCCESSFUL_COPIES[@]}" > /tmp/successful_copies.txt
           printf '%s\n' "${FAILED_COPIES[@]}" > /tmp/failed_copies.txt 2>/dev/null || true
 
@@ -565,13 +845,57 @@ jobs:
           echo "  Failed: ${#FAILED_COPIES[@]}"
           echo "========================================"
 
-          if [ ${#SUCCESSFUL_COPIES[@]} -eq 0 ]; then
+          if [ "${SUBSET}" = "none" ]; then
+            echo "Container subset is 'none'; no images copied (helm-only or wheel/crate-only release)."
+          elif [ ${#SUCCESSFUL_COPIES[@]} -eq 0 ]; then
             echo "ERROR: No images were successfully copied to NGC!"
             exit 1
           fi
 
+      - name: Validate chart/image coherence (on-demand)
+        if: ${{ env.NIGHTLY != 'true' && env.ON_DEMAND == 'true' && env.HELM_SUBSET != 'none' }}
+        run: |
+          set -euo pipefail
+          # A published chart must reference only images published now (in
+          # CONTAINER_SUBSET) or already resolvable at the exact repository the
+          # chart references (the GA registry in values.yaml — NOT the staging
+          # org, where only rc-suffixed tags exist).
+          HELM="${HELM_SUBSET:-all}"
+          CSET="${CONTAINER_SUBSET:-all}"
+          want_helm() { [ "${HELM}" = "all" ] && return 0; case ",${HELM}," in *,"$1",*) return 0 ;; esac; return 1; }
+          want_ctr()  { [ "${CSET}" = "all" ] && return 0; case ",${CSET}," in *,"$1",*) return 0 ;; esac; return 1; }
+          fail=0
+          # Each site: chart_token|container_token|values.yaml|image repo (as referenced by the chart)
+          SITES=(
+            "platform|operator|deploy/helm/charts/platform/components/operator/values.yaml|nvcr.io/nvidia/ai-dynamo/kubernetes-operator"
+            "snapshot|snapshot|deploy/helm/charts/snapshot/values.yaml|nvcr.io/nvidia/ai-dynamo/snapshot-agent"
+          )
+          for SITE in "${SITES[@]}"; do
+            IFS='|' read -r CHART CTOK VALUES REPO <<<"${SITE}"
+            want_helm "${CHART}" || { echo "chart ${CHART} not published; skip coherence check"; continue; }
+            if want_ctr "${CTOK}"; then echo "ok: ${CHART} image ${CTOK} published this release"; continue; fi
+            # Image excluded: read the pinned tag the bump wrote and prove it exists
+            # at the exact repository string the chart will deploy from.
+            TAG=$(awk -v r="${REPO}" '
+              $0 ~ "repository:[[:space:]]*"r {f=1; next}
+              f && /tag:/ { gsub(/[",]/,""); print $2; exit }' "${VALUES}")
+            if [ -z "${TAG}" ]; then
+              echo "::error::could not read pinned tag for ${REPO} in ${VALUES}"; fail=1; continue
+            fi
+            if crane manifest "${REPO}:${TAG}" >/dev/null 2>&1; then
+              echo "ok: ${CHART} references ${REPO}:${TAG} (exists)"
+            else
+              echo "::error::chart ${CHART} references ${REPO}:${TAG}, which is NOT in this release (${CTOK} excluded) and does NOT resolve"
+              fail=1
+            fi
+          done
+          [ "${fail}" -eq 0 ] || { echo "::error::chart/image coherence check failed"; exit 1; }
+
       - name: Package and push Helm charts to NGC
-        if: env.NIGHTLY != 'true'
+        id: helm_push
+        # Skip entirely when no chart is selected — otherwise the step installs the
+        # push plugin and adds credentialed repos just to push nothing.
+        if: ${{ env.NIGHTLY != 'true' && env.HELM_SUBSET != 'none' }}
         env:
           NGC_HELM_REPO: https://helm.ngc.nvidia.com/${{ secrets.NGC_PUBLISH_ORG }}/ai-dynamo
           NGC_TOKEN: ${{ secrets.NGC_PUBLISH_TOKEN }}
@@ -582,7 +906,8 @@ jobs:
 
           REPO_ALIAS="ngc-staging-dynamo"
 
-          helm plugin install https://github.com/chartmuseum/helm-push || true
+          # Pinned: this plugin runs inside the NGC-credentialed publish job.
+          helm plugin install https://github.com/chartmuseum/helm-push --version 0.10.4 || true
 
           echo "${NGC_TOKEN}" | helm repo add "${REPO_ALIAS}" \
             --username="${NGC_USERNAME}" \
@@ -595,22 +920,32 @@ jobs:
           echo "" >> $GITHUB_STEP_SUMMARY
           echo "### Helm Charts" >> $GITHUB_STEP_SUMMARY
 
-          PLATFORM_CHART_DIR="deploy/helm/charts/platform"
-          CHART_NAME=$(awk '/^name:/ {print $2}' "${PLATFORM_CHART_DIR}/Chart.yaml")
-          pushd "${PLATFORM_CHART_DIR}"
-          helm dep build .
-          popd
-
-          echo "Packaging ${CHART_NAME} with version ${HELM_CHART_VERSION}..."
-          helm package \
-            --version "${HELM_CHART_VERSION}" \
-            --app-version "${HELM_CHART_VERSION}" \
-            "${PLATFORM_CHART_DIR}"
-
-          CHART_FILE="${CHART_NAME}-${HELM_CHART_VERSION}.tgz"
-          echo "Pushing ${CHART_FILE} to NGC Helm registry..."
-          helm cm-push "${CHART_FILE}" "${REPO_ALIAS}"
-          echo "- \`${CHART_NAME}:${HELM_CHART_VERSION}\` pushed to NGC Helm registry" >> $GITHUB_STEP_SUMMARY
+          # snapshot + dynamo-platform, mirroring the rcs/dynamo helm runbook.
+          # Only the selected charts are pushed (token = chart dir name);
+          # nightly/rc default to both (HELM_SUBSET defaults to 'all').
+          STAGED_HELM=""
+          for CHART_DIR in deploy/helm/charts/platform deploy/helm/charts/snapshot; do
+            TOKEN="$(basename "${CHART_DIR}")"
+            if [ "${HELM_SUBSET:-all}" != "all" ]; then
+              case ",${HELM_SUBSET}," in *,"${TOKEN}",*) ;; *) echo "skip ${TOKEN} chart (not selected)"; continue ;; esac
+            fi
+            CHART_NAME=$(awk '/^name:/ {print $2; exit}' "${CHART_DIR}/Chart.yaml")
+            ( cd "${CHART_DIR}" && helm dep build . )
+            echo "Packaging ${CHART_NAME} with version ${HELM_CHART_VERSION}..."
+            helm package \
+              --version "${HELM_CHART_VERSION}" \
+              --app-version "${HELM_CHART_VERSION}" \
+              "${CHART_DIR}"
+            CHART_FILE="${CHART_NAME}-${HELM_CHART_VERSION}.tgz"
+            echo "Pushing ${CHART_FILE} to NGC Helm registry..."
+            helm cm-push "${CHART_FILE}" "${REPO_ALIAS}"
+            echo "- \`${CHART_NAME}:${HELM_CHART_VERSION}\` pushed to NGC Helm registry" >> $GITHUB_STEP_SUMMARY
+            STAGED_HELM="${STAGED_HELM:+${STAGED_HELM},}${CHART_NAME}:${HELM_CHART_VERSION}"
+            # Rewrite the output after every chart (later writes win) so a
+            # failure on a subsequent chart still reports what actually landed.
+            # Consumed by GitLab (STAGED_HELM) for exact nSpect chart registration.
+            echo "staged_helm=${STAGED_HELM}" >> "${GITHUB_OUTPUT}"
+          done
 
           helm repo remove "${REPO_ALIAS}"
 
@@ -738,6 +1073,9 @@ jobs:
           NIGHTLY_SKIPPED_HELM: ${{ steps.nightly_helm_push.outputs.skipped_helm }}
           SUCCESSFUL_COUNT: ${{ steps.copy_images.outputs.successful_count }}
           FAILED_COUNT: ${{ steps.copy_images.outputs.failed_count }}
+          STAGED_CONTAINERS: ${{ steps.copy_images.outputs.staged_containers }}
+          STAGED_HELM: ${{ steps.helm_push.outputs.staged_helm }}
+          BRANCH_NAME: ${{ github.ref_name }}
           OPERATOR_STAGED: ${{ steps.copy_images.outputs.operator_staged }}
           PLANNER_STAGED: ${{ steps.copy_images.outputs.planner_staged }}
           FRONTEND_STAGED: ${{ steps.copy_images.outputs.frontend_staged }}
@@ -757,25 +1095,31 @@ jobs:
           echo "| Version | ${VERSION} |" >> $GITHUB_STEP_SUMMARY
           echo "| NGC Version Tag | ${NGC_VERSION_TAG} |" >> $GITHUB_STEP_SUMMARY
           echo "| Source Commit SHA | ${COMMIT_SHA} |" >> $GITHUB_STEP_SUMMARY
-          echo "| Branch | ${{ github.ref_name }} |" >> $GITHUB_STEP_SUMMARY
+          echo "| Branch | ${BRANCH_NAME} |" >> $GITHUB_STEP_SUMMARY
           echo "" >> $GITHUB_STEP_SUMMARY
           echo "### NGC Publishing Results" >> $GITHUB_STEP_SUMMARY
           echo "" >> $GITHUB_STEP_SUMMARY
           echo "- **Successful copies**: ${SUCCESSFUL_COUNT}" >> $GITHUB_STEP_SUMMARY
           echo "- **Failed copies**: ${FAILED_COUNT}" >> $GITHUB_STEP_SUMMARY
           echo "" >> $GITHUB_STEP_SUMMARY
-          # Mirror the NGC_NAME_SUFFIX logic used in the copy step so the summary
-          # advertises the actual repos that were pushed to.
-          NGC_NAME_SUFFIX=""
-          if [[ "${NIGHTLY}" == "true" ]]; then
-            NGC_NAME_SUFFIX="-nightly"
-          fi
-          echo "### Expected Images" >> $GITHUB_STEP_SUMMARY
+          # Report what ACTUALLY landed, not a hardcoded full-release list: a partial
+          # selection must not produce a summary claiming images it never published.
+          echo "### Images published this run" >> $GITHUB_STEP_SUMMARY
           echo "" >> $GITHUB_STEP_SUMMARY
-          echo "Runtime images (CUDA 13 - default, bare tag + explicit -cuda13 alias):" >> $GITHUB_STEP_SUMMARY
-          echo "- \`vllm-runtime${NGC_NAME_SUFFIX}:${NGC_VERSION_TAG}\` / \`vllm-runtime${NGC_NAME_SUFFIX}:${NGC_VERSION_TAG}-cuda13\`" >> $GITHUB_STEP_SUMMARY
-          echo "- \`sglang-runtime${NGC_NAME_SUFFIX}:${NGC_VERSION_TAG}\` / \`sglang-runtime${NGC_NAME_SUFFIX}:${NGC_VERSION_TAG}-cuda13\`" >> $GITHUB_STEP_SUMMARY
-          echo "- \`tensorrtllm-runtime${NGC_NAME_SUFFIX}:${NGC_VERSION_TAG}\` / \`tensorrtllm-runtime${NGC_NAME_SUFFIX}:${NGC_VERSION_TAG}-cuda13\`" >> $GITHUB_STEP_SUMMARY
+          # Actuals-driven list (a partial selection must not claim images it
+          # never published); the nightly-only block below adds the optional
+          # images with their per-build staged/skipped notes.
+          if [ -n "${STAGED_CONTAINERS}" ]; then
+            echo "${STAGED_CONTAINERS}" | tr ',' '\n' | sed 's/^/- `/; s/$/`/' >> $GITHUB_STEP_SUMMARY
+            echo "" >> $GITHUB_STEP_SUMMARY
+            if [[ "${NIGHTLY}" == "true" ]]; then
+              echo "_Each runtime image above also published as a \`-cuda13\` digest alias; floating tags are listed below._" >> $GITHUB_STEP_SUMMARY
+            else
+              echo "_Each runtime image above also published as a \`-cuda13\` digest alias._" >> $GITHUB_STEP_SUMMARY
+            fi
+          else
+            echo "_No container images published (selection: \`${CONTAINER_SUBSET:-all}\`)._" >> $GITHUB_STEP_SUMMARY
+          fi
           if [[ "${NIGHTLY}" == "true" ]]; then
             _note() { [[ "$1" == "true" ]] && echo "" || echo " — **NOT staged this run** (see copy-step warnings)"; }
             OP_NOTE=$(_note "${OPERATOR_STAGED}");   PL_NOTE=$(_note "${PLANNER_STAGED}")
@@ -822,24 +1166,32 @@ jobs:
           fi
           if [[ "${NIGHTLY}" != "true" ]]; then
             echo "" >> $GITHUB_STEP_SUMMARY
-            echo "EFA runtime images (multi-arch):" >> $GITHUB_STEP_SUMMARY
-            echo "- \`vllm-runtime:${NGC_VERSION_TAG}-efa\`" >> $GITHUB_STEP_SUMMARY
-            echo "- \`sglang-runtime:${NGC_VERSION_TAG}-efa\`" >> $GITHUB_STEP_SUMMARY
-            echo "- \`tensorrtllm-runtime:${NGC_VERSION_TAG}-efa\`" >> $GITHUB_STEP_SUMMARY
-            echo "" >> $GITHUB_STEP_SUMMARY
-            echo "Operator image:" >> $GITHUB_STEP_SUMMARY
-            echo "- \`kubernetes-operator:${NGC_VERSION_TAG}\`" >> $GITHUB_STEP_SUMMARY
-            echo "" >> $GITHUB_STEP_SUMMARY
-            echo "Planner image:" >> $GITHUB_STEP_SUMMARY
-            echo "- \`dynamo-planner:${NGC_VERSION_TAG}\` (multi-arch)" >> $GITHUB_STEP_SUMMARY
-            echo "" >> $GITHUB_STEP_SUMMARY
-            echo "Frontend images:" >> $GITHUB_STEP_SUMMARY
-            echo "- \`dynamo-frontend:${NGC_VERSION_TAG}\` (multi-arch)" >> $GITHUB_STEP_SUMMARY
+            echo "### Helm charts published this run" >> $GITHUB_STEP_SUMMARY
             echo "" >> $GITHUB_STEP_SUMMARY
-            echo "Helm chart:" >> $GITHUB_STEP_SUMMARY
-            echo "- \`dynamo-platform:${HELM_CHART_VERSION}\` (pushed to NGC Helm registry)" >> $GITHUB_STEP_SUMMARY
+            if [ -n "${STAGED_HELM}" ]; then
+              echo "${STAGED_HELM}" | tr ',' '\n' | sed 's/^/- `/; s/$/`/' >> $GITHUB_STEP_SUMMARY
+            else
+              echo "_No charts published (selection: \`${HELM_SUBSET:-all}\`)._" >> $GITHUB_STEP_SUMMARY
+            fi
           fi
 
+      # Runs last so charts still publish first. On-demand only: the operator named
+      # an explicit image set, so a partially-copied set must not report success and
+      # let GitLab open promotion MRs for images that never landed. rc/nightly keep
+      # their historical warn-and-continue behavior (a floating-tag hiccup there must
+      # not block the GitLab trigger, which requires this job to succeed).
+      - name: Fail on partially staged container selection (on-demand)
+        if: ${{ env.ON_DEMAND == 'true' && env.CONTAINER_SUBSET != 'none' }}
+        env:
+          FAILED_COUNT: ${{ steps.copy_images.outputs.failed_count }}
+        run: |
+          set -euo pipefail
+          if [ "${FAILED_COUNT:-0}" -ne 0 ]; then
+            echo "::error::${FAILED_COUNT} selected container image(s) failed to copy to NGC — see 'Copy images to NGC'. Re-run with a fresh rc_round once fixed."
+            exit 1
+          fi
+          echo "All selected container images copied."
+
   # ============================================================================
   # ARTIFACTORY STAGING: extract wheels from the dynamo-runtime image in ECR
   # and upload them to JFrog Artifactory so consumers can pip-install them.
@@ -849,14 +1201,21 @@ jobs:
   stage-wheels-artifactory:
     name: Stage Wheels to Artifactory
     needs: [prepare-release]
-    if: needs.prepare-release.result == 'success'
+    # Skip only when no wheels are selected; nightly/rc default to 'all'.
+    if: ${{ needs.prepare-release.result == 'success' && inputs.wheels != 'none' }}
     runs-on: prod-tester-amd-v1
     environment: automated-release
+    outputs:
+      # Comma-separated wheel FILENAMES actually uploaded (NO Artifactory URL — that's a
+      # secret; GitLab prefixes with its own WHEEL_URL_PREFIX). For nspect registration.
+      staged_wheels: ${{ steps.upload_wheels.outputs.staged_wheels }}
     env:
       COMMIT_SHA: ${{ needs.prepare-release.outputs.commit_sha }}
       NIGHTLY: ${{ needs.prepare-release.outputs.nightly }}
+      ON_DEMAND: ${{ needs.prepare-release.outputs.on_demand }}
       VERSION: ${{ needs.prepare-release.outputs.version }}
       RC_NUMBER: ${{ needs.prepare-release.outputs.rc_number }}
+      WHEEL_SUBSET: ${{ inputs.wheels }}
     steps:
       - name: Setup crane
         env:
@@ -930,6 +1289,61 @@ jobs:
 
           echo "upload_dir=${OUT}" >> $GITHUB_OUTPUT
 
+      - name: Inventory selected wheels
+        env:
+          UPLOAD_DIR: ${{ steps.extract.outputs.upload_dir }}
+        run: |
+          set -uo pipefail
+          # Report-only inventory of which selected wheels were extracted. Never fails
+          # (kvbm is built conditionally, so 'all' may legitimately omit it).
+          WSUB="${WHEEL_SUBSET:-all}"
+          want_wheel() {
+            [ "${WSUB}" = "all" ] && return 0
+            case ",${WSUB}," in *,"$1",*) return 0 ;; esac
+            return 1
+          }
+          # token|filename glob (relative to the upload dir)
+          ENTRIES=(
+            "ai-dynamo|ai_dynamo-*-py3-none-any.whl"
+            "ai-dynamo-runtime|ai_dynamo_runtime-*.whl"
+            "kvbm|kvbm-*.whl"
+          )
+
+          {
+            echo "### Wheel availability"
+            echo ""
+            echo "Selection: \`${WSUB}\`"
+            echo ""
+            echo "| Token | Wheels found | Status |"
+            echo "|-------|--------------|--------|"
+          } >> "${GITHUB_STEP_SUMMARY}"
+
+          shopt -s nullglob
+          avail=0; missing=0; selected=0
+          echo "=== Wheel availability (selection: ${WSUB}) ==="
+          for E in "${ENTRIES[@]}"; do
+            IFS='|' read -r TOKEN GLOB <<<"${E}"
+            want_wheel "${TOKEN}" || continue
+            selected=$((selected+1))
+            FILES=( "${UPLOAD_DIR}"/${GLOB} )
+            if [ "${#FILES[@]}" -gt 0 ]; then
+              echo "  available: ${TOKEN} -> ${FILES[*]##*/}"
+              echo "| ${TOKEN} | ${#FILES[@]} | available |" >> "${GITHUB_STEP_SUMMARY}"
+              avail=$((avail+1))
+            else
+              echo "  MISSING:   ${TOKEN} (no ${GLOB})"
+              echo "| ${TOKEN} | 0 | **missing** |" >> "${GITHUB_STEP_SUMMARY}"
+              missing=$((missing+1))
+            fi
+          done
+          shopt -u nullglob
+
+          echo "Summary: ${avail} available, ${missing} missing (of ${selected} selected)."
+          {
+            echo ""
+            echo "**${avail} available, ${missing} missing** (of ${selected} selected)."
+          } >> "${GITHUB_STEP_SUMMARY}"
+
       - name: Verify wheel versions match prepare-release expected tag
         env:
           EXPECTED_WHEEL_VERSION: ${{ needs.prepare-release.outputs.wheel_version }}
@@ -973,6 +1387,7 @@ jobs:
           fi
 
       - name: Upload wheels to Artifactory
+        id: upload_wheels
         env:
           ARTIFACTORY_URL: ${{ secrets.ARTIFACTORY_URL }}
           ARTIFACTORY_USER: ${{ secrets.ARTIFACTORY_USER }}
@@ -983,6 +1398,10 @@ jobs:
 
           if [[ "${NIGHTLY}" == "true" ]]; then
             SUBPATH="nightly/${{ github.run_id }}"
+          elif [[ "${ON_DEMAND}" == "true" ]]; then
+            # Internal staging: rc subpath disambiguates re-stages of the same version
+            # (the wheel filename keeps the clean version). GA promotion drops the rc.
+            SUBPATH="release/${VERSION}/rc${RC_NUMBER}"
           else
             if [ -z "${RC_NUMBER}" ]; then
               echo "Error: RC_NUMBER is empty for a non-nightly release"
@@ -1006,8 +1425,19 @@ jobs:
             exit 1
           fi
 
+          STAGED_WHEELS=""  # filenames actually uploaded (for nspect; URL stays secret)
           for WHEEL in "${WHEELS[@]}"; do
             WHEEL_FILENAME=$(basename "${WHEEL}")
+            # Wheel subset filter (nightly/rc default to all: WHEEL_SUBSET=all).
+            if [ "${WHEEL_SUBSET:-all}" != "all" ]; then
+              case "${WHEEL_FILENAME}" in
+                ai_dynamo_runtime-*) WTOK=ai-dynamo-runtime ;;
+                ai_dynamo-*)         WTOK=ai-dynamo ;;
+                kvbm-*)              WTOK=kvbm ;;
+                *)                   WTOK="" ;;
+              esac
+              case ",${WHEEL_SUBSET}," in *,"${WTOK}",*) ;; *) echo "skip ${WHEEL_FILENAME} (not selected)"; continue ;; esac
+            fi
             TARGET_URL="${BASE_URL}/${WHEEL_FILENAME}"
             BODY_FILE=$(mktemp)
             HTTP_CODE=$(curl -sSL -o "${BODY_FILE}" -w "%{http_code}" \
@@ -1023,15 +1453,159 @@ jobs:
             fi
             echo "Uploaded: ${WHEEL_FILENAME} -> ${TARGET_URL}"
             echo "| \`${WHEEL_FILENAME}\` | ${TARGET_URL} |" >> $GITHUB_STEP_SUMMARY
+            [ -n "${STAGED_WHEELS}" ] && STAGED_WHEELS="${STAGED_WHEELS},"
+            STAGED_WHEELS="${STAGED_WHEELS}${WHEEL_FILENAME}"
             rm -f "${BODY_FILE}"
           done
 
-  # Combined NGC + Artifactory staging reply on the nightly thread.
+          echo "staged_wheels=${STAGED_WHEELS}" >> "$GITHUB_OUTPUT"
+
+          # A subset that matched no wheel in the image would otherwise upload
+          # nothing and still report success.
+          if [ -z "${STAGED_WHEELS}" ]; then
+            echo "::error::wheel subset '${WHEEL_SUBSET:-all}' matched no wheels in the runtime image; nothing was uploaded"
+            exit 1
+          fi
+
+  # ARTIFACTORY STAGING: publish workspace crates to the Artifactory cargo registry
+  # (from ARTIFACTORY_CARGO_INDEX), dependency-ordered. Artifactory only, never crates.io.
+  # cargo needs the native toolchain, so we `docker run` the ECR build image on
+  # prod-builder-amd-v1 (the DinD builder release-publish uses) instead of a job-level
+  # `container:` on a k8s-hooks tester (its per-job pod stayed Pending). On-demand only.
+
+  stage-crates-artifactory:
+    name: Stage Crates to Artifactory
+    needs: [prepare-release]
+    if: ${{ needs.prepare-release.result == 'success' && needs.prepare-release.outputs.on_demand == 'true' && inputs.crates != 'none' && inputs.crates != '' }}
+    runs-on: prod-builder-amd-v1
+    environment: automated-release
+    # Fail-soft: crate staging must never block the release. The job turns red
+    # but the run continues. Consumers must read outputs.crates_status, not the
+    # job result — continue-on-error masks it as 'success' (actions/runner#2495).
+    continue-on-error: true
+    outputs:
+      # Comma-separated crate names actually staged to the cargo registry (set even on
+      # partial failure). Forwarded to GitLab as STAGED_CRATES for nspect registration.
+      staged_crates: ${{ steps.publish_crates.outputs.staged_crates }}
+      # Comma-separated crate names that failed (or were skipped because a
+      # dependency failed). Empty on full success.
+      failed_crates: ${{ steps.publish_crates.outputs.failed_crates }}
+      # success | partial | failed — the authoritative crate-staging outcome.
+      crates_status: ${{ steps.publish_crates.outputs.crates_status }}
+    env:
+      COMMIT_SHA: ${{ needs.prepare-release.outputs.commit_sha }}
+    steps:
+      - name: Checkout at source commit
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        with:
+          ref: ${{ needs.prepare-release.outputs.commit_sha }}
+
+      # Use the workflow commit's copy of stage_crates.py, not the release
+      # tree's: older release branches may predate the script.
+      - name: Fetch crate-staging script from the workflow commit
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        with:
+          ref: ${{ github.sha }}
+          path: .ondemand-tools
+          sparse-checkout: .github/scripts
+          sparse-checkout-cone-mode: false
+
+      - name: Login to ECR
+        run: |
+          set -euo pipefail
+          aws ecr get-login-password --region "${AWS_DEFAULT_REGION}" | docker login --username AWS --password-stdin "${ECR_REGISTRY}"
+          echo "ECR_IMAGE=${ECR_REGISTRY}/${ECR_REPOSITORY}:${COMMIT_SHA}-dynamo-runtime-test" >> "${GITHUB_ENV}"
+
+      - name: Inventory crate build image in ECR
+        env:
+          CRATE_SUBSET: ${{ inputs.crates }}
+        run: |
+          set -uo pipefail
+          # Report-only: is the crate build image in ECR? Never fails (publish step errors if absent).
+          TAG="${COMMIT_SHA}-dynamo-runtime-test"
+          {
+            echo "### Crate build image availability (ECR)"
+            echo ""
+            echo "Crate selection: \`${CRATE_SUBSET:-all}\`"
+            echo ""
+          } >> "${GITHUB_STEP_SUMMARY}"
+          if aws ecr describe-images --repository-name "${ECR_REPOSITORY}" \
+               --image-ids imageTag="${TAG}" --region "${AWS_DEFAULT_REGION}" >/dev/null 2>&1; then
+            echo "available: build image ${ECR_REPOSITORY}:${TAG}"
+            echo "- available: \`${ECR_REPOSITORY}:${TAG}\`" >> "${GITHUB_STEP_SUMMARY}"
+          else
+            echo "MISSING: build image ${ECR_REPOSITORY}:${TAG}"
+            echo "- **missing**: \`${ECR_REPOSITORY}:${TAG}\` (crate publish will fail to pull)" >> "${GITHUB_STEP_SUMMARY}"
+          fi
+
+      - name: Publish crates to Artifactory cargo registry (inside build image)
+        id: publish_crates
+        env:
+          ARTIFACTORY_TOKEN: ${{ secrets.ARTIFACTORY_TOKEN }}
+          ARTIFACTORY_CARGO_INDEX: ${{ secrets.ARTIFACTORY_CARGO_INDEX }}
+          EXPECT_VERSION: ${{ needs.prepare-release.outputs.crate_version }}
+          # Same rc-suffixed value: stage_crates.py rewrites every publishable crate's
+          # version to this before publishing, so re-stages don't collide on the
+          # immutable cargo registry. GA promotion (GitLab) republishes the clean version.
+          STAGE_VERSION: ${{ needs.prepare-release.outputs.crate_version }}
+          CRATE_SUBSET: ${{ inputs.crates }}
+        run: |
+          # No `set -e`: outputs must be set even when the publish fails partway;
+          # we exit with the real code afterwards.
+          set -uo pipefail
+          # Run stage_crates.py inside the ECR build image: repo mounted at /work, cargo uses
+          # the image toolchain, root so it can write Cargo.lock/.cargo and the target dir.
+          docker run --rm --user 0:0 \
+            --network host \
+            --ulimit nofile=65536:65536 \
+            -v "${GITHUB_WORKSPACE}:/work" -w /work \
+            -e HOME=/tmp \
+            -e RUSTFLAGS="--cfg tokio_unstable" \
+            -e CARGO_TARGET_DIR=/tmp/cargo-target \
+            -e ARTIFACTORY_TOKEN \
+            -e ARTIFACTORY_CARGO_INDEX \
+            -e EXPECT_VERSION \
+            -e STAGE_VERSION \
+            -e CRATE_SUBSET \
+            "${ECR_IMAGE}" \
+            bash -lc 'python3 .ondemand-tools/.github/scripts/stage_crates.py --registry artifactory --only "${CRATE_SUBSET}"' \
+            2>&1 | tee /tmp/stage_crates.log
+          rc="${PIPESTATUS[0]}"
+
+          # stage_crates.py emits `STAGED_CRATE=` and `FAILED_CRATE= ` lines.
+          STAGED="$(sed -n 's/^STAGED_CRATE=//p' /tmp/stage_crates.log | paste -sd, - || true)"
+          FAILED="$(sed -n 's/^FAILED_CRATE=\([^ ]*\).*/\1/p' /tmp/stage_crates.log | paste -sd, - || true)"
+          if [ "${rc}" -eq 0 ]; then
+            STATUS="success"
+          elif [ -n "${STAGED}" ]; then
+            STATUS="partial"
+          else
+            STATUS="failed"
+          fi
+          {
+            echo "staged_crates=${STAGED}"
+            echo "failed_crates=${FAILED}"
+            echo "crates_status=${STATUS}"
+          } >> "${GITHUB_OUTPUT}"
+          {
+            echo "### Crate staging: ${STATUS}"
+            echo ""
+            echo "Staged: \`${STAGED:-}\`"
+            echo ""
+            echo "Failed: \`${FAILED:-}\`"
+          } >> "${GITHUB_STEP_SUMMARY}"
+          echo "Staged crates: ${STAGED:-}"
+          if [ "${rc}" -ne 0 ]; then
+            echo "::error::Crate staging ${STATUS}: failed=[${FAILED:-unknown}] staged=[${STAGED:-none}] — the release continues; re-stage crates with a bumped rc_round after fixing."
+          fi
+          exit "${rc}"
+
+  # Combined NGC + Artifactory (wheels + crates) staging reply on the thread.
   notify-slack-staging:
     name: Notify Slack — staging complete
     runs-on: prod-default-small-v2
-    needs: [prepare-release, release-publish, stage-wheels-artifactory]
-    if: ${{ always() && inputs.slack_thread_ts != '' }}
+    needs: [prepare-release, release-publish, stage-wheels-artifactory, stage-crates-artifactory]
+    if: ${{ !cancelled() && inputs.slack_thread_ts != '' }}
     permissions:
       contents: read
     steps:
@@ -1041,11 +1615,16 @@ jobs:
           SLACK_BOT_TOKEN:  ${{ secrets.SLACK_RELEASE_BOT_TOKEN }}
           SLACK_CHANNEL_ID: ${{ secrets.SLACK_RELEASE_CHANNEL_ID }}
           SLACK_THREAD_TS:  ${{ inputs.slack_thread_ts }}
+          PREP_RESULT:      ${{ needs.prepare-release.result }}
           NGC_RESULT:       ${{ needs.release-publish.result }}
           NGC_FAILED_COPIES: ${{ needs.release-publish.outputs.failed_copy_count }}
           ART_RESULT:       ${{ needs.stage-wheels-artifactory.result }}
+          # continue-on-error masks the job result; the output is authoritative.
+          # Empty output = died before the publish step: treat as failed.
+          CRATES_RESULT:    ${{ needs.stage-crates-artifactory.result == 'skipped' && 'skipped' || needs.stage-crates-artifactory.outputs.crates_status || 'failed' }}
           NGC_VERSION_TAG:  ${{ needs.prepare-release.outputs.ngc_version_tag }}
           WHEEL_VERSION:    ${{ needs.prepare-release.outputs.wheel_version }}
+          CRATE_VERSION:    ${{ needs.prepare-release.outputs.crate_version }}
           RUN_URL:          ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
         shell: bash
         run: |
@@ -1054,7 +1633,8 @@ jobs:
           emoji_for() {
             case "$1" in
               success)   echo ":white_check_mark:" ;;
-              failure)   echo ":x:" ;;
+              partial)   echo ":warning:" ;;
+              failure|failed) echo ":x:" ;;
               skipped)   echo ":fast_forward:" ;;
               cancelled) echo ":no_entry_sign:" ;;
               *)         echo ":grey_question:" ;;
@@ -1062,8 +1642,13 @@ jobs:
           }
           NGC_EMOJI=$(emoji_for "${NGC_RESULT}")
           ART_EMOJI=$(emoji_for "${ART_RESULT}")
+          CRATES_EMOJI=$(emoji_for "${CRATES_RESULT:-skipped}")
 
-          if [ "${NGC_RESULT}" = "success" ] && [ "${ART_RESULT}" = "success" ]; then
+          # A deliberately skipped stage (subset 'none') is not an issue.
+          ok() { case "$1" in success|skipped|'') return 0 ;; *) return 1 ;; esac; }
+          if [ "${PREP_RESULT}" != "success" ]; then
+            HEADER=":x: Staging failed before it started (prepare-release: ${PREP_RESULT})"
+          elif ok "${NGC_RESULT}" && ok "${ART_RESULT}" && ok "${CRATES_RESULT:-}"; then
             # Copy failures can be non-fatal (optional-image warn-skips) —
             # a green job with skipped copies must not read as "complete".
             if [ -n "${NGC_FAILED_COPIES:-}" ] && [ "${NGC_FAILED_COPIES}" != "0" ]; then
@@ -1076,16 +1661,19 @@ jobs:
           fi
 
           PAYLOAD=$(jq -n \
-            --arg channel    "${SLACK_CHANNEL_ID}" \
-            --arg thread_ts  "${SLACK_THREAD_TS}" \
-            --arg header     "${HEADER}" \
-            --arg ngc_emoji  "${NGC_EMOJI}" \
-            --arg ngc_result "${NGC_RESULT}" \
-            --arg art_emoji  "${ART_EMOJI}" \
-            --arg art_result "${ART_RESULT}" \
-            --arg ngc_tag    "${NGC_VERSION_TAG}" \
-            --arg wheel      "${WHEEL_VERSION}" \
-            --arg run_url    "${RUN_URL}" \
+            --arg channel       "${SLACK_CHANNEL_ID}" \
+            --arg thread_ts     "${SLACK_THREAD_TS}" \
+            --arg header        "${HEADER}" \
+            --arg ngc_emoji     "${NGC_EMOJI}" \
+            --arg ngc_result    "${NGC_RESULT}" \
+            --arg art_emoji     "${ART_EMOJI}" \
+            --arg art_result    "${ART_RESULT}" \
+            --arg crates_emoji  "${CRATES_EMOJI}" \
+            --arg crates_result "${CRATES_RESULT:-skipped}" \
+            --arg ngc_tag       "${NGC_VERSION_TAG}" \
+            --arg wheel         "${WHEEL_VERSION}" \
+            --arg crate_ver     "${CRATE_VERSION:-n/a}" \
+            --arg run_url       "${RUN_URL}" \
             '{
               channel: $channel,
               thread_ts: $thread_ts,
@@ -1098,7 +1686,9 @@ jobs:
                     { type: "mrkdwn",
                       text: ($ngc_emoji + " *NGC publish:* " + $ngc_result + "\n`" + $ngc_tag + "`") },
                     { type: "mrkdwn",
-                      text: ($art_emoji + " *Artifactory wheels:* " + $art_result + "\n`" + $wheel + "`") }
+                      text: ($art_emoji + " *Artifactory wheels:* " + $art_result + "\n`" + $wheel + "`") },
+                    { type: "mrkdwn",
+                      text: ($crates_emoji + " *Artifactory crates:* " + $crates_result + "\n`" + $crate_ver + "`") }
                   ] },
                 { type: "context",
                   elements: [ { type: "mrkdwn", text: ("<" + $run_url + "|GitHub Actions run>") } ] }
@@ -1128,8 +1718,17 @@ jobs:
 
   trigger-gitlab-release-pipeline:
     name: Trigger GitLab Release Pipeline
-    needs: [prepare-release, release-publish, stage-wheels-artifactory]
-    if: ${{ inputs.skip_gitlab_pipeline != true }}
+    # rc/nightly: every SELECTED stage must succeed (a 'none' subset skips the
+    # stage and counts as satisfied). On-demand: fires even on staging failures —
+    # GitLab's manual approval gates are the real publish gates. !cancelled()
+    # overrides the implicit needs-success(); stage-crates never blocks
+    # (continue-on-error).
+    needs: [prepare-release, release-publish, stage-wheels-artifactory, stage-crates-artifactory]
+    if: >-
+      ${{ !cancelled() && needs.prepare-release.result == 'success' && inputs.skip_gitlab_pipeline != true &&
+          (needs.prepare-release.outputs.on_demand == 'true' ||
+           ((needs.release-publish.result == 'success' || (inputs.containers == 'none' && inputs.helm == 'none')) &&
+            (needs.stage-wheels-artifactory.result == 'success' || inputs.wheels == 'none'))) }}
     runs-on:
       group: gitlab_ci_runners
     environment: automated-release
@@ -1145,25 +1744,52 @@ jobs:
           # rotated without a code change.
           GITLAB_TRIGGER_URL:   ${{ secrets.GITLAB_PIPELINE_URL }}
           GITLAB_TRIGGER_TOKEN: ${{ secrets.GITLAB_TRIGGER_TOKEN }}
+          # release-automation ref to run (var-overridable for testing a GitLab
+          # dev branch). GitLab does NOT force dry-run on non-default refs.
+          # The on-demand support landed on release-automation main (2026-07-08),
+          # so main is the correct default.
+          GITLAB_REF:           ${{ vars.GITLAB_RELEASE_AUTOMATION_REF || 'main' }}
           VERSION:              ${{ needs.prepare-release.outputs.version }}
           NGC_VERSION_TAG:      ${{ needs.prepare-release.outputs.ngc_version_tag }}
           WHEEL_VERSION:        ${{ needs.prepare-release.outputs.wheel_version }}
+          CRATE_VERSION:        ${{ needs.prepare-release.outputs.crate_version }}
           COMMIT_SHA:           ${{ needs.prepare-release.outputs.commit_sha }}
           NIGHTLY:              ${{ needs.prepare-release.outputs.nightly }}
+          ON_DEMAND:            ${{ needs.prepare-release.outputs.on_demand }}
           RC_NUMBER:            ${{ needs.prepare-release.outputs.rc_number }}
-          # chart:version pairs pushed by the nightly helm step.
+          CONTAINER_SUBSET:     ${{ inputs.containers }}
+          WHEEL_SUBSET:         ${{ inputs.wheels }}
+          CRATE_SUBSET:         ${{ inputs.crates }}
+          HELM_SUBSET:          ${{ inputs.helm }}
+          # Artifacts ACTUALLY staged (names/filenames only — URLs/registry/org are secrets,
+          # GitLab re-prefixes with its own bases). Forwarded so GitLab's nspect step
+          # registers exactly what landed, not what was requested. Empty if a stage was
+          # skipped or failed wholesale. STAGED_HELM covers both the rc/on-demand helm
+          # step and the nightly helm step (chart:version pairs).
+          STAGED_CRATES:        ${{ needs.stage-crates-artifactory.outputs.staged_crates }}
+          STAGED_CONTAINERS:    ${{ needs.release-publish.outputs.staged_containers }}
+          STAGED_WHEELS:        ${{ needs.stage-wheels-artifactory.outputs.staged_wheels }}
           STAGED_HELM:          ${{ needs.release-publish.outputs.staged_helm }}
+          # success|partial|failed|skipped (crate staging is fail-soft; empty
+          # output = died before the publish step, treated as failed).
+          CRATES_STATUS:        ${{ needs.stage-crates-artifactory.result == 'skipped' && 'skipped' || needs.stage-crates-artifactory.outputs.crates_status || 'failed' }}
           # Forwarded so the GitLab pipeline can reply on the same thread.
           SLACK_THREAD_TS:      ${{ inputs.slack_thread_ts }}
           SLACK_CHANNEL_ID:     ${{ secrets.SLACK_RELEASE_CHANNEL_ID }}
         run: |
           set -euo pipefail
 
+          if [ "${GITLAB_REF}" != "main" ]; then
+            echo "::warning::GitLab pipeline ref override active: '${GITLAB_REF}' (vars.GITLAB_RELEASE_AUTOMATION_REF). This runs that ref's pipeline definition with production credentials; publish jobs remain manual-gated but are LIVE."
+          fi
+
           # PIPELINE_TYPE / RELEASE_TYPE drive every rule in dynamo.yml.
           # ARTIFACTS gates per-artifact scans and promotions.
           # NIGHTLY_TAG carries the resolved nightly tag (date+shortsha) for
           # the GitLab pipeline summary; the nspect version-name is hardcoded
           # to "Dynamo Nightlies" inside dynamo.yml, so we don't pass it.
+          RELEASE_TAG=""
+          RELEASE_VERSION_VAR="${VERSION}"
           if [ "${NIGHTLY}" = "true" ]; then
             PIPELINE_TYPE="security"
             RELEASE_TYPE="nightly"
@@ -1174,14 +1800,39 @@ jobs:
             fi
             NIGHTLY_TAG="${NGC_VERSION_TAG}"
             RC_TAG=""
+            # VERSION is the literal 'nightly'; empty lets GitLab apply its default.
+            RELEASE_VERSION_VAR=""
+          elif [ "${ON_DEMAND}" = "true" ]; then
+            # On-demand runs the security pipeline (scans + nSpect + manual-gated
+            # publish); RELEASE_TYPE=ondemand (no hyphen) matches dynamo.yml's rules.
+            PIPELINE_TYPE="security"
+            RELEASE_TYPE="ondemand"
+            # Coarse ARTIFACTS list derived from the per-type subsets ('none' omits
+            # the type). The fine-grained *_SUBSET vars below carry the exact picks.
+            ARTIFACTS=""
+            [ "${WHEEL_SUBSET:-all}" != "none" ]     && ARTIFACTS="${ARTIFACTS},wheel"
+            [ "${CONTAINER_SUBSET:-all}" != "none" ] && ARTIFACTS="${ARTIFACTS},container"
+            [ "${CRATE_SUBSET:-all}" != "none" ]     && ARTIFACTS="${ARTIFACTS},crate"
+            [ "${HELM_SUBSET:-all}" != "none" ]      && ARTIFACTS="${ARTIFACTS},helm"
+            ARTIFACTS="${ARTIFACTS#,}"
+            NIGHTLY_TAG=""
+            RC_TAG=""
+            RELEASE_TAG="v${VERSION}"
           else
             PIPELINE_TYPE="rc"
             RELEASE_TYPE="rc"
-            ARTIFACTS="wheel,container,helm"
+            # Subset-derived; defaults produce the historical "wheel,container,helm".
+            ARTIFACTS=""
+            [ "${WHEEL_SUBSET:-all}" != "none" ] && ARTIFACTS="${ARTIFACTS},wheel"
+            [ "${CONTAINER_SUBSET:-all}" != "none" ] && ARTIFACTS="${ARTIFACTS},container"
+            [ "${HELM_SUBSET:-all}" != "none" ] && ARTIFACTS="${ARTIFACTS},helm"
+            ARTIFACTS="${ARTIFACTS#,}"
             NIGHTLY_TAG=""
             RC_TAG="v${VERSION}-rc${RC_NUMBER}"
           fi
 
+          case "${ARTIFACTS}" in *crate*) ENABLE_CRATE_SCAN="true" ;; *) ENABLE_CRATE_SCAN="false" ;; esac
+
           # Pipe the trigger token via @file rather than inline form value so
           # it never appears in process listings or `set -x` output.
           # `trap` is registered BEFORE the write so an error between mktemp
@@ -1191,29 +1842,44 @@ jobs:
           chmod 600 "${TOKEN_FILE}"
           printf '%s' "${GITLAB_TRIGGER_TOKEN}" > "${TOKEN_FILE}"
 
+          # --form-string everywhere: --form treats a leading '<'/'@' as a file
+          # reference. Only the token deliberately uses --form "> "${GITHUB_OUTPUT}"
           echo "GitLab release pipeline triggered: pipeline ID ${PIPELINE_ID}"
 
+          # web_url embeds the internal GitLab host (kept secret) — public
+          # summary carries the numeric ID only; the URL goes to Slack.
           {
             echo "## GitLab Release Pipeline"
             echo ""
             echo "| Property | Value |"
             echo "|----------|-------|"
             echo "| Pipeline ID | ${PIPELINE_ID} |"
-            echo "| Pipeline URL | ${PIPELINE_URL:-n/a} |"
             echo "| PIPELINE_TYPE | ${PIPELINE_TYPE} |"
             echo "| RELEASE_TYPE | ${RELEASE_TYPE} |"
             echo "| Version | ${VERSION} |"
             echo "| NGC tag | ${NGC_VERSION_TAG} |"
             echo "| Commit | ${COMMIT_SHA} |"
             echo "| Artifacts | ${ARTIFACTS} |"
+            echo "| Crate staging | ${CRATES_STATUS:-n/a} |"
           } >> "${GITHUB_STEP_SUMMARY}"
 
       - name: Post GitLab trigger reply on Slack thread
@@ -1300,8 +1968,13 @@ jobs:
     name: Sync to Linear Release
     needs: [prepare-release, release-publish]
     # Skip on nightly: prepare-release sets version=nightly, which would create a
-    # spurious "Dynamo vnightly" release in Linear. Only sync for RC/release runs.
-    if: needs.release-publish.result == 'success' && needs.prepare-release.outputs.nightly != 'true'
+    # spurious "Dynamo vnightly" release in Linear. Also skip on-demand runs:
+    # they re-stage an existing version (possibly a .devN test version) whose
+    # Linear release is owned by the original RC flow. Only sync for RC/release runs.
+    if: >-
+      needs.release-publish.result == 'success' &&
+      needs.prepare-release.outputs.nightly != 'true' &&
+      needs.prepare-release.outputs.on_demand != 'true'
     runs-on: ubuntu-latest
     continue-on-error: true
     steps:

From 3694774ec7b2dc4edde40faca0d37dfe7968ca21 Mon Sep 17 00:00:00 2001
From: pvijayakrish 
Date: Thu, 27 Aug 2026 12:25:19 -0700
Subject: [PATCH 02/29] build: named release branches, ops-gated dispatch,
 PR-ready cleanup

- on-demand-release.yml: new `authorize` job gates every dispatch on
  active membership in the org team `vars.RELEASE_OPS_TEAM` (checked via
  the release app token; fail-closed). Bare named release branches
  v[-] (e.g. v1.3.0-nemotron-ultra-dev.1) are accepted for
  release_branch and source_ref, classified as real releases, and pushed
  with a release/ CI twin (bare refs match no push filter or
  runner admission; staging is SHA-keyed). Branch must start with v +
  the release_version base. Test scaffolding reverted (dispatch-only
  trigger, no version fallbacks, rc_round default 0); await-post-merge
  aligned to vars.AWS_DEFAULT_REGION / vars.ECR_REPOSITORY.
- apply_dev_version.py: fix NameError (PY_RUNTIME_PIN_RE ->
  PY_ROOT_PIN_RE) that crashed every --set-version stamp since the
  workspace-pin change.
- release.yml, nightly-ci.yml, on-demand-release.yml: drop the retired
  snapshot-agent image/chart everywhere, following its removal on main.

Signed-off-by: pvijayakrish 
---
 .github/scripts/apply_dev_version.py    |  12 +-
 .github/workflows/nightly-ci.yml        |   3 +-
 .github/workflows/on-demand-release.yml | 147 ++++++++++++++++--------
 .github/workflows/release.yml           |  23 ++--
 4 files changed, 117 insertions(+), 68 deletions(-)

diff --git a/.github/scripts/apply_dev_version.py b/.github/scripts/apply_dev_version.py
index 3cf57577044d..30252884bdd3 100755
--- a/.github/scripts/apply_dev_version.py
+++ b/.github/scripts/apply_dev_version.py
@@ -75,7 +75,6 @@
 HELM_CHART_TARGETS = [
     ("platform", "deploy/helm/charts/platform/Chart.yaml"),
     ("platform", "deploy/helm/charts/platform/components/operator/Chart.yaml"),
-    ("snapshot", "deploy/helm/charts/snapshot/Chart.yaml"),
 ]
 
 # First-party image `tag:` sites in values.yaml. Each entry is
@@ -92,16 +91,14 @@
      "nvcr.io/nvidia/ai-dynamo/kubernetes-operator"),
     ("operator", "platform", "deploy/helm/charts/platform/components/operator/values.yaml",
      "nvcr.io/nvidia/ai-dynamo/kubernetes-operator"),
-    ("snapshot", "snapshot", "deploy/helm/charts/snapshot/values.yaml",
-     "nvcr.io/nvidia/ai-dynamo/snapshot-agent"),
 ]
 
 # Normalized subset universes for --containers / --helm token validation.
 CONTAINER_TOKENS = {
     "vllm-runtime", "vllm-efa", "sglang-runtime", "sglang-efa",
-    "trtllm-runtime", "trtllm-efa", "frontend", "operator", "planner", "snapshot",
+    "trtllm-runtime", "trtllm-efa", "frontend", "operator", "planner",
 }
-HELM_TOKENS = {"platform", "snapshot"}
+HELM_TOKENS = {"platform"}
 
 # Container token -> the NGC repo release.yml actually publishes at :.
 # Used by --image-refs to rewrite the `my-registry`/`my-tag` placeholders in docs,
@@ -118,7 +115,6 @@
     "frontend": "dynamo-frontend",
     "operator": "kubernetes-operator",
     "planner": "dynamo-planner",
-    "snapshot": "snapshot-agent",
 }
 GA_REGISTRY = "nvcr.io/nvidia/ai-dynamo"
 PLACEHOLDER_REGISTRY = "my-registry"
@@ -273,7 +269,7 @@ def set_pyproject(path: Path, old: str, new: str, is_root: bool) -> None:
         count=1,
     )
     if is_root:
-        text = PY_RUNTIME_PIN_RE.sub(
+        text = PY_ROOT_PIN_RE.sub(
             lambda m: f"{m.group(1)}{new}{m.group(3)}" if m.group(2) == old else m.group(0),
             text,
         )
@@ -531,7 +527,7 @@ def main() -> int:
     ap.add_argument("--containers", default="all",
                     help="normalized container subset (all|none|csv) gating image-tag bumps")
     ap.add_argument("--helm", default="all",
-                    help="helm chart subset (all|none|csv of platform,snapshot) gating chart bumps")
+                    help="helm chart subset (all|none|platform) gating chart bumps")
     ap.add_argument("--image-refs", action="store_true",
                     help="also point the my-registry/my-tag placeholders in docs, examples and "
                          "deploy manifests at the GA registry + release version — only for images "
diff --git a/.github/workflows/nightly-ci.yml b/.github/workflows/nightly-ci.yml
index ba4d21589648..49df675016f6 100644
--- a/.github/workflows/nightly-ci.yml
+++ b/.github/workflows/nightly-ci.yml
@@ -330,8 +330,7 @@ jobs:
     secrets: inherit
 
   # ============================================================================
-  # OPTIONAL NIGHTLY IMAGES (operator, planner, frontend, runtime EFA
-  # variants)
+  # OPTIONAL NIGHTLY IMAGES (operator, planner, frontend, runtime EFA variants)
   # ============================================================================
   # Built HERE — not reused from post-merge — for two reasons:
   #   1. OSRB from the get-go: GitLab's nvbug:attach-compliance resolves
diff --git a/.github/workflows/on-demand-release.yml b/.github/workflows/on-demand-release.yml
index 8802b3cd99d2..a3b0d4716662 100644
--- a/.github/workflows/on-demand-release.yml
+++ b/.github/workflows/on-demand-release.yml
@@ -1,30 +1,22 @@
 # SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 
-# On-demand release: branch from main / a release branch / a main commit SHA,
-# set the release version, commit it to a new release/ branch (which
-# triggers post-merge CI to build the images), then stage all artifacts and
-# trigger the GitLab release-automation pipeline via release.yml.
+# On-demand release: branch from main / a release branch / a commit SHA, set
+# the release version, commit it to a release/ branch or a bare named
+# release branch (v[-], pushed with a release/ CI twin so
+# post-merge CI builds the images), then stage all artifacts and trigger the
+# GitLab release-automation pipeline via release.yml. Dispatch is restricted
+# to active members of the release-ops team (vars.RELEASE_OPS_TEAM).
 
 name: On-Demand Release
 
 # workflow_dispatch is capped at 10 inputs by GitHub — this block is at 10;
 # fold any new knob into an existing input.
 on:
-  # TEMPORARY(test): push events have no inputs, so the fallbacks below run the
-  # test flow as release_version 1.4.0 on the existing release/1.4.0 branch, rc10,
-  # all artifacts (containers/wheels/crates/helm=all). REVERT before merge.
-  push:
-    branches:
-      # copy-pr-bot's reviewed-copy branch. Velonix job admission only admits
-      # push on main, release/*, and pull-request/N, so the test loop runs off
-      # the PR copy: comment `/ok to test ` to refresh it. Same filter
-      # fern-docs.yml uses.
-      - pull-request/[0-9]+
   workflow_dispatch:
     inputs:
       source_ref:
-        description: "Branch to fork from: 'main', a 'release/x.y.z' branch, or a 40-char main commit SHA."
+        description: "Ref to fork from: 'main', a 'release/*' branch, a bare named release branch (v[-]), or a 40-char commit SHA."
         required: false
         type: string
         default: main
@@ -33,17 +25,17 @@ on:
         required: true
         type: string
       release_branch:
-        description: 'Branch to create (default release/). For a throwaway run use e.g. release/0.0.0test — it still matches release/*.*.* so post-merge + release-branch checks run, but the publish jobs skip.'
+        description: 'Branch to create (default release/). Also accepts a bare named release v[-], e.g. v1.3.0-nemotron-ultra-dev.1 — must start with v + the release_version base; a release/ CI twin is pushed for the builds, and artifact versions stay numeric (the name lives only in the branch). For a throwaway run use e.g. release/0.0.0test — post-merge + release-branch checks run, but the publish jobs skip.'
         required: false
         type: string
         default: ''
       rc_round:
-        description: 'Internal staging RC:  or . (e.g. 2 or 2.1 -> staging versions X.Y.Z-rc.2 / X.Y.Z-rc.2.1, a numerically-ordered SemVer pre-release; also the wheel rc subpath). Re-stages of the same version need a distinct value (immutable registries). NO GitHub tag is created. Default 10.'
+        description: 'Internal staging RC:  or . (e.g. 2 or 2.1 -> staging versions X.Y.Z-rc.2 / X.Y.Z-rc.2.1, a numerically-ordered SemVer pre-release; also the wheel rc subpath). Re-stages of the same version need a distinct value (immutable registries). NO GitHub tag is created. Default 0.'
         required: false
         type: string
-        default: '10'
+        default: '0'
       containers:
-        description: 'Containers to release: all | none | csv of vllm[:runtime|:efa],sglang[...],trtllm[...],frontend,operator,planner,snapshot.'
+        description: 'Containers to release: all | none | csv of vllm[:runtime|:efa],sglang[...],trtllm[...],frontend,operator,planner.'
         required: false
         type: string
         default: all
@@ -58,7 +50,7 @@ on:
         type: string
         default: all
       helm:
-        description: 'Helm charts to release: all | none | csv of platform,snapshot.'
+        description: 'Helm charts to release: all | none | platform.'
         required: false
         type: string
         default: all
@@ -77,6 +69,50 @@ permissions:
   contents: read
 
 jobs:
+  # ============================================================================
+  # OPS GATE — only active members of the release-ops GitHub team
+  # (vars.RELEASE_OPS_TEAM, in the repo's org) may run this workflow. Fail
+  # closed: an unset var, missing app org Members:read permission, or any API
+  # error denies. github.actor re-resolves on re-runs, so re-running someone
+  # else's run is re-checked too.
+  # ============================================================================
+  authorize:
+    name: Verify release-ops membership
+    runs-on: prod-default-small-v2
+    environment: automated-release
+    permissions:
+      contents: read
+    steps:
+      - name: Mint app token
+        id: app-token
+        uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1  # v3.2.0
+        with:
+          app-id: ${{ secrets.RELEASE_APP_ID }}
+          private-key: ${{ secrets.RELEASE_APP_PRIVATE_KEY }}
+      - name: Require release-ops team membership
+        env:
+          GH_TOKEN: ${{ steps.app-token.outputs.token }}
+          ORG: ${{ github.repository_owner }}
+          TEAM: ${{ vars.RELEASE_OPS_TEAM }}
+          ACTOR: ${{ github.actor }}
+        run: |
+          set -euo pipefail
+          if [ -z "${TEAM}" ]; then
+            echo "::error::vars.RELEASE_OPS_TEAM is not configured; denying every dispatch (fail closed)."
+            exit 1
+          fi
+          HTTP=$(curl -sS -o "${RUNNER_TEMP}/membership.json" -w '%{http_code}' \
+            -H "Authorization: Bearer ${GH_TOKEN}" \
+            -H "Accept: application/vnd.github+json" \
+            "https://api.github.com/orgs/${ORG}/teams/${TEAM}/memberships/${ACTOR}")
+          STATE=$(python3 -c "import json,sys;print(json.load(open(sys.argv[1])).get('state',''))" \
+            "${RUNNER_TEMP}/membership.json" 2>/dev/null || echo "")
+          if [ "${HTTP}" != "200" ] || [ "${STATE}" != "active" ]; then
+            echo "::error::@${ACTOR} is not an active member of ${ORG}/${TEAM} (HTTP ${HTTP}); only release ops may run this workflow."
+            exit 1
+          fi
+          echo "@${ACTOR} authorized via ${ORG}/${TEAM}"
+
   # ============================================================================
   # CREATE RELEASE BRANCH + VERSION BUMP
   # Pushes with a dedicated token (not GITHUB_TOKEN) so the push triggers
@@ -84,6 +120,7 @@ jobs:
   # ============================================================================
   prepare-branch:
     name: Create release branch + version bump
+    needs: [authorize]
     runs-on: prod-default-small-v2
     environment: automated-release
     permissions:
@@ -100,12 +137,11 @@ jobs:
     steps:
       - name: Validate inputs
         env:
-          # TEMPORARY(test): the || fallbacks cover inputless push events. REVERT
-          # to bare inputs.* before merge.
-          RELEASE_VERSION: ${{ inputs.release_version || '1.4.0' }}
-          SOURCE_REF: ${{ inputs.source_ref || 'main' }}
-          RELEASE_BRANCH: ${{ inputs.release_branch || (github.event_name == 'push' && 'release/1.4.0' || '') }}
-          RC_ROUND: ${{ inputs.rc_round || '10' }}
+          RELEASE_VERSION: ${{ inputs.release_version }}
+          SOURCE_REF: ${{ inputs.source_ref }}
+          RELEASE_BRANCH: ${{ inputs.release_branch }}
+          # || guards a cleared dispatch field (inputs default fills the rest).
+          RC_ROUND: ${{ inputs.rc_round || '0' }}
         run: |
           set -euo pipefail
           if [[ ! "${RELEASE_VERSION}" =~ ^[0-9]+\.[0-9]+\.[0-9]+(\.(dev|post)[0-9]+)?$ ]]; then
@@ -117,12 +153,22 @@ jobs:
             echo "::error::rc_round must be '' or '.' (non-negative integers; got '${RC_ROUND}')"
             exit 1
           fi
-          if [[ -n "${RELEASE_BRANCH}" && ! "${RELEASE_BRANCH}" =~ ^release/[A-Za-z0-9._-]+$ ]]; then
-            echo "::error::release_branch must look like release/ (got '${RELEASE_BRANCH}')"
+          if [[ -n "${RELEASE_BRANCH}" && ! "${RELEASE_BRANCH}" =~ ^release/[A-Za-z0-9._-]+$ \
+                && ! "${RELEASE_BRANCH}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-[A-Za-z0-9._-]+)?$ ]]; then
+            echo "::error::release_branch must be release/ or a bare named release v[-] (got '${RELEASE_BRANCH}')"
             exit 1
           fi
-          if [[ ! "${SOURCE_REF}" =~ ^(main|release/[0-9]+\.[0-9]+\.[0-9]+(\.(dev|post)[0-9]+)?|[0-9a-f]{40})$ ]]; then
-            echo "::error::source_ref must be 'main', a release/x.y.z branch, or a 40-char SHA (got '${SOURCE_REF}')"
+          # A bare named branch must carry the version it stamps (v-);
+          # artifact versions stay numeric, so this prefix is the only tie between them.
+          if [[ "${RELEASE_BRANCH}" =~ ^v[0-9] ]]; then
+            BASE="${RELEASE_VERSION%%.dev*}"; BASE="${BASE%%.post*}"
+            if [[ "${RELEASE_BRANCH}" != "v${BASE}" && "${RELEASE_BRANCH}" != "v${BASE}-"* ]]; then
+              echo "::error::named branch '${RELEASE_BRANCH}' must start with v${BASE} (the release_version base)"
+              exit 1
+            fi
+          fi
+          if [[ ! "${SOURCE_REF}" =~ ^(main|release/[A-Za-z0-9._-]+|v[0-9]+\.[0-9]+\.[0-9]+(-[A-Za-z0-9._-]+)?|[0-9a-f]{40})$ ]]; then
+            echo "::error::source_ref must be 'main', a release/* branch, a named release branch (v[-]), or a 40-char SHA (got '${SOURCE_REF}')"
             exit 1
           fi
 
@@ -153,9 +199,9 @@ jobs:
 
           FRAMEWORKS = {"vllm", "sglang", "trtllm"}
           VARIANTS = {"runtime", "efa"}
-          COMPONENTS = {"frontend", "operator", "planner", "snapshot"}
+          COMPONENTS = {"frontend", "operator", "planner"}
           WHEEL_TOKENS = {"ai-dynamo", "ai-dynamo-runtime", "kvbm"}
-          HELM_TOKENS = {"platform", "snapshot"}
+          HELM_TOKENS = {"platform"}
 
           def fail(msg):
               print(f"::error::{msg}", file=sys.stderr)
@@ -254,17 +300,15 @@ jobs:
       - name: Checkout source
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
         with:
-          # TEMPORARY(test): || 'main' covers inputless push events. REVERT.
-          ref: ${{ inputs.source_ref || 'main' }}
+          ref: ${{ inputs.source_ref }}
           fetch-depth: 0
           token: ${{ steps.app-token.outputs.token }}
 
       - name: Set version and create branch
         id: bump
         env:
-          # TEMPORARY(test): push fallbacks. REVERT to bare inputs.* before merge.
-          RELEASE_VERSION: ${{ inputs.release_version || '1.4.0' }}
-          RELEASE_BRANCH: ${{ inputs.release_branch || (github.event_name == 'push' && 'release/1.4.0' || '') }}
+          RELEASE_VERSION: ${{ inputs.release_version }}
+          RELEASE_BRANCH: ${{ inputs.release_branch }}
           APP_SLUG: ${{ steps.app-token.outputs.app-slug }}
           GH_TOKEN: ${{ steps.app-token.outputs.token }}
           CONTAINERS: ${{ steps.resolve.outputs.containers }}
@@ -277,10 +321,18 @@ jobs:
           set -euo pipefail
 
           BRANCH="${RELEASE_BRANCH:-release/${RELEASE_VERSION}}"
-          # A pure release/X.Y.Z[.devN|.postN] branch is a real release; anything
-          # else (e.g. release/0.0.0test) is a throwaway test branch.
+          # Real releases: pure release/X.Y.Z[.devN|.postN], or a bare named release
+          # v[-] (e.g. v1.3.0-nemotron-ultra-dev.1). Anything else
+          # (e.g. release/0.0.0test) is a throwaway test branch. Bare branches match
+          # no CI push filter (post-merge and runner admission fire only on
+          # main/release/*), so a release/ CI twin is pushed alongside to
+          # build the images; staging consumes the SHA, not the ref.
+          CI_BRANCH="${BRANCH}"
           if [[ "${BRANCH}" =~ ^release/[0-9]+\.[0-9]+\.[0-9]+(\.(dev|post)[0-9]+)?$ ]]; then
             IS_TEST=false
+          elif [[ "${BRANCH}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-[A-Za-z0-9._-]+)?$ ]]; then
+            IS_TEST=false
+            CI_BRANCH="release/${BRANCH}"
           else
             IS_TEST=true
           fi
@@ -346,6 +398,11 @@ jobs:
             git commit -s -m "release: set version ${RELEASE_VERSION}" -m "${SELECTION_MSG}"
             git push origin "${BRANCH}"
           fi
+          # Keep the CI twin of a bare named branch on this commit (fast-forward;
+          # no-op when already there) so post-merge CI builds it.
+          if [ "${CI_BRANCH}" != "${BRANCH}" ]; then
+            git push origin "HEAD:refs/heads/${CI_BRANCH}"
+          fi
 
           echo "branch=${BRANCH}" >> "$GITHUB_OUTPUT"
           echo "bump_sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
@@ -357,7 +414,7 @@ jobs:
         # A dry run sets up the branch/bump only — don't consume an rc.
         if: ${{ !inputs.dry_run }}
         env:
-          RC_ROUND: ${{ inputs.rc_round || '10' }}
+          RC_ROUND: ${{ inputs.rc_round || '0' }}
         run: |
           set -euo pipefail
           # Internal staging registries (crates/helm/wheels) are immutable, so each
@@ -399,7 +456,8 @@ jobs:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           REPO: ${{ github.repository }}
           SHA: ${{ needs.prepare-branch.outputs.bump_sha }}
-          AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
+          AWS_DEFAULT_REGION: ${{ vars.AWS_DEFAULT_REGION }}
+          ECR_REPOSITORY: ${{ vars.ECR_REPOSITORY }}
           CONTAINERS: ${{ needs.prepare-branch.outputs.containers }}
           WHEELS: ${{ needs.prepare-branch.outputs.wheels }}
           CRATES: ${{ needs.prepare-branch.outputs.crates }}
@@ -421,7 +479,7 @@ jobs:
             [vllm-runtime]=vllm-runtime     [vllm-efa]=vllm-runtime-efa
             [sglang-runtime]=sglang-runtime [sglang-efa]=sglang-runtime-efa
             [trtllm-runtime]=trtllm-runtime [trtllm-efa]=trtllm-runtime-efa
-            [frontend]=dynamo-frontend [operator]=operator [planner]=dynamo-planner [snapshot]=snapshot-agent
+            [frontend]=dynamo-frontend [operator]=operator [planner]=dynamo-planner
           )
           declare -a TAGS=()
           for tok in "${!CTR[@]}"; do
@@ -444,7 +502,7 @@ jobs:
           while :; do
             missing=()
             for TAG in "${TAGS[@]}"; do
-              if ! aws ecr describe-images --repository-name "ai-dynamo/dynamo" \
+              if ! aws ecr describe-images --repository-name "${ECR_REPOSITORY}" \
                      --image-ids imageTag="${TAG}" --region "${AWS_DEFAULT_REGION}" >/dev/null 2>&1; then
                 missing+=("${TAG}")
               fi
@@ -514,8 +572,7 @@ jobs:
     with:
       commit_sha: ${{ needs.prepare-branch.outputs.bump_sha }}
       on_demand: true
-      # TEMPORARY(test): || fallback for inputless push events. REVERT.
-      release_version: ${{ inputs.release_version || '1.4.0' }}
+      release_version: ${{ inputs.release_version }}
       # Operator-chosen internal-staging rc (immutable registries can't be overwritten).
       rc_number: ${{ needs.prepare-branch.outputs.rc_number }}
       # Normalized subsets from resolve-selection.
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index c9a24bffb47e..9987dd1de917 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -33,7 +33,7 @@ on:
         type: string
         default: ''
       containers:
-        description: 'Container subset to publish (all|none|csv of vllm-runtime,vllm-efa,sglang-runtime,sglang-efa,trtllm-runtime,trtllm-efa,frontend,operator,planner,snapshot). Partial rc re-publishes supported.'
+        description: 'Container subset to publish (all|none|csv of vllm-runtime,vllm-efa,sglang-runtime,sglang-efa,trtllm-runtime,trtllm-efa,frontend,operator,planner). Partial rc re-publishes supported.'
         required: false
         type: string
         default: all
@@ -48,7 +48,7 @@ on:
         type: string
         default: all
       helm:
-        description: 'Helm chart subset to push (all|none|csv of platform,snapshot).'
+        description: 'Helm chart subset to push (all|none|platform).'
         required: false
         type: string
         default: all
@@ -207,9 +207,9 @@ jobs:
               esac
             done
           }
-          validate_subset containers "${CONTAINER_SUBSET:-all}" "vllm-runtime,vllm-efa,sglang-runtime,sglang-efa,trtllm-runtime,trtllm-efa,frontend,operator,planner,snapshot"
+          validate_subset containers "${CONTAINER_SUBSET:-all}" "vllm-runtime,vllm-efa,sglang-runtime,sglang-efa,trtllm-runtime,trtllm-efa,frontend,operator,planner"
           validate_subset wheels "${WHEEL_SUBSET:-all}" "ai-dynamo,ai-dynamo-runtime,kvbm"
-          validate_subset helm "${HELM_SUBSET:-all}" "platform,snapshot"
+          validate_subset helm "${HELM_SUBSET:-all}" "platform"
           if [[ ! "${CRATE_SUBSET:-all}" =~ ^[A-Za-z0-9_,-]*$ ]]; then
             echo "Error: crate subset may only contain [A-Za-z0-9_-] names separated by commas (got '${CRATE_SUBSET}')"
             exit 1
@@ -489,7 +489,6 @@ jobs:
               "frontend|${COMMIT_SHA}-dynamo-frontend"
               "operator|${COMMIT_SHA}-operator"
               "planner|${COMMIT_SHA}-dynamo-planner"
-              "snapshot|${COMMIT_SHA}-snapshot-agent"
             )
           fi
 
@@ -589,7 +588,7 @@ jobs:
 
           # Copy only the selected container subset; nightly/rc default to the full
           # set (SUBSET=all). Tokens: -runtime (bare + -cuda13 alias),
-          # -efa, frontend, operator, planner, snapshot.
+          # -efa, frontend, operator, planner.
           SUBSET="${CONTAINER_SUBSET:-all}"
           want() {
             [ "${SUBSET}" = "all" ] && return 0
@@ -696,9 +695,8 @@ jobs:
               TARGET="${NGC_REGISTRY}/${NGC_ORG}/ai-dynamo/dynamo-planner:${NGC_VERSION_TAG}"
               copy_image "${SOURCE}" "${TARGET}" "dynamo-planner:${NGC_VERSION_TAG}" || true
             fi
-
           else
-            # Optional nightly images — operator, planner, frontend, and the
+            # Optional nightly images — operator, planner, frontend and the
             # three runtime EFA variants — are built by THIS
             # nightly run (ordering-only needs of the release, deliberately
             # outside its success gate) and staged fail-soft: a build flake
@@ -846,7 +844,6 @@ jobs:
           # Each site: chart_token|container_token|values.yaml|image repo (as referenced by the chart)
           SITES=(
             "platform|operator|deploy/helm/charts/platform/components/operator/values.yaml|nvcr.io/nvidia/ai-dynamo/kubernetes-operator"
-            "snapshot|snapshot|deploy/helm/charts/snapshot/values.yaml|nvcr.io/nvidia/ai-dynamo/snapshot-agent"
           )
           for SITE in "${SITES[@]}"; do
             IFS='|' read -r CHART CTOK VALUES REPO <<<"${SITE}"
@@ -898,11 +895,11 @@ jobs:
           echo "" >> $GITHUB_STEP_SUMMARY
           echo "### Helm Charts" >> $GITHUB_STEP_SUMMARY
 
-          # snapshot + dynamo-platform, mirroring the rcs/dynamo helm runbook.
+          # dynamo-platform, mirroring the rcs/dynamo helm runbook.
           # Only the selected charts are pushed (token = chart dir name);
-          # nightly/rc default to both (HELM_SUBSET defaults to 'all').
+          # nightly/rc default to all (HELM_SUBSET defaults to 'all').
           STAGED_HELM=""
-          for CHART_DIR in deploy/helm/charts/platform deploy/helm/charts/snapshot; do
+          for CHART_DIR in deploy/helm/charts/platform; do
             TOKEN="$(basename "${CHART_DIR}")"
             if [ "${HELM_SUBSET:-all}" != "all" ]; then
               case ",${HELM_SUBSET}," in *,"${TOKEN}",*) ;; *) echo "skip ${TOKEN} chart (not selected)"; continue ;; esac
@@ -927,7 +924,7 @@ jobs:
 
           helm repo remove "${REPO_ALIAS}"
 
-      # Nightly: package the platform chart at the dated pre-release version
+      # Nightly: package platform at the dated pre-release version
       # and dual-push to NGC staging (ChartMuseum + OCI). Values are rewritten
       # in the workspace only, onto the *-nightly repos at this run's tag.
       # Fail-soft per chart via the copy step's *_staged flags; fully fail-soft

From fffbe7005f3ad1d3aa6b868803e79de182dd4bd4 Mon Sep 17 00:00:00 2001
From: pvijayakrish 
Date: Thu, 27 Aug 2026 14:11:59 -0700
Subject: [PATCH 03/29] build: named release branches, ops-gated dispatch,
 PR-ready cleanup

- on-demand-release.yml: new `authorize` job gates every dispatch on
  active membership in the org team `vars.RELEASE_OPS_TEAM` (checked via
  the release app token; fail-closed). Bare named release branches
  v[-] (e.g. v1.3.0-nemotron-ultra-dev.1) are accepted for
  release_branch and source_ref, classified as real releases, and pushed
  with a release/ CI twin (bare refs match no push filter or
  runner admission; staging is SHA-keyed). Branch must start with v +
  the release_version base. Test scaffolding reverted (dispatch-only
  trigger, no version fallbacks, rc_round default 0); await-post-merge
  aligned to vars.AWS_DEFAULT_REGION / vars.ECR_REPOSITORY.
- apply_dev_version.py: fix NameError (PY_RUNTIME_PIN_RE ->
  PY_ROOT_PIN_RE) that crashed every --set-version stamp since the
  workspace-pin change.
- Carried from the retired on-demand-release branch (post-port fixes):
  sync every nested-workspace Cargo.lock on the version bump (root-only
  update shipped three stale locks in the 1.4.1 cut and failed the
  clippy matrix), and skip docs/fern/components/releases.data.ts in
  --image-refs with a loud warning instead of half-rewriting it.
- release.yml, nightly-ci.yml, on-demand-release.yml: drop the retired
  snapshot-agent image/chart everywhere, following its removal on main.

Signed-off-by: pvijayakrish 
---
 .github/scripts/apply_dev_version.py    | 23 +++++++++++++++++++++++
 .github/workflows/on-demand-release.yml | 17 +++++++++++++++--
 2 files changed, 38 insertions(+), 2 deletions(-)

diff --git a/.github/scripts/apply_dev_version.py b/.github/scripts/apply_dev_version.py
index 30252884bdd3..895f8fbb5be3 100755
--- a/.github/scripts/apply_dev_version.py
+++ b/.github/scripts/apply_dev_version.py
@@ -119,6 +119,17 @@
 GA_REGISTRY = "nvcr.io/nvidia/ai-dynamo"
 PLACEHOLDER_REGISTRY = "my-registry"
 
+# Hand-maintained docs data listing MANY releases at once: the current GA rows,
+# per-model dev lines (`tag: "1.4.0-inkling-dev.1"`, `releaseLine: "v1.4.0"`) and
+# dated nightlies (`version: "1.4.0.dev20260803"`) all carry the same version
+# literal, and each row repeats it in a `label:` beside the `clipboard:`.
+# rewrite_image_refs only understands `/:`, so on a re-cut it moved
+# 7 clipboards and left every label behind -- publishing a page whose copy button
+# contradicts its own label, while the historical entries are ones that must never
+# move at all. Telling those apart needs a TS parse, not a regex, so the file is
+# skipped wholesale and the release owner is warned to update it by hand.
+IMAGE_REF_SKIP = frozenset({"docs/fern/components/releases.data.ts"})
+
 # .devN is a PRE-release (sorts before X.Y.Z) -> SemVer '-devN'; .postN is a
 # post-release -> SemVer build metadata '+postN'. Both keep PEP 440 form for Python.
 SET_RE = re.compile(r"^(\d+)\.(\d+)\.(\d+)(?:\.(dev\d+|post\d+))?$")
@@ -409,11 +420,18 @@ def rewrite_image_refs(root: Path, new_version: str, containers: set[str],
     ]
 
     files_changed = refs = 0
+    skipped_stale = []
     for path in _tracked_files(root):
         try:
             text = original = path.read_text()
         except (UnicodeDecodeError, OSError):
             continue  # binary or unreadable — nothing to substitute
+        if path.relative_to(root).as_posix() in IMAGE_REF_SKIP:
+            # All-or-nothing: a partial rewrite here is worse than none (see
+            # IMAGE_REF_SKIP). Record it so the operator gets a loud warning.
+            if any(t in text for t in old_literals if t != "my-tag"):
+                skipped_stale.append(path.relative_to(root).as_posix())
+            continue
         # Fast path — must test every literal the regex can match, including the
         # PEP 440 spelling of the old version and the untagged placeholder registry.
         if not any(t in text for t in old_literals) and PLACEHOLDER_REGISTRY not in text:
@@ -428,6 +446,11 @@ def rewrite_image_refs(root: Path, new_version: str, containers: set[str],
     print(f"rewrite_image_refs: {refs} reference(s) in {files_changed} file(s) -> "
           f"{GA_REGISTRY}/:{new_version} for {images}", file=sys.stderr)
 
+    for rel in skipped_stale:
+        print(f"::warning::{rel} still references {old_version} and was left untouched "
+              f"on purpose (it lists dev-line and nightly releases that must not move). "
+              f"Update the current-release rows to {new_version} by hand.", file=sys.stderr)
+
     # Advisory only: a re-cut of a branch that previously shipped a wider selection
     # legitimately still carries those older refs, so warn rather than fail.
     unselected = sorted(set(IMAGE_REF_TOKENS.values()) - set(images))
diff --git a/.github/workflows/on-demand-release.yml b/.github/workflows/on-demand-release.yml
index a3b0d4716662..d580dbe7c0d9 100644
--- a/.github/workflows/on-demand-release.yml
+++ b/.github/workflows/on-demand-release.yml
@@ -369,8 +369,21 @@ jobs:
               | sh -s -- -y --profile minimal --default-toolchain stable
             source "${HOME}/.cargo/env"
           fi
-          # Keep Cargo.lock in sync so post-merge `cargo test --locked` passes.
-          cargo update --workspace
+          # Keep EVERY Cargo.lock in sync so post-merge `cargo test --locked` and the
+          # pre-merge `cargo metadata --locked` matrix both pass. lib/bindings/python,
+          # lib/bindings/kvbm and lib/runtime/examples are SEPARATE cargo workspaces
+          # with their own lock files, so a root-only `cargo update --workspace` leaves
+          # their workspace-crate pins on the PREVIOUS version. That is what shipped in
+          # the 1.4.1 cut (9f6c6be7): three stale locks, and the clippy matrix — which
+          # runs `cargo metadata --locked` in each of those dirs — failed on all three.
+          # Discovered from git, not hardcoded, so a newly added nested workspace cannot
+          # silently regress the next release.
+          git ls-files | grep -E '(^|/)Cargo\.lock$' | sort | while IFS= read -r lock; do
+            manifest="$(dirname "${lock}")/Cargo.toml"
+            manifest="${manifest#./}"
+            echo "Syncing ${lock}"
+            cargo update --workspace --manifest-path "${manifest}"
+          done
 
           APP_USER_ID=$(curl -fsSL --retry 3 \
             -H "Authorization: Bearer ${GH_TOKEN}" \

From a41b6d7dad69a92735cdbd9a493455ea70aa2e0a Mon Sep 17 00:00:00 2001
From: pvijayakrish 
Date: Fri, 28 Aug 2026 10:04:47 -0700
Subject: [PATCH 04/29] fix(release): address PR review findings across the
 on-demand flow

apply_dev_version.py: match each file's own spelling of the old version
(Cargo holds SemVer '1.4.2-dev1', pyprojects PEP 440 '1.4.2.dev1'), so
re-stamping an already-stamped .devN/.postN branch moves the pyprojects
and the ai-dynamo-runtime pin too. set_pyproject/set_cargo now return
hit counts and set_release_version refuses a partial stamp when a
must-move file carries neither the old nor the new version (independent
versions exempt; pre-pinned or pin-less member manifests tolerated).

stage_crates.py: append the registry config to .cargo/config.toml
instead of overwriting it (rustflags/PCRE2_SYS_STATIC survive;
idempotent via marker; foreign registry tables refused, header-anchored
check). Scrub the registry host from crate_exists error messages.

on-demand-release.yml: ops gate hardcoded to the ai-dynamo/devops team
and checks BOTH github.actor and github.triggering_actor (actor does
not re-resolve on re-runs). refs/heads/ refspecs on fetch/push so a
bare v-branch cannot collide with a same-named tag. Separator-only
subset csv normalizes to 'none' instead of '' (which read as 'all'
downstream). jq failure flags set in the parent shell, not inside
command substitution. Loud warning when a branch classifies as a
throwaway test branch.

release.yml: the partially-staged-containers gate covers rc dispatches
too, not only on-demand (nightly stays warn-skip by design). rc
dispatch now verifies the tree's pyproject version matches the branch
version before staging. Mask the cargo registry host in the crates job
log. Clearer dispatch error for named release branches.

lib/backend-common/Cargo.toml: dynamo-llm pin rides the workspace
version (1.5.0).

Signed-off-by: pvijayakrish 
---
 .github/scripts/apply_dev_version.py    | 69 +++++++++++++++------
 .github/scripts/stage_crates.py         | 36 +++++++----
 .github/workflows/on-demand-release.yml | 79 ++++++++++++++-----------
 .github/workflows/release.yml           | 44 +++++++++-----
 lib/backend-common/Cargo.toml           |  2 +-
 5 files changed, 151 insertions(+), 79 deletions(-)

diff --git a/.github/scripts/apply_dev_version.py b/.github/scripts/apply_dev_version.py
index 895f8fbb5be3..5da2bd55dec6 100755
--- a/.github/scripts/apply_dev_version.py
+++ b/.github/scripts/apply_dev_version.py
@@ -123,11 +123,11 @@
 # per-model dev lines (`tag: "1.4.0-inkling-dev.1"`, `releaseLine: "v1.4.0"`) and
 # dated nightlies (`version: "1.4.0.dev20260803"`) all carry the same version
 # literal, and each row repeats it in a `label:` beside the `clipboard:`.
-# rewrite_image_refs only understands `/:`, so on a re-cut it moved
-# 7 clipboards and left every label behind -- publishing a page whose copy button
-# contradicts its own label, while the historical entries are ones that must never
-# move at all. Telling those apart needs a TS parse, not a regex, so the file is
-# skipped wholesale and the release owner is warned to update it by hand.
+# rewrite_image_refs only understands `/:`, so on a re-cut it would
+# move each row's clipboard ref while leaving the `label:` beside it -- publishing
+# a page whose copy button contradicts its own label -- and the historical rows
+# must never move at all. Telling those apart needs a TS parse, not a regex, so
+# the file is skipped wholesale and the release owner is warned to update it by hand.
 IMAGE_REF_SKIP = frozenset({"docs/fern/components/releases.data.ts"})
 
 # .devN is a PRE-release (sorts before X.Y.Z) -> SemVer '-devN'; .postN is a
@@ -273,27 +273,34 @@ def _semver_form(new: str) -> str:
     return f"{base}-{suffix}" if suffix.startswith("dev") else f"{base}+{suffix}"
 
 
-def set_pyproject(path: Path, old: str, new: str, is_root: bool) -> None:
-    text = VERSION_LINE_RE.sub(
-        lambda m: f"{m.group(1)}{new}{m.group(3)}" if m.group(2) == old else m.group(0),
-        path.read_text(),
-        count=1,
-    )
+def set_pyproject(path: Path, old: str, new: str, is_root: bool) -> int:
+    hits = 0
+
+    def _set(m: re.Match) -> str:
+        nonlocal hits
+        if m.group(2) != old:
+            return m.group(0)
+        hits += 1
+        return f"{m.group(1)}{new}{m.group(3)}"
+
+    text = VERSION_LINE_RE.sub(_set, path.read_text())
     if is_root:
         text = PY_ROOT_PIN_RE.sub(
             lambda m: f"{m.group(1)}{new}{m.group(3)}" if m.group(2) == old else m.group(0),
             text,
         )
     path.write_text(text)
+    return hits
 
 
-def set_cargo(path: Path, old: str, new: str) -> None:
-    text = re.sub(
+def set_cargo(path: Path, old: str, new: str) -> int:
+    text, n = re.subn(
         rf'(\bversion\s*=\s*"){re.escape(old)}(")',
         lambda m: f"{m.group(1)}{new}{m.group(2)}",
         path.read_text(),
     )
     path.write_text(text)
+    return n
 
 
 def set_helm(path: Path, old: str, new: str) -> None:
@@ -495,23 +502,49 @@ def _exists(rel: str) -> bool:
         print(f"set_release_version: skip {rel} (absent at this source ref)", file=sys.stderr)
         return False
 
+    # Cargo.toml holds the workspace version in SemVer form ('1.4.2-dev1'),
+    # pyprojects in PEP 440 form ('1.4.2.dev1'): re-stamping an already-stamped
+    # .devN/.postN branch must match each file's own spelling of the old version.
+    old_py = re.sub(r"[-+](dev|post)", r".\1", old)
+
+    # These files legitimately carry their own version (never the workspace's),
+    # so a zero-hit rewrite is expected for them. Anywhere else, zero hits with
+    # the new version also absent means the file holds some third version and
+    # the release would ship stale metadata.
+    independent = {"lib/gpu_memory_service/pyproject.toml",
+                   "lib/bindings/python/codegen/Cargo.toml"}
+
+    def _require(rel: str, hits: int, want: str) -> None:
+        if rel in independent or hits:
+            return
+        if want not in (root / rel).read_text():
+            raise RuntimeError(
+                f"{rel} carries neither the workspace version ('{old}'/'{old_py}') "
+                f"nor '{want}' -- refusing a partial stamp")
+
     # Package identity -- ALWAYS bumped, regardless of the wheels/crates selection:
     # the containers embed wheels built from this tree, so a container-only release
     # still needs the workspace/pyproject versions stamped or the shipped image would
     # carry the previous version. (wheels/crates are intentionally not passed in.)
     for rel in PYPROJECT_TARGETS:
         if rel == "pyproject.toml" or _exists(rel):
-            set_pyproject(root / rel, old, new_version, is_root=(rel == "pyproject.toml"))
-    set_cargo(root / "Cargo.toml", old, semver)
+            _require(rel, set_pyproject(root / rel, old_py, new_version,
+                                        is_root=(rel == "pyproject.toml")), new_version)
+    _require("Cargo.toml", set_cargo(root / "Cargo.toml", old, semver), semver)
     for rel in SUBCRATE_CARGO_TARGETS:
         if _exists(rel):
-            set_cargo(root / rel, old, semver)
+            _require(rel, set_cargo(root / rel, old, semver), semver)
     # Workspace-version pins in member manifests (inline dep tables, e.g.
     # backend-common's dynamo-llm pin) carry the same literal as the root
-    # pins; set_cargo's exact-string replace stamps them identically.
+    # pins; a pin pre-committed at the new version passes via the _require
+    # already-stamped tolerance. Older refs have a bare path dep with no
+    # version key at all — nothing to stamp (stage_crates injects the version
+    # at publish), so the guard only applies when a pin exists.
     for rel in WORKSPACE_PIN_CARGO_TARGETS:
         if _exists(rel):
-            set_cargo(root / rel, old, semver)
+            n = set_cargo(root / rel, old, semver)
+            if re.search(r"path\s*=[^}\n]*\bversion\s*=", (root / rel).read_text()):
+                _require(rel, n, semver)
     # Chart identity -- only for charts in the --helm subset.
     for token, rel in HELM_CHART_TARGETS:
         if token in helm and _exists(rel):
diff --git a/.github/scripts/stage_crates.py b/.github/scripts/stage_crates.py
index d60cc86b29da..6c3b4d36736d 100644
--- a/.github/scripts/stage_crates.py
+++ b/.github/scripts/stage_crates.py
@@ -32,6 +32,7 @@
 import sys
 import time
 import urllib.error
+import urllib.parse
 import urllib.request
 from pathlib import Path
 
@@ -40,15 +41,14 @@
 # inference-gateway ext-proc) that are not library crates consumers pull from the
 # registry, and whose build scripts (envoy/protoc codegen) aren't wired up here.
 EXCLUDE_PATH_SUBSTRINGS = ("/bindings/", "/examples/", "/deploy/")
-# Workspace members never published from the on-demand flow: internal crates that
-# carry an independent version (not the workspace version) and that no published
-# crate depends on. kvbm-consolidator pins its own 1.2.0, so the version gate would
-# (correctly) abort the release; it is a leaf, so excluding it is closure-safe.
+# Workspace members never published from the on-demand flow: internal
+# service/tool crates that no published crate depends on (leaves — excluding
+# them is closure-safe) and that registry consumers never pull.
 EXCLUDE_NAMES = frozenset({"kvbm-consolidator"})
 # Artifactory < 7.148 rejects the `Content-Type: application/octet-stream` header
 # cargo >= 1.96 sends on publish with a 415 (JFrog RTDEV-83141, rust-lang/cargo#17086),
-# so the upload runs on the last pre-header toolchain. Remove once
-# artifactory.nvidia.com runs Artifactory 7.148+.
+# so the upload runs on the last pre-header toolchain. Remove once the
+# registry host runs Artifactory 7.148+.
 PUBLISH_TOOLCHAIN = "1.95.0"
 # No bare "HTTP/2" alternation here: cargo dumps the raw status line ("HTTP/2 415")
 # on every HTTP error, which made deterministic 4xx failures retry as transient.
@@ -106,9 +106,22 @@ def topo_order(pkgs: dict[str, dict]) -> list[str]:
 
 
 def write_cargo_config(root: Path, alias: str, index: str) -> None:
+    # Append to the workspace .cargo/config.toml -- it carries required build
+    # settings (rustflags, PCRE2_SYS_STATIC) that an overwrite would drop.
+    # The marker makes re-runs idempotent (no duplicated tables).
+    marker = "# --- stage_crates registry (generated) ---"
     cfg = root / ".cargo" / "config.toml"
     cfg.parent.mkdir(exist_ok=True)
+    text = cfg.read_text() if cfg.exists() else ""
+    if marker in text:
+        # Drop our previously generated tail (anything after the marker is ours).
+        text = text.split(marker)[0].rstrip() + "\n"
+    # Header-anchored so [registries.x]/[registry] count but comments don't;
+    # checked AFTER stripping our block so a re-run can't mask a foreign table.
+    if re.search(r"(?m)^\s*\[registr(?:y|ies)[\].]", text):
+        raise RuntimeError(".cargo/config.toml already defines registry tables; refusing to append conflicting ones")
     cfg.write_text(
+        f"{text}\n{marker}\n"
         f'[registries]\n{alias} = {{ index = "{index}" }}\n\n'
         f'[registry]\ndefault = "{alias}"\n'
         'global-credential-providers = ["cargo:token"]\n'
@@ -216,9 +229,11 @@ def crate_exists(raw_base: str, name: str, version: str, token: str) -> bool:
     except urllib.error.HTTPError as e:
         if e.code == 404:
             return False
-        raise RuntimeError(f"registry HEAD {url} failed: HTTP {e.code} (bad ARTIFACTORY_TOKEN or registry outage?)") from e
+        # Path only — the registry host is workflow-secret-derived and error
+        # strings end up in public Actions logs.
+        raise RuntimeError(f"registry HEAD {urllib.parse.urlparse(url).path} failed: HTTP {e.code} (bad ARTIFACTORY_TOKEN or registry outage?)") from e
     except urllib.error.URLError as e:
-        raise RuntimeError(f"registry HEAD {url} unreachable: {e.reason}") from e
+        raise RuntimeError(f"registry HEAD {urllib.parse.urlparse(url).path} unreachable: {e.reason}") from e
 
 
 def publish(manifest: str, alias: str, env: dict) -> str:
@@ -363,8 +378,9 @@ def main() -> int:
         [root / "Cargo.toml", *(Path(pkgs[n]["manifest_path"]) for n in order)],
         order, args.registry, {n: pkgs[n]["version"] for n in order},
     )
-    # cargo publish rejects deps without a registry version; drop the optional,
-    # private git deps (e.g. aiconfigurator-core) from the crates being published.
+    # cargo publish rejects deps without a registry version; drop optional,
+    # private git deps from the crates being published (none on current main;
+    # older release branches still carry e.g. aiconfigurator-core).
     strip_git_deps(root, [Path(pkgs[n]["manifest_path"]) for n in order])
 
     env = dict(os.environ)
diff --git a/.github/workflows/on-demand-release.yml b/.github/workflows/on-demand-release.yml
index d580dbe7c0d9..96c2c0440b07 100644
--- a/.github/workflows/on-demand-release.yml
+++ b/.github/workflows/on-demand-release.yml
@@ -6,7 +6,7 @@
 # release branch (v[-], pushed with a release/ CI twin so
 # post-merge CI builds the images), then stage all artifacts and trigger the
 # GitLab release-automation pipeline via release.yml. Dispatch is restricted
-# to active members of the release-ops team (vars.RELEASE_OPS_TEAM).
+# to active members of the ai-dynamo/devops team.
 
 name: On-Demand Release
 
@@ -70,11 +70,14 @@ permissions:
 
 jobs:
   # ============================================================================
-  # OPS GATE — only active members of the release-ops GitHub team
-  # (vars.RELEASE_OPS_TEAM, in the repo's org) may run this workflow. Fail
-  # closed: an unset var, missing app org Members:read permission, or any API
-  # error denies. github.actor re-resolves on re-runs, so re-running someone
-  # else's run is re-checked too.
+  # OPS GATE — only active members of the org's devops team
+  # (github.com/orgs/ai-dynamo/teams/devops) may run this workflow. Fail
+  # closed: missing app org Members:read permission or any API error denies.
+  # On re-runs github.actor stays the ORIGINAL dispatcher and the
+  # re-runner appears as github.triggering_actor, so BOTH are checked. Caveat:
+  # "Re-run failed jobs" does not re-execute a previously successful authorize —
+  # the manual-release-approval environment is the backstop for replayed
+  # downstream jobs.
   # ============================================================================
   authorize:
     name: Verify release-ops membership
@@ -93,25 +96,24 @@ jobs:
         env:
           GH_TOKEN: ${{ steps.app-token.outputs.token }}
           ORG: ${{ github.repository_owner }}
-          TEAM: ${{ vars.RELEASE_OPS_TEAM }}
+          TEAM: devops
           ACTOR: ${{ github.actor }}
+          TRIGGERING_ACTOR: ${{ github.triggering_actor }}
         run: |
           set -euo pipefail
-          if [ -z "${TEAM}" ]; then
-            echo "::error::vars.RELEASE_OPS_TEAM is not configured; denying every dispatch (fail closed)."
-            exit 1
-          fi
-          HTTP=$(curl -sS -o "${RUNNER_TEMP}/membership.json" -w '%{http_code}' \
-            -H "Authorization: Bearer ${GH_TOKEN}" \
-            -H "Accept: application/vnd.github+json" \
-            "https://api.github.com/orgs/${ORG}/teams/${TEAM}/memberships/${ACTOR}")
-          STATE=$(python3 -c "import json,sys;print(json.load(open(sys.argv[1])).get('state',''))" \
-            "${RUNNER_TEMP}/membership.json" 2>/dev/null || echo "")
-          if [ "${HTTP}" != "200" ] || [ "${STATE}" != "active" ]; then
-            echo "::error::@${ACTOR} is not an active member of ${ORG}/${TEAM} (HTTP ${HTTP}); only release ops may run this workflow."
-            exit 1
-          fi
-          echo "@${ACTOR} authorized via ${ORG}/${TEAM}"
+          for USER in $(printf '%s\n%s\n' "${ACTOR}" "${TRIGGERING_ACTOR}" | sort -u); do
+            HTTP=$(curl -sS -o "${RUNNER_TEMP}/membership.json" -w '%{http_code}' \
+              -H "Authorization: Bearer ${GH_TOKEN}" \
+              -H "Accept: application/vnd.github+json" \
+              "https://api.github.com/orgs/${ORG}/teams/${TEAM}/memberships/${USER}")
+            STATE=$(python3 -c "import json,sys;print(json.load(open(sys.argv[1])).get('state',''))" \
+              "${RUNNER_TEMP}/membership.json" 2>/dev/null || echo "")
+            if [ "${HTTP}" != "200" ] || [ "${STATE}" != "active" ]; then
+              echo "::error::@${USER} is not an active member of ${ORG}/${TEAM} (HTTP ${HTTP}); only release ops may run this workflow."
+              exit 1
+            fi
+            echo "@${USER} authorized via ${ORG}/${TEAM}"
+          done
 
   # ============================================================================
   # CREATE RELEASE BRANCH + VERSION BUMP
@@ -234,7 +236,8 @@ jobs:
                       out.append(f"{fw}-{var}")
                   else:
                       fail(f"unknown variant in '{tok}' (use runtime|efa)")
-              return ",".join(dedupe(out))
+              # A separators-only spec (e.g. ',') is an empty selection, not 'all'.
+              return ",".join(dedupe(out)) or "none"
 
           def norm_simple(spec, universe, label):
               spec = (spec or "all").strip()
@@ -253,7 +256,7 @@ jobs:
               if label == "wheel" and "ai-dynamo" in sel and "ai-dynamo-runtime" not in sel:
                   fail("wheels=ai-dynamo requires ai-dynamo-runtime (ai-dynamo pins "
                        "ai-dynamo-runtime==); add it or use wheels=all")
-              return ",".join(sel)
+              return ",".join(sel) or "none"
 
           def norm_crates(spec):
               spec = (spec or "all").strip()
@@ -267,7 +270,7 @@ jobs:
               bad = [t for t in sel if not re.fullmatch(r"[A-Za-z0-9_-]+", t)]
               if bad:
                   fail(f"invalid crate token(s) {bad}; crate names are [A-Za-z0-9_-]+")
-              return ",".join(sel)
+              return ",".join(sel) or "none"
 
           c = norm_containers(os.environ.get("CONTAINERS", "all"))
           w = norm_simple(os.environ.get("WHEELS", "all"), WHEEL_TOKENS, "wheel")
@@ -335,6 +338,7 @@ jobs:
             CI_BRANCH="release/${BRANCH}"
           else
             IS_TEST=true
+            echo "::warning::branch '${BRANCH}' is not release/X.Y.Z[.devN|.postN] or v[-] — treated as a THROWAWAY TEST branch: it is cut and built, but staging, approval and the GitLab trigger are all skipped."
           fi
 
           # Fetch the version-bump script from THIS workflow's commit (GITHUB_SHA);
@@ -345,8 +349,10 @@ jobs:
           # re-run only adds the diff (e.g. a newly-added script step lands on the branch
           # that was cut before it existed); otherwise base it on the checked-out
           # source_ref. We never delete/clobber — the diff (if any) is committed on top.
+          # refs/heads/ refspecs everywhere: a bare v-branch name can collide with
+          # a GA tag (v1.4.1), and short names would resolve/push ambiguously.
           if git ls-remote --exit-code --heads origin "${BRANCH}" >/dev/null 2>&1; then
-            git fetch origin "${BRANCH}"
+            git fetch origin "refs/heads/${BRANCH}"
             git checkout -B "${BRANCH}" FETCH_HEAD
             EXISTED=1
           else
@@ -373,11 +379,9 @@ jobs:
           # pre-merge `cargo metadata --locked` matrix both pass. lib/bindings/python,
           # lib/bindings/kvbm and lib/runtime/examples are SEPARATE cargo workspaces
           # with their own lock files, so a root-only `cargo update --workspace` leaves
-          # their workspace-crate pins on the PREVIOUS version. That is what shipped in
-          # the 1.4.1 cut (9f6c6be7): three stale locks, and the clippy matrix — which
-          # runs `cargo metadata --locked` in each of those dirs — failed on all three.
-          # Discovered from git, not hardcoded, so a newly added nested workspace cannot
-          # silently regress the next release.
+          # their workspace-crate pins on the PREVIOUS version and their `--locked`
+          # checks fail. Discovered from git, not hardcoded, so a newly added nested
+          # workspace cannot silently regress the next release.
           git ls-files | grep -E '(^|/)Cargo\.lock$' | sort | while IFS= read -r lock; do
             manifest="$(dirname "${lock}")/Cargo.toml"
             manifest="${manifest#./}"
@@ -405,11 +409,11 @@ jobs:
               echo "${BRANCH} already matches the release scripts; nothing to commit (reuse, no push)."
             else
               git commit -s --allow-empty -m "release: set version ${RELEASE_VERSION}" -m "${SELECTION_MSG}"
-              git push origin "${BRANCH}"
+              git push origin "HEAD:refs/heads/${BRANCH}"
             fi
           else
             git commit -s -m "release: set version ${RELEASE_VERSION}" -m "${SELECTION_MSG}"
-            git push origin "${BRANCH}"
+            git push origin "HEAD:refs/heads/${BRANCH}"
           fi
           # Keep the CI twin of a bare named branch on this commit (fast-forward;
           # no-op when already there) so post-merge CI builds it.
@@ -534,8 +538,13 @@ jobs:
               -H "Authorization: Bearer ${GH_TOKEN}" \
               -H "Accept: application/vnd.github+json" \
               "https://api.github.com/repos/${REPO}/actions/workflows/post-merge-ci.yml/runs?head_sha=${SHA}&per_page=1") || { API_OK=false; RUNS='{}'; }
-            COUNT=$(echo "${RUNS}" | jq -r '.workflow_runs | length' 2>/dev/null || { API_OK=false; echo 0; })
-            STATUS=$(echo "${RUNS}" | jq -r '.workflow_runs[0].status // "pending"' 2>/dev/null || { API_OK=false; echo pending; })
+            # Failure flags must be set in the parent shell, not inside $( ).
+            if ! COUNT=$(echo "${RUNS}" | jq -r '.workflow_runs | length' 2>/dev/null); then
+              API_OK=false; COUNT=0
+            fi
+            if ! STATUS=$(echo "${RUNS}" | jq -r '.workflow_runs[0].status // "pending"' 2>/dev/null); then
+              API_OK=false; STATUS=pending
+            fi
             if [ "${API_OK}" = "true" ]; then
               if [ "${COUNT}" = "0" ] && [ "$(date +%s)" -ge "${start_deadline}" ]; then
                 echo "::error::post-merge CI never started for ${SHA} (branch push may not have triggered it); required image(s) missing: ${missing[*]}"
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 9987dd1de917..b2d0a6e8bc81 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -133,8 +133,8 @@ on:
         default: ''
     outputs:
       # The exact NGC docker tag published for this run (nightly: -).
-      # Callers use it to reference what actually shipped instead of recomputing
-      # the date, which can drift across a UTC-midnight boundary.
+      # Informational for callers — recomputing the date can drift across a
+      # UTC-midnight boundary; no current caller consumes it.
       ngc_version_tag:
         description: 'NGC docker version tag published by this run.'
         value: ${{ jobs.prepare-release.outputs.ngc_version_tag }}
@@ -246,6 +246,7 @@ jobs:
             if [[ ! "$BRANCH_NAME" =~ ^release/[0-9]+\.[0-9]+\.[0-9]+(\.(dev|post)[0-9]+)?$ ]]; then
               echo "Error: workflow_dispatch must be triggered from a release/X.Y.Z[.devN|.postN] branch (or main with nightly=true)"
               echo "Current branch: $BRANCH_NAME"
+              echo "Named releases are not dispatchable here — run on-demand-release.yml with a BARE v[-] release_branch (a release/X.Y.Z- branch is treated as a throwaway test branch there too)."
               exit 1
             fi
             VERSION="${BRANCH_NAME#release/}"
@@ -291,6 +292,14 @@ jobs:
             echo "Error: cannot parse base version from pyproject.toml (got '${BASE_VERSION}')"
             exit 1
           fi
+          # rc dispatch derives VERSION from the branch name but stamps wheels from
+          # the tree at COMMIT_SHA — a mismatch ships wheels versioned differently
+          # from the image tags (e.g. a SHA pasted from the wrong branch, or a
+          # branch cut whose version bump hasn't landed yet).
+          if [[ "${NIGHTLY}" != "true" && "${BASE_VERSION}" != "${VERSION}" ]]; then
+            echo "Error: pyproject version '${BASE_VERSION}' at ${COMMIT_SHA:0:12} != release version '${VERSION}' — is the SHA on this release branch (post version-bump)?"
+            exit 1
+          fi
 
           if [[ "${NIGHTLY}" == "true" ]]; then
             DATE_UTC=$(date -u +%Y%m%d)
@@ -1140,19 +1149,19 @@ jobs:
             fi
           fi
 
-      # Runs last so charts still publish first. On-demand only: the operator named
-      # an explicit image set, so a partially-copied set must not report success and
-      # let GitLab open promotion MRs for images that never landed. rc/nightly keep
-      # their historical warn-and-continue behavior (a floating-tag hiccup there must
-      # not block the GitLab trigger, which requires this job to succeed).
-      - name: Fail on partially staged container selection (on-demand)
-        if: ${{ env.ON_DEMAND == 'true' && env.CONTAINER_SUBSET != 'none' }}
+      # Runs last so charts still publish first. rc and on-demand runs name an
+      # explicit image set, so a partially-copied set must not report success and
+      # let GitLab scan/promote images that never landed. Nightly stays warn-skip
+      # by design: its optional images are gated per-build-result upstream and a
+      # flake there must not block the GitLab trigger.
+      - name: Fail on partially staged container selection
+        if: ${{ env.NIGHTLY != 'true' && env.CONTAINER_SUBSET != 'none' }}
         env:
           FAILED_COUNT: ${{ steps.copy_images.outputs.failed_count }}
         run: |
           set -euo pipefail
           if [ "${FAILED_COUNT:-0}" -ne 0 ]; then
-            echo "::error::${FAILED_COUNT} selected container image(s) failed to copy to NGC — see 'Copy images to NGC'. Re-run with a fresh rc_round once fixed."
+            echo "::error::${FAILED_COUNT} selected container image(s) failed to copy to NGC — see 'Copy images to NGC'. Once fixed, re-stage with a fresh round (rc_round on on-demand-release.yml; rc_number on an rc dispatch)."
             exit 1
           fi
           echo "All selected container images copied."
@@ -1435,8 +1444,9 @@ jobs:
   # ARTIFACTORY STAGING: publish workspace crates to the Artifactory cargo registry
   # (from ARTIFACTORY_CARGO_INDEX), dependency-ordered. Artifactory only, never crates.io.
   # cargo needs the native toolchain, so we `docker run` the ECR build image on
-  # prod-builder-amd-v1 (the DinD builder release-publish uses) instead of a job-level
-  # `container:` on a k8s-hooks tester (its per-job pod stayed Pending). On-demand only.
+  # prod-builder-amd-v1 (the DinD builder release-publish uses). A job-level
+  # `container:` cannot pull the image on a delayed release dispatch (registry
+  # creds are only fresh in-step). On-demand only.
 
   stage-crates-artifactory:
     name: Stage Crates to Artifactory
@@ -1518,6 +1528,11 @@ jobs:
           # No `set -e`: outputs must be set even when the publish fails partway;
           # we exit with the real code afterwards.
           set -uo pipefail
+          # Mask the registry host: cargo error output embeds the index URL in
+          # forms (https:///...config.json) that exact-secret masking of
+          # ARTIFACTORY_CARGO_INDEX does not catch.
+          CARGO_HOST=$(printf '%s' "${ARTIFACTORY_CARGO_INDEX}" | sed -E 's#^sparse\+##; s#^https?://##; s#[/:].*$##')
+          if [ -n "${CARGO_HOST}" ]; then echo "::add-mask::${CARGO_HOST}"; fi
           # Run stage_crates.py inside the ECR build image: repo mounted at /work, cargo uses
           # the image toolchain, root so it can write Cargo.lock/.cargo and the target dir.
           docker run --rm --user 0:0 \
@@ -1710,9 +1725,8 @@ jobs:
           GITLAB_TRIGGER_URL:   ${{ secrets.GITLAB_PIPELINE_URL }}
           GITLAB_TRIGGER_TOKEN: ${{ secrets.GITLAB_TRIGGER_TOKEN }}
           # release-automation ref to run (var-overridable for testing a GitLab
-          # dev branch). GitLab does NOT force dry-run on non-default refs.
-          # The on-demand support landed on release-automation main (2026-07-08),
-          # so main is the correct default.
+          # dev branch). GitLab does NOT force dry-run on non-default refs, and
+          # only main carries its dry-run guards — keep the default on main.
           GITLAB_REF:           ${{ vars.GITLAB_RELEASE_AUTOMATION_REF || 'main' }}
           VERSION:              ${{ needs.prepare-release.outputs.version }}
           NGC_VERSION_TAG:      ${{ needs.prepare-release.outputs.ngc_version_tag }}
diff --git a/lib/backend-common/Cargo.toml b/lib/backend-common/Cargo.toml
index 5460a4a70ac4..b508a002d89d 100644
--- a/lib/backend-common/Cargo.toml
+++ b/lib/backend-common/Cargo.toml
@@ -23,7 +23,7 @@ integration = ["dynamo-runtime/integration"]
 
 [dependencies]
 dynamo-runtime = { workspace = true }
-dynamo-llm = { path = "../llm", default-features = false }
+dynamo-llm = { path = "../llm", version = "1.5.0", default-features = false }
 dynamo-protocols = { workspace = true }
 
 anyhow = { workspace = true }

From b9ad3284a2ef232dd2268647967c56f9ba34dacf Mon Sep 17 00:00:00 2001
From: pvijayakrish 
Date: Fri, 28 Aug 2026 20:25:38 -0700
Subject: [PATCH 05/29] feat(release): stage nightly crates at the dotted
 -dev.YYYYMMDD version
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Nightly runs now stage the workspace crates to the internal Artifactory
cargo registry at X.Y.Z-dev.YYYYMMDD. Dotted, so the date is a numeric
SemVer identifier that sorts correctly against every other pre-release
shape (an undotted -devYYYYMMDD is one text identifier: -dev9 would
outrank it). Date-only, matching the wheel's .devYYYYMMDD: a same-day
re-run is an idempotent skip and the day's first staging wins; the step
summary reports the already-present count so a no-op re-stage is
distinguishable from a fresh publish, and reminds nightly consumers to
pin with '=' (a bare pre-release pin resolves upward to a later rc/GA).

The stage-crates job gates on on-demand OR nightly (rc stays crate-less),
pulls the nightly toolchain image -dynamo-runtime-nightly-test (a
missing nightly build must not fall through to the post-merge image and
its stable wheel versions; the inventory step reuses the derived tag),
and stays fail-soft. Fixed on the way: the publish step captured the
docker exit via PIPESTATUS under the implicit `bash -e {0}` shell, so a
failure aborted the step before the output tail — crates_status could
never report 'partial'; `rc=0 ... || rc=$?` keeps the tail reachable.
The nightly GitLab trigger includes 'crate' in ARTIFACTS keyed on
actually-staged crates (like helm), and stage_crates warns when the
stage version does not extend Cargo's workspace version (pyproject vs
Cargo drift has no other gate on the nightly path).

Signed-off-by: pvijayakrish 
---
 .github/scripts/stage_crates.py |  7 ++++
 .github/workflows/release.yml   | 59 ++++++++++++++++++++++++++++-----
 2 files changed, 58 insertions(+), 8 deletions(-)

diff --git a/.github/scripts/stage_crates.py b/.github/scripts/stage_crates.py
index 6c3b4d36736d..18c9b4e3fbdd 100644
--- a/.github/scripts/stage_crates.py
+++ b/.github/scripts/stage_crates.py
@@ -355,6 +355,13 @@ def main() -> int:
             print("::error::--stage-version: cannot read [workspace.package].version", file=sys.stderr)
             return 1
         cur = wm.group(1)
+        # STAGE_VERSION's base comes from pyproject.toml (release.yml); nothing
+        # else gates pyproject against Cargo.toml, so a bump that missed one
+        # file would otherwise publish crates under the wrong base silently.
+        if not (stage_version == cur or stage_version.startswith(f"{cur}-")):
+            print(f"::warning::stage-version '{stage_version}' does not extend the "
+                  f"workspace version '{cur}' — pyproject.toml and Cargo.toml may "
+                  f"have drifted", file=sys.stderr)
         if cur != stage_version:
             n = rewrite_versions([root / "Cargo.toml", *member_manifests], cur, stage_version)
             print(f"stage-version: rewrote {cur} -> {stage_version} across {n} manifest(s)")
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index b2d0a6e8bc81..2a3c6de25d5a 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -313,13 +313,23 @@ jobs:
             # keeps same-day re-runs unique on immutable ChartMuseum; its `g`
             # prefix keeps an all-digit sha from being invalid SemVer.
             HELM_CHART_VERSION="${BASE_VERSION}-dev.${DATE_UTC}.g${SHORT_SHA}"
+            # Crate version: dotted SemVer pre-release identifiers so they sort
+            # numerically (dev.20260828 < dev.20260901; an undotted -devYYYYMMDD
+            # is one TEXT identifier and sorts wrong against any other shape,
+            # e.g. -dev9 > -dev20260828). Date-only, matching the wheel's
+            # .devYYYYMMDD: a same-day re-run at a new SHA is idempotently
+            # SKIPPED by the immutable registry — the day's first staging wins.
+            SEMVER_BASE=$(echo "${BASE_VERSION}" | sed -E 's/^([0-9]+\.[0-9]+\.[0-9]+)\.(.+)/\1-\2/')
+            CRATE_VERSION_NIGHTLY="${SEMVER_BASE}-dev.${DATE_UTC}"
             echo "rc_number=" >> $GITHUB_OUTPUT
             echo "ngc_version_tag=${NGC_VERSION_TAG}" >> $GITHUB_OUTPUT
             echo "helm_chart_version=${HELM_CHART_VERSION}" >> $GITHUB_OUTPUT
             echo "wheel_version=${WHEEL_VERSION}" >> $GITHUB_OUTPUT
+            echo "crate_version=${CRATE_VERSION_NIGHTLY}" >> $GITHUB_OUTPUT
             echo "Nightly NGC tag: ${NGC_VERSION_TAG}"
             echo "Nightly wheel version: ${WHEEL_VERSION}"
             echo "Nightly Helm chart version: ${HELM_CHART_VERSION}"
+            echo "Nightly crate version: ${CRATE_VERSION_NIGHTLY}"
             exit 0
           fi
 
@@ -1446,12 +1456,12 @@ jobs:
   # cargo needs the native toolchain, so we `docker run` the ECR build image on
   # prod-builder-amd-v1 (the DinD builder release-publish uses). A job-level
   # `container:` cannot pull the image on a delayed release dispatch (registry
-  # creds are only fresh in-step). On-demand only.
+  # creds are only fresh in-step). On-demand and nightly; rc stays crate-less.
 
   stage-crates-artifactory:
     name: Stage Crates to Artifactory
     needs: [prepare-release]
-    if: ${{ needs.prepare-release.result == 'success' && needs.prepare-release.outputs.on_demand == 'true' && inputs.crates != 'none' && inputs.crates != '' }}
+    if: ${{ needs.prepare-release.result == 'success' && (needs.prepare-release.outputs.on_demand == 'true' || needs.prepare-release.outputs.nightly == 'true') && inputs.crates != 'none' && inputs.crates != '' }}
     runs-on: prod-builder-amd-v1
     environment: automated-release
     # Fail-soft: crate staging must never block the release. The job turns red
@@ -1469,6 +1479,7 @@ jobs:
       crates_status: ${{ steps.publish_crates.outputs.crates_status }}
     env:
       COMMIT_SHA: ${{ needs.prepare-release.outputs.commit_sha }}
+      NIGHTLY: ${{ needs.prepare-release.outputs.nightly }}
     steps:
       - name: Checkout at source commit
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
@@ -1489,7 +1500,13 @@ jobs:
         run: |
           set -euo pipefail
           aws ecr get-login-password --region "${AWS_DEFAULT_REGION}" | docker login --username AWS --password-stdin "${ECR_REGISTRY}"
-          echo "ECR_IMAGE=${ECR_REGISTRY}/${ECR_REPOSITORY}:${COMMIT_SHA}-dynamo-runtime-test" >> "${GITHUB_ENV}"
+          # Nightly builds tag the toolchain image with the suffix BEFORE -test
+          # (-dynamo-runtime-nightly-test, shared-build-image.yml); a missing
+          # nightly build must not fall through to the post-merge image at the
+          # same SHA (it carries stable, non-dev wheel versions).
+          NIGHTLY_TAG_SUFFIX=""
+          if [ "${NIGHTLY}" = "true" ]; then NIGHTLY_TAG_SUFFIX="-nightly"; fi
+          echo "ECR_IMAGE=${ECR_REGISTRY}/${ECR_REPOSITORY}:${COMMIT_SHA}-dynamo-runtime${NIGHTLY_TAG_SUFFIX}-test" >> "${GITHUB_ENV}"
 
       - name: Inventory crate build image in ECR
         env:
@@ -1497,7 +1514,8 @@ jobs:
         run: |
           set -uo pipefail
           # Report-only: is the crate build image in ECR? Never fails (publish step errors if absent).
-          TAG="${COMMIT_SHA}-dynamo-runtime-test"
+          # Single source of truth: the tag the login step derived into ECR_IMAGE.
+          TAG="${ECR_IMAGE##*:}"
           {
             echo "### Crate build image availability (ECR)"
             echo ""
@@ -1525,8 +1543,10 @@ jobs:
           STAGE_VERSION: ${{ needs.prepare-release.outputs.crate_version }}
           CRATE_SUBSET: ${{ inputs.crates }}
         run: |
-          # No `set -e`: outputs must be set even when the publish fails partway;
-          # we exit with the real code afterwards.
+          # The step shell is GitHub's implicit `bash -e {0}` (set -uo does not
+          # clear -e): every fallible command below is guarded so the outputs are
+          # written even when the publish fails partway; the real exit code is
+          # raised at the end.
           set -uo pipefail
           # Mask the registry host: cargo error output embeds the index URL in
           # forms (https:///...config.json) that exact-secret masking of
@@ -1535,6 +1555,7 @@ jobs:
           if [ -n "${CARGO_HOST}" ]; then echo "::add-mask::${CARGO_HOST}"; fi
           # Run stage_crates.py inside the ECR build image: repo mounted at /work, cargo uses
           # the image toolchain, root so it can write Cargo.lock/.cargo and the target dir.
+          rc=0
           docker run --rm --user 0:0 \
             --network host \
             --ulimit nofile=65536:65536 \
@@ -1549,12 +1570,19 @@ jobs:
             -e CRATE_SUBSET \
             "${ECR_IMAGE}" \
             bash -lc 'python3 .ondemand-tools/.github/scripts/stage_crates.py --registry artifactory --only "${CRATE_SUBSET}"' \
-            2>&1 | tee /tmp/stage_crates.log
-          rc="${PIPESTATUS[0]}"
+            2>&1 | tee /tmp/stage_crates.log || rc=$?
+          # `|| rc=$?` keeps a nonzero exit non-fatal: the step shell is `bash -e {0}`
+          # (set -uo above does NOT clear -e), and errexit here would skip the
+          # output-writing tail — crates_status could never be 'partial' and the
+          # staged list would be lost on failure. pipefail makes rc docker's code.
 
           # stage_crates.py emits `STAGED_CRATE=` and `FAILED_CRATE= ` lines.
           STAGED="$(sed -n 's/^STAGED_CRATE=//p' /tmp/stage_crates.log | paste -sd, - || true)"
           FAILED="$(sed -n 's/^FAILED_CRATE=\([^ ]*\).*/\1/p' /tmp/stage_crates.log | paste -sd, - || true)"
+          # Idempotent skips: the registry is immutable, so an already-present
+          # version keeps its FIRST content — an all-'already' run is a no-op
+          # re-stage, not a fresh publish, and must be tellable from the summary.
+          ALREADY="$(grep -c 'already on the registry (skip)' /tmp/stage_crates.log || true)"
           if [ "${rc}" -eq 0 ]; then
             STATUS="success"
           elif [ -n "${STAGED}" ]; then
@@ -1573,6 +1601,14 @@ jobs:
             echo "Staged: \`${STAGED:-}\`"
             echo ""
             echo "Failed: \`${FAILED:-}\`"
+            if [ "${ALREADY:-0}" -gt 0 ]; then
+              echo ""
+              echo "Already present (idempotent skip, content unchanged from the first staging at this version): ${ALREADY}"
+            fi
+            if [ "${NIGHTLY}" = "true" ]; then
+              echo ""
+              echo "Nightly consumers must pin with '=' (e.g. \`= ${STAGE_VERSION}\`): a bare pre-release pin resolves upward to any later rc/GA on the index."
+            fi
           } >> "${GITHUB_STEP_SUMMARY}"
           echo "Staged crates: ${STAGED:-}"
           if [ "${rc}" -ne 0 ]; then
@@ -1777,6 +1813,13 @@ jobs:
             if [ -n "${STAGED_HELM}" ]; then
               ARTIFACTS="${ARTIFACTS},helm"
             fi
+            # Nightly crates: staged fail-soft to the internal cargo registry at
+            # the dated -dev.YYYYMMDD version. Never crates.io (immutable,
+            # undeletable). Keyed on ACTUALS like helm above — a wholesale-failed
+            # staging must not tell GitLab a crate artifact exists.
+            if [ -n "${STAGED_CRATES}" ]; then
+              ARTIFACTS="${ARTIFACTS},crate"
+            fi
             NIGHTLY_TAG="${NGC_VERSION_TAG}"
             RC_TAG=""
             # VERSION is the literal 'nightly'; empty lets GitLab apply its default.

From 5d1676840529cee38b61300ccab8c6e469a68311 Mon Sep 17 00:00:00 2001
From: pvijayakrish 
Date: Tue, 1 Sep 2026 15:42:24 -0700
Subject: [PATCH 06/29] feat(release): stage the unified sidecar container

post-merge builds the multi-arch dynamo-sidecar image on every push
(-dynamo-sidecar, shared-build-sidecar.yml), including release/*
branches, but nothing staged it. Add the `sidecar` container token to
the on-demand selection grammar, the await-post-merge tag map, the
release.yml subset universe, the NGC copy step
(dynamo-sidecar:), the ECR inventory, and the stamper's
token universe. rc/on-demand only: nightly does not build the sidecar,
so the nightly arm is untouched. GA promotion needs a dynamo-sidecar
entry in ngc-publishing-configs before the GitLab MR button works;
staging is independent of that.

Signed-off-by: pvijayakrish 
---
 .github/scripts/apply_dev_version.py    |  2 ++
 .github/workflows/on-demand-release.yml |  5 +++--
 .github/workflows/release.yml           | 15 +++++++++++++--
 3 files changed, 18 insertions(+), 4 deletions(-)

diff --git a/.github/scripts/apply_dev_version.py b/.github/scripts/apply_dev_version.py
index 5da2bd55dec6..e94a26c7b8b7 100755
--- a/.github/scripts/apply_dev_version.py
+++ b/.github/scripts/apply_dev_version.py
@@ -97,6 +97,7 @@
 CONTAINER_TOKENS = {
     "vllm-runtime", "vllm-efa", "sglang-runtime", "sglang-efa",
     "trtllm-runtime", "trtllm-efa", "frontend", "operator", "planner",
+    "sidecar",
 }
 HELM_TOKENS = {"platform"}
 
@@ -115,6 +116,7 @@
     "frontend": "dynamo-frontend",
     "operator": "kubernetes-operator",
     "planner": "dynamo-planner",
+    "sidecar": "dynamo-sidecar",
 }
 GA_REGISTRY = "nvcr.io/nvidia/ai-dynamo"
 PLACEHOLDER_REGISTRY = "my-registry"
diff --git a/.github/workflows/on-demand-release.yml b/.github/workflows/on-demand-release.yml
index 96c2c0440b07..7df12190df50 100644
--- a/.github/workflows/on-demand-release.yml
+++ b/.github/workflows/on-demand-release.yml
@@ -35,7 +35,7 @@ on:
         type: string
         default: '0'
       containers:
-        description: 'Containers to release: all | none | csv of vllm[:runtime|:efa],sglang[...],trtllm[...],frontend,operator,planner.'
+        description: 'Containers to release: all | none | csv of vllm[:runtime|:efa],sglang[...],trtllm[...],frontend,operator,planner,sidecar.'
         required: false
         type: string
         default: all
@@ -201,7 +201,7 @@ jobs:
 
           FRAMEWORKS = {"vllm", "sglang", "trtllm"}
           VARIANTS = {"runtime", "efa"}
-          COMPONENTS = {"frontend", "operator", "planner"}
+          COMPONENTS = {"frontend", "operator", "planner", "sidecar"}
           WHEEL_TOKENS = {"ai-dynamo", "ai-dynamo-runtime", "kvbm"}
           HELM_TOKENS = {"platform"}
 
@@ -497,6 +497,7 @@ jobs:
             [sglang-runtime]=sglang-runtime [sglang-efa]=sglang-runtime-efa
             [trtllm-runtime]=trtllm-runtime [trtllm-efa]=trtllm-runtime-efa
             [frontend]=dynamo-frontend [operator]=operator [planner]=dynamo-planner
+            [sidecar]=dynamo-sidecar
           )
           declare -a TAGS=()
           for tok in "${!CTR[@]}"; do
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 2a3c6de25d5a..2cc4b658d773 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -33,7 +33,7 @@ on:
         type: string
         default: ''
       containers:
-        description: 'Container subset to publish (all|none|csv of vllm-runtime,vllm-efa,sglang-runtime,sglang-efa,trtllm-runtime,trtllm-efa,frontend,operator,planner). Partial rc re-publishes supported.'
+        description: 'Container subset to publish (all|none|csv of vllm-runtime,vllm-efa,sglang-runtime,sglang-efa,trtllm-runtime,trtllm-efa,frontend,operator,planner,sidecar). Partial rc re-publishes supported.'
         required: false
         type: string
         default: all
@@ -207,7 +207,7 @@ jobs:
               esac
             done
           }
-          validate_subset containers "${CONTAINER_SUBSET:-all}" "vllm-runtime,vllm-efa,sglang-runtime,sglang-efa,trtllm-runtime,trtllm-efa,frontend,operator,planner"
+          validate_subset containers "${CONTAINER_SUBSET:-all}" "vllm-runtime,vllm-efa,sglang-runtime,sglang-efa,trtllm-runtime,trtllm-efa,frontend,operator,planner,sidecar"
           validate_subset wheels "${WHEEL_SUBSET:-all}" "ai-dynamo,ai-dynamo-runtime,kvbm"
           validate_subset helm "${HELM_SUBSET:-all}" "platform"
           if [[ ! "${CRATE_SUBSET:-all}" =~ ^[A-Za-z0-9_,-]*$ ]]; then
@@ -508,6 +508,7 @@ jobs:
               "frontend|${COMMIT_SHA}-dynamo-frontend"
               "operator|${COMMIT_SHA}-operator"
               "planner|${COMMIT_SHA}-dynamo-planner"
+              "sidecar|${COMMIT_SHA}-dynamo-sidecar"
             )
           fi
 
@@ -714,6 +715,16 @@ jobs:
               TARGET="${NGC_REGISTRY}/${NGC_ORG}/ai-dynamo/dynamo-planner:${NGC_VERSION_TAG}"
               copy_image "${SOURCE}" "${TARGET}" "dynamo-planner:${NGC_VERSION_TAG}" || true
             fi
+
+            # ---- Sidecar image (unified, multi-arch; built by post-merge's
+            # sidecar-build as -dynamo-sidecar) ----
+            if want sidecar; then
+              echo ""
+              echo "=== Sidecar Image ==="
+              SOURCE="${ECR_REGISTRY}/${ECR_REPOSITORY}:${COMMIT_SHA}-dynamo-sidecar"
+              TARGET="${NGC_REGISTRY}/${NGC_ORG}/ai-dynamo/dynamo-sidecar:${NGC_VERSION_TAG}"
+              copy_image "${SOURCE}" "${TARGET}" "dynamo-sidecar:${NGC_VERSION_TAG}" || true
+            fi
           else
             # Optional nightly images — operator, planner, frontend and the
             # three runtime EFA variants — are built by THIS

From f6e318a4170008bb255ef1087110b15ad74136b7 Mon Sep 17 00:00:00 2001
From: pvijayakrish 
Date: Tue, 1 Sep 2026 15:42:37 -0700
Subject: [PATCH 07/29] ci(release): trigger the 1.5.0 rc0 branch cut from the
 push loop

TEMPORARY test scaffolding (revert before merge): push trigger on the
copy-pr-bot branch pull-request/13961, input fallbacks
release_version=1.5.0 / source_ref=main / rc0 / all artifacts, and a
push-event bypass of the ops gate (push events have no dispatching
actor). Creates release/1.5.0 from main, stamps it, and stages rc0
after the manual approval.

Signed-off-by: pvijayakrish 
---
 .github/workflows/on-demand-release.yml | 29 ++++++++++++++++++++-----
 1 file changed, 23 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/on-demand-release.yml b/.github/workflows/on-demand-release.yml
index 7df12190df50..84c31baf0312 100644
--- a/.github/workflows/on-demand-release.yml
+++ b/.github/workflows/on-demand-release.yml
@@ -13,6 +13,11 @@ name: On-Demand Release
 # workflow_dispatch is capped at 10 inputs by GitHub — this block is at 10;
 # fold any new knob into an existing input.
 on:
+  # TEMPORARY(1.5.0-rc0): push runs the flow as release_version 1.5.0 from main
+  # (creates release/1.5.0), rc0, all artifacts. REVERT before merge.
+  push:
+    branches:
+      - pull-request/13961
   workflow_dispatch:
     inputs:
       source_ref:
@@ -101,6 +106,12 @@ jobs:
           TRIGGERING_ACTOR: ${{ github.triggering_actor }}
         run: |
           set -euo pipefail
+          # TEMPORARY(1.5.0-rc0): the push test loop has no dispatching actor to
+          # check. REVERT before merge.
+          if [ "${GITHUB_EVENT_NAME}" = "push" ]; then
+            echo "TEMPORARY: push loop — skipping the ops gate"
+            exit 0
+          fi
           for USER in $(printf '%s\n%s\n' "${ACTOR}" "${TRIGGERING_ACTOR}" | sort -u); do
             HTTP=$(curl -sS -o "${RUNNER_TEMP}/membership.json" -w '%{http_code}' \
               -H "Authorization: Bearer ${GH_TOKEN}" \
@@ -139,8 +150,10 @@ jobs:
     steps:
       - name: Validate inputs
         env:
-          RELEASE_VERSION: ${{ inputs.release_version }}
-          SOURCE_REF: ${{ inputs.source_ref }}
+          # TEMPORARY(1.5.0-rc0): || fallbacks cover inputless push events. REVERT
+          # to bare inputs.* before merge.
+          RELEASE_VERSION: ${{ inputs.release_version || '1.5.0' }}
+          SOURCE_REF: ${{ inputs.source_ref || 'main' }}
           RELEASE_BRANCH: ${{ inputs.release_branch }}
           # || guards a cleared dispatch field (inputs default fills the rest).
           RC_ROUND: ${{ inputs.rc_round || '0' }}
@@ -181,7 +194,8 @@ jobs:
       - name: Resolve artifact selection
         id: resolve
         env:
-          RELEASE_VERSION: ${{ inputs.release_version }}
+          # TEMPORARY(1.5.0-rc0): revert to bare inputs.release_version before merge.
+          RELEASE_VERSION: ${{ inputs.release_version || '1.5.0' }}
           CONTAINERS: ${{ inputs.containers || 'all' }}
           WHEELS: ${{ inputs.wheels || 'all' }}
           # Stage ALL publishable workspace crates ('all' = the max). A hardcoded
@@ -303,14 +317,16 @@ jobs:
       - name: Checkout source
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
         with:
-          ref: ${{ inputs.source_ref }}
+          # TEMPORARY(1.5.0-rc0): revert to bare inputs.source_ref before merge.
+          ref: ${{ inputs.source_ref || 'main' }}
           fetch-depth: 0
           token: ${{ steps.app-token.outputs.token }}
 
       - name: Set version and create branch
         id: bump
         env:
-          RELEASE_VERSION: ${{ inputs.release_version }}
+          # TEMPORARY(1.5.0-rc0): revert to bare inputs.release_version before merge.
+          RELEASE_VERSION: ${{ inputs.release_version || '1.5.0' }}
           RELEASE_BRANCH: ${{ inputs.release_branch }}
           APP_SLUG: ${{ steps.app-token.outputs.app-slug }}
           GH_TOKEN: ${{ steps.app-token.outputs.token }}
@@ -595,7 +611,8 @@ jobs:
     with:
       commit_sha: ${{ needs.prepare-branch.outputs.bump_sha }}
       on_demand: true
-      release_version: ${{ inputs.release_version }}
+      # TEMPORARY(1.5.0-rc0): revert to bare inputs.release_version before merge.
+      release_version: ${{ inputs.release_version || '1.5.0' }}
       # Operator-chosen internal-staging rc (immutable registries can't be overwritten).
       rc_number: ${{ needs.prepare-branch.outputs.rc_number }}
       # Normalized subsets from resolve-selection.

From a8536f9e0040ead5a92b03b37cd2700de63d5826 Mon Sep 17 00:00:00 2001
From: pvijayakrish 
Date: Wed, 2 Sep 2026 16:37:31 -0700
Subject: [PATCH 08/29] ci(release): re-stage 1.5.0 rc1 from the push loop
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

rc0 is already staged and the internal registries are immutable, so the
push loop now resolves rc1. The bump is push-gated
(github.event_name == 'push'), so a real dispatch with a cleared
rc_round field still defaults to rc0 — production semantics unchanged.
Both RC_ROUND sites (validate + rc step) move together. The reused
release/1.5.0 branch is re-stamped idempotently (no new commit) and
every artifact re-stages at 1.5.0-rc.1. Still TEMPORARY — revert the
whole push-loop scaffold before merge.

Signed-off-by: pvijayakrish 
---
 .github/workflows/on-demand-release.yml | 24 ++++++++++++++----------
 1 file changed, 14 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/on-demand-release.yml b/.github/workflows/on-demand-release.yml
index 84c31baf0312..78554e43a424 100644
--- a/.github/workflows/on-demand-release.yml
+++ b/.github/workflows/on-demand-release.yml
@@ -13,8 +13,8 @@ name: On-Demand Release
 # workflow_dispatch is capped at 10 inputs by GitHub — this block is at 10;
 # fold any new knob into an existing input.
 on:
-  # TEMPORARY(1.5.0-rc0): push runs the flow as release_version 1.5.0 from main
-  # (creates release/1.5.0), rc0, all artifacts. REVERT before merge.
+  # TEMPORARY(1.5.0-rc1): push runs the flow as release_version 1.5.0 from main
+  # (reuses release/1.5.0), rc1 (rc0 already staged), all artifacts. REVERT before merge.
   push:
     branches:
       - pull-request/13961
@@ -106,7 +106,7 @@ jobs:
           TRIGGERING_ACTOR: ${{ github.triggering_actor }}
         run: |
           set -euo pipefail
-          # TEMPORARY(1.5.0-rc0): the push test loop has no dispatching actor to
+          # TEMPORARY(1.5.0-rc1): the push test loop has no dispatching actor to
           # check. REVERT before merge.
           if [ "${GITHUB_EVENT_NAME}" = "push" ]; then
             echo "TEMPORARY: push loop — skipping the ops gate"
@@ -150,13 +150,15 @@ jobs:
     steps:
       - name: Validate inputs
         env:
-          # TEMPORARY(1.5.0-rc0): || fallbacks cover inputless push events. REVERT
+          # TEMPORARY(1.5.0-rc1): || fallbacks cover inputless push events. REVERT
           # to bare inputs.* before merge.
           RELEASE_VERSION: ${{ inputs.release_version || '1.5.0' }}
           SOURCE_REF: ${{ inputs.source_ref || 'main' }}
           RELEASE_BRANCH: ${{ inputs.release_branch }}
           # || guards a cleared dispatch field (inputs default fills the rest).
-          RC_ROUND: ${{ inputs.rc_round || '0' }}
+          # TEMPORARY(1.5.0-rc1): push stages rc1 — rc0 is already burned on the
+          # immutable registries. Dispatch is unchanged. REVERT to bare || '0'.
+          RC_ROUND: ${{ inputs.rc_round || (github.event_name == 'push' && '1' || '0') }}
         run: |
           set -euo pipefail
           if [[ ! "${RELEASE_VERSION}" =~ ^[0-9]+\.[0-9]+\.[0-9]+(\.(dev|post)[0-9]+)?$ ]]; then
@@ -194,7 +196,7 @@ jobs:
       - name: Resolve artifact selection
         id: resolve
         env:
-          # TEMPORARY(1.5.0-rc0): revert to bare inputs.release_version before merge.
+          # TEMPORARY(1.5.0-rc1): revert to bare inputs.release_version before merge.
           RELEASE_VERSION: ${{ inputs.release_version || '1.5.0' }}
           CONTAINERS: ${{ inputs.containers || 'all' }}
           WHEELS: ${{ inputs.wheels || 'all' }}
@@ -317,7 +319,7 @@ jobs:
       - name: Checkout source
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
         with:
-          # TEMPORARY(1.5.0-rc0): revert to bare inputs.source_ref before merge.
+          # TEMPORARY(1.5.0-rc1): revert to bare inputs.source_ref before merge.
           ref: ${{ inputs.source_ref || 'main' }}
           fetch-depth: 0
           token: ${{ steps.app-token.outputs.token }}
@@ -325,7 +327,7 @@ jobs:
       - name: Set version and create branch
         id: bump
         env:
-          # TEMPORARY(1.5.0-rc0): revert to bare inputs.release_version before merge.
+          # TEMPORARY(1.5.0-rc1): revert to bare inputs.release_version before merge.
           RELEASE_VERSION: ${{ inputs.release_version || '1.5.0' }}
           RELEASE_BRANCH: ${{ inputs.release_branch }}
           APP_SLUG: ${{ steps.app-token.outputs.app-slug }}
@@ -447,7 +449,9 @@ jobs:
         # A dry run sets up the branch/bump only — don't consume an rc.
         if: ${{ !inputs.dry_run }}
         env:
-          RC_ROUND: ${{ inputs.rc_round || '0' }}
+          # TEMPORARY(1.5.0-rc1): push stages rc1 — must match the Validate step
+          # above. REVERT to bare || '0'.
+          RC_ROUND: ${{ inputs.rc_round || (github.event_name == 'push' && '1' || '0') }}
         run: |
           set -euo pipefail
           # Internal staging registries (crates/helm/wheels) are immutable, so each
@@ -611,7 +615,7 @@ jobs:
     with:
       commit_sha: ${{ needs.prepare-branch.outputs.bump_sha }}
       on_demand: true
-      # TEMPORARY(1.5.0-rc0): revert to bare inputs.release_version before merge.
+      # TEMPORARY(1.5.0-rc1): revert to bare inputs.release_version before merge.
       release_version: ${{ inputs.release_version || '1.5.0' }}
       # Operator-chosen internal-staging rc (immutable registries can't be overwritten).
       rc_number: ${{ needs.prepare-branch.outputs.rc_number }}

From 856e1bfe67166da576e9fcce4424af883002ea2d Mon Sep 17 00:00:00 2001
From: pvijayakrish 
Date: Thu, 3 Sep 2026 16:01:20 -0700
Subject: [PATCH 09/29] refactor(release): split cut from RC staging and
 centralize the ops+approval gate
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Per review, separate the one-time branch create + version stamp from the
many-times RC staging path (also dropping each dispatch well under the
10-input cap), and close the release.yml gate bypass.

- cut-release-branch.yml (new, "Cut Release Branch", 3 inputs): authorize
  -> cut-approval -> prepare-branch. Stamps the FULL image-ref set and
  pushes the branch + CI twin. One-time per release.
- on-demand-release.yml ("Stage Release RC", 8 inputs): authorize ->
  resolve-target -> await-post-merge -> invoke-release. Resolves the
  existing branch tip SHA read-only and stages at the chosen rc round via
  release.yml. Its own approval job is dropped — the gate now lives in
  release.yml (below), avoiding a double approval; authorize stays as a
  fast-fail before the ECR wait.
- release.yml: add authorize (ai-dynamo/devops membership) + approval
  (manual-release-approval) gate jobs that prepare-release needs, so EVERY
  caller of this reusable workflow is gated — including a rogue workflow
  that calls it with on_demand=true, and the direct rc workflow_dispatch.
  The gate is condition-scoped to operator runs; the nightly path
  (scheduled / nightly=true) is exempt via a dynamic (empty) environment
  so unattended runs are never blocked.

The in-file gate closes the unmodified-release.yml caller vector; the
server-side controls (environment deployment-branch policy limited to
main + release/*, release/* push restricted to devops, required reviewers
on manual-release-approval, environment-scoped release secrets) remain the
authoritative boundary. nightly-ci and the nightly routing are untouched.

Signed-off-by: pvijayakrish 
---
 .github/workflows/cut-release-branch.yml | 247 ++++++++++++++
 .github/workflows/on-demand-release.yml  | 398 +++++------------------
 .github/workflows/release.yml            |  67 ++++
 3 files changed, 403 insertions(+), 309 deletions(-)
 create mode 100644 .github/workflows/cut-release-branch.yml

diff --git a/.github/workflows/cut-release-branch.yml b/.github/workflows/cut-release-branch.yml
new file mode 100644
index 000000000000..77101f617d31
--- /dev/null
+++ b/.github/workflows/cut-release-branch.yml
@@ -0,0 +1,247 @@
+# SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+# Cut a release branch (one-time): stamp the release version across the tree and
+# push release/ — plus a release/ CI twin for a bare named release
+# so post-merge builds it. RC staging is the separate "Stage Release RC" workflow.
+# Dispatch restricted to the ai-dynamo/devops team.
+
+name: Cut Release Branch
+
+on:
+  workflow_dispatch:
+    inputs:
+      source_ref:
+        description: "Ref to fork from: 'main', a 'release/*' branch, a bare named release branch (v[-]), or a 40-char commit SHA."
+        required: false
+        type: string
+        default: main
+      release_version:
+        description: 'Version to STAMP into the tree, X.Y.Z[.devN|.postN] (PEP 440; .devN = pre-release, .postN = post-release; e.g. 0.0.0, 0.0.0.dev3). Not the branch name.'
+        required: true
+        type: string
+      release_branch:
+        description: 'Branch to create (default release/). Also accepts a bare named release v[-], e.g. v1.3.0-nemotron-ultra-dev.1 — must start with v + the release_version base; a release/ CI twin is pushed for the builds, and artifact versions stay numeric (the name lives only in the branch). For a throwaway rehearsal use e.g. release/0.0.0test — post-merge + release-branch checks run, but it is never staged.'
+        required: false
+        type: string
+        default: ''
+
+permissions:
+  contents: read
+
+jobs:
+  # OPS GATE — only active ai-dynamo/devops members may run this. Fail-closed:
+  # a missing app Members:read permission or any API error denies. Checks BOTH
+  # github.actor and github.triggering_actor (actor does not re-resolve on re-runs).
+  authorize:
+    name: Verify release-ops membership
+    runs-on: prod-default-small-v2
+    environment: automated-release
+    permissions:
+      contents: read
+    steps:
+      - name: Mint app token
+        id: app-token
+        uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1  # v3.2.0
+        with:
+          app-id: ${{ secrets.RELEASE_APP_ID }}
+          private-key: ${{ secrets.RELEASE_APP_PRIVATE_KEY }}
+      - name: Require release-ops team membership
+        env:
+          GH_TOKEN: ${{ steps.app-token.outputs.token }}
+          ORG: ${{ github.repository_owner }}
+          TEAM: devops
+          ACTOR: ${{ github.actor }}
+          TRIGGERING_ACTOR: ${{ github.triggering_actor }}
+        run: |
+          set -euo pipefail
+          for USER in $(printf '%s\n%s\n' "${ACTOR}" "${TRIGGERING_ACTOR}" | sort -u); do
+            HTTP=$(curl -sS -o "${RUNNER_TEMP}/membership.json" -w '%{http_code}' \
+              -H "Authorization: Bearer ${GH_TOKEN}" \
+              -H "Accept: application/vnd.github+json" \
+              "https://api.github.com/orgs/${ORG}/teams/${TEAM}/memberships/${USER}")
+            STATE=$(python3 -c "import json,sys;print(json.load(open(sys.argv[1])).get('state',''))" \
+              "${RUNNER_TEMP}/membership.json" 2>/dev/null || echo "")
+            if [ "${HTTP}" != "200" ] || [ "${STATE}" != "active" ]; then
+              echo "::error::@${USER} is not an active member of ${ORG}/${TEAM} (HTTP ${HTTP}); only release ops may run this workflow."
+              exit 1
+            fi
+            echo "@${USER} authorized via ${ORG}/${TEAM}"
+          done
+
+  # APPROVAL GATE — a release reviewer must approve before any branch is cut.
+  # Enforced by the manual-release-approval environment's required-reviewers rule.
+  cut-approval:
+    name: Cut approval gate
+    needs: [authorize]
+    runs-on: prod-default-small-v2
+    environment: manual-release-approval
+    permissions:
+      contents: read
+    steps:
+      - name: Record approval
+        env:
+          RELEASE_VERSION: ${{ inputs.release_version }}
+          RELEASE_BRANCH: ${{ inputs.release_branch }}
+          # Show the fork point so the reviewer approves a known source, not an
+          # arbitrary ref/SHA.
+          SOURCE_REF: ${{ inputs.source_ref }}
+          ACTOR: ${{ github.actor }}
+        run: echo "Cut ${RELEASE_BRANCH:-release/${RELEASE_VERSION}} (version ${RELEASE_VERSION}) from ${SOURCE_REF} — approved by ${ACTOR}"
+
+  # CREATE RELEASE BRANCH + VERSION BUMP. Pushes with a dedicated app token (not
+  # GITHUB_TOKEN) so the push triggers post-merge CI to build the release images.
+  prepare-branch:
+    name: Create release branch + version bump
+    needs: [authorize, cut-approval]
+    runs-on: prod-default-small-v2
+    environment: automated-release
+    permissions:
+      contents: read
+    outputs:
+      branch: ${{ steps.bump.outputs.branch }}
+      bump_sha: ${{ steps.bump.outputs.bump_sha }}
+      is_test: ${{ steps.bump.outputs.is_test }}
+    steps:
+      - name: Validate inputs
+        env:
+          RELEASE_VERSION: ${{ inputs.release_version }}
+          SOURCE_REF: ${{ inputs.source_ref }}
+          RELEASE_BRANCH: ${{ inputs.release_branch }}
+        run: |
+          set -euo pipefail
+          if [[ ! "${RELEASE_VERSION}" =~ ^[0-9]+\.[0-9]+\.[0-9]+(\.(dev|post)[0-9]+)?$ ]]; then
+            echo "::error::release_version must be X.Y.Z[.devN|.postN] (got '${RELEASE_VERSION}')"
+            exit 1
+          fi
+          if [[ -n "${RELEASE_BRANCH}" && ! "${RELEASE_BRANCH}" =~ ^release/[A-Za-z0-9._-]+$ \
+                && ! "${RELEASE_BRANCH}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-[A-Za-z0-9._-]+)?$ ]]; then
+            echo "::error::release_branch must be release/ or a bare named release v[-] (got '${RELEASE_BRANCH}')"
+            exit 1
+          fi
+          # A bare named branch must carry the version it stamps (v-);
+          # artifact versions stay numeric, so this prefix is the only tie between them.
+          if [[ "${RELEASE_BRANCH}" =~ ^v[0-9] ]]; then
+            BASE="${RELEASE_VERSION%%.dev*}"; BASE="${BASE%%.post*}"
+            if [[ "${RELEASE_BRANCH}" != "v${BASE}" && "${RELEASE_BRANCH}" != "v${BASE}-"* ]]; then
+              echo "::error::named branch '${RELEASE_BRANCH}' must start with v${BASE} (the release_version base)"
+              exit 1
+            fi
+          fi
+          if [[ ! "${SOURCE_REF}" =~ ^(main|release/[A-Za-z0-9._-]+|v[0-9]+\.[0-9]+\.[0-9]+(-[A-Za-z0-9._-]+)?|[0-9a-f]{40})$ ]]; then
+            echo "::error::source_ref must be 'main', a release/* branch, a named release branch (v[-]), or a 40-char SHA (got '${SOURCE_REF}')"
+            exit 1
+          fi
+
+      - name: Mint app token
+        id: app-token
+        uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1  # v3.2.0
+        with:
+          app-id: ${{ secrets.RELEASE_APP_ID }}
+          private-key: ${{ secrets.RELEASE_APP_PRIVATE_KEY }}
+
+      - name: Checkout source
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        with:
+          ref: ${{ inputs.source_ref }}
+          fetch-depth: 0
+          token: ${{ steps.app-token.outputs.token }}
+
+      - name: Set version and create branch
+        id: bump
+        env:
+          RELEASE_VERSION: ${{ inputs.release_version }}
+          RELEASE_BRANCH: ${{ inputs.release_branch }}
+          APP_SLUG: ${{ steps.app-token.outputs.app-slug }}
+          GH_TOKEN: ${{ steps.app-token.outputs.token }}
+        run: |
+          set -euo pipefail
+
+          BRANCH="${RELEASE_BRANCH:-release/${RELEASE_VERSION}}"
+          # Real releases: release/X.Y.Z[.devN|.postN] or bare named v[-];
+          # anything else is a throwaway test branch. A bare v-branch matches no CI
+          # push filter (post-merge fires only on main/release/*), so a release/
+          # CI twin is pushed to build the images.
+          CI_BRANCH="${BRANCH}"
+          if [[ "${BRANCH}" =~ ^release/[0-9]+\.[0-9]+\.[0-9]+(\.(dev|post)[0-9]+)?$ ]]; then
+            IS_TEST=false
+          elif [[ "${BRANCH}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-[A-Za-z0-9._-]+)?$ ]]; then
+            IS_TEST=false
+            CI_BRANCH="release/${BRANCH}"
+          else
+            IS_TEST=true
+            echo "::warning::branch '${BRANCH}' is not release/X.Y.Z[.devN|.postN] or v[-] — treated as a THROWAWAY TEST branch: it is cut and built, but Stage Release RC will refuse to stage it."
+          fi
+
+          # Fetch the version-bump script from THIS workflow's commit (GITHUB_SHA);
+          # source_ref may predate --set-version.
+          git show "${GITHUB_SHA}:.github/scripts/apply_dev_version.py" > "${RUNNER_TEMP}/apply_dev_version.py"
+
+          # Reuse the existing branch when present (a re-run adds only the diff);
+          # else base on source_ref. Never clobber. refs/heads/ refspecs everywhere:
+          # a bare v-branch can collide with a GA tag (v1.4.1).
+          if git ls-remote --exit-code --heads origin "${BRANCH}" >/dev/null 2>&1; then
+            git fetch origin "refs/heads/${BRANCH}"
+            git checkout -B "${BRANCH}" FETCH_HEAD
+            EXISTED=1
+          else
+            git checkout -b "${BRANCH}"
+            EXISTED=0
+          fi
+
+          # Stamp the version + FULL image-ref set: the branch serves every RC and RC
+          # staging never re-stamps, so a subset here would strand un-selected images
+          # on the my-registry/:my-tag placeholder. --containers all also avoids the
+          # partial-selection helm-pin failure. wheels/crates aren't passed — the
+          # package version stamp is unified.
+          python3 "${RUNNER_TEMP}/apply_dev_version.py" "" . --set-version "${RELEASE_VERSION}" \
+            --containers all --helm all --image-refs
+
+          if ! command -v cargo >/dev/null 2>&1; then
+            curl --proto '=https' --tlsv1.2 --retry 5 --retry-delay 2 -sSf https://sh.rustup.rs \
+              | sh -s -- -y --profile minimal --default-toolchain stable
+            source "${HOME}/.cargo/env"
+          fi
+          # Sync EVERY Cargo.lock (root + nested workspaces: bindings/python, kvbm,
+          # runtime/examples): a root-only `cargo update` leaves their pins stale and
+          # fails the `--locked` checks. Discovered from git so a newly added nested
+          # workspace can't silently regress.
+          git ls-files | grep -E '(^|/)Cargo\.lock$' | sort | while IFS= read -r lock; do
+            manifest="$(dirname "${lock}")/Cargo.toml"
+            manifest="${manifest#./}"
+            echo "Syncing ${lock}"
+            cargo update --workspace --manifest-path "${manifest}"
+          done
+
+          APP_USER_ID=$(curl -fsSL --retry 3 \
+            -H "Authorization: Bearer ${GH_TOKEN}" \
+            -H "Accept: application/vnd.github+json" \
+            "https://api.github.com/users/${APP_SLUG}%5Bbot%5D" \
+            | python3 -c "import sys,json;print(json.load(sys.stdin)['id'])")
+          git config user.name "${APP_SLUG}[bot]"
+          git config user.email "${APP_USER_ID}+${APP_SLUG}[bot]@users.noreply.github.com"
+          # Commit + push only on a diff. No diff on an existing branch => reuse (no
+          # push); no diff on a new branch => empty commit so the push fires post-merge.
+          git add -A
+          if git diff --cached --quiet; then
+            if [ "${EXISTED}" = "1" ]; then
+              echo "${BRANCH} already matches the release scripts; nothing to commit (reuse, no push)."
+            else
+              git commit -s --allow-empty -m "release: set version ${RELEASE_VERSION}"
+              git push origin "HEAD:refs/heads/${BRANCH}"
+            fi
+          else
+            git commit -s -m "release: set version ${RELEASE_VERSION}"
+            git push origin "HEAD:refs/heads/${BRANCH}"
+          fi
+          # Keep the CI twin of a bare named branch on this commit (fast-forward;
+          # no-op when already there) so post-merge CI builds it.
+          if [ "${CI_BRANCH}" != "${BRANCH}" ]; then
+            git push origin "HEAD:refs/heads/${CI_BRANCH}"
+          fi
+
+          echo "branch=${BRANCH}" >> "$GITHUB_OUTPUT"
+          echo "bump_sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
+          echo "is_test=${IS_TEST}" >> "$GITHUB_OUTPUT"
+          echo "Prepared ${BRANCH} @ $(git rev-parse HEAD) (is_test=${IS_TEST}, existed=${EXISTED})"
+          echo "Next: run 'Stage Release RC' with release_branch=${BRANCH} once post-merge CI has built the images."
diff --git a/.github/workflows/on-demand-release.yml b/.github/workflows/on-demand-release.yml
index 78554e43a424..2e0215168fe6 100644
--- a/.github/workflows/on-demand-release.yml
+++ b/.github/workflows/on-demand-release.yml
@@ -1,69 +1,49 @@
 # SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 
-# On-demand release: branch from main / a release branch / a commit SHA, set
-# the release version, commit it to a release/ branch or a bare named
-# release branch (v[-], pushed with a release/ CI twin so
-# post-merge CI builds the images), then stage all artifacts and trigger the
-# GitLab release-automation pipeline via release.yml. Dispatch is restricted
-# to active members of the ai-dynamo/devops team.
+# Stage Release RC: stage (or re-stage) an EXISTING release branch at a chosen rc
+# round via release.yml, then trigger the GitLab pipeline. Run once per rc round.
+# The branch + version stamp are created by "Cut Release Branch"; this workflow is
+# read-only on the tree. Dispatch restricted to the ai-dynamo/devops team.
 
-name: On-Demand Release
+name: Stage Release RC
 
-# workflow_dispatch is capped at 10 inputs by GitHub — this block is at 10;
-# fold any new knob into an existing input.
 on:
-  # TEMPORARY(1.5.0-rc1): push runs the flow as release_version 1.5.0 from main
-  # (reuses release/1.5.0), rc1 (rc0 already staged), all artifacts. REVERT before merge.
-  push:
-    branches:
-      - pull-request/13961
   workflow_dispatch:
     inputs:
-      source_ref:
-        description: "Ref to fork from: 'main', a 'release/*' branch, a bare named release branch (v[-]), or a 40-char commit SHA."
-        required: false
+      release_branch:
+        description: 'Existing release branch to stage: release/X.Y.Z[.devN|.postN] or a bare named release v[-]. Create it first with "Cut Release Branch". The tip is staged, so cherry-picks land in the next rc.'
+        required: true
         type: string
-        default: main
       release_version:
-        description: 'Version to STAMP into the tree, X.Y.Z[.devN|.postN] (PEP 440; .devN = pre-release, .postN = post-release; e.g. 0.0.0, 0.0.0.dev3). Not the branch name.'
+        description: 'Numeric version to publish, X.Y.Z[.devN|.postN] (must match the branch; for a named branch it is the base the branch encodes).'
         required: true
         type: string
-      release_branch:
-        description: 'Branch to create (default release/). Also accepts a bare named release v[-], e.g. v1.3.0-nemotron-ultra-dev.1 — must start with v + the release_version base; a release/ CI twin is pushed for the builds, and artifact versions stay numeric (the name lives only in the branch). For a throwaway run use e.g. release/0.0.0test — post-merge + release-branch checks run, but the publish jobs skip.'
-        required: false
-        type: string
-        default: ''
       rc_round:
-        description: 'Internal staging RC:  or . (e.g. 2 or 2.1 -> staging versions X.Y.Z-rc.2 / X.Y.Z-rc.2.1, a numerically-ordered SemVer pre-release; also the wheel rc subpath). Re-stages of the same version need a distinct value (immutable registries). NO GitHub tag is created. Default 0.'
+        description: 'Internal staging RC:  or . (e.g. 2 or 2.1 -> staging versions X.Y.Z-rc.2 / X.Y.Z-rc.2.1, a numerically-ordered SemVer pre-release; also the wheel rc subpath). Re-stages of the same version need a distinct value (immutable registries). NO GitHub tag is created.'
         required: false
         type: string
         default: '0'
       containers:
-        description: 'Containers to release: all | none | csv of vllm[:runtime|:efa],sglang[...],trtllm[...],frontend,operator,planner,sidecar.'
+        description: 'Containers to stage: all | none | csv of vllm[:runtime|:efa],sglang[...],trtllm[...],frontend,operator,planner,sidecar.'
         required: false
         type: string
         default: all
       wheels:
-        description: 'Wheels to release: all | none | csv of ai-dynamo,ai-dynamo-runtime,kvbm.'
+        description: 'Wheels to stage: all | none | csv of ai-dynamo,ai-dynamo-runtime,kvbm.'
         required: false
         type: string
         default: all
       crates:
-        description: 'Crates to release: all | none | csv of crate names (must be dependency-closed).'
+        description: 'Crates to stage: all | none | csv of crate names (must be dependency-closed).'
         required: false
         type: string
         default: all
       helm:
-        description: 'Helm charts to release: all | none | platform.'
+        description: 'Helm charts to stage: all | none | platform.'
         required: false
         type: string
         default: all
-      dry_run:
-        description: 'Create the branch + version bump only; skip staging and the GitLab trigger.'
-        required: false
-        type: boolean
-        default: false
       skip_gitlab_pipeline:
         description: 'Skip the GitLab release automation pipeline trigger. Emergency use only.'
         required: false
@@ -74,16 +54,11 @@ permissions:
   contents: read
 
 jobs:
-  # ============================================================================
-  # OPS GATE — only active members of the org's devops team
-  # (github.com/orgs/ai-dynamo/teams/devops) may run this workflow. Fail
-  # closed: missing app org Members:read permission or any API error denies.
-  # On re-runs github.actor stays the ORIGINAL dispatcher and the
-  # re-runner appears as github.triggering_actor, so BOTH are checked. Caveat:
-  # "Re-run failed jobs" does not re-execute a previously successful authorize —
-  # the manual-release-approval environment is the backstop for replayed
-  # downstream jobs.
-  # ============================================================================
+  # OPS GATE (fast-fail) — reject a non-ai-dynamo/devops dispatch before the ECR
+  # wait. Fail-closed: a missing app Members:read permission or any API error denies.
+  # Checks BOTH github.actor and github.triggering_actor (actor does not re-resolve
+  # on re-runs). The authoritative ops + approval gate for the publish path lives in
+  # release.yml (invoke-release), so it also binds direct and rogue callers.
   authorize:
     name: Verify release-ops membership
     runs-on: prod-default-small-v2
@@ -106,12 +81,6 @@ jobs:
           TRIGGERING_ACTOR: ${{ github.triggering_actor }}
         run: |
           set -euo pipefail
-          # TEMPORARY(1.5.0-rc1): the push test loop has no dispatching actor to
-          # check. REVERT before merge.
-          if [ "${GITHUB_EVENT_NAME}" = "push" ]; then
-            echo "TEMPORARY: push loop — skipping the ops gate"
-            exit 0
-          fi
           for USER in $(printf '%s\n%s\n' "${ACTOR}" "${TRIGGERING_ACTOR}" | sort -u); do
             HTTP=$(curl -sS -o "${RUNNER_TEMP}/membership.json" -w '%{http_code}' \
               -H "Authorization: Bearer ${GH_TOKEN}" \
@@ -126,57 +95,49 @@ jobs:
             echo "@${USER} authorized via ${ORG}/${TEAM}"
           done
 
-  # ============================================================================
-  # CREATE RELEASE BRANCH + VERSION BUMP
-  # Pushes with a dedicated token (not GITHUB_TOKEN) so the push triggers
-  # post-merge CI, which builds the release images for the bump commit.
-  # ============================================================================
-  prepare-branch:
-    name: Create release branch + version bump
+  # RESOLVE TARGET (read-only) — validate inputs; resolve the branch tip SHA (the
+  # commit whose post-merge images this rc stages), the rc round, and the subsets.
+  resolve-target:
+    name: Resolve staging target
     needs: [authorize]
     runs-on: prod-default-small-v2
     environment: automated-release
     permissions:
       contents: read
     outputs:
-      branch: ${{ steps.bump.outputs.branch }}
-      bump_sha: ${{ steps.bump.outputs.bump_sha }}
-      is_test: ${{ steps.bump.outputs.is_test }}
-      rc_number: ${{ steps.rc.outputs.rc_number }}
+      bump_sha: ${{ steps.target.outputs.bump_sha }}
+      rc_number: ${{ steps.target.outputs.rc_number }}
       containers: ${{ steps.resolve.outputs.containers }}
       wheels: ${{ steps.resolve.outputs.wheels }}
       crates: ${{ steps.resolve.outputs.crates }}
       helm: ${{ steps.resolve.outputs.helm }}
     steps:
-      - name: Validate inputs
+      - name: Validate and resolve target SHA
+        id: target
         env:
-          # TEMPORARY(1.5.0-rc1): || fallbacks cover inputless push events. REVERT
-          # to bare inputs.* before merge.
-          RELEASE_VERSION: ${{ inputs.release_version || '1.5.0' }}
-          SOURCE_REF: ${{ inputs.source_ref || 'main' }}
+          RELEASE_VERSION: ${{ inputs.release_version }}
           RELEASE_BRANCH: ${{ inputs.release_branch }}
-          # || guards a cleared dispatch field (inputs default fills the rest).
-          # TEMPORARY(1.5.0-rc1): push stages rc1 — rc0 is already burned on the
-          # immutable registries. Dispatch is unchanged. REVERT to bare || '0'.
-          RC_ROUND: ${{ inputs.rc_round || (github.event_name == 'push' && '1' || '0') }}
+          RC_ROUND: ${{ inputs.rc_round }}
+          REPO: ${{ github.repository }}
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
           set -euo pipefail
           if [[ ! "${RELEASE_VERSION}" =~ ^[0-9]+\.[0-9]+\.[0-9]+(\.(dev|post)[0-9]+)?$ ]]; then
             echo "::error::release_version must be X.Y.Z[.devN|.postN] (got '${RELEASE_VERSION}')"
             exit 1
           fi
-          # Validated before the branch cut — a typo caught after the push wastes hours of CI.
-          if ! [[ "${RC_ROUND}" =~ ^[0-9]+(\.[0-9]+)?$ ]]; then
-            echo "::error::rc_round must be '' or '.' (non-negative integers; got '${RC_ROUND}')"
-            exit 1
-          fi
-          if [[ -n "${RELEASE_BRANCH}" && ! "${RELEASE_BRANCH}" =~ ^release/[A-Za-z0-9._-]+$ \
-                && ! "${RELEASE_BRANCH}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-[A-Za-z0-9._-]+)?$ ]]; then
-            echo "::error::release_branch must be release/ or a bare named release v[-] (got '${RELEASE_BRANCH}')"
+          # Only REAL release branches can be staged. A bare named release builds on
+          # its release/ CI twin (post-merge fires only on main/release/*), so
+          # its images live under the twin's SHA — resolve from there.
+          if [[ "${RELEASE_BRANCH}" =~ ^release/[0-9]+\.[0-9]+\.[0-9]+(\.(dev|post)[0-9]+)?$ ]]; then
+            CI_BRANCH="${RELEASE_BRANCH}"
+          elif [[ "${RELEASE_BRANCH}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-[A-Za-z0-9._-]+)?$ ]]; then
+            CI_BRANCH="release/${RELEASE_BRANCH}"
+          else
+            echo "::error::release_branch must be a real release branch — release/X.Y.Z[.devN|.postN] or a bare named release v[-] (got '${RELEASE_BRANCH}'). Cut it first with 'Cut Release Branch'."
             exit 1
           fi
-          # A bare named branch must carry the version it stamps (v-);
-          # artifact versions stay numeric, so this prefix is the only tie between them.
+          # A named branch encodes only its base; verify it matches release_version's base.
           if [[ "${RELEASE_BRANCH}" =~ ^v[0-9] ]]; then
             BASE="${RELEASE_VERSION%%.dev*}"; BASE="${BASE%%.post*}"
             if [[ "${RELEASE_BRANCH}" != "v${BASE}" && "${RELEASE_BRANCH}" != "v${BASE}-"* ]]; then
@@ -184,32 +145,36 @@ jobs:
               exit 1
             fi
           fi
-          if [[ ! "${SOURCE_REF}" =~ ^(main|release/[A-Za-z0-9._-]+|v[0-9]+\.[0-9]+\.[0-9]+(-[A-Za-z0-9._-]+)?|[0-9a-f]{40})$ ]]; then
-            echo "::error::source_ref must be 'main', a release/* branch, a named release branch (v[-]), or a 40-char SHA (got '${SOURCE_REF}')"
+          if ! [[ "${RC_ROUND}" =~ ^[0-9]+(\.[0-9]+)?$ ]]; then
+            echo "::error::rc_round must be '' or '.' (non-negative integers; got '${RC_ROUND}')"
+            exit 1
+          fi
+          # Resolve the CI branch tip. git/ref/heads (singular) is an EXACT match —
+          # 404 if the branch was never cut.
+          HTTP=$(curl -sS -o "${RUNNER_TEMP}/ref.json" -w '%{http_code}' \
+            -H "Authorization: Bearer ${GH_TOKEN}" \
+            -H "Accept: application/vnd.github+json" \
+            "https://api.github.com/repos/${REPO}/git/ref/heads/${CI_BRANCH}")
+          if [ "${HTTP}" != "200" ]; then
+            echo "::error::branch '${CI_BRANCH}' not found (HTTP ${HTTP}) — cut it first with 'Cut Release Branch'."
             exit 1
           fi
+          SHA=$(python3 -c "import json,sys;print(json.load(open(sys.argv[1]))['object']['sha'])" "${RUNNER_TEMP}/ref.json")
+          echo "bump_sha=${SHA}" >> "$GITHUB_OUTPUT"
+          echo "rc_number=${RC_ROUND}" >> "$GITHUB_OUTPUT"
+          echo "Staging ${RELEASE_BRANCH} (CI branch ${CI_BRANCH}) @ ${SHA} as ${RELEASE_VERSION}-rc.${RC_ROUND}"
 
-      # Normalize the per-type fields into canonical subset strings (all|none|csv
-      # of normalized tokens) consumed by the bump, release.yml, and the GitLab
-      # trigger. The container grammar expands [:runtime|:efa] (bare fw = both)
-      # and validates every token; unknown tokens fail fast.
+      # Normalize the per-type fields to canonical subset strings (all|none|csv)
+      # for await-post-merge, release.yml, and the GitLab trigger. The container
+      # grammar expands [:runtime|:efa] (bare fw = both); unknown tokens fail fast.
       - name: Resolve artifact selection
         id: resolve
         env:
-          # TEMPORARY(1.5.0-rc1): revert to bare inputs.release_version before merge.
-          RELEASE_VERSION: ${{ inputs.release_version || '1.5.0' }}
-          CONTAINERS: ${{ inputs.containers || 'all' }}
-          WHEELS: ${{ inputs.wheels || 'all' }}
-          # Stage ALL publishable workspace crates ('all' = the max). A hardcoded
-          # "public-only" list is infeasible: dynamo-llm depends on dynamo-bench
-          # (lib/llm/Cargo.toml), so excluding bench breaks the dependency closure, and
-          # the publishable graph differs across release branches (e.g. release/1.3.0 has
-          # no dynamo-parsers/protocols/renderer/tokenizers). stage_crates.py publishes
-          # the full publishable set for whichever branch it runs on, topo-ordered, with
-          # dynamo-mocker's aiconfigurator-core git dep stripped at publish (strip_git_deps);
-          # the expect-version gate guards against any crate not at the release version.
-          CRATES: ${{ inputs.crates || 'all' }}
-          HELM: ${{ inputs.helm || 'all' }}
+          RELEASE_VERSION: ${{ inputs.release_version }}
+          CONTAINERS: ${{ inputs.containers }}
+          WHEELS: ${{ inputs.wheels }}
+          CRATES: ${{ inputs.crates }}
+          HELM: ${{ inputs.helm }}
         run: |
           set -euo pipefail
           python3 <<'PY' > "${RUNNER_TEMP}/selection.env"
@@ -309,179 +274,13 @@ jobs:
           cat "${RUNNER_TEMP}/selection.env"
           cat "${RUNNER_TEMP}/selection.env" >> "$GITHUB_OUTPUT"
 
-      - name: Mint app token
-        id: app-token
-        uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1  # v3.2.0
-        with:
-          app-id: ${{ secrets.RELEASE_APP_ID }}
-          private-key: ${{ secrets.RELEASE_APP_PRIVATE_KEY }}
-
-      - name: Checkout source
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
-        with:
-          # TEMPORARY(1.5.0-rc1): revert to bare inputs.source_ref before merge.
-          ref: ${{ inputs.source_ref || 'main' }}
-          fetch-depth: 0
-          token: ${{ steps.app-token.outputs.token }}
-
-      - name: Set version and create branch
-        id: bump
-        env:
-          # TEMPORARY(1.5.0-rc1): revert to bare inputs.release_version before merge.
-          RELEASE_VERSION: ${{ inputs.release_version || '1.5.0' }}
-          RELEASE_BRANCH: ${{ inputs.release_branch }}
-          APP_SLUG: ${{ steps.app-token.outputs.app-slug }}
-          GH_TOKEN: ${{ steps.app-token.outputs.token }}
-          CONTAINERS: ${{ steps.resolve.outputs.containers }}
-          HELM: ${{ steps.resolve.outputs.helm }}
-          # Not passed to the bump (the version stamp is unified) — recorded in the
-          # release commit message so the branch states what it actually ships.
-          WHEELS: ${{ steps.resolve.outputs.wheels }}
-          CRATES: ${{ steps.resolve.outputs.crates }}
-        run: |
-          set -euo pipefail
-
-          BRANCH="${RELEASE_BRANCH:-release/${RELEASE_VERSION}}"
-          # Real releases: pure release/X.Y.Z[.devN|.postN], or a bare named release
-          # v[-] (e.g. v1.3.0-nemotron-ultra-dev.1). Anything else
-          # (e.g. release/0.0.0test) is a throwaway test branch. Bare branches match
-          # no CI push filter (post-merge and runner admission fire only on
-          # main/release/*), so a release/ CI twin is pushed alongside to
-          # build the images; staging consumes the SHA, not the ref.
-          CI_BRANCH="${BRANCH}"
-          if [[ "${BRANCH}" =~ ^release/[0-9]+\.[0-9]+\.[0-9]+(\.(dev|post)[0-9]+)?$ ]]; then
-            IS_TEST=false
-          elif [[ "${BRANCH}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-[A-Za-z0-9._-]+)?$ ]]; then
-            IS_TEST=false
-            CI_BRANCH="release/${BRANCH}"
-          else
-            IS_TEST=true
-            echo "::warning::branch '${BRANCH}' is not release/X.Y.Z[.devN|.postN] or v[-] — treated as a THROWAWAY TEST branch: it is cut and built, but staging, approval and the GitLab trigger are all skipped."
-          fi
-
-          # Fetch the version-bump script from THIS workflow's commit (GITHUB_SHA);
-          # source_ref may predate --set-version.
-          git show "${GITHUB_SHA}:.github/scripts/apply_dev_version.py" > "${RUNNER_TEMP}/apply_dev_version.py"
-
-          # Base the release branch on the existing remote branch when it exists, so a
-          # re-run only adds the diff (e.g. a newly-added script step lands on the branch
-          # that was cut before it existed); otherwise base it on the checked-out
-          # source_ref. We never delete/clobber — the diff (if any) is committed on top.
-          # refs/heads/ refspecs everywhere: a bare v-branch name can collide with
-          # a GA tag (v1.4.1), and short names would resolve/push ambiguously.
-          if git ls-remote --exit-code --heads origin "${BRANCH}" >/dev/null 2>&1; then
-            git fetch origin "refs/heads/${BRANCH}"
-            git checkout -B "${BRANCH}" FETCH_HEAD
-            EXISTED=1
-          else
-            git checkout -b "${BRANCH}"
-            EXISTED=0
-          fi
-
-          # Set the absolute version, then sync the lockfile (below).
-          # --image-refs points the my-registry/:my-tag placeholders in docs, examples
-          # and deploy manifests at the GA registry + release version, but ONLY for the
-          # images in --containers: a partial release must never publish a branch that
-          # tells users to pull an image it didn't build. wheels/crates are deliberately
-          # NOT passed — the package version stamp is unified (the containers embed
-          # wheels built from this tree, so it must move regardless of the selection).
-          python3 "${RUNNER_TEMP}/apply_dev_version.py" "" . --set-version "${RELEASE_VERSION}" \
-            --containers "${CONTAINERS}" --helm "${HELM}" --image-refs
-
-          if ! command -v cargo >/dev/null 2>&1; then
-            curl --proto '=https' --tlsv1.2 --retry 5 --retry-delay 2 -sSf https://sh.rustup.rs \
-              | sh -s -- -y --profile minimal --default-toolchain stable
-            source "${HOME}/.cargo/env"
-          fi
-          # Keep EVERY Cargo.lock in sync so post-merge `cargo test --locked` and the
-          # pre-merge `cargo metadata --locked` matrix both pass. lib/bindings/python,
-          # lib/bindings/kvbm and lib/runtime/examples are SEPARATE cargo workspaces
-          # with their own lock files, so a root-only `cargo update --workspace` leaves
-          # their workspace-crate pins on the PREVIOUS version and their `--locked`
-          # checks fail. Discovered from git, not hardcoded, so a newly added nested
-          # workspace cannot silently regress the next release.
-          git ls-files | grep -E '(^|/)Cargo\.lock$' | sort | while IFS= read -r lock; do
-            manifest="$(dirname "${lock}")/Cargo.toml"
-            manifest="${manifest#./}"
-            echo "Syncing ${lock}"
-            cargo update --workspace --manifest-path "${manifest}"
-          done
-
-          APP_USER_ID=$(curl -fsSL --retry 3 \
-            -H "Authorization: Bearer ${GH_TOKEN}" \
-            -H "Accept: application/vnd.github+json" \
-            "https://api.github.com/users/${APP_SLUG}%5Bbot%5D" \
-            | python3 -c "import sys,json;print(json.load(sys.stdin)['id'])")
-          git config user.name "${APP_SLUG}[bot]"
-          git config user.email "${APP_USER_ID}+${APP_SLUG}[bot]@users.noreply.github.com"
-          # Commit + push only when the scripts produced a diff vs the branch's current
-          # state. No diff on an existing branch => reuse (no push); no diff on a brand-
-          # new branch => still record a release commit so the push fires post-merge CI.
-          # Record the artifact selection in the commit body: the version stamp is
-          # unified, so the diff alone cannot tell a reader which artifacts this
-          # release actually ships.
-          SELECTION_MSG="containers=${CONTAINERS} wheels=${WHEELS} crates=${CRATES} helm=${HELM}"
-          git add -A
-          if git diff --cached --quiet; then
-            if [ "${EXISTED}" = "1" ]; then
-              echo "${BRANCH} already matches the release scripts; nothing to commit (reuse, no push)."
-            else
-              git commit -s --allow-empty -m "release: set version ${RELEASE_VERSION}" -m "${SELECTION_MSG}"
-              git push origin "HEAD:refs/heads/${BRANCH}"
-            fi
-          else
-            git commit -s -m "release: set version ${RELEASE_VERSION}" -m "${SELECTION_MSG}"
-            git push origin "HEAD:refs/heads/${BRANCH}"
-          fi
-          # Keep the CI twin of a bare named branch on this commit (fast-forward;
-          # no-op when already there) so post-merge CI builds it.
-          if [ "${CI_BRANCH}" != "${BRANCH}" ]; then
-            git push origin "HEAD:refs/heads/${CI_BRANCH}"
-          fi
-
-          echo "branch=${BRANCH}" >> "$GITHUB_OUTPUT"
-          echo "bump_sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
-          echo "is_test=${IS_TEST}" >> "$GITHUB_OUTPUT"
-          echo "Prepared ${BRANCH} @ $(git rev-parse HEAD) (is_test=${IS_TEST}, existed=${EXISTED})"
-
-      - name: Resolve internal staging rc
-        id: rc
-        # A dry run sets up the branch/bump only — don't consume an rc.
-        if: ${{ !inputs.dry_run }}
-        env:
-          # TEMPORARY(1.5.0-rc1): push stages rc1 — must match the Validate step
-          # above. REVERT to bare || '0'.
-          RC_ROUND: ${{ inputs.rc_round || (github.event_name == 'push' && '1' || '0') }}
-        run: |
-          set -euo pipefail
-          # Internal staging registries (crates/helm/wheels) are immutable, so each
-          # (re-)stage of the same version needs a distinct rc. We take the rc as an
-          # INPUT (no GitHub tag is created): '' or '.'.
-          # Downstream builds X.Y.Z-rc.[.] — a SemVer pre-release whose
-          # dot-separated numeric identifiers order correctly (rc.2.9 < rc.2.10 <
-          # rc.10.0). GA promotion (GitLab) still uses the clean version.
-          if ! [[ "${RC_ROUND}" =~ ^[0-9]+(\.[0-9]+)?$ ]]; then
-            echo "::error::rc_round must be '' or '.' (non-negative integers; got '${RC_ROUND}')"
-            exit 1
-          fi
-          echo "rc_number=${RC_ROUND}" >> "$GITHUB_OUTPUT"
-          echo "Internal staging rc=${RC_ROUND} (no GitHub tag created)"
-
-  # ============================================================================
-  # WAIT FOR POST-MERGE BUILDS (not tests)
-  # The branch push above triggers post-merge-ci.yml, which builds + pushes the
-  # release images to ECR for the bump commit. We wait only for the ECR images the
-  # SELECTED artifacts consume, so we proceed as soon as those builds finish —
-  # WITHOUT waiting for post-merge tests/compliance/deploy. Tests don't push images,
-  # and the bump SHA is unique, so a present tag is unambiguously this run's build
-  # output. Runs on an ECR-capable runner (same IRSA path as stage-wheels).
-  # ============================================================================
+  # WAIT FOR POST-MERGE BUILDS (not tests) — poll ECR for only the SELECTED
+  # artifacts' image tags at the target SHA, so we proceed as soon as those builds
+  # finish (not the whole post-merge run). The SHA is unique, so a present tag is
+  # unambiguously that commit's output. ECR-capable runner (stage-wheels IRSA path).
   await-post-merge:
     name: Wait for post-merge builds
-    needs: [prepare-branch]
-    # Test branches (is_test) stop at the branch cut: post-merge + release-branch
-    # checks still run on them, but nothing is staged or published.
-    if: ${{ !inputs.dry_run && needs.prepare-branch.outputs.is_test != 'true' }}
+    needs: [resolve-target]
     runs-on: prod-tester-amd-v1
     environment: automated-release
     permissions:
@@ -492,12 +291,12 @@ jobs:
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           REPO: ${{ github.repository }}
-          SHA: ${{ needs.prepare-branch.outputs.bump_sha }}
+          SHA: ${{ needs.resolve-target.outputs.bump_sha }}
           AWS_DEFAULT_REGION: ${{ vars.AWS_DEFAULT_REGION }}
           ECR_REPOSITORY: ${{ vars.ECR_REPOSITORY }}
-          CONTAINERS: ${{ needs.prepare-branch.outputs.containers }}
-          WHEELS: ${{ needs.prepare-branch.outputs.wheels }}
-          CRATES: ${{ needs.prepare-branch.outputs.crates }}
+          CONTAINERS: ${{ needs.resolve-target.outputs.containers }}
+          WHEELS: ${{ needs.resolve-target.outputs.wheels }}
+          CRATES: ${{ needs.resolve-target.outputs.crates }}
         run: |
           set -euo pipefail
           aws sts get-caller-identity >/dev/null || { echo "::error::no AWS credentials on this runner"; exit 1; }
@@ -533,7 +332,7 @@ jobs:
 
           echo "Waiting for post-merge builds to publish ${#TAGS[@]} ECR tag(s): ${TAGS[*]}"
           # 3h budget (runtime builds alone get 120 min); 15-min fast-fail if
-          # post-merge never started for the bump SHA.
+          # post-merge never started for the target SHA.
           start=$(date +%s)
           deadline=$(( start + 10800 ))
           start_deadline=$(( start + 900 ))
@@ -568,7 +367,7 @@ jobs:
             fi
             if [ "${API_OK}" = "true" ]; then
               if [ "${COUNT}" = "0" ] && [ "$(date +%s)" -ge "${start_deadline}" ]; then
-                echo "::error::post-merge CI never started for ${SHA} (branch push may not have triggered it); required image(s) missing: ${missing[*]}"
+                echo "::error::post-merge CI never started for ${SHA}; is the branch tip built? required image(s) missing: ${missing[*]}"
                 exit 1
               fi
               if [ "${COUNT}" != "0" ] && [ "${STATUS}" = "completed" ]; then
@@ -586,43 +385,24 @@ jobs:
             sleep 30
           done
 
-  # ============================================================================
-  # MANUAL APPROVAL GATE before publishing.
-  # ============================================================================
-  release-approval:
-    name: Release approval gate
-    needs: [prepare-branch, await-post-merge]
-    if: ${{ !inputs.dry_run && needs.prepare-branch.outputs.is_test != 'true' }}
-    runs-on: prod-default-small-v2
-    environment: manual-release-approval
-    permissions:
-      contents: read
-    steps:
-      - name: Record approval
-        env:
-          BRANCH: ${{ needs.prepare-branch.outputs.branch }}
-          ACTOR: ${{ github.actor }}
-        run: echo "On-demand release ${BRANCH} approved by ${ACTOR}"
-
-  # ============================================================================
-  # STAGE ARTIFACTS + TRIGGER GITLAB (release.yml in on-demand mode)
-  # ============================================================================
+  # STAGE ARTIFACTS + TRIGGER GITLAB (release.yml in on-demand mode). release.yml's
+  # own authorize + approval gate covers the publish path for every caller, so the
+  # ops/approval gate is NOT duplicated here; this workflow's authorize job stays as
+  # a fast-fail so a non-devops dispatch is rejected before the ECR wait.
   invoke-release:
     name: Stage release
-    needs: [prepare-branch, await-post-merge, release-approval]
-    if: ${{ !inputs.dry_run && needs.prepare-branch.outputs.is_test != 'true' }}
+    needs: [resolve-target, await-post-merge]
     uses: ./.github/workflows/release.yml
     with:
-      commit_sha: ${{ needs.prepare-branch.outputs.bump_sha }}
+      commit_sha: ${{ needs.resolve-target.outputs.bump_sha }}
       on_demand: true
-      # TEMPORARY(1.5.0-rc1): revert to bare inputs.release_version before merge.
-      release_version: ${{ inputs.release_version || '1.5.0' }}
+      release_version: ${{ inputs.release_version }}
       # Operator-chosen internal-staging rc (immutable registries can't be overwritten).
-      rc_number: ${{ needs.prepare-branch.outputs.rc_number }}
-      # Normalized subsets from resolve-selection.
-      containers: ${{ needs.prepare-branch.outputs.containers }}
-      wheels: ${{ needs.prepare-branch.outputs.wheels }}
-      crates: ${{ needs.prepare-branch.outputs.crates }}
-      helm: ${{ needs.prepare-branch.outputs.helm }}
+      rc_number: ${{ needs.resolve-target.outputs.rc_number }}
+      # Normalized subsets from resolve-target.
+      containers: ${{ needs.resolve-target.outputs.containers }}
+      wheels: ${{ needs.resolve-target.outputs.wheels }}
+      crates: ${{ needs.resolve-target.outputs.crates }}
+      helm: ${{ needs.resolve-target.outputs.helm }}
       skip_gitlab_pipeline: ${{ inputs.skip_gitlab_pipeline || false }}
     secrets: inherit
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 2cc4b658d773..f09a7500307a 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -156,8 +156,75 @@ jobs:
   # GATE: Version Extraction
   # ============================================================================
 
+  # OPS GATE — operator-driven runs (on-demand staging and direct rc dispatch)
+  # require an active ai-dynamo/devops member. The nightly path (scheduled /
+  # nightly=true) is exempt so unattended runs are not blocked. Lives here (not
+  # only in the on-demand wrapper) so it binds EVERY caller of this reusable
+  # workflow — including a rogue workflow that calls it with on_demand=true.
+  # In automated-release to reach the RELEASE_APP secrets; carries no approval.
+  authorize:
+    name: Ops gate
+    runs-on: prod-default-small-v2
+    environment: automated-release
+    permissions:
+      contents: read
+    steps:
+      - name: Mint app token
+        if: ${{ inputs.on_demand || (github.event_name == 'workflow_dispatch' && !inputs.nightly) }}
+        id: app-token
+        uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1  # v3.2.0
+        with:
+          app-id: ${{ secrets.RELEASE_APP_ID }}
+          private-key: ${{ secrets.RELEASE_APP_PRIVATE_KEY }}
+      - name: Require release-ops team membership
+        if: ${{ inputs.on_demand || (github.event_name == 'workflow_dispatch' && !inputs.nightly) }}
+        env:
+          GH_TOKEN: ${{ steps.app-token.outputs.token }}
+          ORG: ${{ github.repository_owner }}
+          TEAM: devops
+          ACTOR: ${{ github.actor }}
+          TRIGGERING_ACTOR: ${{ github.triggering_actor }}
+        run: |
+          set -euo pipefail
+          for USER in $(printf '%s\n%s\n' "${ACTOR}" "${TRIGGERING_ACTOR}" | sort -u); do
+            HTTP=$(curl -sS -o "${RUNNER_TEMP}/membership.json" -w '%{http_code}' \
+              -H "Authorization: Bearer ${GH_TOKEN}" \
+              -H "Accept: application/vnd.github+json" \
+              "https://api.github.com/orgs/${ORG}/teams/${TEAM}/memberships/${USER}")
+            STATE=$(python3 -c "import json,sys;print(json.load(open(sys.argv[1])).get('state',''))" \
+              "${RUNNER_TEMP}/membership.json" 2>/dev/null || echo "")
+            if [ "${HTTP}" != "200" ] || [ "${STATE}" != "active" ]; then
+              echo "::error::@${USER} is not an active member of ${ORG}/${TEAM} (HTTP ${HTTP}); only release ops may stage a release."
+              exit 1
+            fi
+            echo "@${USER} authorized via ${ORG}/${TEAM}"
+          done
+      - name: Gate mode
+        run: echo "ops gate applied=${{ inputs.on_demand || (github.event_name == 'workflow_dispatch' && !inputs.nightly) }} (nightly is exempt)"
+
+  # APPROVAL GATE — operator runs also require a manual-release-approval reviewer.
+  # Dynamic environment: empty (no gate) on the nightly path so unattended runs are
+  # not blocked. Enforced by the environment's required-reviewers rule; no secrets
+  # here, so its environment scope is independent of the RELEASE_APP secrets.
+  approval:
+    name: Approval gate
+    needs: [authorize]
+    runs-on: prod-default-small-v2
+    environment: ${{ (inputs.on_demand || (github.event_name == 'workflow_dispatch' && !inputs.nightly)) && 'manual-release-approval' || '' }}
+    permissions:
+      contents: read
+    steps:
+      - name: Record approval
+        env:
+          SHA: ${{ inputs.commit_sha }}
+          VER: ${{ inputs.release_version }}
+          RC: ${{ inputs.rc_number }}
+          ACTOR: ${{ github.actor }}
+        run: echo "Release gate cleared for ${VER:-} rc.${RC} @ ${SHA} by ${ACTOR}"
+
   prepare-release:
     name: Prepare Release
+    needs: [approval]
     runs-on: prod-default-small-v2
     outputs:
       version: ${{ steps.extract.outputs.version }}

From b883f52e72ef04446b4333be3e5de6f0997bae80 Mon Sep 17 00:00:00 2001
From: pvijayakrish 
Date: Thu, 3 Sep 2026 17:09:33 -0700
Subject: [PATCH 10/29] ci(release): push-loop test for 1.5.0 rc2

Signed-off-by: pvijayakrish 
---
 .github/workflows/on-demand-release.yml | 23 ++++++++++++++++++-----
 .github/workflows/release.yml           |  8 +++++---
 2 files changed, 23 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/on-demand-release.yml b/.github/workflows/on-demand-release.yml
index 2e0215168fe6..d02460fa62fd 100644
--- a/.github/workflows/on-demand-release.yml
+++ b/.github/workflows/on-demand-release.yml
@@ -9,6 +9,11 @@
 name: Stage Release RC
 
 on:
+  # TEMPORARY(1.5.0-rc2): push stages release/1.5.0 rc2 (all artifacts) with no
+  # dispatch inputs. REVERT before merge.
+  push:
+    branches:
+      - pull-request/13961
   workflow_dispatch:
     inputs:
       release_branch:
@@ -81,6 +86,11 @@ jobs:
           TRIGGERING_ACTOR: ${{ github.triggering_actor }}
         run: |
           set -euo pipefail
+          # TEMPORARY(1.5.0-rc2): the push test loop has no dispatching actor to check. REVERT.
+          if [ "${GITHUB_EVENT_NAME}" = "push" ]; then
+            echo "TEMPORARY: push loop — skipping the ops gate"
+            exit 0
+          fi
           for USER in $(printf '%s\n%s\n' "${ACTOR}" "${TRIGGERING_ACTOR}" | sort -u); do
             HTTP=$(curl -sS -o "${RUNNER_TEMP}/membership.json" -w '%{http_code}' \
               -H "Authorization: Bearer ${GH_TOKEN}" \
@@ -115,9 +125,10 @@ jobs:
       - name: Validate and resolve target SHA
         id: target
         env:
-          RELEASE_VERSION: ${{ inputs.release_version }}
-          RELEASE_BRANCH: ${{ inputs.release_branch }}
-          RC_ROUND: ${{ inputs.rc_round }}
+          # TEMPORARY(1.5.0-rc2): fallbacks cover inputless push events. REVERT to bare inputs.*.
+          RELEASE_VERSION: ${{ inputs.release_version || '1.5.0' }}
+          RELEASE_BRANCH: ${{ inputs.release_branch || 'release/1.5.0' }}
+          RC_ROUND: ${{ inputs.rc_round || '2' }}
           REPO: ${{ github.repository }}
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
@@ -170,7 +181,8 @@ jobs:
       - name: Resolve artifact selection
         id: resolve
         env:
-          RELEASE_VERSION: ${{ inputs.release_version }}
+          # TEMPORARY(1.5.0-rc2): revert to bare inputs.release_version before merge.
+          RELEASE_VERSION: ${{ inputs.release_version || '1.5.0' }}
           CONTAINERS: ${{ inputs.containers }}
           WHEELS: ${{ inputs.wheels }}
           CRATES: ${{ inputs.crates }}
@@ -396,7 +408,8 @@ jobs:
     with:
       commit_sha: ${{ needs.resolve-target.outputs.bump_sha }}
       on_demand: true
-      release_version: ${{ inputs.release_version }}
+      # TEMPORARY(1.5.0-rc2): revert to bare inputs.release_version before merge.
+      release_version: ${{ inputs.release_version || '1.5.0' }}
       # Operator-chosen internal-staging rc (immutable registries can't be overwritten).
       rc_number: ${{ needs.resolve-target.outputs.rc_number }}
       # Normalized subsets from resolve-target.
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index f09a7500307a..bd8d675e1d15 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -156,6 +156,8 @@ jobs:
   # GATE: Version Extraction
   # ============================================================================
 
+  # TEMPORARY(1.5.0-rc2): the ops-membership steps below carry `&& github.event_name
+  # != 'push'` for the push test loop. REVERT before merge.
   # OPS GATE — operator-driven runs (on-demand staging and direct rc dispatch)
   # require an active ai-dynamo/devops member. The nightly path (scheduled /
   # nightly=true) is exempt so unattended runs are not blocked. Lives here (not
@@ -170,14 +172,14 @@ jobs:
       contents: read
     steps:
       - name: Mint app token
-        if: ${{ inputs.on_demand || (github.event_name == 'workflow_dispatch' && !inputs.nightly) }}
+        if: ${{ (inputs.on_demand || (github.event_name == 'workflow_dispatch' && !inputs.nightly)) && github.event_name != 'push' }}
         id: app-token
         uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1  # v3.2.0
         with:
           app-id: ${{ secrets.RELEASE_APP_ID }}
           private-key: ${{ secrets.RELEASE_APP_PRIVATE_KEY }}
       - name: Require release-ops team membership
-        if: ${{ inputs.on_demand || (github.event_name == 'workflow_dispatch' && !inputs.nightly) }}
+        if: ${{ (inputs.on_demand || (github.event_name == 'workflow_dispatch' && !inputs.nightly)) && github.event_name != 'push' }}
         env:
           GH_TOKEN: ${{ steps.app-token.outputs.token }}
           ORG: ${{ github.repository_owner }}
@@ -200,7 +202,7 @@ jobs:
             echo "@${USER} authorized via ${ORG}/${TEAM}"
           done
       - name: Gate mode
-        run: echo "ops gate applied=${{ inputs.on_demand || (github.event_name == 'workflow_dispatch' && !inputs.nightly) }} (nightly is exempt)"
+        run: echo "ops gate applied=${{ (inputs.on_demand || (github.event_name == 'workflow_dispatch' && !inputs.nightly)) && github.event_name != 'push' }} (nightly exempt)"
 
   # APPROVAL GATE — operator runs also require a manual-release-approval reviewer.
   # Dynamic environment: empty (no gate) on the nightly path so unattended runs are

From 50869080bd6e01918b6cf93a02a4985bbbd4da8f Mon Sep 17 00:00:00 2001
From: pvijayakrish 
Date: Thu, 3 Sep 2026 17:37:05 -0700
Subject: [PATCH 11/29] refactor(release): drop the redundant ops gate from
 Stage Release RC

release.yml's authorize job is the authoritative devops gate for every
caller, so Stage Release RC's own authorize was a duplicate check that
added no security (only a fast-fail). resolve-target is now the first
job. The ops gate and the reviewer approval remain in release.yml.

Signed-off-by: pvijayakrish 
---
 .github/workflows/on-demand-release.yml | 52 ++-----------------------
 1 file changed, 4 insertions(+), 48 deletions(-)

diff --git a/.github/workflows/on-demand-release.yml b/.github/workflows/on-demand-release.yml
index d02460fa62fd..805321d140c6 100644
--- a/.github/workflows/on-demand-release.yml
+++ b/.github/workflows/on-demand-release.yml
@@ -4,7 +4,8 @@
 # Stage Release RC: stage (or re-stage) an EXISTING release branch at a chosen rc
 # round via release.yml, then trigger the GitLab pipeline. Run once per rc round.
 # The branch + version stamp are created by "Cut Release Branch"; this workflow is
-# read-only on the tree. Dispatch restricted to the ai-dynamo/devops team.
+# read-only on the tree. The devops ops gate and the reviewer approval that guard
+# the publish live in release.yml, so they bind this and every other caller.
 
 name: Stage Release RC
 
@@ -59,57 +60,12 @@ permissions:
   contents: read
 
 jobs:
-  # OPS GATE (fast-fail) — reject a non-ai-dynamo/devops dispatch before the ECR
-  # wait. Fail-closed: a missing app Members:read permission or any API error denies.
-  # Checks BOTH github.actor and github.triggering_actor (actor does not re-resolve
-  # on re-runs). The authoritative ops + approval gate for the publish path lives in
-  # release.yml (invoke-release), so it also binds direct and rogue callers.
-  authorize:
-    name: Verify release-ops membership
-    runs-on: prod-default-small-v2
-    environment: automated-release
-    permissions:
-      contents: read
-    steps:
-      - name: Mint app token
-        id: app-token
-        uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1  # v3.2.0
-        with:
-          app-id: ${{ secrets.RELEASE_APP_ID }}
-          private-key: ${{ secrets.RELEASE_APP_PRIVATE_KEY }}
-      - name: Require release-ops team membership
-        env:
-          GH_TOKEN: ${{ steps.app-token.outputs.token }}
-          ORG: ${{ github.repository_owner }}
-          TEAM: devops
-          ACTOR: ${{ github.actor }}
-          TRIGGERING_ACTOR: ${{ github.triggering_actor }}
-        run: |
-          set -euo pipefail
-          # TEMPORARY(1.5.0-rc2): the push test loop has no dispatching actor to check. REVERT.
-          if [ "${GITHUB_EVENT_NAME}" = "push" ]; then
-            echo "TEMPORARY: push loop — skipping the ops gate"
-            exit 0
-          fi
-          for USER in $(printf '%s\n%s\n' "${ACTOR}" "${TRIGGERING_ACTOR}" | sort -u); do
-            HTTP=$(curl -sS -o "${RUNNER_TEMP}/membership.json" -w '%{http_code}' \
-              -H "Authorization: Bearer ${GH_TOKEN}" \
-              -H "Accept: application/vnd.github+json" \
-              "https://api.github.com/orgs/${ORG}/teams/${TEAM}/memberships/${USER}")
-            STATE=$(python3 -c "import json,sys;print(json.load(open(sys.argv[1])).get('state',''))" \
-              "${RUNNER_TEMP}/membership.json" 2>/dev/null || echo "")
-            if [ "${HTTP}" != "200" ] || [ "${STATE}" != "active" ]; then
-              echo "::error::@${USER} is not an active member of ${ORG}/${TEAM} (HTTP ${HTTP}); only release ops may run this workflow."
-              exit 1
-            fi
-            echo "@${USER} authorized via ${ORG}/${TEAM}"
-          done
-
   # RESOLVE TARGET (read-only) — validate inputs; resolve the branch tip SHA (the
   # commit whose post-merge images this rc stages), the rc round, and the subsets.
+  # No ops gate here: the authoritative devops + approval gate for the publish path
+  # lives in release.yml (reached via invoke-release), which binds every caller.
   resolve-target:
     name: Resolve staging target
-    needs: [authorize]
     runs-on: prod-default-small-v2
     environment: automated-release
     permissions:

From 312d1d3f6bea08e32c4526971dc246297b35c812 Mon Sep 17 00:00:00 2001
From: Nate Mailhot 
Date: Fri, 4 Sep 2026 18:09:02 -0700
Subject: [PATCH 12/29] ci(release): push-loop test for 1.5.0 rc3
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

rc2 is already staged and the internal registries are immutable, so
the push loop now resolves rc3. Bumps the RC_ROUND fallback and the
TEMPORARY(1.5.0-rcN) markers in on-demand-release.yml and release.yml
accordingly. Still scaffolding — revert before merge.

Signed-off-by: Nate Mailhot 
---
 .github/workflows/on-demand-release.yml | 10 +++++-----
 .github/workflows/release.yml           |  2 +-
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/on-demand-release.yml b/.github/workflows/on-demand-release.yml
index 805321d140c6..c6efa1650509 100644
--- a/.github/workflows/on-demand-release.yml
+++ b/.github/workflows/on-demand-release.yml
@@ -10,7 +10,7 @@
 name: Stage Release RC
 
 on:
-  # TEMPORARY(1.5.0-rc2): push stages release/1.5.0 rc2 (all artifacts) with no
+  # TEMPORARY(1.5.0-rc3): push stages release/1.5.0 rc3 (all artifacts) with no
   # dispatch inputs. REVERT before merge.
   push:
     branches:
@@ -81,10 +81,10 @@ jobs:
       - name: Validate and resolve target SHA
         id: target
         env:
-          # TEMPORARY(1.5.0-rc2): fallbacks cover inputless push events. REVERT to bare inputs.*.
+          # TEMPORARY(1.5.0-rc3): fallbacks cover inputless push events. REVERT to bare inputs.*.
           RELEASE_VERSION: ${{ inputs.release_version || '1.5.0' }}
           RELEASE_BRANCH: ${{ inputs.release_branch || 'release/1.5.0' }}
-          RC_ROUND: ${{ inputs.rc_round || '2' }}
+          RC_ROUND: ${{ inputs.rc_round || '3' }}
           REPO: ${{ github.repository }}
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
@@ -137,7 +137,7 @@ jobs:
       - name: Resolve artifact selection
         id: resolve
         env:
-          # TEMPORARY(1.5.0-rc2): revert to bare inputs.release_version before merge.
+          # TEMPORARY(1.5.0-rc3): revert to bare inputs.release_version before merge.
           RELEASE_VERSION: ${{ inputs.release_version || '1.5.0' }}
           CONTAINERS: ${{ inputs.containers }}
           WHEELS: ${{ inputs.wheels }}
@@ -364,7 +364,7 @@ jobs:
     with:
       commit_sha: ${{ needs.resolve-target.outputs.bump_sha }}
       on_demand: true
-      # TEMPORARY(1.5.0-rc2): revert to bare inputs.release_version before merge.
+      # TEMPORARY(1.5.0-rc3): revert to bare inputs.release_version before merge.
       release_version: ${{ inputs.release_version || '1.5.0' }}
       # Operator-chosen internal-staging rc (immutable registries can't be overwritten).
       rc_number: ${{ needs.resolve-target.outputs.rc_number }}
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index bd8d675e1d15..ca55c940d2a3 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -156,7 +156,7 @@ jobs:
   # GATE: Version Extraction
   # ============================================================================
 
-  # TEMPORARY(1.5.0-rc2): the ops-membership steps below carry `&& github.event_name
+  # TEMPORARY(1.5.0-rc3): the ops-membership steps below carry `&& github.event_name
   # != 'push'` for the push test loop. REVERT before merge.
   # OPS GATE — operator-driven runs (on-demand staging and direct rc dispatch)
   # require an active ai-dynamo/devops member. The nightly path (scheduled /

From 739c546dc8bb7234679ec9c65aa9f2c28d4da336 Mon Sep 17 00:00:00 2001
From: Nate Mailhot 
Date: Tue, 8 Sep 2026 15:55:36 -0700
Subject: [PATCH 13/29] ci(release): push-loop test for 1.5.0 rc4
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

rc3 is already staged and the internal registries are immutable, so
the push loop now resolves rc4. Bumps the RC_ROUND fallback and the
TEMPORARY(1.5.0-rcN) markers in on-demand-release.yml and release.yml
accordingly. Still scaffolding — revert before merge.

Signed-off-by: Nate Mailhot 
Co-Authored-By: Claude Sonnet 5 
---
 .github/workflows/on-demand-release.yml | 10 +++++-----
 .github/workflows/release.yml           |  2 +-
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/on-demand-release.yml b/.github/workflows/on-demand-release.yml
index c6efa1650509..b254b190acb4 100644
--- a/.github/workflows/on-demand-release.yml
+++ b/.github/workflows/on-demand-release.yml
@@ -10,7 +10,7 @@
 name: Stage Release RC
 
 on:
-  # TEMPORARY(1.5.0-rc3): push stages release/1.5.0 rc3 (all artifacts) with no
+  # TEMPORARY(1.5.0-rc4): push stages release/1.5.0 rc4 (all artifacts) with no
   # dispatch inputs. REVERT before merge.
   push:
     branches:
@@ -81,10 +81,10 @@ jobs:
       - name: Validate and resolve target SHA
         id: target
         env:
-          # TEMPORARY(1.5.0-rc3): fallbacks cover inputless push events. REVERT to bare inputs.*.
+          # TEMPORARY(1.5.0-rc4): fallbacks cover inputless push events. REVERT to bare inputs.*.
           RELEASE_VERSION: ${{ inputs.release_version || '1.5.0' }}
           RELEASE_BRANCH: ${{ inputs.release_branch || 'release/1.5.0' }}
-          RC_ROUND: ${{ inputs.rc_round || '3' }}
+          RC_ROUND: ${{ inputs.rc_round || '4' }}
           REPO: ${{ github.repository }}
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
@@ -137,7 +137,7 @@ jobs:
       - name: Resolve artifact selection
         id: resolve
         env:
-          # TEMPORARY(1.5.0-rc3): revert to bare inputs.release_version before merge.
+          # TEMPORARY(1.5.0-rc4): revert to bare inputs.release_version before merge.
           RELEASE_VERSION: ${{ inputs.release_version || '1.5.0' }}
           CONTAINERS: ${{ inputs.containers }}
           WHEELS: ${{ inputs.wheels }}
@@ -364,7 +364,7 @@ jobs:
     with:
       commit_sha: ${{ needs.resolve-target.outputs.bump_sha }}
       on_demand: true
-      # TEMPORARY(1.5.0-rc3): revert to bare inputs.release_version before merge.
+      # TEMPORARY(1.5.0-rc4): revert to bare inputs.release_version before merge.
       release_version: ${{ inputs.release_version || '1.5.0' }}
       # Operator-chosen internal-staging rc (immutable registries can't be overwritten).
       rc_number: ${{ needs.resolve-target.outputs.rc_number }}
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index ca55c940d2a3..0538e27a0285 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -156,7 +156,7 @@ jobs:
   # GATE: Version Extraction
   # ============================================================================
 
-  # TEMPORARY(1.5.0-rc3): the ops-membership steps below carry `&& github.event_name
+  # TEMPORARY(1.5.0-rc4): the ops-membership steps below carry `&& github.event_name
   # != 'push'` for the push test loop. REVERT before merge.
   # OPS GATE — operator-driven runs (on-demand staging and direct rc dispatch)
   # require an active ai-dynamo/devops member. The nightly path (scheduled /

From 4452936a554ccd96f893da0a6d4831a18ceef221 Mon Sep 17 00:00:00 2001
From: pvijayakrish 
Date: Wed, 9 Sep 2026 19:10:43 -0700
Subject: [PATCH 14/29] ci(release): bump the push-loop test to 1.5.0 rc5

Signed-off-by: pvijayakrish 
---
 .github/workflows/on-demand-release.yml | 10 +++++-----
 .github/workflows/release.yml           |  2 +-
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/on-demand-release.yml b/.github/workflows/on-demand-release.yml
index b254b190acb4..8da681d2d47f 100644
--- a/.github/workflows/on-demand-release.yml
+++ b/.github/workflows/on-demand-release.yml
@@ -10,7 +10,7 @@
 name: Stage Release RC
 
 on:
-  # TEMPORARY(1.5.0-rc4): push stages release/1.5.0 rc4 (all artifacts) with no
+  # TEMPORARY(1.5.0-rc5): push stages release/1.5.0 rc5 (all artifacts) with no
   # dispatch inputs. REVERT before merge.
   push:
     branches:
@@ -81,10 +81,10 @@ jobs:
       - name: Validate and resolve target SHA
         id: target
         env:
-          # TEMPORARY(1.5.0-rc4): fallbacks cover inputless push events. REVERT to bare inputs.*.
+          # TEMPORARY(1.5.0-rc5): fallbacks cover inputless push events. REVERT to bare inputs.*.
           RELEASE_VERSION: ${{ inputs.release_version || '1.5.0' }}
           RELEASE_BRANCH: ${{ inputs.release_branch || 'release/1.5.0' }}
-          RC_ROUND: ${{ inputs.rc_round || '4' }}
+          RC_ROUND: ${{ inputs.rc_round || '5' }}
           REPO: ${{ github.repository }}
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
@@ -137,7 +137,7 @@ jobs:
       - name: Resolve artifact selection
         id: resolve
         env:
-          # TEMPORARY(1.5.0-rc4): revert to bare inputs.release_version before merge.
+          # TEMPORARY(1.5.0-rc5): revert to bare inputs.release_version before merge.
           RELEASE_VERSION: ${{ inputs.release_version || '1.5.0' }}
           CONTAINERS: ${{ inputs.containers }}
           WHEELS: ${{ inputs.wheels }}
@@ -364,7 +364,7 @@ jobs:
     with:
       commit_sha: ${{ needs.resolve-target.outputs.bump_sha }}
       on_demand: true
-      # TEMPORARY(1.5.0-rc4): revert to bare inputs.release_version before merge.
+      # TEMPORARY(1.5.0-rc5): revert to bare inputs.release_version before merge.
       release_version: ${{ inputs.release_version || '1.5.0' }}
       # Operator-chosen internal-staging rc (immutable registries can't be overwritten).
       rc_number: ${{ needs.resolve-target.outputs.rc_number }}
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 0538e27a0285..11616b3dbb9c 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -156,7 +156,7 @@ jobs:
   # GATE: Version Extraction
   # ============================================================================
 
-  # TEMPORARY(1.5.0-rc4): the ops-membership steps below carry `&& github.event_name
+  # TEMPORARY(1.5.0-rc5): the ops-membership steps below carry `&& github.event_name
   # != 'push'` for the push test loop. REVERT before merge.
   # OPS GATE — operator-driven runs (on-demand staging and direct rc dispatch)
   # require an active ai-dynamo/devops member. The nightly path (scheduled /

From b5b1de22a6a5680f5bd98c262991ac66324fa0d3 Mon Sep 17 00:00:00 2001
From: pvijayakrish 
Date: Thu, 10 Sep 2026 18:01:04 -0700
Subject: [PATCH 15/29] ci(release): bump the push-loop test to 1.5.0 rc6

Signed-off-by: pvijayakrish 
---
 .github/workflows/on-demand-release.yml | 10 +++++-----
 .github/workflows/release.yml           |  2 +-
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/on-demand-release.yml b/.github/workflows/on-demand-release.yml
index 8da681d2d47f..a19cc130eb9e 100644
--- a/.github/workflows/on-demand-release.yml
+++ b/.github/workflows/on-demand-release.yml
@@ -10,7 +10,7 @@
 name: Stage Release RC
 
 on:
-  # TEMPORARY(1.5.0-rc5): push stages release/1.5.0 rc5 (all artifacts) with no
+  # TEMPORARY(1.5.0-rc6): push stages release/1.5.0 rc6 (all artifacts) with no
   # dispatch inputs. REVERT before merge.
   push:
     branches:
@@ -81,10 +81,10 @@ jobs:
       - name: Validate and resolve target SHA
         id: target
         env:
-          # TEMPORARY(1.5.0-rc5): fallbacks cover inputless push events. REVERT to bare inputs.*.
+          # TEMPORARY(1.5.0-rc6): fallbacks cover inputless push events. REVERT to bare inputs.*.
           RELEASE_VERSION: ${{ inputs.release_version || '1.5.0' }}
           RELEASE_BRANCH: ${{ inputs.release_branch || 'release/1.5.0' }}
-          RC_ROUND: ${{ inputs.rc_round || '5' }}
+          RC_ROUND: ${{ inputs.rc_round || '6' }}
           REPO: ${{ github.repository }}
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
@@ -137,7 +137,7 @@ jobs:
       - name: Resolve artifact selection
         id: resolve
         env:
-          # TEMPORARY(1.5.0-rc5): revert to bare inputs.release_version before merge.
+          # TEMPORARY(1.5.0-rc6): revert to bare inputs.release_version before merge.
           RELEASE_VERSION: ${{ inputs.release_version || '1.5.0' }}
           CONTAINERS: ${{ inputs.containers }}
           WHEELS: ${{ inputs.wheels }}
@@ -364,7 +364,7 @@ jobs:
     with:
       commit_sha: ${{ needs.resolve-target.outputs.bump_sha }}
       on_demand: true
-      # TEMPORARY(1.5.0-rc5): revert to bare inputs.release_version before merge.
+      # TEMPORARY(1.5.0-rc6): revert to bare inputs.release_version before merge.
       release_version: ${{ inputs.release_version || '1.5.0' }}
       # Operator-chosen internal-staging rc (immutable registries can't be overwritten).
       rc_number: ${{ needs.resolve-target.outputs.rc_number }}
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 11616b3dbb9c..0d201c794c1f 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -156,7 +156,7 @@ jobs:
   # GATE: Version Extraction
   # ============================================================================
 
-  # TEMPORARY(1.5.0-rc5): the ops-membership steps below carry `&& github.event_name
+  # TEMPORARY(1.5.0-rc6): the ops-membership steps below carry `&& github.event_name
   # != 'push'` for the push test loop. REVERT before merge.
   # OPS GATE — operator-driven runs (on-demand staging and direct rc dispatch)
   # require an active ai-dynamo/devops member. The nightly path (scheduled /

From cfbd315ac97b0706294788d56aba4379df1931da Mon Sep 17 00:00:00 2001
From: pvijayakrish 
Date: Fri, 11 Sep 2026 16:59:18 -0700
Subject: [PATCH 16/29] ci(release): bump the push-loop test to 1.5.0 rc7

Signed-off-by: pvijayakrish 
---
 .github/workflows/on-demand-release.yml | 10 +++++-----
 .github/workflows/release.yml           |  2 +-
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/on-demand-release.yml b/.github/workflows/on-demand-release.yml
index a19cc130eb9e..8b35b0a6e7b8 100644
--- a/.github/workflows/on-demand-release.yml
+++ b/.github/workflows/on-demand-release.yml
@@ -10,7 +10,7 @@
 name: Stage Release RC
 
 on:
-  # TEMPORARY(1.5.0-rc6): push stages release/1.5.0 rc6 (all artifacts) with no
+  # TEMPORARY(1.5.0-rc7): push stages release/1.5.0 rc7 (all artifacts) with no
   # dispatch inputs. REVERT before merge.
   push:
     branches:
@@ -81,10 +81,10 @@ jobs:
       - name: Validate and resolve target SHA
         id: target
         env:
-          # TEMPORARY(1.5.0-rc6): fallbacks cover inputless push events. REVERT to bare inputs.*.
+          # TEMPORARY(1.5.0-rc7): fallbacks cover inputless push events. REVERT to bare inputs.*.
           RELEASE_VERSION: ${{ inputs.release_version || '1.5.0' }}
           RELEASE_BRANCH: ${{ inputs.release_branch || 'release/1.5.0' }}
-          RC_ROUND: ${{ inputs.rc_round || '6' }}
+          RC_ROUND: ${{ inputs.rc_round || '7' }}
           REPO: ${{ github.repository }}
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
@@ -137,7 +137,7 @@ jobs:
       - name: Resolve artifact selection
         id: resolve
         env:
-          # TEMPORARY(1.5.0-rc6): revert to bare inputs.release_version before merge.
+          # TEMPORARY(1.5.0-rc7): revert to bare inputs.release_version before merge.
           RELEASE_VERSION: ${{ inputs.release_version || '1.5.0' }}
           CONTAINERS: ${{ inputs.containers }}
           WHEELS: ${{ inputs.wheels }}
@@ -364,7 +364,7 @@ jobs:
     with:
       commit_sha: ${{ needs.resolve-target.outputs.bump_sha }}
       on_demand: true
-      # TEMPORARY(1.5.0-rc6): revert to bare inputs.release_version before merge.
+      # TEMPORARY(1.5.0-rc7): revert to bare inputs.release_version before merge.
       release_version: ${{ inputs.release_version || '1.5.0' }}
       # Operator-chosen internal-staging rc (immutable registries can't be overwritten).
       rc_number: ${{ needs.resolve-target.outputs.rc_number }}
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 0d201c794c1f..ffc49a594bd2 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -156,7 +156,7 @@ jobs:
   # GATE: Version Extraction
   # ============================================================================
 
-  # TEMPORARY(1.5.0-rc6): the ops-membership steps below carry `&& github.event_name
+  # TEMPORARY(1.5.0-rc7): the ops-membership steps below carry `&& github.event_name
   # != 'push'` for the push test loop. REVERT before merge.
   # OPS GATE — operator-driven runs (on-demand staging and direct rc dispatch)
   # require an active ai-dynamo/devops member. The nightly path (scheduled /

From 5462db2acc156bb410abe1582e74a426c9bed469 Mon Sep 17 00:00:00 2001
From: Nate Mailhot 
Date: Mon, 14 Sep 2026 17:51:58 -0700
Subject: [PATCH 17/29] ci(release): bump the push-loop test to 1.5.0 rc8

Co-Authored-By: Claude Sonnet 5 
Signed-off-by: Nate Mailhot 
---
 .github/workflows/on-demand-release.yml | 10 +++++-----
 .github/workflows/release.yml           |  2 +-
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/on-demand-release.yml b/.github/workflows/on-demand-release.yml
index 8b35b0a6e7b8..69cbc55db276 100644
--- a/.github/workflows/on-demand-release.yml
+++ b/.github/workflows/on-demand-release.yml
@@ -10,7 +10,7 @@
 name: Stage Release RC
 
 on:
-  # TEMPORARY(1.5.0-rc7): push stages release/1.5.0 rc7 (all artifacts) with no
+  # TEMPORARY(1.5.0-rc8): push stages release/1.5.0 rc8 (all artifacts) with no
   # dispatch inputs. REVERT before merge.
   push:
     branches:
@@ -81,10 +81,10 @@ jobs:
       - name: Validate and resolve target SHA
         id: target
         env:
-          # TEMPORARY(1.5.0-rc7): fallbacks cover inputless push events. REVERT to bare inputs.*.
+          # TEMPORARY(1.5.0-rc8): fallbacks cover inputless push events. REVERT to bare inputs.*.
           RELEASE_VERSION: ${{ inputs.release_version || '1.5.0' }}
           RELEASE_BRANCH: ${{ inputs.release_branch || 'release/1.5.0' }}
-          RC_ROUND: ${{ inputs.rc_round || '7' }}
+          RC_ROUND: ${{ inputs.rc_round || '8' }}
           REPO: ${{ github.repository }}
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
@@ -137,7 +137,7 @@ jobs:
       - name: Resolve artifact selection
         id: resolve
         env:
-          # TEMPORARY(1.5.0-rc7): revert to bare inputs.release_version before merge.
+          # TEMPORARY(1.5.0-rc8): revert to bare inputs.release_version before merge.
           RELEASE_VERSION: ${{ inputs.release_version || '1.5.0' }}
           CONTAINERS: ${{ inputs.containers }}
           WHEELS: ${{ inputs.wheels }}
@@ -364,7 +364,7 @@ jobs:
     with:
       commit_sha: ${{ needs.resolve-target.outputs.bump_sha }}
       on_demand: true
-      # TEMPORARY(1.5.0-rc7): revert to bare inputs.release_version before merge.
+      # TEMPORARY(1.5.0-rc8): revert to bare inputs.release_version before merge.
       release_version: ${{ inputs.release_version || '1.5.0' }}
       # Operator-chosen internal-staging rc (immutable registries can't be overwritten).
       rc_number: ${{ needs.resolve-target.outputs.rc_number }}
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index ffc49a594bd2..bfa0374b77d7 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -156,7 +156,7 @@ jobs:
   # GATE: Version Extraction
   # ============================================================================
 
-  # TEMPORARY(1.5.0-rc7): the ops-membership steps below carry `&& github.event_name
+  # TEMPORARY(1.5.0-rc8): the ops-membership steps below carry `&& github.event_name
   # != 'push'` for the push test loop. REVERT before merge.
   # OPS GATE — operator-driven runs (on-demand staging and direct rc dispatch)
   # require an active ai-dynamo/devops member. The nightly path (scheduled /

From 6c5350ebd8e718171a1e9e69f83b28d7965ba3a2 Mon Sep 17 00:00:00 2001
From: Nate Mailhot 
Date: Tue, 15 Sep 2026 16:40:36 -0700
Subject: [PATCH 18/29] ci(release): bump the push-loop test to 1.5.0 rc9

Co-Authored-By: Claude Sonnet 5 
Signed-off-by: Nate Mailhot 
---
 .github/workflows/on-demand-release.yml | 10 +++++-----
 .github/workflows/release.yml           |  2 +-
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/on-demand-release.yml b/.github/workflows/on-demand-release.yml
index 69cbc55db276..519bacfd5bf8 100644
--- a/.github/workflows/on-demand-release.yml
+++ b/.github/workflows/on-demand-release.yml
@@ -10,7 +10,7 @@
 name: Stage Release RC
 
 on:
-  # TEMPORARY(1.5.0-rc8): push stages release/1.5.0 rc8 (all artifacts) with no
+  # TEMPORARY(1.5.0-rc9): push stages release/1.5.0 rc9 (all artifacts) with no
   # dispatch inputs. REVERT before merge.
   push:
     branches:
@@ -81,10 +81,10 @@ jobs:
       - name: Validate and resolve target SHA
         id: target
         env:
-          # TEMPORARY(1.5.0-rc8): fallbacks cover inputless push events. REVERT to bare inputs.*.
+          # TEMPORARY(1.5.0-rc9): fallbacks cover inputless push events. REVERT to bare inputs.*.
           RELEASE_VERSION: ${{ inputs.release_version || '1.5.0' }}
           RELEASE_BRANCH: ${{ inputs.release_branch || 'release/1.5.0' }}
-          RC_ROUND: ${{ inputs.rc_round || '8' }}
+          RC_ROUND: ${{ inputs.rc_round || '9' }}
           REPO: ${{ github.repository }}
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
@@ -137,7 +137,7 @@ jobs:
       - name: Resolve artifact selection
         id: resolve
         env:
-          # TEMPORARY(1.5.0-rc8): revert to bare inputs.release_version before merge.
+          # TEMPORARY(1.5.0-rc9): revert to bare inputs.release_version before merge.
           RELEASE_VERSION: ${{ inputs.release_version || '1.5.0' }}
           CONTAINERS: ${{ inputs.containers }}
           WHEELS: ${{ inputs.wheels }}
@@ -364,7 +364,7 @@ jobs:
     with:
       commit_sha: ${{ needs.resolve-target.outputs.bump_sha }}
       on_demand: true
-      # TEMPORARY(1.5.0-rc8): revert to bare inputs.release_version before merge.
+      # TEMPORARY(1.5.0-rc9): revert to bare inputs.release_version before merge.
       release_version: ${{ inputs.release_version || '1.5.0' }}
       # Operator-chosen internal-staging rc (immutable registries can't be overwritten).
       rc_number: ${{ needs.resolve-target.outputs.rc_number }}
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index bfa0374b77d7..7d7a2177cc21 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -156,7 +156,7 @@ jobs:
   # GATE: Version Extraction
   # ============================================================================
 
-  # TEMPORARY(1.5.0-rc8): the ops-membership steps below carry `&& github.event_name
+  # TEMPORARY(1.5.0-rc9): the ops-membership steps below carry `&& github.event_name
   # != 'push'` for the push test loop. REVERT before merge.
   # OPS GATE — operator-driven runs (on-demand staging and direct rc dispatch)
   # require an active ai-dynamo/devops member. The nightly path (scheduled /

From 05a9d78d282459a463d1ba6061e64bd2d7e852ea Mon Sep 17 00:00:00 2001
From: Nate Mailhot 
Date: Tue, 15 Sep 2026 22:11:01 -0700
Subject: [PATCH 19/29] Revert "ci(release): bump the push-loop test to 1.5.0
 rc9"

This reverts commit 6c5350ebd8e718171a1e9e69f83b28d7965ba3a2.

Signed-off-by: Nate Mailhot 
---
 .github/workflows/on-demand-release.yml | 10 +++++-----
 .github/workflows/release.yml           |  2 +-
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/on-demand-release.yml b/.github/workflows/on-demand-release.yml
index 519bacfd5bf8..69cbc55db276 100644
--- a/.github/workflows/on-demand-release.yml
+++ b/.github/workflows/on-demand-release.yml
@@ -10,7 +10,7 @@
 name: Stage Release RC
 
 on:
-  # TEMPORARY(1.5.0-rc9): push stages release/1.5.0 rc9 (all artifacts) with no
+  # TEMPORARY(1.5.0-rc8): push stages release/1.5.0 rc8 (all artifacts) with no
   # dispatch inputs. REVERT before merge.
   push:
     branches:
@@ -81,10 +81,10 @@ jobs:
       - name: Validate and resolve target SHA
         id: target
         env:
-          # TEMPORARY(1.5.0-rc9): fallbacks cover inputless push events. REVERT to bare inputs.*.
+          # TEMPORARY(1.5.0-rc8): fallbacks cover inputless push events. REVERT to bare inputs.*.
           RELEASE_VERSION: ${{ inputs.release_version || '1.5.0' }}
           RELEASE_BRANCH: ${{ inputs.release_branch || 'release/1.5.0' }}
-          RC_ROUND: ${{ inputs.rc_round || '9' }}
+          RC_ROUND: ${{ inputs.rc_round || '8' }}
           REPO: ${{ github.repository }}
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
@@ -137,7 +137,7 @@ jobs:
       - name: Resolve artifact selection
         id: resolve
         env:
-          # TEMPORARY(1.5.0-rc9): revert to bare inputs.release_version before merge.
+          # TEMPORARY(1.5.0-rc8): revert to bare inputs.release_version before merge.
           RELEASE_VERSION: ${{ inputs.release_version || '1.5.0' }}
           CONTAINERS: ${{ inputs.containers }}
           WHEELS: ${{ inputs.wheels }}
@@ -364,7 +364,7 @@ jobs:
     with:
       commit_sha: ${{ needs.resolve-target.outputs.bump_sha }}
       on_demand: true
-      # TEMPORARY(1.5.0-rc9): revert to bare inputs.release_version before merge.
+      # TEMPORARY(1.5.0-rc8): revert to bare inputs.release_version before merge.
       release_version: ${{ inputs.release_version || '1.5.0' }}
       # Operator-chosen internal-staging rc (immutable registries can't be overwritten).
       rc_number: ${{ needs.resolve-target.outputs.rc_number }}
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 7d7a2177cc21..bfa0374b77d7 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -156,7 +156,7 @@ jobs:
   # GATE: Version Extraction
   # ============================================================================
 
-  # TEMPORARY(1.5.0-rc9): the ops-membership steps below carry `&& github.event_name
+  # TEMPORARY(1.5.0-rc8): the ops-membership steps below carry `&& github.event_name
   # != 'push'` for the push test loop. REVERT before merge.
   # OPS GATE — operator-driven runs (on-demand staging and direct rc dispatch)
   # require an active ai-dynamo/devops member. The nightly path (scheduled /

From 883102c7d7dfa9d5cffc61560a3c30cc910d7388 Mon Sep 17 00:00:00 2001
From: Nate Mailhot 
Date: Tue, 15 Sep 2026 22:11:04 -0700
Subject: [PATCH 20/29] Reapply "ci(release): bump the push-loop test to 1.5.0
 rc9"

This reverts commit 05a9d78d282459a463d1ba6061e64bd2d7e852ea.

Signed-off-by: Nate Mailhot 
---
 .github/workflows/on-demand-release.yml | 10 +++++-----
 .github/workflows/release.yml           |  2 +-
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/on-demand-release.yml b/.github/workflows/on-demand-release.yml
index 69cbc55db276..519bacfd5bf8 100644
--- a/.github/workflows/on-demand-release.yml
+++ b/.github/workflows/on-demand-release.yml
@@ -10,7 +10,7 @@
 name: Stage Release RC
 
 on:
-  # TEMPORARY(1.5.0-rc8): push stages release/1.5.0 rc8 (all artifacts) with no
+  # TEMPORARY(1.5.0-rc9): push stages release/1.5.0 rc9 (all artifacts) with no
   # dispatch inputs. REVERT before merge.
   push:
     branches:
@@ -81,10 +81,10 @@ jobs:
       - name: Validate and resolve target SHA
         id: target
         env:
-          # TEMPORARY(1.5.0-rc8): fallbacks cover inputless push events. REVERT to bare inputs.*.
+          # TEMPORARY(1.5.0-rc9): fallbacks cover inputless push events. REVERT to bare inputs.*.
           RELEASE_VERSION: ${{ inputs.release_version || '1.5.0' }}
           RELEASE_BRANCH: ${{ inputs.release_branch || 'release/1.5.0' }}
-          RC_ROUND: ${{ inputs.rc_round || '8' }}
+          RC_ROUND: ${{ inputs.rc_round || '9' }}
           REPO: ${{ github.repository }}
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
@@ -137,7 +137,7 @@ jobs:
       - name: Resolve artifact selection
         id: resolve
         env:
-          # TEMPORARY(1.5.0-rc8): revert to bare inputs.release_version before merge.
+          # TEMPORARY(1.5.0-rc9): revert to bare inputs.release_version before merge.
           RELEASE_VERSION: ${{ inputs.release_version || '1.5.0' }}
           CONTAINERS: ${{ inputs.containers }}
           WHEELS: ${{ inputs.wheels }}
@@ -364,7 +364,7 @@ jobs:
     with:
       commit_sha: ${{ needs.resolve-target.outputs.bump_sha }}
       on_demand: true
-      # TEMPORARY(1.5.0-rc8): revert to bare inputs.release_version before merge.
+      # TEMPORARY(1.5.0-rc9): revert to bare inputs.release_version before merge.
       release_version: ${{ inputs.release_version || '1.5.0' }}
       # Operator-chosen internal-staging rc (immutable registries can't be overwritten).
       rc_number: ${{ needs.resolve-target.outputs.rc_number }}
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index bfa0374b77d7..7d7a2177cc21 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -156,7 +156,7 @@ jobs:
   # GATE: Version Extraction
   # ============================================================================
 
-  # TEMPORARY(1.5.0-rc8): the ops-membership steps below carry `&& github.event_name
+  # TEMPORARY(1.5.0-rc9): the ops-membership steps below carry `&& github.event_name
   # != 'push'` for the push test loop. REVERT before merge.
   # OPS GATE — operator-driven runs (on-demand staging and direct rc dispatch)
   # require an active ai-dynamo/devops member. The nightly path (scheduled /

From ef5d29b4a5361f7390072b35995ae14b7376bea0 Mon Sep 17 00:00:00 2001
From: Nate Mailhot 
Date: Wed, 16 Sep 2026 16:45:44 -0700
Subject: [PATCH 21/29] ci(release): bump the push-loop test to 1.5.0 rc10

Co-Authored-By: Claude Sonnet 5 
Signed-off-by: Nate Mailhot 
---
 .github/workflows/on-demand-release.yml | 10 +++++-----
 .github/workflows/release.yml           |  2 +-
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/on-demand-release.yml b/.github/workflows/on-demand-release.yml
index 519bacfd5bf8..55a26e9dbe53 100644
--- a/.github/workflows/on-demand-release.yml
+++ b/.github/workflows/on-demand-release.yml
@@ -10,7 +10,7 @@
 name: Stage Release RC
 
 on:
-  # TEMPORARY(1.5.0-rc9): push stages release/1.5.0 rc9 (all artifacts) with no
+  # TEMPORARY(1.5.0-rc10): push stages release/1.5.0 rc10 (all artifacts) with no
   # dispatch inputs. REVERT before merge.
   push:
     branches:
@@ -81,10 +81,10 @@ jobs:
       - name: Validate and resolve target SHA
         id: target
         env:
-          # TEMPORARY(1.5.0-rc9): fallbacks cover inputless push events. REVERT to bare inputs.*.
+          # TEMPORARY(1.5.0-rc10): fallbacks cover inputless push events. REVERT to bare inputs.*.
           RELEASE_VERSION: ${{ inputs.release_version || '1.5.0' }}
           RELEASE_BRANCH: ${{ inputs.release_branch || 'release/1.5.0' }}
-          RC_ROUND: ${{ inputs.rc_round || '9' }}
+          RC_ROUND: ${{ inputs.rc_round || '10' }}
           REPO: ${{ github.repository }}
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
@@ -137,7 +137,7 @@ jobs:
       - name: Resolve artifact selection
         id: resolve
         env:
-          # TEMPORARY(1.5.0-rc9): revert to bare inputs.release_version before merge.
+          # TEMPORARY(1.5.0-rc10): revert to bare inputs.release_version before merge.
           RELEASE_VERSION: ${{ inputs.release_version || '1.5.0' }}
           CONTAINERS: ${{ inputs.containers }}
           WHEELS: ${{ inputs.wheels }}
@@ -364,7 +364,7 @@ jobs:
     with:
       commit_sha: ${{ needs.resolve-target.outputs.bump_sha }}
       on_demand: true
-      # TEMPORARY(1.5.0-rc9): revert to bare inputs.release_version before merge.
+      # TEMPORARY(1.5.0-rc10): revert to bare inputs.release_version before merge.
       release_version: ${{ inputs.release_version || '1.5.0' }}
       # Operator-chosen internal-staging rc (immutable registries can't be overwritten).
       rc_number: ${{ needs.resolve-target.outputs.rc_number }}
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 7d7a2177cc21..6f15f8c08613 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -156,7 +156,7 @@ jobs:
   # GATE: Version Extraction
   # ============================================================================
 
-  # TEMPORARY(1.5.0-rc9): the ops-membership steps below carry `&& github.event_name
+  # TEMPORARY(1.5.0-rc10): the ops-membership steps below carry `&& github.event_name
   # != 'push'` for the push test loop. REVERT before merge.
   # OPS GATE — operator-driven runs (on-demand staging and direct rc dispatch)
   # require an active ai-dynamo/devops member. The nightly path (scheduled /

From da9e6d566d068e3298656f113d9b4172d6fc4467 Mon Sep 17 00:00:00 2001
From: Nate Mailhot 
Date: Thu, 17 Sep 2026 17:03:25 -0700
Subject: [PATCH 22/29] ci(release): bump the push-loop test to 1.5.0 rc11

Co-Authored-By: Claude Sonnet 5 
Signed-off-by: Nate Mailhot 
---
 .github/workflows/on-demand-release.yml | 10 +++++-----
 .github/workflows/release.yml           |  2 +-
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/on-demand-release.yml b/.github/workflows/on-demand-release.yml
index 55a26e9dbe53..d19a5e227f12 100644
--- a/.github/workflows/on-demand-release.yml
+++ b/.github/workflows/on-demand-release.yml
@@ -10,7 +10,7 @@
 name: Stage Release RC
 
 on:
-  # TEMPORARY(1.5.0-rc10): push stages release/1.5.0 rc10 (all artifacts) with no
+  # TEMPORARY(1.5.0-rc11): push stages release/1.5.0 rc11 (all artifacts) with no
   # dispatch inputs. REVERT before merge.
   push:
     branches:
@@ -81,10 +81,10 @@ jobs:
       - name: Validate and resolve target SHA
         id: target
         env:
-          # TEMPORARY(1.5.0-rc10): fallbacks cover inputless push events. REVERT to bare inputs.*.
+          # TEMPORARY(1.5.0-rc11): fallbacks cover inputless push events. REVERT to bare inputs.*.
           RELEASE_VERSION: ${{ inputs.release_version || '1.5.0' }}
           RELEASE_BRANCH: ${{ inputs.release_branch || 'release/1.5.0' }}
-          RC_ROUND: ${{ inputs.rc_round || '10' }}
+          RC_ROUND: ${{ inputs.rc_round || '11' }}
           REPO: ${{ github.repository }}
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
@@ -137,7 +137,7 @@ jobs:
       - name: Resolve artifact selection
         id: resolve
         env:
-          # TEMPORARY(1.5.0-rc10): revert to bare inputs.release_version before merge.
+          # TEMPORARY(1.5.0-rc11): revert to bare inputs.release_version before merge.
           RELEASE_VERSION: ${{ inputs.release_version || '1.5.0' }}
           CONTAINERS: ${{ inputs.containers }}
           WHEELS: ${{ inputs.wheels }}
@@ -364,7 +364,7 @@ jobs:
     with:
       commit_sha: ${{ needs.resolve-target.outputs.bump_sha }}
       on_demand: true
-      # TEMPORARY(1.5.0-rc10): revert to bare inputs.release_version before merge.
+      # TEMPORARY(1.5.0-rc11): revert to bare inputs.release_version before merge.
       release_version: ${{ inputs.release_version || '1.5.0' }}
       # Operator-chosen internal-staging rc (immutable registries can't be overwritten).
       rc_number: ${{ needs.resolve-target.outputs.rc_number }}
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 6f15f8c08613..e5a62c71e981 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -156,7 +156,7 @@ jobs:
   # GATE: Version Extraction
   # ============================================================================
 
-  # TEMPORARY(1.5.0-rc10): the ops-membership steps below carry `&& github.event_name
+  # TEMPORARY(1.5.0-rc11): the ops-membership steps below carry `&& github.event_name
   # != 'push'` for the push test loop. REVERT before merge.
   # OPS GATE — operator-driven runs (on-demand staging and direct rc dispatch)
   # require an active ai-dynamo/devops member. The nightly path (scheduled /

From f4f313fccb6684e1bef202e5ab12eb24aafadd0b Mon Sep 17 00:00:00 2001
From: Nate Mailhot 
Date: Thu, 17 Sep 2026 19:36:10 -0700
Subject: [PATCH 23/29] Revert "ci(release): bump the push-loop test to 1.5.0
 rc11"

This reverts commit da9e6d566d068e3298656f113d9b4172d6fc4467.

Signed-off-by: Nate Mailhot 
---
 .github/workflows/on-demand-release.yml | 10 +++++-----
 .github/workflows/release.yml           |  2 +-
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/on-demand-release.yml b/.github/workflows/on-demand-release.yml
index d19a5e227f12..55a26e9dbe53 100644
--- a/.github/workflows/on-demand-release.yml
+++ b/.github/workflows/on-demand-release.yml
@@ -10,7 +10,7 @@
 name: Stage Release RC
 
 on:
-  # TEMPORARY(1.5.0-rc11): push stages release/1.5.0 rc11 (all artifacts) with no
+  # TEMPORARY(1.5.0-rc10): push stages release/1.5.0 rc10 (all artifacts) with no
   # dispatch inputs. REVERT before merge.
   push:
     branches:
@@ -81,10 +81,10 @@ jobs:
       - name: Validate and resolve target SHA
         id: target
         env:
-          # TEMPORARY(1.5.0-rc11): fallbacks cover inputless push events. REVERT to bare inputs.*.
+          # TEMPORARY(1.5.0-rc10): fallbacks cover inputless push events. REVERT to bare inputs.*.
           RELEASE_VERSION: ${{ inputs.release_version || '1.5.0' }}
           RELEASE_BRANCH: ${{ inputs.release_branch || 'release/1.5.0' }}
-          RC_ROUND: ${{ inputs.rc_round || '11' }}
+          RC_ROUND: ${{ inputs.rc_round || '10' }}
           REPO: ${{ github.repository }}
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
@@ -137,7 +137,7 @@ jobs:
       - name: Resolve artifact selection
         id: resolve
         env:
-          # TEMPORARY(1.5.0-rc11): revert to bare inputs.release_version before merge.
+          # TEMPORARY(1.5.0-rc10): revert to bare inputs.release_version before merge.
           RELEASE_VERSION: ${{ inputs.release_version || '1.5.0' }}
           CONTAINERS: ${{ inputs.containers }}
           WHEELS: ${{ inputs.wheels }}
@@ -364,7 +364,7 @@ jobs:
     with:
       commit_sha: ${{ needs.resolve-target.outputs.bump_sha }}
       on_demand: true
-      # TEMPORARY(1.5.0-rc11): revert to bare inputs.release_version before merge.
+      # TEMPORARY(1.5.0-rc10): revert to bare inputs.release_version before merge.
       release_version: ${{ inputs.release_version || '1.5.0' }}
       # Operator-chosen internal-staging rc (immutable registries can't be overwritten).
       rc_number: ${{ needs.resolve-target.outputs.rc_number }}
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index e5a62c71e981..6f15f8c08613 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -156,7 +156,7 @@ jobs:
   # GATE: Version Extraction
   # ============================================================================
 
-  # TEMPORARY(1.5.0-rc11): the ops-membership steps below carry `&& github.event_name
+  # TEMPORARY(1.5.0-rc10): the ops-membership steps below carry `&& github.event_name
   # != 'push'` for the push test loop. REVERT before merge.
   # OPS GATE — operator-driven runs (on-demand staging and direct rc dispatch)
   # require an active ai-dynamo/devops member. The nightly path (scheduled /

From 2461a61b9efecaa5111ff48c1a7265ae67115ecf Mon Sep 17 00:00:00 2001
From: Nate Mailhot 
Date: Thu, 17 Sep 2026 19:36:13 -0700
Subject: [PATCH 24/29] Reapply "ci(release): bump the push-loop test to 1.5.0
 rc11"

This reverts commit f4f313fccb6684e1bef202e5ab12eb24aafadd0b.

Signed-off-by: Nate Mailhot 
---
 .github/workflows/on-demand-release.yml | 10 +++++-----
 .github/workflows/release.yml           |  2 +-
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/on-demand-release.yml b/.github/workflows/on-demand-release.yml
index 55a26e9dbe53..d19a5e227f12 100644
--- a/.github/workflows/on-demand-release.yml
+++ b/.github/workflows/on-demand-release.yml
@@ -10,7 +10,7 @@
 name: Stage Release RC
 
 on:
-  # TEMPORARY(1.5.0-rc10): push stages release/1.5.0 rc10 (all artifacts) with no
+  # TEMPORARY(1.5.0-rc11): push stages release/1.5.0 rc11 (all artifacts) with no
   # dispatch inputs. REVERT before merge.
   push:
     branches:
@@ -81,10 +81,10 @@ jobs:
       - name: Validate and resolve target SHA
         id: target
         env:
-          # TEMPORARY(1.5.0-rc10): fallbacks cover inputless push events. REVERT to bare inputs.*.
+          # TEMPORARY(1.5.0-rc11): fallbacks cover inputless push events. REVERT to bare inputs.*.
           RELEASE_VERSION: ${{ inputs.release_version || '1.5.0' }}
           RELEASE_BRANCH: ${{ inputs.release_branch || 'release/1.5.0' }}
-          RC_ROUND: ${{ inputs.rc_round || '10' }}
+          RC_ROUND: ${{ inputs.rc_round || '11' }}
           REPO: ${{ github.repository }}
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
@@ -137,7 +137,7 @@ jobs:
       - name: Resolve artifact selection
         id: resolve
         env:
-          # TEMPORARY(1.5.0-rc10): revert to bare inputs.release_version before merge.
+          # TEMPORARY(1.5.0-rc11): revert to bare inputs.release_version before merge.
           RELEASE_VERSION: ${{ inputs.release_version || '1.5.0' }}
           CONTAINERS: ${{ inputs.containers }}
           WHEELS: ${{ inputs.wheels }}
@@ -364,7 +364,7 @@ jobs:
     with:
       commit_sha: ${{ needs.resolve-target.outputs.bump_sha }}
       on_demand: true
-      # TEMPORARY(1.5.0-rc10): revert to bare inputs.release_version before merge.
+      # TEMPORARY(1.5.0-rc11): revert to bare inputs.release_version before merge.
       release_version: ${{ inputs.release_version || '1.5.0' }}
       # Operator-chosen internal-staging rc (immutable registries can't be overwritten).
       rc_number: ${{ needs.resolve-target.outputs.rc_number }}
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 6f15f8c08613..e5a62c71e981 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -156,7 +156,7 @@ jobs:
   # GATE: Version Extraction
   # ============================================================================
 
-  # TEMPORARY(1.5.0-rc10): the ops-membership steps below carry `&& github.event_name
+  # TEMPORARY(1.5.0-rc11): the ops-membership steps below carry `&& github.event_name
   # != 'push'` for the push test loop. REVERT before merge.
   # OPS GATE — operator-driven runs (on-demand staging and direct rc dispatch)
   # require an active ai-dynamo/devops member. The nightly path (scheduled /

From 275783ef97243a41c5eccb63e04047da56c366ff Mon Sep 17 00:00:00 2001
From: Nate Mailhot 
Date: Mon, 28 Sep 2026 16:48:57 -0700
Subject: [PATCH 25/29] ci(release): bump the push-loop test to 1.5.1 rc0

Co-Authored-By: Claude Sonnet 5 
Signed-off-by: Nate Mailhot 
---
 .github/workflows/on-demand-release.yml | 18 +++++++++---------
 .github/workflows/release.yml           |  2 +-
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/on-demand-release.yml b/.github/workflows/on-demand-release.yml
index d19a5e227f12..8c6bab02886b 100644
--- a/.github/workflows/on-demand-release.yml
+++ b/.github/workflows/on-demand-release.yml
@@ -10,7 +10,7 @@
 name: Stage Release RC
 
 on:
-  # TEMPORARY(1.5.0-rc11): push stages release/1.5.0 rc11 (all artifacts) with no
+  # TEMPORARY(1.5.1-rc0): push stages release/1.5.1 rc0 (all artifacts) with no
   # dispatch inputs. REVERT before merge.
   push:
     branches:
@@ -81,10 +81,10 @@ jobs:
       - name: Validate and resolve target SHA
         id: target
         env:
-          # TEMPORARY(1.5.0-rc11): fallbacks cover inputless push events. REVERT to bare inputs.*.
-          RELEASE_VERSION: ${{ inputs.release_version || '1.5.0' }}
-          RELEASE_BRANCH: ${{ inputs.release_branch || 'release/1.5.0' }}
-          RC_ROUND: ${{ inputs.rc_round || '11' }}
+          # TEMPORARY(1.5.1-rc0): fallbacks cover inputless push events. REVERT to bare inputs.*.
+          RELEASE_VERSION: ${{ inputs.release_version || '1.5.1' }}
+          RELEASE_BRANCH: ${{ inputs.release_branch || 'release/1.5.1' }}
+          RC_ROUND: ${{ inputs.rc_round || '0' }}
           REPO: ${{ github.repository }}
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
@@ -137,8 +137,8 @@ jobs:
       - name: Resolve artifact selection
         id: resolve
         env:
-          # TEMPORARY(1.5.0-rc11): revert to bare inputs.release_version before merge.
-          RELEASE_VERSION: ${{ inputs.release_version || '1.5.0' }}
+          # TEMPORARY(1.5.1-rc0): revert to bare inputs.release_version before merge.
+          RELEASE_VERSION: ${{ inputs.release_version || '1.5.1' }}
           CONTAINERS: ${{ inputs.containers }}
           WHEELS: ${{ inputs.wheels }}
           CRATES: ${{ inputs.crates }}
@@ -364,8 +364,8 @@ jobs:
     with:
       commit_sha: ${{ needs.resolve-target.outputs.bump_sha }}
       on_demand: true
-      # TEMPORARY(1.5.0-rc11): revert to bare inputs.release_version before merge.
-      release_version: ${{ inputs.release_version || '1.5.0' }}
+      # TEMPORARY(1.5.1-rc0): revert to bare inputs.release_version before merge.
+      release_version: ${{ inputs.release_version || '1.5.1' }}
       # Operator-chosen internal-staging rc (immutable registries can't be overwritten).
       rc_number: ${{ needs.resolve-target.outputs.rc_number }}
       # Normalized subsets from resolve-target.
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index e5a62c71e981..62a1cdd9e9c8 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -156,7 +156,7 @@ jobs:
   # GATE: Version Extraction
   # ============================================================================
 
-  # TEMPORARY(1.5.0-rc11): the ops-membership steps below carry `&& github.event_name
+  # TEMPORARY(1.5.1-rc0): the ops-membership steps below carry `&& github.event_name
   # != 'push'` for the push test loop. REVERT before merge.
   # OPS GATE — operator-driven runs (on-demand staging and direct rc dispatch)
   # require an active ai-dynamo/devops member. The nightly path (scheduled /

From d55ba5aa08c590ac105eece7d1017645cfd03347 Mon Sep 17 00:00:00 2001
From: Nate Mailhot 
Date: Tue, 29 Sep 2026 01:10:28 -0700
Subject: [PATCH 26/29] Revert "ci(release): bump the push-loop test to 1.5.1
 rc0"

This reverts commit 275783ef97243a41c5eccb63e04047da56c366ff.

Signed-off-by: Nate Mailhot 
---
 .github/workflows/on-demand-release.yml | 18 +++++++++---------
 .github/workflows/release.yml           |  2 +-
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/on-demand-release.yml b/.github/workflows/on-demand-release.yml
index 8c6bab02886b..d19a5e227f12 100644
--- a/.github/workflows/on-demand-release.yml
+++ b/.github/workflows/on-demand-release.yml
@@ -10,7 +10,7 @@
 name: Stage Release RC
 
 on:
-  # TEMPORARY(1.5.1-rc0): push stages release/1.5.1 rc0 (all artifacts) with no
+  # TEMPORARY(1.5.0-rc11): push stages release/1.5.0 rc11 (all artifacts) with no
   # dispatch inputs. REVERT before merge.
   push:
     branches:
@@ -81,10 +81,10 @@ jobs:
       - name: Validate and resolve target SHA
         id: target
         env:
-          # TEMPORARY(1.5.1-rc0): fallbacks cover inputless push events. REVERT to bare inputs.*.
-          RELEASE_VERSION: ${{ inputs.release_version || '1.5.1' }}
-          RELEASE_BRANCH: ${{ inputs.release_branch || 'release/1.5.1' }}
-          RC_ROUND: ${{ inputs.rc_round || '0' }}
+          # TEMPORARY(1.5.0-rc11): fallbacks cover inputless push events. REVERT to bare inputs.*.
+          RELEASE_VERSION: ${{ inputs.release_version || '1.5.0' }}
+          RELEASE_BRANCH: ${{ inputs.release_branch || 'release/1.5.0' }}
+          RC_ROUND: ${{ inputs.rc_round || '11' }}
           REPO: ${{ github.repository }}
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
@@ -137,8 +137,8 @@ jobs:
       - name: Resolve artifact selection
         id: resolve
         env:
-          # TEMPORARY(1.5.1-rc0): revert to bare inputs.release_version before merge.
-          RELEASE_VERSION: ${{ inputs.release_version || '1.5.1' }}
+          # TEMPORARY(1.5.0-rc11): revert to bare inputs.release_version before merge.
+          RELEASE_VERSION: ${{ inputs.release_version || '1.5.0' }}
           CONTAINERS: ${{ inputs.containers }}
           WHEELS: ${{ inputs.wheels }}
           CRATES: ${{ inputs.crates }}
@@ -364,8 +364,8 @@ jobs:
     with:
       commit_sha: ${{ needs.resolve-target.outputs.bump_sha }}
       on_demand: true
-      # TEMPORARY(1.5.1-rc0): revert to bare inputs.release_version before merge.
-      release_version: ${{ inputs.release_version || '1.5.1' }}
+      # TEMPORARY(1.5.0-rc11): revert to bare inputs.release_version before merge.
+      release_version: ${{ inputs.release_version || '1.5.0' }}
       # Operator-chosen internal-staging rc (immutable registries can't be overwritten).
       rc_number: ${{ needs.resolve-target.outputs.rc_number }}
       # Normalized subsets from resolve-target.
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 62a1cdd9e9c8..e5a62c71e981 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -156,7 +156,7 @@ jobs:
   # GATE: Version Extraction
   # ============================================================================
 
-  # TEMPORARY(1.5.1-rc0): the ops-membership steps below carry `&& github.event_name
+  # TEMPORARY(1.5.0-rc11): the ops-membership steps below carry `&& github.event_name
   # != 'push'` for the push test loop. REVERT before merge.
   # OPS GATE — operator-driven runs (on-demand staging and direct rc dispatch)
   # require an active ai-dynamo/devops member. The nightly path (scheduled /

From 870cbddb96a8102c655c14cef70b4787ff425d1f Mon Sep 17 00:00:00 2001
From: Nate Mailhot 
Date: Tue, 29 Sep 2026 01:10:29 -0700
Subject: [PATCH 27/29] Reapply "ci(release): bump the push-loop test to 1.5.1
 rc0"

This reverts commit d55ba5aa08c590ac105eece7d1017645cfd03347.

Signed-off-by: Nate Mailhot 
---
 .github/workflows/on-demand-release.yml | 18 +++++++++---------
 .github/workflows/release.yml           |  2 +-
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/on-demand-release.yml b/.github/workflows/on-demand-release.yml
index d19a5e227f12..8c6bab02886b 100644
--- a/.github/workflows/on-demand-release.yml
+++ b/.github/workflows/on-demand-release.yml
@@ -10,7 +10,7 @@
 name: Stage Release RC
 
 on:
-  # TEMPORARY(1.5.0-rc11): push stages release/1.5.0 rc11 (all artifacts) with no
+  # TEMPORARY(1.5.1-rc0): push stages release/1.5.1 rc0 (all artifacts) with no
   # dispatch inputs. REVERT before merge.
   push:
     branches:
@@ -81,10 +81,10 @@ jobs:
       - name: Validate and resolve target SHA
         id: target
         env:
-          # TEMPORARY(1.5.0-rc11): fallbacks cover inputless push events. REVERT to bare inputs.*.
-          RELEASE_VERSION: ${{ inputs.release_version || '1.5.0' }}
-          RELEASE_BRANCH: ${{ inputs.release_branch || 'release/1.5.0' }}
-          RC_ROUND: ${{ inputs.rc_round || '11' }}
+          # TEMPORARY(1.5.1-rc0): fallbacks cover inputless push events. REVERT to bare inputs.*.
+          RELEASE_VERSION: ${{ inputs.release_version || '1.5.1' }}
+          RELEASE_BRANCH: ${{ inputs.release_branch || 'release/1.5.1' }}
+          RC_ROUND: ${{ inputs.rc_round || '0' }}
           REPO: ${{ github.repository }}
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
@@ -137,8 +137,8 @@ jobs:
       - name: Resolve artifact selection
         id: resolve
         env:
-          # TEMPORARY(1.5.0-rc11): revert to bare inputs.release_version before merge.
-          RELEASE_VERSION: ${{ inputs.release_version || '1.5.0' }}
+          # TEMPORARY(1.5.1-rc0): revert to bare inputs.release_version before merge.
+          RELEASE_VERSION: ${{ inputs.release_version || '1.5.1' }}
           CONTAINERS: ${{ inputs.containers }}
           WHEELS: ${{ inputs.wheels }}
           CRATES: ${{ inputs.crates }}
@@ -364,8 +364,8 @@ jobs:
     with:
       commit_sha: ${{ needs.resolve-target.outputs.bump_sha }}
       on_demand: true
-      # TEMPORARY(1.5.0-rc11): revert to bare inputs.release_version before merge.
-      release_version: ${{ inputs.release_version || '1.5.0' }}
+      # TEMPORARY(1.5.1-rc0): revert to bare inputs.release_version before merge.
+      release_version: ${{ inputs.release_version || '1.5.1' }}
       # Operator-chosen internal-staging rc (immutable registries can't be overwritten).
       rc_number: ${{ needs.resolve-target.outputs.rc_number }}
       # Normalized subsets from resolve-target.
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index e5a62c71e981..62a1cdd9e9c8 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -156,7 +156,7 @@ jobs:
   # GATE: Version Extraction
   # ============================================================================
 
-  # TEMPORARY(1.5.0-rc11): the ops-membership steps below carry `&& github.event_name
+  # TEMPORARY(1.5.1-rc0): the ops-membership steps below carry `&& github.event_name
   # != 'push'` for the push test loop. REVERT before merge.
   # OPS GATE — operator-driven runs (on-demand staging and direct rc dispatch)
   # require an active ai-dynamo/devops member. The nightly path (scheduled /

From 91ebf014038c8765f95aba05b484f77a371ab27d Mon Sep 17 00:00:00 2001
From: pvijayakrish 
Date: Wed, 30 Sep 2026 12:25:23 -0700
Subject: [PATCH 28/29] fix(release): address review feedback and remove the rc
 test scaffold

Signed-off-by: pvijayakrish 
---
 .github/scripts/apply_dev_version.py          | 115 +++++++----
 .github/scripts/stage_crates.py               |   2 +-
 .github/workflows/cut-release-branch.yml      |  49 +++--
 .github/workflows/release.yml                 | 178 +++++++++++++-----
 ...emand-release.yml => stage-release-rc.yml} |  95 +++++++---
 lib/backend-common/Cargo.toml                 |   2 +-
 6 files changed, 322 insertions(+), 119 deletions(-)
 rename .github/workflows/{on-demand-release.yml => stage-release-rc.yml} (81%)

diff --git a/.github/scripts/apply_dev_version.py b/.github/scripts/apply_dev_version.py
index e94a26c7b8b7..155f32603b65 100755
--- a/.github/scripts/apply_dev_version.py
+++ b/.github/scripts/apply_dev_version.py
@@ -54,18 +54,20 @@
     "lib/kvbm-physical/Cargo.toml",
 ]
 
-# Member manifests that pin the workspace version inside a dependency
-# inline table, e.g. backend-common's
-# `dynamo-llm = { path = "../llm", version = "1.4.0", default-features = false }`
-# (a direct path dep because cargo cannot express `workspace = true` +
-# `default-features = false`). VERSION_LINE_RE is line-anchored and
-# intentionally skips inline tables, so these files get the same exact-string
-# rewrite as the root path-dep pins (the pinned value always equals
-# [workspace.package].version). Covered by BOTH modes: the dev-suffix stamp
-# (rewrite_root_cargo) and --set-version (set_release_version).
-WORKSPACE_PIN_CARGO_TARGETS = [
-    "lib/backend-common/Cargo.toml",
-]
+# Direct path deps on workspace crates, e.g. backend-common's
+# `dynamo-llm = { path = "../llm", default-features = false }` (cargo cannot
+# express `workspace = true` + `default-features = false`). Main keeps them
+# BARE; `cargo publish` needs a version on each. There is NO hand-kept list:
+# workspace_pin_manifests() discovers every publishable root-workspace member,
+# and stamp_workspace_pin() pins exactly the deps `cargo publish` requires
+# (normal/build deps, relative path, target inherits the workspace version).
+# A new crate is covered by adding it to [workspace] members — nothing here.
+DEP_SECTION_RE = re.compile(
+    r"^\[(?:target\.[^\]]+\.)?(?:build-)?dependencies\]\s*$")
+WORKSPACE_VERSION_RE = re.compile(
+    r"^\s*version\s*(?:\.\s*workspace\s*=\s*true|=\s*\{\s*workspace\s*=\s*true\s*\})",
+    re.MULTILINE)
+PUBLISH_FALSE_RE = re.compile(r"^\s*publish\s*=\s*false\b", re.MULTILINE)
 
 # Helm charts carry the unified version in version / appVersion / dependency
 # version. Each entry is (helm_subset_token, Chart.yaml path); a chart is bumped
@@ -242,17 +244,12 @@ def rewrite_root_cargo(root: Path, suffix: str) -> None:
     text = pin_re.sub(lambda mm: f"{mm.group(1)}{new}{mm.group(2)}", text)
     path.write_text(text)
 
-    # Bump the same literal pin where it lives in member manifests (inline
-    # dep tables that VERSION_LINE_RE deliberately skips). The early
+    # Re-stamp workspace path-dep pins that already carry a version (a release
+    # branch built nightly-style). Bare pins stay bare (inject=False): main's
+    # nightlies get the version from stage_crates at staging time. The early
     # "already stamped" return above keeps this idempotent.
-    for rel in WORKSPACE_PIN_CARGO_TARGETS:
-        p = root / rel
-        if not p.exists():
-            continue
-        t = p.read_text()
-        t2 = pin_re.sub(lambda mm: f"{mm.group(1)}{new}{mm.group(2)}", t)
-        if t2 != t:
-            p.write_text(t2)
+    for p in workspace_pin_manifests(root):
+        stamp_workspace_pin(p, new, inject=False)
 
 
 def _workspace_version(root: Path) -> str:
@@ -305,6 +302,61 @@ def set_cargo(path: Path, old: str, new: str) -> int:
     return n
 
 
+def workspace_pin_manifests(root: Path) -> list[Path]:
+    """Publishable members of the root workspace ([workspace] members, globs
+    expanded; `publish = false` skipped)."""
+    text = (root / "Cargo.toml").read_text()
+    m = re.search(r"^\[workspace\][^\[]*?\bmembers\s*=\s*\[(.*?)\]", text, re.MULTILINE | re.DOTALL)
+    if not m:
+        return []
+    out: list[Path] = []
+    for pat in re.findall(r'"([^"]+)"', m.group(1)):
+        for d in sorted(root.glob(pat)):
+            p = d / "Cargo.toml"
+            if p.is_file() and not PUBLISH_FALSE_RE.search(p.read_text()):
+                out.append(p)
+    return out
+
+
+def stamp_workspace_pin(path: Path, new: str, inject: bool = True) -> list[str]:
+    """Pin `path`'s inline path-deps on workspace-versioned crates to `new`.
+
+    Only deps `cargo publish` needs are touched: inline tables in normal/build
+    dependency sections (dev-deps are stripped on publish), with a relative
+    `path` whose target inherits `version.workspace = true`. An existing version
+    (even a stale one) is overwritten; a bare pin gets one after `path` when
+    `inject`. Registry deps and independently-versioned crates are never
+    touched. Idempotent. Returns the stamped dep names."""
+    stamped: list[str] = []
+    in_deps = False
+    lines = path.read_text().split("\n")
+    for i, line in enumerate(lines):
+        if line.lstrip().startswith("["):
+            in_deps = bool(DEP_SECTION_RE.match(line.strip()))
+            continue
+        dm = re.match(r'^(\s*)([A-Za-z0-9_-]+)(\s*=\s*)(\{[^{}]*\})(.*)$', line)
+        if not in_deps or not dm:
+            continue
+        table = dm.group(4)
+        pm = re.search(r'\bpath\s*=\s*"(\.[^"]*)"', table)
+        if not pm:
+            continue                           # not a relative path dep
+        target = (path.parent / pm.group(1) / "Cargo.toml").resolve()
+        if not target.is_file() or not WORKSPACE_VERSION_RE.search(target.read_text()):
+            continue                           # target has its own version
+        vm = re.search(r'\bversion\s*=\s*"([^"]*)"', table)
+        if vm:
+            table = table[: vm.start(1)] + new + table[vm.end(1):]
+        elif inject:
+            table = table[: pm.end()] + f', version = "{new}"' + table[pm.end():]
+        else:
+            continue
+        lines[i] = f"{dm.group(1)}{dm.group(2)}{dm.group(3)}{table}{dm.group(5)}"
+        stamped.append(dm.group(2))
+    path.write_text("\n".join(lines))
+    return stamped
+
+
 def set_helm(path: Path, old: str, new: str) -> None:
     text = path.read_text()
 
@@ -536,17 +588,14 @@ def _require(rel: str, hits: int, want: str) -> None:
     for rel in SUBCRATE_CARGO_TARGETS:
         if _exists(rel):
             _require(rel, set_cargo(root / rel, old, semver), semver)
-    # Workspace-version pins in member manifests (inline dep tables, e.g.
-    # backend-common's dynamo-llm pin) carry the same literal as the root
-    # pins; a pin pre-committed at the new version passes via the _require
-    # already-stamped tolerance. Older refs have a bare path dep with no
-    # version key at all — nothing to stamp (stage_crates injects the version
-    # at publish), so the guard only applies when a pin exists.
-    for rel in WORKSPACE_PIN_CARGO_TARGETS:
-        if _exists(rel):
-            n = set_cargo(root / rel, old, semver)
-            if re.search(r"path\s*=[^}\n]*\bversion\s*=", (root / rel).read_text()):
-                _require(rel, n, semver)
+    # Workspace-version path-dep pins (e.g. backend-common's dynamo-llm). Main
+    # keeps these BARE; the release cut stamps them so the release branch's
+    # cargo publish (crates.io GA) works. Discovered, not listed.
+    for p in workspace_pin_manifests(root):
+        names = stamp_workspace_pin(p, semver)
+        if names:
+            print(f"set_release_version: pinned {p.relative_to(root)}: {', '.join(names)} -> {semver}",
+                  file=sys.stderr)
     # Chart identity -- only for charts in the --helm subset.
     for token, rel in HELM_CHART_TARGETS:
         if token in helm and _exists(rel):
diff --git a/.github/scripts/stage_crates.py b/.github/scripts/stage_crates.py
index 18c9b4e3fbdd..40aecfbde8d2 100644
--- a/.github/scripts/stage_crates.py
+++ b/.github/scripts/stage_crates.py
@@ -46,7 +46,7 @@
 # them is closure-safe) and that registry consumers never pull.
 EXCLUDE_NAMES = frozenset({"kvbm-consolidator"})
 # Artifactory < 7.148 rejects the `Content-Type: application/octet-stream` header
-# cargo >= 1.96 sends on publish with a 415 (JFrog RTDEV-83141, rust-lang/cargo#17086),
+# cargo >= 1.96 sends on publish with a 415 (rust-lang/cargo#17086),
 # so the upload runs on the last pre-header toolchain. Remove once the
 # registry host runs Artifactory 7.148+.
 PUBLISH_TOOLCHAIN = "1.95.0"
diff --git a/.github/workflows/cut-release-branch.yml b/.github/workflows/cut-release-branch.yml
index 77101f617d31..b8f4ec42c885 100644
--- a/.github/workflows/cut-release-branch.yml
+++ b/.github/workflows/cut-release-branch.yml
@@ -29,10 +29,18 @@ on:
 permissions:
   contents: read
 
+# One cut per branch at a time: a second dispatch queues instead of racing the
+# first on the same branch push.
+concurrency:
+  group: cut-release-branch-${{ inputs.release_branch || inputs.release_version }}
+  cancel-in-progress: false
+
 jobs:
   # OPS GATE — only active ai-dynamo/devops members may run this. Fail-closed:
   # a missing app Members:read permission or any API error denies. Checks BOTH
   # github.actor and github.triggering_actor (actor does not re-resolve on re-runs).
+  # The steps are YAML anchors, repeated in prepare-branch so a "re-run failed
+  # jobs" (which reuses this job's earlier success) re-checks whoever re-ran it.
   authorize:
     name: Verify release-ops membership
     runs-on: prod-default-small-v2
@@ -40,29 +48,37 @@ jobs:
     permissions:
       contents: read
     steps:
-      - name: Mint app token
-        id: app-token
+      - &ops-token
+        name: Mint app token (ops gate)
+        id: ops-token
         uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1  # v3.2.0
         with:
           app-id: ${{ secrets.RELEASE_APP_ID }}
           private-key: ${{ secrets.RELEASE_APP_PRIVATE_KEY }}
-      - name: Require release-ops team membership
+      - &ops-check
+        name: Require release-ops team membership
         env:
-          GH_TOKEN: ${{ steps.app-token.outputs.token }}
+          GH_TOKEN: ${{ steps.ops-token.outputs.token }}
           ORG: ${{ github.repository_owner }}
           TEAM: devops
           ACTOR: ${{ github.actor }}
           TRIGGERING_ACTOR: ${{ github.triggering_actor }}
+          # The ORIGINAL trigger (the caller's, for a reusable workflow; unchanged
+          # on re-runs). A push/PR-triggered run must never reach release secrets,
+          # so a future test hook with a push trigger fails here instead.
+          EVENT: ${{ github.event_name }}
         run: |
           set -euo pipefail
+          case "${EVENT}" in
+            workflow_dispatch) ;;
+            *) echo "::error::Cut Release Branch only runs from a manual dispatch (got '${EVENT}')."; exit 1 ;;
+          esac
           for USER in $(printf '%s\n%s\n' "${ACTOR}" "${TRIGGERING_ACTOR}" | sort -u); do
             HTTP=$(curl -sS -o "${RUNNER_TEMP}/membership.json" -w '%{http_code}' \
               -H "Authorization: Bearer ${GH_TOKEN}" \
               -H "Accept: application/vnd.github+json" \
               "https://api.github.com/orgs/${ORG}/teams/${TEAM}/memberships/${USER}")
-            STATE=$(python3 -c "import json,sys;print(json.load(open(sys.argv[1])).get('state',''))" \
-              "${RUNNER_TEMP}/membership.json" 2>/dev/null || echo "")
-            if [ "${HTTP}" != "200" ] || [ "${STATE}" != "active" ]; then
+            if [ "${HTTP}" != "200" ] || ! grep -Eq '"state"[[:space:]]*:[[:space:]]*"active"' "${RUNNER_TEMP}/membership.json"; then
               echo "::error::@${USER} is not an active member of ${ORG}/${TEAM} (HTTP ${HTTP}); only release ops may run this workflow."
               exit 1
             fi
@@ -103,6 +119,9 @@ jobs:
       bump_sha: ${{ steps.bump.outputs.bump_sha }}
       is_test: ${{ steps.bump.outputs.is_test }}
     steps:
+      # Re-verify the ops gate in-job (see authorize): blocks re-run bypass.
+      - *ops-token
+      - *ops-check
       - name: Validate inputs
         env:
           RELEASE_VERSION: ${{ inputs.release_version }}
@@ -133,6 +152,17 @@ jobs:
             exit 1
           fi
 
+      # Pinned toolchain action instead of `curl | sh`. Runs before the push
+      # token is minted and is never passed a token. override: false keeps the
+      # checked-out rust-toolchain.toml authoritative (a newer cargo could
+      # rewrite Cargo.lock's format); cache off — this job only runs cargo update.
+      - name: Install Rust
+        uses: actions-rust-lang/setup-rust-toolchain@ecabd13d1c56bd1345c230e542e9144811ad706f  # v2.0.0
+        with:
+          toolchain: stable
+          override: false
+          cache: false
+
       - name: Mint app token
         id: app-token
         uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1  # v3.2.0
@@ -197,11 +227,6 @@ jobs:
           python3 "${RUNNER_TEMP}/apply_dev_version.py" "" . --set-version "${RELEASE_VERSION}" \
             --containers all --helm all --image-refs
 
-          if ! command -v cargo >/dev/null 2>&1; then
-            curl --proto '=https' --tlsv1.2 --retry 5 --retry-delay 2 -sSf https://sh.rustup.rs \
-              | sh -s -- -y --profile minimal --default-toolchain stable
-            source "${HOME}/.cargo/env"
-          fi
           # Sync EVERY Cargo.lock (root + nested workspaces: bindings/python, kvbm,
           # runtime/examples): a root-only `cargo update` leaves their pins stale and
           # fails the `--locked` checks. Discovered from git so a newly added nested
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index 62a1cdd9e9c8..27816d5b7d3b 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -5,7 +5,9 @@ name: Release Pipeline
 
 # workflow_dispatch is capped at 10 inputs by GitHub (this block has 9).
 # on_demand/release_version are workflow_call-only: the sole on-demand entry
-# point is on-demand-release.yml, which owns the manual approval gate.
+# point is stage-release-rc.yml. The ops gate and the manual approval gate live
+# HERE (authorize/approval jobs), so they bind every caller; stage-release-rc.yml
+# repeats only the ops gate, as a fast-fail before its long ECR wait.
 on:
   workflow_dispatch:
     inputs:
@@ -156,14 +158,16 @@ jobs:
   # GATE: Version Extraction
   # ============================================================================
 
-  # TEMPORARY(1.5.1-rc0): the ops-membership steps below carry `&& github.event_name
-  # != 'push'` for the push test loop. REVERT before merge.
-  # OPS GATE — operator-driven runs (on-demand staging and direct rc dispatch)
-  # require an active ai-dynamo/devops member. The nightly path (scheduled /
-  # nightly=true) is exempt so unattended runs are not blocked. Lives here (not
-  # only in the on-demand wrapper) so it binds EVERY caller of this reusable
-  # workflow — including a rogue workflow that calls it with on_demand=true.
-  # In automated-release to reach the RELEASE_APP secrets; carries no approval.
+  # OPS GATE — every run requires an active ai-dynamo/devops member, except the
+  # unattended nightly (nightly=true AND a scheduled trigger). A manual nightly
+  # (dispatch of this workflow or of nightly-ci.yml) is gated like any other run.
+  # Lives here (not only in the callers) so it binds EVERY caller of this
+  # reusable workflow — including a rogue workflow that calls it directly.
+  #
+  # The two steps are YAML anchors (&ops-token / &ops-check), repeated as the
+  # first steps of EVERY job that reads automated-release secrets. "Re-run failed
+  # jobs" reuses this job's earlier success, so a job-local check is the only
+  # thing that re-verifies whoever triggered the re-run (github.triggering_actor).
   authorize:
     name: Ops gate
     runs-on: prod-default-small-v2
@@ -171,38 +175,48 @@ jobs:
     permissions:
       contents: read
     steps:
-      - name: Mint app token
-        if: ${{ (inputs.on_demand || (github.event_name == 'workflow_dispatch' && !inputs.nightly)) && github.event_name != 'push' }}
-        id: app-token
+      - &ops-token
+        name: Mint app token (ops gate)
+        if: ${{ !(inputs.nightly && github.event_name == 'schedule') }}
+        id: ops-token
         uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1  # v3.2.0
         with:
           app-id: ${{ secrets.RELEASE_APP_ID }}
           private-key: ${{ secrets.RELEASE_APP_PRIVATE_KEY }}
-      - name: Require release-ops team membership
-        if: ${{ (inputs.on_demand || (github.event_name == 'workflow_dispatch' && !inputs.nightly)) && github.event_name != 'push' }}
+      - &ops-check
+        name: Require release-ops team membership
+        if: ${{ !(inputs.nightly && github.event_name == 'schedule') }}
         env:
-          GH_TOKEN: ${{ steps.app-token.outputs.token }}
+          GH_TOKEN: ${{ steps.ops-token.outputs.token }}
           ORG: ${{ github.repository_owner }}
           TEAM: devops
           ACTOR: ${{ github.actor }}
           TRIGGERING_ACTOR: ${{ github.triggering_actor }}
+          # The ORIGINAL trigger (the caller's, for a reusable workflow; unchanged
+          # on re-runs). A push/PR-triggered run must never reach release secrets,
+          # so a future test hook with a push trigger fails here instead.
+          EVENT: ${{ github.event_name }}
         run: |
           set -euo pipefail
+          case "${EVENT}" in
+            workflow_dispatch|schedule) ;;
+            *) echo "::error::The release pipeline only runs from a manual dispatch or the nightly schedule (got '${EVENT}')."; exit 1 ;;
+          esac
           for USER in $(printf '%s\n%s\n' "${ACTOR}" "${TRIGGERING_ACTOR}" | sort -u); do
             HTTP=$(curl -sS -o "${RUNNER_TEMP}/membership.json" -w '%{http_code}' \
               -H "Authorization: Bearer ${GH_TOKEN}" \
               -H "Accept: application/vnd.github+json" \
               "https://api.github.com/orgs/${ORG}/teams/${TEAM}/memberships/${USER}")
-            STATE=$(python3 -c "import json,sys;print(json.load(open(sys.argv[1])).get('state',''))" \
-              "${RUNNER_TEMP}/membership.json" 2>/dev/null || echo "")
-            if [ "${HTTP}" != "200" ] || [ "${STATE}" != "active" ]; then
-              echo "::error::@${USER} is not an active member of ${ORG}/${TEAM} (HTTP ${HTTP}); only release ops may stage a release."
+            # grep, not python3/jq: this step runs on every secret-using job's
+            # runner, including the gitlab_ci_runners group.
+            if [ "${HTTP}" != "200" ] || ! grep -Eq '"state"[[:space:]]*:[[:space:]]*"active"' "${RUNNER_TEMP}/membership.json"; then
+              echo "::error::@${USER} is not an active member of ${ORG}/${TEAM} (HTTP ${HTTP}); only release ops may run this release job."
               exit 1
             fi
             echo "@${USER} authorized via ${ORG}/${TEAM}"
           done
       - name: Gate mode
-        run: echo "ops gate applied=${{ (inputs.on_demand || (github.event_name == 'workflow_dispatch' && !inputs.nightly)) && github.event_name != 'push' }} (nightly exempt)"
+        run: echo "ops gate applied=${{ !(inputs.nightly && github.event_name == 'schedule') }} (only scheduled nightly is exempt)"
 
   # APPROVAL GATE — operator runs also require a manual-release-approval reviewer.
   # Dynamic environment: empty (no gate) on the nightly path so unattended runs are
@@ -289,7 +303,7 @@ jobs:
           fi
 
           if [[ "${ON_DEMAND}" == "true" ]]; then
-            # Branch + version bump are created/validated by on-demand-release.yml;
+            # Branch + version bump are created/validated by stage-release-rc.yml;
             # the version is passed explicitly rather than derived from the ref.
             if [[ ! "${RELEASE_VERSION}" =~ ^[0-9]+\.[0-9]+\.[0-9]+(\.(dev|post)[0-9]+)?$ ]]; then
               echo "Error: on_demand=true requires release_version X.Y.Z[.devN|.postN] (got: '${RELEASE_VERSION}')"
@@ -315,7 +329,7 @@ jobs:
             if [[ ! "$BRANCH_NAME" =~ ^release/[0-9]+\.[0-9]+\.[0-9]+(\.(dev|post)[0-9]+)?$ ]]; then
               echo "Error: workflow_dispatch must be triggered from a release/X.Y.Z[.devN|.postN] branch (or main with nightly=true)"
               echo "Current branch: $BRANCH_NAME"
-              echo "Named releases are not dispatchable here — run on-demand-release.yml with a BARE v[-] release_branch (a release/X.Y.Z- branch is treated as a throwaway test branch there too)."
+              echo "Named releases are not dispatchable here — run stage-release-rc.yml with a BARE v[-] release_branch (a release/X.Y.Z- branch is treated as a throwaway test branch there too)."
               exit 1
             fi
             VERSION="${BRANCH_NAME#release/}"
@@ -412,7 +426,7 @@ jobs:
             #   helm + crate (SemVer):   ${SEMVER_BASE}-rcN  where .devN/.postN become a
             #                            -devN/-postN pre-release (e.g. 0.1.0-rc0, 0.1.0-post3-rc0)
             #   wheel: clean version, disambiguated by the rcN upload subpath.
-            # Passed through from on-demand-release.yml's rc_number INPUT (default rc0;
+            # Passed through from stage-release-rc.yml's rc_number INPUT (default rc0;
             # NO GitHub tag is created). Require it (don't default to 0) so a broken
             # hand-off fails loud instead of silently colliding on rc0.
             RC_NUM="${INPUT_RC_NUMBER}"
@@ -519,6 +533,9 @@ jobs:
       SGLANG_EFA_BUILD_RESULT: ${{ inputs.sglang_efa_build_result }}
       TRTLLM_EFA_BUILD_RESULT: ${{ inputs.trtllm_efa_build_result }}
     steps:
+      # Re-verify the ops gate in-job (see authorize): blocks re-run bypass.
+      - *ops-token
+      - *ops-check
       - name: Checkout at source commit
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
         with:
@@ -929,25 +946,39 @@ jobs:
 
       - name: Validate chart/image coherence (on-demand)
         if: ${{ env.NIGHTLY != 'true' && env.ON_DEMAND == 'true' && env.HELM_SUBSET != 'none' }}
+        env:
+          STAGED_CONTAINERS: ${{ steps.copy_images.outputs.staged_containers }}
+          NGC_VERSION_TAG: ${{ needs.prepare-release.outputs.ngc_version_tag }}
         run: |
           set -euo pipefail
           # A published chart must reference only images published now (in
-          # CONTAINER_SUBSET) or already resolvable at the exact repository the
-          # chart references (the GA registry in values.yaml — NOT the staging
-          # org, where only rc-suffixed tags exist).
+          # CONTAINER_SUBSET, and proven staged — copy_image failures are
+          # non-fatal) or already resolvable at the exact repository the chart
+          # references (the GA registry in values.yaml — NOT the staging org,
+          # where only rc-suffixed tags exist).
           HELM="${HELM_SUBSET:-all}"
           CSET="${CONTAINER_SUBSET:-all}"
           want_helm() { [ "${HELM}" = "all" ] && return 0; case ",${HELM}," in *,"$1",*) return 0 ;; esac; return 1; }
           want_ctr()  { [ "${CSET}" = "all" ] && return 0; case ",${CSET}," in *,"$1",*) return 0 ;; esac; return 1; }
           fail=0
-          # Each site: chart_token|container_token|values.yaml|image repo (as referenced by the chart)
+          # Each site: chart_token|container_token|values.yaml|image repo (as referenced by the chart)|staged image name
           SITES=(
-            "platform|operator|deploy/helm/charts/platform/components/operator/values.yaml|nvcr.io/nvidia/ai-dynamo/kubernetes-operator"
+            "platform|operator|deploy/helm/charts/platform/components/operator/values.yaml|nvcr.io/nvidia/ai-dynamo/kubernetes-operator|kubernetes-operator"
           )
           for SITE in "${SITES[@]}"; do
-            IFS='|' read -r CHART CTOK VALUES REPO <<<"${SITE}"
+            IFS='|' read -r CHART CTOK VALUES REPO STAGED <<<"${SITE}"
             want_helm "${CHART}" || { echo "chart ${CHART} not published; skip coherence check"; continue; }
-            if want_ctr "${CTOK}"; then echo "ok: ${CHART} image ${CTOK} published this release"; continue; fi
+            if want_ctr "${CTOK}"; then
+              # Selected: the copy step tolerates per-image failures, so require
+              # the image in its successful-copy record before shipping a chart.
+              if [[ ",${STAGED_CONTAINERS}," == *",${STAGED}:${NGC_VERSION_TAG},"* ]]; then
+                echo "ok: ${CHART} image ${CTOK} staged this release (${STAGED}:${NGC_VERSION_TAG})"
+              else
+                echo "::error::chart ${CHART} needs ${CTOK}, which was selected but ${STAGED}:${NGC_VERSION_TAG} failed to copy — see 'Copy images to NGC'"
+                fail=1
+              fi
+              continue
+            fi
             # Image excluded: read the pinned tag the bump wrote and prove it exists
             # at the exact repository string the chart will deploy from.
             TAG=$(awk -v r="${REPO}" '
@@ -1251,7 +1282,7 @@ jobs:
         run: |
           set -euo pipefail
           if [ "${FAILED_COUNT:-0}" -ne 0 ]; then
-            echo "::error::${FAILED_COUNT} selected container image(s) failed to copy to NGC — see 'Copy images to NGC'. Once fixed, re-stage with a fresh round (rc_round on on-demand-release.yml; rc_number on an rc dispatch)."
+            echo "::error::${FAILED_COUNT} selected container image(s) failed to copy to NGC — see 'Copy images to NGC'. Once fixed, re-stage as a sub-round of the same rc (rc_round X.1, X.2, ... on stage-release-rc.yml; a direct rc dispatch still needs a new integer rc_number)."
             exit 1
           fi
           echo "All selected container images copied."
@@ -1281,6 +1312,9 @@ jobs:
       RC_NUMBER: ${{ needs.prepare-release.outputs.rc_number }}
       WHEEL_SUBSET: ${{ inputs.wheels }}
     steps:
+      # Re-verify the ops gate in-job (see authorize): blocks re-run bypass.
+      - *ops-token
+      - *ops-check
       - name: Setup crane
         env:
           CRANE_VERSION: v0.20.2
@@ -1416,21 +1450,39 @@ jobs:
           set -euo pipefail
 
           # Catches UTC-midnight drift between the build-time version stamp and
-          # prepare-release's wheel_version; ai_dynamo and ai_dynamo_runtime must agree.
+          # prepare-release's wheel_version. Every wheel that will be uploaded must agree,
+          # including kvbm (optional: built conditionally, so it is checked only if present).
+          WSUB="${WHEEL_SUBSET:-all}"
+          want_wheel() {
+            [ "${WSUB}" = "all" ] && return 0
+            case ",${WSUB}," in *,"$1",*) return 0 ;; esac
+            return 1
+          }
           shopt -s nullglob
           AI_DYNAMO_WHEELS=("${UPLOAD_DIR}"/ai_dynamo-*-py3-none-any.whl)
           AI_DYNAMO_RUNTIME_WHEELS=("${UPLOAD_DIR}"/ai_dynamo_runtime-*.whl)
+          KVBM_WHEELS=("${UPLOAD_DIR}"/kvbm-*.whl)
           shopt -u nullglob
 
-          if [ ${#AI_DYNAMO_WHEELS[@]} -eq 0 ] || [ ${#AI_DYNAMO_RUNTIME_WHEELS[@]} -eq 0 ]; then
-            echo "::error::Missing ai_dynamo or ai_dynamo_runtime wheel in ${UPLOAD_DIR}"
-            exit 1
+          CHECK=()
+          if want_wheel ai-dynamo; then
+            [ ${#AI_DYNAMO_WHEELS[@]} -gt 0 ] || { echo "::error::Missing ai_dynamo wheel in ${UPLOAD_DIR}"; exit 1; }
+            CHECK+=("${AI_DYNAMO_WHEELS[@]}")
+          fi
+          if want_wheel ai-dynamo-runtime; then
+            [ ${#AI_DYNAMO_RUNTIME_WHEELS[@]} -gt 0 ] || { echo "::error::Missing ai_dynamo_runtime wheel in ${UPLOAD_DIR}"; exit 1; }
+            CHECK+=("${AI_DYNAMO_RUNTIME_WHEELS[@]}")
+          fi
+          if want_wheel kvbm; then CHECK+=("${KVBM_WHEELS[@]}"); fi
+          if [ ${#CHECK[@]} -eq 0 ]; then
+            echo "No selected wheels to verify."
+            exit 0
           fi
 
           echo "Expected wheel version: ${EXPECTED_WHEEL_VERSION}"
 
           mismatch=0
-          for path in "${AI_DYNAMO_WHEELS[@]}" "${AI_DYNAMO_RUNTIME_WHEELS[@]}"; do
+          for path in "${CHECK[@]}"; do
             file="$(basename "${path}")"
             # PEP 427: --.whl; version sits between the 1st and 2nd hyphen.
             rest="${file#*-}"
@@ -1561,6 +1613,9 @@ jobs:
       COMMIT_SHA: ${{ needs.prepare-release.outputs.commit_sha }}
       NIGHTLY: ${{ needs.prepare-release.outputs.nightly }}
     steps:
+      # Re-verify the ops gate in-job (see authorize): blocks re-run bypass.
+      - *ops-token
+      - *ops-check
       - name: Checkout at source commit
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
         with:
@@ -1814,17 +1869,20 @@ jobs:
 
   trigger-gitlab-release-pipeline:
     name: Trigger GitLab Release Pipeline
-    # rc/nightly: every SELECTED stage must succeed (a 'none' subset skips the
-    # stage and counts as satisfied). On-demand: fires even on staging failures —
-    # GitLab's manual approval gates are the real publish gates. !cancelled()
-    # overrides the implicit needs-success(); stage-crates never blocks
-    # (continue-on-error).
+    # Every SELECTED stage must succeed (a 'none' subset skips the stage and
+    # counts as satisfied), so GitLab is never told to scan or promote something
+    # that failed to stage. A failed rc is repaired by re-staging at a new rc
+    # round. On-demand also requires crates to have FULLY staged: stage-crates is
+    # continue-on-error (its job result is masked as success), so crates_status
+    # is the authoritative signal. Nightly crates stay fail-soft (not gated).
+    # !cancelled() overrides the implicit needs-success().
     needs: [prepare-release, release-publish, stage-wheels-artifactory, stage-crates-artifactory]
     if: >-
       ${{ !cancelled() && needs.prepare-release.result == 'success' && inputs.skip_gitlab_pipeline != true &&
-          (needs.prepare-release.outputs.on_demand == 'true' ||
-           ((needs.release-publish.result == 'success' || (inputs.containers == 'none' && inputs.helm == 'none')) &&
-            (needs.stage-wheels-artifactory.result == 'success' || inputs.wheels == 'none'))) }}
+          (needs.release-publish.result == 'success' || (inputs.containers == 'none' && inputs.helm == 'none')) &&
+          (needs.stage-wheels-artifactory.result == 'success' || inputs.wheels == 'none') &&
+          (needs.prepare-release.outputs.on_demand != 'true' || inputs.crates == 'none' || inputs.crates == '' ||
+           needs.stage-crates-artifactory.outputs.crates_status == 'success') }}
     runs-on:
       group: gitlab_ci_runners
     environment: automated-release
@@ -1832,6 +1890,9 @@ jobs:
       pipeline_id:  ${{ steps.trigger.outputs.pipeline_id }}
       pipeline_url: ${{ steps.trigger.outputs.pipeline_url }}
     steps:
+      # Re-verify the ops gate in-job (see authorize): blocks re-run bypass.
+      - *ops-token
+      - *ops-check
       - name: Trigger GitLab pipeline
         id: trigger
         env:
@@ -1909,14 +1970,31 @@ jobs:
             # publish); RELEASE_TYPE=ondemand (no hyphen) matches dynamo.yml's rules.
             PIPELINE_TYPE="security"
             RELEASE_TYPE="ondemand"
-            # Coarse ARTIFACTS list derived from the per-type subsets ('none' omits
-            # the type). The fine-grained *_SUBSET vars below carry the exact picks.
+            # Coarse ARTIFACTS list keyed on what ACTUALLY staged (like nightly),
+            # not on what was requested: GitLab must never scan or promote a type
+            # that did not land. The job-level gate already requires every
+            # selected stage to succeed; this is the second line of defense.
             ARTIFACTS=""
-            [ "${WHEEL_SUBSET:-all}" != "none" ]     && ARTIFACTS="${ARTIFACTS},wheel"
-            [ "${CONTAINER_SUBSET:-all}" != "none" ] && ARTIFACTS="${ARTIFACTS},container"
-            [ "${CRATE_SUBSET:-all}" != "none" ]     && ARTIFACTS="${ARTIFACTS},crate"
-            [ "${HELM_SUBSET:-all}" != "none" ]      && ARTIFACTS="${ARTIFACTS},helm"
+            [ -n "${STAGED_WHEELS}" ]     && ARTIFACTS="${ARTIFACTS},wheel"
+            [ -n "${STAGED_CONTAINERS}" ] && ARTIFACTS="${ARTIFACTS},container"
+            [ -n "${STAGED_CRATES}" ]     && ARTIFACTS="${ARTIFACTS},crate"
+            [ -n "${STAGED_HELM}" ]       && ARTIFACTS="${ARTIFACTS},helm"
             ARTIFACTS="${ARTIFACTS#,}"
+            # A selected type that staged nothing is a failed stage, even when its
+            # job reported success (helm push is continue-on-error). Refuse, so
+            # the operator re-stages as a sub-round (rc_round X.1) instead of a silent gap.
+            MISSING=""
+            for P in "wheel:${WHEEL_SUBSET:-all}:${STAGED_WHEELS}" \
+                     "container:${CONTAINER_SUBSET:-all}:${STAGED_CONTAINERS}" \
+                     "crate:${CRATE_SUBSET:-none}:${STAGED_CRATES}" \
+                     "helm:${HELM_SUBSET:-all}:${STAGED_HELM}"; do
+              IFS=':' read -r KIND SEL GOT <<<"${P}"
+              if [ "${SEL}" != "none" ] && [ -z "${GOT}" ]; then MISSING="${MISSING} ${KIND}"; fi
+            done
+            if [ -n "${MISSING}" ]; then
+              echo "::error::On-demand run selected but did not stage:${MISSING}. Refusing to trigger GitLab; fix and re-stage as a sub-round of the same rc (rc_round X.1, X.2, ...)."
+              exit 1
+            fi
             NIGHTLY_TAG=""
             RC_TAG=""
             RELEASE_TAG="v${VERSION}"
diff --git a/.github/workflows/on-demand-release.yml b/.github/workflows/stage-release-rc.yml
similarity index 81%
rename from .github/workflows/on-demand-release.yml
rename to .github/workflows/stage-release-rc.yml
index 8c6bab02886b..33023fb17448 100644
--- a/.github/workflows/on-demand-release.yml
+++ b/.github/workflows/stage-release-rc.yml
@@ -4,17 +4,14 @@
 # Stage Release RC: stage (or re-stage) an EXISTING release branch at a chosen rc
 # round via release.yml, then trigger the GitLab pipeline. Run once per rc round.
 # The branch + version stamp are created by "Cut Release Branch"; this workflow is
-# read-only on the tree. The devops ops gate and the reviewer approval that guard
-# the publish live in release.yml, so they bind this and every other caller.
+# read-only on the tree. The authoritative devops ops gate and the reviewer
+# approval live in release.yml, so they bind this and every other caller. This
+# workflow repeats the ops gate (authorize job) as a fast-fail, so a non-devops
+# dispatch is rejected before any job reaches the automated-release environment.
 
 name: Stage Release RC
 
 on:
-  # TEMPORARY(1.5.1-rc0): push stages release/1.5.1 rc0 (all artifacts) with no
-  # dispatch inputs. REVERT before merge.
-  push:
-    branches:
-      - pull-request/13961
   workflow_dispatch:
     inputs:
       release_branch:
@@ -26,7 +23,7 @@ on:
         required: true
         type: string
       rc_round:
-        description: 'Internal staging RC:  or . (e.g. 2 or 2.1 -> staging versions X.Y.Z-rc.2 / X.Y.Z-rc.2.1, a numerically-ordered SemVer pre-release; also the wheel rc subpath). Re-stages of the same version need a distinct value (immutable registries). NO GitHub tag is created.'
+        description: 'Internal staging RC:  or . (e.g. 2 or 2.1 -> staging versions X.Y.Z-rc.2 / X.Y.Z-rc.2.1, a numerically-ordered SemVer pre-release; also the wheel rc subpath). To repair a failed stage without bumping the rc QA sees, re-stage as a sub-round (2 -> 2.1 -> 2.2); registries are immutable, so each attempt needs a distinct value. NO GitHub tag is created.'
         required: false
         type: string
         default: '0'
@@ -59,15 +56,70 @@ on:
 permissions:
   contents: read
 
+# One staging run per (branch, rc round): a second dispatch queues instead of
+# racing the first into immutable registries. Never cancel a run mid-stage.
+concurrency:
+  group: stage-release-rc-${{ inputs.release_branch }}-${{ inputs.rc_round }}
+  cancel-in-progress: false
+
 jobs:
+  # OPS GATE (fast-fail) — only active ai-dynamo/devops members may stage. Checks
+  # BOTH github.actor and github.triggering_actor (actor does not re-resolve on
+  # re-runs). The steps are YAML anchors, repeated in await-post-merge (the only
+  # other job here that reads automated-release secrets) so a "re-run failed
+  # jobs" by a non-devops user is re-checked. release.yml re-checks its own jobs.
+  authorize:
+    name: Verify release-ops membership
+    runs-on: prod-default-small-v2
+    environment: automated-release
+    permissions:
+      contents: read
+    steps:
+      - &ops-token
+        name: Mint app token (ops gate)
+        id: ops-token
+        uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1  # v3.2.0
+        with:
+          app-id: ${{ secrets.RELEASE_APP_ID }}
+          private-key: ${{ secrets.RELEASE_APP_PRIVATE_KEY }}
+      - &ops-check
+        name: Require release-ops team membership
+        env:
+          GH_TOKEN: ${{ steps.ops-token.outputs.token }}
+          ORG: ${{ github.repository_owner }}
+          TEAM: devops
+          ACTOR: ${{ github.actor }}
+          TRIGGERING_ACTOR: ${{ github.triggering_actor }}
+          # The ORIGINAL trigger (the caller's, for a reusable workflow; unchanged
+          # on re-runs). A push/PR-triggered run must never reach release secrets,
+          # so a future test hook with a push trigger fails here instead.
+          EVENT: ${{ github.event_name }}
+        run: |
+          set -euo pipefail
+          case "${EVENT}" in
+            workflow_dispatch) ;;
+            *) echo "::error::Stage Release RC only runs from a manual dispatch (got '${EVENT}')."; exit 1 ;;
+          esac
+          for USER in $(printf '%s\n%s\n' "${ACTOR}" "${TRIGGERING_ACTOR}" | sort -u); do
+            HTTP=$(curl -sS -o "${RUNNER_TEMP}/membership.json" -w '%{http_code}' \
+              -H "Authorization: Bearer ${GH_TOKEN}" \
+              -H "Accept: application/vnd.github+json" \
+              "https://api.github.com/orgs/${ORG}/teams/${TEAM}/memberships/${USER}")
+            if [ "${HTTP}" != "200" ] || ! grep -Eq '"state"[[:space:]]*:[[:space:]]*"active"' "${RUNNER_TEMP}/membership.json"; then
+              echo "::error::@${USER} is not an active member of ${ORG}/${TEAM} (HTTP ${HTTP}); only release ops may stage a release."
+              exit 1
+            fi
+            echo "@${USER} authorized via ${ORG}/${TEAM}"
+          done
+
   # RESOLVE TARGET (read-only) — validate inputs; resolve the branch tip SHA (the
   # commit whose post-merge images this rc stages), the rc round, and the subsets.
-  # No ops gate here: the authoritative devops + approval gate for the publish path
-  # lives in release.yml (reached via invoke-release), which binds every caller.
+  # Reads no environment secrets (only GITHUB_TOKEN), so it runs outside
+  # automated-release.
   resolve-target:
     name: Resolve staging target
+    needs: [authorize]
     runs-on: prod-default-small-v2
-    environment: automated-release
     permissions:
       contents: read
     outputs:
@@ -81,10 +133,9 @@ jobs:
       - name: Validate and resolve target SHA
         id: target
         env:
-          # TEMPORARY(1.5.1-rc0): fallbacks cover inputless push events. REVERT to bare inputs.*.
-          RELEASE_VERSION: ${{ inputs.release_version || '1.5.1' }}
-          RELEASE_BRANCH: ${{ inputs.release_branch || 'release/1.5.1' }}
-          RC_ROUND: ${{ inputs.rc_round || '0' }}
+          RELEASE_VERSION: ${{ inputs.release_version }}
+          RELEASE_BRANCH: ${{ inputs.release_branch }}
+          RC_ROUND: ${{ inputs.rc_round }}
           REPO: ${{ github.repository }}
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
@@ -137,8 +188,7 @@ jobs:
       - name: Resolve artifact selection
         id: resolve
         env:
-          # TEMPORARY(1.5.1-rc0): revert to bare inputs.release_version before merge.
-          RELEASE_VERSION: ${{ inputs.release_version || '1.5.1' }}
+          RELEASE_VERSION: ${{ inputs.release_version }}
           CONTAINERS: ${{ inputs.containers }}
           WHEELS: ${{ inputs.wheels }}
           CRATES: ${{ inputs.crates }}
@@ -255,6 +305,9 @@ jobs:
       contents: read
       actions: read
     steps:
+      # Re-verify the ops gate in-job (see authorize): blocks re-run bypass.
+      - *ops-token
+      - *ops-check
       - name: Wait for the selected artifacts' build images in ECR
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -354,9 +407,8 @@ jobs:
           done
 
   # STAGE ARTIFACTS + TRIGGER GITLAB (release.yml in on-demand mode). release.yml's
-  # own authorize + approval gate covers the publish path for every caller, so the
-  # ops/approval gate is NOT duplicated here; this workflow's authorize job stays as
-  # a fast-fail so a non-devops dispatch is rejected before the ECR wait.
+  # own authorize + approval gate covers the publish path for every caller; the
+  # approval is NOT duplicated here.
   invoke-release:
     name: Stage release
     needs: [resolve-target, await-post-merge]
@@ -364,8 +416,7 @@ jobs:
     with:
       commit_sha: ${{ needs.resolve-target.outputs.bump_sha }}
       on_demand: true
-      # TEMPORARY(1.5.1-rc0): revert to bare inputs.release_version before merge.
-      release_version: ${{ inputs.release_version || '1.5.1' }}
+      release_version: ${{ inputs.release_version }}
       # Operator-chosen internal-staging rc (immutable registries can't be overwritten).
       rc_number: ${{ needs.resolve-target.outputs.rc_number }}
       # Normalized subsets from resolve-target.
diff --git a/lib/backend-common/Cargo.toml b/lib/backend-common/Cargo.toml
index a42231b91acf..a6ee0e093ea0 100644
--- a/lib/backend-common/Cargo.toml
+++ b/lib/backend-common/Cargo.toml
@@ -23,7 +23,7 @@ integration = ["dynamo-runtime/integration"]
 
 [dependencies]
 dynamo-runtime = { workspace = true }
-dynamo-llm = { path = "../llm", version = "1.5.0", default-features = false }
+dynamo-llm = { path = "../llm", default-features = false }
 dynamo-protocols = { workspace = true }
 dynamo-rl = { workspace = true }
 

From ab96d257a1306a7eb5511b2b17ba613fd49cdffa Mon Sep 17 00:00:00 2001
From: pvijayakrish 
Date: Wed, 30 Sep 2026 12:41:30 -0700
Subject: [PATCH 29/29] style(release): apply black and mark stage_crates.py
 executable

Signed-off-by: pvijayakrish 
---
 .github/scripts/apply_dev_version.py | 229 +++++++++++++++-------
 .github/scripts/stage_crates.py      | 278 ++++++++++++++++++++-------
 2 files changed, 371 insertions(+), 136 deletions(-)
 mode change 100644 => 100755 .github/scripts/stage_crates.py

diff --git a/.github/scripts/apply_dev_version.py b/.github/scripts/apply_dev_version.py
index 155f32603b65..cdf0bd95588c 100755
--- a/.github/scripts/apply_dev_version.py
+++ b/.github/scripts/apply_dev_version.py
@@ -62,11 +62,11 @@
 # and stamp_workspace_pin() pins exactly the deps `cargo publish` requires
 # (normal/build deps, relative path, target inherits the workspace version).
 # A new crate is covered by adding it to [workspace] members — nothing here.
-DEP_SECTION_RE = re.compile(
-    r"^\[(?:target\.[^\]]+\.)?(?:build-)?dependencies\]\s*$")
+DEP_SECTION_RE = re.compile(r"^\[(?:target\.[^\]]+\.)?(?:build-)?dependencies\]\s*$")
 WORKSPACE_VERSION_RE = re.compile(
     r"^\s*version\s*(?:\.\s*workspace\s*=\s*true|=\s*\{\s*workspace\s*=\s*true\s*\})",
-    re.MULTILINE)
+    re.MULTILINE,
+)
 PUBLISH_FALSE_RE = re.compile(r"^\s*publish\s*=\s*false\b", re.MULTILINE)
 
 # Helm charts carry the unified version in version / appVersion / dependency
@@ -89,16 +89,31 @@
 # from its `tag: "" -> .Chart.AppVersion` inheritance. 3rd-party tags (etcd/nats) are
 # never matched (different repositories).
 HELM_IMAGE_TAG_SITES = [
-    ("operator", "platform", "deploy/helm/charts/platform/values.yaml",
-     "nvcr.io/nvidia/ai-dynamo/kubernetes-operator"),
-    ("operator", "platform", "deploy/helm/charts/platform/components/operator/values.yaml",
-     "nvcr.io/nvidia/ai-dynamo/kubernetes-operator"),
+    (
+        "operator",
+        "platform",
+        "deploy/helm/charts/platform/values.yaml",
+        "nvcr.io/nvidia/ai-dynamo/kubernetes-operator",
+    ),
+    (
+        "operator",
+        "platform",
+        "deploy/helm/charts/platform/components/operator/values.yaml",
+        "nvcr.io/nvidia/ai-dynamo/kubernetes-operator",
+    ),
 ]
 
 # Normalized subset universes for --containers / --helm token validation.
 CONTAINER_TOKENS = {
-    "vllm-runtime", "vllm-efa", "sglang-runtime", "sglang-efa",
-    "trtllm-runtime", "trtllm-efa", "frontend", "operator", "planner",
+    "vllm-runtime",
+    "vllm-efa",
+    "sglang-runtime",
+    "sglang-efa",
+    "trtllm-runtime",
+    "trtllm-efa",
+    "frontend",
+    "operator",
+    "planner",
     "sidecar",
 }
 HELM_TOKENS = {"platform"}
@@ -263,7 +278,9 @@ def _workspace_version(root: Path) -> str:
 def _semver_form(new: str) -> str:
     m = SET_RE.match(new)
     if not m:
-        raise RuntimeError(f"--set-version must be X.Y.Z, X.Y.Z.devN, or X.Y.Z.postN (got '{new}')")
+        raise RuntimeError(
+            f"--set-version must be X.Y.Z, X.Y.Z.devN, or X.Y.Z.postN (got '{new}')"
+        )
     base = f"{m.group(1)}.{m.group(2)}.{m.group(3)}"
     suffix = m.group(4)
     if not suffix:
@@ -285,7 +302,9 @@ def _set(m: re.Match) -> str:
     text = VERSION_LINE_RE.sub(_set, path.read_text())
     if is_root:
         text = PY_ROOT_PIN_RE.sub(
-            lambda m: f"{m.group(1)}{new}{m.group(3)}" if m.group(2) == old else m.group(0),
+            lambda m: f"{m.group(1)}{new}{m.group(3)}"
+            if m.group(2) == old
+            else m.group(0),
             text,
         )
     path.write_text(text)
@@ -306,7 +325,11 @@ def workspace_pin_manifests(root: Path) -> list[Path]:
     """Publishable members of the root workspace ([workspace] members, globs
     expanded; `publish = false` skipped)."""
     text = (root / "Cargo.toml").read_text()
-    m = re.search(r"^\[workspace\][^\[]*?\bmembers\s*=\s*\[(.*?)\]", text, re.MULTILINE | re.DOTALL)
+    m = re.search(
+        r"^\[workspace\][^\[]*?\bmembers\s*=\s*\[(.*?)\]",
+        text,
+        re.MULTILINE | re.DOTALL,
+    )
     if not m:
         return []
     out: list[Path] = []
@@ -334,21 +357,21 @@ def stamp_workspace_pin(path: Path, new: str, inject: bool = True) -> list[str]:
         if line.lstrip().startswith("["):
             in_deps = bool(DEP_SECTION_RE.match(line.strip()))
             continue
-        dm = re.match(r'^(\s*)([A-Za-z0-9_-]+)(\s*=\s*)(\{[^{}]*\})(.*)$', line)
+        dm = re.match(r"^(\s*)([A-Za-z0-9_-]+)(\s*=\s*)(\{[^{}]*\})(.*)$", line)
         if not in_deps or not dm:
             continue
         table = dm.group(4)
         pm = re.search(r'\bpath\s*=\s*"(\.[^"]*)"', table)
         if not pm:
-            continue                           # not a relative path dep
+            continue  # not a relative path dep
         target = (path.parent / pm.group(1) / "Cargo.toml").resolve()
         if not target.is_file() or not WORKSPACE_VERSION_RE.search(target.read_text()):
-            continue                           # target has its own version
+            continue  # target has its own version
         vm = re.search(r'\bversion\s*=\s*"([^"]*)"', table)
         if vm:
-            table = table[: vm.start(1)] + new + table[vm.end(1):]
+            table = table[: vm.start(1)] + new + table[vm.end(1) :]
         elif inject:
-            table = table[: pm.end()] + f', version = "{new}"' + table[pm.end():]
+            table = table[: pm.end()] + f', version = "{new}"' + table[pm.end() :]
         else:
             continue
         lines[i] = f"{dm.group(1)}{dm.group(2)}{dm.group(3)}{table}{dm.group(5)}"
@@ -367,7 +390,8 @@ def set_helm(path: Path, old: str, new: str) -> None:
         re.MULTILINE,
     )
     text, n_top = top.subn(
-        lambda m: f"{m.group('pre')}{m.group('q')}{new}{m.group('q')}{m.group('post')}", text
+        lambda m: f"{m.group('pre')}{m.group('q')}{new}{m.group('q')}{m.group('post')}",
+        text,
     )
     if n_top == 0:
         raise RuntimeError(f"no top-level version/appVersion in {path}")
@@ -375,8 +399,8 @@ def set_helm(path: Path, old: str, new: str) -> None:
     # dynamo-operator (file:// subchart) pin always rides the workspace version;
     # the hop is bounded to the entry so it can't reach nats/etcd/....
     text = re.sub(
-        r'(?m)^(\s*-\s+name:\s*dynamo-operator\s*\n'
-        r'(?:(?!\s*-\s)[^\n]*\n)*?'
+        r"(?m)^(\s*-\s+name:\s*dynamo-operator\s*\n"
+        r"(?:(?!\s*-\s)[^\n]*\n)*?"
         r'\s*version\s*:\s*)("?)[^"\n]*\2(\s*)$',
         lambda m: f"{m.group(1)}{m.group(2)}{new}{m.group(2)}{m.group(3)}",
         text,
@@ -390,14 +414,18 @@ def set_helm(path: Path, old: str, new: str) -> None:
 
 # Bounds the repository->tag hop at the next `repository:` line, so a block
 # with no tag fails loudly instead of rewriting another image's tag.
-_TAG_HOP = r'(?:(?![^\n]*repository:)[^\n]*\n)*?'
+_TAG_HOP = r"(?:(?![^\n]*repository:)[^\n]*\n)*?"
 
 
 def set_helm_values_tag(path: Path, repo: str, new: str) -> None:
     # Set the `tag:` that follows the image `repository: ` line to `new`,
     # regardless of its current value (the published image tag is the release tag).
     pat = re.compile(
-        r'(repository:\s*"?' + re.escape(repo) + r'"?\s*\n' + _TAG_HOP + r'\s*tag:\s*)"?[^"\n]*"?',
+        r'(repository:\s*"?'
+        + re.escape(repo)
+        + r'"?\s*\n'
+        + _TAG_HOP
+        + r'\s*tag:\s*)"?[^"\n]*"?',
         re.MULTILINE,
     )
     text, n = pat.subn(lambda m: f"{m.group(1)}{new}", path.read_text(), count=1)
@@ -411,15 +439,20 @@ def _current_image_tag(path: Path, repo: str) -> str:
     # an empty tag inherits the chart appVersion at deploy time, so there is no
     # recorded last-published tag to pin to).
     m = re.search(
-        r'repository:\s*"?' + re.escape(repo) + r'"?\s*\n' + _TAG_HOP + r'\s*tag:\s*"?([^"\n]*)"?',
+        r'repository:\s*"?'
+        + re.escape(repo)
+        + r'"?\s*\n'
+        + _TAG_HOP
+        + r'\s*tag:\s*"?([^"\n]*)"?',
         path.read_text(),
     )
     return m.group(1).strip() if m else ""
 
 
 def _tracked_files(root: Path) -> list[Path]:
-    out = subprocess.run(["git", "ls-files", "-z"], cwd=root, check=True,
-                         capture_output=True, text=True).stdout
+    out = subprocess.run(
+        ["git", "ls-files", "-z"], cwd=root, check=True, capture_output=True, text=True
+    ).stdout
     files = []
     for rel in out.split("\0"):
         if not rel or rel.startswith(".github/"):
@@ -432,8 +465,9 @@ def _tracked_files(root: Path) -> list[Path]:
     return files
 
 
-def rewrite_image_refs(root: Path, new_version: str, containers: set[str],
-                       old_version: str) -> tuple[int, int]:
+def rewrite_image_refs(
+    root: Path, new_version: str, containers: set[str], old_version: str
+) -> tuple[int, int]:
     """Point first-party image references at the GA registry + release version —
     but ONLY for images this release actually publishes.
 
@@ -447,8 +481,10 @@ def rewrite_image_refs(root: Path, new_version: str, containers: set[str],
     Returns (files_changed, refs_rewritten)."""
     images = sorted({IMAGE_REF_TOKENS[t] for t in containers if t in IMAGE_REF_TOKENS})
     if not images:
-        print("rewrite_image_refs: no publishable image selected; placeholders left intact",
-              file=sys.stderr)
+        print(
+            "rewrite_image_refs: no publishable image selected; placeholders left intact",
+            file=sys.stderr,
+        )
         return (0, 0)
 
     # old_version comes from Cargo.toml, i.e. the SemVer form (1.3.0+post1,
@@ -468,7 +504,11 @@ def rewrite_image_refs(root: Path, new_version: str, containers: set[str],
     # `:1.3.0-cuda13` and `:1.4.0.dev1`, rewriting them to a bare version and
     # silently destroying the variant suffix.
     pats = [
-        (img, re.compile(rf"{reg_alt}/{re.escape(img)}:(?:{tag_alt})(?![\w.-])"), f"{GA_REGISTRY}/{img}:{new_version}")
+        (
+            img,
+            re.compile(rf"{reg_alt}/{re.escape(img)}:(?:{tag_alt})(?![\w.-])"),
+            f"{GA_REGISTRY}/{img}:{new_version}",
+        )
         for img in images
     ]
     # Untagged prose references (`my-registry/vllm-runtime` with no `:tag`) still
@@ -476,7 +516,13 @@ def rewrite_image_refs(root: Path, new_version: str, containers: set[str],
     # image is never given a real registry path. Runs after the tagged patterns,
     # which have already consumed the `/:` forms.
     pats += [
-        (img, re.compile(rf"{re.escape(PLACEHOLDER_REGISTRY)}/{re.escape(img)}(?![\w.:-])"), f"{GA_REGISTRY}/{img}")
+        (
+            img,
+            re.compile(
+                rf"{re.escape(PLACEHOLDER_REGISTRY)}/{re.escape(img)}(?![\w.:-])"
+            ),
+            f"{GA_REGISTRY}/{img}",
+        )
         for img in images
     ]
 
@@ -495,7 +541,10 @@ def rewrite_image_refs(root: Path, new_version: str, containers: set[str],
             continue
         # Fast path — must test every literal the regex can match, including the
         # PEP 440 spelling of the old version and the untagged placeholder registry.
-        if not any(t in text for t in old_literals) and PLACEHOLDER_REGISTRY not in text:
+        if (
+            not any(t in text for t in old_literals)
+            and PLACEHOLDER_REGISTRY not in text
+        ):
             continue
         for _img, pat, repl in pats:
             text, n = pat.subn(repl, text)
@@ -504,13 +553,19 @@ def rewrite_image_refs(root: Path, new_version: str, containers: set[str],
             path.write_text(text)
             files_changed += 1
 
-    print(f"rewrite_image_refs: {refs} reference(s) in {files_changed} file(s) -> "
-          f"{GA_REGISTRY}/:{new_version} for {images}", file=sys.stderr)
+    print(
+        f"rewrite_image_refs: {refs} reference(s) in {files_changed} file(s) -> "
+        f"{GA_REGISTRY}/:{new_version} for {images}",
+        file=sys.stderr,
+    )
 
     for rel in skipped_stale:
-        print(f"::warning::{rel} still references {old_version} and was left untouched "
-              f"on purpose (it lists dev-line and nightly releases that must not move). "
-              f"Update the current-release rows to {new_version} by hand.", file=sys.stderr)
+        print(
+            f"::warning::{rel} still references {old_version} and was left untouched "
+            f"on purpose (it lists dev-line and nightly releases that must not move). "
+            f"Update the current-release rows to {new_version} by hand.",
+            file=sys.stderr,
+        )
 
     # Advisory only: a re-cut of a branch that previously shipped a wider selection
     # legitimately still carries those older refs, so warn rather than fail.
@@ -525,8 +580,11 @@ def rewrite_image_refs(root: Path, new_version: str, containers: set[str],
             if f"{GA_REGISTRY}/{img}:{new_version}" in text:
                 stale.append(f"{path.relative_to(root)} -> {img}")
     if stale:
-        print(f"::warning::{len(stale)} reference(s) point at unselected image(s) at "
-              f"{new_version}: {stale[:8]}{' …' if len(stale) > 8 else ''}", file=sys.stderr)
+        print(
+            f"::warning::{len(stale)} reference(s) point at unselected image(s) at "
+            f"{new_version}: {stale[:8]}{' …' if len(stale) > 8 else ''}",
+            file=sys.stderr,
+        )
     return (files_changed, refs)
 
 
@@ -539,12 +597,19 @@ def _parse_subset(spec: str, universe: set[str]) -> set[str]:
     sel = {t.strip() for t in spec.split(",") if t.strip()}
     unknown = sel - universe
     if unknown:
-        raise RuntimeError(f"unknown subset token(s) {sorted(unknown)}; valid: {sorted(universe)}")
+        raise RuntimeError(
+            f"unknown subset token(s) {sorted(unknown)}; valid: {sorted(universe)}"
+        )
     return sel
 
 
-def set_release_version(root: Path, new_version: str, containers: set[str], helm: set[str],
-                        image_refs: bool = False) -> None:
+def set_release_version(
+    root: Path,
+    new_version: str,
+    containers: set[str],
+    helm: set[str],
+    image_refs: bool = False,
+) -> None:
     old = _workspace_version(root)
     semver = _semver_form(new_version)
 
@@ -553,7 +618,10 @@ def _exists(rel: str) -> bool:
         # a path that doesn't exist there has nothing to stamp.
         if (root / rel).exists():
             return True
-        print(f"set_release_version: skip {rel} (absent at this source ref)", file=sys.stderr)
+        print(
+            f"set_release_version: skip {rel} (absent at this source ref)",
+            file=sys.stderr,
+        )
         return False
 
     # Cargo.toml holds the workspace version in SemVer form ('1.4.2-dev1'),
@@ -565,8 +633,10 @@ def _exists(rel: str) -> bool:
     # so a zero-hit rewrite is expected for them. Anywhere else, zero hits with
     # the new version also absent means the file holds some third version and
     # the release would ship stale metadata.
-    independent = {"lib/gpu_memory_service/pyproject.toml",
-                   "lib/bindings/python/codegen/Cargo.toml"}
+    independent = {
+        "lib/gpu_memory_service/pyproject.toml",
+        "lib/bindings/python/codegen/Cargo.toml",
+    }
 
     def _require(rel: str, hits: int, want: str) -> None:
         if rel in independent or hits:
@@ -574,7 +644,8 @@ def _require(rel: str, hits: int, want: str) -> None:
         if want not in (root / rel).read_text():
             raise RuntimeError(
                 f"{rel} carries neither the workspace version ('{old}'/'{old_py}') "
-                f"nor '{want}' -- refusing a partial stamp")
+                f"nor '{want}' -- refusing a partial stamp"
+            )
 
     # Package identity -- ALWAYS bumped, regardless of the wheels/crates selection:
     # the containers embed wheels built from this tree, so a container-only release
@@ -582,8 +653,13 @@ def _require(rel: str, hits: int, want: str) -> None:
     # carry the previous version. (wheels/crates are intentionally not passed in.)
     for rel in PYPROJECT_TARGETS:
         if rel == "pyproject.toml" or _exists(rel):
-            _require(rel, set_pyproject(root / rel, old_py, new_version,
-                                        is_root=(rel == "pyproject.toml")), new_version)
+            _require(
+                rel,
+                set_pyproject(
+                    root / rel, old_py, new_version, is_root=(rel == "pyproject.toml")
+                ),
+                new_version,
+            )
     _require("Cargo.toml", set_cargo(root / "Cargo.toml", old, semver), semver)
     for rel in SUBCRATE_CARGO_TARGETS:
         if _exists(rel):
@@ -594,8 +670,10 @@ def _require(rel: str, hits: int, want: str) -> None:
     for p in workspace_pin_manifests(root):
         names = stamp_workspace_pin(p, semver)
         if names:
-            print(f"set_release_version: pinned {p.relative_to(root)}: {', '.join(names)} -> {semver}",
-                  file=sys.stderr)
+            print(
+                f"set_release_version: pinned {p.relative_to(root)}: {', '.join(names)} -> {semver}",
+                file=sys.stderr,
+            )
     # Chart identity -- only for charts in the --helm subset.
     for token, rel in HELM_CHART_TARGETS:
         if token in helm and _exists(rel):
@@ -615,10 +693,14 @@ def _require(rel: str, hits: int, want: str) -> None:
                 raise RuntimeError(
                     f"chart '{htoken}' is selected but its image '{ctoken}' is excluded and "
                     f"{rel} records no previously published tag for {repo}; either add "
-                    f"'{ctoken}' to the container selection or drop '{htoken}' from the helm selection")
+                    f"'{ctoken}' to the container selection or drop '{htoken}' from the helm selection"
+                )
         set_helm_values_tag(path, repo, tag)
-    print(f"set_release_version: {old} -> py={new_version} semver={semver} "
-          f"containers={sorted(containers)} helm={sorted(helm)}", file=sys.stderr)
+    print(
+        f"set_release_version: {old} -> py={new_version} semver={semver} "
+        f"containers={sorted(containers)} helm={sorted(helm)}",
+        file=sys.stderr,
+    )
     # Docs / examples / deploy manifests: selection-gated so the release branch never
     # advertises an image tag this release does not publish.
     if image_refs:
@@ -627,18 +709,33 @@ def _require(rel: str, hits: int, want: str) -> None:
 
 def main() -> int:
     ap = argparse.ArgumentParser(description=__doc__)
-    ap.add_argument("suffix", nargs="?", default="", help="e.g. .dev20260423 (empty = no-op)")
+    ap.add_argument(
+        "suffix", nargs="?", default="", help="e.g. .dev20260423 (empty = no-op)"
+    )
     ap.add_argument("root", nargs="?", default=".", help="repo root")
-    ap.add_argument("--set-version", dest="set_version", default="",
-                    help="set an absolute release version X.Y.Z[.devN|.postN] instead of appending a suffix")
-    ap.add_argument("--containers", default="all",
-                    help="normalized container subset (all|none|csv) gating image-tag bumps")
-    ap.add_argument("--helm", default="all",
-                    help="helm chart subset (all|none|platform) gating chart bumps")
-    ap.add_argument("--image-refs", action="store_true",
-                    help="also point the my-registry/my-tag placeholders in docs, examples and "
-                         "deploy manifests at the GA registry + release version — only for images "
-                         "in --containers; unselected images keep their placeholder")
+    ap.add_argument(
+        "--set-version",
+        dest="set_version",
+        default="",
+        help="set an absolute release version X.Y.Z[.devN|.postN] instead of appending a suffix",
+    )
+    ap.add_argument(
+        "--containers",
+        default="all",
+        help="normalized container subset (all|none|csv) gating image-tag bumps",
+    )
+    ap.add_argument(
+        "--helm",
+        default="all",
+        help="helm chart subset (all|none|platform) gating chart bumps",
+    )
+    ap.add_argument(
+        "--image-refs",
+        action="store_true",
+        help="also point the my-registry/my-tag placeholders in docs, examples and "
+        "deploy manifests at the GA registry + release version — only for images "
+        "in --containers; unselected images keep their placeholder",
+    )
     args = ap.parse_args()
 
     root = Path(args.root).resolve()
@@ -646,7 +743,9 @@ def main() -> int:
     if args.set_version:
         containers = _parse_subset(args.containers, CONTAINER_TOKENS)
         helm = _parse_subset(args.helm, HELM_TOKENS)
-        set_release_version(root, args.set_version, containers, helm, image_refs=args.image_refs)
+        set_release_version(
+            root, args.set_version, containers, helm, image_refs=args.image_refs
+        )
         return 0
 
     if not args.suffix:
diff --git a/.github/scripts/stage_crates.py b/.github/scripts/stage_crates.py
old mode 100644
new mode 100755
index 40aecfbde8d2..9c98ae230b4b
--- a/.github/scripts/stage_crates.py
+++ b/.github/scripts/stage_crates.py
@@ -52,7 +52,9 @@
 PUBLISH_TOOLCHAIN = "1.95.0"
 # No bare "HTTP/2" alternation here: cargo dumps the raw status line ("HTTP/2 415")
 # on every HTTP error, which made deterministic 4xx failures retry as transient.
-RETRYABLE = re.compile(r"stream error|INTERNAL_ERROR|connection error|timed out|\b(429|50[0-9])\b")
+RETRYABLE = re.compile(
+    r"stream error|INTERNAL_ERROR|connection error|timed out|\b(429|50[0-9])\b"
+)
 ALREADY = re.compile(r"already exists|409 Conflict|already uploaded")
 # Sparse-index propagation lag after publishing a dep — retried before failing.
 DEP_NOT_INDEXED = re.compile(r"no matching package named")
@@ -61,7 +63,10 @@
 def cargo_metadata(root: Path) -> dict:
     out = subprocess.run(
         ["cargo", "metadata", "--format-version", "1", "--no-deps"],
-        cwd=root, check=True, capture_output=True, text=True,
+        cwd=root,
+        check=True,
+        capture_output=True,
+        text=True,
     ).stdout
     return json.loads(out)
 
@@ -85,8 +90,11 @@ def publishable(meta: dict) -> dict[str, dict]:
 def topo_order(pkgs: dict[str, dict]) -> list[str]:
     names = set(pkgs)
     incoming = {
-        n: {d["name"] for d in pkgs[n]["dependencies"]
-            if d["name"] in names and d["name"] != n and d.get("kind") != "dev"}
+        n: {
+            d["name"]
+            for d in pkgs[n]["dependencies"]
+            if d["name"] in names and d["name"] != n and d.get("kind") != "dev"
+        }
         for n in names
     }
     order: list[str] = []
@@ -101,7 +109,9 @@ def topo_order(pkgs: dict[str, dict]) -> list[str]:
                     ready.append(m)
         ready.sort()
     if len(order) != len(names):
-        raise RuntimeError(f"dependency cycle among crates: {sorted(names - set(order))}")
+        raise RuntimeError(
+            f"dependency cycle among crates: {sorted(names - set(order))}"
+        )
     return order
 
 
@@ -119,7 +129,9 @@ def write_cargo_config(root: Path, alias: str, index: str) -> None:
     # Header-anchored so [registries.x]/[registry] count but comments don't;
     # checked AFTER stripping our block so a re-run can't mask a foreign table.
     if re.search(r"(?m)^\s*\[registr(?:y|ies)[\].]", text):
-        raise RuntimeError(".cargo/config.toml already defines registry tables; refusing to append conflicting ones")
+        raise RuntimeError(
+            ".cargo/config.toml already defines registry tables; refusing to append conflicting ones"
+        )
     cfg.write_text(
         f"{text}\n{marker}\n"
         f'[registries]\n{alias} = {{ index = "{index}" }}\n\n'
@@ -128,7 +140,9 @@ def write_cargo_config(root: Path, alias: str, index: str) -> None:
     )
 
 
-def inject_registry(manifests: list[Path], names: list[str], alias: str, versions: dict[str, str]) -> None:
+def inject_registry(
+    manifests: list[Path], names: list[str], alias: str, versions: dict[str, str]
+) -> None:
     # Internal deps must carry a registry + a version when published, else cargo
     # publish defaults them to crates.io / rejects them. `workspace = true` deps
     # inherit both from the root table (which we also rewrite), so skip them.
@@ -182,27 +196,40 @@ def strip_git_deps(root: Path, manifests: list[Path]) -> None:
     versioned dep redirected by [patch.crates-io] — can't resolve faithfully for
     a registry consumer. Only optional deps are dropped (non-optional is a hard
     error); the feature KEY is kept (emptied) so `/` refs resolve."""
-    git_names = set(re.findall(
-        r'(?m)^\s*([A-Za-z0-9_-]+)\s*=\s*\{[^}\n]*\bgit\s*=',
-        (root / "Cargo.toml").read_text()))
+    git_names = set(
+        re.findall(
+            r"(?m)^\s*([A-Za-z0-9_-]+)\s*=\s*\{[^}\n]*\bgit\s*=",
+            (root / "Cargo.toml").read_text(),
+        )
+    )
     if not git_names:
         return
     for mp in manifests:
         orig = mp.read_text()
         kept, removed = [], set()
         for line in orig.splitlines(keepends=True):
-            m = re.match(r'^[ \t]*([A-Za-z0-9_-]+)[ \t]*=[ \t]*\{(.*)\}[ \t]*$', line)
-            if m and (re.search(r'\bgit\s*=', m.group(2))
-                      or (m.group(1) in git_names and re.search(r'\bworkspace\s*=\s*true', m.group(2)))):
+            m = re.match(r"^[ \t]*([A-Za-z0-9_-]+)[ \t]*=[ \t]*\{(.*)\}[ \t]*$", line)
+            if m and (
+                re.search(r"\bgit\s*=", m.group(2))
+                or (
+                    m.group(1) in git_names
+                    and re.search(r"\bworkspace\s*=\s*true", m.group(2))
+                )
+            ):
                 removed.add(m.group(1))
                 continue
-            if m and m.group(1) in git_names and re.search(r'\bversion\s*=', m.group(2)):
+            if (
+                m
+                and m.group(1) in git_names
+                and re.search(r"\bversion\s*=", m.group(2))
+            ):
                 # Versioned dep git-patched via [patch.crates-io]: droppable only if optional.
-                if not re.search(r'\boptional\s*=\s*true', m.group(2)):
+                if not re.search(r"\boptional\s*=\s*true", m.group(2)):
                     raise RuntimeError(
                         f"{mp}: non-optional dependency '{m.group(1)}' is git-patched in the "
                         "workspace; a registry consumer would resolve different code. "
-                        "Vendor/publish the patched crate or make the dependency optional.")
+                        "Vendor/publish the patched crate or make the dependency optional."
+                    )
                 removed.add(m.group(1))
                 continue
             kept.append(line)
@@ -212,9 +239,9 @@ def strip_git_deps(root: Path, manifests: list[Path]) -> None:
         for nm in removed:
             # scrub feature-array tokens: "dep:nm", "nm", "nm/feat", "nm?/feat"
             text = re.sub(rf'"(?:dep:)?{re.escape(nm)}(?:\?)?(?:/[^"]*)?"', "", text)
-        text = re.sub(r'\[\s*,', "[", text)          # [ , ...   -> [ ...
-        text = re.sub(r',\s*,', ",", text)            # , ,       -> ,
-        text = re.sub(r',(\s*[\]\n])', r'\1', text)   # , ]  / ,\n -> ] / \n
+        text = re.sub(r"\[\s*,", "[", text)  # [ , ...   -> [ ...
+        text = re.sub(r",\s*,", ",", text)  # , ,       -> ,
+        text = re.sub(r",(\s*[\]\n])", r"\1", text)  # , ]  / ,\n -> ] / \n
         mp.write_text(text)
         print(f"strip-git-deps: removed {sorted(removed)} from {mp}")
 
@@ -222,7 +249,9 @@ def strip_git_deps(root: Path, manifests: list[Path]) -> None:
 def crate_exists(raw_base: str, name: str, version: str, token: str) -> bool:
     # 200 -> present, 404 -> absent; anything else (auth/outage) is a hard error.
     url = f"{raw_base}/{name}/{name}-{version}.crate"
-    req = urllib.request.Request(url, method="HEAD", headers={"Authorization": f"Bearer {token}"})
+    req = urllib.request.Request(
+        url, method="HEAD", headers={"Authorization": f"Bearer {token}"}
+    )
     try:
         with urllib.request.urlopen(req, timeout=30) as r:
             return r.status == 200
@@ -231,9 +260,13 @@ def crate_exists(raw_base: str, name: str, version: str, token: str) -> bool:
             return False
         # Path only — the registry host is workflow-secret-derived and error
         # strings end up in public Actions logs.
-        raise RuntimeError(f"registry HEAD {urllib.parse.urlparse(url).path} failed: HTTP {e.code} (bad ARTIFACTORY_TOKEN or registry outage?)") from e
+        raise RuntimeError(
+            f"registry HEAD {urllib.parse.urlparse(url).path} failed: HTTP {e.code} (bad ARTIFACTORY_TOKEN or registry outage?)"
+        ) from e
     except urllib.error.URLError as e:
-        raise RuntimeError(f"registry HEAD {urllib.parse.urlparse(url).path} unreachable: {e.reason}") from e
+        raise RuntimeError(
+            f"registry HEAD {urllib.parse.urlparse(url).path} unreachable: {e.reason}"
+        ) from e
 
 
 def publish(manifest: str, alias: str, env: dict) -> str:
@@ -241,9 +274,20 @@ def publish(manifest: str, alias: str, env: dict) -> str:
     # failure — the caller records it and continues (fail-soft).
     for attempt in range(1, 4):
         r = subprocess.run(
-            ["cargo", f"+{PUBLISH_TOOLCHAIN}", "publish", "--allow-dirty", "--no-verify",
-             "--manifest-path", manifest, "--registry", alias],
-            env=env, capture_output=True, text=True,
+            [
+                "cargo",
+                f"+{PUBLISH_TOOLCHAIN}",
+                "publish",
+                "--allow-dirty",
+                "--no-verify",
+                "--manifest-path",
+                manifest,
+                "--registry",
+                alias,
+            ],
+            env=env,
+            capture_output=True,
+            text=True,
         )
         sys.stdout.write(r.stdout)
         sys.stderr.write(r.stderr)
@@ -254,7 +298,9 @@ def publish(manifest: str, alias: str, env: dict) -> str:
             print(f"  -> {manifest}: skip (already present)")
             return "already"
         if attempt < 3 and DEP_NOT_INDEXED.search(out):
-            print(f"  -> dep not yet indexed (sparse-index lag?), retry in 15s ({attempt}/3)")
+            print(
+                f"  -> dep not yet indexed (sparse-index lag?), retry in 15s ({attempt}/3)"
+            )
             time.sleep(15)
             continue
         if attempt < 3 and RETRYABLE.search(out):
@@ -269,23 +315,35 @@ def main() -> int:
     ap = argparse.ArgumentParser(description=__doc__)
     ap.add_argument("--registry", default="artifactory", help="cargo registry alias")
     ap.add_argument("--root", default=".", help="repo root")
-    ap.add_argument("--expect-version", default=os.environ.get("EXPECT_VERSION", ""),
-                    help="fail fast if any publishable crate's version differs from this")
-    ap.add_argument("--only", default=os.environ.get("CRATE_SUBSET", ""),
-                    help="csv of crate names to publish (strict: must be dependency-closed); "
-                         "empty/'all' = every publishable crate")
-    ap.add_argument("--stage-version", default=os.environ.get("STAGE_VERSION", ""),
-                    help="rewrite every publishable crate's version to this exact value before "
-                         "publishing (internal staging rc disambiguator, e.g. 0.1.0-rc0), so a "
-                         "re-stage of the same release doesn't collide on the immutable registry. "
-                         "Empty = leave the committed versions as-is")
+    ap.add_argument(
+        "--expect-version",
+        default=os.environ.get("EXPECT_VERSION", ""),
+        help="fail fast if any publishable crate's version differs from this",
+    )
+    ap.add_argument(
+        "--only",
+        default=os.environ.get("CRATE_SUBSET", ""),
+        help="csv of crate names to publish (strict: must be dependency-closed); "
+        "empty/'all' = every publishable crate",
+    )
+    ap.add_argument(
+        "--stage-version",
+        default=os.environ.get("STAGE_VERSION", ""),
+        help="rewrite every publishable crate's version to this exact value before "
+        "publishing (internal staging rc disambiguator, e.g. 0.1.0-rc0), so a "
+        "re-stage of the same release doesn't collide on the immutable registry. "
+        "Empty = leave the committed versions as-is",
+    )
     args = ap.parse_args()
 
     # ARTIFACTORY ONLY — never crates.io. Refuse a crates.io alias outright; every
     # `cargo publish` below passes --registry  and write_cargo_config
     # sets it as the default registry, so crates.io is never a publish target.
     if args.registry in ("crates-io", "crates.io", "crates_io"):
-        print("::error::refusing to publish to crates.io; this stages to Artifactory only", file=sys.stderr)
+        print(
+            "::error::refusing to publish to crates.io; this stages to Artifactory only",
+            file=sys.stderr,
+        )
         return 1
 
     # Reuse the wheel-upload token; keep the registry location out of the workflow
@@ -293,11 +351,17 @@ def main() -> int:
     token = os.environ.get("ARTIFACTORY_TOKEN", "")
     index = os.environ.get("ARTIFACTORY_CARGO_INDEX", "")
     if not token or not index:
-        print("::error::ARTIFACTORY_TOKEN and ARTIFACTORY_CARGO_INDEX must be set", file=sys.stderr)
+        print(
+            "::error::ARTIFACTORY_TOKEN and ARTIFACTORY_CARGO_INDEX must be set",
+            file=sys.stderr,
+        )
         return 1
     m = re.match(r"sparse\+(https://.+?)/api/cargo/([^/]+)/index/?$", index)
     if not m:
-        print("::error::ARTIFACTORY_CARGO_INDEX must be 'sparse+https:///.../api/cargo//index/'", file=sys.stderr)
+        print(
+            "::error::ARTIFACTORY_CARGO_INDEX must be 'sparse+https:///.../api/cargo//index/'",
+            file=sys.stderr,
+        )
         return 1
     raw_base = f"{m.group(1)}/{m.group(2)}/crates"
 
@@ -325,20 +389,40 @@ def main() -> int:
         # checked on what actually exists here.
         unknown = selected - set(pkgs)
         if unknown:
-            print(f"::warning::skipping requested crate(s) not publishable on this branch: "
-                  f"{sorted(unknown)} (branch publishable set: {sorted(pkgs)})", file=sys.stderr)
+            print(
+                f"::warning::skipping requested crate(s) not publishable on this branch: "
+                f"{sorted(unknown)} (branch publishable set: {sorted(pkgs)})",
+                file=sys.stderr,
+            )
             selected -= unknown
         if not selected:
-            print("::error::none of the requested crates are publishable on this branch", file=sys.stderr)
+            print(
+                "::error::none of the requested crates are publishable on this branch",
+                file=sys.stderr,
+            )
             return 1
-        gaps = {n: sorted({d["name"] for d in pkgs[n]["dependencies"]
-                           if d["name"] in pkgs and d["name"] != n and d.get("kind") != "dev"} - selected)
-                for n in selected}
+        gaps = {
+            n: sorted(
+                {
+                    d["name"]
+                    for d in pkgs[n]["dependencies"]
+                    if d["name"] in pkgs and d["name"] != n and d.get("kind") != "dev"
+                }
+                - selected
+            )
+            for n in selected
+        }
         gaps = {n: g for n, g in gaps.items() if g}
         if gaps:
             for n, g in gaps.items():
-                print(f"::error::crate {n} depends on un-selected workspace crate(s) {g}", file=sys.stderr)
-            print("::error::crate subset is not dependency-closed; add the missing crate(s) or use 'all'", file=sys.stderr)
+                print(
+                    f"::error::crate {n} depends on un-selected workspace crate(s) {g}",
+                    file=sys.stderr,
+                )
+            print(
+                "::error::crate subset is not dependency-closed; add the missing crate(s) or use 'all'",
+                file=sys.stderr,
+            )
             return 1
         order = [n for n in order if n in selected]
         print("Selected crates (dependency-closed):", " ".join(order))
@@ -349,22 +433,34 @@ def main() -> int:
     # then validates the new value, and cargo publish uploads it (--allow-dirty).
     stage_version = args.stage_version.strip()
     if stage_version:
-        wm = re.search(r'\[workspace\.package\][^\[]*?\n\s*version\s*=\s*"([^"]+)"',
-                       (root / "Cargo.toml").read_text())
+        wm = re.search(
+            r'\[workspace\.package\][^\[]*?\n\s*version\s*=\s*"([^"]+)"',
+            (root / "Cargo.toml").read_text(),
+        )
         if not wm:
-            print("::error::--stage-version: cannot read [workspace.package].version", file=sys.stderr)
+            print(
+                "::error::--stage-version: cannot read [workspace.package].version",
+                file=sys.stderr,
+            )
             return 1
         cur = wm.group(1)
         # STAGE_VERSION's base comes from pyproject.toml (release.yml); nothing
         # else gates pyproject against Cargo.toml, so a bump that missed one
         # file would otherwise publish crates under the wrong base silently.
         if not (stage_version == cur or stage_version.startswith(f"{cur}-")):
-            print(f"::warning::stage-version '{stage_version}' does not extend the "
-                  f"workspace version '{cur}' — pyproject.toml and Cargo.toml may "
-                  f"have drifted", file=sys.stderr)
+            print(
+                f"::warning::stage-version '{stage_version}' does not extend the "
+                f"workspace version '{cur}' — pyproject.toml and Cargo.toml may "
+                f"have drifted",
+                file=sys.stderr,
+            )
         if cur != stage_version:
-            n = rewrite_versions([root / "Cargo.toml", *member_manifests], cur, stage_version)
-            print(f"stage-version: rewrote {cur} -> {stage_version} across {n} manifest(s)")
+            n = rewrite_versions(
+                [root / "Cargo.toml", *member_manifests], cur, stage_version
+            )
+            print(
+                f"stage-version: rewrote {cur} -> {stage_version} across {n} manifest(s)"
+            )
             for nm in pkgs:
                 if pkgs[nm]["version"] == cur:
                     pkgs[nm]["version"] = stage_version
@@ -372,18 +468,30 @@ def main() -> int:
     # Fail fast BEFORE any build/publish if a crate carries an unexpected version
     # (e.g. a hardcoded version the bump missed) — never silently publish wrong tags.
     if args.expect_version:
-        mismatched = [(n, pkgs[n]["version"]) for n in order if pkgs[n]["version"] != args.expect_version]
+        mismatched = [
+            (n, pkgs[n]["version"])
+            for n in order
+            if pkgs[n]["version"] != args.expect_version
+        ]
         if mismatched:
             for n, v in mismatched:
-                print(f"::error::crate {n} is at version {v}, expected {args.expect_version} (hardcoded/un-bumped version)", file=sys.stderr)
-            print(f"::error::aborting before publish: {len(mismatched)} crate(s) carry an unexpected version; "
-                  "set 'version.workspace = true' (or fix Cargo.toml) and re-run", file=sys.stderr)
+                print(
+                    f"::error::crate {n} is at version {v}, expected {args.expect_version} (hardcoded/un-bumped version)",
+                    file=sys.stderr,
+                )
+            print(
+                f"::error::aborting before publish: {len(mismatched)} crate(s) carry an unexpected version; "
+                "set 'version.workspace = true' (or fix Cargo.toml) and re-run",
+                file=sys.stderr,
+            )
             return 1
 
     write_cargo_config(root, args.registry, index)
     inject_registry(
         [root / "Cargo.toml", *(Path(pkgs[n]["manifest_path"]) for n in order)],
-        order, args.registry, {n: pkgs[n]["version"] for n in order},
+        order,
+        args.registry,
+        {n: pkgs[n]["version"] for n in order},
     )
     # cargo publish rejects deps without a registry version; drop optional,
     # private git deps from the crates being published (none on current main;
@@ -395,18 +503,33 @@ def main() -> int:
     env["RUSTFLAGS"] = f"{env.get('RUSTFLAGS', '')} --cfg tokio_unstable".strip()
     # Only the upload PUT runs on PUBLISH_TOOLCHAIN (--no-verify skips compilation);
     # cargo check/update stay on the workspace's rust-toolchain.toml toolchain.
-    r = subprocess.run(["rustup", "toolchain", "install", "--profile", "minimal",
-                        "--no-self-update", PUBLISH_TOOLCHAIN], env=env)
+    r = subprocess.run(
+        [
+            "rustup",
+            "toolchain",
+            "install",
+            "--profile",
+            "minimal",
+            "--no-self-update",
+            PUBLISH_TOOLCHAIN,
+        ],
+        env=env,
+    )
     if r.returncode != 0:
-        print(f"::error::rustup install of publish toolchain {PUBLISH_TOOLCHAIN} failed",
-              file=sys.stderr)
+        print(
+            f"::error::rustup install of publish toolchain {PUBLISH_TOOLCHAIN} failed",
+            file=sys.stderr,
+        )
         return 1
     # Resync Cargo.lock after the stage-version rewrite; fail loudly here rather
     # than as a confusing publish error later.
     if stage_version:
         r = subprocess.run(["cargo", "update", "--workspace"], cwd=root, env=env)
         if r.returncode != 0:
-            print("::error::cargo update --workspace failed after the stage-version rewrite", file=sys.stderr)
+            print(
+                "::error::cargo update --workspace failed after the stage-version rewrite",
+                file=sys.stderr,
+            )
             return 1
 
     # Fail-soft loop: a failed crate skips its dependents but not the rest.
@@ -417,8 +540,11 @@ def main() -> int:
     staged: list[str] = []
     failed: dict[str, str] = {}
     deps_of = {
-        n: {d["name"] for d in pkgs[n]["dependencies"]
-            if d["name"] in pkgs and d["name"] != n and d.get("kind") != "dev"}
+        n: {
+            d["name"]
+            for d in pkgs[n]["dependencies"]
+            if d["name"] in pkgs and d["name"] != n and d.get("kind") != "dev"
+        }
         for n in order
     }
     for name in order:
@@ -438,7 +564,12 @@ def main() -> int:
                 staged.append(name)
                 print(f"STAGED_CRATE={name}", flush=True)  # captured by the staging job
                 continue
-            if subprocess.run(["cargo", "check", "--manifest-path", manifest], cwd=root, env=env).returncode != 0:
+            if (
+                subprocess.run(
+                    ["cargo", "check", "--manifest-path", manifest], cwd=root, env=env
+                ).returncode
+                != 0
+            ):
                 raise RuntimeError(f"cargo check failed for {manifest}")
             status = publish(manifest, args.registry, env)
         except RuntimeError as e:
@@ -453,12 +584,17 @@ def main() -> int:
         staged.append(name)
         print(f"STAGED_CRATE={name}", flush=True)  # captured by the staging job
 
-    print(f"Done: {len(order)} crates ({published} published, {already} already present, "
-          f"{len(staged)} staged, {len(failed)} failed).")
+    print(
+        f"Done: {len(order)} crates ({published} published, {already} already present, "
+        f"{len(staged)} staged, {len(failed)} failed)."
+    )
     print(f"STAGED_CRATES={','.join(staged)}")
     if failed:
         print(f"FAILED_CRATES={','.join(failed)}")
-        print(f"::error::{len(failed)} crate(s) not staged: {', '.join(failed)}", file=sys.stderr)
+        print(
+            f"::error::{len(failed)} crate(s) not staged: {', '.join(failed)}",
+            file=sys.stderr,
+        )
         return 1
     return 0