diff --git a/.github/workflows/fern-docs.yml b/.github/workflows/fern-docs.yml index bec1ace08287..056d56fc1243 100644 --- a/.github/workflows/fern-docs.yml +++ b/.github/workflows/fern-docs.yml @@ -34,6 +34,11 @@ # Note: The publish step is included inline because pushes made with GITHUB_TOKEN # do not trigger other workflows (GitHub's anti-recursion guard), so we cannot # rely on a separate publish-fern-docs.yml workflow for bot-initiated pushes. +# +# Validation: the sync/release composition (rsync scopes, nav path transforms, +# the shared-Reference machinery) only executes on main pushes and tag cuts. +# Before changing it, replay both jobs locally against the docs-website branch: +# docs/fern/scripts/simulate_docs_website.sh name: Fern Docs @@ -149,6 +154,40 @@ jobs: --exclude='watch.sh' \ source-checkout/docs/fern/ docs-checkout/fern/pages-dev/ + # Component *doc* pages (.md/.mdx) live alongside the React .tsx under + # docs/fern/components/. The main rsync excludes components/ wholesale + # so the .tsx don't double-sync into pages-dev (they go to + # fern/components/ for the docs.yml mdx-components + footer + # resolution). But the nav references component doc pages from both + # the Developer Guide tab and the Reference tab's Components variant, + # and the path transform below rewrites those to + # ../pages-dev/components/. Sync ONLY the doc pages here — never the + # .tsx — so the paths resolve and the per-version snapshot + # (cp -r pages-dev) freezes them correctly. + echo "Syncing component doc pages (.md/.mdx only) to pages-dev/components/..." + rsync -a \ + --include='*/' \ + --include='*.md' \ + --include='*.mdx' \ + --exclude='*' \ + --prune-empty-dirs \ + source-checkout/docs/fern/components/ docs-checkout/fern/pages-dev/components/ + + # Backend deploy manifests are embedded into templates/{vllm,sglang, + # trtllm}.mdx via build-time file reads. Those resolve to the repo-root + # examples/ tree in the source layout, which does not exist on + # docs-website — sync just the referenced deploy/ subtrees to the + # branch root so both ../../../examples (from pages-dev) and the + # versioned snapshots resolve. + echo "Syncing examples/backends/*/deploy/ to branch root..." + rsync -a \ + --include='*/' \ + --include='backends/*/deploy/**' \ + --exclude='*' \ + --prune-empty-dirs \ + source-checkout/examples/ docs-checkout/examples/ + # Sync index.yml as versions/dev.yml and transform paths for docs-website layout echo "Syncing index.yml to docs-website branch as versions/dev.yml..." cp source-checkout/docs/fern/index.yml docs-checkout/fern/versions/dev.yml @@ -184,6 +223,15 @@ jobs: cp -r source-checkout/docs/fern/products docs-checkout/fern/products cp source-checkout/docs/fern/welcome.mdx docs-checkout/fern/welcome.mdx + # Sync root-level assets/ (docs.yml logos/fonts and welcome.mdx reference + # ./assets/ relative to the fern root, not pages-dev). Merge-copy without + # deleting so assets still referenced by older versioned pages survive + # even if they are later removed from the source tree. + if [ -d source-checkout/docs/fern/assets ]; then + echo "Syncing assets/ to docs-website branch..." + cp -r source-checkout/docs/fern/assets/. docs-checkout/fern/assets/ + fi + # Community pages move under pages-dev on docs-website. Home and Blog # continue to reference root-level welcome.mdx and digest/, respectively. yq -i '(.. | select(has("path")).path) |= sub("^../", "../pages-dev/")' \ @@ -195,6 +243,10 @@ jobs: # Keep fern/blogs for older versioned docs that still reference ../blogs. rm -rf docs-checkout/fern/digest cp -r source-checkout/docs/fern/digest docs-checkout/fern/digest + # The digest posts were renamed .md -> .mdx at the source; retarget the + # release-managed versions/v*.yml snapshots that still point at the old + # extension. Harmless no-op once every snapshot references .mdx. + sed -i 's|\(path: \.\./digest/.*\)\.md$|\1.mdx|' docs-checkout/fern/versions/v*.yml fi # Sync main.css @@ -220,6 +272,39 @@ jobs: yq -i '(.. | select(has("path")).path) |= sub("^digest/", "../digest/")' docs-checkout/fern/versions/dev.yml yq -i '(.. | select(has("path")).path) |= sub("^([a-zA-Z])", "../pages-dev/${1}")' docs-checkout/fern/versions/dev.yml + - name: Propagate shared Reference nav to released versions + run: | + # The Reference tab's General variant (Compatibility, Release + # Artifacts, Releases, Known Issues, Deprecations, Model EA Builds, + # Glossary) is SHARED across versions: release snapshots keep its nav + # paths on ../pages-dev/ so every version dropdown renders the + # always-current reference. When a page is added to the shared + # reference on main (e.g. a new release-notes page), copy the + # variant's nav block from dev.yml into each released version's yml + # so the new page appears in every dropdown, not just dev. Version + # snapshots cut before the reference rework have no such variant and + # are left untouched (the yq selection matches nothing). + yq '[.navigation[] | select(.tab == "reference") | .variants[] | select(.title == "General")][0]' \ + docs-checkout/fern/versions/dev.yml > /tmp/reference_general_variant.yml + + if [ "$(yq 'length' /tmp/reference_general_variant.yml)" = "0" ] || \ + [ "$(head -c4 /tmp/reference_general_variant.yml)" = "null" ]; then + echo "No shared Reference General variant in dev.yml; skipping propagation" + else + for vfile in docs-checkout/fern/versions/v*.yml; do + [ -e "$vfile" ] || continue + # Only rewrite files that actually carry the shared variant — + # yq -i normalizes whitespace, so touching pre-rework snapshots + # (which have no reference General variant) is pure churn. + if [ "$(yq '[.navigation[] | select(.tab == "reference") | .variants[] | select(.title == "General")] | length' "$vfile")" != "0" ]; then + yq -i '(.navigation[] | select(.tab == "reference") | .variants[] | select(.title == "General")) = load("/tmp/reference_general_variant.yml")' "$vfile" + echo "Synced shared Reference nav into $vfile" + else + echo "Skipped (no shared Reference variant): $vfile" + fi + done + fi + - name: Convert GitHub callouts to Fern format run: | echo "Converting GitHub-style callouts to Fern format in pages/..." @@ -426,6 +511,24 @@ jobs: # Copy current pages-dev/ to pages-vX.Y.Z/ cp -r fern/pages-dev "fern/pages-$TAG" + # The Reference tab's General variant is shared (always-current) + # across versions: its nav paths stay on ../pages-dev/ (see the + # version-config step), so drop exactly the files that variant + # references from the snapshot rather than freezing stale copies. + # Everything else under reference/ (observability pages, config + # references) belongs to versioned tabs and stays in the snapshot. + # If the selectors stop matching (tab or variant renamed), this and + # the version-config revert silently no-op and the reference quietly + # freezes per version again — warn loudly so the rename gets fixed. + if [ "$(yq '[.navigation[] | select(.tab == "reference") | .variants[] | select(.title == "General")] | length' fern/versions/dev.yml)" = "0" ]; then + echo "::warning::No Reference General variant found in dev.yml — the shared-reference exclusion is a no-op and $TAG will freeze its own reference copy. If the tab or variant was renamed, update the yq selectors in this workflow." + fi + yq '.navigation[] | select(.tab == "reference") | .variants[] | select(.title == "General") | .. | select(has("path")) | .path' \ + fern/versions/dev.yml | sed 's|^\.\./pages-dev/||' | while read -r relpath; do + [ -n "$relpath" ] && rm -f "fern/pages-$TAG/$relpath" + done + find "fern/pages-$TAG/reference" -type d -empty -delete 2>/dev/null || true + echo "Created fern/pages-$TAG/" ls -la "fern/pages-$TAG/" | head -20 @@ -479,6 +582,14 @@ jobs: # Update all page paths from ../pages-dev/ to ../pages-vX.Y.Z/ sed -i "s|path: \.\./pages-dev/|path: ../pages-$TAG/|g" "$VERSION_FILE" + # Revert the Reference tab's General variant to the shared source: + # it is always-current across versions (release metadata is + # cumulative — each page carries per-release sections), so its pages + # render from pages-dev in every version. The Kubernetes API and + # Components variants stay on the frozen snapshot (CRD fields and + # config flags are genuinely per-version). + yq -i "(.navigation[] | select(.tab == \"reference\") | .variants[] | select(.title == \"General\") | .. | select(has(\"path\")).path) |= sub(\"\.\./pages-$TAG/\", \"../pages-dev/\")" "$VERSION_FILE" + # Keep cross-navigation links within the selected documentation version. sed -i "s|href: /dynamo/dev/|href: /dynamo/$TAG/|g" "$VERSION_FILE" diff --git a/docs/fern/assets/releases-atom.xml b/docs/fern/assets/releases-atom.xml new file mode 100644 index 000000000000..71c08fbab320 --- /dev/null +++ b/docs/fern/assets/releases-atom.xml @@ -0,0 +1,165 @@ + + + + NVIDIA Dynamo releases + https://docs.nvidia.com/dynamo/dev/reference/releases/release-history + + 2026-07-20T00:00:00Z + NVIDIA Dynamo + + Dynamo v1.3.0 (stable) + https://github.com/ai-dynamo/dynamo/releases/tag/v1.3.0 + + 2026-07-20T00:00:00Z + Tool-calling and reasoning overhaul, RL rollout serving, the largest Router buildout to date, SLA-driven Planner autoscaling, and production GPU Memory Service on Kubernetes. + + + Dynamo v1.2.1 (patch) + https://github.com/ai-dynamo/dynamo/releases/tag/v1.2.1 + + 2026-06-13T00:00:00Z + Patch release. Same backend pins as v1.2.0. + + + Dynamo v1.3.0-dev.1 (platform-preview) + https://github.com/ai-dynamo/dynamo/releases/tag/v1.3.0-dev.1 + + 2026-06-09T00:00:00Z + Full-platform preview of v1.3.0: complete runtime matrix, wheels on pypi.nvidia.com, crates, and Helm charts. Superseded by v1.3.0 GA. + + + Dynamo v1.2.0 (stable) + https://github.com/ai-dynamo/dynamo/releases/tag/v1.2.0 + + 2026-06-02T00:00:00Z + DGD/DGDR v1beta1, CRTC as the default KV router, inter-pod GPU Memory Service, Dynamo Snapshot on CRI-O/OpenShift, and DeepSeek-V4 recipes on vLLM. + + + Dynamo v1.2.0-deepseek-v4-dev.3 (model-build) + https://github.com/ai-dynamo/dynamo/releases/tag/v1.2.0-deepseek-v4-dev.3 + + 2026-05-09T00:00:00Z + DeepSeek-V4 Blackwell preview; vLLM + SGLang containers only. + + + Dynamo v1.1.1 (patch) + https://github.com/ai-dynamo/dynamo/releases/tag/v1.1.1 + + 2026-05-05T00:00:00Z + Patch release. Same backend pins as v1.1.0. + + + Dynamo v1.2.0-deepseek-v4-dev.2 (model-build) + https://github.com/ai-dynamo/dynamo/releases/tag/v1.2.0-deepseek-v4-dev.2 + + 2026-05-01T00:00:00Z + DeepSeek-V4 Blackwell preview; vLLM + SGLang containers only. + + + Dynamo v1.1.0 (stable) + https://github.com/ai-dynamo/dynamo/releases/tag/v1.1.0 + + 2026-05-01T00:00:00Z + Resilient KV routing at scale, Anthropic Messages API support, performance modeling and offline replay, and the multimodal embedding cache. + + + Dynamo v1.0.2 (patch) + https://github.com/ai-dynamo/dynamo/releases/tag/v1.0.2 + + 2026-04-22T00:00:00Z + No artifact additions or removals versus v1.0.0. + + + Dynamo v1.1.0-dev.3 (platform-preview) + https://github.com/ai-dynamo/dynamo/releases/tag/v1.1.0-dev.3 + + 2026-04-18T00:00:00Z + Partial platform preview: TRT-LLM runtime image + wheels only. + + + Dynamo v1.1.0-dev.2 (platform-preview) + https://github.com/ai-dynamo/dynamo/releases/tag/v1.1.0-dev.2 + + 2026-04-09T00:00:00Z + Partial platform preview: SGLang + TRT-LLM runtime images + wheels. + + + Dynamo v1.1.0-dev.1 (platform-preview) + https://github.com/ai-dynamo/dynamo/releases/tag/v1.1.0-dev.1 + + 2026-03-17T00:00:00Z + Platform preview: runtime matrix, wheels on pypi.nvidia.com, Helm charts. + + + Dynamo v1.0.1 (patch) + https://github.com/ai-dynamo/dynamo/releases/tag/v1.0.1 + + 2026-03-16T00:00:00Z + No artifact additions or removals versus v1.0.0. + + + Dynamo v1.0.0 (stable) + https://github.com/ai-dynamo/dynamo/releases/tag/v1.0.0 + + 2026-03-12T00:00:00Z + First GA release: unified configuration, Kubernetes production readiness, multimodal serving, and the agents surface. + + + Dynamo v0.9.1 (patch) + https://github.com/ai-dynamo/dynamo/releases/tag/v0.9.1 + + 2026-03-04T00:00:00Z + No artifact additions or removals versus v0.9.0. + + + Dynamo v0.9.0 (stable) + https://github.com/ai-dynamo/dynamo/releases/tag/v0.9.0 + + 2026-02-11T00:00:00Z + First publish of dynamo-tokens crate. Deprecated dynamo-graph Helm chart dropped from the publish stream. + + + Dynamo v0.8.1 (patch) + https://github.com/ai-dynamo/dynamo/releases/tag/v0.8.1 + + 2026-01-23T00:00:00Z + Post trains .post1/.post2/.post3 republished the TRT-LLM runtime image and PyPI wheels only. + + + Dynamo v0.8.0 (stable) + https://github.com/ai-dynamo/dynamo/releases/tag/v0.8.0 + + 2026-01-15T00:00:00Z + dynamo-frontend image and CUDA 13 variants for vLLM and SGLang. First publish of dynamo-memory and dynamo-config crates. + + + Dynamo v0.7.1 (patch) + https://github.com/ai-dynamo/dynamo/releases/tag/v0.7.1 + + 2025-12-15T00:00:00Z + + + Dynamo v0.7.0 (stable) + https://github.com/ai-dynamo/dynamo/releases/tag/v0.7.0 + + 2025-11-26T00:00:00Z + + + Dynamo v0.6.1 (patch) + https://github.com/ai-dynamo/dynamo/releases/tag/v0.6.1 + + 2025-11-06T00:00:00Z + + + Dynamo v0.6.0 (stable) + https://github.com/ai-dynamo/dynamo/releases/tag/v0.6.0 + + 2025-10-28T00:00:00Z + Oldest release tracked on this page. + + diff --git a/docs/fern/assets/releases.json b/docs/fern/assets/releases.json new file mode 100644 index 000000000000..0c27ff331d61 --- /dev/null +++ b/docs/fern/assets/releases.json @@ -0,0 +1,1716 @@ +{ + "source": "docs/fern/components/releases.data.ts", + "generator": "docs/fern/scripts/gen_llms_tables.py", + "updated": "2026-07-20", + "current": { + "version": "v1.3.0", + "date": "Jul 20, 2026", + "dateIso": "2026-07-20", + "tag": "1.3.0", + "wheel": "1.3.0.post1" + }, + "mainTot": { + "sglang": "0.5.15", + "trtllm": "1.3.0rc21", + "vllm": "0.25.1", + "nixlSglang": "1.3.0", + "nixlTrtllm": "1.0.1", + "nixlVllm": "1.1.0" + }, + "releases": [ + { + "version": "v1.3.0", + "notesHref": "/dynamo/dev/reference/releases/v1-3-0", + "date": "Jul 20, 2026", + "kind": "stable", + "github": "https://github.com/ai-dynamo/dynamo/releases/tag/v1.3.0", + "docs": "https://docs.nvidia.com/dynamo", + "pins": { + "sglang": "0.5.14", + "trtllm": "1.3.0rc19", + "vllm": "0.23.0", + "nixlSglang": "1.3.0", + "nixlTrtllm": "1.0.1", + "nixlVllm": "1.1.0" + }, + "ucx": "1.20.x", + "delta": "CUDA 12 container images discontinued; EFA variants go multi-arch as -efa; GA wheels published as 1.3.0.post1 (containers stay :1.3.0); UCX 1.20.x.", + "notesSummary": "Tool-calling and reasoning overhaul, RL rollout serving, the largest Router buildout to date, SLA-driven Planner autoscaling, and production GPU Memory Service on Kubernetes.", + "dateIso": "2026-07-20", + "notesUrl": "https://docs.nvidia.com/dynamo/dev/reference/releases/v1-3-0" + }, + { + "version": "v1.3.0-dev.1", + "date": "Jun 9, 2026", + "kind": "platform-preview", + "github": "https://github.com/ai-dynamo/dynamo/releases/tag/v1.3.0-dev.1", + "pins": { + "sglang": "0.5.12.post1", + "trtllm": "1.3.0rc17", + "vllm": "0.22.0", + "nixlSglang": "1.0.1", + "nixlTrtllm": "0.10.1", + "nixlVllm": "1.1.0" + }, + "delta": "Full-platform preview of v1.3.0: complete runtime matrix, wheels on pypi.nvidia.com, crates, and Helm charts. Superseded by v1.3.0 GA.", + "dateIso": "2026-06-09", + "notesUrl": "https://github.com/ai-dynamo/dynamo/releases/tag/v1.3.0-dev.1" + }, + { + "version": "v1.2.1", + "notesHref": "/dynamo/dev/reference/releases/v1-2-0", + "date": "Jun 13, 2026", + "kind": "patch", + "github": "https://github.com/ai-dynamo/dynamo/releases/tag/v1.2.1", + "docs": "https://docs.nvidia.com/dynamo", + "pins": { + "sglang": "0.5.11", + "trtllm": "1.3.0rc14", + "vllm": "0.20.1", + "nixlSglang": "1.0.1", + "nixlTrtllm": "0.10.1", + "nixlVllm": "0.10.1" + }, + "delta": "Patch release. Same backend pins as v1.2.0.", + "dateIso": "2026-06-13", + "notesUrl": "https://docs.nvidia.com/dynamo/dev/reference/releases/v1-2-0" + }, + { + "version": "v1.2.0", + "notesHref": "/dynamo/dev/reference/releases/v1-2-0", + "date": "Jun 2, 2026", + "kind": "stable", + "github": "https://github.com/ai-dynamo/dynamo/releases/tag/v1.2.0", + "docs": "https://docs.nvidia.com/dynamo", + "pins": { + "sglang": "0.5.11", + "trtllm": "1.3.0rc14", + "vllm": "0.20.1", + "nixlSglang": "1.0.1", + "nixlTrtllm": "0.10.1", + "nixlVllm": "0.10.1" + }, + "ucx": "1.20.0", + "delta": "603 PRs from 82 authors. DGD/DGDR promoted to v1beta1; CRTC default approximate KV router; inter-pod GMS sidecar; Dynamo Snapshot on CRI-O / OpenShift; UCX 1.20.0.", + "notesSummary": "DGD/DGDR v1beta1, CRTC as the default KV router, inter-pod GPU Memory Service, Dynamo Snapshot on CRI-O/OpenShift, and DeepSeek-V4 recipes on vLLM.", + "dateIso": "2026-06-02", + "notesUrl": "https://docs.nvidia.com/dynamo/dev/reference/releases/v1-2-0" + }, + { + "version": "v1.2.0-deepseek-v4-dev.3", + "date": "May 9, 2026", + "kind": "model-build", + "github": "https://github.com/ai-dynamo/dynamo/releases/tag/v1.2.0-deepseek-v4-dev.3", + "pins": { + "sglang": "upstream DSv4 preview", + "vllm": "0.20.1", + "nixlVllm": "0.10.1" + }, + "partial": true, + "note": "DeepSeek-V4 Blackwell preview; vLLM + SGLang containers only.", + "dateIso": "2026-05-09", + "notesUrl": "https://github.com/ai-dynamo/dynamo/releases/tag/v1.2.0-deepseek-v4-dev.3" + }, + { + "version": "v1.2.0-deepseek-v4-dev.2", + "date": "May 1, 2026", + "kind": "model-build", + "github": "https://github.com/ai-dynamo/dynamo/releases/tag/v1.2.0-deepseek-v4-dev.2", + "pins": { + "sglang": "upstream DSv4 preview", + "vllm": "0.20.0", + "nixlVllm": "0.10.1" + }, + "partial": true, + "note": "DeepSeek-V4 Blackwell preview; vLLM + SGLang containers only.", + "dateIso": "2026-05-01", + "notesUrl": "https://github.com/ai-dynamo/dynamo/releases/tag/v1.2.0-deepseek-v4-dev.2" + }, + { + "version": "v1.1.1", + "notesHref": "/dynamo/dev/reference/releases/v1-1-0", + "date": "May 5, 2026", + "kind": "patch", + "github": "https://github.com/ai-dynamo/dynamo/releases/tag/v1.1.1", + "docs": "https://docs.nvidia.com/dynamo", + "pins": { + "sglang": "0.5.10.post1", + "trtllm": "1.3.0rc11", + "vllm": "0.19.0", + "nixlSglang": "1.0.1", + "nixlTrtllm": "0.10.1", + "nixlVllm": "0.10.1" + }, + "delta": "Patch release. Same backend pins as v1.1.0.", + "dateIso": "2026-05-05", + "notesUrl": "https://docs.nvidia.com/dynamo/dev/reference/releases/v1-1-0" + }, + { + "version": "v1.1.0", + "notesHref": "/dynamo/dev/reference/releases/v1-1-0", + "date": "May 1, 2026", + "kind": "stable", + "github": "https://github.com/ai-dynamo/dynamo/releases/tag/v1.1.0", + "docs": "https://docs.nvidia.com/dynamo", + "pins": { + "sglang": "0.5.10.post1", + "trtllm": "1.3.0rc11", + "vllm": "0.19.0", + "nixlSglang": "1.0.1", + "nixlTrtllm": "0.10.1", + "nixlVllm": "0.10.1" + }, + "ucx": "1.20", + "delta": "Planner split into its own dynamo-planner image (artifact boundary change). First 1.y.z publication of dynamo-protocols on crates.io; dynamo-async-openai deprecated at final 1.0.2.", + "notesSummary": "Resilient KV routing at scale, Anthropic Messages API support, performance modeling and offline replay, and the multimodal embedding cache.", + "dateIso": "2026-05-01", + "notesUrl": "https://docs.nvidia.com/dynamo/dev/reference/releases/v1-1-0" + }, + { + "version": "v1.1.0-dev.3", + "date": "Apr 18, 2026", + "kind": "platform-preview", + "github": "https://github.com/ai-dynamo/dynamo/releases/tag/v1.1.0-dev.3", + "pins": { + "sglang": "0.5.10.post1", + "trtllm": "1.3.0rc11", + "vllm": "0.19.0", + "nixlSglang": "1.0.1", + "nixlTrtllm": "0.10.1", + "nixlVllm": "0.10.1" + }, + "partial": true, + "note": "Partial platform preview: TRT-LLM runtime image + wheels only.", + "dateIso": "2026-04-18", + "notesUrl": "https://github.com/ai-dynamo/dynamo/releases/tag/v1.1.0-dev.3" + }, + { + "version": "v1.1.0-dev.2", + "date": "Apr 9, 2026", + "kind": "platform-preview", + "github": "https://github.com/ai-dynamo/dynamo/releases/tag/v1.1.0-dev.2", + "pins": { + "sglang": "0.5.9", + "trtllm": "1.3.0rc9", + "vllm": "0.19.0", + "nixlSglang": "1.0.1", + "nixlTrtllm": "0.10.1", + "nixlVllm": "0.10.1" + }, + "partial": true, + "note": "Partial platform preview: SGLang + TRT-LLM runtime images + wheels.", + "dateIso": "2026-04-09", + "notesUrl": "https://github.com/ai-dynamo/dynamo/releases/tag/v1.1.0-dev.2" + }, + { + "version": "v1.1.0-dev.1", + "date": "Mar 17, 2026", + "kind": "platform-preview", + "github": "https://github.com/ai-dynamo/dynamo/releases/tag/v1.1.0-dev.1", + "pins": { + "sglang": "0.5.9", + "trtllm": "1.3.0rc5.post1", + "vllm": "0.17.1", + "nixlSglang": "1.0.1", + "nixlTrtllm": "0.10.1", + "nixlVllm": "0.10.1" + }, + "note": "Platform preview: runtime matrix, wheels on pypi.nvidia.com, Helm charts.", + "dateIso": "2026-03-17", + "notesUrl": "https://github.com/ai-dynamo/dynamo/releases/tag/v1.1.0-dev.1" + }, + { + "version": "v1.0.2", + "notesHref": "/dynamo/dev/reference/releases/v1-0-0", + "date": "Apr 22, 2026", + "kind": "patch", + "github": "https://github.com/ai-dynamo/dynamo/releases/tag/v1.0.2", + "docs": "https://docs.nvidia.com/dynamo", + "pins": { + "sglang": "0.5.9", + "trtllm": "1.3.0rc5.post1", + "vllm": "0.16.0", + "nixlSglang": "0.10.1", + "nixlTrtllm": "0.10.1", + "nixlVllm": "0.10.1" + }, + "delta": "No artifact additions or removals versus v1.0.0.", + "dateIso": "2026-04-22", + "notesUrl": "https://docs.nvidia.com/dynamo/dev/reference/releases/v1-0-0" + }, + { + "version": "v1.0.1", + "notesHref": "/dynamo/dev/reference/releases/v1-0-0", + "date": "Mar 16, 2026", + "kind": "patch", + "github": "https://github.com/ai-dynamo/dynamo/releases/tag/v1.0.1", + "docs": "https://docs.nvidia.com/dynamo", + "pins": { + "sglang": "0.5.9", + "trtllm": "1.3.0rc5.post1", + "vllm": "0.16.0", + "nixlSglang": "0.10.1", + "nixlTrtllm": "0.10.1", + "nixlVllm": "0.10.1" + }, + "delta": "No artifact additions or removals versus v1.0.0.", + "dateIso": "2026-03-16", + "notesUrl": "https://docs.nvidia.com/dynamo/dev/reference/releases/v1-0-0" + }, + { + "version": "v1.0.0", + "notesHref": "/dynamo/dev/reference/releases/v1-0-0", + "date": "Mar 12, 2026", + "kind": "stable", + "github": "https://github.com/ai-dynamo/dynamo/releases/tag/v1.0.0", + "docs": "https://docs.nvidia.com/dynamo", + "pins": { + "sglang": "0.5.9", + "trtllm": "1.3.0rc5.post1", + "vllm": "0.16.0", + "nixlSglang": "0.10.1", + "nixlTrtllm": "0.10.1", + "nixlVllm": "0.10.1" + }, + "delta": "snapshot-agent image and EFA variants for vLLM and TRT-LLM (AMD64 only). First publish of dynamo-mocker and dynamo-kv-router crates. snapshot Helm chart added (preview); deprecated dynamo-crds dropped from the publish stream.", + "notesSummary": "First GA release: unified configuration, Kubernetes production readiness, multimodal serving, and the agents surface.", + "dateIso": "2026-03-12", + "notesUrl": "https://docs.nvidia.com/dynamo/dev/reference/releases/v1-0-0" + }, + { + "version": "v0.9.1", + "date": "Mar 4, 2026", + "kind": "patch", + "github": "https://github.com/ai-dynamo/dynamo/releases/tag/v0.9.1", + "docs": "https://docs.nvidia.com/dynamo", + "pins": { + "sglang": "0.5.8", + "trtllm": "1.3.0rc3", + "vllm": "0.14.1", + "nixlSglang": "0.9.0", + "nixlTrtllm": "0.9.0", + "nixlVllm": "0.9.0" + }, + "delta": "No artifact additions or removals versus v0.9.0.", + "dateIso": "2026-03-04", + "notesUrl": "https://github.com/ai-dynamo/dynamo/releases/tag/v0.9.1" + }, + { + "version": "v0.9.0", + "date": "Feb 11, 2026", + "kind": "stable", + "github": "https://github.com/ai-dynamo/dynamo/releases/tag/v0.9.0", + "pins": { + "sglang": "0.5.8", + "trtllm": "1.3.0rc1", + "vllm": "0.14.1", + "nixlSglang": "0.9.0", + "nixlTrtllm": "0.9.0", + "nixlVllm": "0.9.0" + }, + "delta": "First publish of dynamo-tokens crate. Deprecated dynamo-graph Helm chart dropped from the publish stream.", + "dateIso": "2026-02-11", + "notesUrl": "https://github.com/ai-dynamo/dynamo/releases/tag/v0.9.0" + }, + { + "version": "v0.8.1", + "date": "Jan 23, 2026", + "kind": "patch", + "github": "https://github.com/ai-dynamo/dynamo/releases/tag/v0.8.1", + "pins": { + "sglang": "0.5.6.post2", + "trtllm": "1.2.0rc6.post1", + "vllm": "0.12.0", + "nixlSglang": "0.8.0", + "nixlTrtllm": "0.8.0", + "nixlVllm": "0.8.0" + }, + "delta": "Post trains .post1/.post2/.post3 republished the TRT-LLM runtime image and PyPI wheels only.", + "dateIso": "2026-01-23", + "notesUrl": "https://github.com/ai-dynamo/dynamo/releases/tag/v0.8.1" + }, + { + "version": "v0.8.0", + "date": "Jan 15, 2026", + "kind": "stable", + "github": "https://github.com/ai-dynamo/dynamo/releases/tag/v0.8.0", + "pins": { + "sglang": "0.5.6.post2", + "trtllm": "1.2.0rc6.post1", + "vllm": "0.12.0", + "nixlSglang": "0.8.0", + "nixlTrtllm": "0.8.0", + "nixlVllm": "0.8.0" + }, + "delta": "dynamo-frontend image and CUDA 13 variants for vLLM and SGLang. First publish of dynamo-memory and dynamo-config crates.", + "dateIso": "2026-01-15", + "notesUrl": "https://github.com/ai-dynamo/dynamo/releases/tag/v0.8.0" + }, + { + "version": "v0.7.1", + "date": "Dec 15, 2025", + "kind": "patch", + "github": "https://github.com/ai-dynamo/dynamo/releases/tag/v0.7.1", + "pins": { + "sglang": "0.5.4.post3", + "trtllm": "1.2.0rc3", + "vllm": "0.11.0", + "nixlSglang": "0.8.0", + "nixlTrtllm": "0.8.0", + "nixlVllm": "0.8.0" + }, + "dateIso": "2025-12-15", + "notesUrl": "https://github.com/ai-dynamo/dynamo/releases/tag/v0.7.1" + }, + { + "version": "v0.7.0", + "date": "Nov 26, 2025", + "kind": "stable", + "github": "https://github.com/ai-dynamo/dynamo/releases/tag/v0.7.0", + "pins": { + "sglang": "0.5.4.post3", + "trtllm": "1.2.0rc2", + "vllm": "0.11.0", + "nixlSglang": "0.8.0", + "nixlTrtllm": "0.8.0", + "nixlVllm": "0.8.0" + }, + "dateIso": "2025-11-26", + "notesUrl": "https://github.com/ai-dynamo/dynamo/releases/tag/v0.7.0" + }, + { + "version": "v0.6.1", + "date": "Nov 6, 2025", + "kind": "patch", + "github": "https://github.com/ai-dynamo/dynamo/releases/tag/v0.6.1", + "pins": { + "sglang": "0.5.3.post2", + "trtllm": "1.1.0rc5", + "vllm": "0.11.0", + "nixlSglang": "0.6.0", + "nixlTrtllm": "0.6.0", + "nixlVllm": "0.6.0" + }, + "dateIso": "2025-11-06", + "notesUrl": "https://github.com/ai-dynamo/dynamo/releases/tag/v0.6.1" + }, + { + "version": "v0.6.0", + "date": "Oct 28, 2025", + "kind": "stable", + "github": "https://github.com/ai-dynamo/dynamo/releases/tag/v0.6.0", + "pins": { + "sglang": "0.5.3.post2", + "trtllm": "1.1.0rc5", + "vllm": "0.11.0", + "nixlSglang": "0.6.0", + "nixlTrtllm": "0.6.0", + "nixlVllm": "0.6.0" + }, + "delta": "Oldest release tracked on this page.", + "dateIso": "2025-10-28", + "notesUrl": "https://github.com/ai-dynamo/dynamo/releases/tag/v0.6.0" + } + ], + "cudaHistory": [ + { + "version": "1.3.0", + "backend": "SGLang", + "toolkit": "13.0", + "minDriver": "580.xx+" + }, + { + "version": "1.3.0", + "backend": "TensorRT-LLM", + "toolkit": "13.1", + "minDriver": "580.xx+" + }, + { + "version": "1.3.0", + "backend": "vLLM", + "toolkit": "13.0", + "minDriver": "580.xx+" + }, + { + "version": "1.2.1", + "backend": "SGLang", + "toolkit": "12.9", + "minDriver": "575.xx+" + }, + { + "version": "1.2.1", + "backend": "SGLang", + "toolkit": "13.0", + "minDriver": "580.xx+" + }, + { + "version": "1.2.1", + "backend": "TensorRT-LLM", + "toolkit": "13.1", + "minDriver": "580.xx+" + }, + { + "version": "1.2.1", + "backend": "vLLM", + "toolkit": "12.9", + "minDriver": "575.xx+" + }, + { + "version": "1.2.1", + "backend": "vLLM", + "toolkit": "13.0", + "minDriver": "580.xx+" + }, + { + "version": "1.2.0", + "backend": "SGLang", + "toolkit": "12.9", + "minDriver": "575.xx+" + }, + { + "version": "1.2.0", + "backend": "SGLang", + "toolkit": "13.0", + "minDriver": "580.xx+" + }, + { + "version": "1.2.0", + "backend": "TensorRT-LLM", + "toolkit": "13.1", + "minDriver": "580.xx+" + }, + { + "version": "1.2.0", + "backend": "vLLM", + "toolkit": "12.9", + "minDriver": "575.xx+" + }, + { + "version": "1.2.0", + "backend": "vLLM", + "toolkit": "13.0", + "minDriver": "580.xx+" + }, + { + "version": "1.1.1", + "backend": "SGLang", + "toolkit": "12.9", + "minDriver": "575.xx+" + }, + { + "version": "1.1.1", + "backend": "SGLang", + "toolkit": "13.0", + "minDriver": "580.xx+" + }, + { + "version": "1.1.1", + "backend": "TensorRT-LLM", + "toolkit": "13.1", + "minDriver": "580.xx+" + }, + { + "version": "1.1.1", + "backend": "vLLM", + "toolkit": "12.9", + "minDriver": "575.xx+" + }, + { + "version": "1.1.1", + "backend": "vLLM", + "toolkit": "13.0", + "minDriver": "580.xx+" + }, + { + "version": "1.1.0", + "backend": "SGLang", + "toolkit": "12.9", + "minDriver": "575.xx+" + }, + { + "version": "1.1.0", + "backend": "SGLang", + "toolkit": "13.0", + "minDriver": "580.xx+" + }, + { + "version": "1.1.0", + "backend": "TensorRT-LLM", + "toolkit": "13.1", + "minDriver": "580.xx+" + }, + { + "version": "1.1.0", + "backend": "vLLM", + "toolkit": "12.9", + "minDriver": "575.xx+" + }, + { + "version": "1.1.0", + "backend": "vLLM", + "toolkit": "13.0", + "minDriver": "580.xx+" + }, + { + "version": "1.0.2", + "backend": "SGLang", + "toolkit": "12.9", + "minDriver": "575.xx+" + }, + { + "version": "1.0.2", + "backend": "SGLang", + "toolkit": "13.0", + "minDriver": "580.xx+" + }, + { + "version": "1.0.2", + "backend": "TensorRT-LLM", + "toolkit": "13.1", + "minDriver": "580.xx+" + }, + { + "version": "1.0.2", + "backend": "vLLM", + "toolkit": "12.9", + "minDriver": "575.xx+" + }, + { + "version": "1.0.2", + "backend": "vLLM", + "toolkit": "13.0", + "minDriver": "580.xx+" + }, + { + "version": "1.0.1", + "backend": "SGLang", + "toolkit": "12.9", + "minDriver": "575.xx+" + }, + { + "version": "1.0.1", + "backend": "SGLang", + "toolkit": "13.0", + "minDriver": "580.xx+" + }, + { + "version": "1.0.1", + "backend": "TensorRT-LLM", + "toolkit": "13.1", + "minDriver": "580.xx+" + }, + { + "version": "1.0.1", + "backend": "vLLM", + "toolkit": "12.9", + "minDriver": "575.xx+" + }, + { + "version": "1.0.1", + "backend": "vLLM", + "toolkit": "13.0", + "minDriver": "580.xx+" + }, + { + "version": "1.0.0", + "backend": "SGLang", + "toolkit": "12.9", + "minDriver": "575.xx+" + }, + { + "version": "1.0.0", + "backend": "SGLang", + "toolkit": "13.0", + "minDriver": "580.xx+" + }, + { + "version": "1.0.0", + "backend": "TensorRT-LLM", + "toolkit": "13.1", + "minDriver": "580.xx+" + }, + { + "version": "1.0.0", + "backend": "vLLM", + "toolkit": "12.9", + "minDriver": "575.xx+" + }, + { + "version": "1.0.0", + "backend": "vLLM", + "toolkit": "13.0", + "minDriver": "580.xx+" + }, + { + "version": "0.9.1", + "backend": "SGLang", + "toolkit": "12.9", + "minDriver": "575.xx+" + }, + { + "version": "0.9.1", + "backend": "TensorRT-LLM", + "toolkit": "13.0", + "minDriver": "580.xx+" + }, + { + "version": "0.9.1", + "backend": "vLLM", + "toolkit": "12.9", + "minDriver": "575.xx+" + }, + { + "version": "0.9.0", + "backend": "SGLang", + "toolkit": "12.9", + "minDriver": "575.xx+" + }, + { + "version": "0.9.0", + "backend": "TensorRT-LLM", + "toolkit": "13.0", + "minDriver": "580.xx+" + }, + { + "version": "0.9.0", + "backend": "vLLM", + "toolkit": "12.9", + "minDriver": "575.xx+" + }, + { + "version": "0.8.1", + "backend": "SGLang", + "toolkit": "12.9", + "minDriver": "575.xx+" + }, + { + "version": "0.8.1", + "backend": "SGLang", + "toolkit": "13.0", + "minDriver": "580.xx+", + "note": "Experimental" + }, + { + "version": "0.8.1", + "backend": "TensorRT-LLM", + "toolkit": "13.0", + "minDriver": "580.xx+" + }, + { + "version": "0.8.1", + "backend": "vLLM", + "toolkit": "12.9", + "minDriver": "575.xx+" + }, + { + "version": "0.8.1", + "backend": "vLLM", + "toolkit": "13.0", + "minDriver": "580.xx+", + "note": "Experimental" + }, + { + "version": "0.8.0", + "backend": "SGLang", + "toolkit": "12.9", + "minDriver": "575.xx+" + }, + { + "version": "0.8.0", + "backend": "SGLang", + "toolkit": "13.0", + "minDriver": "580.xx+", + "note": "Experimental" + }, + { + "version": "0.8.0", + "backend": "TensorRT-LLM", + "toolkit": "13.0", + "minDriver": "580.xx+" + }, + { + "version": "0.8.0", + "backend": "vLLM", + "toolkit": "12.9", + "minDriver": "575.xx+" + }, + { + "version": "0.8.0", + "backend": "vLLM", + "toolkit": "13.0", + "minDriver": "580.xx+", + "note": "Experimental" + }, + { + "version": "0.7.1", + "backend": "SGLang", + "toolkit": "12.8", + "minDriver": "570.xx+" + }, + { + "version": "0.7.1", + "backend": "TensorRT-LLM", + "toolkit": "13.0", + "minDriver": "580.xx+" + }, + { + "version": "0.7.1", + "backend": "vLLM", + "toolkit": "12.9", + "minDriver": "575.xx+" + }, + { + "version": "0.7.0", + "backend": "SGLang", + "toolkit": "12.9", + "minDriver": "575.xx+" + }, + { + "version": "0.7.0", + "backend": "TensorRT-LLM", + "toolkit": "13.0", + "minDriver": "580.xx+" + }, + { + "version": "0.7.0", + "backend": "vLLM", + "toolkit": "12.8", + "minDriver": "570.xx+" + } + ], + "cudaNotes": [ + "Patch versions (e.g. v0.8.1.post1, v0.7.0.post1) have the same CUDA support as their base version.", + "Early access v1.1.0-dev.* images follow the same CUDA matrix as v1.0.2. The v1.2.0-deepseek-v4-dev.3 vLLM container is CUDA 13.0 multi-arch; the SGLang containers split by arch (CUDA 12.9 on amd64, CUDA 13.0 on arm64).", + "Experimental CUDA 13 images are not published for all versions." + ], + "features": [ + { + "name": "Disaggregated Serving", + "sglang": { + "status": "yes" + }, + "trtllm": { + "status": "yes" + }, + "vllm": { + "status": "yes", + "note": "Prefill/decode separation with NIXL KV transfer" + } + }, + { + "name": "KV-Aware Routing", + "sglang": { + "status": "yes" + }, + "trtllm": { + "status": "yes" + }, + "vllm": { + "status": "yes" + } + }, + { + "name": "SLA-Based Planner", + "sglang": { + "status": "yes" + }, + "trtllm": { + "status": "yes" + }, + "vllm": { + "status": "yes" + } + }, + { + "name": "KV Block Manager", + "sglang": { + "status": "wip", + "note": "Work in progress across all combinations" + }, + "trtllm": { + "status": "yes" + }, + "vllm": { + "status": "yes" + } + }, + { + "name": "Multimodal (Image)", + "sglang": { + "status": "yes", + "note": "Not compatible with KV-aware routing. Disagg patterns: EPD, E/PD, E/P/D (not traditional EP/D)" + }, + "trtllm": { + "status": "yes", + "note": "Image URLs + pre-computed embeddings. Disagg: EP/D + E/P/D. KV-aware routing via dedicated MM Router Worker (requires KV event publishing)" + }, + "vllm": { + "status": "yes", + "note": "With KV-aware routing, image-aware routing on documented paths" + } + }, + { + "name": "Multimodal (Video)", + "sglang": { + "status": "yes" + }, + "trtllm": { + "status": "no" + }, + "vllm": { + "status": "yes", + "note": "Video input with frame sampling" + } + }, + { + "name": "Multimodal (Audio)", + "sglang": { + "status": "no" + }, + "trtllm": { + "status": "no" + }, + "vllm": { + "status": "wip", + "note": "Qwen2-Audio, experimental" + } + }, + { + "name": "Request Migration", + "sglang": { + "status": "yes" + }, + "trtllm": { + "status": "yes", + "note": "Work in progress with multimodal" + }, + "vllm": { + "status": "yes" + } + }, + { + "name": "Request Cancellation", + "sglang": { + "status": "wip", + "note": "Remote-prefill-phase cancellation not supported in disaggregated mode" + }, + "trtllm": { + "status": "caveat", + "note": "Engine temporarily not notified of cancellations — resources for cancelled requests are not freed (known issue)" + }, + "vllm": { + "status": "yes" + } + }, + { + "name": "LoRA", + "sglang": { + "status": "no" + }, + "trtllm": { + "status": "no" + }, + "vllm": { + "status": "yes", + "note": "Dynamic load/unload; KV-aware routing supports adapter affinity" + } + }, + { + "name": "Tool Calling", + "sglang": { + "status": "yes" + }, + "trtllm": { + "status": "yes" + }, + "vllm": { + "status": "yes" + } + }, + { + "name": "Speculative Decoding", + "sglang": { + "status": "wip", + "note": "Code hooks exist; no examples or docs yet" + }, + "trtllm": { + "status": "yes" + }, + "vllm": { + "status": "yes", + "note": "Eagle3" + } + }, + { + "name": "Dynamo Snapshot", + "sglang": { + "status": "yes" + }, + "trtllm": { + "status": "no" + }, + "vllm": { + "status": "yes" + } + } + ], + "artifacts": [ + { + "category": "container", + "group": "runtime", + "name": "vllm-runtime", + "description": "vLLM backend runtime", + "meta": "vLLM v0.23.0 · CUDA 13.0 · AMD64/ARM64", + "href": "https://catalog.ngc.nvidia.com/orgs/nvidia/ai-dynamo/containers/vllm-runtime/tags", + "tags": [ + { + "label": "1.3.0", + "clipboard": "nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.3.0" + }, + { + "label": "1.3.0-efa", + "clipboard": "nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.3.0-efa", + "variant": "experimental" + } + ] + }, + { + "category": "container", + "group": "runtime", + "name": "sglang-runtime", + "description": "SGLang backend runtime", + "meta": "SGLang v0.5.14 · CUDA 13.0 · AMD64/ARM64", + "href": "https://catalog.ngc.nvidia.com/orgs/nvidia/ai-dynamo/containers/sglang-runtime/tags", + "tags": [ + { + "label": "1.3.0", + "clipboard": "nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.3.0" + }, + { + "label": "1.3.0-efa", + "clipboard": "nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.3.0-efa", + "variant": "experimental" + } + ] + }, + { + "category": "container", + "group": "runtime", + "name": "tensorrtllm-runtime", + "description": "TensorRT-LLM backend runtime", + "meta": "TRT-LLM v1.3.0rc19 · CUDA 13.1 · AMD64/ARM64", + "href": "https://catalog.ngc.nvidia.com/orgs/nvidia/ai-dynamo/containers/tensorrtllm-runtime/tags", + "tags": [ + { + "label": "1.3.0", + "clipboard": "nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0" + }, + { + "label": "1.3.0-efa", + "clipboard": "nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-efa", + "variant": "experimental" + } + ] + }, + { + "category": "container", + "group": "component", + "name": "dynamo-frontend", + "description": "OpenAI-compatible API gateway with Endpoint Prediction Protocol (EPP)", + "meta": "AMD64/ARM64", + "href": "https://catalog.ngc.nvidia.com/orgs/nvidia/ai-dynamo/containers/dynamo-frontend/tags", + "tags": [ + { + "label": "1.3.0", + "clipboard": "nvcr.io/nvidia/ai-dynamo/dynamo-frontend:1.3.0" + } + ] + }, + { + "category": "container", + "group": "component", + "name": "dynamo-planner", + "description": "Standalone Planner used by Profiler jobs and Planner pods", + "meta": "AMD64/ARM64", + "href": "https://catalog.ngc.nvidia.com/orgs/nvidia/ai-dynamo/containers/dynamo-planner/tags", + "tags": [ + { + "label": "1.3.0", + "clipboard": "nvcr.io/nvidia/ai-dynamo/dynamo-planner:1.3.0" + } + ] + }, + { + "category": "container", + "group": "component", + "name": "kubernetes-operator", + "description": "Operator that manages Dynamo deployments and CRDs", + "meta": "AMD64/ARM64", + "href": "https://catalog.ngc.nvidia.com/orgs/nvidia/ai-dynamo/containers/kubernetes-operator/tags", + "tags": [ + { + "label": "1.3.0", + "clipboard": "nvcr.io/nvidia/ai-dynamo/kubernetes-operator:1.3.0" + } + ] + }, + { + "category": "container", + "group": "component", + "name": "snapshot-agent", + "description": "Fast GPU worker recovery via CRIU", + "meta": "AMD64/ARM64", + "href": "https://catalog.ngc.nvidia.com/orgs/nvidia/ai-dynamo/containers/snapshot-agent/tags", + "badge": "Preview", + "tags": [ + { + "label": "1.3.0", + "clipboard": "nvcr.io/nvidia/ai-dynamo/snapshot-agent:1.3.0" + } + ] + }, + { + "category": "wheel", + "name": "ai-dynamo", + "description": "Main package with backend integrations (vLLM, SGLang, TRT-LLM)", + "meta": "Python 3.10–3.12 · Linux (glibc v2.28+)", + "href": "https://pypi.org/project/ai-dynamo/1.3.0.post1/", + "tags": [ + { + "label": "uv pip install ai-dynamo==1.3.0.post1", + "clipboard": "uv pip install ai-dynamo==1.3.0.post1" + } + ] + }, + { + "category": "wheel", + "name": "ai-dynamo-runtime", + "description": "Core Python bindings for the Dynamo runtime", + "meta": "Python 3.10–3.12 · Linux (glibc v2.28+)", + "href": "https://pypi.org/project/ai-dynamo-runtime/1.3.0.post1/", + "tags": [ + { + "label": "uv pip install ai-dynamo-runtime==1.3.0.post1", + "clipboard": "uv pip install ai-dynamo-runtime==1.3.0.post1" + } + ] + }, + { + "category": "wheel", + "name": "kvbm", + "description": "KV Block Manager for disaggregated KV cache", + "meta": "Python 3.10–3.12 · Linux (glibc v2.28+)", + "href": "https://pypi.org/project/kvbm/1.3.0.post1/", + "tags": [ + { + "label": "uv pip install kvbm==1.3.0.post1", + "clipboard": "uv pip install kvbm==1.3.0.post1" + } + ] + }, + { + "category": "helm", + "name": "dynamo-platform", + "description": "Platform services (etcd, NATS) and the Dynamo Operator for a Dynamo cluster", + "href": "https://helm.ngc.nvidia.com/nvidia/ai-dynamo/charts/dynamo-platform-1.3.0.tgz", + "tags": [ + { + "label": "helm install · dynamo-platform 1.3.0", + "clipboard": "helm install dynamo-platform oci://helm.ngc.nvidia.com/nvidia/ai-dynamo/charts/dynamo-platform --version 1.3.0" + } + ] + }, + { + "category": "helm", + "name": "snapshot", + "description": "Snapshot DaemonSet for fast GPU worker recovery", + "href": "https://helm.ngc.nvidia.com/nvidia/ai-dynamo/charts/snapshot-1.3.0.tgz", + "tags": [ + { + "label": "helm install · snapshot 1.3.0", + "clipboard": "helm install snapshot oci://helm.ngc.nvidia.com/nvidia/ai-dynamo/charts/snapshot --version 1.3.0" + } + ] + }, + { + "category": "crate", + "name": "dynamo-runtime", + "description": "Core distributed runtime library", + "meta": "MSRV Rust v1.82", + "href": "https://crates.io/crates/dynamo-runtime/1.3.0", + "tags": [ + { + "label": "cargo add dynamo-runtime@1.3.0", + "clipboard": "cargo add dynamo-runtime@1.3.0" + } + ] + }, + { + "category": "crate", + "name": "dynamo-llm", + "description": "LLM inference engine", + "meta": "MSRV Rust v1.82", + "href": "https://crates.io/crates/dynamo-llm/1.3.0", + "tags": [ + { + "label": "cargo add dynamo-llm@1.3.0", + "clipboard": "cargo add dynamo-llm@1.3.0" + } + ] + }, + { + "category": "crate", + "name": "dynamo-protocols", + "description": "Async OpenAI-compatible API client", + "meta": "MSRV Rust v1.82", + "href": "https://crates.io/crates/dynamo-protocols/1.3.0", + "tags": [ + { + "label": "cargo add dynamo-protocols@1.3.0", + "clipboard": "cargo add dynamo-protocols@1.3.0" + } + ] + }, + { + "category": "crate", + "name": "dynamo-async-openai", + "description": "Legacy OpenAI client; use dynamo-protocols", + "meta": "MSRV Rust v1.82 · final release", + "href": "https://crates.io/crates/dynamo-async-openai/1.0.2", + "badge": "Deprecated", + "tags": [ + { + "label": "cargo add dynamo-async-openai@1.0.2", + "clipboard": "cargo add dynamo-async-openai@1.0.2" + } + ] + }, + { + "category": "crate", + "name": "dynamo-parsers", + "description": "Protocol parsers (SSE, JSON streaming)", + "meta": "MSRV Rust v1.82", + "href": "https://crates.io/crates/dynamo-parsers/1.3.0", + "tags": [ + { + "label": "cargo add dynamo-parsers@1.3.0", + "clipboard": "cargo add dynamo-parsers@1.3.0" + } + ] + }, + { + "category": "crate", + "name": "dynamo-memory", + "description": "Memory management utilities", + "meta": "MSRV Rust v1.82", + "href": "https://crates.io/crates/dynamo-memory/1.3.0", + "tags": [ + { + "label": "cargo add dynamo-memory@1.3.0", + "clipboard": "cargo add dynamo-memory@1.3.0" + } + ] + }, + { + "category": "crate", + "name": "dynamo-config", + "description": "Configuration management", + "meta": "MSRV Rust v1.82", + "href": "https://crates.io/crates/dynamo-config/1.3.0", + "tags": [ + { + "label": "cargo add dynamo-config@1.3.0", + "clipboard": "cargo add dynamo-config@1.3.0" + } + ] + }, + { + "category": "crate", + "name": "dynamo-tokens", + "description": "Tokenizer bindings for LLM inference", + "meta": "MSRV Rust v1.82", + "href": "https://crates.io/crates/dynamo-tokens/1.3.0", + "tags": [ + { + "label": "cargo add dynamo-tokens@1.3.0", + "clipboard": "cargo add dynamo-tokens@1.3.0" + } + ] + }, + { + "category": "crate", + "name": "dynamo-tokenizers", + "description": "Tokenizer library for LLM inference", + "meta": "MSRV Rust v1.82", + "href": "https://crates.io/crates/dynamo-tokenizers/1.3.0", + "tags": [ + { + "label": "cargo add dynamo-tokenizers@1.3.0", + "clipboard": "cargo add dynamo-tokenizers@1.3.0" + } + ] + }, + { + "category": "crate", + "name": "dynamo-mocker", + "description": "Inference engine simulator for benchmarking", + "meta": "MSRV Rust v1.82", + "href": "https://crates.io/crates/dynamo-mocker/1.3.0", + "tags": [ + { + "label": "cargo add dynamo-mocker@1.3.0", + "clipboard": "cargo add dynamo-mocker@1.3.0" + } + ] + }, + { + "category": "crate", + "name": "dynamo-kv-router", + "description": "KV-aware request routing library", + "meta": "MSRV Rust v1.82", + "href": "https://crates.io/crates/dynamo-kv-router/1.3.0", + "tags": [ + { + "label": "cargo add dynamo-kv-router@1.3.0", + "clipboard": "cargo add dynamo-kv-router@1.3.0" + } + ] + }, + { + "category": "crate", + "name": "kvbm-logical", + "description": "Logical layer for the KV Block Manager", + "meta": "MSRV Rust v1.82", + "href": "https://crates.io/crates/kvbm-logical/1.3.0", + "tags": [ + { + "label": "cargo add kvbm-logical@1.3.0", + "clipboard": "cargo add kvbm-logical@1.3.0" + } + ] + } + ], + "modelEaBuilds": [ + { + "model": "Inkling", + "tag": "1.4.0-inkling-dev.1", + "releaseLine": "v1.4.0", + "runtimes": [ + "sglang-runtime" + ], + "shipped": "Jul 17, 2026", + "gaPath": "dev-only", + "gaLabel": "Dev-only · v1.4.0 line", + "statusLine": "First build on the v1.4.0 line; targets the next stable release.", + "recipeLabel": "Inkling recipe (main)", + "recipeHref": "https://github.com/ai-dynamo/dynamo/blob/main/docs/recipes/inkling.mdx", + "github": "https://github.com/ai-dynamo/dynamo/releases/tag/v1.4.0-inkling-dev.1", + "coverage": { + "images": true, + "wheels": false, + "helm": false, + "crates": false + }, + "shippedIso": "2026-07-17" + }, + { + "model": "GLM-5.2", + "tag": "1.3.0-glm-5.2-dev.1", + "releaseLine": "v1.3.0", + "runtimes": [ + "sglang-runtime" + ], + "shipped": "Jul 20, 2026", + "gaPath": "dev-only", + "gaLabel": "Dev-only", + "statusLine": "Container carries SGLang cherry-picks (stability, config parsing, model support) opened upstream but not yet in a released SGLang.", + "recipeLabel": "GLM-5 NVFP4 recipe", + "recipeHref": "/dynamo/dev/recipes/glm-5-nvfp4", + "coverage": { + "images": true, + "wheels": false, + "helm": false, + "crates": false + }, + "shippedIso": "2026-07-20" + }, + { + "model": "MiniMax-M3", + "tag": "1.3.0-minimax-m3-dev.1", + "releaseLine": "v1.3.0", + "runtimes": [ + "vllm-runtime", + "sglang-runtime", + "tensorrtllm-runtime" + ], + "shipped": "Jun 12, 2026", + "gaPath": "promoted", + "gaLabel": "Promoted → :1.3.0", + "statusLine": "Dynamo changes and the M2 tool-calling fix are in release/1.3.0; the recipes run on the stock :1.3.0 containers.", + "recipeLabel": "Recipe on release branch", + "recipeHref": "https://github.com/ai-dynamo/dynamo/tree/release/1.3.0-minimax-m3-dev.1/recipes/minimax-m3", + "github": "https://github.com/ai-dynamo/dynamo/releases/tag/v1.3.0-minimax-m3-dev.1", + "coverage": { + "images": true, + "wheels": false, + "helm": false, + "crates": false + }, + "shippedIso": "2026-06-12" + }, + { + "model": "DeepSeek-V4", + "tag": "1.3.0-deepseek-v4-dev.1", + "releaseLine": "v1.3.0", + "runtimes": [ + "tensorrtllm-runtime" + ], + "shipped": "Jun 6, 2026", + "gaPath": "recipe-in-ga", + "gaLabel": "Recipe in v1.3.0", + "statusLine": "DeepSeek-V4 Flash and Pro recipes ship in v1.3.0 on the standard TensorRT-LLM release container.", + "recipeLabel": "recipes/deepseek-v4 (main)", + "recipeHref": "https://github.com/ai-dynamo/dynamo/tree/main/recipes/deepseek-v4", + "github": "https://github.com/ai-dynamo/dynamo/releases/tag/v1.3.0-deepseek-v4-dev.1", + "coverage": { + "images": true, + "wheels": false, + "helm": false, + "crates": false + }, + "shippedIso": "2026-06-06" + }, + { + "model": "Nemotron-3-Ultra", + "tag": "1.3.0-nemotron-ultra-dev.1", + "releaseLine": "v1.3.0", + "runtimes": [ + "vllm-runtime" + ], + "shipped": "Jun 5, 2026", + "gaPath": "dev-only", + "gaLabel": "Dev-only", + "statusLine": "Four un-upstreamed vLLM patches; requires pinned flags VLLM_DISABLED_KERNELS=FlashInferFP8ScaledMMLinearKernel and --no-enable-flashinfer-autotune.", + "recipeLabel": "Nemotron-3-Ultra recipe", + "recipeHref": "/dynamo/dev/recipes/nemotron-3-ultra", + "github": "https://github.com/ai-dynamo/dynamo/releases/tag/v1.3.0-nemotron-ultra-dev.1", + "coverage": { + "images": true, + "wheels": false, + "helm": false, + "crates": false + }, + "shippedIso": "2026-06-05" + }, + { + "model": "Nemotron-3-Super", + "tag": "1.3.0-nemotron-super-dev.1", + "releaseLine": "v1.3.0", + "runtimes": [ + "vllm-runtime" + ], + "shipped": "Jun 4, 2026", + "gaPath": "promoted", + "gaLabel": "Promoted → :1.3.0", + "statusLine": "Both container patches are in the vLLM v0.23.0 that v1.3.0 ships; the recipe runs on the stock vllm-runtime:1.3.0.", + "recipeLabel": "Nemotron-3-Super recipe", + "recipeHref": "/dynamo/dev/recipes/nemotron-3-super", + "github": "https://github.com/ai-dynamo/dynamo/releases/tag/v1.3.0-nemotron-super-dev.1", + "coverage": { + "images": true, + "wheels": false, + "helm": false, + "crates": false + }, + "shippedIso": "2026-06-04" + }, + { + "model": "Kimi-K2.6", + "tag": "1.3.0-kimi-k2.6-dev.1", + "releaseLine": "v1.3.0", + "runtimes": [ + "vllm-runtime" + ], + "shipped": "Jun 4, 2026", + "gaPath": "promoted", + "gaLabel": "Promoted → :1.3.0", + "statusLine": "The build's only container patch is in vLLM v0.23.0; the recipes run on the stock vllm-runtime:1.3.0.", + "recipeLabel": "Kimi-K2.6 recipe", + "recipeHref": "/dynamo/dev/recipes/kimi-k2-6", + "github": "https://github.com/ai-dynamo/dynamo/releases/tag/v1.3.0-kimi-k2.6-dev.1", + "coverage": { + "images": true, + "wheels": false, + "helm": false, + "crates": false + }, + "shippedIso": "2026-06-04" + }, + { + "model": "Cosmos-3", + "tag": "1.3.0-cosmos3-dev.1", + "releaseLine": "v1.3.0", + "runtimes": [ + "vllm-runtime" + ], + "shipped": "Jun 1, 2026", + "gaPath": "dev-only", + "gaLabel": "Dev-only", + "statusLine": "Dynamo #10132 (Cosmos3 support in the vLLM-Omni backend) is open, not merged — v1.3.0 containers cannot run Cosmos3.", + "recipeLabel": "Launch scripts (branch)", + "recipeHref": "https://github.com/ai-dynamo/dynamo/tree/release/1.3.0-cosmos3-dev.1/examples/backends/vllm/launch", + "github": "https://github.com/ai-dynamo/dynamo/releases/tag/v1.3.0-cosmos3-dev.1", + "coverage": { + "images": true, + "wheels": false, + "helm": false, + "crates": false + }, + "shippedIso": "2026-06-01" + }, + { + "model": "DeepSeek-V4 preview", + "tag": "1.2.0-deepseek-v4-dev.3", + "releaseLine": "v1.2.0", + "runtimes": [ + "vllm-runtime", + "sglang-runtime" + ], + "shipped": "May 9, 2026", + "gaPath": "superseded", + "gaLabel": "Superseded — recipe in v1.3.0", + "statusLine": "Blackwell (B200 + GB200) preview; per-arch/CUDA tags (e.g. vllm-runtime:1.2.0-deepseek-v4-cuda13-dev.3). Superseded by the v1.3.0 recipe.", + "github": "https://github.com/ai-dynamo/dynamo/releases/tag/v1.2.0-deepseek-v4-dev.3", + "coverage": { + "images": true, + "wheels": false, + "helm": false, + "crates": false + }, + "shippedIso": "2026-05-09" + }, + { + "model": "DeepSeek-V4 preview", + "tag": "1.2.0-deepseek-v4-dev.2", + "releaseLine": "v1.2.0", + "runtimes": [ + "vllm-runtime", + "sglang-runtime" + ], + "shipped": "May 1, 2026", + "gaPath": "superseded", + "gaLabel": "Superseded — recipe in v1.3.0", + "statusLine": "Blackwell preview on vLLM v0.20.0 (native DSv4 support); superseded by dev.3.", + "github": "https://github.com/ai-dynamo/dynamo/releases/tag/v1.2.0-deepseek-v4-dev.2", + "coverage": { + "images": true, + "wheels": false, + "helm": false, + "crates": false + }, + "shippedIso": "2026-05-01" + }, + { + "model": "DeepSeek-V4 preview", + "tag": "1.2.0-sglang-deepseek-v4-dev.1", + "releaseLine": "v1.2.0", + "runtimes": [ + "sglang-runtime" + ], + "shipped": "Apr 25, 2026", + "gaPath": "superseded", + "gaLabel": "Superseded — recipe in v1.3.0", + "statusLine": "Earliest DSv4 preview (SGLang, B200 only); superseded by dev.2/dev.3.", + "github": "https://github.com/ai-dynamo/dynamo/releases/tag/v1.2.0-sglang-deepseek-v4-dev.1", + "coverage": { + "images": true, + "wheels": false, + "helm": false, + "crates": false + }, + "shippedIso": "2026-04-25" + } + ], + "platformPreviewCoverage": { + "v1.3.0-dev.1": { + "images": true, + "wheels": true, + "helm": true, + "crates": true + }, + "v1.1.0-dev.3": { + "images": true, + "wheels": true, + "helm": false, + "crates": false + }, + "v1.1.0-dev.2": { + "images": true, + "wheels": true, + "helm": false, + "crates": false + }, + "v1.1.0-dev.1": { + "images": true, + "wheels": true, + "helm": true, + "crates": false + } + }, + "platform": { + "gpus": [ + "Blackwell", + "Hopper", + "Ada Lovelace", + "Ampere" + ], + "os": [ + { + "name": "Ubuntu", + "version": "24.04", + "arch": "x86_64, ARM64", + "status": "Supported", + "chip": "ubuntu" + }, + { + "name": "Ubuntu", + "version": "22.04", + "arch": "x86_64", + "status": "Supported", + "chip": "ubuntu" + }, + { + "name": "CentOS Stream", + "version": "9", + "arch": "x86_64", + "status": "Experimental", + "chip": "centos" + } + ], + "arch": [ + "x86_64", + "ARM64 (Ubuntu 24.04 only)" + ], + "wheelsNote": "Wheels are built in a manylinux_2_28-compatible environment and validated on CentOS Stream 9 and Ubuntu 22.04/24.04. Other Linux distributions are expected to work but are not officially verified.", + "csp": [ + { + "provider": "AWS", + "os": "Amazon Linux 2023", + "arch": "x86_64", + "status": "Supported" + } + ] + }, + "knownArtifactIssues": [ + { + "version": "v0.9.0", + "artifact": "dynamo-platform-0.9.0", + "issue": "Helm chart sets operator image to 0.7.1 instead of 0.9.0.", + "status": "Fixed in v0.9.0.post1" + }, + { + "version": "v0.8.1", + "artifact": "vllm-runtime:0.8.1-cuda13", + "issue": "Container fails to launch.", + "status": "Known issue" + }, + { + "version": "v0.8.1", + "artifact": "sglang-runtime:0.8.1-cuda13, vllm-runtime:0.8.1-cuda13", + "issue": "Multimodality not expected to work on ARM64. Works on AMD64.", + "status": "Known limitation" + }, + { + "version": "v0.8.0", + "artifact": "sglang-runtime:0.8.0-cuda13", + "issue": "CuDNN installation issue caused PyTorch v2.9.1 compatibility problems with nn.Conv3d — performance degradation and excessive memory usage in multimodal workloads.", + "status": "Fixed in v0.8.1 (#5461)" + } + ], + "cratesFirstPublished": [ + { + "crate": "dynamo-runtime", + "version": "0.1.0", + "date": "2025-03-18" + }, + { + "crate": "dynamo-llm", + "version": "0.2.0", + "date": "2025-05-01" + }, + { + "crate": "dynamo-async-openai", + "version": "0.4.1", + "date": "2025-08-27" + }, + { + "crate": "dynamo-parsers", + "version": "0.5.0", + "date": "2025-09-18" + }, + { + "crate": "dynamo-memory", + "version": "0.8.0", + "date": "2026-01-15" + }, + { + "crate": "dynamo-config", + "version": "0.8.0", + "date": "2026-01-15" + }, + { + "crate": "dynamo-tokens", + "version": "0.9.0", + "date": "2026-02-12" + }, + { + "crate": "dynamo-mocker", + "version": "1.0.0", + "date": "2026-03-13" + }, + { + "crate": "dynamo-kv-router", + "version": "1.0.0", + "date": "2026-03-13" + }, + { + "crate": "dynamo-protocols", + "version": "1.1.0", + "date": "2026-05-04" + }, + { + "crate": "dynamo-tokenizers", + "version": "1.2.0", + "date": "2026-06-02" + } + ], + "releaseStats": { + "v1.3.0": { + "prs": 930, + "contributors": 125, + "firstTimers": 23, + "breaking": 24, + "knownIssues": 10 + }, + "v1.2.0": { + "prs": 603, + "contributors": 82, + "breaking": 5, + "knownIssues": 11 + }, + "v1.1.0": { + "prs": 896, + "contributors": 113, + "firstTimers": 12, + "breaking": 8, + "knownIssues": 20 + }, + "v1.0.0": { + "contributors": 90, + "firstTimers": 34, + "breaking": 41, + "knownIssues": 14 + } + }, + "nightliesNote": "ai-dynamo and ai-dynamo-runtime nightly builds from main publish wheels tagged *.devYYYYMMDD (since Apr 24, 2026). Install with pip or uv using --pre and the NVIDIA extra-index pattern shown above." +} diff --git a/docs/fern/backends/trtllm/README.md b/docs/fern/backends/trtllm/README.md index abe6198b3ced..23a9267a0e8e 100644 --- a/docs/fern/backends/trtllm/README.md +++ b/docs/fern/backends/trtllm/README.md @@ -71,7 +71,7 @@ TensorRT-LLM delivers maximum inference performance and optimization, with full | `sglang-runtime:1.0.2` | SGLang `v0.5.9` | `v12.9` | `575+` | | `sglang-runtime:1.0.2-cuda13` | SGLang `v0.5.9` | `v13.0` | `580+` | -Source of truth: [`docs/reference/support-matrix.md`](../../reference/support-matrix.md#cuda-and-driver-requirements) and [`docs/reference/release-artifacts.mdx`](../../reference/release-artifacts.mdx). If those differ from the values above, the source-of-truth files win. +Source of truth: [`docs/fern/reference/compatibility.mdx`](../../reference/compatibility.mdx#cuda--driver-requirements) and [`docs/fern/reference/release-artifacts.mdx`](../../reference/release-artifacts.mdx). If those differ from the values above, the source-of-truth files win. ## Quick Start diff --git a/docs/fern/components/ArtifactBrowser.tsx b/docs/fern/components/ArtifactBrowser.tsx new file mode 100644 index 000000000000..7946804e687c --- /dev/null +++ b/docs/fern/components/ArtifactBrowser.tsx @@ -0,0 +1,472 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * ArtifactBrowser — filterable list of every published artifact for the + * current release (containers, wheels, Helm charts, crates) driven entirely + * by releases.data.ts. + * + * Filtering is CSS-only: five hidden radio inputs sit first inside the panel, + * and :checked general-sibling selectors hide the rows (and their group + * headers) whose data-cat does not match — same pattern as the recipe catalog + * in recipes/README.mdx. Server component; shared vocabulary (panel, eyebrow, + * badges, copy buttons) comes from ReferenceStyles — place + * on the page alongside this component. Only the .dynref-ab-* layout classes + * are defined here. + */ + +import { + ARTIFACTS, + CURRENT_TAG, + CURRENT_VERSION, + CURRENT_WHEEL, + RELEASES, + type Artifact, + type ArtifactCategory, +} from "./releases.data"; + +const AB_CSS = ` +/* Inputs are hidden by the shared .dynref-vh (visually hidden, focusable) + class so the filter rail stays keyboard-operable; the :focus-visible rules + below paint the ring on the matching pill. */ + +.dynref-ab-note { + margin: 0; +} + +.dynref-ab-headmeta { + display: flex; + flex-direction: column; + align-items: flex-end; + gap: 8px; +} + +.dynref-ab-links { + display: flex; + flex-wrap: wrap; + gap: 6px; +} + +.dynref-ab-linkchip { + display: inline-flex; + align-items: center; + padding: 4px 10px; + border: 1px solid var(--border, var(--grayscale-a5)); + border-radius: 8px; + color: var(--pst-color-text-muted); + font-size: 12px; + line-height: 1; + text-decoration: none; +} + +.dynref-ab-linkchip:hover { + border-color: var(--nv-color-green, #76B900); + color: var(--pst-color-text-base); +} + +.dark .dynref-ab-linkchip { + border-color: #333; + background: #1c1c1c; +} + +.dynref-ab-rail { + display: flex; + flex-wrap: wrap; + gap: 8px; + margin: 0 0 4px; +} + +.dynref-ab-pill { + display: inline-flex; + align-items: center; + min-height: 30px; + padding: 6px 10px; + border: 1px solid var(--border, var(--grayscale-a5)); + border-radius: var(--rounded, 6px); + background: transparent; + color: var(--pst-color-text-base); + font-size: 12.5px; + line-height: 1; + cursor: pointer; +} + +.dynref-ab-pill:hover { + border-color: var(--nv-color-green, #76B900); +} + +#ab-all:checked ~ .dynref-ab-rail label[for="ab-all"], +#ab-container:checked ~ .dynref-ab-rail label[for="ab-container"], +#ab-wheel:checked ~ .dynref-ab-rail label[for="ab-wheel"], +#ab-helm:checked ~ .dynref-ab-rail label[for="ab-helm"], +#ab-crate:checked ~ .dynref-ab-rail label[for="ab-crate"] { + border-color: var(--nv-color-green, #76B900); + box-shadow: 0 0 0 1px var(--nv-color-green, #76B900); + background: rgba(118, 185, 0, 0.08); + font-weight: 700; +} + +#ab-all:focus-visible ~ .dynref-ab-rail label[for="ab-all"], +#ab-container:focus-visible ~ .dynref-ab-rail label[for="ab-container"], +#ab-wheel:focus-visible ~ .dynref-ab-rail label[for="ab-wheel"], +#ab-helm:focus-visible ~ .dynref-ab-rail label[for="ab-helm"], +#ab-crate:focus-visible ~ .dynref-ab-rail label[for="ab-crate"] { + outline: 2px solid var(--nv-color-green, #76B900); + outline-offset: 1px; +} + +#ab-container:checked ~ .dynref-ab-list > [data-cat]:not([data-cat="container"]), +#ab-wheel:checked ~ .dynref-ab-list > [data-cat]:not([data-cat="wheel"]), +#ab-helm:checked ~ .dynref-ab-list > [data-cat]:not([data-cat="helm"]), +#ab-crate:checked ~ .dynref-ab-list > [data-cat]:not([data-cat="crate"]) { + display: none; +} + +/* Heading recount: the filter is CSS-only, so the heading pre-renders one + span per filter and the same :checked sibling selectors toggle which one + shows. Default (All checked, or nothing checked) shows the All span. */ +.dynref-ab-hspan { + display: none; +} + +.dynref-ab-hspan--all { + display: inline; +} + +#ab-container:checked ~ .dynref-panel-header .dynref-ab-hspan--all, +#ab-wheel:checked ~ .dynref-panel-header .dynref-ab-hspan--all, +#ab-helm:checked ~ .dynref-panel-header .dynref-ab-hspan--all, +#ab-crate:checked ~ .dynref-panel-header .dynref-ab-hspan--all { + display: none; +} + +#ab-container:checked ~ .dynref-panel-header .dynref-ab-hspan--container, +#ab-wheel:checked ~ .dynref-panel-header .dynref-ab-hspan--wheel, +#ab-helm:checked ~ .dynref-panel-header .dynref-ab-hspan--helm, +#ab-crate:checked ~ .dynref-panel-header .dynref-ab-hspan--crate { + display: inline; +} + +.dynref-ab-group { + margin: 0; + padding: 16px 0 6px; +} + +.dynref-ab-row { + /* Fixed first column so names, descriptions, and tag groups align into + clean columns across rows. 230px fits the longest name + (tensorrtllm-runtime + glyph ≈ 171px at 13px mono). */ + display: grid; + grid-template-columns: 230px minmax(0, 1fr) max-content; + gap: 10px; + align-items: center; + padding: 11px 0; + border-bottom: 1px solid var(--border, var(--grayscale-a5)); +} + +.dynref-ab-row:last-child { + border-bottom: 0; +} + +.dynref-ab-name { + display: flex; + align-items: center; + gap: 8px; + min-width: 0; + flex-wrap: wrap; +} + +.dynref-ab-glyph { + flex: 0 0 auto; +} + +.dynref-ab-glyph--container { + color: #76B900; +} + +.dynref-ab-glyph--wheel { + color: var(--dynref-teal-fg); +} + + +.dynref-ab-glyph--helm { + color: var(--dynref-blue-fg); +} + + +.dynref-ab-glyph--crate { + color: var(--dynref-violet-fg); +} + + +.dynref-ab-link { + color: var(--pst-color-text-base); + font-size: 13px; + font-weight: 600; + text-decoration: none; + overflow-wrap: anywhere; +} + +.dynref-ab-link:hover { + text-decoration: underline; + text-decoration-color: var(--nv-color-green, #76B900); +} + +.dynref-ab-desc { + margin: 0; + color: var(--pst-color-text-muted); + font-size: 12.5px; + line-height: 1.4; +} + +.dynref-ab-tags { + display: flex; + flex-wrap: wrap; + justify-content: flex-end; + gap: 6px; +} + +.dynref-ab-tags .dynref-copy { + font-family: var(--pst-font-family-monospace, ui-monospace, SFMono-Regular, Menlo, monospace); + font-size: 11px; +} + +/* Narrow widths: stack each row (name / description / tags in column flow). + Last in the sheet so these override the base rules above. */ +@media (max-width: 640px) { + .dynref-ab-row { + grid-template-columns: minmax(0, 1fr); + gap: 6px; + } + + .dynref-ab-tags { + justify-content: flex-start; + } + + .dynref-ab-headmeta { + align-items: flex-start; + } +} +`; + +/* Simple 15px stroke glyphs per artifact category: box (container), + wheel (Python wheel), anchor (Helm), gear (crate). */ +function Glyph({ category }: { category: ArtifactCategory }) { + const shared = { + className: `dynref-ab-glyph dynref-ab-glyph--${category}`, + width: 15, + height: 15, + viewBox: "0 0 16 16", + fill: "none", + stroke: "currentColor", + strokeWidth: 1.4, + strokeLinecap: "round" as const, + strokeLinejoin: "round" as const, + "aria-hidden": true, + }; + if (category === "container") { + return ( + + + + + ); + } + if (category === "wheel") { + return ( + + + + + + ); + } + if (category === "helm") { + return ( + + + + + ); + } + return ( + + + + + ); +} + +/* Short accessible name for a copy button; the full clipboard payload goes + in the title attribute. */ +function copyAriaLabel(artifact: Artifact, tagLabel: string): string { + if (artifact.category === "container") return `Copy ${artifact.name}:${tagLabel} image reference`; + if (artifact.category === "wheel") return `Copy pip install command for ${artifact.name}`; + if (artifact.category === "helm") return `Copy helm install command for ${artifact.name}`; + return `Copy cargo add command for ${artifact.name}`; +} + +function ArtifactRow({ artifact }: { artifact: Artifact }) { + const badgeVariant = artifact.badge === "Deprecated" ? "red" : "amber"; + return ( +
+
+ + + {artifact.name} + + {artifact.badge && ( + {artifact.badge} + )} +
+

+ {artifact.description} + {artifact.meta ? ` · ${artifact.meta}` : ""} +

+
+ {artifact.tags.map((tag) => ( + + ))} +
+
+ ); +} + +function GroupHeader({ label, cat }: { label: string; cat: ArtifactCategory }) { + return ( +

+ {label} +

+ ); +} + +export function ArtifactBrowser() { + const counts: Record = { container: 0, wheel: 0, helm: 0, crate: 0 }; + for (const artifact of ARTIFACTS) counts[artifact.category] += 1; + + const currentRelease = RELEASES.find((r) => r.version === CURRENT_VERSION); + + const runtimeContainers = ARTIFACTS.filter((a) => a.category === "container" && a.group === "runtime"); + const componentContainers = ARTIFACTS.filter((a) => a.category === "container" && a.group !== "runtime"); + const wheels = ARTIFACTS.filter((a) => a.category === "wheel"); + const helmCharts = ARTIFACTS.filter((a) => a.category === "helm"); + const crates = ARTIFACTS.filter((a) => a.category === "crate"); + + return ( + <> + +
+ + + + + + +
+
+

Release artifacts

+

+ + {ARTIFACTS.length} artifacts for {CURRENT_VERSION} + + + {counts.container} container images for {CURRENT_VERSION} + + + {counts.wheel} Python wheels for {CURRENT_VERSION} + + + {counts.helm} Helm charts for {CURRENT_VERSION} + + + {counts.crate} Rust crates for {CURRENT_VERSION} + +

+
+
+

+ Wheels ship as {CURRENT_WHEEL} · containers stay{" "} + :{CURRENT_TAG} +

+ +
+
+ +
+ + + + + +
+ +
+ + {runtimeContainers.map((a) => ( + + ))} + + {componentContainers.map((a) => ( + + ))} + + {wheels.map((a) => ( + + ))} + + {helmCharts.map((a) => ( + + ))} + + {crates.map((a) => ( + + ))} +
+ +

+ Every tag and command is click-to-copy. For TensorRT-LLM use the NGC container — not the{" "} + ai-dynamo[trtllm] wheel. +

+
+ + ); +} diff --git a/docs/fern/components/BackendVersionMatrix.tsx b/docs/fern/components/BackendVersionMatrix.tsx new file mode 100644 index 000000000000..6c84b2d067d0 --- /dev/null +++ b/docs/fern/components/BackendVersionMatrix.tsx @@ -0,0 +1,276 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * BackendVersionMatrix — Dynamo release × backend-pin table for the + * Compatibility reference page. mode="current" shows main (ToT) and the + * current stable release; mode="all" shows the full release history with a + * diff-strip highlight on every pin that changed versus the chronologically + * previous non-model-build release (RELEASES is newest-first, so "previous" + * is the next comparable entry in the array; model-build side branches are + * skipped both as rows-to-compare-against and never break the main lineage). + * + * Server component (no "use client"); shares .dynref-* base classes from + * ReferenceStyles.tsx and carries only its own .dynref-vm-* layout rules. + */ + + +import { + RELEASES, + MAIN_TOT, + CURRENT_VERSION, + type BackendPins, + type Release, + type ReleaseKind, +} from "./releases.data"; + +const VM_CSS = ` +.dynref-vm-scroll { + overflow-x: auto; + margin: 16px 0 8px; +} + +.dynref-vm-table { + width: 100%; + border-collapse: collapse; + font-size: 13px; +} + +.dynref-vm-table th { + padding: 6px 8px; + text-align: left; + font-size: 12px; + font-weight: 600; + text-transform: uppercase; + letter-spacing: 0.04em; + color: var(--pst-color-text-muted); + border-bottom: 1px solid var(--border, var(--grayscale-a5)); + white-space: nowrap; +} + +.dynref-vm-table td { + padding: 6px 8px; + border-bottom: 1px solid var(--border, var(--grayscale-a5)); + color: var(--pst-color-text-base); + vertical-align: top; +} + +.dynref-vm-version { white-space: nowrap; } + +.dynref-vm-date { + display: block; + margin-top: 2px; + font-size: 12px; + color: var(--pst-color-text-muted); +} + +.dynref-vm-partial { + display: block; + margin-top: 2px; + font-size: 12px; + font-style: italic; + color: var(--pst-color-text-muted); +} + +.dynref-vm-changed { + display: inline-block; + background: rgba(118, 185, 0, 0.1); + padding-inline: 4px; + border-left: 2px solid #76B900; + border-radius: 0 4px 4px 0; +} + +.dynref-vm-nixl { display: inline-block; } + +/* One line per backend, always — the label column keeps the three rows + aligned into a compact sub-table. */ +.dynref-vm-nixl-entry { + display: block; + white-space: nowrap; + line-height: 1.5; +} + +.dynref-vm-nixl-label { + margin-left: 5px; + font-size: 12px; + color: var(--pst-color-text-muted); +} + +.dynref-vm-kind { white-space: nowrap; } + +.dynref-vm-dash { color: var(--pst-color-text-muted); } +`; + +type PinKey = "sglang" | "trtllm" | "vllm"; +type NixlKey = "nixlSglang" | "nixlTrtllm" | "nixlVllm"; + +const PIN_COLUMNS: { key: PinKey; label: string }[] = [ + { key: "sglang", label: "SGLang" }, + { key: "trtllm", label: "TensorRT-LLM" }, + { key: "vllm", label: "vLLM" }, +]; + +/* Sub-entry labels are abbreviated (SGL / TRT / vLLM) so the stacked NIXL + sub-rows stay compact. */ +const NIXL_COLUMNS: { key: NixlKey; label: string }[] = [ + { key: "nixlSglang", label: "SGL" }, + { key: "nixlTrtllm", label: "TRT" }, + { key: "nixlVllm", label: "vLLM" }, +]; + +const KIND_BADGE: Record = { + stable: { variant: "green", label: "GA release" }, + patch: { variant: "gray", label: "Patch" }, + "platform-preview": { variant: "amber", label: "Early access" }, + "model-build": { variant: "amber", label: "Model build" }, +}; + +/** Chronologically previous non-model-build release (RELEASES is newest-first). */ +function previousComparable(index: number): Release | undefined { + for (let j = index + 1; j < RELEASES.length; j++) { + if (RELEASES[j].kind !== "model-build") return RELEASES[j]; + } + return undefined; +} + +function Pin({ value, changed }: { value?: string; changed?: boolean }) { + if (!value) return —; + const pin = {value}; + return changed ? {pin} : pin; +} + +function NixlCell({ pins, prev }: { pins?: BackendPins; prev?: BackendPins }) { + const entries = NIXL_COLUMNS.filter(({ key }) => pins?.[key]); + if (!pins || entries.length === 0) return —; + + const changedFor = (key: NixlKey) => (prev ? prev[key] !== pins[key] : false); + + return ( + + {entries.map(({ key, label }) => ( + + + {label} + + ))} + + ); +} + +function PinCells({ pins, prev }: { pins?: BackendPins; prev?: BackendPins }) { + return ( + <> + {PIN_COLUMNS.map(({ key }) => ( + + + + ))} + + + + + ); +} + +function ReleaseRow({ release, prev }: { release: Release; prev?: Release }) { + const badge = KIND_BADGE[release.kind]; + return ( + + + {release.version} + {release.date && {release.date}} + {release.partial && partial coverage} + + + {badge.label} + + + + + + + ); +} + +export function BackendVersionMatrix({ mode = "current" }: { mode?: "current" | "all" }) { + const current = RELEASES.find((release) => release.version === CURRENT_VERSION); + + return ( +
+ +
+ + + + + + {PIN_COLUMNS.map(({ key, label }) => ( + + ))} + + + + + + {mode === "current" ? ( + <> + + + + + + + {current && ( + + + + + + + )} + + ) : ( + RELEASES.map((release, index) => ( + + )) + )} + +
DynamoType{label}NIXLUCX
main (ToT) + development + + +
+ {current.version} + {current.date && {current.date}} + + GA release + + +
+
+ {mode === "all" && ( + <> +

+ + Highlighted + {" "} + pins changed relative to the previous release; unmarked pins are unchanged (patch + releases typically re-ship their base release’s pins). +

+

+ Early access rows show branch build pins from container/context.yaml; not every backend + ships a published container for those tags. +

+

+ Backend versions listed are the only versions tested and supported for each release. + TensorRT-LLM does not support Python 3.11. +

+ + )} +
+ ); +} diff --git a/docs/fern/components/CompatibilityHero.tsx b/docs/fern/components/CompatibilityHero.tsx new file mode 100644 index 000000000000..4321ec4d917a --- /dev/null +++ b/docs/fern/components/CompatibilityHero.tsx @@ -0,0 +1,222 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * CompatibilityHero — current-release summary panel for the Compatibility page. + * + * Renders the current stable release (version, date, release-notes link), the + * per-backend engine + NIXL + CUDA toolkit pins, and the platform requirement + * rows (GPU, OS, arch) from releases.data.ts. Server component; shared vocabulary + * (panel, eyebrow, label, mono, chips, badges) comes from ReferenceStyles — + * place on the page alongside this component. Only the + * .dynref-hero-* layout classes are defined here. + */ + +import { + RELEASES, + CURRENT_VERSION, + CURRENT_DATE, + CURRENT_TAG, + CUDA_HISTORY, + PLATFORM, +} from "./releases.data"; + +const HERO_CSS = ` +.dynref-hero-header { + display: flex; + flex-wrap: wrap; + align-items: flex-end; + justify-content: space-between; + gap: 8px 16px; + margin-bottom: 16px; +} + +.dynref-hero-title { + display: flex; + flex-wrap: wrap; + align-items: center; + gap: 10px; + margin: 0; + color: var(--pst-color-text-base); + font-size: 22px; + font-weight: 600; + line-height: 1.2; +} + +.dynref-hero-meta { + margin: 0; +} + +.dynref-hero-meta a { + color: inherit; + text-decoration: underline; +} + +.dynref-hero-backends { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(170px, 1fr)); + gap: 10px; +} + +.dynref-hero-backend { + padding: 12px 14px; + border: 1px solid var(--border, var(--grayscale-a5)); + border-radius: 10px; +} + +.dark .dynref-hero-backend { + background: #1d1d1d; + border-color: #2e2e2e; +} + +.dynref-hero-backend-name { + display: block; + color: var(--pst-color-text-base); + font-size: 14px; + font-weight: 600; +} + +.dynref-hero-pin { + display: block; + margin: 4px 0 2px; + color: var(--pst-color-text-base); + font-size: 15px; +} + +.dynref-hero-backend-cuda { + display: flex; + flex-wrap: wrap; + gap: 4px; + margin-top: 6px; +} + +.dynref-hero-reqs { + display: grid; + grid-template-columns: 88px 1fr; + gap: 8px 12px; + align-items: baseline; + margin-top: 16px; + padding-top: 14px; + border-top: 1px solid var(--border, var(--grayscale-a5)); +} + +.dynref-hero-req-values { + display: flex; + flex-wrap: wrap; + align-items: center; + gap: 0 4px; +} +`; + +interface BackendCard { + label: string; + pin?: string; + nixl?: string; +} + +export function CompatibilityHero() { + const current = RELEASES.find((r) => r.version === CURRENT_VERSION); + const pins = current?.pins ?? {}; + + const backends: BackendCard[] = [ + { label: "SGLang", pin: pins.sglang, nixl: pins.nixlSglang }, + { label: "TensorRT-LLM", pin: pins.trtllm, nixl: pins.nixlTrtllm }, + { label: "vLLM", pin: pins.vllm, nixl: pins.nixlVllm }, + ]; + + return ( + <> + +
+
+
+

Current release

+
+ Dynamo {CURRENT_VERSION} + GA release +
+
+

+ Released {current?.date ?? CURRENT_DATE} ·{" "} + Release notes +

+
+ +
+ {backends.map((backend) => ( +
+ {backend.label} + {backend.pin} + + NIXL {backend.nixl} + +
+ {CUDA_HISTORY.filter( + (r) => r.version === CURRENT_TAG && r.backend === backend.label, + ).map((r) => ( + + CUDA {r.toolkit} + + ))} +
+
+ ))} +
+ +
+ GPU +
+ {PLATFORM.gpus.map((gpu) => ( + + {gpu} + + ))} +
+ + OS +
+ {PLATFORM.os.map((row) => ( + + {row.name} {row.version} + {row.status === "Experimental" ? " · experimental" : ""} + + ))} +
+ + Arch +
+ {PLATFORM.arch.map((arch) => ( + + {arch} + + ))} +
+ +
+ +

+ CUDA 12 discontinued as of {CURRENT_VERSION}. +

+ +

+ Early access: model builds are tracked in{" "} + + Model Early Access Builds + + ; platform previews under{" "} + + Early Access Artifacts + + . +

+
+ + ); +} diff --git a/docs/fern/components/CudaDriverMatrix.tsx b/docs/fern/components/CudaDriverMatrix.tsx new file mode 100644 index 000000000000..c19c51cbed84 --- /dev/null +++ b/docs/fern/components/CudaDriverMatrix.tsx @@ -0,0 +1,260 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * CudaDriverMatrix — CUDA toolkit / minimum-driver support for the + * Compatibility reference page. mode="current" is a compact per-backend + * Backend | CUDA toolkit | Min driver table for the current release + * (CURRENT_TAG). mode="all" renders the CUDA driver ladder (toolkit lanes + * derived from the data, one row per version, newest first — the empty + * CUDA 12 lanes above v1.3.0 visualize the CUDA 12 → 13 cutoff; each pill + * carries the lane's minimum driver, e.g. ≥575) followed by the full history + * table and CUDA_NOTES. + * + * Server component (no "use client"); shares .dynref-* base classes from + * ReferenceStyles.tsx and carries only its own .dynref-cuda-* layout rules. + */ + +import { Fragment } from "react"; + +import { CUDA_HISTORY, CUDA_NOTES, CURRENT_TAG, type CudaRow } from "./releases.data"; + +const CUDA_CSS = ` +.dynref-cuda-ladder { + display: grid; + gap: 6px 8px; + align-items: center; + margin: 16px 0 6px; +} + +.dynref-cuda-lane-head { + text-align: center; + font-size: 12px; + font-weight: 600; + text-transform: uppercase; + letter-spacing: 0.04em; + color: var(--pst-color-text-muted); + padding-bottom: 2px; + border-bottom: 1px solid var(--border, var(--grayscale-a5)); +} + +.dynref-cuda-corner { + border-bottom: 1px solid var(--border, var(--grayscale-a5)); +} + +.dynref-cuda-vlabel { + font-size: 12.5px; + color: var(--pst-color-text-base); + white-space: nowrap; +} + +.dynref-cuda-lanecell { + display: flex; + justify-content: center; +} + +.dynref-cuda-exp { border: 1px dashed currentColor; } + +.dynref-cuda-caption { + margin: 6px 0 20px; + font-size: 12px; + color: var(--pst-color-text-muted); +} + +.dynref-cuda-scroll { + overflow-x: auto; + margin: 12px 0 8px; +} + +.dynref-cuda-table { + width: 100%; + border-collapse: collapse; + font-size: 13px; +} + +.dynref-cuda-table th { + padding: 8px 10px; + text-align: left; + font-size: 12px; + font-weight: 600; + text-transform: uppercase; + letter-spacing: 0.04em; + color: var(--pst-color-text-muted); + border-bottom: 1px solid var(--border, var(--grayscale-a5)); + white-space: nowrap; +} + +.dynref-cuda-table td { + padding: 8px 10px; + border-bottom: 1px solid var(--border, var(--grayscale-a5)); + color: var(--pst-color-text-base); + vertical-align: top; +} + +.dynref-cuda-vcell { white-space: nowrap; } + +.dynref-cuda-note-cell { + font-size: 12px; + color: var(--pst-color-text-muted); +} +`; + +/** Unique toolkits, ascending (lanes are derived from the data, never hardcoded). */ +function toolkitLanes(): string[] { + return [...new Set(CUDA_HISTORY.map((row) => row.toolkit))].sort( + (a, b) => parseFloat(a) - parseFloat(b), + ); +} + +/** Unique versions in data order (CUDA_HISTORY is newest-first). */ +function uniqueVersions(): string[] { + return [...new Set(CUDA_HISTORY.map((row) => row.version))]; +} + +function rowsFor(version: string): CudaRow[] { + return CUDA_HISTORY.filter((row) => row.version === version); +} + +/** "575.xx+" → "≥575": ladder-pill label derived from CudaRow.minDriver. */ +function minDriverLabel(minDriver: string): string { + return `≥${minDriver.replace(/\.xx\+$/, "").replace(/\+$/, "")}`; +} + +function CudaChip({ label, experimental }: { label: string; experimental?: boolean }) { + const classes = `dynref-chip dynref-chip--cuda${experimental ? " dynref-cuda-exp" : ""}`; + return {label}; +} + +function CudaLadder() { + const lanes = toolkitLanes(); + const versions = uniqueVersions(); + const gridColumns = { gridTemplateColumns: `60px repeat(${lanes.length}, minmax(64px, 1fr))` }; + + return ( + <> +
+
+ {lanes.map((lane) => ( +
+ CUDA {lane} +
+ ))} + {versions.map((version) => { + const rows = rowsFor(version); + return ( + +
v{version}
+ {lanes.map((lane) => { + const laneRows = rows.filter((row) => row.toolkit === lane); + const experimental = + laneRows.length > 0 && laneRows.every((row) => row.note === "Experimental"); + const drivers = [...new Set(laneRows.map((row) => minDriverLabel(row.minDriver)))]; + return ( +
+ {laneRows.length > 0 && ( + + )} +
+ ); + })} + + ); + })} +
+

+ Pills show the minimum driver for each version × toolkit lane. CUDA 12 lanes end at + v{CURRENT_TAG} — CUDA 12 container images are discontinued. +

+ + ); +} + +function CurrentTable() { + const rows = rowsFor(CURRENT_TAG); + return ( +
+ + + + + + + + + + {rows.map((row) => ( + + + + + + ))} + +
BackendCUDA toolkitMin driver
{row.backend} + + {row.minDriver}
+
+ ); +} + +function HistoryTable() { + const versions = uniqueVersions(); + return ( +
+ + + + + + + + + + + + {versions.map((version) => { + const rows = rowsFor(version); + return rows.map((row, index) => ( + + {index === 0 && ( + + )} + + + + + + )); + })} + +
VersionBackendCUDA toolkitMin driverNotes
+ v{version} + {row.backend} + + {row.minDriver}{row.note ?? ""}
+
+ ); +} + +export function CudaDriverMatrix({ mode = "current" }: { mode?: "current" | "all" }) { + return ( +
+ + {mode === "current" ? ( + + ) : ( + <> + CUDA driver ladder + + Full history + + {CUDA_NOTES.map((note) => ( +

+ {note} +

+ ))} + + )} +
+ ); +} diff --git a/docs/fern/components/FeatureHeatmap.tsx b/docs/fern/components/FeatureHeatmap.tsx new file mode 100644 index 000000000000..aef38ee7fd7d --- /dev/null +++ b/docs/fern/components/FeatureHeatmap.tsx @@ -0,0 +1,352 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * FeatureHeatmap — at-a-glance feature-by-backend support grid for the + * Compatibility reference page. Renders entirely from FEATURES in + * releases.data.ts; per-backend coverage scores are computed, never + * hardcoded. + * + * Status-cell scheme: "yes" and "caveat" cells use the shared tinted-chip + * treatment (translucent fill + 1px border, matching .dynref-badge--green / + * --amber in ReferenceStyles.tsx) so green stays an accent, never a solid + * wallpaper. Experimental keeps the dashed amber outline; not-supported stays + * dim neutral. Cells with a note carry a superscript footnote marker + * (numbered in row-major grid order) resolved in an ordered list below the + * grid; the title attribute is kept as a hover bonus. + * + * Server component (no "use client"); shares .dynref-* base classes from + * ReferenceStyles.tsx and carries only its own .dynref-heat-* layout rules. + */ + +import { Fragment } from "react"; + +import { FEATURES, type FeatureCell } from "./releases.data"; + +const HEAT_CSS = ` +.dynref-heat-legend { + display: flex; + align-items: center; + flex-wrap: wrap; + gap: 12px; + font-size: 12px; + color: var(--pst-color-text-muted); +} + +.dynref-heat-legend-item { + display: inline-flex; + align-items: center; + gap: 5px; +} + +.dynref-heat-swatch { + display: inline-block; + box-sizing: border-box; + width: 10px; + height: 10px; + border-radius: 3px; +} + +.dynref-heat-swatch--yes { + background: var(--dynref-green-bg); + border: 1px solid var(--dynref-green-border); +} +.dark .dynref-heat-swatch--yes { + background: var(--dynref-green-bg); + border-color: var(--dynref-green-border); +} + +.dynref-heat-swatch--caveat { + background: var(--dynref-amber-bg); + border: 1px solid var(--dynref-amber-border); +} +.dark .dynref-heat-swatch--caveat { + background: var(--dynref-amber-bg); + border-color: var(--dynref-amber-border); +} + +.dynref-heat-swatch--wip { + background: transparent; + border: 1.5px dashed #b97a17; +} + +.dynref-heat-swatch--no { background: #ececec; } +.dark .dynref-heat-swatch--no { background: #242424; } + +.dynref-heat-grid { + display: grid; + grid-template-columns: minmax(0, 1.6fr) repeat(3, minmax(64px, 1fr)); + gap: 6px; + font-size: 13px; +} + +.dynref-heat-colhead { + align-self: end; + text-align: center; + font-size: 12.5px; + font-weight: 600; + color: var(--pst-color-text-base); +} + +.dynref-heat-score { + display: block; + margin-top: 2px; + font-size: 11.5px; + font-weight: 400; + color: #5a8c00; +} +.dark .dynref-heat-score { color: #76B900; } + +.dynref-heat-feature { + align-self: center; + min-width: 0; + color: var(--pst-color-text-base); +} + +.dynref-heat-cell { + box-sizing: border-box; + height: 26px; + border-radius: 6px; + display: flex; + align-items: center; + justify-content: center; +} + +.dynref-heat-cell--titled { cursor: help; } + +.dynref-heat-cell--yes { + background: var(--dynref-green-bg); + border: 1px solid var(--dynref-green-border); + color: var(--dynref-green-fg); +} +.dark .dynref-heat-cell--yes { + background: var(--dynref-green-bg); + border-color: var(--dynref-green-border); + color: var(--dynref-green-fg); +} + +.dynref-heat-cell--caveat { + background: var(--dynref-amber-bg); + border: 1px solid var(--dynref-amber-border); + color: var(--dynref-amber-fg); +} +.dark .dynref-heat-cell--caveat { + background: var(--dynref-amber-bg); + border-color: var(--dynref-amber-border); + color: var(--dynref-amber-fg); +} + +.dynref-heat-cell--wip { + background: transparent; + border: 1.5px dashed #b97a17; + color: #B97A17; +} +.dark .dynref-heat-cell--wip { color: #EF9F27; } + +.dynref-heat-cell--no { background: #ececec; } +.dark .dynref-heat-cell--no { background: #242424; } + +.dynref-heat-dash { color: var(--pst-color-text-muted); } + +.dynref-heat-fn { + margin-left: 2px; + font-size: 11px; + font-weight: 600; + line-height: 1; +} + +.dynref-heat-footnotes { + margin: 12px 0 0; + padding-left: 20px; + font-size: 12px; + color: var(--pst-color-text-muted); +} + +.dynref-heat-footnotes li { margin: 2px 0; } +`; + +const BACKENDS = [ + { key: "sglang", label: "SGLang" }, + { key: "trtllm", label: "TRT-LLM" }, + { key: "vllm", label: "vLLM" }, +] as const; + +type BackendKey = (typeof BACKENDS)[number]["key"]; + +const STATUS_LABEL: Record = { + yes: "Supported", + caveat: "Caveat", + wip: "Experimental", + no: "Not supported", +}; + +interface Footnote { + feature: string; + backend: string; + note: string; +} + +/** Every noted cell, numbered in row-major grid order (derived, never hardcoded). */ +const FOOTNOTES: Footnote[] = FEATURES.flatMap((feature) => + BACKENDS.flatMap((backend) => { + const note = feature[backend.key].note; + return note ? [{ feature: feature.name, backend: backend.label, note }] : []; + }), +); + +/** 1-based footnote number for a noted cell; undefined when the cell has no note. */ +function footnoteIndex(feature: string, backend: string): number | undefined { + const i = FOOTNOTES.findIndex((fn) => fn.feature === feature && fn.backend === backend); + return i === -1 ? undefined : i + 1; +} + +function coverageScore(key: BackendKey): string { + const supported = FEATURES.filter((feature) => { + const status = feature[key].status; + return status === "yes" || status === "caveat"; + }).length; + return `${supported} / ${FEATURES.length}`; +} + +function CheckGlyph() { + return ( + + ); +} + +function AlertGlyph() { + return ( + + ); +} + +function FlaskGlyph() { + return ( + + ); +} + +function StatusCellContent({ status }: { status: FeatureCell["status"] }) { + if (status === "yes") return ; + if (status === "caveat") return ; + if (status === "wip") return ; + return —; +} + +function StatusCell({ cell, feature, backend }: { cell: FeatureCell; feature: string; backend: string }) { + const classes = [ + "dynref-heat-cell", + `dynref-heat-cell--${cell.status}`, + cell.note ? "dynref-heat-cell--titled" : "", + ] + .filter(Boolean) + .join(" "); + const label = `${feature} on ${backend}: ${STATUS_LABEL[cell.status]}${cell.note ? ` — ${cell.note}` : ""}`; + const index = cell.note ? footnoteIndex(feature, backend) : undefined; + return ( +
+ + {index !== undefined && {index}} +
+ ); +} + +export function FeatureHeatmap() { + return ( +
+ +
+ Feature support by backend +
+ + + Supported + + + + Caveat + + + + Experimental + + + + Not supported + +
+
+
+
+ {BACKENDS.map((backend) => ( +
+ {backend.label} + {coverageScore(backend.key)} +
+ ))} + {FEATURES.map((feature) => ( + +
{feature.name}
+ {BACKENDS.map((backend) => ( + + ))} + + ))} +
+ {FOOTNOTES.length > 0 && ( +
    + {FOOTNOTES.map((fn) => ( +
  1. + {fn.feature} · {fn.backend}: {fn.note} +
    2. + ))} +
+ )} +

Superscripts reference the numbered notes above; full per-backend detail follows.

+
+ ); +} diff --git a/docs/fern/components/ModelEABuildCards.tsx b/docs/fern/components/ModelEABuildCards.tsx new file mode 100644 index 000000000000..53db564b5ea7 --- /dev/null +++ b/docs/fern/components/ModelEABuildCards.tsx @@ -0,0 +1,348 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * ModelEABuildCards — card grid of model early-access builds from + * releases.data.ts, grouped by release line (or filtered to one line via the + * `line` prop). Each card shows the GA-path badge, a click-to-copy docker + * pull for the primary runtime, runtime/ship metadata, the build status + * line, an Images/Wheels/Helm/Crates coverage-dots row, and the recipe link. + * + * Also exports EaCoverageDots — a standalone inline coverage-dots row for a + * platform-preview version (reads PLATFORM_PREVIEW_COVERAGE), for use next + * to platform-preview mentions on the release-artifacts page. + * + * Server component; shared vocabulary (badges, copy buttons, labels, mono) + * comes from ReferenceStyles — place on the page + * alongside this component. Only the .dynref-ea-* layout classes are + * defined here. + */ + +import { + MODEL_EA_BUILDS, + PLATFORM_PREVIEW_COVERAGE, + type Coverage, + type GaPath, + type ModelEaBuild, +} from "./releases.data"; + +/* Coverage-dots rules are shared by ModelEABuildCards and the standalone + EaCoverageDots, so each component ships them in its own +
+ {builds.map((build) => ( + + ))} +
+ + ); + } + + // Group by release line, preserving first-seen data order. + const groups: { line: string; builds: ModelEaBuild[] }[] = []; + for (const build of MODEL_EA_BUILDS) { + const group = groups.find((g) => g.line === build.releaseLine); + if (group) { + group.builds.push(build); + } else { + groups.push({ line: build.releaseLine, builds: [build] }); + } + } + + return ( + <> + + {groups.map((group) => ( +
+

{group.line} release

+
+ {group.builds.map((build) => ( + + ))} +
+
+ ))} + + ); +} + +/* Standalone inline coverage-dots row for a platform-preview version. + Returns null when the version is not in PLATFORM_PREVIEW_COVERAGE. */ +export function EaCoverageDots({ version }: { version: string }) { + const coverage = PLATFORM_PREVIEW_COVERAGE[version]; + if (!coverage) return null; + return ( + + + + + ); +} diff --git a/docs/fern/components/PinnedEnvironment.tsx b/docs/fern/components/PinnedEnvironment.tsx new file mode 100644 index 000000000000..799c5d10675e --- /dev/null +++ b/docs/fern/components/PinnedEnvironment.tsx @@ -0,0 +1,226 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * PinnedEnvironment — one copy-paste block that pins every install path + * (backend runtime container, frontend + operator images, Helm chart, and + * wheel) to the current release. Every command string is assembled from the + * ARTIFACTS clipboard payloads and CURRENT_* consts in releases.data.ts — + * no registry or version literals live here. + * + * The backend switch is CSS-only: three hidden radios sit first inside the + * panel and :checked general-sibling selectors toggle which pre-rendered + * script block AND which "Copy all" button shows — one of each per backend, + * so the copy payload always matches the visible script exactly (same + * mechanism as ArtifactBrowser's filter rail). TensorRT-LLM ships via the + * NGC container, so its variant omits the wheel line and carries a comment + * instead. + * + * Server component; shared vocabulary (panel, eyebrow, copy buttons) comes + * from ReferenceStyles — place on the page alongside + * this component. Only the .dynref-pe-* layout classes are defined here. + */ + +import { ARTIFACTS, CURRENT_VERSION, CURRENT_WHEEL, CURRENT_TAG } from "./releases.data"; + +const PE_CSS = ` +/* Inputs are hidden by the shared .dynref-vh (visually hidden, focusable) + class so the backend rail stays keyboard-operable; the :focus-visible + rules below paint the ring on the matching pill. */ + +.dynref-pe-rail { + display: flex; + flex-wrap: wrap; + gap: 8px; + margin: 0 0 12px; +} + +.dynref-pe-pill { + display: inline-flex; + align-items: center; + min-height: 30px; + padding: 6px 10px; + border: 1px solid var(--border, var(--grayscale-a5)); + border-radius: var(--rounded, 6px); + background: transparent; + color: var(--pst-color-text-base); + font-size: 12.5px; + line-height: 1; + cursor: pointer; +} + +.dynref-pe-pill:hover { + border-color: var(--nv-color-green, #76B900); +} + +#pe-sglang:checked ~ .dynref-pe-rail label[for="pe-sglang"], +#pe-trtllm:checked ~ .dynref-pe-rail label[for="pe-trtllm"], +#pe-vllm:checked ~ .dynref-pe-rail label[for="pe-vllm"] { + border-color: var(--nv-color-green, #76B900); + box-shadow: 0 0 0 1px var(--nv-color-green, #76B900); + background: rgba(118, 185, 0, 0.08); + font-weight: 700; +} + +#pe-sglang:focus-visible ~ .dynref-pe-rail label[for="pe-sglang"], +#pe-trtllm:focus-visible ~ .dynref-pe-rail label[for="pe-trtllm"], +#pe-vllm:focus-visible ~ .dynref-pe-rail label[for="pe-vllm"] { + outline: 2px solid var(--nv-color-green, #76B900); + outline-offset: 1px; +} + +/* Pre-rendered script blocks and Copy-all buttons: hidden by default, the + checked backend's pair shows. Payload and visible text come from the same + string, so they always match. */ +.dynref-pe-script, +.dynref-pe-copyall { + display: none; +} + +#pe-sglang:checked ~ .dynref-pe-script[data-backend="sglang"], +#pe-trtllm:checked ~ .dynref-pe-script[data-backend="trtllm"], +#pe-vllm:checked ~ .dynref-pe-script[data-backend="vllm"] { + display: block; +} + +#pe-sglang:checked ~ .dynref-panel-header .dynref-pe-copyall[data-backend="sglang"], +#pe-trtllm:checked ~ .dynref-panel-header .dynref-pe-copyall[data-backend="trtllm"], +#pe-vllm:checked ~ .dynref-panel-header .dynref-pe-copyall[data-backend="vllm"] { + display: inline-flex; +} + +/* Prominent green-tinted Copy all — composes with .dynref-copy for the + click-to-copy binder, glyph, and copied-state feedback. */ +.dynref-pe-copyall { + align-items: center; + padding: 8px 14px; + border: 1px solid var(--dynref-green-border); + border-radius: 8px; + background: var(--dynref-green-bg); + color: var(--dynref-green-fg); + font-size: 12.5px; + font-weight: 700; +} + +.dynref-pe-copyall:hover { + border-color: var(--nv-color-green, #76B900); + box-shadow: 0 0 0 1px var(--nv-color-green, #76B900); +} + +.dynref-pe-script { + margin: 0; + padding: 14px 16px; + border: 1px solid var(--border, var(--grayscale-a5)); + border-radius: 8px; + background: #fcfcfc; + color: var(--pst-color-text-base); + font-family: var(--pst-font-family-monospace, ui-monospace, SFMono-Regular, Menlo, monospace); + font-size: 12px; + line-height: 1.7; + white-space: pre; + overflow-x: auto; +} + +.dark .dynref-pe-script { + background: #0f0f0f; + border-color: #2b2b2b; +} +`; + +interface Backend { + id: "sglang" | "trtllm" | "vllm"; + label: string; + runtime: string; + /** ai-dynamo wheel extra; null = ships via container only (comment line instead). */ + extra: string | null; +} + +/* Rail order per the reference pages' backend convention; vLLM is the + default selection. */ +const BACKENDS: Backend[] = [ + { id: "sglang", label: "SGLang", runtime: "sglang-runtime", extra: "sglang" }, + { id: "trtllm", label: "TensorRT-LLM", runtime: "tensorrtllm-runtime", extra: null }, + { id: "vllm", label: "vLLM", runtime: "vllm-runtime", extra: "vllm" }, +]; + +/** docker pull line for a container artifact's CURRENT_TAG image reference. */ +function containerPull(name: string): string | null { + const artifact = ARTIFACTS.find((a) => a.category === "container" && a.name === name); + const tag = artifact?.tags.find((t) => t.label === CURRENT_TAG); + return tag ? `docker pull ${tag.clipboard}` : null; +} + +/** Break the long helm install one-liner with a shell line continuation so + * the script block fits without horizontal clipping. Applied to the shared + * string, so the visible pre and the Copy-all payload stay byte-identical — + * a backslash-continued command is still paste-safe shell. */ +function wrapHelmInstall(line: string): string { + return line.replace(/^(helm install \S+) (oci:)/, "$1 \\\n $2"); +} + +/** Full multi-line pinned-install script for one backend. */ +function buildScript(backend: Backend): string { + const helm = ARTIFACTS.find((a) => a.category === "helm" && a.name === "dynamo-platform"); + const helmLine = helm?.tags[0]?.clipboard; + const lines: (string | null)[] = [ + containerPull(backend.runtime), + containerPull("dynamo-frontend"), + containerPull("kubernetes-operator"), + helmLine ? wrapHelmInstall(helmLine) : null, + backend.extra + ? `uv pip install "ai-dynamo[${backend.extra}]==${CURRENT_WHEEL}"` + : "# TensorRT-LLM ships via the NGC container", + ]; + return lines.filter((line): line is string => line !== null).join("\n"); +} + +export function PinnedEnvironment() { + const scripts = BACKENDS.map((backend) => ({ backend, script: buildScript(backend) })); + + return ( + <> + +
+ + + + +
+
+

Pinned environment

+

Everything pinned to {CURRENT_VERSION}

+
+ {scripts.map(({ backend, script }) => ( + + ))} +
+ +
+ {BACKENDS.map((backend) => ( + + ))} +
+ + {scripts.map(({ backend, script }) => ( + 
+            {script}
+
+ ))} + +

Assembled from the current release's artifact inventory.

+
+ + ); +} diff --git a/docs/fern/components/ReferenceStyles.tsx b/docs/fern/components/ReferenceStyles.tsx new file mode 100644 index 000000000000..8b4f688e691a --- /dev/null +++ b/docs/fern/components/ReferenceStyles.tsx @@ -0,0 +1,391 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Reference page shared component styles. + * + * Shared style vocabulary for the Reference custom components + * (CompatibilityHero and the other reference-page components). Each component + * carries only its own layout classes; the panel, eyebrow, label, mono, chip, + * badge, and click-to-copy treatments all live here so the pages read as one + * system. Chip variants replicate the .dynamo-chip-* palette in main.css. + * + * Delivered as a page-level ; +} diff --git a/docs/fern/components/ReleaseHeader.tsx b/docs/fern/components/ReleaseHeader.tsx new file mode 100644 index 000000000000..5484c95370f6 --- /dev/null +++ b/docs/fern/components/ReleaseHeader.tsx @@ -0,0 +1,217 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * ReleaseHeader — masthead panel for a Release Notes page. + * + * Renders the release title row (version, GA badge, date from releases.data), + * GitHub / Artifacts link chips, and a stat-tile strip (PRs merged, + * contributors, first-time contributors, breaking changes, known issues) + * driven entirely by RELEASE_STATS[version] in releases.data — pages pass + * only the version. Optional stats simply drop their tiles when absent; + * a version without a RELEASE_STATS entry renders the header with no tiles. + * Breaking-changes and known-issues tiles deep-link into the Deprecations / + * Known Issues pages. + * + * Server component (no "use client"); shared vocabulary (panel, eyebrow, + * badges, muted text) comes from ReferenceStyles — place + * on the page alongside this component. Only the .dynref-rh-* layout classes + * are defined here. All stat tiles share the neutral border; semantic color + * lives only in the number (amber for breaking changes / known issues). + */ + +import { RELEASES, RELEASE_STATS } from "./releases.data"; + +const RH_CSS = ` +.dynref-rh-header { + display: flex; + flex-wrap: wrap; + align-items: flex-end; + justify-content: space-between; + gap: 8px 16px; +} + +.dynref-rh-title { + display: flex; + flex-wrap: wrap; + align-items: center; + gap: 10px; + margin: 0; + color: var(--pst-color-text-base); + font-size: 21px; + font-weight: 600; + line-height: 1.2; +} + +.dynref-rh-links { + display: flex; + flex-wrap: wrap; + align-items: center; + gap: 8px; +} + +.dynref-rh-link { + display: inline-flex; + align-items: center; + padding: 3px 10px; + border: 1px solid var(--border, var(--grayscale-a5)); + border-radius: 6px; + color: var(--pst-color-text-muted); + font-size: 12px; + font-weight: 600; + text-decoration: none; + white-space: nowrap; +} + +.dark .dynref-rh-link { + border-color: #383838; +} + +.dynref-rh-link:hover { + border-color: var(--pst-color-text-muted); + color: var(--pst-color-text-base); +} + +.dynref-rh-tiles { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(105px, 1fr)); + gap: 10px; + margin-top: 16px; +} + +/* All tiles share the neutral border — semantic color lives in the number + only. Rendered as when the tile deep-links; same box either way. */ +.dynref-rh-tile { + display: block; + padding: 10px 12px; + border: 1px solid var(--border, var(--grayscale-a5)); + border-radius: 10px; + text-decoration: none; +} + +.dark .dynref-rh-tile { + background: #1d1d1d; + border-color: #2e2e2e; +} + +a.dynref-rh-tile:hover { + border-color: var(--pst-color-text-muted); +} + +.dynref-rh-num { + display: block; + color: var(--pst-color-text-base); + font-size: 18px; + font-weight: 600; + font-variant-numeric: tabular-nums; + line-height: 1.3; +} + +.dynref-rh-num--amber { + color: var(--dynref-amber-fg); +} + +.dynref-rh-tilelabel { + display: block; + margin-top: 2px; + color: var(--pst-color-text-muted); + font-size: 11px; +} +`; + +interface StatTile { + label: string; + value: number; + amber: boolean; + href?: string; +} + +export function ReleaseHeader(props: { version: string }) { + const release = RELEASES.find((r) => r.version === props.version); + /* All counts come from RELEASE_STATS in releases.data — the single source + shared with UpgradePanel and the ledger accordion titles. */ + const stats = RELEASE_STATS[props.version]; + /* v1.3.0 -> "v130" — anchor shape shared with the Deprecations and Known + Issues pages' per-version section ids. */ + const versionAnchor = props.version.replace(/\./g, ""); + + const tiles: StatTile[] = []; + if (stats) { + if (typeof stats.prs === "number") { + tiles.push({ label: "PRs merged", value: stats.prs, amber: false }); + } + if (typeof stats.contributors === "number") { + tiles.push({ label: "Contributors", value: stats.contributors, amber: false }); + } + if (typeof stats.firstTimers === "number") { + tiles.push({ label: "First-time contributors", value: stats.firstTimers, amber: false }); + } + tiles.push({ + label: "Breaking changes", + value: stats.breaking, + amber: true, + href: `/dynamo/dev/reference/releases/deprecations#${versionAnchor}`, + }); + tiles.push({ + label: "Known issues", + value: stats.knownIssues, + amber: true, + href: + stats.knownIssues > 0 + ? `/dynamo/dev/reference/releases/known-issues#${versionAnchor}` + : undefined, + }); + } + + return ( + <> + +
+ + + {tiles.length > 0 && ( +
+ {tiles.map((tile) => { + const body = ( + <> + + {tile.value} + + {tile.label} + + ); + return tile.href ? ( + + {body} + + ) : ( +
+ {body} +
+ ); + })} +
+ )} +
+ + ); +} diff --git a/docs/fern/components/ReleaseSummaryCards.tsx b/docs/fern/components/ReleaseSummaryCards.tsx new file mode 100644 index 000000000000..e8643658db17 --- /dev/null +++ b/docs/fern/components/ReleaseSummaryCards.tsx @@ -0,0 +1,144 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * ReleaseSummaryCards — "what changed" area cards for a Release Notes page. + * + * Renders a responsive grid of per-area summary cards: blue area chip, + * title, plain-text body, and an optional "changes in detail" anchor link + * into the detailed section further down the page. Body text may arrive with + * markdown bold/backticks from the notes pipeline; it is rendered as plain + * text (the ** pairs and backticks are stripped mechanically, not parsed). + * + * Server component (no "use client"); shared vocabulary comes from + * ReferenceStyles — place on the page alongside this + * component. Only the .dynref-sc-* layout classes are defined here. Area + * chips are uniformly blue (the system rule for area badges, matching + * known-issues) using the arch-blue rgba values from ReferenceStyles. + */ + +export interface SummaryCard { + area: string; + title: string; + body: string; + anchor?: string; +} + +const SC_CSS = ` +.dynref-sc-grid { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(280px, 1fr)); + gap: 12px; + margin: 24px 0; +} + +/* Panel recipe, card-sized. */ +.dynref-sc-card { + display: flex; + flex-direction: column; + padding: 16px 18px; + border: 1px solid var(--border, var(--grayscale-a5)); + border-radius: 12px; + background: var(--pst-color-surface); +} + +.dark .dynref-sc-card { + background: #161616; + border-color: #2b2b2b; +} + +.dynref-sc-title { + margin: 8px 0 0; + color: var(--pst-color-text-base); + font-size: 14px; + font-weight: 600; +} + +.dynref-sc-body { + margin: 6px 0 0; + color: var(--pst-color-text-muted); + font-size: 12.5px; + line-height: 1.55; +} + +.dark .dynref-sc-body { + color: #a8a8a8; +} + +.dynref-sc-more { + display: inline-block; + margin-top: 10px; + color: var(--nv-color-green-2, #538300); + font-size: 12px; + font-weight: 600; + text-decoration: none; +} + +.dark .dynref-sc-more { + color: #76B900; +} + +.dynref-sc-more:hover { + text-decoration: underline; +} + +/* Area chip — badge-sized, self-contained, uniformly blue (system rule: + area badges are blue, as on known-issues). Shared --dynref-blue-* tokens + from ReferenceStyles flip for dark mode on their own. */ +.dynref-sc-chip { + display: inline-flex; + align-items: center; + align-self: flex-start; + padding: 1px 8px; + border: 1px solid var(--dynref-blue-border); + border-radius: 6px; + background: var(--dynref-blue-bg); + color: var(--dynref-blue-fg); + font-size: 11.5px; + font-weight: 600; + white-space: nowrap; +} +`; + +/* Bodies may carry markdown bold/code/links from the notes pipeline; render + as plain text — strip the markers mechanically, never parse. Links keep + their visible text ([text](url) → text); the card's anchor link is the + only navigation affordance. */ +function stripInlineMarkdown(text: string): string { + return text + .replace(/\[([^\]]*)\]\([^)]*\)/g, "$1") + .replace(/\*\*/g, "") + .replace(/`/g, ""); +} + +export function ReleaseSummaryCards({ cards }: { cards: SummaryCard[] }) { + return ( + <> + +
+ {cards.map((card) => { + const paragraphs = stripInlineMarkdown(card.body) + .split(/\n{2,}/) + .map((p) => p.trim()) + .filter((p) => p.length > 0); + return ( +
+ {card.area} +

{card.title}

+ {paragraphs.map((paragraph) => ( +

+ {paragraph} +

+ ))} + {card.anchor && ( + + {card.area} changes in detail ↓ + + )} +
+ ); + })} +
+ + ); +} diff --git a/docs/fern/components/ReleaseTimeline.tsx b/docs/fern/components/ReleaseTimeline.tsx new file mode 100644 index 000000000000..2b8dfdaf3ab9 --- /dev/null +++ b/docs/fern/components/ReleaseTimeline.tsx @@ -0,0 +1,271 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * ReleaseTimeline — vertical timeline of every tracked release (newest first, + * data order from releases.data.ts). Node shape encodes the release kind: + * solid green = stable, solid neutral = patch, dashed amber = platform + * preview / model build. Stable releases show their feature-voice + * notesSummary; others show their delta. The Release History landing page + * is the only consumer. (The crates.io first-publication table lives on + * Release Artifacts via the sibling CratesFirstPublished export.) + * + * Server component; shared vocabulary (headings, badges, mono) comes from + * ReferenceStyles — place on the page alongside this + * component. Only the .dynref-tl-* layout classes are defined here. + */ + +import { + CRATES_FIRST_PUBLISHED, + RELEASES, + type Release, + type ReleaseKind, +} from "./releases.data"; + +const TL_CSS = ` +.dynref-tl { + margin: 20px 0; +} + +.dynref-tl-item { + display: grid; + grid-template-columns: 14px minmax(0, 1fr); + gap: 0 12px; +} + +.dynref-tl-rail { + display: flex; + flex-direction: column; + align-items: center; +} + +.dynref-tl-node { + flex: 0 0 auto; + width: 12px; + height: 12px; + margin-top: 4px; + border-radius: 50%; +} + +.dynref-tl-node--stable { + background: #76B900; +} + +.dynref-tl-node--patch { + background: #b5b5b5; +} + +.dark .dynref-tl-node--patch { + background: #3d3d3d; +} + +.dynref-tl-node--preview { + background: transparent; + border: 2px dashed #C77E1B; +} + +.dark .dynref-tl-node--preview { + border-color: #FAC775; +} + +.dynref-tl-line { + flex: 1; + width: 2px; + margin-top: 4px; + background: var(--border, var(--grayscale-a5)); +} + +.dynref-tl-body { + min-width: 0; + padding-bottom: 18px; +} + +.dynref-tl-item:last-child .dynref-tl-body { + padding-bottom: 0; +} + +.dynref-tl-head { + display: flex; + flex-wrap: wrap; + align-items: center; + gap: 8px; +} + +.dynref-tl-version { + color: var(--pst-color-text-base); + font-size: 13.5px; + font-weight: 600; + text-decoration: none; +} + +.dynref-tl-version:hover { + text-decoration: underline; + text-decoration-color: var(--nv-color-green, #76B900); +} + +.dynref-tl-date { + color: var(--pst-color-text-muted); + font-size: 12px; +} + +.dynref-tl-gh { + color: var(--pst-color-text-muted); + font-size: 11.5px; + text-decoration: none; +} + +.dynref-tl-gh:hover { + text-decoration: underline; + text-decoration-color: var(--nv-color-green, #76B900); +} + +.dynref-tl-sum { + margin: 4px 0 0; + color: var(--pst-color-text-muted); + font-size: 12.5px; + line-height: 1.45; +} + +.dynref-tl-crates { + margin-top: 24px; + padding-top: 18px; + border-top: 1px solid var(--border, var(--grayscale-a5)); +} + +.dynref-tl-table { + width: 100%; + margin-top: 10px; + border-collapse: collapse; + font-size: 12.5px; +} + +.dynref-tl-table th { + padding: 4px 16px 6px 0; + border-bottom: 1px solid var(--border, var(--grayscale-a5)); + color: var(--pst-color-text-muted); + font-size: 11px; + font-weight: 700; + letter-spacing: 0.06em; + text-align: left; + text-transform: uppercase; +} + +.dynref-tl-table td { + padding: 6px 16px 6px 0; + border-bottom: 1px solid var(--border, var(--grayscale-a5)); + color: var(--pst-color-text-base); +} + +.dynref-tl-table tbody tr:last-child td { + border-bottom: 0; +} + +.dynref-tl-table td.dynref-tl-cell-muted { + color: var(--pst-color-text-muted); +} +`; + +const KIND_BADGE: Record = { + stable: { label: "GA release", variant: "green" }, + patch: { label: "Patch", variant: "gray" }, + "platform-preview": { label: "Early access", variant: "amber" }, + "model-build": { label: "Model build", variant: "amber" }, +}; + +const NODE_CLASS: Record = { + stable: "dynref-tl-node--stable", + patch: "dynref-tl-node--patch", + "platform-preview": "dynref-tl-node--preview", + "model-build": "dynref-tl-node--preview", +}; + +function TimelineEntry({ release, isLast }: { release: Release; isLast: boolean }) { + const badge = KIND_BADGE[release.kind]; + const summary = + release.kind === "stable" + ? release.notesSummary ?? release.note ?? release.delta + : release.delta ?? release.note; + return ( +
+
+ + {!isLast && } +
+
+
+ {release.notesHref || release.github ? ( + + {release.version} + + ) : ( + {release.version} + )} + {badge.label} + {release.notesHref && release.github && ( + + GitHub ↗ + + )} + {release.date && {release.date}} +
+ {summary &&

{summary}

} +
+
+ ); +} + +export function ReleaseTimeline() { + return ( + <> + +
+ {RELEASES.map((release, index) => ( + + ))} +
+ + ); +} + +/* Crates.io first-publication table — crate-publishing metadata that lives on + the Release Artifacts page (extracted from the timeline so it renders once, + not duplicated alongside the release history). Reuses the .dynref-tl-* table + styles. */ +export function CratesFirstPublished() { + return ( + <> + +
+ + + + + + + + + + {CRATES_FIRST_PUBLISHED.map((entry) => ( + + + + + + ))} + +
CrateFirst versionPublished
{entry.crate}{entry.version}{entry.date}
+

+ dynamo-async-openai is deprecated; 1.0.2 is its final release. Use dynamo-protocols for new + dependencies. +

+
+ + ); +} diff --git a/docs/fern/components/RunsWhereWizard.tsx b/docs/fern/components/RunsWhereWizard.tsx new file mode 100644 index 000000000000..c399f7b9c74e --- /dev/null +++ b/docs/fern/components/RunsWhereWizard.tsx @@ -0,0 +1,421 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * RunsWhereWizard — "what runs where" picker for the Compatibility page. + * Reader states two facts (backend + CUDA driver generation) and gets the + * list of stable/patch Dynamo releases that ship a matching container, with + * the CUDA toolkit and driver floor per release — and the click-to-copy pull + * command for the current release only (older tags are not restated here). + * + * Everything is derived from releases.data.ts: the CUDA options come from + * CUDA_HISTORY's distinct toolkit majors, a release qualifies for a + * (backend, major) pair when CUDA_HISTORY has a row for it, and pull commands + * come from ARTIFACTS (which tracks the current release only). + * + * Filtering is CSS-only, same mechanism as ArtifactBrowser: two hidden radio + * groups sit first inside the panel; each group's :checked sibling selector + * hides the rows whose data-backend / data-cuda does not match, so only rows + * matching BOTH selections stay visible. Combinations with no qualifying + * release pre-render a muted empty-state row driven by the same selectors. + * + * Server component; shared vocabulary (panel, eyebrow, chips, badges, copy + * buttons) comes from ReferenceStyles — place on the page. + * Only the .dynref-ww-* layout classes are defined here. + */ + +import { ARTIFACTS, CUDA_HISTORY, CURRENT_TAG, RELEASES, type CudaRow } from "./releases.data"; + +interface BackendOption { + id: string; + label: string; + backend: CudaRow["backend"]; + runtime: string; +} + +const BACKENDS: BackendOption[] = [ + { id: "sglang", label: "SGLang", backend: "SGLang", runtime: "sglang-runtime" }, + { id: "trtllm", label: "TensorRT-LLM", backend: "TensorRT-LLM", runtime: "tensorrtllm-runtime" }, + { id: "vllm", label: "vLLM", backend: "vLLM", runtime: "vllm-runtime" }, +]; + +interface CudaOption { + major: string; + /** Driver floor for the major's current toolkits, e.g. "580.xx+". */ + floor: string; +} + +/** Distinct CUDA toolkit majors in the history, newest first, each with the + * highest driver floor seen for that major (the floor a reader on that + * driver generation should expect for recent releases). */ +function cudaOptions(): CudaOption[] { + const floors = new Map(); + for (const row of CUDA_HISTORY) { + const major = row.toolkit.split(".")[0]; + const prev = floors.get(major); + if (prev === undefined || row.minDriver > prev) floors.set(major, row.minDriver); + } + return [...floors.entries()] + .map(([major, floor]) => ({ major, floor })) + .sort((a, b) => Number(b.major) - Number(a.major)); +} + +interface WizardRow { + backendId: string; + major: string; + version: string; + href: string; + toolkits: string[]; + minDriver: string; + experimental: boolean; + isCurrent: boolean; + /** Full image reference to copy — current release only. */ + pull?: string; +} + +function pullCommandFor(runtime: string): string | undefined { + const artifact = ARTIFACTS.find((a) => a.category === "container" && a.name === runtime); + const tag = artifact?.tags.find((t) => t.label === CURRENT_TAG); + return tag?.clipboard; +} + +/** Stable + patch releases (newest first, RELEASES order) that CUDA_HISTORY + * lists for the given backend on the given toolkit major. */ +function rowsFor(backend: BackendOption, major: string): WizardRow[] { + const rows: WizardRow[] = []; + for (const release of RELEASES) { + if (release.kind !== "stable" && release.kind !== "patch") continue; + const bare = release.version.replace(/^v/, ""); + const matches = CUDA_HISTORY.filter( + (h) => h.version === bare && h.backend === backend.backend && h.toolkit.split(".")[0] === major, + ); + if (matches.length === 0) continue; + const isCurrent = bare === CURRENT_TAG; + rows.push({ + backendId: backend.id, + major, + version: release.version, + href: release.notesHref ?? release.github ?? "https://github.com/ai-dynamo/dynamo/releases", + toolkits: [...new Set(matches.map((h) => h.toolkit))], + minDriver: matches[0].minDriver, + experimental: matches.every((h) => h.note === "Experimental"), + isCurrent, + pull: isCurrent ? pullCommandFor(backend.runtime) : undefined, + }); + } + return rows; +} + +const WW_CSS = ` +/* Inputs are hidden by the shared .dynref-vh (visually hidden, focusable) + class so the rails stay keyboard-operable; focus rules are generated in + filterCss alongside the :checked rules. */ + +.dynref-ww-rails { + display: flex; + flex-wrap: wrap; + gap: 16px 32px; + margin: 0 0 6px; +} + +.dynref-ww-group { + display: flex; + flex-direction: column; + gap: 6px; +} + +.dynref-ww-rail { + display: flex; + flex-wrap: wrap; + gap: 8px; +} + +.dynref-ww-pill { + display: inline-flex; + align-items: center; + min-height: 30px; + padding: 6px 10px; + border: 1px solid var(--border, var(--grayscale-a5)); + border-radius: var(--rounded, 6px); + background: transparent; + color: var(--pst-color-text-base); + font-size: 12.5px; + line-height: 1; + cursor: pointer; +} + +.dynref-ww-pill:hover { + border-color: var(--nv-color-green, #76B900); +} + +.dynref-ww-row { + display: grid; + grid-template-columns: 96px max-content minmax(0, 1fr) max-content; + gap: 10px; + align-items: center; + padding: 9px 0; + border-bottom: 1px solid var(--border, var(--grayscale-a5)); +} + +.dynref-ww-row:last-child { + border-bottom: 0; +} + +.dynref-ww-version { + display: flex; + align-items: center; + gap: 8px; + white-space: nowrap; +} + +.dynref-ww-link { + color: var(--pst-color-text-base); + font-size: 13px; + font-weight: 600; + text-decoration: none; +} + +.dynref-ww-link:hover { + text-decoration: underline; + text-decoration-color: var(--nv-color-green, #76B900); +} + +.dynref-ww-driver { + color: var(--pst-color-text-base); + font-size: 12.5px; +} + +.dynref-ww-notecol { + margin-left: 6px; + color: var(--pst-color-text-muted); + font-size: 12px; +} + +.dynref-ww-pullslot { + display: flex; + align-items: center; + justify-content: flex-end; + gap: 8px; +} + +.dynref-ww-empty { + padding: 12px 0; + border-bottom: 1px solid var(--border, var(--grayscale-a5)); + color: var(--pst-color-text-muted); + font-size: 13px; +} + +.dynref-ww-emptyhint { + font-size: 12px; +} + +.dynref-ww-empty:last-child { + border-bottom: 0; +} + +@media (max-width: 640px) { + .dynref-ww-row { + grid-template-columns: minmax(0, 1fr); + gap: 6px; + } + + .dynref-ww-pullslot { + justify-content: flex-start; + } +} +`; + +/** :checked and :focus-visible rules are generated from the option lists so + * the selectors always match the rendered inputs — active-pill highlight and + * keyboard focus ring per input, plus the two independent hide rules whose + * intersection leaves only rows matching both the checked backend and the + * checked CUDA major. */ +function filterCss(cuda: CudaOption[]): string { + const inputs = [ + ...BACKENDS.map((b) => ({ id: `ww-b-${b.id}`, attr: "data-backend", value: b.id })), + ...cuda.map((c) => ({ id: `ww-c-${c.major}`, attr: "data-cuda", value: c.major })), + ]; + const pillRules = inputs + .map((i) => `#${i.id}:checked ~ .dynref-ww-rails label[for="${i.id}"]`) + .join(",\n"); + const focusRules = inputs + .map((i) => `#${i.id}:focus-visible ~ .dynref-ww-rails label[for="${i.id}"]`) + .join(",\n"); + const hideRules = inputs + .map( + (i) => + `#${i.id}:checked ~ .dynref-ww-list [${i.attr}]:not([${i.attr}="${i.value}"]) { display: none; }`, + ) + .join("\n"); + return ` +${pillRules} { + border-color: var(--nv-color-green, #76B900); + box-shadow: 0 0 0 1px var(--nv-color-green, #76B900); + background: rgba(118, 185, 0, 0.08); + font-weight: 700; +} +${focusRules} { + outline: 2px solid var(--nv-color-green, #76B900); + outline-offset: 1px; +} +${hideRules} +`; +} + +function WizardDataRow({ row, backendLabel }: { row: WizardRow; backendLabel: string }) { + return ( +
+ + + {row.version} + + + + {row.toolkits.map((toolkit) => ( + + CUDA {toolkit} + + ))} + {row.experimental && Experimental image} + + + driver {row.minDriver} + + + {row.pull && ( + + )} + {/* Trails the row so the fixed 96px version column never has to absorb + it — keeps the CUDA chips column-aligned across rows. */} + {row.isCurrent && Current} + +
+ ); +} + +export function RunsWhereWizard() { + const cuda = cudaOptions(); + const defaultBackend = BACKENDS[0]; + const defaultCuda = cuda[0]; + + const combos = BACKENDS.flatMap((backend) => + cuda.map((option) => ({ backend, option, rows: rowsFor(backend, option.major) })), + ); + + /* CUDA majors (newest first, cudaOptions order) that DO have qualifying + releases per backend — drives the empty-state "switch the driver filter" + hint, so it always reflects the data. */ + const shippedMajors = new Map( + BACKENDS.map((backend) => [ + backend.id, + combos + .filter((combo) => combo.backend.id === backend.id && combo.rows.length > 0) + .map((combo) => combo.option.major), + ]), + ); + + return ( + <> + +
+ {BACKENDS.map((backend) => ( + + ))} + {cuda.map((option) => ( + + ))} + +
+
+

What runs where

+

Releases that match your backend and driver

+
+
+ +
+
+ Backend +
+ {BACKENDS.map((backend) => ( + + ))} +
+
+
+ CUDA driver situation +
+ {cuda.map((option) => ( + + ))} +
+
+
+ +
+ {combos.map(({ backend, option, rows }) => + rows.length > 0 ? ( + rows.map((row) => ( + + )) + ) : ( +
+ No release ships {backend.label} for a CUDA {option.major} driver. + {(shippedMajors.get(backend.id) ?? []).length > 0 && ( + + {" "} + {backend.label} ships CUDA{" "} + {(shippedMajors.get(backend.id) ?? []).join(" and CUDA ")} images only — + switch the driver filter. + + )} +
+ ), + )} +
+ +

+ Driver floors from the CUDA & driver history; pull commands shown for the current release + only. +

+
+ + ); +} diff --git a/docs/fern/components/TagLookup.tsx b/docs/fern/components/TagLookup.tsx new file mode 100644 index 000000000000..3ec59a6d42c3 --- /dev/null +++ b/docs/fern/components/TagLookup.tsx @@ -0,0 +1,381 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * TagLookup — "Have a tag? Look it up." Reverse lookup from a container tag + * to what it is: the release or early-access build it belongs to, the + * runtimes it applies to, ship date, GA-path status (EA builds), and the + * breaking/known-issue chips (stable releases). + * + * The tag set is derived from data only: unique container tag labels for the + * current release from ARTIFACTS, plus every early-access tag in + * MODEL_EA_BUILDS. Lookup is CSS-only: one hidden radio per tag (plus a + * default-checked "none" radio) sits first inside the panel; :checked + * general-sibling selectors highlight the picked pill and reveal its + * pre-rendered detail card. No tag selected shows a muted hint row. + * + * Server component; shared vocabulary (panel, badges, mono, muted) comes + * from ReferenceStyles — place on the page alongside + * this component. Only the .dynref-tg-* layout classes are defined here. + */ + +import { + ARTIFACTS, + CURRENT_VERSION, + MODEL_EA_BUILDS, + RELEASES, + RELEASE_STATS, + type GaPath, + type ModelEaBuild, +} from "./releases.data"; +import { UpgradePanelStyles } from "./UpgradePanel"; + +interface StableTagEntry { + kind: "stable"; + tag: string; + /** Container image names that publish this tag. */ + images: string[]; +} + +interface EaTagEntry { + kind: "ea"; + tag: string; + build: ModelEaBuild; +} + +type TagEntry = StableTagEntry | EaTagEntry; + +/* Current-release container tags from ARTIFACTS, deduped to unique tag + strings (e.g. one "1.3.0" entry across seven images, one "1.3.0-efa" + across the three runtimes), then the EA tags in data order. */ +function deriveEntries(): TagEntry[] { + const stable: StableTagEntry[] = []; + for (const artifact of ARTIFACTS) { + if (artifact.category !== "container") continue; + for (const tag of artifact.tags) { + const existing = stable.find((entry) => entry.tag === tag.label); + if (existing) { + existing.images.push(artifact.name); + } else { + stable.push({ kind: "stable", tag: tag.label, images: [artifact.name] }); + } + } + } + const ea: EaTagEntry[] = MODEL_EA_BUILDS.map((build) => ({ kind: "ea", tag: build.tag, build })); + return [...stable, ...ea]; +} + +const ENTRIES = deriveEntries(); + +/* Same GA-path → badge-variant mapping as ModelEABuildCards, so a build's + status reads identically on both pages. */ +const GA_BADGE_VARIANT: Record = { + promoted: "green", + "recipe-in-ga": "green", + "dev-only": "amber", + superseded: "gray", +}; + +/* Shared "v1.3.0" -> "v130" anchor rule (same as UpgradePanel/ReleaseHeader). */ +function versionAnchor(version: string): string { + return version.replace(/\./g, ""); +} + +/* Per-tag :checked selectors are generated from the entry list so the CSS + always matches the rendered radios. */ +const TG_CSS = ` +/* Inputs are hidden by the shared .dynref-vh (visually hidden, focusable) + class so the tag rail stays keyboard-operable; generated :focus-visible + rules below paint the ring on the matching pill. */ + +.dynref-tg-rail { + display: flex; + flex-wrap: wrap; + gap: 6px; + margin: 0 0 12px; +} + +.dynref-tg-pill { + display: inline-flex; + align-items: center; + min-height: 26px; + padding: 4px 9px; + border: 1px solid var(--border, var(--grayscale-a5)); + border-radius: var(--rounded, 6px); + background: transparent; + color: var(--pst-color-text-base); + font-family: var(--pst-font-family-monospace, ui-monospace, SFMono-Regular, Menlo, monospace); + font-size: 11.5px; + font-variant-numeric: tabular-nums; + line-height: 1; + cursor: pointer; +} + +.dynref-tg-pill:hover { + border-color: var(--nv-color-green, #76B900); +} + +${ENTRIES.map((_, i) => `#tg-t${i}:checked ~ .dynref-tg-rail label[for="tg-t${i}"]`).join(",\n")} { + border-color: var(--nv-color-green, #76B900); + box-shadow: 0 0 0 1px var(--nv-color-green, #76B900); + background: rgba(118, 185, 0, 0.08); + font-weight: 700; +} + +${ENTRIES.map((_, i) => `#tg-t${i}:focus-visible ~ .dynref-tg-rail label[for="tg-t${i}"]`).join(",\n")} { + outline: 2px solid var(--nv-color-green, #76B900); + outline-offset: 1px; +} + +/* The default "none" radio has no pill — while it holds keyboard focus the + hint row carries the ring so focus is never invisible. */ +#tg-none:focus-visible ~ .dynref-tg-cards > .dynref-tg-hint { + outline: 2px solid var(--nv-color-green, #76B900); + outline-offset: 1px; +} + +.dynref-tg-card { + display: none; + padding: 14px 16px; + border: 1px solid var(--border, var(--grayscale-a5)); + border-radius: 8px; + background: #fcfcfc; +} + +.dark .dynref-tg-card { + background: #0f0f0f; + border-color: #2b2b2b; +} + +${ENTRIES.map((_, i) => `#tg-t${i}:checked ~ .dynref-tg-cards > [data-tg="t${i}"]`).join(",\n")} { + display: block; +} + +/* Default state: the hidden "none" radio is checked and only the hint shows. */ +.dynref-tg-hint { + display: none; + margin: 0; + padding: 14px 16px; + border: 1px dashed var(--border, var(--grayscale-a5)); + border-radius: 8px; +} + +#tg-none:checked ~ .dynref-tg-cards > .dynref-tg-hint { + display: block; +} + +.dynref-tg-cardhead { + display: flex; + flex-wrap: wrap; + align-items: center; + gap: 8px; + margin-bottom: 10px; +} + +.dynref-tg-tagname { + color: var(--pst-color-text-base); + font-size: 13px; + font-weight: 700; +} + +.dynref-tg-rows { + display: flex; + flex-direction: column; + gap: 5px; +} + +.dynref-tg-row { + display: flex; + align-items: baseline; + gap: 10px; + font-size: 12.5px; + line-height: 1.45; +} + +.dynref-tg-key { + flex: 0 0 76px; + color: var(--pst-color-text-muted); + font-size: 11px; + font-weight: 700; + letter-spacing: 0.06em; + text-transform: uppercase; +} + +.dynref-tg-val { + color: var(--pst-color-text-base); + overflow-wrap: anywhere; +} + +.dynref-tg-link { + color: var(--nv-color-green-2, #538300); + font-weight: 600; + text-decoration: none; +} + +.dark .dynref-tg-link { + color: var(--nv-color-green, #76B900); +} + +.dynref-tg-link:hover { + text-decoration: underline; +} + +.dynref-tg-note { + margin: 0; +} + +/* Chip visuals come from .dynref-up-read (UpgradePanelStyles) so the + breaking-changes/known-issues links read identically to the Deprecations + reading list; only the container layout lives here. */ +.dynref-tg-chips { + display: flex; + flex-wrap: wrap; + gap: 6px; + margin-top: 10px; +} +`; + +function StableCard({ entry }: { entry: StableTagEntry }) { + const release = RELEASES.find((r) => r.version === CURRENT_VERSION); + const stats = RELEASE_STATS[CURRENT_VERSION]; + const anchor = versionAnchor(CURRENT_VERSION); + return ( + <> +
+ {entry.tag} + Stable · {CURRENT_VERSION} +
+
+
+ Release + + {release?.notesHref ? ( + + {CURRENT_VERSION} release notes + + ) : ( + CURRENT_VERSION + )} + +
+
+ Images + {entry.images.join(", ")} +
+ {release?.date && ( +
+ Shipped + {release.date} +
+ )} +
+ {stats && ( + + )} + + ); +} + +function EaCard({ entry }: { entry: EaTagEntry }) { + const { build } = entry; + const link = build.recipeHref + ? { href: build.recipeHref, label: build.recipeLabel ?? "Recipe" } + : build.github + ? { href: build.github, label: "Release tag on GitHub" } + : null; + return ( + <> +
+ {entry.tag} + + {build.gaLabel} + +
+
+
+ Build + + {build.model} early access · {build.releaseLine} line + {link && ( + <> + {" — "} + + {link.label} + + + )} + +
+
+ Runtimes + {build.runtimes.join(", ")} +
+
+ Shipped + {build.shipped} +
+
+ Status + {build.statusLine} +
+
+ + ); +} + +export function TagLookup() { + return ( + <> + + +
+ + {ENTRIES.map((entry, i) => ( + + ))} + +
+
+

Tag lookup

+

Have a tag? Look it up.

+
+

+ {ENTRIES.length} known tags · current release + early access +

+
+ +
+ {ENTRIES.map((entry, i) => ( + + ))} +
+ +
+

Select a tag.

+ {ENTRIES.map((entry, i) => ( +
+ {entry.kind === "stable" ? : } +
+ ))} +
+
+ + ); +} diff --git a/docs/fern/components/UpgradePanel.tsx b/docs/fern/components/UpgradePanel.tsx new file mode 100644 index 000000000000..cfbbe88d4cb8 --- /dev/null +++ b/docs/fern/components/UpgradePanel.tsx @@ -0,0 +1,337 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * UpgradePanel — "upgrade to this release" panel for a Release Notes page. + * + * Renders the upgrade target, an informational from-version (plain muted + * mono text — deliberately NOT chip-styled, so it does not read as an + * interactive filter), a migration strip computed from releases.data + * (backend pins, NIXL pins, CUDA toolkits + discontinuation badge, minimum + * driver), and a "Read before upgrading" reading list of link chips. Reading + * list items are (version, kind) pairs — labels and counts derive from + * RELEASE_STATS in releases.data, and hrefs from the shared vXYZ anchor rule; + * items whose version has no RELEASE_STATS entry are skipped. + * + * Server component (no "use client"); shared vocabulary comes from + * ReferenceStyles — place on the page alongside this + * component. Only the .dynref-up-* layout classes are defined here. Source + * pills are neutral, target pills green — except when a pin pair is + * identical, where the target pill stays neutral with an "unchanged" note + * (no false green). + * + * The internals — buildRows, the migration strip, the reading-list footer, + * and the .dynref-up-* stylesheet (UpgradePanelStyles) — are exported for + * reuse by UpgradeSelector, which renders one pre-built panel body per + * from-line. The MDX-facing surface of UpgradePanel itself is unchanged. + */ + +import { RELEASES, CUDA_HISTORY, RELEASE_STATS, type BackendPins } from "./releases.data"; + +const UP_CSS = ` +.dynref-up-from { + display: flex; + flex-wrap: wrap; + align-items: baseline; + gap: 6px; +} + +/* Informational, not interactive — no border or background. */ +.dynref-up-fromver { + color: var(--pst-color-text-muted); + font-size: 12.5px; +} + +.dynref-up-strip { + margin-top: 4px; +} + +.dynref-up-row { + display: flex; + flex-wrap: wrap; + align-items: center; + gap: 4px 0; + padding: 5px 0; +} + +/* Single shared label column — pin rows, flattened NIXL rows, CUDA, and + Driver all align on it. */ +.dynref-up-rowlabel { + flex: 0 0 112px; + color: var(--pst-color-text-muted); + font-size: 12px; + font-weight: 600; + white-space: nowrap; +} + +.dynref-up-pill { + display: inline-flex; + align-items: center; + padding: 2px 8px; + border: 1px solid transparent; + border-radius: 6px; + font-family: var(--pst-font-family-monospace, ui-monospace, SFMono-Regular, Menlo, monospace); + font-size: 12px; + font-variant-numeric: tabular-nums; + white-space: nowrap; +} + +/* Source — always neutral; also the target style for unchanged pairs. */ +.dynref-up-pill--src { + background: rgba(120, 120, 120, 0.08); + color: var(--pst-color-text-muted); + border-color: rgba(120, 120, 120, 0.25); +} + +.dark .dynref-up-pill--src { + background: #242424; + color: #a8a8a8; + border-color: #383838; +} + +/* Target — green tint, only when the value actually changes. */ +.dynref-up-pill--dst { + background: var(--dynref-green-bg); + color: var(--dynref-green-fg); + border-color: var(--dynref-green-border); +} + +.dynref-up-arrow { + margin: 0 6px; + color: var(--pst-color-text-muted); +} + +.dynref-up-unchanged { + margin-left: 8px; + color: var(--pst-color-text-muted); + font-size: 11px; +} + +.dynref-up-rowbadge { + margin-left: 8px; +} + +.dynref-up-footer { + display: flex; + flex-wrap: wrap; + align-items: center; + gap: 8px 10px; + margin-top: 14px; + padding-top: 12px; + border-top: 1px solid var(--border, var(--grayscale-a5)); +} + +.dynref-up-footerlabel { + color: var(--pst-color-text-base); + font-size: 12.5px; + font-weight: 500; +} + +/* Reading-list link chips — blue family, matching the badge--blue palette. */ +.dynref-up-read { + display: inline-flex; + align-items: center; + padding: 2px 9px; + border: 1px solid var(--dynref-blue-border); + border-radius: 6px; + background: var(--dynref-blue-bg); + color: var(--dynref-blue-fg); + font-size: 12px; + font-weight: 600; + text-decoration: none; + white-space: nowrap; +} + +.dynref-up-read:hover { + border-color: currentColor; +} +`; + +type PinKey = "sglang" | "trtllm" | "vllm" | "nixlSglang" | "nixlTrtllm" | "nixlVllm"; + +/* TRT-LLM abbreviation keeps the 112px label column tight; NIXL sub-rows are + flattened into the same single column. */ +const PIN_ROWS: { key: PinKey; label: string }[] = [ + { key: "sglang", label: "SGLang" }, + { key: "trtllm", label: "TRT-LLM" }, + { key: "vllm", label: "vLLM" }, + { key: "nixlSglang", label: "NIXL · SGLang" }, + { key: "nixlTrtllm", label: "NIXL · TRT-LLM" }, + { key: "nixlVllm", label: "NIXL · vLLM" }, +]; + +export interface MigrationRow { + label: string; + from: string; + to: string; + badge?: string; +} + +function uniq(values: string[]): string[] { + return values.filter((value, index) => values.indexOf(value) === index); +} + +/* "v1.3.0" -> CUDA_HISTORY rows keyed "1.3.0". */ +function cudaRowsFor(versionTag: string) { + const version = versionTag.replace(/^v/, ""); + return CUDA_HISTORY.filter((row) => row.version === version); +} + +export function buildRows( + fromTag: string, + toTag: string, + fromPins: BackendPins | undefined, + toPins: BackendPins | undefined, +): MigrationRow[] { + const rows: MigrationRow[] = []; + + for (const { key, label } of PIN_ROWS) { + const fromPin = fromPins?.[key]; + const toPin = toPins?.[key]; + if (fromPin && toPin) rows.push({ label, from: fromPin, to: toPin }); + } + + // UCX ships with the NIXL builds, so it sits directly under the NIXL rows. + // Skipped when either side never stated a UCX version (v1.0.0, patches). + const fromUcx = RELEASES.find((r) => r.version === fromTag)?.ucx; + const toUcx = RELEASES.find((r) => r.version === toTag)?.ucx; + if (fromUcx && toUcx) rows.push({ label: "UCX", from: fromUcx, to: toUcx }); + + const fromCuda = cudaRowsFor(fromTag); + const toCuda = cudaRowsFor(toTag); + const fromToolkits = uniq(fromCuda.map((row) => row.toolkit)); + const toToolkits = uniq(toCuda.map((row) => row.toolkit)); + if (fromToolkits.length > 0 && toToolkits.length > 0) { + const has12 = (toolkits: string[]) => toolkits.some((toolkit) => /^12\./.test(toolkit)); + rows.push({ + label: "CUDA", + from: fromToolkits.join(" / "), + to: toToolkits.join(" / "), + badge: has12(fromToolkits) && !has12(toToolkits) ? "CUDA 12 ends" : undefined, + }); + } + + const fromDrivers = uniq(fromCuda.map((row) => row.minDriver)); + const toDrivers = uniq(toCuda.map((row) => row.minDriver)); + if (fromDrivers.length > 0 && toDrivers.length > 0) { + rows.push({ label: "Driver", from: fromDrivers.join(" / "), to: toDrivers.join(" / ") }); + } + + return rows; +} + +/* The .dynref-up-* stylesheet as a component, so UpgradeSelector can carry + the same visual vocabulary without duplicating the CSS. */ +export function UpgradePanelStyles() { + return ; +} + +export interface ReadingItem { + version: string; + kind: "breaking" | "known-issues"; +} + +/* Labels and hrefs derive from RELEASE_STATS + the shared "v1.3.0" -> "v130" + anchor rule (same as ReleaseHeader); versions without stats are skipped. */ +function buildReadingChips(readingList: ReadingItem[]): { label: string; href: string }[] { + return readingList.flatMap((item) => { + const stats = RELEASE_STATS[item.version]; + if (!stats) return []; + const anchor = item.version.replace(/\./g, ""); + return item.kind === "breaking" + ? [ + { + label: `${item.version} breaking changes (${stats.breaking})`, + href: `/dynamo/dev/reference/releases/deprecations#${anchor}`, + }, + ] + : [ + { + label: `${item.version} known issues (${stats.knownIssues})`, + href: `/dynamo/dev/reference/releases/known-issues#${anchor}`, + }, + ]; + }); +} + +/* Migration strip — one from -> to row per pin, CUDA, and driver. Renders + nothing when there are no rows. */ +export function MigrationStrip({ rows }: { rows: MigrationRow[] }) { + if (rows.length === 0) return null; + return ( +
+ {rows.map((row) => { + const unchanged = row.from === row.to; + return ( +
+ {row.label} + {row.from} + → + + {row.to} + + {unchanged && unchanged} + {row.badge && ( + + {row.badge} + + )} +
+ ); + })} +
+ ); +} + +/* "Read before upgrading" footer — link chips derived from RELEASE_STATS. + Renders nothing when every item was skipped (no stats). */ +export function ReadingListFooter({ items }: { items: ReadingItem[] }) { + const readingChips = buildReadingChips(items); + if (readingChips.length === 0) return null; + return ( +
+ Read before upgrading + {readingChips.map((item) => ( + + {item.label} + + ))} +
+ ); +} + +export function UpgradePanel(props: { + toVersion: string; + fromVersion: { version: string; label: string }; + readingList: ReadingItem[]; +}) { + const from = RELEASES.find((r) => r.version === props.fromVersion.version); + const to = RELEASES.find((r) => r.version === props.toVersion); + + const rows = buildRows(props.fromVersion.version, props.toVersion, from?.pins, to?.pins); + + return ( + <> + +
+
+

Upgrade to {props.toVersion}

+
+ from + {props.fromVersion.label} +
+
+ + + + +
+ + ); +} diff --git a/docs/fern/components/UpgradeSelector.tsx b/docs/fern/components/UpgradeSelector.tsx new file mode 100644 index 000000000000..5a04571a5331 --- /dev/null +++ b/docs/fern/components/UpgradeSelector.tsx @@ -0,0 +1,217 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * UpgradeSelector — "coming from" upgrade picker for the Deprecations ledger. + * + * Answers "I run v1.1.x — what must I read to get to the current release?". + * Self-configuring from releases.data: the target is CURRENT_VERSION and the + * from-candidates are every older stable release that has a RELEASE_STATS + * entry, labeled by line ("v1.2.x"). Each line maps to its LATEST release + * including patches (v1.2.x -> v1.2.1) so the migration strip reflects the + * pins the user actually runs. Each panel reuses UpgradePanel's internals + * (buildRows + MigrationStrip + ReadingListFooter + UpgradePanelStyles); the + * reading list is one breaking-changes chip for every stable release strictly + * after the from line's base up to and including the current release, plus + * the current release's known-issues chip (oldest -> newest, known issues + * last). + * + * Switching is CSS-only, same mechanism as ArtifactBrowser: hidden radios + * sit first inside the panel, the label pills live in the header rail, and + * :checked general-sibling selectors reveal the matching pre-rendered panel + * body (newest line default-checked). Server component; shared vocabulary + * comes from ReferenceStyles — place on the page. Only + * the .dynref-us-* layout classes are defined here. + */ + +import { CURRENT_VERSION, RELEASES, RELEASE_STATS, type Release } from "./releases.data"; +import { + buildRows, + MigrationStrip, + ReadingListFooter, + UpgradePanelStyles, + type ReadingItem, +} from "./UpgradePanel"; + +const US_CSS = ` +/* Inputs are hidden by the shared .dynref-vh (visually hidden, focusable) + class so the from-line rail stays keyboard-operable; per-line + :focus-visible rules are generated in wiringCss alongside :checked. */ + +.dynref-us-rail { + display: flex; + flex-wrap: wrap; + align-items: center; + gap: 8px; +} + +.dynref-us-raillabel { + color: var(--pst-color-text-muted); + font-size: 12.5px; +} + +/* Filter pills — same treatment as the artifact browser rail. */ +.dynref-us-pill { + display: inline-flex; + align-items: center; + min-height: 28px; + padding: 5px 10px; + border: 1px solid var(--border, var(--grayscale-a5)); + border-radius: var(--rounded, 6px); + background: transparent; + color: var(--pst-color-text-base); + font-family: var(--pst-font-family-monospace, ui-monospace, SFMono-Regular, Menlo, monospace); + font-size: 12px; + font-variant-numeric: tabular-nums; + line-height: 1; + cursor: pointer; +} + +.dynref-us-pill:hover { + border-color: var(--nv-color-green, #76B900); +} + +.dynref-us-panel { + display: none; +} + +.dynref-us-fromnote { + margin: 0 0 2px; + font-size: 12px; +} +`; + +/* "v1.2.0" -> "us-v120" / "v1.2.x" — shared radio-id and line-label rules. */ +function lineId(baseVersion: string): string { + return `us-${baseVersion.replace(/\./g, "")}`; +} + +function lineLabel(baseVersion: string): string { + return baseVersion.replace(/\.\d+$/, ".x"); +} + +interface FromLine { + /** Stable release the line is named after (v1.2.0 -> "v1.2.x"). */ + base: Release; + /** Latest release on the line including patches — the pins the user runs. */ + latest: Release; + id: string; + label: string; + readingList: ReadingItem[]; +} + +function buildFromLines(): FromLine[] { + const currentIdx = RELEASES.findIndex((r) => r.version === CURRENT_VERSION); + if (currentIdx < 0) return []; + + /* All stable releases carrying RELEASE_STATS, in RELEASES (newest-first) + order, with their array index for older/newer comparisons. */ + const statStables = RELEASES.map((release, index) => ({ release, index })).filter( + ({ release }) => release.kind === "stable" && RELEASE_STATS[release.version] !== undefined, + ); + + return statStables + .filter(({ index }) => index > currentIdx) + .map(({ release: base, index: baseIdx }) => { + /* Latest stable/patch on the line: first RELEASES entry (array order is + newest-first) sharing the "vA.B." prefix. Dev/model builds share the + prefix but are not what operators run — exclude by kind. */ + const linePrefix = base.version.replace(/\d+$/, ""); + const latest = + RELEASES.find( + (r) => + (r.kind === "stable" || r.kind === "patch") && r.version.startsWith(linePrefix), + ) ?? base; + + /* Every stable strictly newer than the line's base, up to and including + current — oldest first — then current's known issues last. */ + const readingList: ReadingItem[] = statStables + .filter(({ index }) => index >= currentIdx && index < baseIdx) + .map(({ release }) => ({ version: release.version, kind: "breaking" as const })) + .reverse(); + readingList.push({ version: CURRENT_VERSION, kind: "known-issues" }); + + return { + base, + latest, + id: lineId(base.version), + label: lineLabel(base.version), + readingList, + }; + }); +} + +export function UpgradeSelector() { + const current = RELEASES.find((r) => r.version === CURRENT_VERSION); + const fromLines = buildFromLines(); + if (!current || fromLines.length === 0) return null; + + /* :checked and :focus-visible wiring is per-line, so the selectors are + generated from the same derived list that renders the radios, pills, and + panels. */ + const wiringCss = fromLines + .map( + (line) => ` +#${line.id}:checked ~ .dynref-panel-header label[for="${line.id}"] { + border-color: var(--nv-color-green, #76B900); + box-shadow: 0 0 0 1px var(--nv-color-green, #76B900); + background: rgba(118, 185, 0, 0.08); + font-weight: 700; +} + +#${line.id}:focus-visible ~ .dynref-panel-header label[for="${line.id}"] { + outline: 2px solid var(--nv-color-green, #76B900); + outline-offset: 1px; +} + +#${line.id}:checked ~ #${line.id}-panel { + display: block; +} +`, + ) + .join(""); + + return ( + <> + + +
+ {fromLines.map((line, index) => ( + + ))} + +
+

Upgrade to {CURRENT_VERSION}

+
+ from + {fromLines.map((line) => ( + + ))} +
+
+ + {fromLines.map((line) => ( +
+

+ Pins shown from {line.latest.version}, the + latest release on the {line.label} line. +

+ + +
+ ))} +
+ + ); +} diff --git a/docs/fern/components/kvbm/kvbm-guide.md b/docs/fern/components/kvbm/kvbm-guide.md index b26899bac9c9..03b889fb72ac 100644 --- a/docs/fern/components/kvbm/kvbm-guide.md +++ b/docs/fern/components/kvbm/kvbm-guide.md @@ -29,7 +29,7 @@ KVBM can be used independently without using the rest of the Dynamo stack: pip install kvbm ``` -See the [support matrix](../../reference/support-matrix.md) for version compatibility. +See the [compatibility page](../../reference/compatibility.mdx) for version compatibility. ### Build from Source diff --git a/docs/fern/components/releases.data.ts b/docs/fern/components/releases.data.ts new file mode 100644 index 000000000000..67f2c698f8cd --- /dev/null +++ b/docs/fern/components/releases.data.ts @@ -0,0 +1,989 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * releases.data.ts — single source of truth for the Reference pages + * (Compatibility, Release Artifacts, Model Early Access Builds). + * + * Every value here is transcribed from the authoritative reference pages on + * main (docs/reference/support-matrix.md, feature-matrix.md, + * release-artifacts.md, model-early-access-builds.md). + * + * PER-RELEASE BUMP CHECKLIST (a release touches more than this file): + * 1. This file: add the RELEASES entry (pins, ucx, date, delta, + * notesSummary, notesHref), CUDA_HISTORY rows, ARTIFACTS tags/versions, MAIN_TOT, + * CURRENT_* consts, MODEL_EA_BUILDS, and the RELEASE_STATS entry + * (counts from the GitHub body) as applicable. + * 2. New page reference/release-notes/vX-Y-Z.mdx (ingest the GitHub body; + * ReleaseHeader and the UpgradePanel readingList read their counts + * from RELEASE_STATS — no per-page count props). + * 3. reference/known-issues.mdx + reference/deprecations.mdx: new vXYZ + * section + accordion retitles (titles read RELEASE_STATS). + * 4. Nav: docs/fern/index.yml Release Notes section (+ explicit slug). + * 5. Regenerate agent twins: python3 scripts/gen_llms_tables.py + * (--check must pass afterwards). + * + * PARSER NOTE: scripts/gen_llms_tables.py parses this file with a + * conservative literal parser — keep it a disciplined literal (no computed + * values, spreads, calls, or ternaries); see the PARSER CONTRACT in that + * script. The parser fails closed on anything it does not understand. + */ + +export type ReleaseKind = "stable" | "patch" | "platform-preview" | "model-build"; + +export interface BackendPins { + sglang?: string; + trtllm?: string; + vllm?: string; + nixlSglang?: string; + nixlTrtllm?: string; + nixlVllm?: string; + pinsNote?: string; +} + +export interface Release { + version: string; + date?: string; + kind: ReleaseKind; + github?: string; + docs?: string; + /** Docs-native release notes page (absolute site path); GitHub link used when absent. */ + notesHref?: string; + pins?: BackendPins; + /** UCX version shipped with the release's NIXL builds — from the release's + * Key Dependencies table; omitted where the source never stated one + * (v1.0.0 and patch releases). */ + ucx?: string; + delta?: string; + note?: string; + /** Feature-voice one-liner for the Release Notes timeline (stable releases); + * composed from the release page's Highlights themes. */ + notesSummary?: string; + partial?: boolean; +} + +export const CURRENT_VERSION = "v1.3.0"; +export const CURRENT_DATE = "Jul 20, 2026"; +export const CURRENT_TAG = "1.3.0"; +export const CURRENT_WHEEL = "1.3.0.post1"; + +export const MAIN_TOT: BackendPins = { + sglang: "0.5.15", + trtllm: "1.3.0rc21", + vllm: "0.25.1", + nixlSglang: "1.3.0", + nixlTrtllm: "1.0.1", + nixlVllm: "1.1.0", +}; + +const GH = "https://github.com/ai-dynamo/dynamo/releases/tag/"; + +export const RELEASES: Release[] = [ + { + version: "v1.3.0", + notesHref: "/dynamo/dev/reference/releases/v1-3-0", + date: "Jul 20, 2026", + kind: "stable", + github: `${GH}v1.3.0`, + docs: "https://docs.nvidia.com/dynamo", + pins: { sglang: "0.5.14", trtllm: "1.3.0rc19", vllm: "0.23.0", nixlSglang: "1.3.0", nixlTrtllm: "1.0.1", nixlVllm: "1.1.0" }, + ucx: "1.20.x", + delta: + "CUDA 12 container images discontinued; EFA variants go multi-arch as -efa; GA wheels published as 1.3.0.post1 (containers stay :1.3.0); UCX 1.20.x.", + notesSummary: + "Tool-calling and reasoning overhaul, RL rollout serving, the largest Router buildout to date, SLA-driven Planner autoscaling, and production GPU Memory Service on Kubernetes.", + }, + { + version: "v1.3.0-dev.1", + date: "Jun 9, 2026", + kind: "platform-preview", + github: `${GH}v1.3.0-dev.1`, + pins: { sglang: "0.5.12.post1", trtllm: "1.3.0rc17", vllm: "0.22.0", nixlSglang: "1.0.1", nixlTrtllm: "0.10.1", nixlVllm: "1.1.0" }, + delta: + "Full-platform preview of v1.3.0: complete runtime matrix, wheels on pypi.nvidia.com, crates, and Helm charts. Superseded by v1.3.0 GA.", + }, + { + version: "v1.2.1", + notesHref: "/dynamo/dev/reference/releases/v1-2-0", + date: "Jun 13, 2026", + kind: "patch", + github: `${GH}v1.2.1`, + docs: "https://docs.nvidia.com/dynamo", + pins: { sglang: "0.5.11", trtllm: "1.3.0rc14", vllm: "0.20.1", nixlSglang: "1.0.1", nixlTrtllm: "0.10.1", nixlVllm: "0.10.1" }, + delta: "Patch release. Same backend pins as v1.2.0.", + }, + { + version: "v1.2.0", + notesHref: "/dynamo/dev/reference/releases/v1-2-0", + date: "Jun 2, 2026", + kind: "stable", + github: `${GH}v1.2.0`, + docs: "https://docs.nvidia.com/dynamo", + pins: { sglang: "0.5.11", trtllm: "1.3.0rc14", vllm: "0.20.1", nixlSglang: "1.0.1", nixlTrtllm: "0.10.1", nixlVllm: "0.10.1" }, + ucx: "1.20.0", + delta: + "603 PRs from 82 authors. DGD/DGDR promoted to v1beta1; CRTC default approximate KV router; inter-pod GMS sidecar; Dynamo Snapshot on CRI-O / OpenShift; UCX 1.20.0.", + notesSummary: + "DGD/DGDR v1beta1, CRTC as the default KV router, inter-pod GPU Memory Service, Dynamo Snapshot on CRI-O/OpenShift, and DeepSeek-V4 recipes on vLLM.", + }, + { + version: "v1.2.0-deepseek-v4-dev.3", + date: "May 9, 2026", + kind: "model-build", + github: `${GH}v1.2.0-deepseek-v4-dev.3`, + pins: { sglang: "upstream DSv4 preview", vllm: "0.20.1", nixlVllm: "0.10.1" }, + partial: true, + note: "DeepSeek-V4 Blackwell preview; vLLM + SGLang containers only.", + }, + { + version: "v1.2.0-deepseek-v4-dev.2", + date: "May 1, 2026", + kind: "model-build", + github: `${GH}v1.2.0-deepseek-v4-dev.2`, + pins: { sglang: "upstream DSv4 preview", vllm: "0.20.0", nixlVllm: "0.10.1" }, + partial: true, + note: "DeepSeek-V4 Blackwell preview; vLLM + SGLang containers only.", + }, + { + version: "v1.1.1", + notesHref: "/dynamo/dev/reference/releases/v1-1-0", + date: "May 5, 2026", + kind: "patch", + github: `${GH}v1.1.1`, + docs: "https://docs.nvidia.com/dynamo", + pins: { sglang: "0.5.10.post1", trtllm: "1.3.0rc11", vllm: "0.19.0", nixlSglang: "1.0.1", nixlTrtllm: "0.10.1", nixlVllm: "0.10.1" }, + delta: "Patch release. Same backend pins as v1.1.0.", + }, + { + version: "v1.1.0", + notesHref: "/dynamo/dev/reference/releases/v1-1-0", + date: "May 1, 2026", + kind: "stable", + github: `${GH}v1.1.0`, + docs: "https://docs.nvidia.com/dynamo", + pins: { sglang: "0.5.10.post1", trtllm: "1.3.0rc11", vllm: "0.19.0", nixlSglang: "1.0.1", nixlTrtllm: "0.10.1", nixlVllm: "0.10.1" }, + ucx: "1.20", + delta: + "Planner split into its own dynamo-planner image (artifact boundary change). First 1.y.z publication of dynamo-protocols on crates.io; dynamo-async-openai deprecated at final 1.0.2.", + notesSummary: + "Resilient KV routing at scale, Anthropic Messages API support, performance modeling and offline replay, and the multimodal embedding cache.", + }, + { + version: "v1.1.0-dev.3", + date: "Apr 18, 2026", + kind: "platform-preview", + github: `${GH}v1.1.0-dev.3`, + pins: { sglang: "0.5.10.post1", trtllm: "1.3.0rc11", vllm: "0.19.0", nixlSglang: "1.0.1", nixlTrtllm: "0.10.1", nixlVllm: "0.10.1" }, + partial: true, + note: "Partial platform preview: TRT-LLM runtime image + wheels only.", + }, + { + version: "v1.1.0-dev.2", + date: "Apr 9, 2026", + kind: "platform-preview", + github: `${GH}v1.1.0-dev.2`, + pins: { sglang: "0.5.9", trtllm: "1.3.0rc9", vllm: "0.19.0", nixlSglang: "1.0.1", nixlTrtllm: "0.10.1", nixlVllm: "0.10.1" }, + partial: true, + note: "Partial platform preview: SGLang + TRT-LLM runtime images + wheels.", + }, + { + version: "v1.1.0-dev.1", + date: "Mar 17, 2026", + kind: "platform-preview", + github: `${GH}v1.1.0-dev.1`, + pins: { sglang: "0.5.9", trtllm: "1.3.0rc5.post1", vllm: "0.17.1", nixlSglang: "1.0.1", nixlTrtllm: "0.10.1", nixlVllm: "0.10.1" }, + note: "Platform preview: runtime matrix, wheels on pypi.nvidia.com, Helm charts.", + }, + { + version: "v1.0.2", + notesHref: "/dynamo/dev/reference/releases/v1-0-0", + date: "Apr 22, 2026", + kind: "patch", + github: `${GH}v1.0.2`, + docs: "https://docs.nvidia.com/dynamo", + pins: { sglang: "0.5.9", trtllm: "1.3.0rc5.post1", vllm: "0.16.0", nixlSglang: "0.10.1", nixlTrtllm: "0.10.1", nixlVllm: "0.10.1" }, + delta: "No artifact additions or removals versus v1.0.0.", + }, + { + version: "v1.0.1", + notesHref: "/dynamo/dev/reference/releases/v1-0-0", + date: "Mar 16, 2026", + kind: "patch", + github: `${GH}v1.0.1`, + docs: "https://docs.nvidia.com/dynamo", + pins: { sglang: "0.5.9", trtllm: "1.3.0rc5.post1", vllm: "0.16.0", nixlSglang: "0.10.1", nixlTrtllm: "0.10.1", nixlVllm: "0.10.1" }, + delta: "No artifact additions or removals versus v1.0.0.", + }, + { + version: "v1.0.0", + notesHref: "/dynamo/dev/reference/releases/v1-0-0", + date: "Mar 12, 2026", + kind: "stable", + github: `${GH}v1.0.0`, + docs: "https://docs.nvidia.com/dynamo", + pins: { sglang: "0.5.9", trtllm: "1.3.0rc5.post1", vllm: "0.16.0", nixlSglang: "0.10.1", nixlTrtllm: "0.10.1", nixlVllm: "0.10.1" }, + delta: + "snapshot-agent image and EFA variants for vLLM and TRT-LLM (AMD64 only). First publish of dynamo-mocker and dynamo-kv-router crates. snapshot Helm chart added (preview); deprecated dynamo-crds dropped from the publish stream.", + notesSummary: + "First GA release: unified configuration, Kubernetes production readiness, multimodal serving, and the agents surface.", + }, + { + version: "v0.9.1", + date: "Mar 4, 2026", + kind: "patch", + github: `${GH}v0.9.1`, + docs: "https://docs.nvidia.com/dynamo", + pins: { sglang: "0.5.8", trtllm: "1.3.0rc3", vllm: "0.14.1", nixlSglang: "0.9.0", nixlTrtllm: "0.9.0", nixlVllm: "0.9.0" }, + delta: "No artifact additions or removals versus v0.9.0.", + }, + { + version: "v0.9.0", + date: "Feb 11, 2026", + kind: "stable", + github: `${GH}v0.9.0`, + pins: { sglang: "0.5.8", trtllm: "1.3.0rc1", vllm: "0.14.1", nixlSglang: "0.9.0", nixlTrtllm: "0.9.0", nixlVllm: "0.9.0" }, + delta: "First publish of dynamo-tokens crate. Deprecated dynamo-graph Helm chart dropped from the publish stream.", + }, + { + version: "v0.8.1", + date: "Jan 23, 2026", + kind: "patch", + github: `${GH}v0.8.1`, + pins: { sglang: "0.5.6.post2", trtllm: "1.2.0rc6.post1", vllm: "0.12.0", nixlSglang: "0.8.0", nixlTrtllm: "0.8.0", nixlVllm: "0.8.0" }, + delta: "Post trains .post1/.post2/.post3 republished the TRT-LLM runtime image and PyPI wheels only.", + }, + { + version: "v0.8.0", + date: "Jan 15, 2026", + kind: "stable", + github: `${GH}v0.8.0`, + pins: { sglang: "0.5.6.post2", trtllm: "1.2.0rc6.post1", vllm: "0.12.0", nixlSglang: "0.8.0", nixlTrtllm: "0.8.0", nixlVllm: "0.8.0" }, + delta: "dynamo-frontend image and CUDA 13 variants for vLLM and SGLang. First publish of dynamo-memory and dynamo-config crates.", + }, + { + version: "v0.7.1", + date: "Dec 15, 2025", + kind: "patch", + github: `${GH}v0.7.1`, + pins: { sglang: "0.5.4.post3", trtllm: "1.2.0rc3", vllm: "0.11.0", nixlSglang: "0.8.0", nixlTrtllm: "0.8.0", nixlVllm: "0.8.0" }, + }, + { + version: "v0.7.0", + date: "Nov 26, 2025", + kind: "stable", + github: `${GH}v0.7.0`, + pins: { sglang: "0.5.4.post3", trtllm: "1.2.0rc2", vllm: "0.11.0", nixlSglang: "0.8.0", nixlTrtllm: "0.8.0", nixlVllm: "0.8.0" }, + }, + { + version: "v0.6.1", + date: "Nov 6, 2025", + kind: "patch", + github: `${GH}v0.6.1`, + pins: { sglang: "0.5.3.post2", trtllm: "1.1.0rc5", vllm: "0.11.0", nixlSglang: "0.6.0", nixlTrtllm: "0.6.0", nixlVllm: "0.6.0" }, + }, + { + version: "v0.6.0", + date: "Oct 28, 2025", + kind: "stable", + github: `${GH}v0.6.0`, + pins: { sglang: "0.5.3.post2", trtllm: "1.1.0rc5", vllm: "0.11.0", nixlSglang: "0.6.0", nixlTrtllm: "0.6.0", nixlVllm: "0.6.0" }, + delta: "Oldest release tracked on this page.", + }, +]; + +export interface CudaRow { + version: string; + backend: "SGLang" | "TensorRT-LLM" | "vLLM"; + toolkit: string; + minDriver: string; + note?: string; +} + +export const CUDA_HISTORY: CudaRow[] = [ + { version: "1.3.0", backend: "SGLang", toolkit: "13.0", minDriver: "580.xx+" }, + { version: "1.3.0", backend: "TensorRT-LLM", toolkit: "13.1", minDriver: "580.xx+" }, + { version: "1.3.0", backend: "vLLM", toolkit: "13.0", minDriver: "580.xx+" }, + { version: "1.2.1", backend: "SGLang", toolkit: "12.9", minDriver: "575.xx+" }, + { version: "1.2.1", backend: "SGLang", toolkit: "13.0", minDriver: "580.xx+" }, + { version: "1.2.1", backend: "TensorRT-LLM", toolkit: "13.1", minDriver: "580.xx+" }, + { version: "1.2.1", backend: "vLLM", toolkit: "12.9", minDriver: "575.xx+" }, + { version: "1.2.1", backend: "vLLM", toolkit: "13.0", minDriver: "580.xx+" }, + { version: "1.2.0", backend: "SGLang", toolkit: "12.9", minDriver: "575.xx+" }, + { version: "1.2.0", backend: "SGLang", toolkit: "13.0", minDriver: "580.xx+" }, + { version: "1.2.0", backend: "TensorRT-LLM", toolkit: "13.1", minDriver: "580.xx+" }, + { version: "1.2.0", backend: "vLLM", toolkit: "12.9", minDriver: "575.xx+" }, + { version: "1.2.0", backend: "vLLM", toolkit: "13.0", minDriver: "580.xx+" }, + { version: "1.1.1", backend: "SGLang", toolkit: "12.9", minDriver: "575.xx+" }, + { version: "1.1.1", backend: "SGLang", toolkit: "13.0", minDriver: "580.xx+" }, + { version: "1.1.1", backend: "TensorRT-LLM", toolkit: "13.1", minDriver: "580.xx+" }, + { version: "1.1.1", backend: "vLLM", toolkit: "12.9", minDriver: "575.xx+" }, + { version: "1.1.1", backend: "vLLM", toolkit: "13.0", minDriver: "580.xx+" }, + { version: "1.1.0", backend: "SGLang", toolkit: "12.9", minDriver: "575.xx+" }, + { version: "1.1.0", backend: "SGLang", toolkit: "13.0", minDriver: "580.xx+" }, + { version: "1.1.0", backend: "TensorRT-LLM", toolkit: "13.1", minDriver: "580.xx+" }, + { version: "1.1.0", backend: "vLLM", toolkit: "12.9", minDriver: "575.xx+" }, + { version: "1.1.0", backend: "vLLM", toolkit: "13.0", minDriver: "580.xx+" }, + { version: "1.0.2", backend: "SGLang", toolkit: "12.9", minDriver: "575.xx+" }, + { version: "1.0.2", backend: "SGLang", toolkit: "13.0", minDriver: "580.xx+" }, + { version: "1.0.2", backend: "TensorRT-LLM", toolkit: "13.1", minDriver: "580.xx+" }, + { version: "1.0.2", backend: "vLLM", toolkit: "12.9", minDriver: "575.xx+" }, + { version: "1.0.2", backend: "vLLM", toolkit: "13.0", minDriver: "580.xx+" }, + { version: "1.0.1", backend: "SGLang", toolkit: "12.9", minDriver: "575.xx+" }, + { version: "1.0.1", backend: "SGLang", toolkit: "13.0", minDriver: "580.xx+" }, + { version: "1.0.1", backend: "TensorRT-LLM", toolkit: "13.1", minDriver: "580.xx+" }, + { version: "1.0.1", backend: "vLLM", toolkit: "12.9", minDriver: "575.xx+" }, + { version: "1.0.1", backend: "vLLM", toolkit: "13.0", minDriver: "580.xx+" }, + { version: "1.0.0", backend: "SGLang", toolkit: "12.9", minDriver: "575.xx+" }, + { version: "1.0.0", backend: "SGLang", toolkit: "13.0", minDriver: "580.xx+" }, + { version: "1.0.0", backend: "TensorRT-LLM", toolkit: "13.1", minDriver: "580.xx+" }, + { version: "1.0.0", backend: "vLLM", toolkit: "12.9", minDriver: "575.xx+" }, + { version: "1.0.0", backend: "vLLM", toolkit: "13.0", minDriver: "580.xx+" }, + { version: "0.9.1", backend: "SGLang", toolkit: "12.9", minDriver: "575.xx+" }, + { version: "0.9.1", backend: "TensorRT-LLM", toolkit: "13.0", minDriver: "580.xx+" }, + { version: "0.9.1", backend: "vLLM", toolkit: "12.9", minDriver: "575.xx+" }, + { version: "0.9.0", backend: "SGLang", toolkit: "12.9", minDriver: "575.xx+" }, + { version: "0.9.0", backend: "TensorRT-LLM", toolkit: "13.0", minDriver: "580.xx+" }, + { version: "0.9.0", backend: "vLLM", toolkit: "12.9", minDriver: "575.xx+" }, + { version: "0.8.1", backend: "SGLang", toolkit: "12.9", minDriver: "575.xx+" }, + { version: "0.8.1", backend: "SGLang", toolkit: "13.0", minDriver: "580.xx+", note: "Experimental" }, + { version: "0.8.1", backend: "TensorRT-LLM", toolkit: "13.0", minDriver: "580.xx+" }, + { version: "0.8.1", backend: "vLLM", toolkit: "12.9", minDriver: "575.xx+" }, + { version: "0.8.1", backend: "vLLM", toolkit: "13.0", minDriver: "580.xx+", note: "Experimental" }, + { version: "0.8.0", backend: "SGLang", toolkit: "12.9", minDriver: "575.xx+" }, + { version: "0.8.0", backend: "SGLang", toolkit: "13.0", minDriver: "580.xx+", note: "Experimental" }, + { version: "0.8.0", backend: "TensorRT-LLM", toolkit: "13.0", minDriver: "580.xx+" }, + { version: "0.8.0", backend: "vLLM", toolkit: "12.9", minDriver: "575.xx+" }, + { version: "0.8.0", backend: "vLLM", toolkit: "13.0", minDriver: "580.xx+", note: "Experimental" }, + { version: "0.7.1", backend: "SGLang", toolkit: "12.8", minDriver: "570.xx+" }, + { version: "0.7.1", backend: "TensorRT-LLM", toolkit: "13.0", minDriver: "580.xx+" }, + { version: "0.7.1", backend: "vLLM", toolkit: "12.9", minDriver: "575.xx+" }, + { version: "0.7.0", backend: "SGLang", toolkit: "12.9", minDriver: "575.xx+" }, + { version: "0.7.0", backend: "TensorRT-LLM", toolkit: "13.0", minDriver: "580.xx+" }, + { version: "0.7.0", backend: "vLLM", toolkit: "12.8", minDriver: "570.xx+" }, +]; + +export const CUDA_NOTES = [ + "Patch versions (e.g. v0.8.1.post1, v0.7.0.post1) have the same CUDA support as their base version.", + "Early access v1.1.0-dev.* images follow the same CUDA matrix as v1.0.2. The v1.2.0-deepseek-v4-dev.3 vLLM container is CUDA 13.0 multi-arch; the SGLang containers split by arch (CUDA 12.9 on amd64, CUDA 13.0 on arm64).", + "Experimental CUDA 13 images are not published for all versions.", +]; + +export type FeatureStatus = "yes" | "caveat" | "wip" | "no"; + +export interface FeatureCell { + status: FeatureStatus; + note?: string; +} + +export interface Feature { + name: string; + sglang: FeatureCell; + trtllm: FeatureCell; + vllm: FeatureCell; +} + +export const FEATURES: Feature[] = [ + { + name: "Disaggregated Serving", + sglang: { status: "yes" }, + trtllm: { status: "yes" }, + vllm: { status: "yes", note: "Prefill/decode separation with NIXL KV transfer" }, + }, + { + name: "KV-Aware Routing", + sglang: { status: "yes" }, + trtllm: { status: "yes" }, + vllm: { status: "yes" }, + }, + { + name: "SLA-Based Planner", + sglang: { status: "yes" }, + trtllm: { status: "yes" }, + vllm: { status: "yes" }, + }, + { + name: "KV Block Manager", + sglang: { status: "wip", note: "Work in progress across all combinations" }, + trtllm: { status: "yes" }, + vllm: { status: "yes" }, + }, + { + name: "Multimodal (Image)", + sglang: { + status: "yes", + note: "Not compatible with KV-aware routing. Disagg patterns: EPD, E/PD, E/P/D (not traditional EP/D)", + }, + trtllm: { + status: "yes", + note: "Image URLs + pre-computed embeddings. Disagg: EP/D + E/P/D. KV-aware routing via dedicated MM Router Worker (requires KV event publishing)", + }, + vllm: { + status: "yes", + note: "With KV-aware routing, image-aware routing on documented paths", + }, + }, + { + name: "Multimodal (Video)", + sglang: { status: "yes" }, + trtllm: { status: "no" }, + vllm: { status: "yes", note: "Video input with frame sampling" }, + }, + { + name: "Multimodal (Audio)", + sglang: { status: "no" }, + trtllm: { status: "no" }, + vllm: { status: "wip", note: "Qwen2-Audio, experimental" }, + }, + { + name: "Request Migration", + sglang: { status: "yes" }, + trtllm: { status: "yes", note: "Work in progress with multimodal" }, + vllm: { status: "yes" }, + }, + { + name: "Request Cancellation", + sglang: { + status: "wip", + note: "Remote-prefill-phase cancellation not supported in disaggregated mode", + }, + trtllm: { + status: "caveat", + note: "Engine temporarily not notified of cancellations — resources for cancelled requests are not freed (known issue)", + }, + vllm: { status: "yes" }, + }, + { + name: "LoRA", + sglang: { status: "no" }, + trtllm: { status: "no" }, + vllm: { status: "yes", note: "Dynamic load/unload; KV-aware routing supports adapter affinity" }, + }, + { + name: "Tool Calling", + sglang: { status: "yes" }, + trtllm: { status: "yes" }, + vllm: { status: "yes" }, + }, + { + name: "Speculative Decoding", + sglang: { status: "wip", note: "Code hooks exist; no examples or docs yet" }, + trtllm: { status: "yes" }, + vllm: { status: "yes", note: "Eagle3" }, + }, + { + name: "Dynamo Snapshot", + sglang: { status: "yes" }, + trtllm: { status: "no" }, + vllm: { status: "yes" }, + }, +]; + +export const BACKEND_BLURBS = { + vllm: "vLLM offers the broadest feature coverage in Dynamo, with full support for disaggregated serving, KV-aware routing, KV block management, LoRA adapters, and multimodal inference including video and audio.", + sglang: + "SGLang is optimized for high-throughput serving with fast primitives, providing robust support for disaggregated serving, KV-aware routing, and request migration.", + trtllm: + "TensorRT-LLM delivers maximum inference performance and optimization, with full KVBM integration and robust disaggregated serving support.", +}; + +export type ArtifactCategory = "container" | "wheel" | "helm" | "crate"; + +export interface Artifact { + category: ArtifactCategory; + group?: "runtime" | "component"; + name: string; + description: string; + meta?: string; + href: string; + tags: { label: string; clipboard: string; variant?: "default" | "experimental" }[]; + badge?: "Preview" | "Experimental" | "Deprecated"; +} + +const NGC_C = "https://catalog.ngc.nvidia.com/orgs/nvidia/ai-dynamo/containers"; + +export const ARTIFACTS: Artifact[] = [ + { + category: "container", + group: "runtime", + name: "vllm-runtime", + description: "vLLM backend runtime", + meta: "vLLM v0.23.0 · CUDA 13.0 · AMD64/ARM64", + href: `${NGC_C}/vllm-runtime/tags`, + tags: [ + { label: "1.3.0", clipboard: "nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.3.0" }, + { label: "1.3.0-efa", clipboard: "nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.3.0-efa", variant: "experimental" }, + ], + }, + { + category: "container", + group: "runtime", + name: "sglang-runtime", + description: "SGLang backend runtime", + meta: "SGLang v0.5.14 · CUDA 13.0 · AMD64/ARM64", + href: `${NGC_C}/sglang-runtime/tags`, + tags: [ + { label: "1.3.0", clipboard: "nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.3.0" }, + { label: "1.3.0-efa", clipboard: "nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.3.0-efa", variant: "experimental" }, + ], + }, + { + category: "container", + group: "runtime", + name: "tensorrtllm-runtime", + description: "TensorRT-LLM backend runtime", + meta: "TRT-LLM v1.3.0rc19 · CUDA 13.1 · AMD64/ARM64", + href: `${NGC_C}/tensorrtllm-runtime/tags`, + tags: [ + { label: "1.3.0", clipboard: "nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0" }, + { label: "1.3.0-efa", clipboard: "nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-efa", variant: "experimental" }, + ], + }, + { + category: "container", + group: "component", + name: "dynamo-frontend", + description: "OpenAI-compatible API gateway with Endpoint Prediction Protocol (EPP)", + meta: "AMD64/ARM64", + href: `${NGC_C}/dynamo-frontend/tags`, + tags: [{ label: "1.3.0", clipboard: "nvcr.io/nvidia/ai-dynamo/dynamo-frontend:1.3.0" }], + }, + { + category: "container", + group: "component", + name: "dynamo-planner", + description: "Standalone Planner used by Profiler jobs and Planner pods", + meta: "AMD64/ARM64", + href: `${NGC_C}/dynamo-planner/tags`, + tags: [{ label: "1.3.0", clipboard: "nvcr.io/nvidia/ai-dynamo/dynamo-planner:1.3.0" }], + }, + { + category: "container", + group: "component", + name: "kubernetes-operator", + description: "Operator that manages Dynamo deployments and CRDs", + meta: "AMD64/ARM64", + href: `${NGC_C}/kubernetes-operator/tags`, + tags: [{ label: "1.3.0", clipboard: "nvcr.io/nvidia/ai-dynamo/kubernetes-operator:1.3.0" }], + }, + { + category: "container", + group: "component", + name: "snapshot-agent", + description: "Fast GPU worker recovery via CRIU", + meta: "AMD64/ARM64", + href: `${NGC_C}/snapshot-agent/tags`, + badge: "Preview", + tags: [{ label: "1.3.0", clipboard: "nvcr.io/nvidia/ai-dynamo/snapshot-agent:1.3.0" }], + }, + { + category: "wheel", + name: "ai-dynamo", + description: "Main package with backend integrations (vLLM, SGLang, TRT-LLM)", + meta: "Python 3.10–3.12 · Linux (glibc v2.28+)", + href: "https://pypi.org/project/ai-dynamo/1.3.0.post1/", + tags: [{ label: "uv pip install ai-dynamo==1.3.0.post1", clipboard: "uv pip install ai-dynamo==1.3.0.post1" }], + }, + { + category: "wheel", + name: "ai-dynamo-runtime", + description: "Core Python bindings for the Dynamo runtime", + meta: "Python 3.10–3.12 · Linux (glibc v2.28+)", + href: "https://pypi.org/project/ai-dynamo-runtime/1.3.0.post1/", + tags: [ + { label: "uv pip install ai-dynamo-runtime==1.3.0.post1", clipboard: "uv pip install ai-dynamo-runtime==1.3.0.post1" }, + ], + }, + { + category: "wheel", + name: "kvbm", + description: "KV Block Manager for disaggregated KV cache", + meta: "Python 3.10–3.12 · Linux (glibc v2.28+)", + href: "https://pypi.org/project/kvbm/1.3.0.post1/", + tags: [{ label: "uv pip install kvbm==1.3.0.post1", clipboard: "uv pip install kvbm==1.3.0.post1" }], + }, + { + category: "helm", + name: "dynamo-platform", + description: "Platform services (etcd, NATS) and the Dynamo Operator for a Dynamo cluster", + href: "https://helm.ngc.nvidia.com/nvidia/ai-dynamo/charts/dynamo-platform-1.3.0.tgz", + tags: [ + { + label: "helm install · dynamo-platform 1.3.0", + clipboard: + "helm install dynamo-platform oci://helm.ngc.nvidia.com/nvidia/ai-dynamo/charts/dynamo-platform --version 1.3.0", + }, + ], + }, + { + category: "helm", + name: "snapshot", + description: "Snapshot DaemonSet for fast GPU worker recovery", + href: "https://helm.ngc.nvidia.com/nvidia/ai-dynamo/charts/snapshot-1.3.0.tgz", + tags: [ + { + label: "helm install · snapshot 1.3.0", + clipboard: "helm install snapshot oci://helm.ngc.nvidia.com/nvidia/ai-dynamo/charts/snapshot --version 1.3.0", + }, + ], + }, + { + category: "crate", + name: "dynamo-runtime", + description: "Core distributed runtime library", + meta: "MSRV Rust v1.82", + href: "https://crates.io/crates/dynamo-runtime/1.3.0", + tags: [{ label: "cargo add dynamo-runtime@1.3.0", clipboard: "cargo add dynamo-runtime@1.3.0" }], + }, + { + category: "crate", + name: "dynamo-llm", + description: "LLM inference engine", + meta: "MSRV Rust v1.82", + href: "https://crates.io/crates/dynamo-llm/1.3.0", + tags: [{ label: "cargo add dynamo-llm@1.3.0", clipboard: "cargo add dynamo-llm@1.3.0" }], + }, + { + category: "crate", + name: "dynamo-protocols", + description: "Async OpenAI-compatible API client", + meta: "MSRV Rust v1.82", + href: "https://crates.io/crates/dynamo-protocols/1.3.0", + tags: [{ label: "cargo add dynamo-protocols@1.3.0", clipboard: "cargo add dynamo-protocols@1.3.0" }], + }, + { + category: "crate", + name: "dynamo-async-openai", + description: "Legacy OpenAI client; use dynamo-protocols", + meta: "MSRV Rust v1.82 · final release", + href: "https://crates.io/crates/dynamo-async-openai/1.0.2", + badge: "Deprecated", + tags: [{ label: "cargo add dynamo-async-openai@1.0.2", clipboard: "cargo add dynamo-async-openai@1.0.2" }], + }, + { + category: "crate", + name: "dynamo-parsers", + description: "Protocol parsers (SSE, JSON streaming)", + meta: "MSRV Rust v1.82", + href: "https://crates.io/crates/dynamo-parsers/1.3.0", + tags: [{ label: "cargo add dynamo-parsers@1.3.0", clipboard: "cargo add dynamo-parsers@1.3.0" }], + }, + { + category: "crate", + name: "dynamo-memory", + description: "Memory management utilities", + meta: "MSRV Rust v1.82", + href: "https://crates.io/crates/dynamo-memory/1.3.0", + tags: [{ label: "cargo add dynamo-memory@1.3.0", clipboard: "cargo add dynamo-memory@1.3.0" }], + }, + { + category: "crate", + name: "dynamo-config", + description: "Configuration management", + meta: "MSRV Rust v1.82", + href: "https://crates.io/crates/dynamo-config/1.3.0", + tags: [{ label: "cargo add dynamo-config@1.3.0", clipboard: "cargo add dynamo-config@1.3.0" }], + }, + { + category: "crate", + name: "dynamo-tokens", + description: "Tokenizer bindings for LLM inference", + meta: "MSRV Rust v1.82", + href: "https://crates.io/crates/dynamo-tokens/1.3.0", + tags: [{ label: "cargo add dynamo-tokens@1.3.0", clipboard: "cargo add dynamo-tokens@1.3.0" }], + }, + { + category: "crate", + name: "dynamo-tokenizers", + description: "Tokenizer library for LLM inference", + meta: "MSRV Rust v1.82", + href: "https://crates.io/crates/dynamo-tokenizers/1.3.0", + tags: [{ label: "cargo add dynamo-tokenizers@1.3.0", clipboard: "cargo add dynamo-tokenizers@1.3.0" }], + }, + { + category: "crate", + name: "dynamo-mocker", + description: "Inference engine simulator for benchmarking", + meta: "MSRV Rust v1.82", + href: "https://crates.io/crates/dynamo-mocker/1.3.0", + tags: [{ label: "cargo add dynamo-mocker@1.3.0", clipboard: "cargo add dynamo-mocker@1.3.0" }], + }, + { + category: "crate", + name: "dynamo-kv-router", + description: "KV-aware request routing library", + meta: "MSRV Rust v1.82", + href: "https://crates.io/crates/dynamo-kv-router/1.3.0", + tags: [{ label: "cargo add dynamo-kv-router@1.3.0", clipboard: "cargo add dynamo-kv-router@1.3.0" }], + }, + { + category: "crate", + name: "kvbm-logical", + description: "Logical layer for the KV Block Manager", + meta: "MSRV Rust v1.82", + href: "https://crates.io/crates/kvbm-logical/1.3.0", + tags: [{ label: "cargo add kvbm-logical@1.3.0", clipboard: "cargo add kvbm-logical@1.3.0" }], + }, +]; + +export type GaPath = "promoted" | "dev-only" | "recipe-in-ga" | "superseded"; + +export interface Coverage { + images: boolean; + wheels: boolean; + helm: boolean; + crates: boolean; +} + +export interface ModelEaBuild { + model: string; + tag: string; + releaseLine: string; + runtimes: string[]; + shipped: string; + gaPath: GaPath; + gaLabel: string; + statusLine: string; + recipeLabel?: string; + recipeHref?: string; + github?: string; + coverage: Coverage; +} + +const MODEL_COVERAGE: Coverage = { images: true, wheels: false, helm: false, crates: false }; + +export const MODEL_EA_BUILDS: ModelEaBuild[] = [ + { + model: "Inkling", + tag: "1.4.0-inkling-dev.1", + releaseLine: "v1.4.0", + runtimes: ["sglang-runtime"], + shipped: "Jul 17, 2026", + gaPath: "dev-only", + gaLabel: "Dev-only · v1.4.0 line", + statusLine: "First build on the v1.4.0 line; targets the next stable release.", + recipeLabel: "Inkling recipe (main)", + recipeHref: "https://github.com/ai-dynamo/dynamo/blob/main/docs/recipes/inkling.mdx", + github: `${GH}v1.4.0-inkling-dev.1`, + coverage: MODEL_COVERAGE, + }, + { + model: "GLM-5.2", + tag: "1.3.0-glm-5.2-dev.1", + releaseLine: "v1.3.0", + runtimes: ["sglang-runtime"], + shipped: "Jul 20, 2026", + gaPath: "dev-only", + gaLabel: "Dev-only", + statusLine: + "Container carries SGLang cherry-picks (stability, config parsing, model support) opened upstream but not yet in a released SGLang.", + recipeLabel: "GLM-5 NVFP4 recipe", + recipeHref: "/dynamo/dev/recipes/glm-5-nvfp4", + coverage: MODEL_COVERAGE, + }, + { + model: "MiniMax-M3", + tag: "1.3.0-minimax-m3-dev.1", + releaseLine: "v1.3.0", + runtimes: ["vllm-runtime", "sglang-runtime", "tensorrtllm-runtime"], + shipped: "Jun 12, 2026", + gaPath: "promoted", + gaLabel: "Promoted → :1.3.0", + statusLine: "Dynamo changes and the M2 tool-calling fix are in release/1.3.0; the recipes run on the stock :1.3.0 containers.", + recipeLabel: "Recipe on release branch", + recipeHref: "https://github.com/ai-dynamo/dynamo/tree/release/1.3.0-minimax-m3-dev.1/recipes/minimax-m3", + github: `${GH}v1.3.0-minimax-m3-dev.1`, + coverage: MODEL_COVERAGE, + }, + { + model: "DeepSeek-V4", + tag: "1.3.0-deepseek-v4-dev.1", + releaseLine: "v1.3.0", + runtimes: ["tensorrtllm-runtime"], + shipped: "Jun 6, 2026", + gaPath: "recipe-in-ga", + gaLabel: "Recipe in v1.3.0", + statusLine: "DeepSeek-V4 Flash and Pro recipes ship in v1.3.0 on the standard TensorRT-LLM release container.", + recipeLabel: "recipes/deepseek-v4 (main)", + recipeHref: "https://github.com/ai-dynamo/dynamo/tree/main/recipes/deepseek-v4", + github: `${GH}v1.3.0-deepseek-v4-dev.1`, + coverage: MODEL_COVERAGE, + }, + { + model: "Nemotron-3-Ultra", + tag: "1.3.0-nemotron-ultra-dev.1", + releaseLine: "v1.3.0", + runtimes: ["vllm-runtime"], + shipped: "Jun 5, 2026", + gaPath: "dev-only", + gaLabel: "Dev-only", + statusLine: + "Four un-upstreamed vLLM patches; requires pinned flags VLLM_DISABLED_KERNELS=FlashInferFP8ScaledMMLinearKernel and --no-enable-flashinfer-autotune.", + recipeLabel: "Nemotron-3-Ultra recipe", + recipeHref: "/dynamo/dev/recipes/nemotron-3-ultra", + github: `${GH}v1.3.0-nemotron-ultra-dev.1`, + coverage: MODEL_COVERAGE, + }, + { + model: "Nemotron-3-Super", + tag: "1.3.0-nemotron-super-dev.1", + releaseLine: "v1.3.0", + runtimes: ["vllm-runtime"], + shipped: "Jun 4, 2026", + gaPath: "promoted", + gaLabel: "Promoted → :1.3.0", + statusLine: "Both container patches are in the vLLM v0.23.0 that v1.3.0 ships; the recipe runs on the stock vllm-runtime:1.3.0.", + recipeLabel: "Nemotron-3-Super recipe", + recipeHref: "/dynamo/dev/recipes/nemotron-3-super", + github: `${GH}v1.3.0-nemotron-super-dev.1`, + coverage: MODEL_COVERAGE, + }, + { + model: "Kimi-K2.6", + tag: "1.3.0-kimi-k2.6-dev.1", + releaseLine: "v1.3.0", + runtimes: ["vllm-runtime"], + shipped: "Jun 4, 2026", + gaPath: "promoted", + gaLabel: "Promoted → :1.3.0", + statusLine: "The build's only container patch is in vLLM v0.23.0; the recipes run on the stock vllm-runtime:1.3.0.", + recipeLabel: "Kimi-K2.6 recipe", + recipeHref: "/dynamo/dev/recipes/kimi-k2-6", + github: `${GH}v1.3.0-kimi-k2.6-dev.1`, + coverage: MODEL_COVERAGE, + }, + { + model: "Cosmos-3", + tag: "1.3.0-cosmos3-dev.1", + releaseLine: "v1.3.0", + runtimes: ["vllm-runtime"], + shipped: "Jun 1, 2026", + gaPath: "dev-only", + gaLabel: "Dev-only", + statusLine: + "Dynamo #10132 (Cosmos3 support in the vLLM-Omni backend) is open, not merged — v1.3.0 containers cannot run Cosmos3.", + recipeLabel: "Launch scripts (branch)", + recipeHref: "https://github.com/ai-dynamo/dynamo/tree/release/1.3.0-cosmos3-dev.1/examples/backends/vllm/launch", + github: `${GH}v1.3.0-cosmos3-dev.1`, + coverage: MODEL_COVERAGE, + }, + { + model: "DeepSeek-V4 preview", + tag: "1.2.0-deepseek-v4-dev.3", + releaseLine: "v1.2.0", + runtimes: ["vllm-runtime", "sglang-runtime"], + shipped: "May 9, 2026", + gaPath: "superseded", + gaLabel: "Superseded — recipe in v1.3.0", + statusLine: + "Blackwell (B200 + GB200) preview; per-arch/CUDA tags (e.g. vllm-runtime:1.2.0-deepseek-v4-cuda13-dev.3). Superseded by the v1.3.0 recipe.", + github: `${GH}v1.2.0-deepseek-v4-dev.3`, + coverage: MODEL_COVERAGE, + }, + { + model: "DeepSeek-V4 preview", + tag: "1.2.0-deepseek-v4-dev.2", + releaseLine: "v1.2.0", + runtimes: ["vllm-runtime", "sglang-runtime"], + shipped: "May 1, 2026", + gaPath: "superseded", + gaLabel: "Superseded — recipe in v1.3.0", + statusLine: "Blackwell preview on vLLM v0.20.0 (native DSv4 support); superseded by dev.3.", + github: `${GH}v1.2.0-deepseek-v4-dev.2`, + coverage: MODEL_COVERAGE, + }, + { + model: "DeepSeek-V4 preview", + tag: "1.2.0-sglang-deepseek-v4-dev.1", + releaseLine: "v1.2.0", + runtimes: ["sglang-runtime"], + shipped: "Apr 25, 2026", + gaPath: "superseded", + gaLabel: "Superseded — recipe in v1.3.0", + statusLine: "Earliest DSv4 preview (SGLang, B200 only); superseded by dev.2/dev.3.", + github: `${GH}v1.2.0-sglang-deepseek-v4-dev.1`, + coverage: MODEL_COVERAGE, + }, +]; + +export const PLATFORM_PREVIEW_COVERAGE: Record = { + "v1.3.0-dev.1": { images: true, wheels: true, helm: true, crates: true }, + "v1.1.0-dev.3": { images: true, wheels: true, helm: false, crates: false }, + "v1.1.0-dev.2": { images: true, wheels: true, helm: false, crates: false }, + "v1.1.0-dev.1": { images: true, wheels: true, helm: true, crates: false }, +}; + +export const PLATFORM = { + gpus: ["Blackwell", "Hopper", "Ada Lovelace", "Ampere"], + os: [ + { name: "Ubuntu", version: "24.04", arch: "x86_64, ARM64", status: "Supported", chip: "ubuntu" }, + { name: "Ubuntu", version: "22.04", arch: "x86_64", status: "Supported", chip: "ubuntu" }, + { name: "CentOS Stream", version: "9", arch: "x86_64", status: "Experimental", chip: "centos" }, + ], + arch: ["x86_64", "ARM64 (Ubuntu 24.04 only)"], + wheelsNote: + "Wheels are built in a manylinux_2_28-compatible environment and validated on CentOS Stream 9 and Ubuntu 22.04/24.04. Other Linux distributions are expected to work but are not officially verified.", + csp: [{ provider: "AWS", os: "Amazon Linux 2023", arch: "x86_64", status: "Supported" }], +}; + +export const KNOWN_ARTIFACT_ISSUES = [ + { + version: "v0.9.0", + artifact: "dynamo-platform-0.9.0", + issue: "Helm chart sets operator image to 0.7.1 instead of 0.9.0.", + status: "Fixed in v0.9.0.post1", + }, + { + version: "v0.8.1", + artifact: "vllm-runtime:0.8.1-cuda13", + issue: "Container fails to launch.", + status: "Known issue", + }, + { + version: "v0.8.1", + artifact: "sglang-runtime:0.8.1-cuda13, vllm-runtime:0.8.1-cuda13", + issue: "Multimodality not expected to work on ARM64. Works on AMD64.", + status: "Known limitation", + }, + { + version: "v0.8.0", + artifact: "sglang-runtime:0.8.0-cuda13", + issue: + "CuDNN installation issue caused PyTorch v2.9.1 compatibility problems with nn.Conv3d — performance degradation and excessive memory usage in multimodal workloads.", + status: "Fixed in v0.8.1 (#5461)", + }, +]; + +export const CRATES_FIRST_PUBLISHED = [ + { crate: "dynamo-runtime", version: "0.1.0", date: "2025-03-18" }, + { crate: "dynamo-llm", version: "0.2.0", date: "2025-05-01" }, + { crate: "dynamo-async-openai", version: "0.4.1", date: "2025-08-27" }, + { crate: "dynamo-parsers", version: "0.5.0", date: "2025-09-18" }, + { crate: "dynamo-memory", version: "0.8.0", date: "2026-01-15" }, + { crate: "dynamo-config", version: "0.8.0", date: "2026-01-15" }, + { crate: "dynamo-tokens", version: "0.9.0", date: "2026-02-12" }, + { crate: "dynamo-mocker", version: "1.0.0", date: "2026-03-13" }, + { crate: "dynamo-kv-router", version: "1.0.0", date: "2026-03-13" }, + { crate: "dynamo-protocols", version: "1.1.0", date: "2026-05-04" }, + { crate: "dynamo-tokenizers", version: "1.2.0", date: "2026-06-02" }, +]; + +/* Per-release ingestion-time stats for the Release Notes pages (ReleaseHeader + tiles, UpgradePanel reading list) and the Deprecations / Known Issues + accordion titles. Counted from each release's GitHub body at ingestion. */ +export interface ReleaseStats { + prs?: number; + contributors?: number; + firstTimers?: number; + breaking: number; + knownIssues: number; +} + +export const RELEASE_STATS: Record = { + "v1.3.0": { prs: 930, contributors: 125, firstTimers: 23, breaking: 24, knownIssues: 10 }, + "v1.2.0": { prs: 603, contributors: 82, breaking: 5, knownIssues: 11 }, + "v1.1.0": { prs: 896, contributors: 113, firstTimers: 12, breaking: 8, knownIssues: 20 }, + "v1.0.0": { contributors: 90, firstTimers: 34, breaking: 41, knownIssues: 14 }, +}; + +export const NIGHTLIES_NOTE = + "ai-dynamo and ai-dynamo-runtime nightly builds from main publish wheels tagged *.devYYYYMMDD (since Apr 24, 2026). Install with pip or uv using --pre and the NVIDIA extra-index pattern shown above."; diff --git a/docs/fern/custom.js b/docs/fern/custom.js index e2f49fd03ad2..c32e0325280f 100644 --- a/docs/fern/custom.js +++ b/docs/fern/custom.js @@ -331,3 +331,118 @@ window.addEventListener("popstate", queueEnhancement); queueEnhancement(); })(); + +// Reference-page click-to-copy: [data-dynref-copy] buttons (styled by +// ReferenceStyles.tsx) copy their payload, flash the .dynref-copied state, +// and swap their label to "Copied" for 1.2s. Buttons must carry plain-text +// labels — the swap replaces textContent. +(() => { + if (typeof document === "undefined") return; + document.addEventListener("click", (event) => { + const el = event.target.closest("[data-dynref-copy]"); + if (!el || !navigator.clipboard) return; + navigator.clipboard.writeText(el.getAttribute("data-dynref-copy")); + if (el.dataset.dynrefRestore === undefined) { + // Narrow chips (short tags) would GROW to fit "Copied", causing a width + // jump — for those, the glyph flip + green state is the whole feedback. + const swapText = el.offsetWidth >= 72; + el.dataset.dynrefRestore = el.textContent; + el.style.minWidth = `${el.offsetWidth}px`; + if (swapText) el.textContent = "Copied"; + el.classList.add("dynref-copied"); + window.setTimeout(() => { + if (swapText) el.textContent = el.dataset.dynrefRestore; + delete el.dataset.dynrefRestore; + el.style.minWidth = ""; + el.classList.remove("dynref-copied"); + }, 1200); + } + }); +})(); + +// Hash deep-links into closed accordions: when the URL hash targets an anchor +// (e.g. #v120) that lives inside — or immediately before — a closed
+// accordion, open it and re-scroll the anchor into view. Generic: no +// page-specific ids; runs on load, on every hashchange, and (because the app +// hydrates after DOMContentLoaded) via a self-disconnecting MutationObserver +// that waits for the anchor to appear. +(() => { + if (typeof document === "undefined") return; + + // Returns true once the anchor exists (whether or not an accordion needed + // opening), so pending observers know to stand down. + function openAccordionForHash() { + const hash = window.location.hash; + if (!hash || hash.length < 2) return true; + const el = document.getElementById(hash.slice(1)); + if (el == null) return false; + + // Ancestor
first; otherwise the first
among the next + // few forward siblings (anchor placed just before its accordion). + let details = el.closest("details"); + if (details == null) { + let sibling = el.nextElementSibling; + for (let i = 0; i < 3 && sibling != null; i += 1) { + const candidate = + sibling.tagName === "DETAILS" ? sibling : sibling.querySelector("details"); + if (candidate != null) { + details = candidate; + break; + } + sibling = sibling.nextElementSibling; + } + } + + if (details != null && !details.open) { + details.open = true; + window.requestAnimationFrame(() => { + el.scrollIntoView(); + }); + // Hydration (and the accordion's own hash rewrite) can reset the + // scroll position after our first scroll — re-scroll once, later, if + // the anchor fell back out of view. + window.setTimeout(() => { + const box = el.getBoundingClientRect(); + if (box.top < 0 || box.top > window.innerHeight) { + el.scrollIntoView(); + } + }, 400); + } + + return true; + } + + let observer = null; + let observerDeadline = null; + + function stopObserving() { + if (observer != null) { + observer.disconnect(); + observer = null; + } + if (observerDeadline != null) { + window.clearTimeout(observerDeadline); + observerDeadline = null; + } + } + + // Try now; if the anchor is not rendered yet (client hydration), watch the + // DOM until it appears, giving up after 10s so the observer never lingers. + function openAccordionWhenReady() { + stopObserving(); + if (openAccordionForHash()) return; + + observer = new MutationObserver(() => { + if (openAccordionForHash()) stopObserving(); + }); + observer.observe(document.documentElement, { childList: true, subtree: true }); + observerDeadline = window.setTimeout(stopObserving, 10000); + } + + if (document.readyState === "loading") { + document.addEventListener("DOMContentLoaded", openAccordionWhenReady); + } else { + openAccordionWhenReady(); + } + window.addEventListener("hashchange", openAccordionWhenReady); +})(); diff --git a/docs/fern/docs.yml b/docs/fern/docs.yml index bc9a85d5b486..fc6f09120efd 100644 --- a/docs/fern/docs.yml +++ b/docs/fern/docs.yml @@ -16,7 +16,8 @@ # yaml-language-server: $schema=https://schema.buildwithfern.dev/docs-yml.json instances: - - url: ai-dynamo.docs.buildwithfern.com/dynamo + - url: dynamo.docs.buildwithfern.com/dynamo + custom-domain: [docs.nvidia.com/dynamo, docs.dynamo.nvidia.com/dynamo] multi-source: true title: NVIDIA Dynamo Documentation @@ -72,6 +73,17 @@ redirects: destination: "/dynamo/dev/reference/compatibility" - source: "/dynamo/resources/release-artifacts" destination: "/dynamo/dev/reference/release-artifacts" + - source: "/dynamo/resources/model-early-access-builds" + destination: "/dynamo/dev/reference/model-early-access-builds" + # Live-site "Resources" section URLs (main's nav) → consolidated reference pages. + - source: "/dynamo/dev/resources/support-matrix" + destination: "/dynamo/dev/reference/compatibility" + - source: "/dynamo/dev/resources/feature-matrix" + destination: "/dynamo/dev/reference/compatibility" + - source: "/dynamo/dev/resources/release-artifacts" + destination: "/dynamo/dev/reference/release-artifacts" + - source: "/dynamo/dev/resources/model-early-access-builds" + destination: "/dynamo/dev/reference/model-early-access-builds" - source: "/dynamo/resources/examples" destination: "/dynamo/dev/resources/examples" # Version-scoped getting-started → resources redirects @@ -318,7 +330,9 @@ experimental: colors: accentPrimary: dark: '#76B900' - light: '#76B900' + # Darkened NVIDIA green for light mode: #76B900 on white is 2.41:1, + # below the 3:1 minimum Fern warns about; #538300 passes (~4.3:1). + light: '#538300' background: light: '#FFFFFF' dark: '#000000' diff --git a/docs/fern/features/lora/README.md b/docs/fern/features/lora/README.md index 695130667f7e..7b88b87db772 100644 --- a/docs/fern/features/lora/README.md +++ b/docs/fern/features/lora/README.md @@ -15,7 +15,7 @@ LoRA (Low-Rank Adaptation) enables efficient fine-tuning and serving of speciali | SGLang | 🚧 | In progress | | TensorRT-LLM | ❌ | Not yet supported | -See the [Feature Matrix](../../reference/feature-matrix.md) for full compatibility details. +See the [feature support matrix](../../reference/compatibility.mdx#feature-support) for full compatibility details. ## Overview @@ -390,7 +390,7 @@ This works end-to-end across the publisher pipeline, the KV consolidator (for de ## See Also -- [Feature Matrix](../../reference/feature-matrix.md) - Backend compatibility overview +- [Compatibility](../../reference/compatibility.mdx#feature-support) - Backend compatibility overview - [vLLM Backend](../../backends/vllm/README.md) - vLLM-specific configuration - [Dynamo Operator](../../kubernetes/dynamo-operator.md) - Kubernetes operator overview - [Routing Concepts](../../components/router/router-concepts.md) - LoRA-aware request routing diff --git a/docs/fern/fern.config.json b/docs/fern/fern.config.json index df6f156bfd72..66f6846691c3 100644 --- a/docs/fern/fern.config.json +++ b/docs/fern/fern.config.json @@ -1,4 +1,4 @@ { - "organization": "ai-dynamo", + "organization": "nvidia", "version": "5.76.0" } diff --git a/docs/fern/getting-started/about.md b/docs/fern/getting-started/about.md index bc0de3c76ada..4acd16cd3716 100644 --- a/docs/fern/getting-started/about.md +++ b/docs/fern/getting-started/about.md @@ -193,6 +193,6 @@ Explore the following resources to go deeper: - [Kubernetes Deployment](../kubernetes/README.md) -- Deploy at scale with Grove - [Inference Gateway (GAIE)](../kubernetes/inference-gateway.md) -- Run Dynamo in gateway mode behind the K8s Inference Gateway - [Overall Architecture](../design-docs/architecture.md) -- Full technical design -- [Support Matrix](../reference/support-matrix.md) -- Check hardware and engine compatibility +- [Support Matrix](../reference/compatibility.mdx) -- Check hardware and engine compatibility **Further reading:** [Dynamo Digest](../digest/index.mdx). diff --git a/docs/fern/getting-started/about.zh-CN.md b/docs/fern/getting-started/about.zh-CN.md index d87255a15c08..f91accfe690a 100644 --- a/docs/fern/getting-started/about.zh-CN.md +++ b/docs/fern/getting-started/about.zh-CN.md @@ -162,6 +162,6 @@ Dynamo 提供内置指标、分布式追踪和日志，用于监控推理部署 - [Planner](../components/planner/planner-guide.zh-CN.md) -- 配置基于 SLA 的自动扩缩容 - [Kubernetes Deployment](../kubernetes/README.md) -- 使用 Grove 进行大规模部署 - [Overall Architecture](../design-docs/architecture.zh-CN.md) -- 完整技术设计 -- [Support Matrix](../reference/support-matrix.md) -- 检查硬件和引擎兼容性 +- [Support Matrix](../reference/compatibility.mdx) -- 检查硬件和引擎兼容性 **延伸阅读：** [Dynamo Digest](../digest/index.mdx)。 diff --git a/docs/fern/getting-started/local-installation.mdx b/docs/fern/getting-started/local-installation.mdx index 5e548779acf7..bfece7f3677d 100644 --- a/docs/fern/getting-started/local-installation.mdx +++ b/docs/fern/getting-started/local-installation.mdx @@ -27,7 +27,7 @@ For production multi-node clusters, see the [Kubernetes Deployment Guide](../kub TensorRT-LLM does not support Python 3.11. -For the full compatibility matrix including backend framework versions, see the [Support Matrix](../reference/support-matrix.md). +For the full compatibility matrix including backend framework versions, see the [Support Matrix](../reference/compatibility.mdx). ## Prerequisites @@ -206,7 +206,7 @@ This applies to **local, single-machine** setups. On Kubernetes, the Dynamo oper **CUDA/driver version mismatch** -Run `nvidia-smi` to check your driver version. Dynamo requires driver 575.51.03+ for CUDA 12 or 580.00.03+ for CUDA 13. B300/GB300 GPUs require CUDA 13. See the [Support Matrix](../reference/support-matrix.md) for full requirements. +Run `nvidia-smi` to check your driver version. Dynamo requires driver 575.51.03+ for CUDA 12 or 580.00.03+ for CUDA 13. B300/GB300 GPUs require CUDA 13. See the [Support Matrix](../reference/compatibility.mdx) for full requirements. **Python 3.11 with TensorRT-LLM** diff --git a/docs/fern/getting-started/local-installation.zh-CN.md b/docs/fern/getting-started/local-installation.zh-CN.md index 16fdb46b3e03..4b6bdb823937 100644 --- a/docs/fern/getting-started/local-installation.zh-CN.md +++ b/docs/fern/getting-started/local-installation.zh-CN.md @@ -28,7 +28,7 @@ description: 使用容器或 PyPI 在本地机器或 VM 上安装并运行 Dynam TensorRT-LLM 不支持 Python 3.11。 -如需查看包含后端框架版本在内的完整兼容性矩阵，请参阅[支持矩阵](../reference/support-matrix.md)。 +如需查看包含后端框架版本在内的完整兼容性矩阵，请参阅[支持矩阵](../reference/compatibility.mdx)。 ## 安装 Dynamo @@ -194,7 +194,7 @@ curl localhost:8000/v1/chat/completions \ **CUDA/驱动版本不匹配** -运行 `nvidia-smi` 检查你的驱动版本。Dynamo 对 CUDA 12 需要驱动 575.51.03+，对 CUDA 13 需要驱动 580.00.03+。B300/GB300 GPU 需要 CUDA 13。完整要求请参阅[支持矩阵](../reference/support-matrix.md)。 +运行 `nvidia-smi` 检查你的驱动版本。Dynamo 对 CUDA 12 需要驱动 575.51.03+，对 CUDA 13 需要驱动 580.00.03+。B300/GB300 GPU 需要 CUDA 13。完整要求请参阅[支持矩阵](../reference/compatibility.mdx)。 **模型无法装入 GPU（OOM）** diff --git a/docs/fern/index.yml b/docs/fern/index.yml index 87fc929afa08..13db22a9d4a5 100644 --- a/docs/fern/index.yml +++ b/docs/fern/index.yml @@ -259,6 +259,18 @@ navigation: path: features/multimodal/encoder-disaggregation.md - page: Multimodal KV Routing path: features/multimodal/multimodal-kv-routing.md + # Per-backend multimodal pages: linked from the Compatibility + # feature tables; registered here (hidden) so those links + # resolve without changing the visible sidebar. + - section: Per-Backend Multimodal + hidden: true + contents: + - page: Multimodal with vLLM + path: features/multimodal/multimodal-vllm.md + - page: Multimodal with SGLang + path: features/multimodal/multimodal-sglang.md + - page: Multimodal with TensorRT-LLM + path: features/multimodal/multimodal-trtllm.md - section: Diffusion slug: diffusion contents: @@ -747,8 +759,36 @@ navigation: path: reference/compatibility.mdx - page: Release Artifacts path: reference/release-artifacts.mdx + - section: Releases + contents: + - page: Release History + path: reference/release-notes/README.mdx + - page: Dynamo v1.3.0 + path: reference/release-notes/v1-3-0.mdx + slug: v1-3-0 + - page: Dynamo v1.2.0 + path: reference/release-notes/v1-2-0.mdx + slug: v1-2-0 + - page: Dynamo v1.1.0 + path: reference/release-notes/v1-1-0.mdx + slug: v1-1-0 + - page: Dynamo v1.0.0 + path: reference/release-notes/v1-0-0.mdx + slug: v1-0-0 + - page: Known Issues + path: reference/known-issues.mdx + - page: Deprecations + path: reference/deprecations.mdx + - page: Model Early Access Builds + path: reference/model-early-access-builds.mdx - page: Glossary path: reference/glossary.md + # Machine-readable mirror of releases.data.ts for agents; reachable + # by URL and llms indexes, not the sidebar. + - page: Releases (machine-readable) + path: reference/releases-data.mdx + slug: releases-data + hidden: true # ==================== Kubernetes API variant ==================== - title: Kubernetes API diff --git a/docs/fern/reference/compatibility.mdx b/docs/fern/reference/compatibility.mdx index e3738947c21a..ef594b1181c1 100644 --- a/docs/fern/reference/compatibility.mdx +++ b/docs/fern/reference/compatibility.mdx @@ -2,77 +2,69 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 title: Compatibility +subtitle: Hardware, platform, and feature support for Dynamo backends --- -## Supported Platforms (At a Glance) +import { ReferenceStyles } from "@/components/ReferenceStyles"; +import { CompatibilityHero } from "@/components/CompatibilityHero"; +import { FeatureHeatmap } from "@/components/FeatureHeatmap"; +import { BackendVersionMatrix } from "@/components/BackendVersionMatrix"; +import { CudaDriverMatrix } from "@/components/CudaDriverMatrix"; +import { RunsWhereWizard } from "@/components/RunsWhereWizard"; -**Latest stable release:** [v1.2.1](https://github.com/ai-dynamo/dynamo/releases/tag/v1.2.1) -- SGLang `0.5.11` (NIXL `1.0.1`) | TensorRT-LLM `1.3.0rc14` (NIXL `0.10.1`) | vLLM `0.20.1` (NIXL `0.10.1`) + -**Experimental release:** [v1.3.0-dev.1](https://github.com/ai-dynamo/dynamo/tree/release/1.3.0-dev.1) *(full-platform preview of v1.3.0 -- all runtime + component containers, wheels, crates, Helm)* -- SGLang `0.5.12.post1` | TensorRT-LLM `1.3.0rc17` | vLLM `0.22.0` | NIXL `1.1.0` (vLLM); `1.0.1` (SGLang); `0.10.1` (TRT-LLM) + -| Requirement | Supported | -| :--- | :--- | -| **GPU** | Ampere Ada Lovelace Hopper Blackwell | -| **OS** | Ubuntu 24.04
Ubuntu 22.04
CentOS Stream 9 (experimental) -| **Arch** | x86_64 ARM64 (Ubuntu 24.04 only) -| **SGLang** | CUDA 12.9 CUDA 13.0 | -| **TensorRT-LLM** | CUDA 13.1 | -| **vLLM** | CUDA 12.9 CUDA 13.0 | + +See [Release Artifacts](release-artifacts.mdx) for the full artifact inventory — container images, wheels, Helm charts, and crates — and [Model Early Access Builds](model-early-access-builds.mdx) for per-model early access container builds. + -Dynamo ships multi-arch (x86_64 + ARM64) container images. For the full per-version artifact history — container images, wheels, Helm charts, and crates — see [Release Artifacts](release-artifacts.mdx). +## Backend Dependencies -### Backend Dependencies +The backend framework versions included with the current Dynamo release. Backend versions listed are the only versions tested and supported for each release. TensorRT-LLM does not support Python 3.11; installation of the `ai-dynamo[trtllm]` wheel will fail on Python 3.11. -The backend framework versions included with the current Dynamo releases: + -| **Dynamo** | **SGLang** | **TensorRT-LLM** | **vLLM** | **NIXL** | -| :--- | :--- | :--- | :--- | :--- | -| **v1.2.1** | `0.5.11` | `1.3.0rc14` | `0.20.1` | `0.10.1` (TRT-LLM, vLLM); `1.0.1` (SGLang) | -| **v1.2.0** | `0.5.11` | `1.3.0rc14` | `0.20.1` | `0.10.1` (TRT-LLM, vLLM); `1.0.1` (SGLang) | -| **v1.1.1** | `0.5.10.post1` | `1.3.0rc11` | `0.19.0` | `0.10.1` (TRT-LLM, vLLM); `1.0.1` (SGLang) | -| **v1.3.0-dev.1** *(experimental)* | `0.5.12.post1` | `1.3.0rc17` | `0.22.0` | `1.1.0` (vLLM); `1.0.1` (SGLang); `0.10.1` (TRT-LLM) | + + + -`v1.3.0-dev.1` is a full-platform experimental preview; not every backend ships a runtime container for every dev tag. See [Release Artifacts](release-artifacts.mdx) for the published images, wheels, charts, and crates. Backend versions listed are the only versions tested and supported for each release. TensorRT-LLM does not support Python 3.11. - -### CUDA & Driver Requirements +## CUDA & Driver Requirements Dynamo container images include CUDA toolkit libraries; the host must have a compatible NVIDIA GPU driver. -| Backend | CUDA Toolkit | Min Driver | -| :--- | :--- | :--- | -| **SGLang** | 12.9 | 575.xx+ | -| | 13.0 | 580.xx+ | -| **TensorRT-LLM** | 13.1 | 580.xx+ | -| **vLLM** | 12.9 | 575.xx+ | -| | 13.0 | 580.xx+ | - -The table above is for **v1.2.0**; `v1.3.0-dev.1` follows the same CUDA matrix. For extended driver compatibility beyond these minimums (forward compatibility, `cuda-compat` packages, troubleshooting), see the [CUDA Compatibility documentation](https://docs.nvidia.com/deploy/cuda-compatibility/). - -## Feature Quick Comparison - -**Legend:** -* ✅ : Supported -* ⚠️ : Supported, with a caveat -* 🚧 : Work in Progress / Experimental -* ✗ : Not Supported - -| Feature | SGLang | TensorRT-LLM | vLLM | Learn More | -| :--- | :---: | :---: | :---: | :--- | -| **Disaggregated Serving** | ✅ | ✅ | ✅ | [Design Doc][disagg] | -| **KV-Aware Routing** | ✅ | ✅ | ✅ | [Router Doc][kv-routing] | -| **SLA-Based Planner** | ✅ | ✅ | ✅ | [Planner Doc][planner] | -| **KV Block Manager** | 🚧 | ✅ | ✅ | [KVBM Doc][kvbm] | -| **Multimodal (Image)** | ✅ | ✅ | ✅ | [Multimodal Doc][mm] | -| **Multimodal (Video)** | ✅ | | ✅ | [Multimodal Doc][mm] | -| **Multimodal (Audio)** | | | 🚧 | [Multimodal Doc][mm] | -| **Request Migration** | ✅ | 🚧 | ✅ | [Migration Doc][migration] | -| **Request Cancellation** | 🚧 | ✅ | ✅ | Backend pages | -| **LoRA** | | | ✅ | [LoRA Doc][lora] | -| **Tool Calling** | ✅ | ✅ | ✅ | [Tool Calling Doc][tools] | -| **Speculative Decoding** | 🚧 | ✅ | ✅ | Backend pages | -| **Dynamo Snapshot** | ✅ | | ✅ | [Snapshot Docs][snapshot] | - -## Per-Backend Feature Support + + +For extended driver compatibility beyond these minimums (forward compatibility, `cuda-compat` packages, troubleshooting), see the [CUDA Compatibility documentation](https://docs.nvidia.com/deploy/cuda-compatibility/). + + + + + +### What Runs Where + +Pick your backend and the CUDA generation your host driver supports to see which Dynamo releases you can run — and what to pull for the current one. + + + +## Platform Notes + +Dynamo ships multi-arch (x86_64 + ARM64) container images. Wheels are built in a manylinux_2_28-compatible environment and validated on CentOS Stream 9 and Ubuntu 22.04/24.04; other Linux distributions are expected to work but are not officially verified. + +### Cloud Service Providers + +**Amazon Linux 2023** (AWS) · x86_64 · Supported + + +**AL2023 TensorRT-LLM limitation:** there is a known issue with the TensorRT-LLM framework when running the AL2023 container locally with `docker run --network host ...` due to a [bug](https://github.com/mpi4py/mpi4py/discussions/491#discussioncomment-12660609) in mpi4py. Replace the `--network host` flag with precise networking configuration by mapping only the necessary ports (4222 for NATS, 2379/2380 for etcd, 8000 for the frontend). + + +## Feature Support + + + +### Per-Backend Detail @@ -83,16 +75,16 @@ vLLM offers the broadest feature coverage in Dynamo, with full support for disag | Feature | Supported? | Notes | | :------------------------ | :--------: | :------------------------------------------------------------------------------------------------------------------------------------------- | -| **Disaggregated Serving** | ✅ | Prefill/decode separation with NIXL KV transfer | -| **KV-Aware Routing** | ✅ | | -| **SLA-Based Planner** | ✅ | | -| **KV Block Manager** | ✅ | | -| **Multimodal** | ✅ | Image + video; audio 🚧 (Qwen2-Audio, experimental). With KV-aware routing, image-aware routing on documented paths ([Source][mm-kv-routing]) | -| **Request Migration** | ✅ | | -| **Request Cancellation** | ✅ | | -| **LoRA** | ✅ | Dynamic load/unload; KV-aware routing supports adapter affinity | -| **Tool Calling** | ✅ | | -| **Speculative Decoding** | ✅ | Eagle3 ([Source][vllm-spec]) | +| **Disaggregated Serving** | ✓ | Prefill/decode separation with NIXL KV transfer | +| **KV-Aware Routing** | ✓ | | +| **SLA-Based Planner** | ✓ | | +| **KV Block Manager** | ✓ | | +| **Multimodal** | ✓ | Image + video; audio experimental (Qwen2-Audio). With KV-aware routing, image-aware routing on documented paths ([Source][mm-kv-routing]) | +| **Request Migration** | ✓ | | +| **Request Cancellation** | ✓ | | +| **LoRA** | ✓ | Dynamic load/unload; KV-aware routing supports adapter affinity | +| **Tool Calling** | ✓ | | +| **Speculative Decoding** | ✓ | Eagle3 ([Source][vllm-spec]) | @@ -103,16 +95,16 @@ SGLang is optimized for high-throughput serving with fast primitives, providing | Feature | Supported? | Notes | | :------------------------ | :--------: | :------------------------------------------------------------------------------------------------------------------------------------ | -| **Disaggregated Serving** | ✅ | | -| **KV-Aware Routing** | ✅ | | -| **SLA-Based Planner** | ✅ | | -| **KV Block Manager** | 🚧 | Work in progress across all combinations | -| **Multimodal** | ✅ | Image + video. ✗ Not compatible with KV-aware routing. Disagg patterns: EPD, E/PD, E/P/D (not traditional EP/D) ([Source][mm-sglang]) | -| **Request Migration** | ✅ | | -| **Request Cancellation** | 🚧 | Remote-prefill-phase cancellation not supported in disaggregated mode ([Source][sglang-readme]) | -| **LoRA** | ✗ | Not supported | -| **Tool Calling** | ✅ | | -| **Speculative Decoding** | 🚧 | Code hooks exist; no examples or docs yet | +| **Disaggregated Serving** | ✓ | | +| **KV-Aware Routing** | ✓ | | +| **SLA-Based Planner** | ✓ | | +| **KV Block Manager** | WIP | Work in progress across all combinations | +| **Multimodal** | ✓ | Image + video. Not compatible with KV-aware routing. Disagg patterns: EPD, E/PD, E/P/D (not traditional EP/D) ([Source][mm-sglang]) | +| **Request Migration** | ✓ | | +| **Request Cancellation** | WIP | Remote-prefill-phase cancellation not supported in disaggregated mode ([Source][sglang-readme]) | +| **LoRA** | — | Not supported | +| **Tool Calling** | ✓ | | +| **Speculative Decoding** | WIP | Code hooks exist; no examples or docs yet | @@ -123,26 +115,157 @@ TensorRT-LLM delivers maximum inference performance and optimization, with full | Feature | Supported? | Notes | | :------------------------ | :--------: | :---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| **Disaggregated Serving** | ✅ | | -| **KV-Aware Routing** | ✅ | | -| **SLA-Based Planner** | ✅ | | -| **KV Block Manager** | ✅ | | -| **Multimodal** | ✅ | Image only (URLs + pre-computed embeddings). Disagg: EP/D + E/P/D. With KV-aware routing, via dedicated MM Router Worker (requires KV event publishing) ([Source][mm-trtllm]) | -| **Request Migration** | ✅ | 🚧 With multimodal | -| **Request Cancellation** | ⚠️ | Engine temporarily not notified of cancellations — resources for cancelled requests are not freed (known issue) | -| **LoRA** | ✗ | Not supported | -| **Tool Calling** | ✅ | | -| **Speculative Decoding** | ✅ | | +| **Disaggregated Serving** | ✓ | | +| **KV-Aware Routing** | ✓ | | +| **SLA-Based Planner** | ✓ | | +| **KV Block Manager** | ✓ | | +| **Multimodal** | ✓ | Image only (URLs + pre-computed embeddings). Disagg: EP/D + E/P/D. With KV-aware routing, via dedicated MM Router Worker (requires KV event publishing) ([Source][mm-trtllm]) | +| **Request Migration** | ✓ | Work in progress with multimodal | +| **Request Cancellation** | ! | Engine temporarily not notified of cancellations — resources for cancelled requests are not freed (known issue) | +| **LoRA** | — | Not supported | +| **Tool Calling** | ✓ | | +| **Speculative Decoding** | ✓ | | -[disagg]: ../design-docs/disagg-serving.md -[kv-routing]: ../components/router/README.md -[planner]: ../components/planner/planner-guide.md -[kvbm]: ../components/kvbm/README.md -[mm]: ../features/multimodal/README.md -[migration]: ../fault-tolerance/request-migration.md -[lora]: ../features/lora/README.md -[tools]: ../tool-calling/README.md -[snapshot]: ../kubernetes/snapshot.md +[vllm-readme]: ../backends/vllm/README.md +[sglang-readme]: ../backends/sglang/README.md +[trtllm-readme]: ../backends/trtllm/README.md +[mm-kv-routing]: ../features/multimodal/multimodal-kv-routing.md +[mm-sglang]: ../features/multimodal/multimodal-sglang.md +[mm-trtllm]: ../features/multimodal/multimodal-trtllm.md +[vllm-spec]: ../features/speculative-decoding/speculative-decoding-vllm.md + +{/* llms-tables:begin — generated by scripts/gen_llms_tables.py, do not edit */} + + +Current stable release: v1.3.0 (container tag `1.3.0`, wheel version `1.3.0.post1`). + +**Backend engine pins per Dynamo release** + +| Dynamo | Type | SGLang | TensorRT-LLM | vLLM | NIXL (SGL / TRT / vLLM) | UCX | +| --- | --- | --- | --- | --- | --- | --- | +| main (ToT) | development head | 0.5.15 | 1.3.0rc21 | 0.25.1 | 1.3.0 / 1.0.1 / 1.1.0 | - | +| v1.3.0 | stable | 0.5.14 | 1.3.0rc19 | 0.23.0 | 1.3.0 / 1.0.1 / 1.1.0 | 1.20.x | +| v1.3.0-dev.1 | platform-preview | 0.5.12.post1 | 1.3.0rc17 | 0.22.0 | 1.0.1 / 0.10.1 / 1.1.0 | - | +| v1.2.1 | patch | 0.5.11 | 1.3.0rc14 | 0.20.1 | 1.0.1 / 0.10.1 / 0.10.1 | - | +| v1.2.0 | stable | 0.5.11 | 1.3.0rc14 | 0.20.1 | 1.0.1 / 0.10.1 / 0.10.1 | 1.20.0 | +| v1.2.0-deepseek-v4-dev.3 | model-build | upstream DSv4 preview | - | 0.20.1 | - / - / 0.10.1 | - | +| v1.2.0-deepseek-v4-dev.2 | model-build | upstream DSv4 preview | - | 0.20.0 | - / - / 0.10.1 | - | +| v1.1.1 | patch | 0.5.10.post1 | 1.3.0rc11 | 0.19.0 | 1.0.1 / 0.10.1 / 0.10.1 | - | +| v1.1.0 | stable | 0.5.10.post1 | 1.3.0rc11 | 0.19.0 | 1.0.1 / 0.10.1 / 0.10.1 | 1.20 | +| v1.1.0-dev.3 | platform-preview | 0.5.10.post1 | 1.3.0rc11 | 0.19.0 | 1.0.1 / 0.10.1 / 0.10.1 | - | +| v1.1.0-dev.2 | platform-preview | 0.5.9 | 1.3.0rc9 | 0.19.0 | 1.0.1 / 0.10.1 / 0.10.1 | - | +| v1.1.0-dev.1 | platform-preview | 0.5.9 | 1.3.0rc5.post1 | 0.17.1 | 1.0.1 / 0.10.1 / 0.10.1 | - | +| v1.0.2 | patch | 0.5.9 | 1.3.0rc5.post1 | 0.16.0 | 0.10.1 / 0.10.1 / 0.10.1 | - | +| v1.0.1 | patch | 0.5.9 | 1.3.0rc5.post1 | 0.16.0 | 0.10.1 / 0.10.1 / 0.10.1 | - | +| v1.0.0 | stable | 0.5.9 | 1.3.0rc5.post1 | 0.16.0 | 0.10.1 / 0.10.1 / 0.10.1 | - | +| v0.9.1 | patch | 0.5.8 | 1.3.0rc3 | 0.14.1 | 0.9.0 / 0.9.0 / 0.9.0 | - | +| v0.9.0 | stable | 0.5.8 | 1.3.0rc1 | 0.14.1 | 0.9.0 / 0.9.0 / 0.9.0 | - | +| v0.8.1 | patch | 0.5.6.post2 | 1.2.0rc6.post1 | 0.12.0 | 0.8.0 / 0.8.0 / 0.8.0 | - | +| v0.8.0 | stable | 0.5.6.post2 | 1.2.0rc6.post1 | 0.12.0 | 0.8.0 / 0.8.0 / 0.8.0 | - | +| v0.7.1 | patch | 0.5.4.post3 | 1.2.0rc3 | 0.11.0 | 0.8.0 / 0.8.0 / 0.8.0 | - | +| v0.7.0 | stable | 0.5.4.post3 | 1.2.0rc2 | 0.11.0 | 0.8.0 / 0.8.0 / 0.8.0 | - | +| v0.6.1 | patch | 0.5.3.post2 | 1.1.0rc5 | 0.11.0 | 0.6.0 / 0.6.0 / 0.6.0 | - | +| v0.6.0 | stable | 0.5.3.post2 | 1.1.0rc5 | 0.11.0 | 0.6.0 / 0.6.0 / 0.6.0 | - | + +**CUDA toolkit and minimum driver per Dynamo release** + +| Dynamo | Backend | CUDA Toolkit | Min Driver | Note | +| --- | --- | --- | --- | --- | +| 1.3.0 | SGLang | 13.0 | 580.xx+ | - | +| 1.3.0 | TensorRT-LLM | 13.1 | 580.xx+ | - | +| 1.3.0 | vLLM | 13.0 | 580.xx+ | - | +| 1.2.1 | SGLang | 12.9 | 575.xx+ | - | +| 1.2.1 | SGLang | 13.0 | 580.xx+ | - | +| 1.2.1 | TensorRT-LLM | 13.1 | 580.xx+ | - | +| 1.2.1 | vLLM | 12.9 | 575.xx+ | - | +| 1.2.1 | vLLM | 13.0 | 580.xx+ | - | +| 1.2.0 | SGLang | 12.9 | 575.xx+ | - | +| 1.2.0 | SGLang | 13.0 | 580.xx+ | - | +| 1.2.0 | TensorRT-LLM | 13.1 | 580.xx+ | - | +| 1.2.0 | vLLM | 12.9 | 575.xx+ | - | +| 1.2.0 | vLLM | 13.0 | 580.xx+ | - | +| 1.1.1 | SGLang | 12.9 | 575.xx+ | - | +| 1.1.1 | SGLang | 13.0 | 580.xx+ | - | +| 1.1.1 | TensorRT-LLM | 13.1 | 580.xx+ | - | +| 1.1.1 | vLLM | 12.9 | 575.xx+ | - | +| 1.1.1 | vLLM | 13.0 | 580.xx+ | - | +| 1.1.0 | SGLang | 12.9 | 575.xx+ | - | +| 1.1.0 | SGLang | 13.0 | 580.xx+ | - | +| 1.1.0 | TensorRT-LLM | 13.1 | 580.xx+ | - | +| 1.1.0 | vLLM | 12.9 | 575.xx+ | - | +| 1.1.0 | vLLM | 13.0 | 580.xx+ | - | +| 1.0.2 | SGLang | 12.9 | 575.xx+ | - | +| 1.0.2 | SGLang | 13.0 | 580.xx+ | - | +| 1.0.2 | TensorRT-LLM | 13.1 | 580.xx+ | - | +| 1.0.2 | vLLM | 12.9 | 575.xx+ | - | +| 1.0.2 | vLLM | 13.0 | 580.xx+ | - | +| 1.0.1 | SGLang | 12.9 | 575.xx+ | - | +| 1.0.1 | SGLang | 13.0 | 580.xx+ | - | +| 1.0.1 | TensorRT-LLM | 13.1 | 580.xx+ | - | +| 1.0.1 | vLLM | 12.9 | 575.xx+ | - | +| 1.0.1 | vLLM | 13.0 | 580.xx+ | - | +| 1.0.0 | SGLang | 12.9 | 575.xx+ | - | +| 1.0.0 | SGLang | 13.0 | 580.xx+ | - | +| 1.0.0 | TensorRT-LLM | 13.1 | 580.xx+ | - | +| 1.0.0 | vLLM | 12.9 | 575.xx+ | - | +| 1.0.0 | vLLM | 13.0 | 580.xx+ | - | +| 0.9.1 | SGLang | 12.9 | 575.xx+ | - | +| 0.9.1 | TensorRT-LLM | 13.0 | 580.xx+ | - | +| 0.9.1 | vLLM | 12.9 | 575.xx+ | - | +| 0.9.0 | SGLang | 12.9 | 575.xx+ | - | +| 0.9.0 | TensorRT-LLM | 13.0 | 580.xx+ | - | +| 0.9.0 | vLLM | 12.9 | 575.xx+ | - | +| 0.8.1 | SGLang | 12.9 | 575.xx+ | - | +| 0.8.1 | SGLang | 13.0 | 580.xx+ | Experimental | +| 0.8.1 | TensorRT-LLM | 13.0 | 580.xx+ | - | +| 0.8.1 | vLLM | 12.9 | 575.xx+ | - | +| 0.8.1 | vLLM | 13.0 | 580.xx+ | Experimental | +| 0.8.0 | SGLang | 12.9 | 575.xx+ | - | +| 0.8.0 | SGLang | 13.0 | 580.xx+ | Experimental | +| 0.8.0 | TensorRT-LLM | 13.0 | 580.xx+ | - | +| 0.8.0 | vLLM | 12.9 | 575.xx+ | - | +| 0.8.0 | vLLM | 13.0 | 580.xx+ | Experimental | +| 0.7.1 | SGLang | 12.8 | 570.xx+ | - | +| 0.7.1 | TensorRT-LLM | 13.0 | 580.xx+ | - | +| 0.7.1 | vLLM | 12.9 | 575.xx+ | - | +| 0.7.0 | SGLang | 12.9 | 575.xx+ | - | +| 0.7.0 | TensorRT-LLM | 13.0 | 580.xx+ | - | +| 0.7.0 | vLLM | 12.8 | 570.xx+ | - | + +- Patch versions (e.g. v0.8.1.post1, v0.7.0.post1) have the same CUDA support as their base version. +- Early access v1.1.0-dev.* images follow the same CUDA matrix as v1.0.2. The v1.2.0-deepseek-v4-dev.3 vLLM container is CUDA 13.0 multi-arch; the SGLang containers split by arch (CUDA 12.9 on amd64, CUDA 13.0 on arm64). +- Experimental CUDA 13 images are not published for all versions. + +**Feature support by backend (v1.3.0)** + +| Feature | SGLang | TensorRT-LLM | vLLM | +| --- | --- | --- | --- | +| Disaggregated Serving | Supported | Supported | Supported (Prefill/decode separation with NIXL KV transfer) | +| KV-Aware Routing | Supported | Supported | Supported | +| SLA-Based Planner | Supported | Supported | Supported | +| KV Block Manager | Experimental (Work in progress across all combinations) | Supported | Supported | +| Multimodal (Image) | Supported (Not compatible with KV-aware routing. Disagg patterns: EPD, E/PD, E/P/D (not traditional EP/D)) | Supported (Image URLs + pre-computed embeddings. Disagg: EP/D + E/P/D. KV-aware routing via dedicated MM Router Worker (requires KV event publishing)) | Supported (With KV-aware routing, image-aware routing on documented paths) | +| Multimodal (Video) | Supported | Not supported | Supported (Video input with frame sampling) | +| Multimodal (Audio) | Not supported | Not supported | Experimental (Qwen2-Audio, experimental) | +| Request Migration | Supported | Supported (Work in progress with multimodal) | Supported | +| Request Cancellation | Experimental (Remote-prefill-phase cancellation not supported in disaggregated mode) | Supported with caveat (Engine temporarily not notified of cancellations — resources for cancelled requests are not freed (known issue)) | Supported | +| LoRA | Not supported | Not supported | Supported (Dynamic load/unload; KV-aware routing supports adapter affinity) | +| Tool Calling | Supported | Supported | Supported | +| Speculative Decoding | Experimental (Code hooks exist; no examples or docs yet) | Supported | Supported (Eagle3) | +| Dynamo Snapshot | Supported | Not supported | Supported | + +**Platform support** + +- GPU architectures: Blackwell, Hopper, Ada Lovelace, Ampere +- OS: Ubuntu 24.04 (x86_64, ARM64) — Supported +- OS: Ubuntu 22.04 (x86_64) — Supported +- OS: CentOS Stream 9 (x86_64) — Experimental +- CSP: AWS — Amazon Linux 2023 (x86_64) — Supported +- CPU architectures: x86_64, ARM64 (Ubuntu 24.04 only) +- Wheels: Wheels are built in a manylinux_2_28-compatible environment and validated on CentOS Stream 9 and Ubuntu 22.04/24.04. Other Linux distributions are expected to work but are not officially verified. + + +{/* llms-tables:end */} diff --git a/docs/fern/reference/deprecations.mdx b/docs/fern/reference/deprecations.mdx new file mode 100644 index 000000000000..ff420c5fe2e4 --- /dev/null +++ b/docs/fern/reference/deprecations.mdx @@ -0,0 +1,346 @@ +--- +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +title: Deprecations +subtitle: Breaking changes, deprecations, and removals across Dynamo releases +--- + +import { ReferenceStyles } from "@/components/ReferenceStyles"; +import { UpgradeSelector } from "@/components/UpgradeSelector"; +import { RELEASE_STATS } from "@/components/releases.data"; + + + +This page is the ledger of breaking changes, behavioral changes, and deprecations and removals for each Dynamo release, mirrored verbatim from the GitHub release notes. Each entry carries a badge: Removed means the surface is gone, Deprecated means it still works but emits warnings, and Behavioral means defaults or behavior changed. + +Upgrading? Pick the version you run today to see the dependency migration and the exact reading list. + + + + + +## v1.3.0 — Jul 20, 2026 + +{/* Count mirrors RELEASE_STATS["v1.3.0"].breaking (24); the v1.3.0 notes page renders it from the data via ReleaseHeader. Kept literal here — a bare MDX expression at paragraph start breaks the page compile. */} +24 entries — 14 behavioral changes, 10 deprecated and removed. + +Behavioral **CUDA 12 container images removed — CUDA 13 only** (`vLLM and SGLang runtime images shipped CUDA 12.9 and CUDA 13 variants` -> `CUDA 13 images only`). + + > **Migrate:** Move all deployments to the CUDA 13 runtime images before upgrading — the CUDA 12.9 variants are no longer published. The bare `vllm-runtime` / `sglang-runtime` / `tensorrtllm-runtime` tags now resolve to CUDA 13, with explicit `-cuda13` aliases on the same digest. + +Behavioral **Multimodal now requires an explicit `--enable-multimodal` flag** (`--multimodal-worker / --multimodal-encode-worker` -> `--enable-multimodal (with --dedicated-mm-encoder / --disaggregation-mode)`); workers without the flag now raise a RuntimeError on multimodal requests instead of silently processing them as text-only ([#10680](https://github.com/ai-dynamo/dynamo/pull/10680)). + + > **Migrate:** Add `--enable-multimodal` (and `--dedicated-mm-encoder` for internal encode-worker topologies) to both prefill and decode workers so multimodal requests are accepted rather than rejected. + +Behavioral **`disaggregation-mode` default changed to `agg`; `prefill_and_decode` deprecated** (`--disaggregation-mode default 'prefill_and_decode' (DisaggregationMode.AGGREGATED)` -> `--disaggregation-mode default 'agg'; 'prefill_and_decode' deprecated, use 'pd'`) ([#10690](https://github.com/ai-dynamo/dynamo/pull/10690)). + + > **Migrate:** Use `--disaggregation-mode agg` for aggregated serving and `pd` for combined prefill+decode; legacy `prefill_and_decode` still works temporarily. + +Behavioral **GPU Memory Service (GMS) DRA now requires Kubernetes 1.34+** — GMS attaches GPUs to workers through the Kubernetes Dynamic Resource Allocation (DRA) API, which must now be the stable `resource.k8s.io/v1` (Kubernetes 1.34+) instead of the older 1.32+ API group; GMS errors if it is unavailable ([#9454](https://github.com/ai-dynamo/dynamo/pull/9454)). + + > **Migrate:** Upgrade the cluster to Kubernetes 1.34+ so `resource.k8s.io/v1` is available before using GMS `ResourceClaimTemplates`. + +Behavioral **GMS now wires GPUs into your declared client containers instead of a sidecar** — setting `enabled` used to start a GMS sidecar and swap the main container's GPUs for a DRA ResourceClaim; it now wires GPUs into the client containers you declare, and the operator no longer auto-creates `gms-loader`/`gms-saver` containers ([#9641](https://github.com/ai-dynamo/dynamo/pull/9641)). + + > **Migrate:** Because the operator no longer injects `gms-loader`/`gms-saver` containers automatically, list the containers that should use GMS-managed GPU memory yourself via `extraClientContainers` (and `checkpoint.job.gmsClientContainers` for checkpoint jobs). + +Behavioral **GMS "shadow mode" is now enabled only with inter-pod failover** — shadow mode runs the GPU Memory Service alongside the live KV path for validation without serving from it; `DYN_VLLM_GMS_SHADOW_MODE=true` is now injected only when inter-pod failover is on, not for every standalone inter-pod GMS deployment ([#10378](https://github.com/ai-dynamo/dynamo/pull/10378)). + + > **Migrate:** If you rely on shadow mode with standalone inter-pod GMS, enable failover (`mode=interPod`) to restore `DYN_VLLM_GMS_SHADOW_MODE` injection. + +Behavioral **Router backpressure defaults raised** — queue threshold `4.0 -> 16.0` and active-prefill-tokens threshold fraction `10.0 -> 64.0` ([#9547](https://github.com/ai-dynamo/dynamo/pull/9547)). + + > **Migrate:** Explicitly set `--router-queue-threshold=4.0` and `--active-prefill-tokens-threshold-frac=10.0` to retain the previous backpressure and prefill busy-detection behavior. + +Behavioral **Removed the forced default of `--runner generate` in vLLM** (`runner defaults to "generate" when not explicitly set` -> `runner defaults to vLLM's own value (auto-detection)`) ([#9710](https://github.com/ai-dynamo/dynamo/pull/9710)). + + > **Migrate:** If you relied on the implicit `runner=generate` default, pass `--runner generate` explicitly. + +Behavioral **Frontend nvext / admin-API master switches renamed to opt-out** (`DYN_ENABLE_FRONTEND_{NVEXT,ADMIN_API}` -> `DYN_DISABLE_*`) ([#11123](https://github.com/ai-dynamo/dynamo/pull/11123)). + + **Migrate:** The `nvext` and frontend admin-API surfaces are now on by default; disable them with `DYN_DISABLE_FRONTEND_NVEXT` / `DYN_DISABLE_FRONTEND_ADMIN_API` instead of enabling with `DYN_ENABLE_*`. + +Behavioral **Requests carrying `nvext.agent_context` are now rejected** — the `nvext` OpenAI-API extension no longer accepts the `agent_context` object (previously used to pass agent/trajectory identity); it is refused as an unknown field ([#10808](https://github.com/ai-dynamo/dynamo/pull/10808)). + + > **Migrate:** Stop sending `agent_context` in `nvext`; use trajectory identity headers instead. + +Behavioral **`spec.restart.id` on DGD create no longer triggers a restart (`Creating a DynamoGraphDeployment with spec.restart.id triggered an immediate restart` -> `spec.restart.id on initial creation is treated as already observed and does not trigger a restart`)** ([#10955](https://github.com/ai-dynamo/dynamo/pull/10955)). + + > **Migrate:** Create the DGD without `spec.restart`, then set `spec.restart.id` after creation to request a restart. + +Behavioral **Snapshot Helm chart defaults changed for live worker checkpointing** — Dynamo checkpoints running workers with CRIU (Checkpoint/Restore In Userspace); the chart now defaults TCP socket handling to `tcpClose=false / tcpEstablished=true` and points the CRIU `libDir` at `/usr/local/lib/snapshot/criu-plugins` instead of leaving it empty ([#10727](https://github.com/ai-dynamo/dynamo/pull/10727)). + + > **Migrate:** To preserve prior behavior, set `config.tcpClose=true`, `config.tcpEstablished=false`, and `config.libDir=""` explicitly in your snapshot values. + +Behavioral **DGD Auto checkpoint identity and status changes** — the DynamoCheckpoint printer column is renamed `Hash -> CheckpointID` (`.status.identityHash` -> `.status.checkpointID`), Auto checkpoints are no longer reused across DGDs via identity hash (now scoped to the owning DGD/component generation), and Manual mode without `checkpointRef` or `identity` now fails fast instead of silently waiting ([#10177](https://github.com/ai-dynamo/dynamo/pull/10177)). + + > **Migrate:** Read `.status.checkpointID` (identityHash remains as a deprecated mirror), set `checkpointRef` to explicitly reuse a named DynamoCheckpoint, and provide `checkpointRef` or `identity` when using Manual mode. + +Behavioral **Router `enforce-disagg` deprecated** ([#11295](https://github.com/ai-dynamo/dynamo/pull/11295)). + + **Migrate:** Remove `enforce-disagg` from router configuration; disaggregated routing is selected through the standard router modes. + +Removed **Removed the standalone `dynamo-gaie` Helm chart and gateway CRD install script for the inference gateway (EPP)** — the Endpoint Picker (EPP) inference-gateway extension now installs through the operator, so the standalone chart (`deploy/inference-gateway/standalone/helm/dynamo-gaie`) and the `install_gaie_crd_kgateway.sh` script are removed ([#10001](https://github.com/ai-dynamo/dynamo/pull/10001)). + + > **Migrate:** Move off the standalone EPP Helm chart to the operator-managed inference gateway path, and use `agentgateway` CRD installation instead of the removed `kgateway` install script. + +Removed **Removed `clear_kv_blocks` endpoint and its router function** (`POST /clear_kv_blocks` and `pub fn clear_kv_blocks_router` removed) ([#10556](https://github.com/ai-dynamo/dynamo/pull/10556)). + + > **Migrate:** Stop calling the `/clear_kv_blocks` HTTP endpoint and remove any callers of `clear_kv_blocks_router`; both the endpoint and the function/module were deleted. + +Removed **Removed `nvext.agent_context` and its session fields** (`nvext.agent_context` request-body field removed; `session_type_id` and `session_id` removed from AgentContext, leaving trajectory fields only) ([#10808](https://github.com/ai-dynamo/dynamo/pull/10808)). + + > **Migrate:** Remove `agent_context` from request bodies and supply trajectory identity via `x-dynamo-trajectory-id` / `x-dynamo-parent-trajectory-id` / `x-dynamo-trajectory-final` headers; only trajectory fields remain on AgentContext. + +Removed **Removed sticky-session routing and its `nvext.session_control` surface** — session affinity (pinning a conversation's requests to one worker) is replaced by trajectory-based KV/radix cache tags; this drops `nvext.session_control`, the worker session-lifecycle endpoint, and the `StickySessionRouter / InMemoryAffinityStore / AffinityBinding / AffinityKind` Rust types ([#10214](https://github.com/ai-dynamo/dynamo/pull/10214)). + + > **Migrate:** Stop sending `nvext.session_control` and migrate to trajectory-based radix cache tags; sticky routing, the worker lifecycle RPC, and the sticky routing types are no longer available. + +Removed **Removed `DYN_AGENT_TRACE` in favor of `DYN_REQUEST_TRACE`** (`DYN_AGENT_TRACE` -> `DYN_REQUEST_TRACE`) ([#10701](https://github.com/ai-dynamo/dynamo/pull/10701)). + + > **Migrate:** Set `DYN_REQUEST_TRACE=1` instead of `DYN_AGENT_TRACE`; agent-context enrichment now emits under the unified request-trace path. + +Removed **vLLM ModelExpress weight loading moved behind a single `--load-format modelexpress`** — ModelExpress streams model weights to workers for faster startup; the split `mx-source`/`mx-target` load formats and the `--model-express-url` / `MODEL_EXPRESS_URL` setting are removed in favor of the plugin-owned `--load-format modelexpress` ([#10049](https://github.com/ai-dynamo/dynamo/pull/10049)). + + > **Migrate:** Switch vLLM deployments to `--load-format modelexpress` and rely on the ModelExpress vLLM plugin; stop relying on `--model-express-url` / `MODEL_EXPRESS_URL`. + +Removed **Removed the diffusion-transformer data-parallel flag `--dit-dp-size`** — this TensorRT-LLM knob set the data-parallel degree for the DiT (Diffusion Transformer) image/video path; the flag and its `DYN_TRTLLM_DIT_DP_SIZE` env var are gone ([#10036](https://github.com/ai-dynamo/dynamo/pull/10036)). + + > **Migrate:** Remove `--dit-dp-size` and `DYN_TRTLLM_DIT_DP_SIZE` from your diffusion launch configuration. + +Deprecated **Deprecated per-backend multimodal flags (`--multimodal-worker` / `--multimodal-encode-worker` / `--modality multimodal`)** — superseded by a single `--enable-multimodal` plus `--disaggregation-mode` (with `--dedicated-mm-encoder` for internal encode workers); the old flags still work but emit deprecation warnings ([#10680](https://github.com/ai-dynamo/dynamo/pull/10680), [#10690](https://github.com/ai-dynamo/dynamo/pull/10690)). + + > **Migrate:** Switch to `--enable-multimodal` plus the appropriate `--disaggregation-mode` / `--dedicated-mm-encoder` combination. + +Removed **Removed the GMS checkpoint `loader`/`saver` override structs** — instead of the built-in `gpuMemoryService.checkpoint.{loader,saver}` containers, declare your own GMS client containers via `extraClientContainers` / `checkpoint.job.gmsClientContainers` ([#9641](https://github.com/ai-dynamo/dynamo/pull/9641)). + + > **Migrate:** Replace the `gpuMemoryService.checkpoint.{loader,saver}` config with user-declared containers referenced via `extraClientContainers` / `checkpoint.job.gmsClientContainers`. + +Removed **Fixed the misspelled Planner autoscaling key `decode_sacle_up_kv_rate`** — this SLA-Planner threshold governs when decode workers scale up under KV-cache pressure; the misspelled alias is removed in favor of the correct `decode_scale_up_kv_rate` ([#10601](https://github.com/ai-dynamo/dynamo/pull/10601)). + + > **Migrate:** Update any configuration using the misspelled `decode_sacle_up_kv_rate` key to the correct `decode_scale_up_kv_rate`. + + + + + +**ACTION REQUIRED:** The following changes require updates to your code, configuration, or deployment manifests before upgrading. + +Behavioral **DGD/DGDR Promoted to `v1beta1` as the Served API** ([#9235](https://github.com/ai-dynamo/dynamo/pull/9235), [#9262](https://github.com/ai-dynamo/dynamo/pull/9262)): The `DynamoGraphDeployment`, `DynamoComponentDeployment`, and `DynamoGraphDeploymentRequest` APIs now serve `v1beta1`. The `v1alpha1` ↔ `v1beta1` round-trip conversion is maintained for backward compatibility, but all docs and examples have moved to `v1beta1`. + + > **Migrate:** Update Helm values, Argo manifests, and any inline YAML to reference `v1beta1`. Existing `v1alpha1` resources continue to work via the conversion path during the transition; remove `v1alpha1` references at your next deploy. + +Behavioral **Duration Config Fields Suffixed with Units** ([#9246](https://github.com/ai-dynamo/dynamo/pull/9246)): Configuration fields representing durations were renamed to make their units explicit (e.g., `*_ttl` → `*_ttl_secs`). + + > **Migrate:** Audit your deployment YAMLs and CLI invocations for any duration field; the new suffix-bearing names are required. The previous unsuffixed names are no longer recognized. + +Behavioral **Concurrent Radix Tree is the Default Approximate Router** ([#9219](https://github.com/ai-dynamo/dynamo/pull/9219), [#9007](https://github.com/ai-dynamo/dynamo/pull/9007)): The default approximate KV routing backend switched to the concurrent radix tree with anchor-aware branch sharding. + + > **Migrate:** Update any custom router instrumentation that targeted the previous radix-tree internals. No action required for default deployments; routing semantics are equivalent. + +Behavioral **Inter-Pod GPU Memory Service Sidecar Model** ([#7777](https://github.com/ai-dynamo/dynamo/pull/7777), [#8829](https://github.com/ai-dynamo/dynamo/pull/8829)): GMS moves to an inter-pod sidecar that multiple workers can share. The Operator gates GMS+Snapshot combinations via Helm config. + + > **Migrate:** If you ran the v1.1.0 per-pod GMS deployment pattern, switch to the new shared-sidecar Helm values. The previous topology continues to work but is no longer the recommended path. + +Deprecated **CUDA 12 Containers Discontinued in v1.3.0:** v1.2.0 is the last release to publish CUDA 12 container images. Starting with v1.3.0, Dynamo will ship CUDA 13 images only. In v1.2.0 the vLLM and SGLang containers ship both CUDA 12.9 and CUDA 13.0 variants, while TensorRT-LLM is already CUDA 13 only. + + > **Migrate:** Move CUDA 12 deployments to the CUDA 13 container variants before upgrading to v1.3.0. + +Deprecated The following warnings from v1.1.0 still apply. Migrate before they are removed: + +- `v1alpha1` DGDR API: now in active conversion to `v1beta1` (see above) +- `enableGpuDiscovery` CRD field has no effect +- `ComponentName` field on `ServiceReplicaStatus`: migrate to `ComponentNames` +- Router CLI flags without the `--router-` prefix +- vLLM `--is-prefill-worker` / `--is-decode-worker`: migrate to `--disaggregation-mode` +- `--router-durable-kv-events`: migrate to the event-plane subscriber + + + + + + + +**ACTION REQUIRED:** The following changes require updates to your code, configuration, or deployment manifests before upgrading. + +Behavioral **`enable_nats` and `use_kv_events` Removed from `DistributedRuntime`** ([#7265](https://github.com/ai-dynamo/dynamo/pull/7265)): Both parameters are removed from `DistributedRuntime`, `create_runtime()`, and the `dynamo_worker()` decorator. NATS is now auto-detected from the event plane: enabled when the request plane is NATS or `NATS_SERVER` is configured. + + > **Migrate:** Drop both arguments from your Python entry points and configure NATS via the `DYN_EVENT_PLANE` and `NATS_SERVER` environment variables instead. + +Behavioral **Experimental `nvext.cache_control` Cache Pinning Removed** ([#7790](https://github.com/ai-dynamo/dynamo/pull/7790)): The experimental cache-pinning feature is removed: the `nvext.cache_control` request field, the `--enable-cache-control` flag, and the `DYN_ENABLE_CACHE_CONTROL` env var are all gone. SGLang upstream chose a different direction, so the v1.0.0 plumbing is being unwound. The Anthropic Messages parser still accepts `cache_control` blocks for protocol compatibility but no longer derives router-pin TTLs from them. + + > **Migrate:** If you depended on cache pinning, track the v1.2.0 sticky-session / session-controller work. There is no drop-in replacement in v1.1.0. + +Behavioral **Cargo-Built `dynamo-kv-indexer` Binary Removed** ([#7338](https://github.com/ai-dynamo/dynamo/pull/7338)): The Cargo-built `dynamo-kv-indexer` binary in `lib/kv-router/target/release/` is removed; the maturin-built binary shipped via the Python wheel is now the single source. + + > **Migrate:** Update launchers and Dockerfiles to point at the wheel-installed `dynamo-kv-indexer` (on `PATH` after `pip install ai-dynamo`). + +Behavioral **LLaVA-Specific EPD Path Removed; EPD Now Single-GPU** ([#6674](https://github.com/ai-dynamo/dynamo/pull/6674)): The LLaVA-specific multimodal EPD code path is removed, EPD is now constrained to single-GPU configurations, and the default multimodal example moved from `Llava-Mistral` to `Qwen/Qwen3-VL-2B-Instruct`. + + > **Migrate:** Switch LLaVA workloads to the aggregated path or to a Qwen3-VL recipe. + +Behavioral **Compressed Concurrent Tree Default** ([#7874](https://github.com/ai-dynamo/dynamo/pull/7874)): The KV router defaults to the compressed concurrent radix tree. Improves resource utilization for multi-threaded indexing; node-allocation semantics differ from the previous tree. + + > **Migrate:** Update any custom instrumentation that targeted the old radix-tree internals. No action required for default deployments. + +Behavioral **MDC Checksum Scoped Per-WorkerSet** ([#7368](https://github.com/ai-dynamo/dynamo/pull/7368)): Model Discovery Card checksum validation moved from per-Model to per-WorkerSet. Different WorkerSets under the same Model can now carry different configuration without forcing workers to drain first. + + > **Migrate:** If you relied on the v1.0.0 strict per-Model behavior, audit your WorkerSet configs before upgrading. + +Removed **vLLM Auto-Enable KV Events Removed** ([#7591](https://github.com/ai-dynamo/dynamo/pull/7591)): The deprecated automatic KV-events config in vLLM is removed; the `DYN_VLLM_KV_EVENT_PORT` env var is also no longer supported. + + > **Migrate:** Set `--kv-events-config` explicitly per the v1.0.0 migration note. + +Removed **Unused `genai-perf` Pin Dropped** ([#8763](https://github.com/ai-dynamo/dynamo/pull/8763)): The unused `genai-perf==0.0.15` pin was removed from `container/deps/requirements.benchmark.txt`. It was not invoked anywhere in the repo. + + > **Migrate:** No action required. `aiperf` is the supported in-container benchmarking tool. + +Deprecated The following warnings from v1.0.0 still apply. Migrate before they are removed: + +- `v1alpha1` DGDR API: migrate to `v1beta1` +- `enableGpuDiscovery` CRD field has no effect +- `ComponentName` field on `ServiceReplicaStatus`: migrate to `ComponentNames` +- Router CLI flags without the `--router-` prefix +- vLLM `--is-prefill-worker`/`--is-decode-worker`: migrate to `--disaggregation-mode` +- `--router-durable-kv-events`: migrate to the event-plane subscriber + + + + + + + +**ACTION REQUIRED:** The following changes require updates to your code, configuration, or deployment manifests before upgrading. + +Removed **KV Router Flags Renamed** ([#6361](https://github.com/ai-dynamo/dynamo/pull/6361)): All KV router CLI flags and env vars now use the `--router-`* / `DYN_ROUTER_`* prefix. + +| Old Flag / Env Var | New Flag / Env Var | +| :---- | :---- | +| `--kv-events` / `DYN_KV_EVENTS` | `--router-kv-events` / `DYN_ROUTER_USE_KV_EVENTS` | +| `--kv-overlap-score-weight` / `DYN_KV_OVERLAP_SCORE_WEIGHT` | `--router-kv-overlap-score-weight` / `DYN_ROUTER_KV_OVERLAP_SCORE_WEIGHT` | +| `--assume-kv-reuse` / `DYN_ASSUME_KV_REUSE` | `--router-assume-kv-reuse` / `DYN_ROUTER_ASSUME_KV_REUSE` | +| `--durable-kv-events` / `DYN_DURABLE_KV_EVENTS` | `--router-durable-kv-events` / `DYN_ROUTER_DURABLE_KV_EVENTS` | +| `--track-active-blocks` / `DYN_TRACK_ACTIVE_BLOCKS` | `--router-track-active-blocks` / `DYN_ROUTER_TRACK_ACTIVE_BLOCKS` | +| `--track-output-blocks` | `--router-track-output-blocks` | +| `--router-ttl` / `DYN_ROUTER_TTL` | `--router-ttl-secs` / `DYN_ROUTER_TTL_SECS` | + +**Migrate:** Update all CLI invocations, env vars, and deployment YAMLs to use the new names. + +Removed **Disagg Flag Inverted** ([#6515](https://github.com/ai-dynamo/dynamo/pull/6515)): `--enforce-disagg` replaced by `--decode-fallback` with inverted semantics — disaggregated mode is now enforced by default. + + **Migrate:** Replace `--enforce-disagg` with `--decode-fallback`. If you need fallback to aggregated mode, explicitly pass `--decode-fallback` or `DYN_DECODE_FALLBACK=true`. In the EPP plugin, update from `DYN_ENFORCE_DISAGG` to `DYN_DECODE_FALLBACK` with inverted boolean. + +Removed **Migration Limit Moved to Frontend** ([#5918](https://github.com/ai-dynamo/dynamo/pull/5918)): The `--migration-limit` CLI flag has been removed from all backend workers (vLLM, SGLang, TRT-LLM) and is now set on the Frontend only. + + **Migrate:** Remove `--migration-limit` from backend launch commands; pass it to the Frontend instead. + +Removed **Connector Flag Replaced** ([#6450](https://github.com/ai-dynamo/dynamo/pull/6450)): The `--connector` flag is removed. Disaggregated prefill workers now require explicit `--kv-transfer-config` with a JSON value. + + **Migrate:** Replace `--connector nixl` with `--kv-transfer-config '{"kv_connector":"NixlConnector","kv_role":"kv_both"}'`. Update all deployment YAMLs and launch scripts accordingly. + +Behavioral **KV Events Now Opt-In** ([#6404](https://github.com/ai-dynamo/dynamo/pull/6404)): KV cache events are no longer auto-created when prefix caching is enabled. Users must explicitly opt in via `--kv-events-config`. + + **Migrate:** Add `--kv-events-config '{"publisher":"zmq","endpoint":"tcp://*:20080","enable_kv_cache_events":true}'` to worker launch commands. Replace `DYN_VLLM_KV_EVENT_PORT` env var with the CLI flag. + +Behavioral **Local Indexer Now Default** ([#5941](https://github.com/ai-dynamo/dynamo/pull/5941), [#6073](https://github.com/ai-dynamo/dynamo/pull/6073)): Default event transport changed from JetStream to NATS Core/Event Plane with Local Indexer. The `--enable-local-indexer` flag is removed. + + **Migrate:** If you relied on JetStream persistence, add `--durable-kv-events` on both frontend and all workers. Remove any `--enable-local-indexer` flags. + +Removed **Omni Flags Prefixed** ([#6476](https://github.com/ai-dynamo/dynamo/pull/6476)): 14 diffusion/omni CLI flags renamed with `--omni-` prefix (e.g., `--enforce-eager` → `--omni-enforce-eager`). + + **Migrate:** Update CLI invocations to use the `--omni-` prefixed names. + +Removed **Multimodal Worker Flag Removed** ([#6060](https://github.com/ai-dynamo/dynamo/pull/6060)): `--multimodal-encode-prefill-worker` removed from vLLM backend. + + **Migrate:** Use `--multimodal-encode-worker`, `--multimodal-worker`, or `--multimodal-decode-worker` instead. + +Behavioral **Output Modalities Required** ([#6270](https://github.com/ai-dynamo/dynamo/pull/6270)): vLLM omni mode no longer auto-registers image endpoints; you must pass `--output-modalities image` explicitly. + +Removed **Media URL Flags Unified** ([#6391](https://github.com/ai-dynamo/dynamo/pull/6391)): SGLang/TRT-LLM flags `--image-diffusion-fs-url`, `--video-generation-fs-url`, and `--output-dir` replaced by `--media-fs-url` and `--media-base-url`. + +Removed **Discovery Backend Simplified** ([#6167](https://github.com/ai-dynamo/dynamo/pull/6167)): `DYN_DISCOVERY_BACKEND` now accepts `kubernetes`, `etcd`, `file`, `mem` directly. Remove `DYN_KV_STORE`; replace `--store-kv` with `--discovery-backend`. + +Removed **Planner CLI Replaced by Config File** ([#6356](https://github.com/ai-dynamo/dynamo/pull/6356)): All individual Planner CLI flags removed in favor of `--config ` pointing to a JSON/YAML configuration file. + +Removed **dynamo-run Removed** ([#6203](https://github.com/ai-dynamo/dynamo/pull/6203)): The `dynamo-run` CLI tool and all its flags have been removed. Migrate to the Python-based deployment approach. + +Removed **Env Var Renames** ([#6358](https://github.com/ai-dynamo/dynamo/pull/6358), [#5882](https://github.com/ai-dynamo/dynamo/pull/5882)): + +| Old | New | +| :---- | :---- | +| `DYNAMO_FATBIN_PATH` | `DYN_FATBIN_PATH` | +| `ENABLE_KVBM_RECORD` | `DYN_KVBM_ENABLE_RECORD` | +| `SPLIT_ENCODE` | `DYN_SPLIT_ENCODE` | +| `DYNAMO_BUSY_THRESHOLD` | `DYN_BUSY_THRESHOLD` | +| `DYNAMO_`* (EPP vars) | `DYN_`* | + +Removed **KVStats Metrics Removed** ([#5704](https://github.com/ai-dynamo/dynamo/pull/5704)): `dynamo_component_kvstats_`* metrics removed. Use `dynamo_frontend_inter_token_latency_seconds` for Decode autoscaling instead of `kvstats_gpu_cache_usage_percent`. + +Removed **Router Metrics Namespace** ([#6227](https://github.com/ai-dynamo/dynamo/pull/6227)): `dynamo_frontend_worker_active_`* → `dynamo_router_worker_active_`*, `dynamo_component_router_*` → `dynamo_router_*`. New `router_id` label added to all Router metrics. + +Behavioral **Frontend Request Counter Label** ([#5568](https://github.com/ai-dynamo/dynamo/pull/5568)): `dynamo_frontend_requests_total` now includes an `error_type` label. Update PromQL queries to account for the new label. + +Removed **SGLang Metric Prefix** ([#5701](https://github.com/ai-dynamo/dynamo/pull/5701)): SGLang metrics now use the native `sglang:` prefix (colon) instead of `sglang_` (underscore). + +Behavioral **etcd Subchart Disabled** ([#6329](https://github.com/ai-dynamo/dynamo/pull/6329)): Bundled etcd is now disabled by default. Set `global.etcd.install: true` if your deployment depends on it. + +Removed **Webhook Key Removed** ([#6441](https://github.com/ai-dynamo/dynamo/pull/6441)): `webhook.enabled` removed from Helm values. Remove it from custom values files. + +Removed **Helm Values Restructured for Snapshot** ([#5946](https://github.com/ai-dynamo/dynamo/pull/5946)): `storage.signalHostPath`, `daemonset.criu.`*, and `daemonset.containerRuntimeSocket` replaced by `config.checkpoint.`* and `config.agent.*` hierarchy. + +Behavioral **DGDR Planner Schema** ([#6463](https://github.com/ai-dynamo/dynamo/pull/6463)): `FeaturesSpec.planner` in DGDR CRD changed from a typed `PlannerSpec` to the PlannerConfig JSON schema. Review DGDR manifests that set `features.planner`. + +Removed **EPP Discovery Timeout** ([#5770](https://github.com/ai-dynamo/dynamo/pull/5770)): `DYN_DISCOVERY_TIMEOUT_SEC` no longer works. Use StartupProbe `failureThreshold` × `periodSeconds` instead. + +Removed **Component/Namespace/CancellationToken Removed** ([#6403](https://github.com/ai-dynamo/dynamo/pull/6403), [#6386](https://github.com/ai-dynamo/dynamo/pull/6386), [#6405](https://github.com/ai-dynamo/dynamo/pull/6405)): `Component`, `Namespace`, and `CancellationToken` classes removed from the Python API. + + **Migrate:** Replace `runtime.namespace('ns').component('comp').endpoint('ep')` with `runtime.endpoint('ns.comp.ep')`. Replace `token.cancel()` with `HttpService.shutdown()`. Pass `DistributedRuntime` directly to service `.run()` methods. + +Removed + +| Old | New | PR | +| :---- | :---- | :---- | +| `client2(router_mode)` | `client(router_mode=router_mode)` | [#6158](https://github.com/ai-dynamo/dynamo/pull/6158) | +| `register_llm` / `unregister_llm` / `fetch_llm` | `register_model` / `unregister_model` / `fetch_model` | [#6268](https://github.com/ai-dynamo/dynamo/pull/6268) | +| `ModelDeploymentCard` in `dynamo.runtime` | Moved to `dynamo._internal` | [#6378](https://github.com/ai-dynamo/dynamo/pull/6378) | +| `EncoderCacheManager` | `MultimodalEmbeddingCacheManager` in `dynamo.common.memory` | [#5962](https://github.com/ai-dynamo/dynamo/pull/5962) | +| `KvPushRouter` / `ZmqKvEventPublisherConfig` | `KvRouter`; pass `zmq_endpoint`/`zmq_topic` directly to `KvEventPublisher()` | [#6238](https://github.com/ai-dynamo/dynamo/pull/6238) | +| `ZmqKvEventPublisher` | `KvEventPublisher(component, zmq_config=config)` | [#6016](https://github.com/ai-dynamo/dynamo/pull/6016) | +| `DYNAMO_ARGS` from `dynamo.sglang.args` | `DynamoSGLangArgGroup` from `dynamo.sglang.backend_args` | [#6280](https://github.com/ai-dynamo/dynamo/pull/6280) | +| `Config` from `dynamo.trtllm.utils.trtllm_utils` / `create_worker(...)` | `Config` in `dynamo.trtllm.args` / `create_llm_worker(...)` | [#6297](https://github.com/ai-dynamo/dynamo/pull/6297) | + +Behavioral **Frontend Config Refactored** ([#6201](https://github.com/ai-dynamo/dynamo/pull/6201)): Frontend CLI now rejects unknown args unless `--chat-processor vllm` is set. + +Behavioral **ModelManager Checksum Enforcement** ([#6054](https://github.com/ai-dynamo/dynamo/pull/6054)): Mismatched MDC checksums across WorkerSets now raise `ChecksumMismatch` instead of being silently accepted. + +Behavioral **Tool Call Parser Separation** ([#5849](https://github.com/ai-dynamo/dynamo/pull/5849)): `--tool-call-parser` alone no longer uses Dynamo's parser. Use `--dyn-tool-call-parser` for Dynamo's pipeline. + +Behavioral **Custom Backend Metrics Removed** ([#5893](https://github.com/ai-dynamo/dynamo/pull/5893)): `custom_backend_metrics_endpoint` and `custom_backend_metrics_polling_interval` removed from `LocalModel` and frontend config. + +Removed **Deprecated Component Removals:** Removed dynamo-run and mistral-rs engine (#6203), standalone FastAPI Router (#5845), media-nixl feature (#5940), and llava-hf recipes (#6961). + +Behavioral **Local Indexers On By Default** ([#5941](https://github.com/ai-dynamo/dynamo/pull/5941)): KV event transport now defaults to NATS Core/Event Plane with Local Indexer instead of JetStream. Pass `--durable-kv-events` on both frontend and workers to restore JetStream behavior. + +Behavioral **GPU Memory Utilization** ([#5755](https://github.com/ai-dynamo/dynamo/pull/5755)): `gpu-memory-utilization` adjusted for vLLM runtime to improve out-of-the-box performance. + +Behavioral **Operator Env Vars Documented** ([#6548](https://github.com/ai-dynamo/dynamo/pull/6548)): All environment variables injected by the Operator are now documented. + +Deprecated **`dynamo-crds` Helm Chart:** The standalone `dynamo-crds` Helm chart is deprecated. CRDs are now embedded in the Dynamo Operator image and applied automatically via an init container on the operator Deployment ([#6466](https://github.com/ai-dynamo/dynamo/pull/6466), [#6780](https://github.com/ai-dynamo/dynamo/pull/6780)). Users should uninstall the `dynamo-crds` Helm release; the operator manages CRD lifecycle directly. + +The following features still work but will be removed in a future release with most targeted to Dynamo v1.1.0. + +Deprecated **`v1alpha1` DGDR API** ([#6352](https://github.com/ai-dynamo/dynamo/pull/6352)): The `v1alpha1` DynamoGraphDeploymentRequest API will be removed in a future release. Migrate to `v1beta1`; automatic conversion maintains backward compatibility during the transition. + +Deprecated **enableGpuDiscovery CRD Field** ([#6224](https://github.com/ai-dynamo/dynamo/pull/6224)): The `enableGpuDiscovery` CRD field no longer has any effect and will be removed in a future release. GPU discovery now runs automatically. + +Deprecated **ComponentName Field** ([#6110](https://github.com/ai-dynamo/dynamo/pull/6110)): The `ComponentName` field on `ServiceReplicaStatus` will be removed in a future release. Migrate to the new `ComponentNames` list field. + +Deprecated **Router Legacy Flag Names** ([#6346](https://github.com/ai-dynamo/dynamo/pull/6346)): Router CLI flags without the `--router-` prefix (e.g., `--block-size`, `--kv-events`) will be removed in a future release. Migrate to the prefixed versions (`--router-block-size`, `--router-kv-events`). + +Deprecated **vLLM KV Auto-Enable** ([#6404](https://github.com/ai-dynamo/dynamo/pull/6404)): vLLM's auto-enabling of KV events when prefix caching is active will be removed in a future release. Use `--kv-events-config` explicitly instead. + +Deprecated **Prefill/Decode Worker Flags** ([#6483](https://github.com/ai-dynamo/dynamo/pull/6483)): The `--is-prefill-worker` and `--is-decode-worker` boolean flags for the vLLM backend will be removed in a future release. Migrate to `--disaggregation-mode`. + +Deprecated **Durable KV Events** ([#6477](https://github.com/ai-dynamo/dynamo/pull/6477)): The `--router-durable-kv-events` CLI flag will be removed in a future release. Migrate to the event-plane subscriber (local_indexer mode). + + diff --git a/docs/fern/reference/known-issues.mdx b/docs/fern/reference/known-issues.mdx new file mode 100644 index 000000000000..afe3fae2cccd --- /dev/null +++ b/docs/fern/reference/known-issues.mdx @@ -0,0 +1,319 @@ +--- +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +title: Known Issues +subtitle: Known issues per Dynamo release, mirrored from the GitHub release notes +--- + +import { ReferenceStyles } from "@/components/ReferenceStyles"; +import { RELEASE_STATS } from "@/components/releases.data"; + + + +Known issues are mirrored verbatim from each release's GitHub release notes. The current release is listed first, with older releases collapsed below; artifact-specific issues sit at the bottom of the page. + + + +## v1.3.0 — current release + +vLLM **GMS + Expert Parallel Crash at Model Load:** When running the GPU Memory Service with Expert Parallel, `Tensor.item()` is called on a meta tensor in `ExpertMapManager` and the worker crashes at model load. The root cause is an upstream vLLM defect (fix in flight as vllm-project/vllm#43928, not yet in v0.23.0 or v0.24.0). **Workaround:** Avoid GMS with Expert Parallel (MoE) on vLLM until the upstream vLLM fix ships. + +vLLM **Mistral Multimodal Image Fetch Fails on vLLM v0.23.0:** Multimodal Mistral image requests fail because `MistralCommonImageProcessor` is missing the `fetch_images` method. This is an upstream vLLM v0.23.0 gap (fixed upstream in v0.24.0); the v0.24.0 pin bump was not taken for v1.3.0. **Workaround:** None in v1.3.0; resolved by a vLLM pin bump to `>=0.24.0` (already fixed on the development tip). + +vLLM **Disagg Decode + EAGLE3 Speculative Decoding Crashes Silently:** In a disaggregated deployment, EAGLE3 speculative decoding crashes silently under large KV-transfer volume (an upstream vLLM defect). The crash reproduces even with speculative decoding enabled on both the prefill and decode workers using an identical `--speculative_config`. **Workaround:** None reliable in v1.3.0 — EAGLE3 speculative decoding with disaggregated serving is unsupported. **Targeted fix:** v1.4.0. + +vLLM **KVBM PdConnector Disaggregated E2E Broken (NIXL VRAM_SEG):** All four disaggregated KVBM/LMCache configurations fail the NIXL `VRAM_SEG` handshake (aggregated paths pass). The KVBM PD connector subclasses vLLM's `MultiConnector`, which in vLLM v0.23.0 inherits `SupportsHMA` and wrongly assumes the PD connector is HMA-capable, tripping the handshake at `MultiConnector.__init__`. **Workaround:** Pass `--disable-hybrid-kv-cache-manager` on both prefill and decode workers. **Targeted fix:** v1.3.1. + +TRT-LLM **Qwen2-VL-7B Multimodal Inference Fails:** Qwen2-VL-7B multimodal inference fails with `AssertionError: Number of mm_embeds (2) does not match expected total (3897)`. This is an upstream TensorRT-LLM regression introduced across the rc14→rc19 bump (transformers 4.57.3→5.5.4); it affects Qwen2-VL-7B only — Qwen3-VL is unaffected. **Workaround:** Use Qwen3-VL. **Targeted fix:** v1.4.0, contingent on the upstream TensorRT-LLM fix. + +TRT-LLM **Crashed TensorRT-LLM Worker Keeps Receiving Traffic and Is Never Restarted:** After a TensorRT-LLM engine-core OOM or SIGKILL, `check_health()` still reports healthy, so `TrtllmEngineMonitor` never trips and the worker keeps receiving requests it can no longer serve while the pod is never restarted. Correct idle-gap fatal-state detection requires an upstream TensorRT-LLM change; this is pre-existing behavior, not a v1.3.0 regression. **Workaround:** Manually restart the affected worker. **Targeted fix:** pending upstream TensorRT-LLM. + +TRT-LLM **Wide-EP Decode Crashes on All Ranks:** Wide Expert-Parallel decode crashes on all WideEP ranks due to a `run_moe()` argument-count mismatch (reproduced since rc1/rc2). The root cause is an upstream TensorRT-LLM ABI skew (a 37-vs-44 argument mismatch present in the rc18–rc20 bases, fixed upstream in rc21); v1.3.0 stays on the rc19 base rather than taking the destabilizing rc21 bump. **Workaround:** Rebuild against a TensorRT-LLM rc21 base, where the upstream `run_moe()` fix is present — v1.3.0 ships the rc21-compatibility changes ([#11769](https://github.com/ai-dynamo/dynamo/pull/11769), [#11799](https://github.com/ai-dynamo/dynamo/pull/11799)) so the Dynamo TensorRT-LLM worker builds against rc21. **Targeted fix:** v1.3.1. + +SGLang **Disaggregated SGLang Inference over EFA Stalls and Returns Empty Response:** On AWS EFA clusters (GB200 and H100 / P5), the first disaggregated end-to-end SGLang request stalls for roughly 300 seconds and then returns HTTP 200 with empty content and zero completion tokens, because the KV cache is never transferred. The root cause is a NIXL `FI_MORE` multi-rail write-batching deadlock in the KV transfer over the EFA LIBFABRIC backend. **Workaround:** Avoid EFA-based disaggregated SGLang serving; use aggregated serving or a non-EFA transport instead. **Targeted fix:** v1.3.1. + +Planner **Planner SLA and Load-Based Autoscaling Does Not Scale Deployments:** SLA and load-based scaling is an advanced, opt-in path; default throughput-based scaling and core disaggregated serving are unaffected. On that path, decode-side KV-rate scaling never fires on TensorRT-LLM (workers do not report `total_kv_blocks` / `kv_cache_block_size`), disagg-prefill scale-up never triggers on SGLang (the prefill-token signal omits the chunked-prefill backlog), the MTP accept-length discount never reaches replica decisions (accept length pins at 1.0), and GlobalPlanner scale operations fail while reading the deprecated DGD `spec.services` field. **Workaround:** Use throughput-based Planner scaling. **Targeted fix:** v1.4.0. + +Planner **AIConfigurator-Backed Estimation Unavailable on v0.9.0:** AIConfigurator `v0.9.0` is missing the native forward-pass performance estimator and the KV-memory estimator (`sdk.memory`), so AIConfigurator-backed estimation is unavailable. The Planner may fall back to FPM regression, AIC-backed Mocker and router modes may fail to initialize, and `dynamo.mocker` crashes on startup with `--aic-perf-model`. The Profiler `thorough` sweep also skips its mocker-mode check and can spawn a real-GPU interpolation pod that fails before falling back. **Workaround:** Run the Mocker and router without AIC-backed modes and use FPM-based Planner scaling. **Targeted fix:** v1.4.0, pending the AIConfigurator `v0.10` upgrade. + + + +{/* +1: the v1.2.1 patch entry at the bottom of this accordion, on top of the v1.2.0 GA count. */} + + +vLLM **Qwen2.5-Omni Text-to-Text Garbage Output:** When running the Qwen2.5-Omni-7B model through the vLLM-Omni backend in a text-to-text (aggregated) configuration, the model produces garbage tokens instead of coherent text output. The vLLM-Omni backend is not natively designed for pure text generation with this model, and the text-to-text example path is being deprecated. **Workaround:** Use the pure vLLM backend for text-to-text generation; the vLLM-Omni text-to-text path produces garbage output and was deprecated for this reason. + +vLLM **Disagg Prefill Sleep Fails Post-Inference:** When using disaggregated prefill with the vLLM backend (observed with Qwen/Qwen3-0.6B, TP=1 on B200), attempting to sleep a prefill worker after inference fails with a "pause_scheduler ... not supported yet" error due to stale KV-transfer state in vLLM's sleep/wake lifecycle. Because the sleep handler unregisters the worker from discovery before calling pause_scheduler, and does not roll back on failure, the prefill worker is permanently dropped from the routing pool until the pod is restarted. This is an upstream vLLM limitation related to sleep/wake KV-transfer state cleanup, not a Dynamo defect. **Workaround:** Restart the affected prefill-worker pod to return it to the routing pool; the worker does not auto-recover after a failed sleep. + +SGLang **Disagg Serving Fails on ARM64:** Disaggregated serving fails on ARM64 platforms (GB200 with EFA) across all backends (vLLM, TensorRT-LLM, and SGLang). The NIXL libfabric backend's `register_memory()` call fails when attempting to register GPU memory for RDMA via the EFA driver. The root cause is an upstream NIXL issue: the bundled libfabric version (2.4.0amzn1.0) lacks the CUDA dmabuf fix required for ARM64 64K-page kernels. The same operation succeeds on x86_64 (H100 + EFA). The fix is available in upstream NIXL v1.1.0 but requires a major version bump with breaking changes that could not be incorporated before the 1.2.0 release. **Workaround:** Run disaggregated serving on an x86_64 platform (e.g., H100 + EFA), where the same NIXL `register_memory` path succeeds; arm64 (GB200) is blocked until upstream NIXL v1.1.0 is integrated. + +SGLang **Multi-Node Streaming Completions Hang:** In a multi-node aggregated deployment with TP=8 using the SGLang backend, both streaming and unary chat completion requests may hang intermittently (approximately 1 in 3 runs) after prefill. All tensor-parallel ranks become stuck at a `torch.cuda.streams.synchronize()` call due to a CUDA-event deadlock in the upstream SGLang overlap scheduler (see https://github.com/sgl-project/sglang/issues/26454). **Workaround:** Launch the SGLang worker with `--disable-overlap-schedule`. This avoids the deadlock but may reduce scheduling throughput. + +SGLang **DeepSeek V4 Unsupported on SGLang:** The SGLang backend shipped in v1.2.0 (SGLang 0.5.11) does not support DeepSeek V4 models; in this release DeepSeek V4 is generally available only on the vLLM backend (vLLM 0.20.1). **Workaround:** Run DeepSeek V4 on the vLLM backend, or use the separate experimental SGLang images at https://github.com/ai-dynamo/dynamo/releases/tag/v1.2.0-deepseek-v4-dev.3. SGLang DeepSeek V4 support is planned for a future release. + +TRT-LLM **Multimodal Disagg Decode Tensor Failure:** When running multimodal disaggregated serving (prefill/decode) with the TensorRT-LLM backend on Kubernetes, the decode worker crashes because it cannot access shared-tensor handles (e.g. multimodal embedding and MRoPE position tensors stored in `/dev/shm`) that were created locally by the prefill worker in a separate pod. Text-only disaggregated serving is unaffected because KV transfer uses NIXL/EFA, but multimodal embedding and MRoPE tensors are passed via local shared-memory references that are not visible across pods. This is a TensorRT-LLM backend limitation where the disaggregated path assumes local-process visibility for multimodal tensor artifacts. **Workaround:** No workaround is available yet. + +Multimodal **Wan2.2 Video Compiled-Path Crash:** Wan2.2 video generation (Wan-AI/Wan2.1-T2V-1.3B-Diffusers) runs in eager mode only. On the `torch.compile` path the vLLM-Omni diffusion worker crashes with a CUDA illegal memory access during the transformer cross-attention forward pass, because `WanSelfAttention.forward` mutates internal module state in a way that is incompatible with `torch.compile`. The root cause is upstream. **Workaround:** The v1.2.0 video launchers set `--enforce-eager` by default to avoid the crash; keep that flag set. The compiled path remains broken pending an upstream vLLM-Omni fix. + +KVBM **TRT-LLM KVBM RequestData AttributeError:** All TensorRT-LLM × KV Block Manager container tests fail with `AttributeError: 'RequestData' object has no attribute 'block_hashes'` at `kvbm_connector_leader.py:138`. The failure is 100% reproducible across both H100 and GB200 (aarch64) hardware and affects every KVBM test configuration (aggregated, disaggregated, offload filtering). The root cause is an API attribute drift in the TensorRT-LLM `RequestData` class between upstream release candidates; the Dynamo KVBM connector expects a `block_hashes` attribute that was removed or renamed in the bundled TRT-LLM version. A fix is expected in a future release. **Workaround:** No workaround is available yet. + +Planner **Scale-Up Gang Roll Strands Planner:** When the Planner issues a latency-mode prefill or decode scale-up in a disaggregated topology (optimization_target=sla, enable_throughput_scaling=True), the resulting DGD patch triggers a full Grove pod-gang roll. All pods are replaced, breaking active connections (observed ~86% request error rate during the roll) and resetting the Planner's in-memory state. After the roll, new workers report zero traffic (wall_time=0.0), the regression model cannot fit, and the throughput scaler enters a permanent model_not_ready hold, leaving the Planner stranded at its post-roll worker count with no further scaling decisions possible. The root cause is non-deterministic PodCliqueSet hash ordering in Grove, which interprets a metadata-only patch as a full topology change. A fix (PodCliqueSet hash stabilization) has been merged upstream in Grove but is not yet included in a tagged Grove release consumed by Dynamo. **Workaround:** No workaround is available yet. + +Planner **SLA Load Scaler Stuck:** Under certain workloads the Planner's SLA-based load scaler remains stuck in an `insufficient_data` state because the decode regression model lacks noise tolerance for non-relaxable coefficients. This is a follow-up to a partially addressed issue where only one of two deadlock paths was resolved; the remaining path (decode regression noise tolerance) can prevent the scaler from advancing past its initial state, leaving autoscaling decisions stalled. Root cause is identified in the Planner regression-model code but the fix is deferred to a future release. **Workaround:** No workaround is available yet. + +Recipes **GPT-OSS 120B GB200 Crash:** When deploying the `gpt-oss-120b-agg` recipe on GB200 systems, TRT-LLM automatically selects the DeepEP communication path, which triggers a `cudaErrorIllegalAddress` in the `intranode_combine` kernel (FmhaAutoTuner). The root cause is an upstream TRT-LLM defect where newer releases (rc11/rc12) incorrectly auto-select DeepEP instead of the working AllGather strategy. **Workaround:** Set `TRTLLM_FORCE_COMM_METHOD=ALLGATHER` in the deployment configuration to bypass the broken DeepEP path on GB200. + +v1.2.1 **SGLang ModelExpress peer-to-peer:** SGLang ModelExpress peer-to-peer transfer is not included in this release; the bundled SGLang 0.5.11 runtime does not carry the required upstream support. + + + + + + + +### NIXL + +#### TRT-LLM and vLLM on NIXL `v0.10.1` Missing EFA Fixes + +TRT-LLM and vLLM ship NIXL `0.10.1`, missing two Libfabric/EFA fixes in NIXL `1.0.1`: an endpoint-thread-safety mutex ([nixl#1457](https://github.com/ai-dynamo/nixl/pull/1457)), without which concurrent I/O can crash the worker, and a notification-override fix ([nixl#1433](https://github.com/ai-dynamo/nixl/pull/1433)), without which reposted transfers can corrupt the KV cache. Dynamo v1.1.1 will upgrade TRT-LLM and vLLM to NIXL `v1.0.1`. + +> **Workaround:** None for v1.1.0. Fix lands in v1.1.1 with the NIXL `v1.0.1` bump. + +### Planner + +#### SLA Mode Throughput-Tick Ignores Decode-Side ITL Violations + +Under `optimization_target=sla` with default scaling settings, the 180 s throughput-tick decouples scaling decisions from the latency signal. Prefill never scales above 1 replica because `demand_rps` is capped under HTTP backpressure; decode never fires its ITL-violation check because `find_best_engine_decode_rps` clamps the returned ITL to ≤ target by construction. In QA, decode hit a 16% client `ReadTimeout` rate at the 50 ms ITL target while the planner's reported `est_itl` stayed flat. Prefill-side fix shipped in RC10 (#8861, cherry-pick #8956); decode-side fix is deferred. + +> **Workaround:** Enable `enable_load_scaling=True` (the README's "advanced" example) so the reactive 5 s load-tick triggers prefill scale-up based on `queued_prefill_tokens`. For decode, pre-provision sufficient `VllmDecodeWorker.replicas` for peak load until the fix lands. + +#### Latency-Mode Prefill Scale-Up Triggers Grove Gang Roll + +On `environment=kubernetes` disagg DGDs with `optimization_target=latency`, planner scale-up patches trigger a Grove full-gang roll that kills and recreates every service in the DGD. The post-roll planner observes one prefill engine and short-circuits its scale-down gate, bricking the loop for the lifetime of the DGD. Throughput mode is unaffected. Tracked as Grove #566 — Grove-side fix. + +> **Workaround:** Use `optimization_target=throughput` or pre-provision prefill replicas at the expected peak count. + +#### GlobalPlanner Endpoint Health Race + +GlobalPlanner can fail to register endpoint health on first reconcile when multiple pool workers come up concurrently. The connector now waits for pool workers ([#8702](https://github.com/ai-dynamo/dynamo/pull/8702), cherry-pick of [#8694](https://github.com/ai-dynamo/dynamo/pull/8694)) and awaits endpoints concurrently ([#8692](https://github.com/ai-dynamo/dynamo/pull/8692), cherry-pick of [#8682](https://github.com/ai-dynamo/dynamo/pull/8682)), but residual races can still surface as a `not ready` status until the next reconcile. + +> **Workaround:** Wait for the next reconcile cycle (typically 30 s) or trigger a manual reconcile by patching the DGDR. + +#### Multi-DGD GlobalPlanner Scaling Edge Case + +Under multi-DGD configurations with GlobalPlanner, readiness gates can transiently report not-ready when one DGD is mid-rollout while a sibling DGD is being scaled. The primary fix landed in [#8514](https://github.com/ai-dynamo/dynamo/pull/8514) (cherry-pick of [#8482](https://github.com/ai-dynamo/dynamo/pull/8482)) but a residual edge case remains. + +> **Workaround:** Stagger DGDR rollouts when scaling sibling deployments under a single GlobalPlanner. + +### TensorRT-LLM + +#### Dynamic Default `max_tokens` Not Effective — Responses Capped at 32 Tokens + +The dynamic-default `max_tokens` path for the TensorRT-LLM backend ([#5152](https://github.com/ai-dynamo/dynamo/pull/5152)) is not effective in the released image. When the client omits `max_tokens` from the OpenAI request, the handler should compute `dynamic_default = max(1, max_seq_len - input_length)`, but in practice requests still produce exactly 32 completion tokens with `finish_reason="length"` — identical to the pre-PR behavior. + +> **Workaround:** Pass `max_seq_length` in the worker config or set `max_tokens` explicitly per request. + +#### Scheduler Deadlock with KV Cache Reuse + Chunked Prefill ([TRT-LLM #13318](https://github.com/NVIDIA/TensorRT-LLM/issues/13318)) + +Dynamo + TRT-LLM servers hit a scheduler deadlock and crash roughly every 30 minutes when KV cache reuse is combined with chunked prefill. Root cause is a token-accounting bug in TRT-LLM's `microBatchScheduler` that hangs the event loop. v1.1.0 ships TRT-LLM 1.3.0rc11; the fix ([TRT-LLM #12976](https://github.com/NVIDIA/TensorRT-LLM/pull/12976)) is merged upstream and lands in rc13. A v1.1.x bump to rc13 is under consideration. + +> **Workaround:** Disable KV cache reuse or chunked prefill until the rc13 bump lands. + +#### Qwen3‑235B‑A22B‑FP8 CuTe Experimental NotImplementedError on Blackwell *(carryover from v1.0.0)* + +Deploying the `qwen3-235b-a22b-fp8` recipes (both agg and disagg) on GB200/Blackwell still fails at runtime with `NotImplementedError: CuTe Experimental module is only supported on Cuda toolkit 13.1 and above!`. Caused by a packaging mismatch in the container image: the bundled `nvidia-cutlass-dsl==4.3.4` wheel is the pre-CUDA-13.1 variant that stubs out `cutlass.cute.experimental`, while the image itself ships CUDA 13.1 and TRT‑LLM's Blackwell FP8 GEMM path requires `cute.experimental` to be functional. + +> **Workaround:** Use the GB200-supported recipe variants until the container packaging is realigned. + +#### TRT-LLM Disaggregated Multimodal Path Edge Cases + +The TRT-LLM disaggregated embeddings/prefill/decode path has had multiple fixes land in v1.1.0 (#6726, #6810, #6840, #6920, #6924) but specific model+config combinations can still raise input-shape errors because the LLM API does not accept the token IDs + multimodal embeddings tuple for all model classes. + +> **Workaround:** Use aggregated mode for affected TRT-LLM multimodal workloads. + +#### `tensorrt_llm` ModuleNotFoundError on Bare Imports *(carryover from v0.5.0)* + +Importing TRT-LLM-backed Python entry points outside the documented worker launch path can raise `ModuleNotFoundError: No module named 'tensorrt_llm'`. Long-standing carryover from v0.5.0; does not affect supported worker launch scripts, which set the import path correctly before invoking `dynamo.trtllm`. + +> **Workaround:** Use the documented TRT-LLM worker launch scripts; do not import `dynamo.trtllm` directly from a bare Python REPL inside the container. + +### vLLM Omni / Diffusion + +#### `disagg_omni_glm_image` Crashes on Startup + +The vLLM disaggregated GLM-Image flow fails at Stage 0 with `Value error, The checkpoint you are trying to load has model type 'glm_image' but Transformers does not recognize this architecture` because the runtime image bundles a `transformers` version that predates `glm_image` support. Stage 0 dies before `/v1/models` becomes healthy and the diffusion stage later fails handshake (exit code 143). + +> **Workaround:** Install `transformers>=5.0` inside the container before launching the GLM-Image flow. A docs note has been added; a packaging fix is planned for the next image refresh. + +#### `vllm_omni` Missing from ARM64 Image — All Omni Tests Fail on GB200 + +The `vllm-runtime` ARM64 image does not include the `vllm_omni` module, so all Omni test cases fail on GB200 / arm64. Deferred from the v1.1.0 fix scope per QA. + +> **Workaround:** Run vLLM Omni workloads on x86_64 hardware, or install `vllm_omni` manually inside the ARM64 container. + +### GMS + +#### Disagg Prefill Worker Sleep Fails After Inference + +In a disaggregated vLLM deployment with GMS and `--enable-sleep-mode`, `/engine/sleep` on the prefill worker fails after any inference request. The `NixlConnector` leaves KV transfer state that prevents `pause_scheduler` from resetting the cache, and the worker becomes unresponsive (subsequent requests fail with `no instances found`). Decode worker sleep/wake is unaffected. + +> **Workaround:** Do not call `/engine/sleep` on prefill workers in disaggregated GMS deployments. Use decode-side sleep only. + +#### GMS Failover Not Supported for Disaggregated Serving + +The operator-managed GMS failover API ([#8157](https://github.com/ai-dynamo/dynamo/pull/8157)) currently targets aggregated single-engine active/standby pairs. Failover is not yet supported for disaggregated serving topologies, and also fails when DRA-managed and non-DRA workloads co-exist on the same GPU. + +> **Workaround:** Use GMS failover only with aggregated deployments where all workloads on a shared GPU use a consistent device-allocation mode. + +### KVBM + +#### CPU/Disk to GPU Reload Never Triggers for Preempted Requests + +The KVBM onboard path (CPU/Disk → GPU reload) does not trigger for preempted requests. Root-caused to an upstream vLLM scheduler bug, not a Dynamo regression. + +> **Workaround:** None for v1.1.0; tracking the upstream vLLM fix. + +#### NUMA-Aware Pinned Memory Fails on Multi-Socket Systems + +KVBM's NUMA worker pool relies on first-touch placement via `cudaHostAlloc()`, but CUDA does not honor per-thread CPU affinity for memory placement. On GB200 NVL4, all pinned memory lands on NUMA node 0, so GPUs 2 and 3 see suboptimal access latency. Does not reproduce on H100. Fix is to call `mbind()` after `cudaHostAlloc()`. + +> **Workaround:** Wrap the worker process with `numactl --membind=N --cpunodebind=N` to set process-level NUMA policy, which CUDA does honor. + +#### PyO3 Panic in `slot.rs` on TRT-LLM + +A PyO3 panic fires in `slot.rs` (`next_position > device_blocks`) under KVBM with the TRT-LLM backend. Deferred from v1.1.0 fix scope. + +> **Workaround:** None for v1.1.0; affected configurations should disable KVBM with `DYN_KVBM_ENABLE=0`. + +#### NCCL Window-Buffer Init Fails on GB200 with CUDA 13 + +KVBM MLA on TRT-LLM fails NCCL window-buffer initialization on GB200 with CUDA 13. Failure is inside the TRT-LLM library, not Dynamo source. Deferred from v1.1.0 fix scope. + +> **Workaround:** Use a non-MLA configuration or run on CUDA 12 hardware until the TRT-LLM-side fix lands. + +#### KVBM Performance vs. Disabled Baseline *(carryover from v1.0.0)* + +KVBM end-to-end performance with offload enabled can still trail the KVBM-disabled baseline on certain workloads. Active investigation across SGLang, TRT-LLM, and vLLM backends; v1.1.0 ships scheduler-aware offload sizing, KV-events on TRT-LLM, and event throughput improvements, but KVBM is not yet a default-on feature. + +> **Workaround:** Benchmark KVBM on/off for your workload before enabling in production. Default disabled. + +### Snapshot + +#### Snapshot + GMS Combination Disabled in Admission + +The Operator admission webhook explicitly rejects DGDs that combine ModelExpress snapshots with GMS storage. Enforced in [#8688](https://github.com/ai-dynamo/dynamo/pull/8688) (cherry-pick of [#8675](https://github.com/ai-dynamo/dynamo/pull/8675)). This is intentional pending design alignment between the snapshot lifecycle and the GMS attachment model. + +> **Workaround:** Use one or the other for v1.1.0; combined support is targeted for a follow-up release. + + + + + + + +### DynamoGraphDeploymentRequest (Preview in v1.0.0) + +#### Planner With Empty Defaults Fails on Non-AIC-Supported Model/Hardware + +Applying a DGDR with `features.planner: {}` (empty defaults) on a model/hardware combination not supported by AIConfigurator causes the profiling job to fail with `ValueError: Throughput-based planner scaling requires AIC support`. The default planner config assumes throughput scaling with rapid in-depth sweeping, which requires AIC support. The Dynamo profiler validation raises a hard error before AIC is called, even though AIC PR#516 added the backend-side fix. + +> **Workaround:** Set `features.planner: {pre_deployment_sweeping_mode: thorough}` to bypass the AIC support gate check. + +#### Profiler Rejects Valid SLA Combination + +Specifying both `optimizationType` and `ttft`/`itl` SLA targets on a `DynamoGraphDeploymentRequest` triggers a Pydantic validation error because the schema treats them as mutually exclusive. The `optimizationType` field is not yet implemented in Dynamo 1.0.0, and any CRDs or manifests that reference it will fail validation. Users who upgrade from earlier versions with existing DGDR specs that include `optimizationType` alongside latency targets will see immediate admission errors. + +> **Workaround:** Remove the `optimizationType` field from SLA specifications. Use only `e2eLatency` or the `ttft`/`itl` pair (which must be specified together) — these two modes are mutually exclusive. + +#### Interpolation Does Not Propagate Tolerations + +Tolerations defined in `overrides.dgd` on a `DynamoGraphDeploymentRequest` are not propagated to candidate `DynamoGraphDeployments` created during the interpolation phase of profiling. This causes worker pods to remain in `Pending` state on clusters with tainted nodes, because the generated deployments lack the required tolerations to schedule onto those nodes. PR [#7226](https://github.com/ai-dynamo/dynamo/pull/7226) moved override application to before the interpolation step, but the fix is incomplete for all override paths and has been reopened. A complete fix is pending for a patch release. + +> **Workaround:** Manually add the required tolerations directly to each generated `DynamoGraphDeployment` after interpolation completes, or remove taints from target nodes during profiling. + +#### Thorough Profiler Generates Infeasible TP=1 for MoE Models + +The profiler's memory estimation does not account for WideEP communication buffers used by Mixture-of-Experts models, causing it to generate TP=1 configurations that are guaranteed to OOM at runtime. When the thorough profiler enumerates candidate configurations, it underestimates peak memory for MoE architectures, and the resulting deployment crashes immediately upon loading the model. + +> **Workaround:** Manually reduce `kv_cache_ratio` to approximately 0.75 in the profiler configuration to reserve headroom for WideEP buffers, or exclude TP=1 from the candidate search space by setting a minimum tensor parallelism degree. + +#### Infeasible SLA Targets Silently Accepted + +When a user specifies SLA targets (TTFT, ITL, or E2E latency) that cannot be met by any profiled configuration, the profiler logs a warning but does not surface it as a Kubernetes condition on the `DynamoGraphDeploymentRequest` status. Operators monitoring the DGDR via `kubectl` or cluster dashboards will see no indication that the requested SLAs are unachievable, leading to deployments that run but never meet their performance objectives. This issue has been moved to the backlog and will not be fixed in 1.0.0. + +> **Workaround:** After profiling completes, manually inspect profiler pod logs for warnings containing "infeasible" or "no valid configuration" to verify that the requested SLA targets are achievable. + +### Multimodal + +#### TRT-LLM Disaggregated Multimodal Raises AttributeError + +Running the disaggregated embeddings/prefill/decode pipeline (`diagg_e_pd.sh`) with TRT-LLM on multimodal models raises `AttributeError: 'NoneType' object has no attribute 'keys'` during input preprocessing. The root cause is that TRT-LLM does not support the token IDs and multimodal embeddings path in its LLM API; the preprocessor must fall back to passing a text prompt via `default_multimodal_input_loader` for the embeddings case. A fix was merged ([#6840](https://github.com/ai-dynamo/dynamo/pull/6840)) and cherry-picked as [#6920](https://github.com/ai-dynamo/dynamo/pull/6920) in RC6, but the fix regressed and the issue persists in the v1.0.0 release. + +> **Workaround:** Use aggregated mode instead of disaggregated embeddings/prefill/decode for TRT-LLM multimodal workloads. A corrected fix is planned for a follow-up patch release. + +#### Wan2.1 Video Diffusion Requires Manual imageio Install + +Deploying `Wan-AI/Wan2.1-T2V-1.3B-Diffusers` for text-to-video generation fails with `ModuleNotFoundError: No module named 'imageio'`. The `imageio` package is intentionally excluded from the TRT-LLM runtime container to reduce image size, as video generation is an experimental feature. This is documented in `docs/backends/trtllm/trtllm-video-diffusion.md`. + +> **Workaround:** Install the package manually inside the container: `pip install imageio imageio-ffmpeg`. + +#### Embeddings Cache with TensorRT-LLM and enable_block_reuse + +Deploying a TensorRT-LLM multimodal workflow with Embeddings Cache and `enable_block_reuse: true` is not supported due to limitations in the backend. This will be supported in upcoming releases. + +> **Workaround:** Use Embeddings Cache with `enable_block_reuse: false`. All existing recipes, benchmarks, and guides already reflect this configuration. + +### Dynamo Snapshot + +#### Snapshot Restore Fails on AKS for vLLM + +Snapshot restore of vLLM workers on AKS does not fully reinitialize model state. A single restored worker appears healthy and passes readiness checks but returns empty responses with no generated tokens. Restoring multiple workers simultaneously can hang, causing inference requests to time out. This issue has only been observed on AKS. + +> **Workaround:** No workaround available. Fix planned for a follow-up patch release. + +### KVBM + +#### Pinned Memory Allocation Failure on Blackwell GPUs + +KVBM initialization may fail on Blackwell GPUs (GB200, B100, B200) with `CUDA_ERROR_INVALID_VALUE` when allocating pinned host memory. The root cause is that the `PinnedAllocator` was hardcoded to `device_id` 0 instead of using the actual device ID, which causes NUMA binding to select the wrong memory node. A partial fix ([#6809](https://github.com/ai-dynamo/dynamo/pull/6809)) corrects the device ID in the allocator, but some Blackwell configurations may still encounter initialization failures depending on the NUMA topology. + +> **Workaround:** Ensure `CUDA_VISIBLE_DEVICES` is set to expose only the intended GPUs, and verify that the NUMA node assignment matches the GPU topology. + +#### Performance Degradation When KVBM Is Enabled + +Enabling KVBM may degrade inference performance compared to running without it — observed in vLLM disaggregated mode and TensorRT-LLM aggregated mode. KVBM is now enabled by default (#5602), so users may see lower throughput out of the box. The overhead comes from KV cache block management and transfer coordination, which adds latency to each request even when KV cache reuse rates are low. + +> **Workaround:** Disable KVBM by unsetting `DYN_KVBM_ENABLE` if KV cache sharing is not needed for your workload. + +### SGLang + +#### HiCache NIXL Storage Backend Crash on Init + +SGLang HiCache with `--hicache-storage-backend nixl` crashes during scheduler initialization with `TypeError: expected str, bytes or os.PathLike object, not MHATokenToKVPoolHost`. The `HiCacheNixl` backend passes the memory pool host object where a file path string is expected. This is an upstream SGLang bug, fixed in [sgl-project/sglang#19517](https://github.com/sgl-project/sglang/pull/19517) but not yet included in the SGLang version pinned by Dynamo 1.0.0. + +> **Workaround:** Use a different HiCache storage backend (e.g., `disk`). HiCache works correctly with non-NIXL backends. + +#### SGLang DSR1 Recipe Model Loading from PVC Failure + +Deploying the SGLang DSR1 recipe or using it as a base config in the SLA profiler may fail because the model-download script downloads the model into a non-standard HuggingFace directory that ModelExpress cannot load, causing prefill and decode workers to enter CrashLoopBackOff. + +> **Workaround:** (1) Download the HF model into a standard HF directory and set `HF_HOME` to the PVC-mounted path, (2) update `--model-path` to point at the directory containing the downloaded HF cache (not supported for SLA profiler), or (3) provide `HF_TOKEN` so the model can be downloaded directly. + +### TensorRT-LLM + +#### Qwen3‑235B‑A22B‑FP8 fails with CuTe Experimental NotImplementedError on Blackwell + +Deploying the `qwen3-235b-a22b-fp8` recipes (both agg and disagg) on GB200/Blackwell fails at runtime with: `NotImplementedError: CuTe Experimental module is only supported on Cuda toolkit 13.1 and above!` + +> **Workaround:** This is caused by a packaging mismatch in the container image: the `nvidia-cutlass-dsl==4.3.4` wheel baked into the image is the CUDA \< 13.1 variant that stubs out `cutlass.cute.experimental` by unconditionally raising `NotImplementedError`, while the image itself ships CUDA 13.1 and TensorRT‑LLM’s Blackwell FP8 GEMM path (`cute_dsl_fp8_gemm_blackwell`) requires `cute.experimental` to be functional + + + +## Known Artifact Issues + +| Version | Artifact | Issue | Status | +| :---- | :---- | :---- | :---- | +| v0.9.0 | dynamo-platform-0.9.0 | Helm chart sets operator image to 0.7.1 instead of 0.9.0. | Fixed in v0.9.0.post1 | +| v0.8.1 | vllm-runtime:0.8.1-cuda13 | Container fails to launch. | Known issue | +| v0.8.1 | sglang-runtime:0.8.1-cuda13, vllm-runtime:0.8.1-cuda13 | Multimodality not expected to work on ARM64. Works on AMD64. | Known limitation | +| v0.8.0 | sglang-runtime:0.8.0-cuda13 | CuDNN installation issue caused PyTorch v2.9.1 compatibility problems with nn.Conv3d — performance degradation and excessive memory usage in multimodal workloads. | Fixed in v0.8.1 (#5461) | diff --git a/docs/fern/reference/model-early-access-builds.mdx b/docs/fern/reference/model-early-access-builds.mdx new file mode 100644 index 000000000000..931c5ed9aa1f --- /dev/null +++ b/docs/fern/reference/model-early-access-builds.mdx @@ -0,0 +1,55 @@ +--- +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +title: Model Early Access Builds +subtitle: Per-model early access container builds shipped ahead of stable releases +--- + +import { ReferenceStyles } from "@/components/ReferenceStyles"; +import { ModelEABuildCards } from "@/components/ModelEABuildCards"; + + + + +**Model early access builds do not go through QA validation.** They are experimental builds intended for early testing of a specific model. They may contain bugs, require pinned runtime flags, and receive no patch support. Use a stable release container for production workloads unless the build's GA path says the recipe is promoted. + + +A **model early access build** packages a single model's recipe on one runtime container, tagged `X.Y.Z--dev.N` and cut from a side branch ahead of that model's launch — independently of the stable release cadence. When the model's backend patches land upstream in the versions a stable release ships, the recipe is **promoted** to the plain `:X.Y.Z` release container and the early access image is no longer needed. + +Full-platform early access builds (`vX.Y.Z-dev.N`, covering all runtimes, wheels, crates, and Helm charts) are platform previews and are documented under [Early Access Artifacts](release-artifacts.mdx#early-access-artifacts) instead. + +**GA path legend:** + +- **Promoted** — the recipe runs on the stock `:X.Y.Z` stable release container; the early access image is superseded. +- **Dev-only** — the model still requires this early access image (its patches are not yet in a stable release). +- **Recipe in GA** — the model ships as a recipe on the standard release container. + +Every card's tag is click-to-copy as a full `docker pull` command. The coverage dots show what each build actually shipped — model builds publish container images only (no wheels, Helm charts, or crates). + + + + +See [Compatibility](compatibility.mdx) for hardware, platform, and backend feature support, and [Release Artifacts](release-artifacts.mdx) for the stable release inventory. + + +{/* llms-tables:begin — generated by scripts/gen_llms_tables.py, do not edit */} + + +**Model early-access builds** + +| Model | Tag | Release line | Runtimes | Shipped | GA path | Status | Coverage (images / wheels / helm / crates) | +| --- | --- | --- | --- | --- | --- | --- | --- | +| Inkling | `1.4.0-inkling-dev.1` | v1.4.0 | sglang-runtime | Jul 17, 2026 | Dev-only · v1.4.0 line | First build on the v1.4.0 line; targets the next stable release. | yes / no / no / no | +| GLM-5.2 | `1.3.0-glm-5.2-dev.1` | v1.3.0 | sglang-runtime | Jul 20, 2026 | Dev-only | Container carries SGLang cherry-picks (stability, config parsing, model support) opened upstream but not yet in a released SGLang. | yes / no / no / no | +| MiniMax-M3 | `1.3.0-minimax-m3-dev.1` | v1.3.0 | vllm-runtime, sglang-runtime, tensorrtllm-runtime | Jun 12, 2026 | Promoted → :1.3.0 | Dynamo changes and the M2 tool-calling fix are in release/1.3.0; the recipes run on the stock :1.3.0 containers. | yes / no / no / no | +| DeepSeek-V4 | `1.3.0-deepseek-v4-dev.1` | v1.3.0 | tensorrtllm-runtime | Jun 6, 2026 | Recipe in v1.3.0 | DeepSeek-V4 Flash and Pro recipes ship in v1.3.0 on the standard TensorRT-LLM release container. | yes / no / no / no | +| Nemotron-3-Ultra | `1.3.0-nemotron-ultra-dev.1` | v1.3.0 | vllm-runtime | Jun 5, 2026 | Dev-only | Four un-upstreamed vLLM patches; requires pinned flags VLLM_DISABLED_KERNELS=FlashInferFP8ScaledMMLinearKernel and --no-enable-flashinfer-autotune. | yes / no / no / no | +| Nemotron-3-Super | `1.3.0-nemotron-super-dev.1` | v1.3.0 | vllm-runtime | Jun 4, 2026 | Promoted → :1.3.0 | Both container patches are in the vLLM v0.23.0 that v1.3.0 ships; the recipe runs on the stock vllm-runtime:1.3.0. | yes / no / no / no | +| Kimi-K2.6 | `1.3.0-kimi-k2.6-dev.1` | v1.3.0 | vllm-runtime | Jun 4, 2026 | Promoted → :1.3.0 | The build's only container patch is in vLLM v0.23.0; the recipes run on the stock vllm-runtime:1.3.0. | yes / no / no / no | +| Cosmos-3 | `1.3.0-cosmos3-dev.1` | v1.3.0 | vllm-runtime | Jun 1, 2026 | Dev-only | Dynamo #10132 (Cosmos3 support in the vLLM-Omni backend) is open, not merged — v1.3.0 containers cannot run Cosmos3. | yes / no / no / no | +| DeepSeek-V4 preview | `1.2.0-deepseek-v4-dev.3` | v1.2.0 | vllm-runtime, sglang-runtime | May 9, 2026 | Superseded — recipe in v1.3.0 | Blackwell (B200 + GB200) preview; per-arch/CUDA tags (e.g. vllm-runtime:1.2.0-deepseek-v4-cuda13-dev.3). Superseded by the v1.3.0 recipe. | yes / no / no / no | +| DeepSeek-V4 preview | `1.2.0-deepseek-v4-dev.2` | v1.2.0 | vllm-runtime, sglang-runtime | May 1, 2026 | Superseded — recipe in v1.3.0 | Blackwell preview on vLLM v0.20.0 (native DSv4 support); superseded by dev.3. | yes / no / no / no | +| DeepSeek-V4 preview | `1.2.0-sglang-deepseek-v4-dev.1` | v1.2.0 | sglang-runtime | Apr 25, 2026 | Superseded — recipe in v1.3.0 | Earliest DSv4 preview (SGLang, B200 only); superseded by dev.2/dev.3. | yes / no / no / no | + + +{/* llms-tables:end */} diff --git a/docs/fern/reference/observability/operator-metrics.mdx b/docs/fern/reference/observability/operator-metrics.mdx index 3663a6b0266b..cd0ff692d59f 100644 --- a/docs/fern/reference/observability/operator-metrics.mdx +++ b/docs/fern/reference/observability/operator-metrics.mdx @@ -5,7 +5,7 @@ title: Operator Metrics subtitle: Kubernetes-only dynamo_operator_* metrics and the annotations and Helm values that enable metrics collection. --- -The Dynamo Operator exposes its own Prometheus metrics for controller reconciliation, webhook validation, and resource inventory. These are **Kubernetes-scoped** — they exist only where the operator runs, and are separate from the application metrics emitted by frontends and workers (see [Metrics Catalog](metrics-catalog.mdx)). For setup and dashboards, see the [Operator Metrics guide](../../kubernetes/observability/operator-metrics.md). +The Dynamo Operator exposes its own Prometheus metrics for controller reconciliation, webhook validation, and resource inventory. These are **Kubernetes-scoped** — they exist only where the operator runs, and are separate from the application metrics emitted by frontends and workers (see [Metrics Catalog](metrics-catalog.mdx)). For enablement and dashboards (including the operator dashboard), see the [Kubernetes observability guide](../../kubernetes/observability/metrics.md). All operator metrics use the `dynamo_operator` prefix. Unlike application metrics (which use a PodMonitor), the operator is scraped via a ServiceMonitor created by the Helm chart. @@ -130,7 +130,7 @@ spec: ## Related -- [Operator Metrics guide](../../kubernetes/observability/operator-metrics.md) — setup, example queries, and the operator Grafana dashboard. +- [Kubernetes observability guide](../../kubernetes/observability/metrics.md) — signal enablement and the operator Grafana dashboard. - [Kubernetes Metrics guide](../../kubernetes/observability/metrics.md) — PodMonitor and Prometheus walkthrough for application metrics. - [Metrics Catalog](metrics-catalog.mdx) — application `dynamo_*` metrics. - [Metric Labels](metric-labels.mdx) — application metric label dimensions. diff --git a/docs/fern/reference/release-artifacts.mdx b/docs/fern/reference/release-artifacts.mdx index 1831856a8a14..7f3e95c00e4e 100644 --- a/docs/fern/reference/release-artifacts.mdx +++ b/docs/fern/reference/release-artifacts.mdx @@ -2,84 +2,144 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 title: Release Artifacts -subtitle: Container images, Python wheels, Helm charts, and Rust crates published for the current Dynamo release. +subtitle: Container images, Python wheels, Helm charts, and Rust crates for the current release, plus early access builds and release history. --- -This page lists the published NVIDIA Dynamo release artifacts for the current stable release and where to find each one. Container images are on NVIDIA NGC under `nvcr.io/nvidia/ai-dynamo/`; each image name links to its NGC listing, where you can browse all published tags. For wheels, Helm charts, and crates, click a command in the **Install** column to copy the exact version-pinned command. +import { ReferenceStyles } from "@/components/ReferenceStyles"; +import { ArtifactBrowser } from "@/components/ArtifactBrowser"; +import { CratesFirstPublished } from "@/components/ReleaseTimeline"; +import { EaCoverageDots } from "@/components/ModelEABuildCards"; +import { PinnedEnvironment } from "@/components/PinnedEnvironment"; +import { TagLookup } from "@/components/TagLookup"; + + + +This page lists the published NVIDIA Dynamo release artifacts for the current stable release and where to find each one. Container images live on NVIDIA NGC under `nvcr.io/nvidia/ai-dynamo/`; every tag and install command below is click-to-copy. -See [Compatibility](compatibility.mdx) for hardware, platform, and backend feature support. +See [Compatibility](compatibility.mdx) for hardware, platform, and backend feature support, and [Model Early Access Builds](model-early-access-builds.mdx) for per-model early access container builds. -## Current Release: Dynamo v1.2.1 +## Current Release -- **GitHub Release:** [v1.2.1](https://github.com/ai-dynamo/dynamo/releases/tag/v1.2.1) -- **Docs:** [v1.2.1](https://docs.nvidia.com/dynamo) -- **NGC Collection:** [ai-dynamo](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/ai-dynamo/collections/ai-dynamo) + -## Container Images +**CUDA and driver requirements:** for the CUDA toolkit versions and minimum drivers for each container image, see [Compatibility](compatibility.mdx#cuda--driver-requirements). -Dynamo publishes seven container images on NVIDIA NGC under `nvcr.io/nvidia/ai-dynamo/` — three backend runtimes and four platform components. Each name links to its full NGC tag listing; the **Tags** column shows the tags published for v1.2.1 — click a tag to copy its full image reference. +### Pinned Environment -### Runtime Containers +One copy-paste block that pins every install path — the backend runtime container, frontend and operator images, Helm chart, and wheel — to the same release. -| Image | What it is | Tags | -|-------|------------|------| -| [`vllm-runtime`](https://catalog.ngc.nvidia.com/orgs/nvidia/ai-dynamo/containers/vllm-runtime/tags) | vLLM backend runtime | 1.2.11.2.1-cuda131.2.1-efa-amd64 | -| [`sglang-runtime`](https://catalog.ngc.nvidia.com/orgs/nvidia/ai-dynamo/containers/sglang-runtime/tags) | SGLang backend runtime | 1.2.11.2.1-cuda131.2.1-efa-amd64 | -| [`tensorrtllm-runtime`](https://catalog.ngc.nvidia.com/orgs/nvidia/ai-dynamo/containers/tensorrtllm-runtime/tags) | TensorRT-LLM backend runtime | 1.2.11.2.1-efa-amd64 | + -### Component Containers +## Known Issues -| Image | What it is | Tags | -|-------|------------|------| -| [`dynamo-frontend`](https://catalog.ngc.nvidia.com/orgs/nvidia/ai-dynamo/containers/dynamo-frontend/tags) | OpenAI-compatible API gateway with Endpoint Prediction Protocol (EPP) | 1.2.1 | -| [`dynamo-planner`](https://catalog.ngc.nvidia.com/orgs/nvidia/ai-dynamo/containers/dynamo-planner/tags) | Standalone Planner used by Profiler jobs and Planner pods | 1.2.1 | -| [`kubernetes-operator`](https://catalog.ngc.nvidia.com/orgs/nvidia/ai-dynamo/containers/kubernetes-operator/tags) | Operator that manages Dynamo deployments and CRDs | 1.2.1 | -| [`snapshot-agent`](https://catalog.ngc.nvidia.com/orgs/nvidia/ai-dynamo/containers/snapshot-agent/tags) Preview | Fast GPU worker recovery via CRIU | 1.2.1 | +Known issues are tracked on the [Known Issues](known-issues.mdx) reference page, including artifact-specific issues for older releases. -## Python Wheels + -Dynamo publishes three wheels on PyPI. All are at `1.2.1`, Python `3.10`–`3.12`, Linux (glibc `v2.28+`). +## Early Access Artifacts -| Package | What it is | Version | Install | -|---------|------------|---------|---------| -| [`ai-dynamo`](https://pypi.org/project/ai-dynamo/1.2.1/) | Main package with backend integrations (vLLM, SGLang, TensorRT-LLM) | `1.2.1` | | -| [`ai-dynamo-runtime`](https://pypi.org/project/ai-dynamo-runtime/1.2.1/) | Core Python bindings for the Dynamo runtime | `1.2.1` | | -| [`kvbm`](https://pypi.org/project/kvbm/1.2.1/) | KV Block Manager for disaggregated KV cache | `1.2.1` | | + +**Early access artifacts do not go through QA validation.** They are experimental previews intended for early testing and feedback, and may contain bugs, breaking changes, or incomplete features. Use stable releases for production workloads. + -For TensorRT-LLM, use the NGC container instead of the `ai-dynamo[trtllm]` wheel. +Model-specific early access builds (`vX.Y.Z--dev.N`) are tracked in [Model Early Access Builds](model-early-access-builds.mdx). Full-platform previews (`vX.Y.Z-dev.N`) are summarized here. -## Helm Charts + -Dynamo publishes two Helm charts, both at `1.2.1` on NGC (OCI). +**Early access Python wheels** are published on the NVIDIA package index at [pypi.nvidia.com](https://pypi.nvidia.com/), not on the public [PyPI](https://pypi.org/) index. Like stable wheels, they are Linux (manylinux) builds for the Python versions in [Compatibility](compatibility.mdx); `pip`/`uv` on macOS or Windows will not find matching wheels. A git tag `v1.3.0-dev.N` maps to a wheel version `1.3.0.devN`: -| Chart | What it is | Install | -|-------|------------|---------| -| [`dynamo-platform`](https://helm.ngc.nvidia.com/nvidia/ai-dynamo/charts/dynamo-platform-1.2.1.tgz) | Platform services (etcd, NATS) and the Dynamo Operator for a Dynamo cluster | | -| [`snapshot`](https://helm.ngc.nvidia.com/nvidia/ai-dynamo/charts/snapshot-1.2.1.tgz) | Snapshot DaemonSet for fast GPU worker recovery | | +```bash +# uv +uv pip install --pre --extra-index-url https://pypi.nvidia.com/ ai-dynamo==1.3.0.dev1 - -The `dynamo-crds` Helm chart is deprecated as of v1.0.0; CRDs are now managed by the Dynamo Operator. The `dynamo-graph` Helm chart is deprecated as of v0.9.0. - +# pip +pip install --pre --extra-index-url https://pypi.nvidia.com ai-dynamo==1.3.0.dev1 +``` -## Rust Crates - -Dynamo publishes its runtime as a set of crates on crates.io. All are at `1.2.1` with MSRV Rust `v1.82`, except `dynamo-async-openai` at `1.0.2`. - -| Crate | What it is | Install | -|-------|------------|---------| -| [`dynamo-runtime`](https://crates.io/crates/dynamo-runtime/1.2.1) | Core distributed runtime library | | -| [`dynamo-llm`](https://crates.io/crates/dynamo-llm/1.2.1) | LLM inference engine | | -| [`dynamo-protocols`](https://crates.io/crates/dynamo-protocols/1.2.1) | Async OpenAI-compatible API client | | -| [`dynamo-async-openai`](https://crates.io/crates/dynamo-async-openai/1.0.2) Deprecated | Legacy OpenAI client; use `dynamo-protocols` | | -| [`dynamo-parsers`](https://crates.io/crates/dynamo-parsers/1.2.1) | Protocol parsers (SSE, JSON streaming) | | -| [`dynamo-memory`](https://crates.io/crates/dynamo-memory/1.2.1) | Memory management utilities | | -| [`dynamo-config`](https://crates.io/crates/dynamo-config/1.2.1) | Configuration management | | -| [`dynamo-tokens`](https://crates.io/crates/dynamo-tokens/1.2.1) | Tokenizer bindings for LLM inference | | -| [`dynamo-tokenizers`](https://crates.io/crates/dynamo-tokenizers/1.2.1) | Tokenizer library for LLM inference | | -| [`dynamo-mocker`](https://crates.io/crates/dynamo-mocker/1.2.1) | Inference engine simulator for benchmarking | | -| [`dynamo-kv-router`](https://crates.io/crates/dynamo-kv-router/1.2.1) | KV-aware request routing library | | -| [`kvbm-logical`](https://crates.io/crates/kvbm-logical/1.2.1) | Logical layer for the KV Block Manager | | +**Nightlies:** `ai-dynamo` and `ai-dynamo-runtime` nightly builds from `main` publish wheels tagged `*.devYYYYMMDD` (since Apr 24, 2026). Install with the same `--pre` + extra-index pattern. -**CUDA and driver requirements:** for the CUDA toolkit versions and minimum drivers for each container image, see [Compatibility](compatibility.mdx#cuda--driver-requirements). + + +### v1.3.0-dev.1 + +Full-platform preview of v1.3.0, cut from `main` after the TensorRT-LLM `1.3.0rc17` upgrade (Jun 9, 2026) and superseded by the v1.3.0 GA release. Backends: SGLang `0.5.12.post1` | TensorRT-LLM `1.3.0rc17` | vLLM `0.22.0`. + + + +Complete runtime and component container matrix, `ai-dynamo` / `ai-dynamo-runtime` / `kvbm` wheels at `1.3.0.dev1` on pypi.nvidia.com, Rust crates at `1.3.0-dev.1` on crates.io, and the `dynamo-platform` and `snapshot` Helm charts. + +### Tag Lookup + +Reverse lookup for any tag on this page or in [Model Early Access Builds](model-early-access-builds.mdx): pick a tag to see the release or build it belongs to, the runtimes it applies to, and its status. + + + +For the full release history — every release newest-first with its notes — see [Releases](release-notes/README.mdx). + + + + + +{/* llms-tables:begin — generated by scripts/gen_llms_tables.py, do not edit */} + + +Current stable release: v1.3.0 (container tag `1.3.0`, wheel version `1.3.0.post1`). + +**Artifact inventory (v1.3.0)** + +| Category | Name | Description | Meta | Tags / install | +| --- | --- | --- | --- | --- | +| container | vllm-runtime | vLLM backend runtime | vLLM v0.23.0 · CUDA 13.0 · AMD64/ARM64 | `nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.3.0`; `nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.3.0-efa` | +| container | sglang-runtime | SGLang backend runtime | SGLang v0.5.14 · CUDA 13.0 · AMD64/ARM64 | `nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.3.0`; `nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.3.0-efa` | +| container | tensorrtllm-runtime | TensorRT-LLM backend runtime | TRT-LLM v1.3.0rc19 · CUDA 13.1 · AMD64/ARM64 | `nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0`; `nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-efa` | +| container | dynamo-frontend | OpenAI-compatible API gateway with Endpoint Prediction Protocol (EPP) | AMD64/ARM64 | `nvcr.io/nvidia/ai-dynamo/dynamo-frontend:1.3.0` | +| container | dynamo-planner | Standalone Planner used by Profiler jobs and Planner pods | AMD64/ARM64 | `nvcr.io/nvidia/ai-dynamo/dynamo-planner:1.3.0` | +| container | kubernetes-operator | Operator that manages Dynamo deployments and CRDs | AMD64/ARM64 | `nvcr.io/nvidia/ai-dynamo/kubernetes-operator:1.3.0` | +| container | snapshot-agent (Preview) | Fast GPU worker recovery via CRIU | AMD64/ARM64 | `nvcr.io/nvidia/ai-dynamo/snapshot-agent:1.3.0` | +| wheel | ai-dynamo | Main package with backend integrations (vLLM, SGLang, TRT-LLM) | Python 3.10–3.12 · Linux (glibc v2.28+) | `uv pip install ai-dynamo==1.3.0.post1` | +| wheel | ai-dynamo-runtime | Core Python bindings for the Dynamo runtime | Python 3.10–3.12 · Linux (glibc v2.28+) | `uv pip install ai-dynamo-runtime==1.3.0.post1` | +| wheel | kvbm | KV Block Manager for disaggregated KV cache | Python 3.10–3.12 · Linux (glibc v2.28+) | `uv pip install kvbm==1.3.0.post1` | +| helm | dynamo-platform | Platform services (etcd, NATS) and the Dynamo Operator for a Dynamo cluster | - | `helm install dynamo-platform oci://helm.ngc.nvidia.com/nvidia/ai-dynamo/charts/dynamo-platform --version 1.3.0` | +| helm | snapshot | Snapshot DaemonSet for fast GPU worker recovery | - | `helm install snapshot oci://helm.ngc.nvidia.com/nvidia/ai-dynamo/charts/snapshot --version 1.3.0` | +| crate | dynamo-runtime | Core distributed runtime library | MSRV Rust v1.82 | `cargo add dynamo-runtime@1.3.0` | +| crate | dynamo-llm | LLM inference engine | MSRV Rust v1.82 | `cargo add dynamo-llm@1.3.0` | +| crate | dynamo-protocols | Async OpenAI-compatible API client | MSRV Rust v1.82 | `cargo add dynamo-protocols@1.3.0` | +| crate | dynamo-async-openai (Deprecated) | Legacy OpenAI client; use dynamo-protocols | MSRV Rust v1.82 · final release | `cargo add dynamo-async-openai@1.0.2` | +| crate | dynamo-parsers | Protocol parsers (SSE, JSON streaming) | MSRV Rust v1.82 | `cargo add dynamo-parsers@1.3.0` | +| crate | dynamo-memory | Memory management utilities | MSRV Rust v1.82 | `cargo add dynamo-memory@1.3.0` | +| crate | dynamo-config | Configuration management | MSRV Rust v1.82 | `cargo add dynamo-config@1.3.0` | +| crate | dynamo-tokens | Tokenizer bindings for LLM inference | MSRV Rust v1.82 | `cargo add dynamo-tokens@1.3.0` | +| crate | dynamo-tokenizers | Tokenizer library for LLM inference | MSRV Rust v1.82 | `cargo add dynamo-tokenizers@1.3.0` | +| crate | dynamo-mocker | Inference engine simulator for benchmarking | MSRV Rust v1.82 | `cargo add dynamo-mocker@1.3.0` | +| crate | dynamo-kv-router | KV-aware request routing library | MSRV Rust v1.82 | `cargo add dynamo-kv-router@1.3.0` | +| crate | kvbm-logical | Logical layer for the KV Block Manager | MSRV Rust v1.82 | `cargo add kvbm-logical@1.3.0` | + +**Known artifact issues** + +| Release | Artifact | Issue | Status | +| --- | --- | --- | --- | +| v0.9.0 | dynamo-platform-0.9.0 | Helm chart sets operator image to 0.7.1 instead of 0.9.0. | Fixed in v0.9.0.post1 | +| v0.8.1 | vllm-runtime:0.8.1-cuda13 | Container fails to launch. | Known issue | +| v0.8.1 | sglang-runtime:0.8.1-cuda13, vllm-runtime:0.8.1-cuda13 | Multimodality not expected to work on ARM64. Works on AMD64. | Known limitation | +| v0.8.0 | sglang-runtime:0.8.0-cuda13 | CuDNN installation issue caused PyTorch v2.9.1 compatibility problems with nn.Conv3d — performance degradation and excessive memory usage in multimodal workloads. | Fixed in v0.8.1 (#5461) | + +**Crates: first published version on crates.io** + +| Crate | First version | Date | +| --- | --- | --- | +| dynamo-runtime | 0.1.0 | 2025-03-18 | +| dynamo-llm | 0.2.0 | 2025-05-01 | +| dynamo-async-openai | 0.4.1 | 2025-08-27 | +| dynamo-parsers | 0.5.0 | 2025-09-18 | +| dynamo-memory | 0.8.0 | 2026-01-15 | +| dynamo-config | 0.8.0 | 2026-01-15 | +| dynamo-tokens | 0.9.0 | 2026-02-12 | +| dynamo-mocker | 1.0.0 | 2026-03-13 | +| dynamo-kv-router | 1.0.0 | 2026-03-13 | +| dynamo-protocols | 1.1.0 | 2026-05-04 | +| dynamo-tokenizers | 1.2.0 | 2026-06-02 | + + +{/* llms-tables:end */} diff --git a/docs/fern/reference/release-notes/README.mdx b/docs/fern/reference/release-notes/README.mdx new file mode 100644 index 000000000000..7ffc484f165b --- /dev/null +++ b/docs/fern/reference/release-notes/README.mdx @@ -0,0 +1,23 @@ +--- +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +title: Releases +subtitle: Release history and per-release notes for every Dynamo release, mirrored from GitHub +--- + +import { ReferenceStyles } from "@/components/ReferenceStyles"; +import { ReleaseTimeline } from "@/components/ReleaseTimeline"; + + + +Release notes for Dynamo v1.0.0 and newer are mirrored here from the [GitHub releases](https://github.com/ai-dynamo/dynamo/releases), with breaking changes tracked on the [Deprecations](../deprecations.mdx) ledger and known issues on the [Known Issues](../known-issues.mdx) page. Patch releases are folded into their base release's page. Earlier releases link to GitHub. + + +For the artifact inventory each release shipped — container images, wheels, Helm charts, crates — see [Release Artifacts](../release-artifacts.mdx). + + +Agents and automation: [Releases (machine-readable)](../releases-data.mdx) renders this release data as generated markdown tables, with JSON and Atom twins in-repo at `docs/fern/assets/releases.json` and `docs/fern/assets/releases-atom.xml`. + +## Releases + + diff --git a/docs/fern/reference/release-notes/v1-0-0.mdx b/docs/fern/reference/release-notes/v1-0-0.mdx new file mode 100644 index 000000000000..143f75245f92 --- /dev/null +++ b/docs/fern/reference/release-notes/v1-0-0.mdx @@ -0,0 +1,442 @@ +--- +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +title: Dynamo v1.0.0 +subtitle: Release notes for Dynamo v1.0.0 (GA Mar 12, 2026), including patch releases v1.0.1, v1.0.2 +--- + +import { ReferenceStyles } from "@/components/ReferenceStyles"; +import { ReleaseHeader } from "@/components/ReleaseHeader"; +import { ReleaseSummaryCards } from "@/components/ReleaseSummaryCards"; + + + + + +Dynamo v1.0.0 is the first major release of the open-source distributed inference platform. This release delivers production-grade disaggregated serving with comprehensive multimodal and omni-model support, KV cache optimizations, improved handling of agentic workloads, Kubernetes-native deployment at scale, and a stabilized public API. + + +Breaking changes and deprecations for this release are tracked on the [Deprecations](../deprecations.mdx#v100) ledger; known issues on the [Known Issues](../known-issues.mdx#v100) page. Key dependency pins live on [Compatibility](../compatibility.mdx); shipped artifacts on [Release Artifacts](../release-artifacts.mdx). Model early access builds (`vX.Y.Z--dev.N`) are tracked in [Model Early Access Builds](../model-early-access-builds.mdx). + + +## Highlights + + + +## Features & Improvements + +### Multimodal & Diffusion + +- **Encoder Cache Infrastructure:** Implemented EncoderCacheManager with async support for caching multimodal encoder outputs (#5632, #5676) and content-addressed hashing for TensorRT-LLM (#5715). +- **TensorRT-LLM Encoder Cache:** Integrated encoder cache into TensorRT-LLM PrefillHandler (#5714), EPD workflow (#5780), and E/PD disaggregated workflow (#5815) for cross-worker multimodal reuse. +- **vLLM Embedding Cache:** Added embedding cache to PD workers (#6029, #6061) and aggregated vLLM nodes (#6153) for reusing multimodal embeddings across requests. +- **Text-to-Image Generation:** Added text-to-image support via vLLM Omni pipeline (#5608, #5912) and SGLang image diffusion (#5609). +- **Text-to-Video Generation:** Added text-to-video support via SGLang T2V (#5793), vLLM Omni pipeline (#6104), and TensorRT-LLM Wan T2V (#5926), with experimental MJPEG video streaming via `/v1/videos/stream` (#6487). +- **Multimodal Embedding Transfer:** Added embedding transfer sender and receiver for cross-worker multimodal data movement (#6098), adopted transfer classes for the EPD pipeline (#6223), and optimized by keeping embeddings on GPU in the Embedding Sender (#6535). +- **Multimodal-Aware KV Cache Routing:** Implemented multimodal-aware request routing for vLLM (#6235) and end-to-end multimodal KV cache routing for TensorRT-LLM (#5480), optimizing request placement based on media content. +- **TensorRT-LLM Multimodal Preprocessor:** Added TensorRT-LLM multimodal preprocessor with backend media decoding (#5910). +- **vLLM Frontend Media Decoding:** Enabled vLLM backend with frontend media decoding for end-to-end multimodal serving (#5781). +- **Batch Image Processing:** Added batch image processing in encode worker and Qwen3 model support (#6021). +- **SGLang MMEncoder in EPD:** Integrated SGLang MMEncoder for multimodal EPD encode worker pipeline (#6162). +- **Multimodal Model Support:** Improved multimodal disaggregation reliability with Qwen2.5 VL 32B support (#5895) and added Qwen3-VL-30B-A3B support for EPD pipeline (#6533). +- **NIXL WRITE Embedding Transfer:** Added NIXL WRITE initiation for cross-node multimodal embedding transfer (#6776). +- **vLLM Omni Container:** Installed vllm-omni in vLLM container for visual generation support (#6458). + +### Frontend & Agents + +- **Reasoning and Tool Call Parsers:** Added reasoning content management for DeepSeek v3.2, GLM-4.7, and Kimi-2.5 (#6107), interleaved thinking support (#6422), and new tool/reasoning parsers for GLM-4.7 (#5897), MiniMax-M2 (#6294), and Kimi K2/K2.5 (#6407). +- **Responses API Compliance:** Implemented Responses API compliance with upstream type alignment for spec conformance (#6089). +- **Anthropic Messages Endpoint:** Added Anthropic Messages API endpoint (`/v1/messages`) for cross-provider compatibility (#6231). +- **Tiktoken Support:** Added Tiktoken tokenizer support for models requiring Tiktoken encoding (#6460). +- **vLLM Chat Path Optimization:** Reduced Python-side overhead in the vLLM chat path for lower latency (#6437). +- **vLLM Pre/Post Processing:** Adopted vLLM for pre- and post-processing in the Frontend for consistency (#5544). +- **Dynamic gRPC Startup:** Made gRPC startup dynamic for high ISL/OSL scenarios in the gRPC Frontend (#5536). + +### Kubernetes Deployment + +#### DynamoGraphDeployment Request + +- **DGDR Deployment Guide:** Added comprehensive Kubernetes deployment guide for DynamoGraphDeploymentRequest (DGDR) workflows covering the golden path from model selection through profiling and autoscaling (#7304). +- **DGDR API Maturation:** Added structured `.status.state` enums for DGD (#6324) and DGDR (#6396), `observedGeneration` for reconciliation tracking (#6398), introduced the `v1beta1` DGDR API with automatic conversion from `v1alpha1` (#6352), and adopted `v1beta1` in the controller (#6498). +- **Model/WorkerSet Architecture:** Introduced hierarchical Model/WorkerSet architecture for multi-namespace support (#6054). +- **Rolling Updates:** Implemented managed rolling updates for DGD worker deployments (#6110). +- **DGD Print Columns:** Added print columns with ready condition for `v1alpha1` API types like DGD (#5542). +- **Optional DGDR Image Field:** Made the image field optional in DGDRs for flexible container configuration (#6557). +- **Operator Version in DGD:** Included Operator version in DGD for version tracking in cluster state (#6121). +- **AIC DGD Generation:** Enabled AIC DGD generation call for automated infrastructure configuration (#6216). +- **Profiler Job Overrides:** Added profiler job overrides for customizable profiling runs (#6641). + +#### Dynamo Snapshot + +- **Dynamo Snapshot:** Introduced Dynamo Snapshot for fast GPU worker recovery (#4978, #7068), refactored configuration with `/dev/shm` support and mount-policy rewrite (#5946), added external restore with signal-based IPC (#6286), and extended to the SGLang backend (#6594). + +#### Gateway API Inference Endpoint (GAIE) + +- **EPP Integration:** Added the EPP component for Kubernetes Gateway API-based inference routing (#5611), implemented the decomposed pipeline for flexible routing stages (#5446), added startup probe for reliable liveness detection (#5770), and enabled the EPP `pods` interface for pod-level traffic management (#6302). + +#### Dynamo Operator + +- **Operator Management Improvements:** Implemented config versioning via ConfigMap injection (#6464), simplified CRD management (#6466), reduced Helm chart dependencies (#7048), and replaced kube-rbac-proxy with controller-runtime authorization (#7069). +- **GPU Discovery Migration:** Migrated GPU discovery from Dynamo Profiler to Operator with automatic injection (#6224). +- **Namespace-Scoped GPU Discovery:** Added optional GPU discovery for namespace-scoped Operators (#6343). +- **Tolerations and Affinity Support:** Added tolerations and affinity support for all platform Helm chart components (#5561). +- **Rolling Updates Documentation:** Added documentation for Operator rolling updates (#6541). + +### Scheduling + +#### Router + +- **Data-Parallel Routing:** Added per-DP-rank gap detection (#5873), TensorRT-LLM DP rank routing (#5936), and RNG tiebreaking for DP routing targets (#6253) for improved data-parallel load distribution. +- **Router Priority Queue:** Implemented request priority queue in the Router (#6010) and plumbed priority through SGLang and vLLM handlers for end-to-end support (#6348). +- **Global Router:** Added global Router for hierarchical Planner topology (#5697). +- **Global Router + vLLM Example:** Added DGD example for global Router + vLLM deployment (#5760). +- **Expert Routing Info:** Enabled returning routed experts info through SGLang for expert-parallel routing visibility (#6137). +- **Prefill Tokens Threshold:** Added prefill tokens threshold based on max batched tokens fraction for adaptive batching (#5867). +- **Default Event Threads:** Defaulted `router_event_threads` to 4 for improved Router throughput (#6724). + +#### Planner + +- **Planner Autoscaling:** Added GlobalPlanner component for centralized cross-cluster scaling (#5702), implemented load-based scaling in SLA Planner (#6145), added throughput metrics source for disaggregated scaling decisions (#6500), and moved core logic from DPP to AIC with static profiling support (#6285). +- **Planner P/D Separation:** Separated Planner into independent prefill/decode Planners (#5622) and automated resource allocation by deriving GPU counts (#5919) and worker counts (#5934) from DGD status. +- **Planner Config Migration:** Migrated Planner from argparse CLI to config file for unified configuration (#6356). +- **Planner Schema in DGDR:** Added Planner schema to DGDR and Profiler input for configuration consistency (#6463). +- **Profiler Model Validation:** Removed default model name in Profiler and added validation for served model name or path (#5950). + +### KV Block Manager + +- **Speculative Prefill:** Implemented speculative prefill for proactive KV cache population (#6230). +- **Flash Indexer Optimizations:** Optimized flash indexer performance for faster KV cache prefix lookups (#6305). +- **Standalone KV Indexer:** Added standalone KV indexer with query endpoint for decoupled prefix matching (#6446). +- **KVBM Priority Offload:** Implemented priority-based KV cache offload filtering (#5563) and optimized by reading the priority env var once at init (#5798). +- **KVBM Logical Abstraction:** Introduced KVBM-logical abstraction layer for flexible KV block management (#6033). +- **Nested KV Index Mapper:** Implemented nested mapper for KV indexing to support hierarchical prefix matching (#5785). +- **KVBM Memory Enhancements:** Added KVBM memory management enhancements for improved allocation and lifecycle (#5532). +- **Default KVBM Enablement:** Enabled lib/memory, media-nixl, and KVBM by default for out-of-the-box disaggregated serving (#5602). +- **KVBM Kernels Crate:** Added `kvbm-kernels` crate and upgraded cudarc to 0.19 for GPU kernel support (#6309). +- **NVTX Annotations:** Added NVTX annotations to KVBM for GPU profiling visibility (#6334). +- **Default KV Events Config:** Defaulted `kv-events-config` to empty to align with vLLM defaults (#6404). +- **KV Hit Rate Histogram:** Exposed predicted KV hit rate as Prometheus histogram for cache efficiency monitoring (#6507). +- **Mocker KV Cache Tracing:** Added optional KV cache allocation/eviction tracing (#6052, #6207), KV transfer latency simulation for disaggregated benchmarks (#6504), and ZMQ-based KV event publishing (#6528) to the mocker. + +### LoRA Support + +- **LoRA Routing and Allocation:** Added LoRA-aware routing hints and tracking (#5875), memory-aware load estimation (#5880), HRW-based optimal adapter allocation (#5992), and LoRA-aware KV cache events (#6517). +- **Multimodal LoRA:** Extended LoRA support to multimodal workloads with protocol-level model identification (#6382), request handling for multimodal workers (#6399), and deployment examples for local (#6400) and Kubernetes (#6452). + +### Infrastructure Modernization + +- **Unified Configuration System:** Introduced a unified configuration system with typed base classes (#5975) and migrated vLLM (#6075) and Frontend CLI (#6201) to the new system. +- **Configuration System Migration:** Migrated SGLang (#6280), TensorRT-LLM (#6297), global Router (#6342), and Router (#6346) to the unified configuration system. +- **Go-to-Definition Support:** Enabled go-to-definition for `dynamo.runtime`, `dynamo.nixl`, and external dependencies (#6026). +- **Standardized Error Type:** Introduced standardized Dynamo error type for consistent error handling across the stack (#6303). +- **AIPerf Client Rate Control:** Added `--request-rate` and `--request-rate-mode` flags to aiper client for flexible load testing (#6585). +- **Disaggregation Mode Enum:** Added `--disaggregation-mode` enum to vLLM backend for explicit mode selection (#6483). +- **vLLM Endpoint Flag:** Added `--endpoint` flag support to `dynamo.vllm` for flexible serving configuration (#6360). + +### Performance + +- **Mocker Performance:** Improved mocker with model pre-fetching, staggered launches, and timing accuracy (#5871, #5808, #6100), and modularized the crate into common/scheduler/kv_manager/cache modules (#6440). + +#### SGLang + +- **SGLang GPU Memory Service:** Integrated SGLang with GPU Memory Service for unified memory management (#5664). +- **SGLang Request Migration:** Implemented request migration for SGLang to support live request handoff (#5659). +- **SGLang Weight Update Endpoints:** Added SGLang `/engine` weight update endpoints for online model updates (#6094). + +#### TensorRT-LLM + +- **TensorRT-LLM Guided Decoding:** Added guided decoding backend config and choice support for TensorRT-LLM (#5762). +- **CUDA IPC for TensorRT-LLM:** Introduced CUDA IPC for TensorRT-LLM PrefillHandler enabling zero-copy cross-process transfers (#5773). +- **NixlConnector Config:** Added `--kv-transfer-config NixlConnector` to disaggregated scripts and recipes (#6560). + +#### vLLM + +- **vLLM Multi-Node Multiprocessing:** Adopted vLLM multiprocessing in multi-node scenarios for improved parallelism (#6191). +- **Headless Multi-Node Mode:** Added `--headless` mode for multi-node TP/PP in `dynamo.vllm` for worker-only deployments (#6204). +- **ModelExpress P2P Weight Transfer:** Enabled ModelExpress P2P weight transfer in Dynamo vLLM worker for faster model loading (#6186). + +### Fault Tolerance & Observability + +- **Router Metrics and Tracing:** Added per-worker load monitoring (#5842), centralized Router-level request tracking (#6146), standardized all Router metrics under the `dynamo_router`_* namespace (#6227), and added OTel tracing for routing overheads (#6194). +- **Engine Prometheus Metrics:** Exposed Python-level engine metrics via LLMComponentMetrics (#5817), added auto/custom label injection (#5989), introduced tokenizer (#6092) and detokenization (#6160) latency metrics, and exposed TensorRT-LLM kv_cache metrics (#6469). +- **NIXL Telemetry Port:** Added NIXL Telemetry Prometheus port for transfer library monitoring (#5567). +- **Error Type Metric Label:** Added `error_type` label to request metrics for fine-grained error classification (#5568). +- **Grafana Dashboard:** Added Grafana dashboard and monitoring setup for comprehensive observability (#4639). +- **NIXL Sanity Check:** Added NIXL availability check to sanity_check for environment validation (#6087). +- **Graceful Shutdown Draining:** Enabled backends to accept new requests during shutdown grace period for graceful draining (#6093). + +### Recipes + +- **GB200 Disagg Recipe:** Added GB200 GPT-oss disaggregated serving recipe for next-gen hardware support (#4954). +- **DeepSeek V3.2 Recipe:** Added DeepSeek V3.2 TensorRT-LLM recipe for optimized serving (#6969). +- **Qwen3-VL-30B Recipe:** Added Qwen3-VL-30B recipe for aggregated and encoder cache deployment with vLLM (#7191). + +## Bug Fixes + +### Multimodal + +- **Multimodal Disaggregated Serving:** Fixed multiple reliability issues in multimodal prefill/decode disaggregated serving and restored EPD pipeline on single-GPU (#5951, #6753, #6978). +- **Multimodal Input Processing:** Fixed multimodal input loader blocking the async event loop, PSD file crash in the image pipeline, and vLLM OmniModel image processing performance (#5945, #6212, #6451). +- **Multimodal API and Stream Handling:** Fixed `stream_options` forwarding through the multimodal request pipeline, CLI flag collisions with `--omni-` prefixes, and `normalize_finish_reason` on the OmniHandler (#6474, #6476, #6896). +- **Multimodal Cross-Node Transfer:** Fixed encode + prefill/decode flow in TensorRT-LLM for multimodal embedding transfer (#6790). +- **Multimodal Video and Audio:** Fixed vLLM chat processor to correctly handle video and audio inputs and resolved invalid UUID errors from empty multimodal inputs (#6708, #6904). +- **Multimodal Router Performance:** Fixed duplicate image downloads and unnecessary image processing in the multimodal Router for vLLM, reducing latency for repeated media content (#7172). +- **Multimodal Pipeline Fixes:** Fixed multiple minor issues in the vLLM multimodal pipeline, worker service registration collisions, Llama 4 aggregated multimodal launch script, and LLaVA model EPD support (#5748, #5986, #6103, #6765). + +### Frontend & Agents + +- **LoRA Endpoint Reliability:** Fixed LoRA load/unload endpoints silently swallowing errors and extended S3 download timeouts to prevent failures with large adapter files (#5626, #6986). +- **Request Sampling Parameters:** Fixed request sampling parameters not being forwarded to backend workers, causing generation settings to be silently ignored (#5797). +- **Reasoning Token Handling:** Fixed reasoning parser propagation from worker runtime config, interleaved reasoning content ordering, and reasoning content being dropped when a tool-call starts mid-stream (#6300, #6442, #7051). +- **Chat Template and Model Fixes:** Fixed DeepSeek V3.2 chat template for function calling and structured output, Nemotron Nano model to use the correct reasoning parser (#6034, #6288), and added `force_nonempty_content` for Nemotron models (#7225). +- **Frontend Stability:** Fixed HTTP request cancellation using a temporary token instead of the real cancellation token, and fixed a Frontend crash when running with the TensorRT-LLM runtime image (#6344, #6481). +- **Model Endpoint Correctness:** Fixed `/v1/models` endpoint exposing inactive models and model name resolution to prefer `--served-model-name` (#5881, #7021). +- **Responses API Compatibility:** Fixed Responses API rejecting valid assistant `output_text` messages that lacked `id`/`status` fields (#7049). +- **vLLM Processor Compatibility:** Fixed vLLM processor compatibility with vLLM 0.16 API changes and incorrect output when `stream_interval` is greater than 1 (#6873, #6874). +- **Prompt Length Validation:** Fixed missing validation for prompts exceeding `max_seq_len`, now returning HTTP 400 instead of silently failing (#6997). +- **Guided Grammar Depth Limit:** Fixed guided grammar to reject schemas with excessive nesting depth, preventing potential resource exhaustion (#7135). + +### Kubernetes Deployment + +#### DynamoGraphDeployment Request + +- **DGD/DGDR Configuration:** Fixed DGD cross-selection, fallback for missing `subComponentType`, service name length validation for DNS compliance, name sanitization for DNS-1035, DGDR prefix for naive fallback (#5449, #6113, #6317, #7062, #6679), and stripped `apiVersion`/`kind`/`metadata` from `overrides.dgd` before merging (#7121). +- **Operator Override Ordering:** Fixed DGD overrides to apply before running interpolation, ensuring tolerations propagate correctly (#7226). + +#### Dynamo Snapshot + +- **Snapshot Checkpoint/Restore:** Fixed Snapshot checkpoint failure handling to use SIGKILL, multi-GPU UUID mapping, restore to correctly pass the checkpoint path (#6478, #6492, #7018), and snapshot children before process group kill to prevent GPU memory leaks (#7232). + +#### Dynamo Operator + +- **Helm Chart Reliability:** Disabled etcd subchart by default, restored Helm docs autogeneration, and reverted a template change that caused deployment failures (#5739, #6329, #6459). +- **Operator Stability:** Fixed restart state tracking for parallel restarts, `DynamoComponentReady` condition updates, `imagePullPolicy` application, etcd cleanup logic, and consolidated discovery backend configuration (#4821, #5051, #5949, #6263, #6167). +- **Multi-Node Deployment Fixes:** Fixed SSH setup for TensorRT-LLM multi-node workers, unquoted mpirun and Ray leader arguments that caused multi-node failures, and added nodeSelector support (#6225, #6248, #6711). +- **Operator GPU Discovery and Tolerations:** Fixed GPU discovery preflight job, correct storage of GPU-equipped nodes, propagation of tolerations with auto-discovered GPU limits, and PVC block emission in configmap (#6640, #6714, #6979, #6755). +- **Operator CRD and API Configuration:** Fixed CRD validation for nil/empty containers, `AutoApply` field type for proper nil handling, webhook version matching for `v1alpha1` DGDR, annotation propagation, EPP config plugin weight support (#6255, #6712, #6808, #6718, #6783), and allowed `x-kubernetes-preserve-unknown-fields` in CRD validation (#7128). + +### Scheduling + +#### Router + +- **Router Startup Race Condition:** Fixed race condition between worker discovery and runtime config discovery in the KV Router that caused routing failures on startup (#5924). +- **Router Stream Panics:** Fixed stream handling in the Router that caused panics when polling after stream termination (#5872). +- **Router Data-Parallel Routing:** Fixed Router to correctly pass the data-parallel rank into the vLLM engine and corrected KV Router discovery name derivation (#6014, #6475). +- **Router Scheduling Backpressure:** Fixed scheduling by folding it into the queue so backpressure propagates correctly (#6470). +- **Router Metrics Collection:** Fixed `RouterRequestMetrics` availability to ensure Router metrics are always collected (#6558). + +#### Planner + +- **Profiler Timeout and Crash Fixes:** Fixed profiler deployment timeout handling for large MoE models and config generation to strip None arguments that caused crashes (#6086, #6887). +- **Profiler DGDR Validation:** Fixed DGDR validator and DGD generation in the profiler, improved service name logging (#6876, #6112), and fixed profiling condition updates to populate results and clear phase after completion (#7195). +- **Planner CLI Configuration:** Fixed `disagg_planner.yaml` and Planner test configs to use the updated CLI format (#6775, #7041, #7042). +- **Planner Backend Resolution:** Fixed propagation of resolved backend and skipped interpolation for aggregated deployments (#7142). +- **Profiler TTFT/ITL Default Handling:** Fixed Profiler validation error by using `model_fields_set` to distinguish TTFT/ITL default usage (#6827). + +### KV Block Manager + +- **KV Cache Sleep/Wake Stability:** Fixed KV cache block allocation signal after sleep/wake cycles and CUDA synchronization race conditions during GPU memory transitions (#5681, #5759). +- **KV Event Propagation and Block Management:** Fixed KV event propagation for data-parallel multi-node deployments and KVBM to read block size from vLLM at runtime instead of using a hardcoded value (#5589, #5713, #5851). +- **KV Cache Memory Leak:** Fixed memory leak where KV cache blocks were not freed on stream drop (#6246). +- **GMS Reliability:** Fixed GMS CLI startup failure, removed unnecessary CUDA synchronize calls that degraded performance, and fixed GMS socket UUID resolution via the CUDA driver API (#5749, #6362, #6914). +- **KVBM CUDA Device Handling:** Fixed PinnedAllocator to use the correct `device_id`, KVBM to respect `CUDA_VISIBLE_DEVICES` for NUMA binding, `device_blocks` double-counting in the TensorRT-LLM connector, and added authorization guards to memory occupation control endpoints (#6877, #6950, #6406, #7023). + +### Performance + +#### SGLang + +- **SGLang Metrics and Monitoring:** Fixed metrics prefix format from `sglang`_ to `sglang:` and `TokenizerMetricsCollector` lazy-import to avoid collector registration errors (#5701, #6269). +- **SGLang Configuration Fixes:** Fixed tool-call-parser flags to prevent configuration conflicts and DeepSeek-R1 recipe with watchdog timeout to prevent hangs (#5849, #6076). +- **SGLang Decode Handler:** Fixed decode handler to ignore empty non-final stream chunks (#6304). +- **SGLang Build and API:** Fixed container build conflict by removing `python3-blinker` and corrected multimodal item keys in the SGLang API (#5995, #5981). + +#### TensorRT-LLM + +- **TensorRT-LLM Stability:** Fixed decode worker stability by temporarily disabling request cancellation and eliminated crashes caused by unsafe `abort()` calls (#5764, #5827). +- **TensorRT-LLM Multimodal Support:** Fixed multimodal flag being silently ignored, multimodal hash support for TRT-LLM 1.3 `apply_mm_hashes` API, skipped encoder LLM creation for unsupported models (#6468, #6907, #6918), and fixed the multimodal preprocessor after the initial approach was reverted (#6920, #6993). +- **TensorRT-LLM Guided Decoding:** Fixed handler to properly convert guided decoding dictionaries to `GuidedDecodingParams` (#6127). +- **TensorRT-LLM Multi-Node Deployment:** Fixed multi-node worker SSH crash in non-root containers and removed deprecated `beam_width` parameter from health check (#6772, #6890). + +#### vLLM + +- **vLLM Worker Stability:** Fixed worker graceful shutdown to prevent orphaned processes, decode worker logging format that caused CrashLoopBackOff, and worker registration for external/hybrid load balancing (#5818, #6267, #6833). +- **vLLM Disaggregated Serving:** Fixed disaggregated serving by adding missing `--is-decode-worker` and `--kv-transfer-config` flags (#5843, #6554). +- **vLLM Launch Script Fixes:** Fixed DeepSeek-R1 recipe checkpoint path, removed an unnecessary bash wrapper, and corrected launch scripts for disaggregated and speculative decoding (#5721, #6035, #6562). +- **vLLM Stream Handling:** Fixed sampling parameter parsing in the EPD flow (#5813). +- **vLLM Performance Configuration:** Fixed Docker image to use the CUDA sampler for better performance and corrected engine stats logging (#5613, #6566). +- **vLLM Multi-Worker Port Collisions:** Fixed HTTP port collisions when multiple workers share a process (#7185). + +### Build & Container + +- **Runtime Image Fixes:** Fixed missing native libraries (nvlink, UCX, NIXL, CRT, Triton paths), corrected image tags across SGLang, TensorRT-LLM, and vLLM Dockerfiles (#6503, #6521, #6958, #6983, #6401), and updated UCX reference for performance (#7218). +- **TensorRT-LLM Dependency Fixes:** Fixed missing `msgpack` dependency and pinned `pydantic-settings` below 2.13.0 for compatibility (#5799, #6339). +- **Build System Fixes:** Fixed cross-platform NUMA module compilation, `ai-dynamo-runtime` wheel packaging to exclude NIXL shared libraries, CI container `GIT_COMMIT_SHA` population, and disabled `media-ffmpeg` feature by default (#6354, #6881, #7016, #6574). + +### Other + +- **Core Infrastructure Fixes:** Fixed ZMQ transport receive timeout to prevent hangs, Prometheus metric collisions via multi-registry scrape, stale NATS consumers, multi-node Slurm launch arguments, performance degradation from excessive logging in the EPD pipeline, and tool call validation (#5685, #5678, #5948, #5861, #6742, #5504). + +## Documentation + +### New Content + +- **AKS Storage Guidance:** Added Azure AKS storage guidance for Dynamo caches (#5581). +- **TensorRT-LLM Known Issues:** Added known issues section for TensorRT-LLM backend (#5801). +- **Mocker Documentation:** Added mocker component documentation (#5832). +- **GPU Memory Service:** Added overview documentation for GPU Memory Service (#5920). +- **Disaggregated Serving Guide:** Added disaggregated serving guide (#6024). +- **Quick Start Sections:** Added quick start sections to KVBM and Router guides (#6043). +- **KVBM Disaggregated Setup:** Added instructions for TensorRT-LLM KVBM disaggregated setup (#6055). +- **Architecture Docs:** Added Discovery Plane documentation and refactored Event Plane with D2 diagrams (#6229). +- **Inference Gateway:** Added inference gateway documentation page (#6319). +- **Agent Docs:** Added agent readme and documentation (#6320). +- **Frontend Configuration:** Documented Frontend requirement for model config file access (#6327). +- **Speculative Prefill Demo:** Added multiturn_bench README with speculative prefill demo (#6502). +- **Dev Containers Troubleshooting:** Documented Docker 29.x Dev Containers hang root cause and fix (#6505). +- **KV Indexer Docs:** Added standalone KV indexer documentation (#6511). +- **Embedding Cache:** Documented embedding cache support in vLLM and TensorRT-LLM (#6555). +- **SGLang Observability:** Expanded SGLang observability guide with tracing and dashboards (#6556). +- **DGDR `v1beta1`:** Documented `v1beta1` DynamoGPUDynamicResource API (#6713). +- **vLLM Multimodal Router:** Added docs for vLLM multimodal Router (#6568). +- **Nemotron-3-Super-FP8 Recipes:** Added Nemotron-3-Super-FP8 deployment recipes for SGLang aggregated, SGLang disaggregated, and TensorRT-LLM disaggregated with model download manifests (#7254). +- **FastVideo Example and Guide:** Added FastVideo text-to-video example with deployment guide and sidebar reorganization (#7283). +- **Getting Started Introduction:** Added introduction page to the Getting Started section with platform overview (#7292). + +## Looking Ahead + +Dynamo v1.1.0 is targeted for April 29, 2026. Here's a small preview of what's already taking shape: + +### Longer Contexts, Lower Cost + +FlexKV manages KV cache across HBM, host memory, and SSD so long-context and high-concurrency workloads don't hit GPU memory limits. Instead of dropping requests when HBM fills up, the system spills KV blocks to cheaper storage tiers and pulls them back on demand. + +### Resilient Routing at Scale + +The KV indexer gains P2P state recovery and automatic ZMQ gap replay, keeping prefix matching correct through node failures without manual intervention. Multi-model and multi-tenant isolation ensures that shared clusters route requests to the right cache even when multiple models share the same infrastructure. + +### Unified Observability + +Forward pass metrics on the event plane and Loki log aggregation with unified OTLP ingestion bring metrics, traces, and logs into a single pipeline. Operators debugging latency or throughput issues in disaggregated deployments no longer need to correlate data across separate tools. + +**Full Changelog**: [https://github.com/ai-dynamo/dynamo/compare/v0.9.1...v1.0.0](https://github.com/ai-dynamo/dynamo/compare/v0.9.1...v1.0.0) + +## Patch releases + +### v1.0.1 — Mar 16, 2026 + +#### Summary + +Dynamo v1.0.1 is a patch release to Dynamo v1.0.0 with **critical bug fixes** and **expanded model support**. Key fixes resolve a TensorRT-LLM startup crash on CUDA 13.1 caused by a `cutlass-dsl` packaging mismatch, and restore OpenAI API logprobs compliance where `bytes` and `token` fields were not populated when routing through Dynamo Frontend. This release also enables experimental Kimi K2.5 model support by adding DeepSeek V3 architecture tokenizer handling and fixing tiktoken multi-byte streaming panics. + +**Base Branch**: `release/1.0.0` + +#### Bug Fixes + +- **TensorRT-LLM CUDA 13.1 Startup Crash:** Fixed startup crash where a `cutlass-dsl` stub crashed the TensorRT-LLM import chain on CUDA 13.1. This was a [known issue in v1.0.0](https://github.com/ai-dynamo/dynamo/releases/tag/v1.0.0) that blocked MoE models on Blackwell GPUs, including the Qwen3-235B-A22B-FP8 TensorRT-LLM recipe. (#7393) +- **Kimi K2.5 Tokenizer and Streaming:** Added DeepSeek V3 architecture support for Kimi's BPE tokenizer pattern and fixed worker thread panic on incomplete multi-byte sequences during streaming inference. Enables serving nvidia/Kimi-K2.5-NVFP4 and other Kimi K2.5 models that use the DeepSeek V3 `model_type` with a tiktoken tokenizer. (#7424) +- **OpenAI Logprobs Fields:** Fixed `bytes` and `token` fields in logprobs responses always returning `None`/empty when routing through Dynamo Frontend. Affected vLLM backend; direct backend queries returned correct values but requests routed through Frontend did not. (#7404) + +### v1.0.2 — Apr 22, 2026 + +#### Summary + +Dynamo v1.0.2 is a patch release focusing on **Frontend correctness fixes**, **DGDR-driven Kubernetes deployment robustness**, **rolling-update flexibility**, and **guided-decoding input hardening**. + +Key fixes restore real stream metadata in non-streaming responses with tool calls, correct Kimi K2.5 tokenizer special-token handling that caused TensorRT-LLM to reject requests, and add byte-length and nesting-depth caps to the OpenAI guided-decoding path. + +On the deployment side, DGDR-created DynamoGraphDeployments now derive their name from the parent DGDR, DGDR-managed ConfigMaps cascade-delete with their parent, the Operator no longer thrashes on foreground cascading deletion, and per-WorkerSet MDC checksum validation enables rolling updates with divergent worker configuration under the same Model. + +**Base Branch**: `release/1.0.1` + +#### Full Changelog + +##### Kubernetes Deployment + +- **DGDR-Driven DGD Naming:** Fixed Profiler-generated DynamoGraphDeployment naming so that DGDs derive their name from the parent DynamoGraphDeploymentRequest (`-dgd`) instead of from topology alone (`-`) (#7835), eliminating namespace-level name collisions when multiple DGDRs share the same backend/topology and respecting user-provided names from `spec.overrides` when present. +- **DGDR ConfigMap Owner References:** Added Kubernetes owner references to ConfigMaps created by DGDR (#7881) so that DGDR-managed ConfigMaps are cascade-deleted with their parent. + +##### Runtime + +- **Per-WorkerSet MDC Checksum Validation:** Scoped Model Discovery Card checksum validation from per-Model to per-WorkerSet (#8278), enabling rolling updates where different WorkerSets under the same Model can carry different configuration (e.g. tool-call parser) without draining existing workers first. Mismatches are still rejected when a new worker joins an existing WorkerSet, but cross-WorkerSet checksum drift is no longer a hard error. + +#### Bug Fixes + +- **DGD Cascading Deletion Thrashing:** Fixed Operator behavior under foreground cascading deletion of DynamoGraphDeployments (#8212) so the Operator no longer thrashes the resource during teardown, ensuring clean DGD deletion in Kubernetes garbage-collection scenarios. +- **Stream Metadata Preservation:** Fixed OpenAI Frontend stream finalization that overwrote real `id`, `model`, and `created` fields with hardcoded placeholders (`stream-end`, `unknown`, `0`) when a tool-call parser combined streamed chunks into a non-streaming response (#8281), restoring correct response metadata for non-streaming tool-call requests. +- **Per-Node GPU Topology in DGD Builder:** Fixed thorough-mode MoE config enumeration in the Planner/Profiler that ignored `numGpusPerNode` and produced unschedulable candidate DGDs on multi-node clusters (#8281). Worker GPU resource limits are now clamped per node and `multinode.nodeCount` is set for workers that span multiple nodes. +- **Kimi Tokenizer Special Tokens:** Fixed Rust tiktoken tokenizer handling of reserved-token fallback names for Kimi K2.5 (#7898), resolving prompt-token inflation that caused TensorRT-LLM to reject requests with negative `default_max_tokens` and enabling correct serving of `nvidia/Kimi-K2.5-NVFP4` and other Kimi K2.5 models. +- **Guided-Decoding Input Bounds:** Added byte-length and nesting-depth caps to OpenAI guided-decoding input validation (#8349) — `guided_grammar` 64 KiB, `guided_regex` 32 KiB, `guided_whitespace_pattern` 1 KiB, `guided_json` 256 KiB serialized with a nesting-depth cap of 64 — bounding pathological inputs before they reach the downstream guided-decoding backend. + +**Full Changelog**: [https://github.com/ai-dynamo/dynamo/compare/v1.0.1...v1.0.2](https://github.com/ai-dynamo/dynamo/compare/v1.0.1...v1.0.2) + + + +Between v0.9.0 and v1.0.0, we merged over 700 commits from over 90 contributors — 34 first-time contributors and 19 external contributors from 12 organizations. + +**First-Time External Contributors** + +- **@devivasudevan** (Microsoft) contributed a PR that adds Azure AKS storage guidance for Dynamo caches (#5581). +- **@maljazaery** (Microsoft) contributed a PR that clarifies DGDSA creation for services is disabled by default (#6389). +- **@dsocek** (Intel) contributed a PR that improves multimodal disaggregation reliability (#5895). +- **@muskansh-google** (Google) contributed a PR that updates build commands for the Dynamo + SGLang container (#5908). +- **@InfraWhisperer** (F5) contributed a PR that fixes a frontend crash when using the TRT-LLM runtime image (#6481). +- **@Kaonael** (Gcore) contributed a PR that adds a status state enum to DynamoGraphDeployment for improved lifecycle tracking (#6324). +- **@Ryan-Amirthan** (Fern) contributed a PR that adds standard NVIDIA Fern styling assets to the documentation site (#6148). +- **@bledden** (Facilitair) contributed a PR that forwards `stream_options` through the multimodal request pipeline (#6474). +- **@advpropsys** (WhiteCircle.ai) contributed a PR that reduces NATS consumer inactive threshold from 1 hour to 2 minutes to prevent stale connections (#5861). +- **@luc-hiverge** (Hiverge) contributed a PR that fixes first token creation signal timing by emitting the signal after sleeping (#5681). +- **@orangeng** contributed a PR that fixes the service name in port-forward documentation (#5527). +- **@huitianbai** contributed a PR that limits bootstrap room ID range to 0–2^63-1 to prevent overflow (#6277). + +**First-Time NVIDIA Contributors:** + +- **@knowicki-nvidia** contributed a PR that adds image diffusion and text-to-image support for the SGLang backend (#5609). +- **@akshatha-k** contributed a PR that restructures KVBM documentation into a three-tier format (#5905). +- **@alexanderbilk** contributed a PR that adds a Prometheus port for NIXL telemetry metrics (#5567). +- **@rwipfelnv** contributed a PR that adds Grafana dashboard and monitoring setup for observability (#4639). +- **@mikwieczorek** contributed a PR that fixes TRT-LLM recipe component type from "main" to "worker" (#5788). +- **@jpohl-nv** contributed a PR that adds experimental MJPEG video streaming via `/v1/videos/stream` (#6487). +- **@rafiw** contributed a PR that adds Triton path environment variables to the vLLM runtime Dockerfile (#6401). + +**Returning External Contributors:** @michaelfeil (Baseten), @vladnosiv (Yandex.Cloud), @Jont828 (Microsoft), @ashnamehrotra (Microsoft), @ls-2018, @AmeenP (PrimeIntellect), @kerthcet (InftyAI/Hiverge). + +>If you would like to get involved, please see our [Contribution Guide](https://docs.nvidia.com/dynamo/dev/getting-started/contribution-guide) + + diff --git a/docs/fern/reference/release-notes/v1-1-0.mdx b/docs/fern/reference/release-notes/v1-1-0.mdx new file mode 100644 index 000000000000..fe5507ae7d14 --- /dev/null +++ b/docs/fern/reference/release-notes/v1-1-0.mdx @@ -0,0 +1,490 @@ +--- +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +title: Dynamo v1.1.0 +subtitle: Release notes for Dynamo v1.1.0 (GA May 1, 2026), including patch release v1.1.1 +--- + +import { ReferenceStyles } from "@/components/ReferenceStyles"; +import { ReleaseHeader } from "@/components/ReleaseHeader"; +import { ReleaseSummaryCards } from "@/components/ReleaseSummaryCards"; + + + + + +Dynamo v1.1.0 is the 14th feature release of the open-source distributed inference platform. It makes the standalone KV indexer recoverable across node failures, brings the Anthropic Messages API to production for Claude Code, lands SGLang multimodal disaggregated serving, and turns the Mocker into a unified performance-modeling and offline-replay engine. + + +Breaking changes and deprecations for this release are tracked on the [Deprecations](../deprecations.mdx#v110) ledger; known issues on the [Known Issues](../known-issues.mdx#v110) page. Key dependency pins live on [Compatibility](../compatibility.mdx); shipped artifacts on [Release Artifacts](../release-artifacts.mdx). Model early access builds (`vX.Y.Z--dev.N`) are tracked in [Model Early Access Builds](../model-early-access-builds.mdx). + + +## Highlights + + + +## Features & Improvements + +### Multimodal & Diffusion + +#### Embedding Cache & E/P/D + +- **SGLang Embedding Cache:** Added an SGLang embedding cache for cross-request reuse ([#7674](https://github.com/ai-dynamo/dynamo/pull/7674)). +- **NIXL WRITE for Embedding Transfer:** Added a NIXL WRITE initiation path for cross-node embedding transfer ([#6651](https://github.com/ai-dynamo/dynamo/pull/6651)). +- **SGLang Embedding Transfer:** Added `NixlEmbeddingSender`/`NixlEmbeddingReceiver` in the SGLang backend for cross-worker multimodal embedding transfer ([#7153](https://github.com/ai-dynamo/dynamo/pull/7153)) and NVTX markers for SGLang EPD profiling ([#7079](https://github.com/ai-dynamo/dynamo/pull/7079)). +- **vLLM Connector Modernization:** Replaced the v1.0.0 vLLM `PersistentConnector` monkey-patch with a proper `nixl_connector` integration ([#6913](https://github.com/ai-dynamo/dynamo/pull/6913)) and retired the aggregated-embedding-cache patches ([#7799](https://github.com/ai-dynamo/dynamo/pull/7799)). +- **vLLM E/P/D Refactor:** Refactored vLLM E/P/D worker init into a factory and abstracted embedding loading into `ImageLoader` for cleaner per-worker bootstrap and pluggable image sources ([#7367](https://github.com/ai-dynamo/dynamo/pull/7367), [#7482](https://github.com/ai-dynamo/dynamo/pull/7482), [#7507](https://github.com/ai-dynamo/dynamo/pull/7507)). +- **Device-Aware EPD Routing:** Added device-type-and-load EPD routing to balance multimodal requests across heterogeneous EPD pools ([#7215](https://github.com/ai-dynamo/dynamo/pull/7215)). +- **Embedding Cache Hit-Rate Metric:** Exposed multimodal embedding cache hit-rate via Prometheus for tuning cache sizing in production ([#8031](https://github.com/ai-dynamo/dynamo/pull/8031)). + +#### Model Coverage + +- **SGLang Multi-Image Requests:** Added multi-image-per-request support on the SGLang backend ([#6068](https://github.com/ai-dynamo/dynamo/pull/6068)). +- **Qwen3-VL Multimodal Expansion:** Expanded Qwen3-VL multimodal support across vLLM to cover the broader Qwen3-VL model family ([#6163](https://github.com/ai-dynamo/dynamo/pull/6163)). +- **Qwen VL Grid for P/D Disagg:** Compute Qwen VL `grid_thw` from PIL images for P/D disaggregation ([#7885](https://github.com/ai-dynamo/dynamo/pull/7885)). + +#### Diffusion & Video + +- **Image-to-Video on vLLM Omni:** Added image-to-video ([#6530](https://github.com/ai-dynamo/dynamo/pull/6530)), a vLLM Omni CLI ([#6788](https://github.com/ai-dynamo/dynamo/pull/6788)), a disagg path ([#7409](https://github.com/ai-dynamo/dynamo/pull/7409)), and end-to-end tests ([#7454](https://github.com/ai-dynamo/dynamo/pull/7454)). +- **vLLM Video Backend Promotion:** Promoted vLLM video support from example to backend ([#7663](https://github.com/ai-dynamo/dynamo/pull/7663)). +- **SGLang Diffusion:** Added image-to-image generation ([#7870](https://github.com/ai-dynamo/dynamo/pull/7870)) and video input for SGLang aggregated serving ([#7941](https://github.com/ai-dynamo/dynamo/pull/7941)). +- **Audio/TTS on vLLM Omni:** Initial audio/TTS pipeline on vLLM Omni ([#7495](https://github.com/ai-dynamo/dynamo/pull/7495)). +- **Audio-in-Video Pipelines:** Added audio-in-video pipeline support ([#8150](https://github.com/ai-dynamo/dynamo/pull/8150)). +- **TRT-LLM Video Diffusion API Alignment:** Aligned the TRT-LLM video diffusion pipeline with 1.3.0rc9 API changes ([#7529](https://github.com/ai-dynamo/dynamo/pull/7529)). +- **FastVideo Example:** Shipped a FastVideo example with a guide and sidebar reorganization ([#7283](https://github.com/ai-dynamo/dynamo/pull/7283)). +- **XPU Aggregated-Video Example:** Added an XPU aggregated-video example ([#7855](https://github.com/ai-dynamo/dynamo/pull/7855)). +- **Flux Benchmarking:** Added Flux benchmarking to track image-diffusion throughput across releases ([#8083](https://github.com/ai-dynamo/dynamo/pull/8083)). +- **GPUDirect for Intel XPU:** Added GPUDirect support for Intel XPU ([#5852](https://github.com/ai-dynamo/dynamo/pull/5852)) and enabled the Intel XPU Dockerfile for dev and runtime targets ([#6109](https://github.com/ai-dynamo/dynamo/pull/6109), [#7134](https://github.com/ai-dynamo/dynamo/pull/7134)). + +#### Plumbing & Observability + +- **Multimodal URL Forwarding:** Forward multimodal URLs from Python preprocessors to backend workers to keep media decoding on the worker side ([#7837](https://github.com/ai-dynamo/dynamo/pull/7837)), and strip inline base64 from TCP transport in frontend decoding to cut request payload sizes ([#7895](https://github.com/ai-dynamo/dynamo/pull/7895)). +- **Omni Post-Processing Formatters:** Moved Omni post-processing to dedicated formatters and processors for cleaner separation between engine output and response shaping ([#7746](https://github.com/ai-dynamo/dynamo/pull/7746)). +- **OTEL Multimodal Trace Context:** Propagated OpenTelemetry trace context across E/P/D multimodal workers ([#7239](https://github.com/ai-dynamo/dynamo/pull/7239)) and TRT-LLM E/P/D workers ([#7592](https://github.com/ai-dynamo/dynamo/pull/7592)). +- **Multimodal Sweep Tooling:** Added a `request_rate`-based multimodal sweep for throughput-vs-latency curves ([#7363](https://github.com/ai-dynamo/dynamo/pull/7363)), seed support to the multimodal sweep and JSONL generator for reproducible runs ([#7995](https://github.com/ai-dynamo/dynamo/pull/7995)), and a generalized multimodal model coverage test framework so new multimodal models inherit a baseline test suite ([#7975](https://github.com/ai-dynamo/dynamo/pull/7975)). + +### Frontend & Agents + +#### Anthropic Messages API + +- **Anthropic `cache_control` Coverage:** Full `cache_control` support at top-level, per-block, and system-block-array forms for prompt-cache parity with Anthropic Messages API ([#6629](https://github.com/ai-dynamo/dynamo/pull/6629)). +- **Anthropic Thinking Blocks:** Added thinking-block support and system-prompt preamble stripping for `/v1/messages` ([#7137](https://github.com/ai-dynamo/dynamo/pull/7137)). +- **Anthropic Vision Input:** Convert Anthropic `image` blocks to OpenAI `image_url` parts with data URIs so multimodal Anthropic requests reach OpenAI-shaped backends ([#7256](https://github.com/ai-dynamo/dynamo/pull/7256)). +- **Anthropic Streaming Tokens & `/v1/models`:** Accurate streaming `input_tokens`, `/v1/models` `context_window`, and the OpenAI model-retrieve endpoint so Claude Code and other clients see correct token counts and model metadata ([#7234](https://github.com/ai-dynamo/dynamo/pull/7234)). +- **Anthropic Streaming Fix:** Fixed streaming double-parsing and `reasoning_content` round-tripping for clean reasoning streams to Claude Code ([#7358](https://github.com/ai-dynamo/dynamo/pull/7358)). + +#### Tokenizers + +- **`fastokens` BPE Backend:** Integrated `fastokens` as a hybrid BPE encoder selectable via `DYN_TOKENIZER=fastokens` for faster BPE tokenization on long prompts ([#7387](https://github.com/ai-dynamo/dynamo/pull/7387)), with benchmarks and docs ([#7388](https://github.com/ai-dynamo/dynamo/pull/7388)). +- **`chat_template.json` Support:** Added support for `chat_template.json` as a prompt-formatter artifact so models that ship a chat template alongside the tokenizer load correctly ([#7785](https://github.com/ai-dynamo/dynamo/pull/7785)), and for passing a local tokenizer path for offline and air-gapped deployments ([#8128](https://github.com/ai-dynamo/dynamo/pull/8128)). +- **Tokenizer Kind Enum:** Converted the tokenizer kind to a strongly typed enum to catch invalid tokenizer kinds at parse time ([#8022](https://github.com/ai-dynamo/dynamo/pull/8022)). +- **Kimi K2.5 Tokenizer Fixes:** Resolved Kimi K2.5 tiktoken multi-byte sequence handling and reserved-token fallback naming so K2.5 prompts no longer panic on multi-byte boundaries ([#6996](https://github.com/ai-dynamo/dynamo/pull/6996), [#7886](https://github.com/ai-dynamo/dynamo/pull/7886)). +- **DeepSeek V3 BPE Reuse:** Allowed DeepSeek V3 architectures to reuse Kimi's BPE pattern to enable DeepSeek V3 tokenization without a separate code path ([#6653](https://github.com/ai-dynamo/dynamo/pull/6653)). +- **GLM-5 Tokenizer Loading:** Fixed GLM-5 custom tokenizer loading ([#8079](https://github.com/ai-dynamo/dynamo/pull/8079)). +- **Qwen3.5 `eos_token_ids`:** Fixed `eos_token_ids` handling for Qwen3.5 ([#8091](https://github.com/ai-dynamo/dynamo/pull/8091)). +- **Null-Byte Prompt Rejection:** Stripped null bytes before tokenization to fix prompt-rejection divergence between Rust and Python tokenizers ([#7694](https://github.com/ai-dynamo/dynamo/pull/7694)). +- **Generic Tokenizer Manager Passthrough:** Added a generic `tokenizer_manager` passthrough route for RL training and other off-path tokenizer consumers ([#6836](https://github.com/ai-dynamo/dynamo/pull/6836)). + +#### Tool Calling & Reasoning + +- **Structured Streaming SSE:** Added structured streaming SSE events for tool calls and reasoning content ([#7114](https://github.com/ai-dynamo/dynamo/pull/7114)). +- **Reasoning + Tool Mid-Chunk:** Preserved reasoning content when a tool call starts mid-chunk so streaming clients see both segments ([#6902](https://github.com/ai-dynamo/dynamo/pull/6902)). +- **`tool_choice=required` Routing:** Routed `tool_choice=required` through the format-specific parser path so required tool calls honor the model's parser instead of the default ([#7589](https://github.com/ai-dynamo/dynamo/pull/7589)). +- **Tool-Call Speculative Decoding Fix:** Fixed tool-call loss under speculative decoding ([#7768](https://github.com/ai-dynamo/dynamo/pull/7768)). +- **LoRA Parser Inheritance:** Inherited tool-call and reasoning parsers when registering LoRA models so LoRA-tuned variants reuse the base model's tool/reasoning parsers ([#7559](https://github.com/ai-dynamo/dynamo/pull/7559)). +- **Dict-Format Tool Args:** Accepted dict-format tool-call arguments in `async-openai` for compatibility with clients that send tool args as objects rather than JSON strings ([#7772](https://github.com/ai-dynamo/dynamo/pull/7772)). +- **Qwen3Coder `anyOf`/`oneOf`:** Handled `anyOf`/`oneOf` parameter schemas in the Qwen3Coder tool parser ([#7847](https://github.com/ai-dynamo/dynamo/pull/7847)). +- **Tool Parser Log Quieting:** Downgraded noisy tool-parser INFO logs to DEBUG to reduce log volume in production frontends ([#7573](https://github.com/ai-dynamo/dynamo/pull/7573)). + +#### Backend Options + +- **SGLang Guided Decoding:** Added SGLang guided-decoding support ([#6620](https://github.com/ai-dynamo/dynamo/pull/6620)). +- **vLLM Chat Processor Flags:** Allowed passing vLLM chat processor-specific flags through the frontend so vLLM-specific options don't have to be configured separately ([#7896](https://github.com/ai-dynamo/dynamo/pull/7896)). + +#### Observability & Lifecycle + +- **Request Lifecycle Logging:** Added end-to-end request-lifecycle logging and metrics capture for end-to-end observability of request paths ([#7840](https://github.com/ai-dynamo/dynamo/pull/7840)). +- **Request Rejection Metrics:** Added request rejection metrics ([#7644](https://github.com/ai-dynamo/dynamo/pull/7644)) and frontend/runtime request cancellation metrics ([#7493](https://github.com/ai-dynamo/dynamo/pull/7493)) to surface client-side cancellations and admission rejections in dashboards. +- **Unified `on_response` and Request ID:** Unified `on_response`, renamed `request_id` fields, and deprecated the `x-dynamo-request-id` header for a single canonical request identity across the request plane ([#7834](https://github.com/ai-dynamo/dynamo/pull/7834)). + +### Kubernetes Deployment + +#### Operator Self-Sufficiency + +- **CRD Apply via Init Container:** Moved CRD apply from a Helm hook Job into an init container on the operator Deployment so CRDs reconcile on every operator restart instead of only at install time ([#6780](https://github.com/ai-dynamo/dynamo/pull/6780)). +- **Webhook Cert Management in Operator:** Moved webhook certificate management and CA injection from Helm hooks into the operator for self-rotating certs without Helm post-install jobs ([#6839](https://github.com/ai-dynamo/dynamo/pull/6839)). +- **MPI SSH Key in Reconciliation:** Moved MPI SSH key generation from a Helm hook Job into operator reconciliation so multinode workers get keys without a separate install hook ([#6940](https://github.com/ai-dynamo/dynamo/pull/6940)). +- **`metrics-reader` Cleanup:** Removed the leftover `metrics-reader` ClusterRole to drop a stale RBAC binding ([#7107](https://github.com/ai-dynamo/dynamo/pull/7107)). +- **Operator Helm `env` Support:** Added `env` support on the operator Helm chart for image-side configuration ([#7081](https://github.com/ai-dynamo/dynamo/pull/7081)). + +#### GPU Memory Service & Failover + +- **GMS Failover API:** Operator `failover` API runs an active/standby pair sharing GPUs and the GMS sidecar so the standby can take over via flock when the active engine fails ([#8157](https://github.com/ai-dynamo/dynamo/pull/8157)). +- **Operator-Managed GMS Checkpoint/Restore:** Added operator-managed GMS checkpoint/restore with a shared runtime helper that keeps the snapshot layer GMS-unaware ([#8153](https://github.com/ai-dynamo/dynamo/pull/8153), [#8194](https://github.com/ai-dynamo/dynamo/pull/8194)). +- **GMS in Snapshot Subsystem:** Wired GMS into the snapshot subsystem to drive memory release and restore through the snapshot lifecycle ([#7026](https://github.com/ai-dynamo/dynamo/pull/7026)). +- **TRT-LLM Sleep/Wake with GMS:** Integrated TRT-LLM sleep/wake with GMS so TRT-LLM workers can release and reclaim GPU memory under GMS control ([#7575](https://github.com/ai-dynamo/dynamo/pull/7575)). +- **Single-Handle Allocation:** Exported a single shareable handle per allocation and reused it for later export RPCs to cut allocation overhead in GMS sidecars ([#8108](https://github.com/ai-dynamo/dynamo/pull/8108)). + +#### Dynamo Snapshot + +- **Renamed from `chrek`:** Renamed Dynamo Snapshot from `chrek` as part of the v1.1.0 productization ([#7028](https://github.com/ai-dynamo/dynamo/pull/7028)). +- **x86_64-Only with Backend Docs:** Restricted to x86_64 with explicit backend-support docs to set clear platform expectations ([#7031](https://github.com/ai-dynamo/dynamo/pull/7031)). +- **Manifest-Based `snapshotctl`:** Introduced a manifest-based `snapshotctl` flow with shared workload builders for repeatable declarative snapshot/restore operations ([#7671](https://github.com/ai-dynamo/dynamo/pull/7671)) and watch-based waits ([#8024](https://github.com/ai-dynamo/dynamo/pull/8024)). +- **File-Sentinel IPC Contract:** Replaced the PID-1 `SIGUSR1`/`SIGCONT` contract with file sentinels for safer container-PID-1 coordination ([#8431](https://github.com/ai-dynamo/dynamo/pull/8431), cherry-pick of [#8403](https://github.com/ai-dynamo/dynamo/pull/8403)). +- **CUDA Namespace PID Persistence:** Persisted CUDA namespace PIDs in the manifest so restores can rebuild CUDA namespaces correctly ([#7539](https://github.com/ai-dynamo/dynamo/pull/7539)). +- **DRA GPU UUID Resolution:** Resolved DRA GPU UUIDs from Kubernetes claims so checkpoints capture the GPU identity used at runtime ([#8425](https://github.com/ai-dynamo/dynamo/pull/8425)). +- **Versioned Snapshot Artifacts:** Versioned snapshot artifacts on readable checkpoints for forward-compatible snapshot upgrades ([#7533](https://github.com/ai-dynamo/dynamo/pull/7533)). +- **Tighter Operator Boundary:** Tightened the operator protocol boundary to keep operator and runtime concerns cleanly separated ([#8018](https://github.com/ai-dynamo/dynamo/pull/8018)). +- **CRIU `criu-dev` Default:** Defaulted CRIU builds to upstream `criu-dev` to track the upstream branch we depend on ([#7744](https://github.com/ai-dynamo/dynamo/pull/7744)). +- **Checkpoint/Restore Timing:** Added checkpoint and restore timing summaries for visibility into snapshot performance ([#7949](https://github.com/ai-dynamo/dynamo/pull/7949)). +- **Snapshot CI Build:** Added a snapshot CI build to keep the subsystem green pre-merge ([#8159](https://github.com/ai-dynamo/dynamo/pull/8159)). + +#### DGD/DGDR Lifecycle + +- **MDC Checksum Per-WorkerSet:** Scoped Model Discovery Card checksum validation from per-Model to per-WorkerSet so different WorkerSets under the same Model can carry different configuration without draining workers first ([#7368](https://github.com/ai-dynamo/dynamo/pull/7368)). +- **Foreground Cascade-Delete Stability:** Prevented operator thrashing under foreground cascading deletion of DGDs ([#8207](https://github.com/ai-dynamo/dynamo/pull/8207)). +- **No Orphaned Worker DCDs:** Prevented orphaned old worker DCDs after a rolling update ([#7939](https://github.com/ai-dynamo/dynamo/pull/7939)). +- **DGDR ConfigMap Owner Refs:** Added Kubernetes owner references to DGDR-created ConfigMaps so they cascade-delete with their parent ([#7782](https://github.com/ai-dynamo/dynamo/pull/7782)). +- **Unique DGD Names per DGDR:** Used unique DGD names following the DGDR to avoid collisions across rolling DGDRs ([#7778](https://github.com/ai-dynamo/dynamo/pull/7778)). +- **DNS-1035 Compliance:** Sanitized dots in DGD names for DNS-1035 compliance ([#7032](https://github.com/ai-dynamo/dynamo/pull/7032)). +- **`model_name` Case Normalization:** Normalized `model_name` case in KubernetesConnector comparisons so case-mismatched DGDs no longer cause spurious reconciliation diffs ([#8401](https://github.com/ai-dynamo/dynamo/pull/8401), cherry-pick of [#8384](https://github.com/ai-dynamo/dynamo/pull/8384)). +- **Spec-Level Annotation/Label Propagation:** Propagated DGD spec-level annotations and labels to child resources so cluster-level metadata reaches every child ([#7326](https://github.com/ai-dynamo/dynamo/pull/7326)). +- **`SyncResource` Spec-Less Handling:** Handled spec-less resources in `SyncResource` to keep sync stable across resource types that omit `spec` ([#7953](https://github.com/ai-dynamo/dynamo/pull/7953)). +- **Webhook SA Allow-List:** Replaced hardcoded SA suffix matching with a config-driven allow-list in the DGD-replicas webhook so cluster admins can authorize new ServiceAccount patterns without a code change ([#7682](https://github.com/ai-dynamo/dynamo/pull/7682)). +- **`wait-for-leader` ConfigMap:** Used a ConfigMap for the vLLM multinode `wait-for-leader` script so operators can patch the script without rebuilding the image ([#7954](https://github.com/ai-dynamo/dynamo/pull/7954)). +- **GPU Resource Requests Recognition:** Recognized GPU resources in `requests` rather than only `limits` for compatibility with schedulers that set GPU `requests` ([#8005](https://github.com/ai-dynamo/dynamo/pull/8005)). + +#### Inference Gateway (GAIE) + +- **EPP Worker Discovery Mode:** Enabled EPP worker discovery mode for the Gateway API Inference Endpoint so the Endpoint Picker can locate Dynamo workers via the GAIE API ([#6592](https://github.com/ai-dynamo/dynamo/pull/6592)). +- **Data-Parallel Routing through GAIE:** Enabled data-parallel routing through GAIE so DP-sharded backends are reachable behind the gateway ([#7741](https://github.com/ai-dynamo/dynamo/pull/7741)). +- **EPP Prefill Race Fix:** Fixed a race condition in EPP prefill ([#7530](https://github.com/ai-dynamo/dynamo/pull/7530)). +- **GAIE Test Coverage:** Added GAIE unit and e2e integration tests ([#7257](https://github.com/ai-dynamo/dynamo/pull/7257)) plus a nightly integration test ([#7458](https://github.com/ai-dynamo/dynamo/pull/7458)) to lock the gateway integration in CI. +- **vCluster in Post-Merge GAIE:** Supported vCluster in post-merge GAIE testing for isolated multi-tenant test runs ([#7743](https://github.com/ai-dynamo/dynamo/pull/7743)). + +### Scheduling + +#### KV Router + +- **Maturin-Built `dynamo-kv-indexer`:** Packaged the indexer as a maturin-built binary with a Python launcher for a standalone, pip-installable indexer ([#7194](https://github.com/ai-dynamo/dynamo/pull/7194), [#7338](https://github.com/ai-dynamo/dynamo/pull/7338), [#7395](https://github.com/ai-dynamo/dynamo/pull/7395)). +- **Standalone KV Indexer Bootstrap:** Added P2P recovery so a new or restarted replica bootstraps its radix-tree state from a healthy peer's `/dump` endpoint before serving queries ([#6934](https://github.com/ai-dynamo/dynamo/pull/6934), [#7596](https://github.com/ai-dynamo/dynamo/pull/7596)). +- **Inline ZMQ Gap Detection & Replay:** Recovered dropped messages from the engine's ring buffer via inline gap detection and replay ([#7209](https://github.com/ai-dynamo/dynamo/pull/7209)). +- **Multi-Model / Multi-Tenant Isolation:** Added multi-model and multi-tenant isolation in the standalone indexer so a single indexer can serve multiple models without cross-tenant cache reuse ([#6830](https://github.com/ai-dynamo/dynamo/pull/6830)). +- **Discovery-Based Worker Management:** Runtime integration with discovery-based worker management and a `Remote` indexer variant so the router learns workers via discovery instead of static config ([#7295](https://github.com/ai-dynamo/dynamo/pull/7295), [#7973](https://github.com/ai-dynamo/dynamo/pull/7973)). +- **Indexer Metrics & Health:** Exposed Prometheus metrics and `/health` endpoints on the standalone indexer for production observability and load-balancer probes ([#7339](https://github.com/ai-dynamo/dynamo/pull/7339)). +- **Monolithic Indexer Split:** Split the monolithic indexer into smaller crates for cleaner ownership and faster compile times ([#6870](https://github.com/ai-dynamo/dynamo/pull/6870), [#7871](https://github.com/ai-dynamo/dynamo/pull/7871)). +- **Power-of-Two-Choices Router:** Added a power-of-two-choices router mode for low-overhead load balancing under heavy fan-out ([#7614](https://github.com/ai-dynamo/dynamo/pull/7614)). +- **Pluggable Router Queue Policy:** Added a pluggable scheduling policy for the router queue so deployments can swap FIFO for fairness or priority ([#7260](https://github.com/ai-dynamo/dynamo/pull/7260)). +- **Priority-Based Pool Routing:** Added priority-based pool routing in the global router that consumes `nvext.agent_hints.priority` and overrides 2D grid-based pool selection on a first-matching-rule basis with full backward compatibility ([#8010](https://github.com/ai-dynamo/dynamo/pull/8010)). +- **Per-Media-Type Worker Placements:** Added per-media-type worker-side placements so multimodal requests prefer workers warm for the same media type ([#7462](https://github.com/ai-dynamo/dynamo/pull/7462)). +- **Compressed Radix Tree:** Added an internal radix tree that compresses single-token chains for faster prefix matching ([#7459](https://github.com/ai-dynamo/dynamo/pull/7459)). +- **Compressed Tree as Default:** Switched to the compressed concurrent radix tree by default so all deployments benefit from the smaller, faster tree ([#7874](https://github.com/ai-dynamo/dynamo/pull/7874)). +- **Compressed-Tree Size Accounting:** Fixed compressed-tree-size accounting so reported tree size matches the real footprint ([#7800](https://github.com/ai-dynamo/dynamo/pull/7800)). +- **170M ops/s Linear-Scan Path:** Added the linear-scan perf path that reaches 170M ops/s ([#6363](https://github.com/ai-dynamo/dynamo/pull/6363)). +- **Concurrent KV Event Consumer:** Added a concurrent KV event consumer to keep up with high-throughput event streams ([#7293](https://github.com/ai-dynamo/dynamo/pull/7293)). +- **Client-Side Event Batching:** Added dynamic batching of client-side events to reduce per-event overhead under burst traffic ([#6733](https://github.com/ai-dynamo/dynamo/pull/6733), [#6741](https://github.com/ai-dynamo/dynamo/pull/6741)). +- **vLLM KV Block Event Dedup:** Added a dedup filter for duplicate vLLM KV block events to prevent double-accounting under upstream replay ([#8012](https://github.com/ai-dynamo/dynamo/pull/8012)). +- **Lazy Worker Registration:** Added lazy worker registration in the slot tracker so workers register only when first scheduled ([#7795](https://github.com/ai-dynamo/dynamo/pull/7795)). +- **Decode Scoring for Unregistered Workers:** Corrected decode-worker scoring for unregistered workers so scoring no longer mis-counts workers that aren't yet visible ([#7919](https://github.com/ai-dynamo/dynamo/pull/7919)). +- **Watermark Cleanup on Deregister:** Cleaned up watermarks on worker deregister to keep watermark state correct under churn ([#7863](https://github.com/ai-dynamo/dynamo/pull/7863)). +- **Unit-Block-Size Handling:** Fixed unit-block-size handling for correct accounting at unit block size ([#8405](https://github.com/ai-dynamo/dynamo/pull/8405)). +- **Queue-Depth Metric & `nvext` Field:** Exposed a queue-depth Prometheus metric and matching `nvext` field for queue-aware client-side routing ([#6786](https://github.com/ai-dynamo/dynamo/pull/6786)). +- **Queue ISL-Token Metrics:** Added queue ISL-token metrics for length-aware queue diagnostics ([#8136](https://github.com/ai-dynamo/dynamo/pull/8136)). +- **`ActiveSequences` Benchmark:** Added an `ActiveSequences` benchmark to track router-side active-request scaling under load ([#6633](https://github.com/ai-dynamo/dynamo/pull/6633)). +- **Router Perf Improvements:** Miscellaneous router performance improvements ([#7477](https://github.com/ai-dynamo/dynamo/pull/7477)). + +#### Planner & Profiler + +- **FPM-Based Regression Unification:** Replaced the dual performance-modeling layer (scipy interpolators for throughput scaling, sklearn regression for load scaling) with a single FPM-based regression model that serves both modes, enabling throughput scaling for aggregated mode ([#7961](https://github.com/ai-dynamo/dynamo/pull/7961)). +- **Discrete-Event Control Loop:** Extracted the planner control loop into an explicit discrete-event state machine for inspectable, deterministic scaling decisions ([#8046](https://github.com/ai-dynamo/dynamo/pull/8046)). +- **Planner Subpackage Restructure:** Restructured the planner package into a subpackage hierarchy for cleaner module boundaries between scaling, profiling, and policy code ([#7689](https://github.com/ai-dynamo/dynamo/pull/7689)). +- **Standalone `dynamo-planner` Image:** Shipped a standalone `dynamo-planner` image and removed planner deps from non-planner images to keep worker images lean ([#7696](https://github.com/ai-dynamo/dynamo/pull/7696), [#7748](https://github.com/ai-dynamo/dynamo/pull/7748)). +- **Planner CI with arm64:** Added a planner CI pipeline with arm64 builds for ARM-architecture coverage ([#7956](https://github.com/ai-dynamo/dynamo/pull/7956)). +- **GlobalPlanner GPU Budget:** Added GlobalPlanner `--max-total-gpus` for cluster-wide GPU budgeting ([#7103](https://github.com/ai-dynamo/dynamo/pull/7103)). +- **Dynamic `--trtllm.*` Config Modifier:** Switched the TRT-LLM config modifier to dynamic `--trtllm.*` flags so new TRT-LLM options propagate without planner code changes ([#7884](https://github.com/ai-dynamo/dynamo/pull/7884)). +- **GPU-VRAM Memory-Fraction Injection:** Added GPU-VRAM profiler memory-fraction injection with profiled test markers for SGLang and vLLM to right-size memory budgets per backend during profiling ([#6719](https://github.com/ai-dynamo/dynamo/pull/6719), [#7508](https://github.com/ai-dynamo/dynamo/pull/7508)). +- **Profiler Job Overrides:** Added profiler job overrides so users can pin specific profiler runs without editing the base config ([#6607](https://github.com/ai-dynamo/dynamo/pull/6607)). +- **Wait-for-Scaling Hang Fix:** Fixed a wait-for-scaling-completion hang ([#8008](https://github.com/ai-dynamo/dynamo/pull/8008)). +- **Planner Test Isolation:** Isolated planner test suites to keep planner tests from interfering with the rest of the suite ([#7723](https://github.com/ai-dynamo/dynamo/pull/7723)). + +### KV Block Manager + +#### Storage & Allocation + +- **KVBM Disk Allocation Variants:** Added KVBM disk allocation for different storage classes including local-SSD and shared-volume tiers ([#7839](https://github.com/ai-dynamo/dynamo/pull/7839)). +- **`PinnedAllocator` Device ID:** Passed `device_id` into the `PinnedAllocator` instead of hardcoding 0 so multi-GPU workers allocate pinned memory on the right device ([#6809](https://github.com/ai-dynamo/dynamo/pull/6809)). +- **`CUDA_VISIBLE_DEVICES` for NUMA:** Respected `CUDA_VISIBLE_DEVICES` for NUMA binding ([#6931](https://github.com/ai-dynamo/dynamo/pull/6931)). +- **MAX_CONCURRENT_TRANSFERS Knob:** Added a `MAX_CONCURRENT_TRANSFERS` env var to throttle outbound KV transfer concurrency on the offload path ([#7527](https://github.com/ai-dynamo/dynamo/pull/7527)). + +#### NCCL Integration + +- **KVBM `nccl` Wheel Feature:** Built KVBM wheels with the `nccl` feature for CUDA containers ([#8120](https://github.com/ai-dynamo/dynamo/pull/8120)). +- **Runtime NCCL Version Query:** Queried the NCCL version at runtime instead of hardcoding so KVBM tracks whatever NCCL ships in the runtime image ([#8130](https://github.com/ai-dynamo/dynamo/pull/8130)). +- **CUDA Device Per Rank:** Set the CUDA device per rank before `ncclCommInitRank` to avoid cross-rank device-binding bugs in NCCL init ([#8147](https://github.com/ai-dynamo/dynamo/pull/8147)). + +#### MLA, Consolidator & Tests + +- **MLA Support for DeepSeek-V2:** Added MLA support to KVBM for DeepSeek-V2-style attention ([#7786](https://github.com/ai-dynamo/dynamo/pull/7786)). +- **KVBM MLA Optimization:** Added KVBM MLA optimization for faster MLA-attention KV ops ([#7015](https://github.com/ai-dynamo/dynamo/pull/7015)). +- **Typed `KvCacheConnectorConfig`:** Handled the typed `KvCacheConnectorConfig` in the consolidator so the consolidator parses connector config without ad-hoc dict handling ([#8117](https://github.com/ai-dynamo/dynamo/pull/8117)). +- **Consolidator e2e Tests:** Fixed consolidator e2e tests to keep consolidator coverage green pre-merge ([#8503](https://github.com/ai-dynamo/dynamo/pull/8503)). +- **Disagg Determinism on GB200:** Stabilized the disagg determinism test on GB200 by setting the KV block size to 32 ([#6980](https://github.com/ai-dynamo/dynamo/pull/6980)). + +### Performance Modeling & Replay + +#### Latency Prediction & Mocker Simulation + +- **AIConfigurator Latency Prediction:** Added AIConfigurator-backed latency prediction with MoE-parallelism support for closed-form latency estimates without running real workloads ([#7505](https://github.com/ai-dynamo/dynamo/pull/7505), [#7856](https://github.com/ai-dynamo/dynamo/pull/7856)). +- **Mocker SGLang Simulation:** Added SGLang engine simulation to the Mocker so the Mocker can model SGLang behaviors offline ([#6977](https://github.com/ai-dynamo/dynamo/pull/6977)). +- **vLLM v1 Scheduler Parity in Mocker:** Aligned the Mocker's vLLM scheduler with v1 semantics — drop watermark, LIFO preemption, retry loop — so simulation matches the v1 scheduler in production ([#7020](https://github.com/ai-dynamo/dynamo/pull/7020)). +- **Speculative Decoding Simulation:** Added `--decode-speedup-ratio` for speculative-decoding simulation ([#7349](https://github.com/ai-dynamo/dynamo/pull/7349)). +- **Mocker FPM Emission:** Forward-pass-metric emission from the Mocker to the event plane for planner consumption ([#8032](https://github.com/ai-dynamo/dynamo/pull/8032)). + +#### Trace Replay + +- **Offline Aggregated & Disaggregated Replay:** Added offline aggregated and disaggregated trace replay including Mooncake-style traces with multi-worker support for repeatable benchmarking against captured traffic ([#7543](https://github.com/ai-dynamo/dynamo/pull/7543), [#7553](https://github.com/ai-dynamo/dynamo/pull/7553), [#7617](https://github.com/ai-dynamo/dynamo/pull/7617), [#7876](https://github.com/ai-dynamo/dynamo/pull/7876)). +- **Shared Loadgen with Multi-Turn:** Shared loadgen and workload paths with multi-turn session support so multi-turn agentic traces replay alongside single-turn workloads ([#7593](https://github.com/ai-dynamo/dynamo/pull/7593)). +- **Planner-in-the-Loop Replay:** Planner-in-the-loop offline replay with dynamic worker pool management to validate planner decisions on captured traffic before deployment ([#8187](https://github.com/ai-dynamo/dynamo/pull/8187)). +- **Dense Replay Optimization:** Dense offline replay optimization for both aggregated and disaggregated topologies for faster offline iteration on large traces ([#7774](https://github.com/ai-dynamo/dynamo/pull/7774)). +- **`replay_optimize` Sweep:** Added a `replay_optimize` sweep README and single-worker soak test to make the optimization sweep reproducible from the docs ([#8195](https://github.com/ai-dynamo/dynamo/pull/8195), [#8234](https://github.com/ai-dynamo/dynamo/pull/8234)). +- **Replay Accounting & Perf Fixes:** Many small replay accounting and performance fixes to keep replay numbers honest under heavy traces ([#6998](https://github.com/ai-dynamo/dynamo/pull/6998), [#7647](https://github.com/ai-dynamo/dynamo/pull/7647), [#7687](https://github.com/ai-dynamo/dynamo/pull/7687), [#7692](https://github.com/ai-dynamo/dynamo/pull/7692), [#7698](https://github.com/ai-dynamo/dynamo/pull/7698), [#7729](https://github.com/ai-dynamo/dynamo/pull/7729), [#7838](https://github.com/ai-dynamo/dynamo/pull/7838), [#7866](https://github.com/ai-dynamo/dynamo/pull/7866), [#7938](https://github.com/ai-dynamo/dynamo/pull/7938), [#8050](https://github.com/ai-dynamo/dynamo/pull/8050), [#8232](https://github.com/ai-dynamo/dynamo/pull/8232)). + +### Infrastructure Modernization + +#### Event & Transport Plane + +- **libzmq Migration Complete:** Finished the libzmq migration and stabilized standalone indexer/replay coverage to land all event and transport code on a single ZMQ binding ([#7871](https://github.com/ai-dynamo/dynamo/pull/7871)). +- **Router ZMQ Port Reservation:** Made router tests reserve contiguous ZMQ ports to eliminate flaky port collisions in router CI ([#7448](https://github.com/ai-dynamo/dynamo/pull/7448)). + +#### Forward-Pass Metrics + +- **vLLM FPM Initial Emission:** Initial ZMQ PUB/SUB emission of forward-pass metrics from the vLLM engine to expose per-step engine state to planner and observability consumers ([#7200](https://github.com/ai-dynamo/dynamo/pull/7200)). +- **vLLM FPM Event-Plane Relay:** Transport-agnostic event-plane relay for vLLM forward-pass metrics, following the same two-layer bridge pattern as KV events ([#7250](https://github.com/ai-dynamo/dynamo/pull/7250)). +- **`ForwardPassMetric` Versioning:** Exposed `inc_id` and a version field on `ForwardPassMetric` so consumers can detect schema upgrades and dropped samples ([#7501](https://github.com/ai-dynamo/dynamo/pull/7501)). +- **FPM Async Scheduling Fix:** Fixed forward-pass-metric emission under async scheduling so async-scheduled workers still emit FPM samples ([#7537](https://github.com/ai-dynamo/dynamo/pull/7537)). +- **FPM `data_parallel_index` Port Offset:** Used `data_parallel_index` for the FPM ZMQ port offset to avoid port collisions between DP ranks on the same node ([#8706](https://github.com/ai-dynamo/dynamo/pull/8706), cherry-pick of [#8696](https://github.com/ai-dynamo/dynamo/pull/8696)). +- **Auto-Injected `worker_id` Label:** Auto-injected a `worker_id` label into all metrics so planner consumers can attribute FPM streams correctly ([#8089](https://github.com/ai-dynamo/dynamo/pull/8089)). + +### Recipes + +#### Kimi K2.5 + +- **Kimi K2.5 Initial Recipe:** Shipped the initial Baseten-based Kimi K2.5 recipe to enable Kimi K2.5 deployment on Dynamo ([#6602](https://github.com/ai-dynamo/dynamo/pull/6602)). +- **Kimi K2.5 NVFP4 Aggregated + KVBM:** Added an aggregated + KVBM recipe with a TRT-LLM patch for `nvidia/Kimi-K2.5-NVFP4` for KVBM-backed NVFP4 deployments ([#6842](https://github.com/ai-dynamo/dynamo/pull/6842)). +- **Kimi K2.5 Speculative Decoding:** Added a performance-optimized speculative-decoding variant for Kimi K2.5 ([#7555](https://github.com/ai-dynamo/dynamo/pull/7555), [#7576](https://github.com/ai-dynamo/dynamo/pull/7576)). +- **Kimi K2.5 Container-Patch Removal:** Removed the container-patch requirement and replaced the KVBM Kimi recipe with a Qwen3 KVBM recipe so the recipe runs on the stock TRT-LLM container ([#8199](https://github.com/ai-dynamo/dynamo/pull/8199), [#8476](https://github.com/ai-dynamo/dynamo/pull/8476)). +- **Kimi K2.5 KVBM Metrics on Kubernetes:** Enabled KVBM metrics for the Kimi K2.5 recipe on Kubernetes so K2.5 deployments expose KVBM telemetry on K8s ([#6963](https://github.com/ai-dynamo/dynamo/pull/6963)). +- **Kimi K2.5 Stabilization Fixes:** Pin reset and patch fixes during Kimi K2.5 stabilization ([#7411](https://github.com/ai-dynamo/dynamo/pull/7411), [#7435](https://github.com/ai-dynamo/dynamo/pull/7435)). +- **Kimi K2.5 Doc Restructuring:** Restructured the Kimi K2.5 recipe documentation for clearer per-variant guidance ([#7412](https://github.com/ai-dynamo/dynamo/pull/7412)). + +#### DeepSeek & Qwen + +- **DeepSeek V3.2 on TRT-LLM:** Added a DeepSeek V3.2 recipe on TensorRT-LLM ([#6688](https://github.com/ai-dynamo/dynamo/pull/6688)). +- **DeepSeek WideEP + Qwen3-235B Recipe Refresh:** Advanced the DeepSeek WideEP and Qwen3-235B recipes to the Dynamo 1.0.1-era TRT-LLM container baseline to keep both recipes runnable on a known-good baseline ([#7479](https://github.com/ai-dynamo/dynamo/pull/7479)); v1.1.0 itself ships TRT-LLM `v1.3.0rc11`. +- **Qwen3-VL-30B Aggregated + Encoder Cache:** Added a Qwen3-VL-30B aggregated recipe with encoder cache on vLLM ([#6919](https://github.com/ai-dynamo/dynamo/pull/6919)). +- **Qwen3-235B DeepGEMM Switch:** Switched the Qwen3-235B recipe to the DeepGEMM backend for higher throughput on Hopper-class GPUs ([#7204](https://github.com/ai-dynamo/dynamo/pull/7204)). + +#### Other Models + +- **GLM-5 NVFP4 on GB200 SGLang:** Added a GLM-5 NVFP4 recipe on GB200 SGLang ([#7780](https://github.com/ai-dynamo/dynamo/pull/7780), [#8098](https://github.com/ai-dynamo/dynamo/pull/8098)). +- **Nemotron-3-Super-FP8:** Added a Nemotron-3-Super-FP8 recipe ([#7216](https://github.com/ai-dynamo/dynamo/pull/7216)). +- **gpt-oss-120b Disaggregated:** Added a gpt-oss-120b disaggregated recipe ([#8133](https://github.com/ai-dynamo/dynamo/pull/8133)). + +--- + +## Bug Fixes + +### Multimodal + +- **TRT-LLM EPD Multimodal Stabilization:** Fixed encoder LLM creation gating, `apply_mm_hashes` 1.3 API alignment, and preprocessor handling for the embeddings case in the TRT-LLM EPD multimodal flow ([#6726](https://github.com/ai-dynamo/dynamo/pull/6726), [#6810](https://github.com/ai-dynamo/dynamo/pull/6810), [#6840](https://github.com/ai-dynamo/dynamo/pull/6840), [#6866](https://github.com/ai-dynamo/dynamo/pull/6866), [#6920](https://github.com/ai-dynamo/dynamo/pull/6920), [#6924](https://github.com/ai-dynamo/dynamo/pull/6924)). +- **LLaVA EPD Path Phase-Out:** Phased out the LLaVA-specific EPD path and constrained EPD to single-GPU ([#6674](https://github.com/ai-dynamo/dynamo/pull/6674)). +- **LLaVA `out_hidden_size`:** Fixed `out_hidden_size` handling for LLaVA in the EPD encode worker ([#6759](https://github.com/ai-dynamo/dynamo/pull/6759)). +- **vLLM-Omni Finish Reason:** Fixed the vLLM-Omni `normalize_finish_reason` call ([#6910](https://github.com/ai-dynamo/dynamo/pull/6910)) and the chat processor for video/audio examples ([#6689](https://github.com/ai-dynamo/dynamo/pull/6689)). +- **Empty Multimodal Input Rejection:** Rejected empty multimodal inputs that triggered invalid-UUID checks ([#6853](https://github.com/ai-dynamo/dynamo/pull/6853)) and local file inputs in the `ImageLoader` ([#8158](https://github.com/ai-dynamo/dynamo/pull/8158)). +- **Modality Mismatch Rejection:** Rejected multimodal requests against workers without `--modality multimodal` ([#7065](https://github.com/ai-dynamo/dynamo/pull/7065)). +- **MM Router Image Dedup:** Skipped duplicate image downloads and unnecessary image processing in the MM Router ([#7080](https://github.com/ai-dynamo/dynamo/pull/7080)). +- **Decode-Side Re-Download Elimination:** Eliminated redundant image re-download on the decode worker in disagg ([#7827](https://github.com/ai-dynamo/dynamo/pull/7827)). +- **Multimodal PD Race Fix:** Fixed a race condition in the multimodal PD worker ([#7679](https://github.com/ai-dynamo/dynamo/pull/7679)) and `ImageLoader` errors ([#7703](https://github.com/ai-dynamo/dynamo/pull/7703)). +- **Dummy-Embedding Value Range:** Restricted the dummy-embedding value range to bypass vLLM checks ([#7117](https://github.com/ai-dynamo/dynamo/pull/7117)). +- **Bounded Media-Fetch Concurrency:** Bounded media-fetch concurrency with decoupled httpx timeouts to prevent fetcher exhaustion ([#8767](https://github.com/ai-dynamo/dynamo/pull/8767)). +- **Vision Model Loader Fixes:** Fixed several vision-model loader paths and the SGLang Eagle bigram tokens KV event report ([#6952](https://github.com/ai-dynamo/dynamo/pull/6952), [#6872](https://github.com/ai-dynamo/dynamo/pull/6872)). +- **Multimodal Media URL Validation:** Added validation on multimodal media URLs and bounded the `MediaConnector` allowed local-media path so an unset or `/` value cannot yield arbitrary `file://` reads ([#9015](https://github.com/ai-dynamo/dynamo/pull/9015), cherry-pick of [#8282](https://github.com/ai-dynamo/dynamo/pull/8282)). +- **TRT-LLM Multimodal Loader Hardening:** Replaced `torch.load` with `safetensors` for TRT-LLM multimodal and routed media decoding through the Rust frontend ([#9016](https://github.com/ai-dynamo/dynamo/pull/9016), cherry-pick of [#8295](https://github.com/ai-dynamo/dynamo/pull/8295)). +- **`ffmpeg-next` 7.1.0 → 8.1:** Bumped the `ffmpeg-next` Rust dependency to harden the Rust-frontend media-decode path against pathological inputs ([#9018](https://github.com/ai-dynamo/dynamo/pull/9018), cherry-pick of [#8452](https://github.com/ai-dynamo/dynamo/pull/8452)). + +### Frontend & Agents + +- **Responses API `output_text` Acceptance:** Fixed the OpenAI Responses API to accept assistant `output_text` messages without `id`/`status` and rejected variants that should never have round-tripped ([#6599](https://github.com/ai-dynamo/dynamo/pull/6599), [#7049](https://github.com/ai-dynamo/dynamo/pull/7049)). +- **Responses Wire-Shape Compliance:** Aligned the Responses wire shape with the OpenResponses spec and added compliance CI ([#8561](https://github.com/ai-dynamo/dynamo/pull/8561), cherry-pick of [#8283](https://github.com/ai-dynamo/dynamo/pull/8283)). +- **`logprobs` Bytes & Token Population:** Populated `logprobs` `bytes` and `token` fields in OpenAI-compatible responses ([#6953](https://github.com/ai-dynamo/dynamo/pull/6953)). +- **DeepSeek V3.2 Content Arrays:** Supported OpenAI content arrays in DeepSeek V3.2 prompt rendering ([#6321](https://github.com/ai-dynamo/dynamo/pull/6321)). +- **DeepSeek V3.2 Thinking Mode kwargs:** Per-request `chat_template_kwargs` for V3.2 thinking mode ([#7286](https://github.com/ai-dynamo/dynamo/pull/7286)). +- **`tool_choice=none` Template Stripping:** Stripped tools from the chat template when `tool_choice=none` ([#7391](https://github.com/ai-dynamo/dynamo/pull/7391)). +- **vLLM-Only Tokenizer Path:** Allowed running without a Rust tokenizer when `dyn-chat-processor` is `vllm` ([#7697](https://github.com/ai-dynamo/dynamo/pull/7697)). +- **`ResponseTimeout` & Worker Quarantine:** Added a `ResponseTimeout` error type with request-plane worker quarantine ([#8011](https://github.com/ai-dynamo/dynamo/pull/8011)). +- **`top_logprobs` Token-ID Detokenization:** Detokenized `top_logprobs` token IDs in the backend so the OpenAI-compatible response surfaces human-readable tokens instead of raw IDs ([#8958](https://github.com/ai-dynamo/dynamo/pull/8958)). +- **LoRA S3 and Streaming:** Extended the LoRA download S3 timeout and streamed large LoRA downloads to disk to prevent OOM on large adapters ([#6544](https://github.com/ai-dynamo/dynamo/pull/6544)). +- **Cancellation in KV Commit Router:** Fixed cancellation handling in the KV-commit router path ([#7178](https://github.com/ai-dynamo/dynamo/pull/7178)). +- **Active Sequences Expiration:** Improved active-sequences request expiration to clear stale slots more aggressively ([#7340](https://github.com/ai-dynamo/dynamo/pull/7340)). + +### vLLM + +- **vLLM Correctness Fixes:** Corrected the KV transfer config plumbing ([#7163](https://github.com/ai-dynamo/dynamo/pull/7163)), the deprecated automatic KV-events config ([#7591](https://github.com/ai-dynamo/dynamo/pull/7591)), the vLLM Omni dependency wiring ([#7683](https://github.com/ai-dynamo/dynamo/pull/7683)), the KV block size derivation from the engine ([#7690](https://github.com/ai-dynamo/dynamo/pull/7690)), and the `distributed-executor-backend` annotation handling ([#6692](https://github.com/ai-dynamo/dynamo/pull/6692)). +- **vLLM Prompt-Embeds Loader:** Routed prompt-embeds loading through vLLM instead of `torch.load` ([#9013](https://github.com/ai-dynamo/dynamo/pull/9013), cherry-pick of [#8228](https://github.com/ai-dynamo/dynamo/pull/8228)) and gated the prompt-embeds request field behind a global enable flag so the loader is only reachable when an operator opts in ([#9014](https://github.com/ai-dynamo/dynamo/pull/9014), cherry-pick of [#8248](https://github.com/ai-dynamo/dynamo/pull/8248)). + +### SGLang + +- **SGLang Stabilization:** Fixed NIXL native libs in the SGLang container ([#6939](https://github.com/ai-dynamo/dynamo/pull/6939)), `served_model_name` resolution ([#8035](https://github.com/ai-dynamo/dynamo/pull/8035)), enabled OTEL trace propagation ([#7592](https://github.com/ai-dynamo/dynamo/pull/7592), [#8361](https://github.com/ai-dynamo/dynamo/pull/8361)), `accelerate` package installation in containers ([#8400](https://github.com/ai-dynamo/dynamo/pull/8400)), and disabled piecewise CUDA graph in launch scripts to work around a backend hang ([#8622](https://github.com/ai-dynamo/dynamo/pull/8622), cherry-pick of [#8609](https://github.com/ai-dynamo/dynamo/pull/8609)). +- **SGLang RC11 Cherry-Picks:** Backported a decode-canary `instance_id-not-found` health fix ([#8816](https://github.com/ai-dynamo/dynamo/pull/8816), cherry-pick of [#8294](https://github.com/ai-dynamo/dynamo/pull/8294)), enabled guided decoding in aggregated serving ([#8874](https://github.com/ai-dynamo/dynamo/pull/8874), cherry-pick of [#8843](https://github.com/ai-dynamo/dynamo/pull/8843)), and registered LoRA with the `Prefill` ModelType in prefill workers ([#8964](https://github.com/ai-dynamo/dynamo/pull/8964), cherry-pick of [#8945](https://github.com/ai-dynamo/dynamo/pull/8945)). + +### TensorRT-LLM + +- **TRT-LLM Metrics & Test Stabilization:** Incremented `kv_transfer_success_total` on the decode side so disagg KV-transfer success is observable end-to-end ([#8873](https://github.com/ai-dynamo/dynamo/pull/8873), cherry-pick of [#8483](https://github.com/ai-dynamo/dynamo/pull/8483)) and stabilized the TRT-LLM router e2e test in the release/1.1.0 CI lane ([#8959](https://github.com/ai-dynamo/dynamo/pull/8959)). + +### Discovery & Transport + +- **ETCD & NATS Hardening:** Fixed an ETCD cluster-unhealthy condition during operator startup ([#6976](https://github.com/ai-dynamo/dynamo/pull/6976)), spawned `handle_put` operations concurrently to avoid head-of-line blocking on the discovery plane ([#7931](https://github.com/ai-dynamo/dynamo/pull/7931)), corrected push-router transport resolution ([#8007](https://github.com/ai-dynamo/dynamo/pull/8007)), and stabilized NATS connection handling in the snapshot path ([#8635](https://github.com/ai-dynamo/dynamo/pull/8635)). + +### Operator + +- **Operator Reliability:** Fixed a multi-node SSH crash during operator startup ([#6694](https://github.com/ai-dynamo/dynamo/pull/6694)), corrected the `chrek` Helm chart version reference ([#6738](https://github.com/ai-dynamo/dynamo/pull/6738)), prevented a nil-pointer panic when DGD service omits replicas ([#6739](https://github.com/ai-dynamo/dynamo/pull/6739)), unblocked PVC handling on restore ([#6752](https://github.com/ai-dynamo/dynamo/pull/6752)), and resolved a startup race condition ([#6929](https://github.com/ai-dynamo/dynamo/pull/6929)). + +### Snapshot + +- **Snapshot Hardening:** Fixed child-snapshot creation to prevent orphaned processes ([#7122](https://github.com/ai-dynamo/dynamo/pull/7122)), corrected the CI image linter ([#7124](https://github.com/ai-dynamo/dynamo/pull/7124)), bounded snapshot GPU memory growth ([#7975](https://github.com/ai-dynamo/dynamo/pull/7975)), and disabled snapshot with GMS in admission to surface unsupported configurations ([#8764](https://github.com/ai-dynamo/dynamo/pull/8764), cherry-pick of [#8689](https://github.com/ai-dynamo/dynamo/pull/8689)). + +### GPU Memory Service + +- **GMS Init & Per-Engine Port Fixes:** Initialized the GPU Memory Service with a scratch-aliased KV cache so SGLang and vLLM engines start cleanly under the operator-managed sidecar ([#8865](https://github.com/ai-dynamo/dynamo/pull/8865), cherry-pick of [#8686](https://github.com/ai-dynamo/dynamo/pull/8686)), and bounded GMS retry plus assigned per-engine FPM ports to eliminate port collisions on multi-engine pods ([#8962](https://github.com/ai-dynamo/dynamo/pull/8962)). + +### Runtime & Lifecycle + +- **Process Lifecycle Correctness:** Propagated child exit codes through `wait_any_exit` so pod restarts surface the real failure code instead of always reporting success ([#8920](https://github.com/ai-dynamo/dynamo/pull/8920), cherry-pick of [#8883](https://github.com/ai-dynamo/dynamo/pull/8883)), and fixed teardown ordering with additional cleanup tests so worker shutdown no longer races against in-flight requests ([#8936](https://github.com/ai-dynamo/dynamo/pull/8936), cherry-pick of [#8857](https://github.com/ai-dynamo/dynamo/pull/8857)). + +### Profiler & Planner Stabilization (release/1.1.0) + +- **Dedicated Planner Image for Profiler:** Used the dedicated `dynamo-planner` image for profiler jobs and planner pods ([#8450](https://github.com/ai-dynamo/dynamo/pull/8450), cherry-pick of [#8407](https://github.com/ai-dynamo/dynamo/pull/8407)). +- **DGD `Recreate` Override Removal:** Dropped a `Recreate` override on restore-target Deployments to avoid downtime during planner-driven scaling ([#8546](https://github.com/ai-dynamo/dynamo/pull/8546), cherry-pick of [#8434](https://github.com/ai-dynamo/dynamo/pull/8434)). +- **Multi-DGD GlobalPlanner Scaling:** Fixed multi-DGD plus GlobalPlanner scaling and readiness ([#8514](https://github.com/ai-dynamo/dynamo/pull/8514), cherry-pick of [#8482](https://github.com/ai-dynamo/dynamo/pull/8482)). +- **MDC Backend-Default Match:** Matched the MDC component field against the backend default rather than the DGD key ([#8512](https://github.com/ai-dynamo/dynamo/pull/8512), cherry-pick of [#8489](https://github.com/ai-dynamo/dynamo/pull/8489)). +- **Prometheus Metric Behaviour:** Restored the documented Prometheus metric behaviour ([#8618](https://github.com/ai-dynamo/dynamo/pull/8618), cherry-pick of [#8575](https://github.com/ai-dynamo/dynamo/pull/8575)). +- **GlobalPlanner Endpoint Concurrency:** Awaited GlobalPlanner endpoints concurrently so health registers ([#8692](https://github.com/ai-dynamo/dynamo/pull/8692), cherry-pick of [#8682](https://github.com/ai-dynamo/dynamo/pull/8682)). +- **GlobalPlanner Pool-Worker Wait:** Waited for pool workers in the GlobalPlanner connector ([#8702](https://github.com/ai-dynamo/dynamo/pull/8702), cherry-pick of [#8694](https://github.com/ai-dynamo/dynamo/pull/8694)). +- **DGD/DGDR Validation Hardening:** Added `optimizationType` enum validation in DGDR so invalid values surface at admission rather than at reconcile ([#8837](https://github.com/ai-dynamo/dynamo/pull/8837), cherry-pick of [#8796](https://github.com/ai-dynamo/dynamo/pull/8796)), surfaced DGD name-length violations as a terminal failure instead of an indefinite retry loop ([#8838](https://github.com/ai-dynamo/dynamo/pull/8838), cherry-pick of [#8807](https://github.com/ai-dynamo/dynamo/pull/8807)), and fixed an "unknown manifest" planner error path that masked the real failure mode ([#8877](https://github.com/ai-dynamo/dynamo/pull/8877)). + +--- + +## Documentation + +### New Content + +- **Mocker & Planner Docs:** Added comprehensive Mocker documentation including planner-in-the-loop and offline replay patterns ([#7488](https://github.com/ai-dynamo/dynamo/pull/7488), [#7610](https://github.com/ai-dynamo/dynamo/pull/7610)), the `replay_optimize` sweep README ([#8195](https://github.com/ai-dynamo/dynamo/pull/8195)), and crate-level READMEs for `dynamo-mocker` and `dynamo-kv-router` ([#7687](https://github.com/ai-dynamo/dynamo/pull/7687)). +- **KV Router A/B Testing Guide:** Added a KV Router A/B testing guide for evaluating routing-policy changes ([#7047](https://github.com/ai-dynamo/dynamo/pull/7047)). +- **Multimodal Documentation Reorg:** Reorganized multimodal documentation around the new EPD pipeline and embedding cache ([#6831](https://github.com/ai-dynamo/dynamo/pull/6831)). +- **KVBM Diagram & Architecture Updates:** Updated KVBM architecture diagrams and the consolidator boundary to match the current code layout ([#7277](https://github.com/ai-dynamo/dynamo/pull/7277), [#7365](https://github.com/ai-dynamo/dynamo/pull/7365)). +- **vLLM Container README:** Documented vLLM container image build and run instructions ([#6793](https://github.com/ai-dynamo/dynamo/pull/6793)). +- **KV Event Replay Comparison:** Documented the KV event replay flow and how it compares with vLLM's native KV event handling ([#6928](https://github.com/ai-dynamo/dynamo/pull/6928)). +- **Snapshot Checkpointing Docs:** Updated Dynamo Snapshot checkpointing docs for the file-sentinel IPC contract ([#7244](https://github.com/ai-dynamo/dynamo/pull/7244)). +- **Profiler & Planner Doc Updates:** Added profiler/planner clarifications ([#7303](https://github.com/ai-dynamo/dynamo/pull/7303)), the `planner-profile-data` ConfigMap clarification ([#8516](https://github.com/ai-dynamo/dynamo/pull/8516), cherry-pick of [#8486](https://github.com/ai-dynamo/dynamo/pull/8486)), and the throughput-scaling SLA requirement ([#8655](https://github.com/ai-dynamo/dynamo/pull/8655), cherry-pick of [#8649](https://github.com/ai-dynamo/dynamo/pull/8649)). +- **Tool & Reasoning Parser Support Docs:** Added documentation for the supported tool and reasoning parsers ([#7605](https://github.com/ai-dynamo/dynamo/pull/7605)). +- **Disaggregated Inference Communication Guide:** Added a disaggregated inference communication guide for Kubernetes deployments ([#6370](https://github.com/ai-dynamo/dynamo/pull/6370)). +- **AIC Disaggregated Serving Guide:** Updated the AI Configurator disaggregated serving guide ([#6553](https://github.com/ai-dynamo/dynamo/pull/6553)). +- **GAIE Fallback Documentation:** Clarified GAIE fallback behavior and source-install flow ([#7077](https://github.com/ai-dynamo/dynamo/pull/7077)). +- **DGDR `v1beta1` Documentation:** Added `v1beta1` DGDR API documentation ([#6647](https://github.com/ai-dynamo/dynamo/pull/6647)). +- **Fern CI / Docs Platform:** First Fern push of the docs site ([#7253](https://github.com/ai-dynamo/dynamo/pull/7253)) and broken-link fixes ([#7322](https://github.com/ai-dynamo/dynamo/pull/7322)). + +--- + +## Looking Ahead + +Dynamo v1.2.0 is targeted for May 27, 2026. Here's what is already taking shape: + +### TensorRT-LLM Backend Refactor + +A larger refactor of the TRT-LLM backend is in implementation, consolidating the disaggregated-multimodal path and the EPD encoder boundary to remove the residual cases that still require workarounds today. Tracked in [#8251](https://github.com/ai-dynamo/dynamo/issues/8251). + +### FlexKV Maturation + +FlexKV (introduced as an integration in [#5858](https://github.com/ai-dynamo/dynamo/pull/5858)) continues to mature toward production use for cross-tier KV cache management on long-context workloads. + +### GlobalPlanner Multi-Tenancy + +GlobalPlanner gains multi-tenant scheduling for shared clusters, building on the v1.1.0 multi-DGD scaling work and the standalone KV indexer's multi-tenant isolation. + +### Continued Diffusion Expansion + +Audio, video, and image-to-image pipelines move from initial support in v1.1.0 to broader model coverage and tighter integration with the EPD pipeline. + +--- + +**Full Changelog**: [https://github.com/ai-dynamo/dynamo/compare/v1.0.2...v1.1.0](https://github.com/ai-dynamo/dynamo/compare/v1.0.2...v1.1.0) + +--- + +## Patch releases + +### v1.1.1 — May 5, 2026 + +#### Summary + +Dynamo v1.1.1 is a patch release that bumps the **TensorRT-LLM** pin from `1.3.0rc11` to `1.3.0rc13` to pick up an upstream fix for a scheduler deadlock that could permanently hang the TRT-LLM engine when KV cache reuse and chunked prefill were enabled together. + +For dependency versions, full feature set, and known issues, see the [Dynamo v1.1.0 release notes](https://github.com/ai-dynamo/dynamo/releases/tag/v1.1.0). + +**Base Branch**: `release/1.1.0` + +#### Bug Fixes + +- **TensorRT-LLM Scheduler Deadlock with KV Reuse + Chunked Prefill:** Resolved a TensorRT-LLM scheduler deadlock where `_prepare_tp_inputs` would trip the `total_num_tokens > max_num_tokens` assertion under KV offload + chunked prefill and permanently hang the event loop, by bumping the TensorRT-LLM pin from `1.3.0rc11` to `1.3.0rc13` (#9182, cherry-pick of #9126) to pick up the upstream fix for [NVIDIA/TensorRT-LLM#13318](https://github.com/NVIDIA/TensorRT-LLM/issues/13318). Reported against TRT-LLM with `--enable_kv_cache_reuse` and `--enable_chunked_context` both on; v1.1.1 resolves it for Dynamo users running the TensorRT-LLM backend with these flags enabled. + +**Full Changelog**: [https://github.com/ai-dynamo/dynamo/compare/v1.1.0...v1.1.1](https://github.com/ai-dynamo/dynamo/compare/v1.1.0...v1.1.1) + + + +Between v1.0.2 and v1.1.0, the project merged 896 PRs from 113 contributors. New first-time external contributors in this release include: + +- **@stevemurr** (Baseten) added a dynamic default `max_tokens` for the TensorRT-LLM backend (#5152) +- **@YconquestY** added FlexKV integration for cross-tier KV cache management (#5858) +- **@Spycsh** (Intel) added GPUDirect support for Intel XPU (#5852) +- **@sywangyi** (Intel) added the SGLang multi-image request path, NIXL EmbeddingSender/Receiver in SGLang, NVTX markers for SGLang EPD, and device-type EPD routing (#6068, #7079, #7153, #7215) +- **@kornelcsernai-harmonic** (Harmonic AI) added a least-loaded router mode (#6314) +- **@danehans** (Tetrate) clarified GAIE fallback behavior and source-install flow (#7077) +- **@jellysnack** added SGLang guided-decoding support (#6620) +- **@blarson-b10** (Baseten) improved active-sequence request expiration (#7340) +- **@simone-chen** updated the AIC disaggregated serving guide (#6553) +- **@yifjiang** added TRT-LLM dynamo-trtllm metrics and fixed guided-decoding arg placement (#6617, #6668) +- **@joshuayao** (Intel) added vLLM aggregated serving examples and unit tests for XPU (#7146, #7078) +- **@ZhengHongming888** (Intel) enabled the Intel XPU Dockerfile for dev targets (#7134) + +Returning external contributors include **@michaelfeil** (Baseten), **@vladnosiv** (Yandex.Cloud), **@dsocek** (Intel), **@AmeenP** (PrimeIntellect), **@huitianbai**, **@InfraWhisperer** (F5), **@devivasudevan** (Microsoft), **@Jont828** (Microsoft), **@ashnamehrotra** (Microsoft), **@Ryan-Amirthan** (Fern), and several others. + +If you would like to get involved, please see our [Contribution Guide](https://docs.nvidia.com/dynamo/dev/getting-started/contribution-guide). + + diff --git a/docs/fern/reference/release-notes/v1-2-0.mdx b/docs/fern/reference/release-notes/v1-2-0.mdx new file mode 100644 index 000000000000..f55fac9e1bfa --- /dev/null +++ b/docs/fern/reference/release-notes/v1-2-0.mdx @@ -0,0 +1,402 @@ +--- +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +title: Dynamo v1.2.0 +subtitle: Release notes for Dynamo v1.2.0 (GA Jun 2, 2026), including patch release v1.2.1 +--- + +import { ReferenceStyles } from "@/components/ReferenceStyles"; +import { ReleaseHeader } from "@/components/ReleaseHeader"; +import { ReleaseSummaryCards } from "@/components/ReleaseSummaryCards"; + + + + + +Dynamo v1.2.0 is the 15th feature release of the open-source distributed inference platform. It expands **multimodal serving** (text-to-image on TensorRT-LLM, formalized audio/video streaming, and media-aware KV routing), scales the standalone KV indexer to **branch-sharded, anchor-aware** routing, and promotes the **DynamoGraphDeployment (DGD)** Kubernetes API to served `v1beta1`. It also productizes the **inter-pod GPU Memory Service**, extends **Dynamo Snapshot** to OpenShift and multi-namespace deployments, and adds **DeepSeek-V4 (Flash and Pro) recipes**, native on vLLM 0.20.1 and on SGLang via an upstream preview image. + + +Breaking changes and deprecations for this release are tracked on the [Deprecations](../deprecations.mdx#v120) ledger; known issues on the [Known Issues](../known-issues.mdx#v120) page. Key dependency pins live on [Compatibility](../compatibility.mdx); shipped artifacts on [Release Artifacts](../release-artifacts.mdx). Model early access builds (`vX.Y.Z--dev.N`) are tracked in [Model Early Access Builds](../model-early-access-builds.mdx). + + +## Highlights + + + +## Features & Improvements + +### Scheduling + +- **KV Router Operational Robustness:** Increased the JSON body-size limit to 8 MiB so the standalone `/query` endpoint can handle 1M-token payloads ([#8315](https://github.com/ai-dynamo/dynamo/pull/8315)), allowed unit block sizes in slot tracking with a bandwidth-flood warning ([#8395](https://github.com/ai-dynamo/dynamo/pull/8395)), added batch-level `start_position` support to flat KV events for absolute position restoration ([#8426](https://github.com/ai-dynamo/dynamo/pull/8426)), introduced a warning-only metrics path for duplicate store replays ([#8473](https://github.com/ai-dynamo/dynamo/pull/8473)), filtered non-main ZMQ KV event groups via `group_idx` to prevent mis-indexing of DeepSeek V4 auxiliary cache state ([#8669](https://github.com/ai-dynamo/dynamo/pull/8669)), added forward-compatible `kv_cache_spec_kind` parsing for hybrid vLLM cache metadata ([#8751](https://github.com/ai-dynamo/dynamo/pull/8751)), resolved a startup crash by polling discovery until a worker model card appears ([#8765](https://github.com/ai-dynamo/dynamo/pull/8765)), and fixed post-clear recovery replay so buffer queries skip events prior to the last clear barrier ([#9296](https://github.com/ai-dynamo/dynamo/pull/9296)). +- **Branch-Sharded KV Router Indexer:** Added `BranchShardedIndexer`, a sharding strategy that partitions the prefix space by conversation branch and routes queries to exactly one shard using FNV-hashed prefix blocks for linear read-throughput scaling ([#7859](https://github.com/ai-dynamo/dynamo/pull/7859)), with a write-triggered cleanup task that prunes stale compressed-tree children to reduce memory footprint ([#8127](https://github.com/ai-dynamo/dynamo/pull/8127)), a fix that registers intermediate FNV values at each depth to prevent false cache misses for short queries ([#8939](https://github.com/ai-dynamo/dynamo/pull/8939)), an opt-in anchor-aware mode that installs shared structural anchors at divergence points and lets compressed radix-tree reads walk anchored suffixes directly ([#9007](https://github.com/ai-dynamo/dynamo/pull/9007)), re-exported the anchor-backed implementation as the default `BranchShardedIndexer` path with corrected anchor bookkeeping on worker removal ([#9318](https://github.com/ai-dynamo/dynamo/pull/9318)), and renamed the internal type while preserving the public alias ([#9333](https://github.com/ai-dynamo/dynamo/pull/9333)). +- **Mooncake Multi-Tier KV Routing:** Added multi-tier KV cache indexing supporting device, host, and disk storage tiers with KV consolidation mode selection and external sequence hash tracking ([#8380](https://github.com/ai-dynamo/dynamo/pull/8380)), extended the remote indexer wire protocol to carry tiered match data end-to-end so host/disk continuation matching works correctly under `use_remote_indexer=true` ([#8632](https://github.com/ai-dynamo/dynamo/pull/8632)), aligned the standalone HTTP indexer with Mooncake RFC #1403 by unifying the tier-walk implementation and reshaping `/query`/`/query_by_hash` responses to expose per-instance tier breakdowns ([#8912](https://github.com/ai-dynamo/dynamo/pull/8912)), and updated the standalone-indexer documentation to reflect the new multi-tier response schema including `instances` map, cumulative tier semantics, and `additional_salt`/`cache_salt` fields ([#9247](https://github.com/ai-dynamo/dynamo/pull/9247)). +- **Global Planner Pool Arbiter:** Added aggregated routing mode to the global router with TTFT × ITL 2D grid pool selection and priority overrides from agent hints ([#8044](https://github.com/ai-dynamo/dynamo/pull/8044)), introduced a `--min-total-gpus` floor with paired intent-cache arbitration that searches for opposite-direction scale intents across managed DGDs to keep the cluster within bounds ([#8633](https://github.com/ai-dynamo/dynamo/pull/8633)), fixed the GlobalPlanner health endpoint never registering by awaiting serve endpoints concurrently instead of sequentially ([#8682](https://github.com/ai-dynamo/dynamo/pull/8682)), and moved per-request target TTFT/ITL fields into the `nvext.router` sub-object with schema validation at deserialization time ([#9845](https://github.com/ai-dynamo/dynamo/pull/9845)). +- **Concurrent Radix-Tree Router:** Defaulted approximate routing to the concurrent radix tree when router event threads exceed one, with support for recording precomputed routing-decision hashes and updated CLI help ([#9219](https://github.com/ai-dynamo/dynamo/pull/9219)), optimized child insertion by gating only the first child with a shape write lock and removed a redundant in-plan shape-version retry check while refactoring mooncake bench helpers for repeated runs with percentile summaries and improving prompt trie stale-lookup repair via BFS ([#9076](https://github.com/ai-dynamo/dynamo/pull/9076)), and implemented `block_count()` and `worker_count()` for the `SyncIndexer` so shard sizes report accurate values in benchmarks ([#8559](https://github.com/ai-dynamo/dynamo/pull/8559)). +- **LoRA Routing Foundation Layer:** Added foundational LoRA allocation primitives including the `LoraAllocator` trait with `compute_replica_set()`, slot-aware state tracking, and shared constants as the base layer for the multi-PR LoRA allocation system ([#8177](https://github.com/ai-dynamo/dynamo/pull/8177)), and fixed LoRA adapter registration in prefill workers to use `ModelType.Prefill` instead of `Chat | Completions`, preventing chat requests from being incorrectly routed to prefill workers that would stall waiting for KV transfers ([#8945](https://github.com/ai-dynamo/dynamo/pull/8945)). +- **KV Router Metrics and Reliability:** Added worker-side ZMQ KV relay counters for received, accepted, and filtered events, exposed normalizer filter reasons for observability, and consolidated KV publisher metrics into the existing KV router metrics module ([#9233](https://github.com/ai-dynamo/dynamo/pull/9233)), with a follow-up that deferred JetStream startup orphan cleanup by snapshotting candidates before a grace period to avoid treating newly started routers as orphans ([#9303](https://github.com/ai-dynamo/dynamo/pull/9303)). +- **Per-Worker Router Configuration:** Exposed an optional `router_config` field in the Model Deployment Card so individual worker sets can override the frontend's global router config, for different routing strategies (e.g., device-aware weighted) for different workers within the same deployment ([#8290](https://github.com/ai-dynamo/dynamo/pull/8290)). +- **Shared KV Cache Routing:** Added an interface for third-party shared KV caches to integrate into the Dynamo router, so routing decisions that account for both local device cache hits and shared cache state using a configurable `shared_cache_multiplier` (default 0.5) can scale shared cache matches relative to device-local matches ([#7536](https://github.com/ai-dynamo/dynamo/pull/7536)). +- **Approximate Pruning TTL-Only Mode:** Refactored KV Router approximate-mode pruning to use exclusively TTL-based eviction, removing `router_max_tree_size` and `router_prune_target_ratio` count-based configuration parameters while keeping normal KV event ingestion fire-and-forget ([#8893](https://github.com/ai-dynamo/dynamo/pull/8893)). +- **Prefill Load Scoring Refinement:** Replaced the hidden `DYN_ROUTER_LOAD_BLOCK_SIZE` scoring override with explicit block-space prefill scoring, added a new `prefill_load_scale` configuration parameter (`DYN_ROUTER_PREFILL_LOAD_SCALE` / `--router-prefill-load-scale`) plumbed through Rust, Python, and C bindings, and introduced a `token-dp-balance` frontend router mode that uses KV scheduler load balancing without prefix matching or KV event/indexer paths ([#9267](https://github.com/ai-dynamo/dynamo/pull/9267)). +- **Profiler and Rapid-Mode Hardening:** Fixed model path doubling when `modelCache` lacks `pvcModelPath` ([#8449](https://github.com/ai-dynamo/dynamo/pull/8449)), wired mocker-rapid to direct AIConfigurator (AIC) flags and removed dead profiler AIC interpolation code ([#8455](https://github.com/ai-dynamo/dynamo/pull/8455)), threaded the deployment's total GPU budget into AIC enumeration so candidates respect cluster size ([#8479](https://github.com/ai-dynamo/dynamo/pull/8479)), enforced `totalGpus` as a hard cap in rapid-mode DGD generation to prevent over-provisioning ([#8617](https://github.com/ai-dynamo/dynamo/pull/8617)), corrected `PickedParallelConfig.num_gpus` to reflect physical GPU count rather than overcounting attention DP ([#8610](https://github.com/ai-dynamo/dynamo/pull/8610)), added an explicit replay optimization objective knob defaulting to throughput ([#8518](https://github.com/ai-dynamo/dynamo/pull/8518)), resolved decode benchmarking failures by using actual deployment component names for log path lookup ([#8733](https://github.com/ai-dynamo/dynamo/pull/8733)), and hardened Pareto computation to skip NaN/inf/non-numeric points gracefully ([#9169](https://github.com/ai-dynamo/dynamo/pull/9169)). +- **Planner Advisory Mode Scaling Logic:** Added an `advisory` scaling mode so operators can observe planner decisions (replica recommendations, structured summary logs, Prometheus metrics, Plotly HTML reports) without applying changes, while the full pipeline runs identically to `active` mode ([#8244](https://github.com/ai-dynamo/dynamo/pull/8244)), with interactive HTML diagnostics and throughput-regression bootstrapping wired into offline planner-in-the-loop replay along with several scaling-logic correctness fixes surfaced by the new reports ([#8280](https://github.com/ai-dynamo/dynamo/pull/8280)), KV-cache hit-rate awareness threaded through load and throughput scaling paths so prefill compute is discounted on reuse-heavy workloads ([#8314](https://github.com/ai-dynamo/dynamo/pull/8314)), AIC interpolation moved in-process to the planner for rapid pre-deployment sweeping and MoE-DEP correctness fixes for models like Qwen3-235B-A22B-FP8 ([#8335](https://github.com/ai-dynamo/dynamo/pull/8335)), and scoped test stubs to prevent advisory-mode fixtures from leaking into other planner test modules ([#8418](https://github.com/ai-dynamo/dynamo/pull/8418)). +- **A30 GPU SKU Support:** Added the NVIDIA A30 to the DGDR `hardware.gpuSku` enum across the Go API, generated Python model, CRD, Helm CRD, and API docs, and taught GPU discovery to infer `a30` from A30 product labels ([#9176](https://github.com/ai-dynamo/dynamo/pull/9176)). +- **Discovery Concurrency Race Fixes:** Addressed multiple concurrency edge cases in the discovery service's `handle_put` lifecycle, including an abort-cleanup race where completed tasks could delete a newer task's handle, a poll timeout guard for `recover_concurrent_registration`, and a delete timeout to avoid blocking the watch loop indefinitely ([#8237](https://github.com/ai-dynamo/dynamo/pull/8237)), added the missing `Audios` variant to `is_model_type_list_empty` which previously caused spurious `ModelUpdate::Removed` emissions for audio models ([#8241](https://github.com/ai-dynamo/dynamo/pull/8241)), moved `ModelUpdate::Added` notification to after `add_worker_set` completes so HTTP endpoints are not exposed before a serving pipeline exists ([#8242](https://github.com/ai-dynamo/dynamo/pull/8242)), and replaced the 100ms polling loop in `recover_concurrent_registration` with a `tokio::sync::Notify` plus `enable()` pattern using an absolute deadline to prevent spurious wakeups from extending the total wait ([#8291](https://github.com/ai-dynamo/dynamo/pull/8291)). +- **Default TCP Worker Pool:** Raised the default TCP worker pool size to 10,000 to prevent unexpected performance drops when benchmarking at concurrency levels above 1,500 ([#9090](https://github.com/ai-dynamo/dynamo/pull/9090)). +- **Lock-Free TCP Pool Infrastructure:** Replaced the mutex-based TCP connection pool with a lock-free LRU design backed by `ArcSwap` + atomic round-robin + `SegQueue`, closing five correctness gaps including unbounded-queue OOM, RAII guard leaks that inflated the inflight counter, and a drop-before-first-poll scheduler leak ([#7806](https://github.com/ai-dynamo/dynamo/pull/7806)), replaced `panic!` calls with `warn + break` on TCP stream read errors (`ECONNRESET`) so connection resets no longer abort Tokio tasks under high concurrency ([#8254](https://github.com/ai-dynamo/dynamo/pull/8254)), and introduced per-stream buffer garbage collection with a configurable threshold to reclaim memory after large transfers while raising the frame-size ceiling for multi-modal embeddings ([#8420](https://github.com/ai-dynamo/dynamo/pull/8420)). + +### Kubernetes Deployment + +- **Snapshot Checkpoint-Restore Hardening:** Added DRA-aware GPU UUID resolution from allocated claims for the snapshot agent ([#8292](https://github.com/ai-dynamo/dynamo/pull/8292)), replaced the PID-1 SIGUSR1/SIGCONT signaling contract with file sentinels and fixed a shell-form entrypoint unwrap panic ([#8403](https://github.com/ai-dynamo/dynamo/pull/8403)), introduced a pluggable `Runtime` interface with containerd and CRI-O backends plus OpenShift chart enablement ([#8427](https://github.com/ai-dynamo/dynamo/pull/8427)), adopted an annotation-driven contract for multi-container checkpoint/restore targeting ([#8631](https://github.com/ai-dynamo/dynamo/pull/8631)), blocked the unsupported snapshot-plus-GPU-Memory-Service combination via admission validation ([#8689](https://github.com/ai-dynamo/dynamo/pull/8689)), enabled a single cross-namespace snapshot-agent DaemonSet using a new `podMount` PVC access mode ([#8740](https://github.com/ai-dynamo/dynamo/pull/8740)), reworked seccomp configuration to support OpenShift checkpoint flows with a structured `seccomp` config block ([#8902](https://github.com/ai-dynamo/dynamo/pull/8902)), and preserved the vLLM torch compile cache in rootfs diffs to prevent CRIU restore failures from missing Triton shared objects ([#9943](https://github.com/ai-dynamo/dynamo/pull/9943)). +- **Operator GPU Hardware Discovery:** Expanded hardware detection to cover Blackwell, Hopper, Ampere, Ada Lovelace, older NVIDIA, and AMD GPU families with interconnect and RDMA/SR-IOV profiling ([#7551](https://github.com/ai-dynamo/dynamo/pull/7551)), fixed a crash when `totalGpus` was missing by failing cleanly instead of proceeding with incomplete hardware info ([#8267](https://github.com/ai-dynamo/dynamo/pull/8267)), added inference fallbacks for unrecognized form factors such as defaulting to SXM when no PCIe variant exists ([#8507](https://github.com/ai-dynamo/dynamo/pull/8507)), hardened enrichment validation with nil guards and a `gpu.DiscoverGPUs` fallback when DCGM fails ([#8508](https://github.com/ai-dynamo/dynamo/pull/8508)), introduced a SKU-filtered GFD node-label discovery path for environments where DCGM pods are unavailable ([#8510](https://github.com/ai-dynamo/dynamo/pull/8510)), and documented the semantics and auto-discovery behavior of the new Interconnect and RDMA HardwareSpec fields ([#8300](https://github.com/ai-dynamo/dynamo/pull/8300)). +- **Operator v1beta1 API Migration:** Introduced `v1beta1` API types for DynamoGraphDeployment, DynamoComponentDeployment, and DynamoGraphDeploymentScalingAdapter with a cleaned-up surface replacing per-component fields with `podTemplate` and renaming `services` to `components` ([#8414](https://github.com/ai-dynamo/dynamo/pull/8414)), implemented lossless `v1alpha1 ↔ v1beta1` conversion and flipped `v1beta1` to served ([#8647](https://github.com/ai-dynamo/dynamo/pull/8647)), added round-trip fuzz coverage and fixed DGDR conversion mutability bugs ([#9164](https://github.com/ai-dynamo/dynamo/pull/9164)), migrated the DGSA controller to reconcile the v1beta1 shape ([#9194](https://github.com/ai-dynamo/dynamo/pull/9194)), migrated the DGD/DCD controllers to read v1beta1 objects while preserving upgrade/downgrade behavior ([#9235](https://github.com/ai-dynamo/dynamo/pull/9235)), and aligned DGDR conversion to the structural rules with sparse annotation preservation for downgrade compatibility ([#9262](https://github.com/ai-dynamo/dynamo/pull/9262)). +- **Istio Mesh Support Fixes:** Added automatic generation of Istio DestinationRule resources for EPP deployments in the Dynamo Operator, preventing double-TLS issues in Istio service meshes that previously required manual DestinationRule creation ([#8270](https://github.com/ai-dynamo/dynamo/pull/8270)). +- **Inter-Pod GMS Failover Support:** Added inter-pod GPU Memory Service failover for vLLM deployments, where each rank receives a dedicated GMS weight-server pod and one or more engine pods sharing GPUs via Kubernetes DRA ResourceClaims and rank-isolated hostPath volumes for CUDA IPC, with a failover cascade controller for fast group-wide cleanup on engine failure ([#7777](https://github.com/ai-dynamo/dynamo/pull/7777)). Introduced scratch-aliased KV cache initialization that preserves VA reservations across sleep/wake cycles so cudagraphs replay correctly during shadow-failover transitions ([#8686](https://github.com/ai-dynamo/dynamo/pull/8686)), fixed SGLang GMS startup by auto-enabling the memory saver path when `setup_gms()` is used ([#9647](https://github.com/ai-dynamo/dynamo/pull/9647)), and corrected the vLLM GMS worker to use the DP-adjusted CUDA device before connecting to GMS, preventing rank collisions under data parallelism ([#9840](https://github.com/ai-dynamo/dynamo/pull/9840)). +- **Helm Operator Configuration Hardening:** Added a temporary admission-level gate that rejects services combining GPU Memory Service with Snapshot unless explicitly opted in via the `dynamo-operator.featureGates.gmsSnapshot` Helm value ([#8829](https://github.com/ai-dynamo/dynamo/pull/8829)), consolidated the previously split NATS subchart flags into a single `global.nats.install` value with validation templates that fail on stale overrides ([#9232](https://github.com/ai-dynamo/dynamo/pull/9232)), and fixed remaining references to the removed flags in the Tiltfile and chart documentation ([#9281](https://github.com/ai-dynamo/dynamo/pull/9281)). + +### Multimodal & Diffusion + +- **Multimodal Media Fetcher Hardening:** Centralized URL validation into a shared `url_validator` module enforcing scheme, local-path, and redirect policies across audio, image, and video loaders ([#8282](https://github.com/ai-dynamo/dynamo/pull/8282)), wired the `DYN_MM_ALLOW_INTERNAL` env var through to the Rust `MediaFetcher` to honor localhost/on-prem semantics and unblock nightly tests ([#8535](https://github.com/ai-dynamo/dynamo/pull/8535)), hardened the Rust side against redirect bypass and DNS rebinding with RFC-based IP blocklists, cloud-metadata hostname checks, and a custom `BlocklistResolver` on the `reqwest::Client` ([#8569](https://github.com/ai-dynamo/dynamo/pull/8569)), introduced a dual-backend HTTP client supporting both httpx and aiohttp to improve throughput under high concurrency ([#8646](https://github.com/ai-dynamo/dynamo/pull/8646)), bounded total in-flight media fetches with a global semaphore and decoupled httpx pool/read timeouts via `DYN_MM_HTTP_*` env vars to eliminate `PoolTimeout` errors ([#8657](https://github.com/ai-dynamo/dynamo/pull/8657)), and added an opt-in byte-budgeted LRU cache to the Rust `MediaLoader` that skips network fetch, decode, and NIXL registration on cache hits ([#8863](https://github.com/ai-dynamo/dynamo/pull/8863)). +- **Multimodal Benchmarks Tooling Enhancements:** Added a sliding-window JSONL generation subcommand for exercising prefix-caching under interleaved multi-user image streams, refactoring argument parsing into composable parent parsers and subcommands ([#8201](https://github.com/ai-dynamo/dynamo/pull/8201)), fixed grouped single-turn sweeps to use `--conversation-num` so sessions run to completion without re-dispatch distortion ([#8458](https://github.com/ai-dynamo/dynamo/pull/8458)), and introduced multithreading with configurable processing delay to the local media server for production-realistic latency simulation ([#8822](https://github.com/ai-dynamo/dynamo/pull/8822)). +- **SGLang Diffusion Video Pipeline Support:** Added video input support for SGLang in the disaggregated prefill/decode path ([#8597](https://github.com/ai-dynamo/dynamo/pull/8597)), fixed a startup crash caused by a missing `enable_trace` field in the diffusion worker `ServerArgs` stub ([#8332](https://github.com/ai-dynamo/dynamo/pull/8332)), added the `accelerate` package to the SGLang runtime Docker image so diffusion pipelines can initialize successfully ([#8357](https://github.com/ai-dynamo/dynamo/pull/8357)), and preserved file permissions in the dev container venv setup to prevent the bundled `ffmpeg` binary from losing execute bits ([#8408](https://github.com/ai-dynamo/dynamo/pull/8408)). +- **Multimodal-Aware KV Routing Path:** Added a Rust-frontend multimodal routing path that extracts per-image hashes, dimensions, and token counts (via per-VLM-family math) before selecting a worker, for KV-cache-aware placement of multimodal workloads ([#9272](https://github.com/ai-dynamo/dynamo/pull/9272)), with a plumbing-only PR that wired the `lightseek-mm` cargo feature and exposed `LightseekMmCounter` and token-ID resolver helpers from the `llm-multimodal` crate ([#9352](https://github.com/ai-dynamo/dynamo/pull/9352)), and documented that Qwen3.5/Qwen3.6 models are not yet supported ([#9572](https://github.com/ai-dynamo/dynamo/pull/9572)). +- **Multimodal Routing Frontend Integration:** Moved multimodal-aware KV routing from the separate MM Router Worker process into the frontend's vLLM processor, eliminating an extra network hop and redundant image processing by transferring pre-processed `mm_kwargs` via shared memory (~2ms) or NIXL RDMA, with automatic fallback to URL reprocessing for cross-node deployments ([#8065](https://github.com/ai-dynamo/dynamo/pull/8065)). +- **TRT-LLM Multimodal Image Support:** Added text-to-image generation capability to the TRT-LLM backend with a new `/v1/images/generations` API endpoint and Flux pipeline integration ([#8200](https://github.com/ai-dynamo/dynamo/pull/8200)), and hardened the multimodal processor by replacing `torch.load()` with safetensors loading and Rust frontend media decoding to eliminate arbitrary code execution risks ([#8295](https://github.com/ai-dynamo/dynamo/pull/8295)). +- **Model Family Lookup Resolver:** Introduced `ModelFamily` and `resolve_model_family` for vLLM multimodal encode-related handling in disaggregated prefill, replacing eight inconsistent per-callsite dispatch paths that failed when models were loaded from local paths not matching HF ids ([#8973](https://github.com/ai-dynamo/dynamo/pull/8973)). +- **NIXL Canonical Memtype XPU Support:** Refactored NIXL memory type classification to use canonical segment names (VRAM/DRAM) instead of device-specific strings for forward compatibility, added Intel XPU (Level-Zero) device support for disaggregated encode/prefill-decode transfers, and fixed XPU kernel synchronization to prevent race conditions before NIXL transfers ([#9073](https://github.com/ai-dynamo/dynamo/pull/9073)). +- **Formalized Video Streaming Protocol:** Introduced a dedicated `data_source` field in the audio protocol to disambiguate the overloaded `response_format` field, added an `output_format` container field to the video protocol, and extended the `/v1/videos` HTTP endpoint to support SSE streaming output in addition to batch JSON responses, for a consistent internal interface across all modalities ([#8491](https://github.com/ai-dynamo/dynamo/pull/8491)). +- **SGLang Frontend Decoding Support:** Added `--frontend-decoding` flag for the SGLang backend so aggregated multimodal requests are decoded by the Rust frontend and shipped via NIXL RDMA, removing the GIL-bound image-decode hop from the worker's P0 thread ([#9405](https://github.com/ai-dynamo/dynamo/pull/9405)). +- **Image UUID Benchmark Emission:** Added optional `--uuid` flag to the multimodal JSONL generator that emits deterministic `image_uuids` alongside `images`, so vLLM's cached-mm processor can short-circuit on cache hits across benchmark restarts and image reuse ([#8854](https://github.com/ai-dynamo/dynamo/pull/8854)). + +### Frontend & Agents + +- **Agent Context and Tool Relay:** Added passive `nvext.agent_context` parsing and a normalized agent trace bus with configurable `jsonl`/`stderr` sinks that emit chat-completion `request_end` records with Dynamo request metrics ([#8789](https://github.com/ai-dynamo/dynamo/pull/8789)), introduced a ZMQ-to-event-plane relay with a narrow `AgentToolEventRelay` Python binding that captures tool lifecycle events from external harnesses ([#8790](https://github.com/ai-dynamo/dynamo/pull/8790)), hardened the ingest topology so Dynamo owns the ZMQ `PULL` bind side, allowing multiple producers to connect without endpoint conflicts, and added multi-producer regression coverage ([#9105](https://github.com/ai-dynamo/dynamo/pull/9105)), and aligned agent context identifiers with ATIF terminology (`session_type_id`, `session_id`, `trajectory_id`, `parent_trajectory_id`) while splitting the Agents documentation into focused pages ([#9140](https://github.com/ai-dynamo/dynamo/pull/9140)). +- **Nvext Response Metadata Gating:** Fixed a regression where plain OpenAI-compatible requests leaked `nvext` response metadata (`worker_id`, `timing`) by default, introducing `NvExtResponseFieldSelection` to gate each response field independently behind `extra_fields` opt-in while preserving the `query_instance_id` exception ([#8252](https://github.com/ai-dynamo/dynamo/pull/8252)). +- **DeepSeek V4 Parser Support:** Added a DeepSeek V4 DSML tool-call parser and reasoning parser to the Rust parser crate, registering canonical names and compatibility aliases (`deepseek_v4`, `deepseek-v4`, `deepseekv4`) with unit coverage for tool parsing, streaming reasoning, and alias resolution ([#8665](https://github.com/ai-dynamo/dynamo/pull/8665)), hardened the V4 formatter so `thinking: false`, `enable_thinking: false`, `thinking_mode: "chat"`, `reasoning_effort="max"`, and per-request `drop_thinking` overrides all route end-to-end correctly ([#8670](https://github.com/ai-dynamo/dynamo/pull/8670)), fixed tool-continuation reasoning parsing where a prompt-injected `` seed caused the closing tag to leak as normal content without regressing Kimi K2.5 behavior ([#8901](https://github.com/ai-dynamo/dynamo/pull/8901)), and deduplicated the DeepSeek V3.2 and V4 prompt encoders into a shared `deepseek_common` module to reduce code duplication ([#9322](https://github.com/ai-dynamo/dynamo/pull/9322)). +- **Self-Host MDC Metadata Pipeline:** Added worker-side metadata artifact hosting via `system_status_server`, where `LocalModel::attach()` registers local-disk MDC files in a process-local registry and rewrites paths to HTTP URLs served by the worker ([#8855](https://github.com/ai-dynamo/dynamo/pull/8855)), implemented a frontend verify-and-cache resolution pipeline that derives URIs per `CheckedFile`, fetches via scheme handlers, blake3-verifies, and atomically publishes to a content-addressed cache ([#9057](https://github.com/ai-dynamo/dynamo/pull/9057)), and fixed a regression where sibling files like `preprocessor_config.json` and `tokenizer.model` were missing from the slug directory by adding a harvest pass that symlinks non-weight siblings from resolved directories ([#9610](https://github.com/ai-dynamo/dynamo/pull/9610)). +- **OpenAI Multi-Choice N Support:** Added OpenAI-compatible `n` parameter plumbing to the Dynamo frontend and response contract so requests with `n > 1` return multiple choices, with vLLM backend integration ([#8744](https://github.com/ai-dynamo/dynamo/pull/8744)), SGLang backend support using its native multi-sequence generation ([#8745](https://github.com/ai-dynamo/dynamo/pull/8745)), and TensorRT-LLM backend support passing `n` through to its sampling params and keeping streamed choices separated ([#8746](https://github.com/ai-dynamo/dynamo/pull/8746)). +- **Cross-Impl Parser Parity Harness:** Added a pytest-driven parity harness that exercises Dynamo, vLLM, and SGLang parsers against shared YAML fixtures and diffs their outputs to surface cross-implementation divergences ([#9186](https://github.com/ai-dynamo/dynamo/pull/9186)), expanded coverage from 7 to all 19 registered Dynamo parser families with new fixtures and xfail-tracked divergences ([#9261](https://github.com/ai-dynamo/dynamo/pull/9261)), and documented the end-to-end workflow for finding, reproducing, fixing, and retiring divergences ([#9394](https://github.com/ai-dynamo/dynamo/pull/9394)). +- **Bidirectional Streaming WebSocket Frontend:** Added a `/v1/realtime` WebSocket endpoint that accepts `NvCreateChatCompletionRequest` JSON frames over a single connection and streams `NvCreateChatCompletionStreamResponse` chunks back, for realtime bidirectional communication as the first slice of the streaming-input feature ([#9079](https://github.com/ai-dynamo/dynamo/pull/9079)). + +### TensorRT-LLM + +- **TRT-LLM Metrics and KV Publisher:** Added ForwardPassMetrics publishing via a PyO3 FpmDirectPublisher for non-attention-DP TensorRT-LLM workers, so the Planner can treat them equivalently to vLLM workers for autoscaling and latency prediction ([#8356](https://github.com/ai-dynamo/dynamo/pull/8356)), fixed the `trtllm_kv_transfer_success_total` Prometheus counter that was never incremented due to a mutually exclusive gate between prefill mode and the decode-side timing check ([#8483](https://github.com/ai-dynamo/dynamo/pull/8483)), and eliminated a ~148 ms TTFT regression when `--publish-events-and-metrics` was enabled by moving the publisher off the request loop and batching KV-cache walks ([#8892](https://github.com/ai-dynamo/dynamo/pull/8892)). +- **TRT-LLM Canary Health Checks:** Set highest priority (1.0) on the canary health-check request to prevent false-negative timeouts when long-context inference requests starved the probe under load ([#8488](https://github.com/ai-dynamo/dynamo/pull/8488)), and fixed disaggregated decode workers staying permanently NotReady by injecting explicit `disaggregated_params` into the canary payload so the handler no longer rejects probes that lack prefill-peer context ([#8521](https://github.com/ai-dynamo/dynamo/pull/8521)). +- **Disable GC for TRT-LLM:** Added support for disabling Python garbage collection in the Dynamo TRT-LLM worker when `TRTLLM_SERVER_DISABLE_GC` or `DYN_TRTLLM_SERVER_DISABLE_GC` is set, preventing uvloop stalls at high concurrency and performance parity with trtllm-serve ([#9096](https://github.com/ai-dynamo/dynamo/pull/9096)). +- **TRT-LLM Wakeup RPC Caching:** Improved wakeup call handling to preserve original exceptions and cache the resolved RPC method name after first invocation, eliminating redundant lookups on subsequent calls ([#8255](https://github.com/ai-dynamo/dynamo/pull/8255)). + +### Performance Modeling & Replay + +- **Performance Modeling Offline Replay:** Added KV Block Manager (KVBM)-backed G1↔G2 offload simulation for the vLLM mocker in both online and offline replay modes ([#8184](https://github.com/ai-dynamo/dynamo/pull/8184)), simulated non-zero worker startup time in the offline discrete-event engine with `WorkerReady` events and `pending_startup` tracking ([#8231](https://github.com/ai-dynamo/dynamo/pull/8231)), hardened replay-router scaling invariants including single-worker queueing, 0→N recovery, and draining-state preservation ([#8236](https://github.com/ai-dynamo/dynamo/pull/8236)), introduced an agentic trace file format with loader and multi-turn smoke coverage ([#8627](https://github.com/ai-dynamo/dynamo/pull/8627)), published tiered `HostPinned` KV events for G2 offload with a new `StorageTier`-aware event sink ([#8961](https://github.com/ai-dynamo/dynamo/pull/8961)), and captured replay-friendly hashes on agent `request_end` records with a converter producing Mooncake JSONL for mocker replay ([#8998](https://github.com/ai-dynamo/dynamo/pull/8998)). +- **Claude Trace Exporter Tool:** Added a privacy-preserving Claude raw-trace exporter under `benchmarks/coding/claude`, supporting autodiscovery, compaction-aware parsing, structural sidecar output, and configurable tokenization for accurate output token computation ([#8096](https://github.com/ai-dynamo/dynamo/pull/8096)). + +### Infrastructure Modernization + +- **Context-Aware Event Plane Defaults:** Made the event-plane selection context-aware so local-only discovery backends (`file`/`mem`) default to ZMQ while distributed backends (`etcd`/`kubernetes`) default to NATS, eliminating unconditional NATS connections at startup ([#8398](https://github.com/ai-dynamo/dynamo/pull/8398)), removed the Python CLI's hard-coded `DYN_EVENT_PLANE=nats` override that still forced NATS connections and caused "Connection refused" failures in local workflows ([#8614](https://github.com/ai-dynamo/dynamo/pull/8614)), and removed hard-coded `NATS_SERVER` and `ETCD_ENDPOINTS` environment variables from profiling job configuration to respect the same infrastructure-aware defaults ([#9271](https://github.com/ai-dynamo/dynamo/pull/9271)). + +### Fault Tolerance & Observability + +- **Frontend Staged Request Gauges:** Added `dynamo_frontend_active_requests` and `dynamo_frontend_stage_requests` gauges with per-stage (preprocess, route, dispatch) and per-phase (prefill/decode/aggregated) inflight counts, backed by a `StageGuard` RAII type that increments on creation and decrements on drop ([#8162](https://github.com/ai-dynamo/dynamo/pull/8162)), and updated the metrics documentation with descriptions of the new gauges and deprecation notes for superseded ones ([#8459](https://github.com/ai-dynamo/dynamo/pull/8459)). +- **Dynamo Local Resource Monitor:** Added a lightweight, high-frequency (200 ms) per-process resource monitor that tracks VRAM, GPU, PCIe, CPU, disk, and network usage for Dynamo processes on a single host, with an accompanying Grafana dashboard integrated into the existing observability stack ([#9055](https://github.com/ai-dynamo/dynamo/pull/9055)). +- **Worker-Pool Saturation Metrics:** Added six Prometheus metrics (`dynamo_work_handler_queue_depth`, `queue_capacity`, `enqueue_rejected_total`, `permit_wait_seconds`, `pool_active_tasks`, `pool_capacity`) to the shared TCP server, so operators can detect queue buildup and permit starvation before workers are OOM-killed ([#8412](https://github.com/ai-dynamo/dynamo/pull/8412)). +- **Reusable Telemetry Bus Primitives:** Added a typed telemetry bus, stream completion helper, and async JSONL sink to the LLM library for shared event-recording infrastructure with configurable append mode, buffer sizing, and periodic flush while preserving the existing recorder JSONL format ([#8788](https://github.com/ai-dynamo/dynamo/pull/8788)). +- **Orphaned Pending Request Handling:** Implemented discovery-plane-driven cancellation of pending response-stream subjects when a worker is removed, preventing indefinite hangs where requests queued in a killed worker's bounded channel became permanently stuck; cancelled streams now return a migratable `Disconnected` error so the migration layer can retry on another worker ([#8182](https://github.com/ai-dynamo/dynamo/pull/8182)). + +### vLLM + +- **vLLM Multinode Elastic EP:** Added test infrastructure for validating vLLM's native elastic expert parallelism across multiple nodes using a warm-standby Ray topology, confirming all scale steps (dp=2→3→4→3→2→4→2) succeed on a 2-node AKS cluster ([#8183](https://github.com/ai-dynamo/dynamo/pull/8183)), and introduced operator support that routes `--enable-elastic-ep` deployments through a cross-node Ray cluster while fixing a concurrent `scale_elastic_ep` race condition that caused 300 s TCPStore timeouts on remote worker nodes during scale-up ([#8216](https://github.com/ai-dynamo/dynamo/pull/8216)). +- **vLLM Start Stop Profile:** Added start/stop profile endpoints (`engine/start_profile` and `engine/stop_profile`) for vLLM to maintain parity with the SGLang implementation, giving a unified profiling interface ([#8068](https://github.com/ai-dynamo/dynamo/pull/8068)). + +### SGLang + +- **SGLang Forward Pass Metrics:** Wired SGLang's ForwardPassMetrics ZMQ publisher into Dynamo's event plane via FpmEventRelay, injecting the endpoint instance ID as worker_id and adding a cross-repo wire-format contract test ([#8154](https://github.com/ai-dynamo/dynamo/pull/8154)), and fixed the previously unpopulated `total_kv_blocks` runtime config by resolving rank-0 scheduler info through SGLang's canonical Engine scheduler path ([#8439](https://github.com/ai-dynamo/dynamo/pull/8439)). +- **SGLang Token ID Logprobs:** Added `return_tokens_as_token_ids` support, integer token-id stop arrays, and gated top-logprobs for SGLang to enable RL workflows while preserving OpenAI-compatible response shapes ([#8119](https://github.com/ai-dynamo/dynamo/pull/8119)). + +### Unified Backend (Preview) + +- **Unified Disaggregated Serving Abstraction:** Added a common backend framework that lets engines plug into Dynamo's disaggregated-serving path through a shared `LLMEngine` trait/ABC, eliminating the need for each backend to reinvent wire format, registration plumbing, or shutdown orchestration ([#9249](https://github.com/ai-dynamo/dynamo/pull/9249)). +- **Rust Backend Common Framework:** Added a shared Rust backend layer (`dynamo-backend-common`) with an `LLMEngine` trait and `Worker` lifecycle driver, for native Rust backend integrations that follow the same lifecycle and cancellation contract as existing Python backends, along with a `mocker` reference engine for end-to-end pipeline testing ([#8584](https://github.com/ai-dynamo/dynamo/pull/8584)). +- **Rust Worker PyO3 Backend:** Moved the unified backend Worker lifecycle from Python into the Rust `dynamo_backend_common` crate and exposed it to Python engines through a new `dynamo._core.backend.Worker` PyO3 binding, shrinking the Python Worker shim from 259 lines to ~115 while consolidating signal handling, discovery unregister, grace-period sleep, drain, cleanup, and 3-phase runtime shutdown entirely in Rust ([#9202](https://github.com/ai-dynamo/dynamo/pull/9202)). +- **Unified Worker Engine Args:** Enabled the `dynamo.trtllm` unified worker to honor `--extra-engine-args` (YAML) and `--override-engine-args` (JSON) flags, bringing it to parity with the existing worker by porting the YAML+JSON merge sequence and adding validation that parsed overrides are JSON objects ([#8886](https://github.com/ai-dynamo/dynamo/pull/8886)). + +### KV Block Manager + +- **KVBM-Logical Mocker Backend Replacement:** Replaced the mocker's manual vLLM block manager and evictor with a new `kvbm-logical::BlockManager` backend using a `Lineage` inactive pool, simplifying block lifecycle management and bridging KVBM's PositionalLineageHash to the router's SequenceHash via a new HashMap ([#8451](https://github.com/ai-dynamo/dynamo/pull/8451)). +- **PositionalLineageHash Ordering Support:** Added `Ord` and `PartialOrd` trait implementations to `PositionalLineageHash`, for deterministic sorting and comparison operations for KVBM token types ([#8687](https://github.com/ai-dynamo/dynamo/pull/8687)). + +### TokenSpeed + +- **Initial TokenSpeed Backend:** Added a TokenSpeed backend integration through the common LLMEngine path, introducing `python -m dynamo.tokenspeed` with argument parsing, request/sampling conversion, streaming token output, abort/cleanup handling, guided decoding support, and focused unit coverage ([#9212](https://github.com/ai-dynamo/dynamo/pull/9212)). + +## Recipes + +- **DeepSeek-V4 Recipe Consolidation:** Restructured the DeepSeek-V4 Flash and Pro recipes into a single `recipes/deepseek-v4/` subtree following the repo-wide convention, deduplicated Dockerfiles, and simplified the SGLang Dockerfile to consume the Dynamo donor image directly with a pinned base digest ([#8735](https://github.com/ai-dynamo/dynamo/pull/8735)), performed a post-base cleanup pass removing unnecessary system dependencies from the B200 SGLang image ([#8929](https://github.com/ai-dynamo/dynamo/pull/8929)), refreshed Python and Rust attribution files based on container scans of the runtime images ([#8948](https://github.com/ai-dynamo/dynamo/pull/8948)), bumped deploy manifests to dev.2 public image tags and consolidated apt purge directives ([#8971](https://github.com/ai-dynamo/dynamo/pull/8971)), refreshed the SGLang base image ([#9002](https://github.com/ai-dynamo/dynamo/pull/9002)), updated all recipe manifests to dev.3 images ([#9356](https://github.com/ai-dynamo/dynamo/pull/9356)), and aligned the docs support-matrix and release-artifacts pages to reflect the dev.3 experimental release ([#9358](https://github.com/ai-dynamo/dynamo/pull/9358)). +- **DeepSeek-V4 SGLang Serving Recipes:** Added SGLang serving recipe for DeepSeek-V4-Flash on B200 GPUs with TP4, MXFP4 MoE, and EAGLE speculative decoding via Dynamo frontend ([#8704](https://github.com/ai-dynamo/dynamo/pull/8704)), with Dockerfile PATH fixes for etcd resolution ([#8713](https://github.com/ai-dynamo/dynamo/pull/8713), [#8716](https://github.com/ai-dynamo/dynamo/pull/8716)), published both Flash and Pro SGLang recipes using public NGC images and updated documentation ([#8734](https://github.com/ai-dynamo/dynamo/pull/8734)), added aggregated and disaggregated recipes for DeepSeek-V4-Pro on GB200 with NIXL KV transfer over GKE RDMA ([#8960](https://github.com/ai-dynamo/dynamo/pull/8960)), and added a disaggregated prefill/decode recipe for DeepSeek-V4-Pro on B200 with InfiniBand RDMA including UCX configuration guidance ([#9278](https://github.com/ai-dynamo/dynamo/pull/9278)). +- **DeepSeek-V4 vLLM Serving Recipes:** Added aggregated vLLM recipes for DeepSeek-V4-Flash (4×B200, DP=4+EP) and DeepSeek-V4-Pro (8×B200, TP=8+EP) ([#8668](https://github.com/ai-dynamo/dynamo/pull/8668)), introduced a disaggregated prefill/decode recipe for V4-Pro on GB200 NVL72 using DRA ComputeDomain for cross-node MNNVL with 16 GPUs total ([#8811](https://github.com/ai-dynamo/dynamo/pull/8811)), extended GB200 aggregated recipes for both V4-Flash (TP=4+EP, FP4 path) and V4-Pro (TP=8+EP) ([#8876](https://github.com/ai-dynamo/dynamo/pull/8876)), and pinned `--no-enable-flashinfer-autotune` across all dsv4 vLLM recipes to prevent accuracy regressions ([#9268](https://github.com/ai-dynamo/dynamo/pull/9268)). +- **Qwen3.6-35B Benchmark Recipe:** Added a 3-way Kubernetes benchmark recipe for Qwen/Qwen3.6-35B-A3B-FP8 on a single H100 or GB200, comparing vanilla vLLM, Dynamo with frontend-decoding, and Dynamo with embedding cache across throughput and latency metrics ([#9392](https://github.com/ai-dynamo/dynamo/pull/9392)). +- **Qwen3-32B-FP8 vLLM Recipe:** Added a production-ready vLLM disaggregated single-node recipe for Qwen3-32B-FP8, including deploy and benchmark manifests with a topology of 2× prefill workers (TP=2) and 1× decode worker (TP=4) using NixlConnector KV transfer ([#7915](https://github.com/ai-dynamo/dynamo/pull/7915)). +- **Qwen3-235B Architecture-Specific Recipes:** Split TensorRT-LLM aggregated and disaggregated deployment recipes for Qwen3-235B-A22B-FP8 into separate Hopper and Blackwell subdirectories, eliminating the need for manual YAML editing when targeting different GPU architectures since TRT-LLM 1.3.x requires different `moe_config` settings on each ([#8470](https://github.com/ai-dynamo/dynamo/pull/8470)). +- **Allow Internal URLs Recipe Fix:** Added `DYN_MM_ALLOW_INTERNAL=1` environment variable to the Qwen3-VL-30B vLLM agg-embedding-cache recipe, preventing multimodal benchmark failures caused by the new default-deny URL validator rejecting COCO dataset image URLs ([#9309](https://github.com/ai-dynamo/dynamo/pull/9309)). +- **vLLM Recipe Flag Rename:** Renamed the deprecated `--disable-log-requests` argument to `--no-enable-log-requests` in recipe deployment YAMLs and benchmark documentation, preventing vLLM worker pods from crash-looping on startup with newer vLLM runtimes (0.19.x+) that removed the old flag ([#8693](https://github.com/ai-dynamo/dynamo/pull/8693)). +- **Qwen3 KVBM Recipe Addition:** Removed the Kimi K2.5 KVBM recipe (which required an unreleased container) and added a new Qwen3-32B KVBM recipe with single-GPU aggregated deployment using vLLM ([#8475](https://github.com/ai-dynamo/dynamo/pull/8475)). +- **Nemotron-3-Nano-Omni Recipe:** Added an experimental recipe for serving `nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-NVFP4` via vLLM in aggregated single-GPU mode, supporting text, image, video, and audio inputs with Kubernetes manifests for model caching and deployment ([#8799](https://github.com/ai-dynamo/dynamo/pull/8799)). +- **Kimi-K2.5 TokenSpeed Aggregated Recipe:** Added a complete deployment recipe for serving `nvidia/Kimi-K2.5-NVFP4` on the TokenSpeed engine under Dynamo's KV-aware aggregated frontend, including a multi-stage local-build Dockerfile, raw Kubernetes Deployments and Services (pending Operator support for TokenSpeed), and a full build/push/deploy walkthrough with TP=4, EP=4, NVFP4 weights, FP8 KV cache, and MLA/MoE plugin configuration ([#9231](https://github.com/ai-dynamo/dynamo/pull/9231)). +- **Empty Reasoning Content Fix:** Fixed the GPT-OSS reasoning parser producing empty `reasoning_content` when used with bounded KV cache by whitelisting special tokens needed by the gpt-oss and harmony parsers in the preprocessor, and resolved a Tokio runtime-drop panic during lazy initialization by moving init to a fresh OS thread ([#9050](https://github.com/ai-dynamo/dynamo/pull/9050)). +- **Pin Transformers for DeepSeek Perf:** Pinned `transformers==4.57.6` in DeepSeek-V3.2-FP4 performance job YAML files to prevent silent upgrades to transformers 5.x, which lacks native `deepseek_v32` model-type support and caused `TokenizerError: Failed to load tokenizer` failures during benchmarking ([#8690](https://github.com/ai-dynamo/dynamo/pull/8690)). +- **GLM-5 Disagg Frontend Flag:** Fixed the GLM-5 SGLang disaggregated deploy manifest to use the correct `--enforce-disagg` flag instead of the non-existent `--no-decode-fallback` flag, which would have caused a startup failure ([#8914](https://github.com/ai-dynamo/dynamo/pull/8914)). +- **Model Cache Resource Limits:** Added Kubernetes `resources.{requests,limits}.memory` blocks and an absolute `HF_XET_RECONSTRUCTION_DOWNLOAD_BUFFER_LIMIT=16GB` environment variable to all 15 model-download recipe Job pods, preventing OOM evictions caused by unbounded HF XET reconstruction buffers when `HF_XET_HIGH_PERFORMANCE=1` is set ([#9884](https://github.com/ai-dynamo/dynamo/pull/9884)). + +## Bug Fixes + +### Kubernetes Deployment + +- **Operator DGD Reconciliation Lifecycle Fixes:** Fixed DGD reconciliation stalling at `Ready=False` by adding a PodCliqueScalingGroup (PCSG) watch so PCSG status changes trigger the controller ([#8328](https://github.com/ai-dynamo/dynamo/pull/8328)), with an analogous fix for the single-node PodClique readiness path that now reconciles on all relevant status fields ([#8423](https://github.com/ai-dynamo/dynamo/pull/8423)). Corrected planner ConfigMap ownership so profiling-generated ConfigMaps survive DGDR deletion and are adopted under the DGD lifecycle ([#8766](https://github.com/ai-dynamo/dynamo/pull/8766)), added profiler validation that surfaces DGD-plus-service name-length violations as terminal failures ([#8807](https://github.com/ai-dynamo/dynamo/pull/8807)), surfaced workload pod image-pull failures as Warning events on the DGDR instead of requiring `kubectl logs --previous` ([#8815](https://github.com/ai-dynamo/dynamo/pull/8815)), and rewrote the rolling-update replica calculation to use an availability-gated scale-down budget mirroring the Kubernetes Deployment controller, preventing maxUnavailable breaches during preemption scenarios ([#8823](https://github.com/ai-dynamo/dynamo/pull/8823)). +- **Planner Container Image Fixes:** Added `tail` and `env` utilities to the planner distroless image so container orchestration tooling and exec-based workflows function correctly ([#8603](https://github.com/ai-dynamo/dynamo/pull/8603)), introduced an inline `planner_test` build stage and unified all framework test image builds through inline `_test` stages routed via the shared `docker-remote-build` action with sccache and registry caching ([#8739](https://github.com/ai-dynamo/dynamo/pull/8739)), and included `aiperf` in the planner builder stage to fix `FileNotFoundError` crashes during thorough profiling benchmarks ([#8769](https://github.com/ai-dynamo/dynamo/pull/8769)). +- **Profiler Sidecar Retry Prevention:** Disabled automatic retries for profiling sidecar failures in the Operator, treating them as deterministic issues that require user intervention and now reporting explicit status and error details for clearer diagnostics ([#9166](https://github.com/ai-dynamo/dynamo/pull/9166)). +- **DGDR Optimization Type Validation:** Added `OptimizationType` as an enum-constrained string field to the DGDR SLASpec, so users can specify whether SLA-based profiling targets latency or throughput optimization with webhook validation enforcing valid values ([#8796](https://github.com/ai-dynamo/dynamo/pull/8796)). +- **Grove Resource Name Truncation:** Shortened TRT-LLM disaggregated worker service names from `TRTLLMPrefillWorker`/`TRTLLMDecodeWorker` to `prefill`/`decode` so that combined Grove resource names fit within the 45-character PodCliqueSet limit, fixing broken DGDR deployments with TRT-LLM disaggregation ([#8563](https://github.com/ai-dynamo/dynamo/pull/8563)). +- **Operator Least-Privilege RBAC Hardening:** Removed the cluster-admin-equivalent `*/*: *` wildcard rule from the Helm chart's manager ClusterRole and replaced it with explicit, audited least-privilege rules for every Kubernetes client operation in the operator codebase ([#8907](https://github.com/ai-dynamo/dynamo/pull/8907)), fixed an RBAC privilege-escalation rejection on fresh installs by adding the missing `pods/log: get` permission required for DGDR profiling job RoleBinding creation ([#9969](https://github.com/ai-dynamo/dynamo/pull/9969)), scoped the Docker secret indexer to the configured restricted namespace so that `mgr.GetAPIReader()` no longer bypasses namespace boundaries ([#9863](https://github.com/ai-dynamo/dynamo/pull/9863)), and added a projected service-account token volume for the profiling output-copier sidecar to keep it authenticated when `automountServiceAccountToken: false` is set without exposing the token to the main container ([#8771](https://github.com/ai-dynamo/dynamo/pull/8771)). +- **Doubled Model Path Fix:** Fixed a bug where `build_dgd_config()` produced doubled model paths (e.g. `/opt/model-cache/opt/model-cache`) when a DGDR specified `modelCache.pvcName` without `pvcModelPath`, by splitting the PVC branch so that an unset `pvcModelPath` now mounts the PVC as an HF cache directory while workers receive the HF model ID instead of the mount path ([#9177](https://github.com/ai-dynamo/dynamo/pull/9177)). +- **Operator v1beta1 Conversion Fixes:** Fixed v1alpha1 ↔ v1beta1 status conversion to preserve legacy scalar `componentName` values during round-trips ([#9184](https://github.com/ai-dynamo/dynamo/pull/9184)), relaxed v1beta1 component-name validation to accept uppercase letters historically allowed in v1alpha1 service keys ([#9192](https://github.com/ai-dynamo/dynamo/pull/9192)), preserved the legacy DGD worker hash across conversion to prevent unnecessary rolling updates of existing v1alpha1 deployments ([#9210](https://github.com/ai-dynamo/dynamo/pull/9210)), and restructured DGD/DCD conversion helpers with exported structural methods and documented conversion rules in `CONVERSION.md` ([#9257](https://github.com/ai-dynamo/dynamo/pull/9257)). +- **LWS Native Scaling Deployment Fixes:** Refactored the DynamoComponentDeployment controller to create a single LeaderWorkerSet object with `Spec.Replicas` set to the desired count instead of one LWS per replica, removing legacy indexed-resource logic and adding cleanup for old resources ([#5468](https://github.com/ai-dynamo/dynamo/pull/5468)), fixed multinode deployments (vLLM / SGLang / TRT-LLM) by emitting `$(LWS_LEADER_ADDRESS)` Kubernetes env-var expansion syntax so the kubelet correctly substitutes the leader hostname in direct-python container args ([#8369](https://github.com/ai-dynamo/dynamo/pull/8369)), and restored the LeaderWorkerSet resource name to the `-0` form to avoid a service name collision between the operator-created ClusterIP service and the LWS-created headless service ([#9612](https://github.com/ai-dynamo/dynamo/pull/9612)). +- **Frontend Sidecar PVC Mounts:** Fixed the auto-generated frontend sidecar container in EPP/inference-gateway deployments to mirror the parent worker container's volume mounts, so PVC-backed model paths (tokenizer, config, chat-template) can resolve correctly instead of failing on file reads ([#8598](https://github.com/ai-dynamo/dynamo/pull/8598)). +- **vLLM Multiprocessing Init Gating:** Fixed vLLM multinode pod rendering so the `wait-for-leader-mp` init container is only injected when the rendered main container actually uses the `--distributed-executor-backend mp` flag, preventing worker pods on data-parallel or Ray-based paths from hanging indefinitely ([#9955](https://github.com/ai-dynamo/dynamo/pull/9955)). +- **Istio Sidecar Injection Exclusion:** Excluded Istio sidecar injection from kgateway-proxy pods by adding a GatewayParameters resource with `sidecar.istio.io/inject: "false"` in the install script, preventing HTTP 500 errors on ext_proc gRPC connections when namespace-level Istio injection is active ([#9839](https://github.com/ai-dynamo/dynamo/pull/9839)). +- **DCGM Discovery Namespace-Scoped Mode:** Enabled DCGM GPU discovery for namespace-scoped operator deployments by removing the hard-coded short-circuit in `validateGPUHardwareInfo` and granting cluster-wide pod read access via the gpu-discovery ClusterRole, for exporter pod detection regardless of operator scope ([#8365](https://github.com/ai-dynamo/dynamo/pull/8365)). +- **Deduplicate Concurrent GPU Discovery:** Added singleflight coordination to DCGM GPU discovery so concurrent reconciles for the same GPU SKU share one in-flight scrape, eliminating redundant DCGM calls and reducing discovery latency under contention ([#8797](https://github.com/ai-dynamo/dynamo/pull/8797)). +- **Istio Sidecar EPP Exclusion:** Excluded Istio sidecar injection from the EPP pod in both the standalone Helm chart and operator-managed deployments, preventing double-TLS handshake failures that caused `cx_connect_fail` errors and HTTP 500 responses when secure serving was enabled ([#9935](https://github.com/ai-dynamo/dynamo/pull/9935)). +- **Persist Discovered Hardware Metadata:** Fixed persistence of auto-discovered DGDR hardware metadata, including interconnect, RDMA capability, GPU SKU, and VRAM, before profiling job creation, then requeued for a clean second reconcile pass to ensure accurate deployment profiling in Kubernetes ([#9890](https://github.com/ai-dynamo/dynamo/pull/9890)). +- **Bounded Retry Per-Engine Port:** Fixed unbounded cuMemCreate OOM retry by adding a default 60s timeout with elapsed and free/total memory logging, and resolved port collisions for failover engines sharing a pod network namespace by overriding the forward-pass metric port per engine ([#8919](https://github.com/ai-dynamo/dynamo/pull/8919)). +- **Operator Go Module Fix:** Fixed the `go.mod` file in the operator deployment to update core dependencies for system stability and compatibility ([#8804](https://github.com/ai-dynamo/dynamo/pull/8804)). Needs verification. +- **Forward Pass Metric Port:** Injected `DYN_FORWARDPASS_METRIC_PORT` environment variable into worker pods, for forward pass metrics collection and Planner load scaling ([#8817](https://github.com/ai-dynamo/dynamo/pull/8817)). +- **Conditional Container Name Injection:** Fixed the Operator to inject `CONTAINER_NAME=main` only when Kubernetes discovery mode is set to `container`, preventing unnecessary rolling updates for deployments using the default pod discovery mode ([#9366](https://github.com/ai-dynamo/dynamo/pull/9366)). +- **ImagePullSecrets Drift Prevention:** Fixed imagePullSecrets drift during operator startup, ensuring consistent secret configuration across reconciliation cycles ([#9841](https://github.com/ai-dynamo/dynamo/pull/9841)). Needs verification. +- **Prefill Worker Failover Initialization:** Fixed prefill worker missing the `FlockFailoverLock` initialization that decode workers already performed, which caused prefill standbys to register immediately alongside the primary instead of waiting for failover ([#8367](https://github.com/ai-dynamo/dynamo/pull/8367)). +- **Drop Recreate Override Restore-Target:** Removed the hardcoded `Recreate` deployment strategy on restore-target worker Deployments in checkpoint-enabled DynamoComponentDeployments, so the default `RollingUpdate` applies so scaling no longer evicts the serving cold-start replica before new restore-target replicas are ready ([#8434](https://github.com/ai-dynamo/dynamo/pull/8434)). +- **Embedded Pod Template Metadata:** Enabled `generateEmbeddedObjectMeta=true` for Operator CRD generation so embedded Kubernetes `ObjectMeta` fields preserve labels and annotations, fixing API server pruning that broke DGD-level annotation propagation and prevented failover engine containers from receiving required environment variables ([#9553](https://github.com/ai-dynamo/dynamo/pull/9553)). +- **Restore Succeeded Condition Write:** Fixed a missing `setSucceededCondition` call in the operator's `updatePhaseWithCondition` path that was accidentally dropped during a cherry-pick to `release/1.2.0`, which caused the aggregate `Succeeded` condition to never be written during phase transitions and broke the post-merge operator CI job ([#10008](https://github.com/ai-dynamo/dynamo/pull/10008)). +- **GMS ResourceClaim Name Normalization:** Fixed GMS ResourceClaimTemplate naming to route CamelCase component names through `NormalizeKubeResourceName`, ensuring generated names comply with DNS-1123 subdomain validation and preventing Grove reconciliation failures ([#9829](https://github.com/ai-dynamo/dynamo/pull/9829)). +- **Pod Discovery Retry Logic:** Fixed a race condition in `get_deployment_logs()` where pod label selectors returned zero results because labels were not yet propagated after deployment readiness, by adding retry logic (up to 12 attempts, 5 seconds apart) with clear warning messages if pods remain undiscovered ([#9171](https://github.com/ai-dynamo/dynamo/pull/9171)). +- **Fail Fast on CrashLoopBackOff:** Fixed deployment wait logic to immediately detect and handle unrecoverable pod states such as CrashLoopBackOff, triggering automatic cleanup to avoid stalled profiling runs ([#9215](https://github.com/ai-dynamo/dynamo/pull/9215)). + +### Frontend & Agents + +- **Tool-Call Streaming Parser Robustness:** Preserved logprobs through the tool-call jailing layer so streaming responses no longer return `logprobs: null` ([#8072](https://github.com/ai-dynamo/dynamo/pull/8072)), fixed guided-decoding handling and corrected `finish_reason` to `tool_calls` per the OpenAI spec along with model-specific parser patches ([#8442](https://github.com/ai-dynamo/dynamo/pull/8442)), reconstructed tool-call arguments that were split across multiple streaming deltas instead of returning empty strings ([#8582](https://github.com/ai-dynamo/dynamo/pull/8582)), bypassed the reasoning parser for Qwen3 `enable_thinking=False` so parallel tool calls are no longer silently dropped ([#8589](https://github.com/ai-dynamo/dynamo/pull/8589)), prevented raw internal protocol tokens from leaking into content when tool-call parsing is interrupted ([#8820](https://github.com/ai-dynamo/dynamo/pull/8820)), and recovered nine silent-drop scenarios across top model parsers when `max_tokens` or EOS truncates the closing fence or JSON arguments mid-value ([#8888](https://github.com/ai-dynamo/dynamo/pull/8888)). +- **XML Parser Whitespace Parity:** Fixed the generic XML parser to drop normal text after tool-call blocks while preserving prefix separator whitespace, aligning qwen3_coder, nemotron_nano, and minimax_m2 parity fixtures with the upstream parser parity contract ([#9350](https://github.com/ai-dynamo/dynamo/pull/9350)). +- **Auto-Detect Force Reasoning Mode:** Added automatic detection of `force_reasoning` when a chat template's generation prompt ends with ``, ensuring the reasoning parser starts in reasoning mode to correctly separate thinking content from normal output ([#8240](https://github.com/ai-dynamo/dynamo/pull/8240)). +- **Responses API Wire Compliance:** Fixed the `/v1/responses` input-chain handling to accept Codex and Agents SDK tool-call round-trip shapes (`function_call` → assistant → `function_call_output`) that previously caused deserialization failures ([#8275](https://github.com/ai-dynamo/dynamo/pull/8275)), aligned `NvResponse` serialization with the OpenResponses spec by emitting `null` for nullable-required fields, injecting missing top-level parameters, defaulting image-input `detail` to `auto`, and adding compliance CI ([#8283](https://github.com/ai-dynamo/dynamo/pull/8283)), and removed the hardcoded 4 096-token `max_output_tokens` default that silently truncated reasoning-model outputs, deferring instead to the engine's own limit ([#9181](https://github.com/ai-dynamo/dynamo/pull/9181)). +- **Structured SSE Error Propagation:** Replaced bare or unspecified SSE error comments with properly structured `data: {"error":{...}}` frames followed by `data: [DONE]` on mid-stream faults, ensuring OpenAI-style clients receive actionable error messages instead of silent hangs ([#8430](https://github.com/ai-dynamo/dynamo/pull/8430)), and propagated actual backend error messages (e.g. multimodal image load failures) through streaming responses that previously surfaced only as "unspecified error" or premature disconnects ([#8674](https://github.com/ai-dynamo/dynamo/pull/8674)). +- **Kimi K2 Parser Correctness:** Fixed Kimi K2 tool-call parsing to recover partial tool calls when the model hits max_tokens before emitting the section_end marker, instead of silently discarding them ([#8208](https://github.com/ai-dynamo/dynamo/pull/8208)), back-ported upstream SGLang fixes to widen the tool-name regex to support hyphenated names and added a force-exit from reasoning mode when tool_start_token appears before `` ([#8532](https://github.com/ai-dynamo/dynamo/pull/8532)), and ensured the preprocessor preserves special tokens required by the kimi_k2 and kimi_k25 parsers so markers like `<|tool_calls_section_begin|>` and `` are not stripped during decoding ([#9227](https://github.com/ai-dynamo/dynamo/pull/9227)). +- **SGLang Tool Calling Update:** Updated the SGLang frontend processor's tool-calling functionalities to match the latest SGLang implementation, including guided decoding integration for required or named tool choices, streaming result buffering with fallback non-stream parsing, trust-remote-code passthrough, and special token preservation for tool-calling requests ([#8269](https://github.com/ai-dynamo/dynamo/pull/8269)). +- **Nemotron V3 Parser Parity:** Added `nemotron_v3` as a reasoning parser alias and ported vLLM Nemotron v3 reasoning parser test coverage into Dynamo's Rust implementation, aligning disabled-thinking behavior and streaming/non-streaming extraction paths with the vLLM contract ([#9058](https://github.com/ai-dynamo/dynamo/pull/9058)). +- **Chat Message Content Serialization:** Fixed non-streaming `/v1/chat/completions` responses to always serialize the `content` key as `null` when absent, instead of omitting it entirely, matching OpenAI's wire format and preventing client-side `KeyError` failures for reasoning-only or tool-call-only responses ([#8372](https://github.com/ai-dynamo/dynamo/pull/8372)). +- **KServe gRPC Readiness Race Fix:** Fixed a race condition in KServe gRPC `model_ready`/`server_ready` endpoints where readiness was reported as soon as a `ModelDeploymentCard` was registered, before the corresponding `WorkerSet` and engines were attached, causing clients to fail with "Connection closed unexpectedly" on immediate inference requests ([#9619](https://github.com/ai-dynamo/dynamo/pull/9619)). +- **Top Logprobs Token Detokenization:** Fixed missing `token` and `bytes` fields in `top_logprobs` responses from the SGLang backend by detokenizing `token_id` values in the Dynamo backend layer when the upstream engine returns `None` for decoded tokens, ensuring correct OpenAI-compatible logprobs output regardless of backend ([#8911](https://github.com/ai-dynamo/dynamo/pull/8911)). +- **Qwen 3.5 Tool Calling:** Fixed assistant tool-call argument handling for Qwen 3.5 models, which require special rendering when tool-calling messages contain a list of dicts, and improved error handling in the chat processor to surface similar issues faster ([#8792](https://github.com/ai-dynamo/dynamo/pull/8792)). +- **Default Skip Special Tokens:** Fixed the Rust backend's streaming detokenizer to default `skip_special_tokens` to `true` when the OpenAI request omits the field, aligning with upstream engines (vLLM, SGLang, TRT-LLM) and preventing special-token text such as `<｜begin▁of▁sentence｜>` from leaking into `content` or `reasoning_content` ([#8780](https://github.com/ai-dynamo/dynamo/pull/8780)). + +### Scheduling + +- **Planner Scheduling Operational Fixes:** Backfilled `max_num_batched_tokens` from discovery model cards when the VirtualConnector left it unset, unblocking load-based and throughput-based scaling in aggregated and prefill modes ([#8042](https://github.com/ai-dynamo/dynamo/pull/8042)), normalized model-name case comparisons in KubernetesConnector to prevent `CrashLoopBackOff` with mixed-case model identifiers ([#8384](https://github.com/ai-dynamo/dynamo/pull/8384)), corrected the profiler Job and example manifests to reference the standalone `dynamo-planner` image after runtime dependencies were split out ([#8407](https://github.com/ai-dynamo/dynamo/pull/8407)), tracked pending scaling targets in `GlobalPlannerConnector` so in-flight rollouts no longer trigger stacked scaling decisions ([#8422](https://github.com/ai-dynamo/dynamo/pull/8422)), exposed `get_worker_info` and fixed readiness signaling in multi-DGD GlobalPlanner topologies ([#8482](https://github.com/ai-dynamo/dynamo/pull/8482)), matched MDC `component` fields against the backend default instead of the PascalCase DGD service key to restore context-length resolution ([#8489](https://github.com/ai-dynamo/dynamo/pull/8489)), moved inventory and GPU-hour gauge publication out of the throughput-only tick path so Prometheus metrics are emitted in all planner modes ([#8575](https://github.com/ai-dynamo/dynamo/pull/8575)), delegated `wait_for_deployment_ready` to the pool-local KubernetesConnector so pool Planners no longer skip worker discovery ([#8694](https://github.com/ai-dynamo/dynamo/pull/8694)), and made SLA-mode scale-down decisions consolidation-aware by re-predicting per-survivor load to eliminate 2↔1 oscillation under steady traffic ([#9294](https://github.com/ai-dynamo/dynamo/pull/9294)). +- **Latency Ratio Scaling Floor:** Used the latency ratio (predicted TTFT divided by target) as a minimum replica floor in the Planner, so that latency-violation signals now drive scale-up decisions instead of being logged but ignored ([#8861](https://github.com/ai-dynamo/dynamo/pull/8861)). +- **Scale Status Budget Rejection:** Added a `ScaleStatus.REJECTED` status for GPU budget exceeded responses from the GlobalPlanner, so over-budget rejections now produce a warning log and continued operation rather than raising a RuntimeError ([#8774](https://github.com/ai-dynamo/dynamo/pull/8774)). +- **SGLang Prefill CUDA Graph:** Fixed SGLang prefill argument normalization to emit `--cuda-graph-bs 1` when prefill uses `--max-running-requests 1`, avoiding duplicated explicit CUDA graph settings in the rapid-path disaggregated generation path ([#9852](https://github.com/ai-dynamo/dynamo/pull/9852)). +- **Planner Config Serialization Fix:** Excluded environment-dynamic fields (namespace and Prometheus endpoint) from `planner_config.json` serialization so they are read from environment variables at Planner startup, and injected `PROMETHEUS_ENDPOINT` into profiler environment from Operator config ([#8805](https://github.com/ai-dynamo/dynamo/pull/8805)). +- **Prefill Avg ISL Decay:** Decayed the prefill regression average ISL tracker with idle Forward Pass Metrics (FPM) samples so stale long-prompt traffic no longer permanently blocks SLA-driven scale-down ([#9759](https://github.com/ai-dynamo/dynamo/pull/9759)). +- **Remove Double Tokenization EPP:** Removed redundant double tokenization in the EPP integration by upgrading the Gateway API Inference Extension (GAIE) dependency from v1.2.1 to v1.5.0-rc.2 and using upstream support for pre-computed token ID injection, so tokens are now computed once in GAIE and read directly in the frontend ([#8093](https://github.com/ai-dynamo/dynamo/pull/8093)). +- **Enable Priority Hints Routing:** Enabled priority hints in the Go Endpoint Picker (EPP) so that requests can forward priority information through the routing system to influence scheduling decisions ([#9353](https://github.com/ai-dynamo/dynamo/pull/9353)). +- **TRT-LLM Override Merge Fix:** Fixed a crash where thorough-mode profiler generated both `--override-engine-args` and `--trtllm.*` flags simultaneously, which TRT-LLM rejects as mutually exclusive, by introducing `_merge_overrides_into_args()` to detect and merge profiler overrides into an existing JSON blob instead of appending conflicting flags ([#9107](https://github.com/ai-dynamo/dynamo/pull/9107)). +- **Persist Endpoint After Handshake:** Fixed a routing-chain stall in the disaggregated frontend where transient decode-pod restarts caused permanent inference outages by persisting the prefill endpoint in the activator map after handshake completion, ensuring decode rebuilds can reactivate the `PrefillRouter` without waiting indefinitely; also resolved a stale `DecodeWaiting` entry left when decode registers before prefill and is removed before prefill arrives ([#8965](https://github.com/ai-dynamo/dynamo/pull/8965)). +- **Router Rejection Threshold Guard:** Guarded rejection logic with `None` when CLI busy-threshold flags (`--active-decode-blocks-threshold`, `--active-prefill-tokens-threshold`, `--active-prefill-tokens-threshold-frac`) are unset, preventing silent fallback values from triggering spurious `503 "All workers are busy"` rejections ([#8333](https://github.com/ai-dynamo/dynamo/pull/8333)). +- **Prefill DP Rank Balancing:** Fixed load imbalance across data-parallel ranks in prefill engines by generating `bootstrap_room` based on the prefill DP rank instead of a random value, ensuring decode engines correctly route to the intended rank ([#9080](https://github.com/ai-dynamo/dynamo/pull/9080)). +- **Cancel Replay Router Tasks:** Fixed the online replay router's LocalScheduler to cancel background tasks during shutdown, preventing replay-scoped tasks from outliving the router in tests and short-lived runtimes ([#8429](https://github.com/ai-dynamo/dynamo/pull/8429)). +- **Orphan Cleanup Race Guard:** Fixed a race condition in the KV Router where simultaneous startup caused `cleanup_orphaned_consumers` to see a transiently-empty active-instance set from discovery and incorrectly delete every peer's NATS consumer ([#9132](https://github.com/ai-dynamo/dynamo/pull/9132)). +- **Prefill Router Hash Mode:** Disabled EAGLE hash mode for the disaggregated prefill router so it uses prefill-side KV hash semantics instead of inheriting decode-only EAGLE settings, preventing prefill KV events from being indexed under one hash mode and queried under another ([#9871](https://github.com/ai-dynamo/dynamo/pull/9871)). +- **FPM ZMQ Port Offset Fix:** Fixed data-parallel ZMQ port assignment in vLLM instrumented scheduler by reading `data_parallel_index` instead of `data_parallel_rank`, which vLLM resets to 0 for dense models in external DP mode, preventing `Address already in use` errors when starting multiple DP children ([#8696](https://github.com/ai-dynamo/dynamo/pull/8696)). + +### Multimodal & Diffusion + +- **Disaggregated Omni Serving Pipeline Fixes:** Fixed multiple bugs preventing disaggregated Qwen2.5-Omni multi-worker serving, including connector routing gated incorrectly by the `final_output` flag and missing chat template handling so the full thinker→talker→code2wav pipeline completes correctly ([#8301](https://github.com/ai-dynamo/dynamo/pull/8301)), preserved caller-provided runtime device mappings for tensor-parallel workers instead of forcibly overwriting them ([#9034](https://github.com/ai-dynamo/dynamo/pull/9034)), added `--enforce-eager` to Wan2.2 video launchers to bypass a CUDA illegal memory access caused by `torch.compile` graph breaks in `WanSelfAttention` ([#9563](https://github.com/ai-dynamo/dynamo/pull/9563)), and removed the obsolete dummy `tokenizer.json` placeholder for TTS models that caused a race condition between cleanup and the frontend watcher ([#9954](https://github.com/ai-dynamo/dynamo/pull/9954)). +- **LLaVA E/P/D Deployment Fix:** Fixed LLaVA Encode/Prefill/Decode (E/P/D) deployment and added LLaVA test coverage for multimodal model profile configurations across multiple topology variants ([#8330](https://github.com/ai-dynamo/dynamo/pull/8330)). +- **Prompt Embeds Loading Fix:** Used vLLM to load prompt embeddings instead of custom logic, improving safety with stricter type and dimension validation checks ([#8228](https://github.com/ai-dynamo/dynamo/pull/8228)). +- **Multimodal Image Hash Collision:** Fixed `compute_mm_uuids_from_images` to include image geometry (width, height) in the blake3 hash preimage, preventing UUID collisions between RGB images with different dimensions but equal pixel counts ([#8341](https://github.com/ai-dynamo/dynamo/pull/8341)). +- **Encoder Model Memory Cap:** Capped `gpu_memory_utilization` to 0.2 for the nested vLLM engine in `load_vision_model()`, preventing a free-memory precheck failure on GPUs with ~24 GiB that caused encoder-only multimodal disaggregated tests to crash on CI ([#8466](https://github.com/ai-dynamo/dynamo/pull/8466)). +- **Qwen3-VL Disagg Multimodal Fix:** Backported vLLM PR #40932 to remove an invalid deepstack boundary check, unblocking Qwen3-VL disaggregated multimodal inference ([#9522](https://github.com/ai-dynamo/dynamo/pull/9522)). +- **SGLang Multimodal Encoder Mode:** Enabled `encoder_only` mode for SGLang multimodal encode workers, preventing the full model (including LLM weights) from loading and avoiding out-of-memory errors on GPUs with limited memory ([#9292](https://github.com/ai-dynamo/dynamo/pull/9292)). + +### Fault Tolerance & Observability + +- **Canary Health Check Reliability:** Made the canary health check the sole authority on endpoint readiness when enabled, preventing crash loops caused by eager `Ready` signals racing against canary verification ([#8165](https://github.com/ai-dynamo/dynamo/pull/8165)), replaced the fragile discovery/routing-based canary path with a direct in-process call via `LocalEndpointRegistry` to eliminate stale-pod and instance-id-mismatch failures in Kubernetes ([#8294](https://github.com/ai-dynamo/dynamo/pull/8294)), and reset the canary timer on request arrival and per streaming chunk to prevent false-negative health check triggers during long-context prefills and extended streaming responses ([#8467](https://github.com/ai-dynamo/dynamo/pull/8467)). +- **Preserve Original Model Casing:** Removed `.to_lowercase()` calls on the `model` label in `InflightGuard`, `ResponseMetricCollector`, and `HttpQueueGuard` so that all `dynamo_frontend_*` metric families emit the same casing as the originally-registered model card, ensuring dashboard filters match consistently across metric families ([#9953](https://github.com/ai-dynamo/dynamo/pull/9953)). +- **KV Publisher Metrics Registration:** Fixed the `kv_publisher` dropped-events counter so it is properly registered and exposed on `/metrics`, by removing a redundant `worker_id` variable label that collided with the runtime's auto-injected constant label of the same name ([#8660](https://github.com/ai-dynamo/dynamo/pull/8660)). + +### SGLang + +- **SGLang Disagg Prefill Health Probes:** Honored the `_HEALTH_CHECK` marker in the SGLang prefill handler so the canary probe detects a hung scheduler instead of returning a false-positive 200, and consolidated `HEALTH_CHECK_KEY` into the shared `dynamo.health_check` module for wire-format consistency across backends ([#8611](https://github.com/ai-dynamo/dynamo/pull/8611)), with a follow-up that aligned the injected `--cuda-graph-bs` value to the effective data-parallel size when `--max-running-requests 1`, preventing a CUDA-graph capture assertion that crashed the prefill worker under DP attention ([#9962](https://github.com/ai-dynamo/dynamo/pull/9962)). +- **SGLang Stop Token Forwarding:** Fixed the SGLang decode handler to pass `stop_token_ids_hidden` from the Rust frontend to the engine's sampling parameters, for proactive EOS detection and avoiding wasted compute on tokens generated past EOS when `skip_tokenizer_init=True` ([#8084](https://github.com/ai-dynamo/dynamo/pull/8084)). +- **Piecewise CUDA Graph Disabled:** Disabled piecewise CUDA graph in SGLang launch scripts (`agg.sh`, `agg_router.sh`, `disagg.sh`, `disagg_router.sh`) to prevent worker crashes caused by `CUBLAS_STATUS_EXECUTION_FAILED` during warmup when `--context-length` is smaller than the default piecewise bucket size ([#8609](https://github.com/ai-dynamo/dynamo/pull/8609)). +- **Guided Decoding Aggregated Serving:** Removed the `--skip-tokenizer-init` flag from the SGLang aggregated-serving launch script so the grammar backend initializes correctly, for `response_format: json_schema` and other constrained-decoding features in aggregated mode ([#8843](https://github.com/ai-dynamo/dynamo/pull/8843)). +- **Preserve Max New Tokens:** Fixed SGLang decode handler to preserve `max_new_tokens=None` in sampling params, preventing a silent 128-token output cap when clients omit `max_tokens` from chat completion requests, so generation now correctly continues until EOS ([#8743](https://github.com/ai-dynamo/dynamo/pull/8743)). +- **NVTX Profiling Dependency Fix:** Fixed the SGLang runtime image to ship the `nvtx` Python package and expose the `nsys` binary on PATH, resolving `ModuleNotFoundError` crashes and `command not found` errors when using `DYN_NVTX=1` profiling ([#8629](https://github.com/ai-dynamo/dynamo/pull/8629)). +- **Gated Routed Experts Kwarg:** Gated the `return_routed_experts` keyword argument behind an explicit `enable_return_routed_experts` opt-in flag and resolved it once at handler initialization rather than per-request, ensuring DeepSeek-V4 compatibility with older SGLang builds that lack the kwarg in their `async_generate` signature ([#8798](https://github.com/ai-dynamo/dynamo/pull/8798)). +- **SGLang Mooncake Dependency Fix:** Fixed a missing `libjsoncpp25` runtime dependency in the SGLang runtime Docker image that caused `ImportError` when importing the bundled Mooncake transfer engine ([#8645](https://github.com/ai-dynamo/dynamo/pull/8645)). + +### vLLM + +- **Stream Interval Flag Propagation:** Propagated vLLM's `--stream-interval` flag to the Dynamo frontend via `ModelRuntimeConfig.set_engine_specific()`, so the flag now takes effect in disaggregated serving mode instead of being silently ignored with a hardcoded default of 20 ([#8101](https://github.com/ai-dynamo/dynamo/pull/8101)). +- **Prefill Cancellation Log Race:** Fixed an asyncio race condition in `_monitor_abort` where `CancelledError` could interrupt `engine_client.abort()` mid-call, leaving the engine with a dangling request and preventing the "Aborted Request ID" completion log from appearing. The fix shields the abort coroutine from cancellation using `asyncio.shield` on the no-guard path and `asyncio.create_task` on the disaggregated-decode guard path, ensuring abort always runs to completion ([#8768](https://github.com/ai-dynamo/dynamo/pull/8768)). +- **Disagg Decode Benchmark Unblocked:** Fixed two independent bugs in `InstrumentedScheduler` decode benchmark mode: attached `kv_connector_metadata` to synthetic `SchedulerOutput`s (preventing `AssertionError` on workers with KV connectors in disaggregated deployments) and padded fake-decode prompts by one token to avoid the async sampler's `-1` placeholder causing out-of-vocabulary embedding lookups at batch sizes greater than one ([#9360](https://github.com/ai-dynamo/dynamo/pull/9360)). +- **Prompt Embeds Feature Guard:** Added validation to reject requests containing prompt embeddings when the feature is not globally enabled, preventing unexpected behavior in deployments that do not use this capability ([#8248](https://github.com/ai-dynamo/dynamo/pull/8248)). +- **Disagg Decode Queue Classification:** Fixed `InstrumentedScheduler._compute_queued` to iterate `self.skipped_waiting` alongside `self.waiting`, correctly classifying `WAITING_FOR_REMOTE_KVS` requests as queued decode work so the planner accurately observes decode-engine pressure in disaggregated serving ([#8471](https://github.com/ai-dynamo/dynamo/pull/8471)). +- **Deferred vLLM Request Cancellation:** Delayed vLLM request cancellation on the decode side until the engine produces its first token, preventing EngineCore crashes caused by aborting a request while NIXL KV transfer is still in flight during disaggregated serving ([#8624](https://github.com/ai-dynamo/dynamo/pull/8624)). +- **Max Thinking Tokens Mapping:** Fixed silent dropping of `max_thinking_tokens` at the vLLM worker by mapping it to vLLM 0.20+'s renamed `thinking_token_budget` field on `SamplingParams`, ensuring the thinking-budget logits processor correctly enforces the configured limit ([#9571](https://github.com/ai-dynamo/dynamo/pull/9571)). +- **Ray Dependency in vLLM Runtime:** Fixed missing Ray binary in the vLLM runtime container image by pinning Ray in `requirements.vllm.txt` and the `[vllm]` pyproject extras, preventing crash on pods using the Ray distributed executor backend after vLLM 0.19.1 removed Ray from its dependency graph ([#8515](https://github.com/ai-dynamo/dynamo/pull/8515)). + +### TensorRT-LLM + +- **NVRTC JIT Include Discovery:** Fixed TRT-LLM NVRTC JIT compilation failures on Blackwell (sm_100a) by installing `pip` into the runtime venv, so `pip show tensorrt_llm` can resolve the correct include path for kernel sources ([#8296](https://github.com/ai-dynamo/dynamo/pull/8296)). +- **Event Buffer Size Preserved:** Fixed the TensorRT-LLM worker to no longer override a user-provided `event_buffer_max_size` with the default value of 1024, ensuring custom engine configurations are respected and event publishing rates are not unintentionally affected ([#9284](https://github.com/ai-dynamo/dynamo/pull/9284)). +- **NIXL Initialization Mode Fix:** Skipped NIXL connector creation in aggregated mode where it is unused, and wrapped disaggregated-mode initialization in error handling so missing IB/RDMA hardware surfaces as a warning at init time instead of crashing on first request ([#9501](https://github.com/ai-dynamo/dynamo/pull/9501)). +- **TRT-LLM MDC Registration Fix:** Fixed TRT-LLM MDC registration so that `max_seq_len`, `max_batch_size`, and `max_num_tokens` supplied via `--extra-engine-args` or `--override-engine-args` are propagated back to `config` before handler and MDC registration, preventing MDC from advertising the model-native context length instead of the engine's actual limits ([#9130](https://github.com/ai-dynamo/dynamo/pull/9130)). +- **Router E2E Test Fix:** Fixed the TRT-LLM disaggregated router end-to-end test that was failing due to a missing `cache_transceiver_config` setting, causing workers to assert on the first request and re-enabled the test in nightly CI ([#8954](https://github.com/ai-dynamo/dynamo/pull/8954)). +- **Diffusion Multi-Image Response Fix:** Removed the restriction that truncated multi-image diffusion responses to a single entry and stopped rejecting `n > 1` requests; the engine now emits a warning when the pipeline's actual batch differs from the requested `num_images_per_prompt`, with a `1 ≤ n ≤ 10` validation guard on the handler side ([#9853](https://github.com/ai-dynamo/dynamo/pull/9853)). + +### KV Block Manager + +- **Single-Block Cache Match Speedup:** Optimized the KVBM prefix-matching path so that single-block lookups probe the first hash before allocating the full result vector, reducing overhead in the common one-block case ([#9196](https://github.com/ai-dynamo/dynamo/pull/9196)). +- **KVBM Consolidator E2E Tests:** Fixed consolidator end-to-end tests by adding the `--kv-events-config '{"enable_kv_cache_events": true}'` flag and setting `DYN_KVBM_DISK_DISABLE_O_DIRECT=true` in worker fixtures, preventing silent consolidator attachment failures and hangs on kernels lacking O_DIRECT support ([#8464](https://github.com/ai-dynamo/dynamo/pull/8464)). +- **KVBM Connector Import Update:** Updated KVBM's kv_cache_connector library imports to align with the module reorganization introduced in TensorRT-LLM 1.3.0rc14, dropping compatibility with earlier TensorRT-LLM versions ([#9622](https://github.com/ai-dynamo/dynamo/pull/9622)). +- **FlexKV Connector PdConnector Support:** Fixed a `TypeError` in `PdConnector` that prevented `FlexKVConnectorV1` from being used as the first connector when launching disaggregated FlexKV serving, by adding it to the allowed first connector types alongside `DynamoConnector` and `LMCacheConnectorV1` ([#8787](https://github.com/ai-dynamo/dynamo/pull/8787)). +- **Main Attention KV Block Size:** Fixed KV event block size resolution for vLLM to use the main-attention KV cache group metadata, ensuring hybrid Nemotron-style events align with the router/indexer block size instead of falling back to `cache_config.block_size` ([#9228](https://github.com/ai-dynamo/dynamo/pull/9228)). + +- **Replay Wall Time Accuracy:** Excluded report bookkeeping from replay wall-time measurement so the metric reflects actual replay execution rather than post-processing overhead ([#9190](https://github.com/ai-dynamo/dynamo/pull/9190)). +- **NIXL SDK Dev Stage:** Fixed the sglang dev container image by copying the NIXL C++ SDK (`libnixl_common`, UCX, libfabric, `gdrapi.h`) into the dev stage, resolving `rust-lld: error: unable to find library -lnixl_common` failures during source compilation ([#9216](https://github.com/ai-dynamo/dynamo/pull/9216)). +- **GAIE kgateway URL Update:** Fixed the GAIE kgateway URL to reflect the changed endpoint, ensuring correct gateway connectivity ([#9331](https://github.com/ai-dynamo/dynamo/pull/9331)). +- **Sync vLLM Version on Ref:** Fixed the `install_vllm.sh` script to derive `VLLM_VER` from `VLLM_REF` when `--vllm-ref` is passed, ensuring the correct vLLM version is installed instead of silently falling back to the default ([#8257](https://github.com/ai-dynamo/dynamo/pull/8257)). +- **Loosened Ray Version Pin:** Relaxed the `ray==2.55.0` dependency to `ray>=2.55.0` in the `dynamo[vllm]` extra and container requirements, so downstream consumers can use any compatible Ray version without version conflicts ([#9439](https://github.com/ai-dynamo/dynamo/pull/9439)). +- **Auditwheel Hidden Directory Copy:** Fixed the container build's dist-packages merge step to include dotfile directories (e.g., `.nixl_cu13.mesonpy.libs/`) by changing the glob from `*` to `.`, ensuring shared libraries like `libnixl.so` and `libserdes.so` are correctly copied into the SGLang venv and `import nixl` no longer fails ([#8966](https://github.com/ai-dynamo/dynamo/pull/8966)). +- **CuPy CUDA Version Matching:** Fixed runtime Dockerfiles for vLLM and TRT-LLM to dynamically select the CuPy package variant matching the build target CUDA version, replacing a hardcoded `cupy-cuda13x` that caused `ImportError: libcudart.so.12` failures on CUDA 12 builds ([#9379](https://github.com/ai-dynamo/dynamo/pull/9379)). +- **XPU Container Build Fix:** Fixed the vLLM runtime Dockerfile for XPU builds by guarding cupy-cuda installation behind a CUDA device check and removing leftover `nixl-cu*` packages that caused import failures in `dynamo.nixl_connect` ([#9419](https://github.com/ai-dynamo/dynamo/pull/9419)). +- **Worker Cleanup Shutdown Ordering:** Fixed shutdown ordering so engine cleanup (releasing GPU memory, tearing down PyTorch process groups) runs before the Rust runtime is shut down, and made cleanup idempotent so the signal-handler path and the Worker.run() finally path cannot execute it twice ([#8857](https://github.com/ai-dynamo/dynamo/pull/8857)). +- **Triton Backend Directory Default:** Fixed the Triton worker container image to set `BACKEND_DIR=/opt/tritonserver/backends` and updated the launch script to honor this environment variable, resolving a startup failure when using the container quick-start path instead of a local source build ([#8697](https://github.com/ai-dynamo/dynamo/pull/8697)). +- **Broken Readme References Removed:** Removed invalid `readme` declarations from `dynamo-runtime`, `dynamo-llm`, and `dynamo-protocols` Cargo manifests that referenced nonexistent files, unblocking `cargo package` and `cargo publish` for those crates ([#9809](https://github.com/ai-dynamo/dynamo/pull/9809)). +- **Cargo Lock Workspace Consistency:** Regenerated `Cargo.lock` to resolve internal version mismatches for `clap` and `rand` dependencies introduced by new workspace members, restoring `cargo metadata --locked` compatibility on `main` ([#8785](https://github.com/ai-dynamo/dynamo/pull/8785)). +- **Child Exit Code Propagation:** Fixed `wait_any_exit` in launch utilities to correctly propagate a failing child's exit code instead of silently overwriting it with `0` due to the EXIT trap's `kill 0` looping SIGTERM back to the script ([#8883](https://github.com/ai-dynamo/dynamo/pull/8883)). + +## Documentation + +### New Content + +- **Tool Calling Docs Reorganization:** Reorganized tool calling and reasoning documentation into dedicated top-level User Guides sections (`docs/tool-calling/`, `docs/reasoning/`), splitting content into Dynamo-native and engine-fallback subpages so each topic is self-contained and easier to discover ([#9400](https://github.com/ai-dynamo/dynamo/pull/9400)). +- **Chat Processor Options Page:** Added a new `chat-processor-options.md` overview page and expanded the tool-calling and reasoning parser tables with per-model detail, upstream name divergences (vLLM/SGLang), and cross-links between documentation pages ([#8497](https://github.com/ai-dynamo/dynamo/pull/8497)). +- **Fastokens Tokenizer User Guide:** Added a user-facing guide for the Fastokens tokenizer backend, covering integration details, enablement criteria, CLI/env-var configuration, model compatibility, verification steps, and troubleshooting, so operators can decide whether and how to enable `fastokens` on the Dynamo frontend ([#9430](https://github.com/ai-dynamo/dynamo/pull/9430)). +- **Agentic Harnesses Blog Post:** Added documentation covering prompt stability, reasoning fidelity, and streaming-dispatch optimizations at the harness boundary, detailing how Dynamo preserves KV prefix reuse, maintains interleaved reasoning/tool-call order, and emits explicit `tool_call_dispatch` SSE events for Claude Code, OpenClaw, and Codex integrations ([#7561](https://github.com/ai-dynamo/dynamo/pull/7561)). +- **Sharded Indexer Benchmarking Docs:** Added benchmarking documentation for the sharded indexer, including trace dataset download instructions, benchmark execution modes, output metrics reference, and CLI flags ([#8695](https://github.com/ai-dynamo/dynamo/pull/8695)). +- **Anchor-Aware BSI Benchmark Docs:** Added anchor-aware BSI benchmark results and commands to the KV router indexer documentation, covering steady-state, sweep, and repeated-overload runs with observed tradeoffs such as stronger routing structure, higher routing cost, and hot-branch collapse on dominant-prefix workloads ([#9275](https://github.com/ai-dynamo/dynamo/pull/9275)). +- **Kubernetes Documentation Restructuring:** Refactored the monolithic AKS guide into focused sub-pages (RDMA/InfiniBand, storage, Spot VMs) and rewrote the main setup flow as a four-step CLI guide, including a full RDMA setup walkthrough with known-issue documentation, and removed deprecated `dynamoNamespace` references now auto-computed by the Operator ([#8777](https://github.com/ai-dynamo/dynamo/pull/8777), [#9101](https://github.com/ai-dynamo/dynamo/pull/9101)). +- **EFA Setup Guide:** Added a guide for using Dynamo with the AWS Elastic Fabric Adapter (EFA) on EKS, covering prerequisites, device-plugin installation, kernel-module verification, LIBFABRIC backend configuration, environment variables, pod security settings, and troubleshooting ([#9521](https://github.com/ai-dynamo/dynamo/pull/9521)). +- **Disaggregated Communication Guide:** Updated the disaggregated communication guide with AWS EFA benchmark results, including 9.6 GB/s KV transfer bandwidth and 31% lower inter-token latency measured on p5.48xlarge instances, along with expanded EFA configuration details, a kernel compatibility matrix, and EAGAIN troubleshooting guidance ([#7764](https://github.com/ai-dynamo/dynamo/pull/7764)). +- **GPU Memory Service Page:** Added a dedicated GPU Memory Service Kubernetes documentation page clarifying its experimental status, current limitations with Snapshot, and decision guidance for users choosing between GMS, Snapshot, and failover ([#9119](https://github.com/ai-dynamo/dynamo/pull/9119)). +- **ModelExpress with ModelStreamer:** Expanded the Kubernetes model-caching guide with customer-facing ModelExpress peer-to-peer instructions, showing how ModelStreamer is selected via `MX_MODEL_URI` for S3, GCS, Azure Blob Storage, and local/PVC safetensors paths ([#9417](https://github.com/ai-dynamo/dynamo/pull/9417)). +- **Planner Replay Benchmarking Guide:** Added a guide for benchmarking the Dynamo Planner in replay mode, covering configuration knobs, aggregated vs disaggregated invocation, output artifacts, remote HTML viewing workflows, parallel sweep tips, and a case study showing an engine-startup-time sweep that reveals an SLA cliff near 100 to 120 s startup ([#8641](https://github.com/ai-dynamo/dynamo/pull/8641)). +- **Unified Backend Guides:** Added Rust and Python unified backend guides, with follow-up fixes correcting a test link, a `CARGO_BIN_NAME` compile error in integration tests, and a hang in the channel-less `generate` template when delay is zero ([#9581](https://github.com/ai-dynamo/dynamo/pull/9581)). +- **Observability Metrics Docs:** Documented the full set of `dynamo_component_*` metric labels, component values, endpoints, and error types to remove ambiguity around Dynamo-emitted versus Kubernetes-injected labels, corrected the `DYN_SYSTEM_PORT` default in the health-checks table, and explained Prometheus label-warmup behavior so users understand why labeled metrics only appear after the first matching request ([#8478](https://github.com/ai-dynamo/dynamo/pull/8478), [#8545](https://github.com/ai-dynamo/dynamo/pull/8545), [#8834](https://github.com/ai-dynamo/dynamo/pull/8834)). + +## Looking Ahead + +The following is planned for the next release, v1.3.0. See the public [Dynamo roadmap](https://github.com/ai-dynamo/dynamo/issues/9178) for the full plan. + +### DeepSeek-V4 for Agentic Coding + +Performant DeepSeek-V4 recipes across vLLM and SGLang on Hopper and Blackwell, tuned for agentic-coding workloads with KV-cache reuse and disaggregated serving. + +### Pluggable, Topology-Aware Planner + +A re-architected Planner with a plugin framework for custom scaling logic, extending toward topology-aware placement that accounts for interconnect and node layout across heterogeneous worker pools. + +### CRIU GPU Snapshots + +Dynamo Snapshot's CRIU-based GPU process checkpoint and restore matures toward faster, model-reload-free recovery at scale, paired with operator-driven checkpoint lifecycle management. + +### Realtime Streaming I/O + +A bidirectional `/v1/realtime` request plane lays the foundation for streaming input and output, including voice and multimodal streaming. + +## Patch releases + +### v1.2.1 — Jun 13, 2026 + +#### Summary + +Dynamo v1.2.1 is a patch release on top of v1.2.0, focused on **ModelExpress 0.4.0 engine-side model loading** (including object-storage model sources), **AMD ROCm / Python 3.10 import compatibility**, **EFA container build fixes**, and backend correctness fixes for **SGLang** and the **gpt-oss-120b recipe**. + +**Base Branch**: `release/1.2.1` + +#### Features & Improvements + +- **ModelExpress 0.4.0 Integration:** Added engine-side ModelExpress model loading to the vLLM and SGLang runtimes (#10578). The runtime images now ship `modelexpress==0.4.0` by default (installed with `--no-deps` so the upstream engine dependency stacks are untouched), the new vLLM path is owned by the ModelExpress vLLM plugin, and the legacy Dynamo-owned `--model-express-url` / `MODEL_EXPRESS_URL` wrapper is retained only as deprecated compatibility parsing. + +#### Bug Fixes + +- **ModelExpress Object-Storage Loading:** Fixed a model-load stall on the vLLM and SGLang ModelExpress / RunAI Model Streamer object-storage path (`--model s3://…`, also `gs://` / `az://`) (#10674). Avoided constructing the vLLM `ModelConfig` twice (which ran duplicate `__post_init__()` side effects during engine startup) and made `register_model()` use the engine's pulled local directory rather than re-resolving the object-storage URI. +- **SGLang Routed-Experts Encoding:** Fixed a crash on the first decoded token when `--enable-return-routed-experts` is used with non-DeepSeek-V4 MoE models on SGLang 0.5.11+ (#10543). SGLang v0.5.11 moved the base64 encoding of `routed_experts` upstream into `tokenizer_manager`, so the Dynamo decode handler no longer re-encodes the already-encoded string; `nvext.routed_experts` is emitted as a base64 UTF-8 string at both emit sites. +- **ROCm / Python 3.10 Import Compatibility:** Fixed two import-time failures that blocked `import dynamo.*` on AMD ROCm / Python 3.10 hosts (#10545). `nixl_connect` now defers the CUDA-only NIXL `ImportError` until first use so the router, planner, and frontend import cleanly on hosts without a NIXL wheel, and `dynamo.common.configuration` uses `typing_extensions.Self` instead of `typing.Self` for Python 3.10. CUDA behavior is unchanged. +- **gpt-oss-120b Recipe Revert:** Reverted the gpt-oss-120b TensorRT-LLM aggregated recipe to runtime image `1.0.0` to restore the prior recipe baseline (#10549). + +#### Build, CI and Test + +- **EFA Container Build Fixes:** Backported two EFA container build fixes to the release branch (#10425): bumped `nixl_gdrcopy_ref` to v2.5.2 for Linux kernel ≥6.15 compatibility, and built and overlaid libfabric v2.5.1 (the first release with the CUDA dmabuf fix for GB200 EFA) onto the EFA installer's stock binary. Together these restore the Dynamo EFA RDMA container build. + + + +Between v1.1.1 and v1.2.0, the project merged 603 PRs from 82 authors. Thank you to the external community contributors in this release (organization confirmed via commit-author email, cross-referenced against the team roster): + +- **Intel:** @dsocek, @sywangyi, @tthakkal, @sandeep-maddipatla, @VincyZhang, @ZhengHongming888. +- **Microsoft:** @Jont828, @ashnamehrotra, @devivasudevan, @avinashpenmetsa. +- **Baseten:** @the-david-oy, @michaelfeil. +- **vLLM:** @ywang96 (Roger Wang). **SGLang:** @ch-wan (Cheng Wan). +- **Roblox:** @navmarri14. +- **Independent:** @Ayobami-00, @Kaonael. + +If you would like to get involved, please see our [Contribution Guide](https://docs.nvidia.com/dynamo/dev/getting-started/contribution-guide). + + diff --git a/docs/fern/reference/release-notes/v1-3-0.mdx b/docs/fern/reference/release-notes/v1-3-0.mdx new file mode 100644 index 000000000000..34a88b8d87ac --- /dev/null +++ b/docs/fern/reference/release-notes/v1-3-0.mdx @@ -0,0 +1,449 @@ +--- +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +title: Dynamo v1.3.0 +subtitle: Release notes for Dynamo v1.3.0 (GA Jul 20, 2026) +--- + +import { ReferenceStyles } from "@/components/ReferenceStyles"; +import { ReleaseHeader } from "@/components/ReleaseHeader"; +import { ReleaseSummaryCards } from "@/components/ReleaseSummaryCards"; +import { UpgradePanel } from "@/components/UpgradePanel"; +import { PinnedEnvironment } from "@/components/PinnedEnvironment"; + + + + + +Dynamo v1.3.0 is the 16th feature release of the open-source distributed inference platform. It delivers the largest **Dynamo Router** buildout to date (a standalone selection service, the **Branch-Sharded KV Indexer**, compressed-radix-tree hot-path speedups, and topology-aware routing), overhauls **tool-calling and reasoning** into a configurable parser layer aligned to vLLM and SGLang across the model fleet, and deepens **Kubernetes deployment** around a production **GPU Memory Service** and DynamoGraphDeployment (DGD) `v1beta1` admission. It also expands **performance modeling and replay** with AIC latency prediction and Mooncake trace replay, and moves the platform to **CUDA 13** with refreshed vLLM, SGLang, and TensorRT-LLM engines and **NIXL v1.x**. + + +Breaking changes and deprecations for this release are tracked on the [Deprecations](../deprecations.mdx#v130) ledger; known issues on the [Known Issues](../known-issues.mdx#v130) page. Key dependency pins live on [Compatibility](../compatibility.mdx); shipped artifacts on [Release Artifacts](../release-artifacts.mdx). Model early access builds (`vX.Y.Z--dev.N`) are tracked in [Model Early Access Builds](../model-early-access-builds.mdx). + + + + +## Get v1.3.0 + +Pull, deploy, and install with every artifact pinned to the v1.3.0 release set. + + + +## Highlights + + + +## Features & Improvements + +### Frontend + +- **GPT-OSS Harmony Parser Fixes:** Corrected the Harmony parser to preserve analysis text as normal content when reasoning parsing is unconfigured ([#9729](https://github.com/ai-dynamo/dynamo/pull/9729)), enabled GPT-OSS tool calling from Codex by retaining the `<|call|>` terminator and resolving Jinja `.items()` schema conflicts ([#9778](https://github.com/ai-dynamo/dynamo/pull/9778)), corrected thinking behavior ([#9844](https://github.com/ai-dynamo/dynamo/pull/9844)), handled bare Harmony stream markers starting directly at the commentary channel ([#9897](https://github.com/ai-dynamo/dynamo/pull/9897)), and routed Harmony reasoning, visible text, and tool-call handoff into the correct Dynamo fields by channel and recipient ([#10054](https://github.com/ai-dynamo/dynamo/pull/10054)). Preserved the directed commentary tool-call handoff to the downstream Harmony tool parser ([#10111](https://github.com/ai-dynamo/dynamo/pull/10111)), and recovered tool calls emitted on the analysis channel using the recipient as the definitive signal ([#10366](https://github.com/ai-dynamo/dynamo/pull/10366)). +- **Experimental Parsers V2 Routing:** Routed Qwen3-Coder and DeepSeek-V4 tool calls through the dynamo-parsers-v2 streaming parser behind a new `DYN_ENABLE_EXPERIMENTAL_PARSERS_V2` flag (off by default); the v2 parser owns incremental tool-call emission and drops parameter values truncated at EOF ([#10853](https://github.com/ai-dynamo/dynamo/pull/10853)). Added reasoning-parser parity cases for the "1 open + 2 close" pattern and fixed BasicReasoningParser and Gemma4ReasoningParser to keep stray close markers out of normal_text ([#10250](https://github.com/ai-dynamo/dynamo/pull/10250)). +- **Structural Tag Tool Calls:** Added automatic xgrammar structural tag generation for tool-calling requests, constraining decoding in each model's native tool-call format to enforce `required`/`named` tool choice, strict function schemas, `parallel_tool_calls=false`, and `tool_choice="none"` token bans. Disabled by default, configurable via `--dyn-enable-structural-tag`, supporting the hermes, qwen3_coder, deepseek_v3_2, and deepseek_v4 parsers ([#9711](https://github.com/ai-dynamo/dynamo/pull/9711)). +- **Skip Special Tokens Warning:** Added a one-time diagnostic warning in the OpenAIPreprocessor when a request forces `skip_special_tokens=true` while a special-token-dependent parser is active, a combination that silently strips parser markers and leaks tool-call or reasoning markup into content ([#10225](https://github.com/ai-dynamo/dynamo/pull/10225)). +- **Realtime API Endpoint Build-Out:** Wired the `/v1/realtime` WebSocket endpoint through ModelManager with a first-class `ModelType::Realtime` endpoint kind and bidirectional PushRouter transport ([#9308](https://github.com/ai-dynamo/dynamo/pull/9308)), and replaced the placeholder chat-completion shape with OpenAI Realtime API event types, exchanging `RealtimeClientEvent` and `RealtimeServerEvent` frames and synthesizing a spec-compliant `session.created` event on connect ([#9205](https://github.com/ai-dynamo/dynamo/pull/9205)). +- **OpenAI Embeddings Format Support:** Added `dimensions` Matryoshka truncation to the vLLM `/v1/embeddings` worker, validating and slicing each embedding ([#9751](https://github.com/ai-dynamo/dynamo/pull/9751)), then added end-to-end `encoding_format=base64` support: the Rust frontend accepts array and base64 shapes and both SGLang and vLLM backends emit the requested form ([#9887](https://github.com/ai-dynamo/dynamo/pull/9887)). +- **MiniMax M3 Frontend Support:** Added MiniMax M3 support to the OpenAI frontend: `thinking`/`thinking_mode` handling for `enabled`, `disabled`, and `adaptive` modes, parser alias normalization, special-token and trailing EOS handling, prior tool-call argument validation, and MiniMax-M3-VL multimodal context-length detection ([#10983](https://github.com/ai-dynamo/dynamo/pull/10983)). +- **Unified Backend Logits and LogProbs:** Added `dynamo.common.backend.logprobs` as the single source of truth for logprob option parsing and per-chunk extraction, wiring logprobs end-to-end on the unified path so `output_options` reaches `sampling_params` and chunks carry `log_probs` and `top_logprobs` keys ([#10149](https://github.com/ai-dynamo/dynamo/pull/10149)), and brought the custom logits-processor smoke path and shared serializable wire format to the unified vLLM and SGLang backends ([#10224](https://github.com/ai-dynamo/dynamo/pull/10224)). +- **Frontend nvext Switches and Relocation:** Added frontend master switches for the `nvext` and admin-API surfaces, letting operators close off non-OpenAI-spec surfaces on the frontend HTTP service without affecting inference, metrics, models, or health probes (both on by default) ([#10556](https://github.com/ai-dynamo/dynamo/pull/10556)), and moved typed `NvExt` handling out of the OpenAI module into the LLM common extension layer, keeping `dynamo-protocols` extension-agnostic and adding Anthropic Messages ingress support for `nvext.agent_context` ([#10784](https://github.com/ai-dynamo/dynamo/pull/10784)). +- **Dynamo Routing Header Standardization:** Added `x-dynamo-request-priority` and `x-dynamo-request-strict-priority` HTTP headers as alternatives to `nvext.agent_hints`, letting OpenAI and Anthropic clients set scheduling priority without modifying request bodies, behind `DYN_DISABLE_FRONTEND_NVEXT` ([#10871](https://github.com/ai-dynamo/dynamo/pull/10871)), then standardized every Dynamo-owned public routing header on the `x-dynamo-*` namespace while keeping the original unprefixed worker and DP-rank names as compatibility aliases across EPP, ext-proc, frontend, tests, and docs ([#10873](https://github.com/ai-dynamo/dynamo/pull/10873)). +- **HTTP Header Metadata Propagation:** Extracted HTTP request headers matching a configurable prefix into the context metadata map, so tenants and routing systems can inject key-value metadata at the HTTP boundary and flow it end-to-end across HTTP, streaming, and gRPC endpoints. Oversized or invalid header metadata now returns a clear "request headers too large" error ([#9726](https://github.com/ai-dynamo/dynamo/pull/9726)). +- **LoRA Controller and Routing:** Added a LoRA controller with HRW/MCF placement, filtered request routing, metrics, discovery, and configuration wiring across the vLLM and SGLang backends. Preserved session affinity when LoRA is disabled, rejected incompatible direct and advanced routes, and returned a resource-exhausted response when every worker in a selected replica set is saturated ([#8180](https://github.com/ai-dynamo/dynamo/pull/8180)). +- **Admission Control Flag Replacement:** Replaced the boolean `--no-admission-control` opt-out flag with an enum-valued `--admission-control {token-capacity,none}` flag defaulting to `none`, so operators must now explicitly opt in to busy-worker rejection. `DYN_NO_ADMISSION_CONTROL` gave way to `DYN_ADMISSION_CONTROL` ([#9694](https://github.com/ai-dynamo/dynamo/pull/9694)). +- **Configurable Overload Status Code:** Added the `DYN_HTTP_OVERLOAD_STATUS_CODE` environment variable to set the frontend's HTTP status code for overload rejections, with the default `529` unchanged. Deployments behind proxies or clients that only understand `503` can now select that code without a code change ([#11420](https://github.com/ai-dynamo/dynamo/pull/11420)). +- **L1 Tokenizer Prefix Cache:** Added an opt-in `CachedTokenizer` that records prefix tokenizations at special-token boundaries and re-encodes only the diverging suffix, enabled per-frontend via `DYN_TOKENIZER_CACHE=1` with a configurable budget and `/metrics` hit and miss counters ([#9742](https://github.com/ai-dynamo/dynamo/pull/9742)), then extended it with partial-hit handling for growing multi-turn conversations, a moka W-TinyLFU backend, single-pass Aho-Corasick boundary detection, and broadened tests across tokenizer families ([#10201](https://github.com/ai-dynamo/dynamo/pull/10201)). +- **Streaming Hot Path Optimization:** Optimized the OpenAI chat SSE streaming path by replacing the per-chunk flat_map to Vec to stream::iter plumbing with a single async_stream adapter that reuses event storage and serializes JSON directly ([#10433](https://github.com/ai-dynamo/dynamo/pull/10433)), applied the same rework to the Responses streaming adapter so it owns the converter and serializes events with borrowed Serialize wrappers ([#10498](https://github.com/ai-dynamo/dynamo/pull/10498)), and to the Anthropic Messages streaming adapter to drain a reusable event buffer from a single async stream while preserving error handling, disconnect monitoring, and keepalive behavior ([#10499](https://github.com/ai-dynamo/dynamo/pull/10499)). +- **Frontend Hot-Path Optimizations:** Applied three profile-guided optimizations to the frontend hot path: returning the shared `Arc` instead of cloning the cached prefix on a tokenizer L1 cache hit, caching per-model Prometheus metric handles in `ResponseMetricCollector` rather than resolving them per chunk and per token, and capping the `log` crate at compile-time `debug` so per-character `trace!` events from the HF tokenizers crate compile to no-ops in release. Stock-tokenizer-path throughput improved roughly 1.45x, from about 189 to about 275 requests per second ([#10273](https://github.com/ai-dynamo/dynamo/pull/10273)). +- **Self-Host Sibling Metadata Harvesting:** Harvested non-weight sibling files (preprocessor_config.json, special_tokens_map.json, added_tokens.json, tokenizer.model) into slug_dir at the end of resolve_metadata_files to fix KV routing regressions for from_pretrained consumers ([#9610](https://github.com/ai-dynamo/dynamo/pull/9610)), had the worker advertise non-typed siblings through a new ModelDeploymentCard.extra_files field for custom directories without an HF source ([#9707](https://github.com/ai-dynamo/dynamo/pull/9707)), made the vLLM and SGLang native preprocessors consume the resolved slug_dir instead of re-running fetch_model ([#10037](https://github.com/ai-dynamo/dynamo/pull/10037)), and had the MetadataArtifactRegistry auto-clean entries on detach to stop leaks on LoRA detach or model reload ([#10351](https://github.com/ai-dynamo/dynamo/pull/10351)). +- **Disagg Processor Engine Support:** Routed the vLLM preprocessor through a new Rust RoutedEngine that wraps Client or KV router with PrefillRouter for disaggregated serving ([#9503](https://github.com/ai-dynamo/dynamo/pull/9503)), extended the same path to the SGLang processor and added structural_tag for guided decoding on the Rust side ([#9577](https://github.com/ai-dynamo/dynamo/pull/9577)), and added request migration failover to the vLLM and SGLang processor paths ([#9617](https://github.com/ai-dynamo/dynamo/pull/9617)). +- **Disaggregated Topology Readiness:** Migrated the worker set key to include `worker_type`, removed the legacy `ModelType::Prefill` bit, registered encode workers, and added per-model topology gating in the frontend ([#9815](https://github.com/ai-dynamo/dynamo/pull/9815)), plus a read-only `GET /v1/models/{model}/ready` endpoint exposing structured per-namespace worker readiness for a model ([#10383](https://github.com/ai-dynamo/dynamo/pull/10383)). +- **Worker Type Topology Scaffolding:** Added a `WorkerType` enum with `Prefill`, `Decode`, `Encode`, and `Aggregated` roles, plus `worker_type` and `needs` fields on the model deployment card and live readiness methods on `Model`, a first step toward making the frontend aware of disaggregated serving topology before serving requests ([#8626](https://github.com/ai-dynamo/dynamo/pull/8626)). +- **Python-Rust PyO3 Bridge:** Exposed the runtime's bidirectional engine path to Python so an async generate worker can be registered and driven end-to-end via PyAsyncRequestStream and serve_bidirectional_endpoint ([#10135](https://github.com/ai-dynamo/dynamo/pull/10135)), propagated HTTP-status-carrying exceptions through the boundary so unsupported image formats return their intended status instead of HTTP 500 ([#10364](https://github.com/ai-dynamo/dynamo/pull/10364)), and removed the cyclic Python to env to Rust config side channels in the frontend by passing resolved config through the PyO3 contract as explicit Rust fields ([#10446](https://github.com/ai-dynamo/dynamo/pull/10446)). + +### Agents + +- **Agent Request Trace Capture:** Added agent traces to the v1/completions endpoint and refactored the internal streaming trace infrastructure ([#9125](https://github.com/ai-dynamo/dynamo/pull/9125)), plus optional finish_reason_metadata recording backend stop metadata, final chat and completions finish metadata, and tool-call metadata without storing arguments, surfaced in the Perfetto conversion ([#9817](https://github.com/ai-dynamo/dynamo/pull/9817)). +- **Header-Only Agent Trajectory Identity:** Added a deterministic program-close signal via `nvext.agent_context.trajectory_final`, with a symmetric resume INFO log and scheduler docs for `dynamo.thunderagent_router` ([#10172](https://github.com/ai-dynamo/dynamo/pull/10172)), and mapped trajectory identity into SGLang as a radix-cache session, normalizing an explicit final header into `kv_hints.evict_trajectory` ([#10214](https://github.com/ai-dynamo/dynamo/pull/10214)). Made `trajectory_id` the only required identity field and added the generic `x-dynamo-trajectory-id` HTTP fallback ([#10800](https://github.com/ai-dynamo/dynamo/pull/10800)), removed `nvext.agent_context` from public request bodies for a header-only contract parsed at the HTTP boundary ([#10808](https://github.com/ai-dynamo/dynamo/pull/10808)), and preserved the immediate Claude parent for nested subagents via `x-claude-code-parent-agent-id` ([#10942](https://github.com/ai-dynamo/dynamo/pull/10942)). + +### Reinforcement Learning + +- **RL Tokens-In-Tokens-Out Support:** Added the Rust `lib/llm` nvext Tokens-in-Tokens-Out protocol, preprocessor, and response plumbing, carrying `nvext.token_data`, `cache_salt`, `extra_fields`, token constraints, `prompt_logprobs`, detokenize, and stop token IDs through the request and response path, validating passthrough `cache_salt` and `stop_token_ids` while rejecting unsupported `truncate_prompt_tokens` ([#9649](https://github.com/ai-dynamo/dynamo/pull/9649)), plus a read-only `GET /v1/rl/workers` discovery endpoint in the new `dynamo-rl` crate gated behind `DYN_ENABLE_RL` and `DYN_RL_PORT` ([#9681](https://github.com/ai-dynamo/dynamo/pull/9681)). +- **vLLM RL Token-In/Token-Out Support:** Added vLLM worker-side support for nvext Tokens-in-Tokens-Out requests, forwarding RL/TITO flags through the CLI and runtime config, preserving cache salt on decode and disaggregated prefill prompts, and exposing completion token IDs, logprobs, and engine data parity fields ([#9651](https://github.com/ai-dynamo/dynamo/pull/9651)), registered worker RL admin routes under `/engine/` with LoRA request validation ([#9680](https://github.com/ai-dynamo/dynamo/pull/9680)), and added NeMo-RL NCCL weight updates ([#11034](https://github.com/ai-dynamo/dynamo/pull/11034)). +- **SGLang RL Metadata Uploads:** Added an opt-in path to upload large reinforcement-learning serving metadata (routed expert captures, top-logprobs, tensors, NumPy arrays) as zstd-compressed msgpack objects through fsspec storage backends. Gated by SGLang RL mode and a per-request `nvext.metadata_upload.url` field, so callers can scope rollout metadata to any installed fsspec target ([#10034](https://github.com/ai-dynamo/dynamo/pull/10034)). + +### Multimodal & Diffusion + +- **Multimodal KV Routing Support:** Added MM-aware KV routing on the SGLang backend via per-image pad_value substitution with auto-detected backend identity ([#9561](https://github.com/ai-dynamo/dynamo/pull/9561)), tightened post-merge tests to fail closed when routing degrades and fixed the Qwen2-VL, Qwen2.5-VL, and fastokens paths ([#9441](https://github.com/ai-dynamo/dynamo/pull/9441)), dropped Phi-3 from MM-aware routing since its sliding-window KV events are not admitted to the contiguous-prefix index ([#10441](https://github.com/ai-dynamo/dynamo/pull/10441)), and added the Qwen3.5/3.6 and Kimi K2 families with a tightened pre-merge gate ([#10999](https://github.com/ai-dynamo/dynamo/pull/10999)). +- **SGLang Multimodal Disaggregated Support:** Forwarded image and video inputs to both SGLang disaggregated prefill and decode workers so media is encoded and KV-aligned over NIXL instead of dropped and decoded as text ([#10643](https://github.com/ai-dynamo/dynamo/pull/10643)), normalized multimodal content shapes on the SGLang chat-processor path so chat templates bind media placeholders ([#10673](https://github.com/ai-dynamo/dynamo/pull/10673)), rejected multimodal requests when a PD worker lacks the multimodal-worker flag ([#10680](https://github.com/ai-dynamo/dynamo/pull/10680)), and unified multimodal E/P/D configuration flags across backends with deprecation warnings for the prior backend-specific flags ([#10690](https://github.com/ai-dynamo/dynamo/pull/10690)). +- **Encoder-Result Handoff Contract:** Added an engine-opaque, object-only `encoder_result` payload to `PreprocessedRequest`, `LLMEngineOutput`, and `BackendOutput`, with producer helpers in Rust and a new `dynamo.common.backend.multimodal` Python module, the foundation for multimodal Encode worker to Prefill peer handoff ([#10969](https://github.com/ai-dynamo/dynamo/pull/10969)). +- **Cache-Aware Embedding Device Routing:** Integrated EmbeddingCache awareness into the heterogeneous EPD weighted device routing so cache-hit requests are excluded from the CUDA-to-CPU ratio accounting and fall back to round-robin ([#8750](https://github.com/ai-dynamo/dynamo/pull/8750)), and added video embedding cache support to the SGLang multimodal encode worker, storing video-specific metadata and config-aware video cache keys while preserving image cache behavior ([#8298](https://github.com/ai-dynamo/dynamo/pull/8298)). +- **OpenAI Embeddings Wire Performance:** Added the OpenAI `/v1/embeddings` `dimensions` field with Matryoshka truncation in the SGLang handler ([#9722](https://github.com/ai-dynamo/dynamo/pull/9722)), submitted batched prompts concurrently via `asyncio.gather` so vLLM's scheduler coalesces them into a single forward pass ([#10117](https://github.com/ai-dynamo/dynamo/pull/10117)), and switched the internal worker-to-frontend wire format to base64-encoded bytes, decoded back to floats at the HTTP boundary, dropping latency at `--batch-size 15` from 215 ms to 124 ms on arm64 GB200 ([#10139](https://github.com/ai-dynamo/dynamo/pull/10139)). +- **Diffusion Unified Backend Onboarding:** Onboarded diffusion image and video generation onto the unified backend, so raw-media engines for TensorRT-LLM and SGLang share the same Worker lifecycle as token LLM engines: signal handling, graceful drain, 3-phase shutdown, and health checks ([#10371](https://github.com/ai-dynamo/dynamo/pull/10371)). + +### Scheduling + +#### KV Router + +- **Branch-Sharded KV Indexer:** Added the `AsyncShardHandle` trait and generalized `BranchShardedIndexer` over in-process and future remote shard backends ([#9763](https://github.com/ai-dynamo/dynamo/pull/9763)); removed the scheduler hop from in-process anchored reads ([#10200](https://github.com/ai-dynamo/dynamo/pull/10200)); added O(1) anchor cleanup and worker-id lookup via per-worker indexes ([#10302](https://github.com/ai-dynamo/dynamo/pull/10302)); and borrowed shard-read suffix slices instead of per-lookup allocations ([#10495](https://github.com/ai-dynamo/dynamo/pull/10495)). +- **Standalone KV-Router Selection Service:** Added a runtime-independent slot-tracker microservice with HTTP worker registration, hash-based lifecycle accounting, and advisory load reads ([#10291](https://github.com/ai-dynamo/dynamo/pull/10291)), plus optional ZMQ replica sync for lifecycle events and dynamic peer management ([#10394](https://github.com/ai-dynamo/dynamo/pull/10394)). Added a runtime-free selection service with worker-catalog reconciliation, selection and reservation APIs, and scoring endpoints ([#10641](https://github.com/ai-dynamo/dynamo/pull/10641)), extended it with startup recovery from HTTP indexer peers, best-effort replica sync, and Mooncake-style token-overlap summaries ([#10745](https://github.com/ai-dynamo/dynamo/pull/10745)), and exposed the selection core as a feature-gated Python class for in-process use ([#10766](https://github.com/ai-dynamo/dynamo/pull/10766)). +- **Router Overlap Score Handling:** Added a KvRouter overlap-scores API and standalone router endpoint returning per-worker tiered KV overlap with optional shared-cache hit context ([#9538](https://github.com/ai-dynamo/dynamo/pull/9538)); refreshed overlap scores at dequeue time for requests whose scheduler queue wait exceeds a threshold so stale scores no longer misroute workers ([#9663](https://github.com/ai-dynamo/dynamo/pull/9663)); and added opt-in decay of device-local overlap credit under active prefill load, defaulting to off and configurable through Rust, Python, CLI, and environment ([#10574](https://github.com/ai-dynamo/dynamo/pull/10574)). +- **Topology-Aware Routing Constraints:** Added worker taints and request-side `RoutingConstraints` for KV router selection, enforcing `required_taints` and biasing traffic with `preferred_taints` ([#9558](https://github.com/ai-dynamo/dynamo/pull/9558)); added typed topology metadata and KV-transfer enforcement fields in `ModelRuntimeConfig` plus canonical topology-taint generation ([#9767](https://github.com/ai-dynamo/dynamo/pull/9767)); and propagated topology constraints to decode from the selected prefill worker, failing closed only for required KV-transfer topology policy when prefill-worker attribution is unavailable ([#9893](https://github.com/ai-dynamo/dynamo/pull/9893)). +- **Global Router SLA Retries:** Added priority-retry configuration for global-router pool failures, retrying prefill, decode, and aggregated requests on faster pools before streaming starts, with documented pool priorities, defaults, and unit coverage ([#9460](https://github.com/ai-dynamo/dynamo/pull/9460)), and moved per-request target TTFT and ITL fields into a schema-checked `router` sub-field under `nvext` ([#9845](https://github.com/ai-dynamo/dynamo/pull/9845)). +- **KV-Router Recovery Hardening:** Added a bounded local pending-live-event buffer for KV-router recovery and split the local-indexer recovery code into focused modules ([#9881](https://github.com/ai-dynamo/dynamo/pull/9881)); added batched replica-sync side effects, cancellation-aware push routing, and ghost-booking prevention ([#10331](https://github.com/ai-dynamo/dynamo/pull/10331)); and added per-rank cancellation tokens that abort in-flight recovery when a worker is removed ([#10616](https://github.com/ai-dynamo/dynamo/pull/10616)). +- **Lower-Tier Cache-Hit Scoring:** Fixed the KV-router lower-tier walk to seed `query_lower_tiers` from primary-only `MatchDetails`, then merge side-indexer scores into the device output, so side-only worker-block credit is no longer dropped from host and disk cache-hit math ([#9797](https://github.com/ai-dynamo/dynamo/pull/9797)); exposed `host_cache_hit_weight` and `disk_cache_hit_weight` through the `dynamo.frontend` and `dynamo.router` CLI and `DYN_ROUTER_*` env vars at existing defaults of 0.75 and 0.25 ([#10157](https://github.com/ai-dynamo/dynamo/pull/10157)); and routed vLLM native CPU-offload KV events to the HostPinned tier with lower-tier applies counted in `kv_cache_events_applied` ([#10368](https://github.com/ai-dynamo/dynamo/pull/10368)). +- **KV-Router Hot-Path Optimizations:** Reduced block-hash cloning in router-lookup and route-time recording paths ([#9960](https://github.com/ai-dynamo/dynamo/pull/9960)); replaced per-block approximate-prune heap entries with 100ms expiry buckets for a 13.7% throughput gain on the bs64 c384 multi-frontend AgentX benchmark ([#10521](https://github.com/ai-dynamo/dynamo/pull/10521)); and reused caller-owned child-traversal storage during CRTC lookup repair while skipping redundant same-node worker updates ([#10540](https://github.com/ai-dynamo/dynamo/pull/10540)). Changed compressed-edge remove bookkeeping to return only the newly uncovered hash range for a ~28x speedup in store/remove event processing ([#10676](https://github.com/ai-dynamo/dynamo/pull/10676)), and batched CRTC remove handling for hashes on the same compressed edge with a split-safe per-hash fallback ([#10858](https://github.com/ai-dynamo/dynamo/pull/10858)). +- **Active Sequence Load Tracking:** Optimized scheduler load projection by replacing the prompt registry's `DashMap` load table with a dense worker load table and updating `active_sequences_bench` to measure the production `project_worker_loads` path ([#10935](https://github.com/ai-dynamo/dynamo/pull/10935)); added incremental worker-overload tracking in `KvWorkerMonitor` via a dedicated `OverloadedWorkerTracker` that updates only the touched worker per `ActiveLoad` event ([#10645](https://github.com/ai-dynamo/dynamo/pull/10645)); and reduced hot-path locking by switching per-worker load entries to `SeqLock` and the worker load table to `IndexMap` ([#10967](https://github.com/ai-dynamo/dynamo/pull/10967)). +- **DP-Rank Session Affinity:** Promoted sticky-session affinity from worker to `(worker, dp_rank)` granularity so a session pinned to a multi-DP-rank engine stayed on the rank where its prefix was warm ([#9920](https://github.com/ai-dynamo/dynamo/pull/9920)); added header-based session identity via `X-Dynamo-Session-ID` with Claude Code, Codex, and OpenCode fallbacks, router-local atomic first-dispatch affinity with idle expiry and active-request leases, and an independent `--router-session-affinity-ttl-secs` setting ([#10875](https://github.com/ai-dynamo/dynamo/pull/10875)). +- **Unified Routing Log Output:** Standardized worker selection across all router modes into one structured info-level "Selected worker" log with consistent fields ([#10554](https://github.com/ai-dynamo/dynamo/pull/10554)), and reclassified prefill capacity rejections from error to warn so routine load-shed backpressure is no longer logged as a failure ([#10561](https://github.com/ai-dynamo/dynamo/pull/10561)). +- **Strict Priority Tiers in KV-Router:** Added `nvext.agent_hints.strict_priority` as a per-request pending-queue tier ordering higher values before lower, preserving FCFS, LCFS, and WSPT as secondary ordering within each tier and propagating the hint through push and picker routing, standalone requests, EPP, C JSON routing, and Python `KvRouter.best_worker` ([#10638](https://github.com/ai-dynamo/dynamo/pull/10638)), and fixed the `strict_priority` call sites in the selection service that broke the build on main ([#10689](https://github.com/ai-dynamo/dynamo/pull/10689)). +- **Tiered Router Queue Backpressure:** Added a configurable KV Router queue-depth backpressure option that caps per-worker queue depth by request cost (missing prefill tokens) and returns immediate rejections when caps are exceeded, shedding expensive low-cache-hit requests first under load. A new Prometheus counter tracks router backpressure by worker type and reason ([#8144](https://github.com/ai-dynamo/dynamo/pull/8144)). +- **Min-Cost Flow LoRA Placement:** Added a churn-minimizing LoRA-placement solver that assigns LoRA adapters to workers via a min-cost flow algorithm with top-M candidate generation, delta freezing, and a configurable cost function ([#8179](https://github.com/ai-dynamo/dynamo/pull/8179)). +- **Router Predict-On-Route Colocation:** Added `--router-predict-on-route` and `--router-predicted-ttl-secs` options so the KV Router co-locates burst-arrival sibling requests (parallel sampling, best-of-N, agent fan-out) on the worker that picked the first sibling, closing the window between the routing decision and the engine's first block-stored event. When KV events are enabled, the Router runs a secondary short-TTL approximate indexer and scores each worker with the per-worker max overlap of both trees ([#8276](https://github.com/ai-dynamo/dynamo/pull/8276)). +- **Experimental ThunderAgent Router:** Added the `thunderagent_router` service that wraps the native KvRouter with program-level pause/resume scheduling for agentic workloads, applying per-worker capacity accounting and request-boundary admission control. It is experimental and opt-in; requests lacking `nvext.agent_context.trajectory_id` pass through unchanged ([#9448](https://github.com/ai-dynamo/dynamo/pull/9448)). +- **KV-Aware Routing Unification:** Added KV event and metrics source descriptors to the unified backend abstraction so engines declare publishers through a shared interface, with the Rust Worker owning publisher construction, the metrics poll loop, and shutdown ordering. Migrated vLLM, SGLang, and TensorRT-LLM onto one component naming convention for KV-aware routing ([#9493](https://github.com/ai-dynamo/dynamo/pull/9493)). +- **Stable Worker Routing ID:** Added a `stable_routing_id` field on `ModelRuntimeConfig` that survives process restarts, sourced from the `DYN_STABLE_ROUTING_ID` or `HOSTNAME` environment variables and published through the worker's existing etcd discovery entry, giving caching layers a persistent worker identity so Kubernetes StatefulSet pod restarts no longer trigger cache rebalancing and cold-start misses ([#9665](https://github.com/ai-dynamo/dynamo/pull/9665)). +- **Workers Endpoint Filtering:** Enriched the GET /workers response with model name, tenant ID, and block size fields, added optional model_name and tenant_id query parameters for filtering, and fixed a block_size=0 time-of-check-to-time-of-use race by skipping mid-deregistration workers ([#9983](https://github.com/ai-dynamo/dynamo/pull/9983)). +- **Optional Binary-Search Indexer Mode:** Added an optional binary-search algorithm to the KV Router's `PositionalIndexer::find_matches`, selectable via the `DYN_ROUTER_POSITIONAL_SEARCH_MODE` environment variable or API, cutting index probes for long query-block lengths while producing byte-identical results to the default strided mode ([#10181](https://github.com/ai-dynamo/dynamo/pull/10181)). +- **Standalone Router Timing Propagation:** Forwarded per-request timing from a standalone KV-router to the frontend on the data payload, so `prefill_time_ms`, `ttft_ms`, `prefill_wait_time_ms`, `kv_hit_rate`, and `queue_depth` reach the nvext `timing` field, Prometheus metrics, and agent traces in split deployments ([#10182](https://github.com/ai-dynamo/dynamo/pull/10182)). +- **Modeled Prefill Time Reads:** Added a Rust-internal slot-tracker read for modeled remaining prefill time in the KV Router, exposing per-worker projected prefill consumption in signed milliseconds for load visibility while keeping unmodeled workers on the fast error path ([#10190](https://github.com/ai-dynamo/dynamo/pull/10190)). +- **Router Load and Request APIs:** Added a potential-loads API, exposed active request counts to external APIs, and allowed markPrefill and markFree to execute without a context ID for uniform behavior, aiding cross-region balancing and detection of overloaded or queueing workers ([#10412](https://github.com/ai-dynamo/dynamo/pull/10412)). +- **KV Indexer Access Logging:** Added structured JSONL access logging to the standalone KV indexer via a `--access-log` option recording timestamp, trace-id, method, path, model, status code, and duration per request, plus `--trace-id-header` and `--access-log-local-time` flags and a `/reopen_logs` endpoint for logrotate integration, improving production observability and request tracing ([#10700](https://github.com/ai-dynamo/dynamo/pull/10700)). +- **Standalone Indexer Metrics Export:** Registered the core KvIndexerMetrics collectors in the standalone indexer's Prometheus registry, so event warnings and errors now export through `/metrics`, matching the native Dynamo path ([#10768](https://github.com/ai-dynamo/dynamo/pull/10768)). + +#### Planner & Profiler + +- **Configurable Prometheus Client Auth:** Made Planner's Prometheus client TLS verification configurable via `PROMETHEUS_SSL_VERIFY`, defaulting to off to preserve existing behavior ([#9510](https://github.com/ai-dynamo/dynamo/pull/9510)); added a custom CA-bundle path for private and internal roots ([#9511](https://github.com/ai-dynamo/dynamo/pull/9511)), a static bearer token ([#9512](https://github.com/ai-dynamo/dynamo/pull/9512)), and a bearer-token file re-read on every request for rotating credentials ([#9513](https://github.com/ai-dynamo/dynamo/pull/9513)); and fixed extra URL query parameters on every PromQL request for tenancy-enforcing front-ends ([#9557](https://github.com/ai-dynamo/dynamo/pull/9557)). +- **MTP Accept Length Scaling:** Added MTP accept-length scaling to the Planner, publishing backend spec-decode metadata in the ModelDeploymentCard and scraping speculative accept length from worker Prometheus metrics to discount decode ITL without altering raw OSL, KV, or context estimates, and rebuilding AIC perf models when capabilities change ([#10435](https://github.com/ai-dynamo/dynamo/pull/10435)); added replay support carrying accept length through mocker engine passes, PSM replay, orchestrator plugin metrics, and the plugin proto schema, and fixed PSM replay tick time to use the scheduled replay clock ([#10501](https://github.com/ai-dynamo/dynamo/pull/10501)). +- **Attention-DP Forward Pass Metrics:** Enabled TensorRT-LLM ForwardPassMetrics under Attention-DP by removing the Dynamo-side gate that disabled them when attention_dp_size exceeded one, so the Planner now receives one metrics stream per Attention-DP rank and can detect load imbalance ([#9059](https://github.com/ai-dynamo/dynamo/pull/9059)). +- **Planner Load Optimization Target:** Added an optimization_target='load' option for the Planner with configurable prefill queue token and decode KV utilization thresholds, forcing reactive load scaling and disabling throughput scaling with a warning ([#9590](https://github.com/ai-dynamo/dynamo/pull/9590)). +- **Planner Gzip Diagnostics Logs:** Added a compressed JSONL diagnostics sidecar written next to each Planner HTML report by default, capturing per-tick snapshots; a `report_write_gzip_log` setting disables it, and writes stay best-effort so gzip failures warn instead of breaking the HTML report ([#9623](https://github.com/ai-dynamo/dynamo/pull/9623)). +- **Planner Recommended Replica Markers:** Added recommended prefill and decode replica markers to the Planner HTML diagnostics Replica Counts plot, and documented local Planner advisory mode as suggestion-only for evaluating configurations without scaling ([#9644](https://github.com/ai-dynamo/dynamo/pull/9644)). +- **SLA Target Metrics Exposure:** Added `sla_target_ttft_ms` and `sla_target_itl_ms` Gauges to the Planner's Prometheus metrics, publishing operator-configured SLA targets so Grafana dashboards can compare observed and estimated latency against the SLA boundary ([#10032](https://github.com/ai-dynamo/dynamo/pull/10032)). +- **Planner Plugin Framework Infrastructure:** Added an opt-in tick-engine path for the Planner that runs decisions through a PREDICT, PROPOSE, RECONCILE, CONSTRAIN pipeline driven by in-process or gRPC plugins, enabled per planner via `scheduling.use_orchestrator=true` while the legacy PlannerStateMachine path stays the default. This infrastructure-only change ships proto and Pydantic types, transport clients, a plugin registry and scheduler, merge algorithms, and static-config and gRPC self-register plugin registration, with builtin plugins to follow ([#10124](https://github.com/ai-dynamo/dynamo/pull/10124)). +- **Rust Engine Perf Shim:** Routed SLA Planner performance queries through a new PlannerEnginePerfModel adapter backed by the Rust engine perf model, with legacy Python regression fallback when the optional Rust shim is unavailable, and relaxed planner bootstrap requirements so missing pre-deployment data no longer blocks throughput mode ([#10229](https://github.com/ai-dynamo/dynamo/pull/10229)). + +### Performance Modeling & Replay + +#### Mocker & Simulation + +- **KVBM Offload Replay Simulation:** Added mocker support for KVBM G3 offload simulation following the G1 to G2 to G3 tiering path with staged promotion, and a first-admission prefix-cache reused-ratio metric in replay reports ([#9337](https://github.com/ai-dynamo/dynamo/pull/9337)); modeled G4 object-storage offload as shared infinite-capacity storage with G2-to-G4 pipeline and bandwidth configs ([#9939](https://github.com/ai-dynamo/dynamo/pull/9939)); and extended the Rust-native offline-replay benchmark with aggregated and disaggregated topology selection, KVBM capacity and bandwidth options, worker-initialization error propagation, and benchmark documentation ([#11010](https://github.com/ai-dynamo/dynamo/pull/11010)). +- **AIC Latency Prediction Integration:** Added a feature-gated AIC forward-pass engine perf shim to the mocker, exposing `RustEnginePerfModel` through PyO3 ([#10150](https://github.com/ai-dynamo/dynamo/pull/10150)), and used AIC memory estimates to size KV cache blocks while preserving the legacy 16384 fallback when AIC is disabled ([#9598](https://github.com/ai-dynamo/dynamo/pull/9598)). Switched latency prediction to the AIC Rust crate as a GIL-free callback on the hot path ([#10615](https://github.com/ai-dynamo/dynamo/pull/10615)), forwarded AIC quantization-mode overrides so replay latency and capacity track quantized deployments ([#10914](https://github.com/ai-dynamo/dynamo/pull/10914)), scaled `num_gpu_blocks` by `attention_dp_size` for DP-attention KV provisioning ([#10964](https://github.com/ai-dynamo/dynamo/pull/10964)), and divided the scheduled batch by `attention_dp_size` for correct per-rank perf queries in offline replay ([#11002](https://github.com/ai-dynamo/dynamo/pull/11002)). +- **TensorRT-LLM Scheduler Simulation:** Added TensorRT-LLM scheduler simulation to the mocker via `engine_type=trtllm`, modeled as a scheduling-policy variant of the vLLM scheduler core with a GUARANTEED_NO_EVICT policy and AIC perf-model integration ([#10193](https://github.com/ai-dynamo/dynamo/pull/10193)), and terminal rejection of unschedulable requests whose footprint exceeds the KV pool, propagated through every replay driver via a new `rejected` flag on `OutputSignal` ([#10287](https://github.com/ai-dynamo/dynamo/pull/10287)). +- **Mocker Replay Performance Tuning:** Routed one-worker aggregated offline replay through `SingleRuntime` for vLLM, SGLang, and TRT-LLM across flat, workload, concurrency, and agentic entrypoints while retaining `AggRuntime` for multi-worker, KV-router, planner-driven, and disaggregated replay, and added cross-runtime parity coverage and an `--engine-type` option to `offline_replay_bench` ([#10629](https://github.com/ai-dynamo/dynamo/pull/10629)); and reduced scheduler and FPM publish overhead by skipping timerfd construction for consumed deadlines, reusing the publisher task's MessagePack buffer, and caching the event-plane subject ([#11025](https://github.com/ai-dynamo/dynamo/pull/11025)). +- **Eagle Speculative-Decoding Perf Model:** Added MTP/Eagle speculative-decoding support to the AIC perf model used by mocker and replay, exposing the `aic_nextn` draft-token count and `aic_nextn_accept_rates` per-position accept rates through the CLI, Python bindings, and mocker serde so AIC applies the speculative-decode decode speedup during latency modeling ([#10197](https://github.com/ai-dynamo/dynamo/pull/10197)). +- **MTP Burst Sampling:** Added speculative-decoding (Multi-Token Prediction) support to the mocker engines with conditional acceptance sampling and atomic burst reservations, exposed through new options `--aic-nextn`, `--aic-nextn-accept-rates`, and `--aic-mtp-seed` across live and replay paths ([#10436](https://github.com/ai-dynamo/dynamo/pull/10436)). +- **Atomic Source Hold Primitive:** Added a scheduler-owned source-hold registry with a stable HandoffId to the mocker, letting vLLM and SGLang retain completed request state and defer terminal cleanup so release and cancel clean up exactly once, even for early or duplicate commands ([#10831](https://github.com/ai-dynamo/dynamo/pull/10831)). +- **Disaggregated Handoff Lifecycle:** Replaced the mocker's synthetic second-request approximation of disaggregated serving with one explicit handoff lifecycle covering source hold, destination reservation, transfer, activation, release, completion, cancellation, and failure, shared across offline replay and live bootstrap. It preserves vLLM source-first and SGLang destination-first admission semantics and surfaced latent fixes in virtual-time visibility, offload wakeups, cancellation cleanup, worker retirement, and queue accounting ([#10915](https://github.com/ai-dynamo/dynamo/pull/10915)). +- **Unified KV-Cache Estimator:** Routed the mocker and replay `num_gpu_blocks` estimator through aiconfigurator's unified `sdk.memory.estimate_num_gpu_blocks` API instead of hand-rolled per-backend budget math, making aiconfigurator the single source of truth. SGLang block counts grew slightly more conservative by subtracting full non-KV memory; vLLM and TensorRT-LLM counts were unchanged ([#10686](https://github.com/ai-dynamo/dynamo/pull/10686)). + +#### Trace Replay + +- **Mooncake Trace Capture and Replay:** Added per-request Mooncake trace capture for Rust OpenAI chat and completion serving via the `DYN_REQUEST_TRACE` environment variable, with rolling sequence hashes, partial output length on cancellation, and configurable trace sinks ([#10381](https://github.com/ai-dynamo/dynamo/pull/10381)), introduced a mooncake-delta trace format that accumulates session input deltas into cumulative prompts before recomputing engine block hashes ([#9653](https://github.com/ai-dynamo/dynamo/pull/9653)), added an experimental agentic Mooncake replay path with dependency edges, branch markers, and tool wait timing ([#9728](https://github.com/ai-dynamo/dynamo/pull/9728)), and fixed mooncake-delta prompt reconstruction to include the previous prompt and generated output in each follow-up turn ([#11060](https://github.com/ai-dynamo/dynamo/pull/11060)). +- **Planner Replay Goodput Metrics:** Added SLA-satisfying goodput and provisioned GPU-hours to the planner-replay report, classifying each completed request against an optional SLA using aiperf-matched ITL, TTFT, and e2e bounds independent of the planner scaling SLA ([#10739](https://github.com/ai-dynamo/dynamo/pull/10739)); exposed the goodput SLA on the plain trace-replay binding so static non-autoscaling replays also report the goodput keys ([#10849](https://github.com/ai-dynamo/dynamo/pull/10849)); and supported concurrency-capped and synthetic workloads in the planner-in-the-loop path beyond arrival-timestamp traces ([#10888](https://github.com/ai-dynamo/dynamo/pull/10888)). +- **Offline Replay Event-Driven Path:** Unified offline replay onto one event-driven run loop where a planner tick is a first-class `PlannerTick` event, fixing an unbounded FPM-snapshot memory leak on the plain run path ([#10940](https://github.com/ai-dynamo/dynamo/pull/10940)), added a `--max-sim-time-minutes` flag that ends the benchmark at the specified simulated wall time across disagg and agg paths ([#9699](https://github.com/ai-dynamo/dynamo/pull/9699)), and changed `--replay-concurrency` to schedule multi-turn sessions depth-first as a session cap, admitting a new session only when an active one finishes ([#10516](https://github.com/ai-dynamo/dynamo/pull/10516)). +- **Replay Per-Request Metrics:** Added a `--report-jsonl` option to offline `dynamo.replay` that emits one JSON object per completed request with TTFT, ITL, ISL/OSL, cache hit rate, and worker index ([#9720](https://github.com/ai-dynamo/dynamo/pull/9720)); added native vLLM/SGLang-style Prometheus metrics for the live mocker covering request timing, cache hit rates, queue status, and preemption counts ([#10056](https://github.com/ai-dynamo/dynamo/pull/10056)); and trimmed online-replay diagnostics by demoting `replay_diag` breadcrumbs to debug level and removing test timing prints ([#9565](https://github.com/ai-dynamo/dynamo/pull/9565)). +- **Output Token Replay:** Added deterministic output-token replay for Dynamo mockers, where enriched Mooncake traces supply exact `output_token_ids` and live requests select a replay row via an `output_replay_id:` annotation, retaining the random-token fallback when no replay plan is present ([#10877](https://github.com/ai-dynamo/dynamo/pull/10877)). +- **Claude Session Trace Export:** Added export of local Claude Code root and subagent sessions as canonical `dynamo.request.trace.v1` request and tool events, with explicit source-to-tool-to-child causality, making historical sessions replayable while an always-on verifier fails on fidelity drift ([#10971](https://github.com/ai-dynamo/dynamo/pull/10971)). + +### Kubernetes Deployment + +- **GMS Weight Loading Integration:** Integrated ModelExpress P2P weight transfer into the GMS loader and worker via a LoadStrategyChain for automatic weight-source detection ([#8218](https://github.com/ai-dynamo/dynamo/pull/8218)), and cut GMS load overhead by skipping torch import on the server startup path and bounding NIXL staging workers ([#9635](https://github.com/ai-dynamo/dynamo/pull/9635)). Auto-enabled the SGLang memory saver path for GMS startup ([#9647](https://github.com/ai-dynamo/dynamo/pull/9647)), used the DP-adjusted device for the early vLLM GMS connection ([#9840](https://github.com/ai-dynamo/dynamo/pull/9840)), refreshed vLLM KV cache wake state after GMS remaps VAs ([#10319](https://github.com/ai-dynamo/dynamo/pull/10319)), set a 512 MiB default scratch alias size for shadow-engine scratch mappings ([#10329](https://github.com/ai-dynamo/dynamo/pull/10329)), and dropped the implicit failover requirement from standalone inter-pod GMS in the Operator ([#10378](https://github.com/ai-dynamo/dynamo/pull/10378)). +- **GMS Sidecar Lifecycle Rework:** Coordinated GMS sidecar lifecycles through the lock state machine instead of kubelet probes and init-phase ordering, dropping the gms-server StartupProbe and fixing a saver Job-completion race and a too-short 30 second read-only reconnect timeout ([#9514](https://github.com/ai-dynamo/dynamo/pull/9514)). Reworked the checkpoint-client API around user-declared Kubernetes containers so the Operator no longer creates gms-loader or gms-saver containers ([#9641](https://github.com/ai-dynamo/dynamo/pull/9641)), and made the _allocator_ext C++ extension PEP 703 free-threading ready by replacing torn-pointer callback globals with a magic-statics singleton ([#9575](https://github.com/ai-dynamo/dynamo/pull/9575)). +- **Snapshot Restore Probing and Runtime:** Reworked snapshot checkpoint and restore handling to gate checkpoints on target-container readiness with an OCI runtime container-ID fallback ([#9534](https://github.com/ai-dynamo/dynamo/pull/9534)), tightened restore-target StartupProbe cadence so containers flip Ready as soon as CRIU restore completes ([#9627](https://github.com/ai-dynamo/dynamo/pull/9627)), and let restore pods start container-ID polling before Kubernetes marks the pod Running ([#9984](https://github.com/ai-dynamo/dynamo/pull/9984)). Refreshed restore-time runtime env before vLLM and SGLang create the DistributedRuntime and split the snapshot utils into focused modules ([#10376](https://github.com/ai-dynamo/dynamo/pull/10376)), added a CRIU UPDATE_INETSK plugin that remaps IPv4 socket addresses across pod IPs and defaulted established TCP socket restore ([#10727](https://github.com/ai-dynamo/dynamo/pull/10727)), and applied fast 1s worker probes only once checkpointInfo is Ready, bounding restore startup probes to 30 minutes ([#10859](https://github.com/ai-dynamo/dynamo/pull/10859)). +- **DGD Auto-Checkpoint Lifecycle Management:** Scoped DynamoGraphDeployment automatic checkpoints to the owning DGD component worker generation instead of legacy identity-hash reuse, removing the need for `checkpoint.identity` in `mode: Auto` ([#10177](https://github.com/ai-dynamo/dynamo/pull/10177)). Added a `checkpoint.startupPolicy` and Pod CREATE mutating webhook that defaults workers to `Immediate` cold start while the checkpoint job runs in the background ([#10179](https://github.com/ai-dynamo/dynamo/pull/10179)), and a `checkpoint.deletionPolicy` with finalizer-based artifact cleanup that defaults to deleting DGD-managed checkpoint CRs and stored artifacts on DGD deletion ([#10183](https://github.com/ai-dynamo/dynamo/pull/10183)). +- **TensorRT-LLM Snapshot Support:** Added TensorRT-LLM support for Dynamo Snapshot restore, delaying `DistributedRuntime` creation until after restore and reloading the Kubernetes restore identity before endpoint registration, plus a Qwen3-0.6B single-GPU example ([#10432](https://github.com/ai-dynamo/dynamo/pull/10432)). Introduced the namespaced `PodSnapshot` and cluster-scoped `PodSnapshotContent` CRDs (`nvidia.com/v1alpha1`) with a `PodSnapshotReconciler` modeled on the CSI VolumeSnapshot pattern ([#10820](https://github.com/ai-dynamo/dynamo/pull/10820)). +- **Grove DGD Integration:** Preserved existing Grove PodCliqueSet replica fields during DGD reconciliation so live scaling stays driven through Grove scale subresources instead of PCS template updates ([#9773](https://github.com/ai-dynamo/dynamo/pull/9773)). Added a DynamoGraphDeployment `priorityClassName` field passed through to the Grove PodCliqueSet and rejected by the webhook unless the Grove pathway is active ([#10217](https://github.com/ai-dynamo/dynamo/pull/10217)), and a component-level `minAvailable` knob for Grove-backed DGDs controlling gang scheduling and termination ([#10532](https://github.com/ai-dynamo/dynamo/pull/10532)). +- **Worker Topology Label Injection:** Injected topology metadata into worker pod templates when `spec.experimental.kvTransferPolicy` is set, adding a Downward API volume and runtime topology env vars while leaving frontend pods and policy-less workers unchanged ([#9792](https://github.com/ai-dynamo/dynamo/pull/9792)). Added a `TopologyLabelReconciler` that watches annotated scheduled pods and copies the configured topology label from their node, marks worker DCDs with `nvidia.com/topology-label-key`, and adds node-read RBAC for cluster-wide and namespace-restricted installs ([#9879](https://github.com/ai-dynamo/dynamo/pull/9879)). +- **Volcano Scheduler Support:** Added Volcano scheduler support for Grove in the Operator ([#11348](https://github.com/ai-dynamo/dynamo/pull/11348)). +- **Rust ext_proc EPP Server:** Added a Rust ext_proc service for the Endpoint Picker (EPP) that Envoy calls to make routing decisions, replacing the llm-d Go implementation and cutting the call path from three language boundaries to one (Envoy C++ to Rust gRPC). The server calls Dynamo Router, tokenizes the request body for KV-aware routing, and locates workers via DistributedRuntime discovery plus Kubernetes pod watches ([#8783](https://github.com/ai-dynamo/dynamo/pull/8783)). +- **KV Transfer Policy API:** Added an experimental topology-aware KV transfer policy under `spec.experimental.kvTransferPolicy` on DynamoGraphDeployment, with `labelKey`, `domain`, `required` or `preferred` enforcement, and worker-side `DYN_KV_TRANSFER_*` env injection ([#9768](https://github.com/ai-dynamo/dynamo/pull/9768)). Added Grove `ClusterTopology` as an alternative topology source via `clusterTopologyName`, projecting topology levels as `nvidia.com/dynamo-topology.` pod labels ([#10041](https://github.com/ai-dynamo/dynamo/pull/10041)). +- **Custom Init Container Image:** Added a Helm value to configure the snapshot DaemonSet init container image, letting users override the default in their charts ([#8432](https://github.com/ai-dynamo/dynamo/pull/8432)). +- **Controller-Manager Pod Metadata:** Added `controllerManager.podLabels` and `controllerManager.podAnnotations` values to the Operator chart, letting users inject pod-level labels and annotations onto the controller-manager Deployment without forking the chart for observability integrations such as Datadog and Prometheus ([#9195](https://github.com/ai-dynamo/dynamo/pull/9195)). +- **Power Agent DaemonSet:** Added a standalone Power Agent DaemonSet for per-node GPU power-cap enforcement, packaged as a Helm chart with DaemonSet, ServiceAccount, and RBAC for cluster-scoped or namespace-restricted mode. Power caps apply through the `dynamo.nvidia.com/gpu-power-limit` pod annotation, with NVML clamping, multi-pod policy, SIGTERM restore, and fail-safe reconcile ([#9682](https://github.com/ai-dynamo/dynamo/pull/9682)). +- **GB10 GPU SKU Support:** Added support for the GB10 GPU SKU type, letting users select and deploy on GB10 hardware in deployment specs ([#9976](https://github.com/ai-dynamo/dynamo/pull/9976)). + +### Fault Tolerance & Observability + +- **Unified Backend Observability Surface:** Added end-to-end OpenTelemetry tracing across the unified backend, forwarding W3C traceparent to vLLM, TRT-LLM, and SGLang with cross-process trace linking for disaggregated serving ([#9543](https://github.com/ai-dynamo/dynamo/pull/9543)). Added Prometheus metric-name parity via a new `register_prometheus` engine hook and Rust `EngineMetrics` handle ([#9586](https://github.com/ai-dynamo/dynamo/pull/9586)), and health-check canary support through `LLMEngine.health_check_payload()` registered with the runtime `HealthCheckManager`, overridable by the `--health-check-payload` flag and `DYN_HEALTH_CHECK_PAYLOAD` env var ([#9642](https://github.com/ai-dynamo/dynamo/pull/9642)). +- **OTLP HTTP and Sampling:** Added OTLP HTTP/protobuf export for traces and logs with protocol and endpoint selection via `OTEL_EXPORTER_OTLP_PROTOCOL` and `OTEL_EXPORTER_OTLP_ENDPOINT`, plus optional trace sampling through `OTEL_TRACES_SAMPLE_RATIO` while keeping the default sampler ([#10576](https://github.com/ai-dynamo/dynamo/pull/10576)), fixed a startup crash when the `http/protobuf` transport selected no default HTTP client ([#10829](https://github.com/ai-dynamo/dynamo/pull/10829)), and preserved W3C `traceparent` sampling flags so inbound sampler decisions propagate to downstream workers ([#10980](https://github.com/ai-dynamo/dynamo/pull/10980)). +- **Embedding Workload Metrics Gating:** Added embedding-shaped Prometheus histograms for batch size and per-request input tokens on the SGLang embedding worker ([#9753](https://github.com/ai-dynamo/dynamo/pull/9753)), plus a `dynamo_embedding_latency_seconds` histogram for end-to-end `/v1/embeddings` latency in the Rust frontend ([#9758](https://github.com/ai-dynamo/dynamo/pull/9758)), and gated the chat-shaped collectors (KV gauges, prefill/decode counters, `LLMBackendMetrics`) so they no longer emit zeros on embedding workers in SGLang ([#9830](https://github.com/ai-dynamo/dynamo/pull/9830)) and vLLM ([#9886](https://github.com/ai-dynamo/dynamo/pull/9886)). +- **Request Trace Unification:** Removed the separate `DYN_AGENT_TRACE` path and made `DYN_REQUEST_TRACE` the single trace switch, emitting enriched `dynamo.request.trace.v1` rows with agent context, replay hashes, timing and token metrics, worker attribution, and finish-reason metadata for requests carrying `nvext.agent_context` ([#10701](https://github.com/ai-dynamo/dynamo/pull/10701)). The Perfetto converter now infers tool slices from `request.finish_reason_metadata.tool_calls` when explicit harness tool events are absent ([#10802](https://github.com/ai-dynamo/dynamo/pull/10802)). +- **Log Crate Trace Capping:** Capped the `log` crate's trace level at compile time via the `release_max_level_debug` feature so `log::trace!` compiles to a no-op in release builds, removing per-character trace spam from the HF tokenizers normalization path ([#10286](https://github.com/ai-dynamo/dynamo/pull/10286)), and removed the `DYNAMO_SKIP_PYTHON_LOG_INIT` env var from the replay and indexer launchers so logging state no longer leaks into child pytest processes ([#9724](https://github.com/ai-dynamo/dynamo/pull/9724)). +- **Chat Stream Metrics Annotations:** Emitted the `llm_metrics` annotation from the vLLM chat processor so per-chunk TTFT and ITL histograms record samples under `--dyn-chat-processor vllm` ([#10019](https://github.com/ai-dynamo/dynamo/pull/10019)), added a typed `LLMMetricAnnotation` fast path that attaches metrics directly to chat stream responses and skips the per-chunk JSON serialize/parse round trip while keeping the legacy path as a fallback ([#10512](https://github.com/ai-dynamo/dynamo/pull/10512)), and emitted token, TTFT, ITL, and cached-token annotations from the SGLang chat processor ([#10779](https://github.com/ai-dynamo/dynamo/pull/10779)). +- **Worker Request Admission Control:** Added worker-side defensive request admission, disabled by default, via `--engine-request-limit` and `--dynamo-request-queue-limit` flags that bound concurrent in-engine and Dynamo-queued requests with fast HTTP 503 shedding on overflow ([#10509](https://github.com/ai-dynamo/dynamo/pull/10509)). Added a `--no-admission-control` escape hatch that clears busy-worker rejection thresholds while keeping router queueing independent, and updated the default queue and prefill-fraction thresholds plus the 503 rejection hint ([#9547](https://github.com/ai-dynamo/dynamo/pull/9547)). +- **Planner SLA Dashboard Updates:** Added SLA target reference lines for TTFT and ITL to the planner Grafana dashboard's Observed Latency panel, matching each target to its observed counterpart's color and axis with a dashed style ([#10712](https://github.com/ai-dynamo/dynamo/pull/10712)), and removed the dead Correction Factors panels that rendered permanent No data once the correction-factor gauges gave way to online FPM regression ([#10824](https://github.com/ai-dynamo/dynamo/pull/10824)). +- **Configurable SGLang Trace Level:** Added a default SGLang trace level of 2 when `--enable-trace` is set, suppressing high-volume scheduler spans such as `decode_loop` while keeping per-request spans, and made the level overridable via `SGLANG_TRACE_LEVEL` ([#9327](https://github.com/ai-dynamo/dynamo/pull/9327)). +- **Standalone Resource Observability Dashboard:** Added a standalone optional WebSocket dashboard served at `/` alongside the `/metrics` endpoint, giving lightweight host-resource debugging without standing up Grafana or Docker Compose. If dashboard packages are missing, `/` shows the install command and `/metrics` still works ([#9780](https://github.com/ai-dynamo/dynamo/pull/9780)). +- **Per-Model Grafana Dashboard:** Added an engine-agnostic per-model Grafana dashboard with 24 panels across Overview, Frontend, KV Routing, and Workers rows, reading the standard `dynamo_frontend_*`, `dynamo_component_*`, and `dynamo_router_*` metrics filtered by a templated model selector. It preserves the existing UID so bookmarks survive ([#9811](https://github.com/ai-dynamo/dynamo/pull/9811)). +- **TensorRT-LLM Engine Health Monitor:** Added `TrtllmEngineMonitor`, which polls TensorRT-LLM's native `check_health` API and shuts down the engine, distributed runtime, and worker process on a fatal health state so Kubernetes can restart the pod ([#10266](https://github.com/ai-dynamo/dynamo/pull/10266)). +- **Registered Worker Metric:** Added a Router component gauge, `dynamo_component_router_worker_registered`, that tracks registered backend workers and updates as topology changes or workers are removed, labeled by router worker ID, data-parallel rank, and worker type for worker-level monitoring ([#10587](https://github.com/ai-dynamo/dynamo/pull/10587)). + +### vLLM + +- **Native vLLM Rust Backend:** Added a native vLLM backend built on the vLLM Rust crates that act as clients of the Python vLLM engine, replacing the older AsyncLLM wrapper and leaving the Dynamo frontend untouched ([#9206](https://github.com/ai-dynamo/dynamo/pull/9206)). +- **Unified vLLM LoRA Support:** Added dynamic LoRA adapter load, unload, and list operations to the unified vLLM engine, plus discovery publishing and per-request LoRA routing, closing the parity gap with the legacy vLLM path. Controls gate on `--enable-lora` and `DYN_LORA_ENABLED`, with a `/v1/loras` compatibility shim preserving the legacy HTTP surface ([#10347](https://github.com/ai-dynamo/dynamo/pull/10347)). +- **vLLM Worker Topology Registration:** Added worker_type and peer-dependency declarations to each vLLM model registration call so aggregated, decode, and prefill workers declare their disaggregation role and dependencies. Re-exported WorkerType through dynamo.llm for backend imports ([#9395](https://github.com/ai-dynamo/dynamo/pull/9395)). +- **Mooncake PD Disaggregation Support:** Let vLLM workers running MooncakeConnector participate in prefill/decode disaggregation by branching the prefill handler to set push-based `kv_transfer_params` and synthesize the decode-side `disaggregated_params` payload. No Rust router changes were needed, since the router treats `disaggregated_params` as opaque ([#9414](https://github.com/ai-dynamo/dynamo/pull/9414)). +- **Aggregated Text-Embedding Worker:** Added an aggregated text-embedding worker shape to the vLLM backend, selected with a new `--embedding-worker` flag, so existing embedding deployments such as `Qwen3-Embedding-0.6B` can move onto Dynamo and vLLM without changing engine args. The worker serves the `/v1/embeddings` route and rejects incompatible combinations like prefill or decode disaggregation, multimodal flags, and benchmark mode at parse time ([#9713](https://github.com/ai-dynamo/dynamo/pull/9713)). +- **Aggregated LMCache MP Mode:** Added a launch script for vLLM aggregated serving that starts an LMCache server sidecar and runs the `LMCacheMPConnector`, replacing the in-process LMCache path on non-Kubernetes environments, and realigned the LMCache integration and observability docs to the multi-process sidecar architecture ([#9982](https://github.com/ai-dynamo/dynamo/pull/9982)). + +### SGLang + +- **SGLang Worker Topology Fields:** Added worker-type and dependency declarations at SGLang LLM registration sites, deriving Decode, Prefill, and Aggregated roles from the serving mode so each worker reports its topology role at registration. Embedding, diffusion, and multimodal-encode workers default to Aggregated, preserving current behavior ([#9397](https://github.com/ai-dynamo/dynamo/pull/9397)). + +### TensorRT-LLM + +- **Worker Type Registration:** Populated worker_type and peer dependencies at the TensorRT-LLM `register_model` call site, deriving the disaggregation role (Prefill, Decode, or Aggregated) and required peers per branch of the configured disaggregation mode. Appended Encode to the dependency set when an encode endpoint is configured ([#9396](https://github.com/ai-dynamo/dynamo/pull/9396)). +- **Unified Backend Logits Processors:** Ported the logits-processor pipeline to the unified TensorRT-LLM backend and added a backend-agnostic `LogitsProcessorSpec` layer that expresses per-engine logits-processor activation as serializable spec entries and applies cross-backend policies for per-request runtime state and PREFILL skip in disaggregated serving ([#10080](https://github.com/ai-dynamo/dynamo/pull/10080)). + +### Infrastructure Modernization + +- **Call-Home Request-Plane Transport:** Implemented the request-stream half of the call-home TCP transport so an upstream server can stream frames to a downstream client over a dedicated socket, settling on a unidirectional-after-handshake design ([#9991](https://github.com/ai-dynamo/dynamo/pull/9991)). Wired end-to-end bidirectional dispatch through `PushRouter::generate(ManyIn)` and `AddressedPushRouter::generate_bidirectional` with a new runtime-side ingress handler ([#9674](https://github.com/ai-dynamo/dynamo/pull/9674)), and added an opt-in `DYN_REQUEST_PLANE_CODEC=json|msgpack` setting carried in the request control message while keeping JSON as the default ([#10437](https://github.com/ai-dynamo/dynamo/pull/10437)). +- **Runtime Event Plane Selection:** Added an explicit `event_plane` parameter to the Python `DistributedRuntime` so callers can request ZMQ and avoid ambient NATS ([#10021](https://github.com/ai-dynamo/dynamo/pull/10021)), then defaulted ZMQ across all discovery backends and kept NATS as an opt-in via `DYN_EVENT_PLANE=nats` ([#10902](https://github.com/ai-dynamo/dynamo/pull/10902)). +- **Configurable ETCD Lease TTL:** Added the ETCD_LEASE_TTL environment variable to control the etcd lease time-to-live, previously hardcoded to 10 seconds, letting users tune lease duration at runtime ([#8076](https://github.com/ai-dynamo/dynamo/pull/8076)). +- **Context Metadata Propagation:** Added a string-keyed metadata field to `Context` that propagates end-to-end through the Dynamo pipeline, from the Python caller through the network into the backend engine, and survives cancellation, retries, and prefill handoffs. Python callers can attach metadata such as tenant or region to requests and read or mutate it on the backend ([#9662](https://github.com/ai-dynamo/dynamo/pull/9662)). + +### Hardware + +#### Intel XPU + +- **XPU Multimodal Nixl Support:** Added Intel XPU aggregated multimodal examples, router launchers, and test profiles using the shared MultimodalModelProfile pattern ([#8112](https://github.com/ai-dynamo/dynamo/pull/8112)), with NIXL canonical VRAM/DRAM memory-type classification, Intel XPU device support for disaggregated E/PD, and XPU encode-worker kernel synchronization ([#9073](https://github.com/ai-dynamo/dynamo/pull/9073)), and aligned DYN_SYSTEM_PORT defaults plus ZE_AFFINITY validation in the XPU launch scripts ([#10488](https://github.com/ai-dynamo/dynamo/pull/10488)). + +### KV Block Manager + +- **KVBM-Logical Backend Rework:** Restructured block lifecycle management into a unified store architecture, replacing the separate pool-based designs and simplifying block state transitions and registration flows under concurrent operations ([#8793](https://github.com/ai-dynamo/dynamo/pull/8793)). Added single-lock prefix matching that resolves an N-hash active-or-inactive prefix under one store-mutex acquisition, plus a Criterion benchmark scaffold covering the lru, multi_lru, and lineage backends ([#9551](https://github.com/ai-dynamo/dynamo/pull/9551)). +- **Universal Hashing Consolidator:** Added a KV cache event consolidator that uses universal hashing to process KVBM v2 event streams and vLLM KV events, deduplicating across sources and publishing a unified stream ([#9480](https://github.com/ai-dynamo/dynamo/pull/9480)). +- **Configurable Block Reset Toggles:** Added pool-level and per-block controls to disable caching of inactive blocks in the KVBM block manager, keeping the reset-on-release preference across cache resurrections ([#9504](https://github.com/ai-dynamo/dynamo/pull/9504)). +- **Python Topology Config Reading:** Added a Python topology environment and file reader for worker startup, and wired topology plus KV-transfer-policy fields into vLLM, SGLang, and TensorRT-LLM runtime config publication through the `ModelRuntimeConfig` binding ([#9878](https://github.com/ai-dynamo/dynamo/pull/9878)). +- **ModelExpress Engine Integration:** Added ModelExpress engine-side weight loading for the vLLM and SGLang runtimes, installing the `modelexpress` client in both images by default and adding an `ignore_weights` option to `register_model()` so registration fetches tokenizer and config metadata without a full weight download. Deprecated the legacy Dynamo-owned vLLM `--model-express-url` wrapper in favor of the plugin-owned `--load-format modelexpress` path ([#10049](https://github.com/ai-dynamo/dynamo/pull/10049)). + +### General + +- **LoRA Load Estimation Pipeline:** Added a self-contained load-tracking pipeline for LoRA adapter allocation, with environment-variable configuration, an exponential-moving-average predictor, and a lock-free bucketed arrival-rate counter for per-LoRA request-rate estimation ([#8178](https://github.com/ai-dynamo/dynamo/pull/8178)). +- **Worker Topology in register_model:** Extended `register_model` and `LocalModel::attach` with optional `worker_type` and `needs` parameters so backends can attach disaggregation role and peer dependencies to a worker's ModelDeploymentCard, rejecting non-canonical combinations at the binding. The card checksum now covers these fields, so a rolling update that changes only topology metadata is rejected as incompatible rather than joining a WorkerSet with a stale card ([#8700](https://github.com/ai-dynamo/dynamo/pull/8700)). +- **Engine Management Routes:** Added unified-backend engine-management plumbing exposed through Dynamo's `/engine/{route}` runtime endpoint, with per-backend callbacks for profiling, sleep/wake and memory release/resume, weight updates, and elastic scaling across vLLM, SGLang, and TensorRT-LLM ([#10094](https://github.com/ai-dynamo/dynamo/pull/10094)). +- **Hugging Face Hub LoRA Sources:** Added native `hf://[@revision]` support for dynamic LoRA loading, downloading adapters into the standard Hugging Face Hub cache and returning the immutable commit snapshot without a second copy under `DYN_LORA_PATH`. Revision-pinned, timeout-bounded snapshot downloads honor Hugging Face cache, token, endpoint, and offline semantics ([#11816](https://github.com/ai-dynamo/dynamo/pull/11816)). + +--- + +## Recipes + +- **Kimi K2 Agentic Recipes:** Replaced outdated Kimi K2.5 recipes with disaggregated TensorRT-LLM deployment using KV-aware routing and Eagle3 speculative decoding, aggregated round-robin variants, and ComputeDomain multi-node configuration tuned on the agentic code dataset ([#9621](https://github.com/ai-dynamo/dynamo/pull/9621)), and added Kimi K2.6 Turbo recipes with Kubernetes deployments for chat and agentic workloads on B200 and H200 GPUs plus a trace-replay benchmarking guide ([#10187](https://github.com/ai-dynamo/dynamo/pull/10187)). +- **Qwen3 vLLM Serving Recipes:** Added an aggregated serving recipe for Qwen3-0.6B with Gateway API Inference Extension (GAIE) integration, including an Istio DestinationRule required for the gateway proxy to reach the EPP service, plus a Qwen3.6-35B-A3B aggregated serving recipe with a standalone frontend ([#8355](https://github.com/ai-dynamo/dynamo/pull/8355)). +- **Qwen3-VL-32B-FP8 Recipe:** Added a production-ready deployment recipe for the Qwen3-VL-32B-Instruct-FP8 vision-language model with aggregated single-GPU and disaggregated cross-vendor modes, the latter using Intel XPU encode and NVIDIA GPU decode with NIXL embedding transfer over RDMA. The recipe includes model-cache jobs and QPS-sweep multimodal benchmarks ([#9252](https://github.com/ai-dynamo/dynamo/pull/9252)). +- **GLM-5-NVFP4 EFA Variant:** Added an EFA variant of the GLM-5-NVFP4 SGLang disaggregated recipe for GB200 clusters on AWS, routing KV transfer over AWS EFA RDMA via NIXL's LIBFABRIC backend instead of UCX for p6e-gb200.36xlarge deployments ([#9712](https://github.com/ai-dynamo/dynamo/pull/9712)). +- **Nemotron-3-Super Turbo Recipes:** Added NIM Turbo deployment recipes for NVIDIA-Nemotron-3-Super-120B-A12B using Dynamo and vLLM, covering NVFP4 on B200 and FP8 on H200 across chat and agentic serving modes, plus model-cache manifests and an AIPerf trace-replay performance job ([#10223](https://github.com/ai-dynamo/dynamo/pull/10223)). +- **Speculative Decoding Hardcoding Fix:** Removed a hardcoded speculative decoding variable from the Kimi K2.5 TensorRT-LLM recipes and corrected the README. ([#9637](https://github.com/ai-dynamo/dynamo/pull/9637)). +- **Model-Cache Memory Limits:** Added per-recipe memory and CPU resource requests and limits plus an absolute 16GB HF XET reconstruction buffer cap to all 15 model-download Jobs, preventing kubelet OOM eviction during downloads on Kubernetes ([#9884](https://github.com/ai-dynamo/dynamo/pull/9884)). +- **DeepSeek V4 SGLang Dependency:** Added an explicit `distro` install to the DeepSeek V4 SGLang B200 runtime image, fixing a `ModuleNotFoundError` that caused the worker to exit during startup when the OpenAI SDK imported the missing package ([#10085](https://github.com/ai-dynamo/dynamo/pull/10085)). +- **gpt-oss-120b Recipe Revert:** Reverted the gpt-oss-120b TensorRT-LLM aggregated recipe runtime image from 1.1.1 to 1.0.0, since the 1.1.1 runtime regressed the recipe on GB200 ([#10147](https://github.com/ai-dynamo/dynamo/pull/10147)). +- **Hugging Face Hub Pins:** Bumped the Hugging Face Hub CLI pin from 1.11.0 to 1.16.4 across 17 model-download recipes and two vLLM LoRA sync jobs, so helper jobs install the `click` dependency required by `hf download` ([#11273](https://github.com/ai-dynamo/dynamo/pull/11273)). + +## Bug Fixes + +### Frontend + +- **Tool-Call & Reasoning Parser Hardening:** Aligned tool-call and reasoning parsers for GLM-4.7, DeepSeek V3/V3.2/V4, Kimi K2, Qwen, Gemma4, Mistral, MiniMax, and Nemotron with the vLLM/SGLang reference contracts, suppressing streaming markup leaks, recovering truncated and EOF-terminated tool calls, restoring chat-template fidelity, and routing structured output correctly ([#9355](https://github.com/ai-dynamo/dynamo/pull/9355), [#9524](https://github.com/ai-dynamo/dynamo/pull/9524), [#9594](https://github.com/ai-dynamo/dynamo/pull/9594), [#9462](https://github.com/ai-dynamo/dynamo/pull/9462), [#9411](https://github.com/ai-dynamo/dynamo/pull/9411), [#10833](https://github.com/ai-dynamo/dynamo/pull/10833)) and related fixes. +- **Worker-Set Readiness Routing:** Confined routing to complete worker sets and added 529/503 codes separating capacity overload from backend unavailability ([#10503](https://github.com/ai-dynamo/dynamo/pull/10503), [#10941](https://github.com/ai-dynamo/dynamo/pull/10941)). +- **KServe gRPC Inference Dispatch:** Fixed a KServe `model_ready`/`server_ready` race that reported ready before engines attached, and centralized dispatch to return clear model-not-found errors ([#9619](https://github.com/ai-dynamo/dynamo/pull/9619), [#10621](https://github.com/ai-dynamo/dynamo/pull/10621)). +- **EOS/BOS Token & Model Discovery:** Resolved `eos_token_id`/`bos_token_id` from tokenizer and special-tokens files when absent from model config ([#10672](https://github.com/ai-dynamo/dynamo/pull/10672)), and accepted the `deepseek_v32` model_type spelling so V3.2 workers register ([#10332](https://github.com/ai-dynamo/dynamo/pull/10332)). +- **Sanitized Errors & API Compatibility:** Stopped HTTP errors from leaking paths, versions, and stack text (CWE-209) ([#10151](https://github.com/ai-dynamo/dynamo/pull/10151)), mapped backend `InvalidArgument` to HTTP 400 ([#11591](https://github.com/ai-dynamo/dynamo/pull/11591)), and relaxed deserialization for multimodal `EasyMessage` and Anthropic message-level system roles so agent clients like Claude Code pass validation ([#9470](https://github.com/ai-dynamo/dynamo/pull/9470), [#10108](https://github.com/ai-dynamo/dynamo/pull/10108)). +- **Request Draining & Context Length:** Drained inflight requests on shutdown so terminating frontend pods finish streaming instead of returning HTTP 500 ([#10862](https://github.com/ai-dynamo/dynamo/pull/10862)), and propagated the backend `context_length` as `max_model_len` so long-context requests reach the backend ([#10827](https://github.com/ai-dynamo/dynamo/pull/10827)). + +### Agents + +- **Coding Agent Header Handling:** Merged runs of leading system messages when converting `/v1/responses` to chat completions, fixing Codex CLI first-turn failures on templates that reject multiple system messages ([#10283](https://github.com/ai-dynamo/dynamo/pull/10283)), and normalized Codex, Claude Code, and OpenCode session headers into `nvext.agent_context`, mapping `x-claude-code-agent-id` to a child `trajectory_id` ([#10782](https://github.com/ai-dynamo/dynamo/pull/10782)). + +### Reinforcement Learning + +- **Routed-Experts Capture for Log-Prob Recompute:** Base64-encoded MoE routed-experts capture with a `start` offset so consumers align routing data to the completion ([#10529](https://github.com/ai-dynamo/dynamo/pull/10529)), and fixed a first-token crash from double re-encoding `routed_experts` on non-DSv4 MoE models under SGLang v0.5.11+ ([#9657](https://github.com/ai-dynamo/dynamo/pull/9657)). +- **Rollout Serving Robustness:** Rejected unsupported RL output fields `completion_token_ids` and `prompt_logprobs` on `/v1/responses` with a field-specific 501 ([#11265](https://github.com/ai-dynamo/dynamo/pull/11265)), added a configurable weight-update init watchdog `DYN_RL_INIT_WEIGHTS_TIMEOUT_S` ([#11692](https://github.com/ai-dynamo/dynamo/pull/11692)), and pinned `zstandard==0.23.0` in the SGLang image to restore the RL metadata upload path ([#11379](https://github.com/ai-dynamo/dynamo/pull/11379)). + +### Multimodal & Diffusion + +- **Diffusion & Omni Model Coverage:** Forced eager mode in the Wan2.2 video and image-to-video launchers to bypass a CUDA illegal memory access ([#9563](https://github.com/ai-dynamo/dynamo/pull/9563)), raised Qwen3-VL `max_seq_len` to 8192 restoring disaggregated multimodal coverage ([#9764](https://github.com/ai-dynamo/dynamo/pull/9764)), and emitted one image per diffusion output instead of capping at n=1 ([#9822](https://github.com/ai-dynamo/dynamo/pull/9822)). +- **Multimodal Request Safety & Limits:** Returned actionable 4xx for SSRF-blocked media URLs instead of 500 ([#11360](https://github.com/ai-dynamo/dynamo/pull/11360)), derived the `max_tokens` default from the image-expanded prompt length so requests no longer overshoot context ([#10757](https://github.com/ai-dynamo/dynamo/pull/10757)), and threaded `trust_remote_code` from engine settings (default `False`) across vLLM, SGLang, and TensorRT-LLM, closing CWE-829 ([#11366](https://github.com/ai-dynamo/dynamo/pull/11366)). +- **Multimodal Rendering & Tensor Path:** Derived CHW tensor image dimensions from `shape[-2:]` to keep SGLang prefill/decode on the exact-token path ([#11299](https://github.com/ai-dynamo/dynamo/pull/11299)), and flattened mixed text and image content in the native Rust renderer for verbatim-echo templates like NVIDIA-Nemotron-Parse ([#10380](https://github.com/ai-dynamo/dynamo/pull/10380)). + +### Scheduling + +- **Router Admission & Backpressure:** Gated prefill admission on overloaded-worker sets, excluded over-threshold workers across routing modes, randomized tie-breaks, and bounded event-subscriber channels to cap frontend memory ([#10631](https://github.com/ai-dynamo/dynamo/pull/10631), [#9688](https://github.com/ai-dynamo/dynamo/pull/9688), [#9516](https://github.com/ai-dynamo/dynamo/pull/9516)). +- **KV-Router Indexing & Attribution:** Fixed prefill-router hash-mode mismatches, invalidated cached hashes on LoRA/multimodal metadata, and reconciled instance state during worker churn ([#9857](https://github.com/ai-dynamo/dynamo/pull/9857), [#10500](https://github.com/ai-dynamo/dynamo/pull/10500), [#9631](https://github.com/ai-dynamo/dynamo/pull/9631)). +- **Planner Load & FPM Accounting:** Counted started requests for throughput load, classified decode-sweep rows in Forward-Pass Metrics, and guarded overlapping disagg scale operations ([#9526](https://github.com/ai-dynamo/dynamo/pull/9526), [#9528](https://github.com/ai-dynamo/dynamo/pull/9528), [#9774](https://github.com/ai-dynamo/dynamo/pull/9774)). + +### KV Router + +- **KV Event & Cancellation Correctness:** Stamped SGLang multinode KV events with routable leader worker IDs so remote DP ranks update Router trees, shared KV hashing via the dynamo-kv-hashing contract, attributed workers only after dispatch, and scoped cancellation to a child token ([#9505](https://github.com/ai-dynamo/dynamo/pull/9505), [#10095](https://github.com/ai-dynamo/dynamo/pull/10095), [#11324](https://github.com/ai-dynamo/dynamo/pull/11324), [#11455](https://github.com/ai-dynamo/dynamo/pull/11455)). + +### Planner & Profiler + +- **Planner Scaling & SLA Correctness:** Scaled down throughput-only SLA deployments after traffic stops, propagated user SLA targets into PlannerConfig, and kept Planner bound to the serving worker namespace through rollouts ([#11296](https://github.com/ai-dynamo/dynamo/pull/11296), [#9537](https://github.com/ai-dynamo/dynamo/pull/9537), [#11622](https://github.com/ai-dynamo/dynamo/pull/11622)). +- **Profiler Sweep & DGDR Fixes:** Preserved DGDR served-model identity across sweeps, made parallel-config labels injective, resolved local PVC model paths (no HF refetch), and normalized SGLang/vLLM/TRT-LLM CUDA-graph arguments ([#11196](https://github.com/ai-dynamo/dynamo/pull/11196), [#10086](https://github.com/ai-dynamo/dynamo/pull/10086), [#9877](https://github.com/ai-dynamo/dynamo/pull/9877)). +- **AIConfigurator Version Fixes:** Capped aiconfigurator below v0.10 to restore the v0.9 API the profiler targets and removed the deprecated profiler WebApp and its Gradio dependency ([#11732](https://github.com/ai-dynamo/dynamo/pull/11732), [#11361](https://github.com/ai-dynamo/dynamo/pull/11361)). + +### Performance Modeling & Replay + +- **Replay & Mocker Correctness:** Preserved zero-duration completions and single-rank KV capture in disagg replay, fixed cached chunked-prefill accounting, gated mocker admission on physically available KV, and hardened KVBM offload simulation to fail fast on invalid states ([#9771](https://github.com/ai-dynamo/dynamo/pull/9771), [#10919](https://github.com/ai-dynamo/dynamo/pull/10919), [#10439](https://github.com/ai-dynamo/dynamo/pull/10439), [#11012](https://github.com/ai-dynamo/dynamo/pull/11012)). + +### Kubernetes Deployment + +- **Inference Gateway & Service Mesh:** Switched the GAIE install and validation flow from kgateway to agentgateway ([#7607](https://github.com/ai-dynamo/dynamo/pull/7607)), fixed EPP DestinationRule generation for `tlsMode=MUTUAL` ([#10068](https://github.com/ai-dynamo/dynamo/pull/10068)), and added `serviceMesh.enabled` to disable Istio where the DestinationRule API is absent ([#11344](https://github.com/ai-dynamo/dynamo/pull/11344)). +- **GMS, DRA & Snapshot/CRIU Restore:** Gated GMS ResourceClaimTemplate sync on DRA `v1` and derived snapshot-restore CUDA order from `nvidia-smi` ([#9454](https://github.com/ai-dynamo/dynamo/pull/9454), [#11325](https://github.com/ai-dynamo/dynamo/pull/11325)) and related fixes. +- **DGD/DGDR/DCD Admission & CRD Versioning:** Ported DynamoGraphDeployment webhooks to `v1beta1` with legacy v1alpha1 conversion and moved CRDs into a `crd-apply` init container to fit the 1 MiB Secret limit ([#10934](https://github.com/ai-dynamo/dynamo/pull/10934), [#11689](https://github.com/ai-dynamo/dynamo/pull/11689)) and related fixes. +- **DGDR & Rollout Lifecycle:** Surfaced specific DGDR failure reasons, fixed a profiling race that stranded DGDRs in Deploying, and preserved availability during overlapping worker rollouts ([#8227](https://github.com/ai-dynamo/dynamo/pull/8227), [#11340](https://github.com/ai-dynamo/dynamo/pull/11340), [#10184](https://github.com/ai-dynamo/dynamo/pull/10184)) and related fixes. +- **Operator Config, Secrets, Volumes & Services:** Fixed spurious rollouts from non-deterministic pull-secret discovery, restored the LWS `-0` service name for worker DNS, and gated vLLM MP-init to the multiprocessing backend ([#9826](https://github.com/ai-dynamo/dynamo/pull/9826), [#9612](https://github.com/ai-dynamo/dynamo/pull/9612)) and related fixes. + +### Fault Tolerance & Observability + +- **Frontend Metrics & Shutdown Hardening:** Normalized `model` metric label casing, collapsed unregistered names to a bounded `unknown_model` sentinel, added a configurable graceful-shutdown drain timeout, and migrated Grafana dashboards to the replacement request gauges ([#9775](https://github.com/ai-dynamo/dynamo/pull/9775), [#9836](https://github.com/ai-dynamo/dynamo/pull/9836), [#10705](https://github.com/ai-dynamo/dynamo/pull/10705), [#11690](https://github.com/ai-dynamo/dynamo/pull/11690)). + +### vLLM + +- **Config & Correctness Fixes:** Stopped overwriting the user `--runner` flag that broke embedding models, removed per-run torch-compile cache regeneration, and aligned the vLLM-Omni pin to fix an import-time crash that left workers unregistered ([#9710](https://github.com/ai-dynamo/dynamo/pull/9710), [#11374](https://github.com/ai-dynamo/dynamo/pull/11374)). +- **Embedding & Multimodal Fixes:** Fixed the text-embedding worker to return one pooled, L2-normalized vector instead of the full per-token matrix, and backported a deepstack fix that crashed EngineCore on disaggregated multimodal Qwen3-VL requests ([#10248](https://github.com/ai-dynamo/dynamo/pull/10248), [#9491](https://github.com/ai-dynamo/dynamo/pull/9491)). + +### SGLang + +- **Correctness & Routing Fixes:** Forwarded dropped OpenAI sampling controls that caused degenerate token repetition, and advertised the global data-parallel rank range so distributed routing targets the correct ranks ([#10310](https://github.com/ai-dynamo/dynamo/pull/10310), [#10196](https://github.com/ai-dynamo/dynamo/pull/10196)). +- **Stability & Backend Fixes:** Fixed an asyncio Future/Task leak in the cancellation monitor causing linear RSS growth and TTFT spikes under load, and resolved NIXL libraries from `nixl_cu13` on cuda13 images ([#10082](https://github.com/ai-dynamo/dynamo/pull/10082), [#10325](https://github.com/ai-dynamo/dynamo/pull/10325)). + +### TensorRT-LLM + +- **Crash & Compatibility Fixes:** Skipped unused NIXL connector creation in aggregated mode, and capped `mpmath<1.4` to prevent an import-time crash from a missing sympy symbol ([#9501](https://github.com/ai-dynamo/dynamo/pull/9501), [#11404](https://github.com/ai-dynamo/dynamo/pull/11404)). +- **Performance & Telemetry Fixes:** Enabled block reuse and CUDA graphs in the qwen2-vl-7b example to close a multimodal TTFT regression, and replaced multi-megabyte KV event dumps with compact summaries plus Prometheus telemetry ([#9942](https://github.com/ai-dynamo/dynamo/pull/9942), [#10298](https://github.com/ai-dynamo/dynamo/pull/10298)). + +### Infrastructure Modernization + +- **Request-Plane & Discovery Hardening:** Rejected oversized NATS payloads with a sanitized `InvalidArgument`, aborted the TCP writer when its reader fails to end a connection-monitor hang, and gave each publisher a fork-safe ID to stop discovery-key collisions from dropping subscribers ([#9769](https://github.com/ai-dynamo/dynamo/pull/9769), [#9716](https://github.com/ai-dynamo/dynamo/pull/9716), [#11311](https://github.com/ai-dynamo/dynamo/pull/11311)). + +### Hardware + +- **Intel XPU & AMD ROCm Import Compatibility:** Added a `MultiModalUUIDDict` import fallback for XPU builds on vLLM v0.16.0, and fixed ROCm import failures on Python 3.10 by deferring the `nixl_connect` import ([#8106](https://github.com/ai-dynamo/dynamo/pull/8106), [#9929](https://github.com/ai-dynamo/dynamo/pull/9929)). + +### KV Block Manager + +- **Leak & Lifecycle Fixes:** Added a prefill-drain hook to prevent a decode-side use-after-free during NIXL transfers ([#9937](https://github.com/ai-dynamo/dynamo/pull/9937)). +- **Compatibility & Accounting Fixes:** Defaulted NIXL install extras to `nixl[cu13]`, and normalized live KV metrics to per-data-parallel-rank units ([#10413](https://github.com/ai-dynamo/dynamo/pull/10413), [#9932](https://github.com/ai-dynamo/dynamo/pull/9932)). + +### General + +- **Model Loading & Storage Fixes:** Made `register_model`/`hub::from_hf` cache-first, and prepended the hidden NIXL wheel directory to `LD_LIBRARY_PATH` so the frontend loads `libnixl.so` ([#10102](https://github.com/ai-dynamo/dynamo/pull/10102), [#9911](https://github.com/ai-dynamo/dynamo/pull/9911)). +- **Concurrency & Store Consistency Fixes:** Centralized `get_lora_manager()` behind a thread-safe `OnceLock` to stop duplicate LoRAManager construction, and fixed FileStore to honor `revision = 0` as create-if-absent instead of overwriting existing files ([#10110](https://github.com/ai-dynamo/dynamo/pull/10110), [#10968](https://github.com/ai-dynamo/dynamo/pull/10968)). + +--- + +## Documentation + +- **Parser & Tool-Calling Docs:** Reorganized tool-call and reasoning parser docs into a single Tool Calling section with a top-level Parser Configuration reference page, split Dynamo-native from engine-fallback paths, and added a `logprobs`-based troubleshooting guide ([#10236](https://github.com/ai-dynamo/dynamo/pull/10236), [#9400](https://github.com/ai-dynamo/dynamo/pull/9400), [#9658](https://github.com/ai-dynamo/dynamo/pull/9658)) and related updates. +- **Recipes & Feature Benchmarks:** Added Recipes and Feature Benchmarks doc surfaces with model-family filters and deployable targets, a DynoSim simulation section, and Nemotron-3-Ultra NIM Turbo recipes ([#10628](https://github.com/ai-dynamo/dynamo/pull/10628), [#10156](https://github.com/ai-dynamo/dynamo/pull/10156), [#10303](https://github.com/ai-dynamo/dynamo/pull/10303)) and related updates. +- **Deployment & Configuration Docs:** Documented Standalone and Gateway (GAIE) deployment modes, consolidated EKS/GKE/AKS/ECS guides under `cloud-providers/`, and expanded DGDR, Planner, and router configuration and KV-transfer guides ([#10059](https://github.com/ai-dynamo/dynamo/pull/10059), [#10348](https://github.com/ai-dynamo/dynamo/pull/10348), [#9428](https://github.com/ai-dynamo/dynamo/pull/9428), [#10123](https://github.com/ai-dynamo/dynamo/pull/10123)) and related updates. + +## Looking Ahead + +The following is planned for the next release, v1.4.0 (targeted 2026-08-12). See the public [Dynamo roadmap](https://github.com/ai-dynamo/dynamo/issues/9178) for the full plan. + +### Self-Tuning Planner & Live Autoscaling + +Dynamo Planner moves from configured scaling to live, self-tuning scaling. Forward-Pass Metrics streamed from the vLLM, SGLang, and TensorRT-LLM engines feed an AIConfigurator performance model that the Planner continuously fine-tunes against real traffic, so scaling decisions track your actual workload rather than a static profile. This release line adds bring-your-own-trace warmup from production traffic replay, a self-tuning remote-prefill policy that shifts local-versus-remote prefill live from prefill-worker load, and a plugin interface for custom scaling logic such as cost-aware or spot-instance strategies. + +### Fault Tolerance & Fast Recovery + +Production resiliency hardens across the stack. GPU Memory Service targets sub-five-second recovery for failed workers through shadow failover backed by a warm compile cache, CRIU-based GPU process snapshots make multi-GPU checkpoint and restore practical, and in-flight requests migrate or cancel cleanly under SLA pressure instead of failing outright. That foundation extends toward WideEP fail-continue, where a single GPU failure lets the healthy ranks keep serving while a replacement rejoins at rank granularity. + +### Voice & Multimodal Pipelining + +Dynamo extends beyond text to orchestrate multi-model pipelines as a single request. Transparent pipelining chains stages such as speech-to-text, an LLM, and text-to-speech (STT → LLM → TTS) into one logical call, with Dynamo handling scheduling, colocation, streaming, and latency budgeting across stages — so real-time voice assistants and audio-to-audio translation no longer need custom orchestration outside the inference stack. Landing alongside are use-case-optimized Omni and vision-language recipes tuned for VLM multi-turn chat and streaming audio. + + + +Between v1.2.1 and v1.3.0 the project merged 930 PRs from 125 contributors. Thank you to the external community contributors in this release (organization identified via commit-author email domain or public GitHub profile, cross-referenced against the team roster): + +- **Intel:** @dsocek, @joshuayao, @pallavijaini0525, @Spycsh, @tthakkal, @VincyZhang, @yuanwu2017. +- **DaoCloud:** @carlory, @flpanbin, @my-git9, @yankay. +- **Inferact:** @BugenZhao, @Dao007forever, @zhewenl. **Gcore:** @kirillemilio. **AMD:** @andyluo7. **Epic Games:** @hatemfaheem. +- **Anyscale:** @jeffreywang88. **CoreWeave:** @ritazh. **Microsoft:** @Jont828. **GMI Cloud:** @GavinZhu-GMI. **NeuReality:** @VadimEisenberg. **Ohio Supercomputer Center:** @treydock. **SK Telecom:** @mcpark84. **Solo.io:** @danehans. +- **University of Washington:** @Shaoting-Feng. **Arizona State University:** @Change72. **East China Normal University:** @Muqi1029. **Rutgers University:** @aryanputta. **AI Labs Taiwan:** @waynehacking8. +- **Independent:** @atomic, @bewestphal, @cmdy, @doujiang24, @esmeetu, @jellysnack, @RitwijParmar, @sungsooha, @xianlubird, @yuanchen8911. + +**New Contributors** — welcome to the 23 first-time contributors in this release: + +- @Dao007forever made their first contribution in [#9195](https://github.com/ai-dynamo/dynamo/pull/9195). +- @zhewenl made their first contribution in [#9414](https://github.com/ai-dynamo/dynamo/pull/9414). +- @kirillemilio made their first contribution in [#9824](https://github.com/ai-dynamo/dynamo/pull/9824). +- @Shaoting-Feng made their first contribution in [#9982](https://github.com/ai-dynamo/dynamo/pull/9982). +- @Harrilee made their first contribution in [#10076](https://github.com/ai-dynamo/dynamo/pull/10076). +- @andyluo7 made their first contribution in [#9929](https://github.com/ai-dynamo/dynamo/pull/9929). +- @Change72 made their first contribution in [#10157](https://github.com/ai-dynamo/dynamo/pull/10157). +- @mvillmow made their first contribution in [#9095](https://github.com/ai-dynamo/dynamo/pull/9095). +- @Muqi1029 made their first contribution in [#10203](https://github.com/ai-dynamo/dynamo/pull/10203). +- @hatemfaheem made their first contribution in [#9556](https://github.com/ai-dynamo/dynamo/pull/9556). +- @yuanchen8911 made their first contribution in [#8776](https://github.com/ai-dynamo/dynamo/pull/8776). +- @ritazh made their first contribution in [#8355](https://github.com/ai-dynamo/dynamo/pull/8355). +- @aryanputta made their first contribution in [#10281](https://github.com/ai-dynamo/dynamo/pull/10281). +- @kangclzjc made their first contribution in [#10124](https://github.com/ai-dynamo/dynamo/pull/10124). +- @RitwijParmar made their first contribution in [#10095](https://github.com/ai-dynamo/dynamo/pull/10095). +- @yuanwu2017 made their first contribution in [#7990](https://github.com/ai-dynamo/dynamo/pull/7990). +- @treydock made their first contribution in [#10597](https://github.com/ai-dynamo/dynamo/pull/10597). +- @VadimEisenberg made their first contribution in [#10595](https://github.com/ai-dynamo/dynamo/pull/10595). +- @BugenZhao made their first contribution in [#9206](https://github.com/ai-dynamo/dynamo/pull/9206). +- @waynehacking8 made their first contribution in [#10616](https://github.com/ai-dynamo/dynamo/pull/10616). +- @atomic made their first contribution in [#10827](https://github.com/ai-dynamo/dynamo/pull/10827). +- @yankay made their first contribution in [#10861](https://github.com/ai-dynamo/dynamo/pull/10861). +- @pallavijaini0525 made their first contribution in [#9252](https://github.com/ai-dynamo/dynamo/pull/9252). + +If you would like to get involved, please see our [Contribution Guide](https://docs.nvidia.com/dynamo/dev/getting-started/contribution-guide). + + diff --git a/docs/fern/reference/releases-data.mdx b/docs/fern/reference/releases-data.mdx new file mode 100644 index 000000000000..ea3120fba3d4 --- /dev/null +++ b/docs/fern/reference/releases-data.mdx @@ -0,0 +1,238 @@ +--- +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +title: Releases (machine-readable) +subtitle: Generated release, compatibility, and artifact data for agents and automation +--- + +This page is a plain-markdown rendering of [`components/releases.data.ts`](https://github.com/ai-dynamo/dynamo/blob/main/docs/fern/components/releases.data.ts), the single source of truth behind [Compatibility](compatibility.mdx), [Release Artifacts](release-artifacts.mdx), [Model Early Access Builds](model-early-access-builds.mdx), and the [Release Notes](release-notes/README.mdx) timeline. It is regenerated by `scripts/gen_llms_tables.py` at every release bump — do not edit the tables below by hand. Append `.md` to this page's URL for a clean markdown export. The same data ships in-repo as JSON (`docs/fern/assets/releases.json`) and as an Atom feed (`docs/fern/assets/releases-atom.xml`). + +{/* llms-tables:begin — generated by scripts/gen_llms_tables.py, do not edit */} + +Current stable release: v1.3.0 (Jul 20, 2026; container tag `1.3.0`, wheel version `1.3.0.post1`). + +## Releases + +| Version | Kind | Date | SGLang | TensorRT-LLM | vLLM | NIXL (SGL / TRT / vLLM) | UCX | Notes | Delta | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | +| main (ToT) | development head | - | 0.5.15 | 1.3.0rc21 | 0.25.1 | 1.3.0 / 1.0.1 / 1.1.0 | - | - | - | +| v1.3.0 | stable | Jul 20, 2026 | 0.5.14 | 1.3.0rc19 | 0.23.0 | 1.3.0 / 1.0.1 / 1.1.0 | 1.20.x | [release notes](https://docs.nvidia.com/dynamo/dev/reference/releases/v1-3-0) | CUDA 12 container images discontinued; EFA variants go multi-arch as -efa; GA wheels published as 1.3.0.post1 (containers stay :1.3.0); UCX 1.20.x. | +| v1.3.0-dev.1 | platform-preview | Jun 9, 2026 | 0.5.12.post1 | 1.3.0rc17 | 0.22.0 | 1.0.1 / 0.10.1 / 1.1.0 | - | [release notes](https://github.com/ai-dynamo/dynamo/releases/tag/v1.3.0-dev.1) | Full-platform preview of v1.3.0: complete runtime matrix, wheels on pypi.nvidia.com, crates, and Helm charts. Superseded by v1.3.0 GA. | +| v1.2.1 | patch | Jun 13, 2026 | 0.5.11 | 1.3.0rc14 | 0.20.1 | 1.0.1 / 0.10.1 / 0.10.1 | - | [release notes](https://docs.nvidia.com/dynamo/dev/reference/releases/v1-2-0) | Patch release. Same backend pins as v1.2.0. | +| v1.2.0 | stable | Jun 2, 2026 | 0.5.11 | 1.3.0rc14 | 0.20.1 | 1.0.1 / 0.10.1 / 0.10.1 | 1.20.0 | [release notes](https://docs.nvidia.com/dynamo/dev/reference/releases/v1-2-0) | 603 PRs from 82 authors. DGD/DGDR promoted to v1beta1; CRTC default approximate KV router; inter-pod GMS sidecar; Dynamo Snapshot on CRI-O / OpenShift; UCX 1.20.0. | +| v1.2.0-deepseek-v4-dev.3 | model-build | May 9, 2026 | upstream DSv4 preview | - | 0.20.1 | - / - / 0.10.1 | - | [release notes](https://github.com/ai-dynamo/dynamo/releases/tag/v1.2.0-deepseek-v4-dev.3) | DeepSeek-V4 Blackwell preview; vLLM + SGLang containers only. | +| v1.2.0-deepseek-v4-dev.2 | model-build | May 1, 2026 | upstream DSv4 preview | - | 0.20.0 | - / - / 0.10.1 | - | [release notes](https://github.com/ai-dynamo/dynamo/releases/tag/v1.2.0-deepseek-v4-dev.2) | DeepSeek-V4 Blackwell preview; vLLM + SGLang containers only. | +| v1.1.1 | patch | May 5, 2026 | 0.5.10.post1 | 1.3.0rc11 | 0.19.0 | 1.0.1 / 0.10.1 / 0.10.1 | - | [release notes](https://docs.nvidia.com/dynamo/dev/reference/releases/v1-1-0) | Patch release. Same backend pins as v1.1.0. | +| v1.1.0 | stable | May 1, 2026 | 0.5.10.post1 | 1.3.0rc11 | 0.19.0 | 1.0.1 / 0.10.1 / 0.10.1 | 1.20 | [release notes](https://docs.nvidia.com/dynamo/dev/reference/releases/v1-1-0) | Planner split into its own dynamo-planner image (artifact boundary change). First 1.y.z publication of dynamo-protocols on crates.io; dynamo-async-openai deprecated at final 1.0.2. | +| v1.1.0-dev.3 | platform-preview | Apr 18, 2026 | 0.5.10.post1 | 1.3.0rc11 | 0.19.0 | 1.0.1 / 0.10.1 / 0.10.1 | - | [release notes](https://github.com/ai-dynamo/dynamo/releases/tag/v1.1.0-dev.3) | Partial platform preview: TRT-LLM runtime image + wheels only. | +| v1.1.0-dev.2 | platform-preview | Apr 9, 2026 | 0.5.9 | 1.3.0rc9 | 0.19.0 | 1.0.1 / 0.10.1 / 0.10.1 | - | [release notes](https://github.com/ai-dynamo/dynamo/releases/tag/v1.1.0-dev.2) | Partial platform preview: SGLang + TRT-LLM runtime images + wheels. | +| v1.1.0-dev.1 | platform-preview | Mar 17, 2026 | 0.5.9 | 1.3.0rc5.post1 | 0.17.1 | 1.0.1 / 0.10.1 / 0.10.1 | - | [release notes](https://github.com/ai-dynamo/dynamo/releases/tag/v1.1.0-dev.1) | Platform preview: runtime matrix, wheels on pypi.nvidia.com, Helm charts. | +| v1.0.2 | patch | Apr 22, 2026 | 0.5.9 | 1.3.0rc5.post1 | 0.16.0 | 0.10.1 / 0.10.1 / 0.10.1 | - | [release notes](https://docs.nvidia.com/dynamo/dev/reference/releases/v1-0-0) | No artifact additions or removals versus v1.0.0. | +| v1.0.1 | patch | Mar 16, 2026 | 0.5.9 | 1.3.0rc5.post1 | 0.16.0 | 0.10.1 / 0.10.1 / 0.10.1 | - | [release notes](https://docs.nvidia.com/dynamo/dev/reference/releases/v1-0-0) | No artifact additions or removals versus v1.0.0. | +| v1.0.0 | stable | Mar 12, 2026 | 0.5.9 | 1.3.0rc5.post1 | 0.16.0 | 0.10.1 / 0.10.1 / 0.10.1 | - | [release notes](https://docs.nvidia.com/dynamo/dev/reference/releases/v1-0-0) | snapshot-agent image and EFA variants for vLLM and TRT-LLM (AMD64 only). First publish of dynamo-mocker and dynamo-kv-router crates. snapshot Helm chart added (preview); deprecated dynamo-crds dropped from the publish stream. | +| v0.9.1 | patch | Mar 4, 2026 | 0.5.8 | 1.3.0rc3 | 0.14.1 | 0.9.0 / 0.9.0 / 0.9.0 | - | [release notes](https://github.com/ai-dynamo/dynamo/releases/tag/v0.9.1) | No artifact additions or removals versus v0.9.0. | +| v0.9.0 | stable | Feb 11, 2026 | 0.5.8 | 1.3.0rc1 | 0.14.1 | 0.9.0 / 0.9.0 / 0.9.0 | - | [release notes](https://github.com/ai-dynamo/dynamo/releases/tag/v0.9.0) | First publish of dynamo-tokens crate. Deprecated dynamo-graph Helm chart dropped from the publish stream. | +| v0.8.1 | patch | Jan 23, 2026 | 0.5.6.post2 | 1.2.0rc6.post1 | 0.12.0 | 0.8.0 / 0.8.0 / 0.8.0 | - | [release notes](https://github.com/ai-dynamo/dynamo/releases/tag/v0.8.1) | Post trains .post1/.post2/.post3 republished the TRT-LLM runtime image and PyPI wheels only. | +| v0.8.0 | stable | Jan 15, 2026 | 0.5.6.post2 | 1.2.0rc6.post1 | 0.12.0 | 0.8.0 / 0.8.0 / 0.8.0 | - | [release notes](https://github.com/ai-dynamo/dynamo/releases/tag/v0.8.0) | dynamo-frontend image and CUDA 13 variants for vLLM and SGLang. First publish of dynamo-memory and dynamo-config crates. | +| v0.7.1 | patch | Dec 15, 2025 | 0.5.4.post3 | 1.2.0rc3 | 0.11.0 | 0.8.0 / 0.8.0 / 0.8.0 | - | [release notes](https://github.com/ai-dynamo/dynamo/releases/tag/v0.7.1) | - | +| v0.7.0 | stable | Nov 26, 2025 | 0.5.4.post3 | 1.2.0rc2 | 0.11.0 | 0.8.0 / 0.8.0 / 0.8.0 | - | [release notes](https://github.com/ai-dynamo/dynamo/releases/tag/v0.7.0) | - | +| v0.6.1 | patch | Nov 6, 2025 | 0.5.3.post2 | 1.1.0rc5 | 0.11.0 | 0.6.0 / 0.6.0 / 0.6.0 | - | [release notes](https://github.com/ai-dynamo/dynamo/releases/tag/v0.6.1) | - | +| v0.6.0 | stable | Oct 28, 2025 | 0.5.3.post2 | 1.1.0rc5 | 0.11.0 | 0.6.0 / 0.6.0 / 0.6.0 | - | [release notes](https://github.com/ai-dynamo/dynamo/releases/tag/v0.6.0) | Oldest release tracked on this page. | + +Release highlights (stable releases): + +- v1.3.0: Tool-calling and reasoning overhaul, RL rollout serving, the largest Router buildout to date, SLA-driven Planner autoscaling, and production GPU Memory Service on Kubernetes. +- v1.2.0: DGD/DGDR v1beta1, CRTC as the default KV router, inter-pod GPU Memory Service, Dynamo Snapshot on CRI-O/OpenShift, and DeepSeek-V4 recipes on vLLM. +- v1.1.0: Resilient KV routing at scale, Anthropic Messages API support, performance modeling and offline replay, and the multimodal embedding cache. +- v1.0.0: First GA release: unified configuration, Kubernetes production readiness, multimodal serving, and the agents surface. + +## CUDA toolkit and minimum driver history + +| Dynamo | Backend | CUDA Toolkit | Min Driver | Note | +| --- | --- | --- | --- | --- | +| 1.3.0 | SGLang | 13.0 | 580.xx+ | - | +| 1.3.0 | TensorRT-LLM | 13.1 | 580.xx+ | - | +| 1.3.0 | vLLM | 13.0 | 580.xx+ | - | +| 1.2.1 | SGLang | 12.9 | 575.xx+ | - | +| 1.2.1 | SGLang | 13.0 | 580.xx+ | - | +| 1.2.1 | TensorRT-LLM | 13.1 | 580.xx+ | - | +| 1.2.1 | vLLM | 12.9 | 575.xx+ | - | +| 1.2.1 | vLLM | 13.0 | 580.xx+ | - | +| 1.2.0 | SGLang | 12.9 | 575.xx+ | - | +| 1.2.0 | SGLang | 13.0 | 580.xx+ | - | +| 1.2.0 | TensorRT-LLM | 13.1 | 580.xx+ | - | +| 1.2.0 | vLLM | 12.9 | 575.xx+ | - | +| 1.2.0 | vLLM | 13.0 | 580.xx+ | - | +| 1.1.1 | SGLang | 12.9 | 575.xx+ | - | +| 1.1.1 | SGLang | 13.0 | 580.xx+ | - | +| 1.1.1 | TensorRT-LLM | 13.1 | 580.xx+ | - | +| 1.1.1 | vLLM | 12.9 | 575.xx+ | - | +| 1.1.1 | vLLM | 13.0 | 580.xx+ | - | +| 1.1.0 | SGLang | 12.9 | 575.xx+ | - | +| 1.1.0 | SGLang | 13.0 | 580.xx+ | - | +| 1.1.0 | TensorRT-LLM | 13.1 | 580.xx+ | - | +| 1.1.0 | vLLM | 12.9 | 575.xx+ | - | +| 1.1.0 | vLLM | 13.0 | 580.xx+ | - | +| 1.0.2 | SGLang | 12.9 | 575.xx+ | - | +| 1.0.2 | SGLang | 13.0 | 580.xx+ | - | +| 1.0.2 | TensorRT-LLM | 13.1 | 580.xx+ | - | +| 1.0.2 | vLLM | 12.9 | 575.xx+ | - | +| 1.0.2 | vLLM | 13.0 | 580.xx+ | - | +| 1.0.1 | SGLang | 12.9 | 575.xx+ | - | +| 1.0.1 | SGLang | 13.0 | 580.xx+ | - | +| 1.0.1 | TensorRT-LLM | 13.1 | 580.xx+ | - | +| 1.0.1 | vLLM | 12.9 | 575.xx+ | - | +| 1.0.1 | vLLM | 13.0 | 580.xx+ | - | +| 1.0.0 | SGLang | 12.9 | 575.xx+ | - | +| 1.0.0 | SGLang | 13.0 | 580.xx+ | - | +| 1.0.0 | TensorRT-LLM | 13.1 | 580.xx+ | - | +| 1.0.0 | vLLM | 12.9 | 575.xx+ | - | +| 1.0.0 | vLLM | 13.0 | 580.xx+ | - | +| 0.9.1 | SGLang | 12.9 | 575.xx+ | - | +| 0.9.1 | TensorRT-LLM | 13.0 | 580.xx+ | - | +| 0.9.1 | vLLM | 12.9 | 575.xx+ | - | +| 0.9.0 | SGLang | 12.9 | 575.xx+ | - | +| 0.9.0 | TensorRT-LLM | 13.0 | 580.xx+ | - | +| 0.9.0 | vLLM | 12.9 | 575.xx+ | - | +| 0.8.1 | SGLang | 12.9 | 575.xx+ | - | +| 0.8.1 | SGLang | 13.0 | 580.xx+ | Experimental | +| 0.8.1 | TensorRT-LLM | 13.0 | 580.xx+ | - | +| 0.8.1 | vLLM | 12.9 | 575.xx+ | - | +| 0.8.1 | vLLM | 13.0 | 580.xx+ | Experimental | +| 0.8.0 | SGLang | 12.9 | 575.xx+ | - | +| 0.8.0 | SGLang | 13.0 | 580.xx+ | Experimental | +| 0.8.0 | TensorRT-LLM | 13.0 | 580.xx+ | - | +| 0.8.0 | vLLM | 12.9 | 575.xx+ | - | +| 0.8.0 | vLLM | 13.0 | 580.xx+ | Experimental | +| 0.7.1 | SGLang | 12.8 | 570.xx+ | - | +| 0.7.1 | TensorRT-LLM | 13.0 | 580.xx+ | - | +| 0.7.1 | vLLM | 12.9 | 575.xx+ | - | +| 0.7.0 | SGLang | 12.9 | 575.xx+ | - | +| 0.7.0 | TensorRT-LLM | 13.0 | 580.xx+ | - | +| 0.7.0 | vLLM | 12.8 | 570.xx+ | - | + +- Patch versions (e.g. v0.8.1.post1, v0.7.0.post1) have the same CUDA support as their base version. +- Early access v1.1.0-dev.* images follow the same CUDA matrix as v1.0.2. The v1.2.0-deepseek-v4-dev.3 vLLM container is CUDA 13.0 multi-arch; the SGLang containers split by arch (CUDA 12.9 on amd64, CUDA 13.0 on arm64). +- Experimental CUDA 13 images are not published for all versions. + +## Feature support by backend (v1.3.0) + +| Feature | SGLang | TensorRT-LLM | vLLM | +| --- | --- | --- | --- | +| Disaggregated Serving | Supported | Supported | Supported (Prefill/decode separation with NIXL KV transfer) | +| KV-Aware Routing | Supported | Supported | Supported | +| SLA-Based Planner | Supported | Supported | Supported | +| KV Block Manager | Experimental (Work in progress across all combinations) | Supported | Supported | +| Multimodal (Image) | Supported (Not compatible with KV-aware routing. Disagg patterns: EPD, E/PD, E/P/D (not traditional EP/D)) | Supported (Image URLs + pre-computed embeddings. Disagg: EP/D + E/P/D. KV-aware routing via dedicated MM Router Worker (requires KV event publishing)) | Supported (With KV-aware routing, image-aware routing on documented paths) | +| Multimodal (Video) | Supported | Not supported | Supported (Video input with frame sampling) | +| Multimodal (Audio) | Not supported | Not supported | Experimental (Qwen2-Audio, experimental) | +| Request Migration | Supported | Supported (Work in progress with multimodal) | Supported | +| Request Cancellation | Experimental (Remote-prefill-phase cancellation not supported in disaggregated mode) | Supported with caveat (Engine temporarily not notified of cancellations — resources for cancelled requests are not freed (known issue)) | Supported | +| LoRA | Not supported | Not supported | Supported (Dynamic load/unload; KV-aware routing supports adapter affinity) | +| Tool Calling | Supported | Supported | Supported | +| Speculative Decoding | Experimental (Code hooks exist; no examples or docs yet) | Supported | Supported (Eagle3) | +| Dynamo Snapshot | Supported | Not supported | Supported | + +## Artifact inventory (v1.3.0) + +| Category | Name | Description | Meta | Tags / install | +| --- | --- | --- | --- | --- | +| container | vllm-runtime | vLLM backend runtime | vLLM v0.23.0 · CUDA 13.0 · AMD64/ARM64 | `nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.3.0`; `nvcr.io/nvidia/ai-dynamo/vllm-runtime:1.3.0-efa` | +| container | sglang-runtime | SGLang backend runtime | SGLang v0.5.14 · CUDA 13.0 · AMD64/ARM64 | `nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.3.0`; `nvcr.io/nvidia/ai-dynamo/sglang-runtime:1.3.0-efa` | +| container | tensorrtllm-runtime | TensorRT-LLM backend runtime | TRT-LLM v1.3.0rc19 · CUDA 13.1 · AMD64/ARM64 | `nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0`; `nvcr.io/nvidia/ai-dynamo/tensorrtllm-runtime:1.3.0-efa` | +| container | dynamo-frontend | OpenAI-compatible API gateway with Endpoint Prediction Protocol (EPP) | AMD64/ARM64 | `nvcr.io/nvidia/ai-dynamo/dynamo-frontend:1.3.0` | +| container | dynamo-planner | Standalone Planner used by Profiler jobs and Planner pods | AMD64/ARM64 | `nvcr.io/nvidia/ai-dynamo/dynamo-planner:1.3.0` | +| container | kubernetes-operator | Operator that manages Dynamo deployments and CRDs | AMD64/ARM64 | `nvcr.io/nvidia/ai-dynamo/kubernetes-operator:1.3.0` | +| container | snapshot-agent (Preview) | Fast GPU worker recovery via CRIU | AMD64/ARM64 | `nvcr.io/nvidia/ai-dynamo/snapshot-agent:1.3.0` | +| wheel | ai-dynamo | Main package with backend integrations (vLLM, SGLang, TRT-LLM) | Python 3.10–3.12 · Linux (glibc v2.28+) | `uv pip install ai-dynamo==1.3.0.post1` | +| wheel | ai-dynamo-runtime | Core Python bindings for the Dynamo runtime | Python 3.10–3.12 · Linux (glibc v2.28+) | `uv pip install ai-dynamo-runtime==1.3.0.post1` | +| wheel | kvbm | KV Block Manager for disaggregated KV cache | Python 3.10–3.12 · Linux (glibc v2.28+) | `uv pip install kvbm==1.3.0.post1` | +| helm | dynamo-platform | Platform services (etcd, NATS) and the Dynamo Operator for a Dynamo cluster | - | `helm install dynamo-platform oci://helm.ngc.nvidia.com/nvidia/ai-dynamo/charts/dynamo-platform --version 1.3.0` | +| helm | snapshot | Snapshot DaemonSet for fast GPU worker recovery | - | `helm install snapshot oci://helm.ngc.nvidia.com/nvidia/ai-dynamo/charts/snapshot --version 1.3.0` | +| crate | dynamo-runtime | Core distributed runtime library | MSRV Rust v1.82 | `cargo add dynamo-runtime@1.3.0` | +| crate | dynamo-llm | LLM inference engine | MSRV Rust v1.82 | `cargo add dynamo-llm@1.3.0` | +| crate | dynamo-protocols | Async OpenAI-compatible API client | MSRV Rust v1.82 | `cargo add dynamo-protocols@1.3.0` | +| crate | dynamo-async-openai (Deprecated) | Legacy OpenAI client; use dynamo-protocols | MSRV Rust v1.82 · final release | `cargo add dynamo-async-openai@1.0.2` | +| crate | dynamo-parsers | Protocol parsers (SSE, JSON streaming) | MSRV Rust v1.82 | `cargo add dynamo-parsers@1.3.0` | +| crate | dynamo-memory | Memory management utilities | MSRV Rust v1.82 | `cargo add dynamo-memory@1.3.0` | +| crate | dynamo-config | Configuration management | MSRV Rust v1.82 | `cargo add dynamo-config@1.3.0` | +| crate | dynamo-tokens | Tokenizer bindings for LLM inference | MSRV Rust v1.82 | `cargo add dynamo-tokens@1.3.0` | +| crate | dynamo-tokenizers | Tokenizer library for LLM inference | MSRV Rust v1.82 | `cargo add dynamo-tokenizers@1.3.0` | +| crate | dynamo-mocker | Inference engine simulator for benchmarking | MSRV Rust v1.82 | `cargo add dynamo-mocker@1.3.0` | +| crate | dynamo-kv-router | KV-aware request routing library | MSRV Rust v1.82 | `cargo add dynamo-kv-router@1.3.0` | +| crate | kvbm-logical | Logical layer for the KV Block Manager | MSRV Rust v1.82 | `cargo add kvbm-logical@1.3.0` | + +## Known artifact issues + +| Release | Artifact | Issue | Status | +| --- | --- | --- | --- | +| v0.9.0 | dynamo-platform-0.9.0 | Helm chart sets operator image to 0.7.1 instead of 0.9.0. | Fixed in v0.9.0.post1 | +| v0.8.1 | vllm-runtime:0.8.1-cuda13 | Container fails to launch. | Known issue | +| v0.8.1 | sglang-runtime:0.8.1-cuda13, vllm-runtime:0.8.1-cuda13 | Multimodality not expected to work on ARM64. Works on AMD64. | Known limitation | +| v0.8.0 | sglang-runtime:0.8.0-cuda13 | CuDNN installation issue caused PyTorch v2.9.1 compatibility problems with nn.Conv3d — performance degradation and excessive memory usage in multimodal workloads. | Fixed in v0.8.1 (#5461) | + +## Crates: first published version on crates.io + +| Crate | First version | Date | +| --- | --- | --- | +| dynamo-runtime | 0.1.0 | 2025-03-18 | +| dynamo-llm | 0.2.0 | 2025-05-01 | +| dynamo-async-openai | 0.4.1 | 2025-08-27 | +| dynamo-parsers | 0.5.0 | 2025-09-18 | +| dynamo-memory | 0.8.0 | 2026-01-15 | +| dynamo-config | 0.8.0 | 2026-01-15 | +| dynamo-tokens | 0.9.0 | 2026-02-12 | +| dynamo-mocker | 1.0.0 | 2026-03-13 | +| dynamo-kv-router | 1.0.0 | 2026-03-13 | +| dynamo-protocols | 1.1.0 | 2026-05-04 | +| dynamo-tokenizers | 1.2.0 | 2026-06-02 | + +## Model early-access builds + +| Model | Tag | Release line | Runtimes | Shipped | GA path | Status | Coverage (images / wheels / helm / crates) | +| --- | --- | --- | --- | --- | --- | --- | --- | +| Inkling | `1.4.0-inkling-dev.1` | v1.4.0 | sglang-runtime | Jul 17, 2026 | Dev-only · v1.4.0 line | First build on the v1.4.0 line; targets the next stable release. | yes / no / no / no | +| GLM-5.2 | `1.3.0-glm-5.2-dev.1` | v1.3.0 | sglang-runtime | Jul 20, 2026 | Dev-only | Container carries SGLang cherry-picks (stability, config parsing, model support) opened upstream but not yet in a released SGLang. | yes / no / no / no | +| MiniMax-M3 | `1.3.0-minimax-m3-dev.1` | v1.3.0 | vllm-runtime, sglang-runtime, tensorrtllm-runtime | Jun 12, 2026 | Promoted → :1.3.0 | Dynamo changes and the M2 tool-calling fix are in release/1.3.0; the recipes run on the stock :1.3.0 containers. | yes / no / no / no | +| DeepSeek-V4 | `1.3.0-deepseek-v4-dev.1` | v1.3.0 | tensorrtllm-runtime | Jun 6, 2026 | Recipe in v1.3.0 | DeepSeek-V4 Flash and Pro recipes ship in v1.3.0 on the standard TensorRT-LLM release container. | yes / no / no / no | +| Nemotron-3-Ultra | `1.3.0-nemotron-ultra-dev.1` | v1.3.0 | vllm-runtime | Jun 5, 2026 | Dev-only | Four un-upstreamed vLLM patches; requires pinned flags VLLM_DISABLED_KERNELS=FlashInferFP8ScaledMMLinearKernel and --no-enable-flashinfer-autotune. | yes / no / no / no | +| Nemotron-3-Super | `1.3.0-nemotron-super-dev.1` | v1.3.0 | vllm-runtime | Jun 4, 2026 | Promoted → :1.3.0 | Both container patches are in the vLLM v0.23.0 that v1.3.0 ships; the recipe runs on the stock vllm-runtime:1.3.0. | yes / no / no / no | +| Kimi-K2.6 | `1.3.0-kimi-k2.6-dev.1` | v1.3.0 | vllm-runtime | Jun 4, 2026 | Promoted → :1.3.0 | The build's only container patch is in vLLM v0.23.0; the recipes run on the stock vllm-runtime:1.3.0. | yes / no / no / no | +| Cosmos-3 | `1.3.0-cosmos3-dev.1` | v1.3.0 | vllm-runtime | Jun 1, 2026 | Dev-only | Dynamo #10132 (Cosmos3 support in the vLLM-Omni backend) is open, not merged — v1.3.0 containers cannot run Cosmos3. | yes / no / no / no | +| DeepSeek-V4 preview | `1.2.0-deepseek-v4-dev.3` | v1.2.0 | vllm-runtime, sglang-runtime | May 9, 2026 | Superseded — recipe in v1.3.0 | Blackwell (B200 + GB200) preview; per-arch/CUDA tags (e.g. vllm-runtime:1.2.0-deepseek-v4-cuda13-dev.3). Superseded by the v1.3.0 recipe. | yes / no / no / no | +| DeepSeek-V4 preview | `1.2.0-deepseek-v4-dev.2` | v1.2.0 | vllm-runtime, sglang-runtime | May 1, 2026 | Superseded — recipe in v1.3.0 | Blackwell preview on vLLM v0.20.0 (native DSv4 support); superseded by dev.3. | yes / no / no / no | +| DeepSeek-V4 preview | `1.2.0-sglang-deepseek-v4-dev.1` | v1.2.0 | sglang-runtime | Apr 25, 2026 | Superseded — recipe in v1.3.0 | Earliest DSv4 preview (SGLang, B200 only); superseded by dev.2/dev.3. | yes / no / no / no | + +## Platform-preview artifact coverage + +| Preview | Images | Wheels | Helm | Crates | +| --- | --- | --- | --- | --- | +| v1.3.0-dev.1 | yes | yes | yes | yes | +| v1.1.0-dev.3 | yes | yes | no | no | +| v1.1.0-dev.2 | yes | yes | no | no | +| v1.1.0-dev.1 | yes | yes | yes | no | + +## Platform support + +- GPU architectures: Blackwell, Hopper, Ada Lovelace, Ampere +- OS: Ubuntu 24.04 (x86_64, ARM64) — Supported +- OS: Ubuntu 22.04 (x86_64) — Supported +- OS: CentOS Stream 9 (x86_64) — Experimental +- CSP: AWS — Amazon Linux 2023 (x86_64) — Supported +- CPU architectures: x86_64, ARM64 (Ubuntu 24.04 only) +- Wheels: Wheels are built in a manylinux_2_28-compatible environment and validated on CentOS Stream 9 and Ubuntu 22.04/24.04. Other Linux distributions are expected to work but are not officially verified. + +## Release statistics + +| Release | PRs | Contributors | First-time contributors | Breaking changes | Known issues | +| --- | --- | --- | --- | --- | --- | +| v1.3.0 | 930 | 125 | 23 | 24 | 10 | +| v1.2.0 | 603 | 82 | - | 5 | 11 | +| v1.1.0 | 896 | 113 | 12 | 8 | 20 | +| v1.0.0 | - | 90 | 34 | 41 | 14 | + +## Nightlies + +ai-dynamo and ai-dynamo-runtime nightly builds from main publish wheels tagged *.devYYYYMMDD (since Apr 24, 2026). Install with pip or uv using --pre and the NVIDIA extra-index pattern shown above. + +{/* llms-tables:end */} diff --git a/docs/fern/scripts/check_reference.sh b/docs/fern/scripts/check_reference.sh new file mode 100755 index 000000000000..9d56b1226255 --- /dev/null +++ b/docs/fern/scripts/check_reference.sh @@ -0,0 +1,126 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# check_reference.sh — one-command gate for the Reference pages. +# +# Run after every release bump (see the PER-RELEASE BUMP CHECKLIST in +# components/releases.data.ts) and before opening a docs PR: +# +# ./scripts/check_reference.sh +# +# Checks, in order: +# 1. Agent twins are fresh (gen_llms_tables.py --check) — also proves +# releases.data.ts still satisfies the generator's parser contract. +# 2. custom.js parses (node --check). +# 3. No stale reference/(support|feature)-matrix repo links outside the +# allowed remnants. +# 4. Every absolute /dynamo/dev/reference/... href in components, the data +# module, generated assets, and reference pages resolves to a URL the +# index.yml Reference General variant actually publishes. Catches nav +# restructures (e.g. pages moving under a new section slug) that +# fern broken-links cannot see because the hrefs live in TSX/JSON. +# 5. Fern broken-links contains zero errors inside reference/ pages +# (skipped with a warning if the fern CLI is unavailable). +set -euo pipefail +cd "$(dirname "$0")/.." + +fail=0 + +echo "== 1/5 agent twins fresh ==" +python3 scripts/gen_llms_tables.py --check || fail=1 + +echo "== 2/5 custom.js parses ==" +node --check custom.js || fail=1 + +echo "== 3/5 no stale matrix links ==" +stale=$(grep -rnE "reference/(support|feature)-matrix" --include="*.md" --include="*.mdx" . \ + | grep -vE "documentation-style-guide|^\./README\.md" || true) +if [[ -n "$stale" ]]; then + echo "$stale" + echo "stale support-matrix/feature-matrix links found" + fail=1 +else + echo "clean" +fi + +echo "== 4/5 absolute reference hrefs match the nav ==" +python3 - <<'PY' || fail=1 +"""Validate /dynamo/dev/reference/... hrefs against index.yml-derived URLs.""" +import pathlib +import re +import sys + +import yaml + +nav = yaml.safe_load(pathlib.Path("index.yml").read_text()) + + +def kebab(title: str) -> str: + return re.sub(r"[^a-z0-9]+", "-", title.lower()).strip("-") + + +def collect(items, prefix, urls): + for item in items: + if "page" in item: + slug = item.get("slug") or kebab(item["page"]) + urls.add(f"{prefix}/{slug}") + elif "section" in item: + slug = item.get("slug") or kebab(item["section"]) + collect(item.get("contents", []), f"{prefix}/{slug}", urls) + + +general = next( + variant + for tab in nav["navigation"] + if tab.get("tab") == "reference" + for variant in tab.get("variants", []) + if variant.get("title") == "General" +) +valid: set[str] = set() +collect(general["layout"], "/dynamo/dev/reference", valid) + +href_re = re.compile(r"/dynamo/dev/reference/[a-z0-9/-]+") +sources = [ + *pathlib.Path("components").glob("*.tsx"), + pathlib.Path("components/releases.data.ts"), + pathlib.Path("scripts/gen_llms_tables.py"), + *pathlib.Path("reference").rglob("*.mdx"), + pathlib.Path("assets/releases.json"), + pathlib.Path("assets/releases-atom.xml"), +] +bad = [] +for source in sources: + if not source.exists(): + continue + for lineno, line in enumerate(source.read_text().splitlines(), 1): + for href in href_re.findall(line): + if href.rstrip("/") not in valid: + bad.append(f"{source}:{lineno}: {href}") + +if bad: + print("\n".join(bad)) + print(f"{len(bad)} absolute reference href(s) do not match any published URL") + sys.exit(1) +print(f"clean ({len(valid)} published reference URLs)") +PY + +echo "== 5/5 fern broken-links (reference/ scope) ==" +if command -v fern >/dev/null 2>&1; then + out=$(fern docs broken-links 2>&1 || true) + scoped=$(echo "$out" | grep -cE "fix here: reference/" || true) + total=$(echo "$out" | grep -c "\[error\]" || true) + echo "total site errors: ${total} (pre-existing baseline elsewhere); in reference/: ${scoped}" + if [[ "${scoped}" != "0" ]]; then + echo "$out" | grep -B2 "fix here: reference/" | head -30 + fail=1 + fi +else + echo "WARNING: fern CLI not found — broken-links check skipped" +fi + +if [[ "$fail" != "0" ]]; then + echo "CHECK FAILED" + exit 1 +fi +echo "ALL CHECKS PASSED" diff --git a/docs/fern/scripts/gen_llms_tables.py b/docs/fern/scripts/gen_llms_tables.py new file mode 100755 index 000000000000..545b7c8ba0c8 --- /dev/null +++ b/docs/fern/scripts/gen_llms_tables.py @@ -0,0 +1,979 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Generate agent-facing twins of the Reference-page component data. + +Three Reference pages (reference/compatibility.mdx, +reference/release-artifacts.mdx, reference/model-early-access-builds.mdx) +render their data through custom React components, whose output may be absent +from Fern's agent-facing markdown exports (.md endpoints, llms.txt). This +script reads the single source of truth ``components/releases.data.ts`` and +emits equivalent markdown tables into each page between idempotent markers, +wrapped in so only agent exports see them. + +It also emits three machine-readable outputs from the same parse: + + * reference/releases-data.mdx — a "Releases (machine-readable)" page whose + body (between the same idempotent markers, NOT -wrapped) is the + full releases.data.ts content as plain markdown tables. + * assets/releases.json — a stable-schema JSON serialization of the parsed + data (current, mainTot, releases, cudaHistory, features, artifacts, + modelEaBuilds, platform, releaseStats, ...). Human dates gain ISO-8601 + ``dateIso`` twins. + * assets/releases-atom.xml — an Atom 1.0 feed, one entry per RELEASES item, + newest first. Entry links resolve notesHref against the canonical prod + base (https://docs.nvidia.com/dynamo), falling back to the GitHub release + URL. The feed ``updated`` stamp is the newest release date (deterministic + — never "now"). + +Usage (from any cwd; paths resolve relative to this file): + + python3 gen_llms_tables.py # write/refresh all outputs + python3 gen_llms_tables.py --check # exit 1 if any output is stale, no writes + +Re-run at every release bump, after editing releases.data.ts. + +The parser is deliberately conservative: releases.data.ts must stay a +disciplined literal (see PARSER CONTRACT below). Any construct the parser +does not understand aborts the run with a clear error -- partial output is +never emitted. + +PARSER CONTRACT (keep releases.data.ts within these bounds): + * Top-level declarations of the form ``[export] const NAME[: Type] = ;`` + * Literal values: object/array literals, string literals ('...' or "..."), + template literals whose only interpolations are ``${IDENT}`` where IDENT + is a previously declared string const (e.g. ``${GH}``, ``${NGC_C}``), + numbers, true/false/null, and bare identifier references to previously + declared consts (e.g. ``coverage: MODEL_COVERAGE``). + * Object keys: bare identifiers or quoted strings. + * No computed values, spreads, function calls, arithmetic, or ternaries. +""" + +from __future__ import annotations + +import argparse +import copy +import json +import re +import sys +from pathlib import Path + +SCRIPT_DIR = Path(__file__).resolve().parent +DATA_TS = SCRIPT_DIR.parent / "components" / "releases.data.ts" +REFERENCE_DIR = SCRIPT_DIR.parent / "reference" +ASSETS_DIR = SCRIPT_DIR.parent / "assets" +JSON_PATH = ASSETS_DIR / "releases.json" +ATOM_PATH = ASSETS_DIR / "releases-atom.xml" + +# Canonical prod host; notesHref values are absolute site paths (/dynamo/...). +PROD_HOST = "https://docs.nvidia.com" +PROD_BASE = f"{PROD_HOST}/dynamo" + +MARKER_BEGIN = ( + "{/* llms-tables:begin — generated by scripts/gen_llms_tables.py, do not edit */}" +) +MARKER_END = "{/* llms-tables:end */}" + +# Exports the generator depends on. Parsing aborts if any is missing. +REQUIRED_EXPORTS = [ + "RELEASES", + "MAIN_TOT", + "CURRENT_VERSION", + "CURRENT_DATE", + "CURRENT_TAG", + "CURRENT_WHEEL", + "CUDA_HISTORY", + "FEATURES", + "ARTIFACTS", + "MODEL_EA_BUILDS", + "PLATFORM", + "PLATFORM_PREVIEW_COVERAGE", + "KNOWN_ARTIFACT_ISSUES", + "CRATES_FIRST_PUBLISHED", + "RELEASE_STATS", +] + + +class TSParseError(Exception): + """releases.data.ts contains a construct the conservative parser rejects.""" + + +# --------------------------------------------------------------------------- +# TypeScript literal parsing (stdlib only, no TS runtime) +# --------------------------------------------------------------------------- + + +def strip_comments(src: str) -> str: + """Remove // and /* */ comments, preserving string/template contents. + + A character walk (not regex) so that ``https://...`` inside strings and + comment-like sequences inside template literals survive intact. + """ + out: list[str] = [] + i, n = 0, len(src) + while i < n: + c = src[i] + if c in "\"'`": + quote = c + out.append(c) + i += 1 + while i < n: + ch = src[i] + out.append(ch) + if ch == "\\" and i + 1 < n: # keep escaped char verbatim + out.append(src[i + 1]) + i += 2 + continue + i += 1 + if ch == quote: + break + continue + if c == "/" and i + 1 < n and src[i + 1] == "/": + while i < n and src[i] != "\n": + i += 1 + continue + if c == "/" and i + 1 < n and src[i + 1] == "*": + end = src.find("*/", i + 2) + if end == -1: + raise TSParseError("unterminated /* */ comment") + # Preserve newlines so any future error positions stay meaningful. + out.append("\n" * src.count("\n", i, end)) + i = end + 2 + continue + out.append(c) + i += 1 + return "".join(out) + + +class LiteralParser: + """Recursive-descent parser for disciplined TS object/array literals.""" + + def __init__(self, src: str, env: dict): + self.src = src + self.i = 0 + self.env = env # previously parsed consts, for ${IDENT} / bare refs + + def error(self, msg: str) -> TSParseError: + line = self.src.count("\n", 0, self.i) + 1 + context = self.src[self.i : self.i + 40].replace("\n", "\\n") + return TSParseError(f"{msg} (near line {line}: {context!r})") + + def ws(self) -> None: + while self.i < len(self.src) and self.src[self.i] in " \t\r\n": + self.i += 1 + + def peek(self) -> str: + return self.src[self.i] if self.i < len(self.src) else "" + + def expect(self, ch: str) -> None: + if self.peek() != ch: + raise self.error(f"expected {ch!r}") + self.i += 1 + + def parse_value(self): + self.ws() + c = self.peek() + if c == "{": + return self.parse_object() + if c == "[": + return self.parse_array() + if c in "\"'": + return self.parse_string(c) + if c == "`": + return self.parse_template() + if c.isdigit() or c == "-": + return self.parse_number() + if c.isalpha() or c == "_": + return self.parse_word() + raise self.error("unexpected token") + + def parse_object(self) -> dict: + self.expect("{") + obj: dict = {} + while True: + self.ws() + if self.peek() == "}": + self.i += 1 + return obj + key = self.parse_key() + self.ws() + self.expect(":") + obj[key] = self.parse_value() + self.ws() + if self.peek() == ",": + self.i += 1 + elif self.peek() != "}": + raise self.error("expected ',' or '}' in object") + + def parse_key(self) -> str: + c = self.peek() + if c in "\"'": + return self.parse_string(c) + m = re.match(r"[A-Za-z_$][\w$]*", self.src[self.i :]) + if not m: + raise self.error("expected object key") + self.i += m.end() + return m.group(0) + + def parse_array(self) -> list: + self.expect("[") + arr: list = [] + while True: + self.ws() + if self.peek() == "]": + self.i += 1 + return arr + arr.append(self.parse_value()) + self.ws() + if self.peek() == ",": + self.i += 1 + elif self.peek() != "]": + raise self.error("expected ',' or ']' in array") + + def parse_string(self, quote: str) -> str: + self.expect(quote) + out: list[str] = [] + while True: + if self.i >= len(self.src): + raise self.error("unterminated string") + c = self.src[self.i] + if c == "\\": + out.append(self.unescape()) + continue + self.i += 1 + if c == quote: + return "".join(out) + out.append(c) + + def parse_template(self) -> str: + """Template literal; only ``${IDENT}`` interpolations of known + string consts are allowed (the ``${GH}`` / ``${NGC_C}`` prefixes).""" + self.expect("`") + out: list[str] = [] + while True: + if self.i >= len(self.src): + raise self.error("unterminated template literal") + c = self.src[self.i] + if c == "\\": + out.append(self.unescape()) + continue + if c == "`": + self.i += 1 + return "".join(out) + if c == "$" and self.src[self.i : self.i + 2] == "${": + m = re.match(r"\$\{([A-Za-z_$][\w$]*)\}", self.src[self.i :]) + if not m: + raise self.error("only ${IDENT} interpolation is supported") + name = m.group(1) + if name not in self.env or not isinstance(self.env[name], str): + raise self.error( + f"${{{name}}}: not a previously declared string const" + ) + out.append(self.env[name]) + self.i += m.end() + continue + out.append(c) + self.i += 1 + + def unescape(self) -> str: + esc = self.src[self.i + 1] if self.i + 1 < len(self.src) else "" + self.i += 2 + return {"n": "\n", "t": "\t", "r": "\r"}.get(esc, esc) + + def parse_number(self): + m = re.match(r"-?\d+(?:\.\d+)?", self.src[self.i :]) + if not m: + raise self.error("bad number") + self.i += m.end() + text = m.group(0) + return float(text) if "." in text else int(text) + + def parse_word(self): + m = re.match(r"[A-Za-z_$][\w$]*", self.src[self.i :]) + word = m.group(0) + self.i += m.end() + if word == "true": + return True + if word == "false": + return False + if word in ("null", "undefined"): + return None + # Bare identifier reference to a previously declared const + # (e.g. ``coverage: MODEL_COVERAGE``). Deep-copied so shared + # references never alias each other in the parsed data. + if word in self.env: + return copy.deepcopy(self.env[word]) + raise self.error(f"unknown identifier {word!r}") + + +DECL_RE = re.compile( + r"^\s*(?:export\s+)?const\s+([A-Za-z_$][\w$]*)\s*(?::[^=;]+?)?=", + re.MULTILINE, +) + + +def parse_data_module(path: Path) -> dict: + """Parse every top-level const in releases.data.ts into a dict.""" + try: + src = strip_comments(path.read_text(encoding="utf-8")) + except OSError as exc: + raise TSParseError(f"cannot read {path}: {exc}") from exc + + env: dict = {} + for m in DECL_RE.finditer(src): + name = m.group(1) + parser = LiteralParser(src, env) + parser.i = m.end() + try: + env[name] = parser.parse_value() + except TSParseError as exc: + raise TSParseError(f"while parsing const {name}: {exc}") from exc + parser.ws() + if parser.peek() == ";": + parser.i += 1 + + missing = [n for n in REQUIRED_EXPORTS if n not in env] + if missing: + raise TSParseError(f"missing required exports: {', '.join(missing)}") + return env + + +# --------------------------------------------------------------------------- +# Markdown rendering +# --------------------------------------------------------------------------- + + +def cell(value) -> str: + """Sanitize a value for a markdown table cell.""" + if value is None or value == "": + return "-" + return str(value).replace("|", "\\|").replace("\n", " ") + + +def md_table(headers: list[str], rows: list[list]) -> str: + lines = [ + "| " + " | ".join(headers) + " |", + "| " + " | ".join("---" for _ in headers) + " |", + ] + for row in rows: + lines.append("| " + " | ".join(cell(v) for v in row) + " |") + return "\n".join(lines) + + +def nixl_cell(pins: dict) -> str: + parts = [pins.get(k) or "-" for k in ("nixlSglang", "nixlTrtllm", "nixlVllm")] + return " / ".join(parts) if any(p != "-" for p in parts) else "-" + + +FEATURE_STATUS_LABEL = { + "yes": "Supported", + "caveat": "Supported with caveat", + "wip": "Experimental", + "no": "Not supported", +} + + +def feature_cell(fc: dict) -> str: + status = fc.get("status") + if status not in FEATURE_STATUS_LABEL: + raise TSParseError(f"unknown feature status {status!r}") + label = FEATURE_STATUS_LABEL[status] + note = fc.get("note") + return f"{label} ({note})" if note else label + + +# --------------------------------------------------------------------------- +# Shared section renderers (composed by both the per-page twins +# and the machine-readable releases-data.mdx page; each returns a headerless +# fragment so every call site can supply its own heading style). +# --------------------------------------------------------------------------- + + +def cuda_table(data: dict) -> str: + """CUDA toolkit and minimum-driver history table.""" + rows = [ + [r["version"], r["backend"], r["toolkit"], r["minDriver"], r.get("note")] + for r in data["CUDA_HISTORY"] + ] + return md_table(["Dynamo", "Backend", "CUDA Toolkit", "Min Driver", "Note"], rows) + + +def feature_table(data: dict) -> str: + """Feature-support-by-backend table.""" + rows = [ + [ + f["name"], + feature_cell(f["sglang"]), + feature_cell(f["trtllm"]), + feature_cell(f["vllm"]), + ] + for f in data["FEATURES"] + ] + return md_table(["Feature", "SGLang", "TensorRT-LLM", "vLLM"], rows) + + +def artifact_table(data: dict) -> str: + """Artifact-inventory table.""" + rows: list[list] = [] + for art in data["ARTIFACTS"]: + name = art["name"] + if art.get("badge"): + name = f"{name} ({art['badge']})" + tags = "; ".join(f"`{t['clipboard']}`" for t in art.get("tags", [])) + rows.append([art["category"], name, art["description"], art.get("meta"), tags]) + return md_table(["Category", "Name", "Description", "Meta", "Tags / install"], rows) + + +def known_issues_table(data: dict) -> str: + """Known-artifact-issues table.""" + rows = [ + [i["version"], i["artifact"], i["issue"], i["status"]] + for i in data["KNOWN_ARTIFACT_ISSUES"] + ] + return md_table(["Release", "Artifact", "Issue", "Status"], rows) + + +def crates_table(data: dict) -> str: + """First-published-version-on-crates.io table.""" + rows = [ + [c["crate"], c["version"], c["date"]] for c in data["CRATES_FIRST_PUBLISHED"] + ] + return md_table(["Crate", "First version", "Date"], rows) + + +def platform_lines(data: dict) -> str: + """Platform-support bullet lines (GPUs / OS / CSP / CPU arch / wheels).""" + plat = data["PLATFORM"] + lines = [f"- GPU architectures: {', '.join(plat['gpus'])}"] + for os_row in plat["os"]: + lines.append( + f"- OS: {os_row['name']} {os_row['version']} ({os_row['arch']}) — {os_row['status']}" + ) + for csp in plat.get("csp", []): + lines.append( + f"- CSP: {csp['provider']} — {csp['os']} ({csp['arch']}) — {csp['status']}" + ) + lines.append(f"- CPU architectures: {', '.join(plat['arch'])}") + if plat.get("wheelsNote"): + lines.append(f"- Wheels: {plat['wheelsNote']}") + return "\n".join(lines) + + +def ea_table(data: dict) -> str: + """Model early-access builds table.""" + rows: list[list] = [] + for b in data["MODEL_EA_BUILDS"]: + cov = b["coverage"] + cov_cell = " / ".join( + "yes" if cov.get(k) else "no" + for k in ("images", "wheels", "helm", "crates") + ) + rows.append( + [ + b["model"], + f"`{b['tag']}`", + b["releaseLine"], + ", ".join(b["runtimes"]), + b["shipped"], + b["gaLabel"], + b["statusLine"], + cov_cell, + ] + ) + return md_table( + [ + "Model", + "Tag", + "Release line", + "Runtimes", + "Shipped", + "GA path", + "Status", + "Coverage (images / wheels / helm / crates)", + ], + rows, + ) + + +def render_compatibility(data: dict) -> str: + parts: list[str] = [] + parts.append( + f"Current stable release: {data['CURRENT_VERSION']} " + f"(container tag `{data['CURRENT_TAG']}`, wheel version `{data['CURRENT_WHEEL']}`)." + ) + + # Backend pins: main (ToT) first, then every tracked release, newest first. + pin_rows: list[list] = [] + tot = data["MAIN_TOT"] + pin_rows.append( + [ + "main (ToT)", + "development head", + tot.get("sglang"), + tot.get("trtllm"), + tot.get("vllm"), + nixl_cell(tot), + None, + ] + ) + for rel in data["RELEASES"]: + pins = rel.get("pins") or {} + pin_rows.append( + [ + rel["version"], + rel.get("kind"), + pins.get("sglang"), + pins.get("trtllm"), + pins.get("vllm"), + nixl_cell(pins), + rel.get("ucx"), + ] + ) + parts.append("**Backend engine pins per Dynamo release**") + parts.append( + md_table( + [ + "Dynamo", + "Type", + "SGLang", + "TensorRT-LLM", + "vLLM", + "NIXL (SGL / TRT / vLLM)", + "UCX", + ], + pin_rows, + ) + ) + + parts.append("**CUDA toolkit and minimum driver per Dynamo release**") + parts.append(cuda_table(data)) + cuda_notes = data.get("CUDA_NOTES") or [] + if cuda_notes: + parts.append("\n".join(f"- {note}" for note in cuda_notes)) + + parts.append(f"**Feature support by backend ({data['CURRENT_VERSION']})**") + parts.append(feature_table(data)) + + # Platform support (GPUs / OS / arch) — rendered by CompatibilityHero. + parts.append("**Platform support**") + parts.append(platform_lines(data)) + + return "\n\n".join(parts) + + +def render_release_artifacts(data: dict) -> str: + parts: list[str] = [] + parts.append( + f"Current stable release: {data['CURRENT_VERSION']} " + f"(container tag `{data['CURRENT_TAG']}`, wheel version `{data['CURRENT_WHEEL']}`)." + ) + + parts.append(f"**Artifact inventory ({data['CURRENT_VERSION']})**") + parts.append(artifact_table(data)) + + parts.append("**Known artifact issues**") + parts.append(known_issues_table(data)) + + parts.append("**Crates: first published version on crates.io**") + parts.append(crates_table(data)) + + return "\n\n".join(parts) + + +def render_model_ea_builds(data: dict) -> str: + parts: list[str] = [] + parts.append("**Model early-access builds**") + parts.append(ea_table(data)) + return "\n\n".join(parts) + + +# --------------------------------------------------------------------------- +# Dates (fixed mapping — content never depends on the wall clock) +# --------------------------------------------------------------------------- + +_MONTHS = { + "Jan": 1, "Feb": 2, "Mar": 3, "Apr": 4, "May": 5, "Jun": 6, + "Jul": 7, "Aug": 8, "Sep": 9, "Oct": 10, "Nov": 11, "Dec": 12, +} # fmt: skip + +_DATE_RE = re.compile(r"^([A-Z][a-z]{2}) (\d{1,2}), (\d{4})$") + + +def iso_date(human: str) -> str: + """'Jul 20, 2026' -> '2026-07-20'. Fails closed on anything else.""" + m = _DATE_RE.match(human.strip()) + if not m or m.group(1) not in _MONTHS: + raise TSParseError(f"unparseable date {human!r} (expected 'Mon D, YYYY')") + return f"{int(m.group(3)):04d}-{_MONTHS[m.group(1)]:02d}-{int(m.group(2)):02d}" + + +def release_link(rel: dict) -> str | None: + """Docs-native notes URL on the canonical prod host; GitHub fallback.""" + href = rel.get("notesHref") + if href: + return PROD_BASE + href.removeprefix("/dynamo") + return rel.get("github") + + +# --------------------------------------------------------------------------- +# Machine-readable page (reference/releases-data.mdx) +# --------------------------------------------------------------------------- + + +def render_releases_data(data: dict) -> str: + """Full releases.data.ts content as plain markdown (not ).""" + parts: list[str] = [] + parts.append( + f"Current stable release: {data['CURRENT_VERSION']} ({data['CURRENT_DATE']}; " + f"container tag `{data['CURRENT_TAG']}`, wheel version `{data['CURRENT_WHEEL']}`)." + ) + + parts.append("## Releases") + rel_rows: list[list] = [] + tot = data["MAIN_TOT"] + rel_rows.append( + [ + "main (ToT)", + "development head", + "-", + tot.get("sglang"), + tot.get("trtllm"), + tot.get("vllm"), + nixl_cell(tot), + None, + "-", + "-", + ] + ) + for rel in data["RELEASES"]: + pins = rel.get("pins") or {} + link = release_link(rel) + rel_rows.append( + [ + rel["version"], + rel.get("kind"), + rel.get("date"), + pins.get("sglang"), + pins.get("trtllm"), + pins.get("vllm"), + nixl_cell(pins), + rel.get("ucx"), + f"[release notes]({link})" if link else None, + rel.get("delta") or rel.get("note"), + ] + ) + parts.append( + md_table( + [ + "Version", + "Kind", + "Date", + "SGLang", + "TensorRT-LLM", + "vLLM", + "NIXL (SGL / TRT / vLLM)", + "UCX", + "Notes", + "Delta", + ], + rel_rows, + ) + ) + summaries = [ + f"- {rel['version']}: {rel['notesSummary']}" + for rel in data["RELEASES"] + if rel.get("notesSummary") + ] + if summaries: + parts.append("Release highlights (stable releases):") + parts.append("\n".join(summaries)) + + parts.append("## CUDA toolkit and minimum driver history") + parts.append(cuda_table(data)) + cuda_notes = data.get("CUDA_NOTES") or [] + if cuda_notes: + parts.append("\n".join(f"- {note}" for note in cuda_notes)) + + parts.append(f"## Feature support by backend ({data['CURRENT_VERSION']})") + parts.append(feature_table(data)) + + parts.append(f"## Artifact inventory ({data['CURRENT_VERSION']})") + parts.append(artifact_table(data)) + + parts.append("## Known artifact issues") + parts.append(known_issues_table(data)) + + parts.append("## Crates: first published version on crates.io") + parts.append(crates_table(data)) + + parts.append("## Model early-access builds") + parts.append(ea_table(data)) + + parts.append("## Platform-preview artifact coverage") + ppc_rows = [ + [ + version, + "yes" if cov.get("images") else "no", + "yes" if cov.get("wheels") else "no", + "yes" if cov.get("helm") else "no", + "yes" if cov.get("crates") else "no", + ] + for version, cov in data["PLATFORM_PREVIEW_COVERAGE"].items() + ] + parts.append(md_table(["Preview", "Images", "Wheels", "Helm", "Crates"], ppc_rows)) + + parts.append("## Platform support") + parts.append(platform_lines(data)) + + parts.append("## Release statistics") + stat_rows = [ + [ + version, + stats.get("prs"), + stats.get("contributors"), + stats.get("firstTimers"), + stats.get("breaking"), + stats.get("knownIssues"), + ] + for version, stats in data["RELEASE_STATS"].items() + ] + parts.append( + md_table( + [ + "Release", + "PRs", + "Contributors", + "First-time contributors", + "Breaking changes", + "Known issues", + ], + stat_rows, + ) + ) + + if data.get("NIGHTLIES_NOTE"): + parts.append("## Nightlies") + parts.append(data["NIGHTLIES_NOTE"]) + + return "\n\n".join(parts) + + +# --------------------------------------------------------------------------- +# JSON payload (assets/releases.json) +# --------------------------------------------------------------------------- + + +def build_json(data: dict) -> str: + """Stable-schema JSON serialization of the parsed releases.data.ts.""" + releases = [] + for rel in data["RELEASES"]: + out = dict(rel) + if rel.get("date"): + out["dateIso"] = iso_date(rel["date"]) + link = release_link(rel) + if link: + out["notesUrl"] = link + releases.append(out) + + ea_builds = [] + for b in data["MODEL_EA_BUILDS"]: + out = dict(b) + out["shippedIso"] = iso_date(b["shipped"]) + ea_builds.append(out) + + if not any("dateIso" in r for r in releases): + raise TSParseError("no dated RELEASES entries — cannot build releases.json") + + payload = { + "source": "docs/fern/components/releases.data.ts", + "generator": "docs/fern/scripts/gen_llms_tables.py", + "updated": max(r["dateIso"] for r in releases if "dateIso" in r), + "current": { + "version": data["CURRENT_VERSION"], + "date": data["CURRENT_DATE"], + "dateIso": iso_date(data["CURRENT_DATE"]), + "tag": data["CURRENT_TAG"], + "wheel": data["CURRENT_WHEEL"], + }, + "mainTot": data["MAIN_TOT"], + "releases": releases, + "cudaHistory": data["CUDA_HISTORY"], + "cudaNotes": data.get("CUDA_NOTES") or [], + "features": data["FEATURES"], + "artifacts": data["ARTIFACTS"], + "modelEaBuilds": ea_builds, + "platformPreviewCoverage": data["PLATFORM_PREVIEW_COVERAGE"], + "platform": data["PLATFORM"], + "knownArtifactIssues": data["KNOWN_ARTIFACT_ISSUES"], + "cratesFirstPublished": data["CRATES_FIRST_PUBLISHED"], + "releaseStats": data["RELEASE_STATS"], + "nightliesNote": data.get("NIGHTLIES_NOTE"), + } + return json.dumps(payload, indent=2, ensure_ascii=False) + "\n" + + +# --------------------------------------------------------------------------- +# Atom feed (assets/releases-atom.xml) +# --------------------------------------------------------------------------- + + +def xml_escape(text: str) -> str: + return ( + text.replace("&", "&") + .replace("<", "<") + .replace(">", ">") + .replace('"', """) + ) + + +def build_atom(data: dict) -> str: + """Atom 1.0 feed of RELEASES, newest first, deterministic timestamps.""" + dated = [] + for index, rel in enumerate(data["RELEASES"]): + if not rel.get("date"): + continue # undated entries cannot be ordered in a feed + dated.append((iso_date(rel["date"]), index, rel)) + if not dated: + raise TSParseError("no dated RELEASES entries — cannot build the Atom feed") + # Newest first; ties keep releases.data.ts order (already newest-first). + dated.sort(key=lambda item: (item[0], -item[1]), reverse=True) + + feed_updated = dated[0][0] + "T00:00:00Z" + notes_index = f"{PROD_BASE}/dev/reference/releases/release-history" + + lines = [ + '', + "", + '', + " NVIDIA Dynamo releases", + f" {xml_escape(notes_index)}", + f' ', + f" {feed_updated}", + " NVIDIA Dynamo", + ] + for date, _index, rel in dated: + link = release_link(rel) + entry_id = rel.get("github") or link + if not entry_id or not link: + raise TSParseError( + f"release {rel['version']}: no notesHref or github link for the feed" + ) + summary = rel.get("notesSummary") or rel.get("delta") or rel.get("note") + lines += [ + " ", + f" Dynamo {xml_escape(rel['version'])} ({xml_escape(rel['kind'])})", + f" {xml_escape(entry_id)}", + f' ', + f" {date}T00:00:00Z", + ] + if summary: + lines.append(f" {xml_escape(summary)}") + lines.append(" ") + lines.append("") + return "\n".join(lines) + "\n" + + +# --------------------------------------------------------------------------- +# Emission +# --------------------------------------------------------------------------- + +# page -> (renderer, wrap_in_llms_only). The three component-backed pages get +# twins (humans see the React components); releases-data.mdx IS +# the page body, human-viewable and machine-consumable alike. +PAGES = { + "compatibility.mdx": (render_compatibility, True), + "release-artifacts.mdx": (render_release_artifacts, True), + "model-early-access-builds.mdx": (render_model_ea_builds, True), + "releases-data.mdx": (render_releases_data, False), +} + +# Standalone machine-readable outputs (path -> builder returning full text). +ASSET_OUTPUTS = { + JSON_PATH: build_json, + ATOM_PATH: build_atom, +} + +# Matches an existing generated span. Tolerant of edits to the note after +# "llms-tables:begin" so a hand-tweaked marker comment is still replaced. +BLOCK_RE = re.compile( + r"\{/\*\s*llms-tables:begin[^*]*\*/\}.*?\{/\*\s*llms-tables:end\s*\*/\}", + re.DOTALL, +) + + +def build_block(body: str, wrap: bool = True) -> str: + if wrap: + return f"{MARKER_BEGIN}\n\n\n{body}\n\n\n{MARKER_END}" + return f"{MARKER_BEGIN}\n\n{body}\n\n{MARKER_END}" + + +def apply_block(page_text: str, block: str) -> str: + if BLOCK_RE.search(page_text): + return BLOCK_RE.sub(lambda _m: block, page_text, count=1) + # No markers yet: append at end of file, before the trailing newline. + return page_text.rstrip("\n") + "\n\n" + block + "\n" + + +def main(argv: list[str]) -> int: + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument( + "--check", + action="store_true", + help="exit 1 if regeneration would change any page; write nothing", + ) + args = ap.parse_args(argv) + + try: + data = parse_data_module(DATA_TS) + except TSParseError as exc: + print(f"error: failed to parse {DATA_TS}: {exc}", file=sys.stderr) + return 2 + + # Render everything up front so a failure on any output emits nothing. + try: + rendered: dict[Path, str] = {} + for page_name, (renderer, wrap) in PAGES.items(): + page_path = REFERENCE_DIR / page_name + if not page_path.is_file(): + print(f"error: page not found: {page_path}", file=sys.stderr) + return 2 + old = page_path.read_text(encoding="utf-8") + rendered[page_path] = apply_block(old, build_block(renderer(data), wrap)) + for asset_path, builder in ASSET_OUTPUTS.items(): + rendered[asset_path] = builder(data) + except (TSParseError, KeyError) as exc: + print(f"error: rendering failed: {exc!r}", file=sys.stderr) + return 2 + + stale = [] + for path, new_text in rendered.items(): + name = path.name + old_text = path.read_text(encoding="utf-8") if path.is_file() else None + if new_text == old_text: + print(f"{name}: unchanged") + continue + stale.append(name) + if args.check: + print(f"{name}: STALE (regeneration would change it)") + else: + path.write_text(new_text, encoding="utf-8") + print(f"{name}: wrote {len(new_text.encode('utf-8'))} bytes") + + if args.check and stale: + print( + f"check failed: {len(stale)} output(s) stale — run gen_llms_tables.py", + file=sys.stderr, + ) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/docs/fern/scripts/simulate_docs_website.sh b/docs/fern/scripts/simulate_docs_website.sh new file mode 100755 index 000000000000..515a2bb21600 --- /dev/null +++ b/docs/fern/scripts/simulate_docs_website.sh @@ -0,0 +1,185 @@ +#!/usr/bin/env bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# simulate_docs_website.sh — local regression harness for the fern-docs.yml +# sync + release-version composition. +# +# The workflow's real jobs only fire on main pushes and tag cuts, so changes +# to the composition (rsync scopes, nav path transforms, the shared-Reference +# machinery) are otherwise unvalidatable before merge. This script replays +# both jobs against a scratch checkout of the local docs-website branch and +# asserts the invariants: +# +# 1. fern check on the composed tree reports 0 errors. +# 2. The generated versions/.yml keeps the Reference General variant +# on ../pages-dev/ (shared, always-current) while the Kubernetes API and +# Components variants point at the frozen ../pages-/ snapshot. +# 3. The pages- snapshot drops exactly the shared reference files and +# keeps the versioned ones (runtime-config, observability). +# 4. No React .tsx leaks into pages-dev/components/ (doc pages only). +# 5. Pre-rework version files gain no shared-reference pointers. +# 6. Round two: a page added to the General variant on a later main push +# propagates into the already-cut version's nav. +# +# Usage: ./scripts/simulate_docs_website.sh [TAG] +# TAG defaults to v9.9.9 (must not exist on docs-website yet). +# +# Requires: git, rsync, perl, yq v4, python3 >= 3.10 (or python3.13), and the +# fern CLI for check 1 (skipped with a warning if unavailable). +set -euo pipefail + +TAG="${1:-v9.9.9}" +REPO_ROOT="$(git rev-parse --show-toplevel)" +SRC="$REPO_ROOT/docs/fern" + +PY="$(command -v python3.13 || command -v python3)" +if ! "$PY" -c 'import sys; sys.exit(0 if sys.version_info >= (3, 10) else 1)'; then + echo "ERROR: python3 >= 3.10 required (convert_callouts.py uses 3.10 syntax)"; exit 1 +fi +command -v yq >/dev/null || { echo "ERROR: yq (v4) required"; exit 1; } + +WT="$(mktemp -d)/docs-checkout" +cleanup() { git -C "$REPO_ROOT" worktree remove --force "$WT" >/dev/null 2>&1 || true; } +trap cleanup EXIT +git -C "$REPO_ROOT" worktree add --quiet --detach "$WT" docs-website + +fail=0 +note() { printf '%-64s %s\n' "$1" "$2"; } +assert() { # assert