refactor(frontend): unify native generate handlers behind format plugins

.github/codeowners/areas.yaml

@@ -10,6 +10,8 @@ areas:
   - components/src/dynamo/frontend/tests/
   - examples/chat_templates/
   - lib/bindings/python/rust/parsers.rs
+  - lib/generate-format-catalog/
+  - lib/llm/src/generate_format.rs
   - lib/llm/src/entrypoint.rs
   - lib/llm/src/entrypoint/
   - lib/llm/src/entrypoint/input.rs
@@ -88,6 +90,7 @@ areas:
   - components/src/dynamo/vllm/tests/
   - examples/backends/vllm/
   - lib/sidecar/vllm/
+  - lib/generate-formats/vllm/
 - label: backend-trtllm
   github_team: '@ai-dynamo/dynamo-backend-trtllm-codeowners'
   path_globs:
@@ -111,6 +114,7 @@ areas:
   - examples/backends/sglang/
   - lib/mocker/servers/sglang/
   - lib/sidecar/sglang/
+  - lib/generate-formats/sglang/
 - label: backend-triton
   github_team: '@ai-dynamo/dynamo-backend-triton-codeowners'
   path_globs:

CODEOWNERS

@@ -66,6 +66,7 @@
 
 # === frontend  (@ai-dynamo/dynamo-frontend-codeowners) ===
 /examples/chat_templates/                                                                               @ai-dynamo/dynamo-frontend-codeowners
+/lib/generate-format-catalog/                                                                           @ai-dynamo/dynamo-frontend-codeowners
 /components/src/dynamo/frontend/                                                                        @ai-dynamo/dynamo-frontend-codeowners
 /components/src/dynamo/frontend/tests/                                                                  @ai-dynamo/dynamo-frontend-codeowners
 
@@ -93,6 +94,7 @@
 # === backend-vllm  (@ai-dynamo/dynamo-backend-vllm-codeowners) ===
 /lib/sidecar/vllm/                                                                                      @ai-dynamo/dynamo-backend-vllm-codeowners
 /examples/backends/vllm/                                                                                @ai-dynamo/dynamo-backend-vllm-codeowners
+/lib/generate-formats/vllm/                                                                             @ai-dynamo/dynamo-backend-vllm-codeowners
 /components/src/dynamo/vllm/                                                                            @ai-dynamo/dynamo-backend-vllm-codeowners
 /components/src/dynamo/vllm/tests/                                                                      @ai-dynamo/dynamo-backend-vllm-codeowners
 
@@ -111,6 +113,7 @@
 # === backend-sglang  (@ai-dynamo/dynamo-backend-sglang-codeowners) ===
 /lib/sidecar/sglang/                                                                                    @ai-dynamo/dynamo-backend-sglang-codeowners
 /examples/backends/sglang/                                                                              @ai-dynamo/dynamo-backend-sglang-codeowners
+/lib/generate-formats/sglang/                                                                           @ai-dynamo/dynamo-backend-sglang-codeowners
 /components/src/dynamo/sglang/                                                                          @ai-dynamo/dynamo-backend-sglang-codeowners
 /components/src/dynamo/sglang/tests/                                                                    @ai-dynamo/dynamo-backend-sglang-codeowners
 /components/src/dynamo/sglang/request_handlers/                                                         @ai-dynamo/dynamo-backend-sglang-codeowners
@@ -520,6 +523,7 @@ requirements.txt
 /lib/llm/src/protocols/common/                                                                          @ai-dynamo/dynamo-frontend-codeowners
 /lib/llm/src/protocols/openai/                                                                          @ai-dynamo/dynamo-frontend-codeowners
 /tests/fault_tolerance/deploy/                                                                          @ai-dynamo/dynamo-fault-tolerance-codeowners
+/lib/llm/src/generate_format.rs                                                                         @ai-dynamo/dynamo-frontend-codeowners
 /lib/llm/src/kv_router/indexer/                                                                         @ai-dynamo/dynamo-router-codeowners
 /tests/fault_tolerance/etcd_ha/                                                                         @ai-dynamo/dynamo-fault-tolerance-codeowners
 /lib/llm/src/block_manager/pool/                                                                        @ai-dynamo/dynamo-kv-memory-codeowners

Cargo.toml

@@ -14,6 +14,9 @@ members = [
     "lib/kv-router",
     "lib/router-plugins/catalog",
     "lib/router-plugins/builtin",
+    "lib/generate-format-catalog",
+    "lib/generate-formats/sglang",
+    "lib/generate-formats/vllm",
     "examples/router/custom-policy-example/soft-pin-repin",
     "examples/router/custom-policy-example/simple-filter-score-pick",
     "examples/router/custom-policy-example/catalog",
@@ -58,6 +61,7 @@ keywords = ["llm", "genai", "inference", "nvidia", "distributed"]
 
 [workspace.dependencies]
 dynamo-custom-policy-builtin = { path = "lib/router-plugins/builtin", version = "1.6.0" }
+dynamo-generate-format-catalog = { path = "lib/generate-format-catalog", version = "1.6.0" }
 # Published separately by https://github.com/ai-dynamo/aisimulate.
 aisimulate-core = { version = "=0.13.0-dev.202610160000000073" }
 

components/src/dynamo/sglang/AGENTS.md

@@ -36,8 +36,8 @@ more reliable since SGLang's internal layout doesn't always match the version st
 
 **When you bump SGLang**: the frontend joins native `/generate` responses (streamed
 chunks into one unary response, prefill's prompt-side outputs into decode's) with rules
-in `lib/llm/src/protocols/sglang/merge_rules.json`. Those rules copy SGLang code listed
-in the header of `lib/llm/src/protocols/sglang/merge.rs`. Re-check that code, and run
+in `lib/generate-formats/sglang/src/merge_rules.json`. Those rules copy SGLang code listed
+in the header of `lib/generate-formats/sglang/src/merge.rs`. Re-check that code, and run
 `tests/test_sglang_generate_merge_contract.py`, which fails on any unclassified response
 field or request flag.
 

components/src/dynamo/sglang/tests/test_sglang_generate_merge_contract.py

@@ -5,8 +5,8 @@
 
 The frontend joins SGLang's streamed chunks into one unary response, and joins
 prefill's prompt-side outputs into decode's response, using the rules in
-``lib/llm/src/protocols/sglang/merge_rules.json``. Those rules copy SGLang code
-listed in the header of ``lib/llm/src/protocols/sglang/merge.rs``.
+``lib/generate-formats/sglang/src/merge_rules.json``. Those rules copy SGLang code
+listed in the header of ``lib/generate-formats/sglang/src/merge.rs``.
 
 These tests fail when an SGLang upgrade adds, renames, or changes a field the
 rules must know about. To fix a failure, read the upstream code named in
@@ -34,16 +34,29 @@
 
 RULES_PATH = (
     Path(__file__).resolve().parents[5]
-    / "lib/llm/src/protocols/sglang/merge_rules.json"
+    / "lib/generate-formats/sglang/src/merge_rules.json"
 )
 CATEGORIES = ("append", "take_prefill", "overlay_base64_prefix", "take_last")
-FIX = f"Classify it in {RULES_PATH.name} (see lib/llm/src/protocols/sglang/merge.rs)."
+FIX = (
+    f"Classify it in {RULES_PATH.name} (see lib/generate-formats/sglang/src/merge.rs)."
+)
 
 
 @pytest.fixture(scope="module")
 def rules() -> dict:
     assert RULES_PATH.is_file(), f"merge rules not found at {RULES_PATH}"
-    return json.loads(RULES_PATH.read_text())
+    rules = json.loads(RULES_PATH.read_text())
+    # Chunk joins by field class: deltas append, prompt values come from
+    # prefill, routed-expert rows overlay, and everything else keeps the last
+    # value. Nested pointers refine a field classified above them.
+    joins = {"delta": "append", "prompt": "take_prefill"}
+    joins["sequence_final"] = "overlay_base64_prefix"
+    for category in CATEGORIES:
+        rules[category] = []
+    for path, field_class in rules["fields"].items():
+        if path.count("/") <= 2:
+            rules[joins.get(field_class, "take_last")].append(path)
+    return rules
 
 
 def _sglang_tree(relative: str) -> ast.Module:

lib/bindings/python/Cargo.toml

@@ -99,6 +99,8 @@ uuid = { version = "1.18.1" }
 dynamo-kv-router = { path = "../../kv-router" }
 dynamo-custom-policy-builtin = { path = "../../router-plugins/builtin" }
 dynamo-worker-selection-policy-catalog = { path = "../../router-plugins/catalog", optional = true }
+# The replaceable /generate format catalog; custom images point this name at their own.
+dynamo-generate-format-catalog = { path = "../../generate-format-catalog" }
 clap = { version = "4.5", features = ["derive"], optional = true }
 tracing-subscriber = { version = "0.3", features = ["env-filter"], optional = true }
 

lib/bindings/python/rust/lib.rs

@@ -11,9 +11,7 @@ use dynamo_runtime::storage::kv;
 use futures::StreamExt;
 use once_cell::sync::OnceCell;
 use pyo3::IntoPyObjectExt;
-#[cfg(feature = "custom-policy")]
-use pyo3::exceptions::PyRuntimeError;
-use pyo3::exceptions::{PyStopAsyncIteration, PyTimeoutError, PyValueError};
+use pyo3::exceptions::{PyRuntimeError, PyStopAsyncIteration, PyTimeoutError, PyValueError};
 use pyo3::types::PyCapsule;
 use pyo3::types::{PyDict, PyString};
 use pyo3::{exceptions::PyException, prelude::*};
@@ -290,6 +288,15 @@ fn register_core(m: &Bound<'_, PyModule>) -> PyResult<()> {
         rs::logging::init();
     }
 
+    let mut formats = dynamo_llm::generate_format::Registry::default();
+    dynamo_generate_format_catalog::register(&mut formats)
+        .map_err(|error| PyRuntimeError::new_err(error.to_string()))?;
+    if !dynamo_llm::generate_format::install(formats) {
+        return Err(PyRuntimeError::new_err(
+            "generate format registry already installed",
+        ));
+    }
+
     // Size the runtime the bridge may build for itself.
     //
     // `DistributedRuntime::new` gives the bridge a configured runtime, but only when it gets

lib/generate-format-catalog/Cargo.toml

@@ -0,0 +1,29 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+[package]
+name = "dynamo-generate-format-catalog"
+version.workspace = true
+edition.workspace = true
+authors.workspace = true
+license.workspace = true
+homepage.workspace = true
+repository.workspace = true
+description = "Default build-time /generate format catalog for Dynamo"
+
+[dependencies]
+dynamo-llm = { path = "../llm", default-features = false }
+anyhow = { workspace = true }
+dynamo-sglang-generate = { path = "../generate-formats/sglang" }
+dynamo-vllm-generate = { path = "../generate-formats/vllm" }
+
+[dev-dependencies]
+dynamo-sglang-generate = { path = "../generate-formats/sglang", features = ["testkit"] }
+dynamo-runtime = { workspace = true }
+axum = { workspace = true }
+async-trait = { workspace = true }
+futures = { workspace = true }
+serde_json = { workspace = true }
+tokio = { workspace = true }
+tower = { workspace = true }
+prometheus = { workspace = true }

lib/generate-format-catalog/src/lib.rs

@@ -0,0 +1,23 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+//! Default catalog of client-facing `/generate` formats.
+//!
+//! Backend maintainers own each implementation in `lib/generate-formats/`.
+//! A custom frontend replaces this dependency in `lib/bindings/python/Cargo.toml`.
+//! Its catalog can depend on this crate under an alias, call [`register`], then
+//! add formats under distinct names. The frontend installs the resulting registry.
+//!
+//! `DYN_GENERATE_FORMAT` selects `/generate`'s format (default `sglang`).
+//! `/inference/v1/generate` selects `vllm`. Enable engine APIs to expose the routes.
+//! Unknown names and duplicate registrations fail at initialization.
+
+use dynamo_llm::generate_format::Registry;
+use std::sync::Arc;
+
+/// Register the built-in formats. Custom catalogs may extend this registry.
+pub fn register(registry: &mut Registry) -> anyhow::Result<()> {
+    registry.register(Arc::new(dynamo_sglang_generate::SglangFormat::native()))?;
+    registry.register(Arc::new(dynamo_vllm_generate::VllmFormat))?;
+    Ok(())
+}