refactor(frontend): unify native generate handlers behind format plugins

.github/codeowners/areas.yaml

@@ -10,6 +10,9 @@ areas:
   - components/src/dynamo/frontend/tests/
   - examples/chat_templates/
   - lib/bindings/python/rust/parsers.rs
+  - lib/generate-format-catalog/
+  - lib/llm/src/generate_format.rs
+  - lib/llm/src/generate_format/
   - lib/llm/src/entrypoint.rs
   - lib/llm/src/entrypoint/
   - lib/llm/src/entrypoint/input.rs

CODEOWNERS

@@ -66,6 +66,7 @@
 
 # === frontend  (@ai-dynamo/dynamo-frontend-codeowners) ===
 /examples/chat_templates/                                                                               @ai-dynamo/dynamo-frontend-codeowners
+/lib/generate-format-catalog/                                                                           @ai-dynamo/dynamo-frontend-codeowners
 /components/src/dynamo/frontend/                                                                        @ai-dynamo/dynamo-frontend-codeowners
 /components/src/dynamo/frontend/tests/                                                                  @ai-dynamo/dynamo-frontend-codeowners
 
@@ -509,6 +510,7 @@ requirements.txt
 /lib/llm/src/preprocessor.rs                                                                            @ai-dynamo/dynamo-frontend-codeowners
 /lib/llm/src/block_manager.md                                                                           @ai-dynamo/dynamo-kv-memory-codeowners
 /lib/llm/src/block_manager.rs                                                                           @ai-dynamo/dynamo-kv-memory-codeowners
+/lib/llm/src/generate_format/                                                                           @ai-dynamo/dynamo-frontend-codeowners
 /tests/benchmarks/multimodal/                                                                           @ai-dynamo/dynamo-multimodal-codeowners
 /tests/serve/test_vllm_xpu.py                                                                           @ai-dynamo/dynamo-xpu-codeowners
 /.github/codeowners/areas.yaml                                                                          @ai-dynamo/dynamo-process-codeowners
@@ -520,6 +522,7 @@ requirements.txt
 /lib/llm/src/protocols/common/                                                                          @ai-dynamo/dynamo-frontend-codeowners
 /lib/llm/src/protocols/openai/                                                                          @ai-dynamo/dynamo-frontend-codeowners
 /tests/fault_tolerance/deploy/                                                                          @ai-dynamo/dynamo-fault-tolerance-codeowners
+/lib/llm/src/generate_format.rs                                                                         @ai-dynamo/dynamo-frontend-codeowners
 /lib/llm/src/kv_router/indexer/                                                                         @ai-dynamo/dynamo-router-codeowners
 /tests/fault_tolerance/etcd_ha/                                                                         @ai-dynamo/dynamo-fault-tolerance-codeowners
 /lib/llm/src/block_manager/pool/                                                                        @ai-dynamo/dynamo-kv-memory-codeowners

Cargo.toml

@@ -14,6 +14,7 @@ members = [
     "lib/kv-router",
     "lib/router-plugins/catalog",
     "lib/router-plugins/builtin",
+    "lib/generate-format-catalog",
     "examples/router/custom-policy-example/soft-pin-repin",
     "examples/router/custom-policy-example/simple-filter-score-pick",
     "examples/router/custom-policy-example/catalog",
@@ -58,6 +59,7 @@ keywords = ["llm", "genai", "inference", "nvidia", "distributed"]
 
 [workspace.dependencies]
 dynamo-custom-policy-builtin = { path = "lib/router-plugins/builtin", version = "1.6.0" }
+dynamo-generate-format-catalog = { path = "lib/generate-format-catalog", version = "1.6.0" }
 # Published separately by https://github.com/ai-dynamo/aisimulate.
 aisimulate-core = { version = "=0.13.0-dev.2" }
 

components/src/dynamo/sglang/tests/test_sglang_generate_merge_contract.py

@@ -43,7 +43,18 @@
 @pytest.fixture(scope="module")
 def rules() -> dict:
     assert RULES_PATH.is_file(), f"merge rules not found at {RULES_PATH}"
-    return json.loads(RULES_PATH.read_text())
+    rules = json.loads(RULES_PATH.read_text())
+    # Chunk joins by field class: deltas append, prompt values come from
+    # prefill, routed-expert rows overlay, and everything else keeps the last
+    # value. Nested pointers refine a field classified above them.
+    joins = {"delta": "append", "prompt": "take_prefill"}
+    joins["sequence_final"] = "overlay_base64_prefix"
+    for category in CATEGORIES:
+        rules[category] = []
+    for path, field_class in rules["fields"].items():
+        if path.count("/") <= 2:
+            rules[joins.get(field_class, "take_last")].append(path)
+    return rules
 
 
 def _sglang_tree(relative: str) -> ast.Module:

lib/bindings/python/Cargo.toml

@@ -99,6 +99,8 @@ uuid = { version = "1.18.1" }
 dynamo-kv-router = { path = "../../kv-router" }
 dynamo-custom-policy-builtin = { path = "../../router-plugins/builtin" }
 dynamo-worker-selection-policy-catalog = { path = "../../router-plugins/catalog", optional = true }
+# The replaceable /generate format catalog; custom images point this name at their own.
+dynamo-generate-format-catalog = { path = "../../generate-format-catalog" }
 clap = { version = "4.5", features = ["derive"], optional = true }
 tracing-subscriber = { version = "0.3", features = ["env-filter"], optional = true }
 

lib/bindings/python/rust/lib.rs

@@ -290,6 +290,11 @@ fn register_core(m: &Bound<'_, PyModule>) -> PyResult<()> {
         rs::logging::init();
     }
 
+    let mut formats = dynamo_llm::generate_format::Registry::default();
+    dynamo_generate_format_catalog::register(&mut formats)
+        .map_err(|error| PyRuntimeError::new_err(error.to_string()))?;
+    dynamo_llm::generate_format::install(formats);
+
     // Size the runtime the bridge may build for itself.
     //
     // `DistributedRuntime::new` gives the bridge a configured runtime, but only when it gets

lib/generate-format-catalog/Cargo.toml

@@ -0,0 +1,16 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+[package]
+name = "dynamo-generate-format-catalog"
+version.workspace = true
+edition.workspace = true
+authors.workspace = true
+license.workspace = true
+homepage.workspace = true
+repository.workspace = true
+description = "Default build-time /generate format catalog for Dynamo"
+
+[dependencies]
+dynamo-llm = { path = "../llm", default-features = false }
+anyhow = { workspace = true }

lib/generate-format-catalog/src/lib.rs

@@ -0,0 +1,28 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+//! The empty catalog used when a custom image does not replace Dynamo's
+//! `/generate` format catalog slot.
+//!
+//! To add a client format, replace this crate's dependency path in
+//! `lib/bindings/python/Cargo.toml` and register an `Arc<dyn GenerateFormat>`
+//! below. Implement `ClientCodec` with `SglangFormat::with_codec` to reuse
+//! SGLang's joining and migration, or implement `GenerateFormat` and
+//! `GenerateExchange` for another engine contract.
+//!
+//! Enable the `/generate` route with `DYN_SGLANG_ENABLE_GENERATE=1` (or
+//! `--enable-engine-apis`) and select the registered name with
+//! `DYN_GENERATE_FORMAT`. The default is `sglang`. The vLLM
+//! `/inference/v1/generate` route always uses the built-in `vllm` format.
+//! Unknown names and duplicate registrations fail at initialization.
+
+use dynamo_llm::generate_format::Registry;
+
+/// Register the formats linked into this image.
+///
+/// Custom catalogs replace this crate under the same dependency name in the
+/// frontend extension and in workers that read native responses. Dynamo's own
+/// formats are registered first, so a catalog cannot shadow one.
+pub fn register(_registry: &mut Registry) -> anyhow::Result<()> {
+    Ok(())
+}

lib/llm/Cargo.toml

@@ -37,6 +37,8 @@ mm-routing = ["dep:llm-multimodal", "dep:llm-tokenizer"]
 request-trace-bench = []
 # S3 destination for the request-trace sink, reusing the existing object_store client.
 request-trace-s3 = []
+# Simulator and property laws for testing /generate formats.
+generate-testkit = ["dep:proptest"]
 
 [[bench]]
 name = "tokenizer_simple"
@@ -118,6 +120,7 @@ tempfile = { workspace = true }
 thiserror = { workspace = true }
 tmq = { workspace = true }
 tokio = { workspace = true }
+proptest = { version = "1.5", optional = true }
 tokio-stream = { workspace = true, features = ["net"] }
 tokio-util = { workspace = true }
 tracing = { workspace = true }

lib/llm/src/discovery/model.rs

@@ -86,6 +86,8 @@ pub(crate) struct GenerateEngineSelection {
     pub(crate) kv_cache_block_size: u32,
     pub(crate) lora_name: Option<String>,
     pub(crate) tower_connector_lora_enabled: bool,
+    /// What the worker set advertises to its `/generate` format.
+    pub(crate) engine_info: serde_json::Value,
 }
 
 /// Readiness facts for one namespace, from [`Model::evaluate_namespace`].
@@ -684,6 +686,13 @@ impl Model {
                         .card()
                         .runtime_config
                         .runtime_flag_enabled(VLLM_ENABLE_TOWER_CONNECTOR_LORA_RUNTIME_KEY),
+                    engine_info: ws
+                        .card()
+                        .runtime_config
+                        .runtime_data
+                        .get(crate::generate_format::ENGINE_INFO_KEY)
+                        .cloned()
+                        .unwrap_or_default(),
                 })
         })
         .ok_or_else(|| self.engine_error(self.has_generate_engine_for_capability(capability)))

lib/llm/src/discovery/watcher.rs

@@ -152,9 +152,6 @@ fn supports_enabled_engine_generate(card: &ModelDeploymentCard, capabilities: &[
         .any(|capability| supports_generate_capability(card, capability))
 }
 
-// Generate's opaque request state is not yet verified for migration replay.
-const GENERATE_MIGRATION_LIMIT: u32 = 0;
-
 /// Resolve the effective [`WorkerType`] for a card during the
 /// cross-version rollout.
 ///
@@ -786,12 +783,7 @@ impl ModelWatcher {
                     anyhow::anyhow!("generate pipeline requires preprocessed routing")
                 })?;
                 let generate_engine = routing
-                    .build_preprocessed_pipeline(
-                        card,
-                        GENERATE_MIGRATION_LIMIT,
-                        None,
-                        self.metrics.clone(),
-                    )
+                    .build_generate_pipeline(card, self.migration_limit, self.metrics.clone())
                     .context("build generate (preprocessed) pipeline")?;
                 worker_set.generate_engine = Some(generate_engine);
                 tracing::info!("Generate (token-in/token-out) is ready");

lib/llm/src/entrypoint/input/common.rs

@@ -524,13 +524,39 @@ impl PreprocessedRouting {
         metrics: Arc<Metrics>,
     ) -> anyhow::Result<
         ServiceEngine<SingleIn<PreprocessedRequest>, ManyOut<Annotated<LLMEngineOutput>>>,
+    > {
+        self.link_preprocessed(Migration::from_mdc(
+            card,
+            migration_limit,
+            migration_max_seq_len,
+            metrics,
+        ))
+    }
+
+    /// The engine-native Generate pipeline: requests migrate only through
+    /// their `/generate` format's continuation.
+    pub fn build_generate_pipeline(
+        &self,
+        card: &ModelDeploymentCard,
+        migration_limit: u32,
+        metrics: Arc<Metrics>,
+    ) -> anyhow::Result<
+        ServiceEngine<SingleIn<PreprocessedRequest>, ManyOut<Annotated<LLMEngineOutput>>>,
+    > {
+        self.link_preprocessed(Migration::for_generate(card, migration_limit, metrics))
+    }
+
+    fn link_preprocessed(
+        &self,
+        migration: Arc<Migration>,
+    ) -> anyhow::Result<
+        ServiceEngine<SingleIn<PreprocessedRequest>, ManyOut<Annotated<LLMEngineOutput>>>,
     > {
         let frontend = SegmentSource::<
             SingleIn<PreprocessedRequest>,
             ManyOut<Annotated<LLMEngineOutput>>,
         >::new();
-        let migration = Migration::from_mdc(card, migration_limit, migration_max_seq_len, metrics)
-            .into_operator_for::<LLMEngineOutput>();
+        let migration = migration.into_operator_for::<LLMEngineOutput>();
         let prefill_op = self.prefill_router.into_operator();
         let encoder_op = self.encoder_router.into_operator();
         let backend = ServiceBackend::from_engine(self.backend_engine.clone());