feat(mm-routing): integrate GLM-5.3-Flash image planner

lib/bindings/python/tests/test_routing_image_token_id.py

@@ -0,0 +1,94 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+
+"""Exercise the worker's real Rust resolver without model weights or services."""
+
+import json
+import shutil
+from pathlib import Path
+
+import pytest
+
+from dynamo import _core
+
+pytestmark = [
+    pytest.mark.pre_merge,
+    pytest.mark.gpu_0,
+    pytest.mark.unit,
+    pytest.mark.multimodal,
+    pytest.mark.parallel,
+    pytest.mark.skipif(
+        not hasattr(_core, "resolve_routing_image_token_id"),
+        reason="requires a wheel built with mm-routing",
+    ),
+]
+
+
+@pytest.fixture(scope="module", autouse=True)
+def nats_and_etcd():
+    """Override the binding suite's service fixture for this pure resolver."""
+    yield
+
+
+@pytest.fixture
+def glm_model_dir(tmp_path):
+    fixture = Path(__file__).resolve().parents[3] / "llm/tests/data/glm5-next-routing"
+    for name in ("config.json", "processor_config.json"):
+        shutil.copyfile(fixture / name, tmp_path / name)
+    return tmp_path
+
+
+@pytest.mark.parametrize("flat", [False, True], ids=["nested", "flat"])
+def test_glm_resolves_checkpoint_placeholder(glm_model_dir, flat):
+    processor_path = glm_model_dir / "processor_config.json"
+    if flat:
+        processor = json.loads(processor_path.read_text())["image_processor"]
+        (glm_model_dir / "preprocessor_config.json").write_text(json.dumps(processor))
+        processor_path.unlink()
+
+    # A custom checkpoint name must work without tokenizer.json or weights.
+    token_id = _core.resolve_routing_image_token_id(
+        "custom-checkpoint", str(glm_model_dir)
+    )
+    assert type(token_id) is int
+    assert token_id == 154854
+
+
+@pytest.mark.parametrize(
+    ("file", "path", "value"),
+    [
+        ("config.json", ("image_token_id",), -1),
+        ("config.json", ("image_token_id",), 2**31),
+        ("config.json", ("vision_config", "patch_size"), 16),
+        ("config.json", ("model_type",), "unsupported_model"),
+        ("processor_config.json", ("image_processor", "do_resize"), False),
+        ("processor_config.json", ("image_processor", "max_image_tokens"), None),
+        ("processor_config.json", ("image_processor", "unknown_geometry_knob"), 1),
+    ],
+)
+def test_glm_unsupported_config_returns_none(glm_model_dir, file, path, value):
+    config_path = glm_model_dir / file
+    config = json.loads(config_path.read_text())
+    target = config
+    for key in path[:-1]:
+        target = target[key]
+    target[path[-1]] = value
+    config_path.write_text(json.dumps(config))
+
+    assert (
+        _core.resolve_routing_image_token_id("custom-checkpoint", str(glm_model_dir))
+        is None
+    )
+
+
+@pytest.mark.parametrize("invalid", [False, True], ids=["missing", "malformed"])
+def test_glm_unreadable_processor_returns_none(glm_model_dir, invalid):
+    processor_path = glm_model_dir / "processor_config.json"
+    if invalid:
+        processor_path.write_text("{")
+    else:
+        processor_path.unlink()
+    assert (
+        _core.resolve_routing_image_token_id("custom-checkpoint", str(glm_model_dir))
+        is None
+    )

lib/llm/Cargo.toml

@@ -33,7 +33,7 @@ bench = ["dynamo-kv-router/bench"]
 # is enabled by normal `dynamo-llm` builds and must not retain diagnostic hot-path work.
 ckf-diagnostics = []
 kv-router-stress = ["dep:clap", "dep:indicatif", "bench"]
-mm-routing = ["dep:llm-multimodal", "dep:llm-tokenizer"]
+mm-routing = ["dep:dynamo-multimodal", "dep:llm-multimodal", "dep:llm-tokenizer"]
 request-trace-bench = []
 # S3 destination for the request-trace sink, reusing the existing object_store client.
 request-trace-s3 = []
@@ -228,6 +228,8 @@ rmp-serde = "1.3"
 # reviewed step paired with re-running our golden cross-check against
 # `transformers`.
 llm-multimodal = { version = "=1.9.0", optional = true }
+# GLM's metadata-only planner; no parallel pixel processing is needed for routing.
+dynamo-multimodal = { version = "=0.2.2", default-features = false, optional = true }
 # Companion to llm-multimodal: provides `HuggingFaceTokenizer` so the
 # model_dir's `tokenizer.json` can be passed to the
 # `ModelRegistry::lookup` for image-placeholder token-id resolution.

lib/llm/src/preprocessor.rs

@@ -14,6 +14,9 @@
 pub mod media;
 #[cfg(feature = "mm-routing")]
 pub mod mm_routing;
+
+#[cfg(all(test, feature = "mm-routing"))]
+mod glm5_next_tests;
 pub mod prompt;
 pub mod speculative_prefill;
 pub(crate) mod structural_tag;
@@ -1765,8 +1768,7 @@ pub struct OpenAIPreprocessor {
     /// Tracks warmups and cancels them when this preprocessor is retired.
     speculative_prefill_tasks: speculative_prefill::PrefillTasks,
     /// Per-image token-count engine. `None` when the feature is disabled, the
-    /// model isn't covered by the registry, or `preprocessor_config.json` is
-    /// unreadable.
+    /// model isn't supported, or its processor configuration is invalid.
     #[cfg(feature = "mm-routing")]
     image_token_counter: Option<mm_routing::image::ImageRoutingProcessor>,
     /// Lightweight model-visible video expansion. Unlike the image counter,
@@ -4329,6 +4331,15 @@ impl OpenAIPreprocessor {
                     return None;
                 }
             };
+            if let Err(error) = counter.validate_image_prompt(token_ids, &dimensions) {
+                tracing::warn!(
+                    target: "mm_routing",
+                    model = counter.model_id(),
+                    %error,
+                    "image prompt layout is unsupported; skipping MM routing info"
+                );
+                return None;
+            }
             let (expanded, expanded_prompt_len) = try_expand_mm_routing_tokens(
                 self.tokenizer.as_ref(),
                 self.routing_image_prompt_layout