Skip to content
Merged
Show file tree
Hide file tree
Changes from 11 commits
Commits
Show all changes
27 commits
Select commit Hold shift + click to select a range
fd0d572
fix(mm-routing): MM-aware KV routing for Phi-3, Qwen2-VL, Qwen2.5-VL
krishung5 May 12, 2026
d52d504
perf(mm-routing): pass formatted_prompt by reference to avoid per-req…
krishung5 May 13, 2026
b216723
test(mm-routing): strong-gate agg_router profiles on lightseek init +…
krishung5 May 14, 2026
253f73e
test(mm-routing): settle kv-router index between repeats to fix Phi-3…
krishung5 May 27, 2026
688b675
Revert "test(mm-routing): settle kv-router index between repeats to f…
krishung5 May 27, 2026
5032de3
fix(mm-routing): mirror vLLM decode-roundtrip for Phi-3 routing tokens
krishung5 May 27, 2026
0af2150
fix(mm-routing): disable MM-aware routing when DYN_TOKENIZER=fastokens
krishung5 May 27, 2026
324fb0f
chore(mm-routing): cargo fmt fixup
krishung5 May 27, 2026
261a871
test(mm-routing): metric-based router_kv_hit_rate gate for MM agg_rou…
krishung5 May 27, 2026
e50b044
Merge remote-tracking branch 'origin/main' into krish/phi3-mm-routing…
krishung5 May 28, 2026
ab42746
Merge remote-tracking branch 'origin/main' into krish/phi3-mm-routing…
krishung5 May 29, 2026
646fe91
Merge remote-tracking branch 'origin/main' into krish/phi3-mm-routing…
krishung5 Jun 1, 2026
19d263d
test(mm-routing): rename require_lightseek_init -> require_rust_proce…
krishung5 Jun 1, 2026
211d975
test(mm-routing): drop lightseek wording from PR-added log strings
krishung5 Jun 1, 2026
523c02b
refactor(mm-routing): centralize HF config parsing in lightseek_mm.rs
krishung5 Jun 1, 2026
79d2057
refactor(mm-routing): collapse redundant image-token fields
krishung5 Jun 1, 2026
4024fdc
refactor(mm-routing): unify BOS handling + Phi-3 family guard in spli…
krishung5 Jun 1, 2026
8c02d52
test(mm-routing): compose router_kv_hit_rate metric name from canonic…
krishung5 Jun 1, 2026
d6a21c4
test(mm-routing): simplify router-histogram scraper to return (sum, c…
krishung5 Jun 1, 2026
d2bef12
test(tokenizers): add merge_special_tokens_from_config gate test + wa…
krishung5 Jun 1, 2026
21f02de
test(mm-routing): narrow _scrape_router_kv_hit_rate except to Request…
krishung5 Jun 1, 2026
b759d58
docs(mm-routing): fix stale Qwen2-VL placeholder-token comment
krishung5 Jun 1, 2026
098de96
fix(mm-routing): apply tokenizer_config.json merge before extracting …
krishung5 Jun 1, 2026
8f6d7a6
Merge remote-tracking branch 'origin/main' into krish/phi3-mm-routing…
krishung5 Jun 1, 2026
ced13a9
style(mm-routing): cargo fmt + clippy fixes for pre-merge gates
krishung5 Jun 1, 2026
2d7a88a
docs(mm-routing): add TODO marker for RouterMetricsAssertion mixin fo…
krishung5 Jun 1, 2026
769f6ea
fix(test): align rust mm-router init-log assertion with renamed log line
krishung5 Jun 2, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion examples/backends/vllm/launch/agg_multimodal_router.sh
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ NATS_SERVER="${NATS_SERVER:-nats://127.0.0.1:4222}"
ETCD_ENDPOINTS="${ETCD_ENDPOINTS:-http://127.0.0.1:2379}"
VLLM_SYSTEM_PORT_BASE="${VLLM_SYSTEM_PORT_BASE:-18081}"
KV_EVENTS_PORT_BASE="${KV_EVENTS_PORT_BASE:-5557}"
DYN_LOG_VAL="${DYN_LOG:-info,lightseek_mm=debug,dynamo_kv_router::scheduling=debug}"
DYN_LOG_VAL="${DYN_LOG:-info,lightseek_mm=debug,dynamo_kv_router::scheduling=debug,dynamo_llm::kv_router=debug}"

# Pass-through extra args for `python -m dynamo.vllm`.
VLLM_EXTRA_ARGS="${VLLM_EXTRA_ARGS:-}"
Expand Down
20 changes: 14 additions & 6 deletions lib/llm/src/model_card.rs
Original file line number Diff line number Diff line change
Expand Up @@ -985,14 +985,24 @@ impl ModelDeploymentCard {
})
.map_err(anyhow::Error::msg)
.with_context(|| p.display().to_string())?;

// Hold onto specials before any move of `hf`.
let specials: Vec<String> = if cache_enabled {
extract_hf_special_tokens(&hf)
} else {
Vec::new()
};

// Wrap an HfTokenizer, merging sibling tokenizer_config.json
// specials when a model_dir is available (needed for Qwen2-VL-2B).
let wrap_hf = |hf: HfTokenizer| match p.parent() {
Some(model_dir) => {
crate::tokenizers::HuggingFaceTokenizer::from_tokenizer_with_model_dir(
Comment thread
rmccorm4 marked this conversation as resolved.
Resolved
hf, model_dir,
)
}
None => crate::tokenizers::HuggingFaceTokenizer::from_tokenizer(hf),
};

// Pick the inner backend.
let raw: Arc<dyn crate::tokenizers::traits::Tokenizer> = if use_fast {
if let Some(path_str) = p.to_str() {
Expand All @@ -1006,20 +1016,18 @@ impl ModelDeploymentCard {
%e,
"Failed to load fastokens, falling back to HuggingFace"
);
Arc::new(crate::tokenizers::HuggingFaceTokenizer::from_tokenizer(
hf,
))
Arc::new(wrap_hf(hf))
}
}
} else {
tracing::warn!(
path = %p.display(),
"Tokenizer path contains non-UTF-8 characters, skipping fastokens; falling back to HuggingFace"
);
Arc::new(crate::tokenizers::HuggingFaceTokenizer::from_tokenizer(hf))
Arc::new(wrap_hf(hf))
}
} else {
Arc::new(crate::tokenizers::HuggingFaceTokenizer::from_tokenizer(hf))
Arc::new(wrap_hf(hf))
};

if cache_enabled {
Expand Down
Loading
Loading