Repository navigation
[Rust Frontend][gRPC] Preserve multimodal metadata for remote-prefill decode #54814
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 1 commit
4e28495
905b964
32b31c9
78a7f2b
21b373e
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -35,6 +35,62 @@ struct PreparedGrpcRequest { | |
| started_at: Instant, | ||
| } | ||
|
|
||
| /// Keep producer metadata for matching EC items; leave unmatched inputs intact. | ||
| fn apply_encoder_cache_placeholders(text_request: &mut TextRequest) { | ||
| let is_decode_kv_consumer = text_request | ||
| .sampling_params | ||
| .vllm_xargs | ||
| .as_ref() | ||
| .and_then(|args| args.get("kv_transfer_params")) | ||
| .and_then(|params| params.get("do_remote_prefill")) | ||
| .and_then(serde_json::Value::as_bool) | ||
| .unwrap_or(false); | ||
| let Some(ec_items) = text_request | ||
| .sampling_params | ||
| .vllm_xargs | ||
| .as_ref() | ||
| .and_then(|args| args.get("ec_transfer_params")) | ||
| .and_then(|params| params.get("ec_items")) | ||
| .and_then(serde_json::Value::as_array) | ||
| .cloned() | ||
| else { | ||
| return; | ||
| }; | ||
| let Some(features) = text_request.mm_features.as_mut() else { | ||
| return; | ||
| }; | ||
|
|
||
| for (feature, item) in features.iter_mut().zip(ec_items) { | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🎯 Functional Correctness | 🟠 Major | ⚡ Quick win Match encoder-cache items by
Index 🤖 Prompt for AI Agents✅ Addressed in commit 905b964
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This seems valid. Shall we index items by hash instead?
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
This makes matching independent of item order. It also keeps unmatched features unchanged for inline processing. Add a multi-image test where 🐇 You are interacting with an AI system.
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Fixed @BugenZhao, along with a small change to the added tests. |
||
| let Some(item) = item.as_object() else { | ||
| continue; | ||
| }; | ||
| if item.get("mm_hash").and_then(serde_json::Value::as_str) | ||
| != Some(feature.identifier.as_str()) | ||
| { | ||
| continue; | ||
| } | ||
| let Some(data) = feature.data.as_mut() else { | ||
| continue; | ||
| }; | ||
| let metadata_keys: Vec<_> = data | ||
| .keys() | ||
| .filter(|key| key.as_str() != "mm_hash" && item.contains_key(key.as_str())) | ||
| .cloned() | ||
| .collect(); | ||
| if metadata_keys.is_empty() { | ||
| continue; | ||
| } | ||
| data.retain(|key, _| metadata_keys.contains(key)); | ||
| } | ||
|
|
||
| // Decode uses EC metadata only to prepare the prompt; EngineCore consumes KV. | ||
| if is_decode_kv_consumer { | ||
| if let Some(args) = text_request.sampling_params.vllm_xargs.as_mut() { | ||
| args.remove("ec_transfer_params"); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| impl InferenceServiceImpl { | ||
| pub fn new(state: Arc<AppState>) -> Self { | ||
| Self { state } | ||
|
|
@@ -105,6 +161,7 @@ impl InferenceServiceImpl { | |
| .map_err(|error| Status::internal(error.to_report_string()))?; | ||
| text_request.prompt = Prompt::TokenIds(token_ids); | ||
| text_request.mm_features = mm_features; | ||
| apply_encoder_cache_placeholders(&mut text_request); | ||
| } | ||
|
|
||
| Ok(text_request) | ||
|
|
||
Uh oh!
There was an error while loading. Please reload this page.