[Rust] multimodal generation back-ported on vllm v0.28.0

rust/proto/control.proto

@@ -39,6 +39,8 @@ message ServerInfo {
   uint64 max_running_requests = 8;
   uint64 max_batched_tokens = 9;
   RlCapabilities rl_capabilities = 11;
+  bool supports_native_sampling_params_json = 12;
+  bool supports_preprocessed_media_features = 13;
 }
 
 message RlCapabilities {
@@ -54,6 +56,8 @@ message ParallelismInfo {
   uint32 data_parallel_size = 3;
   uint32 data_parallel_rank = 4;
   uint32 decode_context_parallel_size = 5;
+  // Process world size for one data-parallel engine (TP * PP * PCP).
+  uint64 world_size = 6;
 }
 
 message GetModelInfoRequest {}

rust/proto/inference.proto

@@ -50,6 +50,11 @@ message GenerateRequest {
 
   // Multimodal inputs aligned with placeholder markers in token_ids.
   repeated MediaItem media = 14;
+
+  // Exact JSON object from native /inference/v1/generate sampling_params.
+  // When non-empty, this is the sampling authority; structured fields remain
+  // the compatibility path for generic gRPC clients.
+  bytes native_sampling_params_json = 16;
 }
 
 message RandomSampling {
@@ -225,7 +230,20 @@ message MediaItem {
     string url = 2;       // http:// or https://
     string data_uri = 3;  // data:
     bytes raw_bytes = 4;
+    PreprocessedMediaFeatures features = 7;
   }
   string mime_type = 5;
   string uuid = 6;
 }
+
+// Engine-ready image features produced by a version-compatible frontend.
+// `kwargs` is the MessagePack wire representation of one MmKwargsItem.
+message PreprocessedMediaFeatures {
+  // Required inline payload. Cache-only references are not accepted.
+  optional bytes kwargs = 1;
+  string identifier = 2;
+  uint64 offset = 3;
+  uint64 length = 4;
+  optional string mm_hash = 5;
+  repeated bool is_embed = 6;
+}

rust/src/engine-core-client/src/protocol/output.rs

@@ -73,6 +73,20 @@ pub struct EngineCoreEvent {
     pub timestamp: f64,
 }
 
+/// Compatibility-only engine output value that is decoded without allocation.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
+pub struct DiscardedEngineOutputValue;
+
+impl<'de> Deserialize<'de> for DiscardedEngineOutputValue {
+    fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
+    where
+        D: Deserializer<'de>,
+    {
+        serde::de::IgnoredAny::deserialize(deserializer)?;
+        Ok(Self)
+    }
+}
+
 /// Engine-core output for a single request.
 ///
 /// Original Python definition:
@@ -125,6 +139,12 @@ pub struct EngineCoreOutput {
     pub mm_cache_miss_hashes: Option<Vec<String>>,
     #[serde(default)]
     pub new_sampling_mask: Option<OpaqueValue>,
+    /// Per-request speculative-decoding metrics appended by newer v0.28 wheels.
+    #[serde(default)]
+    pub spec_decode_metrics: Option<DiscardedEngineOutputValue>,
+    /// Routed-expert metadata appended by newer v0.28 wheels.
+    #[serde(default)]
+    pub routed_experts_payload: Option<DiscardedEngineOutputValue>,
 }
 
 impl EngineCoreOutput {
@@ -443,6 +463,8 @@ mod tests {
                             num_nans_in_logits: 0,
                             mm_cache_miss_hashes: None,
                             new_sampling_mask: None,
+                            spec_decode_metrics: None,
+                            routed_experts_payload: None,
                         },
                     ],
                     scheduler_stats: None,

rust/src/engine-core-client/src/tests/client.rs

@@ -2612,6 +2612,12 @@ fn python_msgpack_fixtures_match_rust_encoding() {
                         num_nans_in_logits: 0,
                         mm_cache_miss_hashes: None,
                         new_sampling_mask: None,
+                        spec_decode_metrics: Some(
+                            DiscardedEngineOutputValue,
+                        ),
+                        routed_experts_payload: Some(
+                            DiscardedEngineOutputValue,
+                        ),
                     },
                 ],
                 scheduler_stats: None,

rust/src/engine-core-client/src/tests/python_compat.py

@@ -99,6 +99,8 @@ class EngineCoreOutput(
     num_nans_in_logits: int = 0
     mm_cache_miss_hashes: list[str] | None = None
     new_sampling_mask: object | None = None
+    spec_decode_metrics: object | None = None
+    routed_experts_payload: object | None = None
 
 
 class EngineCoreOutputs(
@@ -198,6 +200,8 @@ class EngineCoreOutputs(
             request_id="req-1",
             new_token_ids=[7, 8],
             finish_reason=FinishReason.LENGTH,
+            spec_decode_metrics={"accepted": 1},
+            routed_experts_payload={"format": "test"},
         )
     ],
     finished_requests={"req-1"},

rust/src/server/Cargo.toml

@@ -21,6 +21,7 @@ llm-multimodal.workspace = true
 openssl.workspace = true
 prost.workspace = true
 prost-types.workspace = true
+rmp-serde.workspace = true
 rmpv.workspace = true
 serde.workspace = true
 serde_json.workspace = true
@@ -60,7 +61,6 @@ async-openai = { workspace = true, features = ["full"] }
 bytes.workspace = true
 clap.workspace = true
 expect-test.workspace = true
-rmp-serde.workspace = true
 serial_test.workspace = true
 tempfile.workspace = true
 tokio = { workspace = true, features = ["test-util"] }

rust/src/server/src/grpc/control.rs

@@ -46,6 +46,7 @@ impl ControlServiceImpl {
             data_parallel_size: self.state.data_parallel_size().min(u32::MAX as usize) as u32,
             data_parallel_rank: ready.data_parallel_rank,
             decode_context_parallel_size: ready.decode_context_parallel_size,
+            world_size: ready.world_size,
         }
     }
 
@@ -151,6 +152,8 @@ impl pb::control_server::Control for ControlServiceImpl {
             max_running_requests: ready.max_num_seqs,
             max_batched_tokens: ready.max_num_batched_tokens,
             rl_capabilities: Some(self.rl_capabilities()),
+            supports_native_sampling_params_json: true,
+            supports_preprocessed_media_features: true,
         }))
     }