dynamo-prime-rl-integration

rust/README.md

@@ -87,3 +87,18 @@ curl http://127.0.0.1:8000/v1/chat/completions \
     "stream": true
   }'
 ```
+
+### Runtime LoRA adapters
+
+Dynamic LoRA loading requires `VLLM_ALLOW_RUNTIME_LORA_UPDATING=1`. When an adapter is loaded from
+the local filesystem, `VLLM_RUNTIME_LORA_ALLOWED_PATH_PREFIXES` is also required. Set it to a
+platform-separated list of directories that may contain adapters; local paths outside those
+directories are rejected. Hugging Face repository IDs do not require a local path prefix.
+
+For example:
+
+```bash
+VLLM_ALLOW_RUNTIME_LORA_UPDATING=1 \
+VLLM_RUNTIME_LORA_ALLOWED_PATH_PREFIXES=/model-cache/loras:/workspace/loras \
+vllm serve Qwen/Qwen3-0.6B --enable-lora
+```

rust/proto/control.proto

@@ -8,6 +8,12 @@ service Control {
   rpc GetServerInfo (GetServerInfoRequest) returns (ServerInfo) {}
   rpc GetModelInfo (GetModelInfoRequest) returns (ModelInfo) {}
   rpc Abort (AbortRequest) returns (AbortResponse) {}
+  rpc Drain (DrainRequest) returns (DrainResponse) {}
+
+  rpc LoadLora (LoadLoraRequest) returns (LoadLoraResponse) {}
+  rpc UnloadLora (UnloadLoraRequest) returns (UnloadLoraResponse) {}
+  rpc ListLoras (ListLorasRequest) returns (ListLorasResponse) {}
+
   rpc GetKvEventSources (GetKvEventSourcesRequest) returns (GetKvEventSourcesResponse) {}
 }
 
@@ -23,6 +29,8 @@ message ServerInfo {
   uint64 total_kv_blocks = 7;
   uint64 max_running_requests = 8;
   uint64 max_batched_tokens = 9;
+  uint32 max_loras = 10;
+  repeated string capabilities = 11;
 }
 
 message ParallelismInfo {
@@ -31,6 +39,8 @@ message ParallelismInfo {
   uint32 data_parallel_size = 3;
   uint32 data_parallel_rank = 4;
   uint32 decode_context_parallel_size = 5;
+  uint32 data_parallel_start_rank = 6;
+  uint32 managed_data_parallel_size = 7;
 }
 
 message GetModelInfoRequest {}
@@ -39,9 +49,11 @@ message ModelInfo {
   string model_id = 1;
   string served_model_name = 2;
   repeated string served_model_aliases = 3;
+  repeated string tokenizer_modes = 4;
 
   bool supports_text_input = 20;
   bool supports_token_ids_input = 21;
+  bool supports_lora = 22;
   bool supports_multimodal = 23;
   string reasoning_parser = 24;
   string tool_call_parser = 25;
@@ -53,16 +65,59 @@ message AbortRequest {
 
 message AbortResponse {}
 
+message DrainRequest {}
+
+message DrainResponse {
+  DrainState state = 1;
+  uint32 in_flight_requests = 2;
+  string message = 3;
+}
+
+enum DrainState {
+  DRAIN_STATE_UNSPECIFIED = 0;
+  DRAIN_STATE_IN_PROGRESS = 1;
+  DRAIN_STATE_COMPLETE = 2;
+}
+
+// ======================================================================================
+// LoRA lifecycle
+// ======================================================================================
+
+message LoraAdapter {
+  int64 lora_id = 1;
+  string lora_name = 2;
+  string source_path = 3;
+}
+
+message LoadLoraRequest {
+  LoraAdapter adapter = 1;
+  bool load_inplace = 2;
+}
+message LoadLoraResponse {
+  LoraAdapter adapter = 1;
+  bool already_loaded = 2;
+}
+message UnloadLoraRequest { string lora_name = 1; }
+message UnloadLoraResponse { LoraAdapter adapter = 1; }
+message ListLorasRequest {}
+message ListLorasResponse { repeated LoraAdapter adapters = 1; }
+
 // ======================================================================================
 // KV discovery
 // ======================================================================================
 
 message GetKvEventSourcesRequest {}
 message GetKvEventSourcesResponse { repeated KvEventSource sources = 1; }
 
+message KvEventEndpoint {
+  string host = 1;
+  uint32 port = 2;
+  string protocol = 3;
+}
+
 message KvEventSource {
   string transport = 1;
-  string endpoint = 2;
+  KvEventEndpoint endpoint_addr = 2;
   string topic = 3;
   string replay_endpoint = 4;
   optional uint32 data_parallel_rank = 5;

rust/proto/inference.proto

@@ -1,6 +1,3 @@
-// SPDX-License-Identifier: Apache-2.0
-// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-
 syntax = "proto3";
 package vllm;
 
@@ -45,21 +42,32 @@ message GenerateRequest {
   uint32 truncate_prompt_tokens = 11;
 
   int32 priority = 12;
+
+  // Multimodal inputs aligned with placeholder markers in token_ids.
+  repeated MediaItem media = 13;
+  // Loaded LoRA name; empty selects the base model.
+  string lora_name = 14;
+  // Lossless JSON object for engine-specific vLLM extensions.
+  optional bytes vllm_xargs_json = 15;
+  // Already-processed multimodal features from the token-in/token-out renderer.
+  repeated PreprocessedMultimodalFeature mm_features = 16;
+  // Skip this many prompt-token routing rows in the returned expert tensor.
+  uint32 routed_experts_prompt_start = 17;
 }
 
 message RandomSampling {
   uint32 num_sequences = 1;  // "n", default (0) means 1
-  uint32 top_k = 2;  // 0 means default
-  float top_p = 3;  // 0 means default
-  float min_p = 4;  // 0 means default
+  optional int32 top_k = 2;
+  optional float top_p = 3;
+  optional float min_p = 4;
   optional int64 seed = 5;
 }
 
 message DecodingParameters {
   // Penalties
-  float presence_penalty = 1;  // Default (0.0) means no penalty
-  float frequency_penalty = 2;  // Default (0.0) means no penalty
-  float repetition_penalty = 3;  // Default (0.0) means no penalty
+  optional float presence_penalty = 1;
+  optional float frequency_penalty = 2;
+  optional float repetition_penalty = 3;
   map<uint32, float> logit_bias = 4;
   repeated uint32 allowed_token_ids = 5;
 
@@ -76,6 +84,10 @@ message DecodingParameters {
     bool json_object = 10;
     string structural_tag = 11;
   }
+  bool structured_output_disable_any_whitespace = 12;
+  bool structured_output_disable_additional_properties = 13;
+  optional string structured_output_whitespace_pattern = 14;
+  repeated string bad_words = 15;
 }
 
 message StoppingCriteria {
@@ -89,6 +101,7 @@ message StoppingCriteria {
   bool include_stop_strings = 5;
 
   bool ignore_eos = 6;
+  optional int64 thinking_token_budget = 7;
 }
 
 message ResponseOptions {
@@ -110,9 +123,6 @@ message KVCacheParameters {
 
   // KV Connector transfer parameters
   google.protobuf.Struct kv_transfer_params = 3;
-
-  // Encoder cache connector transfer parameters
-  google.protobuf.Struct ec_transfer_params = 4;
 }
 
 // Controls which extra candidate tokens at each position should be returned
@@ -147,6 +157,13 @@ message SequenceOutput {
 
   // Only present in final output for this sequence
   optional FinishInfo finish_info = 8;
+  optional RoutedExpertsTensor routed_experts = 9;
+}
+
+message RoutedExpertsTensor {
+  string dtype = 1;
+  repeated uint64 shape = 2;
+  bytes data = 3;
 }
 
 // Prompt info, returned in the first response
@@ -179,7 +196,6 @@ message FinishInfo {
 
   google.protobuf.Struct kv_transfer_params = 6;
   //uint64 seed = 7;
-  google.protobuf.Struct ec_transfer_params = 8;
 }
 
 // Info for candidate tokens other than the input/sampled
@@ -200,3 +216,44 @@ message CandidateTokenInfo {
 message TokenIds {
   repeated uint32 ids = 1;
 }
+
+// ======================================================================================
+// Media
+// ======================================================================================
+
+enum Modality {
+  MODALITY_UNSPECIFIED = 0;
+  MODALITY_IMAGE = 1;
+  MODALITY_VIDEO = 2;
+  MODALITY_AUDIO = 3;
+}
+
+message MediaItem {
+  Modality modality = 1;
+  oneof source {
+    string url = 2;
+    string data_uri = 3;
+    bytes raw_bytes = 4;
+  }
+  string mime_type = 5;
+  string uuid = 6;
+}
+
+message MultimodalPlaceholder {
+  uint64 offset = 1;
+  uint64 length = 2;
+  // Empty means every position in the placeholder is an embedding position.
+  repeated bool is_embed = 3;
+}
+
+message PreprocessedMultimodalFeature {
+  string modality = 1;
+  // Canonical modality+kwargs identity. Kept equal to cache_identifier so
+  // Python vLLM cannot prefer an unverified renderer-provided cache key.
+  string mm_hash = 2;
+  MultimodalPlaceholder position = 3;
+  // Python vLLM's inline msgpack encoding. The current gRPC server requires it.
+  optional bytes kwargs_msgpack = 4;
+  // Canonical cache identity derived from modality and kwargs_msgpack.
+  string cache_identifier = 5;
+}

rust/src/chat/src/lib.rs

@@ -21,6 +21,7 @@ pub use event::{
     AssistantToolCall, ChatEvent,
 };
 use futures::{StreamExt, TryStreamExt as _};
+pub use llm_multimodal::MediaContentPart;
 pub use output::{
     ChatOutputProcessor, DefaultChatOutputProcessor, DynChatOutputProcessor,
     HarmonyChatOutputProcessor,
@@ -41,6 +42,7 @@ pub use request::{
     ChatToolChoice, GenerationPromptMode, ReasoningEffort, SamplingParams,
 };
 pub use stream::{ChatEventStream, ChatEventStreamTrait, CollectedAssistantMessage};
+pub use vllm_engine_core_client::protocol::multimodal::MmFeatures;
 pub use vllm_llm::FinishReason;
 
 mod backend;
@@ -55,7 +57,6 @@ mod stream;
 
 use vllm_engine_core_client::EngineCoreClient;
 use vllm_engine_core_client::protocol::dtype::ModelDtype;
-use vllm_engine_core_client::protocol::multimodal::MmFeatures;
 use vllm_engine_core_client::protocol::request::ReasoningParserKwargs;
 use vllm_llm::Llm;
 use vllm_text::{Prompt, TextLlm, TextRequest};
@@ -283,6 +284,25 @@ impl ChatLlm {
         }
     }
 
+    /// Prepare media for an already-tokenized request.
+    pub async fn prepare_media(
+        &self,
+        media: Vec<MediaContentPart>,
+        token_ids: &mut Vec<u32>,
+    ) -> Result<Option<MmFeatures>> {
+        if media.is_empty() {
+            return Ok(None);
+        }
+        let info = self
+            .processor
+            .backend
+            .multimodal_model_info()
+            .ok_or(Error::UnsupportedMultimodalRenderer)?;
+        let model_dtype = self.processor.model_dtype.ok_or(Error::UnsupportedMultimodalRenderer)?;
+        let features = info.prepare_multimodal(media, token_ids, model_dtype).await?;
+        Ok(Some(features))
+    }
+
     /// Render, tokenize, and submit one chat request.
     pub async fn chat(&self, request: ChatRequest) -> Result<ChatEventStream> {
         let (text_request, output_processor) = self

rust/src/chat/src/multimodal.rs

@@ -696,7 +696,7 @@ impl MultimodalModelInfo {
     /// `prompt_token_ids` is mutated in place because placeholder expansion
     /// changes both the final prompt and the offsets recorded in
     /// `PlaceholderRange`.
-    async fn prepare_multimodal(
+    pub(crate) async fn prepare_multimodal(
         &self,
         media_parts: Vec<MediaContentPart>,
         prompt_token_ids: &mut Vec<u32>,

rust/src/chat/src/output/default/unified.rs

@@ -392,6 +392,7 @@ mod tests {
                 finish_reason: crate::FinishReason::Stop(None),
                 kv_transfer_params: None,
                 ec_transfer_params: None,
+                routed_experts: None,
             }),
         }
     }

rust/src/chat/src/output/harmony/tests.rs

@@ -56,6 +56,7 @@ fn finished() -> Finished {
         finish_reason: FinishReason::stop_eos(),
         kv_transfer_params: None,
         ec_transfer_params: None,
+        routed_experts: None,
     }
 }
 

rust/src/chat/tests/roundtrip.rs

@@ -722,6 +722,7 @@ fn decoded_completion_stream(
                     finish_reason: FinishReason::stop_eos(),
                     kv_transfer_params: None,
                     ec_transfer_params: None,
+                    routed_experts: None,
                 }),
             }
         });
@@ -733,6 +734,7 @@ fn decoded_completion_stream(
                 finish_reason: FinishReason::stop_eos(),
                 kv_transfer_params: None,
                 ec_transfer_params: None,
+                routed_experts: None,
             });
             events.push(DecodedTextEvent::TextDelta {
                 delta: chunk.delta,

rust/src/cmd/src/cli/tests.rs

@@ -156,6 +156,35 @@ fn serve_args_auto_forward_enable_lora_to_python() {
     .assert_debug_eq(&args.managed_engine.python_args);
 }
 
+#[test]
+fn serve_args_forward_lora_capacity_to_python() {
+    let cli = Cli::try_parse_from([
+        "vllm-rs",
+        "serve",
+        "Qwen/Qwen3-0.6B",
+        "--enable-lora",
+        "--max-loras",
+        "4",
+        "--max-lora-rank",
+        "64",
+    ])
+    .unwrap();
+
+    let Command::Serve(args) = cli.command else {
+        panic!("expected serve args");
+    };
+    expect![[r#"
+        [
+            "--enable-lora",
+            "--max-loras",
+            "4",
+            "--max-lora-rank",
+            "64",
+        ]
+    "#]]
+    .assert_debug_eq(&args.managed_engine.python_args);
+}
+
 #[test]
 fn serve_args_forward_shutdown_timeout_to_managed_engine() {
     let cli = Cli::try_parse_from([