Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 0 additions & 4 deletions rust/src/engine-core-client/src/protocol/lora.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,6 @@

use serde_tuple::{Deserialize_tuple, Serialize_tuple};

use crate::protocol::OpaqueValue;

/// Request for a LoRA adapter.
///
/// Mirrors Python `vllm.lora.request.LoRARequest`, which is a msgspec
Expand All @@ -17,8 +15,6 @@ pub struct LoraRequest {
#[serde(default)]
pub base_model_name: Option<String>,
#[serde(default)]
pub tensorizer_config_dict: Option<OpaqueValue>,
#[serde(default)]
pub load_inplace: bool,
#[serde(default)]
pub is_3d_lora_weight: bool,
Expand Down
27 changes: 27 additions & 0 deletions rust/src/engine-core-client/src/tests/client.rs
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ use zeromq::{DealerSocket, PushSocket, SocketOptions, SubSocket, XPubSocket, Zmq
use crate::protocol::handshake::{EngineCoreReadyResponse, HandshakeInitMessage, ReadyMessage};
use crate::protocol::kv_hints::{KvHintAction, KvHintsEnvelope};
use crate::protocol::logprobs::MaybeWireLogprobs;
use crate::protocol::lora::LoraRequest;
use crate::protocol::multimodal::{
MmFeatureSpec, MmField, MmFieldElem, MmFlatField, MmKwargValue, MmModality, MmSlice,
PlaceholderRange, SliceSpec,
Expand Down Expand Up @@ -2778,6 +2779,7 @@ fn python_msgpack_fixtures_match_rust_encoding() {
lines.next().expect("missing MultiConnector stats fixture line");
let ready_response_hex = lines.next().expect("missing ready response fixture line");
let extended_outputs_hex = lines.next().expect("missing extended outputs fixture line");
let lora_request_hex = lines.next().expect("missing LoRA request fixture line");

let request_bytes = hex::decode(request_hex).unwrap();
let multimodal_request_bytes = hex::decode(multimodal_request_hex).unwrap();
Expand Down Expand Up @@ -2851,6 +2853,31 @@ fn python_msgpack_fixtures_match_rust_encoding() {
decode_value(&rmp_serde::to_vec_named(&expected_multimodal_request.mm_features).unwrap());
assert_eq!(python_mm_features, rust_mm_features);

// Pin the full positional LoRARequest layout against Python in both
// directions: Python wire -> Rust struct, and Rust struct -> wire for the
// `lora_request` slot of the request.
let lora_request_bytes = hex::decode(lora_request_hex).unwrap();
let expected_lora_request = LoraRequest {
lora_name: "alice".to_string(),
lora_int_id: 3,
lora_path: "/adapters/alice".to_string(),
base_model_name: Some("base".to_string()),
load_inplace: true,
is_3d_lora_weight: true,
};
let decoded_lora_request: EngineCoreRequest =
rmp_serde::from_slice(&lora_request_bytes).unwrap();
assert_eq!(
decoded_lora_request.lora_request,
Some(expected_lora_request.clone())
);
let python_lora_slot = match decode_value(&lora_request_bytes) {
Value::Array(items) => items.get(6).cloned().expect("request lora_request slot"),
other => panic!("request should encode as tuple array, got {other:?}"),
};
let rust_lora_slot = decode_value(&rmp_serde::to_vec_named(&expected_lora_request).unwrap());
assert_eq!(python_lora_slot, rust_lora_slot);

let decoded_sampling_mask_outputs =
decode_engine_core_outputs(&[bytes::Bytes::from(sampling_mask_outputs_bytes)]).unwrap();
let sampling_mask_output =
Expand Down
36 changes: 35 additions & 1 deletion rust/src/engine-core-client/src/tests/python_compat.py
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,20 @@ class KvHintsEnvelope(msgspec.Struct, frozen=True):
actions: list[KvHintAction]


# Mirror of `vllm.lora.request.LoRARequest`.
class LoRARequest(
msgspec.Struct,
array_like=True,
omit_defaults=True,
):
lora_name: str
lora_int_id: int
lora_path: str = ""
base_model_name: str | None = None
load_inplace: bool = False
is_3d_lora_weight: bool = False


class EngineCoreRequest(
msgspec.Struct,
array_like=True,
Expand All @@ -76,7 +90,7 @@ class EngineCoreRequest(
sampling_params: EngineCoreSamplingParams | None
pooling_params: object | None
arrival_time: float
lora_request: object | None = None
lora_request: LoRARequest | None = None
cache_salt: str | None = None
data_parallel_rank: int | None = None
prompt_embeds: object | None = None
Expand Down Expand Up @@ -201,6 +215,25 @@ class EngineCoreOutputs(
arrival_time=1.0,
)

# Every LoRARequest field is non-default, so omit_defaults trims nothing and the
# fixture pins the full positional layout.
lora_request = EngineCoreRequest(
request_id="req-lora",
prompt_token_ids=[5, 6, 7],
mm_features=None,
sampling_params=EngineCoreSamplingParams(),
pooling_params=None,
arrival_time=2.0,
lora_request=LoRARequest(
lora_name="alice",
lora_int_id=3,
lora_path="/adapters/alice",
base_model_name="base",
load_inplace=True,
is_3d_lora_weight=True,
),
)

multimodal_tensor = np.array([[1.0, 2.0], [3.5, 4.25]], dtype=np.float32)
multimodal_features = [
{
Expand Down Expand Up @@ -573,3 +606,4 @@ class EngineCoreReadyResponse:
print(msgspec.msgpack.encode(multi_connector_stats).hex())
print(msgspec.msgpack.encode(ready_response).hex())
print(extended_outputs_bytes.hex())
print(msgspec.msgpack.encode(lora_request).hex())
1 change: 0 additions & 1 deletion rust/src/llm/tests/generate.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1133,7 +1133,6 @@ async fn generate_without_engine_stats_exports_only_frontend_observed_metrics()
lora_int_id: 1,
lora_path: "/unused".to_string(),
base_model_name: None,
tensorizer_config_dict: None,
load_inplace: false,
is_3d_lora_weight: false,
});
Expand Down
1 change: 0 additions & 1 deletion rust/src/server/src/lora.rs
Original file line number Diff line number Diff line change
Expand Up @@ -248,7 +248,6 @@ impl LoraManager {
lora_int_id,
lora_path,
base_model_name,
tensorizer_config_dict: None,
load_inplace,
is_3d_lora_weight,
};
Expand Down