Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions rust/src/engine-core-client/src/mock_engine.rs
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@ pub fn default_ready_response() -> EngineCoreReadyResponse {
enable_sleep_mode: false,
supports_draft_weight_updates: false,
effective_attention_block_size: None,
use_spec_decode: false,
}
}

Expand Down
3 changes: 3 additions & 0 deletions rust/src/engine-core-client/src/protocol/handshake.rs
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,9 @@ pub struct EngineCoreReadyResponse {
/// Full-attention block size in tokens after initialization, or unavailable.
#[serde(default)]
pub effective_attention_block_size: Option<u64>,
/// Whether the engine was started with speculative decoding enabled.
#[serde(default)]
pub use_spec_decode: bool,
}

/// Frontend-owned ZMQ addresses that are sent to the engine during startup
Expand Down
1 change: 1 addition & 0 deletions rust/src/engine-core-client/src/tests/client.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2968,6 +2968,7 @@ fn python_msgpack_fixtures_match_rust_encoding() {
assert!(ready_response.enable_sleep_mode);
assert!(ready_response.supports_draft_weight_updates);
assert_eq!(ready_response.effective_attention_block_size, Some(64));
assert!(ready_response.use_spec_decode);
let kv_events_config = ready_response.kv_events_config.expect("KV events config should decode");
assert!(kv_events_config.enable_kv_cache_events);
assert_eq!(kv_events_config.publisher, "zmq");
Expand Down
2 changes: 2 additions & 0 deletions rust/src/engine-core-client/src/tests/python_compat.py
Original file line number Diff line number Diff line change
Expand Up @@ -470,6 +470,7 @@ class EngineCoreReadyResponse:
enable_sleep_mode: bool = False
supports_draft_weight_updates: bool = False
effective_attention_block_size: int | None = None
use_spec_decode: bool = False


ready_response = EngineCoreReadyResponse(
Expand All @@ -494,6 +495,7 @@ class EngineCoreReadyResponse:
enable_sleep_mode=True,
supports_draft_weight_updates=True,
effective_attention_block_size=64,
use_spec_decode=True,
kv_events_config=KVEventsConfig(
enable_kv_cache_events=True,
publisher="zmq",
Expand Down
3 changes: 2 additions & 1 deletion rust/src/server/src/routes.rs
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,8 @@ fn build_router_with_options(
.route("/v1/chat/completions", post(openai::chat_completions))
// vLLM specific endpoints
.route("/tokenize", post(tokenize::tokenize))
.route("/detokenize", post(tokenize::detokenize));
.route("/detokenize", post(tokenize::detokenize))
.route("/generative_scoring", post(inference::generative_scoring));

if scale_out_endpoints_enabled {
router = router.route("/inference/v1/generate", post(inference::generate));
Expand Down
Loading
Loading