Skip to content
Merged
Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions rust/src/chat/src/error.rs
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,8 @@ pub enum Error {
ToolCallStreamInvariant { message: String },
#[error(transparent)]
Text(#[from] vllm_text::Error),
#[error(transparent)]
Tokenizer(#[from] vllm_tokenizer::TokenizerError),
}

pub type Result<T> = std::result::Result<T, Error>;
Expand Down
27 changes: 27 additions & 0 deletions rust/src/chat/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -198,6 +198,33 @@ impl ChatLlm {
Ok(ChatEventStream::new(request.request_id, structured_stream))
}

/// Render through the chat template and tokenize, without submitting to the engine.
///
/// Same render → [`multimodal::finalize_rendered_prompt`] → encode pipeline as
/// [`Self::chat`], but stops after token IDs so `/tokenize` counts match what
/// generation would see. Used by `POST /tokenize` (chat form).
pub async fn tokenize_chat(&self, request: ChatRequest) -> Result<Vec<u32>> {
request.validate()?;

let rendered = self.backend.chat_renderer().render(&request)?;
let (prompt, _mm_features) = multimodal::finalize_rendered_prompt(
Comment thread
TanNgocDo marked this conversation as resolved.
Resolved
&request,
rendered,
self.backend.multimodal_model_info(),
self.model_dtype,
)
.await?;

let tokenizer = self.text.tokenizer();
let token_ids = match prompt {
// Rendered string from the template (usual chat path).
vllm_text::Prompt::Text(text) => tokenizer.encode(&text, request.add_special_tokens)?,
Comment thread
TanNgocDo marked this conversation as resolved.
Resolved
// Already tokenized (e.g. multimodal path); pass through unchanged.
vllm_text::Prompt::TokenIds(ids) => ids,
};
Ok(token_ids)
}

/// Shut down the underlying LLM client and its background tasks.
pub async fn shutdown(self) -> Result<()> {
self.text.shutdown().await?;
Expand Down
2 changes: 2 additions & 0 deletions rust/src/server/src/routes.rs
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,8 @@ fn build_router_with_options(
.route("/v1/models", get(openai::list_models))
.route("/v1/completions", post(openai::completions))
.route("/v1/chat/completions", post(openai::chat_completions))
.route("/tokenize", post(openai::tokenize))
.route("/detokenize", post(openai::detokenize))
Comment thread
TanNgocDo marked this conversation as resolved.
Resolved
// vLLM specific inference endpoints
.route("/inference/v1/generate", post(inference::generate));

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -152,8 +152,7 @@ pub(crate) fn prepare_chat_request(
return_tokens_as_token_ids: request.return_tokens_as_token_ids.unwrap_or(false),
})
}

fn normalize_generation_prompt_mode(
pub(crate) fn normalize_generation_prompt_mode(
add_generation_prompt: Option<bool>,
continue_final_message: bool,
messages: &[VllmChatMessage],
Expand Down Expand Up @@ -200,7 +199,7 @@ fn extract_last_assistant_content(messages: &[ChatMessage]) -> Option<String> {
}

/// Lower one OpenAI chat message into the `vllm-chat` message shape.
fn convert_message(message: ChatMessage) -> Result<VllmChatMessage, ApiError> {
pub(crate) fn convert_message(message: ChatMessage) -> Result<VllmChatMessage, ApiError> {
match message {
ChatMessage::System { content, .. } => {
Ok(VllmChatMessage::system(convert_content(content)?))
Expand Down
32 changes: 3 additions & 29 deletions rust/src/server/src/routes/openai/chat_completions/types.rs
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,9 @@ use vllm_chat::ReasoningEffort;

use crate::routes::openai::utils::structured_outputs::ResponseFormat;
use crate::routes::openai::utils::types::{
ChatLogProbs, ChatMessage, MessageContent, Normalizable, StreamOptions, StringOrArray, Tool,
ToolCall, ToolCallDelta, ToolChoice, ToolChoiceValue, ToolReference, UNKNOWN_MODEL_ID, Usage,
default_true, validate_stop, validate_top_p_value,
ChatLogProbs, ChatMessage, Normalizable, StreamOptions, StringOrArray, Tool, ToolCall,
ToolCallDelta, ToolChoice, ToolChoiceValue, ToolReference, UNKNOWN_MODEL_ID, Usage,
default_true, validate_messages, validate_stop, validate_top_p_value,
};

/// vLLM-compatible request type for the Chat Completions API.
Expand Down Expand Up @@ -430,32 +430,6 @@ fn default_model() -> String {
UNKNOWN_MODEL_ID.to_string()
}

/// Validates messages array is not empty and has valid content
fn validate_messages(messages: &[ChatMessage]) -> Result<(), validator::ValidationError> {
if messages.is_empty() {
return Err(validator::ValidationError::new("messages cannot be empty"));
}

for msg in messages {
if let ChatMessage::User { content, .. } = msg {
match content {
MessageContent::Text(text) if text.is_empty() => {
return Err(validator::ValidationError::new(
"message content cannot be empty",
));
}
MessageContent::Parts(parts) if parts.is_empty() => {
return Err(validator::ValidationError::new(
"message content parts cannot be empty",
));
}
_ => {}
}
}
}
Ok(())
}

/// Schema-level validation for cross-field dependencies
fn validate_chat_cross_parameters(
req: &ChatCompletionRequest,
Expand Down
2 changes: 2 additions & 0 deletions rust/src/server/src/routes/openai/mod.rs
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
pub mod chat_completions;
mod completions;
mod models;
mod tokenize;
pub(crate) mod utils;

pub use chat_completions::chat_completions;
pub use completions::completions;
pub use models::list_models;
pub use tokenize::{detokenize, tokenize};
157 changes: 157 additions & 0 deletions rust/src/server/src/routes/openai/tokenize.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,157 @@
//! `POST /tokenize` and `POST /detokenize` (root paths, matching Python).
//!
//! Encode/decode runs entirely in-process via [`DynTokenizer`]; the inference
//! engine is not involved.

mod types;

use std::sync::Arc;

use axum::Json;
use axum::extract::State;
use axum::http::HeaderMap;
use axum::response::{IntoResponse, Response};
use thiserror_ext::AsReport as _;
use tracing::debug;

use crate::error::{ApiError, server_error};
use crate::routes::openai::tokenize::types::{
DetokenizeRequest, DetokenizeResponse, TokenizeChatRequest, TokenizeCompletionRequest,
TokenizeRequest, TokenizeResponse,
};
use crate::routes::openai::utils::validated_json::ValidatedJson;
use crate::state::AppState;
use crate::utils::resolve_base_request_id;

/// Match Python `tokenize-{base}` where base is `X-Request-Id` or a new UUID.
fn tokenize_request_id(headers: &HeaderMap) -> String {
let base = resolve_base_request_id(
headers.get("X-Request-Id").and_then(|value| value.to_str().ok()),
None,
);
format!("tokenize-{base}")
}

/// Reject an unknown model name, matching the other handlers.
fn check_model(state: &AppState, model: Option<&str>) -> Result<(), ApiError> {
if let Some(model) = model
&& !state.served_model_names().iter().any(|n| n == model)
{
return Err(ApiError::model_not_found(model.to_string()));
}
Ok(())
}

/// Build the `token_strs` vector when requested, via the tokenizer vocab.
fn token_strs(tokenizer: &vllm_text::tokenizer::DynTokenizer, ids: &[u32]) -> Vec<String> {
// Unknown IDs yield "" — intentional; matches Python's convert_ids_to_tokens behaviour.
ids.iter().map(|&id| tokenizer.id_to_token(id).unwrap_or_default()).collect()
}

pub async fn tokenize(
State(state): State<Arc<AppState>>,
headers: HeaderMap,
ValidatedJson(body): ValidatedJson<TokenizeRequest>,
) -> Response {
let request_id = tokenize_request_id(&headers);
let tokenizer = state.chat.text().tokenizer();
let max_model_len = state.chat.engine_core_client().max_model_len();

let result = match body {
// Completion form: encode the raw `prompt` string (no chat template).
TokenizeRequest::Completion(req) => {
tokenize_completion(&state, &tokenizer, &request_id, req)
}
// Chat form: render `messages` through the template, then encode (see `tokenize_chat`).
TokenizeRequest::Chat(req) => tokenize_chat(&state, &request_id, req).await,
};

match result {
Ok((tokens, want_strs)) => {
let token_strs = want_strs.then(|| token_strs(&tokenizer, &tokens));
Json(TokenizeResponse {
count: tokens.len(),
max_model_len,
tokens,
token_strs,
})
.into_response()
}
Err(error) => error.into_response(),
}
}

fn tokenize_completion(
state: &AppState,
tokenizer: &vllm_text::tokenizer::DynTokenizer,
request_id: &str,
req: TokenizeCompletionRequest,
) -> Result<(Vec<u32>, bool), ApiError> {
debug!(%request_id, "tokenize completion");
Comment thread
TanNgocDo marked this conversation as resolved.
Resolved
check_model(state, req.model.as_deref())?;
let tokens = tokenizer
.encode(&req.prompt, req.add_special_tokens)
.map_err(|e| server_error!("tokenize failed: {}", e.to_report_string()))?;
Ok((tokens, req.return_token_strs))
}

/// HTTP adapter for the chat-shaped `/tokenize` body.
///
/// Not [`vllm_chat::ChatLlm::tokenize_chat`]: this checks the model name and maps
/// errors to [`ApiError`]; the chat-crate method does render → finalize → encode.
async fn tokenize_chat(
state: &AppState,
request_id: &str,
req: TokenizeChatRequest,
) -> Result<(Vec<u32>, bool), ApiError> {
debug!(%request_id, "tokenize chat");
check_model(state, req.model.as_deref())?;
let return_token_strs = req.return_token_strs;
// `continue_final_message` / `add_generation_prompt` mutual exclusion is
// enforced in `normalize_generation_prompt_mode` inside `into_chat_request`.
let tokens = state
.chat
.tokenize_chat(req.into_chat_request(request_id.to_string())?)
.await
.map_err(|e| server_error!("tokenize failed: {}", e.to_report_string()))?;
Ok((tokens, return_token_strs))
}

pub async fn detokenize(
State(state): State<Arc<AppState>>,
headers: HeaderMap,
ValidatedJson(body): ValidatedJson<DetokenizeRequest>,
) -> Response {
let request_id = tokenize_request_id(&headers);
debug!(%request_id, "detokenize");
if let Err(error) = check_model(&state, body.model.as_deref()) {
return error.into_response();
}
let tokenizer = state.chat.text().tokenizer();
match tokenizer.decode(&body.tokens, /* skip_special_tokens = */ false) {
Ok(prompt) => Json(DetokenizeResponse { prompt }).into_response(),
Err(e) => server_error!("detokenize failed: {}", e.to_report_string()).into_response(),
}
}

#[cfg(test)]
mod tests {
use axum::http::{HeaderMap, HeaderValue};

use super::tokenize_request_id;

#[test]
fn tokenize_request_id_prefers_x_request_id_header() {
let mut headers = HeaderMap::new();
headers.insert("X-Request-Id", HeaderValue::from_static("client-req-1"));
assert_eq!(tokenize_request_id(&headers), "tokenize-client-req-1");
}

#[test]
fn tokenize_request_id_generates_uuid_when_header_missing() {
let headers = HeaderMap::new();
let id = tokenize_request_id(&headers);
assert!(id.starts_with("tokenize-"));
assert_ne!(id, "tokenize-");
}
}
Loading
Loading