diff --git a/rust/src/chat/src/parser/mod.rs b/rust/src/chat/src/parser/mod.rs index 52e83b3e0471..244a87cc7a76 100644 --- a/rust/src/chat/src/parser/mod.rs +++ b/rust/src/chat/src/parser/mod.rs @@ -6,10 +6,10 @@ use std::convert::Infallible; use std::fmt; use std::str::FromStr; -use serde_with::DeserializeFromStr; +use serde_with::{DeserializeFromStr, SerializeDisplay}; /// Specify which reasoning or tool-call parser implementation to use. -#[derive(Debug, Clone, PartialEq, Eq, Default, DeserializeFromStr)] +#[derive(Debug, Clone, PartialEq, Eq, Default, DeserializeFromStr, SerializeDisplay)] pub enum ParserSelection { /// Use model-based auto-detection. #[default] diff --git a/rust/src/chat/src/renderer/hf/format.rs b/rust/src/chat/src/renderer/hf/format.rs index a9b35d0f41c8..2c990fb37ba3 100644 --- a/rust/src/chat/src/renderer/hf/format.rs +++ b/rust/src/chat/src/renderer/hf/format.rs @@ -5,7 +5,7 @@ use std::str::FromStr; use minijinja::machinery::ast::{Expr, ForLoop, Set, Stmt}; use minijinja::machinery::{WhitespaceConfig, parse}; use minijinja::syntax::SyntaxConfig; -use serde_with::DeserializeFromStr; +use serde_with::{DeserializeFromStr, SerializeDisplay}; /// Chat template content format. #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] @@ -18,7 +18,7 @@ pub enum ChatTemplateContentFormat { } /// Configurable chat-template content format selection. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, DeserializeFromStr)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, DeserializeFromStr, SerializeDisplay)] pub enum ChatTemplateContentFormatOption { /// Detect the format from the template source. #[default] diff --git a/rust/src/chat/src/renderer/selection.rs b/rust/src/chat/src/renderer/selection.rs index f4bd565bafdd..cb22f95de0da 100644 --- a/rust/src/chat/src/renderer/selection.rs +++ b/rust/src/chat/src/renderer/selection.rs @@ -1,10 +1,10 @@ use std::fmt; use std::str::FromStr; -use serde_with::DeserializeFromStr; +use serde_with::{DeserializeFromStr, SerializeDisplay}; /// Specify which chat renderer implementation to use. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, DeserializeFromStr)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, DeserializeFromStr, SerializeDisplay)] pub enum RendererSelection { /// Use model-based auto-detection. #[default] diff --git a/rust/src/engine-core-client/src/client.rs b/rust/src/engine-core-client/src/client.rs index 1e6968555e7f..2a8c3c741884 100644 --- a/rust/src/engine-core-client/src/client.rs +++ b/rust/src/engine-core-client/src/client.rs @@ -2,6 +2,7 @@ use std::sync::Arc; use std::time::Duration; use futures::future::{join_all, try_join_all}; +use serde::Serialize; use tokio::sync::mpsc; use tokio_util::task::AbortOnDropHandle; use tracing::{debug, info, trace}; @@ -23,7 +24,7 @@ pub use stream::{EngineCoreOutputStream, EngineCoreStreamOutput}; /// How the frontend acquires its request/response transport with Python /// `EngineCoreProc`s. -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] pub enum TransportMode { /// The Rust process owns the startup handshake and allocates or binds the /// frontend transport addresses itself before replying to engine diff --git a/rust/src/server/src/config.rs b/rust/src/server/src/config.rs index 6853d9ccab99..f1599d18793f 100644 --- a/rust/src/server/src/config.rs +++ b/rust/src/server/src/config.rs @@ -2,12 +2,13 @@ use std::collections::HashMap; use std::time::Duration; use anyhow::Result; +use serde::Serialize; use serde_json::Value; use vllm_chat::{ChatTemplateContentFormatOption, ParserSelection, RendererSelection}; use vllm_engine_core_client::{CoordinatorMode as EngineCoreCoordinatorMode, TransportMode}; /// How the HTTP server obtains its listening socket. -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] pub enum HttpListenerMode { /// Bind a fresh TCP listener on the given host/port. BindTcp { host: String, port: u16 }, @@ -20,7 +21,7 @@ pub enum HttpListenerMode { /// Which coordinator implementation should be active when one is present for a /// frontend client. -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] pub enum CoordinatorMode { /// Do not run a coordinator at all. None, @@ -32,7 +33,7 @@ pub enum CoordinatorMode { } /// Normalized runtime configuration for the minimal OpenAI-compatible server. -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] pub struct Config { /// Frontend-to-engine transport setup. pub transport_mode: TransportMode, diff --git a/rust/src/server/src/lib.rs b/rust/src/server/src/lib.rs index adfcc60229ee..8d779da132f0 100644 --- a/rust/src/server/src/lib.rs +++ b/rust/src/server/src/lib.rs @@ -7,6 +7,7 @@ mod listener; mod lora; mod middleware; mod routes; +mod server_info; mod state; mod utils; @@ -30,6 +31,7 @@ use vllm_text::TextLlm; use crate::listener::Listener; use crate::routes::build_router; +use crate::server_info::ServerInfoSnapshot; use crate::state::AppState; /// Build the shared application state for one configured model and one engine @@ -88,7 +90,8 @@ async fn build_state(config: &Config) -> Result> { Ok(Arc::new( AppState::new(served_model_names, chat) .with_log_requests(config.enable_log_requests) - .with_request_id_headers(config.enable_request_id_headers), + .with_request_id_headers(config.enable_request_id_headers) + .with_server_info(ServerInfoSnapshot::from_config(config)), )) } diff --git a/rust/src/server/src/routes.rs b/rust/src/server/src/routes.rs index b708b43e66e8..a0473c783a03 100644 --- a/rust/src/server/src/routes.rs +++ b/rust/src/server/src/routes.rs @@ -6,6 +6,7 @@ mod load; mod lora; mod metrics; pub(crate) mod openai; +mod server_info; mod sleep; mod version; @@ -89,6 +90,7 @@ fn build_router_with_options( .route("/sleep", post(sleep::sleep)) .route("/wake_up", post(sleep::wake_up)) .route("/is_sleeping", get(sleep::is_sleeping)) + .route("/server_info", get(server_info::server_info)) } let enable_request_id_headers = state.enable_request_id_headers; diff --git a/rust/src/server/src/routes/server_info.rs b/rust/src/server/src/routes/server_info.rs new file mode 100644 index 000000000000..aefb17a25faf --- /dev/null +++ b/rust/src/server/src/routes/server_info.rs @@ -0,0 +1,47 @@ +use std::sync::Arc; + +use axum::Json; +use axum::extract::{Query, State}; +use axum::http::StatusCode; +use axum::response::{IntoResponse, Response}; +use serde::Deserialize; + +use crate::server_info::ServerInfoConfigFormat; +use crate::state::AppState; + +#[derive(Debug, Clone, Copy, Deserialize)] +#[serde(rename_all = "lowercase")] +enum ConfigFormat { + Text, + Json, +} + +impl From for ServerInfoConfigFormat { + fn from(value: ConfigFormat) -> Self { + match value { + ConfigFormat::Text => Self::Text, + ConfigFormat::Json => Self::Json, + } + } +} + +fn default_config_format() -> ConfigFormat { + ConfigFormat::Text +} + +#[derive(Debug, Deserialize)] +pub(crate) struct ServerInfoParams { + #[serde(default = "default_config_format")] + config_format: ConfigFormat, +} + +/// Get server configuration and environment metadata. +pub async fn server_info( + State(state): State>, + Query(params): Query, +) -> Response { + match state.server_info_response(params.config_format.into()) { + Some(response) => Json(response).into_response(), + None => StatusCode::NOT_FOUND.into_response(), + } +} diff --git a/rust/src/server/src/routes/tests.rs b/rust/src/server/src/routes/tests.rs index 6a33bc7755d2..a3e437e04802 100644 --- a/rust/src/server/src/routes/tests.rs +++ b/rust/src/server/src/routes/tests.rs @@ -742,16 +742,23 @@ async fn test_chat_with_engine_outputs( } async fn test_app() -> axum::Router { + test_app_with_dev_mode(false).await +} + +async fn test_app_with_dev_mode(dev_mode_enabled: bool) -> axum::Router { let (chat, _engine_task) = test_models_with_engine_outputs_and_backend( b"engine-openai", default_stream_output_specs(), Arc::new(FakeChatBackend::new()), ) .await; - build_router(Arc::new(AppState::new( - vec!["Qwen/Qwen1.5-0.5B-Chat".to_string()], - chat, - ))) + build_router_with_dev_mode( + Arc::new(AppState::new( + vec!["Qwen/Qwen1.5-0.5B-Chat".to_string()], + chat, + )), + dev_mode_enabled, + ) } async fn test_app_with_request_id_headers() -> (axum::Router, MockEngineTask) { @@ -1087,6 +1094,23 @@ async fn version_returns_engine_vllm_version() { ); } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial] +async fn server_info_endpoint_is_dev_mode_only() { + let mut app = test_app().await; + let response = app + .call( + Request::builder() + .uri("/server_info") + .body(Body::empty()) + .expect("build request"), + ) + .await + .expect("call app"); + + assert_eq!(response.status(), StatusCode::NOT_FOUND); +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[serial] async fn load_lora_adapter_registers_model_and_forwards_lora_request() { @@ -1302,6 +1326,23 @@ async fn load_lora_adapter_registers_model_and_forwards_lora_request() { engine_task.finish().await; } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial] +async fn server_info_endpoint_returns_not_found_without_snapshot() { + let mut app = test_app_with_dev_mode(true).await; + let response = app + .call( + Request::builder() + .uri("/server_info") + .body(Body::empty()) + .expect("build request"), + ) + .await + .expect("call app"); + + assert_eq!(response.status(), StatusCode::NOT_FOUND); +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[serial] async fn unload_lora_adapter_rejects_mismatched_lora_int_id() { diff --git a/rust/src/server/src/server_info.rs b/rust/src/server/src/server_info.rs new file mode 100644 index 000000000000..cca1b0f37954 --- /dev/null +++ b/rust/src/server/src/server_info.rs @@ -0,0 +1,144 @@ +use std::collections::BTreeMap; + +use serde_json::{Value, json}; + +use crate::config::Config; + +const SENSITIVE_VLLM_ENV_PATTERNS: &[&str] = + &["KEY", "SECRET", "TOKEN", "PASSWORD", "CREDENTIAL", "AUTH"]; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum ServerInfoConfigFormat { + Text, + Json, +} + +/// Snapshot returned by `/server_info`. +#[derive(Debug, Clone)] +pub(crate) struct ServerInfoSnapshot { + vllm_config_text: String, + vllm_config_json: Value, + vllm_env: BTreeMap, + system_env: BTreeMap, +} + +impl ServerInfoSnapshot { + /// Capture the runtime configuration fields available to the Rust frontend. + pub(crate) fn from_config(config: &Config) -> Self { + let vllm_config_json = + serde_json::to_value(config).expect("server info value must serialize"); + + Self { + vllm_config_text: render_config_text(&vllm_config_json), + vllm_config_json, + vllm_env: collect_vllm_env(), + system_env: collect_system_env(), + } + } + + pub(crate) fn response(&self, config_format: ServerInfoConfigFormat) -> Value { + let vllm_config = match config_format { + ServerInfoConfigFormat::Text => Value::String(self.vllm_config_text.clone()), + ServerInfoConfigFormat::Json => self.vllm_config_json.clone(), + }; + + json!({ + "vllm_config": vllm_config, + "vllm_env": self.vllm_env.clone(), + "system_env": self.system_env.clone(), + }) + } +} + +fn render_config_text(config: &Value) -> String { + match config { + Value::Object(fields) => fields + .iter() + .map(|(key, value)| format!("{key}={}", render_config_text_value(value))) + .collect::>() + .join("\n"), + _ => render_config_text_value(config), + } +} + +fn render_config_text_value(value: &Value) -> String { + match value { + Value::Null => "None".to_string(), + Value::String(value) => value.clone(), + _ => value.to_string(), + } +} + +fn collect_vllm_env() -> BTreeMap { + std::env::vars().filter(|(key, _)| is_public_vllm_env_key(key)).collect() +} + +fn is_public_vllm_env_key(key: &str) -> bool { + let key = key.to_ascii_uppercase(); + key.starts_with("VLLM_") + && !SENSITIVE_VLLM_ENV_PATTERNS.iter().any(|pattern| key.contains(pattern)) +} + +fn collect_system_env() -> BTreeMap { + BTreeMap::from([ + ("arch".to_string(), std::env::consts::ARCH.to_string()), + ("family".to_string(), std::env::consts::FAMILY.to_string()), + ("os".to_string(), std::env::consts::OS.to_string()), + ]) +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeSet; + + use serde_json::{Value, json}; + + use super::{is_public_vllm_env_key, render_config_text}; + + #[test] + fn render_config_text_formats_config_snapshot() { + let rendered = render_config_text(&json!({ + "model": "test-model", + "served_model_name": ["served-model"], + "chat_template": null, + "enable_log_requests": true, + })); + let lines = rendered.lines().collect::>(); + + assert_eq!( + lines, + BTreeSet::from([ + "chat_template=None", + "enable_log_requests=true", + "model=test-model", + "served_model_name=[\"served-model\"]", + ]) + ); + assert_eq!( + render_config_text(&Value::String("inline".to_string())), + "inline" + ); + assert_eq!(render_config_text(&Value::Null), "None"); + } + + #[test] + fn server_info_env_filter_excludes_sensitive_vllm_keys() { + for key in [ + "VLLM_API_KEY", + "VLLM_AUTH_TOKEN", + "VLLM_SECRET", + "VLLM_PASSWORD", + "VLLM_CREDENTIAL_FILE", + "vllm_token", + ] { + assert!(!is_public_vllm_env_key(key), "{key}"); + } + } + + #[test] + fn server_info_env_filter_includes_public_vllm_keys() { + assert!(is_public_vllm_env_key("VLLM_LOGGING_LEVEL")); + assert!(is_public_vllm_env_key("VLLM_USE_MODELSCOPE")); + assert!(!is_public_vllm_env_key("OTHER_ENV")); + } +} diff --git a/rust/src/server/src/state.rs b/rust/src/server/src/state.rs index 6863000943af..c73ca04c5d62 100644 --- a/rust/src/server/src/state.rs +++ b/rust/src/server/src/state.rs @@ -1,6 +1,7 @@ use std::sync::Arc; use std::sync::atomic::{AtomicU64, Ordering}; +use serde_json::Value; use tokio::time::{Duration, Instant, sleep_until}; use tracing::warn; use vllm_chat::ChatLlm; @@ -9,6 +10,8 @@ use vllm_engine_core_client::protocol::lora::LoraRequest; use crate::lora::{LoadLoraError, LoraManager, LoraModelResolution, UnloadLoraError}; +use crate::server_info::{ServerInfoConfigFormat, ServerInfoSnapshot}; + const SHUTDOWN_REFCOUNT_POLL_INTERVAL: Duration = Duration::from_millis(100); /// Shared router state for the minimal single-model OpenAI server. @@ -22,6 +25,8 @@ pub struct AppState { pub enable_log_requests: bool, /// Whether to set X-Request-Id on every HTTP response. pub enable_request_id_headers: bool, + /// Runtime server information returned by `/server_info`, when available. + server_info: Option, /// Number of in-flight inference requests currently owned by this frontend. server_load: AtomicU64, /// Dynamic LoRA adapter registry. @@ -47,6 +52,7 @@ impl AppState { chat, enable_log_requests: false, enable_request_id_headers: false, + server_info: None, server_load: AtomicU64::new(0), lora_manager: LoraManager::new(), } @@ -64,6 +70,20 @@ impl AppState { self } + /// Attach the runtime server information snapshot used by `/server_info`. + pub(crate) fn with_server_info(mut self, server_info: ServerInfoSnapshot) -> Self { + self.server_info = Some(server_info); + self + } + + /// Build a `/server_info` response payload. + pub(crate) fn server_info_response( + &self, + config_format: ServerInfoConfigFormat, + ) -> Option { + self.server_info.as_ref().map(|server_info| server_info.response(config_format)) + } + /// The primary model name echoed back in API responses (the first served /// name). pub fn primary_model_name(&self) -> &str {