From eb6af1863237af82e50c11978e04b2cdab402923 Mon Sep 17 00:00:00 2001 From: xunzhuo Date: Fri, 29 May 2026 11:48:38 +0800 Subject: [PATCH 1/8] Add Rust frontend server info endpoint Signed-off-by: xunzhuo --- rust/src/server/src/lib.rs | 6 +- rust/src/server/src/routes.rs | 2 + rust/src/server/src/routes/server_info.rs | 42 +++++++++ rust/src/server/src/routes/tests.rs | 52 +++++++++++ rust/src/server/src/state.rs | 103 ++++++++++++++++++++++ 5 files changed, 203 insertions(+), 2 deletions(-) create mode 100644 rust/src/server/src/routes/server_info.rs diff --git a/rust/src/server/src/lib.rs b/rust/src/server/src/lib.rs index 2b684287ba23..62887cf91218 100644 --- a/rust/src/server/src/lib.rs +++ b/rust/src/server/src/lib.rs @@ -29,7 +29,7 @@ use vllm_text::TextLlm; use crate::listener::Listener; use crate::routes::build_router; -use crate::state::AppState; +use crate::state::{AppState, ServerInfoSnapshot}; /// Build the shared application state for one configured model and one engine /// client. @@ -85,7 +85,9 @@ async fn build_state(config: &Config) -> Result> { }; Ok(Arc::new( - AppState::new(served_model_names, chat).with_log_requests(config.enable_log_requests), + AppState::new(served_model_names, chat) + .with_log_requests(config.enable_log_requests) + .with_server_info(ServerInfoSnapshot::from_config(config)), )) } diff --git a/rust/src/server/src/routes.rs b/rust/src/server/src/routes.rs index b9549e7144ff..3cea31020dca 100644 --- a/rust/src/server/src/routes.rs +++ b/rust/src/server/src/routes.rs @@ -5,6 +5,7 @@ mod inference; mod load; mod metrics; pub(crate) mod openai; +mod server_info; mod sleep; mod version; @@ -37,6 +38,7 @@ fn build_router_with_dev_mode(state: Arc, dev_mode_enabled: bool) -> R .route("/metrics", get(metrics::scrape)) .route("/load", get(load::load)) .route("/version", get(version::version)) + .route("/server_info", get(server_info::server_info)) // OpenAI-compatible endpoints .route("/v1/models", get(openai::list_models)) .route("/v1/completions", post(openai::completions)) diff --git a/rust/src/server/src/routes/server_info.rs b/rust/src/server/src/routes/server_info.rs new file mode 100644 index 000000000000..434765937ca3 --- /dev/null +++ b/rust/src/server/src/routes/server_info.rs @@ -0,0 +1,42 @@ +use std::sync::Arc; + +use axum::Json; +use axum::extract::{Query, State}; +use serde::Deserialize; +use serde_json::Value; + +use crate::state::{AppState, ServerInfoConfigFormat}; + +#[derive(Debug, Clone, Copy, Deserialize)] +#[serde(rename_all = "lowercase")] +enum ConfigFormat { + Text, + Json, +} + +impl From for ServerInfoConfigFormat { + fn from(value: ConfigFormat) -> Self { + match value { + ConfigFormat::Text => Self::Text, + ConfigFormat::Json => Self::Json, + } + } +} + +fn default_config_format() -> ConfigFormat { + ConfigFormat::Text +} + +#[derive(Debug, Deserialize)] +pub(crate) struct ServerInfoParams { + #[serde(default = "default_config_format")] + config_format: ConfigFormat, +} + +/// Get server configuration and environment metadata. +pub async fn server_info( + State(state): State>, + Query(params): Query, +) -> Json { + Json(state.server_info_response(params.config_format.into())) +} diff --git a/rust/src/server/src/routes/tests.rs b/rust/src/server/src/routes/tests.rs index d166b8004470..cdc538da3081 100644 --- a/rust/src/server/src/routes/tests.rs +++ b/rust/src/server/src/routes/tests.rs @@ -1015,6 +1015,58 @@ async fn version_returns_engine_vllm_version() { ); } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial] +async fn server_info_endpoint_returns_text_config_by_default() { + let mut app = test_app().await; + let response = app + .call( + Request::builder() + .uri("/server_info") + .body(Body::empty()) + .expect("build request"), + ) + .await + .expect("call app"); + + assert_eq!(response.status(), StatusCode::OK); + let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body"); + let json: serde_json::Value = serde_json::from_slice(&body).expect("decode json"); + let config = json["vllm_config"].as_str().expect("text config"); + + assert!(config.contains("served_model_name")); + assert!(json["vllm_env"].is_object()); + assert_eq!(json["system_env"]["arch"], std::env::consts::ARCH); + assert_eq!(json["system_env"]["family"], std::env::consts::FAMILY); + assert_eq!(json["system_env"]["os"], std::env::consts::OS); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial] +async fn server_info_endpoint_returns_json_config_when_requested() { + let mut app = test_app().await; + let response = app + .call( + Request::builder() + .uri("/server_info?config_format=json") + .body(Body::empty()) + .expect("build request"), + ) + .await + .expect("call app"); + + assert_eq!(response.status(), StatusCode::OK); + let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body"); + let json: serde_json::Value = serde_json::from_slice(&body).expect("decode json"); + + assert_eq!( + json["vllm_config"]["served_model_name"], + json!(["Qwen/Qwen1.5-0.5B-Chat"]) + ); + assert!(json["vllm_env"].is_object()); + assert!(json["system_env"].is_object()); +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[serial] async fn http_metrics_record_list_models_requests() { diff --git a/rust/src/server/src/state.rs b/rust/src/server/src/state.rs index 04d37f1a5d44..2918a509b718 100644 --- a/rust/src/server/src/state.rs +++ b/rust/src/server/src/state.rs @@ -1,13 +1,101 @@ +use std::collections::BTreeMap; use std::sync::Arc; use std::sync::atomic::{AtomicU64, Ordering}; +use serde_json::{Value, json}; use tokio::time::{Duration, Instant, sleep_until}; use tracing::warn; use vllm_chat::ChatLlm; use vllm_engine_core_client::EngineCoreClient; +use crate::config::Config; + const SHUTDOWN_REFCOUNT_POLL_INTERVAL: Duration = Duration::from_millis(100); +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum ServerInfoConfigFormat { + Text, + Json, +} + +/// Snapshot returned by `/server_info`. +#[derive(Debug, Clone)] +pub(crate) struct ServerInfoSnapshot { + vllm_config_text: String, + vllm_config_json: Value, + vllm_env: BTreeMap, + system_env: BTreeMap, +} + +impl ServerInfoSnapshot { + fn from_served_model_names(served_model_names: &[String]) -> Self { + Self { + vllm_config_text: format!("served_model_name={served_model_names:?}"), + vllm_config_json: json!({ + "served_model_name": served_model_names, + }), + vllm_env: collect_vllm_env(), + system_env: collect_system_env(), + } + } + + /// Capture the runtime configuration fields available to the Rust frontend. + pub(crate) fn from_config(config: &Config) -> Self { + let vllm_config_json = json!({ + "transport_mode": format!("{:?}", config.transport_mode), + "coordinator_mode": format!("{:?}", config.coordinator_mode), + "model": config.model.clone(), + "served_model_name": config.served_model_name.clone(), + "listener_mode": format!("{:?}", config.listener_mode), + "tool_call_parser": format!("{:?}", config.tool_call_parser), + "reasoning_parser": format!("{:?}", config.reasoning_parser), + "renderer": format!("{:?}", config.renderer), + "chat_template": config.chat_template.clone(), + "default_chat_template_kwargs": config.default_chat_template_kwargs.clone(), + "chat_template_content_format": format!("{:?}", config.chat_template_content_format), + "enable_log_requests": config.enable_log_requests, + "disable_log_stats": config.disable_log_stats, + "grpc_port": config.grpc_port, + "shutdown_timeout_secs": config.shutdown_timeout.as_secs_f64(), + "engine_count": config.engine_count(), + }); + + Self { + vllm_config_text: format!("{config:#?}"), + vllm_config_json, + vllm_env: collect_vllm_env(), + system_env: collect_system_env(), + } + } + + pub(crate) fn response(&self, config_format: ServerInfoConfigFormat) -> Value { + let vllm_config = match config_format { + ServerInfoConfigFormat::Text => Value::String(self.vllm_config_text.clone()), + ServerInfoConfigFormat::Json => self.vllm_config_json.clone(), + }; + + json!({ + "vllm_config": vllm_config, + "vllm_env": self.vllm_env.clone(), + "system_env": self.system_env.clone(), + }) + } +} + +fn collect_vllm_env() -> BTreeMap { + std::env::vars() + .filter(|(key, _)| key.starts_with("VLLM_") && !key.contains("KEY")) + .collect() +} + +fn collect_system_env() -> BTreeMap { + BTreeMap::from([ + ("arch".to_string(), std::env::consts::ARCH.to_string()), + ("family".to_string(), std::env::consts::FAMILY.to_string()), + ("os".to_string(), std::env::consts::OS.to_string()), + ]) +} + /// Shared router state for the minimal single-model OpenAI server. pub struct AppState { /// All public model IDs served by this frontend. The first entry is the @@ -17,6 +105,8 @@ pub struct AppState { pub chat: ChatLlm, /// Whether to log a summary line for each completed request. pub enable_log_requests: bool, + /// Runtime server information returned by `/server_info`. + server_info: ServerInfoSnapshot, /// Number of in-flight inference requests currently owned by this frontend. server_load: AtomicU64, } @@ -35,10 +125,12 @@ impl AppState { !served_model_names.is_empty(), "served_model_names must not be empty" ); + let server_info = ServerInfoSnapshot::from_served_model_names(&served_model_names); Self { served_model_names, chat, enable_log_requests: false, + server_info, server_load: AtomicU64::new(0), } } @@ -49,6 +141,17 @@ impl AppState { self } + /// Attach the runtime server information snapshot used by `/server_info`. + pub(crate) fn with_server_info(mut self, server_info: ServerInfoSnapshot) -> Self { + self.server_info = server_info; + self + } + + /// Build a `/server_info` response payload. + pub(crate) fn server_info_response(&self, config_format: ServerInfoConfigFormat) -> Value { + self.server_info.response(config_format) + } + /// The primary model name echoed back in API responses (the first served /// name). pub fn primary_model_name(&self) -> &str { From bef6f08d468752a271b6f536ecc63f0dd6173765 Mon Sep 17 00:00:00 2001 From: xunzhuo Date: Fri, 29 May 2026 12:34:10 +0800 Subject: [PATCH 2/8] Gate server info route behind dev mode Signed-off-by: xunzhuo --- rust/src/server/src/routes.rs | 2 +- rust/src/server/src/routes/tests.rs | 36 ++++++++++++++++++++++++----- 2 files changed, 31 insertions(+), 7 deletions(-) diff --git a/rust/src/server/src/routes.rs b/rust/src/server/src/routes.rs index 3cea31020dca..b37ba714abfb 100644 --- a/rust/src/server/src/routes.rs +++ b/rust/src/server/src/routes.rs @@ -38,7 +38,6 @@ fn build_router_with_dev_mode(state: Arc, dev_mode_enabled: bool) -> R .route("/metrics", get(metrics::scrape)) .route("/load", get(load::load)) .route("/version", get(version::version)) - .route("/server_info", get(server_info::server_info)) // OpenAI-compatible endpoints .route("/v1/models", get(openai::list_models)) .route("/v1/completions", post(openai::completions)) @@ -56,6 +55,7 @@ fn build_router_with_dev_mode(state: Arc, dev_mode_enabled: bool) -> R .route("/sleep", post(sleep::sleep)) .route("/wake_up", post(sleep::wake_up)) .route("/is_sleeping", get(sleep::is_sleeping)) + .route("/server_info", get(server_info::server_info)) } router diff --git a/rust/src/server/src/routes/tests.rs b/rust/src/server/src/routes/tests.rs index cdc538da3081..2802057ab941 100644 --- a/rust/src/server/src/routes/tests.rs +++ b/rust/src/server/src/routes/tests.rs @@ -734,16 +734,23 @@ async fn test_chat_with_engine_outputs( } async fn test_app() -> axum::Router { + test_app_with_dev_mode(false).await +} + +async fn test_app_with_dev_mode(dev_mode_enabled: bool) -> axum::Router { let (chat, _engine_task) = test_models_with_engine_outputs_and_backend( b"engine-openai", default_stream_output_specs(), Arc::new(FakeChatBackend::new()), ) .await; - build_router(Arc::new(AppState::new( - vec!["Qwen/Qwen1.5-0.5B-Chat".to_string()], - chat, - ))) + build_router_with_dev_mode( + Arc::new(AppState::new( + vec!["Qwen/Qwen1.5-0.5B-Chat".to_string()], + chat, + )), + dev_mode_enabled, + ) } async fn test_health_app_with_engine_script( @@ -1017,7 +1024,7 @@ async fn version_returns_engine_vllm_version() { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[serial] -async fn server_info_endpoint_returns_text_config_by_default() { +async fn server_info_endpoint_is_dev_mode_only() { let mut app = test_app().await; let response = app .call( @@ -1029,6 +1036,23 @@ async fn server_info_endpoint_returns_text_config_by_default() { .await .expect("call app"); + assert_eq!(response.status(), StatusCode::NOT_FOUND); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[serial] +async fn server_info_endpoint_returns_text_config_by_default() { + let mut app = test_app_with_dev_mode(true).await; + let response = app + .call( + Request::builder() + .uri("/server_info") + .body(Body::empty()) + .expect("build request"), + ) + .await + .expect("call app"); + assert_eq!(response.status(), StatusCode::OK); let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body"); let json: serde_json::Value = serde_json::from_slice(&body).expect("decode json"); @@ -1044,7 +1068,7 @@ async fn server_info_endpoint_returns_text_config_by_default() { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[serial] async fn server_info_endpoint_returns_json_config_when_requested() { - let mut app = test_app().await; + let mut app = test_app_with_dev_mode(true).await; let response = app .call( Request::builder() From 3dfba11d85cb037cea27f702457faa8af95adae1 Mon Sep 17 00:00:00 2001 From: xunzhuo Date: Fri, 29 May 2026 12:34:48 +0800 Subject: [PATCH 3/8] Filter sensitive server info env vars Signed-off-by: xunzhuo --- rust/src/server/src/state.rs | 38 +++++++++++++++++++++++++++++++++--- 1 file changed, 35 insertions(+), 3 deletions(-) diff --git a/rust/src/server/src/state.rs b/rust/src/server/src/state.rs index 2918a509b718..3ed7b503bfdf 100644 --- a/rust/src/server/src/state.rs +++ b/rust/src/server/src/state.rs @@ -11,6 +11,8 @@ use vllm_engine_core_client::EngineCoreClient; use crate::config::Config; const SHUTDOWN_REFCOUNT_POLL_INTERVAL: Duration = Duration::from_millis(100); +const SENSITIVE_VLLM_ENV_PATTERNS: &[&str] = + &["KEY", "SECRET", "TOKEN", "PASSWORD", "CREDENTIAL", "AUTH"]; #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) enum ServerInfoConfigFormat { @@ -83,9 +85,13 @@ impl ServerInfoSnapshot { } fn collect_vllm_env() -> BTreeMap { - std::env::vars() - .filter(|(key, _)| key.starts_with("VLLM_") && !key.contains("KEY")) - .collect() + std::env::vars().filter(|(key, _)| is_public_vllm_env_key(key)).collect() +} + +fn is_public_vllm_env_key(key: &str) -> bool { + let key = key.to_ascii_uppercase(); + key.starts_with("VLLM_") + && !SENSITIVE_VLLM_ENV_PATTERNS.iter().any(|pattern| key.contains(pattern)) } fn collect_system_env() -> BTreeMap { @@ -221,3 +227,29 @@ impl AppState { } } } + +#[cfg(test)] +mod tests { + use super::is_public_vllm_env_key; + + #[test] + fn server_info_env_filter_excludes_sensitive_vllm_keys() { + for key in [ + "VLLM_API_KEY", + "VLLM_AUTH_TOKEN", + "VLLM_SECRET", + "VLLM_PASSWORD", + "VLLM_CREDENTIAL_FILE", + "vllm_token", + ] { + assert!(!is_public_vllm_env_key(key), "{key}"); + } + } + + #[test] + fn server_info_env_filter_includes_public_vllm_keys() { + assert!(is_public_vllm_env_key("VLLM_LOGGING_LEVEL")); + assert!(is_public_vllm_env_key("VLLM_USE_MODELSCOPE")); + assert!(!is_public_vllm_env_key("OTHER_ENV")); + } +} From f4c271355dfd9bb495055966bdd9de929b5bc192 Mon Sep 17 00:00:00 2001 From: xunzhuo Date: Fri, 29 May 2026 12:35:42 +0800 Subject: [PATCH 4/8] Render server info text from curated config Signed-off-by: xunzhuo --- rust/src/server/src/routes/tests.rs | 3 ++- rust/src/server/src/state.rs | 31 ++++++++++++++++++++++++----- 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/rust/src/server/src/routes/tests.rs b/rust/src/server/src/routes/tests.rs index 2802057ab941..cec2304b116e 100644 --- a/rust/src/server/src/routes/tests.rs +++ b/rust/src/server/src/routes/tests.rs @@ -1058,7 +1058,8 @@ async fn server_info_endpoint_returns_text_config_by_default() { let json: serde_json::Value = serde_json::from_slice(&body).expect("decode json"); let config = json["vllm_config"].as_str().expect("text config"); - assert!(config.contains("served_model_name")); + assert!(config.contains("served_model_name=")); + assert!(!config.contains("Config {")); assert!(json["vllm_env"].is_object()); assert_eq!(json["system_env"]["arch"], std::env::consts::ARCH); assert_eq!(json["system_env"]["family"], std::env::consts::FAMILY); diff --git a/rust/src/server/src/state.rs b/rust/src/server/src/state.rs index 3ed7b503bfdf..d4e26104dac7 100644 --- a/rust/src/server/src/state.rs +++ b/rust/src/server/src/state.rs @@ -31,11 +31,13 @@ pub(crate) struct ServerInfoSnapshot { impl ServerInfoSnapshot { fn from_served_model_names(served_model_names: &[String]) -> Self { + let vllm_config_json = json!({ + "served_model_name": served_model_names, + }); + Self { - vllm_config_text: format!("served_model_name={served_model_names:?}"), - vllm_config_json: json!({ - "served_model_name": served_model_names, - }), + vllm_config_text: render_config_text(&vllm_config_json), + vllm_config_json, vllm_env: collect_vllm_env(), system_env: collect_system_env(), } @@ -63,7 +65,7 @@ impl ServerInfoSnapshot { }); Self { - vllm_config_text: format!("{config:#?}"), + vllm_config_text: render_config_text(&vllm_config_json), vllm_config_json, vllm_env: collect_vllm_env(), system_env: collect_system_env(), @@ -84,6 +86,25 @@ impl ServerInfoSnapshot { } } +fn render_config_text(config: &Value) -> String { + match config { + Value::Object(fields) => fields + .iter() + .map(|(key, value)| format!("{key}={}", render_config_text_value(value))) + .collect::>() + .join("\n"), + _ => render_config_text_value(config), + } +} + +fn render_config_text_value(value: &Value) -> String { + match value { + Value::Null => "None".to_string(), + Value::String(value) => value.clone(), + _ => value.to_string(), + } +} + fn collect_vllm_env() -> BTreeMap { std::env::vars().filter(|(key, _)| is_public_vllm_env_key(key)).collect() } From e500725c7b9b91ab1ab48559cdf6b8e55a33b5b7 Mon Sep 17 00:00:00 2001 From: xunzhuo Date: Fri, 29 May 2026 12:37:07 +0800 Subject: [PATCH 5/8] Use stable server info config values Signed-off-by: xunzhuo --- rust/src/server/src/state.rs | 129 ++++++++++++++++++++++++++++++++--- 1 file changed, 120 insertions(+), 9 deletions(-) diff --git a/rust/src/server/src/state.rs b/rust/src/server/src/state.rs index d4e26104dac7..b52ff6bd30db 100644 --- a/rust/src/server/src/state.rs +++ b/rust/src/server/src/state.rs @@ -6,9 +6,9 @@ use serde_json::{Value, json}; use tokio::time::{Duration, Instant, sleep_until}; use tracing::warn; use vllm_chat::ChatLlm; -use vllm_engine_core_client::EngineCoreClient; +use vllm_engine_core_client::{EngineCoreClient, TransportMode}; -use crate::config::Config; +use crate::config::{Config, CoordinatorMode, HttpListenerMode}; const SHUTDOWN_REFCOUNT_POLL_INTERVAL: Duration = Duration::from_millis(100); const SENSITIVE_VLLM_ENV_PATTERNS: &[&str] = @@ -46,17 +46,17 @@ impl ServerInfoSnapshot { /// Capture the runtime configuration fields available to the Rust frontend. pub(crate) fn from_config(config: &Config) -> Self { let vllm_config_json = json!({ - "transport_mode": format!("{:?}", config.transport_mode), - "coordinator_mode": format!("{:?}", config.coordinator_mode), + "transport_mode": transport_mode_config(&config.transport_mode), + "coordinator_mode": coordinator_mode_config(&config.coordinator_mode), "model": config.model.clone(), "served_model_name": config.served_model_name.clone(), - "listener_mode": format!("{:?}", config.listener_mode), - "tool_call_parser": format!("{:?}", config.tool_call_parser), - "reasoning_parser": format!("{:?}", config.reasoning_parser), - "renderer": format!("{:?}", config.renderer), + "listener_mode": listener_mode_config(&config.listener_mode), + "tool_call_parser": config.tool_call_parser.to_string(), + "reasoning_parser": config.reasoning_parser.to_string(), + "renderer": config.renderer.to_string(), "chat_template": config.chat_template.clone(), "default_chat_template_kwargs": config.default_chat_template_kwargs.clone(), - "chat_template_content_format": format!("{:?}", config.chat_template_content_format), + "chat_template_content_format": config.chat_template_content_format.to_string(), "enable_log_requests": config.enable_log_requests, "disable_log_stats": config.disable_log_stats, "grpc_port": config.grpc_port, @@ -86,6 +86,68 @@ impl ServerInfoSnapshot { } } +fn transport_mode_config(transport_mode: &TransportMode) -> Value { + match transport_mode { + TransportMode::HandshakeOwner { + handshake_address, + advertised_host, + engine_count, + ready_timeout, + local_input_address, + local_output_address, + } => json!({ + "mode": "handshake_owner", + "handshake_address": handshake_address, + "advertised_host": advertised_host, + "engine_count": engine_count, + "ready_timeout_secs": ready_timeout.as_secs_f64(), + "local_input_address": local_input_address, + "local_output_address": local_output_address, + }), + TransportMode::Bootstrapped { + input_address, + output_address, + engine_count, + ready_timeout, + } => json!({ + "mode": "bootstrapped", + "input_address": input_address, + "output_address": output_address, + "engine_count": engine_count, + "ready_timeout_secs": ready_timeout.as_secs_f64(), + }), + } +} + +fn coordinator_mode_config(coordinator_mode: &CoordinatorMode) -> Value { + match coordinator_mode { + CoordinatorMode::None => json!("none"), + CoordinatorMode::MaybeInProc => json!("maybe_in_proc"), + CoordinatorMode::External { address } => json!({ + "mode": "external", + "address": address, + }), + } +} + +fn listener_mode_config(listener_mode: &HttpListenerMode) -> Value { + match listener_mode { + HttpListenerMode::BindTcp { host, port } => json!({ + "mode": "bind_tcp", + "host": host, + "port": port, + }), + HttpListenerMode::BindUnix { path } => json!({ + "mode": "bind_unix", + "path": path, + }), + HttpListenerMode::InheritedFd { fd } => json!({ + "mode": "inherited_fd", + "fd": fd, + }), + } +} + fn render_config_text(config: &Value) -> String { match config { Value::Object(fields) => fields @@ -251,7 +313,14 @@ impl AppState { #[cfg(test)] mod tests { + use std::time::Duration; + + use vllm_chat::{ChatTemplateContentFormatOption, ParserSelection, RendererSelection}; + use vllm_engine_core_client::TransportMode; + use super::is_public_vllm_env_key; + use super::{ServerInfoConfigFormat, ServerInfoSnapshot}; + use crate::config::{Config, CoordinatorMode, HttpListenerMode}; #[test] fn server_info_env_filter_excludes_sensitive_vllm_keys() { @@ -273,4 +342,46 @@ mod tests { assert!(is_public_vllm_env_key("VLLM_USE_MODELSCOPE")); assert!(!is_public_vllm_env_key("OTHER_ENV")); } + + #[test] + fn server_info_config_json_uses_stable_values() { + let config = Config { + transport_mode: TransportMode::Bootstrapped { + input_address: "tcp://127.0.0.1:0".to_string(), + output_address: "tcp://127.0.0.1:1".to_string(), + engine_count: 2, + ready_timeout: Duration::from_secs(5), + }, + coordinator_mode: CoordinatorMode::MaybeInProc, + model: "test-model".to_string(), + served_model_name: vec!["served-model".to_string()], + listener_mode: HttpListenerMode::BindTcp { + host: "127.0.0.1".to_string(), + port: 8000, + }, + tool_call_parser: ParserSelection::Auto, + reasoning_parser: ParserSelection::None, + renderer: RendererSelection::Hf, + chat_template: None, + default_chat_template_kwargs: None, + chat_template_content_format: ChatTemplateContentFormatOption::OpenAi, + enable_log_requests: true, + disable_log_stats: false, + grpc_port: Some(9000), + shutdown_timeout: Duration::from_secs(10), + }; + + let snapshot = ServerInfoSnapshot::from_config(&config); + let response = snapshot.response(ServerInfoConfigFormat::Json); + let vllm_config = &response["vllm_config"]; + + assert_eq!(vllm_config["transport_mode"]["mode"], "bootstrapped"); + assert_eq!(vllm_config["transport_mode"]["engine_count"], 2); + assert_eq!(vllm_config["coordinator_mode"], "maybe_in_proc"); + assert_eq!(vllm_config["listener_mode"]["mode"], "bind_tcp"); + assert_eq!(vllm_config["tool_call_parser"], "auto"); + assert_eq!(vllm_config["reasoning_parser"], "none"); + assert_eq!(vllm_config["renderer"], "hf"); + assert_eq!(vllm_config["chat_template_content_format"], "openai"); + } } From ffceba834af04f71df6999308b4d0856fbc0c771 Mon Sep 17 00:00:00 2001 From: xunzhuo Date: Wed, 3 Jun 2026 14:28:15 +0800 Subject: [PATCH 6/8] Move server info construction out of state Signed-off-by: xunzhuo --- rust/src/server/src/lib.rs | 4 +- rust/src/server/src/routes/server_info.rs | 3 +- rust/src/server/src/server_info.rs | 254 +++++++++++++++++++++ rust/src/server/src/state.rs | 255 +--------------------- 4 files changed, 262 insertions(+), 254 deletions(-) create mode 100644 rust/src/server/src/server_info.rs diff --git a/rust/src/server/src/lib.rs b/rust/src/server/src/lib.rs index 62887cf91218..8ecdc5f774d6 100644 --- a/rust/src/server/src/lib.rs +++ b/rust/src/server/src/lib.rs @@ -6,6 +6,7 @@ mod grpc; mod listener; mod middleware; mod routes; +mod server_info; mod state; mod utils; @@ -29,7 +30,8 @@ use vllm_text::TextLlm; use crate::listener::Listener; use crate::routes::build_router; -use crate::state::{AppState, ServerInfoSnapshot}; +use crate::server_info::ServerInfoSnapshot; +use crate::state::AppState; /// Build the shared application state for one configured model and one engine /// client. diff --git a/rust/src/server/src/routes/server_info.rs b/rust/src/server/src/routes/server_info.rs index 434765937ca3..353885a9626c 100644 --- a/rust/src/server/src/routes/server_info.rs +++ b/rust/src/server/src/routes/server_info.rs @@ -5,7 +5,8 @@ use axum::extract::{Query, State}; use serde::Deserialize; use serde_json::Value; -use crate::state::{AppState, ServerInfoConfigFormat}; +use crate::server_info::ServerInfoConfigFormat; +use crate::state::AppState; #[derive(Debug, Clone, Copy, Deserialize)] #[serde(rename_all = "lowercase")] diff --git a/rust/src/server/src/server_info.rs b/rust/src/server/src/server_info.rs new file mode 100644 index 000000000000..94f75cbde2a1 --- /dev/null +++ b/rust/src/server/src/server_info.rs @@ -0,0 +1,254 @@ +use std::collections::BTreeMap; + +use serde_json::{Value, json}; +use vllm_engine_core_client::TransportMode; + +use crate::config::{Config, CoordinatorMode, HttpListenerMode}; + +const SENSITIVE_VLLM_ENV_PATTERNS: &[&str] = + &["KEY", "SECRET", "TOKEN", "PASSWORD", "CREDENTIAL", "AUTH"]; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum ServerInfoConfigFormat { + Text, + Json, +} + +/// Snapshot returned by `/server_info`. +#[derive(Debug, Clone)] +pub(crate) struct ServerInfoSnapshot { + vllm_config_text: String, + vllm_config_json: Value, + vllm_env: BTreeMap, + system_env: BTreeMap, +} + +impl ServerInfoSnapshot { + pub(crate) fn from_served_model_names(served_model_names: &[String]) -> Self { + let vllm_config_json = json!({ + "served_model_name": served_model_names, + }); + + Self { + vllm_config_text: render_config_text(&vllm_config_json), + vllm_config_json, + vllm_env: collect_vllm_env(), + system_env: collect_system_env(), + } + } + + /// Capture the runtime configuration fields available to the Rust frontend. + pub(crate) fn from_config(config: &Config) -> Self { + let vllm_config_json = json!({ + "transport_mode": transport_mode_config(&config.transport_mode), + "coordinator_mode": coordinator_mode_config(&config.coordinator_mode), + "model": config.model.clone(), + "served_model_name": config.served_model_name.clone(), + "listener_mode": listener_mode_config(&config.listener_mode), + "tool_call_parser": config.tool_call_parser.to_string(), + "reasoning_parser": config.reasoning_parser.to_string(), + "renderer": config.renderer.to_string(), + "chat_template": config.chat_template.clone(), + "default_chat_template_kwargs": config.default_chat_template_kwargs.clone(), + "chat_template_content_format": config.chat_template_content_format.to_string(), + "enable_log_requests": config.enable_log_requests, + "disable_log_stats": config.disable_log_stats, + "grpc_port": config.grpc_port, + "shutdown_timeout_secs": config.shutdown_timeout.as_secs_f64(), + "engine_count": config.engine_count(), + }); + + Self { + vllm_config_text: render_config_text(&vllm_config_json), + vllm_config_json, + vllm_env: collect_vllm_env(), + system_env: collect_system_env(), + } + } + + pub(crate) fn response(&self, config_format: ServerInfoConfigFormat) -> Value { + let vllm_config = match config_format { + ServerInfoConfigFormat::Text => Value::String(self.vllm_config_text.clone()), + ServerInfoConfigFormat::Json => self.vllm_config_json.clone(), + }; + + json!({ + "vllm_config": vllm_config, + "vllm_env": self.vllm_env.clone(), + "system_env": self.system_env.clone(), + }) + } +} + +fn transport_mode_config(transport_mode: &TransportMode) -> Value { + match transport_mode { + TransportMode::HandshakeOwner { + handshake_address, + advertised_host, + engine_count, + ready_timeout, + local_input_address, + local_output_address, + } => json!({ + "mode": "handshake_owner", + "handshake_address": handshake_address, + "advertised_host": advertised_host, + "engine_count": engine_count, + "ready_timeout_secs": ready_timeout.as_secs_f64(), + "local_input_address": local_input_address, + "local_output_address": local_output_address, + }), + TransportMode::Bootstrapped { + input_address, + output_address, + engine_count, + ready_timeout, + } => json!({ + "mode": "bootstrapped", + "input_address": input_address, + "output_address": output_address, + "engine_count": engine_count, + "ready_timeout_secs": ready_timeout.as_secs_f64(), + }), + } +} + +fn coordinator_mode_config(coordinator_mode: &CoordinatorMode) -> Value { + match coordinator_mode { + CoordinatorMode::None => json!("none"), + CoordinatorMode::MaybeInProc => json!("maybe_in_proc"), + CoordinatorMode::External { address } => json!({ + "mode": "external", + "address": address, + }), + } +} + +fn listener_mode_config(listener_mode: &HttpListenerMode) -> Value { + match listener_mode { + HttpListenerMode::BindTcp { host, port } => json!({ + "mode": "bind_tcp", + "host": host, + "port": port, + }), + HttpListenerMode::BindUnix { path } => json!({ + "mode": "bind_unix", + "path": path, + }), + HttpListenerMode::InheritedFd { fd } => json!({ + "mode": "inherited_fd", + "fd": fd, + }), + } +} + +fn render_config_text(config: &Value) -> String { + match config { + Value::Object(fields) => fields + .iter() + .map(|(key, value)| format!("{key}={}", render_config_text_value(value))) + .collect::>() + .join("\n"), + _ => render_config_text_value(config), + } +} + +fn render_config_text_value(value: &Value) -> String { + match value { + Value::Null => "None".to_string(), + Value::String(value) => value.clone(), + _ => value.to_string(), + } +} + +fn collect_vllm_env() -> BTreeMap { + std::env::vars().filter(|(key, _)| is_public_vllm_env_key(key)).collect() +} + +fn is_public_vllm_env_key(key: &str) -> bool { + let key = key.to_ascii_uppercase(); + key.starts_with("VLLM_") + && !SENSITIVE_VLLM_ENV_PATTERNS.iter().any(|pattern| key.contains(pattern)) +} + +fn collect_system_env() -> BTreeMap { + BTreeMap::from([ + ("arch".to_string(), std::env::consts::ARCH.to_string()), + ("family".to_string(), std::env::consts::FAMILY.to_string()), + ("os".to_string(), std::env::consts::OS.to_string()), + ]) +} + +#[cfg(test)] +mod tests { + use std::time::Duration; + + use vllm_chat::{ChatTemplateContentFormatOption, ParserSelection, RendererSelection}; + use vllm_engine_core_client::TransportMode; + + use super::{ServerInfoConfigFormat, ServerInfoSnapshot, is_public_vllm_env_key}; + use crate::config::{Config, CoordinatorMode, HttpListenerMode}; + + #[test] + fn server_info_env_filter_excludes_sensitive_vllm_keys() { + for key in [ + "VLLM_API_KEY", + "VLLM_AUTH_TOKEN", + "VLLM_SECRET", + "VLLM_PASSWORD", + "VLLM_CREDENTIAL_FILE", + "vllm_token", + ] { + assert!(!is_public_vllm_env_key(key), "{key}"); + } + } + + #[test] + fn server_info_env_filter_includes_public_vllm_keys() { + assert!(is_public_vllm_env_key("VLLM_LOGGING_LEVEL")); + assert!(is_public_vllm_env_key("VLLM_USE_MODELSCOPE")); + assert!(!is_public_vllm_env_key("OTHER_ENV")); + } + + #[test] + fn server_info_config_json_uses_stable_values() { + let config = Config { + transport_mode: TransportMode::Bootstrapped { + input_address: "tcp://127.0.0.1:0".to_string(), + output_address: "tcp://127.0.0.1:1".to_string(), + engine_count: 2, + ready_timeout: Duration::from_secs(5), + }, + coordinator_mode: CoordinatorMode::MaybeInProc, + model: "test-model".to_string(), + served_model_name: vec!["served-model".to_string()], + listener_mode: HttpListenerMode::BindTcp { + host: "127.0.0.1".to_string(), + port: 8000, + }, + tool_call_parser: ParserSelection::Auto, + reasoning_parser: ParserSelection::None, + renderer: RendererSelection::Hf, + chat_template: None, + default_chat_template_kwargs: None, + chat_template_content_format: ChatTemplateContentFormatOption::OpenAi, + enable_log_requests: true, + disable_log_stats: false, + grpc_port: Some(9000), + shutdown_timeout: Duration::from_secs(10), + }; + + let snapshot = ServerInfoSnapshot::from_config(&config); + let response = snapshot.response(ServerInfoConfigFormat::Json); + let vllm_config = &response["vllm_config"]; + + assert_eq!(vllm_config["transport_mode"]["mode"], "bootstrapped"); + assert_eq!(vllm_config["transport_mode"]["engine_count"], 2); + assert_eq!(vllm_config["coordinator_mode"], "maybe_in_proc"); + assert_eq!(vllm_config["listener_mode"]["mode"], "bind_tcp"); + assert_eq!(vllm_config["tool_call_parser"], "auto"); + assert_eq!(vllm_config["reasoning_parser"], "none"); + assert_eq!(vllm_config["renderer"], "hf"); + assert_eq!(vllm_config["chat_template_content_format"], "openai"); + } +} diff --git a/rust/src/server/src/state.rs b/rust/src/server/src/state.rs index b52ff6bd30db..a8c71f98c49a 100644 --- a/rust/src/server/src/state.rs +++ b/rust/src/server/src/state.rs @@ -1,189 +1,15 @@ -use std::collections::BTreeMap; use std::sync::Arc; use std::sync::atomic::{AtomicU64, Ordering}; -use serde_json::{Value, json}; +use serde_json::Value; use tokio::time::{Duration, Instant, sleep_until}; use tracing::warn; use vllm_chat::ChatLlm; -use vllm_engine_core_client::{EngineCoreClient, TransportMode}; +use vllm_engine_core_client::EngineCoreClient; -use crate::config::{Config, CoordinatorMode, HttpListenerMode}; +use crate::server_info::{ServerInfoConfigFormat, ServerInfoSnapshot}; const SHUTDOWN_REFCOUNT_POLL_INTERVAL: Duration = Duration::from_millis(100); -const SENSITIVE_VLLM_ENV_PATTERNS: &[&str] = - &["KEY", "SECRET", "TOKEN", "PASSWORD", "CREDENTIAL", "AUTH"]; - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(crate) enum ServerInfoConfigFormat { - Text, - Json, -} - -/// Snapshot returned by `/server_info`. -#[derive(Debug, Clone)] -pub(crate) struct ServerInfoSnapshot { - vllm_config_text: String, - vllm_config_json: Value, - vllm_env: BTreeMap, - system_env: BTreeMap, -} - -impl ServerInfoSnapshot { - fn from_served_model_names(served_model_names: &[String]) -> Self { - let vllm_config_json = json!({ - "served_model_name": served_model_names, - }); - - Self { - vllm_config_text: render_config_text(&vllm_config_json), - vllm_config_json, - vllm_env: collect_vllm_env(), - system_env: collect_system_env(), - } - } - - /// Capture the runtime configuration fields available to the Rust frontend. - pub(crate) fn from_config(config: &Config) -> Self { - let vllm_config_json = json!({ - "transport_mode": transport_mode_config(&config.transport_mode), - "coordinator_mode": coordinator_mode_config(&config.coordinator_mode), - "model": config.model.clone(), - "served_model_name": config.served_model_name.clone(), - "listener_mode": listener_mode_config(&config.listener_mode), - "tool_call_parser": config.tool_call_parser.to_string(), - "reasoning_parser": config.reasoning_parser.to_string(), - "renderer": config.renderer.to_string(), - "chat_template": config.chat_template.clone(), - "default_chat_template_kwargs": config.default_chat_template_kwargs.clone(), - "chat_template_content_format": config.chat_template_content_format.to_string(), - "enable_log_requests": config.enable_log_requests, - "disable_log_stats": config.disable_log_stats, - "grpc_port": config.grpc_port, - "shutdown_timeout_secs": config.shutdown_timeout.as_secs_f64(), - "engine_count": config.engine_count(), - }); - - Self { - vllm_config_text: render_config_text(&vllm_config_json), - vllm_config_json, - vllm_env: collect_vllm_env(), - system_env: collect_system_env(), - } - } - - pub(crate) fn response(&self, config_format: ServerInfoConfigFormat) -> Value { - let vllm_config = match config_format { - ServerInfoConfigFormat::Text => Value::String(self.vllm_config_text.clone()), - ServerInfoConfigFormat::Json => self.vllm_config_json.clone(), - }; - - json!({ - "vllm_config": vllm_config, - "vllm_env": self.vllm_env.clone(), - "system_env": self.system_env.clone(), - }) - } -} - -fn transport_mode_config(transport_mode: &TransportMode) -> Value { - match transport_mode { - TransportMode::HandshakeOwner { - handshake_address, - advertised_host, - engine_count, - ready_timeout, - local_input_address, - local_output_address, - } => json!({ - "mode": "handshake_owner", - "handshake_address": handshake_address, - "advertised_host": advertised_host, - "engine_count": engine_count, - "ready_timeout_secs": ready_timeout.as_secs_f64(), - "local_input_address": local_input_address, - "local_output_address": local_output_address, - }), - TransportMode::Bootstrapped { - input_address, - output_address, - engine_count, - ready_timeout, - } => json!({ - "mode": "bootstrapped", - "input_address": input_address, - "output_address": output_address, - "engine_count": engine_count, - "ready_timeout_secs": ready_timeout.as_secs_f64(), - }), - } -} - -fn coordinator_mode_config(coordinator_mode: &CoordinatorMode) -> Value { - match coordinator_mode { - CoordinatorMode::None => json!("none"), - CoordinatorMode::MaybeInProc => json!("maybe_in_proc"), - CoordinatorMode::External { address } => json!({ - "mode": "external", - "address": address, - }), - } -} - -fn listener_mode_config(listener_mode: &HttpListenerMode) -> Value { - match listener_mode { - HttpListenerMode::BindTcp { host, port } => json!({ - "mode": "bind_tcp", - "host": host, - "port": port, - }), - HttpListenerMode::BindUnix { path } => json!({ - "mode": "bind_unix", - "path": path, - }), - HttpListenerMode::InheritedFd { fd } => json!({ - "mode": "inherited_fd", - "fd": fd, - }), - } -} - -fn render_config_text(config: &Value) -> String { - match config { - Value::Object(fields) => fields - .iter() - .map(|(key, value)| format!("{key}={}", render_config_text_value(value))) - .collect::>() - .join("\n"), - _ => render_config_text_value(config), - } -} - -fn render_config_text_value(value: &Value) -> String { - match value { - Value::Null => "None".to_string(), - Value::String(value) => value.clone(), - _ => value.to_string(), - } -} - -fn collect_vllm_env() -> BTreeMap { - std::env::vars().filter(|(key, _)| is_public_vllm_env_key(key)).collect() -} - -fn is_public_vllm_env_key(key: &str) -> bool { - let key = key.to_ascii_uppercase(); - key.starts_with("VLLM_") - && !SENSITIVE_VLLM_ENV_PATTERNS.iter().any(|pattern| key.contains(pattern)) -} - -fn collect_system_env() -> BTreeMap { - BTreeMap::from([ - ("arch".to_string(), std::env::consts::ARCH.to_string()), - ("family".to_string(), std::env::consts::FAMILY.to_string()), - ("os".to_string(), std::env::consts::OS.to_string()), - ]) -} /// Shared router state for the minimal single-model OpenAI server. pub struct AppState { @@ -310,78 +136,3 @@ impl AppState { } } } - -#[cfg(test)] -mod tests { - use std::time::Duration; - - use vllm_chat::{ChatTemplateContentFormatOption, ParserSelection, RendererSelection}; - use vllm_engine_core_client::TransportMode; - - use super::is_public_vllm_env_key; - use super::{ServerInfoConfigFormat, ServerInfoSnapshot}; - use crate::config::{Config, CoordinatorMode, HttpListenerMode}; - - #[test] - fn server_info_env_filter_excludes_sensitive_vllm_keys() { - for key in [ - "VLLM_API_KEY", - "VLLM_AUTH_TOKEN", - "VLLM_SECRET", - "VLLM_PASSWORD", - "VLLM_CREDENTIAL_FILE", - "vllm_token", - ] { - assert!(!is_public_vllm_env_key(key), "{key}"); - } - } - - #[test] - fn server_info_env_filter_includes_public_vllm_keys() { - assert!(is_public_vllm_env_key("VLLM_LOGGING_LEVEL")); - assert!(is_public_vllm_env_key("VLLM_USE_MODELSCOPE")); - assert!(!is_public_vllm_env_key("OTHER_ENV")); - } - - #[test] - fn server_info_config_json_uses_stable_values() { - let config = Config { - transport_mode: TransportMode::Bootstrapped { - input_address: "tcp://127.0.0.1:0".to_string(), - output_address: "tcp://127.0.0.1:1".to_string(), - engine_count: 2, - ready_timeout: Duration::from_secs(5), - }, - coordinator_mode: CoordinatorMode::MaybeInProc, - model: "test-model".to_string(), - served_model_name: vec!["served-model".to_string()], - listener_mode: HttpListenerMode::BindTcp { - host: "127.0.0.1".to_string(), - port: 8000, - }, - tool_call_parser: ParserSelection::Auto, - reasoning_parser: ParserSelection::None, - renderer: RendererSelection::Hf, - chat_template: None, - default_chat_template_kwargs: None, - chat_template_content_format: ChatTemplateContentFormatOption::OpenAi, - enable_log_requests: true, - disable_log_stats: false, - grpc_port: Some(9000), - shutdown_timeout: Duration::from_secs(10), - }; - - let snapshot = ServerInfoSnapshot::from_config(&config); - let response = snapshot.response(ServerInfoConfigFormat::Json); - let vllm_config = &response["vllm_config"]; - - assert_eq!(vllm_config["transport_mode"]["mode"], "bootstrapped"); - assert_eq!(vllm_config["transport_mode"]["engine_count"], 2); - assert_eq!(vllm_config["coordinator_mode"], "maybe_in_proc"); - assert_eq!(vllm_config["listener_mode"]["mode"], "bind_tcp"); - assert_eq!(vllm_config["tool_call_parser"], "auto"); - assert_eq!(vllm_config["reasoning_parser"], "none"); - assert_eq!(vllm_config["renderer"], "hf"); - assert_eq!(vllm_config["chat_template_content_format"], "openai"); - } -} From e4844cdf4004056a5dd7aef0ce433e9ea89a88fc Mon Sep 17 00:00:00 2001 From: xunzhuo Date: Wed, 3 Jun 2026 14:31:47 +0800 Subject: [PATCH 7/8] Serialize server info config views Signed-off-by: xunzhuo --- rust/src/server/src/server_info.rs | 232 +++++++++++++++++++---------- 1 file changed, 157 insertions(+), 75 deletions(-) diff --git a/rust/src/server/src/server_info.rs b/rust/src/server/src/server_info.rs index 94f75cbde2a1..a385bb2038e9 100644 --- a/rust/src/server/src/server_info.rs +++ b/rust/src/server/src/server_info.rs @@ -1,5 +1,6 @@ -use std::collections::BTreeMap; +use std::collections::{BTreeMap, HashMap}; +use serde::Serialize; use serde_json::{Value, json}; use vllm_engine_core_client::TransportMode; @@ -25,9 +26,7 @@ pub(crate) struct ServerInfoSnapshot { impl ServerInfoSnapshot { pub(crate) fn from_served_model_names(served_model_names: &[String]) -> Self { - let vllm_config_json = json!({ - "served_model_name": served_model_names, - }); + let vllm_config_json = serialize_value(ServedModelNamesConfig { served_model_names }); Self { vllm_config_text: render_config_text(&vllm_config_json), @@ -39,24 +38,7 @@ impl ServerInfoSnapshot { /// Capture the runtime configuration fields available to the Rust frontend. pub(crate) fn from_config(config: &Config) -> Self { - let vllm_config_json = json!({ - "transport_mode": transport_mode_config(&config.transport_mode), - "coordinator_mode": coordinator_mode_config(&config.coordinator_mode), - "model": config.model.clone(), - "served_model_name": config.served_model_name.clone(), - "listener_mode": listener_mode_config(&config.listener_mode), - "tool_call_parser": config.tool_call_parser.to_string(), - "reasoning_parser": config.reasoning_parser.to_string(), - "renderer": config.renderer.to_string(), - "chat_template": config.chat_template.clone(), - "default_chat_template_kwargs": config.default_chat_template_kwargs.clone(), - "chat_template_content_format": config.chat_template_content_format.to_string(), - "enable_log_requests": config.enable_log_requests, - "disable_log_stats": config.disable_log_stats, - "grpc_port": config.grpc_port, - "shutdown_timeout_secs": config.shutdown_timeout.as_secs_f64(), - "engine_count": config.engine_count(), - }); + let vllm_config_json = serialize_value(RuntimeServerInfoConfig::from(config)); Self { vllm_config_text: render_config_text(&vllm_config_json), @@ -80,68 +62,168 @@ impl ServerInfoSnapshot { } } -fn transport_mode_config(transport_mode: &TransportMode) -> Value { - match transport_mode { - TransportMode::HandshakeOwner { - handshake_address, - advertised_host, - engine_count, - ready_timeout, - local_input_address, - local_output_address, - } => json!({ - "mode": "handshake_owner", - "handshake_address": handshake_address, - "advertised_host": advertised_host, - "engine_count": engine_count, - "ready_timeout_secs": ready_timeout.as_secs_f64(), - "local_input_address": local_input_address, - "local_output_address": local_output_address, - }), - TransportMode::Bootstrapped { - input_address, - output_address, - engine_count, - ready_timeout, - } => json!({ - "mode": "bootstrapped", - "input_address": input_address, - "output_address": output_address, - "engine_count": engine_count, - "ready_timeout_secs": ready_timeout.as_secs_f64(), - }), +#[derive(Debug, Serialize)] +struct ServedModelNamesConfig<'a> { + #[serde(rename = "served_model_name")] + served_model_names: &'a [String], +} + +#[derive(Debug, Serialize)] +struct RuntimeServerInfoConfig { + transport_mode: TransportModeInfo, + coordinator_mode: CoordinatorModeInfo, + model: String, + served_model_name: Vec, + listener_mode: HttpListenerModeInfo, + tool_call_parser: String, + reasoning_parser: String, + renderer: String, + chat_template: Option, + default_chat_template_kwargs: Option>, + chat_template_content_format: String, + enable_log_requests: bool, + disable_log_stats: bool, + grpc_port: Option, + shutdown_timeout_secs: f64, + engine_count: usize, +} + +impl From<&Config> for RuntimeServerInfoConfig { + fn from(config: &Config) -> Self { + Self { + transport_mode: TransportModeInfo::from(&config.transport_mode), + coordinator_mode: CoordinatorModeInfo::from(&config.coordinator_mode), + model: config.model.clone(), + served_model_name: config.served_model_name.clone(), + listener_mode: HttpListenerModeInfo::from(&config.listener_mode), + tool_call_parser: config.tool_call_parser.to_string(), + reasoning_parser: config.reasoning_parser.to_string(), + renderer: config.renderer.to_string(), + chat_template: config.chat_template.clone(), + default_chat_template_kwargs: config.default_chat_template_kwargs.clone(), + chat_template_content_format: config.chat_template_content_format.to_string(), + enable_log_requests: config.enable_log_requests, + disable_log_stats: config.disable_log_stats, + grpc_port: config.grpc_port, + shutdown_timeout_secs: config.shutdown_timeout.as_secs_f64(), + engine_count: config.engine_count(), + } + } +} + +#[derive(Debug, Serialize)] +#[serde(tag = "mode", rename_all = "snake_case")] +enum TransportModeInfo { + HandshakeOwner { + handshake_address: String, + advertised_host: String, + engine_count: usize, + ready_timeout_secs: f64, + local_input_address: Option, + local_output_address: Option, + }, + Bootstrapped { + input_address: String, + output_address: String, + engine_count: usize, + ready_timeout_secs: f64, + }, +} + +impl From<&TransportMode> for TransportModeInfo { + fn from(transport_mode: &TransportMode) -> Self { + match transport_mode { + TransportMode::HandshakeOwner { + handshake_address, + advertised_host, + engine_count, + ready_timeout, + local_input_address, + local_output_address, + } => Self::HandshakeOwner { + handshake_address: handshake_address.clone(), + advertised_host: advertised_host.clone(), + engine_count: *engine_count, + ready_timeout_secs: ready_timeout.as_secs_f64(), + local_input_address: local_input_address.clone(), + local_output_address: local_output_address.clone(), + }, + TransportMode::Bootstrapped { + input_address, + output_address, + engine_count, + ready_timeout, + } => Self::Bootstrapped { + input_address: input_address.clone(), + output_address: output_address.clone(), + engine_count: *engine_count, + ready_timeout_secs: ready_timeout.as_secs_f64(), + }, + } } } -fn coordinator_mode_config(coordinator_mode: &CoordinatorMode) -> Value { - match coordinator_mode { - CoordinatorMode::None => json!("none"), - CoordinatorMode::MaybeInProc => json!("maybe_in_proc"), - CoordinatorMode::External { address } => json!({ - "mode": "external", - "address": address, - }), +#[derive(Debug, Serialize)] +#[serde(untagged)] +enum CoordinatorModeInfo { + Named(CoordinatorModeName), + External { + mode: CoordinatorModeExternalTag, + address: String, + }, +} + +impl From<&CoordinatorMode> for CoordinatorModeInfo { + fn from(coordinator_mode: &CoordinatorMode) -> Self { + match coordinator_mode { + CoordinatorMode::None => Self::Named(CoordinatorModeName::None), + CoordinatorMode::MaybeInProc => Self::Named(CoordinatorModeName::MaybeInProc), + CoordinatorMode::External { address } => Self::External { + mode: CoordinatorModeExternalTag::External, + address: address.clone(), + }, + } } } -fn listener_mode_config(listener_mode: &HttpListenerMode) -> Value { - match listener_mode { - HttpListenerMode::BindTcp { host, port } => json!({ - "mode": "bind_tcp", - "host": host, - "port": port, - }), - HttpListenerMode::BindUnix { path } => json!({ - "mode": "bind_unix", - "path": path, - }), - HttpListenerMode::InheritedFd { fd } => json!({ - "mode": "inherited_fd", - "fd": fd, - }), +#[derive(Debug, Serialize)] +#[serde(rename_all = "snake_case")] +enum CoordinatorModeName { + None, + MaybeInProc, +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "snake_case")] +enum CoordinatorModeExternalTag { + External, +} + +#[derive(Debug, Serialize)] +#[serde(tag = "mode", rename_all = "snake_case")] +enum HttpListenerModeInfo { + BindTcp { host: String, port: u16 }, + BindUnix { path: String }, + InheritedFd { fd: i32 }, +} + +impl From<&HttpListenerMode> for HttpListenerModeInfo { + fn from(listener_mode: &HttpListenerMode) -> Self { + match listener_mode { + HttpListenerMode::BindTcp { host, port } => Self::BindTcp { + host: host.clone(), + port: *port, + }, + HttpListenerMode::BindUnix { path } => Self::BindUnix { path: path.clone() }, + HttpListenerMode::InheritedFd { fd } => Self::InheritedFd { fd: *fd }, + } } } +fn serialize_value(value: impl Serialize) -> Value { + serde_json::to_value(value).expect("server info value must serialize") +} + fn render_config_text(config: &Value) -> String { match config { Value::Object(fields) => fields From 10334b082b219780db21923abed01f0de7613066 Mon Sep 17 00:00:00 2001 From: Bugen Zhao Date: Wed, 3 Jun 2026 17:47:38 +0800 Subject: [PATCH 8/8] simplify by directly deriving `Serialize` on actual config structs Signed-off-by: Bugen Zhao --- rust/src/chat/src/parser/mod.rs | 4 +- rust/src/chat/src/renderer/hf/format.rs | 4 +- rust/src/chat/src/renderer/selection.rs | 4 +- rust/src/engine-core-client/src/client.rs | 3 +- rust/src/server/src/config.rs | 7 +- rust/src/server/src/routes/server_info.rs | 10 +- rust/src/server/src/routes/tests.rs | 40 +--- rust/src/server/src/server_info.rs | 259 +++------------------- rust/src/server/src/state.rs | 16 +- 9 files changed, 63 insertions(+), 284 deletions(-) diff --git a/rust/src/chat/src/parser/mod.rs b/rust/src/chat/src/parser/mod.rs index 52e83b3e0471..244a87cc7a76 100644 --- a/rust/src/chat/src/parser/mod.rs +++ b/rust/src/chat/src/parser/mod.rs @@ -6,10 +6,10 @@ use std::convert::Infallible; use std::fmt; use std::str::FromStr; -use serde_with::DeserializeFromStr; +use serde_with::{DeserializeFromStr, SerializeDisplay}; /// Specify which reasoning or tool-call parser implementation to use. -#[derive(Debug, Clone, PartialEq, Eq, Default, DeserializeFromStr)] +#[derive(Debug, Clone, PartialEq, Eq, Default, DeserializeFromStr, SerializeDisplay)] pub enum ParserSelection { /// Use model-based auto-detection. #[default] diff --git a/rust/src/chat/src/renderer/hf/format.rs b/rust/src/chat/src/renderer/hf/format.rs index a9b35d0f41c8..2c990fb37ba3 100644 --- a/rust/src/chat/src/renderer/hf/format.rs +++ b/rust/src/chat/src/renderer/hf/format.rs @@ -5,7 +5,7 @@ use std::str::FromStr; use minijinja::machinery::ast::{Expr, ForLoop, Set, Stmt}; use minijinja::machinery::{WhitespaceConfig, parse}; use minijinja::syntax::SyntaxConfig; -use serde_with::DeserializeFromStr; +use serde_with::{DeserializeFromStr, SerializeDisplay}; /// Chat template content format. #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] @@ -18,7 +18,7 @@ pub enum ChatTemplateContentFormat { } /// Configurable chat-template content format selection. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, DeserializeFromStr)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, DeserializeFromStr, SerializeDisplay)] pub enum ChatTemplateContentFormatOption { /// Detect the format from the template source. #[default] diff --git a/rust/src/chat/src/renderer/selection.rs b/rust/src/chat/src/renderer/selection.rs index f4bd565bafdd..cb22f95de0da 100644 --- a/rust/src/chat/src/renderer/selection.rs +++ b/rust/src/chat/src/renderer/selection.rs @@ -1,10 +1,10 @@ use std::fmt; use std::str::FromStr; -use serde_with::DeserializeFromStr; +use serde_with::{DeserializeFromStr, SerializeDisplay}; /// Specify which chat renderer implementation to use. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, DeserializeFromStr)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, DeserializeFromStr, SerializeDisplay)] pub enum RendererSelection { /// Use model-based auto-detection. #[default] diff --git a/rust/src/engine-core-client/src/client.rs b/rust/src/engine-core-client/src/client.rs index 1e6968555e7f..2a8c3c741884 100644 --- a/rust/src/engine-core-client/src/client.rs +++ b/rust/src/engine-core-client/src/client.rs @@ -2,6 +2,7 @@ use std::sync::Arc; use std::time::Duration; use futures::future::{join_all, try_join_all}; +use serde::Serialize; use tokio::sync::mpsc; use tokio_util::task::AbortOnDropHandle; use tracing::{debug, info, trace}; @@ -23,7 +24,7 @@ pub use stream::{EngineCoreOutputStream, EngineCoreStreamOutput}; /// How the frontend acquires its request/response transport with Python /// `EngineCoreProc`s. -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] pub enum TransportMode { /// The Rust process owns the startup handshake and allocates or binds the /// frontend transport addresses itself before replying to engine diff --git a/rust/src/server/src/config.rs b/rust/src/server/src/config.rs index 6853d9ccab99..f1599d18793f 100644 --- a/rust/src/server/src/config.rs +++ b/rust/src/server/src/config.rs @@ -2,12 +2,13 @@ use std::collections::HashMap; use std::time::Duration; use anyhow::Result; +use serde::Serialize; use serde_json::Value; use vllm_chat::{ChatTemplateContentFormatOption, ParserSelection, RendererSelection}; use vllm_engine_core_client::{CoordinatorMode as EngineCoreCoordinatorMode, TransportMode}; /// How the HTTP server obtains its listening socket. -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] pub enum HttpListenerMode { /// Bind a fresh TCP listener on the given host/port. BindTcp { host: String, port: u16 }, @@ -20,7 +21,7 @@ pub enum HttpListenerMode { /// Which coordinator implementation should be active when one is present for a /// frontend client. -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] pub enum CoordinatorMode { /// Do not run a coordinator at all. None, @@ -32,7 +33,7 @@ pub enum CoordinatorMode { } /// Normalized runtime configuration for the minimal OpenAI-compatible server. -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] pub struct Config { /// Frontend-to-engine transport setup. pub transport_mode: TransportMode, diff --git a/rust/src/server/src/routes/server_info.rs b/rust/src/server/src/routes/server_info.rs index 353885a9626c..aefb17a25faf 100644 --- a/rust/src/server/src/routes/server_info.rs +++ b/rust/src/server/src/routes/server_info.rs @@ -2,8 +2,9 @@ use std::sync::Arc; use axum::Json; use axum::extract::{Query, State}; +use axum::http::StatusCode; +use axum::response::{IntoResponse, Response}; use serde::Deserialize; -use serde_json::Value; use crate::server_info::ServerInfoConfigFormat; use crate::state::AppState; @@ -38,6 +39,9 @@ pub(crate) struct ServerInfoParams { pub async fn server_info( State(state): State>, Query(params): Query, -) -> Json { - Json(state.server_info_response(params.config_format.into())) +) -> Response { + match state.server_info_response(params.config_format.into()) { + Some(response) => Json(response).into_response(), + None => StatusCode::NOT_FOUND.into_response(), + } } diff --git a/rust/src/server/src/routes/tests.rs b/rust/src/server/src/routes/tests.rs index 80687c41e96f..a3e437e04802 100644 --- a/rust/src/server/src/routes/tests.rs +++ b/rust/src/server/src/routes/tests.rs @@ -1328,7 +1328,7 @@ async fn load_lora_adapter_registers_model_and_forwards_lora_request() { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[serial] -async fn server_info_endpoint_returns_text_config_by_default() { +async fn server_info_endpoint_returns_not_found_without_snapshot() { let mut app = test_app_with_dev_mode(true).await; let response = app .call( @@ -1340,43 +1340,7 @@ async fn server_info_endpoint_returns_text_config_by_default() { .await .expect("call app"); - assert_eq!(response.status(), StatusCode::OK); - let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body"); - let json: serde_json::Value = serde_json::from_slice(&body).expect("decode json"); - let config = json["vllm_config"].as_str().expect("text config"); - - assert!(config.contains("served_model_name=")); - assert!(!config.contains("Config {")); - assert!(json["vllm_env"].is_object()); - assert_eq!(json["system_env"]["arch"], std::env::consts::ARCH); - assert_eq!(json["system_env"]["family"], std::env::consts::FAMILY); - assert_eq!(json["system_env"]["os"], std::env::consts::OS); -} - -#[tokio::test(flavor = "multi_thread", worker_threads = 2)] -#[serial] -async fn server_info_endpoint_returns_json_config_when_requested() { - let mut app = test_app_with_dev_mode(true).await; - let response = app - .call( - Request::builder() - .uri("/server_info?config_format=json") - .body(Body::empty()) - .expect("build request"), - ) - .await - .expect("call app"); - - assert_eq!(response.status(), StatusCode::OK); - let body = to_bytes(response.into_body(), usize::MAX).await.expect("read body"); - let json: serde_json::Value = serde_json::from_slice(&body).expect("decode json"); - - assert_eq!( - json["vllm_config"]["served_model_name"], - json!(["Qwen/Qwen1.5-0.5B-Chat"]) - ); - assert!(json["vllm_env"].is_object()); - assert!(json["system_env"].is_object()); + assert_eq!(response.status(), StatusCode::NOT_FOUND); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] diff --git a/rust/src/server/src/server_info.rs b/rust/src/server/src/server_info.rs index 004dd46fb6cd..cca1b0f37954 100644 --- a/rust/src/server/src/server_info.rs +++ b/rust/src/server/src/server_info.rs @@ -1,10 +1,8 @@ -use std::collections::{BTreeMap, HashMap}; +use std::collections::BTreeMap; -use serde::Serialize; use serde_json::{Value, json}; -use vllm_engine_core_client::TransportMode; -use crate::config::{Config, CoordinatorMode, HttpListenerMode}; +use crate::config::Config; const SENSITIVE_VLLM_ENV_PATTERNS: &[&str] = &["KEY", "SECRET", "TOKEN", "PASSWORD", "CREDENTIAL", "AUTH"]; @@ -25,20 +23,10 @@ pub(crate) struct ServerInfoSnapshot { } impl ServerInfoSnapshot { - pub(crate) fn from_served_model_names(served_model_names: &[String]) -> Self { - let vllm_config_json = serialize_value(ServedModelNamesConfig { served_model_names }); - - Self { - vllm_config_text: render_config_text(&vllm_config_json), - vllm_config_json, - vllm_env: collect_vllm_env(), - system_env: collect_system_env(), - } - } - /// Capture the runtime configuration fields available to the Rust frontend. pub(crate) fn from_config(config: &Config) -> Self { - let vllm_config_json = serialize_value(RuntimeServerInfoConfig::from(config)); + let vllm_config_json = + serde_json::to_value(config).expect("server info value must serialize"); Self { vllm_config_text: render_config_text(&vllm_config_json), @@ -62,168 +50,6 @@ impl ServerInfoSnapshot { } } -#[derive(Debug, Serialize)] -struct ServedModelNamesConfig<'a> { - #[serde(rename = "served_model_name")] - served_model_names: &'a [String], -} - -#[derive(Debug, Serialize)] -struct RuntimeServerInfoConfig { - transport_mode: TransportModeInfo, - coordinator_mode: CoordinatorModeInfo, - model: String, - served_model_name: Vec, - listener_mode: HttpListenerModeInfo, - tool_call_parser: String, - reasoning_parser: String, - renderer: String, - chat_template: Option, - default_chat_template_kwargs: Option>, - chat_template_content_format: String, - enable_log_requests: bool, - disable_log_stats: bool, - grpc_port: Option, - shutdown_timeout_secs: f64, - engine_count: usize, -} - -impl From<&Config> for RuntimeServerInfoConfig { - fn from(config: &Config) -> Self { - Self { - transport_mode: TransportModeInfo::from(&config.transport_mode), - coordinator_mode: CoordinatorModeInfo::from(&config.coordinator_mode), - model: config.model.clone(), - served_model_name: config.served_model_name.clone(), - listener_mode: HttpListenerModeInfo::from(&config.listener_mode), - tool_call_parser: config.tool_call_parser.to_string(), - reasoning_parser: config.reasoning_parser.to_string(), - renderer: config.renderer.to_string(), - chat_template: config.chat_template.clone(), - default_chat_template_kwargs: config.default_chat_template_kwargs.clone(), - chat_template_content_format: config.chat_template_content_format.to_string(), - enable_log_requests: config.enable_log_requests, - disable_log_stats: config.disable_log_stats, - grpc_port: config.grpc_port, - shutdown_timeout_secs: config.shutdown_timeout.as_secs_f64(), - engine_count: config.engine_count(), - } - } -} - -#[derive(Debug, Serialize)] -#[serde(tag = "mode", rename_all = "snake_case")] -enum TransportModeInfo { - HandshakeOwner { - handshake_address: String, - advertised_host: String, - engine_count: usize, - ready_timeout_secs: f64, - local_input_address: Option, - local_output_address: Option, - }, - Bootstrapped { - input_address: String, - output_address: String, - engine_count: usize, - ready_timeout_secs: f64, - }, -} - -impl From<&TransportMode> for TransportModeInfo { - fn from(transport_mode: &TransportMode) -> Self { - match transport_mode { - TransportMode::HandshakeOwner { - handshake_address, - advertised_host, - engine_count, - ready_timeout, - local_input_address, - local_output_address, - } => Self::HandshakeOwner { - handshake_address: handshake_address.clone(), - advertised_host: advertised_host.clone(), - engine_count: *engine_count, - ready_timeout_secs: ready_timeout.as_secs_f64(), - local_input_address: local_input_address.clone(), - local_output_address: local_output_address.clone(), - }, - TransportMode::Bootstrapped { - input_address, - output_address, - engine_count, - ready_timeout, - } => Self::Bootstrapped { - input_address: input_address.clone(), - output_address: output_address.clone(), - engine_count: *engine_count, - ready_timeout_secs: ready_timeout.as_secs_f64(), - }, - } - } -} - -#[derive(Debug, Serialize)] -#[serde(untagged)] -enum CoordinatorModeInfo { - Named(CoordinatorModeName), - External { - mode: CoordinatorModeExternalTag, - address: String, - }, -} - -impl From<&CoordinatorMode> for CoordinatorModeInfo { - fn from(coordinator_mode: &CoordinatorMode) -> Self { - match coordinator_mode { - CoordinatorMode::None => Self::Named(CoordinatorModeName::None), - CoordinatorMode::MaybeInProc => Self::Named(CoordinatorModeName::MaybeInProc), - CoordinatorMode::External { address } => Self::External { - mode: CoordinatorModeExternalTag::External, - address: address.clone(), - }, - } - } -} - -#[derive(Debug, Serialize)] -#[serde(rename_all = "snake_case")] -enum CoordinatorModeName { - None, - MaybeInProc, -} - -#[derive(Debug, Serialize)] -#[serde(rename_all = "snake_case")] -enum CoordinatorModeExternalTag { - External, -} - -#[derive(Debug, Serialize)] -#[serde(tag = "mode", rename_all = "snake_case")] -enum HttpListenerModeInfo { - BindTcp { host: String, port: u16 }, - BindUnix { path: String }, - InheritedFd { fd: i32 }, -} - -impl From<&HttpListenerMode> for HttpListenerModeInfo { - fn from(listener_mode: &HttpListenerMode) -> Self { - match listener_mode { - HttpListenerMode::BindTcp { host, port } => Self::BindTcp { - host: host.clone(), - port: *port, - }, - HttpListenerMode::BindUnix { path } => Self::BindUnix { path: path.clone() }, - HttpListenerMode::InheritedFd { fd } => Self::InheritedFd { fd: *fd }, - } - } -} - -fn serialize_value(value: impl Serialize) -> Value { - serde_json::to_value(value).expect("server info value must serialize") -} - fn render_config_text(config: &Value) -> String { match config { Value::Object(fields) => fields @@ -263,13 +89,37 @@ fn collect_system_env() -> BTreeMap { #[cfg(test)] mod tests { - use std::time::Duration; + use std::collections::BTreeSet; - use vllm_chat::{ChatTemplateContentFormatOption, ParserSelection, RendererSelection}; - use vllm_engine_core_client::TransportMode; + use serde_json::{Value, json}; - use super::{ServerInfoConfigFormat, ServerInfoSnapshot, is_public_vllm_env_key}; - use crate::config::{Config, CoordinatorMode, HttpListenerMode}; + use super::{is_public_vllm_env_key, render_config_text}; + + #[test] + fn render_config_text_formats_config_snapshot() { + let rendered = render_config_text(&json!({ + "model": "test-model", + "served_model_name": ["served-model"], + "chat_template": null, + "enable_log_requests": true, + })); + let lines = rendered.lines().collect::>(); + + assert_eq!( + lines, + BTreeSet::from([ + "chat_template=None", + "enable_log_requests=true", + "model=test-model", + "served_model_name=[\"served-model\"]", + ]) + ); + assert_eq!( + render_config_text(&Value::String("inline".to_string())), + "inline" + ); + assert_eq!(render_config_text(&Value::Null), "None"); + } #[test] fn server_info_env_filter_excludes_sensitive_vllm_keys() { @@ -291,47 +141,4 @@ mod tests { assert!(is_public_vllm_env_key("VLLM_USE_MODELSCOPE")); assert!(!is_public_vllm_env_key("OTHER_ENV")); } - - #[test] - fn server_info_config_json_uses_stable_values() { - let config = Config { - transport_mode: TransportMode::Bootstrapped { - input_address: "tcp://127.0.0.1:0".to_string(), - output_address: "tcp://127.0.0.1:1".to_string(), - engine_count: 2, - ready_timeout: Duration::from_secs(5), - }, - coordinator_mode: CoordinatorMode::MaybeInProc, - model: "test-model".to_string(), - served_model_name: vec!["served-model".to_string()], - listener_mode: HttpListenerMode::BindTcp { - host: "127.0.0.1".to_string(), - port: 8000, - }, - tool_call_parser: ParserSelection::Auto, - reasoning_parser: ParserSelection::None, - renderer: RendererSelection::Hf, - chat_template: None, - default_chat_template_kwargs: None, - chat_template_content_format: ChatTemplateContentFormatOption::OpenAi, - enable_log_requests: true, - enable_request_id_headers: false, - disable_log_stats: false, - grpc_port: Some(9000), - shutdown_timeout: Duration::from_secs(10), - }; - - let snapshot = ServerInfoSnapshot::from_config(&config); - let response = snapshot.response(ServerInfoConfigFormat::Json); - let vllm_config = &response["vllm_config"]; - - assert_eq!(vllm_config["transport_mode"]["mode"], "bootstrapped"); - assert_eq!(vllm_config["transport_mode"]["engine_count"], 2); - assert_eq!(vllm_config["coordinator_mode"], "maybe_in_proc"); - assert_eq!(vllm_config["listener_mode"]["mode"], "bind_tcp"); - assert_eq!(vllm_config["tool_call_parser"], "auto"); - assert_eq!(vllm_config["reasoning_parser"], "none"); - assert_eq!(vllm_config["renderer"], "hf"); - assert_eq!(vllm_config["chat_template_content_format"], "openai"); - } } diff --git a/rust/src/server/src/state.rs b/rust/src/server/src/state.rs index fa744914cb66..c73ca04c5d62 100644 --- a/rust/src/server/src/state.rs +++ b/rust/src/server/src/state.rs @@ -25,8 +25,8 @@ pub struct AppState { pub enable_log_requests: bool, /// Whether to set X-Request-Id on every HTTP response. pub enable_request_id_headers: bool, - /// Runtime server information returned by `/server_info`. - server_info: ServerInfoSnapshot, + /// Runtime server information returned by `/server_info`, when available. + server_info: Option, /// Number of in-flight inference requests currently owned by this frontend. server_load: AtomicU64, /// Dynamic LoRA adapter registry. @@ -47,13 +47,12 @@ impl AppState { !served_model_names.is_empty(), "served_model_names must not be empty" ); - let server_info = ServerInfoSnapshot::from_served_model_names(&served_model_names); Self { served_model_names, chat, enable_log_requests: false, enable_request_id_headers: false, - server_info, + server_info: None, server_load: AtomicU64::new(0), lora_manager: LoraManager::new(), } @@ -73,13 +72,16 @@ impl AppState { /// Attach the runtime server information snapshot used by `/server_info`. pub(crate) fn with_server_info(mut self, server_info: ServerInfoSnapshot) -> Self { - self.server_info = server_info; + self.server_info = Some(server_info); self } /// Build a `/server_info` response payload. - pub(crate) fn server_info_response(&self, config_format: ServerInfoConfigFormat) -> Value { - self.server_info.response(config_format) + pub(crate) fn server_info_response( + &self, + config_format: ServerInfoConfigFormat, + ) -> Option { + self.server_info.as_ref().map(|server_info| server_info.response(config_format)) } /// The primary model name echoed back in API responses (the first served