From 9628a8ac4626c5cb5cd9411f3a8b666dfdf908eb Mon Sep 17 00:00:00 2001 From: nnshah1 Date: Wed, 10 Jun 2026 13:27:43 -0700 Subject: [PATCH 1/8] feat(frontend): add DYN_ENABLE_NVEXT and DYN_ENABLE_FRONTEND_ADMIN_API master switches MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses PSIRT findings 6067559/6067562 (unauthenticated busy_threshold) and the 6067564 by-design routing-header surface by giving operators two binary master switches that fully close off the nvext extension protocol and the frontend admin API. DYN_ENABLE_NVEXT (default true) When false at frontend startup, the chat/completions/responses handlers drop request.nvext at the boundary and ignore the routing-override headers (x-worker-instance-id, x-prefill-instance-id, x-dp-rank, x-prefill-dp-rank). Response-side extra_fields opt-in is silently ignored. This shuts off worker-targeting via header AND body in one switch. Compatibility: disables EPP/GAIE serving, Prime-RL-style training that uses nvext.cache_salt for KV isolation, multi-tenant agent platforms that forward agent_hints/agent_context, and clients that opt into response disclosure via nvext.extra_fields. DYN_ENABLE_FRONTEND_ADMIN_API (default true) When false, the GET /busy_threshold and POST /busy_threshold routes are not registered. Inference, metrics, models, health, and liveness routes are unaffected. (The historical POST /clear_kv_blocks route was already unwired in #1629; not re-exposed.) Design notes - Both flags are read once at HttpService::run() via env_is_falsey on DYN_ENABLE_NVEXT / DYN_ENABLE_FRONTEND_ADMIN_API, ANDed with the HttpServiceConfig builder field. Env-falsey only — unset env keeps the default-true behavior intact. - State carries the nvext_enabled flag as a plain bool field set at construction; handlers read it via state.nvext_enabled() — single struct-field load on the hot path, no atomic, no per-request env lookup, no OnceLock. Matches the existing pattern used for StateFlags + the enable_rl_router gate around line 698. - The admin-api gate is purely at the route-registration site, so disabled mode means the route literally doesn't exist (404) rather than running a handler that 503s. Tested with cargo check and cargo clippy --no-deps on dynamo-llm. Signed-off-by: nnshah1 --- lib/llm/src/http/service/openai.rs | 33 +++++++- lib/llm/src/http/service/service_v2.rs | 85 ++++++++++++++++++++- lib/runtime/src/config/environment_names.rs | 29 +++++++ 3 files changed, 140 insertions(+), 7 deletions(-) diff --git a/lib/llm/src/http/service/openai.rs b/lib/llm/src/http/service/openai.rs index 4f14cb6845c1..1f9fb3167eda 100644 --- a/lib/llm/src/http/service/openai.rs +++ b/lib/llm/src/http/service/openai.rs @@ -551,7 +551,16 @@ async fn handler_completions( check_ready(&state)?; check_model_serving_ready(&state, &request.inner.model)?; - request.nvext = apply_header_routing_overrides(request.nvext.take(), &headers); + // DYN_ENABLE_NVEXT master switch (default on). When the operator turns + // it off, drop any client-supplied nvext and ignore routing-override + // headers — closes off the entire nvext surface for default-secure + // deployments. Read from State (set once at service construction), + // so this is a single bool deref, no atomic, on the hot path. + if state.nvext_enabled() { + request.nvext = apply_header_routing_overrides(request.nvext.take(), &headers); + } else { + request.nvext = None; + } // create the context for the request let request_id = get_or_create_request_id(&headers); @@ -1142,7 +1151,16 @@ async fn handler_chat_completions( check_model_serving_ready(&state, resolved_model)?; } - request.nvext = apply_header_routing_overrides(request.nvext.take(), &headers); + // DYN_ENABLE_NVEXT master switch (default on). When the operator turns + // it off, drop any client-supplied nvext and ignore routing-override + // headers — closes off the entire nvext surface for default-secure + // deployments. Read from State (set once at service construction), + // so this is a single bool deref, no atomic, on the hot path. + if state.nvext_enabled() { + request.nvext = apply_header_routing_overrides(request.nvext.take(), &headers); + } else { + request.nvext = None; + } // create the context for the request let request_id = get_or_create_request_id(&headers); @@ -1891,7 +1909,16 @@ async fn handler_responses( check_model_serving_ready(&state, resolved_model)?; } - request.nvext = apply_header_routing_overrides(request.nvext.take(), &headers); + // DYN_ENABLE_NVEXT master switch (default on). When the operator turns + // it off, drop any client-supplied nvext and ignore routing-override + // headers — closes off the entire nvext surface for default-secure + // deployments. Read from State (set once at service construction), + // so this is a single bool deref, no atomic, on the hot path. + if state.nvext_enabled() { + request.nvext = apply_header_routing_overrides(request.nvext.take(), &headers); + } else { + request.nvext = None; + } // create the context for the request let request_id = get_or_create_request_id(&headers); diff --git a/lib/llm/src/http/service/service_v2.rs b/lib/llm/src/http/service/service_v2.rs index f07b309224c9..c2385fee959b 100644 --- a/lib/llm/src/http/service/service_v2.rs +++ b/lib/llm/src/http/service/service_v2.rs @@ -26,7 +26,7 @@ use anyhow::Result; use axum_server::tls_rustls::RustlsConfig; use derive_builder::Builder; use dynamo_runtime::DistributedRuntime; -use dynamo_runtime::config::env_is_truthy; +use dynamo_runtime::config::{env_is_falsey, env_is_truthy}; use dynamo_runtime::config::environment_names::llm as env_llm; use dynamo_runtime::discovery::Discovery; use dynamo_runtime::logging::{make_inference_request_span, make_system_request_span}; @@ -61,6 +61,11 @@ pub struct State { discovery_client: Arc, flags: StateFlags, cancel_token: CancellationToken, + /// When false, the chat / completions / responses handlers drop + /// `request.nvext` at the boundary and ignore the routing-override + /// headers. Read once at construction from `DYN_ENABLE_NVEXT` (default + /// `true`); never mutated after that, so a plain `bool` suffices. + nvext_enabled: bool, } #[derive(Default, Debug)] @@ -131,11 +136,24 @@ impl State { manager: Arc, discovery_client: Arc, cancel_token: CancellationToken, + ) -> Self { + Self::new_with_flags(manager, discovery_client, cancel_token, true) + } + + /// Constructor that lets the caller seed the runtime-immutable + /// `nvext_enabled` master switch. `State::new` defaults it to `true` + /// for callers that don't care. + pub fn new_with_flags( + manager: Arc, + discovery_client: Arc, + cancel_token: CancellationToken, + nvext_enabled: bool, ) -> Self { Self { manager, metrics: Arc::new(Metrics::default()), discovery_client, + nvext_enabled, flags: StateFlags { chat_endpoints_enabled: AtomicBool::new(false), cmpl_endpoints_enabled: AtomicBool::new(false), @@ -173,6 +191,14 @@ impl State { self.cancel_token.is_cancelled() } + /// Whether the `nvext` extension protocol is active for this service. + /// `false` ⇒ chat / completions / responses handlers drop `request.nvext` + /// and ignore routing-override headers (`DYN_ENABLE_NVEXT=false`). + #[inline] + pub fn nvext_enabled(&self) -> bool { + self.nvext_enabled + } + /// Get the cancellation token pub fn cancel_token(&self) -> &CancellationToken { &self.cancel_token @@ -279,6 +305,20 @@ pub struct HttpServiceConfig { #[builder(default = "false")] enable_rl: bool, + /// Master switch for the `nvext` extension protocol. Default `true`. + /// Mirrored via the `DYN_ENABLE_NVEXT` env var at service-construction + /// time (env-falsy ⇒ disabled). When disabled, the frontend handlers + /// drop `request.nvext` and ignore routing-override headers. + #[builder(default = "true")] + enable_nvext: bool, + + /// Master switch for the frontend's HTTP admin API surface + /// (`POST/GET /busy_threshold`, `POST /clear_kv_blocks`). Default `true`. + /// Mirrored via `DYN_ENABLE_FRONTEND_ADMIN_API`. When disabled, those + /// routes are not registered. + #[builder(default = "true")] + enable_admin_api: bool, + /// Port for the RL worker discovery listener. Defaults to `DYN_RL_PORT` or 8001. #[builder(default = "default_rl_port()")] rl_port: u16, @@ -567,7 +607,23 @@ impl HttpServiceConfigBuilder { cancel_token.child_token(), )) as Arc }); - let state = Arc::new(State::new(model_manager, discovery_client, cancel_token)); + // Env-var mirrors for the two master switches. Read once at + // service-construction time so the per-request hot path is a single + // struct-field load (no env::var on every request, no atomics). + // `enable_nvext` / `enable_admin_api` come from the builder + // (Python frontend_args sets them from CLI); the env vars give + // operators a way to flip the switch without a CLI change. + let nvext_enabled = config.enable_nvext + && !env_is_falsey(env_llm::DYN_ENABLE_NVEXT); + let admin_api_enabled = config.enable_admin_api + && !env_is_falsey(env_llm::DYN_ENABLE_FRONTEND_ADMIN_API); + + let state = Arc::new(State::new_with_flags( + model_manager, + discovery_client, + cancel_token, + nvext_enabled, + )); state .flags .set(&EndpointType::Chat, config.enable_chat_endpoints); @@ -641,7 +697,7 @@ impl HttpServiceConfigBuilder { }; // System routes (health, metrics, models) — debug-level spans - let system_routes = vec![ + let mut system_routes = vec![ metrics::router( registry, var(HTTP_SVC_METRICS_PATH_ENV).ok(), @@ -657,8 +713,29 @@ impl HttpServiceConfigBuilder { }, super::health::health_check_router(state.clone(), var(HTTP_SVC_HEALTH_PATH_ENV).ok()), super::health::live_check_router(state.clone(), var(HTTP_SVC_LIVE_PATH_ENV).ok()), - super::busy_threshold::busy_threshold_router(state.clone(), None), ]; + if admin_api_enabled { + // GET /busy_threshold + POST /busy_threshold are the + // cluster-mutating admin surface on the frontend. Gated under a + // single switch so operators can close off "out-of-band + // scheduler / worker control" without affecting inference, + // metrics, models, or health/liveness probes. + // + // Note: the historical `POST /clear_kv_blocks` route was removed + // in PR #1629 ("fix: remove http endpoint for clearing kv + // blocks"); the handler module is orphaned and not re-exposed + // here. + system_routes.push(super::busy_threshold::busy_threshold_router( + state.clone(), + None, + )); + } else { + tracing::info!( + "frontend admin API disabled — busy_threshold routes not \ + registered (DYN_ENABLE_FRONTEND_ADMIN_API=false or \ + builder enable_admin_api=false)" + ); + } let mut system_router = axum::Router::new(); for (route_docs, route) in system_routes { system_router = system_router.merge(route); diff --git a/lib/runtime/src/config/environment_names.rs b/lib/runtime/src/config/environment_names.rs index de60b53e46c3..89e44bb8ee4b 100644 --- a/lib/runtime/src/config/environment_names.rs +++ b/lib/runtime/src/config/environment_names.rs @@ -284,6 +284,35 @@ pub mod llm { /// Enable the experimental Anthropic Messages API endpoint (/v1/messages) pub const DYN_ENABLE_ANTHROPIC_API: &str = "DYN_ENABLE_ANTHROPIC_API"; + /// Master switch for the `nvext` extension protocol on the frontend. + /// + /// Default `true` — preserves current behavior. When set to a falsy value + /// (`0` / `false` / `no` / `off`, case-insensitive), the frontend drops + /// `request.nvext` at handler entry and ignores the routing-override + /// headers (`x-worker-instance-id`, `x-prefill-instance-id`, `x-dp-rank`, + /// `x-prefill-dp-rank`). Response-side `extra_fields` opt-in is also + /// silently ignored. Useful as a default-secure stance for deployments + /// that do not run an EPP-fronted GAIE protocol. + /// + /// Compatibility note: disabling this breaks EPP / GAIE serving, + /// Prime-RL-style training that relies on `nvext.cache_salt` for KV cache + /// isolation, multi-tenant agent platforms that forward + /// `nvext.agent_hints` / `nvext.agent_context`, and clients that opt into + /// response disclosure via `nvext.extra_fields`. + pub const DYN_ENABLE_NVEXT: &str = "DYN_ENABLE_NVEXT"; + + /// Master switch for the frontend's HTTP admin API surface. + /// + /// Default `true` — preserves current behavior. When set to a falsy value, + /// the frontend does not register routes that mutate or read worker / + /// scheduler state from outside the request plane: + /// - `POST /busy_threshold` (set busy thresholds on workers) + /// - `GET /busy_threshold` (read current busy thresholds) + /// - `POST /clear_kv_blocks` (clear all KV cache blocks on workers) + /// + /// Inference, metrics, models, health, and liveness routes are unaffected. + pub const DYN_ENABLE_FRONTEND_ADMIN_API: &str = "DYN_ENABLE_FRONTEND_ADMIN_API"; + /// Strip the Claude Code billing preamble (`x-anthropic-billing-header: ...`) /// from the system prompt before forwarding to the target model. The preamble /// varies per session and per release, wasting tokens and breaking prompt caching. From c54df2ff6c4d8e39b6ad59df2b840266815c20ad Mon Sep 17 00:00:00 2001 From: nnshah1 Date: Wed, 10 Jun 2026 15:37:15 -0700 Subject: [PATCH 2/8] review fixes for #10556 - Drop POST /clear_kv_blocks from env-var docs (it's not a registered route) and delete the orphaned lib/llm/src/http/service/clear_kv_blocks.rs source file (dead since PR #1629 removed the route registration; no mod declaration, so it wasn't even compiled). - Rename State::new_with_flags -> State::new_with_nvext_enabled (only one flag for now). - Collapse the 6-line copy-pasted comment block at the 3 openai.rs handler call sites to a one-liner; rewrite the if/else as an expression assignment. - Trim const docs in environment_names.rs to behavior + default; the EPP/GAIE/Prime-RL compatibility paragraph is design rationale and lives in the PR description, not the public env-var reference. - Trim the env-mirror comment, the field/accessor doc duplication, and switch the admin-API-disabled log to structured fields. - Add two tests: enable_nvext propagates through the builder onto State.nvext_enabled (default + on + off), and enable_admin_api=false causes GET /busy_threshold to 404 while /live still serves 200. Signed-off-by: nnshah1 --- lib/llm/src/http/service/clear_kv_blocks.rs | 238 -------------------- lib/llm/src/http/service/openai.rs | 42 ++-- lib/llm/src/http/service/service_v2.rs | 123 ++++++---- lib/runtime/src/config/environment_names.rs | 32 +-- 4 files changed, 101 insertions(+), 334 deletions(-) delete mode 100644 lib/llm/src/http/service/clear_kv_blocks.rs diff --git a/lib/llm/src/http/service/clear_kv_blocks.rs b/lib/llm/src/http/service/clear_kv_blocks.rs deleted file mode 100644 index dcd4352ffb17..000000000000 --- a/lib/llm/src/http/service/clear_kv_blocks.rs +++ /dev/null @@ -1,238 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -use super::{service_v2, RouteDoc}; -use axum::{http::Method, response::IntoResponse, routing::post, Json, Router}; -use serde_json::json; -use std::sync::Arc; - -use dynamo_runtime::{discovery::DiscoveryQuery, pipeline::PushRouter, stream::StreamExt}; - -pub const CLEAR_KV_ENDPOINT: &str = "clear_kv_blocks"; - -pub fn clear_kv_blocks_router( - state: Arc, - path: Option, -) -> (Vec, Router) { - let path = path.unwrap_or_else(|| "/clear_kv_blocks".to_string()); - - let docs: Vec = vec![RouteDoc::new(Method::POST, &path)]; - - let router = Router::new() - .route(&path, post(clear_kv_blocks_handler)) - .with_state(state); - - (docs, router) -} - -async fn clear_kv_blocks_handler( - axum::extract::State(state): axum::extract::State>, -) -> impl IntoResponse { - let model_entries = state.manager().get_model_entries(); - - // if there are no active workers - if model_entries.is_empty() { - return Json(serde_json::json!({ - "message": "No active worker groups found" - })); - } - - let distributed = match state.runtime() { - Some(runtime) => runtime, - None => { - return Json(serde_json::json!({ - "message": "Failed to create distributed runtime", - })); - } - }; - - let mut cleared_workers = Vec::new(); - let mut failed_workers = Vec::new(); - - // update cleared and failed workers - let mut add_worker_result = |success: bool, - name: String, - status: &str, - ns: &str, - comp: &str, - message: Option| { - let mut result = json!({ - "name": name, - "endpoint": format!("{}/{}/{}", ns, comp, CLEAR_KV_ENDPOINT), - "status": status, - }); - if success { - if let Some(m) = message { - result["response"] = json!(m); - } - cleared_workers.push(result); - } else { - if let Some(m) = message { - result["error"] = json!(m); - } - failed_workers.push(result); - } - }; - - // create client for each model entry - for entry in &model_entries { - let namespace = &entry.endpoint_id.namespace; - let component = &entry.endpoint_id.component; - let entry_name = entry.name.to_string(); - - tracing::debug!("Processing worker group: {}/{}", namespace, component); - - let namespace_obj = match distributed.namespace(namespace) { - Ok(ns) => ns, - Err(e) => { - add_worker_result( - false, - entry_name, - "Failed to get namespace", - namespace, - component, - Some(e.to_string()), - ); - continue; - } - }; - - let component_obj = match namespace_obj.component(component) { - Ok(comp) => comp, - Err(e) => { - add_worker_result( - false, - entry_name, - "Failed to get component", - namespace, - component, - Some(e.to_string()), - ); - continue; - } - }; - - let endpoint: dynamo_runtime::component::Endpoint = - component_obj.endpoint(CLEAR_KV_ENDPOINT); - - let client = match endpoint.client().await { - Ok(c) => c, - Err(e) => { - add_worker_result( - false, - entry_name, - "Failed to get client", - namespace, - component, - Some(e.to_string()), - ); - continue; - } - }; - - let router = match PushRouter::<(), serde_json::Value>::from_client( - client.clone(), - Default::default(), - ) - .await - { - Ok(r) => r, - Err(e) => { - add_worker_result( - false, - entry_name, - "Failed to create router", - namespace, - component, - Some(e.to_string()), - ); - continue; - } - }; - - let discovery_client = distributed.discovery(); - let discovery_key = DiscoveryQuery::Endpoint { - namespace: namespace.clone(), - component: component.clone(), - endpoint: CLEAR_KV_ENDPOINT.to_string(), - }; - - let discovery_instances = match discovery_client.list(discovery_key).await { - Ok(instances) => instances, - Err(e) => { - add_worker_result( - false, - entry_name, - "Failed to get instances for worker group", - namespace, - component, - Some(e.to_string()), - ); - continue; - } - }; - - if discovery_instances.is_empty() { - add_worker_result( - false, - entry_name, - "No instances found for clear_kv_blocks endpoint", - namespace, - component, - None, - ); - continue; - } - - let instances_filtered: Vec = discovery_instances - .into_iter() - .filter_map(|di| match di { - dynamo_runtime::discovery::DiscoveryInstance::Endpoint(instance) => Some(instance), - _ => None, - }) - .collect(); - - for instance in &instances_filtered { - let instance_name = format!("{}-instance-{}", entry.name, instance.id()); - match router.direct(().into(), instance.id()).await { - Ok(mut stream) => match stream.next().await { - Some(response) => { - add_worker_result( - true, - instance_name, - "Successfully cleared kv blocks for instance", - namespace, - component, - Some(response.to_string()), - ); - } - None => { - add_worker_result( - false, - instance_name, - "No response from instance", - namespace, - component, - None, - ); - } - }, - Err(e) => { - add_worker_result( - false, - instance_name, - "Failed to send request for instance", - namespace, - component, - Some(e.to_string()), - ); - } - } - } - } - - Json(serde_json::json!({ - "cleared_workers": cleared_workers, - "failed_workers": failed_workers - })) -} diff --git a/lib/llm/src/http/service/openai.rs b/lib/llm/src/http/service/openai.rs index 1f9fb3167eda..1b789a02a07e 100644 --- a/lib/llm/src/http/service/openai.rs +++ b/lib/llm/src/http/service/openai.rs @@ -551,16 +551,12 @@ async fn handler_completions( check_ready(&state)?; check_model_serving_ready(&state, &request.inner.model)?; - // DYN_ENABLE_NVEXT master switch (default on). When the operator turns - // it off, drop any client-supplied nvext and ignore routing-override - // headers — closes off the entire nvext surface for default-secure - // deployments. Read from State (set once at service construction), - // so this is a single bool deref, no atomic, on the hot path. - if state.nvext_enabled() { - request.nvext = apply_header_routing_overrides(request.nvext.take(), &headers); + // nvext master switch — see State::nvext_enabled. + request.nvext = if state.nvext_enabled() { + apply_header_routing_overrides(request.nvext.take(), &headers) } else { - request.nvext = None; - } + None + }; // create the context for the request let request_id = get_or_create_request_id(&headers); @@ -1151,16 +1147,12 @@ async fn handler_chat_completions( check_model_serving_ready(&state, resolved_model)?; } - // DYN_ENABLE_NVEXT master switch (default on). When the operator turns - // it off, drop any client-supplied nvext and ignore routing-override - // headers — closes off the entire nvext surface for default-secure - // deployments. Read from State (set once at service construction), - // so this is a single bool deref, no atomic, on the hot path. - if state.nvext_enabled() { - request.nvext = apply_header_routing_overrides(request.nvext.take(), &headers); + // nvext master switch — see State::nvext_enabled. + request.nvext = if state.nvext_enabled() { + apply_header_routing_overrides(request.nvext.take(), &headers) } else { - request.nvext = None; - } + None + }; // create the context for the request let request_id = get_or_create_request_id(&headers); @@ -1909,16 +1901,12 @@ async fn handler_responses( check_model_serving_ready(&state, resolved_model)?; } - // DYN_ENABLE_NVEXT master switch (default on). When the operator turns - // it off, drop any client-supplied nvext and ignore routing-override - // headers — closes off the entire nvext surface for default-secure - // deployments. Read from State (set once at service construction), - // so this is a single bool deref, no atomic, on the hot path. - if state.nvext_enabled() { - request.nvext = apply_header_routing_overrides(request.nvext.take(), &headers); + // nvext master switch — see State::nvext_enabled. + request.nvext = if state.nvext_enabled() { + apply_header_routing_overrides(request.nvext.take(), &headers) } else { - request.nvext = None; - } + None + }; // create the context for the request let request_id = get_or_create_request_id(&headers); diff --git a/lib/llm/src/http/service/service_v2.rs b/lib/llm/src/http/service/service_v2.rs index c2385fee959b..229a20c14aff 100644 --- a/lib/llm/src/http/service/service_v2.rs +++ b/lib/llm/src/http/service/service_v2.rs @@ -26,8 +26,8 @@ use anyhow::Result; use axum_server::tls_rustls::RustlsConfig; use derive_builder::Builder; use dynamo_runtime::DistributedRuntime; -use dynamo_runtime::config::{env_is_falsey, env_is_truthy}; use dynamo_runtime::config::environment_names::llm as env_llm; +use dynamo_runtime::config::{env_is_falsey, env_is_truthy}; use dynamo_runtime::discovery::Discovery; use dynamo_runtime::logging::{make_inference_request_span, make_system_request_span}; use dynamo_runtime::metrics::{ @@ -61,10 +61,6 @@ pub struct State { discovery_client: Arc, flags: StateFlags, cancel_token: CancellationToken, - /// When false, the chat / completions / responses handlers drop - /// `request.nvext` at the boundary and ignore the routing-override - /// headers. Read once at construction from `DYN_ENABLE_NVEXT` (default - /// `true`); never mutated after that, so a plain `bool` suffices. nvext_enabled: bool, } @@ -137,13 +133,10 @@ impl State { discovery_client: Arc, cancel_token: CancellationToken, ) -> Self { - Self::new_with_flags(manager, discovery_client, cancel_token, true) + Self::new_with_nvext_enabled(manager, discovery_client, cancel_token, true) } - /// Constructor that lets the caller seed the runtime-immutable - /// `nvext_enabled` master switch. `State::new` defaults it to `true` - /// for callers that don't care. - pub fn new_with_flags( + pub fn new_with_nvext_enabled( manager: Arc, discovery_client: Arc, cancel_token: CancellationToken, @@ -191,9 +184,8 @@ impl State { self.cancel_token.is_cancelled() } - /// Whether the `nvext` extension protocol is active for this service. - /// `false` ⇒ chat / completions / responses handlers drop `request.nvext` - /// and ignore routing-override headers (`DYN_ENABLE_NVEXT=false`). + /// Master switch for the `nvext` extension protocol (see + /// [`environment_names::llm::DYN_ENABLE_NVEXT`]). #[inline] pub fn nvext_enabled(&self) -> bool { self.nvext_enabled @@ -305,17 +297,14 @@ pub struct HttpServiceConfig { #[builder(default = "false")] enable_rl: bool, - /// Master switch for the `nvext` extension protocol. Default `true`. - /// Mirrored via the `DYN_ENABLE_NVEXT` env var at service-construction - /// time (env-falsy ⇒ disabled). When disabled, the frontend handlers - /// drop `request.nvext` and ignore routing-override headers. + /// Master switch for the `nvext` extension protocol. Default `true`, + /// env-falsey on `DYN_ENABLE_NVEXT` overrides to `false`. #[builder(default = "true")] enable_nvext: bool, - /// Master switch for the frontend's HTTP admin API surface - /// (`POST/GET /busy_threshold`, `POST /clear_kv_blocks`). Default `true`. - /// Mirrored via `DYN_ENABLE_FRONTEND_ADMIN_API`. When disabled, those - /// routes are not registered. + /// Master switch for the frontend admin API surface (`GET` / + /// `POST /busy_threshold`). Default `true`, env-falsey on + /// `DYN_ENABLE_FRONTEND_ADMIN_API` overrides to `false`. #[builder(default = "true")] enable_admin_api: bool, @@ -607,18 +596,12 @@ impl HttpServiceConfigBuilder { cancel_token.child_token(), )) as Arc }); - // Env-var mirrors for the two master switches. Read once at - // service-construction time so the per-request hot path is a single - // struct-field load (no env::var on every request, no atomics). - // `enable_nvext` / `enable_admin_api` come from the builder - // (Python frontend_args sets them from CLI); the env vars give - // operators a way to flip the switch without a CLI change. - let nvext_enabled = config.enable_nvext - && !env_is_falsey(env_llm::DYN_ENABLE_NVEXT); - let admin_api_enabled = config.enable_admin_api - && !env_is_falsey(env_llm::DYN_ENABLE_FRONTEND_ADMIN_API); - - let state = Arc::new(State::new_with_flags( + // Env-falsey overrides the builder; unset preserves the builder default. + let nvext_enabled = config.enable_nvext && !env_is_falsey(env_llm::DYN_ENABLE_NVEXT); + let admin_api_enabled = + config.enable_admin_api && !env_is_falsey(env_llm::DYN_ENABLE_FRONTEND_ADMIN_API); + + let state = Arc::new(State::new_with_nvext_enabled( model_manager, discovery_client, cancel_token, @@ -715,25 +698,14 @@ impl HttpServiceConfigBuilder { super::health::live_check_router(state.clone(), var(HTTP_SVC_LIVE_PATH_ENV).ok()), ]; if admin_api_enabled { - // GET /busy_threshold + POST /busy_threshold are the - // cluster-mutating admin surface on the frontend. Gated under a - // single switch so operators can close off "out-of-band - // scheduler / worker control" without affecting inference, - // metrics, models, or health/liveness probes. - // - // Note: the historical `POST /clear_kv_blocks` route was removed - // in PR #1629 ("fix: remove http endpoint for clearing kv - // blocks"); the handler module is orphaned and not re-exposed - // here. system_routes.push(super::busy_threshold::busy_threshold_router( state.clone(), None, )); } else { tracing::info!( - "frontend admin API disabled — busy_threshold routes not \ - registered (DYN_ENABLE_FRONTEND_ADMIN_API=false or \ - builder enable_admin_api=false)" + env = env_llm::DYN_ENABLE_FRONTEND_ADMIN_API, + "frontend admin API disabled — busy_threshold routes not registered" ); } let mut system_router = axum::Router::new(); @@ -938,4 +910,63 @@ mod tests { // Clean up handle.abort(); } + + /// `enable_admin_api=false` ⇒ `GET /busy_threshold` is not registered and + /// returns 404, not 503 or 405. Inference is unaffected (covered by other + /// tests). + #[tokio::test] + async fn test_admin_api_disabled_404s_busy_threshold() { + let cancel_token = Arc::new(CancellationToken::new()); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + let service = HttpService::builder() + .port(port) + .enable_admin_api(false) + .build() + .unwrap(); + + let service_token = cancel_token.clone(); + let handle = tokio::spawn(async move { + service + .run_with_listener((*service_token).clone(), listener) + .await + .unwrap(); + }); + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + + let resp = reqwest::Client::new() + .get(format!("http://localhost:{}/busy_threshold", port)) + .send() + .await + .expect("request failed"); + assert_eq!(resp.status(), reqwest::StatusCode::NOT_FOUND); + + // And /live still works (sanity: only the admin surface is gated). + let live = reqwest::Client::new() + .get(format!("http://localhost:{}/live", port)) + .send() + .await + .expect("request failed"); + assert_eq!(live.status(), reqwest::StatusCode::OK); + + cancel_token.cancel(); + handle.abort(); + } + + /// `enable_nvext` is wired from the builder onto `State.nvext_enabled` and + /// exposed via the accessor used by the openai handlers. + #[test] + fn test_enable_nvext_propagates_through_builder_to_state() { + let on = HttpService::builder().enable_nvext(true).build().unwrap(); + assert!(on.state.nvext_enabled()); + + let off = HttpService::builder().enable_nvext(false).build().unwrap(); + assert!(!off.state.nvext_enabled()); + + let default = HttpService::builder().build().unwrap(); + assert!( + default.state.nvext_enabled(), + "default should preserve current behavior (nvext on)" + ); + } } diff --git a/lib/runtime/src/config/environment_names.rs b/lib/runtime/src/config/environment_names.rs index 89e44bb8ee4b..c7cfb455dc7e 100644 --- a/lib/runtime/src/config/environment_names.rs +++ b/lib/runtime/src/config/environment_names.rs @@ -285,32 +285,18 @@ pub mod llm { pub const DYN_ENABLE_ANTHROPIC_API: &str = "DYN_ENABLE_ANTHROPIC_API"; /// Master switch for the `nvext` extension protocol on the frontend. - /// - /// Default `true` — preserves current behavior. When set to a falsy value - /// (`0` / `false` / `no` / `off`, case-insensitive), the frontend drops - /// `request.nvext` at handler entry and ignores the routing-override - /// headers (`x-worker-instance-id`, `x-prefill-instance-id`, `x-dp-rank`, - /// `x-prefill-dp-rank`). Response-side `extra_fields` opt-in is also - /// silently ignored. Useful as a default-secure stance for deployments - /// that do not run an EPP-fronted GAIE protocol. - /// - /// Compatibility note: disabling this breaks EPP / GAIE serving, - /// Prime-RL-style training that relies on `nvext.cache_salt` for KV cache - /// isolation, multi-tenant agent platforms that forward - /// `nvext.agent_hints` / `nvext.agent_context`, and clients that opt into - /// response disclosure via `nvext.extra_fields`. + /// Default `true`. Falsy values (`0` / `false` / `no` / `off`, + /// case-insensitive) cause the frontend to drop `request.nvext` at + /// handler entry, ignore the routing-override headers + /// (`x-worker-instance-id`, `x-prefill-instance-id`, `x-dp-rank`, + /// `x-prefill-dp-rank`), and silently ignore the response-side + /// `extra_fields` opt-in. pub const DYN_ENABLE_NVEXT: &str = "DYN_ENABLE_NVEXT"; /// Master switch for the frontend's HTTP admin API surface. - /// - /// Default `true` — preserves current behavior. When set to a falsy value, - /// the frontend does not register routes that mutate or read worker / - /// scheduler state from outside the request plane: - /// - `POST /busy_threshold` (set busy thresholds on workers) - /// - `GET /busy_threshold` (read current busy thresholds) - /// - `POST /clear_kv_blocks` (clear all KV cache blocks on workers) - /// - /// Inference, metrics, models, health, and liveness routes are unaffected. + /// Default `true`. Falsy values prevent registration of `GET` / + /// `POST /busy_threshold`. Inference, metrics, models, health, and + /// liveness routes are unaffected. pub const DYN_ENABLE_FRONTEND_ADMIN_API: &str = "DYN_ENABLE_FRONTEND_ADMIN_API"; /// Strip the Claude Code billing preamble (`x-anthropic-billing-header: ...`) From 435b21c0c1feb9cc87214217ab3660042d617095 Mon Sep 17 00:00:00 2001 From: nnshah1 Date: Wed, 10 Jun 2026 15:43:01 -0700 Subject: [PATCH 3/8] review: drop self-documenting comment at handler nvext-gate sites MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Graham-review followup: the one-liner 'nvext master switch — see State::nvext_enabled' was copy-pasted at three handler call sites. state.nvext_enabled() is already self-documenting and the accessor's own /// doc points at the env var definition; the call-site comment adds nothing. Signed-off-by: nnshah1 --- lib/llm/src/http/service/openai.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/llm/src/http/service/openai.rs b/lib/llm/src/http/service/openai.rs index 1b789a02a07e..f7b5249cf7a8 100644 --- a/lib/llm/src/http/service/openai.rs +++ b/lib/llm/src/http/service/openai.rs @@ -551,7 +551,6 @@ async fn handler_completions( check_ready(&state)?; check_model_serving_ready(&state, &request.inner.model)?; - // nvext master switch — see State::nvext_enabled. request.nvext = if state.nvext_enabled() { apply_header_routing_overrides(request.nvext.take(), &headers) } else { @@ -1147,7 +1146,6 @@ async fn handler_chat_completions( check_model_serving_ready(&state, resolved_model)?; } - // nvext master switch — see State::nvext_enabled. request.nvext = if state.nvext_enabled() { apply_header_routing_overrides(request.nvext.take(), &headers) } else { @@ -1901,7 +1899,6 @@ async fn handler_responses( check_model_serving_ready(&state, resolved_model)?; } - // nvext master switch — see State::nvext_enabled. request.nvext = if state.nvext_enabled() { apply_header_routing_overrides(request.nvext.take(), &headers) } else { From fd3a977a6e3a1a5b378ce9e609624930834b0c96 Mon Sep 17 00:00:00 2001 From: nnshah1 Date: Wed, 10 Jun 2026 23:29:50 -0700 Subject: [PATCH 4/8] review #10556: rename DYN_ENABLE_FRONTEND_NVEXT, drop embedding nvext, add env-var test, consolidate docs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses biswapanda's review feedback on the master-switch PR: 1. ("keep them consistent") Rename DYN_ENABLE_NVEXT to DYN_ENABLE_FRONTEND_NVEXT to match the DYN_ENABLE_FRONTEND_ADMIN_API sibling. Only env-var name changes; internal API (enable_nvext field, nvext_enabled() accessor) stays. 2. ("embeddings has no nvext_enabled() gate") KISS — instead of gating the embedding handler, remove nvext from the embedding protocol entirely. The embedding-local NvExt only carried an annotations field (SSE event triggers, useless for typically-non-streaming embeddings) and never carried any routing fields. Delete lib/llm/src/protocols/openai/embeddings/nvext.rs and drop the nvext: Option field from NvCreateEmbeddingRequest plus the AnnotationsProvider impl. preprocessor.rs's preprocess_embedding_request no longer reads request annotations. 3. ("add a test case for unset/true/false env-var with temp_env::with_vars") Add test_dyn_enable_frontend_nvext_env_var_mirror covering unset, truthy, falsey, and the builder-overrides-env case. 4. ("we'd update user-facing docs") Add a Frontend feature switches section under docs/components/frontend/configuration.md with one consolidated table covering DYN_ENABLE_FRONTEND_NVEXT and DYN_ENABLE_FRONTEND_ADMIN_API: default values, behavior when false, and compatibility notes. The existing /busy_threshold rows in the Infrastructure routes table cross-reference the new section. cargo check / cargo fmt --check / cargo clippy --tests --no-deps all clean. 4 unit tests pass: - test_liveness_endpoint_reflects_cancellation - test_enable_nvext_propagates_through_builder_to_state - test_dyn_enable_frontend_nvext_env_var_mirror (new) - test_admin_api_disabled_404s_busy_threshold Signed-off-by: nnshah1 --- docs/components/frontend/configuration.md | 13 +++- lib/llm/src/http/service/service_v2.rs | 51 ++++++++++++++- lib/llm/src/preprocessor.rs | 13 ++-- lib/llm/src/protocols/openai/embeddings.rs | 42 ------------- .../src/protocols/openai/embeddings/nvext.rs | 62 ------------------- lib/runtime/src/config/environment_names.rs | 2 +- 6 files changed, 64 insertions(+), 119 deletions(-) delete mode 100644 lib/llm/src/protocols/openai/embeddings/nvext.rs diff --git a/docs/components/frontend/configuration.md b/docs/components/frontend/configuration.md index 2d81352b81af..cfb36b5144bc 100644 --- a/docs/components/frontend/configuration.md +++ b/docs/components/frontend/configuration.md @@ -161,8 +161,17 @@ The frontend exposes the following HTTP endpoints: | `GET` | `/metrics` | Prometheus metrics | | `GET` | `/openapi.json` | OpenAPI specification | | `GET` | `/docs` | Swagger UI | -| `POST` | `/busy_threshold` | Set busy thresholds | -| `GET` | `/busy_threshold` | Get current busy thresholds | +| `POST` | `/busy_threshold` | Set busy thresholds (gated by `DYN_ENABLE_FRONTEND_ADMIN_API`, see below) | +| `GET` | `/busy_threshold` | Get current busy thresholds (gated by `DYN_ENABLE_FRONTEND_ADMIN_API`, see below) | + +### Frontend feature switches + +Binary master switches that operators can flip to close off non-OpenAI-spec surfaces on a default-secure deployment. All default to `true`; an env value of `0` / `false` / `no` / `off` (case-insensitive) overrides to `false`. Unset env preserves the builder default. + +| Env Var | Default | Behavior when `false` | +|---------|---------|------------------------| +| `DYN_ENABLE_FRONTEND_NVEXT` | `true` | Frontend drops `request.nvext` at handler entry on `/v1/chat/completions`, `/v1/completions`, and `/v1/responses`; ignores routing-override headers (`x-worker-instance-id`, `x-prefill-instance-id`, `x-dp-rank`, `x-data-parallel-rank`, `x-prefill-dp-rank`); silently ignores the response-side `nvext.extra_fields` opt-in. Embeddings are unaffected (do not carry `nvext`). Note: disabling this breaks EPP / GAIE serving, Prime-RL-style training that uses `nvext.cache_salt`, multi-tenant agent platforms that forward `nvext.agent_hints` / `nvext.agent_context`, and clients that opt into response disclosure via `nvext.extra_fields`. | +| `DYN_ENABLE_FRONTEND_ADMIN_API` | `true` | `GET /busy_threshold` and `POST /busy_threshold` are not registered (404 instead of 503). Inference, metrics, models, health, and liveness routes are unaffected. | ### Endpoint Path Customization diff --git a/lib/llm/src/http/service/service_v2.rs b/lib/llm/src/http/service/service_v2.rs index 229a20c14aff..7a297252c546 100644 --- a/lib/llm/src/http/service/service_v2.rs +++ b/lib/llm/src/http/service/service_v2.rs @@ -185,7 +185,7 @@ impl State { } /// Master switch for the `nvext` extension protocol (see - /// [`environment_names::llm::DYN_ENABLE_NVEXT`]). + /// [`environment_names::llm::DYN_ENABLE_FRONTEND_NVEXT`]). #[inline] pub fn nvext_enabled(&self) -> bool { self.nvext_enabled @@ -298,7 +298,7 @@ pub struct HttpServiceConfig { enable_rl: bool, /// Master switch for the `nvext` extension protocol. Default `true`, - /// env-falsey on `DYN_ENABLE_NVEXT` overrides to `false`. + /// env-falsey on `DYN_ENABLE_FRONTEND_NVEXT` overrides to `false`. #[builder(default = "true")] enable_nvext: bool, @@ -597,7 +597,8 @@ impl HttpServiceConfigBuilder { )) as Arc }); // Env-falsey overrides the builder; unset preserves the builder default. - let nvext_enabled = config.enable_nvext && !env_is_falsey(env_llm::DYN_ENABLE_NVEXT); + let nvext_enabled = + config.enable_nvext && !env_is_falsey(env_llm::DYN_ENABLE_FRONTEND_NVEXT); let admin_api_enabled = config.enable_admin_api && !env_is_falsey(env_llm::DYN_ENABLE_FRONTEND_ADMIN_API); @@ -969,4 +970,48 @@ mod tests { "default should preserve current behavior (nvext on)" ); } + + /// `DYN_ENABLE_FRONTEND_NVEXT` is the env-var mirror of the builder + /// flag. Unset → builder default wins (on). Truthy strings → on. + /// Falsey strings (`0` / `false` / `no` / `off`, case-insensitive) → + /// off, regardless of what the builder asked for. + #[test] + fn test_dyn_enable_frontend_nvext_env_var_mirror() { + use dynamo_runtime::config::environment_names::llm::DYN_ENABLE_FRONTEND_NVEXT; + + // Unset → builder default (true) wins. + temp_env::with_var_unset(DYN_ENABLE_FRONTEND_NVEXT, || { + let svc = HttpService::builder().build().unwrap(); + assert!( + svc.state.nvext_enabled(), + "unset env + default builder = on" + ); + }); + + // Explicit truthy → on (builder default also on; env doesn't flip it off). + temp_env::with_var(DYN_ENABLE_FRONTEND_NVEXT, Some("true"), || { + let svc = HttpService::builder().build().unwrap(); + assert!(svc.state.nvext_enabled(), "env=true + default builder = on"); + }); + + // Explicit falsey → off, even though the builder default is on. + for falsey in ["false", "0", "no", "off", "FALSE"] { + temp_env::with_var(DYN_ENABLE_FRONTEND_NVEXT, Some(falsey), || { + let svc = HttpService::builder().build().unwrap(); + assert!( + !svc.state.nvext_enabled(), + "env={falsey:?} should override builder default to off" + ); + }); + } + + // Builder=false short-circuits regardless of env. + temp_env::with_var(DYN_ENABLE_FRONTEND_NVEXT, Some("true"), || { + let svc = HttpService::builder().enable_nvext(false).build().unwrap(); + assert!( + !svc.state.nvext_enabled(), + "builder=false wins even if env=true" + ); + }); + } } diff --git a/lib/llm/src/preprocessor.rs b/lib/llm/src/preprocessor.rs index 62b3d1d880bd..8dadfd3b9579 100644 --- a/lib/llm/src/preprocessor.rs +++ b/lib/llm/src/preprocessor.rs @@ -1770,7 +1770,7 @@ impl OpenAIPreprocessor { request: &NvCreateEmbeddingRequest, ) -> Result<(PreprocessedEmbeddingRequest, HashMap)> { let _stage_guard = StageGuard::new(STAGE_PREPROCESS, ""); - let mut annotations = HashMap::new(); + let annotations = HashMap::new(); let mut builder = PreprocessedEmbeddingRequest::builder(); let all_token_ids = match &request.inner.input { @@ -1802,13 +1802,7 @@ impl OpenAIPreprocessor { } }; - // Handle annotations - if request.has_annotation(ANNOTATION_TOKEN_IDS) { - annotations.insert( - ANNOTATION_TOKEN_IDS.to_string(), - serde_json::to_string(&all_token_ids)?, - ); - } + // Embeddings no longer carry per-request annotations (nvext removed). builder.token_ids(all_token_ids); builder.model(request.inner.model.clone()); @@ -1818,7 +1812,8 @@ impl OpenAIPreprocessor { })); builder.dimensions(request.inner.dimensions); - builder.annotations(request.annotations().unwrap_or_default()); + // Embeddings don't carry per-request annotations (nvext removed). + builder.annotations(Vec::new()); builder.mdc_sum(Some(self.mdcsum.clone())); Ok((builder.build()?, annotations)) diff --git a/lib/llm/src/protocols/openai/embeddings.rs b/lib/llm/src/protocols/openai/embeddings.rs index 68c349a7a528..7c2a291ef3c6 100644 --- a/lib/llm/src/protocols/openai/embeddings.rs +++ b/lib/llm/src/protocols/openai/embeddings.rs @@ -1,24 +1,17 @@ // SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -use dynamo_runtime::protocols::annotated::AnnotationsProvider; use serde::{Deserialize, Serialize}; use utoipa::ToSchema; use validator::Validate; mod aggregator; -mod nvext; - -pub use nvext::{NvExt, NvExtProvider}; #[derive(ToSchema, Serialize, Deserialize, Validate, Debug, Clone)] pub struct NvCreateEmbeddingRequest { #[serde(flatten)] #[schema(value_type = Object)] pub inner: dynamo_protocols::types::CreateEmbeddingRequest, - - #[serde(skip_serializing_if = "Option::is_none")] - pub nvext: Option, } /// A response structure for unary chat completion responses, embedding OpenAI's @@ -49,38 +42,3 @@ impl NvCreateEmbeddingResponse { } } } - -/// Implements `NvExtProvider` for `NvCreateEmbeddingRequest`, -/// providing access to NVIDIA-specific extensions. -impl NvExtProvider for NvCreateEmbeddingRequest { - /// Returns a reference to the optional `NvExt` extension, if available. - fn nvext(&self) -> Option<&NvExt> { - self.nvext.as_ref() - } -} - -/// Implements `AnnotationsProvider` for `NvCreateEmbeddingRequest`, -/// enabling retrieval and management of request annotations. -impl AnnotationsProvider for NvCreateEmbeddingRequest { - /// Retrieves the list of annotations from `NvExt`, if present. - fn annotations(&self) -> Option> { - self.nvext - .as_ref() - .and_then(|nvext| nvext.annotations.clone()) - } - - /// Checks whether a specific annotation exists in the request. - /// - /// # Arguments - /// * `annotation` - A string slice representing the annotation to check. - /// - /// # Returns - /// `true` if the annotation exists, `false` otherwise. - fn has_annotation(&self, annotation: &str) -> bool { - self.nvext - .as_ref() - .and_then(|nvext| nvext.annotations.as_ref()) - .map(|annotations| annotations.contains(&annotation.to_string())) - .unwrap_or(false) - } -} diff --git a/lib/llm/src/protocols/openai/embeddings/nvext.rs b/lib/llm/src/protocols/openai/embeddings/nvext.rs deleted file mode 100644 index 3808cb3d4b6b..000000000000 --- a/lib/llm/src/protocols/openai/embeddings/nvext.rs +++ /dev/null @@ -1,62 +0,0 @@ -// SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -// SPDX-License-Identifier: Apache-2.0 - -use derive_builder::Builder; -use serde::{Deserialize, Serialize}; -use utoipa::ToSchema; -use validator::{Validate, ValidationError}; - -pub trait NvExtProvider { - fn nvext(&self) -> Option<&NvExt>; -} - -/// NVIDIA LLM extensions to the OpenAI API -#[derive(ToSchema, Serialize, Deserialize, Builder, Validate, Debug, Clone)] -#[validate(schema(function = "validate_nv_ext"))] -pub struct NvExt { - /// Annotations - /// User requests triggers which result in the request issue back out-of-band information in the SSE - /// stream using the `event:` field. - #[serde(default, skip_serializing_if = "Option::is_none")] - #[builder(default, setter(strip_option))] - pub annotations: Option>, -} - -impl Default for NvExt { - fn default() -> Self { - NvExt::builder().build().unwrap() - } -} - -impl NvExt { - pub fn builder() -> NvExtBuilder { - NvExtBuilder::default() - } -} - -fn validate_nv_ext(_nv_ext: &NvExt) -> Result<(), ValidationError> { - Ok(()) -} - -impl NvExtBuilder { - pub fn add_annotation(&mut self, annotation: impl Into) -> &mut Self { - self.annotations - .get_or_insert_with(|| Some(vec![])) - .as_mut() - .expect("stop should always be Some(Vec)") - .push(annotation.into()); - self - } -} - -#[cfg(test)] -mod tests { - use super::*; - - // Test default builder configuration - #[test] - fn test_nv_ext_builder_default() { - let nv_ext = NvExt::builder().build().unwrap(); - assert_eq!(nv_ext.annotations, None); - } -} diff --git a/lib/runtime/src/config/environment_names.rs b/lib/runtime/src/config/environment_names.rs index c7cfb455dc7e..5463921436f1 100644 --- a/lib/runtime/src/config/environment_names.rs +++ b/lib/runtime/src/config/environment_names.rs @@ -291,7 +291,7 @@ pub mod llm { /// (`x-worker-instance-id`, `x-prefill-instance-id`, `x-dp-rank`, /// `x-prefill-dp-rank`), and silently ignore the response-side /// `extra_fields` opt-in. - pub const DYN_ENABLE_NVEXT: &str = "DYN_ENABLE_NVEXT"; + pub const DYN_ENABLE_FRONTEND_NVEXT: &str = "DYN_ENABLE_FRONTEND_NVEXT"; /// Master switch for the frontend's HTTP admin API surface. /// Default `true`. Falsy values prevent registration of `GET` / From e333bc6540b4e13bfaeefd9c03594bbf72c33b76 Mon Sep 17 00:00:00 2001 From: nnshah1 Date: Wed, 10 Jun 2026 23:52:49 -0700 Subject: [PATCH 5/8] review #10556: restore embeddings nvext, gate it instead of deleting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reversal of the deletion in fd3a977a6e. Per offline discussion with @nnshah1: even though the embedding-local NvExt currently only exposes 'annotations', deleting it removes a contract clients may be relying on. Keep the surface, gate its consumption — same pattern as the three other handlers — so DYN_ENABLE_FRONTEND_NVEXT=false closes off the *behavior* without breaking the request shape. Embedding handler now does the same one-line clear at handler entry: if !state.nvext_enabled() { request.nvext = None; } Routing-override headers don't apply to embeddings, so no apply_header_routing_overrides call to gate. Restored: - lib/llm/src/protocols/openai/embeddings/nvext.rs - lib/llm/src/protocols/openai/embeddings.rs (nvext field + NvExtProvider + AnnotationsProvider impls) - lib/llm/src/preprocessor.rs (request.has_annotation() + request.annotations() calls) Doc table updated to list /v1/embeddings alongside the other three gated handlers. Signed-off-by: nnshah1 --- docs/components/frontend/configuration.md | 2 +- lib/llm/src/http/service/openai.rs | 6 +- lib/llm/src/preprocessor.rs | 13 ++-- lib/llm/src/protocols/openai/embeddings.rs | 42 +++++++++++++ .../src/protocols/openai/embeddings/nvext.rs | 62 +++++++++++++++++++ 5 files changed, 119 insertions(+), 6 deletions(-) create mode 100644 lib/llm/src/protocols/openai/embeddings/nvext.rs diff --git a/docs/components/frontend/configuration.md b/docs/components/frontend/configuration.md index cfb36b5144bc..286dcc5f7407 100644 --- a/docs/components/frontend/configuration.md +++ b/docs/components/frontend/configuration.md @@ -170,7 +170,7 @@ Binary master switches that operators can flip to close off non-OpenAI-spec surf | Env Var | Default | Behavior when `false` | |---------|---------|------------------------| -| `DYN_ENABLE_FRONTEND_NVEXT` | `true` | Frontend drops `request.nvext` at handler entry on `/v1/chat/completions`, `/v1/completions`, and `/v1/responses`; ignores routing-override headers (`x-worker-instance-id`, `x-prefill-instance-id`, `x-dp-rank`, `x-data-parallel-rank`, `x-prefill-dp-rank`); silently ignores the response-side `nvext.extra_fields` opt-in. Embeddings are unaffected (do not carry `nvext`). Note: disabling this breaks EPP / GAIE serving, Prime-RL-style training that uses `nvext.cache_salt`, multi-tenant agent platforms that forward `nvext.agent_hints` / `nvext.agent_context`, and clients that opt into response disclosure via `nvext.extra_fields`. | +| `DYN_ENABLE_FRONTEND_NVEXT` | `true` | Frontend drops `request.nvext` at handler entry on `/v1/chat/completions`, `/v1/completions`, `/v1/responses`, and `/v1/embeddings`; ignores routing-override headers (`x-worker-instance-id`, `x-prefill-instance-id`, `x-dp-rank`, `x-data-parallel-rank`, `x-prefill-dp-rank`); silently ignores the response-side `nvext.extra_fields` opt-in. Note: disabling this breaks EPP / GAIE serving, Prime-RL-style training that uses `nvext.cache_salt`, multi-tenant agent platforms that forward `nvext.agent_hints` / `nvext.agent_context`, and clients that opt into response disclosure via `nvext.extra_fields`. | | `DYN_ENABLE_FRONTEND_ADMIN_API` | `true` | `GET /busy_threshold` and `POST /busy_threshold` are not registered (404 instead of 503). Inference, metrics, models, health, and liveness routes are unaffected. | ### Endpoint Path Customization diff --git a/lib/llm/src/http/service/openai.rs b/lib/llm/src/http/service/openai.rs index f7b5249cf7a8..84ce3423fa4b 100644 --- a/lib/llm/src/http/service/openai.rs +++ b/lib/llm/src/http/service/openai.rs @@ -975,12 +975,16 @@ async fn completions_batch( async fn embeddings( State(state): State>, headers: HeaderMap, - Json(request): Json, + Json(mut request): Json, ) -> Result { // return a 503 if the service or per-model topology is not ready check_ready(&state)?; check_model_serving_ready(&state, &request.inner.model)?; + if !state.nvext_enabled() { + request.nvext = None; + } + let request_id = get_or_create_request_id(&headers); let request = context_from_headers(request, request_id, &headers)?; let request_id = request.id().to_string(); diff --git a/lib/llm/src/preprocessor.rs b/lib/llm/src/preprocessor.rs index 8dadfd3b9579..62b3d1d880bd 100644 --- a/lib/llm/src/preprocessor.rs +++ b/lib/llm/src/preprocessor.rs @@ -1770,7 +1770,7 @@ impl OpenAIPreprocessor { request: &NvCreateEmbeddingRequest, ) -> Result<(PreprocessedEmbeddingRequest, HashMap)> { let _stage_guard = StageGuard::new(STAGE_PREPROCESS, ""); - let annotations = HashMap::new(); + let mut annotations = HashMap::new(); let mut builder = PreprocessedEmbeddingRequest::builder(); let all_token_ids = match &request.inner.input { @@ -1802,7 +1802,13 @@ impl OpenAIPreprocessor { } }; - // Embeddings no longer carry per-request annotations (nvext removed). + // Handle annotations + if request.has_annotation(ANNOTATION_TOKEN_IDS) { + annotations.insert( + ANNOTATION_TOKEN_IDS.to_string(), + serde_json::to_string(&all_token_ids)?, + ); + } builder.token_ids(all_token_ids); builder.model(request.inner.model.clone()); @@ -1812,8 +1818,7 @@ impl OpenAIPreprocessor { })); builder.dimensions(request.inner.dimensions); - // Embeddings don't carry per-request annotations (nvext removed). - builder.annotations(Vec::new()); + builder.annotations(request.annotations().unwrap_or_default()); builder.mdc_sum(Some(self.mdcsum.clone())); Ok((builder.build()?, annotations)) diff --git a/lib/llm/src/protocols/openai/embeddings.rs b/lib/llm/src/protocols/openai/embeddings.rs index 7c2a291ef3c6..68c349a7a528 100644 --- a/lib/llm/src/protocols/openai/embeddings.rs +++ b/lib/llm/src/protocols/openai/embeddings.rs @@ -1,17 +1,24 @@ // SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 +use dynamo_runtime::protocols::annotated::AnnotationsProvider; use serde::{Deserialize, Serialize}; use utoipa::ToSchema; use validator::Validate; mod aggregator; +mod nvext; + +pub use nvext::{NvExt, NvExtProvider}; #[derive(ToSchema, Serialize, Deserialize, Validate, Debug, Clone)] pub struct NvCreateEmbeddingRequest { #[serde(flatten)] #[schema(value_type = Object)] pub inner: dynamo_protocols::types::CreateEmbeddingRequest, + + #[serde(skip_serializing_if = "Option::is_none")] + pub nvext: Option, } /// A response structure for unary chat completion responses, embedding OpenAI's @@ -42,3 +49,38 @@ impl NvCreateEmbeddingResponse { } } } + +/// Implements `NvExtProvider` for `NvCreateEmbeddingRequest`, +/// providing access to NVIDIA-specific extensions. +impl NvExtProvider for NvCreateEmbeddingRequest { + /// Returns a reference to the optional `NvExt` extension, if available. + fn nvext(&self) -> Option<&NvExt> { + self.nvext.as_ref() + } +} + +/// Implements `AnnotationsProvider` for `NvCreateEmbeddingRequest`, +/// enabling retrieval and management of request annotations. +impl AnnotationsProvider for NvCreateEmbeddingRequest { + /// Retrieves the list of annotations from `NvExt`, if present. + fn annotations(&self) -> Option> { + self.nvext + .as_ref() + .and_then(|nvext| nvext.annotations.clone()) + } + + /// Checks whether a specific annotation exists in the request. + /// + /// # Arguments + /// * `annotation` - A string slice representing the annotation to check. + /// + /// # Returns + /// `true` if the annotation exists, `false` otherwise. + fn has_annotation(&self, annotation: &str) -> bool { + self.nvext + .as_ref() + .and_then(|nvext| nvext.annotations.as_ref()) + .map(|annotations| annotations.contains(&annotation.to_string())) + .unwrap_or(false) + } +} diff --git a/lib/llm/src/protocols/openai/embeddings/nvext.rs b/lib/llm/src/protocols/openai/embeddings/nvext.rs new file mode 100644 index 000000000000..3808cb3d4b6b --- /dev/null +++ b/lib/llm/src/protocols/openai/embeddings/nvext.rs @@ -0,0 +1,62 @@ +// SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use derive_builder::Builder; +use serde::{Deserialize, Serialize}; +use utoipa::ToSchema; +use validator::{Validate, ValidationError}; + +pub trait NvExtProvider { + fn nvext(&self) -> Option<&NvExt>; +} + +/// NVIDIA LLM extensions to the OpenAI API +#[derive(ToSchema, Serialize, Deserialize, Builder, Validate, Debug, Clone)] +#[validate(schema(function = "validate_nv_ext"))] +pub struct NvExt { + /// Annotations + /// User requests triggers which result in the request issue back out-of-band information in the SSE + /// stream using the `event:` field. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[builder(default, setter(strip_option))] + pub annotations: Option>, +} + +impl Default for NvExt { + fn default() -> Self { + NvExt::builder().build().unwrap() + } +} + +impl NvExt { + pub fn builder() -> NvExtBuilder { + NvExtBuilder::default() + } +} + +fn validate_nv_ext(_nv_ext: &NvExt) -> Result<(), ValidationError> { + Ok(()) +} + +impl NvExtBuilder { + pub fn add_annotation(&mut self, annotation: impl Into) -> &mut Self { + self.annotations + .get_or_insert_with(|| Some(vec![])) + .as_mut() + .expect("stop should always be Some(Vec)") + .push(annotation.into()); + self + } +} + +#[cfg(test)] +mod tests { + use super::*; + + // Test default builder configuration + #[test] + fn test_nv_ext_builder_default() { + let nv_ext = NvExt::builder().build().unwrap(); + assert_eq!(nv_ext.annotations, None); + } +} From 844ace9914dbd391454fa3635930992547b2890e Mon Sep 17 00:00:00 2001 From: nnshah1 Date: Thu, 11 Jun 2026 10:52:57 -0700 Subject: [PATCH 6/8] test(frontend): e2e gate check + warn when nvext arrives disabled MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add an end-to-end integration test asserting that when the frontend nvext extension is disabled, a request asking for response-side extra_fields (plus a routing header and body routing field) produces no nvext field in the response — proving the request- and response-side strips are wired together, not just unit-tested in isolation. Also warn once at the handler when a request carries nvext data (body or routing headers) while the extension is disabled, so the otherwise silent strip is observable in logs. The check lives only in the disabled branch, so the default request path is unaffected. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: nnshah1 --- lib/llm/src/http/service/openai.rs | 34 ++++++++++++++++++ lib/llm/tests/http-service.rs | 55 ++++++++++++++++++++++++++++++ 2 files changed, 89 insertions(+) diff --git a/lib/llm/src/http/service/openai.rs b/lib/llm/src/http/service/openai.rs index 84ce3423fa4b..2db60cfe9a62 100644 --- a/lib/llm/src/http/service/openai.rs +++ b/lib/llm/src/http/service/openai.rs @@ -537,6 +537,36 @@ fn copy_x_request_id( /// OpenAI Completions Request Handler /// /// This method will handle the incoming request for the `/v1/completions endpoint`. The endpoint is a "source" +/// Emit a single warning when a request carries `nvext` data — either a body +/// `nvext` object or one of the routing-override headers — while the frontend +/// `nvext` extension is disabled (`DYN_ENABLE_FRONTEND_NVEXT`). The data is +/// dropped regardless; this just makes the otherwise-silent strip observable. +/// +/// Only ever called from the disabled branch, so it adds nothing to the +/// default (nvext-enabled) request path. +fn warn_nvext_disabled(endpoint: &str, nvext_present: bool, headers: &HeaderMap) { + use crate::protocols::openai::nvext::{ + HEADER_DP_RANK, HEADER_DP_RANK_ALIAS, HEADER_PREFILL_DP_RANK, HEADER_PREFILL_INSTANCE_ID, + HEADER_WORKER_INSTANCE_ID, + }; + let header_present = [ + HEADER_WORKER_INSTANCE_ID, + HEADER_PREFILL_INSTANCE_ID, + HEADER_DP_RANK, + HEADER_DP_RANK_ALIAS, + HEADER_PREFILL_DP_RANK, + ] + .iter() + .any(|h| headers.contains_key(*h)); + + if nvext_present || header_present { + tracing::warn!( + endpoint, + "request carried nvext data (body and/or routing headers) but DYN_ENABLE_FRONTEND_NVEXT is disabled; dropping it" + ); + } +} + /// for an [`super::OpenAICompletionsStreamingEngine`] and will return a stream of /// responses which will be forward to the client. /// @@ -554,6 +584,7 @@ async fn handler_completions( request.nvext = if state.nvext_enabled() { apply_header_routing_overrides(request.nvext.take(), &headers) } else { + warn_nvext_disabled("completions", request.nvext.is_some(), &headers); None }; @@ -982,6 +1013,7 @@ async fn embeddings( check_model_serving_ready(&state, &request.inner.model)?; if !state.nvext_enabled() { + warn_nvext_disabled("embeddings", request.nvext.is_some(), &headers); request.nvext = None; } @@ -1153,6 +1185,7 @@ async fn handler_chat_completions( request.nvext = if state.nvext_enabled() { apply_header_routing_overrides(request.nvext.take(), &headers) } else { + warn_nvext_disabled("chat_completions", request.nvext.is_some(), &headers); None }; @@ -1906,6 +1939,7 @@ async fn handler_responses( request.nvext = if state.nvext_enabled() { apply_header_routing_overrides(request.nvext.take(), &headers) } else { + warn_nvext_disabled("responses", request.nvext.is_some(), &headers); None }; diff --git a/lib/llm/tests/http-service.rs b/lib/llm/tests/http-service.rs index 398c3e71e934..56793af85a33 100644 --- a/lib/llm/tests/http-service.rs +++ b/lib/llm/tests/http-service.rs @@ -908,3 +908,58 @@ async fn test_request_id_annotation() { cancel_token.cancel(); task.await.unwrap().unwrap(); } + +/// End-to-end gate check for `DYN_ENABLE_FRONTEND_NVEXT`: when the master +/// switch is off, a request asking for response-side `nvext.extra_fields` +/// must not produce any `nvext` field in the streamed response. +#[tokio::test] +async fn test_nvext_disabled_strips_request_and_response() { + dynamo_runtime::logging::init(); + + let (listener, port) = bind_random_port().await; + let service = HttpService::builder() + .enable_chat_endpoints(true) + .enable_nvext(false) + .port(port) + .build() + .unwrap(); + let state = service.state_clone(); + let manager = state.manager(); + + let token = CancellationToken::new(); + let cancel_token = token.clone(); + let task = tokio::spawn(async move { service.run_with_listener(token, listener).await }); + wait_for_service_ready(port).await; + + let card = ModelDeploymentCard::with_name_only("test-model"); + manager + .add_chat_completions_model("test-model", card.mdcsum(), Arc::new(CounterEngine {})) + .unwrap(); + + let response = reqwest::Client::new() + .post(format!("http://localhost:{port}/v1/chat/completions")) + .header("x-worker-instance-id", "42") + .json(&serde_json::json!({ + "model": "test-model", + "messages": [{"role": "user", "content": "hi"}], + "stream": true, + "max_tokens": 1, + "nvext": { + "extra_fields": ["worker_id", "timing", "engine_data"], + "backend_instance_id": 99 + } + })) + .send() + .await + .expect("request should succeed"); + assert!(response.status().is_success()); + + let body = response.text().await.expect("read body"); + assert!( + !body.contains("\"nvext\""), + "nvext gate off: response must not contain an `nvext` field, got: {body}" + ); + + cancel_token.cancel(); + task.await.unwrap().unwrap(); +} From 42a9c751607a910bec61c5b68af134323eea490a Mon Sep 17 00:00:00 2001 From: nnshah1 Date: Thu, 11 Jun 2026 11:42:42 -0700 Subject: [PATCH 7/8] review: relocate nvext warn helper, trim comments Move warn_nvext_disabled above the handler_completions doc block (it had been inserted mid-doc-comment, splitting it) and trim verbose docstrings on the helper and the new test per review. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: nnshah1 --- lib/llm/src/http/service/openai.rs | 17 ++++++----------- lib/llm/tests/http-service.rs | 5 ++--- 2 files changed, 8 insertions(+), 14 deletions(-) diff --git a/lib/llm/src/http/service/openai.rs b/lib/llm/src/http/service/openai.rs index 2db60cfe9a62..4620941095fd 100644 --- a/lib/llm/src/http/service/openai.rs +++ b/lib/llm/src/http/service/openai.rs @@ -534,16 +534,8 @@ fn copy_x_request_id( } } -/// OpenAI Completions Request Handler -/// -/// This method will handle the incoming request for the `/v1/completions endpoint`. The endpoint is a "source" -/// Emit a single warning when a request carries `nvext` data — either a body -/// `nvext` object or one of the routing-override headers — while the frontend -/// `nvext` extension is disabled (`DYN_ENABLE_FRONTEND_NVEXT`). The data is -/// dropped regardless; this just makes the otherwise-silent strip observable. -/// -/// Only ever called from the disabled branch, so it adds nothing to the -/// default (nvext-enabled) request path. +/// Warn (once per request) when nvext data is dropped because the extension is +/// disabled. Only called from the disabled branch, so the default path is free. fn warn_nvext_disabled(endpoint: &str, nvext_present: bool, headers: &HeaderMap) { use crate::protocols::openai::nvext::{ HEADER_DP_RANK, HEADER_DP_RANK_ALIAS, HEADER_PREFILL_DP_RANK, HEADER_PREFILL_INSTANCE_ID, @@ -562,11 +554,14 @@ fn warn_nvext_disabled(endpoint: &str, nvext_present: bool, headers: &HeaderMap) if nvext_present || header_present { tracing::warn!( endpoint, - "request carried nvext data (body and/or routing headers) but DYN_ENABLE_FRONTEND_NVEXT is disabled; dropping it" + "request carried nvext data but DYN_ENABLE_FRONTEND_NVEXT is disabled; dropping it" ); } } +/// OpenAI Completions Request Handler +/// +/// This method will handle the incoming request for the `/v1/completions endpoint`. The endpoint is a "source" /// for an [`super::OpenAICompletionsStreamingEngine`] and will return a stream of /// responses which will be forward to the client. /// diff --git a/lib/llm/tests/http-service.rs b/lib/llm/tests/http-service.rs index 56793af85a33..792bfab7b7bd 100644 --- a/lib/llm/tests/http-service.rs +++ b/lib/llm/tests/http-service.rs @@ -909,9 +909,8 @@ async fn test_request_id_annotation() { task.await.unwrap().unwrap(); } -/// End-to-end gate check for `DYN_ENABLE_FRONTEND_NVEXT`: when the master -/// switch is off, a request asking for response-side `nvext.extra_fields` -/// must not produce any `nvext` field in the streamed response. +/// With nvext disabled, a request asking for response `extra_fields` must not +/// produce any `nvext` field in the response. #[tokio::test] async fn test_nvext_disabled_strips_request_and_response() { dynamo_runtime::logging::init(); From ab67297a2038cb0ee230f82fea6d59684e3c2cbb Mon Sep 17 00:00:00 2001 From: nnshah1 Date: Thu, 11 Jun 2026 11:44:10 -0700 Subject: [PATCH 8/8] docs(frontend): simplify feature-switch intro Replace the verbose master-switch preamble with a short deployment-focused note: extensions are on by default; disable the ones a deployment doesn't need to prevent accidental abuse. Co-Authored-By: Claude Opus 4.8 (1M context) Signed-off-by: nnshah1 --- docs/components/frontend/configuration.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/components/frontend/configuration.md b/docs/components/frontend/configuration.md index 286dcc5f7407..874a92eb022f 100644 --- a/docs/components/frontend/configuration.md +++ b/docs/components/frontend/configuration.md @@ -166,7 +166,9 @@ The frontend exposes the following HTTP endpoints: ### Frontend feature switches -Binary master switches that operators can flip to close off non-OpenAI-spec surfaces on a default-secure deployment. All default to `true`; an env value of `0` / `false` / `no` / `off` (case-insensitive) overrides to `false`. Unset env preserves the builder default. +Environment variables controlling frontend extensions. Extensions are enabled by default. When deploying, consider whether each is needed for your use case; if not, disable it to prevent accidental abuse. + +Set an env value of `0` / `false` / `no` / `off` (case-insensitive) to disable. | Env Var | Default | Behavior when `false` | |---------|---------|------------------------|