diff --git a/lib/bindings/python/rust/parsers.rs b/lib/bindings/python/rust/parsers.rs index 1132e374c0f9..41d57f8216ba 100644 --- a/lib/bindings/python/rust/parsers.rs +++ b/lib/bindings/python/rust/parsers.rs @@ -202,6 +202,7 @@ fn make_stream_chunk(chunk: &CaptureChunk) -> Annotated( response_collector: &mut ResponseMetricCollector, http_queue_guard: &mut Option, ) { - use crate::preprocessor::LLMMetricAnnotation; - - // update metrics if let Ok(Some(metrics)) = LLMMetricAnnotation::from_annotation(annotated) { - response_collector.observe_current_osl(metrics.output_tokens); - response_collector.observe_cached_tokens(metrics.cached_tokens); - response_collector.observe_tokenize_latencies( - metrics.tokenize_latency, - metrics.detokenize_total_latency, - metrics.detokenize_count, - ); - response_collector.set_worker_info( - metrics.prefill_worker_id, - metrics.prefill_dp_rank, - metrics.prefill_worker_type, - metrics.decode_worker_id, - metrics.decode_dp_rank, - metrics.decode_worker_type, - ); - - // Drop http_queue_guard on first token for non-streaming (same as streaming) - if response_collector.is_first_token() - && metrics.chunk_tokens > 0 - && let Some(guard) = http_queue_guard.take() - { - drop(guard); - } + observe_llm_metrics(&metrics, response_collector, http_queue_guard); + } +} - response_collector.observe_response(metrics.input_tokens, metrics.chunk_tokens); +/// Process streaming metrics for chat-derived responses. +/// +/// The typed metrics field is the hot path. Annotation parsing remains as a compatibility +/// fallback for legacy/generated annotation frames. +pub fn process_chat_response_and_observe_metrics( + annotated: &crate::types::Annotated, + response_collector: &mut ResponseMetricCollector, + http_queue_guard: &mut Option, +) { + if let Some(metrics) = annotated + .data + .as_ref() + .and_then(|data| data.llm_metrics.as_ref()) + { + observe_llm_metrics(metrics, response_collector, http_queue_guard); + } else { + process_response_and_observe_metrics(annotated, response_collector, http_queue_guard); } } @@ -1670,57 +1667,53 @@ fn sse_json_data(event: Event, data: &T) -> Result( - annotated: EventConverter, +fn observe_llm_metrics( + metrics: &LLMMetricAnnotation, response_collector: &mut ResponseMetricCollector, http_queue_guard: &mut Option, -) -> Result, axum::Error> { - use crate::preprocessor::LLMMetricAnnotation; - - let mut annotated = annotated.0; - - // update metrics - if let Ok(Some(metrics)) = LLMMetricAnnotation::from_annotation(&annotated) { - response_collector.observe_current_osl(metrics.output_tokens); - response_collector.observe_cached_tokens(metrics.cached_tokens); - response_collector.observe_tokenize_latencies( - metrics.tokenize_latency, - metrics.detokenize_total_latency, - metrics.detokenize_count, - ); - response_collector.set_worker_info( - metrics.prefill_worker_id, - metrics.prefill_dp_rank, - metrics.prefill_worker_type, - metrics.decode_worker_id, - metrics.decode_dp_rank, - metrics.decode_worker_type, - ); - - // Drop http_queue_guard on first token for streaming - if response_collector.is_first_token() - && metrics.chunk_tokens > 0 - && let Some(guard) = http_queue_guard.take() - { - drop(guard); - } - - response_collector.observe_response(metrics.input_tokens, metrics.chunk_tokens); +) { + response_collector.observe_current_osl(metrics.output_tokens); + response_collector.observe_cached_tokens(metrics.cached_tokens); + response_collector.observe_tokenize_latencies( + metrics.tokenize_latency, + metrics.detokenize_total_latency, + metrics.detokenize_count, + ); + response_collector.set_worker_info( + metrics.prefill_worker_id, + metrics.prefill_dp_rank, + metrics.prefill_worker_type.clone(), + metrics.decode_worker_id, + metrics.decode_dp_rank, + metrics.decode_worker_type.clone(), + ); + + if response_collector.is_first_token() + && metrics.chunk_tokens > 0 + && let Some(guard) = http_queue_guard.take() + { + drop(guard); + } + + response_collector.observe_response(metrics.input_tokens, metrics.chunk_tokens); +} - // Chomp the LLMMetricAnnotation so it's not returned in the response stream - // TODO: add a flag to control what is returned in the SSE stream - if annotated.event.as_deref() == Some(crate::preprocessor::ANNOTATION_LLM_METRICS) { - annotated.event = None; - annotated.comment = None; - } +fn observe_annotation_metrics( + annotated: &crate::types::Annotated, + response_collector: &mut ResponseMetricCollector, + http_queue_guard: &mut Option, +) -> bool { + if let Ok(Some(metrics)) = LLMMetricAnnotation::from_annotation(annotated) { + observe_llm_metrics(&metrics, response_collector, http_queue_guard); + true + } else { + false } +} +fn annotated_to_sse_event( + annotated: crate::types::Annotated, +) -> Result, axum::Error> { let mut event = Event::default(); if let Some(ref data) = annotated.data { @@ -1763,6 +1756,60 @@ pub fn process_response_using_event_converter_and_observe_metrics( } } +/// Process streaming response with event conversion for SSE +/// +/// This function handles metrics collection, http_queue_guard management, and converts +/// annotated responses to SSE events for streaming responses. +/// +/// Returns None for metrics annotation events (events without SSE data payload). +pub fn process_response_using_event_converter_and_observe_metrics( + annotated: EventConverter, + response_collector: &mut ResponseMetricCollector, + http_queue_guard: &mut Option, +) -> Result, axum::Error> { + let mut annotated = annotated.0; + + if observe_annotation_metrics(&annotated, response_collector, http_queue_guard) { + // Preserve the previous SSE behavior: observe legacy metrics annotation + // frames internally, then strip them before building the outbound event. + if annotated.event.as_deref() == Some(ANNOTATION_LLM_METRICS) { + annotated.event = None; + annotated.comment = None; + } + } + + annotated_to_sse_event(annotated) +} + +/// Process chat-derived streaming responses with typed metrics before SSE conversion. +/// +/// If the typed field is absent, this falls back to the same legacy annotation +/// frame parsing and stripping behavior as `process_response_using_event_converter_and_observe_metrics`. +pub fn process_chat_response_using_event_converter_and_observe_metrics( + annotated: EventConverter, + response_collector: &mut ResponseMetricCollector, + http_queue_guard: &mut Option, +) -> Result, axum::Error> { + let mut annotated = annotated.0; + + if let Some(metrics) = annotated + .data + .as_ref() + .and_then(|data| data.llm_metrics.as_ref()) + { + observe_llm_metrics(metrics, response_collector, http_queue_guard); + } else if observe_annotation_metrics(&annotated, response_collector, http_queue_guard) + && annotated.event.as_deref() == Some(ANNOTATION_LLM_METRICS) + { + // Legacy compatibility path: annotation frames were never emitted as + // client-visible SSE payloads after metrics collection. + annotated.event = None; + annotated.comment = None; + } + + annotated_to_sse_event(annotated) +} + /// Create a new router with optional DRT metrics integration. /// /// When `drt_metrics` is provided, the `/metrics` handler will also include @@ -2389,6 +2436,117 @@ mod tests { ); } + #[test] + fn test_chat_typed_metrics_fast_path_observes_without_annotation() { + use crate::protocols::common::metrics::LLMMetricAnnotation; + + let metrics = Arc::new(Metrics::new()); + let registry = prometheus::Registry::new(); + metrics.register(®istry).unwrap(); + + let model = "test-model"; + let expected_metric_name = "dynamo_frontend_cached_tokens"; + let expected_tokenizer_metric_name = "dynamo_frontend_tokenizer_latency_ms"; + let mut collector = metrics.clone().create_response_collector(model); + let mut annotated = make_chat_stream_annotated("hello"); + annotated.data.as_mut().unwrap().llm_metrics = Some(LLMMetricAnnotation { + input_tokens: 10, + output_tokens: 20, + chunk_tokens: 5, + cached_tokens: Some(15), + prefill_worker_id: Some(11), + prefill_dp_rank: Some(1), + prefill_worker_type: Some(WORKER_TYPE_PREFILL.to_string()), + decode_worker_id: Some(22), + decode_dp_rank: Some(2), + decode_worker_type: Some(WORKER_TYPE_DECODE.to_string()), + tokenize_latency: Some(Duration::from_millis(8)), + detokenize_total_latency: Some(Duration::from_micros(100)), + detokenize_count: Some(2), + }); + + assert!(annotated.event.is_none()); + assert!(annotated.comment.is_none()); + + let mut http_queue_guard = Some(metrics.clone().create_http_queue_guard(model)); + let result = process_chat_response_using_event_converter_and_observe_metrics( + EventConverter::from(annotated), + &mut collector, + &mut http_queue_guard, + ); + + assert!( + http_queue_guard.is_none(), + "first positive chunk should release HTTP queue guard" + ); + let event = result + .unwrap() + .expect("typed metrics data chunk should still produce an SSE event"); + let json = extract_sse_data_json(event); + assert!( + json.get("llm_metrics").is_none(), + "typed metrics must be skipped on the SSE wire" + ); + + drop(collector); + + let metric_families = registry.gather(); + let histogram_family = metric_families + .iter() + .find(|mf| mf.name() == expected_metric_name) + .expect("histogram should be registered"); + assert_eq!( + histogram_family.get_metric()[0] + .get_histogram() + .get_sample_count(), + 1 + ); + + let histogram_family = metric_families + .iter() + .find(|mf| mf.name() == expected_tokenizer_metric_name) + .expect("histogram should be registered"); + let tokenize_metric = histogram_family + .get_metric() + .iter() + .find(|m| m.get_label().iter().any(|l| l.value() == "tokenize")) + .expect("tokenize metric should exist"); + assert_eq!(tokenize_metric.get_histogram().get_sample_count(), 1); + assert!( + (tokenize_metric.get_histogram().get_sample_sum() - 8.0).abs() < 0.001, + "tokenize latency should be 8.0ms" + ); + } + + #[test] + fn test_chat_stream_typed_metrics_are_skipped_in_json() { + use crate::protocols::common::metrics::LLMMetricAnnotation; + + let without_metrics = make_chat_stream_annotated("hello").data.unwrap(); + let mut with_metrics = without_metrics.clone(); + with_metrics.llm_metrics = Some(LLMMetricAnnotation { + input_tokens: 1, + output_tokens: 2, + chunk_tokens: 1, + cached_tokens: Some(1), + prefill_worker_id: None, + prefill_dp_rank: None, + prefill_worker_type: None, + decode_worker_id: None, + decode_dp_rank: None, + decode_worker_type: None, + tokenize_latency: None, + detokenize_total_latency: None, + detokenize_count: None, + }); + + let without_json = serde_json::to_value(&without_metrics).unwrap(); + let with_json = serde_json::to_value(&with_metrics).unwrap(); + + assert_eq!(with_json, without_json); + assert!(with_json.get("llm_metrics").is_none()); + } + #[test] fn test_non_streaming_path_observes_cached_tokens() { use crate::preprocessor::LLMMetricAnnotation; @@ -2859,7 +3017,7 @@ mod tests { let metrics = Arc::new(Metrics::new()); let mut collector = ResponseMetricCollector::new(metrics, "test-model".to_string()); let mut http_queue_guard: Option = None; - process_response_using_event_converter_and_observe_metrics( + process_chat_response_using_event_converter_and_observe_metrics( EventConverter::from(annotated), &mut collector, &mut http_queue_guard, @@ -2906,6 +3064,7 @@ mod tests { service_tier: None, }, nvext: None, + llm_metrics: None, }, ), event: None, diff --git a/lib/llm/src/http/service/openai.rs b/lib/llm/src/http/service/openai.rs index 809be84ce0fc..3469febd7c06 100644 --- a/lib/llm/src/http/service/openai.rs +++ b/lib/llm/src/http/service/openai.rs @@ -38,6 +38,8 @@ use super::{ metadata::{attach_x_request_id, extract_metadata_from_http}, metrics::{ CancellationLabels, Endpoint, ErrorType, EventConverter, + process_chat_response_and_observe_metrics, + process_chat_response_using_event_converter_and_observe_metrics, process_response_and_observe_metrics, process_response_using_event_converter_and_observe_metrics, }, @@ -1769,7 +1771,7 @@ async fn chat_completions( // Convert to SSE event (this consumes the response). // EventConverter will detect `event: "error"` and convert to SSE error events. - let sse_result = process_response_using_event_converter_and_observe_metrics( + let sse_result = process_chat_response_using_event_converter_and_observe_metrics( EventConverter::from(response), &mut response_collector, &mut http_queue_guard, @@ -1811,7 +1813,7 @@ async fn chat_completions( let mut http_queue_guard = Some(http_queue_guard); let stream = stream_with_check.inspect(move |response| { // Calls observe_response() on each token - drops http_queue_guard on first token - process_response_and_observe_metrics( + process_chat_response_and_observe_metrics( response, &mut response_collector, &mut http_queue_guard, @@ -2206,7 +2208,7 @@ async fn responses( let mut saw_error = false; while let Some(annotated_chunk) = engine_stream.next().await { - process_response_and_observe_metrics( + process_chat_response_and_observe_metrics( &annotated_chunk, &mut response_collector, &mut http_queue_guard, @@ -2262,7 +2264,7 @@ async fn responses( let mut http_queue_guard = Some(http_queue_guard); let stream = stream_with_check.inspect(move |response| { - process_response_and_observe_metrics( + process_chat_response_and_observe_metrics( response, &mut response_collector, &mut http_queue_guard, @@ -4324,6 +4326,7 @@ mod tests { usage: None, }, nvext: None, + llm_metrics: None, }), id: Some("msg-1".to_string()), event: None, @@ -4363,6 +4366,7 @@ mod tests { usage: None, }, nvext: None, + llm_metrics: None, }), id: Some("msg-1".to_string()), event: None, @@ -4662,6 +4666,7 @@ mod tests { service_tier: None, }, nvext: None, + llm_metrics: None, }; Annotated { id: Some("test-id".to_string()), @@ -5295,6 +5300,7 @@ mod tests { service_tier: None, }, nvext: None, + llm_metrics: None, } } diff --git a/lib/llm/src/perf/logprobs.rs b/lib/llm/src/perf/logprobs.rs index 6873ad49a27c..5869734a0366 100644 --- a/lib/llm/src/perf/logprobs.rs +++ b/lib/llm/src/perf/logprobs.rs @@ -976,6 +976,7 @@ mod tests { usage: None, }, nvext: None, + llm_metrics: None, } } @@ -1017,6 +1018,7 @@ mod tests { usage: None, }, nvext: None, + llm_metrics: None, } } @@ -1361,6 +1363,7 @@ mod tests { usage: None, }, nvext: None, + llm_metrics: None, }; let logprobs = response.extract_logprobs_by_choice(); @@ -1578,6 +1581,7 @@ mod tests { usage: None, }, nvext: None, + llm_metrics: None, } } diff --git a/lib/llm/src/preprocessor.rs b/lib/llm/src/preprocessor.rs index 63f924855525..0be94a82a4e9 100644 --- a/lib/llm/src/preprocessor.rs +++ b/lib/llm/src/preprocessor.rs @@ -31,7 +31,7 @@ use dynamo_renderer::OAIPromptFormatter; use dynamo_runtime::error::{DynamoError, ErrorType}; use futures::Stream; use futures::stream::{self, StreamExt}; -use std::time::{Duration, Instant}; +use std::time::Instant; use dynamo_runtime::dynamo_nvtx_range; use dynamo_runtime::metrics::frontend_perf::{ @@ -39,7 +39,7 @@ use dynamo_runtime::metrics::frontend_perf::{ StageGuard, TEMPLATE_SECONDS, TOKENIZE_SECONDS, }; use std::borrow::Cow; -use std::{collections::HashMap, pin::Pin, sync::Arc}; +use std::{any::Any, collections::HashMap, pin::Pin, sync::Arc}; use tracing; #[cfg(feature = "mm-routing")] @@ -82,6 +82,7 @@ use crate::preprocessor::prompt::{MediaRequestExt, prompt_formatter_from_mdc}; use dynamo_renderer::{OAIChatLikeRequest, PromptFormatter, PromptInput, TextInput, TokenInput}; pub use crate::protocols::common::llm_backend::{BackendOutput, PreprocessedRequest}; +pub use crate::protocols::common::metrics::{ANNOTATION_LLM_METRICS, LLMMetricAnnotation}; pub use crate::protocols::common::preprocessor::PreprocessedEmbeddingRequest; use crate::protocols::common::llm_backend::EmbeddingsEngineOutput; @@ -112,68 +113,6 @@ fn encode_floats_to_base64(floats: &[f32]) -> String { pub const ANNOTATION_FORMATTED_PROMPT: &str = "formatted_prompt"; pub const ANNOTATION_TOKEN_IDS: &str = "token_ids"; -pub const ANNOTATION_LLM_METRICS: &str = "llm_metrics"; -#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] -pub struct LLMMetricAnnotation { - pub input_tokens: usize, - pub output_tokens: usize, - pub chunk_tokens: usize, - pub cached_tokens: Option, - /// Prefill worker ID (for TTFT attribution in disaggregated mode) - #[serde(default, skip_serializing_if = "Option::is_none")] - pub prefill_worker_id: Option, - /// Prefill worker DP rank - #[serde(default, skip_serializing_if = "Option::is_none")] - pub prefill_dp_rank: Option, - /// Prefill worker type ("prefill" or "decode") for Prometheus metric labeling. - /// Stored at routing time to avoid expensive MDC lookup when updating TTFT metrics. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub prefill_worker_type: Option, - /// Decode worker ID (for ITL attribution in disaggregated mode) - #[serde(default, skip_serializing_if = "Option::is_none")] - pub decode_worker_id: Option, - /// Decode worker DP rank - #[serde(default, skip_serializing_if = "Option::is_none")] - pub decode_dp_rank: Option, - /// Decode worker type ("prefill" or "decode") for Prometheus metric labeling. - /// Stored at routing time to avoid expensive MDC lookup when updating ITL metrics. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub decode_worker_type: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub tokenize_latency: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub detokenize_total_latency: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub detokenize_count: Option, -} - -impl LLMMetricAnnotation { - /// Convert this metrics struct to an Annotated event - pub fn to_annotation(&self) -> Result, serde_json::Error> { - Annotated::from_annotation(ANNOTATION_LLM_METRICS, self) - } - - /// Extract LLM metrics from an Annotated event, if present - pub fn from_annotation( - annotation: &Annotated, - ) -> Result, Box> { - if annotation.event.is_none() { - return Ok(None); - } - if annotation.event.as_ref().unwrap() != ANNOTATION_LLM_METRICS { - return Ok(None); - } - let comments = annotation - .comment - .as_ref() - .ok_or("missing comments block")?; - if comments.len() != 1 { - return Err("malformed comments block - expected exactly 1 comment".into()); - } - let metrics: LLMMetricAnnotation = serde_json::from_str(&comments[0])?; - Ok(Some(metrics)) - } -} /// Drain a standalone router's forwarded `routing_data` onto this request's tracker so the /// frontend's timing/worker/token surfaces populate, then drop the field to keep it off the @@ -201,6 +140,34 @@ fn drain_router_routing_data( } } +fn attach_metrics_annotation(response: &mut Annotated, metrics: &LLMMetricAnnotation) { + if let Ok(metrics_annotated) = metrics.to_annotation::<()>() { + response.event = metrics_annotated.event; + response.comment = metrics_annotated.comment; + } else { + tracing::warn!("Failed to serialize LLM metrics annotation"); + } +} + +fn attach_llm_metrics(response: &mut Annotated, metrics: LLMMetricAnnotation) +where + Resp: 'static, +{ + if response.event.is_some() { + return; + } + + if let Some(data) = response.data.as_mut() + && let Some(chat_response) = + (data as &mut dyn Any).downcast_mut::() + { + chat_response.llm_metrics = Some(metrics); + return; + } + + attach_metrics_annotation(response, &metrics); +} + // Reasoning State for reasoning parsing transformation step struct ReasoningState { stream: Pin> + Send>>, @@ -1533,7 +1500,7 @@ impl OpenAIPreprocessor { static DIM_CACHE: LazyLock> = LazyLock::new(|| { Cache::builder() .max_capacity(100_000) - .time_to_live(Duration::from_secs(24 * 60 * 60)) + .time_to_live(std::time::Duration::from_secs(24 * 60 * 60)) .build() }); @@ -1572,7 +1539,7 @@ impl OpenAIPreprocessor { // Per-Range tighter bound than MediaFetcher's 30 s default — dim // fetch is best-effort; on a slow remote we'd rather skip MM // routing for this image than starve the request. - const DIM_FETCH_TIMEOUT: Duration = Duration::from_secs(10); + const DIM_FETCH_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(10); if let Some(rest) = url.strip_prefix("data:") { let comma = rest @@ -2178,13 +2145,7 @@ impl OpenAIPreprocessor { DETOKENIZE_TOKEN_COUNT.inc_by(t.detokenize_count() as f64); } - if let Ok(metrics_annotated) = llm_metrics.to_annotation::<()>() { - // Only set event if not already set to avoid overriding existing events (like errors) - if response.event.is_none() { - response.event = metrics_annotated.event; - response.comment = metrics_annotated.comment; - } - } + attach_llm_metrics(&mut response, llm_metrics); // Mark if we've seen a finish_reason if has_finish_reason { @@ -2261,12 +2222,6 @@ impl OpenAIPreprocessor { DETOKENIZE_TOKEN_COUNT.inc_by(t.detokenize_count() as f64); } - // Create annotation string - let annotation = llm_metrics.to_annotation::<()>().unwrap_or_else(|e| { - tracing::warn!("Failed to serialize metrics: {}", e); - Annotated::<()>::from_data(()) - }); - // Send the usage chunk if needed let data = if inner.response_generator.is_usage_enabled() { Some(usage_chunk) @@ -2274,13 +2229,14 @@ impl OpenAIPreprocessor { None }; - let annotated_usage = Annotated:: { + let mut annotated_usage = Annotated:: { id: None, data, - event: Some(ANNOTATION_LLM_METRICS.to_string()), - comment: annotation.comment, + event: None, + comment: None, error: None, }; + attach_llm_metrics(&mut annotated_usage, llm_metrics); tracing::trace!( request_id = inner.context.id(), diff --git a/lib/llm/src/protocols/anthropic/stream_converter.rs b/lib/llm/src/protocols/anthropic/stream_converter.rs index bcd31efc55e5..ed002c8f0bb7 100644 --- a/lib/llm/src/protocols/anthropic/stream_converter.rs +++ b/lib/llm/src/protocols/anthropic/stream_converter.rs @@ -790,6 +790,7 @@ mod tests { usage: None, }, nvext: None, + llm_metrics: None, } } @@ -832,6 +833,7 @@ mod tests { usage: None, }, nvext: None, + llm_metrics: None, } } @@ -1021,6 +1023,7 @@ mod tests { usage: None, }, nvext: None, + llm_metrics: None, } } diff --git a/lib/llm/src/protocols/common.rs b/lib/llm/src/protocols/common.rs index 1682a870e7d1..49dcd143e52a 100644 --- a/lib/llm/src/protocols/common.rs +++ b/lib/llm/src/protocols/common.rs @@ -25,6 +25,7 @@ const MAX_GRAMMAR_NESTING_DEPTH: usize = 500; pub mod extensions; pub mod llm_backend; +pub mod metrics; pub mod postprocessor; pub mod preprocessor; pub mod timing; diff --git a/lib/llm/src/protocols/common/metrics.rs b/lib/llm/src/protocols/common/metrics.rs new file mode 100644 index 000000000000..ea6b3e6f9f0b --- /dev/null +++ b/lib/llm/src/protocols/common/metrics.rs @@ -0,0 +1,70 @@ +// SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::time::Duration; + +use dynamo_runtime::protocols::annotated::Annotated; + +pub const ANNOTATION_LLM_METRICS: &str = "llm_metrics"; + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq)] +pub struct LLMMetricAnnotation { + pub input_tokens: usize, + pub output_tokens: usize, + pub chunk_tokens: usize, + pub cached_tokens: Option, + /// Prefill worker ID (for TTFT attribution in disaggregated mode) + #[serde(default, skip_serializing_if = "Option::is_none")] + pub prefill_worker_id: Option, + /// Prefill worker DP rank + #[serde(default, skip_serializing_if = "Option::is_none")] + pub prefill_dp_rank: Option, + /// Prefill worker type ("prefill" or "decode") for Prometheus metric labeling. + /// Stored at routing time to avoid expensive MDC lookup when updating TTFT metrics. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub prefill_worker_type: Option, + /// Decode worker ID (for ITL attribution in disaggregated mode) + #[serde(default, skip_serializing_if = "Option::is_none")] + pub decode_worker_id: Option, + /// Decode worker DP rank + #[serde(default, skip_serializing_if = "Option::is_none")] + pub decode_dp_rank: Option, + /// Decode worker type ("prefill" or "decode") for Prometheus metric labeling. + /// Stored at routing time to avoid expensive MDC lookup when updating ITL metrics. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub decode_worker_type: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tokenize_latency: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub detokenize_total_latency: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub detokenize_count: Option, +} + +impl LLMMetricAnnotation { + /// Convert this metrics struct to an Annotated event. + pub fn to_annotation(&self) -> Result, serde_json::Error> { + Annotated::from_annotation(ANNOTATION_LLM_METRICS, self) + } + + /// Extract LLM metrics from an Annotated event, if present. + pub fn from_annotation( + annotation: &Annotated, + ) -> Result, Box> { + if annotation.event.is_none() { + return Ok(None); + } + if annotation.event.as_ref().unwrap() != ANNOTATION_LLM_METRICS { + return Ok(None); + } + let comments = annotation + .comment + .as_ref() + .ok_or("missing comments block")?; + if comments.len() != 1 { + return Err("malformed comments block - expected exactly 1 comment".into()); + } + let metrics: LLMMetricAnnotation = serde_json::from_str(&comments[0])?; + Ok(Some(metrics)) + } +} diff --git a/lib/llm/src/protocols/openai/chat_completions.rs b/lib/llm/src/protocols/openai/chat_completions.rs index 2066fe6f5ad3..3b9b27643cfc 100644 --- a/lib/llm/src/protocols/openai/chat_completions.rs +++ b/lib/llm/src/protocols/openai/chat_completions.rs @@ -201,6 +201,10 @@ pub struct NvCreateChatCompletionStreamResponse { pub inner: dynamo_protocols::types::CreateChatCompletionStreamResponse, #[serde(skip_serializing_if = "Option::is_none")] pub nvext: Option, + /// Internal frontend metrics payload. This must never be serialized to + /// client-facing OpenAI-compatible streams. + #[serde(skip)] + pub llm_metrics: Option, } /// Implements `NvExtProvider` for `NvCreateChatCompletionRequest`, diff --git a/lib/llm/src/protocols/openai/chat_completions/aggregator.rs b/lib/llm/src/protocols/openai/chat_completions/aggregator.rs index 2b96f4561302..fc8fe3630cd7 100644 --- a/lib/llm/src/protocols/openai/chat_completions/aggregator.rs +++ b/lib/llm/src/protocols/openai/chat_completions/aggregator.rs @@ -610,6 +610,7 @@ mod tests { object: "chat.completion".to_string(), }, nvext: None, + llm_metrics: None, }; Annotated { @@ -658,6 +659,7 @@ mod tests { object: "chat.completion".to_string(), }, nvext: None, + llm_metrics: None, }; Annotated { data: Some(data), @@ -1173,6 +1175,7 @@ mod tests { object: "chat.completion".to_string(), }, nvext: None, + llm_metrics: None, }; // Wrap it in Annotated and create a stream diff --git a/lib/llm/src/protocols/openai/chat_completions/delta.rs b/lib/llm/src/protocols/openai/chat_completions/delta.rs index 25f3eb83a5c8..dbe4f12652ec 100644 --- a/lib/llm/src/protocols/openai/chat_completions/delta.rs +++ b/lib/llm/src/protocols/openai/chat_completions/delta.rs @@ -190,6 +190,7 @@ impl DeltaGenerator { service_tier: self.service_tier.clone(), }, nvext: None, // Will be populated by router layer if needed + llm_metrics: None, } } @@ -213,6 +214,7 @@ impl DeltaGenerator { service_tier: self.service_tier.clone(), }, nvext: None, + llm_metrics: None, } } diff --git a/lib/llm/src/protocols/openai/chat_completions/jail.rs b/lib/llm/src/protocols/openai/chat_completions/jail.rs index 91e6f2f0af53..543c19c694c7 100644 --- a/lib/llm/src/protocols/openai/chat_completions/jail.rs +++ b/lib/llm/src/protocols/openai/chat_completions/jail.rs @@ -847,6 +847,7 @@ impl JailedStream { system_fingerprint: None, }, nvext: None, + llm_metrics: None, }; let final_metadata = (last_annotated_id, last_annotated_event, last_annotated_comment); @@ -1765,6 +1766,7 @@ mod tests { system_fingerprint: None, }, nvext: None, + llm_metrics: None, }), id: None, event: None, @@ -1865,6 +1867,7 @@ mod tests { system_fingerprint: None, }, nvext: None, + llm_metrics: None, }), id: None, event: None, diff --git a/lib/llm/src/protocols/openai/responses/stream_converter.rs b/lib/llm/src/protocols/openai/responses/stream_converter.rs index 42e4607a6772..883626ba358d 100644 --- a/lib/llm/src/protocols/openai/responses/stream_converter.rs +++ b/lib/llm/src/protocols/openai/responses/stream_converter.rs @@ -925,6 +925,7 @@ mod tests { usage: None, }, nvext: None, + llm_metrics: None, } } @@ -954,6 +955,7 @@ mod tests { usage: None, }, nvext: None, + llm_metrics: None, } } diff --git a/lib/llm/src/request_trace/agent_context.rs b/lib/llm/src/request_trace/agent_context.rs index 103d4677ca74..d83eca5e1a4c 100644 --- a/lib/llm/src/request_trace/agent_context.rs +++ b/lib/llm/src/request_trace/agent_context.rs @@ -676,6 +676,7 @@ mod tests { usage: None, }, nvext: None, + llm_metrics: None, }), Annotated::from_data(NvCreateChatCompletionStreamResponse { inner: CreateChatCompletionStreamResponse { @@ -701,6 +702,7 @@ mod tests { usage: None, }, nvext: None, + llm_metrics: None, }), ]; for response in &responses { diff --git a/lib/llm/tests/aggregators.rs b/lib/llm/tests/aggregators.rs index 367522b89118..765e48c3c7f8 100644 --- a/lib/llm/tests/aggregators.rs +++ b/lib/llm/tests/aggregators.rs @@ -185,6 +185,7 @@ fn make_stream_delta( usage: None, }, nvext, + llm_metrics: None, }) } diff --git a/lib/llm/tests/logprob_analysis_integration.rs b/lib/llm/tests/logprob_analysis_integration.rs index f684991fbf5d..ca50aed425a4 100644 --- a/lib/llm/tests/logprob_analysis_integration.rs +++ b/lib/llm/tests/logprob_analysis_integration.rs @@ -406,6 +406,7 @@ fn create_response_with_linear_probs( usage: None, }, nvext: None, + llm_metrics: None, } } @@ -488,5 +489,6 @@ fn create_multi_choice_response( usage: None, }, nvext: None, + llm_metrics: None, } } diff --git a/lib/llm/tests/postprocessor_parsing_stream.rs b/lib/llm/tests/postprocessor_parsing_stream.rs index 69fd0df1569c..5e575bec7190 100644 --- a/lib/llm/tests/postprocessor_parsing_stream.rs +++ b/lib/llm/tests/postprocessor_parsing_stream.rs @@ -304,6 +304,7 @@ fn mock_content_chunk(content: &str) -> NvCreateChatCompletionStreamResponse { service_tier: None, }, nvext: None, + llm_metrics: None, } } @@ -346,6 +347,7 @@ fn mock_multi_choice_content_chunk( service_tier: None, }, nvext: None, + llm_metrics: None, } } @@ -385,6 +387,7 @@ fn mock_reasoning_only_chunk(reasoning: &str) -> NvCreateChatCompletionStreamRes service_tier: None, }, nvext: None, + llm_metrics: None, } } @@ -419,6 +422,7 @@ fn mock_final_chunk() -> NvCreateChatCompletionStreamResponse { service_tier: None, }, nvext: None, + llm_metrics: None, } } diff --git a/lib/llm/tests/test_jail.rs b/lib/llm/tests/test_jail.rs index 7d7af707c9b8..21e02b8411af 100644 --- a/lib/llm/tests/test_jail.rs +++ b/lib/llm/tests/test_jail.rs @@ -60,6 +60,7 @@ mod tests { service_tier: None, }, nvext: None, + llm_metrics: None, }; Annotated { @@ -102,6 +103,7 @@ mod tests { service_tier: None, }, nvext: None, + llm_metrics: None, }; Annotated { @@ -148,6 +150,7 @@ mod tests { service_tier: None, }, nvext: None, + llm_metrics: None, }; Annotated { @@ -195,6 +198,7 @@ mod tests { service_tier: None, }, nvext: None, + llm_metrics: None, }; Annotated { @@ -242,6 +246,7 @@ mod tests { service_tier: None, }, nvext: None, + llm_metrics: None, }; Annotated { @@ -2556,6 +2561,7 @@ mod parallel_jail_tests { service_tier: None, }, nvext: None, + llm_metrics: None, }; Annotated { diff --git a/lib/llm/tests/test_reasoning_parser.rs b/lib/llm/tests/test_reasoning_parser.rs index 7cbe805abfd2..18cd3ef82714 100644 --- a/lib/llm/tests/test_reasoning_parser.rs +++ b/lib/llm/tests/test_reasoning_parser.rs @@ -49,6 +49,7 @@ fn create_mock_response_chunk( service_tier: None, }, nvext: None, + llm_metrics: None, }; Annotated { diff --git a/lib/llm/tests/test_streaming_tool_parsers.rs b/lib/llm/tests/test_streaming_tool_parsers.rs index ab2c2ebd2b2c..9e276bfacb64 100644 --- a/lib/llm/tests/test_streaming_tool_parsers.rs +++ b/lib/llm/tests/test_streaming_tool_parsers.rs @@ -118,6 +118,7 @@ fn load_test_data(file_path: &str) -> TestData { service_tier: None, }, nvext: None, + llm_metrics: None, }; Annotated { @@ -1425,6 +1426,7 @@ mod tests { service_tier: None, }, nvext: None, + llm_metrics: None, }), event: None, comment: None, diff --git a/lib/llm/tests/tool_choice.rs b/lib/llm/tests/tool_choice.rs index 1459b1ba0ef8..1d441c867554 100644 --- a/lib/llm/tests/tool_choice.rs +++ b/lib/llm/tests/tool_choice.rs @@ -510,6 +510,7 @@ fn make_text_chunk( service_tier: None, }, nvext: None, + llm_metrics: None, } }