Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 18 additions & 0 deletions crates/aisix-obs/src/otlp_http_sink.rs
Original file line number Diff line number Diff line change
Expand Up @@ -312,6 +312,9 @@ fn build_otlp_traces_payload(event: &UsageEvent, exporter_name: &str) -> Value {
}
attributes.push(attr_string("aisix.exporter_name", exporter_name));
attributes.push(attr_string("aisix.request_id", &event.request_id));
if event.ttft_ms > 0 {
attributes.push(attr_int("aisix.ttft_ms", event.ttft_ms as i64));
}

json!({
"resourceSpans": [{
Expand Down Expand Up @@ -557,6 +560,21 @@ mod tests {
assert!(!keys.contains(&"gen_ai.response.id"));
assert!(!keys.contains(&"gen_ai.response.model"));
assert!(!keys.contains(&"gen_ai.response.finish_reasons"));
// ttft_ms = 0 (default) → omitted
assert!(!keys.contains(&"aisix.ttft_ms"));
}

#[test]
fn payload_includes_ttft_when_set() {
let mut ev = sample_event();
ev.ttft_ms = 42;
let body = build_otlp_traces_payload(&ev, "test-exp");
let attrs = body["resourceSpans"][0]["scopeSpans"][0]["spans"][0]["attributes"]
.as_array()
.unwrap();
let ttft_attr = attrs.iter().find(|a| a["key"] == "aisix.ttft_ms");
assert!(ttft_attr.is_some(), "aisix.ttft_ms should be present");
assert_eq!(ttft_attr.unwrap()["value"]["intValue"], "42");
}

#[test]
Expand Down
10 changes: 10 additions & 0 deletions crates/aisix-obs/src/usage.rs
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,13 @@ pub struct UsageEvent {

pub latency_ms: u32,

/// Time to first token in milliseconds. Only meaningful on the
/// streaming path — measures elapsed time from request entry to
/// the first upstream SSE chunk. 0 on non-streaming, error, and
/// cache-hit paths (omitted from the wire via skip_serializing_if).
#[serde(default, skip_serializing_if = "is_zero_u32")]
pub ttft_ms: u32,

/// HTTP status code the proxy returned to the downstream caller.
pub status_code: u16,

Expand Down Expand Up @@ -310,6 +317,7 @@ mod tests {
assert!(!json.contains("provider_request_id"));
assert!(!json.contains("provider_model_version"));
assert!(!json.contains("finish_reason"));
assert!(!json.contains("ttft_ms"));
}

#[test]
Expand All @@ -325,6 +333,7 @@ mod tests {
provider_request_id: "chatcmpl-abc".into(),
provider_model_version: "gpt-4o-2024-08-06".into(),
finish_reason: "stop".into(),
ttft_ms: 123,
..Default::default()
};
let json = serde_json::to_string(&ev).unwrap();
Expand All @@ -335,6 +344,7 @@ mod tests {
assert!(json.contains(r#""provider_request_id":"chatcmpl-abc""#));
assert!(json.contains(r#""provider_model_version":"gpt-4o-2024-08-06""#));
assert!(json.contains(r#""finish_reason":"stop""#));
assert!(json.contains(r#""ttft_ms":123"#));
}

fn sample_event(id: &str) -> UsageEvent {
Expand Down
16 changes: 16 additions & 0 deletions crates/aisix-proxy/src/chat.rs
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,7 @@ pub async fn chat_completions(
cache_status: success.cache_status.as_str().to_string(),
cache_hit_saved_input_tokens: success.cache_hit_saved_input_tokens,
cache_hit_saved_output_tokens: success.cache_hit_saved_output_tokens,
ttft_ms: 0,
},
success.cost_usd,
/* guardrail_blocked */ false,
Expand Down Expand Up @@ -212,6 +213,7 @@ pub async fn chat_completions(
cache_status: c.cache_status.as_str().to_string(),
cache_hit_saved_input_tokens: 0,
cache_hit_saved_output_tokens: 0,
ttft_ms: 0,
},
),
None => (0, 0, UsageExtras::default()),
Expand Down Expand Up @@ -567,6 +569,7 @@ async fn dispatch(
upstream,
now,
stream_guardrail,
started,
move |comp: StreamCompletion| {
// Rate-limit accounting (TPM cap) for all layers.
for key in &post_stream_keys {
Expand Down Expand Up @@ -613,6 +616,7 @@ async fn dispatch(
cache_status: CacheStatus::Disabled.as_str().to_string(),
cache_hit_saved_input_tokens: 0,
cache_hit_saved_output_tokens: 0,
ttft_ms: comp.ttft_ms,
},
/* cost_usd */ 0.0,
comp.guardrail_blocked,
Expand Down Expand Up @@ -1164,6 +1168,7 @@ fn emit_usage_event(
cache_status: extras.cache_status,
cache_hit_saved_input_tokens: extras.cache_hit_saved_input_tokens,
cache_hit_saved_output_tokens: extras.cache_hit_saved_output_tokens,
ttft_ms: extras.ttft_ms,
// chat.rs is the OpenAI-shape /v1/chat/completions handler.
// /v1/responses / /v1/embeddings / /v1/audio* / /v1/images* /
// /v1/rerank don't emit UsageEvents today; when they do they
Expand Down Expand Up @@ -1212,6 +1217,7 @@ struct UsageExtras {
/// ingest from these + its pricing catalog (see #88).
cache_hit_saved_input_tokens: u32,
cache_hit_saved_output_tokens: u32,
ttft_ms: u32,
}

fn record_error(metrics: &Metrics, err: &ProxyError, model: &str, status: u16, elapsed: Duration) {
Expand Down Expand Up @@ -1316,6 +1322,9 @@ struct StreamCompletion {
/// fail-opened on a streamed response. Empty string = no bypass.
/// First-bypass-wins matches the non-streaming convention.
bypass_reason: String,
/// Time to first token in milliseconds. Set once when the first
/// Ok(chunk) arrives in `build_sse_stream`.
ttft_ms: u32,
}

/// Parameters needed to run output-guardrail evaluation at
Expand Down Expand Up @@ -1375,6 +1384,7 @@ fn build_sse_stream<F>(
upstream: aisix_gateway::ChatChunkStream,
created: i64,
output_guardrail: Option<StreamGuardrailContext>,
started: Instant,
on_complete: F,
) -> impl Stream<Item = Result<Event, Infallible>>
where
Expand Down Expand Up @@ -1416,9 +1426,15 @@ where
// treats `[DONE]` as clean-completion would mis-interpret
// the truncated response as a successful one.
let mut errored = false;
let mut first_chunk_seen = false;
while let Some(item) = upstream.next().await {
let ev = match item {
Ok(chunk) => {
if !first_chunk_seen {
first_chunk_seen = true;
Comment thread
nic-6443 marked this conversation as resolved.
Resolved
guard.comp().ttft_ms =
started.elapsed().as_millis().min(u32::MAX as u128) as u32;
}
let comp = guard.comp();
if !chunk.id.is_empty() {
comp.provider_request_id = chunk.id.clone();
Expand Down
Loading