From a3455fd8b84d0d5eb9483ec75cca597c158fe460 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Wed, 30 Sep 2026 09:41:44 -0700 Subject: [PATCH 01/12] feat(tracing): add tenant scoped agent tracing integration --- litellm-rust/Cargo.lock | 67 +- litellm-rust/crates/python-bridge/src/lib.rs | 7 +- .../crates/python-bridge/src/routes/traces.rs | 163 +- litellm-rust/crates/traces/Cargo.toml | 4 + litellm-rust/crates/traces/src/error.rs | 14 + litellm-rust/crates/traces/src/insert.rs | 137 +- litellm-rust/crates/traces/src/lib.rs | 35 +- litellm-rust/crates/traces/src/otlp.rs | 221 + .../crates/traces/tests/migrations.rs | 2 +- litellm-rust/crates/traces/tests/otlp.rs | 47 + litellm/constants.py | 13 + .../clickhouse/clickhouse_batch_logger.py | 101 + litellm/integrations/clickhouse/schema.py | 11 + litellm/proxy/_types.py | 4 + .../proxy/common_utils/http_parsing_utils.py | 18 +- litellm/proxy/proxy_server.py | 23 + litellm/proxy/tracing_endpoints.py | 120 + litellm/rust_bridge/_native.pyi | 28 +- litellm/rust_bridge/traces.py | 105 +- litellm/tracing/AGENTS.md | 6 + litellm/tracing/__init__.py | 16 + litellm/tracing/decode.py | 245 + litellm/tracing/receiver.py | 117 + litellm/tracing/store.py | 271 ++ litellm/tracing/types.py | 119 + .../test_clickhouse_batch_logger.py | 71 + .../common_utils/test_http_parsing_utils.py | 12 + .../proxy/test_tracing_endpoints.py | 160 + .../fixtures/langsmith_deep_agent_export.json | 924 ++++ tests/test_litellm/tracing/test_decode.py | 301 ++ tests/test_litellm/tracing/test_receiver.py | 120 + tests/test_litellm/tracing/test_store.py | 297 ++ tests/test_litellm_rust/test_traces.py | 42 +- .../src/components/networking.tsx | 28 + .../components/view_logs/RequestLogsPanel.tsx | 95 +- .../view_logs/TraceView/AgentGraph.tsx | 207 + .../TraceView/AgentTracesSection.test.tsx | 80 + .../TraceView/AgentTracesSection.tsx | 98 + .../view_logs/TraceView/AgentTracesTable.tsx | 256 ++ .../view_logs/TraceView/SpanDetail.tsx | 148 + .../view_logs/TraceView/SpanTree.tsx | 198 + .../view_logs/TraceView/TraceDrawer.test.tsx | 109 + .../view_logs/TraceView/TraceDrawer.tsx | 313 ++ .../view_logs/TraceView/TracePills.tsx | 58 + .../view_logs/TraceView/TraceSteps.tsx | 107 + .../view_logs/TraceView/TracingSetupCard.tsx | 33 + .../__fixtures__/deep_agent_trace.json | 2037 +++++++++ .../__fixtures__/research_trace.json | 3606 +++++++++++++++ .../TraceView/__fixtures__/swarm_trace.json | 3960 +++++++++++++++++ .../TraceView/__fixtures__/trace_list.json | 59 + .../view_logs/TraceView/traceTypes.ts | 89 + .../view_logs/TraceView/traceUtils.test.ts | 262 ++ .../view_logs/TraceView/traceUtils.ts | 513 +++ .../view_logs/TraceView/useAgentTraces.ts | 79 + .../view_logs/TraceView/useTraceNavigation.ts | 33 + 55 files changed, 15985 insertions(+), 204 deletions(-) create mode 100644 litellm-rust/crates/traces/src/otlp.rs create mode 100644 litellm-rust/crates/traces/tests/otlp.rs create mode 100644 litellm/integrations/clickhouse/clickhouse_batch_logger.py create mode 100644 litellm/integrations/clickhouse/schema.py create mode 100644 litellm/proxy/tracing_endpoints.py create mode 100644 litellm/tracing/AGENTS.md create mode 100644 litellm/tracing/__init__.py create mode 100644 litellm/tracing/decode.py create mode 100644 litellm/tracing/receiver.py create mode 100644 litellm/tracing/store.py create mode 100644 litellm/tracing/types.py create mode 100644 tests/test_litellm/integrations/clickhouse/test_clickhouse_batch_logger.py create mode 100644 tests/test_litellm/proxy/test_tracing_endpoints.py create mode 100644 tests/test_litellm/tracing/fixtures/langsmith_deep_agent_export.json create mode 100644 tests/test_litellm/tracing/test_decode.py create mode 100644 tests/test_litellm/tracing/test_receiver.py create mode 100644 tests/test_litellm/tracing/test_store.py create mode 100644 ui/litellm-dashboard/src/components/view_logs/TraceView/AgentGraph.tsx create mode 100644 ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesSection.test.tsx create mode 100644 ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesSection.tsx create mode 100644 ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesTable.tsx create mode 100644 ui/litellm-dashboard/src/components/view_logs/TraceView/SpanDetail.tsx create mode 100644 ui/litellm-dashboard/src/components/view_logs/TraceView/SpanTree.tsx create mode 100644 ui/litellm-dashboard/src/components/view_logs/TraceView/TraceDrawer.test.tsx create mode 100644 ui/litellm-dashboard/src/components/view_logs/TraceView/TraceDrawer.tsx create mode 100644 ui/litellm-dashboard/src/components/view_logs/TraceView/TracePills.tsx create mode 100644 ui/litellm-dashboard/src/components/view_logs/TraceView/TraceSteps.tsx create mode 100644 ui/litellm-dashboard/src/components/view_logs/TraceView/TracingSetupCard.tsx create mode 100644 ui/litellm-dashboard/src/components/view_logs/TraceView/__fixtures__/deep_agent_trace.json create mode 100644 ui/litellm-dashboard/src/components/view_logs/TraceView/__fixtures__/research_trace.json create mode 100644 ui/litellm-dashboard/src/components/view_logs/TraceView/__fixtures__/swarm_trace.json create mode 100644 ui/litellm-dashboard/src/components/view_logs/TraceView/__fixtures__/trace_list.json create mode 100644 ui/litellm-dashboard/src/components/view_logs/TraceView/traceTypes.ts create mode 100644 ui/litellm-dashboard/src/components/view_logs/TraceView/traceUtils.test.ts create mode 100644 ui/litellm-dashboard/src/components/view_logs/TraceView/traceUtils.ts create mode 100644 ui/litellm-dashboard/src/components/view_logs/TraceView/useAgentTraces.ts create mode 100644 ui/litellm-dashboard/src/components/view_logs/TraceView/useTraceNavigation.ts diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index 5fdf0266e8c8..474f45ac0cdb 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -1274,6 +1274,18 @@ version = "0.4.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e8ccc4ea9f6acc32d102c0f6d471d11d913ad15f20c04de743374861fa1d414" +[[package]] +name = "const-hex" +version = "1.19.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e59eef12462b0f9b0a3620219be5d639afd79fe39dff0a42c3997061f9298b4" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "proptest", + "serde_core", +] + [[package]] name = "const-oid" version = "0.9.6" @@ -2372,9 +2384,9 @@ dependencies = [ "http-body-util", "hyper 1.10.1", "lazy_static", - "opentelemetry", + "opentelemetry 0.32.0", "opentelemetry-semantic-conventions", - "opentelemetry_sdk", + "opentelemetry_sdk 0.32.1", "percent-encoding", "pin-project", "prost", @@ -4356,7 +4368,11 @@ dependencies = [ name = "litellm-traces" version = "0.1.0" dependencies = [ + "base64 0.22.1", + "flate2", "litellm-http", + "opentelemetry-proto", + "prost", "rstest", "serde", "serde_json", @@ -4776,6 +4792,33 @@ dependencies = [ "tracing", ] +[[package]] +name = "opentelemetry" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cdb0b1b267eb9db3331b434ed9ddab10d50e280a9adf9d13e5233e2002b61b5" +dependencies = [ + "futures-core", + "futures-sink", + "js-sys", + "pin-project-lite", + "thiserror 2.0.19", +] + +[[package]] +name = "opentelemetry-proto" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25da1ac11a0aeccf38d7f77ee0348715adaf8340f65ad46c94a02c6b20e2f65d" +dependencies = [ + "base64 0.22.1", + "const-hex", + "opentelemetry 0.33.0", + "opentelemetry_sdk 0.33.0", + "prost", + "serde", +] + [[package]] name = "opentelemetry-semantic-conventions" version = "0.32.1" @@ -4791,7 +4834,23 @@ dependencies = [ "futures-channel", "futures-executor", "futures-util", - "opentelemetry", + "opentelemetry 0.32.0", + "percent-encoding", + "portable-atomic", + "rand 0.9.5", + "thiserror 2.0.19", +] + +[[package]] +name = "opentelemetry_sdk" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb39533d9d1c912123efd7d41d7e0c29d16917b60ce15b4c8d87cb1af7f67520" +dependencies = [ + "futures-channel", + "futures-executor", + "futures-util", + "opentelemetry 0.33.0", "percent-encoding", "portable-atomic", "rand 0.9.5", @@ -7524,7 +7583,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "adbc64cba7137545b8044cb1fe9814f7aacf3c6b5f9b45be8bb5db538befdb26" dependencies = [ "js-sys", - "opentelemetry", + "opentelemetry 0.32.0", "tracing", "tracing-core", "tracing-subscriber", diff --git a/litellm-rust/crates/python-bridge/src/lib.rs b/litellm-rust/crates/python-bridge/src/lib.rs index f65cb350cfe6..0d4df996552a 100644 --- a/litellm-rust/crates/python-bridge/src/lib.rs +++ b/litellm-rust/crates/python-bridge/src/lib.rs @@ -44,7 +44,7 @@ mod _native { #[pymodule_export] use crate::routes::token_counter::TokenCounter; #[pymodule_export] - use crate::routes::traces::{trace_encode_rows, trace_ensure_schema, trace_query}; + use crate::routes::traces::{NativeTraceStorage, trace_decode_otlp}; #[cfg(feature = "huggingface")] #[pymodule_export] use crate::tokenizer::HuggingFaceEncoding; @@ -109,9 +109,8 @@ mod tests { "aresponses", "ResponsesWebSocketConnection", "NativeDiagnosticProcessor", - "trace_encode_rows", - "trace_ensure_schema", - "trace_query", + "NativeTraceStorage", + "trace_decode_otlp", "TokenCounter", "Tokenizer", "gil_stats", diff --git a/litellm-rust/crates/python-bridge/src/routes/traces.rs b/litellm-rust/crates/python-bridge/src/routes/traces.rs index fdd2a7eb93d8..3f2589e4b0ea 100644 --- a/litellm-rust/crates/python-bridge/src/routes/traces.rs +++ b/litellm-rust/crates/python-bridge/src/routes/traces.rs @@ -1,19 +1,21 @@ use std::collections::BTreeMap; use litellm_http::ClientVariant; -use litellm_traces::{Connection, Error, Parameter}; +use litellm_traces::{Connection, Error, InsertTable, Parameter}; use pyo3::{ - exceptions::{PyRuntimeError, PyValueError}, + exceptions::{PyOverflowError, PyRuntimeError, PyValueError}, prelude::*, }; fn map_error(error: Error) -> PyErr { match error { - Error::InvalidRow | Error::InvalidSchema | Error::EmptySql => { + Error::InvalidRow | Error::InvalidTable | Error::InvalidSchema | Error::EmptySql => { PyValueError::new_err(error.to_string()) } + Error::InsertTooLarge => PyOverflowError::new_err(error.to_string()), Error::InvalidUrl | Error::QueryFailed(_) + | Error::InsertFailed(_) | Error::SchemaFailed(_) | Error::ResponseTooLarge | Error::InvalidResponse @@ -21,63 +23,112 @@ fn map_error(error: Error) -> PyErr { } } -#[pyfunction] -pub fn trace_ensure_schema<'py>( - py: Python<'py>, - url: &str, +#[pyclass] +pub struct NativeTraceStorage { database: String, - user: &str, - password: &str, - trace_retention_days: u32, - spend_log_retention_days: u32, -) -> PyResult> { - let connection = Connection::writer(url, user, password).map_err(map_error)?; - let client = crate::http::host_client(py, ClientVariant::NoRedirect)?; - crate::execution::run_async( - py, - async move { - litellm_traces::ensure_schema( - &client, - &connection, - &database, - trace_retention_days, - spend_log_retention_days, - ) - .await - }, - map_error, - ) + writer: Connection, + reader: Option, } -#[pyfunction] -pub fn trace_query<'py>( - py: Python<'py>, - url: &str, - database: &str, - user: &str, - password: &str, - sql: String, - #[pyo3(from_py_with = litellm_host_python::from_py_argument)] parameters: BTreeMap< - String, - Parameter, - >, -) -> PyResult> { - let connection = Connection::configured(url, database, user, password).map_err(map_error)?; - let client = crate::http::host_client(py, ClientVariant::NoRedirect)?; - crate::execution::run_async( - py, - async move { litellm_traces::execute_read(&client, &connection, &sql, ¶meters).await }, - map_error, - ) +#[pymethods] +impl NativeTraceStorage { + #[new] + fn new(database: String, url: &str, reader_url: Option<&str>) -> PyResult { + litellm_traces::schema_statements(&database, 1, 1).map_err(map_error)?; + Ok(Self { + writer: Connection::writer(url).map_err(map_error)?, + reader: reader_url + .map(|value| Connection::reader(value, &database)) + .transpose() + .map_err(map_error)?, + database, + }) + } + + fn ensure_schema<'py>( + &self, + py: Python<'py>, + trace_retention_days: u32, + spend_log_retention_days: u32, + ) -> PyResult> { + let client = crate::http::host_client(py, ClientVariant::NoRedirect)?; + let connection = self.writer.clone(); + let database = self.database.clone(); + crate::execution::run_async( + py, + async move { + litellm_traces::ensure_schema( + &client, + &connection, + &database, + trace_retention_days, + spend_log_retention_days, + ) + .await + }, + map_error, + ) + } + + fn insert_rows<'py>( + &self, + py: Python<'py>, + table: &str, + #[pyo3(from_py_with = litellm_host_python::from_py_argument)] rows: Vec< + BTreeMap, + >, + ) -> PyResult> { + let table = InsertTable::parse(table).map_err(map_error)?; + let client = crate::http::host_client(py, ClientVariant::NoRedirect)?; + let connection = self.writer.clone(); + let database = self.database.clone(); + crate::execution::run_async( + py, + async move { + litellm_traces::insert_rows(&client, &connection, &database, table, rows).await + }, + map_error, + ) + } + + fn query<'py>( + &self, + py: Python<'py>, + sql: String, + #[pyo3(from_py_with = litellm_host_python::from_py_argument)] parameters: BTreeMap< + String, + Parameter, + >, + ) -> PyResult> { + let connection = self.reader.clone().ok_or_else(|| { + PyRuntimeError::new_err("Trace reads require a separate ClickHouse reader URL") + })?; + let client = crate::http::host_client(py, ClientVariant::NoRedirect)?; + crate::execution::run_async( + py, + async move { litellm_traces::execute_read(&client, &connection, &sql, ¶meters).await }, + map_error, + ) + } } #[pyfunction] -pub fn trace_encode_rows( - py: Python<'_>, - #[pyo3(from_py_with = litellm_host_python::from_py_argument)] rows: Vec< - BTreeMap, - >, -) -> PyResult { - py.detach(|| litellm_traces::encode_rows(rows)) - .map_err(map_error) +pub fn trace_decode_otlp<'py>( + py: Python<'py>, + body: &[u8], + content_type: Option<&str>, + content_encoding: Option<&str>, + max_decompressed_bytes: usize, +) -> PyResult> { + let spans = py + .detach(|| { + litellm_traces::decode_otlp( + body, + content_type, + content_encoding, + max_decompressed_bytes, + ) + }) + .map_err(|error| PyValueError::new_err(error.to_string()))?; + litellm_host_python::Pythonized(spans).into_pyobject(py) } diff --git a/litellm-rust/crates/traces/Cargo.toml b/litellm-rust/crates/traces/Cargo.toml index a8091cf9ec80..0aeec4c8276e 100644 --- a/litellm-rust/crates/traces/Cargo.toml +++ b/litellm-rust/crates/traces/Cargo.toml @@ -6,6 +6,10 @@ license.workspace = true repository.workspace = true [dependencies] +base64.workspace = true +flate2.workspace = true +opentelemetry-proto = { version = "0.33.0", default-features = false, features = ["gen-tonic-messages", "trace", "with-serde"] } +prost = "0.14.4" time = { workspace = true, features = ["formatting"] } litellm-http.workspace = true serde.workspace = true diff --git a/litellm-rust/crates/traces/src/error.rs b/litellm-rust/crates/traces/src/error.rs index a06eb27397a9..125edc354224 100644 --- a/litellm-rust/crates/traces/src/error.rs +++ b/litellm-rust/crates/traces/src/error.rs @@ -2,6 +2,8 @@ pub enum Error { #[error("invalid ClickHouse insert row")] InvalidRow, + #[error("invalid ClickHouse insert table")] + InvalidTable, #[error("invalid ClickHouse HTTP URL")] InvalidUrl, #[error("database must be a nonempty SQL identifier and retention must be positive")] @@ -10,6 +12,10 @@ pub enum Error { EmptySql, #[error("ClickHouse query failed with HTTP status {0}")] QueryFailed(u16), + #[error("ClickHouse insert failed with HTTP status {0}")] + InsertFailed(u16), + #[error("ClickHouse insert exceeds the encoded size limit")] + InsertTooLarge, #[error("ClickHouse schema setup failed with HTTP status {0}")] SchemaFailed(u16), #[error("ClickHouse query exceeded the response size limit")] @@ -19,3 +25,11 @@ pub enum Error { #[error("ClickHouse query transport failed")] Transport, } + +#[derive(Debug, thiserror::Error)] +pub enum DecodeError { + #[error("invalid OTLP trace payload")] + InvalidPayload, + #[error("OTLP trace payload exceeds the decompressed size limit")] + TooLarge, +} diff --git a/litellm-rust/crates/traces/src/insert.rs b/litellm-rust/crates/traces/src/insert.rs index 05b14651ce07..e6b58903964e 100644 --- a/litellm-rust/crates/traces/src/insert.rs +++ b/litellm-rust/crates/traces/src/insert.rs @@ -1,21 +1,108 @@ -use std::collections::BTreeMap; +use std::{collections::BTreeMap, io::Write, time::Duration}; +use flate2::{Compression, write::GzEncoder}; +use litellm_http::Client; use serde_json::Value; use time::{OffsetDateTime, format_description::well_known::Rfc3339}; -use crate::Error; +use crate::{Connection, Error}; + +const MAX_INSERT_BYTES: usize = 64 * 1024 * 1024; +const INSERT_TIMEOUT: Duration = Duration::from_secs(30); + +pub enum InsertTable { + OtelTraces, + SpendLogs, +} + +impl InsertTable { + pub fn parse(value: &str) -> Result { + match value { + "otel_traces" => Ok(Self::OtelTraces), + "spend_logs" => Ok(Self::SpendLogs), + _ => Err(Error::InvalidTable), + } + } + + fn name(&self) -> &'static str { + match self { + Self::OtelTraces => "otel_traces", + Self::SpendLogs => "spend_logs", + } + } +} + +pub async fn insert_rows( + client: &Client, + connection: &Connection, + database: &str, + table: InsertTable, + rows: Vec>, +) -> Result<(), Error> { + if rows.is_empty() { + return Ok(()); + } + let encoded = encode_rows_with_limit(rows, MAX_INSERT_BYTES)?; + let mut encoder = GzEncoder::new(Vec::new(), Compression::default()); + encoder + .write_all(encoded.as_bytes()) + .map_err(|_| Error::InvalidRow)?; + let body = encoder.finish().map_err(|_| Error::InvalidRow)?; + let mut url = connection.url().clone(); + url.query_pairs_mut() + .append_pair( + "query", + &format!( + "INSERT INTO `{database}`.{} FORMAT JSONEachRow", + table.name() + ), + ) + .append_pair("async_insert", "1") + .append_pair("wait_for_async_insert", "1") + .append_pair("date_time_input_format", "best_effort"); + let response = client + .post(url) + .timeout(INSERT_TIMEOUT) + .header("Content-Encoding", "gzip") + .body(body) + .send() + .await + .map_err(|_| Error::Transport)?; + if !response.status().is_success() { + return Err(Error::InsertFailed(response.status().as_u16())); + } + Ok(()) +} pub fn encode_rows(rows: Vec>) -> Result { - rows.into_iter() - .map(|row| { - let encoded = row - .into_iter() - .map(|(name, value)| insert_value(&name, value).map(|value| (name, value))) - .collect::, _>>()?; - serde_json::to_string(&encoded).map_err(|_| Error::InvalidRow) - }) - .collect::, _>>() - .map(|rows| rows.join("\n")) + encode_rows_with_limit(rows, usize::MAX) +} + +fn encode_rows_with_limit( + rows: Vec>, + limit: usize, +) -> Result { + let mut body = Vec::new(); + for row in rows { + let encoded = row + .into_iter() + .map(|(name, value)| insert_value(&name, value).map(|value| (name, value))) + .collect::, _>>()?; + let record = serde_json::to_vec(&encoded).map_err(|_| Error::InvalidRow)?; + let size = body + .len() + .checked_add(record.len()) + .and_then(|size| size.checked_add(usize::from(!body.is_empty()))) + .ok_or(Error::InsertTooLarge)?; + if size > limit { + return Err(Error::InsertTooLarge); + } + if !body.is_empty() { + body.push(b'\n'); + } + body.extend_from_slice(&record); + } + String::from_utf8(body).map_err(|_| Error::InvalidRow) } fn insert_value(name: &str, value: Value) -> Result { @@ -35,3 +122,29 @@ fn insert_value(name: &str, value: Value) -> Result { .map(Value::String) .map_err(|_| Error::InvalidRow) } + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + + use rstest::rstest; + use serde_json::json; + + use super::encode_rows_with_limit; + use crate::Error; + + #[rstest] + fn encoded_limit_counts_utf8_bytes_across_rows() { + let rows = vec![ + BTreeMap::from([("Input".to_owned(), json!("雪"))]), + BTreeMap::from([("Input".to_owned(), json!("雪"))]), + ]; + let encoded = encode_rows_with_limit(rows.clone(), usize::MAX).expect("valid rows"); + + assert!(encode_rows_with_limit(rows.clone(), encoded.len()).is_ok()); + assert!(matches!( + encode_rows_with_limit(rows, encoded.len() - 1), + Err(Error::InsertTooLarge) + )); + } +} diff --git a/litellm-rust/crates/traces/src/lib.rs b/litellm-rust/crates/traces/src/lib.rs index 1f56bb5b3a6a..279afb20e9bb 100644 --- a/litellm-rust/crates/traces/src/lib.rs +++ b/litellm-rust/crates/traces/src/lib.rs @@ -1,10 +1,12 @@ mod error; mod insert; +mod otlp; mod schema; mod sql; -pub use error::Error; -pub use insert::encode_rows; +pub use error::{DecodeError, Error}; +pub use insert::{InsertTable, encode_rows, insert_rows}; +pub use otlp::{DecodedSpan, decode_otlp}; pub use schema::{ensure_schema, schema_statements}; pub use sql::{Parameter, execute_read}; use url::Url; @@ -53,17 +55,32 @@ impl Connection { Ok(connection) } - pub fn writer(url: &str, user: &str, password: &str) -> Result { + pub fn writer(url: &str) -> Result { let mut connection = Self::parse(url)?; - connection + let pairs: Vec<_> = connection .url - .set_username(user) - .map_err(|_| Error::InvalidUrl)?; + .query_pairs() + .filter(|(key, _)| !matches!(key.as_ref(), "database" | "readonly" | "query")) + .map(|(key, value)| (key.into_owned(), value.into_owned())) + .collect(); + connection.url.query_pairs_mut().clear().extend_pairs(pairs); + Ok(connection) + } + + pub fn reader(url: &str, database: &str) -> Result { + let mut connection = Self::parse(url)?; + let pairs: Vec<_> = connection + .url + .query_pairs() + .filter(|(key, _)| key != "database") + .map(|(key, value)| (key.into_owned(), value.into_owned())) + .collect(); connection .url - .set_password(Some(password)) - .map_err(|_| Error::InvalidUrl)?; - connection.url.set_query(None); + .query_pairs_mut() + .clear() + .extend_pairs(pairs) + .append_pair("database", database); Ok(connection) } diff --git a/litellm-rust/crates/traces/src/otlp.rs b/litellm-rust/crates/traces/src/otlp.rs new file mode 100644 index 000000000000..f162256ef1f0 --- /dev/null +++ b/litellm-rust/crates/traces/src/otlp.rs @@ -0,0 +1,221 @@ +use std::{collections::BTreeMap, io::Read}; + +use base64::Engine; +use flate2::read::GzDecoder; +use opentelemetry_proto::tonic::{ + collector::trace::v1::ExportTraceServiceRequest, + common::v1::{AnyValue, KeyValue, any_value::Value as AttributeValue}, + trace::v1::{Span, span::SpanKind, status::StatusCode}, +}; +use prost::Message; +use serde::Serialize; +use serde_json::Value; + +use crate::DecodeError; + +#[derive(Serialize)] +pub struct DecodedEvent { + pub name: String, + pub attributes: BTreeMap, +} + +#[derive(Serialize)] +pub struct DecodedSpan { + pub trace_id: String, + pub span_id: String, + pub parent_span_id: String, + pub trace_state: String, + pub name: String, + pub kind: String, + pub resource_attributes: BTreeMap, + pub scope_name: String, + pub scope_version: String, + pub attributes: BTreeMap, + pub start_ns: u64, + pub end_ns: u64, + pub status_code: String, + pub status_message: String, + pub events: Vec, +} + +pub fn decode_otlp( + body: &[u8], + content_type: Option<&str>, + content_encoding: Option<&str>, + max_decompressed_bytes: usize, +) -> Result, DecodeError> { + let payload = if content_encoding == Some("gzip") || body.starts_with(&[0x1f, 0x8b]) { + let limit = u64::try_from(max_decompressed_bytes).map_err(|_| DecodeError::TooLarge)?; + let mut decoded = Vec::new(); + GzDecoder::new(body) + .take(limit + 1) + .read_to_end(&mut decoded) + .map_err(|_| DecodeError::InvalidPayload)?; + decoded + } else { + body.to_vec() + }; + if payload.len() > max_decompressed_bytes { + return Err(DecodeError::TooLarge); + } + let request = if content_type.is_some_and(|value| value.contains("json")) { + let value: Value = + serde_json::from_slice(&payload).map_err(|_| DecodeError::InvalidPayload)?; + serde_json::from_value(normalize_json_ids(value)?) + .map_err(|_| DecodeError::InvalidPayload)? + } else { + ExportTraceServiceRequest::decode(payload.as_slice()) + .map_err(|_| DecodeError::InvalidPayload)? + }; + Ok(request + .resource_spans + .into_iter() + .flat_map(|resource_spans| { + let resource_attributes = attributes( + resource_spans + .resource + .map(|resource| resource.attributes) + .unwrap_or_default(), + ); + resource_spans + .scope_spans + .into_iter() + .flat_map(move |scope_spans| { + let scope = scope_spans.scope.unwrap_or_default(); + let resource_attributes = resource_attributes.clone(); + scope_spans.spans.into_iter().map(move |span| { + decoded_span(span, &resource_attributes, &scope.name, &scope.version) + }) + }) + }) + .collect()) +} + +fn normalize_json_ids(value: Value) -> Result { + match value { + Value::Object(fields) => fields + .into_iter() + .map(|(name, value)| { + let normalized = if matches!(name.as_str(), "traceId" | "spanId" | "parentSpanId") { + let encoded = value.as_str().ok_or(DecodeError::InvalidPayload)?; + let bytes = base64::engine::general_purpose::STANDARD + .decode(encoded) + .map_err(|_| DecodeError::InvalidPayload)?; + Value::String(hex_bytes(&bytes)) + } else if name == "kind" && value.is_string() { + let kind = SpanKind::from_str_name(value.as_str().unwrap_or_default()) + .ok_or(DecodeError::InvalidPayload)?; + Value::from(kind as i32) + } else if name == "code" && value.is_string() { + let code = StatusCode::from_str_name(value.as_str().unwrap_or_default()) + .ok_or(DecodeError::InvalidPayload)?; + Value::from(code as i32) + } else { + normalize_json_ids(value)? + }; + Ok((name, normalized)) + }) + .collect::, _>>() + .map(Value::Object), + Value::Array(values) => values + .into_iter() + .map(normalize_json_ids) + .collect::, _>>() + .map(Value::Array), + value => Ok(value), + } +} + +fn hex_bytes(bytes: &[u8]) -> String { + bytes.iter().map(|byte| format!("{byte:02x}")).collect() +} + +fn decoded_span( + span: Span, + resource_attributes: &BTreeMap, + scope_name: &str, + scope_version: &str, +) -> DecodedSpan { + let status = span.status.unwrap_or_default(); + DecodedSpan { + trace_id: hex_bytes(&span.trace_id), + span_id: hex_bytes(&span.span_id), + parent_span_id: hex_bytes(&span.parent_span_id), + trace_state: span.trace_state, + name: span.name, + kind: SpanKind::try_from(span.kind) + .unwrap_or(SpanKind::Unspecified) + .as_str_name() + .to_owned(), + resource_attributes: resource_attributes.clone(), + scope_name: scope_name.to_owned(), + scope_version: scope_version.to_owned(), + attributes: attributes(span.attributes), + start_ns: span.start_time_unix_nano, + end_ns: span.end_time_unix_nano, + status_code: StatusCode::try_from(status.code) + .unwrap_or(StatusCode::Unset) + .as_str_name() + .to_owned(), + status_message: status.message, + events: span + .events + .into_iter() + .map(|event| DecodedEvent { + name: event.name, + attributes: attributes(event.attributes), + }) + .collect(), + } +} + +fn attributes(values: Vec) -> BTreeMap { + values + .into_iter() + .map(|entry| { + ( + entry.key, + entry.value.as_ref().map(attribute_text).unwrap_or_default(), + ) + }) + .collect() +} + +fn attribute_text(value: &AnyValue) -> String { + match value.value.as_ref() { + Some(AttributeValue::StringValue(value)) => value.clone(), + Some(AttributeValue::BoolValue(value)) => value.to_string(), + Some(AttributeValue::IntValue(value)) => value.to_string(), + Some(AttributeValue::DoubleValue(value)) => { + serde_json::to_string(value).unwrap_or_default() + } + Some(AttributeValue::BytesValue(value)) => String::from_utf8_lossy(value).into_owned(), + Some(AttributeValue::ArrayValue(value)) => format!( + "[{}]", + value + .values + .iter() + .map(|value| serde_json::to_string(&attribute_text(value)).unwrap_or_default()) + .collect::>() + .join(", ") + ), + Some(AttributeValue::KvlistValue(value)) => format!( + "{{{}}}", + value + .values + .iter() + .map(|entry| format!( + "{}: {}", + serde_json::to_string(&entry.key).unwrap_or_default(), + serde_json::to_string( + &entry.value.as_ref().map(attribute_text).unwrap_or_default() + ) + .unwrap_or_default() + )) + .collect::>() + .join(", ") + ), + Some(AttributeValue::StringValueStrindex(value)) => value.to_string(), + None => String::new(), + } +} diff --git a/litellm-rust/crates/traces/tests/migrations.rs b/litellm-rust/crates/traces/tests/migrations.rs index a77d57cfb0fe..2652a3e3cce5 100644 --- a/litellm-rust/crates/traces/tests/migrations.rs +++ b/litellm-rust/crates/traces/tests/migrations.rs @@ -25,7 +25,7 @@ async fn schema_supports_span_rollups_and_spend_joins() -> Result<(), Box, + #[case] content_encoding: Option<&str>, +) { + let payload = if content_encoding == Some("gzip") { + let mut encoder = GzEncoder::new(Vec::new(), Compression::default()); + encoder.write_all(body).expect("gzip input"); + encoder.finish().expect("gzip payload") + } else { + body.to_vec() + }; + let spans = decode_otlp(&payload, content_type, content_encoding, 8 * 1024 * 1024) + .expect("valid OTLP export"); + assert_eq!(spans.len(), 6); + assert_eq!(spans[0].trace_id, "4bad42b84e9de3ba46fc870185f8f023"); + assert_eq!(spans[0].resource_attributes["service.name"], "agent-demo"); + assert_eq!(spans[0].scope_name, "langsmith"); + assert!( + spans + .iter() + .any(|span| span.attributes.contains_key("gen_ai.prompt")) + ); +} + +#[rstest] +#[case::invalid(b"not protobuf", None, 8 * 1024 * 1024)] +#[case::too_large(FIXTURE, Some("application/json"), 1)] +fn rejects_invalid_or_oversized_payload( + #[case] body: &[u8], + #[case] content_type: Option<&str>, + #[case] limit: usize, +) { + assert!(decode_otlp(body, content_type, None, limit).is_err()); +} diff --git a/litellm/constants.py b/litellm/constants.py index 530d678457d0..8a74ca21ff97 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -46,6 +46,19 @@ ) DEFAULT_BATCH_SIZE: Final = int(os.getenv("DEFAULT_BATCH_SIZE", 512)) DEFAULT_FLUSH_INTERVAL_SECONDS: Final = int(os.getenv("DEFAULT_FLUSH_INTERVAL_SECONDS", 5)) +# Agent tracing / ClickHouse +CLICKHOUSE_BATCH_SIZE: Final = int(os.getenv("CLICKHOUSE_BATCH_SIZE", 10_000)) +CLICKHOUSE_FLUSH_INTERVAL_SECONDS: Final = float(os.getenv("CLICKHOUSE_FLUSH_INTERVAL_SECONDS", 1.0)) +CLICKHOUSE_MAX_BUFFERED_ROWS: Final = int(os.getenv("CLICKHOUSE_MAX_BUFFERED_ROWS", 200_000)) +CLICKHOUSE_MAX_RETRIES: Final = int(os.getenv("CLICKHOUSE_MAX_RETRIES", 3)) +AGENT_TRACING_RETENTION_DAYS: Final = int(os.getenv("AGENT_TRACING_RETENTION_DAYS", 30)) +AGENT_TRACING_SPEND_LOG_RETENTION_DAYS: Final = int(os.getenv("AGENT_TRACING_SPEND_LOG_RETENTION_DAYS", 90)) +OTLP_MAX_BODY_BYTES: Final = int(os.getenv("OTLP_MAX_BODY_BYTES", 8 * 1024 * 1024)) +OTLP_MAX_ATTRIBUTE_VALUE_BYTES: Final = int(os.getenv("OTLP_MAX_ATTRIBUTE_VALUE_BYTES", 64 * 1024)) +OTLP_RETRY_AFTER_SECONDS: Final = int(os.getenv("OTLP_RETRY_AFTER_SECONDS", 2)) +OTLP_OFFLOAD_DECODE_BYTES: Final = int(os.getenv("OTLP_OFFLOAD_DECODE_BYTES", 256 * 1024)) +AGENT_TRACING_INPUT_PREVIEW_CHARS: Final = int(os.getenv("AGENT_TRACING_INPUT_PREVIEW_CHARS", 240)) +AGENT_TRACING_LIST_PAGE_SIZE: Final = int(os.getenv("AGENT_TRACING_LIST_PAGE_SIZE", 50)) DEFAULT_S3_FLUSH_INTERVAL_SECONDS: Final = int(os.getenv("DEFAULT_S3_FLUSH_INTERVAL_SECONDS", 10)) DEFAULT_S3_BATCH_SIZE: Final = int(os.getenv("DEFAULT_S3_BATCH_SIZE", 512)) DEFAULT_S3_MAX_CONCURRENT_UPLOADS: Final = int(os.getenv("DEFAULT_S3_MAX_CONCURRENT_UPLOADS", "16")) diff --git a/litellm/integrations/clickhouse/clickhouse_batch_logger.py b/litellm/integrations/clickhouse/clickhouse_batch_logger.py new file mode 100644 index 000000000000..2c55e3ca4d99 --- /dev/null +++ b/litellm/integrations/clickhouse/clickhouse_batch_logger.py @@ -0,0 +1,101 @@ +""" +Shared base for everything LiteLLM writes to ClickHouse. + +Built on `CustomBatchLogger`: rows accumulate in `log_queue` and are flushed as one +gzip JSONEachRow insert, either every `CLICKHOUSE_FLUSH_INTERVAL_SECONDS` or as soon as +`batch_size` rows are queued. Subclasses only pick the table and build rows: + +- `ClickHouseSpendLogger` -> spend_logs (LiteLLM requests, via the `clickhouse` callback) +""" + +import asyncio +import os +from typing import Any, ClassVar + +from litellm._logging import verbose_logger +from litellm.constants import ( + CLICKHOUSE_BATCH_SIZE, + CLICKHOUSE_FLUSH_INTERVAL_SECONDS, + CLICKHOUSE_MAX_BUFFERED_ROWS, + CLICKHOUSE_MAX_RETRIES, +) +from litellm.integrations.custom_batch_logger import CustomBatchLogger +from litellm.rust_bridge.traces import TraceStorage + + +def clickhouse_storage_from_env() -> TraceStorage: + return TraceStorage( + database=os.getenv("CLICKHOUSE_DATABASE", "litellm"), + url=os.getenv("CLICKHOUSE_URL", ""), + ) + + +class ClickHouseBatchLogger(CustomBatchLogger): + table: ClassVar[str] + + def __init__(self, storage: TraceStorage | None = None, **kwargs: Any) -> None: + self.storage = storage or clickhouse_storage_from_env() + self.rows_written = 0 + self.rows_dropped = 0 + self._failed_attempts = 0 + super().__init__( + flush_lock=asyncio.Lock(), + batch_size=CLICKHOUSE_BATCH_SIZE, + flush_interval=CLICKHOUSE_FLUSH_INTERVAL_SECONDS, # type: ignore[arg-type] + **kwargs, + ) + try: + asyncio.get_running_loop().create_task(self.periodic_flush()) + except RuntimeError: # no loop yet (e.g. sync config load); proxy startup calls start() + pass + + def start(self) -> None: + asyncio.get_running_loop().create_task(self.periodic_flush()) + + def is_full(self) -> bool: + """Backpressure signal: producers should reject (429) instead of enqueueing.""" + return len(self.log_queue) >= CLICKHOUSE_MAX_BUFFERED_ROWS + + def enqueue(self, rows: list[dict[str, Any]]) -> None: + """Never awaits ClickHouse. Kicks off an early flush once a full batch is queued.""" + self.log_queue.extend(rows) + if len(self.log_queue) >= self.batch_size: + asyncio.get_running_loop().create_task(self.flush_queue()) + + async def flush_queue(self) -> None: + # Swap the queue under the lock so rows enqueued during the insert are kept. + if self.flush_lock is None: + return + async with self.flush_lock: + while self.log_queue: + batch = self.log_queue[: self.batch_size] + self.log_queue = self.log_queue[len(batch) :] + if not await self._insert(batch): + break + + async def async_send_batch(self, *args: Any, **kwargs: Any) -> None: + await self.flush_queue() + + async def _insert(self, batch: list[dict[str, Any]]) -> bool: + try: + await self.storage.insert_rows(self.table, batch) + self.rows_written += len(batch) + self._failed_attempts = 0 + return True + except Exception as e: + self._failed_attempts += 1 + if self._failed_attempts >= CLICKHOUSE_MAX_RETRIES: + self.rows_dropped += len(batch) + self._failed_attempts = 0 + verbose_logger.error( + "ClickHouse: dropped %s rows for %s after %s attempts: %s", + len(batch), + self.table, + CLICKHOUSE_MAX_RETRIES, + e, + ) + else: + # put it back; the next periodic flush retries it + self.log_queue = batch + self.log_queue + verbose_logger.warning("ClickHouse: insert into %s failed, will retry: %s", self.table, e) + return False diff --git a/litellm/integrations/clickhouse/schema.py b/litellm/integrations/clickhouse/schema.py new file mode 100644 index 000000000000..f9e48d3b3545 --- /dev/null +++ b/litellm/integrations/clickhouse/schema.py @@ -0,0 +1,11 @@ +from typing import Final + +from litellm.rust_bridge.traces import TraceStorage + +OTEL_TRACES_TABLE: Final = "otel_traces" +AGENT_TRACES_TABLE: Final = "agent_traces" +SPEND_LOGS_TABLE: Final = "spend_logs" + + +async def ensure_schema(storage: TraceStorage, trace_retention_days: int, spend_log_retention_days: int) -> None: + await storage.ensure_schema(trace_retention_days, spend_log_retention_days) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index be06d2e73217..5aa617734b45 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -520,6 +520,10 @@ class LiteLLMRoutes(enum.Enum): "/v1/rag/ingest", "/rag/query", "/v1/rag/query", + # agent tracing: OTLP ingest + reads (scoped to the caller's team in the handler) + "/v1/traces", + "/v1/traces/{trace_id}", + "/v1/traces/{trace_id}/spans/{span_id}", ] anthropic_routes = [ diff --git a/litellm/proxy/common_utils/http_parsing_utils.py b/litellm/proxy/common_utils/http_parsing_utils.py index 1c2bd7ea2179..13a9b7ebf2ba 100644 --- a/litellm/proxy/common_utils/http_parsing_utils.py +++ b/litellm/proxy/common_utils/http_parsing_utils.py @@ -21,6 +21,9 @@ from litellm.types.router import Deployment _FORM_CONTENT_TYPES: Final[frozenset[str]] = frozenset({"application/x-www-form-urlencoded", "multipart/form-data"}) +# Binary bodies (e.g. OTLP trace exports on POST /v1/traces) are not JSON: arbitrary bytes used to +# hit the JSON surrogate-repair path and fail auth with a 400. JSON under these types still parses. +_BINARY_CONTENT_TYPES: Final[frozenset[str]] = frozenset({"application/x-protobuf", "application/protobuf"}) _ANNOTATION_QUALIFIERS: Final[frozenset[object]] = frozenset({Annotated, NotRequired, ReadOnly, Required}) @@ -119,6 +122,17 @@ def coerce_numeric_form_fields( } +def _parse_binary_body(body: bytes) -> dict: + """JSON sent under a binary content type still parses; real binary (protobuf) carries no params -> {}.""" + if not body: + return {} + try: + parsed: Final = orjson.loads(body) + except orjson.JSONDecodeError: + return {} + return parsed if isinstance(parsed, dict) else {} + + async def _read_request_body(request: Request | None) -> dict: """ Safely read the request body and parse it as JSON. @@ -141,7 +155,9 @@ async def _read_request_body(request: Request | None) -> dict: _request_headers: Final[dict] = _safe_get_request_headers(request=request) content_type: Final = _request_headers.get("content-type", "") - if _is_form_content_type(content_type): + if _normalize_media_type(content_type) in _BINARY_CONTENT_TYPES: + parsed_body = _parse_binary_body(await request.body()) + elif _is_form_content_type(content_type): try: form_data: Final = await request.form() except Exception as e: diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index a994293a4d89..7f2742fcffc2 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -713,6 +713,7 @@ def generate_feedback_box(): except ImportError: build_billing_metrics_recorder = None shutdown_billing_metrics_recorder = None +from litellm.proxy import tracing_endpoints from litellm.proxy.middleware.admission_control_middleware import ( AdmissionControlMiddleware, admission_control_state, @@ -844,6 +845,7 @@ def generate_feedback_box(): secret_manager_would_be_consulted, str_to_bool, ) +from litellm.tracing import TraceReceiver from litellm.types.integrations.slack_alerting import AlertType, SlackAlertingArgs from litellm.types.llms.anthropic import ( AnthropicMessagesRequest, @@ -1520,6 +1522,9 @@ async def _run_agent_grant_id_migration() -> None: _tagged.strategy._state_loaded = True asyncio.create_task(_adaptive_router_flusher_loop()) + ## [Optional] Initialize agent tracing + await ProxyStartupEvent._init_tracing(general_settings) + ## [Optional] Initialize dd tracer ProxyStartupEvent._init_dd_tracer() @@ -11309,6 +11314,23 @@ async def _setup_prisma_client( ) return connected_client + @classmethod + async def _init_tracing(cls, general_settings: dict) -> None: + """ + Enable agent tracing (`POST/GET /v1/traces`) when configured: + + general_settings: + tracing: + store: clickhouse # CLICKHOUSE_URL / _USER / _PASSWORD / _DATABASE + """ + settings = general_settings.get("tracing") or {} + if settings.get("store") != "clickhouse": + return + tracing = TraceReceiver.from_env() + await tracing.start() + tracing_endpoints.receiver = tracing + verbose_proxy_logger.info("Agent tracing enabled (store=clickhouse)") + @classmethod def _init_dd_tracer(cls): """ @@ -19860,6 +19882,7 @@ async def get_routes(): app.include_router(video_router) app.include_router(container_router) app.include_router(search_router) +app.include_router(tracing_endpoints.router) app.include_router(image_router) app.include_router(fine_tuning_router) app.include_router(credential_router) diff --git a/litellm/proxy/tracing_endpoints.py b/litellm/proxy/tracing_endpoints.py new file mode 100644 index 000000000000..83fc4c12e31a --- /dev/null +++ b/litellm/proxy/tracing_endpoints.py @@ -0,0 +1,120 @@ +""" +Agent tracing endpoints. Thin wrappers over `TraceReceiver`: auth -> tenant/scope -> one call. + +POST /v1/traces OTLP/HTTP trace export (protobuf or JSON) +GET /v1/traces TracePage +GET /v1/traces/{trace_id} Trace +GET /v1/traces/{trace_id}/spans/{span_id} SpanDetail +""" + +import time +from typing import Final + +from fastapi import APIRouter, Depends, HTTPException, Query, Request, Response + +from litellm.constants import OTLP_RETRY_AFTER_SECONDS +from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.tracing import ( + Tenant, + TraceReceiver, + TracingPayloadTooLargeError, +) +from litellm.tracing.decode import encode_otlp_response +from litellm.tracing.types import SpanDetail, Trace, TracePage, TraceScope + +router = APIRouter(tags=["agent tracing"]) + +MS_PER_DAY: Final = 24 * 60 * 60 * 1000 +_ADMIN_ROLES: Final = (LitellmUserRoles.PROXY_ADMIN, LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY) + +receiver: TraceReceiver | None = None # set at proxy startup when agent tracing is enabled + + +def get_receiver() -> TraceReceiver: + if receiver is None: + raise HTTPException( + status_code=501, + detail="Agent tracing is not enabled. Set `tracing:` in general_settings and CLICKHOUSE_URL.", + ) + return receiver + + +def tenant_for(user_api_key_dict: UserAPIKeyAuth) -> Tenant: + return Tenant( + team_id=user_api_key_dict.team_id or "", + api_key_hash=user_api_key_dict.token or "", + org_id=user_api_key_dict.org_id or "", + ) + + +def scope_for(user_api_key_dict: UserAPIKeyAuth) -> TraceScope: + """Admins see everything; team members see their team; team-less keys see their own traces.""" + if user_api_key_dict.user_role in _ADMIN_ROLES: + return TraceScope(team_ids=[], api_key_hash="") + if user_api_key_dict.team_id: + return TraceScope(team_ids=[user_api_key_dict.team_id], api_key_hash="") + if not user_api_key_dict.token: + raise HTTPException(status_code=403, detail="Not allowed to view agent traces") + return TraceScope(team_ids=[""], api_key_hash=user_api_key_dict.token) + + +@router.post("/v1/traces", include_in_schema=False) +async def ingest_otlp_traces( + request: Request, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +) -> Response: + tracing: Final = get_receiver() + content_type: Final = request.headers.get("content-type") + try: + await tracing.ingest( + body=await request.body(), + content_type=content_type, + content_encoding=request.headers.get("content-encoding"), + tenant=tenant_for(user_api_key_dict), + ) + except RuntimeError: + raise HTTPException(status_code=503, headers={"Retry-After": str(OTLP_RETRY_AFTER_SECONDS)}) + except TracingPayloadTooLargeError as e: + raise HTTPException(status_code=413, detail=str(e)) + body, media_type = encode_otlp_response(content_type) + return Response(content=body, media_type=media_type) + + +@router.get("/v1/traces", response_model=None) +async def list_agent_traces( + start_ms: int | None = Query(None, description="Window start, unix ms. Default: 24h ago"), + end_ms: int | None = Query(None, description="Window end, unix ms. Default: now"), + cursor: str | None = Query(None), + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +) -> TracePage: + now_ms: Final = int(time.time() * 1000) + return await get_receiver().list_traces( + scope=scope_for(user_api_key_dict), + start_ms=start_ms if start_ms is not None else now_ms - MS_PER_DAY, + end_ms=end_ms if end_ms is not None else now_ms, + cursor=cursor, + ) + + +@router.get("/v1/traces/{trace_id}", response_model=None) +async def get_agent_trace( + trace_id: str, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +) -> Trace: + trace: Final = await get_receiver().get_trace(trace_id, scope_for(user_api_key_dict)) + if trace is None: + raise HTTPException(status_code=404, detail=f"Trace {trace_id} not found") + return trace + + +@router.get("/v1/traces/{trace_id}/spans/{span_id}", response_model=None) +async def get_agent_trace_span( + trace_id: str, + span_id: str, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +) -> SpanDetail: + span: Final = await get_receiver().get_span(trace_id, span_id, scope_for(user_api_key_dict)) + if span is None: + raise HTTPException(status_code=404, detail=f"Span {span_id} not found") + return span diff --git a/litellm/rust_bridge/_native.pyi b/litellm/rust_bridge/_native.pyi index 5d2af93a9bdc..30d4bbfb68e1 100644 --- a/litellm/rust_bridge/_native.pyi +++ b/litellm/rust_bridge/_native.pyi @@ -11,6 +11,7 @@ from litellm.rust_bridge.embeddings.entrypoints import LiteLLMEmbeddingRequest from litellm.rust_bridge.messages.entrypoints import LiteLLMMessagesRequest from litellm.rust_bridge.ocr.entrypoints import LiteLLMOcrRequest from litellm.rust_bridge.responses.entrypoints import LiteLLMResponsesRequest +from litellm.rust_bridge.traces import DecodedSpan from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicMessagesResponse from litellm.types.llms.openai import ResponsesAPIResponse from litellm.types.utils import EmbeddingResponse, ModelResponse @@ -20,18 +21,16 @@ class RustUpstreamError(Exception): ... class ForkedAfterNativeRuntimeStarted(RuntimeError): ... class ProcessReservedForForking(RuntimeError): ... -def trace_encode_rows(rows: Sequence[Mapping[str, JsonValue]]) -> str: ... -def trace_ensure_schema( - url: str, database: str, user: str, password: str, trace_retention_days: int, spend_log_retention_days: int -) -> Future[None]: ... -def trace_query( - url: str, - database: str, - user: str, - password: str, - sql: str, - parameters: Mapping[str, str | int | Sequence[str]], -) -> Future[str]: ... +def trace_decode_otlp( + body: bytes, content_type: str | None, content_encoding: str | None, max_decompressed_bytes: int +) -> list[DecodedSpan]: ... + +@final +class NativeTraceStorage: + def __new__(cls, database: str, url: str, reader_url: str | None = None) -> NativeTraceStorage: ... + def ensure_schema(self, trace_retention_days: int, spend_log_retention_days: int) -> Future[None]: ... + def insert_rows(self, table: str, rows: Sequence[Mapping[str, JsonValue]]) -> Future[None]: ... + def query(self, sql: str, parameters: Mapping[str, str | int | Sequence[str]]) -> Future[str]: ... @final class NativeDiagnosticProcessor: @@ -327,6 +326,7 @@ __all__ = [ "ForkedAfterNativeRuntimeStarted", "HuggingFaceEncoding", "NativeDiagnosticProcessor", + "NativeTraceStorage", "ProcessReservedForForking", "ResponsesWebSocketConnection", "RustBridgeDeclined", @@ -351,9 +351,7 @@ __all__ = [ "process_state_started", "reserve_process_for_forking", "responses", - "trace_encode_rows", - "trace_ensure_schema", - "trace_query", + "trace_decode_otlp", "transcription", ] diff --git a/litellm/rust_bridge/traces.py b/litellm/rust_bridge/traces.py index cb4fd4a5bd3e..6e64422d1be4 100644 --- a/litellm/rust_bridge/traces.py +++ b/litellm/rust_bridge/traces.py @@ -1,40 +1,65 @@ from collections.abc import Awaitable, Mapping, Sequence -from typing import Final, Protocol, cast +from typing import Final, Protocol, TypedDict, cast -from pydantic import BaseModel, ConfigDict, JsonValue +from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter +from typing_extensions import ReadOnly from litellm.rust_bridge.loader import get_native_bridge +class DecodedEvent(TypedDict): + name: ReadOnly[str] + attributes: ReadOnly[dict[str, str]] + + +class DecodedSpan(TypedDict): + trace_id: ReadOnly[str] + span_id: ReadOnly[str] + parent_span_id: ReadOnly[str] + trace_state: ReadOnly[str] + name: ReadOnly[str] + kind: ReadOnly[str] + resource_attributes: ReadOnly[dict[str, str]] + scope_name: ReadOnly[str] + scope_version: ReadOnly[str] + attributes: ReadOnly[dict[str, str]] + start_ns: ReadOnly[int] + end_ns: ReadOnly[int] + status_code: ReadOnly[str] + status_message: ReadOnly[str] + events: ReadOnly[list[DecodedEvent]] + + +class NativeStore(Protocol): + def __init__(self, database: str, url: str, reader_url: str | None = None) -> None: ... + + def ensure_schema(self, trace_retention_days: int, spend_log_retention_days: int) -> Awaitable[None]: ... + + def insert_rows(self, table: str, rows: Sequence[Mapping[str, JsonValue]]) -> Awaitable[None]: ... + + def query(self, sql: str, parameters: Mapping[str, str | int | Sequence[str]]) -> Awaitable[str]: ... + + class NativeTraces(Protocol): - def trace_encode_rows(self, rows: Sequence[Mapping[str, JsonValue]]) -> str: ... + NativeTraceStorage: type[NativeStore] - def trace_ensure_schema( + def trace_decode_otlp( self, - url: str, - database: str, - user: str, - password: str, - trace_retention_days: int, - spend_log_retention_days: int, - ) -> Awaitable[None]: ... - - def trace_query( - self, - url: str, - database: str, - user: str, - password: str, - sql: str, - parameters: Mapping[str, str | int | Sequence[str]], - ) -> Awaitable[str]: ... - + body: bytes, + content_type: str | None, + content_encoding: str | None, + max_decompressed_bytes: int, + ) -> list[DecodedSpan]: ... class QueryResponse(BaseModel): model_config = ConfigDict(frozen=True) data: list[dict[str, JsonValue]] +INSERT_ROWS: Final = TypeAdapter(list[dict[str, JsonValue]]) +QUERY_PARAMETERS: Final = TypeAdapter(dict[str, str | int | list[str]]) + + def _native() -> NativeTraces: native: Final = get_native_bridge() if native is None: @@ -42,28 +67,22 @@ def _native() -> NativeTraces: return cast(NativeTraces, native) # cast-ok: the native extension is validated against this protocol at call sites -async def ensure_schema( - url: str, - database: str, - user: str, - password: str, - trace_retention_days: int, - spend_log_retention_days: int, -) -> None: - await _native().trace_ensure_schema(url, database, user, password, trace_retention_days, spend_log_retention_days) +def decode_otlp( + body: bytes, content_type: str | None, content_encoding: str | None, max_decompressed_bytes: int +) -> list[DecodedSpan]: + return _native().trace_decode_otlp(body, content_type, content_encoding, max_decompressed_bytes) + +class TraceStorage: + def __init__(self, database: str, url: str, reader_url: str | None = None) -> None: + self._native: Final = _native().NativeTraceStorage(database, url, reader_url) -async def query( - url: str, - database: str, - user: str, - password: str, - sql: str, - parameters: Mapping[str, str | int | Sequence[str]], -) -> list[dict[str, JsonValue]]: - result: Final = await _native().trace_query(url, database, user, password, sql, parameters) - return QueryResponse.model_validate_json(result).data + async def ensure_schema(self, trace_retention_days: int, spend_log_retention_days: int) -> None: + await self._native.ensure_schema(trace_retention_days, spend_log_retention_days) + async def insert_rows(self, table: str, rows: Sequence[Mapping[str, object]]) -> None: + await self._native.insert_rows(table, INSERT_ROWS.validate_python(rows)) -def encode_rows(rows: Sequence[Mapping[str, JsonValue]]) -> bytes: - return _native().trace_encode_rows(rows).encode("utf-8") + async def query(self, sql: str, parameters: Mapping[str, object] | None = None) -> list[dict[str, JsonValue]]: + result: Final = await self._native.query(sql, QUERY_PARAMETERS.validate_python(parameters or {})) + return QueryResponse.model_validate_json(result).data diff --git a/litellm/tracing/AGENTS.md b/litellm/tracing/AGENTS.md new file mode 100644 index 000000000000..f69866c04198 --- /dev/null +++ b/litellm/tracing/AGENTS.md @@ -0,0 +1,6 @@ +- Python owns tracing endpoints, authenticated tenant scope, framework normalization and API response shaping +- Trace ingestion awaits `TraceStorage.insert_rows` before returning success; propagate storage failures so OTLP exporters can retry +- Spend logging keeps its separate batch queue in `litellm/integrations/clickhouse` +- Use `litellm.rust_bridge.traces.TraceStorage` for ClickHouse; keep schema, SQL, encoding and transport in `litellm-traces` +- Derive tenant fields from authentication and overwrite matching fields supplied by the exporter +- Test confirmed writes, failures, tenant isolation and read behavior through public functions diff --git a/litellm/tracing/__init__.py b/litellm/tracing/__init__.py new file mode 100644 index 000000000000..fdde4322b7c3 --- /dev/null +++ b/litellm/tracing/__init__.py @@ -0,0 +1,16 @@ +""" +LiteLLM agent tracing: OTLP traces from agents, joined to LiteLLM spend logs, in ClickHouse. + +""" + +from litellm.tracing.receiver import ( + Tenant, + TraceReceiver, + TracingPayloadTooLargeError, +) + +__all__ = [ + "Tenant", + "TraceReceiver", + "TracingPayloadTooLargeError", +] diff --git a/litellm/tracing/decode.py b/litellm/tracing/decode.py new file mode 100644 index 000000000000..daf15778e498 --- /dev/null +++ b/litellm/tracing/decode.py @@ -0,0 +1,245 @@ +""" +OTLP/HTTP trace export -> `SpanRow`s. + +Pure functions, no I/O. Two steps: +1. `decode_otlp()` protobuf / JSON / gzip `ExportTraceServiceRequest` -> flat spans +2. `normalize()` framework conventions -> LiteLLM columns (type, agent, input/output, + LiteLLM request id). Supported: LangSmith (LangChain, LangGraph, + Deep Agents), OTEL GenAI semconv, OpenInference. +""" + +import json +from collections.abc import Callable, Mapping +from typing import Any, Final + +from litellm.constants import OTLP_MAX_ATTRIBUTE_VALUE_BYTES, OTLP_MAX_BODY_BYTES +from litellm.rust_bridge.traces import DecodedSpan +from litellm.rust_bridge.traces import decode_otlp as native_decode_otlp +from litellm.tracing.types import SpanRow, SpanType + +# attributes whose content we lift into Input/Output and drop from SpanAttributes +_HEAVY_ATTRIBUTES: Final = frozenset( + { + "gen_ai.prompt", + "gen_ai.completion", + "gen_ai.tool.definitions", + "gen_ai.input.messages", + "gen_ai.output.messages", + "input.value", + "output.value", + } +) +# LangChain / Deep Agents middleware wrappers: real spans, but noise in the UI +_FRAMEWORK_SUFFIXES: Final = ( + ".wrap_model_call", + ".wrap_tool_call", + ".before_agent", + ".after_agent", + ".before_model", + ".after_model", +) +_LLM_OPERATIONS: Final = frozenset({"chat", "text_completion", "generate_content"}) +_LC_ROLES: Final = {"human": "user", "ai": "assistant", "system": "system", "tool": "tool"} +_OPENINFERENCE_TYPES: Final[dict[str, SpanType]] = {"AGENT": "agent", "LLM": "llm", "TOOL": "tool"} + + +# ---------------------------------------------------------------- decode + + +def _truncate(value: str) -> str: + size = len(value.encode("utf-8")) + if size <= OTLP_MAX_ATTRIBUTE_VALUE_BYTES: + return value + kept = value.encode("utf-8")[:OTLP_MAX_ATTRIBUTE_VALUE_BYTES].decode("utf-8", "ignore") + return f"{kept}…[truncated {size - OTLP_MAX_ATTRIBUTE_VALUE_BYTES} bytes]" + + +def decode_otlp(body: bytes, content_type: str | None = None, content_encoding: str | None = None) -> list[SpanRow]: + """Decode an OTLP trace export and normalize every span.""" + return [_span_row(span) for span in native_decode_otlp(body, content_type, content_encoding, OTLP_MAX_BODY_BYTES)] + + +def _exception_message(span: DecodedSpan) -> str: + """`span.record_exception()` writes an `exception` event; surface it when status.message is empty.""" + for event in span["events"]: + if event["name"] == "exception": + attributes = event["attributes"] + return attributes.get("exception.message") or attributes.get("exception.type", "") + return "" + + +def _span_row(span: DecodedSpan) -> SpanRow: + attributes = span["attributes"] + resource = span["resource_attributes"] + row = SpanRow( + Timestamp=span["start_ns"], + TraceId=span["trace_id"], + SpanId=span["span_id"], + ParentSpanId=span["parent_span_id"], + TraceState=span["trace_state"], + SpanName=span["name"], + SpanKind=span["kind"], + ServiceName=resource.get("service.name", ""), + ResourceAttributes=resource, + ScopeName=span["scope_name"], + ScopeVersion=span["scope_version"], + SpanAttributes={}, + Duration=max(span["end_ns"] - span["start_ns"], 0), + StatusCode=span["status_code"], + StatusMessage=span["status_message"] or _exception_message(span), + TeamId="", + ApiKeyHash="", + ObservationType="chain", + AgentName="", + LiteLLMRequestId="", + Model="", + InputTokens=0, + OutputTokens=0, + Input="", + Output="", + ) + normalize(row, attributes) + row["SpanAttributes"] = {k: _truncate(v) for k, v in attributes.items() if k not in _HEAVY_ATTRIBUTES} + row["Input"], row["Output"] = _truncate(row["Input"]), _truncate(row["Output"]) + return row + + +# ---------------------------------------------------------------- normalize + + +def _loads(value: str) -> Any: + try: + return json.loads(value) + except (ValueError, TypeError): + return None + + +def _lc_message(message: Mapping[str, Any]) -> dict[str, Any]: + """LangChain serialized message (or plain {role, content}) -> {role, content, tool_calls?}.""" + kwargs = message.get("kwargs", message) + role = _LC_ROLES.get(kwargs.get("type") or kwargs.get("role"), kwargs.get("role") or kwargs.get("type") or "") + content = kwargs.get("content", "") + out: dict[str, Any] = {"role": role, "content": content if isinstance(content, str) else json.dumps(content)} + if kwargs.get("tool_calls"): + out["tool_calls"] = [{"name": t.get("name"), "args": t.get("args")} for t in kwargs["tool_calls"]] + if role == "tool" and kwargs.get("name"): + out["name"] = kwargs["name"] + return out + + +def _langsmith_type(row: SpanRow, attributes: Mapping[str, str]) -> SpanType: + kind = attributes.get("langsmith.span.kind", "chain") + name = row["SpanName"] + if not row["ParentSpanId"] or name == attributes.get("langsmith.metadata.lc_agent_name"): + return "agent" + if kind in ("llm", "tool"): + return kind # type: ignore[return-value] + if name.endswith(_FRAMEWORK_SUFFIXES): + return "framework" + return "chain" + + +def _langsmith_io(row: SpanRow, attributes: Mapping[str, str]) -> None: + prompt = _loads(attributes.get("gen_ai.prompt", "")) + completion = _loads(attributes.get("gen_ai.completion", "")) + if row["ObservationType"] == "llm" and isinstance(completion, dict): + messages = (prompt or {}).get("messages") or [[]] + batch = messages[0] if messages and isinstance(messages[0], list) else messages + row["Input"] = json.dumps([_lc_message(m) for m in batch]) + generation = completion["generations"][0][0]["message"]["kwargs"] + row["Output"] = json.dumps(_lc_message({"kwargs": generation})) + row["LiteLLMRequestId"] = (generation.get("response_metadata") or {}).get("id") or "" + return + if row["ObservationType"] == "tool": + output = (completion or {}).get("output", completion) if isinstance(completion, dict) else completion + if isinstance(output, dict) and "update" in output: # LangGraph Command, e.g. Deep Agents `task` + update_messages = (output.get("update") or {}).get("messages") or [] + output = update_messages[-1] if update_messages else output + if isinstance(output, dict): + output = output.get("content", output) + row["Input"] = attributes.get("gen_ai.prompt", "") + row["Output"] = output if isinstance(output, str) else json.dumps(output) + return + if row["ObservationType"] == "agent": + input_messages = (prompt or {}).get("messages") if isinstance(prompt, dict) else None + output_messages = (completion or {}).get("messages") if isinstance(completion, dict) else None + # agents built with @traceable take arbitrary args, not a message list: keep the raw payload then + row["Input"] = ( + json.dumps([_lc_message(m) for m in input_messages if isinstance(m, dict)]) + if input_messages + else attributes.get("gen_ai.prompt", "") + ) + row["Output"] = ( + json.dumps(_lc_message(output_messages[-1])) + if output_messages and isinstance(output_messages[-1], dict) + else attributes.get("gen_ai.completion", "") + ) + return + row["Input"] = attributes.get("gen_ai.prompt", "") + row["Output"] = attributes.get("gen_ai.completion", "") + + +def normalize_langsmith(row: SpanRow, attributes: Mapping[str, str]) -> None: + row["ObservationType"] = _langsmith_type(row, attributes) + row["AgentName"] = attributes.get("langsmith.metadata.lc_agent_name", "") + row["Model"] = attributes.get("gen_ai.request.model", "") + _langsmith_io(row, attributes) + + +def normalize_genai(row: SpanRow, attributes: Mapping[str, str]) -> None: + operation = attributes.get("gen_ai.operation.name", "") + if operation == "invoke_agent" or not row["ParentSpanId"]: + row["ObservationType"] = "agent" + elif operation in _LLM_OPERATIONS: + row["ObservationType"] = "llm" + elif operation == "execute_tool": + row["ObservationType"] = "tool" + row["AgentName"] = attributes.get("gen_ai.agent.name", "") + row["Model"] = attributes.get("gen_ai.request.model") or attributes.get("gen_ai.response.model", "") + row["LiteLLMRequestId"] = attributes.get("gen_ai.response.id", "") + row["Input"] = attributes.get("gen_ai.input.messages") or attributes.get("gen_ai.tool.call.arguments", "") + row["Output"] = attributes.get("gen_ai.output.messages") or attributes.get("gen_ai.tool.call.result", "") + + +def normalize_openinference(row: SpanRow, attributes: Mapping[str, str]) -> None: + kind = attributes.get("openinference.span.kind", "").upper() + row["ObservationType"] = _OPENINFERENCE_TYPES.get(kind, "agent" if not row["ParentSpanId"] else "chain") + row["AgentName"] = attributes.get("agent.name", "") + row["Model"] = attributes.get("llm.model_name", "") + row["Input"] = attributes.get("input.value", "") + row["Output"] = attributes.get("output.value", "") + row["InputTokens"] = _to_int(attributes.get("llm.token_count.prompt")) + row["OutputTokens"] = _to_int(attributes.get("llm.token_count.completion")) + + +def _set_tokens(row: SpanRow, attributes: Mapping[str, str]) -> None: + row["InputTokens"] = _to_int(attributes.get("gen_ai.usage.input_tokens")) + row["OutputTokens"] = _to_int(attributes.get("gen_ai.usage.output_tokens")) + + +def _to_int(value: str | None) -> int: + try: + return int(value) if value else 0 + except ValueError: + return 0 + + +def select_normalizer(scope_name: str, attributes: Mapping[str, str]) -> Callable[[SpanRow, Mapping[str, str]], None]: + if scope_name == "langsmith" or "langsmith.span.kind" in attributes: + return normalize_langsmith + if "openinference.span.kind" in attributes: + return normalize_openinference + return normalize_genai + + +def normalize(row: SpanRow, attributes: Mapping[str, str]) -> None: + select_normalizer(row["ScopeName"], attributes)(row, attributes) + if not row["InputTokens"] and not row["OutputTokens"]: + _set_tokens(row, attributes) + + +def encode_otlp_response(content_type: str | None) -> tuple[bytes, str]: + """Empty ExportTraceServiceResponse in the caller's encoding.""" + if content_type and "json" in content_type: + return b"{}", "application/json" + return b"", "application/x-protobuf" diff --git a/litellm/tracing/receiver.py b/litellm/tracing/receiver.py new file mode 100644 index 000000000000..196860078daf --- /dev/null +++ b/litellm/tracing/receiver.py @@ -0,0 +1,117 @@ +""" +`TraceReceiver`: the one entry point for agent tracing. + + tracing = TraceReceiver.from_env() # or TraceReceiver(store=...) + await tracing.start() # create tables if missing + + tracing.ingest(otlp_body, content_type, content_encoding, tenant) # POST /v1/traces + await tracing.list_traces(scope, start_ms, end_ms, cursor) # GET /v1/traces + await tracing.get_trace(trace_id, scope) # GET /v1/traces/{id} + await tracing.get_span(trace_id, span_id, scope) # GET /v1/traces/{id}/spans/{span_id} + +The proxy endpoints are thin wrappers: auth -> build tenant/scope -> call one method. +""" + +import asyncio +import os +from typing import Final + +from litellm.constants import ( + AGENT_TRACING_RETENTION_DAYS, + AGENT_TRACING_SPEND_LOG_RETENTION_DAYS, + OTLP_MAX_BODY_BYTES, + OTLP_OFFLOAD_DECODE_BYTES, +) +from litellm.integrations.clickhouse.schema import ensure_schema +from litellm.rust_bridge.traces import TraceStorage +from litellm.tracing.decode import decode_otlp +from litellm.tracing.store import ClickHouseTraceStore +from litellm.tracing.types import ( + SpanDetail, + SpanRow, + Trace, + TracePage, + TraceScope, +) + + +class TracingPayloadTooLargeError(Exception): + pass + + +class Tenant: + """Who sent the spans. Always taken from auth, never from span attributes.""" + + def __init__(self, team_id: str, api_key_hash: str, org_id: str = ""): + self.team_id = team_id + self.api_key_hash = api_key_hash + self.org_id = org_id + + def stamp(self, row: SpanRow) -> SpanRow: + row["TeamId"] = self.team_id + row["ApiKeyHash"] = self.api_key_hash + row["ResourceAttributes"] = { + **row["ResourceAttributes"], + "litellm.team_id": self.team_id, + "litellm.api_key_hash": self.api_key_hash, + "litellm.org_id": self.org_id, + } + return row + + +class TraceReceiver: + def __init__(self, store: ClickHouseTraceStore): + self.store = store + + @classmethod + def from_env(cls) -> "TraceReceiver": + return cls( + store=ClickHouseTraceStore( + TraceStorage( + database=os.getenv("CLICKHOUSE_DATABASE", "litellm"), + url=os.environ["CLICKHOUSE_URL"], + reader_url=os.environ["CLICKHOUSE_READER_URL"], + ) + ) + ) + + async def start(self) -> None: + await ensure_schema( + self.store.storage, + trace_retention_days=AGENT_TRACING_RETENTION_DAYS, + spend_log_retention_days=AGENT_TRACING_SPEND_LOG_RETENTION_DAYS, + ) + + # ------------------------------------------------------------ write + + async def ingest( + self, + body: bytes, + content_type: str | None, + content_encoding: str | None, + tenant: Tenant, + ) -> int: + """Decode an OTLP trace export and store its authenticated spans.""" + if len(body) > OTLP_MAX_BODY_BYTES: + raise TracingPayloadTooLargeError(f"OTLP body exceeds {OTLP_MAX_BODY_BYTES} bytes") + rows: Final = ( + await asyncio.to_thread(decode_otlp, body, content_type, content_encoding) + if len(body) > OTLP_OFFLOAD_DECODE_BYTES + else decode_otlp(body, content_type, content_encoding) + ) + try: + await self.store.insert_spans([tenant.stamp(r) for r in rows]) + except OverflowError as error: + raise TracingPayloadTooLargeError(str(error)) from error + return len(rows) + + # ------------------------------------------------------------ read + + async def list_traces(self, scope: TraceScope, start_ms: int, end_ms: int, cursor: str | None = None) -> TracePage: + return await self.store.list_traces(scope, start_ms, end_ms, cursor) + + async def get_trace(self, trace_id: str, scope: TraceScope) -> Trace | None: + return await self.store.get_trace(trace_id, scope) + + async def get_span(self, trace_id: str, span_id: str, scope: TraceScope) -> SpanDetail | None: + return await self.store.get_span(trace_id, span_id, scope) diff --git a/litellm/tracing/store.py b/litellm/tracing/store.py new file mode 100644 index 000000000000..e6814084c0fb --- /dev/null +++ b/litellm/tracing/store.py @@ -0,0 +1,271 @@ +"""ClickHouse-backed trace store: batched span writes and scoped reads.""" + +import base64 +import json +from datetime import UTC, datetime +from typing import Any, Final + +from litellm.constants import AGENT_TRACING_LIST_PAGE_SIZE +from litellm.integrations.clickhouse.schema import ( + AGENT_TRACES_TABLE, + OTEL_TRACES_TABLE, +) +from litellm.rust_bridge.traces import TraceStorage +from litellm.tracing.types import ( + AgentNode, + Span, + SpanDetail, + SpanRow, + SpanStatus, + Trace, + TracePage, + TraceScope, + TraceSummary, +) + +NANOS_PER_MS: Final = 1_000_000 +_STATUS: Final[dict[str, SpanStatus]] = {"STATUS_CODE_OK": "ok", "STATUS_CODE_ERROR": "error"} + +_SCOPE_OTEL: Final = ( + "(empty({team_ids:Array(String)}) OR TeamId IN {team_ids:Array(String)})" + " AND ({api_key_hash:String} = '' OR ApiKeyHash = {api_key_hash:String})" +) +# Page of traces from the per-trace MV. +# The MV writes one partial row per insert, so root fields come from the partial that saw the root span. +# agent_traces has no ApiKeyHash, so key-scoped (team-less) reads filter trace ids through otel_traces. +LIST_TRACES_SQL: Final = f""" +SELECT t.TraceId AS trace_id, any(t.RootName) AS name, any(t.ServiceName) AS service, + any(t.RootInput) AS input_preview, any(t.RootStatus) AS status, + toUnixTimestamp64Milli(any(t.StartTs)) AS start_ms, + dateDiff('millisecond', any(t.StartTs), any(t.EndTs)) AS duration_ms, + any(t.SpanCount) AS span_count, length(any(t.AgentNames)) AS agent_count, + any(t.AgentCount) AS agent_invocations, + any(t.LlmCount) AS llm_calls, any(t.ToolCount) AS tool_calls, + any(t.InputTokens) AS input_tokens, any(t.OutputTokens) AS output_tokens, + any(t.Models) AS models, any(t.ErrorCount) AS error_count +FROM ( + SELECT TeamId, TraceId, min(StartTs) AS StartTs, max(EndTs) AS EndTs, + any(ServiceName) AS ServiceName, anyLastIf(a.RootName, a.RootName != '') AS RootName, + anyLastIf(a.RootInput, a.RootName != '') AS RootInput, + anyLastIf(a.RootStatus, a.RootName != '') AS RootStatus, + sum(SpanCount) AS SpanCount, sum(AgentCount) AS AgentCount, sum(LlmCount) AS LlmCount, + sum(ToolCount) AS ToolCount, sum(ErrorCount) AS ErrorCount, sum(InputTokens) AS InputTokens, + sum(OutputTokens) AS OutputTokens, groupUniqArrayArray(Models) AS Models, + groupUniqArrayArray(AgentNames) AS AgentNames + FROM {AGENT_TRACES_TABLE} AS a + WHERE (empty({{team_ids:Array(String)}}) OR TeamId IN {{team_ids:Array(String)}}) + AND ({{api_key_hash:String}} = '' OR TraceId IN ( + SELECT TraceId FROM {OTEL_TRACES_TABLE} WHERE {_SCOPE_OTEL})) + GROUP BY TeamId, TraceId + HAVING StartTs >= fromUnixTimestamp64Milli({{start_ms:Int64}}) + AND StartTs < fromUnixTimestamp64Milli({{end_ms:Int64}}) + AND (({{cursor_ms:Int64}} = 0) OR (toUnixTimestamp64Milli(StartTs), TraceId) + < ({{cursor_ms:Int64}}, {{cursor_trace_id:String}})) + ORDER BY StartTs DESC, TraceId DESC + LIMIT {{limit:UInt32}} +) AS t +GROUP BY t.TraceId +ORDER BY start_ms DESC, t.TraceId DESC +""" + +TRACE_SPANS_SQL: Final = f""" +SELECT o.SpanId AS span_id, o.ParentSpanId AS parent_span_id, o.SpanName AS name, + o.ObservationType AS type, o.AgentName AS agent, o.StatusCode AS status, + o.StatusMessage AS status_message, + toUnixTimestamp64Nano(o.Timestamp) AS start_ns, o.Duration AS duration_ns, + o.ServiceName AS service, o.InputPreview AS input_preview, o.Model AS model, + o.InputTokens AS input_tokens, o.OutputTokens AS output_tokens, + o.LiteLLMRequestId AS litellm_request_id +FROM {OTEL_TRACES_TABLE} AS o +WHERE o.TraceId = {{trace_id:String}} AND {_SCOPE_OTEL} +ORDER BY o.Timestamp +LIMIT 1 BY o.SpanId +""" + +SPAN_DETAIL_SQL: Final = f""" +SELECT SpanId AS span_id, Input AS input, Output AS output, SpanAttributes AS attributes +FROM {OTEL_TRACES_TABLE} +WHERE TraceId = {{trace_id:String}} AND SpanId = {{span_id:String}} AND {_SCOPE_OTEL} +LIMIT 1 +""" + + +def encode_cursor(start_ms: int, trace_id: str) -> str: + return base64.urlsafe_b64encode(json.dumps([start_ms, trace_id]).encode()).decode() + + +def decode_cursor(cursor: str | None) -> tuple[int, str]: + if not cursor: + return 0, "" + start_ms, trace_id = json.loads(base64.urlsafe_b64decode(cursor.encode())) + return int(start_ms), str(trace_id) + + +def _iso(ms: int) -> str: + return datetime.fromtimestamp(ms / 1000, tz=UTC).isoformat() + + +def _status(code: str) -> SpanStatus: + return _STATUS.get(code, "unset") + + +def trace_summary_from_row(row: dict[str, Any]) -> TraceSummary: + return TraceSummary( + trace_id=row["trace_id"], + name=row["name"], + service=row["service"], + input_preview=row["input_preview"], + start_time=_iso(int(row["start_ms"])), + duration_ms=float(row["duration_ms"]), + status=_status(row["status"]), + span_count=int(row["span_count"]), + agent_count=int(row["agent_count"]), + agent_invocations=int(row.get("agent_invocations") or row["agent_count"]), + llm_calls=int(row["llm_calls"]), + tool_calls=int(row["tool_calls"]), + error_count=int(row.get("error_count") or 0), + input_tokens=int(row["input_tokens"]), + output_tokens=int(row["output_tokens"]), + models=list(row["models"]), + ) + + +def span_from_row(row: dict[str, Any], trace_start_ns: int) -> Span: + return Span( + span_id=row["span_id"], + parent_span_id=row["parent_span_id"] or None, + name=row["name"], + type=row["type"], + agent=row["agent"], + start_offset_ms=(int(row["start_ns"]) - trace_start_ns) / NANOS_PER_MS, + duration_ms=int(row["duration_ns"]) / NANOS_PER_MS, + status=_status(row["status"]), + error=row.get("status_message") or None, + input_preview=row["input_preview"], + model=row["model"] or None, + input_tokens=int(row["input_tokens"]), + output_tokens=int(row["output_tokens"]), + litellm_request_id=row["litellm_request_id"] or None, + ) + + +def _parent_agent_of(span: Span, by_id: dict[str, Span]) -> str | None: + parent_id = span["parent_span_id"] + while parent_id is not None and parent_id in by_id: + parent = by_id[parent_id] + if parent["type"] == "agent" and parent["name"] != span["name"]: + return parent["name"] + parent_id = parent["parent_span_id"] + return None + + +def agent_nodes(spans: list[Span]) -> list[AgentNode]: + """One node per distinct agent name (200 `researcher` invocations = 1 node), with who invoked it.""" + by_id: Final = {s["span_id"]: s for s in spans} + agents: dict[str, AgentNode] = {} + for span in spans: + if span["type"] != "agent": + continue + node = agents.setdefault( + span["name"], + AgentNode( + name=span["name"], + parent_agent=_parent_agent_of(span, by_id), + invocations=0, + llm_calls=0, + tool_calls=0, + duration_ms=0.0, + ), + ) + node["invocations"] += 1 + node["duration_ms"] += span["duration_ms"] + for span in spans: + owner = agents.get(span["agent"]) + if owner is None: + continue + if span["type"] == "llm": + owner["llm_calls"] += 1 + elif span["type"] == "tool": + owner["tool_calls"] += 1 + return list(agents.values()) + + +def trace_from_rows(trace_id: str, rows: list[dict[str, Any]]) -> Trace | None: + if not rows: + return None + trace_start_ns: Final = min(int(r["start_ns"]) for r in rows) + trace_end_ns: Final = max(int(r["start_ns"]) + int(r["duration_ns"]) for r in rows) + spans: Final = [span_from_row(r, trace_start_ns) for r in rows] + root: Final = next((s for s in spans if s["parent_span_id"] is None), spans[0]) + agents: Final = agent_nodes(spans) + llm_spans: Final = [s for s in spans if s["type"] == "llm"] + return Trace( + summary=TraceSummary( + trace_id=trace_id, + name=root["name"], + service=rows[0]["service"], + input_preview=root["input_preview"], + start_time=_iso(trace_start_ns // NANOS_PER_MS), + duration_ms=(trace_end_ns - trace_start_ns) / NANOS_PER_MS, + status=root["status"], + span_count=len(spans), + agent_count=len(agents), + agent_invocations=sum(a["invocations"] for a in agents), + llm_calls=len(llm_spans), + tool_calls=sum(1 for s in spans if s["type"] == "tool"), + error_count=sum(1 for s in spans if s["status"] == "error"), + input_tokens=sum(s["input_tokens"] for s in spans), + output_tokens=sum(s["output_tokens"] for s in spans), + models=sorted({s["model"] for s in llm_spans if s["model"]}), + ), + agents=agents, + spans=spans, + ) + + +class ClickHouseTraceStore: + """Stores spans and runs scoped trace reads.""" + + def __init__(self, storage: TraceStorage): + self.storage = storage + + async def insert_spans(self, rows: list[SpanRow]) -> None: + await self.storage.insert_rows(OTEL_TRACES_TABLE, [dict(row) for row in rows]) + + async def list_traces( + self, + scope: TraceScope, + start_ms: int, + end_ms: int, + cursor: str | None = None, + limit: int = AGENT_TRACING_LIST_PAGE_SIZE, + ) -> TracePage: + cursor_ms, cursor_trace_id = decode_cursor(cursor) + rows = await self.storage.query( + LIST_TRACES_SQL, + { + **scope, + "start_ms": start_ms, + "end_ms": end_ms, + "cursor_ms": cursor_ms, + "cursor_trace_id": cursor_trace_id, + "limit": limit, + }, + ) + next_cursor = encode_cursor(int(rows[-1]["start_ms"]), rows[-1]["trace_id"]) if len(rows) == limit else None + return TracePage(data=[trace_summary_from_row(r) for r in rows], next_cursor=next_cursor) + + async def get_trace(self, trace_id: str, scope: TraceScope) -> Trace | None: + rows = await self.storage.query(TRACE_SPANS_SQL, {**scope, "trace_id": trace_id}) + return trace_from_rows(trace_id, rows) + + async def get_span(self, trace_id: str, span_id: str, scope: TraceScope) -> SpanDetail | None: + rows = await self.storage.query(SPAN_DETAIL_SQL, {**scope, "trace_id": trace_id, "span_id": span_id}) + if not rows: + return None + return SpanDetail( + span_id=rows[0]["span_id"], + input=rows[0]["input"], + output=rows[0]["output"], + attributes=dict(rows[0]["attributes"]), + ) diff --git a/litellm/tracing/types.py b/litellm/tracing/types.py new file mode 100644 index 000000000000..bf0e5b3a722c --- /dev/null +++ b/litellm/tracing/types.py @@ -0,0 +1,119 @@ +""" +Agent tracing types. + +A trace is one agent run. It's made of spans (agent / llm / tool / chain / framework). + Trace + ├── summary: TraceSummary + ├── agents: list[AgentNode] one per distinct agent name (for the agent graph) + └── spans: list[Span] flat, linked by parent_span_id + +""" + +from typing import Literal + +from typing_extensions import TypedDict + +SpanType = Literal["agent", "llm", "tool", "chain", "framework"] +SpanStatus = Literal["ok", "error", "unset"] + + +class Span(TypedDict): + span_id: str + parent_span_id: str | None + name: str + type: SpanType + agent: str # the agent this span runs inside, e.g. "researcher" + start_offset_ms: float # relative to trace start + duration_ms: float + status: SpanStatus + error: str | None # exception message when status == "error" + input_preview: str + model: str | None + input_tokens: int + output_tokens: int + litellm_request_id: str | None + + +class AgentNode(TypedDict): + """One distinct agent in a trace. 200 invocations of `researcher` = one node.""" + + name: str + parent_agent: str | None + invocations: int + llm_calls: int + tool_calls: int + duration_ms: float + + +class TraceSummary(TypedDict): + trace_id: str + name: str + service: str + input_preview: str + start_time: str # ISO 8601 + duration_ms: float + status: SpanStatus + span_count: int + agent_count: int # distinct agent names (researcher x200 counts once) + agent_invocations: int # agent spans (researcher x200 counts 200) + llm_calls: int + tool_calls: int + error_count: int # spans with an error status; > 0 means the run shows as failed + input_tokens: int + output_tokens: int + models: list[str] + + +class Trace(TypedDict): + summary: TraceSummary + agents: list[AgentNode] + spans: list[Span] + + +class TracePage(TypedDict): + data: list[TraceSummary] + next_cursor: str | None + + +class SpanDetail(TypedDict): + span_id: str + input: str + output: str + attributes: dict[str, str] + + +class TraceScope(TypedDict): + """Who is asking. Empty team_ids = all teams (admins only).""" + + team_ids: list[str] + api_key_hash: str + + +class SpanRow(TypedDict): + """One stored span (ClickHouse `otel_traces` row). Produced by `litellm.tracing.decode`.""" + + Timestamp: int # unix ns + TraceId: str + SpanId: str + ParentSpanId: str + TraceState: str + SpanName: str + SpanKind: str + ServiceName: str + ResourceAttributes: dict[str, str] + ScopeName: str + ScopeVersion: str + SpanAttributes: dict[str, str] + Duration: int # ns + StatusCode: str + StatusMessage: str + TeamId: str + ApiKeyHash: str + ObservationType: SpanType + AgentName: str + LiteLLMRequestId: str + Model: str + InputTokens: int + OutputTokens: int + Input: str + Output: str diff --git a/tests/test_litellm/integrations/clickhouse/test_clickhouse_batch_logger.py b/tests/test_litellm/integrations/clickhouse/test_clickhouse_batch_logger.py new file mode 100644 index 000000000000..47085002116d --- /dev/null +++ b/tests/test_litellm/integrations/clickhouse/test_clickhouse_batch_logger.py @@ -0,0 +1,71 @@ +""" +Tests for the CustomBatchLogger-based ClickHouse base logger. +""" + +import os +import sys +from unittest.mock import AsyncMock, MagicMock, patch + +sys.path.insert(0, os.path.abspath("../../..")) + +import pytest + +from litellm.integrations.clickhouse import clickhouse_batch_logger as module +from litellm.integrations.clickhouse.clickhouse_batch_logger import ClickHouseBatchLogger +from litellm.integrations.custom_batch_logger import CustomBatchLogger + + +class _TestLogger(ClickHouseBatchLogger): + table = "test_table" + + +def _logger(insert: AsyncMock) -> _TestLogger: + storage = MagicMock() + storage.insert_rows = insert + return _TestLogger(storage=storage) + + +def test_is_a_custom_batch_logger(): + assert issubclass(ClickHouseBatchLogger, CustomBatchLogger) + + +@pytest.mark.asyncio +async def test_flush_splits_into_batches_and_empties_queue(): + insert = AsyncMock() + logger = _logger(insert) + logger.batch_size = 2 + logger.log_queue.extend([{"i": i} for i in range(5)]) + + await logger.flush_queue() + + assert [len(c.args[1]) for c in insert.await_args_list] == [2, 2, 1] + assert all(c.args[0] == "test_table" for c in insert.await_args_list) + assert logger.log_queue == [] + assert logger.rows_written == 5 + + +@pytest.mark.asyncio +async def test_is_full_signals_backpressure(): + logger = _logger(AsyncMock()) + with patch.object(module, "CLICKHOUSE_MAX_BUFFERED_ROWS", 3): + logger.log_queue.extend([{}, {}]) + assert logger.is_full() is False + logger.log_queue.append({}) + assert logger.is_full() is True + + +@pytest.mark.asyncio +async def test_failed_insert_is_requeued_then_dropped(): + insert = AsyncMock(side_effect=RuntimeError("clickhouse down")) + logger = _logger(insert) + logger.log_queue.extend([{"request_id": "a"}, {"request_id": "b"}]) + + with patch.object(module, "CLICKHOUSE_MAX_RETRIES", 2): + await logger.flush_queue() + assert len(logger.log_queue) == 2 # kept for retry + await logger.flush_queue() + + assert insert.await_count == 2 + assert logger.rows_dropped == 2 + assert logger.rows_written == 0 + assert logger.log_queue == [] diff --git a/tests/test_litellm/proxy/common_utils/test_http_parsing_utils.py b/tests/test_litellm/proxy/common_utils/test_http_parsing_utils.py index 7929a0b21af9..d4c2fb3b9a1c 100644 --- a/tests/test_litellm/proxy/common_utils/test_http_parsing_utils.py +++ b/tests/test_litellm/proxy/common_utils/test_http_parsing_utils.py @@ -71,6 +71,18 @@ async def test_read_raw_json_body_is_none_for_form_bodies(): assert await read_raw_json_body(request) is None +@pytest.mark.asyncio +@pytest.mark.parametrize("content_type", ["application/x-protobuf", "application/protobuf; charset=binary"]) +async def test_protobuf_body_is_not_parsed_as_json(content_type): + # OTLP trace exports (POST /v1/traces) are binary protobuf; arbitrary bytes like these + # used to hit the JSON surrogate-repair path and fail auth with a 400. + body = b"\n\xa2\x01\n\x1c\n\x0cservice.name\x12\x0c\n\nswarm\xed\xa0\x80\xff" + request = _starlette_request(body, content_type) + + assert await _read_request_body(request) == {} + assert await request.body() == body # body is still readable by the endpoint + + @pytest.mark.asyncio async def test_read_raw_json_body_is_none_for_a_request_that_only_mocks_the_parsed_body_path(): mock_request = MagicMock() diff --git a/tests/test_litellm/proxy/test_tracing_endpoints.py b/tests/test_litellm/proxy/test_tracing_endpoints.py new file mode 100644 index 000000000000..7b46beec2821 --- /dev/null +++ b/tests/test_litellm/proxy/test_tracing_endpoints.py @@ -0,0 +1,160 @@ +""" +Tests for the agent tracing endpoints (litellm/proxy/tracing_endpoints.py). +""" + +import os +import sys +from unittest.mock import AsyncMock, MagicMock + +sys.path.insert(0, os.path.abspath("../../..")) + +import pytest +from fastapi import FastAPI, HTTPException +from fastapi.testclient import TestClient + +from litellm.proxy import tracing_endpoints +from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.tracing import TracingPayloadTooLargeError + +TEAM_KEY = UserAPIKeyAuth( + token="hashed-key", team_id="team-research", org_id="org-1", user_role=LitellmUserRoles.INTERNAL_USER +) + + +# ---------------------------------------------------------------- scope / tenant + + +def test_scope_for_admin_sees_everything(): + for role in (LitellmUserRoles.PROXY_ADMIN, LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY): + auth = UserAPIKeyAuth(token="k", team_id="team-a", user_role=role) + assert tracing_endpoints.scope_for(auth) == {"team_ids": [], "api_key_hash": ""} + + +def test_scope_for_team_key_sees_its_team(): + assert tracing_endpoints.scope_for(TEAM_KEY) == {"team_ids": ["team-research"], "api_key_hash": ""} + + +def test_scope_for_teamless_key_sees_only_its_own_traces(): + auth = UserAPIKeyAuth(token="hashed-key", user_role=LitellmUserRoles.INTERNAL_USER) + assert tracing_endpoints.scope_for(auth) == {"team_ids": [""], "api_key_hash": "hashed-key"} + + +def test_scope_for_no_team_no_token_is_forbidden(): + with pytest.raises(HTTPException) as e: + tracing_endpoints.scope_for(UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER)) + assert e.value.status_code == 403 + + +def test_tenant_for_comes_from_auth(): + tenant = tracing_endpoints.tenant_for(TEAM_KEY) + assert (tenant.team_id, tenant.api_key_hash, tenant.org_id) == ("team-research", "hashed-key", "org-1") + blank = tracing_endpoints.tenant_for(UserAPIKeyAuth()) + assert (blank.team_id, blank.api_key_hash, blank.org_id) == ("", "", "") + + +# ---------------------------------------------------------------- endpoints + + +@pytest.fixture +def receiver(monkeypatch) -> MagicMock: + fake = MagicMock() + fake.ingest = AsyncMock(return_value=1) + fake.list_traces = AsyncMock(return_value={"data": [], "next_cursor": None}) + fake.get_trace = AsyncMock(return_value=None) + fake.get_span = AsyncMock(return_value=None) + monkeypatch.setattr(tracing_endpoints, "receiver", fake) + return fake + + +@pytest.fixture +def client() -> TestClient: + app = FastAPI() + app.include_router(tracing_endpoints.router) + app.dependency_overrides[user_api_key_auth] = lambda: TEAM_KEY + return TestClient(app) + + +def test_501_when_tracing_not_enabled(client, monkeypatch): + monkeypatch.setattr(tracing_endpoints, "receiver", None) + assert client.post("/v1/traces", content=b"").status_code == 501 + assert client.get("/v1/traces").status_code == 501 + + +def test_post_protobuf_returns_empty_protobuf(client, receiver): + response = client.post( + "/v1/traces", + content=b"\x0a\x00", + headers={"content-type": "application/x-protobuf", "content-encoding": "gzip"}, + ) + assert response.status_code == 200 + assert response.content == b"" + assert response.headers["content-type"] == "application/x-protobuf" + kwargs = receiver.ingest.call_args.kwargs + assert kwargs["body"] == b"\x0a\x00" + assert kwargs["content_type"] == "application/x-protobuf" + assert kwargs["content_encoding"] == "gzip" + assert kwargs["tenant"].team_id == "team-research" + + +def test_post_json_returns_empty_json(client, receiver): + response = client.post("/v1/traces", content=b"{}", headers={"content-type": "application/json"}) + assert response.status_code == 200 + assert response.json() == {} + + +def test_post_clickhouse_failure_is_503_with_retry_after(client, receiver): + receiver.ingest.side_effect = RuntimeError("ClickHouse unavailable") + response = client.post("/v1/traces", content=b"", headers={"content-type": "application/x-protobuf"}) + assert response.status_code == 503 + assert response.headers["retry-after"] == str(tracing_endpoints.OTLP_RETRY_AFTER_SECONDS) + + +def test_post_too_large_is_413(client, receiver): + receiver.ingest.side_effect = TracingPayloadTooLargeError("OTLP body exceeds 10 bytes") + response = client.post("/v1/traces", content=b"x" * 20) + assert response.status_code == 413 + assert "exceeds" in response.json()["detail"] + + +def test_list_traces_passes_scope_window_and_cursor(client, receiver): + response = client.get("/v1/traces", params={"start_ms": 1, "end_ms": 2, "cursor": "abc"}) + assert response.status_code == 200 + assert response.json() == {"data": [], "next_cursor": None} + receiver.list_traces.assert_awaited_once_with( + scope={"team_ids": ["team-research"], "api_key_hash": ""}, start_ms=1, end_ms=2, cursor="abc" + ) + + +def test_list_traces_defaults_to_last_24h(client, receiver): + client.get("/v1/traces") + kwargs = receiver.list_traces.call_args.kwargs + assert kwargs["end_ms"] - kwargs["start_ms"] == tracing_endpoints.MS_PER_DAY + assert kwargs["cursor"] is None + + +def test_get_trace_404_and_200(client, receiver): + assert client.get("/v1/traces/missing").status_code == 404 + trace = {"summary": {"trace_id": "t1"}, "agents": [], "spans": []} + receiver.get_trace.return_value = trace + response = client.get("/v1/traces/t1") + assert response.status_code == 200 + assert response.json() == trace + receiver.get_trace.assert_awaited_with("t1", {"team_ids": ["team-research"], "api_key_hash": ""}) + + +def test_get_span_404_and_200(client, receiver): + assert client.get("/v1/traces/t1/spans/s1").status_code == 404 + receiver.get_span.return_value = {"span_id": "s1", "input": "", "output": "", "attributes": {}} + response = client.get("/v1/traces/t1/spans/s1") + assert response.status_code == 200 + assert response.json()["span_id"] == "s1" + receiver.get_span.assert_awaited_with("t1", "s1", {"team_ids": ["team-research"], "api_key_hash": ""}) + + +def test_teamless_key_without_token_gets_403_on_reads(client, receiver): + client.app.dependency_overrides[user_api_key_auth] = lambda: UserAPIKeyAuth( + user_role=LitellmUserRoles.INTERNAL_USER + ) + assert client.get("/v1/traces").status_code == 403 + receiver.list_traces.assert_not_called() diff --git a/tests/test_litellm/tracing/fixtures/langsmith_deep_agent_export.json b/tests/test_litellm/tracing/fixtures/langsmith_deep_agent_export.json new file mode 100644 index 000000000000..9bd8e67633b2 --- /dev/null +++ b/tests/test_litellm/tracing/fixtures/langsmith_deep_agent_export.json @@ -0,0 +1,924 @@ +{ + "resourceSpans": [ + { + "resource": { + "attributes": [ + { + "key": "telemetry.sdk.language", + "value": { + "stringValue": "python" + } + }, + { + "key": "telemetry.sdk.name", + "value": { + "stringValue": "opentelemetry" + } + }, + { + "key": "telemetry.sdk.version", + "value": { + "stringValue": "1.45.0" + } + }, + { + "key": "service.instance.id", + "value": { + "stringValue": "86db1687-77ed-422d-a6f7-0319594d9158" + } + }, + { + "key": "service.name", + "value": { + "stringValue": "agent-demo" + } + } + ] + }, + "scopeSpans": [ + { + "scope": { + "name": "langsmith" + }, + "spans": [ + { + "traceId": "S61CuE6d47pG/IcBhfjwIw==", + "spanId": "XnnztbUEmF4=", + "name": "deep_research_agent", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1790742989377137920", + "endTimeUnixNano": "1790743040762587136", + "attributes": [ + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "chain" + } + }, + { + "key": "langsmith.span.kind", + "value": { + "stringValue": "chain" + } + }, + { + "key": "langsmith.trace.name", + "value": { + "stringValue": "deep_research_agent" + } + }, + { + "key": "langsmith.trace.session_name", + "value": { + "stringValue": "default" + } + }, + { + "key": "gen_ai.system", + "value": { + "stringValue": "langchain" + } + }, + { + "key": "langsmith.metadata.ls_integration", + "value": { + "stringValue": "deepagents" + } + }, + { + "key": "langsmith.metadata.lc_agent_name", + "value": { + "stringValue": "deep_research_agent" + } + }, + { + "key": "langsmith.metadata.lc_versions", + "value": { + "stringValue": "{\"deepagents\":\"0.7.20\"}" + } + }, + { + "key": "langsmith.metadata.LANGSMITH_TRACING", + "value": { + "stringValue": "true" + } + }, + { + "key": "langsmith.metadata.LANGSMITH_TRACING_MODE", + "value": { + "stringValue": "otel" + } + }, + { + "key": "gen_ai.prompt", + "value": { + "bytesValue": "eyJtZXNzYWdlcyI6W3siY29udGVudCI6IlNob3VsZCB3ZSBzdG9yZSBPVEVMIGFnZW50IHNwYW5zIGluIENsaWNrSG91c2Ugb3IgUG9zdGdyZXMgYXQgNTBrIHNwYW5zL3NlYz8iLCJhZGRpdGlvbmFsX2t3YXJncyI6e30sInJlc3BvbnNlX21ldGFkYXRhIjp7fSwidHlwZSI6Imh1bWFuIiwiaWQiOiJiMTljODgzMS0wOWIwLTQ5ZjYtYjdlYS05YzQ3ZTM4OWNjMDAifV19" + } + }, + { + "key": "gen_ai.completion", + "value": { + "bytesValue": "eyJtZXNzYWdlcyI6W3siY29udGVudCI6IlNob3VsZCB3ZSBzdG9yZSBPVEVMIGFnZW50IHNwYW5zIGluIENsaWNrSG91c2Ugb3IgUG9zdGdyZXMgYXQgNTBrIHNwYW5zL3NlYz8iLCJhZGRpdGlvbmFsX2t3YXJncyI6e30sInJlc3BvbnNlX21ldGFkYXRhIjp7fSwidHlwZSI6Imh1bWFuIiwiaWQiOiJiMTljODgzMS0wOWIwLTQ5ZjYtYjdlYS05YzQ3ZTM4OWNjMDAifSx7ImNvbnRlbnQiOiJJJ2xsIGhlbHAgeW91IGRlY2lkZSBiZXR3ZWVuIENsaWNrSG91c2UgYW5kIFBvc3RncmVzIGZvciBzdG9yaW5nIE9wZW5UZWxlbWV0cnkgc3BhbnMgYXQgNTBrIHNwYW5zL3NlYy4gTGV0IG1lIHJlc2VhcmNoIHRoaXMgc3lzdGVtYXRpY2FsbHkuIiwiYWRkaXRpb25hbF9rd2FyZ3MiOnsicmVmdXNhbCI6bnVsbH0sInJlc3BvbnNlX21ldGFkYXRhIjp7InRva2VuX3VzYWdlIjp7ImNvbXBsZXRpb25fdG9rZW5zIjo0NjcsInByb21wdF90b2tlbnMiOjMzMzIsInRvdGFsX3Rva2VucyI6Mzc5OSwiY29tcGxldGlvbl90b2tlbnNfZGV0YWlscyI6eyJhY2NlcHRlZF9wcmVkaWN0aW9uX3Rva2VucyI6bnVsbCwiYXVkaW9fdG9rZW5zIjpudWxsLCJyZWFzb25pbmdfdG9rZW5zIjowLCJyZWplY3RlZF9wcmVkaWN0aW9uX3Rva2VucyI6bnVsbCwidGV4dF90b2tlbnMiOjQ2N30sInByb21wdF90b2tlbnNfZGV0YWlscyI6eyJhdWRpb190b2tlbnMiOm51bGwsImNhY2hlX3dyaXRlX3Rva2VucyI6MzMyOSwiY2FjaGVkX3Rva2VucyI6MCwiaW1hZ2VfdG9rZW5zIjpudWxsLCJ0ZXh0X3Rva2VucyI6MywiY2FjaGVfY3JlYXRpb25fdG9rZW5zIjozMzI5LCJjYWNoZV9jcmVhdGlvbl90b2tlbl9kZXRhaWxzIjp7ImVwaGVtZXJhbF81bV9pbnB1dF90b2tlbnMiOjMzMjksImVwaGVtZXJhbF8xaF9pbnB1dF90b2tlbnMiOjB9fSwiY2FjaGVfY3JlYXRpb25faW5wdXRfdG9rZW5zIjozMzI5LCJjYWNoZV9yZWFkX2lucHV0X3Rva2VucyI6MCwiaW5mZXJlbmNlX2dlbyI6Im5vdF9hdmFpbGFibGUiLCJzZXJ2aWNlX3RpZXIiOiJzdGFuZGFyZCJ9LCJtb2RlbF9wcm92aWRlciI6Im9wZW5haSIsIm1vZGVsX25hbWUiOiJjbGF1ZGUtc29ubmV0LTQtNSIsInN5c3RlbV9maW5nZXJwcmludCI6bnVsbCwiaWQiOiJjaGF0Y21wbC00MDc3YmIzNi05MzgwLTRhM2ItOTQ4MS0yNDU3MDBjZWYwOWEiLCJmaW5pc2hfcmVhc29uIjoidG9vbF9jYWxscyIsImxvZ3Byb2JzIjpudWxsfSwidHlwZSI6ImFpIiwibmFtZSI6ImRlZXBfcmVzZWFyY2hfYWdlbnQiLCJpZCI6ImxjX3J1bi0tMDFhMGYwOTktOGE0Ny03ZTQyLWE1ZjQtNWM0N2RlM2QxY2VjLTAiLCJ0b29sX2NhbGxzIjpbeyJuYW1lIjoid3JpdGVfZmlsZSIsImFyZ3MiOnsiZmlsZV9wYXRoIjoiL3RtcC9yZXNlYXJjaF90b2Rvcy5tZCIsImNvbnRlbnQiOiIjIFJlc2VhcmNoIFBsYW46IENsaWNrSG91c2UgdnMgUG9zdGdyZXMgZm9yIE9URUwgU3BhbnMgKDUway9zZWMpXG5cbiMjIFRhc2tzXG4tIFsgXSBSZXNlYXJjaCBDbGlja0hvdXNlIGFuZCBQb3N0Z3JlcyBjYXBhYmlsaXRpZXMgZm9yIGhpZ2gtdm9sdW1lIHRpbWUtc2VyaWVzIGRhdGFcbi4uLiJ9LCJpZCI6InRvb2x1XzAxNjFYaFlQM0I1Zmc0VTFwc1QzcGNpUiIsInR5cGUiOiJ0b29sX2NhbGwifSx7Im5hbWUiOiJ0YXNrIiwiYXJncyI6eyJzdWJhZ2VudF90eXBlIjoicmVzZWFyY2hlciIsImRlc2NyaXB0aW9uIjoiUmVzZWFyY2ggYW5kIGNvbXBhcmUgQ2xpY2tIb3VzZSB2cyBQb3N0Z3JlcyBmb3Igc3RvcmluZyBPcGVuVGVsZW1ldHJ5IChPVEVMKSBhZ2VudCBzcGFucyBhdCA1MCwwMDAgc3BhbnMgcGVyIHNlY29uZC5cblxuRm9jdXMgb246XG4xLiBXcml0ZSB0aHJvdWdocHV0IGNhcGFiaWxpdGllcy4uLiJ9LCJpZCI6InRvb2x1XzAxUEx5bzhUS0tUcFhSNGZwOTZEbjkzVyIsInR5cGUiOiJ0b29sX2NhbGwifV0sImludmFsaWRfdG9vbF9jYWxscyI6W10sInVzYWdlX21ldGFkYXRhIjp7ImlucHV0X3Rva2VucyI6MzMzMiwib3V0cHV0X3Rva2VucyI6NDY3LCJ0b3RhbF90b2tlbnMiOjM3OTksImlucHV0X3Rva2VuX2RldGFpbHMiOnsiY2FjaGVfcmVhZCI6MCwiY2FjaGVfY3JlYXRpb24iOjMzMjl9LCJvdXRwdXRfdG9rZW5fZGV0YWlscyI6eyJyZWFzb25pbmciOjB9fX0seyJjb250ZW50IjoiQmFzZWQgb24gbXkgcmVzZWFyY2gsIGhlcmUncyBhIGNvbXByZWhlbnNpdmUgY29tcGFyaXNvbiBvZiAqKkNsaWNrSG91c2UgdnMgUG9zdGdyZXMqKiBmb3Igc3RvcmluZyBPcGVuVGVsZW1ldHJ5IHNwYW5zIGF0IDUwLDAwMCBzcGFucy9zZWNvbmQ6XG5cbiMjICoqMS4gV3JpdGUgVGhyLi4uIiwiYWRkaXRpb25hbF9rd2FyZ3MiOnt9LCJyZXNwb25zZV9tZXRhZGF0YSI6e30sInR5cGUiOiJ0b29sIiwibmFtZSI6InRhc2siLCJpZCI6IjE0MzVkZTNjLWI4NzktNDQ2YS04MDU0LTFiMGI4MjQ1YmZhZSIsInRvb2xfY2FsbF9pZCI6InRvb2x1XzAxUEx5bzhUS0tUcFhSNGZwOTZEbjkzVyIsInN0YXR1cyI6InN1Y2Nlc3MifSx7ImNvbnRlbnQiOiJCYXNlZCBvbiB0aGUgcmVzZWFyY2ggZmluZGluZ3MsIGhlcmUncyBteSByZWNvbW1lbmRhdGlvbjpcblxuIyMgUmVjb21tZW5kYXRpb246ICoqVXNlIENsaWNrSG91c2UqKlxuXG4qKkNsaWNrSG91c2UgaXMgdGhlIGNsZWFyIGNob2ljZSoqIGZvciBzdG9yaW5nIDUwayBPVEVMIHNwYW5zLy4uLiIsImFkZGl0aW9uYWxfa3dhcmdzIjp7InJlZnVzYWwiOm51bGx9LCJyZXNwb25zZV9tZXRhZGF0YSI6eyJ0b2tlbl91c2FnZSI6eyJjb21wbGV0aW9uX3Rva2VucyI6MjQxLCJwcm9tcHRfdG9rZW5zIjo0NTcwLCJ0b3RhbF90b2tlbnMiOjQ4MTEsImNvbXBsZXRpb25fdG9rZW5zX2RldGFpbHMiOnsiYWNjZXB0ZWRfcHJlZGljdGlvbl90b2tlbnMiOm51bGwsImF1ZGlvX3Rva2VucyI6bnVsbCwicmVhc29uaW5nX3Rva2VucyI6MCwicmVqZWN0ZWRfcHJlZGljdGlvbl90b2tlbnMiOm51bGwsInRleHRfdG9rZW5zIjoyNDF9LCJwcm9tcHRfdG9rZW5zX2RldGFpbHMiOnsiYXVkaW9fdG9rZW5zIjpudWxsLCJjYWNoZV93cml0ZV90b2tlbnMiOjEyMzQsImNhY2hlZF90b2tlbnMiOjMzMjksImltYWdlX3Rva2VucyI6bnVsbCwidGV4dF90b2tlbnMiOjcsImNhY2hlX2NyZWF0aW9uX3Rva2VucyI6MTIzNCwiY2FjaGVfY3JlYXRpb25fdG9rZW5fZGV0YWlscyI6eyJlcGhlbWVyYWxfNW1faW5wdXRfdG9rZW5zIjoxMjM0LCJlcGhlbWVyYWxfMWhfaW5wdXRfdG9rZW5zIjowfX0sImNhY2hlX2NyZWF0aW9uX2lucHV0X3Rva2VucyI6MTIzNCwiY2FjaGVfcmVhZF9pbnB1dF90b2tlbnMiOjMzMjksImluZmVyZW5jZV9nZW8iOiJub3RfYXZhaWxhYmxlIiwic2VydmljZV90aWVyIjoic3RhbmRhcmQifSwibW9kZWxfcHJvdmlkZXIiOiJvcGVuYWkiLCJtb2RlbF9uYW1lIjoiY2xhdWRlLXNvbm5ldC00LTUiLCJzeXN0ZW1fZmluZ2VycHJpbnQiOm51bGwsImlkIjoiY2hhdGNtcGwtZjI2Y2NiNDUtYWIxYi00NGM2LWJkOWUtNDFhMDJjYTVmMTRkIiwiZmluaXNoX3JlYXNvbiI6InN0b3AiLCJsb2dwcm9icyI6bnVsbH0sInR5cGUiOiJhaSIsIm5hbWUiOiJkZWVwX3Jlc2VhcmNoX2FnZW50IiwiaWQiOiJsY19ydW4tLTAxYTBmMDlhLTM4ZTEtNzc0My04ZGIwLTNjNjU5YjdlMGY2MC0wIiwidG9vbF9jYWxscyI6W10sImludmFsaWRfdG9vbF9jYWxscyI6W10sInVzYWdlX21ldGFkYXRhIjp7ImlucHV0X3Rva2VucyI6NDU3MCwib3V0cHV0X3Rva2VucyI6MjQxLCJ0b3RhbF90b2tlbnMiOjQ4MTEsImlucHV0X3Rva2VuX2RldGFpbHMiOnsiY2FjaGVfcmVhZCI6MzMyOSwiY2FjaGVfY3JlYXRpb24iOjEyMzR9LCJvdXRwdXRfdG9rZW5fZGV0YWlscyI6eyJyZWFzb25pbmciOjB9fX1dLCJmaWxlcyI6eyIvdG1wL3Jlc2VhcmNoX3RvZG9zLm1kIjp7ImNvbnRlbnQiOiIjIFJlc2VhcmNoIFBsYW46IENsaWNrSG91c2UgdnMgUG9zdGdyZXMgZm9yIE9URUwgU3BhbnMgKDUway9zZWMpXG5cbiMjIFRhc2tzXG4tIFsgXSBSZXNlYXJjaCBDbGlja0hvdXNlIGFuZCBQb3N0Z3JlcyBjYXBhYmlsaXRpZXMgZm9yIGhpZ2gtdm9sdW1lIHRpbWUtc2VyaWVzIGRhdGFcbi4uLiIsImVuY29kaW5nIjoidXRmLTgiLCJjcmVhdGVkX2F0IjoiMjAyNi0wOS0zMFQwNDozNjozOC44OTkwNTArMDA6MDAiLCJtb2RpZmllZF9hdCI6IjIwMjYtMDktMzBUMDQ6MzY6MzguODk5MDUwKzAwOjAwIn19fQ==" + } + } + ], + "status": { + "code": "STATUS_CODE_OK" + }, + "flags": 256 + }, + { + "traceId": "S61CuE6d47pG/IcBhfjwIw==", + "spanId": "imocMZQNB68=", + "parentSpanId": "Hfr3D90RhPI=", + "name": "ChatOpenAI", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1790742989383207936", + "endTimeUnixNano": "1790742998893985024", + "attributes": [ + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "chat" + } + }, + { + "key": "gen_ai.serialized.name", + "value": { + "stringValue": "ChatOpenAI" + } + }, + { + "key": "langsmith.span.kind", + "value": { + "stringValue": "llm" + } + }, + { + "key": "langsmith.trace.name", + "value": { + "stringValue": "ChatOpenAI" + } + }, + { + "key": "langsmith.trace.session_name", + "value": { + "stringValue": "default" + } + }, + { + "key": "gen_ai.system", + "value": { + "stringValue": "anthropic" + } + }, + { + "key": "gen_ai.request.model", + "value": { + "stringValue": "claude-sonnet-4-5" + } + }, + { + "key": "gen_ai.tool.definitions", + "value": { + "stringValue": "[{\"type\":\"function\",\"function\":{\"name\":\"ls\",\"description\":\"Lists all files in a directory.\\n\\nThis is useful for exploring the filesystem and finding the right file to read or edit.\\nYou should almost ALWAYS use this tool before using the read_file or edit_file tools.\",\"parameters\":{\"properties\":{\"path\":{\"description\":\"Absolute path to the directory to list. Must be absolute, not relative.\",\"type\":\"string\"}},\"required\":[\"path\"],\"type\":\"object\"}}}]" + } + }, + { + "key": "langsmith.metadata.ls_integration", + "value": { + "stringValue": "langchain_chat_model" + } + }, + { + "key": "langsmith.metadata.lc_agent_name", + "value": { + "stringValue": "deep_research_agent" + } + }, + { + "key": "langsmith.metadata.lc_versions", + "value": { + "stringValue": "{\"deepagents\":\"0.7.20\",\"langchain-core\":\"1.6.6\",\"langchain\":\"1.4.3\",\"langchain-openai\":\"1.6.6\"}" + } + }, + { + "key": "langsmith.metadata.langgraph_step", + "value": { + "intValue": "2" + } + }, + { + "key": "langsmith.metadata.langgraph_node", + "value": { + "stringValue": "model" + } + }, + { + "key": "langsmith.metadata.langgraph_triggers", + "value": { + "stringValue": "[\"branch:to:model\"]" + } + }, + { + "key": "langsmith.metadata.langgraph_path", + "value": { + "stringValue": "[\"__pregel_pull\",\"model\"]" + } + }, + { + "key": "langsmith.metadata.langgraph_checkpoint_ns", + "value": { + "stringValue": "model:9abb6d12-32f9-4289-15b6-36ac41ba926c" + } + }, + { + "key": "langsmith.metadata.checkpoint_ns", + "value": { + "stringValue": "model:9abb6d12-32f9-4289-15b6-36ac41ba926c" + } + }, + { + "key": "langsmith.metadata.ls_provider", + "value": { + "stringValue": "openai" + } + }, + { + "key": "langsmith.metadata.ls_model_name", + "value": { + "stringValue": "claude-sonnet-4-5" + } + }, + { + "key": "langsmith.metadata.ls_model_type", + "value": { + "stringValue": "chat" + } + }, + { + "key": "langsmith.metadata.ls_max_tokens", + "value": { + "intValue": "700" + } + }, + { + "key": "langsmith.metadata.ls_method", + "value": { + "stringValue": "traceable" + } + }, + { + "key": "langsmith.metadata.model", + "value": { + "stringValue": "claude-sonnet-4-5" + } + }, + { + "key": "langsmith.metadata.model_name", + "value": { + "stringValue": "claude-sonnet-4-5" + } + }, + { + "key": "langsmith.metadata.stream", + "value": { + "boolValue": false + } + }, + { + "key": "langsmith.metadata.max_completion_tokens", + "value": { + "intValue": "700" + } + }, + { + "key": "langsmith.metadata._type", + "value": { + "stringValue": "openai-chat" + } + }, + { + "key": "langsmith.metadata.usage_metadata", + "value": { + "stringValue": "{\"input_tokens\":3332,\"output_tokens\":467,\"total_tokens\":3799,\"input_token_details\":{\"cache_read\":0,\"cache_creation\":3329},\"output_token_details\":{\"reasoning\":0}}" + } + }, + { + "key": "langsmith.metadata.LANGSMITH_TRACING", + "value": { + "stringValue": "true" + } + }, + { + "key": "langsmith.metadata.LANGSMITH_TRACING_MODE", + "value": { + "stringValue": "otel" + } + }, + { + "key": "langsmith.span.tags", + "value": { + "stringValue": "seq:step:1" + } + }, + { + "key": "gen_ai.prompt", + "value": { + "bytesValue": "eyJtZXNzYWdlcyI6W1t7ImxjIjoxLCJ0eXBlIjoiY29uc3RydWN0b3IiLCJpZCI6WyJsYW5nY2hhaW4iLCJzY2hlbWEiLCJtZXNzYWdlcyIsIlN5c3RlbU1lc3NhZ2UiXSwia3dhcmdzIjp7ImNvbnRlbnQiOiJZb3UgYXJlIGEgcmVzZWFyY2ggbGVhZC4gUGxhbiB3aXRoIHdyaXRlX3RvZG9zLCBkZWxlZ2F0ZSBvbmUgcXVlc3Rpb24gdG8gdGhlIHJlc2VhcmNoZXIgc3ViYWdlbnQgdmlhIHRhc2ssIHRoZW4gd3JpdGUgYSBzaG9ydCByZWNvbW1lbmRhdGlvbiAoPD01IHNlbnRlbmNlcykuIiwidHlwZSI6InN5c3RlbSJ9fSx7ImxjIjoxLCJ0eXBlIjoiY29uc3RydWN0b3IiLCJpZCI6WyJsYW5nY2hhaW4iLCJzY2hlbWEiLCJtZXNzYWdlcyIsIkh1bWFuTWVzc2FnZSJdLCJrd2FyZ3MiOnsiY29udGVudCI6IlNob3VsZCB3ZSBzdG9yZSBPVEVMIGFnZW50IHNwYW5zIGluIENsaWNrSG91c2Ugb3IgUG9zdGdyZXMgYXQgNTBrIHNwYW5zL3NlYz8iLCJ0eXBlIjoiaHVtYW4iLCJpZCI6ImIxOWM4ODMxLTA5YjAtNDlmNi1iN2VhLTljNDdlMzg5Y2MwMCJ9fV1dfQ==" + } + }, + { + "key": "gen_ai.usage.input_tokens", + "value": { + "intValue": "3332" + } + }, + { + "key": "gen_ai.usage.output_tokens", + "value": { + "intValue": "467" + } + }, + { + "key": "gen_ai.usage.total_tokens", + "value": { + "intValue": "3799" + } + }, + { + "key": "gen_ai.completion", + "value": { + "bytesValue": "eyJnZW5lcmF0aW9ucyI6W1t7InRleHQiOiJJJ2xsIGhlbHAgeW91IGRlY2lkZSBiZXR3ZWVuIENsaWNrSG91c2UgYW5kIFBvc3RncmVzIGZvciBzdG9yaW5nIE9wZW5UZWxlbWV0cnkgc3BhbnMgYXQgNTBrIHNwYW5zL3NlYy4gTGV0IG1lIHJlc2VhcmNoIHRoaXMgc3lzdGVtYXRpY2FsbHkuIiwiZ2VuZXJhdGlvbl9pbmZvIjp7ImZpbmlzaF9yZWFzb24iOiJ0b29sX2NhbGxzIiwibG9ncHJvYnMiOm51bGx9LCJ0eXBlIjoiQ2hhdEdlbmVyYXRpb24iLCJtZXNzYWdlIjp7ImxjIjoxLCJ0eXBlIjoiY29uc3RydWN0b3IiLCJpZCI6WyJsYW5nY2hhaW4iLCJzY2hlbWEiLCJtZXNzYWdlcyIsIkFJTWVzc2FnZSJdLCJrd2FyZ3MiOnsiY29udGVudCI6IkknbGwgaGVscCB5b3UgZGVjaWRlIGJldHdlZW4gQ2xpY2tIb3VzZSBhbmQgUG9zdGdyZXMgZm9yIHN0b3JpbmcgT3BlblRlbGVtZXRyeSBzcGFucyBhdCA1MGsgc3BhbnMvc2VjLiBMZXQgbWUgcmVzZWFyY2ggdGhpcyBzeXN0ZW1hdGljYWxseS4iLCJhZGRpdGlvbmFsX2t3YXJncyI6eyJyZWZ1c2FsIjpudWxsfSwicmVzcG9uc2VfbWV0YWRhdGEiOnsidG9rZW5fdXNhZ2UiOnsiY29tcGxldGlvbl90b2tlbnMiOjQ2NywicHJvbXB0X3Rva2VucyI6MzMzMiwidG90YWxfdG9rZW5zIjozNzk5LCJjb21wbGV0aW9uX3Rva2Vuc19kZXRhaWxzIjp7ImFjY2VwdGVkX3ByZWRpY3Rpb25fdG9rZW5zIjpudWxsLCJhdWRpb190b2tlbnMiOm51bGwsInJlYXNvbmluZ190b2tlbnMiOjAsInJlamVjdGVkX3ByZWRpY3Rpb25fdG9rZW5zIjpudWxsLCJ0ZXh0X3Rva2VucyI6NDY3fSwicHJvbXB0X3Rva2Vuc19kZXRhaWxzIjp7ImF1ZGlvX3Rva2VucyI6bnVsbCwiY2FjaGVfd3JpdGVfdG9rZW5zIjozMzI5LCJjYWNoZWRfdG9rZW5zIjowLCJpbWFnZV90b2tlbnMiOm51bGwsInRleHRfdG9rZW5zIjozLCJjYWNoZV9jcmVhdGlvbl90b2tlbnMiOjMzMjksImNhY2hlX2NyZWF0aW9uX3Rva2VuX2RldGFpbHMiOnsiZXBoZW1lcmFsXzVtX2lucHV0X3Rva2VucyI6MzMyOSwiZXBoZW1lcmFsXzFoX2lucHV0X3Rva2VucyI6MH19LCJjYWNoZV9jcmVhdGlvbl9pbnB1dF90b2tlbnMiOjMzMjksImNhY2hlX3JlYWRfaW5wdXRfdG9rZW5zIjowLCJpbmZlcmVuY2VfZ2VvIjoibm90X2F2YWlsYWJsZSIsInNlcnZpY2VfdGllciI6InN0YW5kYXJkIn0sIm1vZGVsX3Byb3ZpZGVyIjoib3BlbmFpIiwibW9kZWxfbmFtZSI6ImNsYXVkZS1zb25uZXQtNC01Iiwic3lzdGVtX2ZpbmdlcnByaW50IjpudWxsLCJpZCI6ImNoYXRjbXBsLTQwNzdiYjM2LTkzODAtNGEzYi05NDgxLTI0NTcwMGNlZjA5YSIsImZpbmlzaF9yZWFzb24iOiJ0b29sX2NhbGxzIiwibG9ncHJvYnMiOm51bGx9LCJ0eXBlIjoiYWkiLCJpZCI6ImxjX3J1bi0tMDFhMGYwOTktOGE0Ny03ZTQyLWE1ZjQtNWM0N2RlM2QxY2VjLTAiLCJ0b29sX2NhbGxzIjpbeyJuYW1lIjoid3JpdGVfZmlsZSIsImFyZ3MiOnsiZmlsZV9wYXRoIjoiL3RtcC9yZXNlYXJjaF90b2Rvcy5tZCIsImNvbnRlbnQiOiIjIFJlc2VhcmNoIFBsYW46IENsaWNrSG91c2UgdnMgUG9zdGdyZXMgZm9yIE9URUwgU3BhbnMgKDUway9zZWMpXG5cbiMjIFRhc2tzXG4tIFsgXSBSZXNlYXJjaCBDbGlja0hvdXNlIGFuZCBQb3N0Z3JlcyBjYXBhYmlsaXRpZXMgZm9yIGhpZ2gtdm9sdW1lIHRpbWUtc2VyaWVzIGRhdGFcbi4uLiJ9LCJpZCI6InRvb2x1XzAxNjFYaFlQM0I1Zmc0VTFwc1QzcGNpUiIsInR5cGUiOiJ0b29sX2NhbGwifSx7Im5hbWUiOiJ0YXNrIiwiYXJncyI6eyJzdWJhZ2VudF90eXBlIjoicmVzZWFyY2hlciIsImRlc2NyaXB0aW9uIjoiUmVzZWFyY2ggYW5kIGNvbXBhcmUgQ2xpY2tIb3VzZSB2cyBQb3N0Z3JlcyBmb3Igc3RvcmluZyBPcGVuVGVsZW1ldHJ5IChPVEVMKSBhZ2VudCBzcGFucyBhdCA1MCwwMDAgc3BhbnMgcGVyIHNlY29uZC5cblxuRm9jdXMgb246XG4xLiBXcml0ZSB0aHJvdWdocHV0IGNhcGFiaWxpdGllcy4uLiJ9LCJpZCI6InRvb2x1XzAxUEx5bzhUS0tUcFhSNGZwOTZEbjkzVyIsInR5cGUiOiJ0b29sX2NhbGwifV0sInVzYWdlX21ldGFkYXRhIjp7ImlucHV0X3Rva2VucyI6MzMzMiwib3V0cHV0X3Rva2VucyI6NDY3LCJ0b3RhbF90b2tlbnMiOjM3OTksImlucHV0X3Rva2VuX2RldGFpbHMiOnsiY2FjaGVfcmVhZCI6MCwiY2FjaGVfY3JlYXRpb24iOjMzMjl9LCJvdXRwdXRfdG9rZW5fZGV0YWlscyI6eyJyZWFzb25pbmciOjB9fSwiaW52YWxpZF90b29sX2NhbGxzIjpbXX19fV1dLCJsbG1fb3V0cHV0Ijp7InRva2VuX3VzYWdlIjp7ImNvbXBsZXRpb25fdG9rZW5zIjo0NjcsInByb21wdF90b2tlbnMiOjMzMzIsInRvdGFsX3Rva2VucyI6Mzc5OSwiY29tcGxldGlvbl90b2tlbnNfZGV0YWlscyI6eyJhY2NlcHRlZF9wcmVkaWN0aW9uX3Rva2VucyI6bnVsbCwiYXVkaW9fdG9rZW5zIjpudWxsLCJyZWFzb25pbmdfdG9rZW5zIjowLCJyZWplY3RlZF9wcmVkaWN0aW9uX3Rva2VucyI6bnVsbCwidGV4dF90b2tlbnMiOjQ2N30sInByb21wdF90b2tlbnNfZGV0YWlscyI6eyJhdWRpb190b2tlbnMiOm51bGwsImNhY2hlX3dyaXRlX3Rva2VucyI6MzMyOSwiY2FjaGVkX3Rva2VucyI6MCwiaW1hZ2VfdG9rZW5zIjpudWxsLCJ0ZXh0X3Rva2VucyI6MywiY2FjaGVfY3JlYXRpb25fdG9rZW5zIjozMzI5LCJjYWNoZV9jcmVhdGlvbl90b2tlbl9kZXRhaWxzIjp7ImVwaGVtZXJhbF81bV9pbnB1dF90b2tlbnMiOjMzMjksImVwaGVtZXJhbF8xaF9pbnB1dF90b2tlbnMiOjB9fSwiY2FjaGVfY3JlYXRpb25faW5wdXRfdG9rZW5zIjozMzI5LCJjYWNoZV9yZWFkX2lucHV0X3Rva2VucyI6MCwiaW5mZXJlbmNlX2dlbyI6Im5vdF9hdmFpbGFibGUiLCJzZXJ2aWNlX3RpZXIiOiJzdGFuZGFyZCJ9LCJtb2RlbF9wcm92aWRlciI6Im9wZW5haSIsIm1vZGVsX25hbWUiOiJjbGF1ZGUtc29ubmV0LTQtNSIsInN5c3RlbV9maW5nZXJwcmludCI6bnVsbCwiaWQiOiJjaGF0Y21wbC00MDc3YmIzNi05MzgwLTRhM2ItOTQ4MS0yNDU3MDBjZWYwOWEifSwicnVuIjpudWxsLCJ0eXBlIjoiTExNUmVzdWx0In0=" + } + } + ], + "status": { + "code": "STATUS_CODE_OK" + }, + "flags": 256 + }, + { + "traceId": "S61CuE6d47pG/IcBhfjwIw==", + "spanId": "zwThqgPzRPo=", + "parentSpanId": "g0UfMjWEf2w=", + "name": "FilesystemMiddleware.wrap_model_call", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1790742989379030016", + "endTimeUnixNano": "1790742998895730944", + "attributes": [ + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "chain" + } + }, + { + "key": "gen_ai.prompt", + "value": { + "bytesValue": "e30=" + } + }, + { + "key": "langsmith.span.kind", + "value": { + "stringValue": "chain" + } + }, + { + "key": "langsmith.trace.name", + "value": { + "stringValue": "FilesystemMiddleware.wrap_model_call" + } + }, + { + "key": "langsmith.trace.session_name", + "value": { + "stringValue": "default" + } + }, + { + "key": "gen_ai.system", + "value": { + "stringValue": "langchain" + } + }, + { + "key": "langsmith.metadata.ls_integration", + "value": { + "stringValue": "deepagents" + } + }, + { + "key": "langsmith.metadata.lc_agent_name", + "value": { + "stringValue": "deep_research_agent" + } + }, + { + "key": "langsmith.metadata.lc_versions", + "value": { + "stringValue": "{\"deepagents\":\"0.7.20\"}" + } + }, + { + "key": "langsmith.metadata.langgraph_step", + "value": { + "intValue": "2" + } + }, + { + "key": "langsmith.metadata.langgraph_node", + "value": { + "stringValue": "model" + } + }, + { + "key": "langsmith.metadata.langgraph_triggers", + "value": { + "stringValue": "[\"branch:to:model\"]" + } + }, + { + "key": "langsmith.metadata.langgraph_path", + "value": { + "stringValue": "[\"__pregel_pull\",\"model\"]" + } + }, + { + "key": "langsmith.metadata.langgraph_checkpoint_ns", + "value": { + "stringValue": "model:9abb6d12-32f9-4289-15b6-36ac41ba926c" + } + }, + { + "key": "langsmith.metadata.ls_method", + "value": { + "stringValue": "traceable" + } + }, + { + "key": "langsmith.metadata.LANGSMITH_TRACING", + "value": { + "stringValue": "true" + } + }, + { + "key": "langsmith.metadata.LANGSMITH_TRACING_MODE", + "value": { + "stringValue": "otel" + } + }, + { + "key": "gen_ai.completion", + "value": { + "bytesValue": "eyJvdXRwdXQiOnsicmVzdWx0IjpbeyJjb250ZW50IjoiSSdsbCBoZWxwIHlvdSBkZWNpZGUgYmV0d2VlbiBDbGlja0hvdXNlIGFuZCBQb3N0Z3JlcyBmb3Igc3RvcmluZyBPcGVuVGVsZW1ldHJ5IHNwYW5zIGF0IDUwayBzcGFucy9zZWMuIExldCBtZSByZXNlYXJjaCB0aGlzIHN5c3RlbWF0aWNhbGx5LiIsImFkZGl0aW9uYWxfa3dhcmdzIjp7InJlZnVzYWwiOm51bGx9LCJyZXNwb25zZV9tZXRhZGF0YSI6eyJ0b2tlbl91c2FnZSI6eyJjb21wbGV0aW9uX3Rva2VucyI6NDY3LCJwcm9tcHRfdG9rZW5zIjozMzMyLCJ0b3RhbF90b2tlbnMiOjM3OTksImNvbXBsZXRpb25fdG9rZW5zX2RldGFpbHMiOnsiYWNjZXB0ZWRfcHJlZGljdGlvbl90b2tlbnMiOm51bGwsImF1ZGlvX3Rva2VucyI6bnVsbCwicmVhc29uaW5nX3Rva2VucyI6MCwicmVqZWN0ZWRfcHJlZGljdGlvbl90b2tlbnMiOm51bGwsInRleHRfdG9rZW5zIjo0Njd9LCJwcm9tcHRfdG9rZW5zX2RldGFpbHMiOnsiYXVkaW9fdG9rZW5zIjpudWxsLCJjYWNoZV93cml0ZV90b2tlbnMiOjMzMjksImNhY2hlZF90b2tlbnMiOjAsImltYWdlX3Rva2VucyI6bnVsbCwidGV4dF90b2tlbnMiOjMsImNhY2hlX2NyZWF0aW9uX3Rva2VucyI6MzMyOSwiY2FjaGVfY3JlYXRpb25fdG9rZW5fZGV0YWlscyI6eyJlcGhlbWVyYWxfNW1faW5wdXRfdG9rZW5zIjozMzI5LCJlcGhlbWVyYWxfMWhfaW5wdXRfdG9rZW5zIjowfX0sImNhY2hlX2NyZWF0aW9uX2lucHV0X3Rva2VucyI6MzMyOSwiY2FjaGVfcmVhZF9pbnB1dF90b2tlbnMiOjAsImluZmVyZW5jZV9nZW8iOiJub3RfYXZhaWxhYmxlIiwic2VydmljZV90aWVyIjoic3RhbmRhcmQifSwibW9kZWxfcHJvdmlkZXIiOiJvcGVuYWkiLCJtb2RlbF9uYW1lIjoiY2xhdWRlLXNvbm5ldC00LTUiLCJzeXN0ZW1fZmluZ2VycHJpbnQiOm51bGwsImlkIjoiY2hhdGNtcGwtNDA3N2JiMzYtOTM4MC00YTNiLTk0ODEtMjQ1NzAwY2VmMDlhIiwiZmluaXNoX3JlYXNvbiI6InRvb2xfY2FsbHMiLCJsb2dwcm9icyI6bnVsbH0sInR5cGUiOiJhaSIsIm5hbWUiOiJkZWVwX3Jlc2VhcmNoX2FnZW50IiwiaWQiOiJsY19ydW4tLTAxYTBmMDk5LThhNDctN2U0Mi1hNWY0LTVjNDdkZTNkMWNlYy0wIiwidG9vbF9jYWxscyI6W3sibmFtZSI6IndyaXRlX2ZpbGUiLCJhcmdzIjp7ImZpbGVfcGF0aCI6Ii90bXAvcmVzZWFyY2hfdG9kb3MubWQiLCJjb250ZW50IjoiIyBSZXNlYXJjaCBQbGFuOiBDbGlja0hvdXNlIHZzIFBvc3RncmVzIGZvciBPVEVMIFNwYW5zICg1MGsvc2VjKVxuXG4jIyBUYXNrc1xuLSBbIF0gUmVzZWFyY2ggQ2xpY2tIb3VzZSBhbmQgUG9zdGdyZXMgY2FwYWJpbGl0aWVzIGZvciBoaWdoLXZvbHVtZSB0aW1lLXNlcmllcyBkYXRhXG4uLi4ifSwiaWQiOiJ0b29sdV8wMTYxWGhZUDNCNWZnNFUxcHNUM3BjaVIiLCJ0eXBlIjoidG9vbF9jYWxsIn0seyJuYW1lIjoidGFzayIsImFyZ3MiOnsic3ViYWdlbnRfdHlwZSI6InJlc2VhcmNoZXIiLCJkZXNjcmlwdGlvbiI6IlJlc2VhcmNoIGFuZCBjb21wYXJlIENsaWNrSG91c2UgdnMgUG9zdGdyZXMgZm9yIHN0b3JpbmcgT3BlblRlbGVtZXRyeSAoT1RFTCkgYWdlbnQgc3BhbnMgYXQgNTAsMDAwIHNwYW5zIHBlciBzZWNvbmQuXG5cbkZvY3VzIG9uOlxuMS4gV3JpdGUgdGhyb3VnaHB1dCBjYXBhYmlsaXRpZXMuLi4ifSwiaWQiOiJ0b29sdV8wMVBMeW84VEtLVHBYUjRmcDk2RG45M1ciLCJ0eXBlIjoidG9vbF9jYWxsIn1dLCJpbnZhbGlkX3Rvb2xfY2FsbHMiOltdLCJ1c2FnZV9tZXRhZGF0YSI6eyJpbnB1dF90b2tlbnMiOjMzMzIsIm91dHB1dF90b2tlbnMiOjQ2NywidG90YWxfdG9rZW5zIjozNzk5LCJpbnB1dF90b2tlbl9kZXRhaWxzIjp7ImNhY2hlX3JlYWQiOjAsImNhY2hlX2NyZWF0aW9uIjozMzI5fSwib3V0cHV0X3Rva2VuX2RldGFpbHMiOnsicmVhc29uaW5nIjowfX19XSwic3RydWN0dXJlZF9yZXNwb25zZSI6bnVsbH19" + } + } + ], + "status": { + "code": "STATUS_CODE_OK" + }, + "flags": 256 + }, + { + "traceId": "S61CuE6d47pG/IcBhfjwIw==", + "spanId": "svs6j1ovzgE=", + "parentSpanId": "Vt73x+GSQ0o=", + "name": "task", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1790742998900896000", + "endTimeUnixNano": "1790743034076956160", + "attributes": [ + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "execute_tool" + } + }, + { + "key": "langsmith.span.kind", + "value": { + "stringValue": "tool" + } + }, + { + "key": "langsmith.trace.name", + "value": { + "stringValue": "task" + } + }, + { + "key": "langsmith.trace.session_name", + "value": { + "stringValue": "default" + } + }, + { + "key": "gen_ai.system", + "value": { + "stringValue": "langchain" + } + }, + { + "key": "gen_ai.tool.name", + "value": { + "stringValue": "task" + } + }, + { + "key": "gen_ai.tool.call.id", + "value": { + "stringValue": "toolu_01PLyo8TKKTpXR4fp96Dn93W" + } + }, + { + "key": "langsmith.metadata.ls_integration", + "value": { + "stringValue": "deepagents" + } + }, + { + "key": "langsmith.metadata.lc_agent_name", + "value": { + "stringValue": "deep_research_agent" + } + }, + { + "key": "langsmith.metadata.lc_versions", + "value": { + "stringValue": "{\"deepagents\":\"0.7.20\"}" + } + }, + { + "key": "langsmith.metadata.langgraph_step", + "value": { + "intValue": "3" + } + }, + { + "key": "langsmith.metadata.langgraph_node", + "value": { + "stringValue": "tools" + } + }, + { + "key": "langsmith.metadata.langgraph_triggers", + "value": { + "stringValue": "[\"__pregel_push\"]" + } + }, + { + "key": "langsmith.metadata.langgraph_path", + "value": { + "stringValue": "[\"__pregel_push\",1,false]" + } + }, + { + "key": "langsmith.metadata.langgraph_checkpoint_ns", + "value": { + "stringValue": "tools:800ca7c3-441c-7ae3-0a5b-ea3fb69766fc" + } + }, + { + "key": "langsmith.metadata.checkpoint_ns", + "value": { + "stringValue": "tools:800ca7c3-441c-7ae3-0a5b-ea3fb69766fc" + } + }, + { + "key": "langsmith.metadata.ls_method", + "value": { + "stringValue": "traceable" + } + }, + { + "key": "langsmith.metadata.LANGSMITH_TRACING", + "value": { + "stringValue": "true" + } + }, + { + "key": "langsmith.metadata.LANGSMITH_TRACING_MODE", + "value": { + "stringValue": "otel" + } + }, + { + "key": "langsmith.span.tags", + "value": { + "stringValue": "seq:step:1" + } + }, + { + "key": "gen_ai.prompt", + "value": { + "bytesValue": "eyJzdWJhZ2VudF90eXBlIjoicmVzZWFyY2hlciIsImRlc2NyaXB0aW9uIjoiUmVzZWFyY2ggYW5kIGNvbXBhcmUgQ2xpY2tIb3VzZSB2cyBQb3N0Z3JlcyBmb3Igc3RvcmluZyBPcGVuVGVsZW1ldHJ5IChPVEVMKSBhZ2VudCBzcGFucyBhdCA1MCwwMDAgc3BhbnMgcGVyIHNlY29uZC5cblxuRm9jdXMgb246XG4xLiBXcml0ZSB0aHJvdWdocHV0IGNhcGFiaWxpdGllcy4uLiJ9" + } + }, + { + "key": "gen_ai.completion", + "value": { + "bytesValue": "eyJvdXRwdXQiOnsiZ3JhcGgiOm51bGwsInVwZGF0ZSI6eyJmaWxlcyI6e30sIm1lc3NhZ2VzIjpbeyJjb250ZW50IjoiQmFzZWQgb24gbXkgcmVzZWFyY2gsIGhlcmUncyBhIGNvbXByZWhlbnNpdmUgY29tcGFyaXNvbiBvZiAqKkNsaWNrSG91c2UgdnMgUG9zdGdyZXMqKiBmb3Igc3RvcmluZyBPcGVuVGVsZW1ldHJ5IHNwYW5zIGF0IDUwLDAwMCBzcGFucy9zZWNvbmQ6XG5cbiMjICoqMS4gV3JpdGUgVGhyLi4uIiwiYWRkaXRpb25hbF9rd2FyZ3MiOnt9LCJyZXNwb25zZV9tZXRhZGF0YSI6e30sInR5cGUiOiJ0b29sIiwidG9vbF9jYWxsX2lkIjoidG9vbHVfMDFQTHlvOFRLS1RwWFI0ZnA5NkRuOTNXIiwic3RhdHVzIjoic3VjY2VzcyJ9XX0sInJlc3VtZSI6bnVsbCwiZ290byI6W119fQ==" + } + } + ], + "status": { + "code": "STATUS_CODE_OK" + }, + "flags": 256 + }, + { + "traceId": "S61CuE6d47pG/IcBhfjwIw==", + "spanId": "gUmbSS/ZP4U=", + "parentSpanId": "svs6j1ovzgE=", + "name": "researcher", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1790742998901422080", + "endTimeUnixNano": "1790743034076699904", + "attributes": [ + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "chain" + } + }, + { + "key": "langsmith.span.kind", + "value": { + "stringValue": "chain" + } + }, + { + "key": "langsmith.trace.name", + "value": { + "stringValue": "researcher" + } + }, + { + "key": "langsmith.trace.session_name", + "value": { + "stringValue": "default" + } + }, + { + "key": "gen_ai.system", + "value": { + "stringValue": "langchain" + } + }, + { + "key": "langsmith.metadata.ls_integration", + "value": { + "stringValue": "langchain_create_agent" + } + }, + { + "key": "langsmith.metadata.lc_agent_name", + "value": { + "stringValue": "researcher" + } + }, + { + "key": "langsmith.metadata.lc_versions", + "value": { + "stringValue": "{\"deepagents\":\"0.7.20\"}" + } + }, + { + "key": "langsmith.metadata.langgraph_step", + "value": { + "intValue": "3" + } + }, + { + "key": "langsmith.metadata.langgraph_node", + "value": { + "stringValue": "tools" + } + }, + { + "key": "langsmith.metadata.langgraph_triggers", + "value": { + "stringValue": "[\"__pregel_push\"]" + } + }, + { + "key": "langsmith.metadata.langgraph_path", + "value": { + "stringValue": "[\"__pregel_push\",1,false]" + } + }, + { + "key": "langsmith.metadata.langgraph_checkpoint_ns", + "value": { + "stringValue": "tools:800ca7c3-441c-7ae3-0a5b-ea3fb69766fc" + } + }, + { + "key": "langsmith.metadata.checkpoint_ns", + "value": { + "stringValue": "tools:800ca7c3-441c-7ae3-0a5b-ea3fb69766fc" + } + }, + { + "key": "langsmith.metadata.ls_method", + "value": { + "stringValue": "traceable" + } + }, + { + "key": "langsmith.metadata.ls_agent_type", + "value": { + "stringValue": "subagent" + } + }, + { + "key": "langsmith.metadata.LANGSMITH_TRACING", + "value": { + "stringValue": "true" + } + }, + { + "key": "langsmith.metadata.LANGSMITH_TRACING_MODE", + "value": { + "stringValue": "otel" + } + }, + { + "key": "gen_ai.prompt", + "value": { + "bytesValue": "eyJmaWxlcyI6e30sIm1lc3NhZ2VzIjpbeyJjb250ZW50IjoiUmVzZWFyY2ggYW5kIGNvbXBhcmUgQ2xpY2tIb3VzZSB2cyBQb3N0Z3JlcyBmb3Igc3RvcmluZyBPcGVuVGVsZW1ldHJ5IChPVEVMKSBhZ2VudCBzcGFucyBhdCA1MCwwMDAgc3BhbnMgcGVyIHNlY29uZC5cblxuRm9jdXMgb246XG4xLiBXcml0ZSB0aHJvdWdocHV0IGNhcGFiaWxpdGllcy4uLiIsImFkZGl0aW9uYWxfa3dhcmdzIjp7fSwicmVzcG9uc2VfbWV0YWRhdGEiOnt9LCJ0eXBlIjoiaHVtYW4iLCJpZCI6ImFmOGRiNzQ5LTBiNTYtNGEzMi1hZGZlLTdmYzViOTRmZDAwMyJ9XX0=" + } + }, + { + "key": "gen_ai.completion", + "value": { + "bytesValue": "eyJtZXNzYWdlcyI6W3siY29udGVudCI6IlJlc2VhcmNoIGFuZCBjb21wYXJlIENsaWNrSG91c2UgdnMgUG9zdGdyZXMgZm9yIHN0b3JpbmcgT3BlblRlbGVtZXRyeSAoT1RFTCkgYWdlbnQgc3BhbnMgYXQgNTAsMDAwIHNwYW5zIHBlciBzZWNvbmQuXG5cbkZvY3VzIG9uOlxuMS4gV3JpdGUgdGhyb3VnaHB1dCBjYXBhYmlsaXRpZXMuLi4iLCJhZGRpdGlvbmFsX2t3YXJncyI6e30sInJlc3BvbnNlX21ldGFkYXRhIjp7fSwidHlwZSI6Imh1bWFuIiwiaWQiOiJhZjhkYjc0OS0wYjU2LTRhMzItYWRmZS03ZmM1Yjk0ZmQwMDMifSx7ImNvbnRlbnQiOiJJJ2xsIHJlc2VhcmNoIHRoZSBjb21wYXJpc29uIGJldHdlZW4gQ2xpY2tIb3VzZSBhbmQgUG9zdGdyZXMgZm9yIHN0b3JpbmcgT3BlblRlbGVtZXRyeSBzcGFucyBhdCBoaWdoIHZvbHVtZS4iLCJhZGRpdGlvbmFsX2t3YXJncyI6eyJyZWZ1c2FsIjpudWxsfSwicmVzcG9uc2VfbWV0YWRhdGEiOnsidG9rZW5fdXNhZ2UiOnsiY29tcGxldGlvbl90b2tlbnMiOjQyNywicHJvbXB0X3Rva2VucyI6Mjk4NiwidG90YWxfdG9rZW5zIjozNDEzLCJjb21wbGV0aW9uX3Rva2Vuc19kZXRhaWxzIjp7ImFjY2VwdGVkX3ByZWRpY3Rpb25fdG9rZW5zIjpudWxsLCJhdWRpb190b2tlbnMiOm51bGwsInJlYXNvbmluZ190b2tlbnMiOjAsInJlamVjdGVkX3ByZWRpY3Rpb25fdG9rZW5zIjpudWxsLCJ0ZXh0X3Rva2VucyI6NDI3fSwicHJvbXB0X3Rva2Vuc19kZXRhaWxzIjp7ImF1ZGlvX3Rva2VucyI6bnVsbCwiY2FjaGVfd3JpdGVfdG9rZW5zIjoyOTgzLCJjYWNoZWRfdG9rZW5zIjowLCJpbWFnZV90b2tlbnMiOm51bGwsInRleHRfdG9rZW5zIjozLCJjYWNoZV9jcmVhdGlvbl90b2tlbnMiOjI5ODMsImNhY2hlX2NyZWF0aW9uX3Rva2VuX2RldGFpbHMiOnsiZXBoZW1lcmFsXzVtX2lucHV0X3Rva2VucyI6Mjk4MywiZXBoZW1lcmFsXzFoX2lucHV0X3Rva2VucyI6MH19LCJjYWNoZV9jcmVhdGlvbl9pbnB1dF90b2tlbnMiOjI5ODMsImNhY2hlX3JlYWRfaW5wdXRfdG9rZW5zIjowLCJpbmZlcmVuY2VfZ2VvIjoibm90X2F2YWlsYWJsZSIsInNlcnZpY2VfdGllciI6InN0YW5kYXJkIn0sIm1vZGVsX3Byb3ZpZGVyIjoib3BlbmFpIiwibW9kZWxfbmFtZSI6ImNsYXVkZS1zb25uZXQtNC01Iiwic3lzdGVtX2ZpbmdlcnByaW50IjpudWxsLCJpZCI6ImNoYXRjbXBsLWFhYWE0Yjc4LTE3ZGMtNDM2NC04ZmE1LTJkODMzNjlmMWRiYyIsImZpbmlzaF9yZWFzb24iOiJ0b29sX2NhbGxzIiwibG9ncHJvYnMiOm51bGx9LCJ0eXBlIjoiYWkiLCJuYW1lIjoicmVzZWFyY2hlciIsImlkIjoibGNfcnVuLS0wMWEwZjA5OS1hZjdlLTc5ZTAtYTMzMy03MDdjMzQ5N2M3MzAtMCIsInRvb2xfY2FsbHMiOlt7Im5hbWUiOiJzZWFyY2hfZG9jcyIsImFyZ3MiOnsicXVlcnkiOiJDbGlja0hvdXNlIFBvc3RncmVzIE9wZW5UZWxlbWV0cnkgT1RFTCBzcGFucyBwZXJmb3JtYW5jZSBjb21wYXJpc29uIn0sImlkIjoidG9vbHVfMDFKc2pIRkZmcHN3NG9wbUs5VVppOFZOIiwidHlwZSI6InRvb2xfY2FsbCJ9LHsibmFtZSI6InNlYXJjaF9kb2NzIiwiYXJncyI6eyJxdWVyeSI6IkNsaWNrSG91c2Ugd3JpdGUgdGhyb3VnaHB1dCA1MDAwMCBzcGFucyBwZXIgc2Vjb25kIHRlbGVtZXRyeSJ9LCJpZCI6InRvb2x1XzAxS05ZcUhKS2kzcExlZU1RaEc1VDl1ZSIsInR5cGUiOiJ0b29sX2NhbGwifSx7Im5hbWUiOiJzZWFyY2hfZG9jcyIsImFyZ3MiOnsicXVlcnkiOiJQb3N0Z3JlcyB2cyBDbGlja0hvdXNlIG9ic2VydmFiaWxpdHkgbWV0cmljcyB0cmFjZXMifSwiaWQiOiJ0b29sdV8wMUpoRjh6NDQ0U1dVM0VXM2hQUUtkMlciLCJ0eXBlIjoidG9vbF9jYWxsIn0seyJuYW1lIjoic2VhcmNoX2RvY3MiLCJhcmdzIjp7InF1ZXJ5IjoiQ2xpY2tIb3VzZSBpbnNlcnQgcGVyZm9ybWFuY2UgYmF0Y2ggd3JpdGVzIHN1c3RhaW5lZCB0aHJvdWdocHV0In0sImlkIjoidG9vbHVfMDFXdXFyNTZKVHhDSllRUFMxWkZQbkg2IiwidHlwZSI6InRvb2xfY2FsbCJ9XSwiaW52YWxpZF90b29sX2NhbGxzIjpbXSwidXNhZ2VfbWV0YWRhdGEiOnsiaW5wdXRfdG9rZW5zIjoyOTg2LCJvdXRwdXRfdG9rZW5zIjo0MjcsInRvdGFsX3Rva2VucyI6MzQxMywiaW5wdXRfdG9rZW5fZGV0YWlscyI6eyJjYWNoZV9yZWFkIjowLCJjYWNoZV9jcmVhdGlvbiI6Mjk4M30sIm91dHB1dF90b2tlbl9kZXRhaWxzIjp7InJlYXNvbmluZyI6MH19fSx7ImNvbnRlbnQiOiJObyByZXN1bHRzLiIsImFkZGl0aW9uYWxfa3dhcmdzIjp7fSwicmVzcG9uc2VfbWV0YWRhdGEiOnt9LCJ0eXBlIjoidG9vbCIsIm5hbWUiOiJzZWFyY2hfZG9jcyIsImlkIjoiYTNhMDQxMmUtMWMxNS00ODk3LThjMmQtZGM0NmMwYmRlYzM2IiwidG9vbF9jYWxsX2lkIjoidG9vbHVfMDFVQmFYd0JQTmRxUkhHYmJhbmdLTFpVIiwic3RhdHVzIjoic3VjY2VzcyJ9LHsiY29udGVudCI6IkJhc2VkIG9uIG15IHJlc2VhcmNoLCBoZXJlJ3MgYSBjb21wcmVoZW5zaXZlIGNvbXBhcmlzb24gb2YgKipDbGlja0hvdXNlIHZzIFBvc3RncmVzKiogZm9yIHN0b3JpbmcgT3BlblRlbGVtZXRyeSBzcGFucyBhdCA1MCwwMDAgc3BhbnMvc2Vjb25kOlxuXG4jIyAqKjEuIFdyaXRlIFRoci4uLiIsImFkZGl0aW9uYWxfa3dhcmdzIjp7InJlZnVzYWwiOm51bGx9LCJyZXNwb25zZV9tZXRhZGF0YSI6eyJ0b2tlbl91c2FnZSI6eyJjb21wbGV0aW9uX3Rva2VucyI6NzAwLCJwcm9tcHRfdG9rZW5zIjo1NTM2LCJ0b3RhbF90b2tlbnMiOjYyMzYsImNvbXBsZXRpb25fdG9rZW5zX2RldGFpbHMiOnsiYWNjZXB0ZWRfcHJlZGljdGlvbl90b2tlbnMiOm51bGwsImF1ZGlvX3Rva2VucyI6bnVsbCwicmVhc29uaW5nX3Rva2VucyI6MCwicmVqZWN0ZWRfcHJlZGljdGlvbl90b2tlbnMiOm51bGwsInRleHRfdG9rZW5zIjo3MDB9LCJwcm9tcHRfdG9rZW5zX2RldGFpbHMiOnsiYXVkaW9fdG9rZW5zIjpudWxsLCJjYWNoZV93cml0ZV90b2tlbnMiOjQzMiwiY2FjaGVkX3Rva2VucyI6NTA5NywiaW1hZ2VfdG9rZW5zIjpudWxsLCJ0ZXh0X3Rva2VucyI6NywiY2FjaGVfY3JlYXRpb25fdG9rZW5zIjo0MzIsImNhY2hlX2NyZWF0aW9uX3Rva2VuX2RldGFpbHMiOnsiZXBoZW1lcmFsXzVtX2lucHV0X3Rva2VucyI6NDMyLCJlcGhlbWVyYWxfMWhfaW5wdXRfdG9rZW5zIjowfX0sImNhY2hlX2NyZWF0aW9uX2lucHV0X3Rva2VucyI6NDMyLCJjYWNoZV9yZWFkX2lucHV0X3Rva2VucyI6NTA5NywiaW5mZXJlbmNlX2dlbyI6Im5vdF9hdmFpbGFibGUiLCJzZXJ2aWNlX3RpZXIiOiJzdGFuZGFyZCJ9LCJtb2RlbF9wcm92aWRlciI6Im9wZW5haSIsIm1vZGVsX25hbWUiOiJjbGF1ZGUtc29ubmV0LTQtNSIsInN5c3RlbV9maW5nZXJwcmludCI6bnVsbCwiaWQiOiJjaGF0Y21wbC0zYzIwZTgwOC05YjE2LTQ0MjctOTk0Zi01Y2U3ZThiMWI5NGQiLCJmaW5pc2hfcmVhc29uIjoibGVuZ3RoIiwibG9ncHJvYnMiOm51bGx9LCJ0eXBlIjoiYWkiLCJuYW1lIjoicmVzZWFyY2hlciIsImlkIjoibGNfcnVuLS0wMWEwZjA5OS1mYmMyLTc5NjMtOWViYy1kYWUzNzZkYmJhMzktMCIsInRvb2xfY2FsbHMiOltdLCJpbnZhbGlkX3Rvb2xfY2FsbHMiOltdLCJ1c2FnZV9tZXRhZGF0YSI6eyJpbnB1dF90b2tlbnMiOjU1MzYsIm91dHB1dF90b2tlbnMiOjcwMCwidG90YWxfdG9rZW5zIjo2MjM2LCJpbnB1dF90b2tlbl9kZXRhaWxzIjp7ImNhY2hlX3JlYWQiOjUwOTcsImNhY2hlX2NyZWF0aW9uIjo0MzJ9LCJvdXRwdXRfdG9rZW5fZGV0YWlscyI6eyJyZWFzb25pbmciOjB9fX1dLCJmaWxlcyI6e319" + } + } + ], + "status": { + "code": "STATUS_CODE_OK" + }, + "flags": 256 + }, + { + "traceId": "S61CuE6d47pG/IcBhfjwIw==", + "spanId": "/mLyrQOgEWw=", + "parentSpanId": "SUm+6tN4+TU=", + "name": "search_docs", + "kind": "SPAN_KIND_INTERNAL", + "startTimeUnixNano": "1790743004976721920", + "endTimeUnixNano": "1790743004977214208", + "attributes": [ + { + "key": "langsmith.span.kind", + "value": { + "stringValue": "tool" + } + }, + { + "key": "langsmith.trace.name", + "value": { + "stringValue": "search_docs" + } + }, + { + "key": "langsmith.trace.session_name", + "value": { + "stringValue": "default" + } + }, + { + "key": "gen_ai.operation.name", + "value": { + "stringValue": "execute_tool" + } + }, + { + "key": "gen_ai.system", + "value": { + "stringValue": "langchain" + } + }, + { + "key": "gen_ai.tool.name", + "value": { + "stringValue": "search_docs" + } + }, + { + "key": "gen_ai.tool.call.id", + "value": { + "stringValue": "toolu_01JsjHFFfpsw4opmK9UZi8VN" + } + }, + { + "key": "langsmith.metadata.ls_integration", + "value": { + "stringValue": "langchain_create_agent" + } + }, + { + "key": "langsmith.metadata.lc_agent_name", + "value": { + "stringValue": "researcher" + } + }, + { + "key": "langsmith.metadata.lc_versions", + "value": { + "stringValue": "{\"deepagents\":\"0.7.20\"}" + } + }, + { + "key": "langsmith.metadata.langgraph_step", + "value": { + "intValue": "3" + } + }, + { + "key": "langsmith.metadata.langgraph_node", + "value": { + "stringValue": "tools" + } + }, + { + "key": "langsmith.metadata.langgraph_triggers", + "value": { + "stringValue": "[\"__pregel_push\"]" + } + }, + { + "key": "langsmith.metadata.langgraph_path", + "value": { + "stringValue": "[\"__pregel_push\",0,false]" + } + }, + { + "key": "langsmith.metadata.langgraph_checkpoint_ns", + "value": { + "stringValue": "tools:800ca7c3-441c-7ae3-0a5b-ea3fb69766fc|tools:49218779-253b-df87-734a-cfd23327bc5d" + } + }, + { + "key": "langsmith.metadata.checkpoint_ns", + "value": { + "stringValue": "tools:800ca7c3-441c-7ae3-0a5b-ea3fb69766fc" + } + }, + { + "key": "langsmith.metadata.ls_method", + "value": { + "stringValue": "traceable" + } + }, + { + "key": "langsmith.metadata.ls_agent_type", + "value": { + "stringValue": "subagent" + } + }, + { + "key": "langsmith.metadata.LANGSMITH_TRACING", + "value": { + "stringValue": "true" + } + }, + { + "key": "langsmith.metadata.LANGSMITH_TRACING_MODE", + "value": { + "stringValue": "otel" + } + }, + { + "key": "langsmith.span.tags", + "value": { + "stringValue": "seq:step:1" + } + }, + { + "key": "gen_ai.prompt", + "value": { + "bytesValue": "eyJxdWVyeSI6IkNsaWNrSG91c2UgUG9zdGdyZXMgT3BlblRlbGVtZXRyeSBPVEVMIHNwYW5zIHBlcmZvcm1hbmNlIGNvbXBhcmlzb24ifQ==" + } + }, + { + "key": "gen_ai.completion", + "value": { + "bytesValue": "eyJvdXRwdXQiOnsiY29udGVudCI6IkNsaWNrSG91c2UgaW5nZXN0cyAxTSsgcm93cy9zIHBlciBub2RlIHdpdGggYmF0Y2hlZCBpbnNlcnRzOyB1c2UgTWVyZ2VUcmVlIG9yZGVyZWQgYnkgKHRlbmFudCwgc2VydmljZSwgdGltZSkgYW5kIGEgYmxvb20gZmlsdGVyIGluZGV4IG9uIFRyYWNlSWQuXG5Qb3N0Z3JlcyBoYW5kLi4uIiwiYWRkaXRpb25hbF9rd2FyZ3MiOnt9LCJyZXNwb25zZV9tZXRhZGF0YSI6e30sInR5cGUiOiJ0b29sIiwibmFtZSI6InNlYXJjaF9kb2NzIiwidG9vbF9jYWxsX2lkIjoidG9vbHVfMDFKc2pIRkZmcHN3NG9wbUs5VVppOFZOIiwic3RhdHVzIjoic3VjY2VzcyJ9fQ==" + } + } + ], + "status": { + "code": "STATUS_CODE_OK" + }, + "flags": 256 + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/test_litellm/tracing/test_decode.py b/tests/test_litellm/tracing/test_decode.py new file mode 100644 index 000000000000..ea252f21e162 --- /dev/null +++ b/tests/test_litellm/tracing/test_decode.py @@ -0,0 +1,301 @@ +""" +Tests for OTLP decode + normalization (litellm/tracing/decode.py). + +The fixture is a trimmed real export from a Deep Agents run (LangSmith OTEL mode): +deep_research_agent -> task (tool) -> researcher (subagent) -> search_docs (tool). +""" + +import gzip +import json +import os +import sys +from pathlib import Path +from unittest.mock import patch + +sys.path.insert(0, os.path.abspath("../../..")) + +import pytest +from google.protobuf.json_format import Parse +from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest +from opentelemetry.proto.common.v1.common_pb2 import AnyValue, KeyValue +from opentelemetry.proto.trace.v1.trace_pb2 import ResourceSpans, ScopeSpans, Span, Status + +from litellm.tracing import decode +from litellm.tracing.decode import decode_otlp, encode_otlp_response + +pytestmark = pytest.mark.requires_rust_extension + +FIXTURE = Path(__file__).parent / "fixtures" / "langsmith_deep_agent_export.json" +TRACE_ID = "4bad42b84e9de3ba46fc870185f8f023" + + +def _fixture_json() -> bytes: + return FIXTURE.read_bytes() + + +def _fixture_protobuf() -> bytes: + request = ExportTraceServiceRequest() + Parse(_fixture_json().decode(), request) + return request.SerializeToString() + + +@pytest.fixture +def rows_by_name() -> dict: + rows = decode_otlp(_fixture_json(), "application/json") + return {r["SpanName"]: r for r in rows} + + +def _kv(key: str, value: str | int) -> KeyValue: + if isinstance(value, int): + return KeyValue(key=key, value=AnyValue(int_value=value)) + return KeyValue(key=key, value=AnyValue(string_value=value)) + + +def _export(*spans: Span, service: str = "svc", scope: str = "test") -> bytes: + resource_spans = ResourceSpans(scope_spans=[ScopeSpans(spans=list(spans))]) + resource_spans.resource.attributes.append(_kv("service.name", service)) + resource_spans.scope_spans[0].scope.name = scope + return ExportTraceServiceRequest(resource_spans=[resource_spans]).SerializeToString() + + +def _span(name: str, span_id: bytes, parent: bytes = b"", **attributes: str | int) -> Span: + return Span( + trace_id=bytes.fromhex(TRACE_ID), + span_id=span_id, + parent_span_id=parent, + name=name, + start_time_unix_nano=1_000, + end_time_unix_nano=5_000, + attributes=[_kv(k.replace("__", "."), v) for k, v in attributes.items()], + ) + + +# ---------------------------------------------------------------- LangSmith / Deep Agents fixture + + +def test_classifies_every_langsmith_span(rows_by_name): + assert {name: r["ObservationType"] for name, r in rows_by_name.items()} == { + "deep_research_agent": "agent", + "ChatOpenAI": "llm", + "FilesystemMiddleware.wrap_model_call": "framework", + "task": "tool", + "researcher": "agent", + "search_docs": "tool", + } + + +def test_agent_name_is_the_enclosing_agent(rows_by_name): + assert rows_by_name["task"]["AgentName"] == "deep_research_agent" + assert rows_by_name["ChatOpenAI"]["AgentName"] == "deep_research_agent" + assert rows_by_name["researcher"]["AgentName"] == "researcher" + assert rows_by_name["search_docs"]["AgentName"] == "researcher" + + +def test_subagent_is_nested_under_task_tool(rows_by_name): + assert rows_by_name["researcher"]["ParentSpanId"] == rows_by_name["task"]["SpanId"] + assert rows_by_name["deep_research_agent"]["ParentSpanId"] == "" + + +def test_llm_span_carries_litellm_request_id_model_and_tokens(rows_by_name): + llm = rows_by_name["ChatOpenAI"] + assert llm["LiteLLMRequestId"] == "chatcmpl-4077bb36-9380-4a3b-9481-245700cef09a" + assert llm["Model"] == "claude-sonnet-4-5" + assert (llm["InputTokens"], llm["OutputTokens"]) == (3332, 467) + + +def test_llm_input_output_are_normalized_messages(rows_by_name): + llm = rows_by_name["ChatOpenAI"] + messages = json.loads(llm["Input"]) + assert [m["role"] for m in messages][:2] == ["system", "user"] + assert "research lead" in messages[0]["content"] + output = json.loads(llm["Output"]) + assert output["role"] == "assistant" + assert output["tool_calls"][0]["name"] + + +def test_task_tool_output_is_subagent_final_message_text(rows_by_name): + task = rows_by_name["task"] + assert json.loads(task["Input"])["subagent_type"] == "researcher" + assert task["Output"].startswith("Based on my research") + assert not task["Output"].startswith("{") + + +def test_agent_input_output(rows_by_name): + root = rows_by_name["deep_research_agent"] + assert json.loads(root["Input"]) == [ + {"role": "user", "content": "Should we store OTEL agent spans in ClickHouse or Postgres at 50k spans/sec?"} + ] + assert json.loads(root["Output"])["role"] == "assistant" + + +def test_plain_tool_input_output(rows_by_name): + tool = rows_by_name["search_docs"] + assert json.loads(tool["Input"]) == {"query": "ClickHouse Postgres OpenTelemetry OTEL spans performance comparison"} + assert tool["Output"].startswith("ClickHouse ingests") + + +def test_heavy_attributes_are_lifted_out_of_span_attributes(rows_by_name): + for row in rows_by_name.values(): + assert not set(row["SpanAttributes"]) & decode._HEAVY_ATTRIBUTES + assert rows_by_name["ChatOpenAI"]["SpanAttributes"]["langsmith.span.kind"] == "llm" + + +def test_ids_are_hex_and_resource_is_kept(rows_by_name): + root = rows_by_name["deep_research_agent"] + assert root["TraceId"] == TRACE_ID + assert root["SpanId"] == "5e79f3b5b504985e" + assert root["ServiceName"] == "agent-demo" + assert root["ScopeName"] == "langsmith" + assert root["SpanKind"] == "SPAN_KIND_INTERNAL" + assert root["StatusCode"] == "STATUS_CODE_OK" + assert root["Duration"] > 0 + + +def test_protobuf_and_json_decode_identically(): + from_json = decode_otlp(_fixture_json(), "application/json") + from_protobuf = decode_otlp(_fixture_protobuf(), "application/x-protobuf") + assert from_json == from_protobuf + assert len(from_json) == 6 + + +def test_content_type_defaults_to_protobuf(): + assert len(decode_otlp(_fixture_protobuf(), None)) == 6 + + +@pytest.mark.parametrize("content_encoding", ["gzip", None]) +def test_gzip_body_by_header_or_magic_bytes(content_encoding): + rows = decode_otlp(gzip.compress(_fixture_protobuf()), "application/x-protobuf", content_encoding) + assert len(rows) == 6 + + +def test_long_values_are_truncated_with_marker(): + with patch.object(decode, "OTLP_MAX_ATTRIBUTE_VALUE_BYTES", 100): + rows = {r["SpanName"]: r for r in decode_otlp(_fixture_json(), "application/json")} + task = rows["task"] + assert "…[truncated " in task["Input"] + assert task["Input"].encode().startswith(task["Input"].split("…")[0].encode()) + assert len(task["Input"].split("…")[0].encode()) <= 100 + + +# ---------------------------------------------------------------- status / exceptions + + +def test_exception_event_fills_status_message(): + span = _span("get_customer_plan", b"\x01" * 8, b"\x02" * 8) + span.status.CopyFrom(Status(code=Status.STATUS_CODE_ERROR)) + event = span.events.add() + event.name = "exception" + event.attributes.extend( + [_kv("exception.type", "KeyError"), _kv("exception.message", "customer acme-404 not found")] + ) + (row,) = decode_otlp(_export(span)) + assert row["StatusCode"] == "STATUS_CODE_ERROR" + assert row["StatusMessage"] == "customer acme-404 not found" + + +def test_status_message_wins_over_exception_event(): + span = _span("tool", b"\x01" * 8, b"\x02" * 8) + span.status.CopyFrom(Status(code=Status.STATUS_CODE_ERROR, message="boom")) + event = span.events.add() + event.name = "exception" + event.attributes.append(_kv("exception.message", "other")) + (row,) = decode_otlp(_export(span)) + assert row["StatusMessage"] == "boom" + + +# ---------------------------------------------------------------- GenAI semconv / OpenInference + + +def test_genai_semconv_spans(): + root = _span( + "invoke_agent planner", b"\x01" * 8, gen_ai__operation__name="invoke_agent", gen_ai__agent__name="planner" + ) + chat = _span( + "chat gpt-4o", + b"\x02" * 8, + b"\x01" * 8, + gen_ai__operation__name="chat", + gen_ai__agent__name="planner", + gen_ai__request__model="gpt-4o", + gen_ai__response__id="chatcmpl-abc", + gen_ai__usage__input_tokens=12, + gen_ai__usage__output_tokens=3, + gen_ai__input__messages='[{"role":"user","content":"hi"}]', + gen_ai__output__messages='[{"role":"assistant","content":"hello"}]', + ) + tool = _span( + "execute_tool search", + b"\x03" * 8, + b"\x01" * 8, + gen_ai__operation__name="execute_tool", + gen_ai__tool__call__arguments='{"q":"x"}', + gen_ai__tool__call__result="found", + ) + rows = {r["SpanName"]: r for r in decode_otlp(_export(root, chat, tool))} + assert rows["invoke_agent planner"]["ObservationType"] == "agent" + assert rows["invoke_agent planner"]["AgentName"] == "planner" + llm = rows["chat gpt-4o"] + assert (llm["ObservationType"], llm["Model"], llm["LiteLLMRequestId"]) == ("llm", "gpt-4o", "chatcmpl-abc") + assert (llm["InputTokens"], llm["OutputTokens"]) == (12, 3) + assert json.loads(llm["Input"])[0]["content"] == "hi" + assert "gen_ai.input.messages" not in llm["SpanAttributes"] + assert (rows["execute_tool search"]["ObservationType"], rows["execute_tool search"]["Output"]) == ("tool", "found") + + +def test_openinference_spans(): + root = _span("agent", b"\x01" * 8, openinference__span__kind="AGENT", agent__name="writer", input__value="task") + llm = _span( + "llm", + b"\x02" * 8, + b"\x01" * 8, + openinference__span__kind="LLM", + llm__model_name="claude-sonnet-4-5", + llm__token_count__prompt=40, + llm__token_count__completion=8, + input__value="prompt", + output__value="answer", + ) + chain = _span("retriever", b"\x03" * 8, b"\x01" * 8, openinference__span__kind="RETRIEVER") + rows = {r["SpanName"]: r for r in decode_otlp(_export(root, llm, chain))} + assert (rows["agent"]["ObservationType"], rows["agent"]["AgentName"], rows["agent"]["Input"]) == ( + "agent", + "writer", + "task", + ) + assert rows["llm"]["ObservationType"] == "llm" + assert (rows["llm"]["Model"], rows["llm"]["InputTokens"], rows["llm"]["OutputTokens"]) == ( + "claude-sonnet-4-5", + 40, + 8, + ) + assert (rows["llm"]["Input"], rows["llm"]["Output"]) == ("prompt", "answer") + assert "input.value" not in rows["llm"]["SpanAttributes"] + assert rows["retriever"]["ObservationType"] == "chain" + + +def test_non_string_attribute_values_are_stringified(): + span = _span("root", b"\x01" * 8) + span.attributes.extend( + [ + KeyValue(key="flag", value=AnyValue(bool_value=True)), + KeyValue(key="ratio", value=AnyValue(double_value=0.5)), + KeyValue(key="raw", value=AnyValue(bytes_value=b"abc")), + ] + ) + array = KeyValue(key="list") + array.value.array_value.values.extend([AnyValue(string_value="a"), AnyValue(int_value=1)]) + span.attributes.append(array) + (row,) = decode_otlp(_export(span)) + assert row["SpanAttributes"]["flag"] == "true" + assert row["SpanAttributes"]["ratio"] == "0.5" + assert row["SpanAttributes"]["raw"] == "abc" + assert json.loads(row["SpanAttributes"]["list"]) == ["a", "1"] + + +# ---------------------------------------------------------------- helpers + + +def test_encode_otlp_response_matches_request_encoding(): + assert encode_otlp_response("application/json") == (b"{}", "application/json") + assert encode_otlp_response("application/x-protobuf") == (b"", "application/x-protobuf") + assert encode_otlp_response(None) == (b"", "application/x-protobuf") diff --git a/tests/test_litellm/tracing/test_receiver.py b/tests/test_litellm/tracing/test_receiver.py new file mode 100644 index 000000000000..53e319fc2ba7 --- /dev/null +++ b/tests/test_litellm/tracing/test_receiver.py @@ -0,0 +1,120 @@ +""" +Tests for TraceReceiver.ingest (litellm/tracing/receiver.py) with a fake store. +""" + +import os +import sys +from pathlib import Path +from unittest.mock import AsyncMock, MagicMock, patch + +sys.path.insert(0, os.path.abspath("../../..")) + +import pytest +from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest +from opentelemetry.proto.common.v1.common_pb2 import AnyValue, KeyValue +from opentelemetry.proto.trace.v1.trace_pb2 import ResourceSpans, ScopeSpans, Span + +from litellm.tracing import Tenant, TraceReceiver, TracingPayloadTooLargeError +from litellm.tracing import receiver as receiver_module + +pytestmark = pytest.mark.requires_rust_extension + +FIXTURE = Path(__file__).parent / "fixtures" / "langsmith_deep_agent_export.json" +TENANT = Tenant(team_id="team-research", api_key_hash="hashed-key", org_id="org-1") + + +def _fake_store() -> MagicMock: + store = MagicMock() + store.insert_spans = AsyncMock() + store.get_trace = AsyncMock(return_value=None) + return store + + +def _spoofed_export() -> bytes: + """A client that tries to claim another team via resource attributes.""" + resource_spans = ResourceSpans(scope_spans=[ScopeSpans(spans=[Span(trace_id=b"\x01" * 16, span_id=b"\x02" * 8)])]) + resource_spans.resource.attributes.extend( + [ + KeyValue(key="service.name", value=AnyValue(string_value="svc")), + KeyValue(key="litellm.team_id", value=AnyValue(string_value="someone-elses-team")), + KeyValue(key="litellm.api_key_hash", value=AnyValue(string_value="someone-elses-key")), + ] + ) + return ExportTraceServiceRequest(resource_spans=[resource_spans]).SerializeToString() + + +@pytest.mark.asyncio +async def test_ingest_returns_span_count_and_writes_stamped_rows(): + store = _fake_store() + count = await TraceReceiver(store).ingest(FIXTURE.read_bytes(), "application/json", None, TENANT) + assert count == 6 + (rows,) = store.insert_spans.await_args.args + assert len(rows) == 6 + for row in rows: + assert (row["TeamId"], row["ApiKeyHash"]) == ("team-research", "hashed-key") + assert row["ResourceAttributes"]["litellm.org_id"] == "org-1" + assert row["ResourceAttributes"]["service.name"] == "agent-demo" + + +@pytest.mark.asyncio +async def test_ingest_overwrites_client_supplied_tenant_attributes(): + store = _fake_store() + await TraceReceiver(store).ingest(_spoofed_export(), "application/x-protobuf", None, TENANT) + ((row,),) = store.insert_spans.await_args.args + assert row["TeamId"] == "team-research" + assert row["ResourceAttributes"]["litellm.team_id"] == "team-research" + assert row["ResourceAttributes"]["litellm.api_key_hash"] == "hashed-key" + + +@pytest.mark.asyncio +async def test_ingest_does_not_acknowledge_failed_clickhouse_write(): + store = _fake_store() + store.insert_spans.side_effect = RuntimeError("ClickHouse unavailable") + with pytest.raises(RuntimeError, match="ClickHouse unavailable"): + await TraceReceiver(store).ingest(FIXTURE.read_bytes(), "application/json", None, TENANT) + store.insert_spans.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_ingest_rejects_oversized_encoded_batch(): + store = _fake_store() + store.insert_spans.side_effect = OverflowError("ClickHouse insert exceeds the encoded size limit") + with pytest.raises(TracingPayloadTooLargeError, match="encoded size limit"): + await TraceReceiver(store).ingest(FIXTURE.read_bytes(), "application/json", None, TENANT) + + +@pytest.mark.asyncio +async def test_ingest_rejects_oversized_body(): + store = _fake_store() + with patch.object(receiver_module, "OTLP_MAX_BODY_BYTES", 10): + with pytest.raises(TracingPayloadTooLargeError): + await TraceReceiver(store).ingest(FIXTURE.read_bytes(), "application/json", None, TENANT) + store.insert_spans.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_large_body_is_decoded_off_the_event_loop(): + store = _fake_store() + with ( + patch.object(receiver_module, "OTLP_OFFLOAD_DECODE_BYTES", 0), + patch.object(receiver_module.asyncio, "to_thread", wraps=receiver_module.asyncio.to_thread) as to_thread, + ): + count = await TraceReceiver(store).ingest(FIXTURE.read_bytes(), "application/json", None, TENANT) + assert count == 6 + to_thread.assert_called_once() + + +@pytest.mark.asyncio +async def test_empty_export_writes_nothing(): + store = _fake_store() + assert await TraceReceiver(store).ingest(b"", "application/x-protobuf", None, TENANT) == 0 + store.insert_spans.assert_awaited_once_with([]) + + +@pytest.mark.asyncio +async def test_reads_delegate_to_store(): + store = _fake_store() + tracing = TraceReceiver(store) + scope = {"team_ids": ["team-research"], "api_key_hash": ""} + assert await tracing.get_trace("t1", scope) is None # type: ignore[arg-type] + store.get_trace.assert_awaited_once_with("t1", scope) diff --git a/tests/test_litellm/tracing/test_store.py b/tests/test_litellm/tracing/test_store.py new file mode 100644 index 000000000000..e7235c3a3269 --- /dev/null +++ b/tests/test_litellm/tracing/test_store.py @@ -0,0 +1,297 @@ +""" +Tests for the pure read-side helpers in litellm/tracing/store.py (no ClickHouse needed). +""" + +import os +import sys +from typing import Any +from unittest.mock import AsyncMock, MagicMock + +sys.path.insert(0, os.path.abspath("../../..")) + +import pytest + +from litellm.tracing.store import ( + ClickHouseTraceStore, + agent_nodes, + decode_cursor, + encode_cursor, + span_from_row, + trace_from_rows, + trace_summary_from_row, +) + +T0 = 1_790_742_989_000_000_000 # ns +MS = 1_000_000 + + +def _row( + span_id: str, + parent: str, + name: str, + type_: str, + agent: str, + start_ms: float = 0, + duration_ms: float = 10, + status: str = "STATUS_CODE_OK", + **extra: Any, +) -> dict[str, Any]: + return { + "span_id": span_id, + "parent_span_id": parent, + "name": name, + "type": type_, + "agent": agent, + "status": status, + "start_ns": T0 + int(start_ms * MS), + "duration_ns": int(duration_ms * MS), + "service": "agent-demo", + "input_preview": f"input of {name}", + "model": "", + "input_tokens": 0, + "output_tokens": 0, + "litellm_request_id": "", + **extra, + } + + +def _llm_row(span_id: str, parent: str, agent: str, request_id: str, start_ms: float = 1) -> dict: + return _row( + span_id, + parent, + "ChatOpenAI", + "llm", + agent, + start_ms=start_ms, + duration_ms=100, + model="claude-sonnet-4-5", + input_tokens=100, + output_tokens=20, + litellm_request_id=request_id, + ) + + +def _deep_agent_rows(researcher_invocations: int = 1) -> list[dict[str, Any]]: + """root agent -> llm, task tool -> researcher subagent (N times) -> llm + search_docs tool.""" + rows = [ + _row("root", "", "deep_research_agent", "agent", "deep_research_agent", duration_ms=1000), + _llm_row("llm-root", "root", "deep_research_agent", "chatcmpl-root"), + _row("task", "root", "task", "tool", "deep_research_agent", start_ms=200, duration_ms=700), + ] + for i in range(researcher_invocations): + rows += [ + _row(f"res-{i}", "task", "researcher", "agent", "researcher", start_ms=201, duration_ms=5), + _llm_row(f"res-llm-{i}", f"res-{i}", "researcher", f"chatcmpl-res-{i}", start_ms=202), + _row(f"res-tool-{i}", f"res-{i}", "search_docs", "tool", "researcher", start_ms=203, duration_ms=1), + _row(f"res-mw-{i}", f"res-{i}", "FilesystemMiddleware.wrap_model_call", "framework", "researcher"), + ] + return rows + + +# ---------------------------------------------------------------- trace_from_rows + + +def test_empty_rows_is_none(): + assert trace_from_rows("abc", []) is None + + +def test_llm_response_id_is_preserved_without_spend_enrichment(): + trace = trace_from_rows("t1", _deep_agent_rows()) + assert trace is not None + spans = {span["span_id"]: span for span in trace["spans"]} + assert spans["llm-root"]["litellm_request_id"] == "chatcmpl-root" + assert spans["task"]["litellm_request_id"] is None + assert "spend" not in trace["summary"] + + +def test_summary_totals(): + trace = trace_from_rows("t1", _deep_agent_rows()) + assert trace is not None + summary = trace["summary"] + assert summary["trace_id"] == "t1" + assert summary["name"] == "deep_research_agent" + assert summary["service"] == "agent-demo" + assert summary["input_preview"] == "input of deep_research_agent" + assert summary["status"] == "ok" + assert summary["span_count"] == 7 + assert summary["agent_count"] == 2 + assert summary["llm_calls"] == 2 + assert summary["tool_calls"] == 2 + assert summary["error_count"] == 0 + assert (summary["input_tokens"], summary["output_tokens"]) == (200, 40) + assert summary["models"] == ["claude-sonnet-4-5"] + assert summary["duration_ms"] == 1000 + assert summary["start_time"].startswith("2026-09-30T") + + +def test_error_count_counts_error_spans(): + rows = _deep_agent_rows() + rows[2]["status"] = "STATUS_CODE_ERROR" + trace = trace_from_rows("t1", rows) + assert trace is not None + assert trace["summary"]["error_count"] == 1 + assert trace["summary"]["status"] == "ok" # root span status; the UI uses error_count for "failed" + assert trace["spans"][2]["status"] == "error" + + +def test_offsets_are_relative_to_trace_start_in_ms(): + trace = trace_from_rows("t1", _deep_agent_rows()) + assert trace is not None + spans = {s["span_id"]: s for s in trace["spans"]} + assert spans["root"]["start_offset_ms"] == 0 + assert spans["task"]["start_offset_ms"] == 200 + assert spans["task"]["duration_ms"] == 700 + assert spans["root"]["parent_span_id"] is None + assert spans["task"]["parent_span_id"] == "root" + + +def test_span_from_row_optional_fields(): + span = span_from_row(_row("s", "", "x", "chain", "a", status="STATUS_CODE_UNSET"), T0) + assert (span["model"], span["parent_span_id"], span["status"], span["litellm_request_id"]) == ( + None, + None, + "unset", + None, + ) + + +def test_agent_nodes_parent_and_per_agent_counts(): + trace = trace_from_rows("t1", _deep_agent_rows()) + assert trace is not None + assert trace["agents"] == [ + { + "name": "deep_research_agent", + "parent_agent": None, + "invocations": 1, + "llm_calls": 1, + "tool_calls": 1, + "duration_ms": 1000, + }, + { + "name": "researcher", + "parent_agent": "deep_research_agent", + "invocations": 1, + "llm_calls": 1, + "tool_calls": 1, + "duration_ms": 5, + }, + ] + + +def test_200_subagent_invocations_aggregate_into_one_node(): + trace = trace_from_rows("t1", _deep_agent_rows(researcher_invocations=200)) + assert trace is not None + assert [a["name"] for a in trace["agents"]] == ["deep_research_agent", "researcher"] + researcher = trace["agents"][1] + assert researcher["parent_agent"] == "deep_research_agent" + assert researcher["invocations"] == 200 + assert researcher["llm_calls"] == 200 + assert researcher["tool_calls"] == 200 + assert researcher["duration_ms"] == pytest.approx(1000) + assert trace["summary"]["agent_count"] == 2 + assert trace["summary"]["span_count"] == 3 + 4 * 200 + + +def test_parent_agent_skips_same_name_ancestors(): + """A recursive agent (researcher -> researcher) still reports the nearest *different* agent.""" + rows = [ + _row("root", "", "lead", "agent", "lead"), + _row("r1", "root", "researcher", "agent", "researcher"), + _row("r2", "r1", "researcher", "agent", "researcher"), + ] + spans = [span_from_row(r, T0) for r in rows] + nodes = {n["name"]: n for n in agent_nodes(spans)} + assert nodes["researcher"]["parent_agent"] == "lead" + assert nodes["researcher"]["invocations"] == 2 + + +def test_agent_nodes_ignores_spans_of_unknown_agents(): + spans = [span_from_row(_row("t", "", "tool", "tool", "ghost"), T0)] + assert agent_nodes(spans) == [] + + +# ---------------------------------------------------------------- list helpers + + +def test_cursor_round_trip(): + cursor = encode_cursor(1790742989377, "4bad42b84e9de3ba46fc870185f8f023") + assert decode_cursor(cursor) == (1790742989377, "4bad42b84e9de3ba46fc870185f8f023") + assert decode_cursor(None) == (0, "") + assert decode_cursor("") == (0, "") + + +def test_trace_summary_from_row(): + summary = trace_summary_from_row( + { + "trace_id": "t1", + "name": "deep_research_agent", + "service": "agent-demo", + "input_preview": "hi", + "start_ms": 1790742989377, + "duration_ms": 51385, + "status": "STATUS_CODE_OK", + "span_count": "126", + "agent_count": "2", + "llm_calls": "7", + "tool_calls": "26", + "error_count": "1", + "input_tokens": "30175", + "output_tokens": "2620", + "models": ["claude-sonnet-4-5"], + } + ) + assert summary["status"] == "ok" + assert (summary["span_count"], summary["error_count"]) == (126, 1) + assert summary["start_time"] == "2026-09-30T04:36:29.377000+00:00" + + +@pytest.mark.asyncio +async def test_list_traces_sets_next_cursor_on_full_page(): + client = MagicMock() + row = { + "trace_id": "t2", + "name": "a", + "service": "s", + "input_preview": "", + "start_ms": 1000, + "duration_ms": 1, + "status": "STATUS_CODE_OK", + "span_count": 1, + "agent_count": 1, + "llm_calls": 0, + "tool_calls": 0, + "error_count": 0, + "input_tokens": 0, + "output_tokens": 0, + "models": [], + } + client.query = AsyncMock(return_value=[row, {**row, "trace_id": "t1", "start_ms": 900}]) + store = ClickHouseTraceStore(client) + scope = {"team_ids": ["team-a"], "api_key_hash": ""} + + page = await store.list_traces(scope, 0, 2000, limit=2) # type: ignore[arg-type] + assert [t["trace_id"] for t in page["data"]] == ["t2", "t1"] + assert page["next_cursor"] is not None + assert decode_cursor(page["next_cursor"]) == (900, "t1") + params = client.query.call_args.args[1] + assert params["team_ids"] == ["team-a"] and params["limit"] == 2 and params["cursor_ms"] == 0 + + page = await store.list_traces(scope, 0, 2000, cursor=page["next_cursor"], limit=3) # type: ignore[arg-type] + assert page["next_cursor"] is None + assert client.query.call_args.args[1]["cursor_trace_id"] == "t1" + + +@pytest.mark.asyncio +async def test_get_span_not_found_and_found(): + client = MagicMock() + client.query = AsyncMock(return_value=[]) + store = ClickHouseTraceStore(client) + scope = {"team_ids": [], "api_key_hash": ""} + assert await store.get_span("t", "s", scope) is None # type: ignore[arg-type] + client.query = AsyncMock(return_value=[{"span_id": "s", "input": "i", "output": "o", "attributes": {"k": "v"}}]) + assert await store.get_span("t", "s", scope) == { # type: ignore[arg-type] + "span_id": "s", + "input": "i", + "output": "o", + "attributes": {"k": "v"}, + } diff --git a/tests/test_litellm_rust/test_traces.py b/tests/test_litellm_rust/test_traces.py index 21f61ad24bce..78071bccec8b 100644 --- a/tests/test_litellm_rust/test_traces.py +++ b/tests/test_litellm_rust/test_traces.py @@ -1,11 +1,12 @@ import base64 +import gzip import json from typing import Final from urllib.parse import parse_qs, urlsplit import pytest -from litellm.rust_bridge.traces import encode_rows, ensure_schema, query +from litellm.rust_bridge._native import NativeTraceStorage from tests.test_litellm_rust.support.recording_server import RecordingServer, ResponseSpec pytestmark = pytest.mark.requires_rust_extension @@ -14,14 +15,9 @@ @pytest.mark.asyncio async def test_trace_reader_projects_connection_and_parameters(recording_server: RecordingServer) -> None: recording_server.enqueue(ResponseSpec(body={"data": [{"trace_id": "trace-1"}]})) - rows: Final = await query( - recording_server.base_url + "?database=wrong&user=wrong&password=wrong", - "trace_test", - "reader", - "p@ss/word%", - "SELECT {trace_id:String} AS trace_id", - {"trace_id": "trace-1"}, - ) + reader_url: Final = recording_server.base_url.replace("http://", "http://reader:p%40ss%2Fword%25@") + storage: Final = NativeTraceStorage("trace_test", recording_server.base_url, reader_url + "?database=wrong") + rows: Final = json.loads(await storage.query("SELECT {trace_id:String} AS trace_id", {"trace_id": "trace-1"})) request: Final = recording_server.requests[0] parameters: Final = parse_qs(urlsplit(request.path).query) assert rows == [{"trace_id": "trace-1"}] @@ -37,15 +33,17 @@ async def test_trace_reader_projects_connection_and_parameters(recording_server: @pytest.mark.asyncio async def test_trace_reader_rejects_success_status_with_embedded_error(recording_server: RecordingServer) -> None: recording_server.enqueue(ResponseSpec(body={"data": [], "exception": "query failed"})) + storage: Final = NativeTraceStorage("trace_test", recording_server.base_url, recording_server.base_url) with pytest.raises(RuntimeError, match="invalid or failed JSON"): - await query(recording_server.base_url, "trace_test", "reader", "password", "SELECT 1", {}) + await storage.query("SELECT 1", {}) @pytest.mark.asyncio @pytest.mark.parametrize("database,retention", [("db; DROP DATABASE default", 7), ("traces", 0)]) async def test_schema_binding_preserves_configuration_validation(database: str, retention: int) -> None: with pytest.raises(ValueError, match=r"database.*retention"): - await ensure_schema("http://localhost:8123", database, "writer", "password", retention, 14) + storage: Final = NativeTraceStorage(database, "http://localhost:8123") + await storage.ensure_schema(retention, 14) @pytest.mark.asyncio @@ -53,15 +51,10 @@ async def test_schema_setup_uses_writer_credentials_and_rejects_failed_statement recording_server.expected_requests = 2 recording_server.enqueue(ResponseSpec(body="")) recording_server.enqueue(ResponseSpec(status=403, body="denied")) + writer_url: Final = recording_server.base_url.replace("http://", "http://writer:p%40ss%2Fword%25@") + storage: Final = NativeTraceStorage("trace_test", writer_url + "?database=wrong&readonly=1") with pytest.raises(RuntimeError, match="schema setup failed with HTTP status 403"): - await ensure_schema( - recording_server.base_url + "?database=wrong&readonly=1", - "trace_test", - "writer", - "p@ss/word%", - 7, - 14, - ) + await storage.ensure_schema(7, 14) assert len(recording_server.requests) == 2 assert recording_server.requests[0].raw_body.startswith(b"CREATE DATABASE IF NOT EXISTS") assert recording_server.requests[1].raw_body.startswith(b"CREATE TABLE IF NOT EXISTS") @@ -71,8 +64,15 @@ async def test_schema_setup_uses_writer_credentials_and_rejects_failed_statement ).decode() -def test_insert_encoding_preserves_nanoseconds_through_bridge() -> None: - assert json.loads(encode_rows([{"Timestamp": 1_234_567_890, "Input": "hello"}])) == { +@pytest.mark.asyncio +async def test_insert_encodes_and_sends_rows(recording_server: RecordingServer) -> None: + recording_server.enqueue(ResponseSpec(body="")) + storage: Final = NativeTraceStorage("trace_test", recording_server.base_url) + await storage.insert_rows("otel_traces", [{"Timestamp": 1_234_567_890, "Input": "hello"}]) + request: Final = recording_server.requests[0] + assert json.loads(gzip.decompress(request.raw_body)) == { "Input": "hello", "Timestamp": "1970-01-01T00:00:01.23456789Z", } + assert parse_qs(urlsplit(request.path).query)["query"] == ["INSERT INTO `trace_test`.otel_traces FORMAT JSONEachRow"] + assert request.headers["content-encoding"] == "gzip" diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index 42dc5350b495..d0a5349364f6 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -115,6 +115,7 @@ import type { ComplexityRouterConfigPayload } from "./add_model/build_complexity import type { AutoRouterPresetsResponse } from "@/lib/autorouter_presets"; import type { VectorStoreIndex } from "@/app/(dashboard)/vector-stores/_components/IndexesTab"; import type { RoutingDecision } from "./view_logs/LogDetailsDrawer/RoutingDecisionCard"; +import type { SpanDetail, Trace, TracePage } from "./view_logs/TraceView/traceTypes"; import { createApiClient, deriveErrorMessage, @@ -2101,6 +2102,33 @@ export const uiSpendLogsCall = async ({ } }; +/** + * Agent tracing. All three respond 501 `{detail}` when `general_settings.tracing` is not + * configured; callers can detect that through the thrown `ApiError`'s `status`. + */ +export const agentTraceListCall = async ({ + accessToken, + startMs, + endMs, + cursor, +}: { + accessToken: string; + startMs: number; + endMs: number; + cursor?: string | null; +}): Promise => { + const query = { start_ms: startMs, end_ms: endMs, cursor: cursor ?? undefined }; + return apiClient.get(`/v1/traces`, { accessToken, query }); +}; + +export const agentTraceCall = async (accessToken: string, traceId: string): Promise => + apiClient.get(`/v1/traces/${encodeURIComponent(traceId)}`, { accessToken }); + +export const agentTraceSpanCall = async (accessToken: string, traceId: string, spanId: string): Promise => + apiClient.get(`/v1/traces/${encodeURIComponent(traceId)}/spans/${encodeURIComponent(spanId)}`, { + accessToken, + }); + export const adminSpendLogsCall = async (accessToken: string) => { try { const data = await apiClient.get(`/global/spend/logs`, { accessToken }); diff --git a/ui/litellm-dashboard/src/components/view_logs/RequestLogsPanel.tsx b/ui/litellm-dashboard/src/components/view_logs/RequestLogsPanel.tsx index 0b5e5ff96163..c6280a6f4eb3 100644 --- a/ui/litellm-dashboard/src/components/view_logs/RequestLogsPanel.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/RequestLogsPanel.tsx @@ -25,6 +25,7 @@ import { useLogDetailRouting } from "./logDetailRouting"; import { LogDetailsDrawer } from "./LogDetailsDrawer"; import { LiveTailBanner, LogsTableToolbar } from "./LogsTableToolbar"; import { RequestLogsTable } from "./RequestLogsTable"; +import { AgentTracesSection, LogsViewSwitch, type LogsView } from "./TraceView/AgentTracesSection"; const PAGE_SIZE = DEFAULT_PAGE_SIZE_OPTIONS[0]; const DEFAULT_INTERVAL = { value: 24, unit: "hours" }; @@ -53,6 +54,7 @@ export default function RequestLogsPanel({ accessToken, token, userRole, userID, const [selectedKeyIdInfoView, setSelectedKeyIdInfoView] = useState(null); const [selectedLog, setSelectedLog] = useState(null); + const [logsView, setLogsView] = useState("all"); const { logId: urlLogId, @@ -96,7 +98,7 @@ export default function RequestLogsPanel({ accessToken, token, userRole, userID, userRole, userID, columnFilters: queryColumnFilters, - activeTab: isActive ? "request logs" : "inactive", + activeTab: isActive && logsView !== "traces" ? "request logs" : "inactive", isLiveTail, excludeInternalHealthChecks, startTime, @@ -286,53 +288,70 @@ export default function RequestLogsPanel({ accessToken, token, userRole, userID, ); } + const logsToolbar = ( + + ); + return (

Request Logs

+
{isLiveTail && pagination.pageIndex === 0 && setIsLiveTail(false)} />} - void logsQuery.refetch()} - onRowClick={handleRowClick} - onKeyHashClick={handleKeyHashClick} - onSessionClick={handleSessionClick} - teams={allTeams ?? []} - logsWindow={logsWindow} - toolbarChildren={ - - } + {logsView === "traces" &&
{logsToolbar}
} + + + {logsView !== "traces" && ( + void logsQuery.refetch()} + onRowClick={handleRowClick} + onKeyHashClick={handleKeyHashClick} + onSessionClick={handleSessionClick} + teams={allTeams ?? []} + logsWindow={logsWindow} + toolbarChildren={logsToolbar} + /> + )} + void; +} + +const INVOCATIONS_PAGE_SIZE = 10; + +function Edge({ edge }: { edge: GraphEdge }) { + const x1 = edge.from.x + edge.from.width; + const y1 = edge.from.y + GRAPH_NODE_HEIGHT / 2; + const x2 = edge.to.x; + const y2 = edge.to.y + GRAPH_NODE_HEIGHT / 2; + const mid = (x1 + x2) / 2; + return ( + + + + {edge.label} + + + ); +} + +interface NodeProps { + node: GraphNode; + hasError: boolean; + selected: boolean; + onClick: (name: string) => void; +} + +function Node({ node, hasError, selected, onClick }: NodeProps) { + const { agent } = node; + return ( + onClick(agent.name)} + onKeyDown={(event) => event.key === "Enter" && onClick(agent.name)} + > + + + {agent.name} + + + {`${agent.llm_calls} LLM · ${agent.tool_calls} tool`} + + + ); +} + +function InvocationList({ + agentName, + spans, + onOpenInvocation, +}: { + agentName: string; + spans: Span[]; + onOpenInvocation: (spanId: string) => void; +}) { + const [page, setPage] = useState(0); + const invocations = useMemo(() => invocationsOf(spans, agentName), [spans, agentName]); + const pages = Math.max(1, Math.ceil(invocations.length / INVOCATIONS_PAGE_SIZE)); + const visible = invocations.slice(page * INVOCATIONS_PAGE_SIZE, (page + 1) * INVOCATIONS_PAGE_SIZE); + return ( +
+
+ + {agentName} · {invocations.length} invocation{invocations.length === 1 ? "" : "s"} + + {pages > 1 && ( + + + {page + 1} / {pages} + + + )} +
+
    + {visible.map((span, i) => ( +
  • + +
    • + ))} +
+
+ ); +} + +/** One node per distinct agent, edges parent → child labelled with invocation count. */ +export function AgentGraph({ agents, spans, onOpenInvocation }: AgentGraphProps) { + const [selected, setSelected] = useState(null); + const layout = useMemo(() => layoutAgentGraph(agents), [agents]); + const failing = useMemo(() => agentsWithErrors(spans), [spans]); + + if (agents.length === 0) { + return
No agent spans in this trace.
; + } + + return ( +
+ + + + + + + {layout.edges.map((edge) => ( + ${edge.to.agent.name}`} edge={edge} /> + ))} + {layout.nodes.map((node) => ( + setSelected(selected === name ? null : name)} + /> + ))} + +

+ Click an agent to list its invocations. +

+ {selected && ( + + )} +
+ ); +} diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesSection.test.tsx b/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesSection.test.tsx new file mode 100644 index 000000000000..f4a45c87b949 --- /dev/null +++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesSection.test.tsx @@ -0,0 +1,80 @@ +import { screen } from "@testing-library/react"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import { ApiError } from "@/lib/http/client"; + +import { renderWithProviders, testQueryClient } from "../../../../tests/test-utils"; +import traceList from "./__fixtures__/trace_list.json"; +import { AgentTracesSection } from "./AgentTracesSection"; +import type { TracePage } from "./traceTypes"; + +vi.mock("../../networking", () => ({ + agentTraceListCall: vi.fn(), + agentTraceCall: vi.fn(), + agentTraceSpanCall: vi.fn(), +})); + +import { agentTraceListCall } from "../../networking"; + +const renderSection = (view: "all" | "traces" | "llm") => + renderWithProviders( + , + ); + +describe("AgentTracesSection", () => { + beforeEach(() => { + testQueryClient.clear(); + vi.mocked(agentTraceListCall).mockReset(); + }); + + it("renders the setup snippet when the proxy answers 501", async () => { + vi.mocked(agentTraceListCall).mockRejectedValue( + new ApiError("Agent tracing is not enabled", 501, { detail: "Agent tracing is not enabled" }), + ); + renderSection("traces"); + + const card = await screen.findByTestId("tracing-setup-card"); + expect(card).toHaveTextContent("Agent tracing is not enabled"); + expect(card).toHaveTextContent("store: clickhouse"); + expect(card).toHaveTextContent("LANGSMITH_TRACING_MODE=otel"); + expect(card).toHaveTextContent('OTEL_EXPORTER_OTLP_HEADERS="Authorization=Bearer "'); + expect(agentTraceListCall).toHaveBeenCalledTimes(1); + }); + + it("renders nothing in All when tracing is off, so request logs stay front and centre", async () => { + vi.mocked(agentTraceListCall).mockRejectedValue(new ApiError("off", 501, { detail: "off" })); + renderSection("all"); + await vi.waitFor(() => expect(agentTraceListCall).toHaveBeenCalled()); + expect(screen.queryByTestId("tracing-setup-card")).not.toBeInTheDocument(); + expect(screen.queryByRole("table", { name: "Agent traces" })).not.toBeInTheDocument(); + }); + + it("lists traces with the agent badge, red status for runs with errors", async () => { + vi.mocked(agentTraceListCall).mockResolvedValue(traceList as TracePage); + renderSection("traces"); + + const rows = await screen.findAllByTestId("agent-trace-row"); + expect(rows).toHaveLength((traceList as TracePage).data.length); + const lead = rows.find((row) => row.textContent?.includes("research_lead")); + expect(lead).toHaveTextContent("◆ 6 agents · 21 LLM · 25 tool"); + expect(lead).toHaveTextContent("Success"); + expect(screen.queryByRole("columnheader", { name: "Cost" })).not.toBeInTheDocument(); + expect(lead).toHaveTextContent("Should we store OTEL agent spans in ClickHouse or Postgres at 50k spans/sec?"); + const failed = rows.find((row) => row.textContent?.includes("acme-404")); + expect(failed).toHaveTextContent("Failure"); + expect(failed).toHaveTextContent("2 err"); + }); + + it("does not query traces in the LLM requests view", () => { + renderSection("llm"); + expect(agentTraceListCall).not.toHaveBeenCalled(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesSection.tsx b/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesSection.tsx new file mode 100644 index 000000000000..6bb2e7cefd78 --- /dev/null +++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesSection.tsx @@ -0,0 +1,98 @@ +"use client"; + +import { useState } from "react"; + +import { Button } from "@/components/ui/button"; +import { ButtonGroup } from "@/components/ui/button-group"; + +import { AgentTracesTable } from "./AgentTracesTable"; +import { TraceDrawer } from "./TraceDrawer"; +import { TracingSetupCard } from "./TracingSetupCard"; +import { useAgentTraces } from "./useAgentTraces"; + +export type LogsView = "all" | "traces" | "llm"; + +const VIEWS: { id: LogsView; label: string }[] = [ + { id: "all", label: "All" }, + { id: "traces", label: "Agent traces" }, + { id: "llm", label: "LLM requests" }, +]; + +export function LogsViewSwitch({ value, onChange }: { value: LogsView; onChange: (view: LogsView) => void }) { + return ( + + {VIEWS.map((view) => ( + + ))} + + ); +} + +interface AgentTracesSectionProps { + view: LogsView; + accessToken: string; + isActive: boolean; + startTime: string; + endTime: string; + isCustomDate: boolean; + isLiveTail: boolean; +} + +/** + * Agent trace rows for the Logs page. In "Agent traces" it is the whole list (or the + * setup card on 501); in "All" the current trace page sits above the request logs. + */ +export function AgentTracesSection({ + view, + accessToken, + isActive, + startTime, + endTime, + isCustomDate, + isLiveTail, +}: AgentTracesSectionProps) { + const [openTrace, setOpenTrace] = useState<{ traceId: string; spanId: string | null } | null>(null); + const traces = useAgentTraces({ + accessToken, + startTime, + endTime, + isCustomDate, + isLiveTail, + enabled: isActive && view !== "llm", + }); + + if (view === "llm") return null; + + return ( + <> + {view === "traces" && traces.notEnabledDetail !== null && } + {traces.notEnabledDetail === null && ( + setOpenTrace({ traceId, spanId: spanId ?? null })} + compact={view === "all"} + /> + )} + setOpenTrace(null)} + /> + + ); +} diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesTable.tsx b/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesTable.tsx new file mode 100644 index 000000000000..edf67b95f1b6 --- /dev/null +++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesTable.tsx @@ -0,0 +1,256 @@ +"use client"; + +import { useQuery } from "@tanstack/react-query"; +import { Workflow } from "lucide-react"; +import { Fragment, useState } from "react"; + +import { Button } from "@/components/ui/button"; +import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table"; +import { formatCellDate } from "@/components/shared/table_cells/date_cell"; +import { cn } from "@/lib/cva.config"; + +import { agentTraceCall } from "../../networking"; +import { AgentTracePill, SpanStatusBadge, SpanTypePill } from "./TracePills"; +import type { Span, TraceSummary } from "./traceTypes"; +import { + agentBadgeLabel, + fmtMs, + fmtTok, + llmSpans, + previewText, + shortId, + spanLabel, + summaryHasErrors, + traceDisplayName, +} from "./traceUtils"; + +interface AgentTracesTableProps { + accessToken: string; + traces: TraceSummary[]; + isLoading: boolean; + error: Error | null; + hasMore: boolean; + onLoadMore: () => void; + /** Open the trace drawer, optionally focused on one span. */ + onOpenTrace: (traceId: string, spanId?: string) => void; + /** Compact mode is used in "All" where the table sits above the request logs. */ + compact?: boolean; +} + +const COLUMNS = ["", "Time", "Type", "Status", "Trace", "Input", "Duration", "Model", "Tokens"] as const; + +const TRUNC = "block max-w-[280px] truncate"; + +function ChildRows({ + accessToken, + traceId, + onOpenTrace, +}: { + accessToken: string; + traceId: string; + onOpenTrace: AgentTracesTableProps["onOpenTrace"]; +}) { + const traceQuery = useQuery({ + queryKey: ["agentTrace", traceId, accessToken], + queryFn: () => agentTraceCall(accessToken, traceId), + staleTime: 30_000, + }); + if (traceQuery.isLoading || traceQuery.isError) { + return ( + + + {traceQuery.isError ? `Could not load trace: ${traceQuery.error.message}` : "Loading LLM calls…"} + + + ); + } + const spans = llmSpans(traceQuery.data?.spans ?? []); + return ( + <> + {spans.map((span) => ( + onOpenTrace(traceId, span.span_id)} /> + ))} + + ); +} + +function ChildRow({ span, onClick }: { span: Span; onClick: () => void }) { + const tokens = span.input_tokens + span.output_tokens; + return ( + + + +{fmtMs(span.start_offset_ms)} + + + + + + + + {span.litellm_request_id ? ( + shortId(span.litellm_request_id, 22) + ) : ( + {span.agent} + )} + + + {previewText(span.input_preview)} + + {fmtMs(span.duration_ms)} + {spanLabel(span)} + {fmtTok(tokens)} + + ); +} + +function TraceRow({ + trace, + expanded, + onToggle, + onOpen, +}: { + trace: TraceSummary; + expanded: boolean; + onToggle: () => void; + onOpen: () => void; +}) { + const failed = summaryHasErrors(trace); + // Roots that never closed report "unset"; with no errors the run reads as a success. + const status = failed ? "error" : "ok"; + const handleToggle = (event: React.MouseEvent) => { + event.stopPropagation(); + onToggle(); + }; + return ( + + + + + {formatCellDate(new Date(trace.start_time), "datetime")} + + + + + + {trace.error_count > 0 && {trace.error_count} err} + + +
{traceDisplayName(trace)}
+
{shortId(trace.trace_id)}
+ + + + {previewText(trace.input_preview)} + + + {fmtMs(trace.duration_ms)} + + {trace.models.join(", ")} + + + {fmtTok(trace.input_tokens + trace.output_tokens)}{" "} + + ({fmtTok(trace.input_tokens)}+{fmtTok(trace.output_tokens)}) + + + + ); +} + +function EmptyState() { + return ( +
+
+ +
+
No agent traces in this time range
+
+ Agent runs exported over OTLP to this proxy will appear here. +
+
+ ); +} + +/** Agent trace rows: violet badge, ▸ expands the trace's LLM calls, click opens the trace drawer. */ +export function AgentTracesTable({ + accessToken, + traces, + isLoading, + error, + hasMore, + onLoadMore, + onOpenTrace, + compact = false, +}: AgentTracesTableProps) { + const [expanded, setExpanded] = useState>(new Set()); + const toggle = (traceId: string) => + setExpanded((prev) => { + const next = new Set(prev); + if (next.has(traceId)) next.delete(traceId); + else next.add(traceId); + return next; + }); + + if (compact && traces.length === 0) return null; + + return ( +
+ + + + {COLUMNS.map((column, i) => ( + + {column} + + ))} + + + + {traces.map((trace) => ( + + toggle(trace.trace_id)} + onOpen={() => onOpenTrace(trace.trace_id)} + /> + {expanded.has(trace.trace_id) && ( + + )} + + ))} + +
+ {isLoading &&
Loading agent traces…
} + {error &&
Could not load traces: {error.message}
} + {!isLoading && !error && traces.length === 0 && } + {hasMore && ( +
+ +
+ )} +
+ ); +} diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/SpanDetail.tsx b/ui/litellm-dashboard/src/components/view_logs/TraceView/SpanDetail.tsx new file mode 100644 index 000000000000..7e22613d83e0 --- /dev/null +++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/SpanDetail.tsx @@ -0,0 +1,148 @@ +"use client"; + +import { useQuery } from "@tanstack/react-query"; +import { useState } from "react"; + +import { cn } from "@/lib/cva.config"; + +import { agentTraceSpanCall } from "../../networking"; +import { SPAN_PILL_CLASS, SpanStatusBadge, SpanTypePill } from "./TracePills"; +import type { Span, TraceMessage } from "./traceTypes"; +import { fmtMs, parseMessages, prettyPayload, spanLabel } from "./traceUtils"; + +interface SpanDetailProps { + accessToken: string; + traceId: string; + span: Span; +} + +const LONG_MESSAGE_CHARS = 360; + +function MessageBlock({ message }: { message: TraceMessage }) { + const long = (message.content || "").length > LONG_MESSAGE_CHARS; + const [expanded, setExpanded] = useState(!long); + return ( +
+ + {message.content && ( +
+ {message.content} +
+ )} + {(message.tool_calls ?? []).map((call, i) => ( +
+ {call.name}({JSON.stringify(call.args, null, 1)}) +
+ ))} +
+ ); +} + +function Payload({ title, value }: { title: string; value: string }) { + if (!value) return null; + const messages = parseMessages(value); + return ( +
+

+ {title} + {messages && messages.length > 1 ? ` · ${messages.length} messages` : ""} +

+ {messages ? ( +
+ {messages.map((message, i) => ( + + ))} +
+ ) : ( + 
+          {prettyPayload(value)}
+
+ )} +
+ ); +} + +/** Right pane: span header, joined LiteLLM request, lazily fetched input / output / attributes. */ +export function SpanDetail({ accessToken, traceId, span }: SpanDetailProps) { + const detailQuery = useQuery({ + queryKey: ["agentTraceSpan", traceId, span.span_id, accessToken], + queryFn: () => agentTraceSpanCall(accessToken, traceId, span.span_id), + staleTime: Infinity, + }); + const detail = detailQuery.data; + + return ( +
+
+ + {spanLabel(span)} + +
+
+ + start +{fmtMs(span.start_offset_ms)} + + + duration {fmtMs(span.duration_ms)} + + + agent {span.agent || "—"} + + + span {span.span_id} + +
+ {span.status === "error" && span.error && ( +
+ {span.error} +
+ )} + {detailQuery.isLoading &&
Loading span input/output…
} + {detailQuery.isError && ( +
Could not load span details: {detailQuery.error.message}
+ )} + {detail && ( + <> + + +
+

OTEL attributes

+ 
+              {JSON.stringify(
+                { trace_id: traceId, span_id: span.span_id, parent_span_id: span.parent_span_id, ...detail.attributes },
+                null,
+                2,
+              )}
+
+
+ + )} +
+ ); +} diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/SpanTree.tsx b/ui/litellm-dashboard/src/components/view_logs/TraceView/SpanTree.tsx new file mode 100644 index 000000000000..6da0b6854661 --- /dev/null +++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/SpanTree.tsx @@ -0,0 +1,198 @@ +"use client"; + +import { useEffect, useRef } from "react"; + +import { cn } from "@/lib/cva.config"; + +import { SPAN_BAR_CLASS, SpanTypePill } from "./TracePills"; +import type { Span } from "./traceTypes"; +import { fmtMs, GROUP_PAGE_SIZE, spanLabel, type SpanGroup, type TreeRow } from "./traceUtils"; + +interface SpanTreeProps { + rows: TreeRow[]; + traceDurationMs: number; + selectedId: string | null; + onSelect: (spanId: string) => void; + onToggleCollapse: (spanId: string) => void; + /** Reveal `shown` invocations of a collapsed agent group (0 hides them again). */ + onGroupShow: (groupKey: string, shown: number) => void; +} + +const MIN_BAR_PERCENT = 0.3; + +const spanMeta = (span: Span): string => fmtMs(span.duration_ms); + +export const groupSummary = (group: SpanGroup): string => { + const errors = group.errors ? ` · ${group.errors} error${group.errors === 1 ? "" : "s"}` : ""; + return `${group.name} ×${group.spans.length} · p50 ${fmtMs(group.p50Ms)}${errors}`; +}; + +function Guides({ depth }: { depth: number }) { + return ( + <> + {Array.from({ length: depth }, (_, i) => ( + + ))} + + ); +} + +function WaterfallBar({ span, total }: { span: Span; total: number }) { + const left = total > 0 ? (span.start_offset_ms / total) * 100 : 0; + const width = total > 0 ? Math.max((span.duration_ms / total) * 100, MIN_BAR_PERCENT) : MIN_BAR_PERCENT; + return ( +
+ +
+ ); +} + +interface SpanRowProps { + row: Extract; + total: number; + selected: boolean; + onSelect: (spanId: string) => void; + onToggleCollapse: (spanId: string) => void; +} + +function SpanRow({ row, total, selected, onSelect, onToggleCollapse }: SpanRowProps) { + const { span } = row; + const handleToggle = (event: React.MouseEvent) => { + event.stopPropagation(); + onToggleCollapse(span.span_id); + }; + return ( +
onSelect(span.span_id)} + className={cn( + "grid min-h-7 cursor-pointer grid-cols-[minmax(0,1fr)_28%] items-center gap-2.5 border-l-2 border-transparent px-3 py-0.5 hover:bg-muted/60", + selected && "border-l-primary bg-primary/5", + )} + > +
+ + + + {spanLabel(span)} + {spanMeta(span)} +
+ +
+ ); +} + +function GroupRow({ + row, + onGroupShow, +}: { + row: Extract; + onGroupShow: SpanTreeProps["onGroupShow"]; +}) { + const open = row.shown > 0; + return ( +
onGroupShow(row.group.key, open ? 0 : GROUP_PAGE_SIZE)} + className="flex min-h-7 cursor-pointer items-center gap-1.5 px-3 py-0.5 hover:bg-muted/60" + > + + {open ? "▾" : "▸"} + + 0 && "text-destructive")}> + {groupSummary(row.group)} + +
+ ); +} + +function MoreRow({ + row, + onGroupShow, +}: { + row: Extract; + onGroupShow: SpanTreeProps["onGroupShow"]; +}) { + const remaining = row.group.spans.length - row.shown; + return ( +
+ + +
+ ); +} + +const rowKey = (row: TreeRow): string => (row.kind === "span" ? row.span.span_id : `${row.kind}:${row.group.key}`); + +/** Waterfall tree of visible spans. Rows are computed by `flattenTree` in the drawer. */ +export function SpanTree({ + rows, + traceDurationMs, + selectedId, + onSelect, + onToggleCollapse, + onGroupShow, +}: SpanTreeProps) { + const scrollRef = useRef(null); + + useEffect(() => { + if (!selectedId) return; + const node = scrollRef.current?.querySelector(`[data-span-id="${selectedId}"]`); + if (node && "scrollIntoView" in node) (node as HTMLElement).scrollIntoView({ block: "nearest" }); + }, [selectedId, rows]); + + return ( +
+
+ Span + + 0 + {fmtMs(traceDurationMs / 2)} + {fmtMs(traceDurationMs)} + +
+
+ {rows.map((row) => { + if (row.kind === "group") return ; + if (row.kind === "more") return ; + return ( + + ); + })} +
+
+ ); +} diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/TraceDrawer.test.tsx b/ui/litellm-dashboard/src/components/view_logs/TraceView/TraceDrawer.test.tsx new file mode 100644 index 000000000000..4beb10df8344 --- /dev/null +++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/TraceDrawer.test.tsx @@ -0,0 +1,109 @@ +import { screen, within } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import { renderWithProviders, testQueryClient } from "../../../../tests/test-utils"; +import researchTrace from "./__fixtures__/research_trace.json"; +import swarmTrace from "./__fixtures__/swarm_trace.json"; +import { initialTraceView, TraceDrawer } from "./TraceDrawer"; +import type { SpanDetail, Trace } from "./traceTypes"; + +vi.mock("../../networking", () => ({ + agentTraceCall: vi.fn(), + agentTraceSpanCall: vi.fn(), +})); + +import { agentTraceCall, agentTraceSpanCall } from "../../networking"; + +const swarm = swarmTrace as Trace; +const research = researchTrace as Trace; + +const spanDetail = (spanId: string): SpanDetail => ({ + span_id: spanId, + input: JSON.stringify([{ role: "user", content: "Compare ClickHouse and Postgres ingest" }]), + output: JSON.stringify({ + role: "assistant", + content: "Calling tools", + tool_calls: [{ name: "lookup_benchmark", args: { db: "clickhouse" } }], + }), + attributes: { "gen_ai.system": "openai" }, +}); + +const renderDrawer = (trace: Trace, props: Partial> = {}) => { + vi.mocked(agentTraceCall).mockResolvedValue(trace); + return renderWithProviders( + , + ); +}; + +describe("TraceDrawer", () => { + beforeEach(() => { + testQueryClient.clear(); + vi.mocked(agentTraceSpanCall).mockImplementation(async (_token, _trace, spanId) => spanDetail(spanId)); + }); + + it("opens a healthy trace on the Steps view with the stats strip", async () => { + renderDrawer(research); + + expect(await screen.findByRole("heading", { name: "research_lead" })).toBeInTheDocument(); + const stats = screen.getByLabelText("Trace stats"); + expect(stats).toHaveTextContent("40.20s"); + expect(stats).toHaveTextContent("LLM calls21"); + expect(stats).toHaveTextContent("Tool calls25"); + expect(stats).not.toHaveTextContent("Cost"); + expect(screen.getByRole("button", { name: "Steps" })).toHaveAttribute("aria-pressed", "true"); + expect(screen.getByRole("list", { name: "Trace steps" })).toBeInTheDocument(); + }); + + it("opens a failed trace on the Tree view with the first failed span selected", async () => { + renderDrawer(swarm); + + const tree = await screen.findByRole("tree", { name: "Span tree" }); + expect(screen.getByRole("button", { name: "Tree" })).toHaveAttribute("aria-pressed", "true"); + const selected = within(tree).getByRole("treeitem", { selected: true }); + expect(selected).toHaveTextContent("lookup_benchmark"); + // the 12 researcher invocations collapse into one group row + expect(within(tree).getByText(/researcher ×12 · p50/)).toBeInTheDocument(); + }); + + it("hides framework spans until the toggle is checked", async () => { + const user = userEvent.setup(); + renderDrawer(research); + await user.click(await screen.findByRole("button", { name: "Tree" })); + + const total = research.spans.length; + expect(screen.getByText(new RegExp(`^\\d+ of ${total} spans$`))).not.toHaveTextContent(new RegExp(`^${total} of`)); + await user.click(screen.getByRole("checkbox")); + expect(screen.getByText(`${total} of ${total} spans`)).toBeInTheDocument(); + }); + + it("moves the selection with J and K", async () => { + const user = userEvent.setup(); + renderDrawer(research); + const steps = await screen.findByRole("list", { name: "Trace steps" }); + const items = within(steps).getAllByRole("listitem"); + expect(items[0]).toHaveAttribute("aria-current", "step"); + await user.keyboard("j"); + expect(items[1]).toHaveAttribute("aria-current", "step"); + await user.keyboard("k"); + expect(items[0]).toHaveAttribute("aria-current", "step"); + }); + + it("jumps from an agent graph invocation to the tree focused on that span", async () => { + const user = userEvent.setup(); + renderDrawer(swarm); + await user.click(await screen.findByRole("button", { name: "Graph" })); + await user.click(screen.getByRole("button", { name: "Agent critic" })); + await user.click(within(screen.getByRole("list", { name: "critic invocations" })).getByRole("button")); + + const tree = screen.getByRole("tree", { name: "Span tree" }); + expect(within(tree).getByRole("treeitem", { selected: true })).toHaveTextContent("critic"); + }); +}); + +describe("initialTraceView", () => { + it("prefers an explicitly requested span", () => { + const llm = swarm.spans.find((s) => s.type === "llm") as Trace["spans"][number]; + expect(initialTraceView(swarm, llm.span_id)).toEqual({ mode: "steps", selectedId: llm.span_id }); + }); +}); diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/TraceDrawer.tsx b/ui/litellm-dashboard/src/components/view_logs/TraceView/TraceDrawer.tsx new file mode 100644 index 000000000000..67006ef191a7 --- /dev/null +++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/TraceDrawer.tsx @@ -0,0 +1,313 @@ +"use client"; + +import { useQuery } from "@tanstack/react-query"; +import { Copy } from "lucide-react"; +import { useCallback, useMemo, useState } from "react"; + +import { Button } from "@/components/ui/button"; +import { ButtonGroup } from "@/components/ui/button-group"; +import { Checkbox } from "@/components/ui/checkbox"; +import { Sheet, SheetContent, SheetTitle } from "@/components/ui/sheet"; +import { UiLoadingSpinner } from "@/components/ui/ui-loading-spinner"; +import { copyToClipboard } from "@/utils/dataUtils"; + +import { agentTraceCall } from "../../networking"; +import { AgentGraph } from "./AgentGraph"; +import { SpanDetail } from "./SpanDetail"; +import { SpanTree } from "./SpanTree"; +import { AgentTracePill, SpanStatusBadge } from "./TracePills"; +import { TraceSteps } from "./TraceSteps"; +import type { Trace } from "./traceTypes"; +import { + buildVisibleTree, + firstErrorSpan, + flattenTree, + fmtMs, + fmtTok, + revealSpan, + spanRowIds, + stepsFromSpans, + subtreeStats, + traceHasErrors, + type TreeUiState, +} from "./traceUtils"; +import { stepSelection, useTraceNavigation } from "./useTraceNavigation"; + +export type TraceViewMode = "steps" | "tree" | "graph"; + +interface TraceDrawerProps { + open: boolean; + traceId: string | null; + /** Span to select on open (e.g. a child LLM row was clicked in the table). */ + initialSpanId?: string | null; + accessToken: string; + onClose: () => void; +} + +const DRAWER_WIDTH = "min(1180px, 80vw)"; +const MODES: { id: TraceViewMode; label: string }[] = [ + { id: "steps", label: "Steps" }, + { id: "tree", label: "Tree" }, + { id: "graph", label: "Graph" }, +]; +const EMPTY_UI: TreeUiState = { collapsed: new Set(), groupShown: {} }; + +interface InitialView { + mode: TraceViewMode; + selectedId: string | null; +} + +/** Errors open on the first failed span in the Tree; otherwise the Steps narrative. */ +export function initialTraceView(trace: Trace, initialSpanId?: string | null): InitialView { + if (initialSpanId && trace.spans.some((s) => s.span_id === initialSpanId)) { + return { mode: "steps", selectedId: initialSpanId }; + } + if (traceHasErrors(trace)) { + const failed = firstErrorSpan(trace.spans); + if (failed) return { mode: "tree", selectedId: failed.span_id }; + } + const firstStep = stepsFromSpans(trace.spans)[0]?.span ?? trace.spans[0]; + return { mode: "steps", selectedId: firstStep?.span_id ?? null }; +} + +/** Tree state with the initially selected span (e.g. the first failure) scrolled into reach. */ +function initialTreeUi(trace: Trace, selectedId: string | null): TreeUiState { + if (!selectedId) return EMPTY_UI; + const { children } = buildVisibleTree(trace.spans, false); + return revealSpan(children, subtreeStats(trace.spans), EMPTY_UI, selectedId); +} + +function Stat({ label, value, sub }: { label: string; value: string; sub?: string }) { + return ( +
+
{label}
+
+ {value} {sub && {sub}} +
+
+ ); +} + +export function TraceStats({ trace }: { trace: Trace }) { + const { summary } = trace; + return ( +
+ + + + + +
+ ); +} + +interface TraceBodyProps { + trace: Trace; + accessToken: string; + initialSpanId?: string | null; +} + +function TraceBody({ trace, accessToken, initialSpanId }: TraceBodyProps) { + const [initial] = useState(() => initialTraceView(trace, initialSpanId)); + const [mode, setMode] = useState(initial.mode); + const [selectedId, setSelectedId] = useState(initial.selectedId); + const [showFramework, setShowFramework] = useState(false); + const [ui, setUi] = useState(() => initialTreeUi(trace, initial.selectedId)); + + const { children, visibleCount } = useMemo( + () => buildVisibleTree(trace.spans, showFramework), + [trace.spans, showFramework], + ); + const stats = useMemo(() => subtreeStats(trace.spans), [trace.spans]); + const rows = useMemo(() => flattenTree(children, stats, ui), [children, stats, ui]); + const steps = useMemo(() => stepsFromSpans(trace.spans), [trace.spans]); + const selectedSpan = useMemo( + () => trace.spans.find((s) => s.span_id === selectedId) ?? null, + [trace.spans, selectedId], + ); + + const order = useMemo( + () => (mode === "steps" ? steps.map((s) => s.span.span_id) : spanRowIds(rows)), + [mode, steps, rows], + ); + const handleStep = useCallback( + (delta: number) => setSelectedId((current) => stepSelection(order, current, delta)), + [order], + ); + useTraceNavigation(mode !== "graph", handleStep); + + const toggleCollapse = useCallback((spanId: string) => { + setUi((prev) => { + const collapsed = new Set(prev.collapsed); + if (collapsed.has(spanId)) collapsed.delete(spanId); + else collapsed.add(spanId); + return { ...prev, collapsed }; + }); + }, []); + const showGroup = useCallback((key: string, shown: number) => { + setUi((prev) => ({ ...prev, groupShown: { ...prev.groupShown, [key]: shown } })); + }, []); + const openInvocation = useCallback( + (spanId: string) => { + setUi((prev) => revealSpan(children, stats, prev, spanId)); + setSelectedId(spanId); + setMode("tree"); + }, + [children, stats], + ); + const changeMode = useCallback( + (next: TraceViewMode) => { + if (next === "tree" && selectedId) setUi((prev) => revealSpan(children, stats, prev, selectedId)); + setMode(next); + }, + [children, stats, selectedId], + ); + + return ( +
+
+
+ + {MODES.map((m) => ( + + ))} + + {mode === "tree" && ( + + )} + + {mode === "tree" ? `${visibleCount} of ${trace.spans.length} spans` : `${steps.length} steps`} + +
+ {mode === "steps" && } + {mode === "tree" && ( + + )} + {mode === "graph" && } +
+ {selectedSpan ? ( + + ) : ( +
Select a span to see its details.
+ )} +
+ ); +} + +function TraceHeader({ trace, onClose }: { trace: Trace; onClose: () => void }) { + const { summary } = trace; + return ( +
+
+ + {summary.name} + + + J /{" "} + K to move + + +
+
+ + + service {summary.service} + + · + trace {summary.trace_id} + + · + {new Date(summary.start_time).toLocaleString()} +
+
+ ); +} + +/** Right-side drawer for one agent trace: header, stats strip, Steps / Tree / Graph views and span details. */ +export function TraceDrawer({ + open, + traceId, + initialSpanId, + accessToken, + onClose, +}: TraceDrawerProps) { + const traceQuery = useQuery({ + queryKey: ["agentTrace", traceId, accessToken], + queryFn: () => agentTraceCall(accessToken, traceId as string), + enabled: open && traceId !== null, + staleTime: 30_000, + }); + const trace = traceQuery.data; + + return ( + !nextOpen && onClose()}> + + {traceQuery.isLoading && ( +
+ Loading trace + +
+ )} + {traceQuery.isError && ( +
+ Could not load trace + {traceQuery.error.message} +
+ )} + {trace && ( +
+ + + +
+ )} + + + ); +} diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/TracePills.tsx b/ui/litellm-dashboard/src/components/view_logs/TraceView/TracePills.tsx new file mode 100644 index 000000000000..4039b62a611f --- /dev/null +++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/TracePills.tsx @@ -0,0 +1,58 @@ +"use client"; + +import { cn } from "@/lib/cva.config"; + +import type { SpanStatus, SpanType } from "./traceTypes"; + +/** Pill colours per span type: agent violet, llm blue, tool amber, chain/framework slate (matches TypeBadges). */ +export const SPAN_PILL_CLASS: Record = { + agent: + "bg-violet-50 text-violet-700 border-violet-200 dark:bg-violet-950 dark:text-violet-300 dark:border-violet-800", + llm: "bg-info/10 text-info border-info/20", + tool: "bg-amber-50 text-amber-700 border-amber-200 dark:bg-amber-950 dark:text-amber-300 dark:border-amber-800", + chain: "bg-slate-100 text-slate-600 border-slate-200 dark:bg-slate-800 dark:text-slate-300 dark:border-slate-700", + framework: "bg-slate-100 text-slate-600 border-slate-200 dark:bg-slate-800 dark:text-slate-300 dark:border-slate-700", +}; + +/** Waterfall bar fill per span type. */ +export const SPAN_BAR_CLASS: Record = { + agent: "bg-violet-500", + llm: "bg-info", + tool: "bg-amber-500", + chain: "bg-slate-400", + framework: "bg-slate-300 dark:bg-slate-600", +}; + +const PILL_BASE = + "inline-flex shrink-0 items-center gap-1 whitespace-nowrap rounded-full border px-2 py-px text-[11px] font-medium"; + +export function SpanTypePill({ type, className }: { type: SpanType; className?: string }) { + return {type === "llm" ? "LLM" : type}; +} + +/** The violet "◆ 2 agents · 7 LLM · 26 tool" badge used on agent trace rows. */ +export function AgentTracePill({ label }: { label: string }) { + return {label}; +} + +const STATUS_CLASS: Record = { + ok: "border-success/20 bg-success/10 text-success", + error: "border-destructive/20 bg-destructive/10 text-destructive", + unset: "border-border bg-muted text-muted-foreground", +}; + +const STATUS_LABEL: Record = { ok: "Success", error: "Failure", unset: "Unset" }; + +export function SpanStatusBadge({ status, compact = false }: { status: SpanStatus; compact?: boolean }) { + return ( + + {compact && status === "ok" ? "OK" : STATUS_LABEL[status]} + + ); +} diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/TraceSteps.tsx b/ui/litellm-dashboard/src/components/view_logs/TraceView/TraceSteps.tsx new file mode 100644 index 000000000000..1c2819f965a2 --- /dev/null +++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/TraceSteps.tsx @@ -0,0 +1,107 @@ +"use client"; + +import { useEffect, useRef } from "react"; + +import { cn } from "@/lib/cva.config"; + +import { SPAN_PILL_CLASS, SpanTypePill } from "./TracePills"; +import { fmtMs, previewText, spanLabel, type TraceStep } from "./traceUtils"; + +interface TraceStepsProps { + steps: TraceStep[]; + selectedId: string | null; + onSelect: (spanId: string) => void; +} + +const SUBAGENT_INDENT_PX = 28; +const BASE_PADDING_PX = 12; + +function StepTitle({ step }: { step: TraceStep }) { + const { span } = step; + if (span.type === "agent") { + return ( + <> + + subagent: {span.name} + {fmtMs(span.duration_ms)} + + ); + } + return ( + <> + + {spanLabel(span)} + + {fmtMs(span.duration_ms)} + + + ); +} + +/** Numbered narrative: every LLM decision and tool result in time order, subagent steps indented. */ +export function TraceSteps({ steps, selectedId, onSelect }: TraceStepsProps) { + const listRef = useRef(null); + + useEffect(() => { + if (!selectedId) return; + const node = listRef.current?.querySelector(`[data-span-id="${selectedId}"]`); + if (node && "scrollIntoView" in node) (node as HTMLElement).scrollIntoView({ block: "nearest" }); + }, [selectedId]); + + if (steps.length === 0) { + return
This trace has no LLM or tool spans.
; + } + + return ( +
    + {steps.map((step) => { + const pill = step.span.type === "agent" ? "agent" : step.span.type; + return ( +
  1. onSelect(step.span.span_id)} + style={{ paddingLeft: BASE_PADDING_PX + step.depth * SUBAGENT_INDENT_PX }} + className={cn( + "grid cursor-pointer grid-cols-[22px_1fr] gap-2.5 py-2 pr-3 hover:bg-muted/60", + step.span.span_id === selectedId && "bg-primary/5", + )} + > + + {step.number} + +
    +
    + + {step.subagent && in {step.subagent}} +
    + {step.span.input_preview && ( +
    + {previewText(step.span.input_preview)} +
    + )} + {step.toolNames.length > 0 && ( +
    + {step.toolNames.map((name, i) => ( + + {name}() + + ))} +
    + )} +
    +
    2. + ); + })} +
+ ); +} diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/TracingSetupCard.tsx b/ui/litellm-dashboard/src/components/view_logs/TraceView/TracingSetupCard.tsx new file mode 100644 index 000000000000..1c8b4a646f8f --- /dev/null +++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/TracingSetupCard.tsx @@ -0,0 +1,33 @@ +"use client"; + +import { Info } from "lucide-react"; + +import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert"; + +export const TRACING_SETUP_SNIPPET = `# proxy config +general_settings: + tracing: + store: clickhouse # + CLICKHOUSE_URL env var +# your agent app +export LANGSMITH_TRACING=true LANGSMITH_TRACING_MODE=otel +export OTEL_EXPORTER_OTLP_ENDPOINT= +export OTEL_EXPORTER_OTLP_HEADERS="Authorization=Bearer "`; + +/** Shown when GET /v1/traces answers 501 (agent tracing not configured on this proxy). */ +export function TracingSetupCard({ detail }: { detail?: string }) { + return ( + + + Agent tracing is not enabled + +

+ Point your agent's OpenTelemetry exporter at this proxy to see agent runs here, joined to their LLM + request logs.{detail ? ` (${detail})` : ""} +

+ 
+          {TRACING_SETUP_SNIPPET}
+
+ + + ); +} diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/__fixtures__/deep_agent_trace.json b/ui/litellm-dashboard/src/components/view_logs/TraceView/__fixtures__/deep_agent_trace.json new file mode 100644 index 000000000000..df808727309c --- /dev/null +++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/__fixtures__/deep_agent_trace.json @@ -0,0 +1,2037 @@ +{ + "summary": { + "trace_id": "4bad42b84e9de3ba46fc870185f8f023", + "name": "deep_research_agent", + "service": "agent-demo", + "input_preview": "Should we store OTEL agent spans in ClickHouse or Postgres at 50k spans/sec?", + "start_time": "2026-09-30T04:36:29.377138Z", + "duration_ms": 51385.449, + "status": "ok", + "span_count": 126, + "agent_count": 2, + "llm_calls": 7, + "tool_calls": 26, + "error_count": 0, + "input_tokens": 30175, + "output_tokens": 2620, + "spend": 0.0832806, + "models": ["claude-sonnet-4-5"] + }, + "agents": [ + { + "name": "deep_research_agent", + "parent_agent": null, + "invocations": 1, + "llm_calls": 2, + "tool_calls": 2, + "spend": 0.02875995, + "duration_ms": 51385.449 + }, + { + "name": "researcher", + "parent_agent": "deep_research_agent", + "invocations": 1, + "llm_calls": 5, + "tool_calls": 24, + "spend": 0.05452065, + "duration_ms": 35175.278 + } + ], + "spans": [ + { + "span_id": "5e79f3b5b504985e", + "parent_span_id": null, + "name": "deep_research_agent", + "type": "agent", + "agent": "deep_research_agent", + "start_offset_ms": 0.0, + "duration_ms": 51385.449, + "status": "ok", + "input_preview": "Should we store OTEL agent spans in ClickHouse or Postgres at 50k spans/sec?", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "f6e5c97125fa4e7d", + "parent_span_id": "5e79f3b5b504985e", + "name": "PatchToolCallsMiddleware.before_agent", + "type": "framework", + "agent": "deep_research_agent", + "start_offset_ms": 0.538, + "duration_ms": 0.089, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "83451f3235847f6c", + "parent_span_id": "5e79f3b5b504985e", + "name": "model", + "type": "chain", + "agent": "deep_research_agent", + "start_offset_ms": 0.743, + "duration_ms": 9518.225, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "cf04e1aa03f344fa", + "parent_span_id": "83451f3235847f6c", + "name": "FilesystemMiddleware.wrap_model_call", + "type": "framework", + "agent": "deep_research_agent", + "start_offset_ms": 1.892, + "duration_ms": 9516.701, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "e491eb7ac3b968ae", + "parent_span_id": "cf04e1aa03f344fa", + "name": "SubAgentMiddleware.wrap_model_call", + "type": "framework", + "agent": "deep_research_agent", + "start_offset_ms": 2.113, + "duration_ms": 9516.359, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "56ebca946acf9ccb", + "parent_span_id": "e491eb7ac3b968ae", + "name": "SummarizationMiddleware.wrap_model_call", + "type": "framework", + "agent": "deep_research_agent", + "start_offset_ms": 2.243, + "duration_ms": 9515.82, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "1458b553bf3f622b", + "parent_span_id": "56ebca946acf9ccb", + "name": "AnthropicPromptCachingMiddleware.wrap_model_call", + "type": "framework", + "agent": "deep_research_agent", + "start_offset_ms": 5.493, + "duration_ms": 9512.377, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "1dfaf70fdd1184f2", + "parent_span_id": "1458b553bf3f622b", + "name": "UnsupportedContentMiddleware.wrap_model_call", + "type": "framework", + "agent": "deep_research_agent", + "start_offset_ms": 5.669, + "duration_ms": 9511.945, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "8a6a1c31940d07af", + "parent_span_id": "1dfaf70fdd1184f2", + "name": "ChatOpenAI", + "type": "llm", + "agent": "deep_research_agent", + "start_offset_ms": 6.07, + "duration_ms": 9510.777, + "status": "ok", + "input_preview": "Should we store OTEL agent spans in ClickHouse or Postgres at 50k spans/sec?", + "model": "claude-sonnet-4-5", + "input_tokens": 3332, + "output_tokens": 467, + "litellm": { + "request_id": "chatcmpl-4077bb36-9380-4a3b-9481-245700cef09a", + "model": "claude-sonnet-4-5", + "model_group": "claude-sonnet-4-5", + "provider": "anthropic", + "key_alias": "agent-demo", + "team_alias": "research-agents", + "spend": 0.01949775, + "prompt_tokens": 3332, + "completion_tokens": 467, + "cache_read_tokens": 0, + "cache_write_tokens": 3329, + "latency_ms": 9510, + "ttft_ms": null, + "status": "success" + } + }, + { + "span_id": "f6fdd164d528fee5", + "parent_span_id": "5e79f3b5b504985e", + "name": "tools", + "type": "chain", + "agent": "deep_research_agent", + "start_offset_ms": 9519.796, + "duration_ms": 2.662, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "33c7678eb38a8670", + "parent_span_id": "f6fdd164d528fee5", + "name": "FilesystemMiddleware.wrap_tool_call", + "type": "framework", + "agent": "deep_research_agent", + "start_offset_ms": 9520.719, + "duration_ms": 1.361, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "1526d46d48d29a09", + "parent_span_id": "33c7678eb38a8670", + "name": "write_file", + "type": "tool", + "agent": "deep_research_agent", + "start_offset_ms": 9521.356, + "duration_ms": 0.622, + "status": "ok", + "input_preview": "{\"file_path\": \"/tmp/research_todos.md\", \"content\": \"# Research Plan: ClickHouse vs Postgres for OTEL Spans (50k/sec)\\n\\n## Tasks\\n- [ ] Research ClickHouse and Postgres capabilities for high-volume ti\u2026", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "2697122e295d91b6", + "parent_span_id": "5e79f3b5b504985e", + "name": "tools", + "type": "chain", + "agent": "deep_research_agent", + "start_offset_ms": 9522.666, + "duration_ms": 35177.678, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "56def7c7e192434a", + "parent_span_id": "2697122e295d91b6", + "name": "FilesystemMiddleware.wrap_tool_call", + "type": "framework", + "agent": "deep_research_agent", + "start_offset_ms": 9523.385, + "duration_ms": 35176.632, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "b2fb3a8f5a2fce01", + "parent_span_id": "56def7c7e192434a", + "name": "task", + "type": "tool", + "agent": "deep_research_agent", + "start_offset_ms": 9523.758, + "duration_ms": 35176.06, + "status": "ok", + "input_preview": "{\"subagent_type\": \"researcher\", \"description\": \"Research and compare ClickHouse vs Postgres for storing OpenTelemetry (OTEL) agent spans at 50,000 spans per second.\\n\\nFocus on:\\n1. Write throughput c\u2026", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "81499b492fd93f85", + "parent_span_id": "b2fb3a8f5a2fce01", + "name": "researcher", + "type": "agent", + "agent": "researcher", + "start_offset_ms": 9524.284, + "duration_ms": 35175.278, + "status": "ok", + "input_preview": "Research and compare ClickHouse vs Postgres for storing OpenTelemetry (OTEL) agent spans at 50,000 spans per second.\n\nFocus on:\n1. Write throughput capabilities - can each handle 50k spans/sec sustain\u2026", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "edabd9535a4b5eab", + "parent_span_id": "81499b492fd93f85", + "name": "PatchToolCallsMiddleware.before_agent", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 9525.39, + "duration_ms": 0.185, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "64a2c760897f3310", + "parent_span_id": "81499b492fd93f85", + "name": "model", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 9525.814, + "duration_ms": 6071.296, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "beadbe4afce32dfa", + "parent_span_id": "64a2c760897f3310", + "name": "FilesystemMiddleware.wrap_model_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 9526.227, + "duration_ms": 6070.569, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "6ff260ca90f57faf", + "parent_span_id": "beadbe4afce32dfa", + "name": "SummarizationMiddleware.wrap_model_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 9526.755, + "duration_ms": 6069.949, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "9f92adb4d5271faf", + "parent_span_id": "6ff260ca90f57faf", + "name": "AnthropicPromptCachingMiddleware.wrap_model_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 9532.039, + "duration_ms": 6064.571, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "b00edbc03c811428", + "parent_span_id": "9f92adb4d5271faf", + "name": "UnsupportedContentMiddleware.wrap_model_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 9532.387, + "duration_ms": 6064.093, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "3cebcc140afc30f9", + "parent_span_id": "b00edbc03c811428", + "name": "ChatOpenAI", + "type": "llm", + "agent": "researcher", + "start_offset_ms": 9532.919, + "duration_ms": 6063.326, + "status": "ok", + "input_preview": "Research and compare ClickHouse vs Postgres for storing OpenTelemetry (OTEL) agent spans at 50,000 spans per second.\n\nFocus on:\n1. Write throughput capabilities - can each handle 50k spans/sec sustain\u2026", + "model": "claude-sonnet-4-5", + "input_tokens": 2986, + "output_tokens": 427, + "litellm": { + "request_id": "chatcmpl-aaaa4b78-17dc-4364-8fa5-2d83369f1dbc", + "model": "claude-sonnet-4-5", + "model_group": "claude-sonnet-4-5", + "provider": "anthropic", + "key_alias": "agent-demo", + "team_alias": "research-agents", + "spend": 0.01760025, + "prompt_tokens": 2986, + "completion_tokens": 427, + "cache_read_tokens": 0, + "cache_write_tokens": 2983, + "latency_ms": 6063, + "ttft_ms": null, + "status": "success" + } + }, + { + "span_id": "21e6ff34da1cdb18", + "parent_span_id": "81499b492fd93f85", + "name": "tools", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 15598.294, + "duration_ms": 2.406, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "4949beead378f935", + "parent_span_id": "21e6ff34da1cdb18", + "name": "FilesystemMiddleware.wrap_tool_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 15599.204, + "duration_ms": 1.048, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "fe62f2ad03a0116c", + "parent_span_id": "4949beead378f935", + "name": "search_docs", + "type": "tool", + "agent": "researcher", + "start_offset_ms": 15599.584, + "duration_ms": 0.492, + "status": "ok", + "input_preview": "{\"query\": \"ClickHouse Postgres OpenTelemetry OTEL spans performance comparison\"}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "837c7d187c132fb3", + "parent_span_id": "81499b492fd93f85", + "name": "tools", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 15600.875, + "duration_ms": 1.641, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "deccff55c4fd4e1c", + "parent_span_id": "837c7d187c132fb3", + "name": "FilesystemMiddleware.wrap_tool_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 15601.505, + "duration_ms": 0.71, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "83388a7eab791953", + "parent_span_id": "deccff55c4fd4e1c", + "name": "search_docs", + "type": "tool", + "agent": "researcher", + "start_offset_ms": 15601.783, + "duration_ms": 0.344, + "status": "ok", + "input_preview": "{\"query\": \"ClickHouse write throughput 50000 spans per second telemetry\"}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "5a7c1909a7b6070a", + "parent_span_id": "81499b492fd93f85", + "name": "tools", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 15602.648, + "duration_ms": 2.664, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "fc789eb6dfefe6c3", + "parent_span_id": "81499b492fd93f85", + "name": "tools", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 15602.886, + "duration_ms": 3.473, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "8d63b242d67f1501", + "parent_span_id": "5a7c1909a7b6070a", + "name": "FilesystemMiddleware.wrap_tool_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 15603.582, + "duration_ms": 0.583, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "594d2d442764411c", + "parent_span_id": "8d63b242d67f1501", + "name": "search_docs", + "type": "tool", + "agent": "researcher", + "start_offset_ms": 15603.797, + "duration_ms": 0.284, + "status": "ok", + "input_preview": "{\"query\": \"Postgres time-series observability tracing performance\"}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "110d913a5ba8d617", + "parent_span_id": "fc789eb6dfefe6c3", + "name": "FilesystemMiddleware.wrap_tool_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 15604.55, + "duration_ms": 0.581, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "b5b9908c8f4133c8", + "parent_span_id": "110d913a5ba8d617", + "name": "search_docs", + "type": "tool", + "agent": "researcher", + "start_offset_ms": 15604.775, + "duration_ms": 0.278, + "status": "ok", + "input_preview": "{\"query\": \"ClickHouse compression storage efficiency time-series data\"}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "0c9973c7cc376699", + "parent_span_id": "81499b492fd93f85", + "name": "tools", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 15605.598, + "duration_ms": 3.401, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "ea13fd361da7da39", + "parent_span_id": "81499b492fd93f85", + "name": "tools", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 15605.981, + "duration_ms": 3.366, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "b0ef93a5f95ea00b", + "parent_span_id": "81499b492fd93f85", + "name": "tools", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 15606.532, + "duration_ms": 3.926, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "b4133562d7f3f1c4", + "parent_span_id": "0c9973c7cc376699", + "name": "FilesystemMiddleware.wrap_tool_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 15607.106, + "duration_ms": 0.695, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "994b1cf39376a0e3", + "parent_span_id": "b4133562d7f3f1c4", + "name": "search_docs", + "type": "tool", + "agent": "researcher", + "start_offset_ms": 15607.354, + "duration_ms": 0.347, + "status": "ok", + "input_preview": "{\"query\": \"OpenTelemetry trace data storage backend ClickHouse\"}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "f884a82a7cac90da", + "parent_span_id": "ea13fd361da7da39", + "name": "FilesystemMiddleware.wrap_tool_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 15608.205, + "duration_ms": 0.598, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "fcc7087fe59f84ce", + "parent_span_id": "f884a82a7cac90da", + "name": "search_docs", + "type": "tool", + "agent": "researcher", + "start_offset_ms": 15608.424, + "duration_ms": 0.297, + "status": "ok", + "input_preview": "{\"query\": \"ClickHouse columnar storage OLAP query performance aggregations\"}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "298c83b12c06a0a3", + "parent_span_id": "b0ef93a5f95ea00b", + "name": "FilesystemMiddleware.wrap_tool_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 15609.655, + "duration_ms": 0.533, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "602ed4ff76e7243f", + "parent_span_id": "298c83b12c06a0a3", + "name": "search_docs", + "type": "tool", + "agent": "researcher", + "start_offset_ms": 15609.865, + "duration_ms": 0.256, + "status": "ok", + "input_preview": "{\"query\": \"Postgres vs ClickHouse observability metrics traces\"}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "aef73f54fb954970", + "parent_span_id": "81499b492fd93f85", + "name": "tools", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 15610.576, + "duration_ms": 1.089, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "0f257b3a9aa1a26a", + "parent_span_id": "aef73f54fb954970", + "name": "FilesystemMiddleware.wrap_tool_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 15610.952, + "duration_ms": 0.479, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "8d20153626cba902", + "parent_span_id": "0f257b3a9aa1a26a", + "name": "search_docs", + "type": "tool", + "agent": "researcher", + "start_offset_ms": 15611.125, + "duration_ms": 0.234, + "status": "ok", + "input_preview": "{\"query\": \"ClickHouse insert performance batch writes sustained throughput\"}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "05114d7d3c246e6b", + "parent_span_id": "81499b492fd93f85", + "name": "model", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 15612.076, + "duration_ms": 4879.181, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "95e74cb656606e14", + "parent_span_id": "05114d7d3c246e6b", + "name": "FilesystemMiddleware.wrap_model_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 15612.405, + "duration_ms": 4878.621, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "d67fe2f9eb270591", + "parent_span_id": "95e74cb656606e14", + "name": "SummarizationMiddleware.wrap_model_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 15612.747, + "duration_ms": 4878.212, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "5205f67bffb4617b", + "parent_span_id": "d67fe2f9eb270591", + "name": "AnthropicPromptCachingMiddleware.wrap_model_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 15614.444, + "duration_ms": 4876.442, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "b60e34f522a4a74d", + "parent_span_id": "5205f67bffb4617b", + "name": "UnsupportedContentMiddleware.wrap_model_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 15614.655, + "duration_ms": 4876.123, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "319a549ffff70b5b", + "parent_span_id": "b60e34f522a4a74d", + "name": "ChatOpenAI", + "type": "llm", + "agent": "researcher", + "start_offset_ms": 15615.144, + "duration_ms": 4875.436, + "status": "ok", + "input_preview": "ClickHouse ingests 1M+ rows/s per node with batched inserts; use MergeTree ordered by (tenant, service, time) and a bloom filter index on TraceId.", + "model": "claude-sonnet-4-5", + "input_tokens": 4013, + "output_tokens": 297, + "litellm": { + "request_id": "chatcmpl-09ff3e21-7ac8-452f-bc08-71651da0ffee", + "model": "claude-sonnet-4-5", + "model_group": "claude-sonnet-4-5", + "provider": "anthropic", + "key_alias": "agent-demo", + "team_alias": "research-agents", + "spend": 0.00920715, + "prompt_tokens": 4013, + "completion_tokens": 297, + "cache_read_tokens": 2983, + "cache_write_tokens": 1023, + "latency_ms": 4875, + "ttft_ms": null, + "status": "success" + } + }, + { + "span_id": "d9175dbeb8cf2bf4", + "parent_span_id": "81499b492fd93f85", + "name": "tools", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 20492.092, + "duration_ms": 3.862, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "9711d7caa64f983b", + "parent_span_id": "81499b492fd93f85", + "name": "tools", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 20492.471, + "duration_ms": 3.823, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "81f6605116388677", + "parent_span_id": "81499b492fd93f85", + "name": "tools", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 20492.78, + "duration_ms": 4.79, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "338e443ef6992f98", + "parent_span_id": "81499b492fd93f85", + "name": "tools", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 20493.062, + "duration_ms": 6.429, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "e286dc1a2580db1d", + "parent_span_id": "d9175dbeb8cf2bf4", + "name": "FilesystemMiddleware.wrap_tool_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 20493.764, + "duration_ms": 0.767, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "9c0a8b4cd5e79898", + "parent_span_id": "e286dc1a2580db1d", + "name": "search_docs", + "type": "tool", + "agent": "researcher", + "start_offset_ms": 20494.085, + "duration_ms": 0.346, + "status": "ok", + "input_preview": "{\"query\": \"ClickHouse data compression ratio time-series storage\"}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "d364b99422266c18", + "parent_span_id": "81499b492fd93f85", + "name": "tools", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 20494.615, + "duration_ms": 5.279, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "ead94fe51eb865ca", + "parent_span_id": "9711d7caa64f983b", + "name": "FilesystemMiddleware.wrap_tool_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 20495.151, + "duration_ms": 0.621, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "2f556851946544ca", + "parent_span_id": "ead94fe51eb865ca", + "name": "search_docs", + "type": "tool", + "agent": "researcher", + "start_offset_ms": 20495.39, + "duration_ms": 0.304, + "status": "ok", + "input_preview": "{\"query\": \"trace reconstruction query performance TraceId filtering\"}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "ca3e2f0e602c4f22", + "parent_span_id": "81499b492fd93f85", + "name": "tools", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 20496.423, + "duration_ms": 4.49, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "545a060525cee8b9", + "parent_span_id": "81f6605116388677", + "name": "FilesystemMiddleware.wrap_tool_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 20496.822, + "duration_ms": 0.502, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "561c6db3f6b4a1b4", + "parent_span_id": "545a060525cee8b9", + "name": "search_docs", + "type": "tool", + "agent": "researcher", + "start_offset_ms": 20497.01, + "duration_ms": 0.245, + "status": "ok", + "input_preview": "{\"query\": \"ClickHouse operational complexity maintenance resource requirements\"}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "59eaeb0a2148a4fc", + "parent_span_id": "338e443ef6992f98", + "name": "FilesystemMiddleware.wrap_tool_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 20497.905, + "duration_ms": 0.477, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "bf7f4ce484795c2d", + "parent_span_id": "59eaeb0a2148a4fc", + "name": "search_docs", + "type": "tool", + "agent": "researcher", + "start_offset_ms": 20498.081, + "duration_ms": 0.235, + "status": "ok", + "input_preview": "{\"query\": \"Postgres TimescaleDB hypertables time-series partitioning\"}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "7be82d50e4acd54d", + "parent_span_id": "d364b99422266c18", + "name": "FilesystemMiddleware.wrap_tool_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 20498.729, + "duration_ms": 0.536, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "cea1504e1f9ca159", + "parent_span_id": "7be82d50e4acd54d", + "name": "search_docs", + "type": "tool", + "agent": "researcher", + "start_offset_ms": 20498.933, + "duration_ms": 0.258, + "status": "ok", + "input_preview": "{\"query\": \"observability backend Jaeger Tempo Signoz storage engine\"}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "f2dc8e9e8a205d6d", + "parent_span_id": "ca3e2f0e602c4f22", + "name": "FilesystemMiddleware.wrap_tool_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 20500.169, + "duration_ms": 0.483, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "a3221021b0534e6f", + "parent_span_id": "f2dc8e9e8a205d6d", + "name": "search_docs", + "type": "tool", + "agent": "researcher", + "start_offset_ms": 20500.348, + "duration_ms": 0.236, + "status": "ok", + "input_preview": "{\"query\": \"ClickHouse MergeTree TTL retention policy data lifecycle\"}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "ea7b4372ee86b40b", + "parent_span_id": "81499b492fd93f85", + "name": "model", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 20501.344, + "duration_ms": 4146.99, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "32e693f696ee8e09", + "parent_span_id": "ea7b4372ee86b40b", + "name": "FilesystemMiddleware.wrap_model_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 20501.773, + "duration_ms": 4144.753, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "6c2857333d95a7b5", + "parent_span_id": "32e693f696ee8e09", + "name": "SummarizationMiddleware.wrap_model_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 20502.135, + "duration_ms": 4143.95, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "dc46587f667ba574", + "parent_span_id": "6c2857333d95a7b5", + "name": "AnthropicPromptCachingMiddleware.wrap_model_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 20503.637, + "duration_ms": 4142.004, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "c1ad676c00916e76", + "parent_span_id": "dc46587f667ba574", + "name": "UnsupportedContentMiddleware.wrap_model_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 20503.86, + "duration_ms": 4141.488, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "ac941208ba750376", + "parent_span_id": "c1ad676c00916e76", + "name": "ChatOpenAI", + "type": "llm", + "agent": "researcher", + "start_offset_ms": 20504.499, + "duration_ms": 4139.454, + "status": "ok", + "input_preview": "ClickHouse ingests 1M+ rows/s per node with batched inserts; use MergeTree ordered by (tenant, service, time) and a bloom filter index on TraceId.", + "model": "claude-sonnet-4-5", + "input_tokens": 4634, + "output_tokens": 246, + "litellm": { + "request_id": "chatcmpl-67cda920-0fea-4e56-8e37-d1bb2061bb0c", + "model": "claude-sonnet-4-5", + "model_group": "claude-sonnet-4-5", + "provider": "anthropic", + "key_alias": "agent-demo", + "team_alias": "research-agents", + "spend": 0.00724155, + "prompt_tokens": 4634, + "completion_tokens": 246, + "cache_read_tokens": 4006, + "cache_write_tokens": 621, + "latency_ms": 4139, + "ttft_ms": null, + "status": "success" + } + }, + { + "span_id": "c08547669a2510b2", + "parent_span_id": "81499b492fd93f85", + "name": "tools", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 24652.82, + "duration_ms": 13.612, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "bd63f327de3102e9", + "parent_span_id": "c08547669a2510b2", + "name": "FilesystemMiddleware.wrap_tool_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 24655.114, + "duration_ms": 2.154, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "5145888189f26430", + "parent_span_id": "bd63f327de3102e9", + "name": "search_docs", + "type": "tool", + "agent": "researcher", + "start_offset_ms": 24656.316, + "duration_ms": 0.713, + "status": "ok", + "input_preview": "{\"query\": \"columnar database analytical queries aggregation performance\"}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "cf5c2a5726620c6d", + "parent_span_id": "81499b492fd93f85", + "name": "tools", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 24657.803, + "duration_ms": 7.245, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "b66b214e1eed463c", + "parent_span_id": "81499b492fd93f85", + "name": "tools", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 24658.91, + "duration_ms": 9.157, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "54a17e39aa2da7bc", + "parent_span_id": "cf5c2a5726620c6d", + "name": "FilesystemMiddleware.wrap_tool_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 24661.804, + "duration_ms": 1.678, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "90d5ea4b66156997", + "parent_span_id": "54a17e39aa2da7bc", + "name": "search_docs", + "type": "tool", + "agent": "researcher", + "start_offset_ms": 24662.798, + "duration_ms": 0.54, + "status": "ok", + "input_preview": "{\"query\": \"OLTP vs OLAP database workload write-heavy time-series\"}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "2eeee520cb394aae", + "parent_span_id": "81499b492fd93f85", + "name": "tools", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 24663.682, + "duration_ms": 5.76, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "9b38b070edfc6711", + "parent_span_id": "81499b492fd93f85", + "name": "tools", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 24664.352, + "duration_ms": 5.445, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "71e43febd60525c1", + "parent_span_id": "b66b214e1eed463c", + "name": "FilesystemMiddleware.wrap_tool_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 24665.558, + "duration_ms": 0.741, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "9afc16768c8fd206", + "parent_span_id": "71e43febd60525c1", + "name": "search_docs", + "type": "tool", + "agent": "researcher", + "start_offset_ms": 24665.846, + "duration_ms": 0.354, + "status": "ok", + "input_preview": "{\"query\": \"bloom filter index TraceId span lookup performance\"}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "27a60dcb575d4570", + "parent_span_id": "2eeee520cb394aae", + "name": "FilesystemMiddleware.wrap_tool_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 24667.07, + "duration_ms": 0.771, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "24aef47986e5ea47", + "parent_span_id": "27a60dcb575d4570", + "name": "search_docs", + "type": "tool", + "agent": "researcher", + "start_offset_ms": 24667.372, + "duration_ms": 0.365, + "status": "ok", + "input_preview": "{\"query\": \"Postgres vacuum bloat write amplification high insert rate\"}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "7d90328211921de8", + "parent_span_id": "9b38b070edfc6711", + "name": "FilesystemMiddleware.wrap_tool_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 24668.608, + "duration_ms": 0.63, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "15ff1df26a3bdd19", + "parent_span_id": "7d90328211921de8", + "name": "search_docs", + "type": "tool", + "agent": "researcher", + "start_offset_ms": 24668.855, + "duration_ms": 0.3, + "status": "ok", + "input_preview": "{\"query\": \"batched inserts bulk loading streaming data ingestion\"}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "22935852b890a769", + "parent_span_id": "81499b492fd93f85", + "name": "model", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 24670.265, + "duration_ms": 4373.43, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "46edf02452728f7b", + "parent_span_id": "22935852b890a769", + "name": "FilesystemMiddleware.wrap_model_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 24670.767, + "duration_ms": 4372.593, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "e405386748bc7fc6", + "parent_span_id": "46edf02452728f7b", + "name": "SummarizationMiddleware.wrap_model_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 24671.171, + "duration_ms": 4372.11, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "ea469e74ff5f8b33", + "parent_span_id": "e405386748bc7fc6", + "name": "AnthropicPromptCachingMiddleware.wrap_model_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 24673.157, + "duration_ms": 4370.031, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "e8d172adbef06ea4", + "parent_span_id": "ea469e74ff5f8b33", + "name": "UnsupportedContentMiddleware.wrap_model_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 24673.434, + "duration_ms": 4369.627, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "dd0a07cd84bbac0c", + "parent_span_id": "e8d172adbef06ea4", + "name": "ChatOpenAI", + "type": "llm", + "agent": "researcher", + "start_offset_ms": 24674.097, + "duration_ms": 4368.626, + "status": "ok", + "input_preview": "No results.", + "model": "claude-sonnet-4-5", + "input_tokens": 5104, + "output_tokens": 242, + "litellm": { + "request_id": "chatcmpl-706f846d-b28a-40e4-8db8-c5bbd9f29eb2", + "model": "claude-sonnet-4-5", + "model_group": "claude-sonnet-4-5", + "provider": "anthropic", + "key_alias": "agent-demo", + "team_alias": "research-agents", + "spend": 0.0068016, + "prompt_tokens": 5104, + "completion_tokens": 242, + "cache_read_tokens": 4627, + "cache_write_tokens": 470, + "latency_ms": 4368, + "ttft_ms": null, + "status": "success" + } + }, + { + "span_id": "809cf9e03fa89682", + "parent_span_id": "81499b492fd93f85", + "name": "tools", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 29044.691, + "duration_ms": 4.366, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "7059f01a979b7c48", + "parent_span_id": "81499b492fd93f85", + "name": "tools", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 29045.222, + "duration_ms": 4.334, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "da423a9f87165ead", + "parent_span_id": "809cf9e03fa89682", + "name": "FilesystemMiddleware.wrap_tool_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 29045.774, + "duration_ms": 0.843, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "bd0135c8a0b822ab", + "parent_span_id": "da423a9f87165ead", + "name": "search_docs", + "type": "tool", + "agent": "researcher", + "start_offset_ms": 29046.098, + "duration_ms": 0.393, + "status": "ok", + "input_preview": "{\"query\": \"ClickHouse real-world production use cases metrics logging\"}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "2078d337f17556b2", + "parent_span_id": "81499b492fd93f85", + "name": "tools", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 29046.713, + "duration_ms": 4.328, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "5e01965b501bce97", + "parent_span_id": "81499b492fd93f85", + "name": "tools", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 29047.0, + "duration_ms": 6.192, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "f9984abf3a97e304", + "parent_span_id": "81499b492fd93f85", + "name": "tools", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 29047.506, + "duration_ms": 5.967, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "7fffb63a4b4681fa", + "parent_span_id": "7059f01a979b7c48", + "name": "FilesystemMiddleware.wrap_tool_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 29048.086, + "duration_ms": 0.729, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "64a68e4bab8bfadd", + "parent_span_id": "7fffb63a4b4681fa", + "name": "search_docs", + "type": "tool", + "agent": "researcher", + "start_offset_ms": 29048.364, + "duration_ms": 0.352, + "status": "ok", + "input_preview": "{\"query\": \"time-range filtering query optimization index performance\"}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "e0a57007e35f8e0a", + "parent_span_id": "2078d337f17556b2", + "name": "FilesystemMiddleware.wrap_tool_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 29049.983, + "duration_ms": 0.742, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "32743c0d0d2e322d", + "parent_span_id": "e0a57007e35f8e0a", + "name": "search_docs", + "type": "tool", + "agent": "researcher", + "start_offset_ms": 29050.268, + "duration_ms": 0.36, + "status": "ok", + "input_preview": "{\"query\": \"storage compression LZ4 ZSTD codec efficiency\"}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "250398aa6572e9c5", + "parent_span_id": "5e01965b501bce97", + "name": "FilesystemMiddleware.wrap_tool_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 29051.476, + "duration_ms": 0.604, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "7da2a2672db0d368", + "parent_span_id": "250398aa6572e9c5", + "name": "search_docs", + "type": "tool", + "agent": "researcher", + "start_offset_ms": 29051.705, + "duration_ms": 0.294, + "status": "ok", + "input_preview": "{\"query\": \"distributed tracing backend database selection criteria\"}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "433f566b47440ac3", + "parent_span_id": "f9984abf3a97e304", + "name": "FilesystemMiddleware.wrap_tool_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 29052.424, + "duration_ms": 0.583, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "cba353fa39655c62", + "parent_span_id": "433f566b47440ac3", + "name": "search_docs", + "type": "tool", + "agent": "researcher", + "start_offset_ms": 29052.646, + "duration_ms": 0.28, + "status": "ok", + "input_preview": "{\"query\": \"resource requirements memory CPU disk IOPS high throughput\"}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "1c1ffab6777cfd82", + "parent_span_id": "81499b492fd93f85", + "name": "model", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 29053.891, + "duration_ms": 15645.011, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "82c4159bd2b62f0d", + "parent_span_id": "1c1ffab6777cfd82", + "name": "FilesystemMiddleware.wrap_model_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 29054.329, + "duration_ms": 15644.333, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "5e6739c7831893cc", + "parent_span_id": "82c4159bd2b62f0d", + "name": "SummarizationMiddleware.wrap_model_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 29054.676, + "duration_ms": 15643.931, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "f3890ceb4757c0e3", + "parent_span_id": "5e6739c7831893cc", + "name": "AnthropicPromptCachingMiddleware.wrap_model_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 29056.46, + "duration_ms": 15642.083, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "b9af74d13c4cd463", + "parent_span_id": "f3890ceb4757c0e3", + "name": "UnsupportedContentMiddleware.wrap_model_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 29056.707, + "duration_ms": 15641.734, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "163d3c4f27176d45", + "parent_span_id": "b9af74d13c4cd463", + "name": "ChatOpenAI", + "type": "llm", + "agent": "researcher", + "start_offset_ms": 29057.391, + "duration_ms": 15640.826, + "status": "ok", + "input_preview": "No results.", + "model": "claude-sonnet-4-5", + "input_tokens": 5536, + "output_tokens": 700, + "litellm": { + "request_id": "chatcmpl-3c20e808-9b16-4427-994f-5ce7e8b1b94d", + "model": "claude-sonnet-4-5", + "model_group": "claude-sonnet-4-5", + "provider": "anthropic", + "key_alias": "agent-demo", + "team_alias": "research-agents", + "spend": 0.0136701, + "prompt_tokens": 5536, + "completion_tokens": 700, + "cache_read_tokens": 5097, + "cache_write_tokens": 432, + "latency_ms": 15640, + "ttft_ms": null, + "status": "success" + } + }, + { + "span_id": "4eacac6e510c033e", + "parent_span_id": "5e79f3b5b504985e", + "name": "model", + "type": "chain", + "agent": "deep_research_agent", + "start_offset_ms": 44700.902, + "duration_ms": 6684.266, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "42f62d678007a842", + "parent_span_id": "4eacac6e510c033e", + "name": "FilesystemMiddleware.wrap_model_call", + "type": "framework", + "agent": "deep_research_agent", + "start_offset_ms": 44701.401, + "duration_ms": 6683.636, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "98f3151c49a85f4f", + "parent_span_id": "42f62d678007a842", + "name": "SubAgentMiddleware.wrap_model_call", + "type": "framework", + "agent": "deep_research_agent", + "start_offset_ms": 44701.808, + "duration_ms": 6683.194, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "6e9a065125a09f98", + "parent_span_id": "98f3151c49a85f4f", + "name": "SummarizationMiddleware.wrap_model_call", + "type": "framework", + "agent": "deep_research_agent", + "start_offset_ms": 44702.068, + "duration_ms": 6682.9, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "7ed1526630e248a6", + "parent_span_id": "6e9a065125a09f98", + "name": "AnthropicPromptCachingMiddleware.wrap_model_call", + "type": "framework", + "agent": "deep_research_agent", + "start_offset_ms": 44704.008, + "duration_ms": 6680.917, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "100f2545e642b97b", + "parent_span_id": "7ed1526630e248a6", + "name": "UnsupportedContentMiddleware.wrap_model_call", + "type": "framework", + "agent": "deep_research_agent", + "start_offset_ms": 44704.268, + "duration_ms": 6680.599, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "ecf35e7f10d8428a", + "parent_span_id": "100f2545e642b97b", + "name": "ChatOpenAI", + "type": "llm", + "agent": "deep_research_agent", + "start_offset_ms": 44704.789, + "duration_ms": 6679.937, + "status": "ok", + "input_preview": "Based on my research, here's a comprehensive comparison of **ClickHouse vs Postgres** for storing OpenTelemetry spans at 50,000 spans/second:\n\n## **1. Write Throughput Capabilities**\n\n\u2022 **ClickHouse**\u2026", + "model": "claude-sonnet-4-5", + "input_tokens": 4570, + "output_tokens": 241, + "litellm": { + "request_id": "chatcmpl-f26ccb45-ab1b-44c6-bd9e-41a02ca5f14d", + "model": "claude-sonnet-4-5", + "model_group": "claude-sonnet-4-5", + "provider": "anthropic", + "key_alias": "agent-demo", + "team_alias": "research-agents", + "spend": 0.0092622, + "prompt_tokens": 4570, + "completion_tokens": 241, + "cache_read_tokens": 3329, + "cache_write_tokens": 1234, + "latency_ms": 6679, + "ttft_ms": null, + "status": "success" + } + } + ] +} diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/__fixtures__/research_trace.json b/ui/litellm-dashboard/src/components/view_logs/TraceView/__fixtures__/research_trace.json new file mode 100644 index 000000000000..679a928d3168 --- /dev/null +++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/__fixtures__/research_trace.json @@ -0,0 +1,3606 @@ +{ + "summary": { + "trace_id": "e309a123963901e74c29cd2d3c86ff9e", + "name": "research_lead", + "service": "research-agent", + "input_preview": "[{\"role\": \"user\", \"content\": \"Should we store OTEL agent spans in ClickHouse or Postgres at 50k spans/sec?\"}]", + "start_time": "2026-09-30T06:43:54.291000+00:00", + "duration_ms": 40198.10688, + "status": "ok", + "span_count": 216, + "agent_count": 3, + "llm_calls": 21, + "tool_calls": 25, + "error_count": 0, + "input_tokens": 69506, + "output_tokens": 2960, + "spend": 0.12833025, + "models": ["claude-sonnet-4-5"] + }, + "agents": [ + { + "name": "research_lead", + "parent_agent": null, + "invocations": 1, + "llm_calls": 3, + "tool_calls": 5, + "spend": 0.025758000000000003, + "duration_ms": 40198.10688 + }, + { + "name": "researcher", + "parent_agent": "research_lead", + "invocations": 4, + "llm_calls": 17, + "tool_calls": 20, + "spend": 0.089172, + "duration_ms": 41634.618112 + }, + { + "name": "critic", + "parent_agent": "research_lead", + "invocations": 1, + "llm_calls": 1, + "tool_calls": 0, + "spend": 0.01340025, + "duration_ms": 6897.236224 + } + ], + "spans": [ + { + "span_id": "3586edf49d446541", + "parent_span_id": null, + "name": "research_lead", + "type": "agent", + "agent": "research_lead", + "start_offset_ms": 0.0, + "duration_ms": 40198.10688, + "status": "ok", + "input_preview": "[{\"role\": \"user\", \"content\": \"Should we store OTEL agent spans in ClickHouse or Postgres at 50k spans/sec?\"}]", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "21583099385b82ce", + "parent_span_id": "3586edf49d446541", + "name": "PatchToolCallsMiddleware.before_agent", + "type": "framework", + "agent": "research_lead", + "start_offset_ms": 0.457984, + "duration_ms": 0.14208, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "4e01052d4b06c859", + "parent_span_id": "3586edf49d446541", + "name": "model", + "type": "chain", + "agent": "research_lead", + "start_offset_ms": 0.71808, + "duration_ms": 8289.062912, + "status": "ok", + "input_preview": "{\"messages\":[{\"content\":\"Should we store OTEL agent spans in ClickHouse or Postgres at 50k spans/sec?\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"61d4ee17-48af-45b2-b02d-0b78aa43a534\"}],\"files\":{}}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "586bc4a78e64db27", + "parent_span_id": "4e01052d4b06c859", + "name": "FilesystemMiddleware.wrap_model_call", + "type": "framework", + "agent": "research_lead", + "start_offset_ms": 1.564928, + "duration_ms": 8288.058112, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "9e1c1bda9a35de71", + "parent_span_id": "586bc4a78e64db27", + "name": "SubAgentMiddleware.wrap_model_call", + "type": "framework", + "agent": "research_lead", + "start_offset_ms": 1.81504, + "duration_ms": 8287.769856, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "7664dbfe677a9c5e", + "parent_span_id": "9e1c1bda9a35de71", + "name": "SummarizationMiddleware.wrap_model_call", + "type": "framework", + "agent": "research_lead", + "start_offset_ms": 1.959936, + "duration_ms": 8287.579904, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "45c4bd55d643e5d3", + "parent_span_id": "7664dbfe677a9c5e", + "name": "AnthropicPromptCachingMiddleware.wrap_model_call", + "type": "framework", + "agent": "research_lead", + "start_offset_ms": 4.811008, + "duration_ms": 8284.677888, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "ae58781d8e2e70eb", + "parent_span_id": "45c4bd55d643e5d3", + "name": "UnsupportedContentMiddleware.wrap_model_call", + "type": "framework", + "agent": "research_lead", + "start_offset_ms": 4.976896, + "duration_ms": 8284.438016, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "2c51e6ddab97ed21", + "parent_span_id": "ae58781d8e2e70eb", + "name": "ChatOpenAI", + "type": "llm", + "agent": "research_lead", + "start_offset_ms": 5.357056, + "duration_ms": 8283.9168, + "status": "ok", + "input_preview": "[{\"role\": \"system\", \"content\": \"You are a research lead. Split the question into exactly 4 narrow sub-questions and delegate EACH one to the researcher subagent via task, in parallel. Then write a 3-sentence draft, send it to the critic sub", + "model": "claude-sonnet-4-5", + "input_tokens": 3378, + "output_tokens": 514, + "litellm": { + "request_id": "chatcmpl-028008eb-a34b-4132-97c2-526b707e961e", + "model": "openai/claude-sonnet-4-5", + "model_group": "claude-sonnet-4-5", + "provider": "openai", + "key_alias": "research-bot", + "team_alias": "research-agents", + "spend": 0.0087315, + "prompt_tokens": 3378, + "completion_tokens": 514, + "cache_read_tokens": 3375, + "cache_write_tokens": 0, + "latency_ms": 8278, + "ttft_ms": 8278, + "status": "success" + } + }, + { + "span_id": "b632d56816eccdae", + "parent_span_id": "3586edf49d446541", + "name": "tools", + "type": "chain", + "agent": "research_lead", + "start_offset_ms": 8290.258176, + "duration_ms": 6625.607936, + "status": "ok", + "input_preview": "{\"input\":[{\"name\":\"task\",\"args\":{\"subagent_type\":\"researcher\",\"description\":\"Research and answer this narrow question: What are the write performance characteristics and benchmarks of ClickHouse specifically for time-series data like OpenTe", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "dcf2045cfb34c569", + "parent_span_id": "b632d56816eccdae", + "name": "FilesystemMiddleware.wrap_tool_call", + "type": "framework", + "agent": "research_lead", + "start_offset_ms": 8290.758912, + "duration_ms": 6624.966912, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "53e84f1d468bc2a6", + "parent_span_id": "dcf2045cfb34c569", + "name": "task", + "type": "tool", + "agent": "research_lead", + "start_offset_ms": 8290.994944, + "duration_ms": 6624.634112, + "status": "ok", + "input_preview": "{\"subagent_type\":\"researcher\",\"description\":\"Research and answer this narrow question: What are the write performance characteristics and benchmarks of ClickHouse specifically for time-series data like OpenTelemetry spans at high ingestion ", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "8febe34dc2549d84", + "parent_span_id": "53e84f1d468bc2a6", + "name": "researcher", + "type": "agent", + "agent": "researcher", + "start_offset_ms": 8291.454976, + "duration_ms": 6624.08704, + "status": "ok", + "input_preview": "[{\"role\": \"user\", \"content\": \"Research and answer this narrow question: What are the write performance characteristics and benchmarks of ClickHouse specifically for time-series data like OpenTelemetry spans at high ingestion rates (around 5", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "cddd94e6c89deef9", + "parent_span_id": "8febe34dc2549d84", + "name": "PatchToolCallsMiddleware.before_agent", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 8291.94496, + "duration_ms": 0.247808, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "1f76f6dea3bfdd04", + "parent_span_id": "8febe34dc2549d84", + "name": "model", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 8292.339968, + "duration_ms": 3149.536, + "status": "ok", + "input_preview": "{\"messages\":[{\"content\":\"Research and answer this narrow question: What are the write performance characteristics and benchmarks of ClickHouse specifically for time-series data like OpenTelemetry spans at high ingestion rates (around 50k in", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "410375c7f58f1f43", + "parent_span_id": "1f76f6dea3bfdd04", + "name": "FilesystemMiddleware.wrap_model_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 8292.577024, + "duration_ms": 3149.182976, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "bae8d0d50862f54b", + "parent_span_id": "410375c7f58f1f43", + "name": "SummarizationMiddleware.wrap_model_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 8292.935936, + "duration_ms": 3148.79104, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "311aefedd3c6be21", + "parent_span_id": "bae8d0d50862f54b", + "name": "AnthropicPromptCachingMiddleware.wrap_model_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 8296.100096, + "duration_ms": 3145.58976, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "0ad9d77b910af51d", + "parent_span_id": "311aefedd3c6be21", + "name": "UnsupportedContentMiddleware.wrap_model_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 8296.327936, + "duration_ms": 3145.309952, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "7e5df9f2d820e2e6", + "parent_span_id": "0ad9d77b910af51d", + "name": "ChatOpenAI", + "type": "llm", + "agent": "researcher", + "start_offset_ms": 8296.672, + "duration_ms": 3144.837888, + "status": "ok", + "input_preview": "[{\"role\": \"system\", \"content\": \"Answer the question using search_docs. Reply with 2 short bullets.\"}, {\"role\": \"user\", \"content\": \"Research and answer this narrow question: What are the write performance characteristics and benchmarks of Cl", + "model": "claude-sonnet-4-5", + "input_tokens": 2890, + "output_tokens": 175, + "litellm": { + "request_id": "chatcmpl-7a706b7b-02ef-466c-92d7-41a78c9e4fd2", + "model": "openai/claude-sonnet-4-5", + "model_group": "claude-sonnet-4-5", + "provider": "openai", + "key_alias": "research-bot", + "team_alias": "research-agents", + "spend": 0.013460250000000002, + "prompt_tokens": 2890, + "completion_tokens": 175, + "cache_read_tokens": 0, + "cache_write_tokens": 2887, + "latency_ms": 3132, + "ttft_ms": 3132, + "status": "success" + } + }, + { + "span_id": "5ea0111d986e34ad", + "parent_span_id": "3586edf49d446541", + "name": "tools", + "type": "chain", + "agent": "research_lead", + "start_offset_ms": 8297.884928, + "duration_ms": 10856.022016, + "status": "ok", + "input_preview": "{\"input\":[{\"name\":\"task\",\"args\":{\"subagent_type\":\"researcher\",\"description\":\"Research and answer this narrow question: What are the write performance characteristics and limitations of Postgres for high-volume time-series data ingestion at ", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "5fcad8b07c383cca", + "parent_span_id": "5ea0111d986e34ad", + "name": "FilesystemMiddleware.wrap_tool_call", + "type": "framework", + "agent": "research_lead", + "start_offset_ms": 8298.398976, + "duration_ms": 10855.315968, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "c9d82ef1972cee03", + "parent_span_id": "5fcad8b07c383cca", + "name": "task", + "type": "tool", + "agent": "research_lead", + "start_offset_ms": 8298.578944, + "duration_ms": 10855.035904, + "status": "ok", + "input_preview": "{\"subagent_type\":\"researcher\",\"description\":\"Research and answer this narrow question: What are the write performance characteristics and limitations of Postgres for high-volume time-series data ingestion at rates around 50k inserts/sec? Re", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "d0e6fec2e9ef645f", + "parent_span_id": "c9d82ef1972cee03", + "name": "researcher", + "type": "agent", + "agent": "researcher", + "start_offset_ms": 8298.937856, + "duration_ms": 10854.58304, + "status": "ok", + "input_preview": "[{\"role\": \"user\", \"content\": \"Research and answer this narrow question: What are the write performance characteristics and limitations of Postgres for high-volume time-series data ingestion at rates around 50k inserts/sec? Report 2 bullets ", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "ae518ba9a91ed090", + "parent_span_id": "d0e6fec2e9ef645f", + "name": "PatchToolCallsMiddleware.before_agent", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 8299.330816, + "duration_ms": 0.09216, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "b99c05775f3efc9a", + "parent_span_id": "d0e6fec2e9ef645f", + "name": "model", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 8299.545856, + "duration_ms": 1781.35808, + "status": "ok", + "input_preview": "{\"messages\":[{\"content\":\"Research and answer this narrow question: What are the write performance characteristics and limitations of Postgres for high-volume time-series data ingestion at rates around 50k inserts/sec? Report 2 bullets about", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "f0296e7c6e5e1cae", + "parent_span_id": "b99c05775f3efc9a", + "name": "FilesystemMiddleware.wrap_model_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 8299.73504, + "duration_ms": 1780.889088, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "afe678420ec73b49", + "parent_span_id": "f0296e7c6e5e1cae", + "name": "SummarizationMiddleware.wrap_model_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 8299.952896, + "duration_ms": 1780.59904, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "639fe351c3e68fb2", + "parent_span_id": "afe678420ec73b49", + "name": "AnthropicPromptCachingMiddleware.wrap_model_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 8300.909824, + "duration_ms": 1779.555072, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "80cbae3d4b5b835d", + "parent_span_id": "639fe351c3e68fb2", + "name": "UnsupportedContentMiddleware.wrap_model_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 8301.693952, + "duration_ms": 1778.65216, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "a1ecd455b3c8a492", + "parent_span_id": "80cbae3d4b5b835d", + "name": "ChatOpenAI", + "type": "llm", + "agent": "researcher", + "start_offset_ms": 8301.971968, + "duration_ms": 1778.086144, + "status": "ok", + "input_preview": "[{\"role\": \"system\", \"content\": \"Answer the question using search_docs. Reply with 2 short bullets.\"}, {\"role\": \"user\", \"content\": \"Research and answer this narrow question: What are the write performance characteristics and limitations of P", + "model": "claude-sonnet-4-5", + "input_tokens": 2878, + "output_tokens": 72, + "litellm": { + "request_id": "chatcmpl-64493c16-aed8-4abf-92cc-5aa6753a3ea4", + "model": "openai/claude-sonnet-4-5", + "model_group": "claude-sonnet-4-5", + "provider": "openai", + "key_alias": "research-bot", + "team_alias": "research-agents", + "spend": 0.01187025, + "prompt_tokens": 2878, + "completion_tokens": 72, + "cache_read_tokens": 0, + "cache_write_tokens": 2875, + "latency_ms": 1766, + "ttft_ms": 1766, + "status": "success" + } + }, + { + "span_id": "4017c649a1b8a6de", + "parent_span_id": "3586edf49d446541", + "name": "tools", + "type": "chain", + "agent": "research_lead", + "start_offset_ms": 8303.038976, + "duration_ms": 11230.483968, + "status": "ok", + "input_preview": "{\"input\":[{\"name\":\"task\",\"args\":{\"subagent_type\":\"researcher\",\"description\":\"Research and answer this narrow question: What are the specific data access patterns and query requirements typical for OpenTelemetry span data (e.g., trace aggreg", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "8a0f7ae876128076", + "parent_span_id": "4017c649a1b8a6de", + "name": "FilesystemMiddleware.wrap_tool_call", + "type": "framework", + "agent": "research_lead", + "start_offset_ms": 8303.544832, + "duration_ms": 11229.657344, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "cb62778d79cec332", + "parent_span_id": "8a0f7ae876128076", + "name": "task", + "type": "tool", + "agent": "research_lead", + "start_offset_ms": 8303.700992, + "duration_ms": 11229.302016, + "status": "ok", + "input_preview": "{\"subagent_type\":\"researcher\",\"description\":\"Research and answer this narrow question: What are the specific data access patterns and query requirements typical for OpenTelemetry span data (e.g., trace aggregation, time-range queries, filte", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "8d9dc2aaa6c8f3a3", + "parent_span_id": "cb62778d79cec332", + "name": "researcher", + "type": "agent", + "agent": "researcher", + "start_offset_ms": 8303.95776, + "duration_ms": 11228.84608, + "status": "ok", + "input_preview": "[{\"role\": \"user\", \"content\": \"Research and answer this narrow question: What are the specific data access patterns and query requirements typical for OpenTelemetry span data (e.g., trace aggregation, time-range queries, filtering)? Report 2", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "38938f77ba141238", + "parent_span_id": "8d9dc2aaa6c8f3a3", + "name": "PatchToolCallsMiddleware.before_agent", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 8334.684928, + "duration_ms": 0.130048, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "58fe37b65d9f7a86", + "parent_span_id": "8d9dc2aaa6c8f3a3", + "name": "model", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 8334.936064, + "duration_ms": 1766.2848, + "status": "ok", + "input_preview": "{\"messages\":[{\"content\":\"Research and answer this narrow question: What are the specific data access patterns and query requirements typical for OpenTelemetry span data (e.g., trace aggregation, time-range queries, filtering)? Report 2 bull", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "d1433394ac5bdb00", + "parent_span_id": "58fe37b65d9f7a86", + "name": "FilesystemMiddleware.wrap_model_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 8335.143936, + "duration_ms": 1765.91104, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "f75d1be9ac1947d3", + "parent_span_id": "d1433394ac5bdb00", + "name": "SummarizationMiddleware.wrap_model_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 8335.355904, + "duration_ms": 1765.654016, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "d5c084f9ab6e23e3", + "parent_span_id": "f75d1be9ac1947d3", + "name": "AnthropicPromptCachingMiddleware.wrap_model_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 8336.306944, + "duration_ms": 1764.64384, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "5d718e2115861da6", + "parent_span_id": "d5c084f9ab6e23e3", + "name": "UnsupportedContentMiddleware.wrap_model_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 8336.446976, + "duration_ms": 1764.434944, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "eefd457b75d7c70a", + "parent_span_id": "5d718e2115861da6", + "name": "ChatOpenAI", + "type": "llm", + "agent": "researcher", + "start_offset_ms": 8336.706048, + "duration_ms": 1764.03584, + "status": "ok", + "input_preview": "[{\"role\": \"system\", \"content\": \"Answer the question using search_docs. Reply with 2 short bullets.\"}, {\"role\": \"user\", \"content\": \"Research and answer this narrow question: What are the specific data access patterns and query requirements t", + "model": "claude-sonnet-4-5", + "input_tokens": 2881, + "output_tokens": 70, + "litellm": { + "request_id": "chatcmpl-2b5813a1-a4d0-4abe-afee-d60951430963", + "model": "openai/claude-sonnet-4-5", + "model_group": "claude-sonnet-4-5", + "provider": "openai", + "key_alias": "research-bot", + "team_alias": "research-agents", + "spend": 0.011851500000000001, + "prompt_tokens": 2881, + "completion_tokens": 70, + "cache_read_tokens": 0, + "cache_write_tokens": 2878, + "latency_ms": 1754, + "ttft_ms": 1754, + "status": "success" + } + }, + { + "span_id": "25aadc9977ff9c18", + "parent_span_id": "3586edf49d446541", + "name": "tools", + "type": "chain", + "agent": "research_lead", + "start_offset_ms": 8337.646848, + "duration_ms": 12928.301056, + "status": "ok", + "input_preview": "{\"input\":[{\"name\":\"task\",\"args\":{\"subagent_type\":\"researcher\",\"description\":\"Research and answer this narrow question: What are the storage efficiency and data retention considerations for ClickHouse versus Postgres when storing large volum", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "26c7d32fd6431206", + "parent_span_id": "25aadc9977ff9c18", + "name": "FilesystemMiddleware.wrap_tool_call", + "type": "framework", + "agent": "research_lead", + "start_offset_ms": 8338.075904, + "duration_ms": 12927.710208, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "fa754e1794d2f80e", + "parent_span_id": "26c7d32fd6431206", + "name": "task", + "type": "tool", + "agent": "research_lead", + "start_offset_ms": 8338.21184, + "duration_ms": 12927.47008, + "status": "ok", + "input_preview": "{\"subagent_type\":\"researcher\",\"description\":\"Research and answer this narrow question: What are the storage efficiency and data retention considerations for ClickHouse versus Postgres when storing large volumes of observability/telemetry da", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "c2f4c93af99a664c", + "parent_span_id": "fa754e1794d2f80e", + "name": "researcher", + "type": "agent", + "agent": "researcher", + "start_offset_ms": 8338.432, + "duration_ms": 12927.101952, + "status": "ok", + "input_preview": "[{\"role\": \"user\", \"content\": \"Research and answer this narrow question: What are the storage efficiency and data retention considerations for ClickHouse versus Postgres when storing large volumes of observability/telemetry data? Report 2 bu", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "7958498c8902047d", + "parent_span_id": "c2f4c93af99a664c", + "name": "PatchToolCallsMiddleware.before_agent", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 8338.735104, + "duration_ms": 0.070912, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "225f39859d0b1797", + "parent_span_id": "c2f4c93af99a664c", + "name": "model", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 8338.904064, + "duration_ms": 3404.400896, + "status": "ok", + "input_preview": "{\"messages\":[{\"content\":\"Research and answer this narrow question: What are the storage efficiency and data retention considerations for ClickHouse versus Postgres when storing large volumes of observability/telemetry data? Report 2 bullets", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "f091b4570ff57cfc", + "parent_span_id": "225f39859d0b1797", + "name": "FilesystemMiddleware.wrap_model_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 8339.051008, + "duration_ms": 3404.1408, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "e906eb238cb8e5d0", + "parent_span_id": "f091b4570ff57cfc", + "name": "SummarizationMiddleware.wrap_model_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 8339.220992, + "duration_ms": 3403.942144, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "26181ec6f6aa4e5e", + "parent_span_id": "e906eb238cb8e5d0", + "name": "AnthropicPromptCachingMiddleware.wrap_model_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 8340.04096, + "duration_ms": 3403.091968, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "5057524339d490c5", + "parent_span_id": "26181ec6f6aa4e5e", + "name": "UnsupportedContentMiddleware.wrap_model_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 8340.16512, + "duration_ms": 3402.922752, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "11290db27f2eeea0", + "parent_span_id": "5057524339d490c5", + "name": "ChatOpenAI", + "type": "llm", + "agent": "researcher", + "start_offset_ms": 8340.397056, + "duration_ms": 3402.59584, + "status": "ok", + "input_preview": "[{\"role\": \"system\", \"content\": \"Answer the question using search_docs. Reply with 2 short bullets.\"}, {\"role\": \"user\", \"content\": \"Research and answer this narrow question: What are the storage efficiency and data retention considerations f", + "model": "claude-sonnet-4-5", + "input_tokens": 2875, + "output_tokens": 197, + "litellm": { + "request_id": "chatcmpl-cd4dcc18-4665-43f0-b22c-4685fec2aa33", + "model": "openai/claude-sonnet-4-5", + "model_group": "claude-sonnet-4-5", + "provider": "openai", + "key_alias": "research-bot", + "team_alias": "research-agents", + "spend": 0.013734, + "prompt_tokens": 2875, + "completion_tokens": 197, + "cache_read_tokens": 0, + "cache_write_tokens": 2872, + "latency_ms": 3397, + "ttft_ms": 3397, + "status": "success" + } + }, + { + "span_id": "d9dad6ec7330b73f", + "parent_span_id": "d0e6fec2e9ef645f", + "name": "tools", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 10081.412096, + "duration_ms": 2.178816, + "status": "ok", + "input_preview": "{\"input\":[{\"name\":\"search_docs\",\"args\":{\"query\":\"Postgres write performance time-series data ingestion 50k inserts per second high-volume\"},\"id\":\"toolu_017EZMv245fwxq2BWyzieBhq\",\"type\":\"tool_call\"}]}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "96abed5e22e54c83", + "parent_span_id": "d9dad6ec7330b73f", + "name": "FilesystemMiddleware.wrap_tool_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 10082.205952, + "duration_ms": 1.025024, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "ca8488012a54b761", + "parent_span_id": "96abed5e22e54c83", + "name": "search_docs", + "type": "tool", + "agent": "researcher", + "start_offset_ms": 10082.61504, + "duration_ms": 0.475904, + "status": "ok", + "input_preview": "{\"query\":\"Postgres write performance time-series data ingestion 50k inserts per second high-volume\"}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "0cde1462b20eb64e", + "parent_span_id": "d0e6fec2e9ef645f", + "name": "model", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 10083.95904, + "duration_ms": 1856.2368, + "status": "ok", + "input_preview": "{\"messages\":[{\"content\":\"Research and answer this narrow question: What are the write performance characteristics and limitations of Postgres for high-volume time-series data ingestion at rates around 50k inserts/sec? Report 2 bullets about", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "09bcb03638055e6f", + "parent_span_id": "0cde1462b20eb64e", + "name": "FilesystemMiddleware.wrap_model_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 10084.428032, + "duration_ms": 1855.6608, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "5f6058ccf4810d98", + "parent_span_id": "09bcb03638055e6f", + "name": "SummarizationMiddleware.wrap_model_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 10084.84992, + "duration_ms": 1855.212032, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "3c01c335af52bfaa", + "parent_span_id": "5f6058ccf4810d98", + "name": "AnthropicPromptCachingMiddleware.wrap_model_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 10086.747136, + "duration_ms": 1853.28384, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "0d72911998165ec9", + "parent_span_id": "3c01c335af52bfaa", + "name": "UnsupportedContentMiddleware.wrap_model_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 10087.028992, + "duration_ms": 1852.96, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "82056db608f1c211", + "parent_span_id": "0d72911998165ec9", + "name": "ChatOpenAI", + "type": "llm", + "agent": "researcher", + "start_offset_ms": 10087.582976, + "duration_ms": 1852.304896, + "status": "ok", + "input_preview": "[{\"role\": \"system\", \"content\": \"Answer the question using search_docs. Reply with 2 short bullets.\"}, {\"role\": \"user\", \"content\": \"Research and answer this narrow question: What are the write performance characteristics and limitations of P", + "model": "claude-sonnet-4-5", + "input_tokens": 2985, + "output_tokens": 65, + "litellm": { + "request_id": "chatcmpl-81dd6157-0aaf-4d19-835a-6dcee93884e8", + "model": "openai/claude-sonnet-4-5", + "model_group": "claude-sonnet-4-5", + "provider": "openai", + "key_alias": "research-bot", + "team_alias": "research-agents", + "spend": 0.00224625, + "prompt_tokens": 2985, + "completion_tokens": 65, + "cache_read_tokens": 2875, + "cache_write_tokens": 105, + "latency_ms": 1845, + "ttft_ms": 1845, + "status": "success" + } + }, + { + "span_id": "d7401760f3f5a01e", + "parent_span_id": "8d9dc2aaa6c8f3a3", + "name": "tools", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 10101.520896, + "duration_ms": 1.287936, + "status": "ok", + "input_preview": "{\"input\":[{\"name\":\"search_docs\",\"args\":{\"query\":\"OpenTelemetry span data access patterns query requirements trace aggregation time-range filtering\"},\"id\":\"toolu_01Sa3MPe67WdezvSNxYVCTZ7\",\"type\":\"tool_call\"}]}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "85f208624a903f82", + "parent_span_id": "d7401760f3f5a01e", + "name": "FilesystemMiddleware.wrap_tool_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 10102.009856, + "duration_ms": 0.573184, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "72fcc3295390f481", + "parent_span_id": "85f208624a903f82", + "name": "search_docs", + "type": "tool", + "agent": "researcher", + "start_offset_ms": 10102.22592, + "duration_ms": 0.272128, + "status": "ok", + "input_preview": "{\"query\":\"OpenTelemetry span data access patterns query requirements trace aggregation time-range filtering\"}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "bd75dbd12044c1c3", + "parent_span_id": "8d9dc2aaa6c8f3a3", + "name": "model", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 10103.049984, + "duration_ms": 1676.137984, + "status": "ok", + "input_preview": "{\"messages\":[{\"content\":\"Research and answer this narrow question: What are the specific data access patterns and query requirements typical for OpenTelemetry span data (e.g., trace aggregation, time-range queries, filtering)? Report 2 bull", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "87a3f5bcdc02321a", + "parent_span_id": "bd75dbd12044c1c3", + "name": "FilesystemMiddleware.wrap_model_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 10103.357952, + "duration_ms": 1675.725824, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "fefc0d7b6a5d355d", + "parent_span_id": "87a3f5bcdc02321a", + "name": "SummarizationMiddleware.wrap_model_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 10103.67872, + "duration_ms": 1675.377152, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "bae2312368589d86", + "parent_span_id": "fefc0d7b6a5d355d", + "name": "AnthropicPromptCachingMiddleware.wrap_model_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 10105.19808, + "duration_ms": 1673.82784, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "044e406836b59c6d", + "parent_span_id": "bae2312368589d86", + "name": "UnsupportedContentMiddleware.wrap_model_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 10105.406976, + "duration_ms": 1673.577984, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "26fd0196ea969838", + "parent_span_id": "044e406836b59c6d", + "name": "ChatOpenAI", + "type": "llm", + "agent": "researcher", + "start_offset_ms": 10105.803776, + "duration_ms": 1673.087232, + "status": "ok", + "input_preview": "[{\"role\": \"system\", \"content\": \"Answer the question using search_docs. Reply with 2 short bullets.\"}, {\"role\": \"user\", \"content\": \"Research and answer this narrow question: What are the specific data access patterns and query requirements t", + "model": "claude-sonnet-4-5", + "input_tokens": 2966, + "output_tokens": 59, + "litellm": { + "request_id": "chatcmpl-320edff6-4f0b-4d67-b6e5-539c68ca2a82", + "model": "openai/claude-sonnet-4-5", + "model_group": "claude-sonnet-4-5", + "provider": "openai", + "key_alias": "research-bot", + "team_alias": "research-agents", + "spend": 0.00207465, + "prompt_tokens": 2966, + "completion_tokens": 59, + "cache_read_tokens": 2878, + "cache_write_tokens": 83, + "latency_ms": 1667, + "ttft_ms": 1667, + "status": "success" + } + }, + { + "span_id": "7d66952eb3149ff8", + "parent_span_id": "8febe34dc2549d84", + "name": "tools", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 11442.199808, + "duration_ms": 1.049088, + "status": "ok", + "input_preview": "{\"input\":[{\"name\":\"search_docs\",\"args\":{\"query\":\"ClickHouse write performance time-series OpenTelemetry spans ingestion rate 50k inserts per second\"},\"id\":\"toolu_01Dg4aLbkY4pXiQMWew5maNs\",\"type\":\"tool_call\"}]}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "2accd598b191224f", + "parent_span_id": "7d66952eb3149ff8", + "name": "FilesystemMiddleware.wrap_tool_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 11442.521856, + "duration_ms": 0.385024, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "910bd68064a40985", + "parent_span_id": "2accd598b191224f", + "name": "search_docs", + "type": "tool", + "agent": "researcher", + "start_offset_ms": 11442.671872, + "duration_ms": 0.179968, + "status": "ok", + "input_preview": "{\"query\":\"ClickHouse write performance time-series OpenTelemetry spans ingestion rate 50k inserts per second\"}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "3e3b69eab1b21370", + "parent_span_id": "8febe34dc2549d84", + "name": "tools", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 11443.020032, + "duration_ms": 1.074944, + "status": "ok", + "input_preview": "{\"input\":[{\"name\":\"search_docs\",\"args\":{\"query\":\"ClickHouse high volume writes benchmarks throughput inserts per second\"},\"id\":\"toolu_016UXr44rs5u7ZLpy7cSoVnd\",\"type\":\"tool_call\"}]}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "f68c7136c22a27c0", + "parent_span_id": "3e3b69eab1b21370", + "name": "FilesystemMiddleware.wrap_tool_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 11443.524864, + "duration_ms": 0.391168, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "ec44997ed84dd458", + "parent_span_id": "f68c7136c22a27c0", + "name": "search_docs", + "type": "tool", + "agent": "researcher", + "start_offset_ms": 11443.681792, + "duration_ms": 0.182272, + "status": "ok", + "input_preview": "{\"query\":\"ClickHouse high volume writes benchmarks throughput inserts per second\"}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "89230f61ca3d25bb", + "parent_span_id": "8febe34dc2549d84", + "name": "tools", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 11444.18304, + "duration_ms": 0.926976, + "status": "ok", + "input_preview": "{\"input\":[{\"name\":\"search_docs\",\"args\":{\"query\":\"ClickHouse time-series data performance characteristics batch inserts\"},\"id\":\"toolu_01EmCdBSeJKEZt1A8vprJDcf\",\"type\":\"tool_call\"}]}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "b8323e5ea41e7d44", + "parent_span_id": "89230f61ca3d25bb", + "name": "FilesystemMiddleware.wrap_tool_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 11444.523008, + "duration_ms": 0.352, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "a5c73b39062741a5", + "parent_span_id": "b8323e5ea41e7d44", + "name": "search_docs", + "type": "tool", + "agent": "researcher", + "start_offset_ms": 11444.66304, + "duration_ms": 0.160768, + "status": "ok", + "input_preview": "{\"query\":\"ClickHouse time-series data performance characteristics batch inserts\"}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "c503d0b6702f171d", + "parent_span_id": "8febe34dc2549d84", + "name": "model", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 11445.348864, + "duration_ms": 3469.96608, + "status": "ok", + "input_preview": "{\"messages\":[{\"content\":\"Research and answer this narrow question: What are the write performance characteristics and benchmarks of ClickHouse specifically for time-series data like OpenTelemetry spans at high ingestion rates (around 50k in", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "1aa2a868683855dc", + "parent_span_id": "c503d0b6702f171d", + "name": "FilesystemMiddleware.wrap_model_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 11445.566976, + "duration_ms": 3469.622784, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "6fa486e85cc4b885", + "parent_span_id": "1aa2a868683855dc", + "name": "SummarizationMiddleware.wrap_model_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 11445.743872, + "duration_ms": 3469.415936, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "a85cb07902b96d84", + "parent_span_id": "6fa486e85cc4b885", + "name": "AnthropicPromptCachingMiddleware.wrap_model_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 11446.690048, + "duration_ms": 3468.435968, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "b5f3affc5eac5183", + "parent_span_id": "a85cb07902b96d84", + "name": "UnsupportedContentMiddleware.wrap_model_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 11446.808064, + "duration_ms": 3468.271872, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "89af13298c71e8d3", + "parent_span_id": "b5f3affc5eac5183", + "name": "ChatOpenAI", + "type": "llm", + "agent": "researcher", + "start_offset_ms": 11447.05792, + "duration_ms": 3467.904, + "status": "ok", + "input_preview": "[{\"role\": \"system\", \"content\": \"Answer the question using search_docs. Reply with 2 short bullets.\"}, {\"role\": \"user\", \"content\": \"Research and answer this narrow question: What are the write performance characteristics and benchmarks of Cl", + "model": "claude-sonnet-4-5", + "input_tokens": 3279, + "output_tokens": 113, + "litellm": { + "request_id": "chatcmpl-fa113ca8-9194-443e-b5b1-d009979ae7a8", + "model": "openai/claude-sonnet-4-5", + "model_group": "claude-sonnet-4-5", + "provider": "openai", + "key_alias": "research-bot", + "team_alias": "research-agents", + "spend": 0.00402585, + "prompt_tokens": 3279, + "completion_tokens": 113, + "cache_read_tokens": 2887, + "cache_write_tokens": 385, + "latency_ms": 3463, + "ttft_ms": 3463, + "status": "success" + } + }, + { + "span_id": "f0e6ba4d76959b0f", + "parent_span_id": "c2f4c93af99a664c", + "name": "tools", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 11743.6288, + "duration_ms": 1.161216, + "status": "ok", + "input_preview": "{\"input\":[{\"name\":\"search_docs\",\"args\":{\"query\":\"ClickHouse Postgres storage efficiency compression observability telemetry data retention\"},\"id\":\"toolu_01LpCf52iSMhb4hPvohUvKXo\",\"type\":\"tool_call\"}]}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "dbdf1fe23f0b7608", + "parent_span_id": "f0e6ba4d76959b0f", + "name": "FilesystemMiddleware.wrap_tool_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 11743.971072, + "duration_ms": 0.377856, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "89478eee5f16f7bc", + "parent_span_id": "dbdf1fe23f0b7608", + "name": "search_docs", + "type": "tool", + "agent": "researcher", + "start_offset_ms": 11744.114944, + "duration_ms": 0.179968, + "status": "ok", + "input_preview": "{\"query\":\"ClickHouse Postgres storage efficiency compression observability telemetry data retention\"}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "c7a58fd2d402ca41", + "parent_span_id": "c2f4c93af99a664c", + "name": "tools", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 11744.491008, + "duration_ms": 1.030912, + "status": "ok", + "input_preview": "{\"input\":[{\"name\":\"search_docs\",\"args\":{\"query\":\"ClickHouse compression storage footprint column-oriented telemetry\"},\"id\":\"toolu_01Ac1WiKauEA8EJTfYRx8xnu\",\"type\":\"tool_call\"}]}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "4e57e56745c075dc", + "parent_span_id": "c7a58fd2d402ca41", + "name": "FilesystemMiddleware.wrap_tool_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 11745.036032, + "duration_ms": 0.356864, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "332e55f64c2cf833", + "parent_span_id": "4e57e56745c075dc", + "name": "search_docs", + "type": "tool", + "agent": "researcher", + "start_offset_ms": 11745.176832, + "duration_ms": 0.168192, + "status": "ok", + "input_preview": "{\"query\":\"ClickHouse compression storage footprint column-oriented telemetry\"}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "ae7ebc9a1d85d06f", + "parent_span_id": "c2f4c93af99a664c", + "name": "tools", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 11745.652992, + "duration_ms": 0.724736, + "status": "ok", + "input_preview": "{\"input\":[{\"name\":\"search_docs\",\"args\":{\"query\":\"Postgres ClickHouse data retention TTL observability metrics logs\"},\"id\":\"toolu_019cQbrLgYaBHvDPNiHQKk8q\",\"type\":\"tool_call\"}]}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "30534e0e14e0976d", + "parent_span_id": "ae7ebc9a1d85d06f", + "name": "FilesystemMiddleware.wrap_tool_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 11745.918976, + "duration_ms": 0.348928, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "7135e0cf258b90c2", + "parent_span_id": "30534e0e14e0976d", + "name": "search_docs", + "type": "tool", + "agent": "researcher", + "start_offset_ms": 11746.05184, + "duration_ms": 0.167168, + "status": "ok", + "input_preview": "{\"query\":\"Postgres ClickHouse data retention TTL observability metrics logs\"}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "ce0587c4a969adab", + "parent_span_id": "c2f4c93af99a664c", + "name": "model", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 11746.582016, + "duration_ms": 2849.200896, + "status": "ok", + "input_preview": "{\"messages\":[{\"content\":\"Research and answer this narrow question: What are the storage efficiency and data retention considerations for ClickHouse versus Postgres when storing large volumes of observability/telemetry data? Report 2 bullets", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "d4d8a5bfedbb5dff", + "parent_span_id": "ce0587c4a969adab", + "name": "FilesystemMiddleware.wrap_model_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 11746.767872, + "duration_ms": 2848.839936, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "ffed87b10acccc61", + "parent_span_id": "d4d8a5bfedbb5dff", + "name": "SummarizationMiddleware.wrap_model_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 11746.927872, + "duration_ms": 2848.633088, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "08d0d119b42f65cf", + "parent_span_id": "ffed87b10acccc61", + "name": "AnthropicPromptCachingMiddleware.wrap_model_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 11747.862016, + "duration_ms": 2847.648, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "32947733f303bef9", + "parent_span_id": "08d0d119b42f65cf", + "name": "UnsupportedContentMiddleware.wrap_model_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 11747.977984, + "duration_ms": 2847.454976, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "a5139c89995770e0", + "parent_span_id": "32947733f303bef9", + "name": "ChatOpenAI", + "type": "llm", + "agent": "researcher", + "start_offset_ms": 11748.226048, + "duration_ms": 2847.051776, + "status": "ok", + "input_preview": "[{\"role\": \"system\", \"content\": \"Answer the question using search_docs. Reply with 2 short bullets.\"}, {\"role\": \"user\", \"content\": \"Research and answer this narrow question: What are the storage efficiency and data retention considerations f", + "model": "claude-sonnet-4-5", + "input_tokens": 3302, + "output_tokens": 162, + "litellm": { + "request_id": "chatcmpl-ba8dcfcc-6ce3-48d4-88a1-46e07f2e3a5b", + "model": "openai/claude-sonnet-4-5", + "model_group": "claude-sonnet-4-5", + "provider": "openai", + "key_alias": "research-bot", + "team_alias": "research-agents", + "spend": 0.00489885, + "prompt_tokens": 3302, + "completion_tokens": 162, + "cache_read_tokens": 2872, + "cache_write_tokens": 423, + "latency_ms": 2841, + "ttft_ms": 2841, + "status": "success" + } + }, + { + "span_id": "ba1b8bea84bb3419", + "parent_span_id": "8d9dc2aaa6c8f3a3", + "name": "tools", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 11779.385088, + "duration_ms": 0.870912, + "status": "ok", + "input_preview": "{\"input\":[{\"name\":\"search_docs\",\"args\":{\"query\":\"span data query patterns trace search filtering\"},\"id\":\"toolu_017Hv7XgHkS9cYQRvEGwK6zF\",\"type\":\"tool_call\"}]}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "74635fe5c444a65f", + "parent_span_id": "ba1b8bea84bb3419", + "name": "FilesystemMiddleware.wrap_tool_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 11779.719936, + "duration_ms": 0.380928, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "941f5d334364eba3", + "parent_span_id": "74635fe5c444a65f", + "name": "search_docs", + "type": "tool", + "agent": "researcher", + "start_offset_ms": 11779.874048, + "duration_ms": 0.171776, + "status": "ok", + "input_preview": "{\"query\":\"span data query patterns trace search filtering\"}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "e664750f1fd77b25", + "parent_span_id": "8d9dc2aaa6c8f3a3", + "name": "model", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 11780.430848, + "duration_ms": 1400.856064, + "status": "ok", + "input_preview": "{\"messages\":[{\"content\":\"Research and answer this narrow question: What are the specific data access patterns and query requirements typical for OpenTelemetry span data (e.g., trace aggregation, time-range queries, filtering)? Report 2 bull", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "71fc36d2515d599f", + "parent_span_id": "e664750f1fd77b25", + "name": "FilesystemMiddleware.wrap_model_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 11780.673792, + "duration_ms": 1400.436224, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "1b100724c30e4584", + "parent_span_id": "71fc36d2515d599f", + "name": "SummarizationMiddleware.wrap_model_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 11780.86784, + "duration_ms": 1400.175104, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "248a4bb6d8182204", + "parent_span_id": "1b100724c30e4584", + "name": "AnthropicPromptCachingMiddleware.wrap_model_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 11781.968128, + "duration_ms": 1399.019776, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "aa46be42752e8a7d", + "parent_span_id": "248a4bb6d8182204", + "name": "UnsupportedContentMiddleware.wrap_model_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 11782.093824, + "duration_ms": 1398.818048, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "aa10c9eeb130add5", + "parent_span_id": "aa46be42752e8a7d", + "name": "ChatOpenAI", + "type": "llm", + "agent": "researcher", + "start_offset_ms": 11782.33984, + "duration_ms": 1398.417152, + "status": "ok", + "input_preview": "[{\"role\": \"system\", \"content\": \"Answer the question using search_docs. Reply with 2 short bullets.\"}, {\"role\": \"user\", \"content\": \"Research and answer this narrow question: What are the specific data access patterns and query requirements t", + "model": "claude-sonnet-4-5", + "input_tokens": 3039, + "output_tokens": 61, + "litellm": { + "request_id": "chatcmpl-f24284ae-daf1-4848-b58c-16939577b86a", + "model": "openai/claude-sonnet-4-5", + "model_group": "claude-sonnet-4-5", + "provider": "openai", + "key_alias": "research-bot", + "team_alias": "research-agents", + "spend": 0.00209205, + "prompt_tokens": 3039, + "completion_tokens": 61, + "cache_read_tokens": 2961, + "cache_write_tokens": 73, + "latency_ms": 1392, + "ttft_ms": 1392, + "status": "success" + } + }, + { + "span_id": "ff9c60db21b2487f", + "parent_span_id": "d0e6fec2e9ef645f", + "name": "tools", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 11940.40192, + "duration_ms": 0.845824, + "status": "ok", + "input_preview": "{\"input\":[{\"name\":\"search_docs\",\"args\":{\"query\":\"PostgreSQL insert performance throughput limitations batch inserts time-series\"},\"id\":\"toolu_01UUhyrPeiB41ruKWhD5h7Lt\",\"type\":\"tool_call\"}]}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "b581e9dae9a8e65a", + "parent_span_id": "ff9c60db21b2487f", + "name": "FilesystemMiddleware.wrap_tool_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 11940.723968, + "duration_ms": 0.363008, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "99b3c54206ab172e", + "parent_span_id": "b581e9dae9a8e65a", + "name": "search_docs", + "type": "tool", + "agent": "researcher", + "start_offset_ms": 11940.865792, + "duration_ms": 0.168192, + "status": "ok", + "input_preview": "{\"query\":\"PostgreSQL insert performance throughput limitations batch inserts time-series\"}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "674fbd4c084f1ae9", + "parent_span_id": "d0e6fec2e9ef645f", + "name": "model", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 11941.427968, + "duration_ms": 1514.968064, + "status": "ok", + "input_preview": "{\"messages\":[{\"content\":\"Research and answer this narrow question: What are the write performance characteristics and limitations of Postgres for high-volume time-series data ingestion at rates around 50k inserts/sec? Report 2 bullets about", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "1b77ec92eb9e3df7", + "parent_span_id": "674fbd4c084f1ae9", + "name": "FilesystemMiddleware.wrap_model_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 11941.666048, + "duration_ms": 1514.493952, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "a6b0961929376ce4", + "parent_span_id": "1b77ec92eb9e3df7", + "name": "SummarizationMiddleware.wrap_model_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 11941.878016, + "duration_ms": 1514.230016, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "2bccf2bcd2554900", + "parent_span_id": "a6b0961929376ce4", + "name": "AnthropicPromptCachingMiddleware.wrap_model_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 11942.840832, + "duration_ms": 1513.211136, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "47c68249883b82f6", + "parent_span_id": "2bccf2bcd2554900", + "name": "UnsupportedContentMiddleware.wrap_model_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 11942.966784, + "duration_ms": 1513.001216, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "f871e3307724bf1b", + "parent_span_id": "47c68249883b82f6", + "name": "ChatOpenAI", + "type": "llm", + "agent": "researcher", + "start_offset_ms": 11943.206912, + "duration_ms": 1512.521984, + "status": "ok", + "input_preview": "[{\"role\": \"system\", \"content\": \"Answer the question using search_docs. Reply with 2 short bullets.\"}, {\"role\": \"user\", \"content\": \"Research and answer this narrow question: What are the write performance characteristics and limitations of P", + "model": "claude-sonnet-4-5", + "input_tokens": 3084, + "output_tokens": 70, + "litellm": { + "request_id": "chatcmpl-31172160-15f3-4a0b-bb95-f054c1f49c96", + "model": "openai/claude-sonnet-4-5", + "model_group": "claude-sonnet-4-5", + "provider": "openai", + "key_alias": "research-bot", + "team_alias": "research-agents", + "spend": 0.00233025, + "prompt_tokens": 3084, + "completion_tokens": 70, + "cache_read_tokens": 2980, + "cache_write_tokens": 99, + "latency_ms": 1506, + "ttft_ms": 1506, + "status": "success" + } + }, + { + "span_id": "9ce1da40c0fb0cff", + "parent_span_id": "8d9dc2aaa6c8f3a3", + "name": "tools", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 13181.638912, + "duration_ms": 1.458944, + "status": "ok", + "input_preview": "{\"input\":[{\"name\":\"search_docs\",\"args\":{\"query\":\"OpenTelemetry traces spans storage query\"},\"id\":\"toolu_01DLFewyFmJArEjsbHzEi9UK\",\"type\":\"tool_call\"}]}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "44411aefd212169b", + "parent_span_id": "9ce1da40c0fb0cff", + "name": "FilesystemMiddleware.wrap_tool_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 13182.208, + "duration_ms": 0.637952, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "f870964607787ed0", + "parent_span_id": "44411aefd212169b", + "name": "search_docs", + "type": "tool", + "agent": "researcher", + "start_offset_ms": 13182.466048, + "duration_ms": 0.29184, + "status": "ok", + "input_preview": "{\"query\":\"OpenTelemetry traces spans storage query\"}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "ba0687166794762c", + "parent_span_id": "8d9dc2aaa6c8f3a3", + "name": "model", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 13183.383808, + "duration_ms": 1531.270144, + "status": "ok", + "input_preview": "{\"messages\":[{\"content\":\"Research and answer this narrow question: What are the specific data access patterns and query requirements typical for OpenTelemetry span data (e.g., trace aggregation, time-range queries, filtering)? Report 2 bull", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "2465b30b775cc0f9", + "parent_span_id": "ba0687166794762c", + "name": "FilesystemMiddleware.wrap_model_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 13183.76704, + "duration_ms": 1530.689792, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "d417013a93db586b", + "parent_span_id": "2465b30b775cc0f9", + "name": "SummarizationMiddleware.wrap_model_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 13184.08704, + "duration_ms": 1530.317056, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "a47137f00b51744c", + "parent_span_id": "d417013a93db586b", + "name": "AnthropicPromptCachingMiddleware.wrap_model_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 13185.6, + "duration_ms": 1528.73984, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "33c55af3901c059f", + "parent_span_id": "a47137f00b51744c", + "name": "UnsupportedContentMiddleware.wrap_model_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 13185.821952, + "duration_ms": 1528.440832, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "2bb5a3398293f7cc", + "parent_span_id": "33c55af3901c059f", + "name": "ChatOpenAI", + "type": "llm", + "agent": "researcher", + "start_offset_ms": 13186.283008, + "duration_ms": 1527.795968, + "status": "ok", + "input_preview": "[{\"role\": \"system\", \"content\": \"Answer the question using search_docs. Reply with 2 short bullets.\"}, {\"role\": \"user\", \"content\": \"Research and answer this narrow question: What are the specific data access patterns and query requirements t", + "model": "claude-sonnet-4-5", + "input_tokens": 3114, + "output_tokens": 58, + "litellm": { + "request_id": "chatcmpl-46c76f65-1f14-43fd-a270-102c535f4306", + "model": "openai/claude-sonnet-4-5", + "model_group": "claude-sonnet-4-5", + "provider": "openai", + "key_alias": "research-bot", + "team_alias": "research-agents", + "spend": 0.00207645, + "prompt_tokens": 3114, + "completion_tokens": 58, + "cache_read_tokens": 3034, + "cache_write_tokens": 75, + "latency_ms": 1519, + "ttft_ms": 1519, + "status": "success" + } + }, + { + "span_id": "620a349bde8eeae1", + "parent_span_id": "d0e6fec2e9ef645f", + "name": "tools", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 13456.779776, + "duration_ms": 1.46432, + "status": "ok", + "input_preview": "{\"input\":[{\"name\":\"search_docs\",\"args\":{\"query\":\"Postgres high throughput write workload bottlenecks WAL autovacuum\"},\"id\":\"toolu_01MaMP4TQi5nppnzwyLFRj63\",\"type\":\"tool_call\"}]}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "41f7c2f062e8a80b", + "parent_span_id": "620a349bde8eeae1", + "name": "FilesystemMiddleware.wrap_tool_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 13457.309952, + "duration_ms": 0.667904, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "f458cf58f874d59a", + "parent_span_id": "41f7c2f062e8a80b", + "name": "search_docs", + "type": "tool", + "agent": "researcher", + "start_offset_ms": 13457.581824, + "duration_ms": 0.304128, + "status": "ok", + "input_preview": "{\"query\":\"Postgres high throughput write workload bottlenecks WAL autovacuum\"}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "0bee3eae4b1a5a9b", + "parent_span_id": "d0e6fec2e9ef645f", + "name": "model", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 13458.531072, + "duration_ms": 1662.05696, + "status": "ok", + "input_preview": "{\"messages\":[{\"content\":\"Research and answer this narrow question: What are the write performance characteristics and limitations of Postgres for high-volume time-series data ingestion at rates around 50k inserts/sec? Report 2 bullets about", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "ca15856d85c17d15", + "parent_span_id": "0bee3eae4b1a5a9b", + "name": "FilesystemMiddleware.wrap_model_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 13458.91072, + "duration_ms": 1661.483264, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "b62d3dd6fafc6bd2", + "parent_span_id": "ca15856d85c17d15", + "name": "SummarizationMiddleware.wrap_model_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 13459.238912, + "duration_ms": 1661.106944, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "578ab81ae856aec8", + "parent_span_id": "b62d3dd6fafc6bd2", + "name": "AnthropicPromptCachingMiddleware.wrap_model_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 13460.74496, + "duration_ms": 1659.54688, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "6ad32c56b2a6e93c", + "parent_span_id": "578ab81ae856aec8", + "name": "UnsupportedContentMiddleware.wrap_model_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 13460.962048, + "duration_ms": 1659.25504, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "87a70110c1b9ba4a", + "parent_span_id": "6ad32c56b2a6e93c", + "name": "ChatOpenAI", + "type": "llm", + "agent": "researcher", + "start_offset_ms": 13461.531904, + "duration_ms": 1658.518016, + "status": "ok", + "input_preview": "[{\"role\": \"system\", \"content\": \"Answer the question using search_docs. Reply with 2 short bullets.\"}, {\"role\": \"user\", \"content\": \"Research and answer this narrow question: What are the write performance characteristics and limitations of P", + "model": "claude-sonnet-4-5", + "input_tokens": 3188, + "output_tokens": 68, + "litellm": { + "request_id": "chatcmpl-a8cf94e1-5991-44b7-bedf-099ebf7865a6", + "model": "openai/claude-sonnet-4-5", + "model_group": "claude-sonnet-4-5", + "provider": "openai", + "key_alias": "research-bot", + "team_alias": "research-agents", + "spend": 0.0023487, + "prompt_tokens": 3188, + "completion_tokens": 68, + "cache_read_tokens": 3079, + "cache_write_tokens": 104, + "latency_ms": 1650, + "ttft_ms": 1650, + "status": "success" + } + }, + { + "span_id": "5ec46fcb9874c3f1", + "parent_span_id": "c2f4c93af99a664c", + "name": "tools", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 14596.340992, + "duration_ms": 2.127104, + "status": "ok", + "input_preview": "{\"input\":[{\"name\":\"search_docs\",\"args\":{\"query\":\"ClickHouse storage compression ratio LZ4 ZSTD column storage\"},\"id\":\"toolu_01PPB1nanXw7YTgoyJ4A2UUT\",\"type\":\"tool_call\"}]}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "e96cd58675e125a2", + "parent_span_id": "c2f4c93af99a664c", + "name": "tools", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 14596.64512, + "duration_ms": 3.00288, + "status": "ok", + "input_preview": "{\"input\":[{\"name\":\"search_docs\",\"args\":{\"query\":\"ClickHouse TTL data retention policy automatic deletion\"},\"id\":\"toolu_013HjnuDx4nHSW6MztW7Tg7q\",\"type\":\"tool_call\"}]}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "dfcf3f95f7122c63", + "parent_span_id": "c2f4c93af99a664c", + "name": "tools", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 14596.90496, + "duration_ms": 3.590144, + "status": "ok", + "input_preview": "{\"input\":[{\"name\":\"search_docs\",\"args\":{\"query\":\"Postgres storage size disk space observability time-series data\"},\"id\":\"toolu_01NGvQyJtnkBpwya3cY992A9\",\"type\":\"tool_call\"}]}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "912d3b24265d6516", + "parent_span_id": "5ec46fcb9874c3f1", + "name": "FilesystemMiddleware.wrap_tool_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 14597.38112, + "duration_ms": 0.86784, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "2da674074a3fb829", + "parent_span_id": "912d3b24265d6516", + "name": "search_docs", + "type": "tool", + "agent": "researcher", + "start_offset_ms": 14597.650944, + "duration_ms": 0.504064, + "status": "ok", + "input_preview": "{\"query\":\"ClickHouse storage compression ratio LZ4 ZSTD column storage\"}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "94864c972b7fbfa7", + "parent_span_id": "e96cd58675e125a2", + "name": "FilesystemMiddleware.wrap_tool_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 14598.811136, + "duration_ms": 0.539904, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "534787a0806803cb", + "parent_span_id": "94864c972b7fbfa7", + "name": "search_docs", + "type": "tool", + "agent": "researcher", + "start_offset_ms": 14599.01312, + "duration_ms": 0.265984, + "status": "ok", + "input_preview": "{\"query\":\"ClickHouse TTL data retention policy automatic deletion\"}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "e31ef2e09e031c06", + "parent_span_id": "dfcf3f95f7122c63", + "name": "FilesystemMiddleware.wrap_tool_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 14599.888896, + "duration_ms": 0.410112, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "d416a275d48fd41b", + "parent_span_id": "e31ef2e09e031c06", + "name": "search_docs", + "type": "tool", + "agent": "researcher", + "start_offset_ms": 14600.044032, + "duration_ms": 0.197888, + "status": "ok", + "input_preview": "{\"query\":\"Postgres storage size disk space observability time-series data\"}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "3dc6317b14b8b60b", + "parent_span_id": "c2f4c93af99a664c", + "name": "model", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 14600.783872, + "duration_ms": 2737.964032, + "status": "ok", + "input_preview": "{\"messages\":[{\"content\":\"Research and answer this narrow question: What are the storage efficiency and data retention considerations for ClickHouse versus Postgres when storing large volumes of observability/telemetry data? Report 2 bullets", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "c03864e0e6e0498b", + "parent_span_id": "3dc6317b14b8b60b", + "name": "FilesystemMiddleware.wrap_model_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 14601.085952, + "duration_ms": 2737.357056, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "e4f30d6cb14fd53b", + "parent_span_id": "c03864e0e6e0498b", + "name": "SummarizationMiddleware.wrap_model_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 14601.357824, + "duration_ms": 2737.00992, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "78906777d7f40bef", + "parent_span_id": "e4f30d6cb14fd53b", + "name": "AnthropicPromptCachingMiddleware.wrap_model_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 14602.676992, + "duration_ms": 2735.609088, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "453af7c9bf4368ba", + "parent_span_id": "78906777d7f40bef", + "name": "UnsupportedContentMiddleware.wrap_model_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 14602.868992, + "duration_ms": 2735.300096, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "93c56b7c2d751e44", + "parent_span_id": "453af7c9bf4368ba", + "name": "ChatOpenAI", + "type": "llm", + "agent": "researcher", + "start_offset_ms": 14603.279872, + "duration_ms": 2734.586112, + "status": "ok", + "input_preview": "[{\"role\": \"system\", \"content\": \"Answer the question using search_docs. Reply with 2 short bullets.\"}, {\"role\": \"user\", \"content\": \"Research and answer this narrow question: What are the storage efficiency and data retention considerations f", + "model": "claude-sonnet-4-5", + "input_tokens": 3638, + "output_tokens": 103, + "litellm": { + "request_id": "chatcmpl-5f57c315-3c80-41f2-9ecb-2dca0f03737b", + "model": "openai/claude-sonnet-4-5", + "model_group": "claude-sonnet-4-5", + "provider": "openai", + "key_alias": "research-bot", + "team_alias": "research-agents", + "spend": 0.0038144999999999997, + "prompt_tokens": 3638, + "completion_tokens": 103, + "cache_read_tokens": 3295, + "cache_write_tokens": 336, + "latency_ms": 2722, + "ttft_ms": 2722, + "status": "success" + } + }, + { + "span_id": "e9e68bf9e4c120ec", + "parent_span_id": "8d9dc2aaa6c8f3a3", + "name": "tools", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 14715.046144, + "duration_ms": 1.582848, + "status": "ok", + "input_preview": "{\"input\":[{\"name\":\"search_docs\",\"args\":{\"query\":\"trace aggregation time series data\"},\"id\":\"toolu_01USMJKAZMEvh5zpkAYBL4pd\",\"type\":\"tool_call\"}]}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "8758a1659a817971", + "parent_span_id": "e9e68bf9e4c120ec", + "name": "FilesystemMiddleware.wrap_tool_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 14715.649024, + "duration_ms": 0.72704, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "486548b06249d120", + "parent_span_id": "8758a1659a817971", + "name": "search_docs", + "type": "tool", + "agent": "researcher", + "start_offset_ms": 14715.938048, + "duration_ms": 0.336896, + "status": "ok", + "input_preview": "{\"query\":\"trace aggregation time series data\"}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "32756d86f68a95f8", + "parent_span_id": "8d9dc2aaa6c8f3a3", + "name": "model", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 14716.905984, + "duration_ms": 1941.266176, + "status": "ok", + "input_preview": "{\"messages\":[{\"content\":\"Research and answer this narrow question: What are the specific data access patterns and query requirements typical for OpenTelemetry span data (e.g., trace aggregation, time-range queries, filtering)? Report 2 bull", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "82913eb816195416", + "parent_span_id": "32756d86f68a95f8", + "name": "FilesystemMiddleware.wrap_model_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 14717.276928, + "duration_ms": 1940.743936, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "6952c979b8c040e2", + "parent_span_id": "82913eb816195416", + "name": "SummarizationMiddleware.wrap_model_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 14717.60896, + "duration_ms": 1940.370944, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "472f7491e326c68c", + "parent_span_id": "6952c979b8c040e2", + "name": "AnthropicPromptCachingMiddleware.wrap_model_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 14719.175936, + "duration_ms": 1938.759168, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "0ce2ac29e47d07fb", + "parent_span_id": "472f7491e326c68c", + "name": "UnsupportedContentMiddleware.wrap_model_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 14719.395072, + "duration_ms": 1938.475008, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "f6db3994aabaf84e", + "parent_span_id": "0ce2ac29e47d07fb", + "name": "ChatOpenAI", + "type": "llm", + "agent": "researcher", + "start_offset_ms": 14720.00896, + "duration_ms": 1937.707008, + "status": "ok", + "input_preview": "[{\"role\": \"system\", \"content\": \"Answer the question using search_docs. Reply with 2 short bullets.\"}, {\"role\": \"user\", \"content\": \"Research and answer this narrow question: What are the specific data access patterns and query requirements t", + "model": "claude-sonnet-4-5", + "input_tokens": 3186, + "output_tokens": 57, + "litellm": { + "request_id": "chatcmpl-6d7b2d42-6476-46c2-b07f-b2cff372cc51", + "model": "openai/claude-sonnet-4-5", + "model_group": "claude-sonnet-4-5", + "provider": "openai", + "key_alias": "research-bot", + "team_alias": "research-agents", + "spend": 0.0020727000000000002, + "prompt_tokens": 3186, + "completion_tokens": 57, + "cache_read_tokens": 3109, + "cache_write_tokens": 72, + "latency_ms": 1930, + "ttft_ms": 1930, + "status": "success" + } + }, + { + "span_id": "378ba6e4ded8b83f", + "parent_span_id": "d0e6fec2e9ef645f", + "name": "tools", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 15120.956928, + "duration_ms": 1.57696, + "status": "ok", + "input_preview": "{\"input\":[{\"name\":\"search_docs\",\"args\":{\"query\":\"TimescaleDB Postgres time series performance optimization partitioning hypertables\"},\"id\":\"toolu_01JU1EhmJ5jQgDAvS9DhG8Yk\",\"type\":\"tool_call\"}]}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "22e75b96f36ac970", + "parent_span_id": "378ba6e4ded8b83f", + "name": "FilesystemMiddleware.wrap_tool_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 15121.536768, + "duration_ms": 0.732416, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "32f6c2e979c945ad", + "parent_span_id": "22e75b96f36ac970", + "name": "search_docs", + "type": "tool", + "agent": "researcher", + "start_offset_ms": 15121.831936, + "duration_ms": 0.33792, + "status": "ok", + "input_preview": "{\"query\":\"TimescaleDB Postgres time series performance optimization partitioning hypertables\"}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "76e48aa2e94e4608", + "parent_span_id": "d0e6fec2e9ef645f", + "name": "model", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 15122.853888, + "duration_ms": 4030.482176, + "status": "ok", + "input_preview": "{\"messages\":[{\"content\":\"Research and answer this narrow question: What are the write performance characteristics and limitations of Postgres for high-volume time-series data ingestion at rates around 50k inserts/sec? Report 2 bullets about", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "5b4eb8c646e5ea38", + "parent_span_id": "76e48aa2e94e4608", + "name": "FilesystemMiddleware.wrap_model_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 15123.289088, + "duration_ms": 4029.922048, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "ecbf621d117d41ef", + "parent_span_id": "5b4eb8c646e5ea38", + "name": "SummarizationMiddleware.wrap_model_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 15123.646976, + "duration_ms": 4029.532928, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "8d411a01f0f40e17", + "parent_span_id": "ecbf621d117d41ef", + "name": "AnthropicPromptCachingMiddleware.wrap_model_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 15125.184, + "duration_ms": 4027.95904, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "82c68ac8749083a6", + "parent_span_id": "8d411a01f0f40e17", + "name": "UnsupportedContentMiddleware.wrap_model_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 15125.409024, + "duration_ms": 4027.684096, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "ca5f50b3b8879759", + "parent_span_id": "82c68ac8749083a6", + "name": "ChatOpenAI", + "type": "llm", + "agent": "researcher", + "start_offset_ms": 15125.856, + "duration_ms": 4027.105024, + "status": "ok", + "input_preview": "[{\"role\": \"system\", \"content\": \"Answer the question using search_docs. Reply with 2 short bullets.\"}, {\"role\": \"user\", \"content\": \"Research and answer this narrow question: What are the write performance characteristics and limitations of P", + "model": "claude-sonnet-4-5", + "input_tokens": 3290, + "output_tokens": 124, + "litellm": { + "request_id": "chatcmpl-c86e1321-87e9-4599-a46e-e23dd0005e78", + "model": "openai/claude-sonnet-4-5", + "model_group": "claude-sonnet-4-5", + "provider": "openai", + "key_alias": "research-bot", + "team_alias": "research-agents", + "spend": 0.0032124000000000002, + "prompt_tokens": 3290, + "completion_tokens": 124, + "cache_read_tokens": 3183, + "cache_write_tokens": 102, + "latency_ms": 4019, + "ttft_ms": 4019, + "status": "success" + } + }, + { + "span_id": "2122f08271342ae9", + "parent_span_id": "8d9dc2aaa6c8f3a3", + "name": "tools", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 16658.482944, + "duration_ms": 1.293056, + "status": "ok", + "input_preview": "{\"input\":[{\"name\":\"search_docs\",\"args\":{\"query\":\"span attributes filtering time range\"},\"id\":\"toolu_01AdKnUtJyMFY6vE1Q44ZmF7\",\"type\":\"tool_call\"}]}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "c0c1df069970a064", + "parent_span_id": "2122f08271342ae9", + "name": "FilesystemMiddleware.wrap_tool_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 16658.987008, + "duration_ms": 0.587008, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "1a0cea88f0e7fcab", + "parent_span_id": "c0c1df069970a064", + "name": "search_docs", + "type": "tool", + "agent": "researcher", + "start_offset_ms": 16659.219968, + "duration_ms": 0.271104, + "status": "ok", + "input_preview": "{\"query\":\"span attributes filtering time range\"}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "a7abcb9eb548822e", + "parent_span_id": "8d9dc2aaa6c8f3a3", + "name": "model", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 16659.99104, + "duration_ms": 2872.426752, + "status": "ok", + "input_preview": "{\"messages\":[{\"content\":\"Research and answer this narrow question: What are the specific data access patterns and query requirements typical for OpenTelemetry span data (e.g., trace aggregation, time-range queries, filtering)? Report 2 bull", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "4905e77b56824800", + "parent_span_id": "a7abcb9eb548822e", + "name": "FilesystemMiddleware.wrap_model_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 16660.3072, + "duration_ms": 2871.8528, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "8c5744fb3a7b0b0b", + "parent_span_id": "4905e77b56824800", + "name": "SummarizationMiddleware.wrap_model_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 16660.57088, + "duration_ms": 2871.527168, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "d6c5eb192e2e391f", + "parent_span_id": "8c5744fb3a7b0b0b", + "name": "AnthropicPromptCachingMiddleware.wrap_model_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 16661.803776, + "duration_ms": 2870.225408, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "ce347268a1f62ffb", + "parent_span_id": "d6c5eb192e2e391f", + "name": "UnsupportedContentMiddleware.wrap_model_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 16662.00192, + "duration_ms": 2869.926144, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "a2c15c6ccb466632", + "parent_span_id": "ce347268a1f62ffb", + "name": "ChatOpenAI", + "type": "llm", + "agent": "researcher", + "start_offset_ms": 16662.38208, + "duration_ms": 2869.247744, + "status": "ok", + "input_preview": "[{\"role\": \"system\", \"content\": \"Answer the question using search_docs. Reply with 2 short bullets.\"}, {\"role\": \"user\", \"content\": \"Research and answer this narrow question: What are the specific data access patterns and query requirements t", + "model": "claude-sonnet-4-5", + "input_tokens": 3257, + "output_tokens": 124, + "litellm": { + "request_id": "chatcmpl-37698b76-1b34-4414-adf1-fb9cdc40765e", + "model": "openai/claude-sonnet-4-5", + "model_group": "claude-sonnet-4-5", + "provider": "openai", + "key_alias": "research-bot", + "team_alias": "research-agents", + "spend": 0.00309555, + "prompt_tokens": 3257, + "completion_tokens": 124, + "cache_read_tokens": 3181, + "cache_write_tokens": 71, + "latency_ms": 2859, + "ttft_ms": 2859, + "status": "success" + } + }, + { + "span_id": "2e507cfbc4fe5c42", + "parent_span_id": "c2f4c93af99a664c", + "name": "tools", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 17339.544064, + "duration_ms": 3.478016, + "status": "ok", + "input_preview": "{\"input\":[{\"name\":\"search_docs\",\"args\":{\"query\":\"column-oriented database compression benefits storage savings\"},\"id\":\"toolu_01FshP24x5QcUUgJ8J4V6Mff\",\"type\":\"tool_call\"}]}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "a313420c8b250d41", + "parent_span_id": "2e507cfbc4fe5c42", + "name": "FilesystemMiddleware.wrap_tool_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 17340.464128, + "duration_ms": 0.952832, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "35761025656d39a1", + "parent_span_id": "a313420c8b250d41", + "name": "search_docs", + "type": "tool", + "agent": "researcher", + "start_offset_ms": 17340.865024, + "duration_ms": 0.422144, + "status": "ok", + "input_preview": "{\"query\":\"column-oriented database compression benefits storage savings\"}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "f85b480c8efa8f56", + "parent_span_id": "c2f4c93af99a664c", + "name": "tools", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 17341.547776, + "duration_ms": 1.922304, + "status": "ok", + "input_preview": "{\"input\":[{\"name\":\"search_docs\",\"args\":{\"query\":\"time-series database storage efficiency retention management\"},\"id\":\"toolu_01U9JwQstrUzHJFWYA6PJah1\",\"type\":\"tool_call\"}]}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "1b4facb81cfb56fe", + "parent_span_id": "f85b480c8efa8f56", + "name": "FilesystemMiddleware.wrap_tool_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 17342.16576, + "duration_ms": 0.634368, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "48d97fb6a419e1d6", + "parent_span_id": "1b4facb81cfb56fe", + "name": "search_docs", + "type": "tool", + "agent": "researcher", + "start_offset_ms": 17342.4128, + "duration_ms": 0.300288, + "status": "ok", + "input_preview": "{\"query\":\"time-series database storage efficiency retention management\"}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "2d1062cce334c01b", + "parent_span_id": "c2f4c93af99a664c", + "name": "model", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 17343.940864, + "duration_ms": 3921.300224, + "status": "ok", + "input_preview": "{\"messages\":[{\"content\":\"Research and answer this narrow question: What are the storage efficiency and data retention considerations for ClickHouse versus Postgres when storing large volumes of observability/telemetry data? Report 2 bullets", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "6a88e16cbd043f5e", + "parent_span_id": "2d1062cce334c01b", + "name": "FilesystemMiddleware.wrap_model_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 17344.491008, + "duration_ms": 3920.632064, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "d94f78d8de8ab127", + "parent_span_id": "6a88e16cbd043f5e", + "name": "SummarizationMiddleware.wrap_model_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 17344.94976, + "duration_ms": 3920.13824, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "e35e83849f3a70e9", + "parent_span_id": "d94f78d8de8ab127", + "name": "AnthropicPromptCachingMiddleware.wrap_model_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 17347.136, + "duration_ms": 3917.87392, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "738d993abf7b7974", + "parent_span_id": "e35e83849f3a70e9", + "name": "UnsupportedContentMiddleware.wrap_model_call", + "type": "framework", + "agent": "researcher", + "start_offset_ms": 17347.393792, + "duration_ms": 3917.497344, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "a1e9092a218f684e", + "parent_span_id": "738d993abf7b7974", + "name": "ChatOpenAI", + "type": "llm", + "agent": "researcher", + "start_offset_ms": 17347.923968, + "duration_ms": 3916.752128, + "status": "ok", + "input_preview": "[{\"role\": \"system\", \"content\": \"Answer the question using search_docs. Reply with 2 short bullets.\"}, {\"role\": \"user\", \"content\": \"Research and answer this narrow question: What are the storage efficiency and data retention considerations f", + "model": "claude-sonnet-4-5", + "input_tokens": 3808, + "output_tokens": 148, + "litellm": { + "request_id": "chatcmpl-d9fb74a4-58ad-4ae3-9d0b-21b06c6b5145", + "model": "openai/claude-sonnet-4-5", + "model_group": "claude-sonnet-4-5", + "provider": "openai", + "key_alias": "research-bot", + "team_alias": "research-agents", + "spend": 0.0039678000000000005, + "prompt_tokens": 3808, + "completion_tokens": 148, + "cache_read_tokens": 3631, + "cache_write_tokens": 170, + "latency_ms": 3906, + "ttft_ms": 3906, + "status": "success" + } + }, + { + "span_id": "7d729d7fe5527e09", + "parent_span_id": "3586edf49d446541", + "name": "model", + "type": "chain", + "agent": "research_lead", + "start_offset_ms": 21266.221056, + "duration_ms": 8096.979968, + "status": "ok", + "input_preview": "{\"messages\":[{\"content\":\"Should we store OTEL agent spans in ClickHouse or Postgres at 50k spans/sec?\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"61d4ee17-48af-45b2-b02d-0b78aa43a534\"},{\"content\":\"I'll help you deter", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "e1c1e76ef490f593", + "parent_span_id": "7d729d7fe5527e09", + "name": "FilesystemMiddleware.wrap_model_call", + "type": "framework", + "agent": "research_lead", + "start_offset_ms": 21266.488832, + "duration_ms": 8096.516352, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "5414fd0da858774d", + "parent_span_id": "e1c1e76ef490f593", + "name": "SubAgentMiddleware.wrap_model_call", + "type": "framework", + "agent": "research_lead", + "start_offset_ms": 21266.683904, + "duration_ms": 8096.268032, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "dda7d2699a1778df", + "parent_span_id": "5414fd0da858774d", + "name": "SummarizationMiddleware.wrap_model_call", + "type": "framework", + "agent": "research_lead", + "start_offset_ms": 21266.811904, + "duration_ms": 8096.086016, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "d7159bb909d27803", + "parent_span_id": "dda7d2699a1778df", + "name": "AnthropicPromptCachingMiddleware.wrap_model_call", + "type": "framework", + "agent": "research_lead", + "start_offset_ms": 21267.846912, + "duration_ms": 8094.988288, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "036bad1c98591c3b", + "parent_span_id": "d7159bb909d27803", + "name": "UnsupportedContentMiddleware.wrap_model_call", + "type": "framework", + "agent": "research_lead", + "start_offset_ms": 21267.974912, + "duration_ms": 8094.769152, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "a7c9fedd896e4ddc", + "parent_span_id": "036bad1c98591c3b", + "name": "ChatOpenAI", + "type": "llm", + "agent": "research_lead", + "start_offset_ms": 21268.27392, + "duration_ms": 8094.284032, + "status": "ok", + "input_preview": "[{\"role\": \"system\", \"content\": \"You are a research lead. Split the question into exactly 4 narrow sub-questions and delegate EACH one to the researcher subagent via task, in parallel. Then write a 3-sentence draft, send it to the critic sub", + "model": "claude-sonnet-4-5", + "input_tokens": 4492, + "output_tokens": 388, + "litellm": { + "request_id": "chatcmpl-6ca81bcb-a605-4067-87df-8b2ba13d34be", + "model": "openai/claude-sonnet-4-5", + "model_group": "claude-sonnet-4-5", + "provider": "openai", + "key_alias": "research-bot", + "team_alias": "research-agents", + "spend": 0.011016000000000001, + "prompt_tokens": 4492, + "completion_tokens": 388, + "cache_read_tokens": 3375, + "cache_write_tokens": 1110, + "latency_ms": 8086, + "ttft_ms": 8086, + "status": "success" + } + }, + { + "span_id": "96021b36c22e3de5", + "parent_span_id": "3586edf49d446541", + "name": "tools", + "type": "chain", + "agent": "research_lead", + "start_offset_ms": 29363.598848, + "duration_ms": 6899.43424, + "status": "ok", + "input_preview": "{\"input\":[{\"name\":\"task\",\"args\":{\"subagent_type\":\"critic\",\"description\":\"Review this draft recommendation and list up to 2 weaknesses:\\n\\n\\\"ClickHouse is the clear choice for storing OTEL spans at 50k spans/sec, as it can handle 1M+ rows/se", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "ec41b5953e07ddf6", + "parent_span_id": "96021b36c22e3de5", + "name": "FilesystemMiddleware.wrap_tool_call", + "type": "framework", + "agent": "research_lead", + "start_offset_ms": 29364.22016, + "duration_ms": 6898.466816, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "aea43e4e80c09950", + "parent_span_id": "ec41b5953e07ddf6", + "name": "task", + "type": "tool", + "agent": "research_lead", + "start_offset_ms": 29364.556032, + "duration_ms": 6897.913856, + "status": "ok", + "input_preview": "{\"subagent_type\":\"critic\",\"description\":\"Review this draft recommendation and list up to 2 weaknesses:\\n\\n\\\"ClickHouse is the clear choice for storing OTEL spans at 50k spans/sec, as it can handle 1M+ rows/second per node with batched inser", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "db4e1f0a6675441c", + "parent_span_id": "aea43e4e80c09950", + "name": "critic", + "type": "agent", + "agent": "critic", + "start_offset_ms": 29365.08288, + "duration_ms": 6897.236224, + "status": "ok", + "input_preview": "[{\"role\": \"user\", \"content\": \"Review this draft recommendation and list up to 2 weaknesses:\\n\\n\\\"ClickHouse is the clear choice for storing OTEL spans at 50k spans/sec, as it can handle 1M+ rows/second per node with batched inserts, while s", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "aecf500a40176e62", + "parent_span_id": "db4e1f0a6675441c", + "name": "PatchToolCallsMiddleware.before_agent", + "type": "framework", + "agent": "critic", + "start_offset_ms": 29365.89312, + "duration_ms": 0.16768, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "8776ee8a5ec58f7a", + "parent_span_id": "db4e1f0a6675441c", + "name": "model", + "type": "chain", + "agent": "critic", + "start_offset_ms": 29366.297856, + "duration_ms": 6895.67616, + "status": "ok", + "input_preview": "{\"messages\":[{\"content\":\"Review this draft recommendation and list up to 2 weaknesses:\\n\\n\\\"ClickHouse is the clear choice for storing OTEL spans at 50k spans/sec, as it can handle 1M+ rows/second per node with batched inserts, while standa", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "100d096f30b12ba7", + "parent_span_id": "8776ee8a5ec58f7a", + "name": "FilesystemMiddleware.wrap_model_call", + "type": "framework", + "agent": "critic", + "start_offset_ms": 29366.657024, + "duration_ms": 6895.030784, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "d562a157f40ce8ec", + "parent_span_id": "100d096f30b12ba7", + "name": "SummarizationMiddleware.wrap_model_call", + "type": "framework", + "agent": "critic", + "start_offset_ms": 29367.105024, + "duration_ms": 6894.512896, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "3539d8541097d392", + "parent_span_id": "d562a157f40ce8ec", + "name": "AnthropicPromptCachingMiddleware.wrap_model_call", + "type": "framework", + "agent": "critic", + "start_offset_ms": 29373.202944, + "duration_ms": 6888.335104, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "2b1e5b780d6e0868", + "parent_span_id": "3539d8541097d392", + "name": "UnsupportedContentMiddleware.wrap_model_call", + "type": "framework", + "agent": "critic", + "start_offset_ms": 29373.569024, + "duration_ms": 6887.84896, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "7b0fd995de99bb3a", + "parent_span_id": "2b1e5b780d6e0868", + "name": "ChatOpenAI", + "type": "llm", + "agent": "critic", + "start_offset_ms": 29374.034944, + "duration_ms": 6887.087872, + "status": "ok", + "input_preview": "[{\"role\": \"system\", \"content\": \"Critique the draft in <=2 bullets. Do not use tools.\"}, {\"role\": \"user\", \"content\": \"Review this draft recommendation and list up to 2 weaknesses:\\n\\n\\\"ClickHouse is the clear choice for storing OTEL spans at", + "model": "claude-sonnet-4-5", + "input_tokens": 2926, + "output_tokens": 162, + "litellm": { + "request_id": "chatcmpl-51dc714f-2a85-4f89-aa60-d30e5ea3aff3", + "model": "openai/claude-sonnet-4-5", + "model_group": "claude-sonnet-4-5", + "provider": "openai", + "key_alias": "research-bot", + "team_alias": "research-agents", + "spend": 0.01340025, + "prompt_tokens": 2926, + "completion_tokens": 162, + "cache_read_tokens": 0, + "cache_write_tokens": 2923, + "latency_ms": 6876, + "ttft_ms": 6876, + "status": "success" + } + }, + { + "span_id": "8693e8571891da82", + "parent_span_id": "3586edf49d446541", + "name": "model", + "type": "chain", + "agent": "research_lead", + "start_offset_ms": 36263.529984, + "duration_ms": 3934.251008, + "status": "ok", + "input_preview": "{\"messages\":[{\"content\":\"Should we store OTEL agent spans in ClickHouse or Postgres at 50k spans/sec?\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"61d4ee17-48af-45b2-b02d-0b78aa43a534\"},{\"content\":\"I'll help you deter", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "e8a056a3f5fa8064", + "parent_span_id": "8693e8571891da82", + "name": "FilesystemMiddleware.wrap_model_call", + "type": "framework", + "agent": "research_lead", + "start_offset_ms": 36264.162816, + "duration_ms": 3933.501952, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "ff113c9de01aa964", + "parent_span_id": "e8a056a3f5fa8064", + "name": "SubAgentMiddleware.wrap_model_call", + "type": "framework", + "agent": "research_lead", + "start_offset_ms": 36264.601856, + "duration_ms": 3933.031168, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "bcdae244a30669fe", + "parent_span_id": "ff113c9de01aa964", + "name": "SummarizationMiddleware.wrap_model_call", + "type": "framework", + "agent": "research_lead", + "start_offset_ms": 36264.86016, + "duration_ms": 3932.74496, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "17958e38044e83b1", + "parent_span_id": "bcdae244a30669fe", + "name": "AnthropicPromptCachingMiddleware.wrap_model_call", + "type": "framework", + "agent": "research_lead", + "start_offset_ms": 36266.764032, + "duration_ms": 3930.806016, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "8e18dafd8aaf0af4", + "parent_span_id": "17958e38044e83b1", + "name": "UnsupportedContentMiddleware.wrap_model_call", + "type": "framework", + "agent": "research_lead", + "start_offset_ms": 36267.02208, + "duration_ms": 3930.494976, + "status": "ok", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "4942936a6ea8b578", + "parent_span_id": "8e18dafd8aaf0af4", + "name": "ChatOpenAI", + "type": "llm", + "agent": "research_lead", + "start_offset_ms": 36267.64288, + "duration_ms": 3929.739008, + "status": "ok", + "input_preview": "[{\"role\": \"system\", \"content\": \"You are a research lead. Split the question into exactly 4 narrow sub-questions and delegate EACH one to the researcher subagent via task, in parallel. Then write a 3-sentence draft, send it to the critic sub", + "model": "claude-sonnet-4-5", + "input_tokens": 5050, + "output_tokens": 170, + "litellm": { + "request_id": "chatcmpl-7531fb51-2940-4e7a-9cf4-ff071e171f6b", + "model": "openai/claude-sonnet-4-5", + "model_group": "claude-sonnet-4-5", + "provider": "openai", + "key_alias": "research-bot", + "team_alias": "research-agents", + "spend": 0.0060105, + "prompt_tokens": 5050, + "completion_tokens": 170, + "cache_read_tokens": 4485, + "cache_write_tokens": 560, + "latency_ms": 3920, + "ttft_ms": 3920, + "status": "success" + } + } + ] +} diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/__fixtures__/swarm_trace.json b/ui/litellm-dashboard/src/components/view_logs/TraceView/__fixtures__/swarm_trace.json new file mode 100644 index 000000000000..9a3cee6924e5 --- /dev/null +++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/__fixtures__/swarm_trace.json @@ -0,0 +1,3960 @@ +{ + "summary": { + "trace_id": "0602f23f8fa5d3a2c1a12521739ca866", + "name": "orchestrator", + "service": "swarm-orchestrator", + "input_preview": "", + "start_time": "2026-09-30T06:48:02.550000+00:00", + "duration_ms": 17145.690112, + "status": "ok", + "span_count": 207, + "agent_count": 4, + "llm_calls": 53, + "tool_calls": 45, + "error_count": 23, + "input_tokens": 42351, + "output_tokens": 4356, + "spend": 0.065457, + "models": ["claude-haiku-4-5", "claude-sonnet-4-5"] + }, + "agents": [ + { + "name": "orchestrator", + "parent_agent": null, + "invocations": 1, + "llm_calls": 0, + "tool_calls": 0, + "spend": 0.0, + "duration_ms": 17145.690112 + }, + { + "name": "researcher", + "parent_agent": "orchestrator", + "invocations": 12, + "llm_calls": 40, + "tool_calls": 35, + "spend": 0.049728, + "duration_ms": 43891.366656 + }, + { + "name": "fact_checker", + "parent_agent": "researcher", + "invocations": 2, + "llm_calls": 12, + "tool_calls": 10, + "spend": 0.013739999999999999, + "duration_ms": 11556.862976 + }, + { + "name": "critic", + "parent_agent": "orchestrator", + "invocations": 1, + "llm_calls": 1, + "tool_calls": 0, + "spend": 0.0019890000000000003, + "duration_ms": 5460.011008 + } + ], + "spans": [ + { + "span_id": "b7b1053f6c95fd70", + "parent_span_id": null, + "name": "orchestrator", + "type": "agent", + "agent": "orchestrator", + "start_offset_ms": 0.0, + "duration_ms": 17145.690112, + "status": "ok", + "input_preview": "", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "ffb1aafc123f7cbf", + "parent_span_id": "b7b1053f6c95fd70", + "name": "researcher", + "type": "agent", + "agent": "researcher", + "start_offset_ms": 54.492928, + "duration_ms": 8857.8112, + "status": "ok", + "input_preview": "[{\"role\": \"user\", \"content\": \"Compare ingest throughput (question 0).\"}]", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "f977e58a340dbc6b", + "parent_span_id": "ffb1aafc123f7cbf", + "name": "model", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 55.320064, + "duration_ms": 1161.488896, + "status": "ok", + "input_preview": "{\"messages\":[{\"content\":\"Compare ingest throughput (question 0).\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"8bd0af23-8ed1-4178-aff9-7a5763b1c928\"}]}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "03f611cf9bfc9ea4", + "parent_span_id": "b7b1053f6c95fd70", + "name": "researcher", + "type": "agent", + "agent": "researcher", + "start_offset_ms": 55.501056, + "duration_ms": 3226.066944, + "status": "ok", + "input_preview": "[{\"role\": \"user\", \"content\": \"Compare multi-tenancy (question 4).\"}]", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "9d124d6bce3185dd", + "parent_span_id": "03f611cf9bfc9ea4", + "name": "model", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 55.905024, + "duration_ms": 1136.644096, + "status": "ok", + "input_preview": "{\"messages\":[{\"content\":\"Compare multi-tenancy (question 4).\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"6f39cb28-7f23-4cd1-b74a-98eb05d175fd\"}]}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "6fbea322f87ea515", + "parent_span_id": "b7b1053f6c95fd70", + "name": "researcher", + "type": "agent", + "agent": "researcher", + "start_offset_ms": 56.007168, + "duration_ms": 3131.008, + "status": "ok", + "input_preview": "[{\"role\": \"user\", \"content\": \"Compare compression (question 5).\"}]", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "2714afb89a510e4a", + "parent_span_id": "b7b1053f6c95fd70", + "name": "researcher", + "type": "agent", + "agent": "researcher", + "start_offset_ms": 56.350976, + "duration_ms": 271.669248, + "status": "ok", + "input_preview": "[{\"role\": \"user\", \"content\": \"Compare backups (question 7).\"}]", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "d23937c848659551", + "parent_span_id": "2714afb89a510e4a", + "name": "model", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 56.684032, + "duration_ms": 192.15616, + "status": "ok", + "input_preview": "{\"messages\":[{\"content\":\"Compare backups (question 7).\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"ae6337ae-8401-4264-a955-836b6b09c7a3\"}]}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "f060fb4662d99991", + "parent_span_id": "b7b1053f6c95fd70", + "name": "researcher", + "type": "agent", + "agent": "researcher", + "start_offset_ms": 56.779264, + "duration_ms": 1167.341824, + "status": "ok", + "input_preview": "[{\"role\": \"user\", \"content\": \"Compare storage cost (question 1).\"}]", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "6ba3ba633024d9e4", + "parent_span_id": "f060fb4662d99991", + "name": "model", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 57.080064, + "duration_ms": 1166.923008, + "status": "ok", + "input_preview": "{\"messages\":[{\"content\":\"Compare storage cost (question 1).\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"01a2d57c-f807-42eb-bda0-ed3b3399a41b\"}]}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "93e72d093e644a1d", + "parent_span_id": "6fbea322f87ea515", + "name": "model", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 57.244928, + "duration_ms": 1178.156288, + "status": "ok", + "input_preview": "{\"messages\":[{\"content\":\"Compare compression (question 5).\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"2701c52d-8721-4a2f-9673-e46179f6a853\"}]}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "4bc4c26013a33115", + "parent_span_id": "b7b1053f6c95fd70", + "name": "researcher", + "type": "agent", + "agent": "researcher", + "start_offset_ms": 57.316096, + "duration_ms": 11627.85792, + "status": "ok", + "input_preview": "[{\"role\": \"user\", \"content\": \"Compare ingest throughput (question 10).\"}]", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "ebd769f68fb432b3", + "parent_span_id": "4bc4c26013a33115", + "name": "model", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 57.57312, + "duration_ms": 1486.716928, + "status": "ok", + "input_preview": "{\"messages\":[{\"content\":\"Compare ingest throughput (question 10).\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"c93e34d9-26a7-4f16-af29-04ad300f0905\"}]}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "71903d779d129c68", + "parent_span_id": "b7b1053f6c95fd70", + "name": "researcher", + "type": "agent", + "agent": "researcher", + "start_offset_ms": 57.648128, + "duration_ms": 2114.355968, + "status": "ok", + "input_preview": "[{\"role\": \"user\", \"content\": \"Compare query latency (question 2).\"}]", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "b33e0f6b6f8a6ef0", + "parent_span_id": "71903d779d129c68", + "name": "model", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 57.89312, + "duration_ms": 188.8128, + "status": "ok", + "input_preview": "{\"messages\":[{\"content\":\"Compare query latency (question 2).\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"c9cf9a10-ff05-4215-8626-9fa60808ae97\"}]}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "a1e8b8b06038a09e", + "parent_span_id": "b7b1053f6c95fd70", + "name": "researcher", + "type": "agent", + "agent": "researcher", + "start_offset_ms": 57.960192, + "duration_ms": 3083.454976, + "status": "ok", + "input_preview": "[{\"role\": \"user\", \"content\": \"Compare storage cost (question 11).\"}]", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "8db5b2b7871b58a6", + "parent_span_id": "a1e8b8b06038a09e", + "name": "model", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 58.30016, + "duration_ms": 1105.966848, + "status": "ok", + "input_preview": "{\"messages\":[{\"content\":\"Compare storage cost (question 11).\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"6c39cf1d-8ce5-459d-96a6-e2f6ba6fccfe\"}]}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "41fa112865f7b1f5", + "parent_span_id": "b7b1053f6c95fd70", + "name": "researcher", + "type": "agent", + "agent": "researcher", + "start_offset_ms": 58.381056, + "duration_ms": 1185.950976, + "status": "ok", + "input_preview": "[{\"role\": \"user\", \"content\": \"Compare replication (question 6).\"}]", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "be4ac7802a570aa4", + "parent_span_id": "41fa112865f7b1f5", + "name": "model", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 58.62912, + "duration_ms": 193.360896, + "status": "ok", + "input_preview": "{\"messages\":[{\"content\":\"Compare replication (question 6).\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"1b0c8ffc-23f5-430f-83e8-c3a28091d5b7\"}]}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "042d555505c3a65a", + "parent_span_id": "b7b1053f6c95fd70", + "name": "researcher", + "type": "agent", + "agent": "researcher", + "start_offset_ms": 58.695168, + "duration_ms": 3089.047808, + "status": "ok", + "input_preview": "[{\"role\": \"user\", \"content\": \"Compare schema changes (question 8).\"}]", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "3e4cf4753e6183c1", + "parent_span_id": "042d555505c3a65a", + "name": "model", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 58.941184, + "duration_ms": 1278.447872, + "status": "ok", + "input_preview": "{\"messages\":[{\"content\":\"Compare schema changes (question 8).\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"4558e9f1-9a1e-4c7b-9df5-14e457ad4f82\"}]}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "49397f8d7856d2ba", + "parent_span_id": "b7b1053f6c95fd70", + "name": "researcher", + "type": "agent", + "agent": "researcher", + "start_offset_ms": 59.060224, + "duration_ms": 3135.371776, + "status": "ok", + "input_preview": "[{\"role\": \"user\", \"content\": \"Compare retention (question 3).\"}]", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "991c19abc6a2001c", + "parent_span_id": "49397f8d7856d2ba", + "name": "model", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 59.379968, + "duration_ms": 1142.242048, + "status": "ok", + "input_preview": "{\"messages\":[{\"content\":\"Compare retention (question 3).\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"c672c207-0adb-458a-abd5-e12496da0886\"}]}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "d9786287209602ae", + "parent_span_id": "f977e58a340dbc6b", + "name": "ChatOpenAI", + "type": "llm", + "agent": "researcher", + "start_offset_ms": 60.77312, + "duration_ms": 1155.918848, + "status": "ok", + "input_preview": "[{\"role\": \"system\", \"content\": \"Call lookup_benchmark once for ClickHouse and once for Postgres, then call check_fact on your conclusion, then answer in ONE short sentence.\"}, {\"role\": \"user\", \"content\": \"Compare ingest throughput (question", + "model": "claude-haiku-4-5", + "input_tokens": 676, + "output_tokens": 120, + "litellm": { + "request_id": "chatcmpl-f4447165-878b-4bf3-ae07-0a43b98482e9", + "model": "openai/claude-haiku-4-5-20251001", + "model_group": "claude-haiku-4-5", + "provider": "openai", + "key_alias": "research-bot", + "team_alias": "research-agents", + "spend": 0.001276, + "prompt_tokens": 676, + "completion_tokens": 120, + "cache_read_tokens": 0, + "cache_write_tokens": 0, + "latency_ms": 1107, + "ttft_ms": 1107, + "status": "success" + } + }, + { + "span_id": "8a556ee09900bc3c", + "parent_span_id": "b7b1053f6c95fd70", + "name": "researcher", + "type": "agent", + "agent": "researcher", + "start_offset_ms": 60.95104, + "duration_ms": 3001.430016, + "status": "ok", + "input_preview": "[{\"role\": \"user\", \"content\": \"Compare joins (question 9).\"}]", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "5afec084c27fe7e2", + "parent_span_id": "8a556ee09900bc3c", + "name": "model", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 61.202176, + "duration_ms": 1148.244992, + "status": "ok", + "input_preview": "{\"messages\":[{\"content\":\"Compare joins (question 9).\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"83bc8232-0b19-43d3-9a5e-6d78a5d4c6b5\"}]}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "8a62792e0140b33c", + "parent_span_id": "5afec084c27fe7e2", + "name": "ChatOpenAI", + "type": "llm", + "agent": "researcher", + "start_offset_ms": 61.479168, + "duration_ms": 1147.847936, + "status": "ok", + "input_preview": "[{\"role\": \"system\", \"content\": \"Call lookup_benchmark once for ClickHouse and once for Postgres, then answer in ONE short sentence.\"}, {\"role\": \"user\", \"content\": \"Compare joins (question 9).\"}]", + "model": "claude-haiku-4-5", + "input_tokens": 604, + "output_tokens": 120, + "litellm": { + "request_id": "chatcmpl-74c4de8a-56f5-41f9-8b61-ed64736ac886", + "model": "openai/claude-haiku-4-5-20251001", + "model_group": "claude-haiku-4-5", + "provider": "openai", + "key_alias": "research-bot", + "team_alias": "research-agents", + "spend": 0.001204, + "prompt_tokens": 604, + "completion_tokens": 120, + "cache_read_tokens": 0, + "cache_write_tokens": 0, + "latency_ms": 1093, + "ttft_ms": 1093, + "status": "success" + } + }, + { + "span_id": "61f05341d2eddb8a", + "parent_span_id": "9d124d6bce3185dd", + "name": "ChatOpenAI", + "type": "llm", + "agent": "researcher", + "start_offset_ms": 61.836032, + "duration_ms": 1130.572032, + "status": "ok", + "input_preview": "[{\"role\": \"system\", \"content\": \"Call lookup_benchmark once for ClickHouse and once for Postgres, then answer in ONE short sentence.\"}, {\"role\": \"user\", \"content\": \"Compare multi-tenancy (question 4).\"}]", + "model": "claude-haiku-4-5", + "input_tokens": 607, + "output_tokens": 120, + "litellm": { + "request_id": "chatcmpl-e987a212-12de-404f-bf30-db1db933c1bc", + "model": "openai/claude-haiku-4-5-20251001", + "model_group": "claude-haiku-4-5", + "provider": "openai", + "key_alias": "research-bot", + "team_alias": "research-agents", + "spend": 0.0012070000000000002, + "prompt_tokens": 607, + "completion_tokens": 120, + "cache_read_tokens": 0, + "cache_write_tokens": 0, + "latency_ms": 1083, + "ttft_ms": 1083, + "status": "success" + } + }, + { + "span_id": "c076a3e9e3b197ae", + "parent_span_id": "d23937c848659551", + "name": "ChatOpenAI", + "type": "llm", + "agent": "researcher", + "start_offset_ms": 62.268928, + "duration_ms": 186.465024, + "status": "ok", + "input_preview": "[{\"role\": \"system\", \"content\": \"Call lookup_benchmark once for ClickHouse and once for Postgres, then answer in ONE short sentence.\"}, {\"role\": \"user\", \"content\": \"Compare backups (question 7).\"}]", + "model": "claude-haiku-4-5", + "input_tokens": 605, + "output_tokens": 120, + "litellm": { + "request_id": "chatcmpl-e7cb41a6-6020-4cdc-9d8e-1516de91bc11", + "model": "openai/claude-haiku-4-5-20251001", + "model_group": "claude-haiku-4-5", + "provider": "openai", + "key_alias": "research-bot", + "team_alias": "research-agents", + "spend": 0.001205, + "prompt_tokens": 605, + "completion_tokens": 120, + "cache_read_tokens": 0, + "cache_write_tokens": 0, + "latency_ms": 1472, + "ttft_ms": 1472, + "status": "success" + } + }, + { + "span_id": "c076a3e9e3b197ae", + "parent_span_id": "d23937c848659551", + "name": "ChatOpenAI", + "type": "llm", + "agent": "researcher", + "start_offset_ms": 62.268928, + "duration_ms": 186.465024, + "status": "ok", + "input_preview": "[{\"role\": \"system\", \"content\": \"Call lookup_benchmark once for ClickHouse and once for Postgres, then answer in ONE short sentence.\"}, {\"role\": \"user\", \"content\": \"Compare backups (question 7).\"}]", + "model": "claude-haiku-4-5", + "input_tokens": 605, + "output_tokens": 120, + "litellm": { + "request_id": "chatcmpl-e7cb41a6-6020-4cdc-9d8e-1516de91bc11", + "model": "openai/claude-haiku-4-5-20251001", + "model_group": "claude-haiku-4-5", + "provider": "openai", + "key_alias": "research-bot", + "team_alias": "research-agents", + "spend": 0.001205, + "prompt_tokens": 605, + "completion_tokens": 120, + "cache_read_tokens": 0, + "cache_write_tokens": 0, + "latency_ms": 135, + "ttft_ms": 135, + "status": "success" + } + }, + { + "span_id": "638912f9c31b4936", + "parent_span_id": "ebd769f68fb432b3", + "name": "ChatOpenAI", + "type": "llm", + "agent": "researcher", + "start_offset_ms": 62.682112, + "duration_ms": 1481.43616, + "status": "ok", + "input_preview": "[{\"role\": \"system\", \"content\": \"Call lookup_benchmark once for ClickHouse and once for Postgres, then call check_fact on your conclusion, then answer in ONE short sentence.\"}, {\"role\": \"user\", \"content\": \"Compare ingest throughput (question", + "model": "claude-haiku-4-5", + "input_tokens": 676, + "output_tokens": 120, + "litellm": { + "request_id": "chatcmpl-5891711d-025f-4e93-b2f7-240a3f6b0c84", + "model": "openai/claude-haiku-4-5-20251001", + "model_group": "claude-haiku-4-5", + "provider": "openai", + "key_alias": "research-bot", + "team_alias": "research-agents", + "spend": 0.001276, + "prompt_tokens": 676, + "completion_tokens": 120, + "cache_read_tokens": 0, + "cache_write_tokens": 0, + "latency_ms": 1428, + "ttft_ms": 1428, + "status": "success" + } + }, + { + "span_id": "c124c17c8d44876e", + "parent_span_id": "be4ac7802a570aa4", + "name": "ChatOpenAI", + "type": "llm", + "agent": "researcher", + "start_offset_ms": 65.745152, + "duration_ms": 186.13376, + "status": "ok", + "input_preview": "[{\"role\": \"system\", \"content\": \"Call lookup_benchmark once for ClickHouse and once for Postgres, then answer in ONE short sentence.\"}, {\"role\": \"user\", \"content\": \"Compare replication (question 6).\"}]", + "model": "claude-haiku-4-5", + "input_tokens": 605, + "output_tokens": 120, + "litellm": { + "request_id": "chatcmpl-613f5748-76a7-4b30-ac6a-4003a58f9490", + "model": "openai/claude-haiku-4-5-20251001", + "model_group": "claude-haiku-4-5", + "provider": "openai", + "key_alias": "research-bot", + "team_alias": "research-agents", + "spend": 0.001205, + "prompt_tokens": 605, + "completion_tokens": 120, + "cache_read_tokens": 0, + "cache_write_tokens": 0, + "latency_ms": 1495, + "ttft_ms": 1495, + "status": "success" + } + }, + { + "span_id": "c124c17c8d44876e", + "parent_span_id": "be4ac7802a570aa4", + "name": "ChatOpenAI", + "type": "llm", + "agent": "researcher", + "start_offset_ms": 65.745152, + "duration_ms": 186.13376, + "status": "ok", + "input_preview": "[{\"role\": \"system\", \"content\": \"Call lookup_benchmark once for ClickHouse and once for Postgres, then answer in ONE short sentence.\"}, {\"role\": \"user\", \"content\": \"Compare replication (question 6).\"}]", + "model": "claude-haiku-4-5", + "input_tokens": 605, + "output_tokens": 120, + "litellm": { + "request_id": "chatcmpl-613f5748-76a7-4b30-ac6a-4003a58f9490", + "model": "openai/claude-haiku-4-5-20251001", + "model_group": "claude-haiku-4-5", + "provider": "openai", + "key_alias": "research-bot", + "team_alias": "research-agents", + "spend": 0.001205, + "prompt_tokens": 605, + "completion_tokens": 120, + "cache_read_tokens": 0, + "cache_write_tokens": 0, + "latency_ms": 141, + "ttft_ms": 141, + "status": "success" + } + }, + { + "span_id": "791584567230ec00", + "parent_span_id": "6ba3ba633024d9e4", + "name": "ChatOpenAI", + "type": "llm", + "agent": "researcher", + "start_offset_ms": 69.241088, + "duration_ms": 1154.654976, + "status": "ok", + "input_preview": "[{\"role\": \"system\", \"content\": \"Call lookup_benchmark once for ClickHouse and once for Postgres, then answer in ONE short sentence.\"}, {\"role\": \"user\", \"content\": \"Compare storage cost (question 1).\"}]", + "model": "claude-haiku-4-5", + "input_tokens": 605, + "output_tokens": 70, + "litellm": { + "request_id": "chatcmpl-bb6a3f5d-8947-4b51-8bb0-4c8997e05bc1", + "model": "openai/claude-haiku-4-5-20251001", + "model_group": "claude-haiku-4-5", + "provider": "openai", + "key_alias": "research-bot", + "team_alias": "research-agents", + "spend": 0.000955, + "prompt_tokens": 605, + "completion_tokens": 70, + "cache_read_tokens": 0, + "cache_write_tokens": 0, + "latency_ms": 1110, + "ttft_ms": 1110, + "status": "success" + } + }, + { + "span_id": "99016550b1f4b1d8", + "parent_span_id": "8db5b2b7871b58a6", + "name": "ChatOpenAI", + "type": "llm", + "agent": "researcher", + "start_offset_ms": 70.228224, + "duration_ms": 1093.913856, + "status": "ok", + "input_preview": "[{\"role\": \"system\", \"content\": \"Call lookup_benchmark once for ClickHouse and once for Postgres, then answer in ONE short sentence.\"}, {\"role\": \"user\", \"content\": \"Compare storage cost (question 11).\"}]", + "model": "claude-haiku-4-5", + "input_tokens": 605, + "output_tokens": 120, + "litellm": { + "request_id": "chatcmpl-96f92920-92c9-4848-bf5c-30633117837c", + "model": "openai/claude-haiku-4-5-20251001", + "model_group": "claude-haiku-4-5", + "provider": "openai", + "key_alias": "research-bot", + "team_alias": "research-agents", + "spend": 0.001205, + "prompt_tokens": 605, + "completion_tokens": 120, + "cache_read_tokens": 0, + "cache_write_tokens": 0, + "latency_ms": 1049, + "ttft_ms": 1049, + "status": "success" + } + }, + { + "span_id": "663415cd473e52bd", + "parent_span_id": "93e72d093e644a1d", + "name": "ChatOpenAI", + "type": "llm", + "agent": "researcher", + "start_offset_ms": 71.22816, + "duration_ms": 1164.059904, + "status": "ok", + "input_preview": "[{\"role\": \"system\", \"content\": \"Call lookup_benchmark once for ClickHouse and once for Postgres, then answer in ONE short sentence.\"}, {\"role\": \"user\", \"content\": \"Compare compression (question 5).\"}]", + "model": "claude-haiku-4-5", + "input_tokens": 604, + "output_tokens": 120, + "litellm": { + "request_id": "chatcmpl-efb08407-99c5-41e5-affc-e7d3f7298f41", + "model": "openai/claude-haiku-4-5-20251001", + "model_group": "claude-haiku-4-5", + "provider": "openai", + "key_alias": "research-bot", + "team_alias": "research-agents", + "spend": 0.001204, + "prompt_tokens": 604, + "completion_tokens": 120, + "cache_read_tokens": 0, + "cache_write_tokens": 0, + "latency_ms": 1118, + "ttft_ms": 1118, + "status": "success" + } + }, + { + "span_id": "f0ac4348db8afae0", + "parent_span_id": "991c19abc6a2001c", + "name": "ChatOpenAI", + "type": "llm", + "agent": "researcher", + "start_offset_ms": 72.411136, + "duration_ms": 1129.085952, + "status": "ok", + "input_preview": "[{\"role\": \"system\", \"content\": \"Call lookup_benchmark once for ClickHouse and once for Postgres, then answer in ONE short sentence.\"}, {\"role\": \"user\", \"content\": \"Compare retention (question 3).\"}]", + "model": "claude-haiku-4-5", + "input_tokens": 604, + "output_tokens": 120, + "litellm": { + "request_id": "chatcmpl-ec0d2784-5f4c-4b9f-b88e-50e3ca221f41", + "model": "openai/claude-haiku-4-5-20251001", + "model_group": "claude-haiku-4-5", + "provider": "openai", + "key_alias": "research-bot", + "team_alias": "research-agents", + "spend": 0.001204, + "prompt_tokens": 604, + "completion_tokens": 120, + "cache_read_tokens": 0, + "cache_write_tokens": 0, + "latency_ms": 1090, + "ttft_ms": 1090, + "status": "success" + } + }, + { + "span_id": "cdd663e40c465dd9", + "parent_span_id": "3e4cf4753e6183c1", + "name": "ChatOpenAI", + "type": "llm", + "agent": "researcher", + "start_offset_ms": 73.424128, + "duration_ms": 1263.8208, + "status": "ok", + "input_preview": "[{\"role\": \"system\", \"content\": \"Call lookup_benchmark once for ClickHouse and once for Postgres, then answer in ONE short sentence.\"}, {\"role\": \"user\", \"content\": \"Compare schema changes (question 8).\"}]", + "model": "claude-haiku-4-5", + "input_tokens": 605, + "output_tokens": 120, + "litellm": { + "request_id": "chatcmpl-e72cb04b-a5c7-4975-84de-cb4f9ea55573", + "model": "openai/claude-haiku-4-5-20251001", + "model_group": "claude-haiku-4-5", + "provider": "openai", + "key_alias": "research-bot", + "team_alias": "research-agents", + "spend": 0.001205, + "prompt_tokens": 605, + "completion_tokens": 120, + "cache_read_tokens": 0, + "cache_write_tokens": 0, + "latency_ms": 1220, + "ttft_ms": 1220, + "status": "success" + } + }, + { + "span_id": "f08c5c831bf4bd72", + "parent_span_id": "b33e0f6b6f8a6ef0", + "name": "ChatOpenAI", + "type": "llm", + "agent": "researcher", + "start_offset_ms": 74.630144, + "duration_ms": 171.833856, + "status": "ok", + "input_preview": "[{\"role\": \"system\", \"content\": \"Call lookup_benchmark once for ClickHouse and once for Postgres, then answer in ONE short sentence.\"}, {\"role\": \"user\", \"content\": \"Compare query latency (question 2).\"}]", + "model": "claude-haiku-4-5", + "input_tokens": 606, + "output_tokens": 120, + "litellm": { + "request_id": "chatcmpl-0e56e75e-8d08-4423-a7e5-003a81ef6046", + "model": "openai/claude-haiku-4-5-20251001", + "model_group": "claude-haiku-4-5", + "provider": "openai", + "key_alias": "research-bot", + "team_alias": "research-agents", + "spend": 0.001206, + "prompt_tokens": 606, + "completion_tokens": 120, + "cache_read_tokens": 0, + "cache_write_tokens": 0, + "latency_ms": 133, + "ttft_ms": 133, + "status": "success" + } + }, + { + "span_id": "f08c5c831bf4bd72", + "parent_span_id": "b33e0f6b6f8a6ef0", + "name": "ChatOpenAI", + "type": "llm", + "agent": "researcher", + "start_offset_ms": 74.630144, + "duration_ms": 171.833856, + "status": "ok", + "input_preview": "[{\"role\": \"system\", \"content\": \"Call lookup_benchmark once for ClickHouse and once for Postgres, then answer in ONE short sentence.\"}, {\"role\": \"user\", \"content\": \"Compare query latency (question 2).\"}]", + "model": "claude-haiku-4-5", + "input_tokens": 606, + "output_tokens": 120, + "litellm": { + "request_id": "chatcmpl-0e56e75e-8d08-4423-a7e5-003a81ef6046", + "model": "openai/claude-haiku-4-5-20251001", + "model_group": "claude-haiku-4-5", + "provider": "openai", + "key_alias": "research-bot", + "team_alias": "research-agents", + "spend": 0.001206, + "prompt_tokens": 606, + "completion_tokens": 120, + "cache_read_tokens": 0, + "cache_write_tokens": 0, + "latency_ms": 1723, + "ttft_ms": 1723, + "status": "success" + } + }, + { + "span_id": "202780b67dcbe6ec", + "parent_span_id": "71903d779d129c68", + "name": "tools", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 247.129088, + "duration_ms": 1.22112, + "status": "ok", + "input_preview": "{\"input\":[{\"name\":\"lookup_benchmark\",\"args\":{\"topic\":\"question 2\",\"store\":\"ClickHouse\"},\"id\":\"toolu_01QpRUG3ZDA2iCbyd3R1bvu7\",\"type\":\"tool_call\"}]}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "a822744d88aa4631", + "parent_span_id": "202780b67dcbe6ec", + "name": "lookup_benchmark", + "type": "tool", + "agent": "researcher", + "start_offset_ms": 247.52512, + "duration_ms": 0.240896, + "status": "ok", + "input_preview": "{\"topic\":\"question 2\",\"store\":\"ClickHouse\"}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "4f4840b705cf0258", + "parent_span_id": "71903d779d129c68", + "name": "tools", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 248.105984, + "duration_ms": 9.818112, + "status": "ok", + "input_preview": "{\"input\":[{\"name\":\"lookup_benchmark\",\"args\":{},\"id\":\"toolu_01Y5NpMECPckqZiivRNbis8y\",\"type\":\"tool_call\"}]}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "f81d2982c71d49ab", + "parent_span_id": "4f4840b705cf0258", + "name": "lookup_benchmark", + "type": "tool", + "agent": "researcher", + "start_offset_ms": 249.108992, + "duration_ms": 5.188096, + "status": "error", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "8da3c27f759a9a70", + "parent_span_id": "2714afb89a510e4a", + "name": "tools", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 249.403136, + "duration_ms": 0.691968, + "status": "ok", + "input_preview": "{\"input\":[{\"name\":\"lookup_benchmark\",\"args\":{\"topic\":\"backups\",\"store\":\"ClickHouse\"},\"id\":\"toolu_01NsgVkiKqRHCMy4TCqCAdwo\",\"type\":\"tool_call\"}]}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "1ae5b620a479299e", + "parent_span_id": "8da3c27f759a9a70", + "name": "lookup_benchmark", + "type": "tool", + "agent": "researcher", + "start_offset_ms": 249.64992, + "duration_ms": 0.270336, + "status": "ok", + "input_preview": "{\"topic\":\"backups\",\"store\":\"ClickHouse\"}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "0e78db2488523ae2", + "parent_span_id": "2714afb89a510e4a", + "name": "tools", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 250.228224, + "duration_ms": 4.8448, + "status": "ok", + "input_preview": "{\"input\":[{\"name\":\"lookup_benchmark\",\"args\":{\"topic\":\"backups\"},\"id\":\"toolu_01VbdVXDtw8UTMViJ7gokomj\",\"type\":\"tool_call\"}]}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "6e60f7c6e87a7d8e", + "parent_span_id": "0e78db2488523ae2", + "name": "lookup_benchmark", + "type": "tool", + "agent": "researcher", + "start_offset_ms": 250.409984, + "duration_ms": 4.124928, + "status": "error", + "input_preview": "{\"topic\":\"backups\"}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "ec9dd8a7195a53c4", + "parent_span_id": "41fa112865f7b1f5", + "name": "tools", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 252.23808, + "duration_ms": 0.653056, + "status": "ok", + "input_preview": "{\"input\":[{\"name\":\"lookup_benchmark\",\"args\":{\"topic\":\"replication\",\"store\":\"ClickHouse\"},\"id\":\"toolu_018LjjywXXsaTX4fFcyoWDMc\",\"type\":\"tool_call\"}]}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "3aaf7251e50f38b0", + "parent_span_id": "ec9dd8a7195a53c4", + "name": "lookup_benchmark", + "type": "tool", + "agent": "researcher", + "start_offset_ms": 252.502016, + "duration_ms": 0.173056, + "status": "ok", + "input_preview": "{\"topic\":\"replication\",\"store\":\"ClickHouse\"}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "0f63b2ed7ede23d9", + "parent_span_id": "41fa112865f7b1f5", + "name": "tools", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 253.079296, + "duration_ms": 1.87264, + "status": "ok", + "input_preview": "{\"input\":[{\"name\":\"lookup_benchmark\",\"args\":{},\"id\":\"toolu_01AFJaxyTcM2JitwHJCbMX9x\",\"type\":\"tool_call\"}]}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "008eaf94b0a32842", + "parent_span_id": "0f63b2ed7ede23d9", + "name": "lookup_benchmark", + "type": "tool", + "agent": "researcher", + "start_offset_ms": 253.319936, + "duration_ms": 1.388032, + "status": "error", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "31780bf2416808de", + "parent_span_id": "41fa112865f7b1f5", + "name": "model", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 255.284992, + "duration_ms": 66.174208, + "status": "ok", + "input_preview": "{\"messages\":[{\"content\":\"Compare replication (question 6).\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"1b0c8ffc-23f5-430f-83e8-c3a28091d5b7\"},{\"content\":\"I'll look up the replication benchmarks for both ClickHouse an", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "58ca5d41dba8391d", + "parent_span_id": "31780bf2416808de", + "name": "ChatOpenAI", + "type": "llm", + "agent": "researcher", + "start_offset_ms": 255.536128, + "duration_ms": 65.780736, + "status": "ok", + "input_preview": "[{\"role\": \"system\", \"content\": \"Call lookup_benchmark once for ClickHouse and once for Postgres, then answer in ONE short sentence.\"}, {\"role\": \"user\", \"content\": \"Compare replication (question 6).\"}, {\"role\": \"assistant\", \"content\": \"I'll ", + "model": "claude-haiku-4-5", + "input_tokens": 831, + "output_tokens": 86, + "litellm": { + "request_id": "chatcmpl-95594677-f719-46bb-b5ac-4120df774a00", + "model": "openai/claude-haiku-4-5-20251001", + "model_group": "claude-haiku-4-5", + "provider": "openai", + "key_alias": "research-bot", + "team_alias": "research-agents", + "spend": 0.001261, + "prompt_tokens": 831, + "completion_tokens": 86, + "cache_read_tokens": 0, + "cache_write_tokens": 0, + "latency_ms": 56, + "ttft_ms": 56, + "status": "success" + } + }, + { + "span_id": "58ca5d41dba8391d", + "parent_span_id": "31780bf2416808de", + "name": "ChatOpenAI", + "type": "llm", + "agent": "researcher", + "start_offset_ms": 255.536128, + "duration_ms": 65.780736, + "status": "ok", + "input_preview": "[{\"role\": \"system\", \"content\": \"Call lookup_benchmark once for ClickHouse and once for Postgres, then answer in ONE short sentence.\"}, {\"role\": \"user\", \"content\": \"Compare replication (question 6).\"}, {\"role\": \"assistant\", \"content\": \"I'll ", + "model": "claude-haiku-4-5", + "input_tokens": 831, + "output_tokens": 86, + "litellm": { + "request_id": "chatcmpl-95594677-f719-46bb-b5ac-4120df774a00", + "model": "openai/claude-haiku-4-5-20251001", + "model_group": "claude-haiku-4-5", + "provider": "openai", + "key_alias": "research-bot", + "team_alias": "research-agents", + "spend": 0.001261, + "prompt_tokens": 831, + "completion_tokens": 86, + "cache_read_tokens": 0, + "cache_write_tokens": 0, + "latency_ms": 1205, + "ttft_ms": 1205, + "status": "success" + } + }, + { + "span_id": "7c54980e8bad2655", + "parent_span_id": "2714afb89a510e4a", + "name": "model", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 256.829952, + "duration_ms": 61.04704, + "status": "ok", + "input_preview": "{\"messages\":[{\"content\":\"Compare backups (question 7).\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"ae6337ae-8401-4264-a955-836b6b09c7a3\"},{\"content\":\"\",\"additional_kwargs\":{\"refusal\":null},\"response_metadata\":{\"token", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "315fe6ce2f65b09c", + "parent_span_id": "7c54980e8bad2655", + "name": "ChatOpenAI", + "type": "llm", + "agent": "researcher", + "start_offset_ms": 257.037312, + "duration_ms": 60.630784, + "status": "ok", + "input_preview": "[{\"role\": \"system\", \"content\": \"Call lookup_benchmark once for ClickHouse and once for Postgres, then answer in ONE short sentence.\"}, {\"role\": \"user\", \"content\": \"Compare backups (question 7).\"}, {\"role\": \"assistant\", \"content\": \"\", \"tool_", + "model": "claude-haiku-4-5", + "input_tokens": 833, + "output_tokens": 84, + "litellm": { + "request_id": "chatcmpl-24edbb47-a7f5-403c-80c0-88ec1eac0dd7", + "model": "openai/claude-haiku-4-5-20251001", + "model_group": "claude-haiku-4-5", + "provider": "openai", + "key_alias": "research-bot", + "team_alias": "research-agents", + "spend": 0.001253, + "prompt_tokens": 833, + "completion_tokens": 84, + "cache_read_tokens": 0, + "cache_write_tokens": 0, + "latency_ms": 52, + "ttft_ms": 52, + "status": "success" + } + }, + { + "span_id": "315fe6ce2f65b09c", + "parent_span_id": "7c54980e8bad2655", + "name": "ChatOpenAI", + "type": "llm", + "agent": "researcher", + "start_offset_ms": 257.037312, + "duration_ms": 60.630784, + "status": "ok", + "input_preview": "[{\"role\": \"system\", \"content\": \"Call lookup_benchmark once for ClickHouse and once for Postgres, then answer in ONE short sentence.\"}, {\"role\": \"user\", \"content\": \"Compare backups (question 7).\"}, {\"role\": \"assistant\", \"content\": \"\", \"tool_", + "model": "claude-haiku-4-5", + "input_tokens": 833, + "output_tokens": 84, + "litellm": { + "request_id": "chatcmpl-24edbb47-a7f5-403c-80c0-88ec1eac0dd7", + "model": "openai/claude-haiku-4-5-20251001", + "model_group": "claude-haiku-4-5", + "provider": "openai", + "key_alias": "research-bot", + "team_alias": "research-agents", + "spend": 0.001253, + "prompt_tokens": 833, + "completion_tokens": 84, + "cache_read_tokens": 0, + "cache_write_tokens": 0, + "latency_ms": 1507, + "ttft_ms": 1507, + "status": "success" + } + }, + { + "span_id": "f3645483ea551b89", + "parent_span_id": "71903d779d129c68", + "name": "model", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 258.110976, + "duration_ms": 872.749056, + "status": "ok", + "input_preview": "{\"messages\":[{\"content\":\"Compare query latency (question 2).\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"c9cf9a10-ff05-4215-8626-9fa60808ae97\"},{\"content\":\"I'll look up the query latency benchmarks for both ClickHous", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "1ea79fa9c6374836", + "parent_span_id": "f3645483ea551b89", + "name": "ChatOpenAI", + "type": "llm", + "agent": "researcher", + "start_offset_ms": 258.290176, + "duration_ms": 872.399104, + "status": "ok", + "input_preview": "[{\"role\": \"system\", \"content\": \"Call lookup_benchmark once for ClickHouse and once for Postgres, then answer in ONE short sentence.\"}, {\"role\": \"user\", \"content\": \"Compare query latency (question 2).\"}, {\"role\": \"assistant\", \"content\": \"I'l", + "model": "claude-haiku-4-5", + "input_tokens": 835, + "output_tokens": 82, + "litellm": { + "request_id": "chatcmpl-c4972857-6f99-4aba-9840-9a8eb8919371", + "model": "openai/claude-haiku-4-5-20251001", + "model_group": "claude-haiku-4-5", + "provider": "openai", + "key_alias": "research-bot", + "team_alias": "research-agents", + "spend": 0.001245, + "prompt_tokens": 835, + "completion_tokens": 82, + "cache_read_tokens": 0, + "cache_write_tokens": 0, + "latency_ms": 865, + "ttft_ms": 865, + "status": "success" + } + }, + { + "span_id": "c7afe4c441a0fd73", + "parent_span_id": "2714afb89a510e4a", + "name": "tools", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 318.098176, + "duration_ms": 6.99904, + "status": "error", + "input_preview": "{\"input\":[{\"name\":\"lookup_benchmark\",\"args\":{\"topic\":\"backups\",\"store\":\"Postgres\"},\"id\":\"toolu_01G9NCc3ydqBTtP2oL5Dm6Sa\",\"type\":\"tool_call\"}]}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "285a73be5eeb1854", + "parent_span_id": "c7afe4c441a0fd73", + "name": "lookup_benchmark", + "type": "tool", + "agent": "researcher", + "start_offset_ms": 318.509056, + "duration_ms": 0.802048, + "status": "error", + "input_preview": "{\"topic\":\"backups\",\"store\":\"Postgres\"}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "efd995d7612509bf", + "parent_span_id": "41fa112865f7b1f5", + "name": "tools", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 321.657088, + "duration_ms": 0.692992, + "status": "ok", + "input_preview": "{\"input\":[{\"name\":\"lookup_benchmark\",\"args\":{\"topic\":\"replication\",\"store\":\"Postgres\"},\"id\":\"toolu_019ZnuDWZ4iesRCHsqAJbDR5\",\"type\":\"tool_call\"}]}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "829d07185a7ad5ae", + "parent_span_id": "efd995d7612509bf", + "name": "lookup_benchmark", + "type": "tool", + "agent": "researcher", + "start_offset_ms": 321.92512, + "duration_ms": 0.188928, + "status": "ok", + "input_preview": "{\"topic\":\"replication\",\"store\":\"Postgres\"}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "4a3842bd69cae949", + "parent_span_id": "41fa112865f7b1f5", + "name": "model", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 322.51904, + "duration_ms": 921.635072, + "status": "ok", + "input_preview": "{\"messages\":[{\"content\":\"Compare replication (question 6).\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"1b0c8ffc-23f5-430f-83e8-c3a28091d5b7\"},{\"content\":\"I'll look up the replication benchmarks for both ClickHouse an", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "8645585126f22d7b", + "parent_span_id": "4a3842bd69cae949", + "name": "ChatOpenAI", + "type": "llm", + "agent": "researcher", + "start_offset_ms": 322.769152, + "duration_ms": 921.25696, + "status": "ok", + "input_preview": "[{\"role\": \"system\", \"content\": \"Call lookup_benchmark once for ClickHouse and once for Postgres, then answer in ONE short sentence.\"}, {\"role\": \"user\", \"content\": \"Compare replication (question 6).\"}, {\"role\": \"assistant\", \"content\": \"I'll ", + "model": "claude-haiku-4-5", + "input_tokens": 941, + "output_tokens": 49, + "litellm": { + "request_id": "chatcmpl-0990e3ad-175e-46df-bf19-68177c9e426a", + "model": "openai/claude-haiku-4-5-20251001", + "model_group": "claude-haiku-4-5", + "provider": "openai", + "key_alias": "research-bot", + "team_alias": "research-agents", + "spend": 0.001186, + "prompt_tokens": 941, + "completion_tokens": 49, + "cache_read_tokens": 0, + "cache_write_tokens": 0, + "latency_ms": 915, + "ttft_ms": 915, + "status": "success" + } + }, + { + "span_id": "cb2c4553753e56f5", + "parent_span_id": "71903d779d129c68", + "name": "tools", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 1131.075072, + "duration_ms": 0.651008, + "status": "ok", + "input_preview": "{\"input\":[{\"name\":\"lookup_benchmark\",\"args\":{\"topic\":\"question 2\",\"store\":\"Postgres\"},\"id\":\"toolu_01G3oiAoW5NPsSL2eJCagf1U\",\"type\":\"tool_call\"}]}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "8b32aaa41ba4f1df", + "parent_span_id": "cb2c4553753e56f5", + "name": "lookup_benchmark", + "type": "tool", + "agent": "researcher", + "start_offset_ms": 1131.366912, + "duration_ms": 0.198144, + "status": "ok", + "input_preview": "{\"topic\":\"question 2\",\"store\":\"Postgres\"}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "915aa192d636e089", + "parent_span_id": "71903d779d129c68", + "name": "model", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 1131.901184, + "duration_ms": 1039.888128, + "status": "ok", + "input_preview": "{\"messages\":[{\"content\":\"Compare query latency (question 2).\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"c9cf9a10-ff05-4215-8626-9fa60808ae97\"},{\"content\":\"I'll look up the query latency benchmarks for both ClickHous", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "be4cad07067533b9", + "parent_span_id": "915aa192d636e089", + "name": "ChatOpenAI", + "type": "llm", + "agent": "researcher", + "start_offset_ms": 1132.164096, + "duration_ms": 1039.467008, + "status": "ok", + "input_preview": "[{\"role\": \"system\", \"content\": \"Call lookup_benchmark once for ClickHouse and once for Postgres, then answer in ONE short sentence.\"}, {\"role\": \"user\", \"content\": \"Compare query latency (question 2).\"}, {\"role\": \"assistant\", \"content\": \"I'l", + "model": "claude-haiku-4-5", + "input_tokens": 942, + "output_tokens": 48, + "litellm": { + "request_id": "chatcmpl-86ca494e-bc4a-4c54-ad78-b90745af11b0", + "model": "openai/claude-haiku-4-5-20251001", + "model_group": "claude-haiku-4-5", + "provider": "openai", + "key_alias": "research-bot", + "team_alias": "research-agents", + "spend": 0.0011819999999999999, + "prompt_tokens": 942, + "completion_tokens": 48, + "cache_read_tokens": 0, + "cache_write_tokens": 0, + "latency_ms": 1034, + "ttft_ms": 1034, + "status": "success" + } + }, + { + "span_id": "09fab5aee6d32ab2", + "parent_span_id": "a1e8b8b06038a09e", + "name": "tools", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 1164.51712, + "duration_ms": 0.731904, + "status": "ok", + "input_preview": "{\"input\":[{\"name\":\"lookup_benchmark\",\"args\":{\"topic\":\"storage cost\",\"store\":\"ClickHouse\"},\"id\":\"toolu_018MuiQt9fmjtqVz2RXCHZhy\",\"type\":\"tool_call\"}]}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "b43b5e1d2cd57e76", + "parent_span_id": "09fab5aee6d32ab2", + "name": "lookup_benchmark", + "type": "tool", + "agent": "researcher", + "start_offset_ms": 1164.958976, + "duration_ms": 0.16896, + "status": "ok", + "input_preview": "{\"topic\":\"storage cost\",\"store\":\"ClickHouse\"}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "c97b01c60c5d1217", + "parent_span_id": "a1e8b8b06038a09e", + "name": "tools", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 1165.370112, + "duration_ms": 0.771072, + "status": "ok", + "input_preview": "{\"input\":[{\"name\":\"lookup_benchmark\",\"args\":{},\"id\":\"toolu_01HD8LARpprgDitPW6nfTZZe\",\"type\":\"tool_call\"}]}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "8a3858ffa83cf12f", + "parent_span_id": "c97b01c60c5d1217", + "name": "lookup_benchmark", + "type": "tool", + "agent": "researcher", + "start_offset_ms": 1165.632, + "duration_ms": 0.303104, + "status": "error", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "53b54096466355c5", + "parent_span_id": "a1e8b8b06038a09e", + "name": "model", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 1166.361088, + "duration_ms": 1261.767936, + "status": "ok", + "input_preview": "{\"messages\":[{\"content\":\"Compare storage cost (question 11).\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"6c39cf1d-8ce5-459d-96a6-e2f6ba6fccfe\"},{\"content\":\"I'll look up the storage cost benchmark for both ClickHouse ", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "336aad6fad874893", + "parent_span_id": "53b54096466355c5", + "name": "ChatOpenAI", + "type": "llm", + "agent": "researcher", + "start_offset_ms": 1166.622208, + "duration_ms": 1261.346816, + "status": "ok", + "input_preview": "[{\"role\": \"system\", \"content\": \"Call lookup_benchmark once for ClickHouse and once for Postgres, then answer in ONE short sentence.\"}, {\"role\": \"user\", \"content\": \"Compare storage cost (question 11).\"}, {\"role\": \"assistant\", \"content\": \"I'l", + "model": "claude-haiku-4-5", + "input_tokens": 830, + "output_tokens": 84, + "litellm": { + "request_id": "chatcmpl-e2f2e18d-7b17-4c6f-8bd0-5acffbf7e3ef", + "model": "openai/claude-haiku-4-5-20251001", + "model_group": "claude-haiku-4-5", + "provider": "openai", + "key_alias": "research-bot", + "team_alias": "research-agents", + "spend": 0.00125, + "prompt_tokens": 830, + "completion_tokens": 84, + "cache_read_tokens": 0, + "cache_write_tokens": 0, + "latency_ms": 1257, + "ttft_ms": 1257, + "status": "success" + } + }, + { + "span_id": "319e1d5201f7ecf8", + "parent_span_id": "03f611cf9bfc9ea4", + "name": "tools", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 1192.827136, + "duration_ms": 0.879872, + "status": "ok", + "input_preview": "{\"input\":[{\"name\":\"lookup_benchmark\",\"args\":{\"topic\":\"multi-tenancy\",\"store\":\"ClickHouse\"},\"id\":\"toolu_01KWSWhiKjDWaLmoVLm9NXB6\",\"type\":\"tool_call\"}]}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "33dc6bb0071e9f68", + "parent_span_id": "319e1d5201f7ecf8", + "name": "lookup_benchmark", + "type": "tool", + "agent": "researcher", + "start_offset_ms": 1193.089024, + "duration_ms": 0.297984, + "status": "ok", + "input_preview": "{\"topic\":\"multi-tenancy\",\"store\":\"ClickHouse\"}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "e8c538553372ff0e", + "parent_span_id": "03f611cf9bfc9ea4", + "name": "tools", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 1193.542144, + "duration_ms": 0.775936, + "status": "ok", + "input_preview": "{\"input\":[{\"name\":\"lookup_benchmark\",\"args\":{},\"id\":\"toolu_01Bof3pVkxMr4giscKi69THD\",\"type\":\"tool_call\"}]}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "8f066d65f6e6ba90", + "parent_span_id": "e8c538553372ff0e", + "name": "lookup_benchmark", + "type": "tool", + "agent": "researcher", + "start_offset_ms": 1193.885184, + "duration_ms": 0.267008, + "status": "error", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "1fed547ebbc3c231", + "parent_span_id": "03f611cf9bfc9ea4", + "name": "model", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 1194.514944, + "duration_ms": 1037.73824, + "status": "ok", + "input_preview": "{\"messages\":[{\"content\":\"Compare multi-tenancy (question 4).\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"6f39cb28-7f23-4cd1-b74a-98eb05d175fd\"},{\"content\":\"I'll look up the multi-tenancy benchmarks for both ClickHous", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "72f543f3390d665f", + "parent_span_id": "1fed547ebbc3c231", + "name": "ChatOpenAI", + "type": "llm", + "agent": "researcher", + "start_offset_ms": 1194.735104, + "duration_ms": 1037.377024, + "status": "ok", + "input_preview": "[{\"role\": \"system\", \"content\": \"Call lookup_benchmark once for ClickHouse and once for Postgres, then answer in ONE short sentence.\"}, {\"role\": \"user\", \"content\": \"Compare multi-tenancy (question 4).\"}, {\"role\": \"assistant\", \"content\": \"I'l", + "model": "claude-haiku-4-5", + "input_tokens": 839, + "output_tokens": 85, + "litellm": { + "request_id": "chatcmpl-1c04a496-4e08-4531-8ae1-3751e17e97bf", + "model": "openai/claude-haiku-4-5-20251001", + "model_group": "claude-haiku-4-5", + "provider": "openai", + "key_alias": "research-bot", + "team_alias": "research-agents", + "spend": 0.001264, + "prompt_tokens": 839, + "completion_tokens": 85, + "cache_read_tokens": 0, + "cache_write_tokens": 0, + "latency_ms": 1032, + "ttft_ms": 1032, + "status": "success" + } + }, + { + "span_id": "ef050fdc5fcbc5f6", + "parent_span_id": "49397f8d7856d2ba", + "name": "tools", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 1201.884928, + "duration_ms": 0.799232, + "status": "ok", + "input_preview": "{\"input\":[{\"name\":\"lookup_benchmark\",\"args\":{\"store\":\"ClickHouse\",\"topic\":\"retention (question 3)\"},\"id\":\"toolu_014st8iy9P8SuxXiZMkh32tw\",\"type\":\"tool_call\"}]}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "8104b956bb5589ab", + "parent_span_id": "ef050fdc5fcbc5f6", + "name": "lookup_benchmark", + "type": "tool", + "agent": "researcher", + "start_offset_ms": 1202.150144, + "duration_ms": 0.17792, + "status": "ok", + "input_preview": "{\"store\":\"ClickHouse\",\"topic\":\"retention (question 3)\"}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "e94883ee4a0748f4", + "parent_span_id": "49397f8d7856d2ba", + "name": "tools", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 1202.44608, + "duration_ms": 0.976128, + "status": "ok", + "input_preview": "{\"input\":[{\"name\":\"lookup_benchmark\",\"args\":{},\"id\":\"toolu_01PhBp6nuzjb7wbTvVD6r53e\",\"type\":\"tool_call\"}]}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "86ce9dc920022f5c", + "parent_span_id": "e94883ee4a0748f4", + "name": "lookup_benchmark", + "type": "tool", + "agent": "researcher", + "start_offset_ms": 1202.829056, + "duration_ms": 0.418048, + "status": "error", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "98a82b26135e44b8", + "parent_span_id": "49397f8d7856d2ba", + "name": "model", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 1203.599104, + "duration_ms": 1051.271936, + "status": "ok", + "input_preview": "{\"messages\":[{\"content\":\"Compare retention (question 3).\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"c672c207-0adb-458a-abd5-e12496da0886\"},{\"content\":\"I'll look up the retention benchmark (question 3) for both Click", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "57be1131bf87528d", + "parent_span_id": "98a82b26135e44b8", + "name": "ChatOpenAI", + "type": "llm", + "agent": "researcher", + "start_offset_ms": 1203.835904, + "duration_ms": 1050.923264, + "status": "ok", + "input_preview": "[{\"role\": \"system\", \"content\": \"Call lookup_benchmark once for ClickHouse and once for Postgres, then answer in ONE short sentence.\"}, {\"role\": \"user\", \"content\": \"Compare retention (question 3).\"}, {\"role\": \"assistant\", \"content\": \"I'll lo", + "model": "claude-haiku-4-5", + "input_tokens": 839, + "output_tokens": 87, + "litellm": { + "request_id": "chatcmpl-d78f00eb-0f7e-4b29-a1c1-91a709f357de", + "model": "openai/claude-haiku-4-5-20251001", + "model_group": "claude-haiku-4-5", + "provider": "openai", + "key_alias": "research-bot", + "team_alias": "research-agents", + "spend": 0.001274, + "prompt_tokens": 839, + "completion_tokens": 87, + "cache_read_tokens": 0, + "cache_write_tokens": 0, + "latency_ms": 1046, + "ttft_ms": 1046, + "status": "success" + } + }, + { + "span_id": "e2a9435a1c14cea3", + "parent_span_id": "8a556ee09900bc3c", + "name": "tools", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 1209.68704, + "duration_ms": 0.747008, + "status": "ok", + "input_preview": "{\"input\":[{\"name\":\"lookup_benchmark\",\"args\":{\"topic\":\"joins\",\"store\":\"ClickHouse\"},\"id\":\"toolu_01RvNzdrWuJkVnodMpMZSLAz\",\"type\":\"tool_call\"}]}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "4f93b6871b48ecef", + "parent_span_id": "e2a9435a1c14cea3", + "name": "lookup_benchmark", + "type": "tool", + "agent": "researcher", + "start_offset_ms": 1209.918976, + "duration_ms": 0.168192, + "status": "ok", + "input_preview": "{\"topic\":\"joins\",\"store\":\"ClickHouse\"}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "cf5525d2d0f3040e", + "parent_span_id": "8a556ee09900bc3c", + "name": "tools", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 1210.278912, + "duration_ms": 0.760064, + "status": "ok", + "input_preview": "{\"input\":[{\"name\":\"lookup_benchmark\",\"args\":{},\"id\":\"toolu_01K4qPmXFz6qj3LLDsvPhcNy\",\"type\":\"tool_call\"}]}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "059af97bec295cf0", + "parent_span_id": "cf5525d2d0f3040e", + "name": "lookup_benchmark", + "type": "tool", + "agent": "researcher", + "start_offset_ms": 1210.6112, + "duration_ms": 0.24704, + "status": "error", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "7c3f6bcdea7df18c", + "parent_span_id": "8a556ee09900bc3c", + "name": "model", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 1211.208192, + "duration_ms": 968.805888, + "status": "ok", + "input_preview": "{\"messages\":[{\"content\":\"Compare joins (question 9).\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"83bc8232-0b19-43d3-9a5e-6d78a5d4c6b5\"},{\"content\":\"I'll look up the benchmark data for joins on both ClickHouse and Pos", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "66575f940e1d8d9f", + "parent_span_id": "7c3f6bcdea7df18c", + "name": "ChatOpenAI", + "type": "llm", + "agent": "researcher", + "start_offset_ms": 1212.324096, + "duration_ms": 967.556096, + "status": "ok", + "input_preview": "[{\"role\": \"system\", \"content\": \"Call lookup_benchmark once for ClickHouse and once for Postgres, then answer in ONE short sentence.\"}, {\"role\": \"user\", \"content\": \"Compare joins (question 9).\"}, {\"role\": \"assistant\", \"content\": \"I'll look u", + "model": "claude-haiku-4-5", + "input_tokens": 828, + "output_tokens": 83, + "litellm": { + "request_id": "chatcmpl-563f2658-f904-44af-875d-9a9643ea50ee", + "model": "openai/claude-haiku-4-5-20251001", + "model_group": "claude-haiku-4-5", + "provider": "openai", + "key_alias": "research-bot", + "team_alias": "research-agents", + "spend": 0.001243, + "prompt_tokens": 828, + "completion_tokens": 83, + "cache_read_tokens": 0, + "cache_write_tokens": 0, + "latency_ms": 960, + "ttft_ms": 960, + "status": "success" + } + }, + { + "span_id": "ee0f8aad942106ae", + "parent_span_id": "ffb1aafc123f7cbf", + "name": "tools", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 1217.080064, + "duration_ms": 0.70784, + "status": "ok", + "input_preview": "{\"input\":[{\"name\":\"lookup_benchmark\",\"args\":{\"store\":\"ClickHouse\",\"topic\":\"ingest throughput question 0\"},\"id\":\"toolu_01UaLUswR7HVY93frdKhzHX4\",\"type\":\"tool_call\"}]}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "95db7339488d7721", + "parent_span_id": "ee0f8aad942106ae", + "name": "lookup_benchmark", + "type": "tool", + "agent": "researcher", + "start_offset_ms": 1217.318144, + "duration_ms": 0.175872, + "status": "ok", + "input_preview": "{\"store\":\"ClickHouse\",\"topic\":\"ingest throughput question 0\"}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "398d8a15feeb0db9", + "parent_span_id": "ffb1aafc123f7cbf", + "name": "tools", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 1217.599232, + "duration_ms": 0.760832, + "status": "ok", + "input_preview": "{\"input\":[{\"name\":\"lookup_benchmark\",\"args\":{\"store\":\"Postgres\"},\"id\":\"toolu_012iYU68ZdNkQXxbA7ka5GvP\",\"type\":\"tool_call\"}]}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "98e38544cc4ed2a8", + "parent_span_id": "398d8a15feeb0db9", + "name": "lookup_benchmark", + "type": "tool", + "agent": "researcher", + "start_offset_ms": 1217.953024, + "duration_ms": 0.250112, + "status": "error", + "input_preview": "{\"store\":\"Postgres\"}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "6f3d60ffdf7ef9ce", + "parent_span_id": "ffb1aafc123f7cbf", + "name": "model", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 1218.537216, + "duration_ms": 1044.436736, + "status": "ok", + "input_preview": "{\"messages\":[{\"content\":\"Compare ingest throughput (question 0).\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"8bd0af23-8ed1-4178-aff9-7a5763b1c928\"},{\"content\":\"\",\"additional_kwargs\":{\"refusal\":null},\"response_metadat", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "0e34ca6259e3e8c0", + "parent_span_id": "6f3d60ffdf7ef9ce", + "name": "ChatOpenAI", + "type": "llm", + "agent": "researcher", + "start_offset_ms": 1218.773248, + "duration_ms": 1044.088832, + "status": "ok", + "input_preview": "[{\"role\": \"system\", \"content\": \"Call lookup_benchmark once for ClickHouse and once for Postgres, then call check_fact on your conclusion, then answer in ONE short sentence.\"}, {\"role\": \"user\", \"content\": \"Compare ingest throughput (question", + "model": "claude-haiku-4-5", + "input_tokens": 914, + "output_tokens": 89, + "litellm": { + "request_id": "chatcmpl-46e407e8-e0c7-45ad-96bc-8f61270a6640", + "model": "openai/claude-haiku-4-5-20251001", + "model_group": "claude-haiku-4-5", + "provider": "openai", + "key_alias": "research-bot", + "team_alias": "research-agents", + "spend": 0.001359, + "prompt_tokens": 914, + "completion_tokens": 89, + "cache_read_tokens": 0, + "cache_write_tokens": 0, + "latency_ms": 1036, + "ttft_ms": 1036, + "status": "success" + } + }, + { + "span_id": "e6f3479907471837", + "parent_span_id": "6fbea322f87ea515", + "name": "tools", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 1235.660032, + "duration_ms": 0.740096, + "status": "ok", + "input_preview": "{\"input\":[{\"name\":\"lookup_benchmark\",\"args\":{\"store\":\"ClickHouse\",\"topic\":\"compression\"},\"id\":\"toolu_01R6ayWaGRufB8hhTzz2eCBk\",\"type\":\"tool_call\"}]}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "53dd0c2ef37cc475", + "parent_span_id": "e6f3479907471837", + "name": "lookup_benchmark", + "type": "tool", + "agent": "researcher", + "start_offset_ms": 1235.916288, + "duration_ms": 0.177664, + "status": "ok", + "input_preview": "{\"store\":\"ClickHouse\",\"topic\":\"compression\"}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "37bc9ac680a894b3", + "parent_span_id": "6fbea322f87ea515", + "name": "tools", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 1236.209152, + "duration_ms": 0.708864, + "status": "ok", + "input_preview": "{\"input\":[{\"name\":\"lookup_benchmark\",\"args\":{},\"id\":\"toolu_01MVe4SLPi34HqRLCu3Cj9rH\",\"type\":\"tool_call\"}]}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "84306979d3b3f910", + "parent_span_id": "37bc9ac680a894b3", + "name": "lookup_benchmark", + "type": "tool", + "agent": "researcher", + "start_offset_ms": 1236.538112, + "duration_ms": 0.229888, + "status": "error", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "a8dded09bd8531ad", + "parent_span_id": "6fbea322f87ea515", + "name": "model", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 1237.088, + "duration_ms": 972.432128, + "status": "ok", + "input_preview": "{\"messages\":[{\"content\":\"Compare compression (question 5).\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"2701c52d-8721-4a2f-9673-e46179f6a853\"},{\"content\":\"I'll look up the compression benchmarks for both ClickHouse an", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "0ebb8092f3f97294", + "parent_span_id": "a8dded09bd8531ad", + "name": "ChatOpenAI", + "type": "llm", + "agent": "researcher", + "start_offset_ms": 1237.296128, + "duration_ms": 972.062976, + "status": "ok", + "input_preview": "[{\"role\": \"system\", \"content\": \"Call lookup_benchmark once for ClickHouse and once for Postgres, then answer in ONE short sentence.\"}, {\"role\": \"user\", \"content\": \"Compare compression (question 5).\"}, {\"role\": \"assistant\", \"content\": \"I'll ", + "model": "claude-haiku-4-5", + "input_tokens": 827, + "output_tokens": 79, + "litellm": { + "request_id": "chatcmpl-b710911c-5c59-4010-a305-eaa1f2567f31", + "model": "openai/claude-haiku-4-5-20251001", + "model_group": "claude-haiku-4-5", + "provider": "openai", + "key_alias": "research-bot", + "team_alias": "research-agents", + "spend": 0.001222, + "prompt_tokens": 827, + "completion_tokens": 79, + "cache_read_tokens": 0, + "cache_write_tokens": 0, + "latency_ms": 967, + "ttft_ms": 967, + "status": "success" + } + }, + { + "span_id": "c204916b6acf6662", + "parent_span_id": "042d555505c3a65a", + "name": "tools", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 1337.686016, + "duration_ms": 0.606208, + "status": "ok", + "input_preview": "{\"input\":[{\"name\":\"lookup_benchmark\",\"args\":{\"topic\":\"schema changes\",\"store\":\"ClickHouse\"},\"id\":\"toolu_01BSaAngjGswgo1rfiMqBR38\",\"type\":\"tool_call\"}]}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "62ff82b718533251", + "parent_span_id": "c204916b6acf6662", + "name": "lookup_benchmark", + "type": "tool", + "agent": "researcher", + "start_offset_ms": 1337.97504, + "duration_ms": 0.193024, + "status": "ok", + "input_preview": "{\"topic\":\"schema changes\",\"store\":\"ClickHouse\"}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "7f81614e4ddc7a77", + "parent_span_id": "042d555505c3a65a", + "name": "tools", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 1338.407936, + "duration_ms": 0.702208, + "status": "ok", + "input_preview": "{\"input\":[{\"name\":\"lookup_benchmark\",\"args\":{},\"id\":\"toolu_01TY1tyXwoQU12aQXrddEs4v\",\"type\":\"tool_call\"}]}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "5e82032b7c3fb5e3", + "parent_span_id": "7f81614e4ddc7a77", + "name": "lookup_benchmark", + "type": "tool", + "agent": "researcher", + "start_offset_ms": 1338.639104, + "duration_ms": 0.268032, + "status": "error", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "4fa3cdf571fc30fb", + "parent_span_id": "042d555505c3a65a", + "name": "model", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 1339.326208, + "duration_ms": 907.485952, + "status": "ok", + "input_preview": "{\"messages\":[{\"content\":\"Compare schema changes (question 8).\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"4558e9f1-9a1e-4c7b-9df5-14e457ad4f82\"},{\"content\":\"I'll look up the benchmark data for schema changes on both ", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "fa68e281815fde27", + "parent_span_id": "4fa3cdf571fc30fb", + "name": "ChatOpenAI", + "type": "llm", + "agent": "researcher", + "start_offset_ms": 1339.57504, + "duration_ms": 907.12704, + "status": "ok", + "input_preview": "[{\"role\": \"system\", \"content\": \"Call lookup_benchmark once for ClickHouse and once for Postgres, then answer in ONE short sentence.\"}, {\"role\": \"user\", \"content\": \"Compare schema changes (question 8).\"}, {\"role\": \"assistant\", \"content\": \"I'", + "model": "claude-haiku-4-5", + "input_tokens": 832, + "output_tokens": 80, + "litellm": { + "request_id": "chatcmpl-afe091b4-18b5-49c6-8092-4160d185abf2", + "model": "openai/claude-haiku-4-5-20251001", + "model_group": "claude-haiku-4-5", + "provider": "openai", + "key_alias": "research-bot", + "team_alias": "research-agents", + "spend": 0.001232, + "prompt_tokens": 832, + "completion_tokens": 80, + "cache_read_tokens": 0, + "cache_write_tokens": 0, + "latency_ms": 903, + "ttft_ms": 903, + "status": "success" + } + }, + { + "span_id": "0f3c8c6d1b95b8f7", + "parent_span_id": "4bc4c26013a33115", + "name": "tools", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 1544.611072, + "duration_ms": 0.68992, + "status": "ok", + "input_preview": "{\"input\":[{\"name\":\"lookup_benchmark\",\"args\":{\"store\":\"ClickHouse\",\"topic\":\"ingest throughput\"},\"id\":\"toolu_017XDB7FAud42UVNwNFAYaDo\",\"type\":\"tool_call\"}]}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "845d9db17364829c", + "parent_span_id": "0f3c8c6d1b95b8f7", + "name": "lookup_benchmark", + "type": "tool", + "agent": "researcher", + "start_offset_ms": 1544.907008, + "duration_ms": 0.197888, + "status": "ok", + "input_preview": "{\"store\":\"ClickHouse\",\"topic\":\"ingest throughput\"}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "d84e2d5d80a1e5cd", + "parent_span_id": "4bc4c26013a33115", + "name": "tools", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 1545.382912, + "duration_ms": 0.96128, + "status": "ok", + "input_preview": "{\"input\":[{\"name\":\"lookup_benchmark\",\"args\":{},\"id\":\"toolu_016U8j45zbsYtH3UYf3DL51N\",\"type\":\"tool_call\"}]}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "dfe4e1fd79458a7d", + "parent_span_id": "d84e2d5d80a1e5cd", + "name": "lookup_benchmark", + "type": "tool", + "agent": "researcher", + "start_offset_ms": 1545.853184, + "duration_ms": 0.299008, + "status": "error", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "1f1988c9693fede7", + "parent_span_id": "4bc4c26013a33115", + "name": "model", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 1546.571008, + "duration_ms": 980.190976, + "status": "ok", + "input_preview": "{\"messages\":[{\"content\":\"Compare ingest throughput (question 10).\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"c93e34d9-26a7-4f16-af29-04ad300f0905\"},{\"content\":\"I'll look up the ingest throughput benchmark for both C", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "e46d723f65245490", + "parent_span_id": "1f1988c9693fede7", + "name": "ChatOpenAI", + "type": "llm", + "agent": "researcher", + "start_offset_ms": 1546.854144, + "duration_ms": 979.710976, + "status": "ok", + "input_preview": "[{\"role\": \"system\", \"content\": \"Call lookup_benchmark once for ClickHouse and once for Postgres, then call check_fact on your conclusion, then answer in ONE short sentence.\"}, {\"role\": \"user\", \"content\": \"Compare ingest throughput (question", + "model": "claude-haiku-4-5", + "input_tokens": 912, + "output_tokens": 86, + "litellm": { + "request_id": "chatcmpl-44ba88cc-2a57-4159-baf0-4ff9963f0c5c", + "model": "openai/claude-haiku-4-5-20251001", + "model_group": "claude-haiku-4-5", + "provider": "openai", + "key_alias": "research-bot", + "team_alias": "research-agents", + "spend": 0.0013419999999999999, + "prompt_tokens": 912, + "completion_tokens": 86, + "cache_read_tokens": 0, + "cache_write_tokens": 0, + "latency_ms": 974, + "ttft_ms": 974, + "status": "success" + } + }, + { + "span_id": "eb1eea92dffec096", + "parent_span_id": "8a556ee09900bc3c", + "name": "tools", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 2180.201984, + "duration_ms": 0.648192, + "status": "ok", + "input_preview": "{\"input\":[{\"name\":\"lookup_benchmark\",\"args\":{\"topic\":\"joins\",\"store\":\"Postgres\"},\"id\":\"toolu_01JHhKgKTUs2WS7qtKM3wCj1\",\"type\":\"tool_call\"}]}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "3ca66f2ec7e25124", + "parent_span_id": "eb1eea92dffec096", + "name": "lookup_benchmark", + "type": "tool", + "agent": "researcher", + "start_offset_ms": 2180.49408, + "duration_ms": 0.21504, + "status": "ok", + "input_preview": "{\"topic\":\"joins\",\"store\":\"Postgres\"}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "1012810e08a39c3a", + "parent_span_id": "8a556ee09900bc3c", + "name": "model", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 2180.987136, + "duration_ms": 881.167872, + "status": "ok", + "input_preview": "{\"messages\":[{\"content\":\"Compare joins (question 9).\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"83bc8232-0b19-43d3-9a5e-6d78a5d4c6b5\"},{\"content\":\"I'll look up the benchmark data for joins on both ClickHouse and Pos", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "debcd76428458d09", + "parent_span_id": "1012810e08a39c3a", + "name": "ChatOpenAI", + "type": "llm", + "agent": "researcher", + "start_offset_ms": 2181.210112, + "duration_ms": 880.774912, + "status": "ok", + "input_preview": "[{\"role\": \"system\", \"content\": \"Call lookup_benchmark once for ClickHouse and once for Postgres, then answer in ONE short sentence.\"}, {\"role\": \"user\", \"content\": \"Compare joins (question 9).\"}, {\"role\": \"assistant\", \"content\": \"I'll look u", + "model": "claude-haiku-4-5", + "input_tokens": 934, + "output_tokens": 48, + "litellm": { + "request_id": "chatcmpl-7a5abd7c-f49a-4f64-8d45-b2c839c1cbde", + "model": "openai/claude-haiku-4-5-20251001", + "model_group": "claude-haiku-4-5", + "provider": "openai", + "key_alias": "research-bot", + "team_alias": "research-agents", + "spend": 0.001174, + "prompt_tokens": 934, + "completion_tokens": 48, + "cache_read_tokens": 0, + "cache_write_tokens": 0, + "latency_ms": 875, + "ttft_ms": 875, + "status": "success" + } + }, + { + "span_id": "33f806f197bb8f31", + "parent_span_id": "6fbea322f87ea515", + "name": "tools", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 2209.719296, + "duration_ms": 0.584704, + "status": "ok", + "input_preview": "{\"input\":[{\"name\":\"lookup_benchmark\",\"args\":{\"store\":\"Postgres\",\"topic\":\"compression\"},\"id\":\"toolu_01HHBqgQqcHMd8evwgJuam2z\",\"type\":\"tool_call\"}]}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "1b360c8d6e6c787a", + "parent_span_id": "33f806f197bb8f31", + "name": "lookup_benchmark", + "type": "tool", + "agent": "researcher", + "start_offset_ms": 2209.995008, + "duration_ms": 0.182272, + "status": "ok", + "input_preview": "{\"store\":\"Postgres\",\"topic\":\"compression\"}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "09d645b80fbb604e", + "parent_span_id": "6fbea322f87ea515", + "name": "model", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 2210.438144, + "duration_ms": 976.382976, + "status": "ok", + "input_preview": "{\"messages\":[{\"content\":\"Compare compression (question 5).\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"2701c52d-8721-4a2f-9673-e46179f6a853\"},{\"content\":\"I'll look up the compression benchmarks for both ClickHouse an", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "13b096b9dae2d701", + "parent_span_id": "09d645b80fbb604e", + "name": "ChatOpenAI", + "type": "llm", + "agent": "researcher", + "start_offset_ms": 2210.657024, + "duration_ms": 976.033024, + "status": "ok", + "input_preview": "[{\"role\": \"system\", \"content\": \"Call lookup_benchmark once for ClickHouse and once for Postgres, then answer in ONE short sentence.\"}, {\"role\": \"user\", \"content\": \"Compare compression (question 5).\"}, {\"role\": \"assistant\", \"content\": \"I'll ", + "model": "claude-haiku-4-5", + "input_tokens": 929, + "output_tokens": 46, + "litellm": { + "request_id": "chatcmpl-5c9e8dbb-ecc2-429f-a9c0-a32a6dd21561", + "model": "openai/claude-haiku-4-5-20251001", + "model_group": "claude-haiku-4-5", + "provider": "openai", + "key_alias": "research-bot", + "team_alias": "research-agents", + "spend": 0.0011589999999999999, + "prompt_tokens": 929, + "completion_tokens": 46, + "cache_read_tokens": 0, + "cache_write_tokens": 0, + "latency_ms": 971, + "ttft_ms": 971, + "status": "success" + } + }, + { + "span_id": "9668389035ccda64", + "parent_span_id": "03f611cf9bfc9ea4", + "name": "tools", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 2232.43904, + "duration_ms": 0.559104, + "status": "ok", + "input_preview": "{\"input\":[{\"name\":\"lookup_benchmark\",\"args\":{\"topic\":\"multi-tenancy\",\"store\":\"Postgres\"},\"id\":\"toolu_01JAfdCsAttQJNk9XYxypyYa\",\"type\":\"tool_call\"}]}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "c3d2f25141a98eb8", + "parent_span_id": "9668389035ccda64", + "name": "lookup_benchmark", + "type": "tool", + "agent": "researcher", + "start_offset_ms": 2232.690176, + "duration_ms": 0.180992, + "status": "ok", + "input_preview": "{\"topic\":\"multi-tenancy\",\"store\":\"Postgres\"}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "67b2045e76a73cc7", + "parent_span_id": "03f611cf9bfc9ea4", + "name": "model", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 2233.128192, + "duration_ms": 1048.211968, + "status": "ok", + "input_preview": "{\"messages\":[{\"content\":\"Compare multi-tenancy (question 4).\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"6f39cb28-7f23-4cd1-b74a-98eb05d175fd\"},{\"content\":\"I'll look up the multi-tenancy benchmarks for both ClickHous", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "c55bf581d431c15d", + "parent_span_id": "67b2045e76a73cc7", + "name": "ChatOpenAI", + "type": "llm", + "agent": "researcher", + "start_offset_ms": 2233.336064, + "duration_ms": 1047.846912, + "status": "ok", + "input_preview": "[{\"role\": \"system\", \"content\": \"Call lookup_benchmark once for ClickHouse and once for Postgres, then answer in ONE short sentence.\"}, {\"role\": \"user\", \"content\": \"Compare multi-tenancy (question 4).\"}, {\"role\": \"assistant\", \"content\": \"I'l", + "model": "claude-haiku-4-5", + "input_tokens": 950, + "output_tokens": 66, + "litellm": { + "request_id": "chatcmpl-4c7b01ab-f44c-4b29-9a90-b3501daee307", + "model": "openai/claude-haiku-4-5-20251001", + "model_group": "claude-haiku-4-5", + "provider": "openai", + "key_alias": "research-bot", + "team_alias": "research-agents", + "spend": 0.00128, + "prompt_tokens": 950, + "completion_tokens": 66, + "cache_read_tokens": 0, + "cache_write_tokens": 0, + "latency_ms": 1042, + "ttft_ms": 1042, + "status": "success" + } + }, + { + "span_id": "40823f352c0bc917", + "parent_span_id": "042d555505c3a65a", + "name": "tools", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 2246.971136, + "duration_ms": 0.545024, + "status": "ok", + "input_preview": "{\"input\":[{\"name\":\"lookup_benchmark\",\"args\":{\"topic\":\"schema changes\",\"store\":\"Postgres\"},\"id\":\"toolu_01UebHCJ4T4tV9bvRQLxLKi5\",\"type\":\"tool_call\"}]}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "298b6cf2f3be5dce", + "parent_span_id": "40823f352c0bc917", + "name": "lookup_benchmark", + "type": "tool", + "agent": "researcher", + "start_offset_ms": 2247.197184, + "duration_ms": 0.189952, + "status": "ok", + "input_preview": "{\"topic\":\"schema changes\",\"store\":\"Postgres\"}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "b7c7d3bb5226d077", + "parent_span_id": "042d555505c3a65a", + "name": "model", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 2247.642112, + "duration_ms": 899.931904, + "status": "ok", + "input_preview": "{\"messages\":[{\"content\":\"Compare schema changes (question 8).\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"4558e9f1-9a1e-4c7b-9df5-14e457ad4f82\"},{\"content\":\"I'll look up the benchmark data for schema changes on both ", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "b7c18743204be618", + "parent_span_id": "b7c7d3bb5226d077", + "name": "ChatOpenAI", + "type": "llm", + "agent": "researcher", + "start_offset_ms": 2247.833088, + "duration_ms": 899.610112, + "status": "ok", + "input_preview": "[{\"role\": \"system\", \"content\": \"Call lookup_benchmark once for ClickHouse and once for Postgres, then answer in ONE short sentence.\"}, {\"role\": \"user\", \"content\": \"Compare schema changes (question 8).\"}, {\"role\": \"assistant\", \"content\": \"I'", + "model": "claude-haiku-4-5", + "input_tokens": 936, + "output_tokens": 47, + "litellm": { + "request_id": "chatcmpl-c7e73498-e07d-4ee7-804a-8c8fcb518108", + "model": "openai/claude-haiku-4-5-20251001", + "model_group": "claude-haiku-4-5", + "provider": "openai", + "key_alias": "research-bot", + "team_alias": "research-agents", + "spend": 0.001171, + "prompt_tokens": 936, + "completion_tokens": 47, + "cache_read_tokens": 0, + "cache_write_tokens": 0, + "latency_ms": 895, + "ttft_ms": 895, + "status": "success" + } + }, + { + "span_id": "3c16f5aa4a7be5a3", + "parent_span_id": "49397f8d7856d2ba", + "name": "tools", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 2255.033088, + "duration_ms": 0.540928, + "status": "ok", + "input_preview": "{\"input\":[{\"name\":\"lookup_benchmark\",\"args\":{\"store\":\"Postgres\",\"topic\":\"retention (question 3)\"},\"id\":\"toolu_015RETCBXbiEpLSJrMxs9Zmo\",\"type\":\"tool_call\"}]}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "105b73f1d31ee52a", + "parent_span_id": "3c16f5aa4a7be5a3", + "name": "lookup_benchmark", + "type": "tool", + "agent": "researcher", + "start_offset_ms": 2255.257088, + "duration_ms": 0.188928, + "status": "ok", + "input_preview": "{\"store\":\"Postgres\",\"topic\":\"retention (question 3)\"}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "fadbe6d96a1873f4", + "parent_span_id": "49397f8d7856d2ba", + "name": "model", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 2255.70432, + "duration_ms": 938.539776, + "status": "ok", + "input_preview": "{\"messages\":[{\"content\":\"Compare retention (question 3).\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"c672c207-0adb-458a-abd5-e12496da0886\"},{\"content\":\"I'll look up the retention benchmark (question 3) for both Click", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "0fba6a6a6f83d0b8", + "parent_span_id": "fadbe6d96a1873f4", + "name": "ChatOpenAI", + "type": "llm", + "agent": "researcher", + "start_offset_ms": 2255.902976, + "duration_ms": 938.221056, + "status": "ok", + "input_preview": "[{\"role\": \"system\", \"content\": \"Call lookup_benchmark once for ClickHouse and once for Postgres, then answer in ONE short sentence.\"}, {\"role\": \"user\", \"content\": \"Compare retention (question 3).\"}, {\"role\": \"assistant\", \"content\": \"I'll lo", + "model": "claude-haiku-4-5", + "input_tokens": 953, + "output_tokens": 50, + "litellm": { + "request_id": "chatcmpl-079b3671-6e0e-4296-abf7-e9e3e1c27fcf", + "model": "openai/claude-haiku-4-5-20251001", + "model_group": "claude-haiku-4-5", + "provider": "openai", + "key_alias": "research-bot", + "team_alias": "research-agents", + "spend": 0.001203, + "prompt_tokens": 953, + "completion_tokens": 50, + "cache_read_tokens": 0, + "cache_write_tokens": 0, + "latency_ms": 933, + "ttft_ms": 933, + "status": "success" + } + }, + { + "span_id": "6110ac9f90c2752e", + "parent_span_id": "ffb1aafc123f7cbf", + "name": "tools", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 2263.136256, + "duration_ms": 0.522752, + "status": "ok", + "input_preview": "{\"input\":[{\"name\":\"lookup_benchmark\",\"args\":{\"store\":\"Postgres\",\"topic\":\"ingest throughput question 0\"},\"id\":\"toolu_01TBx89BvBRuExmz2svZ1yif\",\"type\":\"tool_call\"}]}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "a25f4e268076e364", + "parent_span_id": "6110ac9f90c2752e", + "name": "lookup_benchmark", + "type": "tool", + "agent": "researcher", + "start_offset_ms": 2263.355136, + "duration_ms": 0.163072, + "status": "ok", + "input_preview": "{\"store\":\"Postgres\",\"topic\":\"ingest throughput question 0\"}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "63ab0a445ddf082b", + "parent_span_id": "ffb1aafc123f7cbf", + "name": "model", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 2263.825152, + "duration_ms": 1080.289792, + "status": "ok", + "input_preview": "{\"messages\":[{\"content\":\"Compare ingest throughput (question 0).\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"8bd0af23-8ed1-4178-aff9-7a5763b1c928\"},{\"content\":\"\",\"additional_kwargs\":{\"refusal\":null},\"response_metadat", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "f0ca43516ddabae8", + "parent_span_id": "63ab0a445ddf082b", + "name": "ChatOpenAI", + "type": "llm", + "agent": "researcher", + "start_offset_ms": 2264.07808, + "duration_ms": 1079.866112, + "status": "ok", + "input_preview": "[{\"role\": \"system\", \"content\": \"Call lookup_benchmark once for ClickHouse and once for Postgres, then call check_fact on your conclusion, then answer in ONE short sentence.\"}, {\"role\": \"user\", \"content\": \"Compare ingest throughput (question", + "model": "claude-haiku-4-5", + "input_tokens": 1032, + "output_tokens": 102, + "litellm": { + "request_id": "chatcmpl-a6a337fb-ed87-4945-8d38-c2618dbf7bb5", + "model": "openai/claude-haiku-4-5-20251001", + "model_group": "claude-haiku-4-5", + "provider": "openai", + "key_alias": "research-bot", + "team_alias": "research-agents", + "spend": 0.001542, + "prompt_tokens": 1032, + "completion_tokens": 102, + "cache_read_tokens": 0, + "cache_write_tokens": 0, + "latency_ms": 1075, + "ttft_ms": 1075, + "status": "success" + } + }, + { + "span_id": "3e6908531fde1417", + "parent_span_id": "a1e8b8b06038a09e", + "name": "tools", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 2428.333056, + "duration_ms": 0.584192, + "status": "ok", + "input_preview": "{\"input\":[{\"name\":\"lookup_benchmark\",\"args\":{\"topic\":\"storage cost\",\"store\":\"Postgres\"},\"id\":\"toolu_0179gVEsfBWz2fJApaowwUsR\",\"type\":\"tool_call\"}]}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "8604b658e1d09660", + "parent_span_id": "3e6908531fde1417", + "name": "lookup_benchmark", + "type": "tool", + "agent": "researcher", + "start_offset_ms": 2428.614144, + "duration_ms": 0.176896, + "status": "ok", + "input_preview": "{\"topic\":\"storage cost\",\"store\":\"Postgres\"}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "5e866cac131c336b", + "parent_span_id": "a1e8b8b06038a09e", + "name": "model", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 2429.062144, + "duration_ms": 712.163072, + "status": "ok", + "input_preview": "{\"messages\":[{\"content\":\"Compare storage cost (question 11).\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"6c39cf1d-8ce5-459d-96a6-e2f6ba6fccfe\"},{\"content\":\"I'll look up the storage cost benchmark for both ClickHouse ", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "0b2947532de8748d", + "parent_span_id": "5e866cac131c336b", + "name": "ChatOpenAI", + "type": "llm", + "agent": "researcher", + "start_offset_ms": 2429.29408, + "duration_ms": 711.782144, + "status": "ok", + "input_preview": "[{\"role\": \"system\", \"content\": \"Call lookup_benchmark once for ClickHouse and once for Postgres, then answer in ONE short sentence.\"}, {\"role\": \"user\", \"content\": \"Compare storage cost (question 11).\"}, {\"role\": \"assistant\", \"content\": \"I'l", + "model": "claude-haiku-4-5", + "input_tokens": 938, + "output_tokens": 23, + "litellm": { + "request_id": "chatcmpl-c4edc774-323f-45e1-bc3a-8d0eb97d16f0", + "model": "openai/claude-haiku-4-5-20251001", + "model_group": "claude-haiku-4-5", + "provider": "openai", + "key_alias": "research-bot", + "team_alias": "research-agents", + "spend": 0.001053, + "prompt_tokens": 938, + "completion_tokens": 23, + "cache_read_tokens": 0, + "cache_write_tokens": 0, + "latency_ms": 706, + "ttft_ms": 706, + "status": "success" + } + }, + { + "span_id": "b8bca8de1d96cc40", + "parent_span_id": "4bc4c26013a33115", + "name": "tools", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 2526.972928, + "duration_ms": 0.666368, + "status": "ok", + "input_preview": "{\"input\":[{\"name\":\"lookup_benchmark\",\"args\":{\"store\":\"Postgres\",\"topic\":\"ingest throughput\"},\"id\":\"toolu_011QUPLx4iP9BABp9m4hKz5x\",\"type\":\"tool_call\"}]}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "659942f79f12abd7", + "parent_span_id": "b8bca8de1d96cc40", + "name": "lookup_benchmark", + "type": "tool", + "agent": "researcher", + "start_offset_ms": 2527.28192, + "duration_ms": 0.211968, + "status": "ok", + "input_preview": "{\"store\":\"Postgres\",\"topic\":\"ingest throughput\"}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "f25fc8653bc1fb53", + "parent_span_id": "4bc4c26013a33115", + "name": "model", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 2527.796992, + "duration_ms": 1241.76, + "status": "ok", + "input_preview": "{\"messages\":[{\"content\":\"Compare ingest throughput (question 10).\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"c93e34d9-26a7-4f16-af29-04ad300f0905\"},{\"content\":\"I'll look up the ingest throughput benchmark for both C", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "48939b5fea74fe64", + "parent_span_id": "f25fc8653bc1fb53", + "name": "ChatOpenAI", + "type": "llm", + "agent": "researcher", + "start_offset_ms": 2528.058112, + "duration_ms": 1241.334784, + "status": "ok", + "input_preview": "[{\"role\": \"system\", \"content\": \"Call lookup_benchmark once for ClickHouse and once for Postgres, then call check_fact on your conclusion, then answer in ONE short sentence.\"}, {\"role\": \"user\", \"content\": \"Compare ingest throughput (question", + "model": "claude-haiku-4-5", + "input_tokens": 1024, + "output_tokens": 90, + "litellm": { + "request_id": "chatcmpl-35af2db3-6d09-48c8-8833-6343af4ba2fd", + "model": "openai/claude-haiku-4-5-20251001", + "model_group": "claude-haiku-4-5", + "provider": "openai", + "key_alias": "research-bot", + "team_alias": "research-agents", + "spend": 0.001474, + "prompt_tokens": 1024, + "completion_tokens": 90, + "cache_read_tokens": 0, + "cache_write_tokens": 0, + "latency_ms": 1235, + "ttft_ms": 1235, + "status": "success" + } + }, + { + "span_id": "9c182a974b27b339", + "parent_span_id": "ffb1aafc123f7cbf", + "name": "tools", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 3344.328192, + "duration_ms": 4698.794752, + "status": "ok", + "input_preview": "{\"input\":[{\"name\":\"check_fact\",\"args\":{\"claim\":\"ClickHouse has faster ingest throughput than Postgres, with ClickHouse rated as fast (p50 84ms) compared to Postgres rated as ok (p50 84ms)\"},\"id\":\"toolu_01RYBo12yuVNeAMGPKeeTU8W\",\"type\":\"tool", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "16d2a4ddd9cd75eb", + "parent_span_id": "9c182a974b27b339", + "name": "check_fact", + "type": "tool", + "agent": "researcher", + "start_offset_ms": 3344.649984, + "duration_ms": 4698.307072, + "status": "ok", + "input_preview": "{\"claim\":\"ClickHouse has faster ingest throughput than Postgres, with ClickHouse rated as fast (p50 84ms) compared to Postgres rated as ok (p50 84ms)\"}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "51a224d31e19f7f3", + "parent_span_id": "16d2a4ddd9cd75eb", + "name": "fact_checker", + "type": "agent", + "agent": "fact_checker", + "start_offset_ms": 3344.890112, + "duration_ms": 4697.988096, + "status": "ok", + "input_preview": "[{\"role\": \"user\", \"content\": \"ClickHouse has faster ingest throughput than Postgres, with ClickHouse rated as fast (p50 84ms) compared to Postgres rated as ok (p50 84ms)\"}]", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "263738613df3c778", + "parent_span_id": "51a224d31e19f7f3", + "name": "model", + "type": "chain", + "agent": "fact_checker", + "start_offset_ms": 3345.30816, + "duration_ms": 786.53312, + "status": "ok", + "input_preview": "{\"messages\":[{\"content\":\"ClickHouse has faster ingest throughput than Postgres, with ClickHouse rated as fast (p50 84ms) compared to Postgres rated as ok (p50 84ms)\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"c1a8658", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "26162e38f6e4fb66", + "parent_span_id": "263738613df3c778", + "name": "ChatOpenAI", + "type": "llm", + "agent": "fact_checker", + "start_offset_ms": 3346.041088, + "duration_ms": 785.647104, + "status": "ok", + "input_preview": "[{\"role\": \"system\", \"content\": \"Verify the claim with verify_claim once, then answer 'ok' or 'wrong' in one word.\"}, {\"role\": \"user\", \"content\": \"ClickHouse has faster ingest throughput than Postgres, with ClickHouse rated as fast (p50 84ms", + "model": "claude-haiku-4-5", + "input_tokens": 627, + "output_tokens": 60, + "litellm": { + "request_id": "chatcmpl-edbca34e-9078-48c7-abd7-8a9f4582870f", + "model": "openai/claude-haiku-4-5-20251001", + "model_group": "claude-haiku-4-5", + "provider": "openai", + "key_alias": "research-bot", + "team_alias": "research-agents", + "spend": 0.000927, + "prompt_tokens": 627, + "completion_tokens": 60, + "cache_read_tokens": 0, + "cache_write_tokens": 0, + "latency_ms": 781, + "ttft_ms": 781, + "status": "success" + } + }, + { + "span_id": "bf30867ce7910d16", + "parent_span_id": "4bc4c26013a33115", + "name": "tools", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 3769.766144, + "duration_ms": 6859.677952, + "status": "ok", + "input_preview": "{\"input\":[{\"name\":\"check_fact\",\"args\":{\"claim\":\"ClickHouse has faster ingest throughput (fast, p50 51ms) than Postgres (ok, p50 51ms)\"},\"id\":\"toolu_0156JMekNsuByLWS9LgD467H\",\"type\":\"tool_call\"}]}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "5c98a6ea86360aa7", + "parent_span_id": "bf30867ce7910d16", + "name": "check_fact", + "type": "tool", + "agent": "researcher", + "start_offset_ms": 3770.066176, + "duration_ms": 6859.212032, + "status": "ok", + "input_preview": "{\"claim\":\"ClickHouse has faster ingest throughput (fast, p50 51ms) than Postgres (ok, p50 51ms)\"}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "8feaefa0bc4965be", + "parent_span_id": "5c98a6ea86360aa7", + "name": "fact_checker", + "type": "agent", + "agent": "fact_checker", + "start_offset_ms": 3770.310144, + "duration_ms": 6858.87488, + "status": "ok", + "input_preview": "[{\"role\": \"user\", \"content\": \"ClickHouse has faster ingest throughput (fast, p50 51ms) than Postgres (ok, p50 51ms)\"}]", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "b746d9d381022c15", + "parent_span_id": "8feaefa0bc4965be", + "name": "model", + "type": "chain", + "agent": "fact_checker", + "start_offset_ms": 3770.621952, + "duration_ms": 796.114176, + "status": "ok", + "input_preview": "{\"messages\":[{\"content\":\"ClickHouse has faster ingest throughput (fast, p50 51ms) than Postgres (ok, p50 51ms)\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"9d0c66ae-ed19-44c9-8b7e-be0ae90b6019\"}]}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "e4b9cea76a0d8d31", + "parent_span_id": "b746d9d381022c15", + "name": "ChatOpenAI", + "type": "llm", + "agent": "fact_checker", + "start_offset_ms": 3770.832128, + "duration_ms": 795.727872, + "status": "ok", + "input_preview": "[{\"role\": \"system\", \"content\": \"Verify the claim with verify_claim once, then answer 'ok' or 'wrong' in one word.\"}, {\"role\": \"user\", \"content\": \"ClickHouse has faster ingest throughput (fast, p50 51ms) than Postgres (ok, p50 51ms)\"}]", + "model": "claude-haiku-4-5", + "input_tokens": 615, + "output_tokens": 60, + "litellm": { + "request_id": "chatcmpl-855924db-53e0-4e18-8f06-8d2cb0b58007", + "model": "openai/claude-haiku-4-5-20251001", + "model_group": "claude-haiku-4-5", + "provider": "openai", + "key_alias": "research-bot", + "team_alias": "research-agents", + "spend": 0.000915, + "prompt_tokens": 615, + "completion_tokens": 60, + "cache_read_tokens": 0, + "cache_write_tokens": 0, + "latency_ms": 790, + "ttft_ms": 790, + "status": "success" + } + }, + { + "span_id": "1c17cffbd43b34d7", + "parent_span_id": "51a224d31e19f7f3", + "name": "tools", + "type": "chain", + "agent": "fact_checker", + "start_offset_ms": 4132.04096, + "duration_ms": 0.795136, + "status": "ok", + "input_preview": "{\"input\":[{\"name\":\"verify_claim\",\"args\":{},\"id\":\"toolu_01PmKqDzGSF7Ro3VkLMxRv7v\",\"type\":\"tool_call\"}]}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "89f28493ea2cdb94", + "parent_span_id": "1c17cffbd43b34d7", + "name": "verify_claim", + "type": "tool", + "agent": "fact_checker", + "start_offset_ms": 4132.351232, + "duration_ms": 0.283904, + "status": "error", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "eb7c697a8563ac30", + "parent_span_id": "51a224d31e19f7f3", + "name": "model", + "type": "chain", + "agent": "fact_checker", + "start_offset_ms": 4132.987136, + "duration_ms": 901.867008, + "status": "ok", + "input_preview": "{\"messages\":[{\"content\":\"ClickHouse has faster ingest throughput than Postgres, with ClickHouse rated as fast (p50 84ms) compared to Postgres rated as ok (p50 84ms)\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"c1a8658", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "5feef5ad987b1f20", + "parent_span_id": "eb7c697a8563ac30", + "name": "ChatOpenAI", + "type": "llm", + "agent": "fact_checker", + "start_offset_ms": 4133.216256, + "duration_ms": 901.469952, + "status": "ok", + "input_preview": "[{\"role\": \"system\", \"content\": \"Verify the claim with verify_claim once, then answer 'ok' or 'wrong' in one word.\"}, {\"role\": \"user\", \"content\": \"ClickHouse has faster ingest throughput than Postgres, with ClickHouse rated as fast (p50 84ms", + "model": "claude-haiku-4-5", + "input_tokens": 707, + "output_tokens": 60, + "litellm": { + "request_id": "chatcmpl-657fe258-62c2-44cd-b2fc-ac49db6fabe3", + "model": "openai/claude-haiku-4-5-20251001", + "model_group": "claude-haiku-4-5", + "provider": "openai", + "key_alias": "research-bot", + "team_alias": "research-agents", + "spend": 0.001007, + "prompt_tokens": 707, + "completion_tokens": 60, + "cache_read_tokens": 0, + "cache_write_tokens": 0, + "latency_ms": 891, + "ttft_ms": 891, + "status": "success" + } + }, + { + "span_id": "449f25412adbd28f", + "parent_span_id": "8feaefa0bc4965be", + "name": "tools", + "type": "chain", + "agent": "fact_checker", + "start_offset_ms": 4566.943232, + "duration_ms": 0.877824, + "status": "ok", + "input_preview": "{\"input\":[{\"name\":\"verify_claim\",\"args\":{},\"id\":\"toolu_017eXQyypsE9G9zXsj9Y8ajF\",\"type\":\"tool_call\"}]}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "aef1a1f15b5d7f5b", + "parent_span_id": "449f25412adbd28f", + "name": "verify_claim", + "type": "tool", + "agent": "fact_checker", + "start_offset_ms": 4567.258112, + "duration_ms": 0.318976, + "status": "error", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "46818b4693a3f76d", + "parent_span_id": "8feaefa0bc4965be", + "name": "model", + "type": "chain", + "agent": "fact_checker", + "start_offset_ms": 4568.013056, + "duration_ms": 865.807104, + "status": "ok", + "input_preview": "{\"messages\":[{\"content\":\"ClickHouse has faster ingest throughput (fast, p50 51ms) than Postgres (ok, p50 51ms)\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"9d0c66ae-ed19-44c9-8b7e-be0ae90b6019\"},{\"content\":\"\",\"additio", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "7e5425f54c786eb7", + "parent_span_id": "46818b4693a3f76d", + "name": "ChatOpenAI", + "type": "llm", + "agent": "fact_checker", + "start_offset_ms": 4568.477184, + "duration_ms": 865.179904, + "status": "ok", + "input_preview": "[{\"role\": \"system\", \"content\": \"Verify the claim with verify_claim once, then answer 'ok' or 'wrong' in one word.\"}, {\"role\": \"user\", \"content\": \"ClickHouse has faster ingest throughput (fast, p50 51ms) than Postgres (ok, p50 51ms)\"}, {\"rol", + "model": "claude-haiku-4-5", + "input_tokens": 695, + "output_tokens": 60, + "litellm": { + "request_id": "chatcmpl-3c2957fe-732d-4f5a-ba69-070abdcc38c0", + "model": "openai/claude-haiku-4-5-20251001", + "model_group": "claude-haiku-4-5", + "provider": "openai", + "key_alias": "research-bot", + "team_alias": "research-agents", + "spend": 0.000995, + "prompt_tokens": 695, + "completion_tokens": 60, + "cache_read_tokens": 0, + "cache_write_tokens": 0, + "latency_ms": 859, + "ttft_ms": 859, + "status": "success" + } + }, + { + "span_id": "ab19a3a60dea94a5", + "parent_span_id": "51a224d31e19f7f3", + "name": "tools", + "type": "chain", + "agent": "fact_checker", + "start_offset_ms": 5035.05408, + "duration_ms": 0.785152, + "status": "ok", + "input_preview": "{\"input\":[{\"name\":\"verify_claim\",\"args\":{},\"id\":\"toolu_01ATAjLVTdnytZoCuos7HQs7\",\"type\":\"tool_call\"}]}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "5f53e6faca69b7de", + "parent_span_id": "ab19a3a60dea94a5", + "name": "verify_claim", + "type": "tool", + "agent": "fact_checker", + "start_offset_ms": 5035.332096, + "duration_ms": 0.305152, + "status": "error", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "4391ec180f69212e", + "parent_span_id": "51a224d31e19f7f3", + "name": "model", + "type": "chain", + "agent": "fact_checker", + "start_offset_ms": 5036.003072, + "duration_ms": 856.84224, + "status": "ok", + "input_preview": "{\"messages\":[{\"content\":\"ClickHouse has faster ingest throughput than Postgres, with ClickHouse rated as fast (p50 84ms) compared to Postgres rated as ok (p50 84ms)\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"c1a8658", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "9b352033ec99891b", + "parent_span_id": "4391ec180f69212e", + "name": "ChatOpenAI", + "type": "llm", + "agent": "fact_checker", + "start_offset_ms": 5036.255232, + "duration_ms": 856.406016, + "status": "ok", + "input_preview": "[{\"role\": \"system\", \"content\": \"Verify the claim with verify_claim once, then answer 'ok' or 'wrong' in one word.\"}, {\"role\": \"user\", \"content\": \"ClickHouse has faster ingest throughput than Postgres, with ClickHouse rated as fast (p50 84ms", + "model": "claude-haiku-4-5", + "input_tokens": 795, + "output_tokens": 60, + "litellm": { + "request_id": "chatcmpl-5911e772-31c2-4b17-b468-6a227c48ee6a", + "model": "openai/claude-haiku-4-5-20251001", + "model_group": "claude-haiku-4-5", + "provider": "openai", + "key_alias": "research-bot", + "team_alias": "research-agents", + "spend": 0.001095, + "prompt_tokens": 795, + "completion_tokens": 60, + "cache_read_tokens": 0, + "cache_write_tokens": 0, + "latency_ms": 846, + "ttft_ms": 846, + "status": "success" + } + }, + { + "span_id": "68a537f3863b70da", + "parent_span_id": "8feaefa0bc4965be", + "name": "tools", + "type": "chain", + "agent": "fact_checker", + "start_offset_ms": 5434.029056, + "duration_ms": 0.772096, + "status": "ok", + "input_preview": "{\"input\":[{\"name\":\"verify_claim\",\"args\":{},\"id\":\"toolu_01GJGayEbSUaraG6jQsLqHHV\",\"type\":\"tool_call\"}]}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "720f818c5e48ed15", + "parent_span_id": "68a537f3863b70da", + "name": "verify_claim", + "type": "tool", + "agent": "fact_checker", + "start_offset_ms": 5434.324992, + "duration_ms": 0.281088, + "status": "error", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "8fa0feeeb5905661", + "parent_span_id": "8feaefa0bc4965be", + "name": "model", + "type": "chain", + "agent": "fact_checker", + "start_offset_ms": 5434.936064, + "duration_ms": 897.363968, + "status": "ok", + "input_preview": "{\"messages\":[{\"content\":\"ClickHouse has faster ingest throughput (fast, p50 51ms) than Postgres (ok, p50 51ms)\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"9d0c66ae-ed19-44c9-8b7e-be0ae90b6019\"},{\"content\":\"\",\"additio", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "ed9378cc67e1d2bc", + "parent_span_id": "8fa0feeeb5905661", + "name": "ChatOpenAI", + "type": "llm", + "agent": "fact_checker", + "start_offset_ms": 5435.160064, + "duration_ms": 896.971008, + "status": "ok", + "input_preview": "[{\"role\": \"system\", \"content\": \"Verify the claim with verify_claim once, then answer 'ok' or 'wrong' in one word.\"}, {\"role\": \"user\", \"content\": \"ClickHouse has faster ingest throughput (fast, p50 51ms) than Postgres (ok, p50 51ms)\"}, {\"rol", + "model": "claude-haiku-4-5", + "input_tokens": 783, + "output_tokens": 60, + "litellm": { + "request_id": "chatcmpl-e0aa7109-6024-4604-8a3b-48158cf4129f", + "model": "openai/claude-haiku-4-5-20251001", + "model_group": "claude-haiku-4-5", + "provider": "openai", + "key_alias": "research-bot", + "team_alias": "research-agents", + "spend": 0.001083, + "prompt_tokens": 783, + "completion_tokens": 60, + "cache_read_tokens": 0, + "cache_write_tokens": 0, + "latency_ms": 891, + "ttft_ms": 891, + "status": "success" + } + }, + { + "span_id": "b8568ad437cc7c6f", + "parent_span_id": "51a224d31e19f7f3", + "name": "tools", + "type": "chain", + "agent": "fact_checker", + "start_offset_ms": 5893.080064, + "duration_ms": 0.772864, + "status": "ok", + "input_preview": "{\"input\":[{\"name\":\"verify_claim\",\"args\":{},\"id\":\"toolu_019xqED1EHc6Z6VwWY2zuPVv\",\"type\":\"tool_call\"}]}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "e89ae09beec1b46c", + "parent_span_id": "b8568ad437cc7c6f", + "name": "verify_claim", + "type": "tool", + "agent": "fact_checker", + "start_offset_ms": 5893.383936, + "duration_ms": 0.272128, + "status": "error", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "3572e33ffdea5492", + "parent_span_id": "51a224d31e19f7f3", + "name": "model", + "type": "chain", + "agent": "fact_checker", + "start_offset_ms": 5894.023168, + "duration_ms": 934.276864, + "status": "ok", + "input_preview": "{\"messages\":[{\"content\":\"ClickHouse has faster ingest throughput than Postgres, with ClickHouse rated as fast (p50 84ms) compared to Postgres rated as ok (p50 84ms)\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"c1a8658", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "c19ded5674612356", + "parent_span_id": "3572e33ffdea5492", + "name": "ChatOpenAI", + "type": "llm", + "agent": "fact_checker", + "start_offset_ms": 5894.546176, + "duration_ms": 933.578752, + "status": "ok", + "input_preview": "[{\"role\": \"system\", \"content\": \"Verify the claim with verify_claim once, then answer 'ok' or 'wrong' in one word.\"}, {\"role\": \"user\", \"content\": \"ClickHouse has faster ingest throughput than Postgres, with ClickHouse rated as fast (p50 84ms", + "model": "claude-haiku-4-5", + "input_tokens": 883, + "output_tokens": 60, + "litellm": { + "request_id": "chatcmpl-8740212c-796e-4bc0-b334-2f2be82819f5", + "model": "openai/claude-haiku-4-5-20251001", + "model_group": "claude-haiku-4-5", + "provider": "openai", + "key_alias": "research-bot", + "team_alias": "research-agents", + "spend": 0.001183, + "prompt_tokens": 883, + "completion_tokens": 60, + "cache_read_tokens": 0, + "cache_write_tokens": 0, + "latency_ms": 928, + "ttft_ms": 928, + "status": "success" + } + }, + { + "span_id": "5fb9c34e4ba07882", + "parent_span_id": "8feaefa0bc4965be", + "name": "tools", + "type": "chain", + "agent": "fact_checker", + "start_offset_ms": 6332.523264, + "duration_ms": 0.802816, + "status": "ok", + "input_preview": "{\"input\":[{\"name\":\"verify_claim\",\"args\":{},\"id\":\"toolu_01MYBaPSYtDjeSm3vZxhNFZ5\",\"type\":\"tool_call\"}]}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "e199fcc910071a4e", + "parent_span_id": "5fb9c34e4ba07882", + "name": "verify_claim", + "type": "tool", + "agent": "fact_checker", + "start_offset_ms": 6332.82304, + "duration_ms": 0.286208, + "status": "error", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "3b186a2923d5b458", + "parent_span_id": "8feaefa0bc4965be", + "name": "model", + "type": "chain", + "agent": "fact_checker", + "start_offset_ms": 6333.495296, + "duration_ms": 1013.369856, + "status": "ok", + "input_preview": "{\"messages\":[{\"content\":\"ClickHouse has faster ingest throughput (fast, p50 51ms) than Postgres (ok, p50 51ms)\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"9d0c66ae-ed19-44c9-8b7e-be0ae90b6019\"},{\"content\":\"\",\"additio", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "fab16e12209104f4", + "parent_span_id": "3b186a2923d5b458", + "name": "ChatOpenAI", + "type": "llm", + "agent": "fact_checker", + "start_offset_ms": 6333.748224, + "duration_ms": 1012.94592, + "status": "ok", + "input_preview": "[{\"role\": \"system\", \"content\": \"Verify the claim with verify_claim once, then answer 'ok' or 'wrong' in one word.\"}, {\"role\": \"user\", \"content\": \"ClickHouse has faster ingest throughput (fast, p50 51ms) than Postgres (ok, p50 51ms)\"}, {\"rol", + "model": "claude-haiku-4-5", + "input_tokens": 883, + "output_tokens": 60, + "litellm": { + "request_id": "chatcmpl-ee1c82af-b979-47c0-a2fb-94e4f9b3b6ea", + "model": "openai/claude-haiku-4-5-20251001", + "model_group": "claude-haiku-4-5", + "provider": "openai", + "key_alias": "research-bot", + "team_alias": "research-agents", + "spend": 0.001183, + "prompt_tokens": 883, + "completion_tokens": 60, + "cache_read_tokens": 0, + "cache_write_tokens": 0, + "latency_ms": 1008, + "ttft_ms": 1008, + "status": "success" + } + }, + { + "span_id": "aa4b97926e556afe", + "parent_span_id": "51a224d31e19f7f3", + "name": "tools", + "type": "chain", + "agent": "fact_checker", + "start_offset_ms": 6828.529152, + "duration_ms": 0.78208, + "status": "ok", + "input_preview": "{\"input\":[{\"name\":\"verify_claim\",\"args\":{},\"id\":\"toolu_01T67jNUGCh7eg6wiaHKmgHH\",\"type\":\"tool_call\"}]}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "43f9afdb2a488daf", + "parent_span_id": "aa4b97926e556afe", + "name": "verify_claim", + "type": "tool", + "agent": "fact_checker", + "start_offset_ms": 6828.822272, + "duration_ms": 0.28672, + "status": "error", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "a0a80112bb486b0f", + "parent_span_id": "51a224d31e19f7f3", + "name": "model", + "type": "chain", + "agent": "fact_checker", + "start_offset_ms": 6829.462016, + "duration_ms": 1213.248, + "status": "ok", + "input_preview": "{\"messages\":[{\"content\":\"ClickHouse has faster ingest throughput than Postgres, with ClickHouse rated as fast (p50 84ms) compared to Postgres rated as ok (p50 84ms)\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"c1a8658", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "4d1d711bb127ba1d", + "parent_span_id": "a0a80112bb486b0f", + "name": "ChatOpenAI", + "type": "llm", + "agent": "fact_checker", + "start_offset_ms": 6829.707264, + "duration_ms": 1212.832, + "status": "ok", + "input_preview": "[{\"role\": \"system\", \"content\": \"Verify the claim with verify_claim once, then answer 'ok' or 'wrong' in one word.\"}, {\"role\": \"user\", \"content\": \"ClickHouse has faster ingest throughput than Postgres, with ClickHouse rated as fast (p50 84ms", + "model": "claude-haiku-4-5", + "input_tokens": 974, + "output_tokens": 60, + "litellm": { + "request_id": "chatcmpl-1ef874c3-55cd-4c60-8fc1-c1bccd2a4e88", + "model": "openai/claude-haiku-4-5-20251001", + "model_group": "claude-haiku-4-5", + "provider": "openai", + "key_alias": "research-bot", + "team_alias": "research-agents", + "spend": 0.001274, + "prompt_tokens": 974, + "completion_tokens": 60, + "cache_read_tokens": 0, + "cache_write_tokens": 0, + "latency_ms": 1207, + "ttft_ms": 1207, + "status": "success" + } + }, + { + "span_id": "aeb2460c46a7b885", + "parent_span_id": "8feaefa0bc4965be", + "name": "tools", + "type": "chain", + "agent": "fact_checker", + "start_offset_ms": 7347.08608, + "duration_ms": 0.802048, + "status": "ok", + "input_preview": "{\"input\":[{\"name\":\"verify_claim\",\"args\":{},\"id\":\"toolu_018V9AomXZBTgkUVnsJJUmSf\",\"type\":\"tool_call\"}]}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "d57d0138262e6459", + "parent_span_id": "aeb2460c46a7b885", + "name": "verify_claim", + "type": "tool", + "agent": "fact_checker", + "start_offset_ms": 7347.385088, + "duration_ms": 0.29184, + "status": "error", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "61eef30374dafead", + "parent_span_id": "8feaefa0bc4965be", + "name": "model", + "type": "chain", + "agent": "fact_checker", + "start_offset_ms": 7348.064256, + "duration_ms": 1277.523968, + "status": "ok", + "input_preview": "{\"messages\":[{\"content\":\"ClickHouse has faster ingest throughput (fast, p50 51ms) than Postgres (ok, p50 51ms)\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"9d0c66ae-ed19-44c9-8b7e-be0ae90b6019\"},{\"content\":\"\",\"additio", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "994b738b34f44318", + "parent_span_id": "61eef30374dafead", + "name": "ChatOpenAI", + "type": "llm", + "agent": "fact_checker", + "start_offset_ms": 7349.096192, + "duration_ms": 1276.313856, + "status": "ok", + "input_preview": "[{\"role\": \"system\", \"content\": \"Verify the claim with verify_claim once, then answer 'ok' or 'wrong' in one word.\"}, {\"role\": \"user\", \"content\": \"ClickHouse has faster ingest throughput (fast, p50 51ms) than Postgres (ok, p50 51ms)\"}, {\"rol", + "model": "claude-haiku-4-5", + "input_tokens": 969, + "output_tokens": 60, + "litellm": { + "request_id": "chatcmpl-1585a7bc-85cf-4948-ace7-8685141196e0", + "model": "openai/claude-haiku-4-5-20251001", + "model_group": "claude-haiku-4-5", + "provider": "openai", + "key_alias": "research-bot", + "team_alias": "research-agents", + "spend": 0.001269, + "prompt_tokens": 969, + "completion_tokens": 60, + "cache_read_tokens": 0, + "cache_write_tokens": 0, + "latency_ms": 1270, + "ttft_ms": 1270, + "status": "success" + } + }, + { + "span_id": "99305ec77a4b0db8", + "parent_span_id": "ffb1aafc123f7cbf", + "name": "model", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 8043.282176, + "duration_ms": 868.797952, + "status": "ok", + "input_preview": "{\"messages\":[{\"content\":\"Compare ingest throughput (question 0).\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"8bd0af23-8ed1-4178-aff9-7a5763b1c928\"},{\"content\":\"\",\"additional_kwargs\":{\"refusal\":null},\"response_metadat", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "bac938a998d22f5f", + "parent_span_id": "99305ec77a4b0db8", + "name": "ChatOpenAI", + "type": "llm", + "agent": "researcher", + "start_offset_ms": 8043.563264, + "duration_ms": 868.342784, + "status": "ok", + "input_preview": "[{\"role\": \"system\", \"content\": \"Call lookup_benchmark once for ClickHouse and once for Postgres, then call check_fact on your conclusion, then answer in ONE short sentence.\"}, {\"role\": \"user\", \"content\": \"Compare ingest throughput (question", + "model": "claude-haiku-4-5", + "input_tokens": 1206, + "output_tokens": 49, + "litellm": { + "request_id": "chatcmpl-733eeb0e-1466-486e-b7a8-c659a3e13fd4", + "model": "openai/claude-haiku-4-5-20251001", + "model_group": "claude-haiku-4-5", + "provider": "openai", + "key_alias": "research-bot", + "team_alias": "research-agents", + "spend": 0.001451, + "prompt_tokens": 1206, + "completion_tokens": 49, + "cache_read_tokens": 0, + "cache_write_tokens": 0, + "latency_ms": 863, + "ttft_ms": 863, + "status": "success" + } + }, + { + "span_id": "76dd59068fe39775", + "parent_span_id": "8feaefa0bc4965be", + "name": "tools", + "type": "chain", + "agent": "fact_checker", + "start_offset_ms": 8625.82016, + "duration_ms": 0.784896, + "status": "ok", + "input_preview": "{\"input\":[{\"name\":\"verify_claim\",\"args\":{},\"id\":\"toolu_01DHE5bKweeAAYdUqyUEksxG\",\"type\":\"tool_call\"}]}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "14dbb727670638dc", + "parent_span_id": "76dd59068fe39775", + "name": "verify_claim", + "type": "tool", + "agent": "fact_checker", + "start_offset_ms": 8626.10816, + "duration_ms": 0.290816, + "status": "error", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "820968a1a530b332", + "parent_span_id": "8feaefa0bc4965be", + "name": "model", + "type": "chain", + "agent": "fact_checker", + "start_offset_ms": 8626.76224, + "duration_ms": 920.998656, + "status": "ok", + "input_preview": "{\"messages\":[{\"content\":\"ClickHouse has faster ingest throughput (fast, p50 51ms) than Postgres (ok, p50 51ms)\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"9d0c66ae-ed19-44c9-8b7e-be0ae90b6019\"},{\"content\":\"\",\"additio", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "d980027f6fef58d6", + "parent_span_id": "820968a1a530b332", + "name": "ChatOpenAI", + "type": "llm", + "agent": "fact_checker", + "start_offset_ms": 8627.012096, + "duration_ms": 920.585984, + "status": "ok", + "input_preview": "[{\"role\": \"system\", \"content\": \"Verify the claim with verify_claim once, then answer 'ok' or 'wrong' in one word.\"}, {\"role\": \"user\", \"content\": \"ClickHouse has faster ingest throughput (fast, p50 51ms) than Postgres (ok, p50 51ms)\"}, {\"rol", + "model": "claude-haiku-4-5", + "input_tokens": 1060, + "output_tokens": 60, + "litellm": { + "request_id": "chatcmpl-1bc15649-313e-487d-bf02-ea93f63cc126", + "model": "openai/claude-haiku-4-5-20251001", + "model_group": "claude-haiku-4-5", + "provider": "openai", + "key_alias": "research-bot", + "team_alias": "research-agents", + "spend": 0.00136, + "prompt_tokens": 1060, + "completion_tokens": 60, + "cache_read_tokens": 0, + "cache_write_tokens": 0, + "latency_ms": 915, + "ttft_ms": 915, + "status": "success" + } + }, + { + "span_id": "6b5359ea58dbc7df", + "parent_span_id": "8feaefa0bc4965be", + "name": "tools", + "type": "chain", + "agent": "fact_checker", + "start_offset_ms": 9548.065024, + "duration_ms": 0.884992, + "status": "ok", + "input_preview": "{\"input\":[{\"name\":\"verify_claim\",\"args\":{},\"id\":\"toolu_01EMWtXbGHXpq6EL7jEbPsQX\",\"type\":\"tool_call\"}]}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "14c958250e70669e", + "parent_span_id": "6b5359ea58dbc7df", + "name": "verify_claim", + "type": "tool", + "agent": "fact_checker", + "start_offset_ms": 9548.434944, + "duration_ms": 0.303104, + "status": "error", + "input_preview": "{}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "46e1d0320d1c63e2", + "parent_span_id": "8feaefa0bc4965be", + "name": "model", + "type": "chain", + "agent": "fact_checker", + "start_offset_ms": 9549.12, + "duration_ms": 1079.88608, + "status": "ok", + "input_preview": "{\"messages\":[{\"content\":\"ClickHouse has faster ingest throughput (fast, p50 51ms) than Postgres (ok, p50 51ms)\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"9d0c66ae-ed19-44c9-8b7e-be0ae90b6019\"},{\"content\":\"\",\"additio", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "a9c4ba6ba55a93ef", + "parent_span_id": "46e1d0320d1c63e2", + "name": "ChatOpenAI", + "type": "llm", + "agent": "fact_checker", + "start_offset_ms": 9549.393152, + "duration_ms": 1079.435776, + "status": "ok", + "input_preview": "[{\"role\": \"system\", \"content\": \"Verify the claim with verify_claim once, then answer 'ok' or 'wrong' in one word.\"}, {\"role\": \"user\", \"content\": \"ClickHouse has faster ingest throughput (fast, p50 51ms) than Postgres (ok, p50 51ms)\"}, {\"rol", + "model": "claude-haiku-4-5", + "input_tokens": 1149, + "output_tokens": 60, + "litellm": { + "request_id": "chatcmpl-ada1126b-e136-4cb4-92bd-b6d1419fdf09", + "model": "openai/claude-haiku-4-5-20251001", + "model_group": "claude-haiku-4-5", + "provider": "openai", + "key_alias": "research-bot", + "team_alias": "research-agents", + "spend": 0.0014489999999999998, + "prompt_tokens": 1149, + "completion_tokens": 60, + "cache_read_tokens": 0, + "cache_write_tokens": 0, + "latency_ms": 1073, + "ttft_ms": 1073, + "status": "success" + } + }, + { + "span_id": "725ee1f23444ee13", + "parent_span_id": "4bc4c26013a33115", + "name": "model", + "type": "chain", + "agent": "researcher", + "start_offset_ms": 10629.611008, + "duration_ms": 1055.321856, + "status": "ok", + "input_preview": "{\"messages\":[{\"content\":\"Compare ingest throughput (question 10).\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"c93e34d9-26a7-4f16-af29-04ad300f0905\"},{\"content\":\"I'll look up the ingest throughput benchmark for both C", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "6b5c44b97608a665", + "parent_span_id": "725ee1f23444ee13", + "name": "ChatOpenAI", + "type": "llm", + "agent": "researcher", + "start_offset_ms": 10629.89312, + "duration_ms": 1054.866176, + "status": "ok", + "input_preview": "[{\"role\": \"system\", \"content\": \"Call lookup_benchmark once for ClickHouse and once for Postgres, then call check_fact on your conclusion, then answer in ONE short sentence.\"}, {\"role\": \"user\", \"content\": \"Compare ingest throughput (question", + "model": "claude-haiku-4-5", + "input_tokens": 1186, + "output_tokens": 48, + "litellm": { + "request_id": "chatcmpl-dc15e33e-0526-4017-8359-1ed0a0b5bfb7", + "model": "openai/claude-haiku-4-5-20251001", + "model_group": "claude-haiku-4-5", + "provider": "openai", + "key_alias": "research-bot", + "team_alias": "research-agents", + "spend": 0.001426, + "prompt_tokens": 1186, + "completion_tokens": 48, + "cache_read_tokens": 0, + "cache_write_tokens": 0, + "latency_ms": 1049, + "ttft_ms": 1049, + "status": "success" + } + }, + { + "span_id": "ec33160969513265", + "parent_span_id": "b7b1053f6c95fd70", + "name": "critic", + "type": "agent", + "agent": "critic", + "start_offset_ms": 11685.611008, + "duration_ms": 5460.011008, + "status": "ok", + "input_preview": "[{\"role\": \"user\", \"content\": \"12 findings, 1 failed. ClickHouse wins on ingest and cost.\"}]", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "5c43936b0d83e971", + "parent_span_id": "ec33160969513265", + "name": "model", + "type": "chain", + "agent": "critic", + "start_offset_ms": 11686.049024, + "duration_ms": 5459.417088, + "status": "ok", + "input_preview": "{\"messages\":[{\"content\":\"12 findings, 1 failed. ClickHouse wins on ingest and cost.\",\"additional_kwargs\":{},\"response_metadata\":{},\"type\":\"human\",\"id\":\"9b909661-205a-4ad5-a595-358342a07c35\"}]}", + "model": null, + "input_tokens": 0, + "output_tokens": 0, + "litellm": null + }, + { + "span_id": "82692e8e9ec9105c", + "parent_span_id": "5c43936b0d83e971", + "name": "ChatOpenAI", + "type": "llm", + "agent": "critic", + "start_offset_ms": 11686.348032, + "duration_ms": 5458.984192, + "status": "ok", + "input_preview": "[{\"role\": \"system\", \"content\": \"Critique the summary in 2 bullets.\"}, {\"role\": \"user\", \"content\": \"12 findings, 1 failed. ClickHouse wins on ingest and cost.\"}]", + "model": "claude-sonnet-4-5", + "input_tokens": 38, + "output_tokens": 125, + "litellm": { + "request_id": "chatcmpl-0f035e16-9296-469a-b925-d00aed5a1fed", + "model": "openai/claude-sonnet-4-5", + "model_group": "claude-sonnet-4-5", + "provider": "openai", + "key_alias": "research-bot", + "team_alias": "research-agents", + "spend": 0.0019890000000000003, + "prompt_tokens": 38, + "completion_tokens": 125, + "cache_read_tokens": 0, + "cache_write_tokens": 0, + "latency_ms": 5454, + "ttft_ms": 5454, + "status": "success" + } + } + ] +} diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/__fixtures__/trace_list.json b/ui/litellm-dashboard/src/components/view_logs/TraceView/__fixtures__/trace_list.json new file mode 100644 index 000000000000..f4188aa98ad6 --- /dev/null +++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/__fixtures__/trace_list.json @@ -0,0 +1,59 @@ +{ + "data": [ + { + "trace_id": "e309a123963901e74c29cd2d3c86ff9e", + "name": "research_lead", + "service": "research-agent", + "input_preview": "[{\"role\": \"user\", \"content\": \"Should we store OTEL agent spans in ClickHouse or Postgres at 50k spans/sec?\"}]", + "start_time": "2026-09-30T06:43:54.291000+00:00", + "duration_ms": 40198.0, + "status": "ok", + "span_count": 216, + "agent_count": 6, + "llm_calls": 21, + "tool_calls": 25, + "error_count": 0, + "input_tokens": 69506, + "output_tokens": 2960, + "spend": 0.12833024999999998, + "models": ["claude-sonnet-4-5"] + }, + { + "trace_id": "f78f6df35480060fafadac887e234241", + "name": "support_triage_agent", + "service": "research-agent", + "input_preview": "[{\"role\": \"user\", \"content\": \"Customer acme-404 says billing is wrong. What plan are they on?\"}]", + "start_time": "2026-09-30T06:43:52.928000+00:00", + "duration_ms": 1315.0, + "status": "ok", + "span_count": 5, + "agent_count": 1, + "llm_calls": 1, + "tool_calls": 1, + "error_count": 2, + "input_tokens": 659, + "output_tokens": 60, + "spend": 0.0028770000000000002, + "models": ["claude-sonnet-4-5"] + }, + { + "trace_id": "71498cec128bbea430f01de04b972f36", + "name": "support_triage_agent", + "service": "research-agent", + "input_preview": "[{\"role\": \"user\", \"content\": \"Customer acme-42 gets 429s after upgrading. What plan are they on and what should they check?\"}]", + "start_time": "2026-09-30T06:43:47.373000+00:00", + "duration_ms": 5551.0, + "status": "ok", + "span_count": 9, + "agent_count": 1, + "llm_calls": 2, + "tool_calls": 2, + "error_count": 0, + "input_tokens": 1552, + "output_tokens": 202, + "spend": 0.007686, + "models": ["claude-sonnet-4-5"] + } + ], + "next_cursor": null +} diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/traceTypes.ts b/ui/litellm-dashboard/src/components/view_logs/TraceView/traceTypes.ts new file mode 100644 index 000000000000..f2611b17ad56 --- /dev/null +++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/traceTypes.ts @@ -0,0 +1,89 @@ +/** + * Agent tracing types. Mirrors `litellm/tracing/types.py` exactly. + * + * A trace is one agent run made of spans (agent / llm / tool / chain / framework). + */ + +export type SpanType = "agent" | "llm" | "tool" | "chain" | "framework"; +export type SpanStatus = "ok" | "error" | "unset"; + +export interface Span { + span_id: string; + parent_span_id: string | null; + name: string; + type: SpanType; + /** The agent this span runs inside, e.g. "researcher". */ + agent: string; + /** Relative to trace start. */ + start_offset_ms: number; + duration_ms: number; + status: SpanStatus; + /** Exception message when status is "error". */ + error?: string | null; + input_preview: string; + model: string | null; + input_tokens: number; + output_tokens: number; + litellm_request_id: string | null; +} + +/** One distinct agent in a trace. 200 invocations of `researcher` = one node. */ +export interface AgentNode { + name: string; + parent_agent: string | null; + invocations: number; + llm_calls: number; + tool_calls: number; + duration_ms: number; +} + +export interface TraceSummary { + trace_id: string; + name: string; + service: string; + input_preview: string; + /** ISO 8601 */ + start_time: string; + duration_ms: number; + status: SpanStatus; + span_count: number; + agent_count: number; + llm_calls: number; + tool_calls: number; + /** Spans with an error status; > 0 means the run shows as failed. */ + error_count: number; + input_tokens: number; + output_tokens: number; + models: string[]; +} + +export interface Trace { + summary: TraceSummary; + agents: AgentNode[]; + spans: Span[]; +} + +export interface TracePage { + data: TraceSummary[]; + next_cursor: string | null; +} + +/** `input` / `output` are JSON strings (messages for llm spans, raw args / result for tools). */ +export interface SpanDetail { + span_id: string; + input: string; + output: string; + attributes: Record; +} + +export interface TraceToolCall { + name: string; + args: unknown; +} + +export interface TraceMessage { + role: string; + content: string; + name?: string; + tool_calls?: TraceToolCall[]; +} diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/traceUtils.test.ts b/ui/litellm-dashboard/src/components/view_logs/TraceView/traceUtils.test.ts new file mode 100644 index 000000000000..37da685bb5d2 --- /dev/null +++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/traceUtils.test.ts @@ -0,0 +1,262 @@ +import { describe, expect, it } from "vitest"; + +import deepAgentTrace from "./__fixtures__/deep_agent_trace.json"; +import researchTrace from "./__fixtures__/research_trace.json"; +import swarmTrace from "./__fixtures__/swarm_trace.json"; +import type { Span, Trace } from "./traceTypes"; +import { + agentBadgeLabel, + buildVisibleTree, + firstErrorSpan, + flattenTree, + fmtMs, + GROUP_PAGE_SIZE, + groupSiblingAgents, + interleaveByTime, + isFrameworkSpan, + layoutAgentGraph, + median, + parseMessages, + previewText, + revealSpan, + ROOT_KEY, + spanLabel, + spanRowIds, + stepsFromSpans, + subtreeStats, + traceHasErrors, + type TreeUiState, +} from "./traceUtils"; + +const swarm = swarmTrace as Trace; +const research = researchTrace as Trace; +const deepAgent = deepAgentTrace as Trace; +const EMPTY_UI: TreeUiState = { collapsed: new Set(), groupShown: {} }; + +const span = (overrides: Partial & Pick): Span => ({ + parent_span_id: null, + name: overrides.span_id, + type: "chain", + agent: "root", + start_offset_ms: 0, + duration_ms: 1, + status: "ok", + input_preview: "", + model: null, + input_tokens: 0, + output_tokens: 0, + litellm_request_id: null, + ...overrides, +}); + +describe("formatting", () => { + it("formats durations the way the table shows them", () => { + expect(fmtMs(4.25)).toBe("4.3ms"); + expect(fmtMs(950)).toBe("950ms"); + expect(fmtMs(51386)).toBe("51.39s"); + }); + + it("labels agent rows with agent, LLM and tool counts", () => { + expect(agentBadgeLabel(swarm.summary)).toBe("◆ 4 agents · 53 LLM · 45 tool"); + expect(agentBadgeLabel({ agent_count: 1, llm_calls: 2, tool_calls: 1 })).toBe("◆ Agent · 2 LLM · 1 tool"); + }); + + it("headlines LLM spans with the OTLP model", () => { + expect(spanLabel(span({ span_id: "llm", type: "llm", name: "generation", model: "model-a" }))).toBe("model-a"); + }); + + it("pulls the user message out of a truncated JSON preview", () => { + expect(previewText('[{"role": "user", "content": "Customer acme-404 says billing is wrong."}]')).toBe( + "Customer acme-404 says billing is wrong.", + ); + expect(previewText('[{"role": "system", "content": "sys"}, {"role": "user", "content": "Compare ingest thr')).toBe( + "Compare ingest thr", + ); + expect(previewText("plain text")).toBe("plain text"); + }); + + it("takes the median of an even and an odd list", () => { + expect(median([3, 1, 2])).toBe(2); + expect(median([4, 1, 3, 2])).toBe(2.5); + expect(median([])).toBe(0); + }); +}); + +describe("buildVisibleTree", () => { + it("hides framework spans and re-parents their children to the nearest visible ancestor", () => { + const spans = [ + span({ span_id: "root", type: "agent" }), + span({ span_id: "mw", parent_span_id: "root", type: "framework", name: "X.wrap_model_call" }), + span({ span_id: "model", parent_span_id: "mw", type: "chain", name: "model" }), + span({ span_id: "llm", parent_span_id: "model", type: "llm", start_offset_ms: 5 }), + span({ span_id: "step", parent_span_id: "root", type: "chain", name: "planner", start_offset_ms: 1 }), + ]; + const compact = buildVisibleTree(spans, false); + expect(compact.visibleCount).toBe(3); + expect(compact.children.get(ROOT_KEY)?.map((s) => s.span_id)).toEqual(["root"]); + expect(compact.children.get("root")?.map((s) => s.span_id)).toEqual(["step", "llm"]); + + const raw = buildVisibleTree(spans, true); + expect(raw.visibleCount).toBe(5); + expect(raw.children.get("model")?.map((s) => s.span_id)).toEqual(["llm"]); + }); + + it("never hides the root span even if it looks like framework plumbing", () => { + expect(isFrameworkSpan(span({ span_id: "r", type: "framework" }))).toBe(false); + }); + + it("drops every framework / graph-node span of the real Deep Agents trace", () => { + const { visibleCount, children } = buildVisibleTree(research.spans, false); + const hidden = research.spans.filter(isFrameworkSpan).length; + expect(hidden).toBeGreaterThan(100); + expect(visibleCount).toBe(research.spans.length - hidden); + const visibleIds = new Set([...children.values()].flat().map((s) => s.span_id)); + expect(research.spans.filter((s) => s.type === "llm").every((s) => visibleIds.has(s.span_id))).toBe(true); + }); +}); + +describe("groupSiblingAgents", () => { + const stats = subtreeStats(swarm.spans); + const children = buildVisibleTree(swarm.spans, false).children; + const root = swarm.spans.find((s) => s.parent_span_id === null) as Span; + + it("collapses more than 10 same-named sibling agents into one group row", () => { + const items = groupSiblingAgents(root.span_id, children.get(root.span_id) ?? [], stats); + const groups = items.filter((i) => i.kind === "group"); + expect(groups).toHaveLength(1); + const group = groups[0].kind === "group" ? groups[0].group : null; + expect(group?.name).toBe("researcher"); + expect(group?.spans).toHaveLength(12); + // critic (1 invocation) stays a plain row + expect(items.some((i) => i.kind === "span" && i.span.name === "critic")).toBe(true); + }); + + it("counts grouped invocations that contain errors", () => { + const items = groupSiblingAgents(root.span_id, children.get(root.span_id) ?? [], stats); + const group = items.find((i) => i.kind === "group"); + if (group?.kind !== "group") throw new Error("expected a group"); + expect(group.group.errors).toBeGreaterThan(0); + expect(group.group.p50Ms).toBeGreaterThan(0); + }); + + it("leaves groups of 10 or fewer alone", () => { + const parent = span({ span_id: "p", type: "agent" }); + const kids = Array.from({ length: 10 }, (_, i) => + span({ span_id: `k${i}`, parent_span_id: "p", type: "agent", name: "worker" }), + ); + const items = groupSiblingAgents("p", kids, subtreeStats([parent, ...kids])); + expect(items.every((i) => i.kind === "span")).toBe(true); + }); +}); + +describe("flattenTree + revealSpan", () => { + const children = buildVisibleTree(swarm.spans, false).children; + const stats = subtreeStats(swarm.spans); + + it("shows the group summary row but none of its invocations by default", () => { + const rows = flattenTree(children, stats, EMPTY_UI); + expect(rows.filter((r) => r.kind === "group")).toHaveLength(1); + expect(rows.some((r) => r.kind === "span" && r.span.name === "researcher")).toBe(false); + }); + + it("pages group invocations 20 at a time with a 'more' row until all are shown", () => { + const parent = span({ span_id: "p", type: "agent" }); + const kids = Array.from({ length: 45 }, (_, i) => + span({ span_id: `k${i}`, parent_span_id: "p", type: "agent", name: "worker", start_offset_ms: i }), + ); + const all = [parent, ...kids]; + const tree = buildVisibleTree(all, false).children; + const key = "p::worker"; + const page1 = flattenTree(tree, subtreeStats(all), { + collapsed: new Set(), + groupShown: { [key]: GROUP_PAGE_SIZE }, + }); + expect(page1.filter((r) => r.kind === "span" && r.span.name === "worker")).toHaveLength(20); + expect(page1.at(-1)?.kind).toBe("more"); + const page3 = flattenTree(tree, subtreeStats(all), { collapsed: new Set(), groupShown: { [key]: 60 } }); + expect(page3.filter((r) => r.kind === "span" && r.span.name === "worker")).toHaveLength(45); + expect(page3.some((r) => r.kind === "more")).toBe(false); + }); + + it("reveals a span nested inside a collapsed group so the drawer can open on it", () => { + const failed = firstErrorSpan(swarm.spans) as Span; + const ui = revealSpan(children, stats, EMPTY_UI, failed.span_id); + expect(spanRowIds(flattenTree(children, stats, ui))).toContain(failed.span_id); + }); + + it("hides children of collapsed spans", () => { + const root = swarm.spans.find((s) => s.parent_span_id === null) as Span; + const rows = flattenTree(children, stats, { collapsed: new Set([root.span_id]), groupShown: {} }); + expect(rows).toHaveLength(1); + }); +}); + +describe("stepsFromSpans", () => { + it("numbers every LLM call, tool call and subagent invocation in time order", () => { + const steps = stepsFromSpans(deepAgent.spans); + expect(steps).toHaveLength(7 + 26 + 1); + expect(steps.map((s) => s.number)).toEqual(steps.map((_, i) => i + 1)); + const starts = steps.map((s) => s.span.start_offset_ms); + expect(starts).toEqual([...starts].sort((a, b) => a - b)); + }); + + it("marks steps that ran inside the researcher subagent and indents them", () => { + const steps = stepsFromSpans(deepAgent.spans); + const inResearcher = steps.filter((s) => s.subagent === "researcher"); + expect(inResearcher.length).toBeGreaterThan(0); + expect(inResearcher.every((s) => s.depth === 1)).toBe(true); + const first = steps[0]; + expect(first.span.type).toBe("llm"); + expect(first.subagent).toBeNull(); + // the lead's first decision runs write_file and task + expect(first.toolNames).toEqual(["write_file", "task"]); + }); + +}); + +describe("trace-level helpers", () => { + it("detects errors and finds the earliest failing non-root span", () => { + expect(traceHasErrors(swarm)).toBe(true); + expect(traceHasErrors(research)).toBe(false); + const failed = firstErrorSpan(swarm.spans); + expect(failed?.status).toBe("error"); + expect(failed?.parent_span_id).not.toBeNull(); + }); + + it("lays out one node per agent with an edge labelled by invocations", () => { + const layout = layoutAgentGraph(swarm.agents); + expect(layout.nodes).toHaveLength(4); + const labels = layout.edges.map((e) => `${e.from.agent.name}->${e.to.agent.name} ${e.label}`); + expect(labels).toEqual( + expect.arrayContaining([ + "orchestrator->researcher ×12", + "researcher->fact_checker ×2", + "orchestrator->critic ×1", + ]), + ); + const researcherNode = layout.nodes.find((n) => n.agent.name === "researcher"); + const criticNode = layout.nodes.find((n) => n.agent.name === "critic"); + expect(researcherNode?.width).toBe(criticNode?.width); + expect(researcherNode?.x).toBeGreaterThan(layout.nodes[0].x); + }); + + it("parses llm message payloads and rejects non-message JSON", () => { + expect(parseMessages('[{"role":"user","content":"hi"}]')).toEqual([{ role: "user", content: "hi" }]); + expect( + parseMessages('{"role":"assistant","content":"","tool_calls":[{"name":"t","args":{}}]}')?.[0].tool_calls, + ).toHaveLength(1); + expect(parseMessages('{"file_path":"/tmp/x"}')).toBeNull(); + expect(parseMessages("not json")).toBeNull(); + }); + + it("interleaves two newest-first lists by time", () => { + const merged = interleaveByTime( + [ + { at: 30, item: "t1" }, + { at: 10, item: "t2" }, + ], + [{ at: 20, item: "r1" }], + ); + expect(merged.map((m) => m.item)).toEqual(["t1", "r1", "t2"]); + }); +}); diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/traceUtils.ts b/ui/litellm-dashboard/src/components/view_logs/TraceView/traceUtils.ts new file mode 100644 index 000000000000..116a822a0442 --- /dev/null +++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/traceUtils.ts @@ -0,0 +1,513 @@ +/** + * Pure helpers for the agent trace views. No React in here: everything the tree, + * steps and graph views compute lives here so it can be unit-tested directly. + */ +import type { AgentNode, Span, SpanStatus, Trace, TraceMessage, TraceSummary } from "./traceTypes"; + +/* ------------------------------------------------------------------ */ +/* Formatting */ +/* ------------------------------------------------------------------ */ + +export const fmtMs = (ms: number): string => { + if (ms >= 60_000) return `${(ms / 60_000).toFixed(1)}m`; + if (ms >= 1000) return `${(ms / 1000).toFixed(2)}s`; + return `${Math.max(ms, 0).toFixed(ms < 10 ? 1 : 0)}ms`; +}; + +export const fmtTok = (n: number): string => (n >= 1000 ? `${(n / 1000).toFixed(1)}k` : String(n)); + +export const shortId = (id: string, length = 16): string => (id.length > length ? `${id.slice(0, length)}…` : id); + +const plural = (n: number, word: string): string => `${n} ${word}${n === 1 ? "" : "s"}`; + +/** "◆ Agent · 7 LLM · 26 tool" or "◆ 2 agents · 7 LLM · 26 tool". */ +export const agentBadgeLabel = (summary: Pick): string => { + const agents = summary.agent_count > 1 ? plural(summary.agent_count, "agent") : "Agent"; + return `◆ ${agents} · ${summary.llm_calls} LLM · ${summary.tool_calls} tool`; +}; + +/** LLM spans are labelled by model group (what the caller asked for); everything else by span name. */ +export const spanLabel = (span: Span): string => (span.type === "llm" ? span.model || span.name : span.name); + +/* ------------------------------------------------------------------ */ +/* Visible tree (framework spans hidden + children re-parented) */ +/* ------------------------------------------------------------------ */ + +/** Root-level key in the children map. */ +export const ROOT_KEY = "__root__"; + +export type ChildrenMap = Map; + +export interface VisibleTree { + children: ChildrenMap; + visibleCount: number; +} + +/** + * Framework plumbing: middleware wrappers plus LangGraph's generic "model" / "tools" + * graph nodes. The root span is never hidden. + */ +export const isFrameworkSpan = (span: Span): boolean => + span.parent_span_id !== null && + (span.type === "framework" || (span.type === "chain" && (span.name === "model" || span.name === "tools"))); + +const byStart = (a: Span, b: Span): number => a.start_offset_ms - b.start_offset_ms; + +export const indexSpans = (spans: readonly Span[]): Map => new Map(spans.map((s) => [s.span_id, s])); + +const pushChild = (children: ChildrenMap, key: string, span: Span): void => { + const list = children.get(key); + if (list) list.push(span); + else children.set(key, [span]); +}; + +/** + * Children map for the waterfall. With `showFramework` off, framework spans are + * dropped and their children attach to the nearest visible ancestor. Spans whose + * parent is missing from the trace attach to the root level. + */ +export function buildVisibleTree(spans: readonly Span[], showFramework: boolean): VisibleTree { + const byId = indexSpans(spans); + const hidden = (span: Span) => !showFramework && isFrameworkSpan(span); + const visibleParentKey = (span: Span): string => { + let parent = span.parent_span_id ? byId.get(span.parent_span_id) : undefined; + while (parent && hidden(parent)) { + parent = parent.parent_span_id ? byId.get(parent.parent_span_id) : undefined; + } + return parent ? parent.span_id : ROOT_KEY; + }; + const children: ChildrenMap = new Map(); + let visibleCount = 0; + for (const span of spans) { + if (hidden(span)) continue; + visibleCount++; + pushChild(children, visibleParentKey(span), span); + } + children.forEach((list) => list.sort(byStart)); + return { children, visibleCount }; +} + +/* ------------------------------------------------------------------ */ +/* Subtree rollups */ +/* ------------------------------------------------------------------ */ + +export interface SubtreeStats { + errors: number; +} + +/** Error count of every span's full subtree, keyed by span id. */ +export function subtreeStats(spans: readonly Span[]): Map { + const raw = buildVisibleTree(spans, true).children; + const stats = new Map(); + const visit = (span: Span): SubtreeStats => { + const cached = stats.get(span.span_id); + if (cached) return cached; + const own: SubtreeStats = { errors: span.status === "error" ? 1 : 0 }; + for (const child of raw.get(span.span_id) ?? []) { + const childStats = visit(child); + own.errors += childStats.errors; + } + stats.set(span.span_id, own); + return own; + }; + spans.forEach(visit); + return stats; +} + +/* ------------------------------------------------------------------ */ +/* Sibling agent grouping (auto-collapse of fan-out) */ +/* ------------------------------------------------------------------ */ + +/** A parent with more than this many same-named agent children renders them as one group row. */ +export const GROUP_THRESHOLD = 10; +/** Group rows expand this many invocations at a time. */ +export const GROUP_PAGE_SIZE = 20; + +export interface SpanGroup { + key: string; + name: string; + spans: Span[]; + p50Ms: number; + errors: number; +} + +export type TreeItem = { kind: "span"; span: Span } | { kind: "group"; group: SpanGroup }; + +export const median = (values: readonly number[]): number => { + if (values.length === 0) return 0; + const sorted = [...values].sort((a, b) => a - b); + const mid = Math.floor(sorted.length / 2); + return sorted.length % 2 ? sorted[mid] : (sorted[mid - 1] + sorted[mid]) / 2; +}; + +export const groupKey = (parentKey: string, name: string): string => `${parentKey}::${name}`; + +const buildGroup = (parentKey: string, name: string, members: Span[], stats: Map): SpanGroup => ({ + key: groupKey(parentKey, name), + name, + spans: members, + p50Ms: median(members.map((s) => s.duration_ms)), + errors: members.filter((s) => (stats.get(s.span_id)?.errors ?? 0) > 0).length, +}); + +/** + * Collapse runs of same-named agent siblings (more than `threshold`) into one group + * item, placed where the first member would have been. Other children pass through. + */ +export function groupSiblingAgents( + parentKey: string, + children: readonly Span[], + stats: Map, + threshold = GROUP_THRESHOLD, +): TreeItem[] { + const agentsByName = new Map(); + for (const child of children) { + if (child.type !== "agent") continue; + const list = agentsByName.get(child.name); + if (list) list.push(child); + else agentsByName.set(child.name, [child]); + } + const grouped = new Set([...agentsByName].filter(([, list]) => list.length > threshold).map(([name]) => name)); + const emitted = new Set(); + const items: TreeItem[] = []; + for (const child of children) { + if (child.type !== "agent" || !grouped.has(child.name)) { + items.push({ kind: "span", span: child }); + continue; + } + if (emitted.has(child.name)) continue; + emitted.add(child.name); + items.push({ kind: "group", group: buildGroup(parentKey, child.name, agentsByName.get(child.name) ?? [], stats) }); + } + return items; +} + +/* ------------------------------------------------------------------ */ +/* Flattening the tree into rows */ +/* ------------------------------------------------------------------ */ + +export interface TreeUiState { + /** Span ids whose children are hidden. */ + collapsed: ReadonlySet; + /** Group key -> how many invocations are revealed (0 = just the summary row). */ + groupShown: Readonly>; +} + +export type TreeRow = + | { kind: "span"; span: Span; depth: number; hasChildren: boolean; isCollapsed: boolean } + | { kind: "group"; group: SpanGroup; depth: number; shown: number } + | { kind: "more"; group: SpanGroup; depth: number; shown: number }; + +interface FlattenContext { + children: ChildrenMap; + stats: Map; + ui: TreeUiState; + rows: TreeRow[]; +} + +function pushSpanRow(ctx: FlattenContext, span: Span, depth: number): void { + const hasChildren = (ctx.children.get(span.span_id)?.length ?? 0) > 0; + const isCollapsed = ctx.ui.collapsed.has(span.span_id); + ctx.rows.push({ kind: "span", span, depth, hasChildren, isCollapsed }); + if (hasChildren && !isCollapsed) walkChildren(ctx, span.span_id, depth + 1); +} + +function pushGroupRows(ctx: FlattenContext, group: SpanGroup, depth: number): void { + const shown = Math.min(ctx.ui.groupShown[group.key] ?? 0, group.spans.length); + ctx.rows.push({ kind: "group", group, depth, shown }); + if (shown === 0) return; + group.spans.slice(0, shown).forEach((span) => pushSpanRow(ctx, span, depth + 1)); + if (shown < group.spans.length) ctx.rows.push({ kind: "more", group, depth: depth + 1, shown }); +} + +function walkChildren(ctx: FlattenContext, parentKey: string, depth: number): void { + for (const item of groupSiblingAgents(parentKey, ctx.children.get(parentKey) ?? [], ctx.stats)) { + if (item.kind === "span") pushSpanRow(ctx, item.span, depth); + else pushGroupRows(ctx, item.group, depth); + } +} + +/** Depth-first rows for the waterfall, honouring collapsed spans and group pagination. */ +export function flattenTree(children: ChildrenMap, stats: Map, ui: TreeUiState): TreeRow[] { + const ctx: FlattenContext = { children, stats, ui, rows: [] }; + walkChildren(ctx, ROOT_KEY, 0); + return ctx.rows; +} + +/** Ids of the span rows, in render order (drives J/K navigation). */ +export const spanRowIds = (rows: readonly TreeRow[]): string[] => + rows.flatMap((row) => (row.kind === "span" ? [row.span.span_id] : [])); + +/** + * Tree UI state with `spanId` guaranteed visible: its visible ancestors are + * expanded and any group containing it (or an ancestor) reveals enough pages. + */ +export function revealSpan( + children: ChildrenMap, + stats: Map, + ui: TreeUiState, + spanId: string, +): TreeUiState { + const parentOf = new Map(); + children.forEach((list, key) => list.forEach((s) => parentOf.set(s.span_id, key))); + if (!parentOf.has(spanId)) return ui; + const collapsed = new Set(ui.collapsed); + const groupShown = { ...ui.groupShown }; + let current = spanId; + while (parentOf.has(current)) { + const parentKey = parentOf.get(current) as string; + collapsed.delete(parentKey); + const items = groupSiblingAgents(parentKey, children.get(parentKey) ?? [], stats); + for (const item of items) { + if (item.kind !== "group") continue; + const index = item.group.spans.findIndex((s) => s.span_id === current); + if (index < 0) continue; + const needed = Math.ceil((index + 1) / GROUP_PAGE_SIZE) * GROUP_PAGE_SIZE; + groupShown[item.group.key] = Math.max(groupShown[item.group.key] ?? 0, needed); + } + current = parentKey; + } + return { collapsed, groupShown }; +} + +/* ------------------------------------------------------------------ */ +/* Steps view */ +/* ------------------------------------------------------------------ */ + +export interface TraceStep { + span: Span; + /** 1-based, in time order. */ + number: number; + /** Nearest enclosing non-root agent span name, e.g. "researcher". */ + subagent: string | null; + /** Nesting level of subagents (0 = root agent). */ + depth: number; + /** For LLM steps: the tools the agent ran right after this decision. */ + toolNames: string[]; +} + +const isStepSpan = (span: Span): boolean => + span.type === "llm" || span.type === "tool" || (span.type === "agent" && span.parent_span_id !== null); + +interface AgentContext { + subagent: string | null; + depth: number; + invocationId: string | null; +} + +function agentContext(span: Span, byId: Map): AgentContext { + let parent = span.parent_span_id ? byId.get(span.parent_span_id) : undefined; + let subagent: string | null = null; + let invocationId: string | null = null; + let depth = 0; + while (parent) { + if (parent.type === "agent") { + invocationId ??= parent.span_id; + if (parent.parent_span_id !== null) { + subagent ??= parent.name; + depth++; + } + } + parent = parent.parent_span_id ? byId.get(parent.parent_span_id) : undefined; + } + return { subagent, depth, invocationId }; +} + +/** Every LLM decision, tool call and subagent invocation, in time order. */ +export function stepsFromSpans(spans: readonly Span[]): TraceStep[] { + const byId = indexSpans(spans); + const ordered = spans.filter(isStepSpan).sort(byStart); + const contexts = ordered.map((span) => agentContext(span, byId)); + return ordered.map((span, i) => { + const toolNames: string[] = []; + if (span.type === "llm") { + for (let j = i + 1; j < ordered.length; j++) { + if (contexts[j].invocationId !== contexts[i].invocationId) continue; + if (ordered[j].type === "llm") break; + if (ordered[j].type === "tool") toolNames.push(ordered[j].name); + } + } + return { span, number: i + 1, subagent: contexts[i].subagent, depth: contexts[i].depth, toolNames }; + }); +} + +/* ------------------------------------------------------------------ */ +/* Trace-level rollups */ +/* ------------------------------------------------------------------ */ + +export const summaryHasErrors = (summary: Pick): boolean => + summary.status === "error" || summary.error_count > 0; + +export const traceHasErrors = (trace: Trace): boolean => + summaryHasErrors(trace.summary) || trace.spans.some((s) => s.status === "error"); + +/** Earliest failing non-root span (the root just echoes its children), else the root. */ +export function firstErrorSpan(spans: readonly Span[]): Span | null { + const failed = spans.filter((s) => s.status === "error").sort(byStart); + return failed.find((s) => s.parent_span_id !== null) ?? failed[0] ?? null; +} + +export const agentsWithErrors = (spans: readonly Span[]): Set => + new Set(spans.filter((s) => s.status === "error").map((s) => s.agent)); + +export const invocationsOf = (spans: readonly Span[], agentName: string): Span[] => + spans.filter((s) => s.type === "agent" && s.name === agentName).sort(byStart); + +export const llmSpans = (spans: readonly Span[]): Span[] => spans.filter((s) => s.type === "llm").sort(byStart); + +export const statusTone = (status: SpanStatus): "success" | "error" | "neutral" => { + if (status === "error") return "error"; + return status === "ok" ? "success" : "neutral"; +}; + +/* ------------------------------------------------------------------ */ +/* Agent graph layout */ +/* ------------------------------------------------------------------ */ + +export const GRAPH_NODE_HEIGHT = 48; +export const GRAPH_MIN_NODE_WIDTH = 132; +export const GRAPH_MAX_NODE_WIDTH = 232; +const GRAPH_COLUMN_GAP = 96; +const GRAPH_ROW_GAP = 28; +const GRAPH_PADDING = 16; + +export interface GraphNode { + agent: AgentNode; + x: number; + y: number; + width: number; +} + +export interface GraphEdge { + from: GraphNode; + to: GraphNode; + label: string; +} + +export interface GraphLayout { + nodes: GraphNode[]; + edges: GraphEdge[]; + width: number; + height: number; +} + +function agentDepths(agents: readonly AgentNode[]): Map { + const byName = new Map(agents.map((a) => [a.name, a])); + const depths = new Map(); + const depthOf = (agent: AgentNode, seen: Set): number => { + const known = depths.get(agent.name); + if (known !== undefined) return known; + const parent = agent.parent_agent ? byName.get(agent.parent_agent) : undefined; + const depth = parent && !seen.has(parent.name) ? depthOf(parent, new Set([...seen, agent.name])) + 1 : 0; + depths.set(agent.name, depth); + return depth; + }; + agents.forEach((a) => depthOf(a, new Set([a.name]))); + return depths; +} + +/** Left-to-right layered layout: one column per nesting level. */ +export function layoutAgentGraph(agents: readonly AgentNode[]): GraphLayout { + const depths = agentDepths(agents); + const columnWidth = GRAPH_MAX_NODE_WIDTH + GRAPH_COLUMN_GAP; + const rowsPerColumn = new Map(); + const nodes: GraphNode[] = agents.map((agent) => { + const depth = depths.get(agent.name) ?? 0; + const row = rowsPerColumn.get(depth) ?? 0; + rowsPerColumn.set(depth, row + 1); + return { + agent, + x: GRAPH_PADDING + depth * columnWidth, + y: GRAPH_PADDING + row * (GRAPH_NODE_HEIGHT + GRAPH_ROW_GAP), + width: GRAPH_MIN_NODE_WIDTH, + }; + }); + const byName = new Map(nodes.map((n) => [n.agent.name, n])); + const edges: GraphEdge[] = nodes.flatMap((node) => { + const parent = node.agent.parent_agent ? byName.get(node.agent.parent_agent) : undefined; + return parent ? [{ from: parent, to: node, label: `×${node.agent.invocations}` }] : []; + }); + const columns = Math.max(1, ...[...rowsPerColumn.keys()].map((d) => d + 1)); + const rows = Math.max(1, ...rowsPerColumn.values()); + return { + nodes, + edges, + width: GRAPH_PADDING * 2 + (columns - 1) * columnWidth + GRAPH_MAX_NODE_WIDTH, + height: GRAPH_PADDING * 2 + rows * GRAPH_NODE_HEIGHT + (rows - 1) * GRAPH_ROW_GAP, + }; +} + +/* ------------------------------------------------------------------ */ +/* Span detail payloads */ +/* ------------------------------------------------------------------ */ + +export const parseJson = (value: string): unknown => { + if (!value) return null; + try { + return JSON.parse(value); + } catch { + return null; + } +}; + +const isMessage = (value: unknown): value is TraceMessage => + typeof value === "object" && value !== null && "role" in value && typeof (value as TraceMessage).role === "string"; + +/** An llm span's input (array of messages) or output (one message); null when it isn't one. */ +export function parseMessages(value: string): TraceMessage[] | null { + const parsed = parseJson(value); + if (Array.isArray(parsed)) return parsed.every(isMessage) ? parsed : null; + return isMessage(parsed) ? [parsed] : null; +} + +/** Pretty JSON when the payload is JSON, else the raw string. */ +export const prettyPayload = (value: string): string => { + const parsed = parseJson(value); + if (parsed === null || typeof parsed === "string") return typeof parsed === "string" ? parsed : value; + return JSON.stringify(parsed, null, 2); +}; + +/* ------------------------------------------------------------------ */ +/* List view helpers */ +/* ------------------------------------------------------------------ */ + +export const traceStartMs = (summary: TraceSummary): number => Date.parse(summary.start_time); + +const PREVIEW_USER_CONTENT = /"role":\s*"(?:user|human)",\s*"content":\s*"((?:[^"\\]|\\.)*)/; + +/** + * Human text for an input preview. Previews are often a (possibly truncated) JSON + * message array; show the last user/tool message's content when we can find it. + */ +export function previewText(preview: string): string { + if (!preview) return ""; + const messages = parseMessages(preview); + if (messages) { + const last = [...messages].reverse().find((m) => m.role === "user" || m.role === "tool") ?? messages.at(-1); + return last?.content || preview; + } + if (!preview.trimStart().startsWith("[")) return preview; + const match = PREVIEW_USER_CONTENT.exec(preview); + return match ? match[1].replace(/\\n/g, " ").replace(/\\"/g, '"') : preview; +} + +/** Trace display name; root spans without a name fall back to the service. */ +export const traceDisplayName = (summary: Pick): string => + summary.name || summary.service || "(unnamed trace)"; + +export interface TimedItem { + at: number; + item: T; +} + +/** Merge two newest-first lists into one newest-first list. */ +export function interleaveByTime( + a: readonly TimedItem[], + b: readonly TimedItem[], +): ({ kind: "a"; at: number; item: A } | { kind: "b"; at: number; item: B })[] { + const merged: ({ kind: "a"; at: number; item: A } | { kind: "b"; at: number; item: B })[] = [ + ...a.map((x) => ({ kind: "a" as const, ...x })), + ...b.map((x) => ({ kind: "b" as const, ...x })), + ]; + return merged.sort((x, y) => y.at - x.at); +} diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/useAgentTraces.ts b/ui/litellm-dashboard/src/components/view_logs/TraceView/useAgentTraces.ts new file mode 100644 index 000000000000..c0be361c02e3 --- /dev/null +++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/useAgentTraces.ts @@ -0,0 +1,79 @@ +import { useInfiniteQuery } from "@tanstack/react-query"; +import moment from "moment"; +import { useMemo } from "react"; + +import { ApiError } from "@/lib/http/client"; + +import { agentTraceListCall } from "../../networking"; +import { LIVE_TAIL_INTERVAL_MS } from "../log_filter_logic"; +import type { TracePage, TraceSummary } from "./traceTypes"; + +export const TRACING_NOT_ENABLED_STATUS = 501; + +export const isTracingNotEnabled = (error: unknown): error is ApiError => + error instanceof ApiError && error.status === TRACING_NOT_ENABLED_STATUS; + +interface UseAgentTracesOptions { + accessToken: string; + startTime: string; + endTime: string; + isCustomDate: boolean; + isLiveTail: boolean; + enabled: boolean; +} + +export interface AgentTracesResult { + traces: TraceSummary[]; + isLoading: boolean; + isFetching: boolean; + /** Set when the proxy answered 501: tracing isn't configured. */ + notEnabledDetail: string | null; + error: Error | null; + hasMore: boolean; + loadMore: () => void; + refetch: () => void; +} + +/** + * GET /v1/traces for the Logs page time range, cursor-paginated ("Load more"). + * Preset ranges re-read "now" on every fetch so live tail keeps moving the end bound. + */ +export function useAgentTraces({ + accessToken, + startTime, + endTime, + isCustomDate, + isLiveTail, + enabled, +}: UseAgentTracesOptions): AgentTracesResult { + const query = useInfiniteQuery({ + queryKey: ["agentTraces", accessToken, startTime, endTime, isCustomDate], + queryFn: ({ pageParam }) => + agentTraceListCall({ + accessToken, + startMs: moment(startTime).valueOf(), + endMs: isCustomDate ? moment(endTime).valueOf() : Date.now(), + cursor: pageParam as string | null, + }), + initialPageParam: null, + getNextPageParam: (lastPage) => lastPage.next_cursor ?? undefined, + enabled, + retry: (failureCount, error) => !isTracingNotEnabled(error) && failureCount < 1, + refetchInterval: (q) => (isLiveTail && !isTracingNotEnabled(q.state.error) ? LIVE_TAIL_INTERVAL_MS : false), + refetchIntervalInBackground: false, + }); + + const traces = useMemo(() => query.data?.pages.flatMap((page) => page.data) ?? [], [query.data]); + const notEnabled = isTracingNotEnabled(query.error); + + return { + traces, + isLoading: query.isLoading, + isFetching: query.isFetching, + notEnabledDetail: notEnabled ? query.error?.message || "Agent tracing is not enabled" : null, + error: notEnabled ? null : query.error, + hasMore: query.hasNextPage, + loadMore: () => void query.fetchNextPage(), + refetch: () => void query.refetch(), + }; +} diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/useTraceNavigation.ts b/ui/litellm-dashboard/src/components/view_logs/TraceView/useTraceNavigation.ts new file mode 100644 index 000000000000..7910a306aaa8 --- /dev/null +++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/useTraceNavigation.ts @@ -0,0 +1,33 @@ +import { useEffect } from "react"; + +import { KEY_J_LOWER, KEY_J_UPPER, KEY_K_LOWER, KEY_K_UPPER } from "../LogDetailsDrawer/constants"; + +const isTyping = (target: EventTarget | null): boolean => + target instanceof HTMLInputElement || target instanceof HTMLTextAreaElement; + +/** Id `delta` steps away from `current` in `order`, clamped to the ends. */ +export const stepSelection = (order: readonly string[], current: string | null, delta: number): string | null => { + if (order.length === 0) return null; + const index = current ? order.indexOf(current) : -1; + if (index < 0) return order[0]; + return order[Math.min(Math.max(index + delta, 0), order.length - 1)]; +}; + +/** J / ArrowDown selects the next span, K / ArrowUp the previous one. Esc is handled by the Sheet. */ +export function useTraceNavigation(active: boolean, onStep: (delta: number) => void): void { + useEffect(() => { + if (!active) return; + const handleKeyDown = (event: KeyboardEvent) => { + if (isTyping(event.target)) return; + if ([KEY_J_LOWER, KEY_J_UPPER, "ArrowDown"].includes(event.key)) { + event.preventDefault(); + onStep(1); + } else if ([KEY_K_LOWER, KEY_K_UPPER, "ArrowUp"].includes(event.key)) { + event.preventDefault(); + onStep(-1); + } + }; + window.addEventListener("keydown", handleKeyDown); + return () => window.removeEventListener("keydown", handleKeyDown); + }, [active, onStep]); +} From 473faeca1f2014b0c3bf73e8923308a6465c705e Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Wed, 30 Sep 2026 10:09:28 -0700 Subject: [PATCH 02/12] fix(tracing): include new tests in CI and meet UI lint budgets --- .github/workflows/test-unit.yml | 4 +- .../components/view_logs/RequestLogsPanel.tsx | 5 ++- .../view_logs/TraceView/AgentGraph.tsx | 4 +- .../TraceView/AgentTracesSection.tsx | 5 ++- .../view_logs/TraceView/SpanDetail.tsx | 9 ++-- .../view_logs/TraceView/TraceDrawer.tsx | 13 ++---- .../view_logs/TraceView/TraceSteps.tsx | 4 +- .../view_logs/TraceView/traceUtils.test.ts | 42 ++++++++++++------- .../view_logs/TraceView/traceUtils.ts | 30 ++++++++----- 9 files changed, 66 insertions(+), 50 deletions(-) diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index f55e186e3b22..d3604a81e8fc 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -79,7 +79,9 @@ jobs: - shard: integrations artifact-name: integrations - test-path: "" + test-path: >- + tests/test_litellm/integrations + tests/test_litellm/tracing unit-flag: integrations workers: 2 reruns: 3 diff --git a/ui/litellm-dashboard/src/components/view_logs/RequestLogsPanel.tsx b/ui/litellm-dashboard/src/components/view_logs/RequestLogsPanel.tsx index c6280a6f4eb3..dc0f84c1d56f 100644 --- a/ui/litellm-dashboard/src/components/view_logs/RequestLogsPanel.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/RequestLogsPanel.tsx @@ -92,7 +92,7 @@ export default function RequestLogsPanel({ accessToken, token, userRole, userID, return debouncedSearch === "" ? others : [...others, { id: LOG_FILTER_IDS.SEARCH, value: debouncedSearch }]; }, [columnFilters, debouncedSearch]); - const { logsQuery, filteredLogs, allTeams, usesSessionCursor } = useLogFilterLogic({ + const filterOptions: Parameters[0] = { accessToken, token, userRole, @@ -107,7 +107,8 @@ export default function RequestLogsPanel({ accessToken, token, userRole, userID, isCustomDate, sorting, sessionCursors, - }); + }; + const { logsQuery, filteredLogs, allTeams, usesSessionCursor } = useLogFilterLogic(filterOptions); // Follow the table's own last fetch so a live-tail refresh carries the filter // window with it; before the first fetch, fall back to the stored end time. diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentGraph.tsx b/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentGraph.tsx index 20c0434b7b73..e730b5d3d98d 100644 --- a/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentGraph.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentGraph.tsx @@ -196,9 +196,7 @@ export function AgentGraph({ agents, spans, onOpenInvocation }: AgentGraphProps) /> ))} -

- Click an agent to list its invocations. -

+

Click an agent to list its invocations.

{selected && ( )} diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesSection.tsx b/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesSection.tsx index 6bb2e7cefd78..7e012310d474 100644 --- a/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesSection.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesSection.tsx @@ -60,14 +60,15 @@ export function AgentTracesSection({ isLiveTail, }: AgentTracesSectionProps) { const [openTrace, setOpenTrace] = useState<{ traceId: string; spanId: string | null } | null>(null); - const traces = useAgentTraces({ + const traceOptions: Parameters[0] = { accessToken, startTime, endTime, isCustomDate, isLiveTail, enabled: isActive && view !== "llm", - }); + }; + const traces = useAgentTraces(traceOptions); if (view === "llm") return null; diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/SpanDetail.tsx b/ui/litellm-dashboard/src/components/view_logs/TraceView/SpanDetail.tsx index 7e22613d83e0..817e4151271e 100644 --- a/ui/litellm-dashboard/src/components/view_logs/TraceView/SpanDetail.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/SpanDetail.tsx @@ -93,6 +93,9 @@ export function SpanDetail({ accessToken, traceId, span }: SpanDetailProps) { staleTime: Infinity, }); const detail = detailQuery.data; + const attributes = detail + ? { trace_id: traceId, span_id: span.span_id, parent_span_id: span.parent_span_id, ...detail.attributes } + : null; return (
@@ -134,11 +137,7 @@ export function SpanDetail({ accessToken, traceId, span }: SpanDetailProps) {

OTEL attributes

-              {JSON.stringify(
-                { trace_id: traceId, span_id: span.span_id, parent_span_id: span.parent_span_id, ...detail.attributes },
-                null,
-                2,
-              )}
+              {JSON.stringify(attributes, null, 2)}
diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/TraceDrawer.tsx b/ui/litellm-dashboard/src/components/view_logs/TraceView/TraceDrawer.tsx index 67006ef191a7..13f5ac79bba1 100644 --- a/ui/litellm-dashboard/src/components/view_logs/TraceView/TraceDrawer.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/TraceDrawer.tsx @@ -260,19 +260,14 @@ function TraceHeader({ trace, onClose }: { trace: Trace; onClose: () => void }) } /** Right-side drawer for one agent trace: header, stats strip, Steps / Tree / Graph views and span details. */ -export function TraceDrawer({ - open, - traceId, - initialSpanId, - accessToken, - onClose, -}: TraceDrawerProps) { - const traceQuery = useQuery({ +export function TraceDrawer({ open, traceId, initialSpanId, accessToken, onClose }: TraceDrawerProps) { + const traceQueryOptions = { queryKey: ["agentTrace", traceId, accessToken], queryFn: () => agentTraceCall(accessToken, traceId as string), enabled: open && traceId !== null, staleTime: 30_000, - }); + }; + const traceQuery = useQuery(traceQueryOptions); const trace = traceQuery.data; return ( diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/TraceSteps.tsx b/ui/litellm-dashboard/src/components/view_logs/TraceView/TraceSteps.tsx index 1c2819f965a2..b17a3e067c74 100644 --- a/ui/litellm-dashboard/src/components/view_logs/TraceView/TraceSteps.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/TraceSteps.tsx @@ -31,9 +31,7 @@ function StepTitle({ step }: { step: TraceStep }) { <> {spanLabel(span)} - - {fmtMs(span.duration_ms)} - + {fmtMs(span.duration_ms)} ); } diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/traceUtils.test.ts b/ui/litellm-dashboard/src/components/view_logs/TraceView/traceUtils.test.ts index 37da685bb5d2..12d6c53491f0 100644 --- a/ui/litellm-dashboard/src/components/view_logs/TraceView/traceUtils.test.ts +++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/traceUtils.test.ts @@ -62,7 +62,8 @@ describe("formatting", () => { }); it("headlines LLM spans with the OTLP model", () => { - expect(spanLabel(span({ span_id: "llm", type: "llm", name: "generation", model: "model-a" }))).toBe("model-a"); + const llm = { span_id: "llm", type: "llm", name: "generation", model: "model-a" } as const; + expect(spanLabel(span(llm))).toBe("model-a"); }); it("pulls the user message out of a truncated JSON preview", () => { @@ -84,13 +85,17 @@ describe("formatting", () => { describe("buildVisibleTree", () => { it("hides framework spans and re-parents their children to the nearest visible ancestor", () => { - const spans = [ - span({ span_id: "root", type: "agent" }), - span({ span_id: "mw", parent_span_id: "root", type: "framework", name: "X.wrap_model_call" }), - span({ span_id: "model", parent_span_id: "mw", type: "chain", name: "model" }), - span({ span_id: "llm", parent_span_id: "model", type: "llm", start_offset_ms: 5 }), - span({ span_id: "step", parent_span_id: "root", type: "chain", name: "planner", start_offset_ms: 1 }), - ]; + const middleware = { span_id: "mw", parent_span_id: "root", type: "framework", name: "X.wrap_model_call" } as const; + const model = { span_id: "model", parent_span_id: "mw", type: "chain", name: "model" } as const; + const llm = { span_id: "llm", parent_span_id: "model", type: "llm", start_offset_ms: 5 } as const; + const step = { + span_id: "step", + parent_span_id: "root", + type: "chain", + name: "planner", + start_offset_ms: 1, + } as const; + const spans = [span({ span_id: "root", type: "agent" }), span(middleware), span(model), span(llm), span(step)]; const compact = buildVisibleTree(spans, false); expect(compact.visibleCount).toBe(3); expect(compact.children.get(ROOT_KEY)?.map((s) => s.span_id)).toEqual(["root"]); @@ -141,9 +146,10 @@ describe("groupSiblingAgents", () => { it("leaves groups of 10 or fewer alone", () => { const parent = span({ span_id: "p", type: "agent" }); - const kids = Array.from({ length: 10 }, (_, i) => - span({ span_id: `k${i}`, parent_span_id: "p", type: "agent", name: "worker" }), - ); + const kids = Array.from({ length: 10 }, (_, i) => { + const props = { span_id: `k${i}`, parent_span_id: "p", type: "agent", name: "worker" } as const; + return span(props); + }); const items = groupSiblingAgents("p", kids, subtreeStats([parent, ...kids])); expect(items.every((i) => i.kind === "span")).toBe(true); }); @@ -161,9 +167,16 @@ describe("flattenTree + revealSpan", () => { it("pages group invocations 20 at a time with a 'more' row until all are shown", () => { const parent = span({ span_id: "p", type: "agent" }); - const kids = Array.from({ length: 45 }, (_, i) => - span({ span_id: `k${i}`, parent_span_id: "p", type: "agent", name: "worker", start_offset_ms: i }), - ); + const kids = Array.from({ length: 45 }, (_, i) => { + const props = { + span_id: `k${i}`, + parent_span_id: "p", + type: "agent", + name: "worker", + start_offset_ms: i, + } as const; + return span(props); + }); const all = [parent, ...kids]; const tree = buildVisibleTree(all, false).children; const key = "p::worker"; @@ -211,7 +224,6 @@ describe("stepsFromSpans", () => { // the lead's first decision runs write_file and task expect(first.toolNames).toEqual(["write_file", "task"]); }); - }); describe("trace-level helpers", () => { diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/traceUtils.ts b/ui/litellm-dashboard/src/components/view_logs/TraceView/traceUtils.ts index 116a822a0442..08a176e5bbdd 100644 --- a/ui/litellm-dashboard/src/components/view_logs/TraceView/traceUtils.ts +++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/traceUtils.ts @@ -47,9 +47,10 @@ export interface VisibleTree { * Framework plumbing: middleware wrappers plus LangGraph's generic "model" / "tools" * graph nodes. The root span is never hidden. */ -export const isFrameworkSpan = (span: Span): boolean => - span.parent_span_id !== null && - (span.type === "framework" || (span.type === "chain" && (span.name === "model" || span.name === "tools"))); +export const isFrameworkSpan = (span: Span): boolean => { + const isGraphNode = span.type === "chain" && (span.name === "model" || span.name === "tools"); + return span.parent_span_id !== null && (span.type === "framework" || isGraphNode); +}; const byStart = (a: Span, b: Span): number => a.start_offset_ms - b.start_offset_ms; @@ -208,16 +209,21 @@ interface FlattenContext { function pushSpanRow(ctx: FlattenContext, span: Span, depth: number): void { const hasChildren = (ctx.children.get(span.span_id)?.length ?? 0) > 0; const isCollapsed = ctx.ui.collapsed.has(span.span_id); - ctx.rows.push({ kind: "span", span, depth, hasChildren, isCollapsed }); + const row: TreeRow = { kind: "span", span, depth, hasChildren, isCollapsed }; + ctx.rows.push(row); if (hasChildren && !isCollapsed) walkChildren(ctx, span.span_id, depth + 1); } function pushGroupRows(ctx: FlattenContext, group: SpanGroup, depth: number): void { const shown = Math.min(ctx.ui.groupShown[group.key] ?? 0, group.spans.length); - ctx.rows.push({ kind: "group", group, depth, shown }); + const groupRow: TreeRow = { kind: "group", group, depth, shown }; + ctx.rows.push(groupRow); if (shown === 0) return; group.spans.slice(0, shown).forEach((span) => pushSpanRow(ctx, span, depth + 1)); - if (shown < group.spans.length) ctx.rows.push({ kind: "more", group, depth: depth + 1, shown }); + if (shown < group.spans.length) { + const moreRow: TreeRow = { kind: "more", group, depth: depth + 1, shown }; + ctx.rows.push(moreRow); + } } function walkChildren(ctx: FlattenContext, parentKey: string, depth: number): void { @@ -286,8 +292,10 @@ export interface TraceStep { toolNames: string[]; } -const isStepSpan = (span: Span): boolean => - span.type === "llm" || span.type === "tool" || (span.type === "agent" && span.parent_span_id !== null); +const isStepSpan = (span: Span): boolean => { + const isNestedAgent = span.type === "agent" && span.parent_span_id !== null; + return span.type === "llm" || span.type === "tool" || isNestedAgent; +}; interface AgentContext { subagent: string | null; @@ -450,8 +458,10 @@ export const parseJson = (value: string): unknown => { } }; -const isMessage = (value: unknown): value is TraceMessage => - typeof value === "object" && value !== null && "role" in value && typeof (value as TraceMessage).role === "string"; +const isMessage = (value: unknown): value is TraceMessage => { + if (typeof value !== "object" || value === null || !("role" in value)) return false; + return typeof value.role === "string"; +}; /** An llm span's input (array of messages) or output (one message); null when it isn't one. */ export function parseMessages(value: string): TraceMessage[] | null { From 7b0493563ba95dc66c8371ab43ad69a777d8247b Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Wed, 30 Sep 2026 10:18:54 -0700 Subject: [PATCH 03/12] style(tracing): format Rust bridge facade --- litellm/rust_bridge/traces.py | 1 + 1 file changed, 1 insertion(+) diff --git a/litellm/rust_bridge/traces.py b/litellm/rust_bridge/traces.py index 6e64422d1be4..5d0fa7ee05d5 100644 --- a/litellm/rust_bridge/traces.py +++ b/litellm/rust_bridge/traces.py @@ -51,6 +51,7 @@ def trace_decode_otlp( max_decompressed_bytes: int, ) -> list[DecodedSpan]: ... + class QueryResponse(BaseModel): model_config = ConfigDict(frozen=True) data: list[dict[str, JsonValue]] From b7ab95078c4a88986ac908b47eaeddfa56975cf7 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Wed, 30 Sep 2026 18:47:16 +0000 Subject: [PATCH 04/12] fix(tracing): restore Python 3.10 compatibility Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/tracing/store.py | 4 +- tests/test_litellm_rust/test_traces.py | 13 +- ui/litellm-dashboard/src/lib/http/schema.d.ts | 170 ++++++++++++++++++ 3 files changed, 181 insertions(+), 6 deletions(-) diff --git a/litellm/tracing/store.py b/litellm/tracing/store.py index e6814084c0fb..acb473870699 100644 --- a/litellm/tracing/store.py +++ b/litellm/tracing/store.py @@ -2,7 +2,7 @@ import base64 import json -from datetime import UTC, datetime +from datetime import datetime, timezone from typing import Any, Final from litellm.constants import AGENT_TRACING_LIST_PAGE_SIZE @@ -102,7 +102,7 @@ def decode_cursor(cursor: str | None) -> tuple[int, str]: def _iso(ms: int) -> str: - return datetime.fromtimestamp(ms / 1000, tz=UTC).isoformat() + return datetime.fromtimestamp(ms / 1000, tz=timezone.utc).isoformat() def _status(code: str) -> SpanStatus: diff --git a/tests/test_litellm_rust/test_traces.py b/tests/test_litellm_rust/test_traces.py index 78071bccec8b..447ca2ce4bbd 100644 --- a/tests/test_litellm_rust/test_traces.py +++ b/tests/test_litellm_rust/test_traces.py @@ -39,11 +39,16 @@ async def test_trace_reader_rejects_success_status_with_embedded_error(recording @pytest.mark.asyncio -@pytest.mark.parametrize("database,retention", [("db; DROP DATABASE default", 7), ("traces", 0)]) -async def test_schema_binding_preserves_configuration_validation(database: str, retention: int) -> None: +async def test_schema_binding_rejects_invalid_database() -> None: with pytest.raises(ValueError, match=r"database.*retention"): - storage: Final = NativeTraceStorage(database, "http://localhost:8123") - await storage.ensure_schema(retention, 14) + NativeTraceStorage("db; DROP DATABASE default", "http://localhost:8123") + + +@pytest.mark.asyncio +async def test_schema_binding_rejects_non_positive_retention() -> None: + storage: Final = NativeTraceStorage("traces", "http://localhost:8123") + with pytest.raises(ValueError, match=r"database.*retention"): + await storage.ensure_schema(0, 14) @pytest.mark.asyncio diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index cc0435c3d5e7..f9cf4232cf74 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -21262,6 +21262,58 @@ export interface paths { patch?: never; trace?: never; }; + "/v1/traces": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** List Agent Traces */ + get: operations["list_agent_traces_v1_traces_get"]; + put?: never; + /** Ingest Otlp Traces */ + post: operations["ingest_otlp_traces_v1_traces_post"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/v1/traces/{trace_id}": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Get Agent Trace */ + get: operations["get_agent_trace_v1_traces__trace_id__get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/v1/traces/{trace_id}/spans/{span_id}": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Get Agent Trace Span */ + get: operations["get_agent_trace_span_v1_traces__trace_id__spans__span_id__get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/v1/unified_access_group": { parameters: { query?: never; @@ -74649,6 +74701,124 @@ export interface operations { }; }; }; + list_agent_traces_v1_traces_get: { + parameters: { + query?: { + /** @description Window start, unix ms. Default: 24h ago */ + start_ms?: number | null; + /** @description Window end, unix ms. Default: now */ + end_ms?: number | null; + cursor?: string | null; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + ingest_otlp_traces_v1_traces_post: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown; + }; + }; + }; + }; + get_agent_trace_v1_traces__trace_id__get: { + parameters: { + query?: never; + header?: never; + path: { + trace_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + get_agent_trace_span_v1_traces__trace_id__spans__span_id__get: { + parameters: { + query?: never; + header?: never; + path: { + trace_id: string; + span_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; list_access_groups_v1_unified_access_group_get: { parameters: { query?: never; From 2f0bcc7f4a87e1742e7861641c389a0d5fae1e1a Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Tue, 29 Sep 2026 23:34:25 -0700 Subject: [PATCH 05/12] wip --- litellm-rust/crates/traces/README.md | 13 +++ litellm/proxy/_types.py | 1 + litellm/proxy/logging_endpoints/traces.py | 95 +++++++++++++++++++ litellm/proxy/proxy_server.py | 3 + litellm/rust_bridge/_native.pyi | 12 +++ .../proxy/logging_endpoints/test_traces.py | 45 +++++++++ 6 files changed, 169 insertions(+) create mode 100644 litellm-rust/crates/traces/README.md create mode 100644 litellm/proxy/logging_endpoints/traces.py create mode 100644 tests/unit/proxy/logging_endpoints/test_traces.py diff --git a/litellm-rust/crates/traces/README.md b/litellm-rust/crates/traces/README.md new file mode 100644 index 000000000000..e3d4e2bc50f6 --- /dev/null +++ b/litellm-rust/crates/traces/README.md @@ -0,0 +1,13 @@ +# Trace queries + +Use a dedicated ClickHouse reader account for `general_settings.clickhouse_url`. The SQL helper sends the supplied SQL to ClickHouse without parsing it. Its `readonly=1` request setting does not make an account with write or administrative grants safe + +Pass `execute_admin_sql` a shared HTTP client obtained from `HttpClientPool` with `ClientVariant::NoRedirect` and the host's HTTP configuration. The helper applies a 15-second request timeout and validates the bounded JSON response, including errors returned with HTTP 200 + +For self-hosted ClickHouse, install [config/reader.xml](config/reader.xml) as `/etc/clickhouse-server/users.d/litellm-traces-reader.xml` and provide `LITELLM_TRACES_READER_PASSWORD` in the ClickHouse server environment. Restrict the configured network range to your proxy network. The configuration grants SELECT only on `default.otel_traces` and locks read-only mode, execution time, result rows and bytes, overflow behavior and memory limits. Adjust the database name if traces live elsewhere + +Set `clickhouse_url` to `https://litellm_traces_reader:@:8443/?database=default`. Keep ingestion and migration credentials separate. For managed ClickHouse, provision the equivalent SELECT grant and settings constraints through its access-management interface + +Do not grant write, backup, named-collection management or grant-option privileges to the reader, directly or through roles. Do not make `readonly` changeable in read-only mode. ClickHouse documents exceptions to `readonly` in [query permissions](https://clickhouse.com/docs/concepts/features/configuration/settings/permissions-for-queries) and explains locked settings in [settings constraints](https://clickhouse.com/docs/concepts/features/configuration/settings/constraints-on-settings) + +Run the reader permission, authentication and response-limit tests with `cargo test -p litellm-traces --test admin_sql -- --test-threads=2` from `litellm-rust`. They start disposable ClickHouse instances through Testcontainers and load the same reader configuration diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 5aa617734b45..d1770da5e1c0 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -2628,6 +2628,7 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase): Documents all the fields supported by `general_settings` in config.yaml """ + clickhouse_url: str | None = Field(None, description="ClickHouse HTTP URL for trace queries") completion_model: str | None = Field(None, description="proxy level default model for all chat completion calls") max_in_flight_requests_per_worker: int | None = Field( None, gt=0, description="maximum concurrent requests handled by each worker" diff --git a/litellm/proxy/logging_endpoints/traces.py b/litellm/proxy/logging_endpoints/traces.py new file mode 100644 index 000000000000..ac82f8cb0227 --- /dev/null +++ b/litellm/proxy/logging_endpoints/traces.py @@ -0,0 +1,95 @@ +from collections.abc import Awaitable +from datetime import datetime, timezone +from typing import Final, Protocol, cast + +from fastapi import APIRouter, Depends, HTTPException, Query + +from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.rust_bridge.loader import get_native_bridge + +router: Final = APIRouter(prefix="/v1/traces", tags=["Traces"]) + + +class NativeTraces(Protocol): + def list_traces( + self, + clickhouse_url: str, + start_ms: int, + end_ms: int, + service: str | None, + status: str | None, + search: str | None, + cursor: str | None, + limit: int, + ) -> Awaitable[object]: ... + + def get_trace(self, clickhouse_url: str, trace_id: str) -> Awaitable[object]: ... + + def get_span(self, clickhouse_url: str, trace_id: str, span_id: str) -> Awaitable[object]: ... + + +def _connection(user_api_key_dict: UserAPIKeyAuth) -> tuple[str, NativeTraces]: + if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN: + raise HTTPException(status_code=403, detail="Trace queries require proxy admin access") + + from litellm.proxy.proxy_server import general_settings_view + + clickhouse_url: Final = general_settings_view().get("clickhouse_url") + if not isinstance(clickhouse_url, str) or not clickhouse_url: + raise HTTPException(status_code=503, detail="ClickHouse trace queries are not configured") + + native: Final = get_native_bridge() + if native is None or not all(hasattr(native, name) for name in ("list_traces", "get_trace", "get_span")): + raise HTTPException(status_code=503, detail="Rust trace query bridge is unavailable") + return clickhouse_url, cast(NativeTraces, native) + + +async def _result(awaitable: Awaitable[object]) -> object: + try: + return await awaitable + except NotImplementedError as exc: + raise HTTPException(status_code=501, detail=str(exc)) from exc + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + except RuntimeError as exc: + raise HTTPException(status_code=503, detail=str(exc)) from exc + + +def _utc_ms(value: datetime) -> int: + utc_value: Final = value.replace(tzinfo=timezone.utc) if value.tzinfo is None else value.astimezone(timezone.utc) + return int(utc_value.timestamp() * 1000) + + +@router.get("") +async def list_traces( + start: datetime, + end: datetime, + service: str | None = None, + status: str | None = None, + q: str | None = None, + cursor: str | None = None, + limit: int = Query(default=50, ge=1, le=100), + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +) -> object: + clickhouse_url, native = _connection(user_api_key_dict) + return await _result(native.list_traces(clickhouse_url, _utc_ms(start), _utc_ms(end), service, status, q, cursor, limit)) + + +@router.get("/{trace_id}") +async def get_trace( + trace_id: str, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +) -> object: + clickhouse_url, native = _connection(user_api_key_dict) + return await _result(native.get_trace(clickhouse_url, trace_id)) + + +@router.get("/{trace_id}/spans/{span_id}") +async def get_span( + trace_id: str, + span_id: str, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +) -> object: + clickhouse_url, native = _connection(user_api_key_dict) + return await _result(native.get_span(clickhouse_url, trace_id, span_id)) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 7f2742fcffc2..bd47f51aa7b8 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -568,6 +568,7 @@ def generate_feedback_box(): from litellm.proxy.logging_endpoints.callback_logs_endpoints import ( rust_control_plane_router, ) +from litellm.proxy.logging_endpoints.traces import router as traces_router from litellm.proxy.management_endpoints.auto_router_endpoints import ( router as auto_router_management_router, ) @@ -974,6 +975,7 @@ def generate_feedback_box(): _EXTRA_SECRET_GENERAL_SETTINGS_FIELDS: Final = frozenset( { "database_url", + "clickhouse_url", "database_extra_connection_params", "pass_through_endpoints", "alert_to_webhook_url", @@ -19907,6 +19909,7 @@ async def get_routes(): app.include_router(callback_management_endpoints_router) app.include_router(debugging_endpoints_router) app.include_router(rust_control_plane_router) +app.include_router(traces_router) app.include_router(ui_crud_endpoints_router) app.include_router(user_banner_endpoints_router) app.include_router(latest_release_endpoints_router) diff --git a/litellm/rust_bridge/_native.pyi b/litellm/rust_bridge/_native.pyi index 30d4bbfb68e1..8104ca366818 100644 --- a/litellm/rust_bridge/_native.pyi +++ b/litellm/rust_bridge/_native.pyi @@ -20,6 +20,18 @@ class RustBridgeDeclined(Exception): ... class RustUpstreamError(Exception): ... class ForkedAfterNativeRuntimeStarted(RuntimeError): ... class ProcessReservedForForking(RuntimeError): ... +def list_traces( + clickhouse_url: str, + start_ms: int, + end_ms: int, + service: str | None = None, + status: str | None = None, + search: str | None = None, + cursor: str | None = None, + limit: int = 50, +) -> Future[None]: ... +def get_trace(clickhouse_url: str, trace_id: str) -> Future[None]: ... +def get_span(clickhouse_url: str, trace_id: str, span_id: str) -> Future[None]: ... def trace_decode_otlp( body: bytes, content_type: str | None, content_encoding: str | None, max_decompressed_bytes: int diff --git a/tests/unit/proxy/logging_endpoints/test_traces.py b/tests/unit/proxy/logging_endpoints/test_traces.py new file mode 100644 index 000000000000..2c92e47d78bb --- /dev/null +++ b/tests/unit/proxy/logging_endpoints/test_traces.py @@ -0,0 +1,45 @@ +import asyncio +from collections.abc import Callable, Coroutine +from datetime import datetime, timezone + +import pytest +from fastapi import HTTPException + +from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth +from litellm.proxy.logging_endpoints.traces import get_span, get_trace, list_traces + + +def _list(auth: UserAPIKeyAuth) -> Coroutine[object, object, object]: + return list_traces( + start=datetime(2026, 1, 1, tzinfo=timezone.utc), + end=datetime(2026, 1, 2, tzinfo=timezone.utc), + limit=50, + user_api_key_dict=auth, + ) + + +def _trace(auth: UserAPIKeyAuth) -> Coroutine[object, object, object]: + return get_trace(trace_id="trace", user_api_key_dict=auth) + + +def _span(auth: UserAPIKeyAuth) -> Coroutine[object, object, object]: + return get_span(trace_id="trace", span_id="span", user_api_key_dict=auth) + + +@pytest.mark.parametrize( + "run_route", + ( + _list, + _trace, + _span, + ), +) +def test_trace_routes_reject_non_admin_callers_before_accessing_storage( + run_route: Callable[[UserAPIKeyAuth], Coroutine[object, object, object]], +) -> None: + auth = UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER) + + with pytest.raises(HTTPException) as error: + asyncio.run(run_route(auth)) + + assert error.value.status_code == 403 From b88ab3e894a3a5f2a03deacfd7ec3e6ad219bc54 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Wed, 30 Sep 2026 07:11:55 -0700 Subject: [PATCH 06/12] feat(traces): establish shared Rust storage foundation --- litellm-rust/crates/traces/README.md | 16 ++-- litellm/proxy/_types.py | 1 - litellm/proxy/logging_endpoints/traces.py | 95 ------------------- litellm/proxy/proxy_server.py | 3 - litellm/rust_bridge/_native.pyi | 22 ++--- .../proxy/logging_endpoints/test_traces.py | 45 --------- 6 files changed, 19 insertions(+), 163 deletions(-) delete mode 100644 litellm/proxy/logging_endpoints/traces.py delete mode 100644 tests/unit/proxy/logging_endpoints/test_traces.py diff --git a/litellm-rust/crates/traces/README.md b/litellm-rust/crates/traces/README.md index e3d4e2bc50f6..657bdc58d6ce 100644 --- a/litellm-rust/crates/traces/README.md +++ b/litellm-rust/crates/traces/README.md @@ -1,13 +1,15 @@ -# Trace queries +# Trace storage foundation -Use a dedicated ClickHouse reader account for `general_settings.clickhouse_url`. The SQL helper sends the supplied SQL to ClickHouse without parsing it. Its `readonly=1` request setting does not make an account with write or administrative grants safe +This crate owns the initial ClickHouse schema and parameterized read transport. Python adapters use `trace_schema_statements` and `trace_query` from the native bridge. HTTP endpoints, ingestion and trace response assembly belong to the integration built on this foundation -Pass `execute_admin_sql` a shared HTTP client obtained from `HttpClientPool` with `ClientVariant::NoRedirect` and the host's HTTP configuration. The helper applies a 15-second request timeout and validates the bounded JSON response, including errors returned with HTTP 200 +The SQL files under `migrations/` are the canonical definitions for `otel_traces`, `agent_traces`, its materialized view, and `spend_logs`. `schema_statements` supplies the database and retention settings. Apply the returned statements in order using migration credentials. Setup is repeatable for a new installation; these initial `CREATE IF NOT EXISTS` statements do not upgrade an existing incompatible table. Future schema changes need explicit upgrade statements -For self-hosted ClickHouse, install [config/reader.xml](config/reader.xml) as `/etc/clickhouse-server/users.d/litellm-traces-reader.xml` and provide `LITELLM_TRACES_READER_PASSWORD` in the ClickHouse server environment. Restrict the configured network range to your proxy network. The configuration grants SELECT only on `default.otel_traces` and locks read-only mode, execution time, result rows and bytes, overflow behavior and memory limits. Adjust the database name if traces live elsewhere +`execute_read` uses ClickHouse typed placeholders and binds text, integer and string-array parameters separately from SQL. It obtains no privileges itself: callers must use a dedicated SELECT-only reader. The helper enforces a 15-second HTTP timeout, a 10-second query setting, 1,000 result rows and a 4 MiB response cap, and rejects errors embedded in HTTP 200 JSON responses. Large traces fail explicitly at those limits; span pagination is future work -Set `clickhouse_url` to `https://litellm_traces_reader:@:8443/?database=default`. Keep ingestion and migration credentials separate. For managed ClickHouse, provision the equivalent SELECT grant and settings constraints through its access-management interface +Pass a shared `HttpClientPool` client using `ClientVariant::NoRedirect` and the host HTTP settings. The native bridge already does this. The SQL helper is internal infrastructure, not a public arbitrary-SQL endpoint -Do not grant write, backup, named-collection management or grant-option privileges to the reader, directly or through roles. Do not make `readonly` changeable in read-only mode. ClickHouse documents exceptions to `readonly` in [query permissions](https://clickhouse.com/docs/concepts/features/configuration/settings/permissions-for-queries) and explains locked settings in [settings constraints](https://clickhouse.com/docs/concepts/features/configuration/settings/constraints-on-settings) +For self-hosted ClickHouse, install [config/reader.xml](config/reader.xml) as `/etc/clickhouse-server/users.d/litellm-traces-reader.xml` and set `LITELLM_TRACES_READER_PASSWORD` on the server. Change `default` in the three SELECT grants to the configured database and restrict the network range to the proxy network. The profile locks read-only mode, execution time, result rows and bytes, overflow behavior and memory limits. Managed ClickHouse deployments need equivalent grants and settings constraints -Run the reader permission, authentication and response-limit tests with `cargo test -p litellm-traces --test admin_sql -- --test-threads=2` from `litellm-rust`. They start disposable ClickHouse instances through Testcontainers and load the same reader configuration +Keep reader, ingestion and migration credentials separate. Do not grant the reader write, backup, named-collection management or grant-option privileges, including through roles. `readonly=1` alone is insufficient for privileged accounts; see [query permissions](https://clickhouse.com/docs/concepts/features/configuration/settings/permissions-for-queries) and [settings constraints](https://clickhouse.com/docs/concepts/features/configuration/settings/constraints-on-settings) + +Run `cargo test -p litellm-traces -- --test-threads=2` from `litellm-rust`. Testcontainers applies the canonical schema, checks span rollups and spend joins, and exercises reader permissions, parameter binding, credentials and response limits diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index d1770da5e1c0..5aa617734b45 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -2628,7 +2628,6 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase): Documents all the fields supported by `general_settings` in config.yaml """ - clickhouse_url: str | None = Field(None, description="ClickHouse HTTP URL for trace queries") completion_model: str | None = Field(None, description="proxy level default model for all chat completion calls") max_in_flight_requests_per_worker: int | None = Field( None, gt=0, description="maximum concurrent requests handled by each worker" diff --git a/litellm/proxy/logging_endpoints/traces.py b/litellm/proxy/logging_endpoints/traces.py deleted file mode 100644 index ac82f8cb0227..000000000000 --- a/litellm/proxy/logging_endpoints/traces.py +++ /dev/null @@ -1,95 +0,0 @@ -from collections.abc import Awaitable -from datetime import datetime, timezone -from typing import Final, Protocol, cast - -from fastapi import APIRouter, Depends, HTTPException, Query - -from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth -from litellm.proxy.auth.user_api_key_auth import user_api_key_auth -from litellm.rust_bridge.loader import get_native_bridge - -router: Final = APIRouter(prefix="/v1/traces", tags=["Traces"]) - - -class NativeTraces(Protocol): - def list_traces( - self, - clickhouse_url: str, - start_ms: int, - end_ms: int, - service: str | None, - status: str | None, - search: str | None, - cursor: str | None, - limit: int, - ) -> Awaitable[object]: ... - - def get_trace(self, clickhouse_url: str, trace_id: str) -> Awaitable[object]: ... - - def get_span(self, clickhouse_url: str, trace_id: str, span_id: str) -> Awaitable[object]: ... - - -def _connection(user_api_key_dict: UserAPIKeyAuth) -> tuple[str, NativeTraces]: - if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN: - raise HTTPException(status_code=403, detail="Trace queries require proxy admin access") - - from litellm.proxy.proxy_server import general_settings_view - - clickhouse_url: Final = general_settings_view().get("clickhouse_url") - if not isinstance(clickhouse_url, str) or not clickhouse_url: - raise HTTPException(status_code=503, detail="ClickHouse trace queries are not configured") - - native: Final = get_native_bridge() - if native is None or not all(hasattr(native, name) for name in ("list_traces", "get_trace", "get_span")): - raise HTTPException(status_code=503, detail="Rust trace query bridge is unavailable") - return clickhouse_url, cast(NativeTraces, native) - - -async def _result(awaitable: Awaitable[object]) -> object: - try: - return await awaitable - except NotImplementedError as exc: - raise HTTPException(status_code=501, detail=str(exc)) from exc - except ValueError as exc: - raise HTTPException(status_code=400, detail=str(exc)) from exc - except RuntimeError as exc: - raise HTTPException(status_code=503, detail=str(exc)) from exc - - -def _utc_ms(value: datetime) -> int: - utc_value: Final = value.replace(tzinfo=timezone.utc) if value.tzinfo is None else value.astimezone(timezone.utc) - return int(utc_value.timestamp() * 1000) - - -@router.get("") -async def list_traces( - start: datetime, - end: datetime, - service: str | None = None, - status: str | None = None, - q: str | None = None, - cursor: str | None = None, - limit: int = Query(default=50, ge=1, le=100), - user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), -) -> object: - clickhouse_url, native = _connection(user_api_key_dict) - return await _result(native.list_traces(clickhouse_url, _utc_ms(start), _utc_ms(end), service, status, q, cursor, limit)) - - -@router.get("/{trace_id}") -async def get_trace( - trace_id: str, - user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), -) -> object: - clickhouse_url, native = _connection(user_api_key_dict) - return await _result(native.get_trace(clickhouse_url, trace_id)) - - -@router.get("/{trace_id}/spans/{span_id}") -async def get_span( - trace_id: str, - span_id: str, - user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), -) -> object: - clickhouse_url, native = _connection(user_api_key_dict) - return await _result(native.get_span(clickhouse_url, trace_id, span_id)) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index bd47f51aa7b8..7f2742fcffc2 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -568,7 +568,6 @@ def generate_feedback_box(): from litellm.proxy.logging_endpoints.callback_logs_endpoints import ( rust_control_plane_router, ) -from litellm.proxy.logging_endpoints.traces import router as traces_router from litellm.proxy.management_endpoints.auto_router_endpoints import ( router as auto_router_management_router, ) @@ -975,7 +974,6 @@ def generate_feedback_box(): _EXTRA_SECRET_GENERAL_SETTINGS_FIELDS: Final = frozenset( { "database_url", - "clickhouse_url", "database_extra_connection_params", "pass_through_endpoints", "alert_to_webhook_url", @@ -19909,7 +19907,6 @@ async def get_routes(): app.include_router(callback_management_endpoints_router) app.include_router(debugging_endpoints_router) app.include_router(rust_control_plane_router) -app.include_router(traces_router) app.include_router(ui_crud_endpoints_router) app.include_router(user_banner_endpoints_router) app.include_router(latest_release_endpoints_router) diff --git a/litellm/rust_bridge/_native.pyi b/litellm/rust_bridge/_native.pyi index 8104ca366818..507ab441f8ca 100644 --- a/litellm/rust_bridge/_native.pyi +++ b/litellm/rust_bridge/_native.pyi @@ -20,18 +20,16 @@ class RustBridgeDeclined(Exception): ... class RustUpstreamError(Exception): ... class ForkedAfterNativeRuntimeStarted(RuntimeError): ... class ProcessReservedForForking(RuntimeError): ... -def list_traces( - clickhouse_url: str, - start_ms: int, - end_ms: int, - service: str | None = None, - status: str | None = None, - search: str | None = None, - cursor: str | None = None, - limit: int = 50, -) -> Future[None]: ... -def get_trace(clickhouse_url: str, trace_id: str) -> Future[None]: ... -def get_span(clickhouse_url: str, trace_id: str, span_id: str) -> Future[None]: ... + +def trace_schema_statements(database: str, trace_retention_days: int, spend_log_retention_days: int) -> list[str]: ... +def trace_query( + url: str, + database: str, + user: str, + password: str, + sql: str, + parameters: Mapping[str, str | int | Sequence[str]], +) -> Future[str]: ... def trace_decode_otlp( body: bytes, content_type: str | None, content_encoding: str | None, max_decompressed_bytes: int diff --git a/tests/unit/proxy/logging_endpoints/test_traces.py b/tests/unit/proxy/logging_endpoints/test_traces.py deleted file mode 100644 index 2c92e47d78bb..000000000000 --- a/tests/unit/proxy/logging_endpoints/test_traces.py +++ /dev/null @@ -1,45 +0,0 @@ -import asyncio -from collections.abc import Callable, Coroutine -from datetime import datetime, timezone - -import pytest -from fastapi import HTTPException - -from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth -from litellm.proxy.logging_endpoints.traces import get_span, get_trace, list_traces - - -def _list(auth: UserAPIKeyAuth) -> Coroutine[object, object, object]: - return list_traces( - start=datetime(2026, 1, 1, tzinfo=timezone.utc), - end=datetime(2026, 1, 2, tzinfo=timezone.utc), - limit=50, - user_api_key_dict=auth, - ) - - -def _trace(auth: UserAPIKeyAuth) -> Coroutine[object, object, object]: - return get_trace(trace_id="trace", user_api_key_dict=auth) - - -def _span(auth: UserAPIKeyAuth) -> Coroutine[object, object, object]: - return get_span(trace_id="trace", span_id="span", user_api_key_dict=auth) - - -@pytest.mark.parametrize( - "run_route", - ( - _list, - _trace, - _span, - ), -) -def test_trace_routes_reject_non_admin_callers_before_accessing_storage( - run_route: Callable[[UserAPIKeyAuth], Coroutine[object, object, object]], -) -> None: - auth = UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER) - - with pytest.raises(HTTPException) as error: - asyncio.run(run_route(auth)) - - assert error.value.status_code == 403 From 3e74da0d0235beadeb98c565fec4da3e64477141 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Wed, 30 Sep 2026 09:42:17 -0700 Subject: [PATCH 07/12] feat(tracing): enrich agent traces with spend logs --- .../crates/python-bridge/src/routes/traces.rs | 17 +- litellm-rust/crates/traces/src/error.rs | 2 + litellm-rust/crates/traces/src/lib.rs | 2 +- .../crates/traces/src/list_traces.sql | 37 ++++ .../crates/traces/src/span_detail.sql | 4 + litellm-rust/crates/traces/src/sql.rs | 46 +++- .../crates/traces/src/trace_spans.sql | 27 +++ litellm/__init__.py | 1 + litellm/constants.py | 1 + .../clickhouse/clickhouse_spend_logger.py | 155 +++++++++++++ litellm/litellm_core_utils/litellm_logging.py | 13 ++ litellm/rust_bridge/traces.py | 15 +- litellm/tracing/store.py | 109 ++++----- litellm/tracing/types.py | 66 +++++- .../test_clickhouse_spend_logger.py | 209 ++++++++++++++++++ tests/test_litellm/tracing/test_store.py | 99 +++++++-- .../components/view_logs/RequestLogsPanel.tsx | 1 + .../view_logs/TraceView/AgentGraph.tsx | 7 +- .../TraceView/AgentTracesSection.test.tsx | 5 +- .../TraceView/AgentTracesSection.tsx | 9 + .../view_logs/TraceView/AgentTracesTable.tsx | 9 +- .../view_logs/TraceView/SpanDetail.tsx | 93 +++++++- .../view_logs/TraceView/SpanTree.tsx | 12 +- .../view_logs/TraceView/TraceDrawer.test.tsx | 22 +- .../view_logs/TraceView/TraceDrawer.tsx | 55 ++++- .../view_logs/TraceView/TraceSteps.tsx | 8 +- .../view_logs/TraceView/traceTypes.ts | 23 +- .../view_logs/TraceView/traceUtils.test.ts | 81 ++++--- .../view_logs/TraceView/traceUtils.ts | 45 +++- 29 files changed, 1005 insertions(+), 168 deletions(-) create mode 100644 litellm-rust/crates/traces/src/list_traces.sql create mode 100644 litellm-rust/crates/traces/src/span_detail.sql create mode 100644 litellm-rust/crates/traces/src/trace_spans.sql create mode 100644 litellm/integrations/clickhouse/clickhouse_spend_logger.py create mode 100644 tests/test_litellm/integrations/clickhouse/test_clickhouse_spend_logger.py diff --git a/litellm-rust/crates/python-bridge/src/routes/traces.rs b/litellm-rust/crates/python-bridge/src/routes/traces.rs index 3f2589e4b0ea..3e82cd8d3d1a 100644 --- a/litellm-rust/crates/python-bridge/src/routes/traces.rs +++ b/litellm-rust/crates/python-bridge/src/routes/traces.rs @@ -1,7 +1,7 @@ use std::collections::BTreeMap; use litellm_http::ClientVariant; -use litellm_traces::{Connection, Error, InsertTable, Parameter}; +use litellm_traces::{Connection, Error, InsertTable, Parameter, ReadQuery}; use pyo3::{ exceptions::{PyOverflowError, PyRuntimeError, PyValueError}, prelude::*, @@ -9,9 +9,11 @@ use pyo3::{ fn map_error(error: Error) -> PyErr { match error { - Error::InvalidRow | Error::InvalidTable | Error::InvalidSchema | Error::EmptySql => { - PyValueError::new_err(error.to_string()) - } + Error::InvalidRow + | Error::InvalidTable + | Error::InvalidSchema + | Error::EmptySql + | Error::InvalidQuery => PyValueError::new_err(error.to_string()), Error::InsertTooLarge => PyOverflowError::new_err(error.to_string()), Error::InvalidUrl | Error::QueryFailed(_) @@ -94,19 +96,22 @@ impl NativeTraceStorage { fn query<'py>( &self, py: Python<'py>, - sql: String, + query: &str, #[pyo3(from_py_with = litellm_host_python::from_py_argument)] parameters: BTreeMap< String, Parameter, >, ) -> PyResult> { + let query = ReadQuery::parse(query).map_err(map_error)?; let connection = self.reader.clone().ok_or_else(|| { PyRuntimeError::new_err("Trace reads require a separate ClickHouse reader URL") })?; let client = crate::http::host_client(py, ClientVariant::NoRedirect)?; crate::execution::run_async( py, - async move { litellm_traces::execute_read(&client, &connection, &sql, ¶meters).await }, + async move { + litellm_traces::execute_named_read(&client, &connection, query, ¶meters).await + }, map_error, ) } diff --git a/litellm-rust/crates/traces/src/error.rs b/litellm-rust/crates/traces/src/error.rs index 125edc354224..1b3b0caf3b89 100644 --- a/litellm-rust/crates/traces/src/error.rs +++ b/litellm-rust/crates/traces/src/error.rs @@ -10,6 +10,8 @@ pub enum Error { InvalidSchema, #[error("SQL query must not be empty")] EmptySql, + #[error("invalid trace read query")] + InvalidQuery, #[error("ClickHouse query failed with HTTP status {0}")] QueryFailed(u16), #[error("ClickHouse insert failed with HTTP status {0}")] diff --git a/litellm-rust/crates/traces/src/lib.rs b/litellm-rust/crates/traces/src/lib.rs index 279afb20e9bb..5402b54385e7 100644 --- a/litellm-rust/crates/traces/src/lib.rs +++ b/litellm-rust/crates/traces/src/lib.rs @@ -8,7 +8,7 @@ pub use error::{DecodeError, Error}; pub use insert::{InsertTable, encode_rows, insert_rows}; pub use otlp::{DecodedSpan, decode_otlp}; pub use schema::{ensure_schema, schema_statements}; -pub use sql::{Parameter, execute_read}; +pub use sql::{Parameter, ReadQuery, execute_named_read, execute_read}; use url::Url; #[derive(Clone)] diff --git a/litellm-rust/crates/traces/src/list_traces.sql b/litellm-rust/crates/traces/src/list_traces.sql new file mode 100644 index 000000000000..1fff341e4ad2 --- /dev/null +++ b/litellm-rust/crates/traces/src/list_traces.sql @@ -0,0 +1,37 @@ +SELECT t.TraceId AS trace_id, any(t.RootName) AS name, any(t.ServiceName) AS service, + any(t.RootInput) AS input_preview, any(t.RootStatus) AS status, + toUnixTimestamp64Milli(any(t.StartTs)) AS start_ms, + dateDiff('millisecond', any(t.StartTs), any(t.EndTs)) AS duration_ms, + any(t.SpanCount) AS span_count, length(any(t.AgentNames)) AS agent_count, + any(t.AgentCount) AS agent_invocations, + any(t.LlmCount) AS llm_calls, any(t.ToolCount) AS tool_calls, + any(t.InputTokens) AS input_tokens, any(t.OutputTokens) AS output_tokens, + any(t.Models) AS models, any(t.ErrorCount) AS error_count, + if(countIf(s.response_id != '') = 0, NULL, sum(s.spend)) AS spend +FROM ( + SELECT TeamId, TraceId, min(StartTs) AS StartTs, max(EndTs) AS EndTs, + any(ServiceName) AS ServiceName, anyLastIf(a.RootName, a.RootName != '') AS RootName, + anyLastIf(a.RootInput, a.RootName != '') AS RootInput, + anyLastIf(a.RootStatus, a.RootName != '') AS RootStatus, + sum(SpanCount) AS SpanCount, sum(AgentCount) AS AgentCount, sum(LlmCount) AS LlmCount, + sum(ToolCount) AS ToolCount, sum(ErrorCount) AS ErrorCount, sum(InputTokens) AS InputTokens, + sum(OutputTokens) AS OutputTokens, groupUniqArrayArray(Models) AS Models, + groupUniqArrayArray(AgentNames) AS AgentNames, groupArrayArray(RequestIds) AS RequestIds + FROM agent_traces AS a + WHERE (empty({team_ids:Array(String)}) OR TeamId IN {team_ids:Array(String)}) + AND ({api_key_hash:String} = '' OR TraceId IN ( + SELECT TraceId FROM otel_traces WHERE (empty({team_ids:Array(String)}) OR TeamId IN {team_ids:Array(String)}) AND ({api_key_hash:String} = '' OR ApiKeyHash = {api_key_hash:String}))) + GROUP BY TeamId, TraceId + HAVING StartTs >= fromUnixTimestamp64Milli({start_ms:Int64}) + AND StartTs < fromUnixTimestamp64Milli({end_ms:Int64}) + AND (({cursor_ms:Int64} = 0) OR (toUnixTimestamp64Milli(StartTs), TraceId) + < ({cursor_ms:Int64}, {cursor_trace_id:String})) + ORDER BY StartTs DESC, TraceId DESC + LIMIT {limit:UInt32} +) AS t +LEFT ARRAY JOIN t.RequestIds AS request_id +LEFT JOIN ( + SELECT response_id, any(spend) AS spend FROM spend_logs FINAL WHERE (empty({team_ids:Array(String)}) OR team_id IN {team_ids:Array(String)}) AND ({api_key_hash:String} = '' OR api_key = {api_key_hash:String}) GROUP BY response_id +) AS s ON s.response_id = request_id +GROUP BY t.TraceId +ORDER BY start_ms DESC, t.TraceId DESC diff --git a/litellm-rust/crates/traces/src/span_detail.sql b/litellm-rust/crates/traces/src/span_detail.sql new file mode 100644 index 000000000000..ed68fcd07811 --- /dev/null +++ b/litellm-rust/crates/traces/src/span_detail.sql @@ -0,0 +1,4 @@ +SELECT SpanId AS span_id, Input AS input, Output AS output, SpanAttributes AS attributes +FROM otel_traces +WHERE TraceId = {trace_id:String} AND SpanId = {span_id:String} AND (empty({team_ids:Array(String)}) OR TeamId IN {team_ids:Array(String)}) AND ({api_key_hash:String} = '' OR ApiKeyHash = {api_key_hash:String}) +LIMIT 1 diff --git a/litellm-rust/crates/traces/src/sql.rs b/litellm-rust/crates/traces/src/sql.rs index 1aa21a59caad..ff61d377ad8c 100644 --- a/litellm-rust/crates/traces/src/sql.rs +++ b/litellm-rust/crates/traces/src/sql.rs @@ -8,6 +8,31 @@ use crate::{Connection, Error}; const MAX_RESPONSE_BYTES: usize = 4 * 1024 * 1024; +pub enum ReadQuery { + ListTraces, + TraceSpans, + SpanDetail, +} + +impl ReadQuery { + pub fn parse(value: &str) -> Result { + match value { + "list_traces" => Ok(Self::ListTraces), + "trace_spans" => Ok(Self::TraceSpans), + "span_detail" => Ok(Self::SpanDetail), + _ => Err(Error::InvalidQuery), + } + } + + fn sql(&self) -> &'static str { + match self { + Self::ListTraces => include_str!("list_traces.sql"), + Self::TraceSpans => include_str!("trace_spans.sql"), + Self::SpanDetail => include_str!("span_detail.sql"), + } + } +} + #[derive(Debug, Deserialize)] #[serde(untagged)] pub enum Parameter { @@ -51,7 +76,6 @@ pub async fn execute_read( if sql.trim().is_empty() { return Err(Error::EmptySql); } - let mut url = connection.url().clone(); let existing_pairs: Vec<(String, String)> = url @@ -112,3 +136,23 @@ pub async fn execute_read( } String::from_utf8(body).map_err(|_| Error::InvalidResponse) } + +pub async fn execute_named_read( + client: &Client, + connection: &Connection, + query: ReadQuery, + parameters: &BTreeMap, +) -> Result { + execute_read(client, connection, query.sql(), parameters).await +} + +#[cfg(test)] +mod tests { + use super::ReadQuery; + + #[test] + fn named_reads_reject_caller_sql() { + assert!(ReadQuery::parse("list_traces").is_ok()); + assert!(ReadQuery::parse("SELECT * FROM otel_traces").is_err()); + } +} diff --git a/litellm-rust/crates/traces/src/trace_spans.sql b/litellm-rust/crates/traces/src/trace_spans.sql new file mode 100644 index 000000000000..ec16f68e2225 --- /dev/null +++ b/litellm-rust/crates/traces/src/trace_spans.sql @@ -0,0 +1,27 @@ +SELECT o.SpanId AS span_id, o.ParentSpanId AS parent_span_id, o.SpanName AS name, + o.ObservationType AS type, o.AgentName AS agent, o.StatusCode AS status, + o.StatusMessage AS status_message, + toUnixTimestamp64Nano(o.Timestamp) AS start_ns, o.Duration AS duration_ns, + o.ServiceName AS service, o.InputPreview AS input_preview, o.Model AS model, + o.InputTokens AS input_tokens, o.OutputTokens AS output_tokens, + o.LiteLLMRequestId AS litellm_request_id, + s.request_id AS s_request_id, s.model AS s_model, s.model_group AS s_model_group, + s.custom_llm_provider AS s_provider, s.api_base AS s_api_base, s.key_alias AS s_key_alias, + s.team_alias AS s_team_alias, s.spend AS s_spend, s.prompt_tokens AS s_prompt_tokens, + s.completion_tokens AS s_completion_tokens, s.cache_read_tokens AS s_cache_read_tokens, + s.cache_write_tokens AS s_cache_write_tokens, + toUnixTimestamp64Milli(s.start_time) AS s_start_ms, toUnixTimestamp64Milli(s.end_time) AS s_end_ms, + if(isNull(s.completion_start_time), 0, toUnixTimestamp64Milli(assumeNotNull(s.completion_start_time))) + AS s_ttft_start_ms, + s.status AS s_status +FROM otel_traces AS o +LEFT JOIN ( + SELECT * FROM spend_logs FINAL + WHERE response_id IN ( + SELECT LiteLLMRequestId FROM otel_traces + WHERE TraceId = {trace_id:String} AND LiteLLMRequestId != '' AND (empty({team_ids:Array(String)}) OR TeamId IN {team_ids:Array(String)}) AND ({api_key_hash:String} = '' OR ApiKeyHash = {api_key_hash:String})) + AND (empty({team_ids:Array(String)}) OR team_id IN {team_ids:Array(String)}) AND ({api_key_hash:String} = '' OR api_key = {api_key_hash:String}) +) AS s ON s.response_id = o.LiteLLMRequestId +WHERE o.TraceId = {trace_id:String} AND (empty({team_ids:Array(String)}) OR TeamId IN {team_ids:Array(String)}) AND ({api_key_hash:String} = '' OR ApiKeyHash = {api_key_hash:String}) +ORDER BY o.Timestamp, abs(toUnixTimestamp64Milli(s.start_time) - toUnixTimestamp64Milli(o.Timestamp)) +LIMIT 1 BY o.SpanId diff --git a/litellm/__init__.py b/litellm/__init__.py index e1da202b9eee..58827b60a987 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -157,6 +157,7 @@ def _dev_env_hot_reload_enabled() -> bool: "smtp_email", "deepeval", "s3_v2", + "clickhouse", "pointfive", "zerobus", "aws_sqs", diff --git a/litellm/constants.py b/litellm/constants.py index 8a74ca21ff97..62759471dba1 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -51,6 +51,7 @@ CLICKHOUSE_FLUSH_INTERVAL_SECONDS: Final = float(os.getenv("CLICKHOUSE_FLUSH_INTERVAL_SECONDS", 1.0)) CLICKHOUSE_MAX_BUFFERED_ROWS: Final = int(os.getenv("CLICKHOUSE_MAX_BUFFERED_ROWS", 200_000)) CLICKHOUSE_MAX_RETRIES: Final = int(os.getenv("CLICKHOUSE_MAX_RETRIES", 3)) +MILLISECONDS_PER_SECOND: Final = 1_000 AGENT_TRACING_RETENTION_DAYS: Final = int(os.getenv("AGENT_TRACING_RETENTION_DAYS", 30)) AGENT_TRACING_SPEND_LOG_RETENTION_DAYS: Final = int(os.getenv("AGENT_TRACING_SPEND_LOG_RETENTION_DAYS", 90)) OTLP_MAX_BODY_BYTES: Final = int(os.getenv("OTLP_MAX_BODY_BYTES", 8 * 1024 * 1024)) diff --git a/litellm/integrations/clickhouse/clickhouse_spend_logger.py b/litellm/integrations/clickhouse/clickhouse_spend_logger.py new file mode 100644 index 000000000000..1d23fe759e03 --- /dev/null +++ b/litellm/integrations/clickhouse/clickhouse_spend_logger.py @@ -0,0 +1,155 @@ +""" +`clickhouse` logging callback: one `spend_logs` row per LiteLLM request. + +Agent LLM spans join to these rows on `otel_traces.LiteLLMRequestId = spend_logs.response_id`, +so `response_id` is always the raw provider response id (cache-hit suffix stripped). +""" + +import json +import re +from collections.abc import Mapping +from typing import Any, Final + +import litellm +from litellm._logging import verbose_logger +from litellm.constants import MILLISECONDS_PER_SECOND +from litellm.integrations.clickhouse.clickhouse_batch_logger import ClickHouseBatchLogger +from litellm.integrations.clickhouse.schema import SPEND_LOGS_TABLE +from litellm.tracing.types import SpendLogRecord +from litellm.types.utils import StandardLoggingPayload + +# litellm_logging.py rewrites cache-hit ids as f"{id}_cache_hit{time.time()}" +_CACHE_HIT_SUFFIX: Final = re.compile(r"_cache_hit[0-9.]*$") +# W3C trace context: version-traceid-parentid-flags +_TRACEPARENT: Final = re.compile(r"^[0-9a-f]{2}-([0-9a-f]{32})-([0-9a-f]{16})-[0-9a-f]{2}$") +_INVALID_TRACE_ID: Final = "0" * 32 +_INVALID_SPAN_ID: Final = "0" * 16 +TRACE_INGEST_ROUTE: Final = "/v1/traces" + + +def strip_cache_hit_suffix(request_id: str) -> str: + return _CACHE_HIT_SUFFIX.sub("", request_id) + + +def parse_traceparent(value: object) -> tuple[str, str]: + """(trace_id, span_id) from a W3C `traceparent` header, or ("", "") if absent/invalid.""" + if not isinstance(value, str): + return "", "" + match = _TRACEPARENT.match(value.strip().lower()) + if match is None or match.group(1) == _INVALID_TRACE_ID or match.group(2) == _INVALID_SPAN_ID: + return "", "" + return match.group(1), match.group(2) + + +def _to_ms(seconds: object) -> int | None: + return int(float(seconds) * MILLISECONDS_PER_SECOND) if isinstance(seconds, (int, float)) else None + + +def _int(value: object) -> int: + return value if isinstance(value, int) and not isinstance(value, bool) else 0 + + +def _json(value: object) -> str: + if value is None or value == "": + return "" + return value if isinstance(value, str) else json.dumps(value, default=str) + + +def _find_traceparent(metadata: Mapping[str, Any], kwargs: Mapping[str, Any]) -> tuple[str, str]: + custom_headers = metadata.get("requester_custom_headers") or {} + proxy_request = (kwargs.get("litellm_params") or {}).get("proxy_server_request") or {} + request_headers = proxy_request.get("headers") or {} + for headers in (custom_headers, request_headers): + for name, value in headers.items(): + if str(name).lower() == "traceparent": + return parse_traceparent(value) + return "", "" + + +def _cache_tokens(usage: Mapping[str, Any]) -> tuple[int, int]: + """(cache_read, cache_write) from a Usage dict: OpenAI prompt_tokens_details first, Anthropic fields as fallback.""" + details = usage.get("prompt_tokens_details") or {} + cache_read = _int(details.get("cached_tokens")) or _int(usage.get("cache_read_input_tokens")) + cache_write = ( + _int(details.get("cache_write_tokens")) + or _int(details.get("cache_creation_tokens")) + or _int(usage.get("cache_creation_input_tokens")) + ) + return cache_read, cache_write + + +def _session_id(payload: StandardLoggingPayload, kwargs: Mapping[str, Any]) -> str: + """Mirrors proxy `_get_session_id_for_spend_log`: explicit session id, else the payload trace id.""" + request_metadata = (kwargs.get("litellm_params") or {}).get("metadata") or {} + return str(payload.get("session_id") or request_metadata.get("session_id") or payload.get("trace_id") or "") + + +def _is_trace_ingest(payload: StandardLoggingPayload) -> bool: + """OTLP exports to POST /v1/traces are not LLM requests; don't write them as spend rows.""" + return str(payload.get("call_type") or "").startswith(TRACE_INGEST_ROUTE) + + +def spend_log_row_from_payload(payload: StandardLoggingPayload, kwargs: Mapping[str, Any]) -> SpendLogRecord: + metadata: Mapping[str, Any] = payload.get("metadata") or {} + hidden_params: Mapping[str, Any] = payload.get("hidden_params") or {} + usage: Mapping[str, Any] = metadata.get("usage_object") or hidden_params.get("usage_object") or {} + cache_read_tokens, cache_write_tokens = _cache_tokens(usage) + trace_id, span_id = _find_traceparent(metadata, kwargs) + request_id = str(payload.get("id") or "") + redact = litellm.turn_off_message_logging is True + completion_start_ms = _to_ms(payload.get("completionStartTime")) + return SpendLogRecord( + request_id=request_id, + response_id=strip_cache_hit_suffix(request_id), + call_type=payload.get("call_type") or "", + api_key=metadata.get("user_api_key_hash") or "", + key_alias=metadata.get("user_api_key_alias") or "", + team_id=metadata.get("user_api_key_team_id") or metadata.get("team_id") or "", + team_alias=metadata.get("user_api_key_team_alias") or metadata.get("team_alias") or "", + organization_id=metadata.get("user_api_key_org_id") or "", + user=metadata.get("user_api_key_user_id") or "", + end_user=payload.get("end_user") or metadata.get("user_api_key_end_user_id") or "", + model=payload.get("model") or "", + model_group=payload.get("model_group") or "", + model_id=payload.get("model_id") or "", + custom_llm_provider=payload.get("custom_llm_provider") or "", + api_base=payload.get("api_base") or "", + spend=float(payload.get("response_cost") or 0.0), + prompt_tokens=_int(payload.get("prompt_tokens")), + completion_tokens=_int(payload.get("completion_tokens")), + total_tokens=_int(payload.get("total_tokens")), + cache_read_tokens=cache_read_tokens, + cache_write_tokens=cache_write_tokens, + start_time=_to_ms(payload.get("startTime")) or 0, + end_time=_to_ms(payload.get("endTime")) or 0, + completion_start_time=completion_start_ms or None, + status=payload.get("status") or "", + error_str=payload.get("error_str") or "", + cache_hit=payload.get("cache_hit") is True, + session_id=_session_id(payload, kwargs), + trace_id=trace_id, + span_id=span_id, + request_tags=[str(tag) for tag in payload.get("request_tags") or []], + metadata=_json(dict(metadata)), + messages="" if redact else _json(payload.get("messages")), + response="" if redact else _json(payload.get("response")), + ) + + +class ClickHouseSpendLogger(ClickHouseBatchLogger): + table = SPEND_LOGS_TABLE + + async def async_log_success_event(self, kwargs, response_obj, start_time, end_time) -> None: + self._log(kwargs) + + async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time) -> None: + self._log(kwargs) + + def _log(self, kwargs: Mapping[str, Any]) -> None: + try: + payload = kwargs.get("standard_logging_object") + if payload is None or _is_trace_ingest(payload): + return + self.enqueue([dict(spend_log_row_from_payload(payload, kwargs))]) + except Exception as e: + verbose_logger.exception("ClickHouseSpendLogger: failed to log request: %s", e) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index e292ab7b2ec2..2162a200565b 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -180,6 +180,7 @@ from ..integrations.athina import AthinaLogger from ..integrations.azure_sentinel.azure_sentinel import AzureSentinelLogger from ..integrations.azure_storage.azure_storage import AzureBlobStorageLogger +from ..integrations.clickhouse.clickhouse_spend_logger import ClickHouseSpendLogger from ..integrations.custom_prompt_management import CustomPromptManagement from ..integrations.datadog.datadog import DataDogLogger from ..integrations.datadog.datadog_llm_obs import DataDogLLMObsLogger @@ -4638,6 +4639,14 @@ def _init_custom_logger_compatible_class( _s3_v2_logger: Final = S3V2Logger() _in_memory_loggers.append(_s3_v2_logger) return _s3_v2_logger + elif logging_integration == "clickhouse": + for callback in _in_memory_loggers: + if isinstance(callback, ClickHouseSpendLogger): + return callback + + _clickhouse_spend_logger: Final = ClickHouseSpendLogger() + _in_memory_loggers.append(_clickhouse_spend_logger) + return _clickhouse_spend_logger elif logging_integration == "pointfive": for callback in _in_memory_loggers: if isinstance(callback, PointFiveLogger): @@ -5374,6 +5383,10 @@ def get_custom_logger_compatible_class( for callback in _in_memory_loggers: if isinstance(callback, S3V2Logger): return callback + elif logging_integration == "clickhouse": + for callback in _in_memory_loggers: + if isinstance(callback, ClickHouseSpendLogger): + return callback elif logging_integration == "pointfive": for callback in _in_memory_loggers: if isinstance(callback, PointFiveLogger): diff --git a/litellm/rust_bridge/traces.py b/litellm/rust_bridge/traces.py index 5d0fa7ee05d5..2dc78ad418b8 100644 --- a/litellm/rust_bridge/traces.py +++ b/litellm/rust_bridge/traces.py @@ -37,7 +37,7 @@ def ensure_schema(self, trace_retention_days: int, spend_log_retention_days: int def insert_rows(self, table: str, rows: Sequence[Mapping[str, JsonValue]]) -> Awaitable[None]: ... - def query(self, sql: str, parameters: Mapping[str, str | int | Sequence[str]]) -> Awaitable[str]: ... + def query(self, name: str, parameters: Mapping[str, str | int | Sequence[str]]) -> Awaitable[str]: ... class NativeTraces(Protocol): @@ -84,6 +84,15 @@ async def ensure_schema(self, trace_retention_days: int, spend_log_retention_day async def insert_rows(self, table: str, rows: Sequence[Mapping[str, object]]) -> None: await self._native.insert_rows(table, INSERT_ROWS.validate_python(rows)) - async def query(self, sql: str, parameters: Mapping[str, object] | None = None) -> list[dict[str, JsonValue]]: - result: Final = await self._native.query(sql, QUERY_PARAMETERS.validate_python(parameters or {})) + async def _query(self, name: str, parameters: Mapping[str, object]) -> list[dict[str, JsonValue]]: + result: Final = await self._native.query(name, QUERY_PARAMETERS.validate_python(parameters)) return QueryResponse.model_validate_json(result).data + + async def list_traces(self, parameters: Mapping[str, object]) -> list[dict[str, JsonValue]]: + return await self._query("list_traces", parameters) + + async def trace_spans(self, parameters: Mapping[str, object]) -> list[dict[str, JsonValue]]: + return await self._query("trace_spans", parameters) + + async def span_detail(self, parameters: Mapping[str, object]) -> list[dict[str, JsonValue]]: + return await self._query("span_detail", parameters) diff --git a/litellm/tracing/store.py b/litellm/tracing/store.py index acb473870699..c443f54b726b 100644 --- a/litellm/tracing/store.py +++ b/litellm/tracing/store.py @@ -1,4 +1,9 @@ -"""ClickHouse-backed trace store: batched span writes and scoped reads.""" +""" +ClickHouse-backed trace store: confirmed span writes and scoped reads. + +Reads join agent spans (otel_traces) to LiteLLM requests (spend_logs) on +`otel_traces.LiteLLMRequestId = spend_logs.response_id`. +""" import base64 import json @@ -7,12 +12,12 @@ from litellm.constants import AGENT_TRACING_LIST_PAGE_SIZE from litellm.integrations.clickhouse.schema import ( - AGENT_TRACES_TABLE, OTEL_TRACES_TABLE, ) from litellm.rust_bridge.traces import TraceStorage from litellm.tracing.types import ( AgentNode, + LiteLLMRequest, Span, SpanDetail, SpanRow, @@ -26,69 +31,9 @@ NANOS_PER_MS: Final = 1_000_000 _STATUS: Final[dict[str, SpanStatus]] = {"STATUS_CODE_OK": "ok", "STATUS_CODE_ERROR": "error"} -_SCOPE_OTEL: Final = ( - "(empty({team_ids:Array(String)}) OR TeamId IN {team_ids:Array(String)})" - " AND ({api_key_hash:String} = '' OR ApiKeyHash = {api_key_hash:String})" -) -# Page of traces from the per-trace MV. +# Page of traces from the per-trace MV, then cost from spend logs via ARRAY JOIN on request ids. # The MV writes one partial row per insert, so root fields come from the partial that saw the root span. # agent_traces has no ApiKeyHash, so key-scoped (team-less) reads filter trace ids through otel_traces. -LIST_TRACES_SQL: Final = f""" -SELECT t.TraceId AS trace_id, any(t.RootName) AS name, any(t.ServiceName) AS service, - any(t.RootInput) AS input_preview, any(t.RootStatus) AS status, - toUnixTimestamp64Milli(any(t.StartTs)) AS start_ms, - dateDiff('millisecond', any(t.StartTs), any(t.EndTs)) AS duration_ms, - any(t.SpanCount) AS span_count, length(any(t.AgentNames)) AS agent_count, - any(t.AgentCount) AS agent_invocations, - any(t.LlmCount) AS llm_calls, any(t.ToolCount) AS tool_calls, - any(t.InputTokens) AS input_tokens, any(t.OutputTokens) AS output_tokens, - any(t.Models) AS models, any(t.ErrorCount) AS error_count -FROM ( - SELECT TeamId, TraceId, min(StartTs) AS StartTs, max(EndTs) AS EndTs, - any(ServiceName) AS ServiceName, anyLastIf(a.RootName, a.RootName != '') AS RootName, - anyLastIf(a.RootInput, a.RootName != '') AS RootInput, - anyLastIf(a.RootStatus, a.RootName != '') AS RootStatus, - sum(SpanCount) AS SpanCount, sum(AgentCount) AS AgentCount, sum(LlmCount) AS LlmCount, - sum(ToolCount) AS ToolCount, sum(ErrorCount) AS ErrorCount, sum(InputTokens) AS InputTokens, - sum(OutputTokens) AS OutputTokens, groupUniqArrayArray(Models) AS Models, - groupUniqArrayArray(AgentNames) AS AgentNames - FROM {AGENT_TRACES_TABLE} AS a - WHERE (empty({{team_ids:Array(String)}}) OR TeamId IN {{team_ids:Array(String)}}) - AND ({{api_key_hash:String}} = '' OR TraceId IN ( - SELECT TraceId FROM {OTEL_TRACES_TABLE} WHERE {_SCOPE_OTEL})) - GROUP BY TeamId, TraceId - HAVING StartTs >= fromUnixTimestamp64Milli({{start_ms:Int64}}) - AND StartTs < fromUnixTimestamp64Milli({{end_ms:Int64}}) - AND (({{cursor_ms:Int64}} = 0) OR (toUnixTimestamp64Milli(StartTs), TraceId) - < ({{cursor_ms:Int64}}, {{cursor_trace_id:String}})) - ORDER BY StartTs DESC, TraceId DESC - LIMIT {{limit:UInt32}} -) AS t -GROUP BY t.TraceId -ORDER BY start_ms DESC, t.TraceId DESC -""" - -TRACE_SPANS_SQL: Final = f""" -SELECT o.SpanId AS span_id, o.ParentSpanId AS parent_span_id, o.SpanName AS name, - o.ObservationType AS type, o.AgentName AS agent, o.StatusCode AS status, - o.StatusMessage AS status_message, - toUnixTimestamp64Nano(o.Timestamp) AS start_ns, o.Duration AS duration_ns, - o.ServiceName AS service, o.InputPreview AS input_preview, o.Model AS model, - o.InputTokens AS input_tokens, o.OutputTokens AS output_tokens, - o.LiteLLMRequestId AS litellm_request_id -FROM {OTEL_TRACES_TABLE} AS o -WHERE o.TraceId = {{trace_id:String}} AND {_SCOPE_OTEL} -ORDER BY o.Timestamp -LIMIT 1 BY o.SpanId -""" - -SPAN_DETAIL_SQL: Final = f""" -SELECT SpanId AS span_id, Input AS input, Output AS output, SpanAttributes AS attributes -FROM {OTEL_TRACES_TABLE} -WHERE TraceId = {{trace_id:String}} AND SpanId = {{span_id:String}} AND {_SCOPE_OTEL} -LIMIT 1 -""" - def encode_cursor(start_ms: int, trace_id: str) -> str: return base64.urlsafe_b64encode(json.dumps([start_ms, trace_id]).encode()).decode() @@ -126,10 +71,33 @@ def trace_summary_from_row(row: dict[str, Any]) -> TraceSummary: error_count=int(row.get("error_count") or 0), input_tokens=int(row["input_tokens"]), output_tokens=int(row["output_tokens"]), + spend=float(row["spend"]) if row["spend"] is not None else None, models=list(row["models"]), ) +def _litellm_request(row: dict[str, Any]) -> LiteLLMRequest | None: + if not row.get("s_request_id"): + return None + ttft_start: Final = int(row["s_ttft_start_ms"] or 0) + return LiteLLMRequest( + request_id=row["s_request_id"], + model=row["s_model"], + model_group=row["s_model_group"], + provider=row["s_provider"], + key_alias=row["s_key_alias"], + team_alias=row["s_team_alias"], + spend=float(row["s_spend"]), + prompt_tokens=int(row["s_prompt_tokens"]), + completion_tokens=int(row["s_completion_tokens"]), + cache_read_tokens=int(row["s_cache_read_tokens"]), + cache_write_tokens=int(row["s_cache_write_tokens"]), + latency_ms=int(row["s_end_ms"]) - int(row["s_start_ms"]), + ttft_ms=(ttft_start - int(row["s_start_ms"])) if ttft_start else None, + status=row["s_status"], + ) + + def span_from_row(row: dict[str, Any], trace_start_ns: int) -> Span: return Span( span_id=row["span_id"], @@ -145,7 +113,7 @@ def span_from_row(row: dict[str, Any], trace_start_ns: int) -> Span: model=row["model"] or None, input_tokens=int(row["input_tokens"]), output_tokens=int(row["output_tokens"]), - litellm_request_id=row["litellm_request_id"] or None, + litellm=_litellm_request(row), ) @@ -174,6 +142,7 @@ def agent_nodes(spans: list[Span]) -> list[AgentNode]: invocations=0, llm_calls=0, tool_calls=0, + spend=None, duration_ms=0.0, ), ) @@ -185,6 +154,8 @@ def agent_nodes(spans: list[Span]) -> list[AgentNode]: continue if span["type"] == "llm": owner["llm_calls"] += 1 + if span["litellm"]: + owner["spend"] = (owner["spend"] or 0.0) + span["litellm"]["spend"] elif span["type"] == "tool": owner["tool_calls"] += 1 return list(agents.values()) @@ -216,6 +187,9 @@ def trace_from_rows(trace_id: str, rows: list[dict[str, Any]]) -> Trace | None: error_count=sum(1 for s in spans if s["status"] == "error"), input_tokens=sum(s["input_tokens"] for s in spans), output_tokens=sum(s["output_tokens"] for s in spans), + spend=sum(s["litellm"]["spend"] for s in llm_spans if s["litellm"]) + if any(s["litellm"] for s in llm_spans) + else None, models=sorted({s["model"] for s in llm_spans if s["model"]}), ), agents=agents, @@ -241,8 +215,7 @@ async def list_traces( limit: int = AGENT_TRACING_LIST_PAGE_SIZE, ) -> TracePage: cursor_ms, cursor_trace_id = decode_cursor(cursor) - rows = await self.storage.query( - LIST_TRACES_SQL, + rows = await self.storage.list_traces( { **scope, "start_ms": start_ms, @@ -256,11 +229,11 @@ async def list_traces( return TracePage(data=[trace_summary_from_row(r) for r in rows], next_cursor=next_cursor) async def get_trace(self, trace_id: str, scope: TraceScope) -> Trace | None: - rows = await self.storage.query(TRACE_SPANS_SQL, {**scope, "trace_id": trace_id}) + rows = await self.storage.trace_spans({**scope, "trace_id": trace_id}) return trace_from_rows(trace_id, rows) async def get_span(self, trace_id: str, span_id: str, scope: TraceScope) -> SpanDetail | None: - rows = await self.storage.query(SPAN_DETAIL_SQL, {**scope, "trace_id": trace_id, "span_id": span_id}) + rows = await self.storage.span_detail({**scope, "trace_id": trace_id, "span_id": span_id}) if not rows: return None return SpanDetail( diff --git a/litellm/tracing/types.py b/litellm/tracing/types.py index bf0e5b3a722c..b5bd26f4f963 100644 --- a/litellm/tracing/types.py +++ b/litellm/tracing/types.py @@ -2,10 +2,14 @@ Agent tracing types. A trace is one agent run. It's made of spans (agent / llm / tool / chain / framework). +LLM spans that went through LiteLLM carry a `LiteLLMRequest`: the spend-log row for that +call, joined on the provider response id. + Trace ├── summary: TraceSummary ├── agents: list[AgentNode] one per distinct agent name (for the agent graph) └── spans: list[Span] flat, linked by parent_span_id + └── litellm: LiteLLMRequest | None (llm spans only) """ @@ -17,6 +21,25 @@ SpanStatus = Literal["ok", "error", "unset"] +class LiteLLMRequest(TypedDict): + """The LiteLLM side of an LLM span: joined from spend logs by response id.""" + + request_id: str + model: str + model_group: str + provider: str + key_alias: str + team_alias: str + spend: float + prompt_tokens: int + completion_tokens: int + cache_read_tokens: int + cache_write_tokens: int + latency_ms: int + ttft_ms: int | None + status: str + + class Span(TypedDict): span_id: str parent_span_id: str | None @@ -31,7 +54,7 @@ class Span(TypedDict): model: str | None input_tokens: int output_tokens: int - litellm_request_id: str | None + litellm: LiteLLMRequest | None class AgentNode(TypedDict): @@ -42,6 +65,7 @@ class AgentNode(TypedDict): invocations: int llm_calls: int tool_calls: int + spend: float | None duration_ms: float @@ -61,6 +85,7 @@ class TraceSummary(TypedDict): error_count: int # spans with an error status; > 0 means the run shows as failed input_tokens: int output_tokens: int + spend: float | None models: list[str] @@ -117,3 +142,42 @@ class SpanRow(TypedDict): OutputTokens: int Input: str Output: str + + +class SpendLogRecord(TypedDict): + """One LiteLLM request, as written by the `clickhouse` logging callback.""" + + request_id: str + response_id: str + call_type: str + api_key: str + key_alias: str + team_id: str + team_alias: str + organization_id: str + user: str + end_user: str + model: str + model_group: str + model_id: str + custom_llm_provider: str + api_base: str + spend: float + prompt_tokens: int + completion_tokens: int + total_tokens: int + cache_read_tokens: int + cache_write_tokens: int + start_time: int # unix ms + end_time: int # unix ms + completion_start_time: int | None + status: str + error_str: str + cache_hit: bool + session_id: str + trace_id: str # from an incoming W3C traceparent, if any + span_id: str + request_tags: list[str] + metadata: str + messages: str + response: str diff --git a/tests/test_litellm/integrations/clickhouse/test_clickhouse_spend_logger.py b/tests/test_litellm/integrations/clickhouse/test_clickhouse_spend_logger.py new file mode 100644 index 000000000000..4a15bdd6dc66 --- /dev/null +++ b/tests/test_litellm/integrations/clickhouse/test_clickhouse_spend_logger.py @@ -0,0 +1,209 @@ +""" +Tests for the `clickhouse` spend-log callback. +""" + +import json +import os +import sys +from typing import Any +from unittest.mock import AsyncMock, MagicMock, patch + +sys.path.insert(0, os.path.abspath("../../..")) + +import pytest + +import litellm +from litellm.integrations.clickhouse.clickhouse_spend_logger import ( + ClickHouseSpendLogger, + parse_traceparent, + spend_log_row_from_payload, + strip_cache_hit_suffix, +) +from litellm.integrations.clickhouse.schema import SPEND_LOGS_TABLE +from litellm.integrations.custom_batch_logger import CustomBatchLogger +from litellm.litellm_core_utils import litellm_logging +from litellm.tracing.types import SpendLogRecord + +TRACE_ID = "4bf92f3577b34da6a3ce929d0e0e4736" +SPAN_ID = "00f067aa0ba902b7" +TRACEPARENT = f"00-{TRACE_ID}-{SPAN_ID}-01" + + +def _payload(**overrides: Any) -> dict[str, Any]: + payload: dict[str, Any] = { + "id": "chatcmpl-abc123", + "trace_id": "trace-1", + "session_id": "", + "call_type": "acompletion", + "response_cost": 0.00042, + "status": "success", + "custom_llm_provider": "openai", + "total_tokens": 30, + "prompt_tokens": 20, + "completion_tokens": 10, + "startTime": 1_700_000_000.123, + "endTime": 1_700_000_001.456, + "completionStartTime": 1_700_000_000.5, + "model": "gpt-4o", + "model_id": "model-uuid", + "model_group": "gpt-4o-group", + "api_base": "https://api.openai.com/v1", + "metadata": { + "user_api_key_hash": "hashed-key", + "user_api_key_alias": "my-key", + "user_api_key_team_id": "team-1", + "user_api_key_team_alias": "Team One", + "user_api_key_org_id": "org-1", + "user_api_key_user_id": "user-1", + "user_api_key_end_user_id": None, + "requester_custom_headers": {"traceparent": TRACEPARENT}, + "usage_object": { + "prompt_tokens": 20, + "completion_tokens": 10, + "total_tokens": 30, + "prompt_tokens_details": {"cached_tokens": 5, "cache_write_tokens": 7}, + }, + }, + "cache_hit": None, + "request_tags": ["prod", "agent"], + "end_user": "end-user-1", + "messages": [{"role": "user", "content": "hi"}], + "response": {"choices": [{"message": {"content": "hello"}}]}, + "error_str": None, + "hidden_params": {"usage_object": None}, + } + return {**payload, **overrides} + + +def test_is_a_custom_batch_logger(): + assert issubclass(ClickHouseSpendLogger, CustomBatchLogger) + assert ClickHouseSpendLogger.table == SPEND_LOGS_TABLE + + +def test_success_row_mapping(): + row = spend_log_row_from_payload(_payload(), {}) # type: ignore[arg-type] + + assert set(row) == set(SpendLogRecord.__annotations__) + assert row["request_id"] == "chatcmpl-abc123" + assert row["response_id"] == "chatcmpl-abc123" + assert row["spend"] == 0.00042 + assert (row["prompt_tokens"], row["completion_tokens"], row["total_tokens"]) == (20, 10, 30) + assert (row["cache_read_tokens"], row["cache_write_tokens"]) == (5, 7) + assert row["start_time"] == 1_700_000_000_123 + assert row["end_time"] == 1_700_000_001_456 + assert row["completion_start_time"] == 1_700_000_000_500 + assert row["status"] == "success" + assert row["cache_hit"] is False + assert row["api_key"] == "hashed-key" + assert row["key_alias"] == "my-key" + assert row["team_id"] == "team-1" + assert row["team_alias"] == "Team One" + assert row["organization_id"] == "org-1" + assert row["user"] == "user-1" + assert row["end_user"] == "end-user-1" + assert row["model_group"] == "gpt-4o-group" + assert row["session_id"] == "trace-1" + assert (row["trace_id"], row["span_id"]) == (TRACE_ID, SPAN_ID) + assert row["request_tags"] == ["prod", "agent"] + assert json.loads(row["messages"]) == [{"role": "user", "content": "hi"}] + assert json.loads(row["metadata"])["user_api_key_alias"] == "my-key" + + +def test_anthropic_cache_fields_are_used_as_fallback(): + usage = {"cache_read_input_tokens": 11, "cache_creation_input_tokens": 3} + payload = _payload() + payload["metadata"] = {**payload["metadata"], "usage_object": usage} + + row = spend_log_row_from_payload(payload, {}) # type: ignore[arg-type] + + assert (row["cache_read_tokens"], row["cache_write_tokens"]) == (11, 3) + + +def test_explicit_session_id_wins_over_trace_id(): + row = spend_log_row_from_payload( + _payload(), # type: ignore[arg-type] + {"litellm_params": {"metadata": {"session_id": "sess-9"}}}, + ) + assert row["session_id"] == "sess-9" + + +def test_cache_hit_id_is_stripped_for_response_id(): + row = spend_log_row_from_payload( + _payload(id="chatcmpl-abc123_cache_hit1727600000.123456", cache_hit=True), # type: ignore[arg-type] + {}, + ) + assert row["request_id"] == "chatcmpl-abc123_cache_hit1727600000.123456" + assert row["response_id"] == "chatcmpl-abc123" + assert row["cache_hit"] is True + assert strip_cache_hit_suffix("chatcmpl-xyz") == "chatcmpl-xyz" + + +def test_parse_traceparent_valid_missing_malformed(): + assert parse_traceparent(TRACEPARENT) == (TRACE_ID, SPAN_ID) + assert parse_traceparent(None) == ("", "") + assert parse_traceparent("") == ("", "") + assert parse_traceparent("not-a-traceparent") == ("", "") + assert parse_traceparent(f"00-{TRACE_ID}-{SPAN_ID}") == ("", "") + assert parse_traceparent(f"00-{'0' * 32}-{SPAN_ID}-01") == ("", "") + + +def test_traceparent_from_proxy_server_request_headers(): + payload = _payload() + payload["metadata"] = {**payload["metadata"], "requester_custom_headers": None} + kwargs = {"litellm_params": {"proxy_server_request": {"headers": {"Traceparent": TRACEPARENT}}}} + + row = spend_log_row_from_payload(payload, kwargs) # type: ignore[arg-type] + + assert (row["trace_id"], row["span_id"]) == (TRACE_ID, SPAN_ID) + + +def test_turn_off_message_logging_blanks_messages_and_response(): + with patch.object(litellm, "turn_off_message_logging", True): + row = spend_log_row_from_payload(_payload(), {}) # type: ignore[arg-type] + assert row["messages"] == "" + assert row["response"] == "" + + +@pytest.mark.asyncio +async def test_failure_event_maps_status_and_error(): + client = MagicMock() + client.insert_json_each_row = AsyncMock() + logger = ClickHouseSpendLogger(client=client) + payload = _payload(status="failure", error_str="RateLimitError: slow down", response_cost=0.0) + + await logger.async_log_failure_event({"standard_logging_object": payload}, None, None, None) + + assert len(logger.log_queue) == 1 + row = logger.log_queue[0] + assert row["status"] == "failure" + assert row["error_str"] == "RateLimitError: slow down" + + +@pytest.mark.asyncio +async def test_missing_payload_and_bad_payload_never_raise(): + logger = ClickHouseSpendLogger(client=MagicMock()) + await logger.async_log_success_event({}, None, None, None) + await logger.async_log_success_event({"standard_logging_object": "garbage"}, None, None, None) + assert logger.log_queue == [] + + +@pytest.mark.asyncio +async def test_trace_ingest_requests_are_not_logged_as_spend(): + # OTLP exports hit POST /v1/traces; they are not LLM calls and must not create spend rows + logger = ClickHouseSpendLogger(client=MagicMock()) + payload = _payload(call_type="/v1/traces", status="failure") + + await logger.async_log_failure_event({"standard_logging_object": payload}, None, None, None) + + assert logger.log_queue == [] + + +@pytest.mark.asyncio +async def test_clickhouse_callback_resolves_via_factory(monkeypatch): + monkeypatch.setenv("CLICKHOUSE_URL", "http://localhost:8123") + monkeypatch.setattr(litellm_logging, "_in_memory_loggers", []) + + created = litellm_logging._init_custom_logger_compatible_class("clickhouse", None, None) + assert isinstance(created, ClickHouseSpendLogger) + assert litellm_logging._init_custom_logger_compatible_class("clickhouse", None, None) is created + assert litellm_logging.get_custom_logger_compatible_class("clickhouse") is created diff --git a/tests/test_litellm/tracing/test_store.py b/tests/test_litellm/tracing/test_store.py index e7235c3a3269..7ddf9499ff39 100644 --- a/tests/test_litellm/tracing/test_store.py +++ b/tests/test_litellm/tracing/test_store.py @@ -36,6 +36,7 @@ def _row( status: str = "STATUS_CODE_OK", **extra: Any, ) -> dict[str, Any]: + """A TRACE_SPANS_SQL result row; `s_*` columns empty unless the span joined a spend log.""" return { "span_id": span_id, "parent_span_id": parent, @@ -51,11 +52,12 @@ def _row( "input_tokens": 0, "output_tokens": 0, "litellm_request_id": "", + "s_request_id": "", **extra, } -def _llm_row(span_id: str, parent: str, agent: str, request_id: str, start_ms: float = 1) -> dict: +def _llm_row(span_id: str, parent: str, agent: str, request_id: str, spend: float, start_ms: float = 1) -> dict: return _row( span_id, parent, @@ -68,6 +70,22 @@ def _llm_row(span_id: str, parent: str, agent: str, request_id: str, start_ms: f input_tokens=100, output_tokens=20, litellm_request_id=request_id, + s_request_id=request_id, + s_model="claude-sonnet-4-5", + s_model_group="claude-sonnet-4-5", + s_provider="anthropic", + s_api_base="https://api.anthropic.com", + s_key_alias="research-bot", + s_team_alias="research-agents", + s_spend=spend, + s_prompt_tokens=100, + s_completion_tokens=20, + s_cache_read_tokens=5, + s_cache_write_tokens=0, + s_start_ms=T0 // MS, + s_end_ms=T0 // MS + 100, + s_ttft_start_ms=T0 // MS + 30, + s_status="success", ) @@ -75,13 +93,13 @@ def _deep_agent_rows(researcher_invocations: int = 1) -> list[dict[str, Any]]: """root agent -> llm, task tool -> researcher subagent (N times) -> llm + search_docs tool.""" rows = [ _row("root", "", "deep_research_agent", "agent", "deep_research_agent", duration_ms=1000), - _llm_row("llm-root", "root", "deep_research_agent", "chatcmpl-root"), + _llm_row("llm-root", "root", "deep_research_agent", "chatcmpl-root", 0.01), _row("task", "root", "task", "tool", "deep_research_agent", start_ms=200, duration_ms=700), ] for i in range(researcher_invocations): rows += [ _row(f"res-{i}", "task", "researcher", "agent", "researcher", start_ms=201, duration_ms=5), - _llm_row(f"res-llm-{i}", f"res-{i}", "researcher", f"chatcmpl-res-{i}", start_ms=202), + _llm_row(f"res-llm-{i}", f"res-{i}", "researcher", f"chatcmpl-res-{i}", 0.001, start_ms=202), _row(f"res-tool-{i}", f"res-{i}", "search_docs", "tool", "researcher", start_ms=203, duration_ms=1), _row(f"res-mw-{i}", f"res-{i}", "FilesystemMiddleware.wrap_model_call", "framework", "researcher"), ] @@ -95,13 +113,44 @@ def test_empty_rows_is_none(): assert trace_from_rows("abc", []) is None -def test_llm_response_id_is_preserved_without_spend_enrichment(): +def test_litellm_request_joined_onto_llm_spans_only(): trace = trace_from_rows("t1", _deep_agent_rows()) assert trace is not None - spans = {span["span_id"]: span for span in trace["spans"]} - assert spans["llm-root"]["litellm_request_id"] == "chatcmpl-root" - assert spans["task"]["litellm_request_id"] is None - assert "spend" not in trace["summary"] + spans = {s["span_id"]: s for s in trace["spans"]} + llm = spans["llm-root"]["litellm"] + assert llm == { + "request_id": "chatcmpl-root", + "model": "claude-sonnet-4-5", + "model_group": "claude-sonnet-4-5", + "provider": "anthropic", + "key_alias": "research-bot", + "team_alias": "research-agents", + "spend": 0.01, + "prompt_tokens": 100, + "completion_tokens": 20, + "cache_read_tokens": 5, + "cache_write_tokens": 0, + "latency_ms": 100, + "ttft_ms": 30, + "status": "success", + } + assert spans["task"]["litellm"] is None + assert spans["root"]["litellm"] is None + + +def test_llm_span_without_spend_log_has_no_litellm(): + rows = [_row("root", "", "a", "agent", "a"), _row("llm", "root", "ChatOpenAI", "llm", "a", litellm_request_id="x")] + trace = trace_from_rows("t1", rows) + assert trace is not None + assert trace["spans"][1]["litellm"] is None + assert trace["summary"]["spend"] is None + + +def test_zero_cost_spend_log_is_distinct_from_missing_spend_log(): + trace = trace_from_rows("t1", [_row("root", "", "a", "agent", "a"), _llm_row("llm", "root", "a", "req", 0.0)]) + assert trace is not None + assert trace["summary"]["spend"] == 0.0 + assert trace["agents"][0]["spend"] == 0.0 def test_summary_totals(): @@ -119,6 +168,7 @@ def test_summary_totals(): assert summary["tool_calls"] == 2 assert summary["error_count"] == 0 assert (summary["input_tokens"], summary["output_tokens"]) == (200, 40) + assert summary["spend"] == pytest.approx(0.011) assert summary["models"] == ["claude-sonnet-4-5"] assert summary["duration_ms"] == 1000 assert summary["start_time"].startswith("2026-09-30T") @@ -147,12 +197,16 @@ def test_offsets_are_relative_to_trace_start_in_ms(): def test_span_from_row_optional_fields(): span = span_from_row(_row("s", "", "x", "chain", "a", status="STATUS_CODE_UNSET"), T0) - assert (span["model"], span["parent_span_id"], span["status"], span["litellm_request_id"]) == ( - None, - None, - "unset", - None, - ) + assert (span["model"], span["parent_span_id"], span["status"], span["litellm"]) == (None, None, "unset", None) + + +def test_ttft_is_none_without_completion_start_time(): + row = _llm_row("l", "", "a", "req", 0.1) + row["s_ttft_start_ms"] = 0 + assert span_from_row(row, T0)["litellm"]["ttft_ms"] is None # type: ignore[index] + + +# ---------------------------------------------------------------- agent_nodes def test_agent_nodes_parent_and_per_agent_counts(): @@ -165,6 +219,7 @@ def test_agent_nodes_parent_and_per_agent_counts(): "invocations": 1, "llm_calls": 1, "tool_calls": 1, + "spend": 0.01, "duration_ms": 1000, }, { @@ -173,6 +228,7 @@ def test_agent_nodes_parent_and_per_agent_counts(): "invocations": 1, "llm_calls": 1, "tool_calls": 1, + "spend": 0.001, "duration_ms": 5, }, ] @@ -187,6 +243,7 @@ def test_200_subagent_invocations_aggregate_into_one_node(): assert researcher["invocations"] == 200 assert researcher["llm_calls"] == 200 assert researcher["tool_calls"] == 200 + assert researcher["spend"] == pytest.approx(0.2) assert researcher["duration_ms"] == pytest.approx(1000) assert trace["summary"]["agent_count"] == 2 assert trace["summary"]["span_count"] == 3 + 4 * 200 @@ -238,10 +295,11 @@ def test_trace_summary_from_row(): "input_tokens": "30175", "output_tokens": "2620", "models": ["claude-sonnet-4-5"], + "spend": None, } ) assert summary["status"] == "ok" - assert (summary["span_count"], summary["error_count"]) == (126, 1) + assert (summary["span_count"], summary["error_count"], summary["spend"]) == (126, 1, None) assert summary["start_time"] == "2026-09-30T04:36:29.377000+00:00" @@ -264,8 +322,9 @@ async def test_list_traces_sets_next_cursor_on_full_page(): "input_tokens": 0, "output_tokens": 0, "models": [], + "spend": 0, } - client.query = AsyncMock(return_value=[row, {**row, "trace_id": "t1", "start_ms": 900}]) + client.list_traces = AsyncMock(return_value=[row, {**row, "trace_id": "t1", "start_ms": 900}]) store = ClickHouseTraceStore(client) scope = {"team_ids": ["team-a"], "api_key_hash": ""} @@ -273,22 +332,22 @@ async def test_list_traces_sets_next_cursor_on_full_page(): assert [t["trace_id"] for t in page["data"]] == ["t2", "t1"] assert page["next_cursor"] is not None assert decode_cursor(page["next_cursor"]) == (900, "t1") - params = client.query.call_args.args[1] + params = client.list_traces.call_args.args[0] assert params["team_ids"] == ["team-a"] and params["limit"] == 2 and params["cursor_ms"] == 0 page = await store.list_traces(scope, 0, 2000, cursor=page["next_cursor"], limit=3) # type: ignore[arg-type] assert page["next_cursor"] is None - assert client.query.call_args.args[1]["cursor_trace_id"] == "t1" + assert client.list_traces.call_args.args[0]["cursor_trace_id"] == "t1" @pytest.mark.asyncio async def test_get_span_not_found_and_found(): client = MagicMock() - client.query = AsyncMock(return_value=[]) + client.span_detail = AsyncMock(return_value=[]) store = ClickHouseTraceStore(client) scope = {"team_ids": [], "api_key_hash": ""} assert await store.get_span("t", "s", scope) is None # type: ignore[arg-type] - client.query = AsyncMock(return_value=[{"span_id": "s", "input": "i", "output": "o", "attributes": {"k": "v"}}]) + client.span_detail = AsyncMock(return_value=[{"span_id": "s", "input": "i", "output": "o", "attributes": {"k": "v"}}]) assert await store.get_span("t", "s", scope) == { # type: ignore[arg-type] "span_id": "s", "input": "i", diff --git a/ui/litellm-dashboard/src/components/view_logs/RequestLogsPanel.tsx b/ui/litellm-dashboard/src/components/view_logs/RequestLogsPanel.tsx index dc0f84c1d56f..4471fbf930d5 100644 --- a/ui/litellm-dashboard/src/components/view_logs/RequestLogsPanel.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/RequestLogsPanel.tsx @@ -327,6 +327,7 @@ export default function RequestLogsPanel({ accessToken, token, userRole, userID, endTime={endTime} isCustomDate={isCustomDate} isLiveTail={isLiveTail} + onOpenRequestLog={openLog} /> {logsView !== "traces" && ( diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentGraph.tsx b/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentGraph.tsx index e730b5d3d98d..5b8704259fea 100644 --- a/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentGraph.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentGraph.tsx @@ -8,6 +8,7 @@ import { cn } from "@/lib/cva.config"; import type { AgentNode, Span } from "./traceTypes"; import { agentsWithErrors, + fmtCost, fmtMs, GRAPH_NODE_HEIGHT, invocationsOf, @@ -91,7 +92,7 @@ function Node({ node, hasError, selected, onClick }: NodeProps) { {agent.name} - {`${agent.llm_calls} LLM · ${agent.tool_calls} tool`} + {`${fmtCost(agent.spend)} · ${agent.llm_calls} LLM · ${agent.tool_calls} tool`} ); @@ -196,7 +197,9 @@ export function AgentGraph({ agents, spans, onOpenInvocation }: AgentGraphProps) /> ))} -

Click an agent to list its invocations.

+

+ Node width scales with spend. Click an agent to list its invocations. +

{selected && ( )} diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesSection.test.tsx b/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesSection.test.tsx index f4a45c87b949..c192cd057877 100644 --- a/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesSection.test.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesSection.test.tsx @@ -26,6 +26,7 @@ const renderSection = (view: "all" | "traces" | "llm") => endTime="2026-09-30T00:00" isCustomDate={false} isLiveTail={false} + onOpenRequestLog={vi.fn()} />, ); @@ -57,7 +58,7 @@ describe("AgentTracesSection", () => { expect(screen.queryByRole("table", { name: "Agent traces" })).not.toBeInTheDocument(); }); - it("lists traces with the agent badge, red status for runs with errors", async () => { + it("lists traces with the agent badge, red status for runs with errors and cost", async () => { vi.mocked(agentTraceListCall).mockResolvedValue(traceList as TracePage); renderSection("traces"); @@ -66,7 +67,7 @@ describe("AgentTracesSection", () => { const lead = rows.find((row) => row.textContent?.includes("research_lead")); expect(lead).toHaveTextContent("◆ 6 agents · 21 LLM · 25 tool"); expect(lead).toHaveTextContent("Success"); - expect(screen.queryByRole("columnheader", { name: "Cost" })).not.toBeInTheDocument(); + expect(lead).toHaveTextContent("$0.1283"); expect(lead).toHaveTextContent("Should we store OTEL agent spans in ClickHouse or Postgres at 50k spans/sec?"); const failed = rows.find((row) => row.textContent?.includes("acme-404")); expect(failed).toHaveTextContent("Failure"); diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesSection.tsx b/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesSection.tsx index 7e012310d474..df9431dc430f 100644 --- a/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesSection.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesSection.tsx @@ -44,6 +44,8 @@ interface AgentTracesSectionProps { endTime: string; isCustomDate: boolean; isLiveTail: boolean; + /** Open the request-log drawer for a LiteLLM request id (from a span's LiteLLM request card). */ + onOpenRequestLog: (requestId: string) => void; } /** @@ -58,6 +60,7 @@ export function AgentTracesSection({ endTime, isCustomDate, isLiveTail, + onOpenRequestLog, }: AgentTracesSectionProps) { const [openTrace, setOpenTrace] = useState<{ traceId: string; spanId: string | null } | null>(null); const traceOptions: Parameters[0] = { @@ -72,6 +75,11 @@ export function AgentTracesSection({ if (view === "llm") return null; + const handleOpenRequestLog = (requestId: string) => { + setOpenTrace(null); + onOpenRequestLog(requestId); + }; + return ( <> {view === "traces" && traces.notEnabledDetail !== null && } @@ -93,6 +101,7 @@ export function AgentTracesSection({ initialSpanId={openTrace?.spanId} accessToken={accessToken} onClose={() => setOpenTrace(null)} + onOpenRequestLog={handleOpenRequestLog} /> ); diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesTable.tsx b/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesTable.tsx index edf67b95f1b6..4bfc85ef3231 100644 --- a/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesTable.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesTable.tsx @@ -14,6 +14,7 @@ import { AgentTracePill, SpanStatusBadge, SpanTypePill } from "./TracePills"; import type { Span, TraceSummary } from "./traceTypes"; import { agentBadgeLabel, + fmtCost, fmtMs, fmtTok, llmSpans, @@ -37,7 +38,7 @@ interface AgentTracesTableProps { compact?: boolean; } -const COLUMNS = ["", "Time", "Type", "Status", "Trace", "Input", "Duration", "Model", "Tokens"] as const; +const COLUMNS = ["", "Time", "Type", "Status", "Trace", "Input", "Cost", "Duration", "Model", "Tokens"] as const; const TRUNC = "block max-w-[280px] truncate"; @@ -90,8 +91,8 @@ function ChildRow({ span, onClick }: { span: Span; onClick: () => void }) { - {span.litellm_request_id ? ( - shortId(span.litellm_request_id, 22) + {span.litellm ? ( + shortId(span.litellm.request_id, 22) ) : ( {span.agent} )} @@ -99,6 +100,7 @@ function ChildRow({ span, onClick }: { span: Span; onClick: () => void }) { {previewText(span.input_preview)} + {fmtCost(span.litellm?.spend)} {fmtMs(span.duration_ms)} {spanLabel(span)} {fmtTok(tokens)} @@ -158,6 +160,7 @@ function TraceRow({ {previewText(trace.input_preview)} + {fmtCost(trace.spend)} {fmtMs(trace.duration_ms)} {trace.models.join(", ")} diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/SpanDetail.tsx b/ui/litellm-dashboard/src/components/view_logs/TraceView/SpanDetail.tsx index 817e4151271e..f3ab89c954d6 100644 --- a/ui/litellm-dashboard/src/components/view_logs/TraceView/SpanDetail.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/SpanDetail.tsx @@ -3,21 +3,107 @@ import { useQuery } from "@tanstack/react-query"; import { useState } from "react"; +import { Button } from "@/components/ui/button"; import { cn } from "@/lib/cva.config"; import { agentTraceSpanCall } from "../../networking"; import { SPAN_PILL_CLASS, SpanStatusBadge, SpanTypePill } from "./TracePills"; -import type { Span, TraceMessage } from "./traceTypes"; -import { fmtMs, parseMessages, prettyPayload, spanLabel } from "./traceUtils"; +import type { LiteLLMRequest, Span, TraceMessage } from "./traceTypes"; +import { fmtCost, fmtMs, parseMessages, prettyPayload, spanLabel } from "./traceUtils"; interface SpanDetailProps { accessToken: string; traceId: string; span: Span; + /** Open the existing request-log drawer for a LiteLLM request id. */ + onOpenRequestLog?: (requestId: string) => void; } const LONG_MESSAGE_CHARS = 360; +function Field({ label, value, mono = false }: { label: string; value: string; mono?: boolean }) { + return ( +
+
{label}
+
+ {value || "—"} +
+
+ ); +} + +function TokenFlowBar({ request }: { request: LiteLLMRequest }) { + const fresh = Math.max(request.prompt_tokens - request.cache_read_tokens - request.cache_write_tokens, 0); + const total = request.prompt_tokens + request.completion_tokens || 1; + const parts = [ + { label: "input", value: fresh, className: "bg-info" }, + { label: "cache read", value: request.cache_read_tokens, className: "bg-cyan-500" }, + { label: "cache write", value: request.cache_write_tokens, className: "bg-violet-500" }, + { label: "output", value: request.completion_tokens, className: "bg-amber-500" }, + ]; + return ( + <> +
+ {parts.map((part) => ( + + ))} +
+
+ {parts.map((part) => ( + + + {part.label} {part.value.toLocaleString()} + + ))} +
+ + ); +} + +function LiteLLMRequestCard({ + request, + onOpenRequestLog, +}: { + request: LiteLLMRequest; + onOpenRequestLog?: (requestId: string) => void; +}) { + return ( +
+
+ + LiteLLM request · joined from spend log + + {onOpenRequestLog && ( + + )} +
+
+ + + + + + + + + +
+ +
+ ); +} + function MessageBlock({ message }: { message: TraceMessage }) { const long = (message.content || "").length > LONG_MESSAGE_CHARS; const [expanded, setExpanded] = useState(!long); @@ -86,7 +172,7 @@ function Payload({ title, value }: { title: string; value: string }) { } /** Right pane: span header, joined LiteLLM request, lazily fetched input / output / attributes. */ -export function SpanDetail({ accessToken, traceId, span }: SpanDetailProps) { +export function SpanDetail({ accessToken, traceId, span, onOpenRequestLog }: SpanDetailProps) { const detailQuery = useQuery({ queryKey: ["agentTraceSpan", traceId, span.span_id, accessToken], queryFn: () => agentTraceSpanCall(accessToken, traceId, span.span_id), @@ -126,6 +212,7 @@ export function SpanDetail({ accessToken, traceId, span }: SpanDetailProps) { {span.error}
)} + {span.litellm && } {detailQuery.isLoading &&
Loading span input/output…
} {detailQuery.isError && (
Could not load span details: {detailQuery.error.message}
diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/SpanTree.tsx b/ui/litellm-dashboard/src/components/view_logs/TraceView/SpanTree.tsx index 6da0b6854661..d19aae67310c 100644 --- a/ui/litellm-dashboard/src/components/view_logs/TraceView/SpanTree.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/SpanTree.tsx @@ -6,7 +6,7 @@ import { cn } from "@/lib/cva.config"; import { SPAN_BAR_CLASS, SpanTypePill } from "./TracePills"; import type { Span } from "./traceTypes"; -import { fmtMs, GROUP_PAGE_SIZE, spanLabel, type SpanGroup, type TreeRow } from "./traceUtils"; +import { fmtCost, fmtMs, fmtTok, GROUP_PAGE_SIZE, spanLabel, type SpanGroup, type TreeRow } from "./traceUtils"; interface SpanTreeProps { rows: TreeRow[]; @@ -20,11 +20,17 @@ interface SpanTreeProps { const MIN_BAR_PERCENT = 0.3; -const spanMeta = (span: Span): string => fmtMs(span.duration_ms); +const spanMeta = (span: Span): string => { + if (span.litellm) { + return `${fmtTok(span.litellm.prompt_tokens)}→${fmtTok(span.litellm.completion_tokens)} · ${fmtCost(span.litellm.spend)}`; + } + return fmtMs(span.duration_ms); +}; export const groupSummary = (group: SpanGroup): string => { const errors = group.errors ? ` · ${group.errors} error${group.errors === 1 ? "" : "s"}` : ""; - return `${group.name} ×${group.spans.length} · p50 ${fmtMs(group.p50Ms)}${errors}`; + const cost = group.hasSpend ? ` · ${fmtCost(group.spend)}` : ""; + return `${group.name} ×${group.spans.length}${cost} · p50 ${fmtMs(group.p50Ms)}${errors}`; }; function Guides({ depth }: { depth: number }) { diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/TraceDrawer.test.tsx b/ui/litellm-dashboard/src/components/view_logs/TraceView/TraceDrawer.test.tsx index 4beb10df8344..2fb840331d2b 100644 --- a/ui/litellm-dashboard/src/components/view_logs/TraceView/TraceDrawer.test.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/TraceDrawer.test.tsx @@ -42,7 +42,7 @@ describe("TraceDrawer", () => { vi.mocked(agentTraceSpanCall).mockImplementation(async (_token, _trace, spanId) => spanDetail(spanId)); }); - it("opens a healthy trace on the Steps view with the stats strip", async () => { + it("opens a healthy trace on the Steps view with the stats strip and per-agent cost split", async () => { renderDrawer(research); expect(await screen.findByRole("heading", { name: "research_lead" })).toBeInTheDocument(); @@ -50,7 +50,8 @@ describe("TraceDrawer", () => { expect(stats).toHaveTextContent("40.20s"); expect(stats).toHaveTextContent("LLM calls21"); expect(stats).toHaveTextContent("Tool calls25"); - expect(stats).not.toHaveTextContent("Cost"); + expect(stats).toHaveTextContent("$0.1283"); + expect(screen.getByLabelText("Cost by agent")).toHaveTextContent("researcher $0.0892"); expect(screen.getByRole("button", { name: "Steps" })).toHaveAttribute("aria-pressed", "true"); expect(screen.getByRole("list", { name: "Trace steps" })).toBeInTheDocument(); }); @@ -63,7 +64,22 @@ describe("TraceDrawer", () => { const selected = within(tree).getByRole("treeitem", { selected: true }); expect(selected).toHaveTextContent("lookup_benchmark"); // the 12 researcher invocations collapse into one group row - expect(within(tree).getByText(/researcher ×12 · p50/)).toBeInTheDocument(); + expect(within(tree).getByText(/researcher ×12 · \$0\.0635 · p50/)).toBeInTheDocument(); + }); + + it("shows the LiteLLM request card with model group and deployment for an LLM span", async () => { + const user = userEvent.setup(); + const onOpenRequestLog = vi.fn(); + const llm = research.spans.find((s) => s.type === "llm" && s.litellm) as Trace["spans"][number]; + renderDrawer(research, { initialSpanId: llm.span_id, onOpenRequestLog }); + + const card = await screen.findByRole("region", { name: "LiteLLM request" }); + expect(card).toHaveTextContent(llm.litellm?.request_id ?? ""); + expect(card).toHaveTextContent("claude-sonnet-4-5"); + expect(card).toHaveTextContent("openai/claude-sonnet-4-5 (openai)"); + await user.click(within(card).getByRole("button", { name: "Open request log →" })); + expect(onOpenRequestLog).toHaveBeenCalledWith(llm.litellm?.request_id); + expect(await screen.findByText(/lookup_benchmark/, { selector: "b" })).toBeInTheDocument(); }); it("hides framework spans until the toggle is checked", async () => { diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/TraceDrawer.tsx b/ui/litellm-dashboard/src/components/view_logs/TraceView/TraceDrawer.tsx index 13f5ac79bba1..7346f74ca4be 100644 --- a/ui/litellm-dashboard/src/components/view_logs/TraceView/TraceDrawer.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/TraceDrawer.tsx @@ -22,12 +22,15 @@ import { buildVisibleTree, firstErrorSpan, flattenTree, + fmtCost, fmtMs, fmtTok, revealSpan, + sortStepsByCost, spanRowIds, stepsFromSpans, subtreeStats, + totalCacheRead, traceHasErrors, type TreeUiState, } from "./traceUtils"; @@ -42,6 +45,7 @@ interface TraceDrawerProps { initialSpanId?: string | null; accessToken: string; onClose: () => void; + onOpenRequestLog?: (requestId: string) => void; } const DRAWER_WIDTH = "min(1180px, 80vw)"; @@ -91,7 +95,7 @@ function Stat({ label, value, sub }: { label: string; value: string; sub?: strin export function TraceStats({ trace }: { trace: Trace }) { const { summary } = trace; return ( -
+
@@ -101,6 +105,23 @@ export function TraceStats({ trace }: { trace: Trace }) { value={fmtTok(summary.input_tokens + summary.output_tokens)} sub={`${fmtTok(summary.input_tokens)} in · ${fmtTok(summary.output_tokens)} out`} /> + + +
+ ); +} + +function AgentCostSplit({ trace }: { trace: Trace }) { + if (trace.agents.length < 2) return null; + const sorted = [...trace.agents].sort((a, b) => (b.spend ?? 0) - (a.spend ?? 0)); + return ( +
+ Cost by agent: + {sorted.map((agent) => ( + + {agent.name} {fmtCost(agent.spend)} + + ))}
); } @@ -109,13 +130,15 @@ interface TraceBodyProps { trace: Trace; accessToken: string; initialSpanId?: string | null; + onOpenRequestLog?: (requestId: string) => void; } -function TraceBody({ trace, accessToken, initialSpanId }: TraceBodyProps) { +function TraceBody({ trace, accessToken, initialSpanId, onOpenRequestLog }: TraceBodyProps) { const [initial] = useState(() => initialTraceView(trace, initialSpanId)); const [mode, setMode] = useState(initial.mode); const [selectedId, setSelectedId] = useState(initial.selectedId); const [showFramework, setShowFramework] = useState(false); + const [sortByCost, setSortByCost] = useState(false); const [ui, setUi] = useState(() => initialTreeUi(trace, initial.selectedId)); const { children, visibleCount } = useMemo( @@ -124,7 +147,8 @@ function TraceBody({ trace, accessToken, initialSpanId }: TraceBodyProps) { ); const stats = useMemo(() => subtreeStats(trace.spans), [trace.spans]); const rows = useMemo(() => flattenTree(children, stats, ui), [children, stats, ui]); - const steps = useMemo(() => stepsFromSpans(trace.spans), [trace.spans]); + const timeSteps = useMemo(() => stepsFromSpans(trace.spans), [trace.spans]); + const steps = useMemo(() => (sortByCost ? sortStepsByCost(timeSteps) : timeSteps), [timeSteps, sortByCost]); const selectedSpan = useMemo( () => trace.spans.find((s) => s.span_id === selectedId) ?? null, [trace.spans, selectedId], @@ -190,6 +214,12 @@ function TraceBody({ trace, accessToken, initialSpanId }: TraceBodyProps) { Show framework spans )} + {mode === "steps" && ( + + )} {mode === "tree" ? `${visibleCount} of ${trace.spans.length} spans` : `${steps.length} steps`} @@ -213,6 +243,7 @@ function TraceBody({ trace, accessToken, initialSpanId }: TraceBodyProps) { accessToken={accessToken} traceId={trace.summary.trace_id} span={selectedSpan} + onOpenRequestLog={onOpenRequestLog} /> ) : (
Select a span to see its details.
@@ -255,19 +286,28 @@ function TraceHeader({ trace, onClose }: { trace: Trace; onClose: () => void }) · {new Date(summary.start_time).toLocaleString()}
+
+ +
); } /** Right-side drawer for one agent trace: header, stats strip, Steps / Tree / Graph views and span details. */ -export function TraceDrawer({ open, traceId, initialSpanId, accessToken, onClose }: TraceDrawerProps) { - const traceQueryOptions = { +export function TraceDrawer({ + open, + traceId, + initialSpanId, + accessToken, + onClose, + onOpenRequestLog, +}: TraceDrawerProps) { + const traceQuery = useQuery({ queryKey: ["agentTrace", traceId, accessToken], queryFn: () => agentTraceCall(accessToken, traceId as string), enabled: open && traceId !== null, staleTime: 30_000, - }; - const traceQuery = useQuery(traceQueryOptions); + }); const trace = traceQuery.data; return ( @@ -299,6 +339,7 @@ export function TraceDrawer({ open, traceId, initialSpanId, accessToken, onClose trace={trace} accessToken={accessToken} initialSpanId={initialSpanId} + onOpenRequestLog={onOpenRequestLog} /> )} diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/TraceSteps.tsx b/ui/litellm-dashboard/src/components/view_logs/TraceView/TraceSteps.tsx index b17a3e067c74..acf463e1289f 100644 --- a/ui/litellm-dashboard/src/components/view_logs/TraceView/TraceSteps.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/TraceSteps.tsx @@ -5,7 +5,7 @@ import { useEffect, useRef } from "react"; import { cn } from "@/lib/cva.config"; import { SPAN_PILL_CLASS, SpanTypePill } from "./TracePills"; -import { fmtMs, previewText, spanLabel, type TraceStep } from "./traceUtils"; +import { fmtCost, fmtMs, previewText, spanLabel, type TraceStep } from "./traceUtils"; interface TraceStepsProps { steps: TraceStep[]; @@ -27,11 +27,15 @@ function StepTitle({ step }: { step: TraceStep }) { ); } + const cost = span.type === "llm" ? ` · ${fmtCost(span.litellm?.spend)}` : ""; return ( <> {spanLabel(span)} - {fmtMs(span.duration_ms)} + + {fmtMs(span.duration_ms)} + {cost} + ); } diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/traceTypes.ts b/ui/litellm-dashboard/src/components/view_logs/TraceView/traceTypes.ts index f2611b17ad56..436b927b19e5 100644 --- a/ui/litellm-dashboard/src/components/view_logs/TraceView/traceTypes.ts +++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/traceTypes.ts @@ -2,11 +2,30 @@ * Agent tracing types. Mirrors `litellm/tracing/types.py` exactly. * * A trace is one agent run made of spans (agent / llm / tool / chain / framework). + * LLM spans that went through LiteLLM carry a `LiteLLMRequest`: the spend-log row + * for that call, joined on the provider response id. */ export type SpanType = "agent" | "llm" | "tool" | "chain" | "framework"; export type SpanStatus = "ok" | "error" | "unset"; +export interface LiteLLMRequest { + request_id: string; + model: string; + model_group: string; + provider: string; + key_alias: string; + team_alias: string; + spend: number; + prompt_tokens: number; + completion_tokens: number; + cache_read_tokens: number; + cache_write_tokens: number; + latency_ms: number; + ttft_ms: number | null; + status: string; +} + export interface Span { span_id: string; parent_span_id: string | null; @@ -24,7 +43,7 @@ export interface Span { model: string | null; input_tokens: number; output_tokens: number; - litellm_request_id: string | null; + litellm: LiteLLMRequest | null; } /** One distinct agent in a trace. 200 invocations of `researcher` = one node. */ @@ -34,6 +53,7 @@ export interface AgentNode { invocations: number; llm_calls: number; tool_calls: number; + spend: number | null; duration_ms: number; } @@ -54,6 +74,7 @@ export interface TraceSummary { error_count: number; input_tokens: number; output_tokens: number; + spend: number | null; models: string[]; } diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/traceUtils.test.ts b/ui/litellm-dashboard/src/components/view_logs/TraceView/traceUtils.test.ts index 12d6c53491f0..75489d69b378 100644 --- a/ui/litellm-dashboard/src/components/view_logs/TraceView/traceUtils.test.ts +++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/traceUtils.test.ts @@ -3,12 +3,14 @@ import { describe, expect, it } from "vitest"; import deepAgentTrace from "./__fixtures__/deep_agent_trace.json"; import researchTrace from "./__fixtures__/research_trace.json"; import swarmTrace from "./__fixtures__/swarm_trace.json"; +import { groupSummary } from "./SpanTree"; import type { Span, Trace } from "./traceTypes"; import { agentBadgeLabel, buildVisibleTree, firstErrorSpan, flattenTree, + fmtCost, fmtMs, GROUP_PAGE_SIZE, groupSiblingAgents, @@ -20,6 +22,7 @@ import { previewText, revealSpan, ROOT_KEY, + sortStepsByCost, spanLabel, spanRowIds, stepsFromSpans, @@ -45,15 +48,18 @@ const span = (overrides: Partial & Pick): Span => ({ model: null, input_tokens: 0, output_tokens: 0, - litellm_request_id: null, + litellm: null, ...overrides, }); describe("formatting", () => { - it("formats durations the way the table shows them", () => { + it("formats durations and costs the way the table shows them", () => { expect(fmtMs(4.25)).toBe("4.3ms"); expect(fmtMs(950)).toBe("950ms"); expect(fmtMs(51386)).toBe("51.39s"); + expect(fmtCost(0.0832806)).toBe("$0.0833"); + expect(fmtCost(0.0004)).toBe("$0.00040"); + expect(fmtCost(null)).toBe("—"); }); it("labels agent rows with agent, LLM and tool counts", () => { @@ -61,9 +67,10 @@ describe("formatting", () => { expect(agentBadgeLabel({ agent_count: 1, llm_calls: 2, tool_calls: 1 })).toBe("◆ Agent · 2 LLM · 1 tool"); }); - it("headlines LLM spans with the OTLP model", () => { - const llm = { span_id: "llm", type: "llm", name: "generation", model: "model-a" } as const; - expect(spanLabel(span(llm))).toBe("model-a"); + it("headlines LLM spans with the model group, not the deployment", () => { + const llm = swarm.spans.find((s) => s.litellm?.model.startsWith("openai/")); + expect(llm).toBeDefined(); + expect(spanLabel(llm as Span)).toBe((llm as Span).litellm?.model_group); }); it("pulls the user message out of a truncated JSON preview", () => { @@ -85,17 +92,13 @@ describe("formatting", () => { describe("buildVisibleTree", () => { it("hides framework spans and re-parents their children to the nearest visible ancestor", () => { - const middleware = { span_id: "mw", parent_span_id: "root", type: "framework", name: "X.wrap_model_call" } as const; - const model = { span_id: "model", parent_span_id: "mw", type: "chain", name: "model" } as const; - const llm = { span_id: "llm", parent_span_id: "model", type: "llm", start_offset_ms: 5 } as const; - const step = { - span_id: "step", - parent_span_id: "root", - type: "chain", - name: "planner", - start_offset_ms: 1, - } as const; - const spans = [span({ span_id: "root", type: "agent" }), span(middleware), span(model), span(llm), span(step)]; + const spans = [ + span({ span_id: "root", type: "agent" }), + span({ span_id: "mw", parent_span_id: "root", type: "framework", name: "X.wrap_model_call" }), + span({ span_id: "model", parent_span_id: "mw", type: "chain", name: "model" }), + span({ span_id: "llm", parent_span_id: "model", type: "llm", start_offset_ms: 5 }), + span({ span_id: "step", parent_span_id: "root", type: "chain", name: "planner", start_offset_ms: 1 }), + ]; const compact = buildVisibleTree(spans, false); expect(compact.visibleCount).toBe(3); expect(compact.children.get(ROOT_KEY)?.map((s) => s.span_id)).toEqual(["root"]); @@ -136,23 +139,37 @@ describe("groupSiblingAgents", () => { expect(items.some((i) => i.kind === "span" && i.span.name === "critic")).toBe(true); }); - it("counts grouped invocations that contain errors", () => { + it("rolls up subtree spend and counts invocations that contain errors", () => { const items = groupSiblingAgents(root.span_id, children.get(root.span_id) ?? [], stats); const group = items.find((i) => i.kind === "group"); if (group?.kind !== "group") throw new Error("expected a group"); + const researcherAgent = swarm.agents.find((a) => a.name === "researcher"); + const factChecker = swarm.agents.find((a) => a.name === "fact_checker"); + // researcher subtrees include the nested fact_checker invocations + expect(group.group.spend).toBeCloseTo((researcherAgent?.spend ?? 0) + (factChecker?.spend ?? 0), 6); expect(group.group.errors).toBeGreaterThan(0); expect(group.group.p50Ms).toBeGreaterThan(0); }); it("leaves groups of 10 or fewer alone", () => { const parent = span({ span_id: "p", type: "agent" }); - const kids = Array.from({ length: 10 }, (_, i) => { - const props = { span_id: `k${i}`, parent_span_id: "p", type: "agent", name: "worker" } as const; - return span(props); - }); + const kids = Array.from({ length: 10 }, (_, i) => + span({ span_id: `k${i}`, parent_span_id: "p", type: "agent", name: "worker" }), + ); const items = groupSiblingAgents("p", kids, subtreeStats([parent, ...kids])); expect(items.every((i) => i.kind === "span")).toBe(true); }); + + it("does not show a zero cost when grouped spans have no spend rows", () => { + const parent = span({ span_id: "p", type: "agent" }); + const children = Array.from({ length: 11 }, (_, index) => + span({ span_id: `child-${index}`, parent_span_id: "p", type: "agent", name: "worker" }), + ); + const items = groupSiblingAgents("p", children, subtreeStats([parent, ...children])); + const group = items.find((item) => item.kind === "group"); + if (group?.kind !== "group") throw new Error("expected a group"); + expect(groupSummary(group.group)).toBe("worker ×11 · p50 1.0ms"); + }); }); describe("flattenTree + revealSpan", () => { @@ -167,16 +184,9 @@ describe("flattenTree + revealSpan", () => { it("pages group invocations 20 at a time with a 'more' row until all are shown", () => { const parent = span({ span_id: "p", type: "agent" }); - const kids = Array.from({ length: 45 }, (_, i) => { - const props = { - span_id: `k${i}`, - parent_span_id: "p", - type: "agent", - name: "worker", - start_offset_ms: i, - } as const; - return span(props); - }); + const kids = Array.from({ length: 45 }, (_, i) => + span({ span_id: `k${i}`, parent_span_id: "p", type: "agent", name: "worker", start_offset_ms: i }), + ); const all = [parent, ...kids]; const tree = buildVisibleTree(all, false).children; const key = "p::worker"; @@ -224,6 +234,13 @@ describe("stepsFromSpans", () => { // the lead's first decision runs write_file and task expect(first.toolNames).toEqual(["write_file", "task"]); }); + + it("sorts by cost without renumbering", () => { + const sorted = sortStepsByCost(stepsFromSpans(deepAgent.spans)); + const spend = sorted.map((s) => s.span.litellm?.spend ?? 0); + expect(spend).toEqual([...spend].sort((a, b) => b - a)); + expect(sorted[0].span.type).toBe("llm"); + }); }); describe("trace-level helpers", () => { @@ -248,7 +265,7 @@ describe("trace-level helpers", () => { ); const researcherNode = layout.nodes.find((n) => n.agent.name === "researcher"); const criticNode = layout.nodes.find((n) => n.agent.name === "critic"); - expect(researcherNode?.width).toBe(criticNode?.width); + expect(researcherNode?.width).toBeGreaterThan(criticNode?.width ?? Infinity); expect(researcherNode?.x).toBeGreaterThan(layout.nodes[0].x); }); diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/traceUtils.ts b/ui/litellm-dashboard/src/components/view_logs/TraceView/traceUtils.ts index 08a176e5bbdd..98a38fc8f52c 100644 --- a/ui/litellm-dashboard/src/components/view_logs/TraceView/traceUtils.ts +++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/traceUtils.ts @@ -14,6 +14,12 @@ export const fmtMs = (ms: number): string => { return `${Math.max(ms, 0).toFixed(ms < 10 ? 1 : 0)}ms`; }; +export const fmtCost = (cost: number | null | undefined): string => { + if (cost == null) return "—"; + if (cost === 0) return "$0"; + return cost < 0.001 ? `$${cost.toFixed(5)}` : `$${cost.toFixed(4)}`; +}; + export const fmtTok = (n: number): string => (n >= 1000 ? `${(n / 1000).toFixed(1)}k` : String(n)); export const shortId = (id: string, length = 16): string => (id.length > length ? `${id.slice(0, length)}…` : id); @@ -27,7 +33,10 @@ export const agentBadgeLabel = (summary: Pick (span.type === "llm" ? span.model || span.name : span.name); +export const spanLabel = (span: Span): string => + span.type === "llm" ? span.litellm?.model_group || span.model || span.litellm?.model || span.name : span.name; + +export const spanSpend = (span: Span): number => span.litellm?.spend ?? 0; /* ------------------------------------------------------------------ */ /* Visible tree (framework spans hidden + children re-parented) */ @@ -93,21 +102,24 @@ export function buildVisibleTree(spans: readonly Span[], showFramework: boolean) /* ------------------------------------------------------------------ */ export interface SubtreeStats { + spend: number; + hasSpend: boolean; errors: number; } -/** Error count of every span's full subtree, keyed by span id. */ +/** Spend and error count of every span's full (raw) subtree, keyed by span id. */ export function subtreeStats(spans: readonly Span[]): Map { const raw = buildVisibleTree(spans, true).children; const stats = new Map(); const visit = (span: Span): SubtreeStats => { const cached = stats.get(span.span_id); if (cached) return cached; - const own: SubtreeStats = { errors: span.status === "error" ? 1 : 0 }; - for (const child of raw.get(span.span_id) ?? []) { - const childStats = visit(child); - own.errors += childStats.errors; - } + const children = (raw.get(span.span_id) ?? []).map(visit); + const own: SubtreeStats = { + spend: spanSpend(span) + children.reduce((total, child) => total + child.spend, 0), + hasSpend: span.litellm !== null || children.some((child) => child.hasSpend), + errors: Number(span.status === "error") + children.reduce((total, child) => total + child.errors, 0), + }; stats.set(span.span_id, own); return own; }; @@ -128,6 +140,8 @@ export interface SpanGroup { key: string; name: string; spans: Span[]; + spend: number; + hasSpend: boolean; p50Ms: number; errors: number; } @@ -147,6 +161,8 @@ const buildGroup = (parentKey: string, name: string, members: Span[], stats: Map key: groupKey(parentKey, name), name, spans: members, + spend: members.reduce((sum, s) => sum + (stats.get(s.span_id)?.spend ?? 0), 0), + hasSpend: members.some((s) => stats.get(s.span_id)?.hasSpend), p50Ms: median(members.map((s) => s.duration_ms)), errors: members.filter((s) => (stats.get(s.span_id)?.errors ?? 0) > 0).length, }); @@ -282,7 +298,7 @@ export function revealSpan( export interface TraceStep { span: Span; - /** 1-based, in time order. */ + /** 1-based, in time order (stable when re-sorted by cost). */ number: number; /** Nearest enclosing non-root agent span name, e.g. "researcher". */ subagent: string | null; @@ -339,6 +355,10 @@ export function stepsFromSpans(spans: readonly Span[]): TraceStep[] { }); } +/** Most expensive first; ties keep time order. */ +export const sortStepsByCost = (steps: readonly TraceStep[]): TraceStep[] => + [...steps].sort((a, b) => spanSpend(b.span) - spanSpend(a.span) || a.number - b.number); + /* ------------------------------------------------------------------ */ /* Trace-level rollups */ /* ------------------------------------------------------------------ */ @@ -355,6 +375,9 @@ export function firstErrorSpan(spans: readonly Span[]): Span | null { return failed.find((s) => s.parent_span_id !== null) ?? failed[0] ?? null; } +export const totalCacheRead = (spans: readonly Span[]): number => + spans.reduce((sum, s) => sum + (s.litellm?.cache_read_tokens ?? 0), 0); + export const agentsWithErrors = (spans: readonly Span[]): Set => new Set(spans.filter((s) => s.status === "error").map((s) => s.agent)); @@ -414,20 +437,22 @@ function agentDepths(agents: readonly AgentNode[]): Map { return depths; } -/** Left-to-right layered layout: one column per nesting level. */ +/** Left-to-right layered layout: one column per nesting level, node width by spend. */ export function layoutAgentGraph(agents: readonly AgentNode[]): GraphLayout { const depths = agentDepths(agents); + const maxSpend = Math.max(0, ...agents.map((a) => a.spend ?? 0)); const columnWidth = GRAPH_MAX_NODE_WIDTH + GRAPH_COLUMN_GAP; const rowsPerColumn = new Map(); const nodes: GraphNode[] = agents.map((agent) => { const depth = depths.get(agent.name) ?? 0; const row = rowsPerColumn.get(depth) ?? 0; rowsPerColumn.set(depth, row + 1); + const share = maxSpend > 0 ? (agent.spend ?? 0) / maxSpend : 0; return { agent, x: GRAPH_PADDING + depth * columnWidth, y: GRAPH_PADDING + row * (GRAPH_NODE_HEIGHT + GRAPH_ROW_GAP), - width: GRAPH_MIN_NODE_WIDTH, + width: Math.round(GRAPH_MIN_NODE_WIDTH + share * (GRAPH_MAX_NODE_WIDTH - GRAPH_MIN_NODE_WIDTH)), }; }); const byName = new Map(nodes.map((n) => [n.agent.name, n])); From 338702e2a03ffcc7a4c949deda8c7462bb29f747 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Wed, 30 Sep 2026 09:55:26 -0700 Subject: [PATCH 08/12] test(tracing): verify strict inserts and named native reads --- .../crates/python-bridge/src/routes/traces.rs | 1 + litellm-rust/crates/traces/src/insert.rs | 17 +++++ .../crates/traces/tests/migrations.rs | 74 +++++++++++++++---- litellm/rust_bridge/_native.pyi | 2 +- tests/test_litellm_rust/test_traces.py | 28 +++++-- 5 files changed, 99 insertions(+), 23 deletions(-) diff --git a/litellm-rust/crates/python-bridge/src/routes/traces.rs b/litellm-rust/crates/python-bridge/src/routes/traces.rs index 3e82cd8d3d1a..c3ad96523f76 100644 --- a/litellm-rust/crates/python-bridge/src/routes/traces.rs +++ b/litellm-rust/crates/python-bridge/src/routes/traces.rs @@ -35,6 +35,7 @@ pub struct NativeTraceStorage { #[pymethods] impl NativeTraceStorage { #[new] + #[pyo3(signature = (database, url, reader_url=None))] fn new(database: String, url: &str, reader_url: Option<&str>) -> PyResult { litellm_traces::schema_statements(&database, 1, 1).map_err(map_error)?; Ok(Self { diff --git a/litellm-rust/crates/traces/src/insert.rs b/litellm-rust/crates/traces/src/insert.rs index e6b58903964e..f4fd0d16f34b 100644 --- a/litellm-rust/crates/traces/src/insert.rs +++ b/litellm-rust/crates/traces/src/insert.rs @@ -49,7 +49,23 @@ pub async fn insert_rows( .map_err(|_| Error::InvalidRow)?; let body = encoder.finish().map_err(|_| Error::InvalidRow)?; let mut url = connection.url().clone(); + let existing_pairs: Vec<(String, String)> = url + .query_pairs() + .filter(|(key, _)| { + !matches!( + key.as_ref(), + "query" + | "async_insert" + | "wait_for_async_insert" + | "input_format_skip_unknown_fields" + | "date_time_input_format" + ) + }) + .map(|(key, value)| (key.into_owned(), value.into_owned())) + .collect(); url.query_pairs_mut() + .clear() + .extend_pairs(existing_pairs) .append_pair( "query", &format!( @@ -59,6 +75,7 @@ pub async fn insert_rows( ) .append_pair("async_insert", "1") .append_pair("wait_for_async_insert", "1") + .append_pair("input_format_skip_unknown_fields", "0") .append_pair("date_time_input_format", "best_effort"); let response = client .post(url) diff --git a/litellm-rust/crates/traces/tests/migrations.rs b/litellm-rust/crates/traces/tests/migrations.rs index 2652a3e3cce5..de296ee2d52e 100644 --- a/litellm-rust/crates/traces/tests/migrations.rs +++ b/litellm-rust/crates/traces/tests/migrations.rs @@ -1,7 +1,10 @@ use std::collections::BTreeMap; use litellm_http::Client; -use litellm_traces::{Connection, encode_rows, ensure_schema, execute_read, schema_statements}; +use litellm_traces::{ + Connection, Error, InsertTable, Parameter, ReadQuery, ensure_schema, execute_named_read, + execute_read, insert_rows, schema_statements, +}; use rstest::rstest; use testcontainers_modules::{ clickhouse::ClickHouse, @@ -40,22 +43,61 @@ async fn schema_supports_span_rollups_and_spend_joins() -> Result<(), Box NativeTraceStorage: ... def ensure_schema(self, trace_retention_days: int, spend_log_retention_days: int) -> Future[None]: ... def insert_rows(self, table: str, rows: Sequence[Mapping[str, JsonValue]]) -> Future[None]: ... - def query(self, sql: str, parameters: Mapping[str, str | int | Sequence[str]]) -> Future[str]: ... + def query(self, name: str, parameters: Mapping[str, str | int | Sequence[str]]) -> Future[str]: ... @final class NativeDiagnosticProcessor: diff --git a/tests/test_litellm_rust/test_traces.py b/tests/test_litellm_rust/test_traces.py index 447ca2ce4bbd..9ca7858ec858 100644 --- a/tests/test_litellm_rust/test_traces.py +++ b/tests/test_litellm_rust/test_traces.py @@ -17,11 +17,11 @@ async def test_trace_reader_projects_connection_and_parameters(recording_server: recording_server.enqueue(ResponseSpec(body={"data": [{"trace_id": "trace-1"}]})) reader_url: Final = recording_server.base_url.replace("http://", "http://reader:p%40ss%2Fword%25@") storage: Final = NativeTraceStorage("trace_test", recording_server.base_url, reader_url + "?database=wrong") - rows: Final = json.loads(await storage.query("SELECT {trace_id:String} AS trace_id", {"trace_id": "trace-1"})) + rows: Final = json.loads(await storage.query("span_detail", {"trace_id": "trace-1", "span_id": "span-1"}))["data"] request: Final = recording_server.requests[0] parameters: Final = parse_qs(urlsplit(request.path).query) assert rows == [{"trace_id": "trace-1"}] - assert request.raw_body == b"SELECT {trace_id:String} AS trace_id" + assert b"WHERE TraceId = {trace_id:String} AND SpanId = {span_id:String}" in request.raw_body assert parameters["database"] == ["trace_test"] assert parameters["param_trace_id"] == ["trace-1"] assert parameters["readonly"] == ["1"] @@ -35,7 +35,16 @@ async def test_trace_reader_rejects_success_status_with_embedded_error(recording recording_server.enqueue(ResponseSpec(body={"data": [], "exception": "query failed"})) storage: Final = NativeTraceStorage("trace_test", recording_server.base_url, recording_server.base_url) with pytest.raises(RuntimeError, match="invalid or failed JSON"): - await storage.query("SELECT 1", {}) + await storage.query("span_detail", {}) + + +@pytest.mark.asyncio +async def test_trace_reader_rejects_raw_sql(recording_server: RecordingServer) -> None: + recording_server.expected_requests = 0 + storage: Final = NativeTraceStorage("trace_test", recording_server.base_url, recording_server.base_url) + with pytest.raises(ValueError, match="invalid trace read query"): + await storage.query("SELECT * FROM otel_traces", {}) + assert recording_server.requests == [] @pytest.mark.asyncio @@ -52,7 +61,9 @@ async def test_schema_binding_rejects_non_positive_retention() -> None: @pytest.mark.asyncio -async def test_schema_setup_uses_writer_credentials_and_rejects_failed_statement(recording_server: RecordingServer) -> None: +async def test_schema_setup_uses_writer_credentials_and_rejects_failed_statement( + recording_server: RecordingServer, +) -> None: recording_server.expected_requests = 2 recording_server.enqueue(ResponseSpec(body="")) recording_server.enqueue(ResponseSpec(status=403, body="denied")) @@ -72,12 +83,17 @@ async def test_schema_setup_uses_writer_credentials_and_rejects_failed_statement @pytest.mark.asyncio async def test_insert_encodes_and_sends_rows(recording_server: RecordingServer) -> None: recording_server.enqueue(ResponseSpec(body="")) - storage: Final = NativeTraceStorage("trace_test", recording_server.base_url) + storage: Final = NativeTraceStorage( + "trace_test", recording_server.base_url + "?input_format_skip_unknown_fields=1" + ) await storage.insert_rows("otel_traces", [{"Timestamp": 1_234_567_890, "Input": "hello"}]) request: Final = recording_server.requests[0] assert json.loads(gzip.decompress(request.raw_body)) == { "Input": "hello", "Timestamp": "1970-01-01T00:00:01.23456789Z", } - assert parse_qs(urlsplit(request.path).query)["query"] == ["INSERT INTO `trace_test`.otel_traces FORMAT JSONEachRow"] + assert parse_qs(urlsplit(request.path).query)["query"] == [ + "INSERT INTO `trace_test`.otel_traces FORMAT JSONEachRow" + ] + assert parse_qs(urlsplit(request.path).query)["input_format_skip_unknown_fields"] == ["0"] assert request.headers["content-encoding"] == "gzip" From 1fd9e84d233571306dffb022f3e36e4392f7b505 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Wed, 30 Sep 2026 10:17:01 -0700 Subject: [PATCH 09/12] fix(tracing): preserve UI lint headroom after spend enrichment --- .../view_logs/TraceView/TraceDrawer.tsx | 5 ++- .../view_logs/TraceView/traceUtils.test.ts | 45 ++++++++++++------- .../view_logs/TraceView/traceUtils.ts | 8 +++- 3 files changed, 38 insertions(+), 20 deletions(-) diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/TraceDrawer.tsx b/ui/litellm-dashboard/src/components/view_logs/TraceView/TraceDrawer.tsx index 7346f74ca4be..3ee3cd5a891f 100644 --- a/ui/litellm-dashboard/src/components/view_logs/TraceView/TraceDrawer.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/TraceDrawer.tsx @@ -302,12 +302,13 @@ export function TraceDrawer({ onClose, onOpenRequestLog, }: TraceDrawerProps) { - const traceQuery = useQuery({ + const traceQueryOptions = { queryKey: ["agentTrace", traceId, accessToken], queryFn: () => agentTraceCall(accessToken, traceId as string), enabled: open && traceId !== null, staleTime: 30_000, - }); + }; + const traceQuery = useQuery(traceQueryOptions); const trace = traceQuery.data; return ( diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/traceUtils.test.ts b/ui/litellm-dashboard/src/components/view_logs/TraceView/traceUtils.test.ts index 75489d69b378..26386caf0a61 100644 --- a/ui/litellm-dashboard/src/components/view_logs/TraceView/traceUtils.test.ts +++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/traceUtils.test.ts @@ -92,13 +92,17 @@ describe("formatting", () => { describe("buildVisibleTree", () => { it("hides framework spans and re-parents their children to the nearest visible ancestor", () => { - const spans = [ - span({ span_id: "root", type: "agent" }), - span({ span_id: "mw", parent_span_id: "root", type: "framework", name: "X.wrap_model_call" }), - span({ span_id: "model", parent_span_id: "mw", type: "chain", name: "model" }), - span({ span_id: "llm", parent_span_id: "model", type: "llm", start_offset_ms: 5 }), - span({ span_id: "step", parent_span_id: "root", type: "chain", name: "planner", start_offset_ms: 1 }), - ]; + const middleware = { span_id: "mw", parent_span_id: "root", type: "framework", name: "X.wrap_model_call" } as const; + const model = { span_id: "model", parent_span_id: "mw", type: "chain", name: "model" } as const; + const llm = { span_id: "llm", parent_span_id: "model", type: "llm", start_offset_ms: 5 } as const; + const step = { + span_id: "step", + parent_span_id: "root", + type: "chain", + name: "planner", + start_offset_ms: 1, + } as const; + const spans = [span({ span_id: "root", type: "agent" }), span(middleware), span(model), span(llm), span(step)]; const compact = buildVisibleTree(spans, false); expect(compact.visibleCount).toBe(3); expect(compact.children.get(ROOT_KEY)?.map((s) => s.span_id)).toEqual(["root"]); @@ -153,18 +157,20 @@ describe("groupSiblingAgents", () => { it("leaves groups of 10 or fewer alone", () => { const parent = span({ span_id: "p", type: "agent" }); - const kids = Array.from({ length: 10 }, (_, i) => - span({ span_id: `k${i}`, parent_span_id: "p", type: "agent", name: "worker" }), - ); + const kids = Array.from({ length: 10 }, (_, i) => { + const props = { span_id: `k${i}`, parent_span_id: "p", type: "agent", name: "worker" } as const; + return span(props); + }); const items = groupSiblingAgents("p", kids, subtreeStats([parent, ...kids])); expect(items.every((i) => i.kind === "span")).toBe(true); }); it("does not show a zero cost when grouped spans have no spend rows", () => { const parent = span({ span_id: "p", type: "agent" }); - const children = Array.from({ length: 11 }, (_, index) => - span({ span_id: `child-${index}`, parent_span_id: "p", type: "agent", name: "worker" }), - ); + const children = Array.from({ length: 11 }, (_, index) => { + const props = { span_id: `child-${index}`, parent_span_id: "p", type: "agent", name: "worker" } as const; + return span(props); + }); const items = groupSiblingAgents("p", children, subtreeStats([parent, ...children])); const group = items.find((item) => item.kind === "group"); if (group?.kind !== "group") throw new Error("expected a group"); @@ -184,9 +190,16 @@ describe("flattenTree + revealSpan", () => { it("pages group invocations 20 at a time with a 'more' row until all are shown", () => { const parent = span({ span_id: "p", type: "agent" }); - const kids = Array.from({ length: 45 }, (_, i) => - span({ span_id: `k${i}`, parent_span_id: "p", type: "agent", name: "worker", start_offset_ms: i }), - ); + const kids = Array.from({ length: 45 }, (_, i) => { + const props = { + span_id: `k${i}`, + parent_span_id: "p", + type: "agent", + name: "worker", + start_offset_ms: i, + } as const; + return span(props); + }); const all = [parent, ...kids]; const tree = buildVisibleTree(all, false).children; const key = "p::worker"; diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/traceUtils.ts b/ui/litellm-dashboard/src/components/view_logs/TraceView/traceUtils.ts index 98a38fc8f52c..1b5ada2d7ea1 100644 --- a/ui/litellm-dashboard/src/components/view_logs/TraceView/traceUtils.ts +++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/traceUtils.ts @@ -33,8 +33,12 @@ export const agentBadgeLabel = (summary: Pick - span.type === "llm" ? span.litellm?.model_group || span.model || span.litellm?.model || span.name : span.name; +export const spanLabel = (span: Span): string => { + if (span.type !== "llm") return span.name; + if (span.litellm?.model_group) return span.litellm.model_group; + if (span.model) return span.model; + return span.litellm?.model || span.name; +}; export const spanSpend = (span: Span): number => span.litellm?.spend ?? 0; From f099316e1bbe3ecd62333eb0511d1b3400f598d5 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Wed, 30 Sep 2026 11:13:01 -0700 Subject: [PATCH 10/12] feat: add local tracing stack and shared query registry --- docker/docker-compose.tracing.yml | 72 +++++++++++++++++++ docker/tracing-config.yaml | 14 ++++ .../traces/{src => query}/list_traces.sql | 0 .../traces/{src => query}/span_detail.sql | 0 .../traces/{src => query}/trace_spans.sql | 0 litellm-rust/crates/traces/src/sql.rs | 44 ++++++------ litellm/proxy/proxy_server.py | 9 ++- litellm/proxy/tracing_endpoints.py | 5 +- tests/test_litellm/proxy/test_proxy_server.py | 35 +++++++++ .../proxy/test_tracing_endpoints.py | 10 ++- .../TraceView/AgentTracesSection.test.tsx | 11 +++ .../view_logs/TraceView/TracingSetupCard.tsx | 4 +- .../view_logs/TraceView/useAgentTraces.ts | 20 +++--- 13 files changed, 191 insertions(+), 33 deletions(-) create mode 100644 docker/docker-compose.tracing.yml create mode 100644 docker/tracing-config.yaml rename litellm-rust/crates/traces/{src => query}/list_traces.sql (100%) rename litellm-rust/crates/traces/{src => query}/span_detail.sql (100%) rename litellm-rust/crates/traces/{src => query}/trace_spans.sql (100%) diff --git a/docker/docker-compose.tracing.yml b/docker/docker-compose.tracing.yml new file mode 100644 index 000000000000..1536c6566b70 --- /dev/null +++ b/docker/docker-compose.tracing.yml @@ -0,0 +1,72 @@ +name: litellm-tracing + +services: + litellm: + build: + context: .. + target: runtime + command: ["--config", "/app/tracing-config.yaml", "--port", "4000"] + environment: + LITELLM_MASTER_KEY: test + LITELLM_SALT_KEY: sk-local-tracing-salt-key + DATABASE_URL: postgresql://litellm:litellm@db:5432/litellm + STORE_MODEL_IN_DB: "True" + CLICKHOUSE_URL: http://default:local-tracing-write@clickhouse:8123 + CLICKHOUSE_READER_URL: http://litellm_traces_reader:local-tracing-read@clickhouse:8123 + CLICKHOUSE_DATABASE: default + OPENAI_API_KEY: ${OPENAI_API_KEY:-} + volumes: + - ./tracing-config.yaml:/app/tracing-config.yaml:ro + ports: + - "127.0.0.1:4002:4000" + depends_on: + db: + condition: service_healthy + clickhouse: + condition: service_healthy + healthcheck: + test: + - CMD-SHELL + - python3 -c "import urllib.request; urllib.request.urlopen('http://localhost:4000/health/liveliness')" + interval: 10s + timeout: 5s + retries: 12 + start_period: 30s + + db: + image: postgres:16 + environment: + POSTGRES_DB: litellm + POSTGRES_USER: litellm + POSTGRES_PASSWORD: litellm + volumes: + - postgres_data:/var/lib/postgresql/data + ports: + - "127.0.0.1:5433:5432" + healthcheck: + test: ["CMD-SHELL", "pg_isready -U litellm -d litellm"] + interval: 5s + timeout: 5s + retries: 10 + + clickhouse: + image: clickhouse/clickhouse-server:26.9.6.6 + environment: + CLICKHOUSE_USER: default + CLICKHOUSE_PASSWORD: local-tracing-write + CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: "1" + LITELLM_TRACES_READER_PASSWORD: local-tracing-read + volumes: + - clickhouse_data:/var/lib/clickhouse + - ./../litellm-rust/crates/traces/config/reader.xml:/etc/clickhouse-server/users.d/litellm-reader.xml:ro + ports: + - "127.0.0.1:18123:8123" + healthcheck: + test: ["CMD", "clickhouse-client", "--user", "default", "--password", "local-tracing-write", "--query", "SELECT 1"] + interval: 5s + timeout: 5s + retries: 20 + +volumes: + postgres_data: + clickhouse_data: diff --git a/docker/tracing-config.yaml b/docker/tracing-config.yaml new file mode 100644 index 000000000000..238a1e16bbc3 --- /dev/null +++ b/docker/tracing-config.yaml @@ -0,0 +1,14 @@ +model_list: + - model_name: gpt-5.6 + litellm_params: + model: openai/gpt-5.6 + api_key: os.environ/OPENAI_API_KEY + +general_settings: + master_key: os.environ/LITELLM_MASTER_KEY + tracing: + store: clickhouse + +litellm_settings: + success_callback: + - clickhouse diff --git a/litellm-rust/crates/traces/src/list_traces.sql b/litellm-rust/crates/traces/query/list_traces.sql similarity index 100% rename from litellm-rust/crates/traces/src/list_traces.sql rename to litellm-rust/crates/traces/query/list_traces.sql diff --git a/litellm-rust/crates/traces/src/span_detail.sql b/litellm-rust/crates/traces/query/span_detail.sql similarity index 100% rename from litellm-rust/crates/traces/src/span_detail.sql rename to litellm-rust/crates/traces/query/span_detail.sql diff --git a/litellm-rust/crates/traces/src/trace_spans.sql b/litellm-rust/crates/traces/query/trace_spans.sql similarity index 100% rename from litellm-rust/crates/traces/src/trace_spans.sql rename to litellm-rust/crates/traces/query/trace_spans.sql diff --git a/litellm-rust/crates/traces/src/sql.rs b/litellm-rust/crates/traces/src/sql.rs index ff61d377ad8c..f0eae9dda285 100644 --- a/litellm-rust/crates/traces/src/sql.rs +++ b/litellm-rust/crates/traces/src/sql.rs @@ -8,29 +8,33 @@ use crate::{Connection, Error}; const MAX_RESPONSE_BYTES: usize = 4 * 1024 * 1024; -pub enum ReadQuery { - ListTraces, - TraceSpans, - SpanDetail, -} - -impl ReadQuery { - pub fn parse(value: &str) -> Result { - match value { - "list_traces" => Ok(Self::ListTraces), - "trace_spans" => Ok(Self::TraceSpans), - "span_detail" => Ok(Self::SpanDetail), - _ => Err(Error::InvalidQuery), +macro_rules! read_queries { + ($($variant:ident => $name:literal => $file:literal),+ $(,)?) => { + pub enum ReadQuery { + $($variant),+ } - } - fn sql(&self) -> &'static str { - match self { - Self::ListTraces => include_str!("list_traces.sql"), - Self::TraceSpans => include_str!("trace_spans.sql"), - Self::SpanDetail => include_str!("span_detail.sql"), + impl ReadQuery { + pub fn parse(value: &str) -> Result { + match value { + $($name => Ok(Self::$variant),)+ + _ => Err(Error::InvalidQuery), + } + } + + fn sql(&self) -> &'static str { + match self { + $(Self::$variant => include_str!(concat!("../query/", $file)),)+ + } + } } - } + }; +} + +read_queries! { + ListTraces => "list_traces" => "list_traces.sql", + TraceSpans => "trace_spans" => "trace_spans.sql", + SpanDetail => "span_detail" => "span_detail.sql", } #[derive(Debug, Deserialize)] diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 7f2742fcffc2..eb3db04ae27c 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -11324,10 +11324,17 @@ async def _init_tracing(cls, general_settings: dict) -> None: store: clickhouse # CLICKHOUSE_URL / _USER / _PASSWORD / _DATABASE """ settings = general_settings.get("tracing") or {} + tracing_endpoints.receiver = None if settings.get("store") != "clickhouse": return + missing: Final = tuple(name for name in ("CLICKHOUSE_URL", "CLICKHOUSE_READER_URL") if not os.getenv(name)) + if missing: + raise ValueError(f"Agent tracing requires {', '.join(missing)} when tracing.store is clickhouse") tracing = TraceReceiver.from_env() - await tracing.start() + try: + await tracing.start() + except Exception as exc: + raise RuntimeError("Agent tracing could not initialize the ClickHouse schema") from exc tracing_endpoints.receiver = tracing verbose_proxy_logger.info("Agent tracing enabled (store=clickhouse)") diff --git a/litellm/proxy/tracing_endpoints.py b/litellm/proxy/tracing_endpoints.py index 83fc4c12e31a..1f0bd07097f5 100644 --- a/litellm/proxy/tracing_endpoints.py +++ b/litellm/proxy/tracing_endpoints.py @@ -35,7 +35,10 @@ def get_receiver() -> TraceReceiver: if receiver is None: raise HTTPException( status_code=501, - detail="Agent tracing is not enabled. Set `tracing:` in general_settings and CLICKHOUSE_URL.", + detail=( + "Agent tracing is not enabled. Set general_settings.tracing.store to clickhouse " + "and configure CLICKHOUSE_URL and CLICKHOUSE_READER_URL on the proxy." + ), ) return receiver diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index 23d319159cac..81bc9f3838df 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -46,6 +46,41 @@ from litellm.proxy.proxy_server import app, initialize, openai_exception_handler from litellm.utils import _invalidate_model_cost_lowercase_map + +@pytest.mark.asyncio +async def test_tracing_without_clickhouse_disables_receiver(monkeypatch): + monkeypatch.setattr(proxy_server_module.tracing_endpoints, "receiver", object()) + + await proxy_server_module.ProxyStartupEvent._init_tracing({}) + + assert proxy_server_module.tracing_endpoints.receiver is None + + +@pytest.mark.asyncio +async def test_tracing_requires_both_clickhouse_urls(monkeypatch): + monkeypatch.delenv("CLICKHOUSE_URL", raising=False) + monkeypatch.delenv("CLICKHOUSE_READER_URL", raising=False) + + with pytest.raises(ValueError, match="CLICKHOUSE_URL, CLICKHOUSE_READER_URL"): + await proxy_server_module.ProxyStartupEvent._init_tracing({"tracing": {"store": "clickhouse"}}) + + assert proxy_server_module.tracing_endpoints.receiver is None + + +@pytest.mark.asyncio +async def test_tracing_fails_startup_when_clickhouse_schema_cannot_initialize(monkeypatch): + receiver = MagicMock() + receiver.start = AsyncMock(side_effect=RuntimeError("connection failed")) + monkeypatch.setattr(proxy_server_module, "TraceReceiver", MagicMock(from_env=MagicMock(return_value=receiver))) + monkeypatch.setenv("CLICKHOUSE_URL", "http://clickhouse:8123") + monkeypatch.setenv("CLICKHOUSE_READER_URL", "http://reader:8123") + + with pytest.raises(RuntimeError, match="could not initialize the ClickHouse schema"): + await proxy_server_module.ProxyStartupEvent._init_tracing({"tracing": {"store": "clickhouse"}}) + + assert proxy_server_module.tracing_endpoints.receiver is None + + example_embedding_result = { "object": "list", "data": [ diff --git a/tests/test_litellm/proxy/test_tracing_endpoints.py b/tests/test_litellm/proxy/test_tracing_endpoints.py index 7b46beec2821..24e554bdd99f 100644 --- a/tests/test_litellm/proxy/test_tracing_endpoints.py +++ b/tests/test_litellm/proxy/test_tracing_endpoints.py @@ -77,8 +77,14 @@ def client() -> TestClient: def test_501_when_tracing_not_enabled(client, monkeypatch): monkeypatch.setattr(tracing_endpoints, "receiver", None) - assert client.post("/v1/traces", content=b"").status_code == 501 - assert client.get("/v1/traces").status_code == 501 + for response in ( + client.post("/v1/traces", content=b""), + client.get("/v1/traces"), + client.get("/v1/traces/trace-id"), + client.get("/v1/traces/trace-id/spans/span-id"), + ): + assert response.status_code == 501 + assert "CLICKHOUSE_URL and CLICKHOUSE_READER_URL" in response.json()["detail"] def test_post_protobuf_returns_empty_protobuf(client, receiver): diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesSection.test.tsx b/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesSection.test.tsx index c192cd057877..b0fc10110e4c 100644 --- a/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesSection.test.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesSection.test.tsx @@ -45,11 +45,22 @@ describe("AgentTracesSection", () => { const card = await screen.findByTestId("tracing-setup-card"); expect(card).toHaveTextContent("Agent tracing is not enabled"); expect(card).toHaveTextContent("store: clickhouse"); + expect(card).toHaveTextContent("CLICKHOUSE_URL="); + expect(card).toHaveTextContent("CLICKHOUSE_READER_URL="); expect(card).toHaveTextContent("LANGSMITH_TRACING_MODE=otel"); expect(card).toHaveTextContent('OTEL_EXPORTER_OTLP_HEADERS="Authorization=Bearer "'); expect(agentTraceListCall).toHaveBeenCalledTimes(1); }); + it("shows the tracing setup when the proxy has no traces route", async () => { + vi.mocked(agentTraceListCall).mockRejectedValue(new ApiError("Not Found", 404, { detail: "Not Found" })); + renderSection("traces"); + + const card = await screen.findByTestId("tracing-setup-card"); + expect(card).toHaveTextContent("This proxy does not expose /v1/traces"); + expect(screen.queryByRole("table", { name: "Agent traces" })).not.toBeInTheDocument(); + }); + it("renders nothing in All when tracing is off, so request logs stay front and centre", async () => { vi.mocked(agentTraceListCall).mockRejectedValue(new ApiError("off", 501, { detail: "off" })); renderSection("all"); diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/TracingSetupCard.tsx b/ui/litellm-dashboard/src/components/view_logs/TraceView/TracingSetupCard.tsx index 1c8b4a646f8f..7169eac10386 100644 --- a/ui/litellm-dashboard/src/components/view_logs/TraceView/TracingSetupCard.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/TracingSetupCard.tsx @@ -7,7 +7,9 @@ import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert"; export const TRACING_SETUP_SNIPPET = `# proxy config general_settings: tracing: - store: clickhouse # + CLICKHOUSE_URL env var + store: clickhouse +export CLICKHOUSE_URL= +export CLICKHOUSE_READER_URL= # your agent app export LANGSMITH_TRACING=true LANGSMITH_TRACING_MODE=otel export OTEL_EXPORTER_OTLP_ENDPOINT= diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/useAgentTraces.ts b/ui/litellm-dashboard/src/components/view_logs/TraceView/useAgentTraces.ts index c0be361c02e3..322db6472504 100644 --- a/ui/litellm-dashboard/src/components/view_logs/TraceView/useAgentTraces.ts +++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/useAgentTraces.ts @@ -8,10 +8,11 @@ import { agentTraceListCall } from "../../networking"; import { LIVE_TAIL_INTERVAL_MS } from "../log_filter_logic"; import type { TracePage, TraceSummary } from "./traceTypes"; -export const TRACING_NOT_ENABLED_STATUS = 501; +const TRACING_NOT_ENABLED_STATUS = 501; +const TRACING_ROUTE_MISSING_STATUS = 404; -export const isTracingNotEnabled = (error: unknown): error is ApiError => - error instanceof ApiError && error.status === TRACING_NOT_ENABLED_STATUS; +const isTracingUnavailable = (error: unknown): error is ApiError => + error instanceof ApiError && (error.status === TRACING_NOT_ENABLED_STATUS || error.status === TRACING_ROUTE_MISSING_STATUS); interface UseAgentTracesOptions { accessToken: string; @@ -26,7 +27,6 @@ export interface AgentTracesResult { traces: TraceSummary[]; isLoading: boolean; isFetching: boolean; - /** Set when the proxy answered 501: tracing isn't configured. */ notEnabledDetail: string | null; error: Error | null; hasMore: boolean; @@ -58,19 +58,23 @@ export function useAgentTraces({ initialPageParam: null, getNextPageParam: (lastPage) => lastPage.next_cursor ?? undefined, enabled, - retry: (failureCount, error) => !isTracingNotEnabled(error) && failureCount < 1, - refetchInterval: (q) => (isLiveTail && !isTracingNotEnabled(q.state.error) ? LIVE_TAIL_INTERVAL_MS : false), + retry: (failureCount, error) => !isTracingUnavailable(error) && failureCount < 1, + refetchInterval: (q) => (isLiveTail && !isTracingUnavailable(q.state.error) ? LIVE_TAIL_INTERVAL_MS : false), refetchIntervalInBackground: false, }); const traces = useMemo(() => query.data?.pages.flatMap((page) => page.data) ?? [], [query.data]); - const notEnabled = isTracingNotEnabled(query.error); + const notEnabled = isTracingUnavailable(query.error); + const unavailableDetail = + query.error instanceof ApiError && query.error.status === TRACING_ROUTE_MISSING_STATUS + ? "This proxy does not expose /v1/traces. Update the proxy to enable agent tracing." + : query.error?.message || "Agent tracing is not enabled"; return { traces, isLoading: query.isLoading, isFetching: query.isFetching, - notEnabledDetail: notEnabled ? query.error?.message || "Agent tracing is not enabled" : null, + notEnabledDetail: notEnabled ? unavailableDetail : null, error: notEnabled ? null : query.error, hasMore: query.hasNextPage, loadMore: () => void query.fetchNextPage(), From 914450a590fdb09ef95ea35681aaac12c921f301 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Wed, 30 Sep 2026 11:33:11 -0700 Subject: [PATCH 11/12] refactor(tracing): use one ClickHouse URL for reads and writes --- docker/docker-compose.tracing.yml | 3 --- .../crates/python-bridge/src/routes/traces.rs | 14 ++++--------- litellm/proxy/proxy_server.py | 5 ++--- litellm/proxy/tracing_endpoints.py | 2 +- litellm/rust_bridge/_native.pyi | 2 +- litellm/rust_bridge/traces.py | 6 +++--- litellm/tracing/receiver.py | 1 - tests/test_litellm/proxy/test_proxy_server.py | 20 +++++++++++++++---- .../proxy/test_tracing_endpoints.py | 2 +- tests/test_litellm_rust/test_traces.py | 10 +++++----- .../TraceView/AgentTracesSection.test.tsx | 3 +-- .../view_logs/TraceView/TracingSetupCard.tsx | 3 +-- 12 files changed, 35 insertions(+), 36 deletions(-) diff --git a/docker/docker-compose.tracing.yml b/docker/docker-compose.tracing.yml index 1536c6566b70..1620d52d7129 100644 --- a/docker/docker-compose.tracing.yml +++ b/docker/docker-compose.tracing.yml @@ -12,7 +12,6 @@ services: DATABASE_URL: postgresql://litellm:litellm@db:5432/litellm STORE_MODEL_IN_DB: "True" CLICKHOUSE_URL: http://default:local-tracing-write@clickhouse:8123 - CLICKHOUSE_READER_URL: http://litellm_traces_reader:local-tracing-read@clickhouse:8123 CLICKHOUSE_DATABASE: default OPENAI_API_KEY: ${OPENAI_API_KEY:-} volumes: @@ -55,10 +54,8 @@ services: CLICKHOUSE_USER: default CLICKHOUSE_PASSWORD: local-tracing-write CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT: "1" - LITELLM_TRACES_READER_PASSWORD: local-tracing-read volumes: - clickhouse_data:/var/lib/clickhouse - - ./../litellm-rust/crates/traces/config/reader.xml:/etc/clickhouse-server/users.d/litellm-reader.xml:ro ports: - "127.0.0.1:18123:8123" healthcheck: diff --git a/litellm-rust/crates/python-bridge/src/routes/traces.rs b/litellm-rust/crates/python-bridge/src/routes/traces.rs index c3ad96523f76..dd15310ff1b2 100644 --- a/litellm-rust/crates/python-bridge/src/routes/traces.rs +++ b/litellm-rust/crates/python-bridge/src/routes/traces.rs @@ -29,21 +29,17 @@ fn map_error(error: Error) -> PyErr { pub struct NativeTraceStorage { database: String, writer: Connection, - reader: Option, + reader: Connection, } #[pymethods] impl NativeTraceStorage { #[new] - #[pyo3(signature = (database, url, reader_url=None))] - fn new(database: String, url: &str, reader_url: Option<&str>) -> PyResult { + fn new(database: String, url: &str) -> PyResult { litellm_traces::schema_statements(&database, 1, 1).map_err(map_error)?; Ok(Self { writer: Connection::writer(url).map_err(map_error)?, - reader: reader_url - .map(|value| Connection::reader(value, &database)) - .transpose() - .map_err(map_error)?, + reader: Connection::reader(url, &database).map_err(map_error)?, database, }) } @@ -104,9 +100,7 @@ impl NativeTraceStorage { >, ) -> PyResult> { let query = ReadQuery::parse(query).map_err(map_error)?; - let connection = self.reader.clone().ok_or_else(|| { - PyRuntimeError::new_err("Trace reads require a separate ClickHouse reader URL") - })?; + let connection = self.reader.clone(); let client = crate::http::host_client(py, ClientVariant::NoRedirect)?; crate::execution::run_async( py, diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index eb3db04ae27c..f212263f0d9c 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -11327,9 +11327,8 @@ async def _init_tracing(cls, general_settings: dict) -> None: tracing_endpoints.receiver = None if settings.get("store") != "clickhouse": return - missing: Final = tuple(name for name in ("CLICKHOUSE_URL", "CLICKHOUSE_READER_URL") if not os.getenv(name)) - if missing: - raise ValueError(f"Agent tracing requires {', '.join(missing)} when tracing.store is clickhouse") + if not os.getenv("CLICKHOUSE_URL"): + raise ValueError("Agent tracing requires CLICKHOUSE_URL when tracing.store is clickhouse") tracing = TraceReceiver.from_env() try: await tracing.start() diff --git a/litellm/proxy/tracing_endpoints.py b/litellm/proxy/tracing_endpoints.py index 1f0bd07097f5..604628d46d7a 100644 --- a/litellm/proxy/tracing_endpoints.py +++ b/litellm/proxy/tracing_endpoints.py @@ -37,7 +37,7 @@ def get_receiver() -> TraceReceiver: status_code=501, detail=( "Agent tracing is not enabled. Set general_settings.tracing.store to clickhouse " - "and configure CLICKHOUSE_URL and CLICKHOUSE_READER_URL on the proxy." + "and configure CLICKHOUSE_URL on the proxy." ), ) return receiver diff --git a/litellm/rust_bridge/_native.pyi b/litellm/rust_bridge/_native.pyi index 591a7c8c4cfe..e00792618d9d 100644 --- a/litellm/rust_bridge/_native.pyi +++ b/litellm/rust_bridge/_native.pyi @@ -37,7 +37,7 @@ def trace_decode_otlp( @final class NativeTraceStorage: - def __new__(cls, database: str, url: str, reader_url: str | None = None) -> NativeTraceStorage: ... + def __new__(cls, database: str, url: str) -> NativeTraceStorage: ... def ensure_schema(self, trace_retention_days: int, spend_log_retention_days: int) -> Future[None]: ... def insert_rows(self, table: str, rows: Sequence[Mapping[str, JsonValue]]) -> Future[None]: ... def query(self, name: str, parameters: Mapping[str, str | int | Sequence[str]]) -> Future[str]: ... diff --git a/litellm/rust_bridge/traces.py b/litellm/rust_bridge/traces.py index 2dc78ad418b8..dfc8533cc499 100644 --- a/litellm/rust_bridge/traces.py +++ b/litellm/rust_bridge/traces.py @@ -31,7 +31,7 @@ class DecodedSpan(TypedDict): class NativeStore(Protocol): - def __init__(self, database: str, url: str, reader_url: str | None = None) -> None: ... + def __init__(self, database: str, url: str) -> None: ... def ensure_schema(self, trace_retention_days: int, spend_log_retention_days: int) -> Awaitable[None]: ... @@ -75,8 +75,8 @@ def decode_otlp( class TraceStorage: - def __init__(self, database: str, url: str, reader_url: str | None = None) -> None: - self._native: Final = _native().NativeTraceStorage(database, url, reader_url) + def __init__(self, database: str, url: str) -> None: + self._native: Final = _native().NativeTraceStorage(database, url) async def ensure_schema(self, trace_retention_days: int, spend_log_retention_days: int) -> None: await self._native.ensure_schema(trace_retention_days, spend_log_retention_days) diff --git a/litellm/tracing/receiver.py b/litellm/tracing/receiver.py index 196860078daf..79e65de13aec 100644 --- a/litellm/tracing/receiver.py +++ b/litellm/tracing/receiver.py @@ -70,7 +70,6 @@ def from_env(cls) -> "TraceReceiver": TraceStorage( database=os.getenv("CLICKHOUSE_DATABASE", "litellm"), url=os.environ["CLICKHOUSE_URL"], - reader_url=os.environ["CLICKHOUSE_READER_URL"], ) ) ) diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index 81bc9f3838df..9466dafbd1e5 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -57,23 +57,35 @@ async def test_tracing_without_clickhouse_disables_receiver(monkeypatch): @pytest.mark.asyncio -async def test_tracing_requires_both_clickhouse_urls(monkeypatch): +async def test_tracing_requires_clickhouse_url(monkeypatch): monkeypatch.delenv("CLICKHOUSE_URL", raising=False) - monkeypatch.delenv("CLICKHOUSE_READER_URL", raising=False) - with pytest.raises(ValueError, match="CLICKHOUSE_URL, CLICKHOUSE_READER_URL"): + with pytest.raises(ValueError, match="CLICKHOUSE_URL"): await proxy_server_module.ProxyStartupEvent._init_tracing({"tracing": {"store": "clickhouse"}}) assert proxy_server_module.tracing_endpoints.receiver is None +@pytest.mark.asyncio +async def test_tracing_starts_with_only_clickhouse_url(monkeypatch): + receiver = MagicMock() + receiver.start = AsyncMock() + monkeypatch.setattr(proxy_server_module, "TraceReceiver", MagicMock(from_env=MagicMock(return_value=receiver))) + monkeypatch.setenv("CLICKHOUSE_URL", "http://clickhouse:8123") + monkeypatch.delenv("CLICKHOUSE_READER_URL", raising=False) + + await proxy_server_module.ProxyStartupEvent._init_tracing({"tracing": {"store": "clickhouse"}}) + + assert proxy_server_module.tracing_endpoints.receiver is receiver + receiver.start.assert_awaited_once() + + @pytest.mark.asyncio async def test_tracing_fails_startup_when_clickhouse_schema_cannot_initialize(monkeypatch): receiver = MagicMock() receiver.start = AsyncMock(side_effect=RuntimeError("connection failed")) monkeypatch.setattr(proxy_server_module, "TraceReceiver", MagicMock(from_env=MagicMock(return_value=receiver))) monkeypatch.setenv("CLICKHOUSE_URL", "http://clickhouse:8123") - monkeypatch.setenv("CLICKHOUSE_READER_URL", "http://reader:8123") with pytest.raises(RuntimeError, match="could not initialize the ClickHouse schema"): await proxy_server_module.ProxyStartupEvent._init_tracing({"tracing": {"store": "clickhouse"}}) diff --git a/tests/test_litellm/proxy/test_tracing_endpoints.py b/tests/test_litellm/proxy/test_tracing_endpoints.py index 24e554bdd99f..434610f9c068 100644 --- a/tests/test_litellm/proxy/test_tracing_endpoints.py +++ b/tests/test_litellm/proxy/test_tracing_endpoints.py @@ -84,7 +84,7 @@ def test_501_when_tracing_not_enabled(client, monkeypatch): client.get("/v1/traces/trace-id/spans/span-id"), ): assert response.status_code == 501 - assert "CLICKHOUSE_URL and CLICKHOUSE_READER_URL" in response.json()["detail"] + assert "CLICKHOUSE_URL" in response.json()["detail"] def test_post_protobuf_returns_empty_protobuf(client, receiver): diff --git a/tests/test_litellm_rust/test_traces.py b/tests/test_litellm_rust/test_traces.py index 9ca7858ec858..a0215dc68272 100644 --- a/tests/test_litellm_rust/test_traces.py +++ b/tests/test_litellm_rust/test_traces.py @@ -15,8 +15,8 @@ @pytest.mark.asyncio async def test_trace_reader_projects_connection_and_parameters(recording_server: RecordingServer) -> None: recording_server.enqueue(ResponseSpec(body={"data": [{"trace_id": "trace-1"}]})) - reader_url: Final = recording_server.base_url.replace("http://", "http://reader:p%40ss%2Fword%25@") - storage: Final = NativeTraceStorage("trace_test", recording_server.base_url, reader_url + "?database=wrong") + url: Final = recording_server.base_url.replace("http://", "http://writer:p%40ss%2Fword%25@") + storage: Final = NativeTraceStorage("trace_test", url + "?database=wrong") rows: Final = json.loads(await storage.query("span_detail", {"trace_id": "trace-1", "span_id": "span-1"}))["data"] request: Final = recording_server.requests[0] parameters: Final = parse_qs(urlsplit(request.path).query) @@ -27,13 +27,13 @@ async def test_trace_reader_projects_connection_and_parameters(recording_server: assert parameters["readonly"] == ["1"] assert "user" not in parameters assert "password" not in parameters - assert request.headers["authorization"] == "Basic " + base64.b64encode(b"reader:p@ss/word%").decode() + assert request.headers["authorization"] == "Basic " + base64.b64encode(b"writer:p@ss/word%").decode() @pytest.mark.asyncio async def test_trace_reader_rejects_success_status_with_embedded_error(recording_server: RecordingServer) -> None: recording_server.enqueue(ResponseSpec(body={"data": [], "exception": "query failed"})) - storage: Final = NativeTraceStorage("trace_test", recording_server.base_url, recording_server.base_url) + storage: Final = NativeTraceStorage("trace_test", recording_server.base_url) with pytest.raises(RuntimeError, match="invalid or failed JSON"): await storage.query("span_detail", {}) @@ -41,7 +41,7 @@ async def test_trace_reader_rejects_success_status_with_embedded_error(recording @pytest.mark.asyncio async def test_trace_reader_rejects_raw_sql(recording_server: RecordingServer) -> None: recording_server.expected_requests = 0 - storage: Final = NativeTraceStorage("trace_test", recording_server.base_url, recording_server.base_url) + storage: Final = NativeTraceStorage("trace_test", recording_server.base_url) with pytest.raises(ValueError, match="invalid trace read query"): await storage.query("SELECT * FROM otel_traces", {}) assert recording_server.requests == [] diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesSection.test.tsx b/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesSection.test.tsx index b0fc10110e4c..a51450c3746e 100644 --- a/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesSection.test.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/AgentTracesSection.test.tsx @@ -45,8 +45,7 @@ describe("AgentTracesSection", () => { const card = await screen.findByTestId("tracing-setup-card"); expect(card).toHaveTextContent("Agent tracing is not enabled"); expect(card).toHaveTextContent("store: clickhouse"); - expect(card).toHaveTextContent("CLICKHOUSE_URL="); - expect(card).toHaveTextContent("CLICKHOUSE_READER_URL="); + expect(card).toHaveTextContent("CLICKHOUSE_URL="); expect(card).toHaveTextContent("LANGSMITH_TRACING_MODE=otel"); expect(card).toHaveTextContent('OTEL_EXPORTER_OTLP_HEADERS="Authorization=Bearer "'); expect(agentTraceListCall).toHaveBeenCalledTimes(1); diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/TracingSetupCard.tsx b/ui/litellm-dashboard/src/components/view_logs/TraceView/TracingSetupCard.tsx index 7169eac10386..7b1fc53066c1 100644 --- a/ui/litellm-dashboard/src/components/view_logs/TraceView/TracingSetupCard.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/TracingSetupCard.tsx @@ -8,8 +8,7 @@ export const TRACING_SETUP_SNIPPET = `# proxy config general_settings: tracing: store: clickhouse -export CLICKHOUSE_URL= -export CLICKHOUSE_READER_URL= +export CLICKHOUSE_URL= # your agent app export LANGSMITH_TRACING=true LANGSMITH_TRACING_MODE=otel export OTEL_EXPORTER_OTLP_ENDPOINT= From 0841d9168fe76d0e8fe59c52c5fbf0df11bb4a5f Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Wed, 30 Sep 2026 18:58:45 +0000 Subject: [PATCH 12/12] fix(tracing): satisfy stack checks Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../clickhouse/clickhouse_spend_logger.py | 35 ++- litellm/rust_bridge/_native.pyi | 2 +- litellm/tracing/store.py | 6 +- litellm/tracing/types.py | 246 +++++++++--------- .../view_logs/TraceView/useAgentTraces.ts | 3 +- 5 files changed, 154 insertions(+), 138 deletions(-) diff --git a/litellm/integrations/clickhouse/clickhouse_spend_logger.py b/litellm/integrations/clickhouse/clickhouse_spend_logger.py index 1d23fe759e03..0ddb91bfaf20 100644 --- a/litellm/integrations/clickhouse/clickhouse_spend_logger.py +++ b/litellm/integrations/clickhouse/clickhouse_spend_logger.py @@ -8,6 +8,7 @@ import json import re from collections.abc import Mapping +from types import MappingProxyType from typing import Any, Final import litellm @@ -55,10 +56,16 @@ def _json(value: object) -> str: return value if isinstance(value, str) else json.dumps(value, default=str) +def _json_mapping(value: Mapping[str, Any]) -> str: + return _json(dict(value)) # mutable-ok: [LIT002] JSON serialization requires a dict + + def _find_traceparent(metadata: Mapping[str, Any], kwargs: Mapping[str, Any]) -> tuple[str, str]: - custom_headers = metadata.get("requester_custom_headers") or {} - proxy_request = (kwargs.get("litellm_params") or {}).get("proxy_server_request") or {} - request_headers = proxy_request.get("headers") or {} + custom_headers = metadata.get("requester_custom_headers") or MappingProxyType({}) + proxy_request = (kwargs.get("litellm_params") or MappingProxyType({})).get( + "proxy_server_request" + ) or MappingProxyType({}) + request_headers = proxy_request.get("headers") or MappingProxyType({}) for headers in (custom_headers, request_headers): for name, value in headers.items(): if str(name).lower() == "traceparent": @@ -68,7 +75,7 @@ def _find_traceparent(metadata: Mapping[str, Any], kwargs: Mapping[str, Any]) -> def _cache_tokens(usage: Mapping[str, Any]) -> tuple[int, int]: """(cache_read, cache_write) from a Usage dict: OpenAI prompt_tokens_details first, Anthropic fields as fallback.""" - details = usage.get("prompt_tokens_details") or {} + details = usage.get("prompt_tokens_details") or MappingProxyType({}) cache_read = _int(details.get("cached_tokens")) or _int(usage.get("cache_read_input_tokens")) cache_write = ( _int(details.get("cache_write_tokens")) @@ -78,9 +85,15 @@ def _cache_tokens(usage: Mapping[str, Any]) -> tuple[int, int]: return cache_read, cache_write +def _request_tags(value: object) -> list[str]: + if not isinstance(value, list): + return [] # mutable-ok: [LIT002] empty spend-log tag payload + return [str(tag) for tag in value] # mutable-ok: [LIT002] SpendLogRecord schema + + def _session_id(payload: StandardLoggingPayload, kwargs: Mapping[str, Any]) -> str: """Mirrors proxy `_get_session_id_for_spend_log`: explicit session id, else the payload trace id.""" - request_metadata = (kwargs.get("litellm_params") or {}).get("metadata") or {} + request_metadata = (kwargs.get("litellm_params") or MappingProxyType({})).get("metadata") or MappingProxyType({}) return str(payload.get("session_id") or request_metadata.get("session_id") or payload.get("trace_id") or "") @@ -90,9 +103,9 @@ def _is_trace_ingest(payload: StandardLoggingPayload) -> bool: def spend_log_row_from_payload(payload: StandardLoggingPayload, kwargs: Mapping[str, Any]) -> SpendLogRecord: - metadata: Mapping[str, Any] = payload.get("metadata") or {} - hidden_params: Mapping[str, Any] = payload.get("hidden_params") or {} - usage: Mapping[str, Any] = metadata.get("usage_object") or hidden_params.get("usage_object") or {} + metadata: Mapping[str, Any] = payload.get("metadata") or MappingProxyType({}) + hidden_params: Mapping[str, Any] = payload.get("hidden_params") or MappingProxyType({}) + usage: Mapping[str, Any] = metadata.get("usage_object") or hidden_params.get("usage_object") or MappingProxyType({}) cache_read_tokens, cache_write_tokens = _cache_tokens(usage) trace_id, span_id = _find_traceparent(metadata, kwargs) request_id = str(payload.get("id") or "") @@ -129,8 +142,8 @@ def spend_log_row_from_payload(payload: StandardLoggingPayload, kwargs: Mapping[ session_id=_session_id(payload, kwargs), trace_id=trace_id, span_id=span_id, - request_tags=[str(tag) for tag in payload.get("request_tags") or []], - metadata=_json(dict(metadata)), + request_tags=_request_tags(payload.get("request_tags")), + metadata=_json_mapping(metadata), messages="" if redact else _json(payload.get("messages")), response="" if redact else _json(payload.get("response")), ) @@ -150,6 +163,6 @@ def _log(self, kwargs: Mapping[str, Any]) -> None: payload = kwargs.get("standard_logging_object") if payload is None or _is_trace_ingest(payload): return - self.enqueue([dict(spend_log_row_from_payload(payload, kwargs))]) + self.enqueue([dict(spend_log_row_from_payload(payload, kwargs))]) # mutable-ok: [LIT002] batch logger API except Exception as e: verbose_logger.exception("ClickHouseSpendLogger: failed to log request: %s", e) diff --git a/litellm/rust_bridge/_native.pyi b/litellm/rust_bridge/_native.pyi index e00792618d9d..dd90a243eeae 100644 --- a/litellm/rust_bridge/_native.pyi +++ b/litellm/rust_bridge/_native.pyi @@ -40,7 +40,7 @@ class NativeTraceStorage: def __new__(cls, database: str, url: str) -> NativeTraceStorage: ... def ensure_schema(self, trace_retention_days: int, spend_log_retention_days: int) -> Future[None]: ... def insert_rows(self, table: str, rows: Sequence[Mapping[str, JsonValue]]) -> Future[None]: ... - def query(self, name: str, parameters: Mapping[str, str | int | Sequence[str]]) -> Future[str]: ... + def query(self, query: str, parameters: Mapping[str, str | int | Sequence[str]]) -> Future[str]: ... @final class NativeDiagnosticProcessor: diff --git a/litellm/tracing/store.py b/litellm/tracing/store.py index c443f54b726b..6bc62cb42922 100644 --- a/litellm/tracing/store.py +++ b/litellm/tracing/store.py @@ -8,6 +8,7 @@ import base64 import json from datetime import datetime, timezone +from types import MappingProxyType from typing import Any, Final from litellm.constants import AGENT_TRACING_LIST_PAGE_SIZE @@ -35,6 +36,7 @@ # The MV writes one partial row per insert, so root fields come from the partial that saw the root span. # agent_traces has no ApiKeyHash, so key-scoped (team-less) reads filter trace ids through otel_traces. + def encode_cursor(start_ms: int, trace_id: str) -> str: return base64.urlsafe_b64encode(json.dumps([start_ms, trace_id]).encode()).decode() @@ -229,11 +231,11 @@ async def list_traces( return TracePage(data=[trace_summary_from_row(r) for r in rows], next_cursor=next_cursor) async def get_trace(self, trace_id: str, scope: TraceScope) -> Trace | None: - rows = await self.storage.trace_spans({**scope, "trace_id": trace_id}) + rows = await self.storage.trace_spans(MappingProxyType({**scope, "trace_id": trace_id})) return trace_from_rows(trace_id, rows) async def get_span(self, trace_id: str, span_id: str, scope: TraceScope) -> SpanDetail | None: - rows = await self.storage.span_detail({**scope, "trace_id": trace_id, "span_id": span_id}) + rows = await self.storage.span_detail(MappingProxyType({**scope, "trace_id": trace_id, "span_id": span_id})) if not rows: return None return SpanDetail( diff --git a/litellm/tracing/types.py b/litellm/tracing/types.py index b5bd26f4f963..fa46913bc626 100644 --- a/litellm/tracing/types.py +++ b/litellm/tracing/types.py @@ -15,7 +15,7 @@ from typing import Literal -from typing_extensions import TypedDict +from typing_extensions import ReadOnly, TypedDict SpanType = Literal["agent", "llm", "tool", "chain", "framework"] SpanStatus = Literal["ok", "error", "unset"] @@ -24,160 +24,160 @@ class LiteLLMRequest(TypedDict): """The LiteLLM side of an LLM span: joined from spend logs by response id.""" - request_id: str - model: str - model_group: str - provider: str - key_alias: str - team_alias: str - spend: float - prompt_tokens: int - completion_tokens: int - cache_read_tokens: int - cache_write_tokens: int - latency_ms: int - ttft_ms: int | None - status: str + request_id: ReadOnly[str] + model: ReadOnly[str] + model_group: ReadOnly[str] + provider: ReadOnly[str] + key_alias: ReadOnly[str] + team_alias: ReadOnly[str] + spend: ReadOnly[float] + prompt_tokens: ReadOnly[int] + completion_tokens: ReadOnly[int] + cache_read_tokens: ReadOnly[int] + cache_write_tokens: ReadOnly[int] + latency_ms: ReadOnly[int] + ttft_ms: ReadOnly[int | None] + status: ReadOnly[str] class Span(TypedDict): - span_id: str - parent_span_id: str | None - name: str - type: SpanType - agent: str # the agent this span runs inside, e.g. "researcher" - start_offset_ms: float # relative to trace start - duration_ms: float - status: SpanStatus - error: str | None # exception message when status == "error" - input_preview: str - model: str | None - input_tokens: int - output_tokens: int - litellm: LiteLLMRequest | None + span_id: ReadOnly[str] + parent_span_id: ReadOnly[str | None] + name: ReadOnly[str] + type: ReadOnly[SpanType] + agent: ReadOnly[str] # the agent this span runs inside, e.g. "researcher" + start_offset_ms: ReadOnly[float] # relative to trace start + duration_ms: ReadOnly[float] + status: ReadOnly[SpanStatus] + error: ReadOnly[str | None] # exception message when status == "error" + input_preview: ReadOnly[str] + model: ReadOnly[str | None] + input_tokens: ReadOnly[int] + output_tokens: ReadOnly[int] + litellm: ReadOnly[LiteLLMRequest | None] class AgentNode(TypedDict): """One distinct agent in a trace. 200 invocations of `researcher` = one node.""" - name: str - parent_agent: str | None - invocations: int - llm_calls: int - tool_calls: int - spend: float | None - duration_ms: float + name: ReadOnly[str] + parent_agent: ReadOnly[str | None] + invocations: ReadOnly[int] + llm_calls: ReadOnly[int] + tool_calls: ReadOnly[int] + spend: ReadOnly[float | None] + duration_ms: ReadOnly[float] class TraceSummary(TypedDict): - trace_id: str - name: str - service: str - input_preview: str - start_time: str # ISO 8601 - duration_ms: float - status: SpanStatus - span_count: int - agent_count: int # distinct agent names (researcher x200 counts once) - agent_invocations: int # agent spans (researcher x200 counts 200) - llm_calls: int - tool_calls: int - error_count: int # spans with an error status; > 0 means the run shows as failed - input_tokens: int - output_tokens: int - spend: float | None - models: list[str] + trace_id: ReadOnly[str] + name: ReadOnly[str] + service: ReadOnly[str] + input_preview: ReadOnly[str] + start_time: ReadOnly[str] # ISO 8601 + duration_ms: ReadOnly[float] + status: ReadOnly[SpanStatus] + span_count: ReadOnly[int] + agent_count: ReadOnly[int] # distinct agent names (researcher x200 counts once) + agent_invocations: ReadOnly[int] # agent spans (researcher x200 counts 200) + llm_calls: ReadOnly[int] + tool_calls: ReadOnly[int] + error_count: ReadOnly[int] # spans with an error status; > 0 means the run shows as failed + input_tokens: ReadOnly[int] + output_tokens: ReadOnly[int] + spend: ReadOnly[float | None] + models: ReadOnly[list[str]] class Trace(TypedDict): - summary: TraceSummary - agents: list[AgentNode] - spans: list[Span] + summary: ReadOnly[TraceSummary] + agents: ReadOnly[list[AgentNode]] + spans: ReadOnly[list[Span]] class TracePage(TypedDict): - data: list[TraceSummary] - next_cursor: str | None + data: ReadOnly[list[TraceSummary]] + next_cursor: ReadOnly[str | None] class SpanDetail(TypedDict): - span_id: str - input: str - output: str - attributes: dict[str, str] + span_id: ReadOnly[str] + input: ReadOnly[str] + output: ReadOnly[str] + attributes: ReadOnly[dict[str, str]] class TraceScope(TypedDict): """Who is asking. Empty team_ids = all teams (admins only).""" - team_ids: list[str] - api_key_hash: str + team_ids: ReadOnly[list[str]] + api_key_hash: ReadOnly[str] class SpanRow(TypedDict): """One stored span (ClickHouse `otel_traces` row). Produced by `litellm.tracing.decode`.""" - Timestamp: int # unix ns - TraceId: str - SpanId: str - ParentSpanId: str - TraceState: str - SpanName: str - SpanKind: str - ServiceName: str - ResourceAttributes: dict[str, str] - ScopeName: str - ScopeVersion: str - SpanAttributes: dict[str, str] - Duration: int # ns - StatusCode: str - StatusMessage: str - TeamId: str - ApiKeyHash: str - ObservationType: SpanType - AgentName: str - LiteLLMRequestId: str - Model: str - InputTokens: int - OutputTokens: int - Input: str - Output: str + Timestamp: ReadOnly[int] # unix ns + TraceId: ReadOnly[str] + SpanId: ReadOnly[str] + ParentSpanId: ReadOnly[str] + TraceState: ReadOnly[str] + SpanName: ReadOnly[str] + SpanKind: ReadOnly[str] + ServiceName: ReadOnly[str] + ResourceAttributes: ReadOnly[dict[str, str]] + ScopeName: ReadOnly[str] + ScopeVersion: ReadOnly[str] + SpanAttributes: ReadOnly[dict[str, str]] + Duration: ReadOnly[int] # ns + StatusCode: ReadOnly[str] + StatusMessage: ReadOnly[str] + TeamId: ReadOnly[str] + ApiKeyHash: ReadOnly[str] + ObservationType: ReadOnly[SpanType] + AgentName: ReadOnly[str] + LiteLLMRequestId: ReadOnly[str] + Model: ReadOnly[str] + InputTokens: ReadOnly[int] + OutputTokens: ReadOnly[int] + Input: ReadOnly[str] + Output: ReadOnly[str] class SpendLogRecord(TypedDict): """One LiteLLM request, as written by the `clickhouse` logging callback.""" - request_id: str - response_id: str - call_type: str - api_key: str - key_alias: str - team_id: str - team_alias: str - organization_id: str - user: str - end_user: str - model: str - model_group: str - model_id: str - custom_llm_provider: str - api_base: str - spend: float - prompt_tokens: int - completion_tokens: int - total_tokens: int - cache_read_tokens: int - cache_write_tokens: int - start_time: int # unix ms - end_time: int # unix ms - completion_start_time: int | None - status: str - error_str: str - cache_hit: bool - session_id: str - trace_id: str # from an incoming W3C traceparent, if any - span_id: str - request_tags: list[str] - metadata: str - messages: str - response: str + request_id: ReadOnly[str] + response_id: ReadOnly[str] + call_type: ReadOnly[str] + api_key: ReadOnly[str] + key_alias: ReadOnly[str] + team_id: ReadOnly[str] + team_alias: ReadOnly[str] + organization_id: ReadOnly[str] + user: ReadOnly[str] + end_user: ReadOnly[str] + model: ReadOnly[str] + model_group: ReadOnly[str] + model_id: ReadOnly[str] + custom_llm_provider: ReadOnly[str] + api_base: ReadOnly[str] + spend: ReadOnly[float] + prompt_tokens: ReadOnly[int] + completion_tokens: ReadOnly[int] + total_tokens: ReadOnly[int] + cache_read_tokens: ReadOnly[int] + cache_write_tokens: ReadOnly[int] + start_time: ReadOnly[int] # unix ms + end_time: ReadOnly[int] # unix ms + completion_start_time: ReadOnly[int | None] + status: ReadOnly[str] + error_str: ReadOnly[str] + cache_hit: ReadOnly[bool] + session_id: ReadOnly[str] + trace_id: ReadOnly[str] # from an incoming W3C traceparent, if any + span_id: ReadOnly[str] + request_tags: ReadOnly[list[str]] + metadata: ReadOnly[str] + messages: ReadOnly[str] + response: ReadOnly[str] diff --git a/ui/litellm-dashboard/src/components/view_logs/TraceView/useAgentTraces.ts b/ui/litellm-dashboard/src/components/view_logs/TraceView/useAgentTraces.ts index 322db6472504..def279c0c71a 100644 --- a/ui/litellm-dashboard/src/components/view_logs/TraceView/useAgentTraces.ts +++ b/ui/litellm-dashboard/src/components/view_logs/TraceView/useAgentTraces.ts @@ -12,7 +12,8 @@ const TRACING_NOT_ENABLED_STATUS = 501; const TRACING_ROUTE_MISSING_STATUS = 404; const isTracingUnavailable = (error: unknown): error is ApiError => - error instanceof ApiError && (error.status === TRACING_NOT_ENABLED_STATUS || error.status === TRACING_ROUTE_MISSING_STATUS); + error instanceof ApiError && + (error.status === TRACING_NOT_ENABLED_STATUS || error.status === TRACING_ROUTE_MISSING_STATUS); interface UseAgentTracesOptions { accessToken: string;