Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

17 changes: 17 additions & 0 deletions crates/libsy-llm-client/src/client.rs
Original file line number Diff line number Diff line change
Expand Up @@ -178,6 +178,7 @@ impl TranslatingLlmClient {
let streaming = endpoint.allows_streaming()
&& body.get("stream").and_then(Value::as_bool).unwrap_or(false);
let url = endpoint.url(backend);
record_gen_ai_request(&url, model, streaming);

let max_retries = u64::from(backend.max_retries());
let max_attempts = max_retries + 1;
Expand Down Expand Up @@ -601,6 +602,22 @@ fn duration_millis(duration: Duration) -> u64 {
u64::try_from(duration.as_millis()).unwrap_or(u64::MAX)
}

fn record_gen_ai_request(url: &str, model: &str, streaming: bool) {
let span = tracing::Span::current();
span.record("gen_ai.request.model", model);
if streaming {
span.record("gen_ai.request.stream", true);
}
if let Ok(url) = reqwest::Url::parse(url) {
if let Some(host) = url.host_str() {
span.record("server.address", host);
}
if let Some(port) = url.port_or_known_default() {
span.record("server.port", i64::from(port));
}
}
}

fn convert_reqwest_error(error: reqwest::Error) -> LlmClientError {
// Reqwest labels truncated or otherwise unreadable response bodies as decode
// errors, so distinguish them from serde JSON failures at the call site.
Expand Down
3 changes: 2 additions & 1 deletion crates/libsy/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -26,10 +26,11 @@ thiserror.workspace = true
tokio.workspace = true
tokio-stream = "0.1"
tracing.workspace = true
tracing-opentelemetry = "0.33"

[dev-dependencies]
# SDK + in-memory exporter to assert what the observability layer records.
opentelemetry_sdk = { version = "0.32", features = ["metrics", "testing"] }
opentelemetry_sdk = { version = "0.32", features = ["metrics", "testing", "trace"] }
switchyard-llm-client.workspace = true
tokio.workspace = true
tracing-subscriber = "0.3"
43 changes: 41 additions & 2 deletions crates/libsy/src/core/algorithm.rs
Original file line number Diff line number Diff line change
Expand Up @@ -618,7 +618,7 @@ pub trait Algorithm: Send + Sync + 'static {
// One `libsy.run` span covers the whole algorithm task; the driver's
// `libsy.llm_call` spans and decision logs nest inside it via `tracing`'s
// contextual parenting.
let span = observability::run_span(self.name(), request.metadata.as_ref());
let span = observability::run_span(self.name(), &request);
let observed_driver = task_driver.clone();
let handle = tokio::spawn(
async move {
Expand Down Expand Up @@ -684,12 +684,51 @@ pub trait Algorithm: Send + Sync + 'static {
skip_all,
fields(
algorithm = observability::algorithm_label(&call.get_routed().ctx),
switchyard.algorithm = observability::algorithm_label(&call.get_routed().ctx),
switchyard.routing.tier = tracing::field::Empty,
selected_model = call.get_decision().selected_model(),
otel.kind = "client",
otel.name = %format_args!("chat {}", call.get_decision().selected_model()),
gen_ai.operation.name = "chat",
gen_ai.request.model = call.get_decision().selected_model(),
gen_ai.request.stream = tracing::field::Empty,
gen_ai.request.temperature = tracing::field::Empty,
gen_ai.request.top_p = tracing::field::Empty,
gen_ai.request.top_k = tracing::field::Empty,
gen_ai.request.max_tokens = tracing::field::Empty,
gen_ai.request.reasoning.level = tracing::field::Empty,
gen_ai.output.type = tracing::field::Empty,
gen_ai.conversation.id = tracing::field::Empty,
server.address = tracing::field::Empty,
server.port = tracing::field::Empty,
gen_ai.response.id = tracing::field::Empty,
gen_ai.response.model = tracing::field::Empty,
gen_ai.usage.input_tokens = tracing::field::Empty,
gen_ai.usage.output_tokens = tracing::field::Empty,
gen_ai.usage.cache_read.input_tokens = tracing::field::Empty,
gen_ai.usage.cache_creation.input_tokens = tracing::field::Empty,
gen_ai.usage.reasoning.output_tokens = tracing::field::Empty,
outcome = tracing::field::Empty,
otel.status_code = tracing::field::Empty,
error.type = tracing::field::Empty,
error = tracing::field::Empty,
Comment thread
grahamking marked this conversation as resolved.
)
)]
async fn serve(call: CallLlmRequest) -> Result<()> {
let span = tracing::Span::current();
observability::record_gen_ai_request(&span, &call.get_routed().request.llm_request);
if let Some(tier) = call.get_decision().routing_tier() {
span.record("switchyard.routing.tier", tier);
}
if let Some(session_id) = call
.get_routed()
.request
.metadata
.as_ref()
.and_then(|metadata| metadata.session_id.as_deref())
{
span.record("gen_ai.conversation.id", session_id);
}
let routed = call.get_routed().clone();
let target = routed.decision.selected_model().to_string();
let client =
Expand All @@ -703,7 +742,7 @@ pub trait Algorithm: Send + Sync + 'static {
.call(routed.ctx, routed.request, routed.decision)
.await
.map_err(|source| LibsyError::client_call(target, source));
observability::record_client_call(&result);
let result = observability::observe_client_call(result);
call.respond(result)
}

Expand Down
Loading
Loading