Skip to content
6 changes: 5 additions & 1 deletion crates/switchyard-server/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,7 @@ documented in [Stage-Router Routing](../../docs/routing_algorithms/stage_router_
| `POST` | `/v1/responses` | OpenAI Responses |
| `POST` | `/v1/messages/count_tokens` | Token count from a route's Anthropic target |
| `GET` | `/v1/models` | Routes served by this deployment |
| `GET` | `/v1/stats` | Per-model request, token, and cost totals |
| `GET` | `/v1/stats` | Per-model usage plus curated algorithm stats |
| `POST` | `/v1/stats/reset` | Clear accumulated stats |
| `GET` | `/metrics` | Prometheus text, see [Metrics](#metrics) |
| `GET` | `/health` | Liveness |
Expand All @@ -131,6 +131,10 @@ Requests name a route by its `id`, so `POST /v1/chat/completions` with `"model":
routes through the `[routes.general]` entry above. Any of the three request formats can address any
route, and the server translates between them.

For `stage_router`, `algorithm_stats.stage_router` groups routing decisions by source and semantic
target and summarizes its score, confidence, and input-dimension histograms. These values reset
with `/v1/stats/reset`; the process-lifetime counters on `/metrics` remain cumulative.

Token counting selects an Anthropic-format completion target, preferring target names or model IDs
containing `opus`, `sonnet`, then `haiku`. Other ties preserve the route's target order.

Expand Down
6 changes: 5 additions & 1 deletion crates/switchyard-server/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -219,10 +219,14 @@ impl ServerState {
return Err(ServerError::new("at least one algorithm route is required"));
}
let metrics = metrics::registry().map_err(ServerError::new)?;
let stats = StatsAccumulator::new(
metrics.clone(),
entries.values().map(|entry| entry.algorithm.name()),
);
Ok(Self {
routes: Arc::new(entries),
metrics,
stats: StatsAccumulator::default(),
stats,
routing_log: None,
track_cache_eligibility: tracking_enabled_from_env(),
})
Expand Down
14 changes: 8 additions & 6 deletions crates/switchyard-server/src/metrics.rs
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ fn initialize() -> Result<Metrics, String> {
.map_err(|error| format!("failed to initialize Prometheus metrics: {error}"))?;
let mut builder = SdkMeterProvider::builder()
.with_reader(exporter)
.with_view(routing_overhead_buckets)
.with_view(histogram_buckets)
.with_resource(crate::observability::resource());
if crate::observability::otlp_enabled("METRICS") {
let exporter = opentelemetry_otlp::MetricExporter::builder()
Expand Down Expand Up @@ -70,13 +70,15 @@ pub(crate) fn flush() {
}
}

fn routing_overhead_buckets(instrument: &Instrument) -> Option<Stream> {
if instrument.name() != "switchyard.routing_overhead_ms" {
return None;
}
fn histogram_buckets(instrument: &Instrument) -> Option<Stream> {
let boundaries = if instrument.name() == "switchyard.routing_overhead_ms" {
ROUTING_OVERHEAD_BUCKETS_MS
} else {
crate::stats::algorithm_histogram_buckets(instrument.name())?
};
Stream::builder()
.with_aggregation(Aggregation::ExplicitBucketHistogram {
boundaries: ROUTING_OVERHEAD_BUCKETS_MS.to_vec(),
boundaries: boundaries.to_vec(),
// Cumulative min/max cover the whole process, so they aren't useful.
record_min_max: false,
})
Expand Down
2 changes: 2 additions & 0 deletions crates/switchyard-server/src/stats.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,9 @@
//! Process-local JSON stats accounting for the Rust server.

mod accumulator;
mod algorithms;
mod cache_eligibility;

pub(crate) use accumulator::{StatsAccumulator, StatsSnapshot, TokenUsage};
pub(crate) use algorithms::histogram_buckets as algorithm_histogram_buckets;
pub(crate) use cache_eligibility::{prefix_probe, tracking_enabled_from_env};
55 changes: 50 additions & 5 deletions crates/switchyard-server/src/stats/accumulator.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,10 @@ use std::collections::{BTreeMap, HashSet};
use std::sync::Arc;

use parking_lot::{Mutex, MutexGuard};
use prometheus::Registry;
use serde::Serialize;

use super::algorithms::{AlgorithmStats, AlgorithmStatsSnapshot};
use super::cache_eligibility::PrefixProbe;

const MAX_LATENCY_SAMPLES: usize = 10_000;
Expand All @@ -25,12 +27,28 @@ pub(crate) struct TokenUsage {
}

/// Thread-safe process-local stats store.
#[derive(Clone, Debug, Default)]
#[derive(Clone)]
pub(crate) struct StatsAccumulator {
inner: Arc<Mutex<StatsAccumulatorInner>>,
}

impl Default for StatsAccumulator {
fn default() -> Self {
Self::new(Registry::new(), std::iter::empty())
}
}

impl StatsAccumulator {
/// Creates a stats store for the supplied algorithm names.
pub(crate) fn new<'a>(
registry: Registry,
algorithms: impl IntoIterator<Item = &'a str>,
) -> Self {
Self {
inner: Arc::new(Mutex::new(StatsAccumulatorInner::new(registry, algorithms))),
}
}

/// Records one successful routed backend call.
pub(crate) fn record_success(&self, model: impl Into<String>, backend_latency_ms: f64) {
let mut inner = self.lock();
Expand Down Expand Up @@ -115,21 +133,19 @@ impl StatsAccumulator {

/// Returns a serializable point-in-time snapshot.
pub(crate) fn snapshot(&self) -> StatsSnapshot {
let inner = self.lock().clone();
inner.snapshot()
self.lock().snapshot()
}

/// Clears all accumulated stats.
pub(crate) fn reset(&self) {
*self.lock() = StatsAccumulatorInner::default();
self.lock().reset();
}

fn lock(&self) -> MutexGuard<'_, StatsAccumulatorInner> {
self.inner.lock()
}
}

#[derive(Clone, Debug, Default)]
struct StatsAccumulatorInner {
by_model: BTreeMap<String, ModelStats>,
total_requests: u64,
Expand All @@ -139,9 +155,24 @@ struct StatsAccumulatorInner {
by_classifier: BTreeMap<String, ModelStats>,
classifier_requests: u64,
classifier_errors: u64,
algorithm_stats: AlgorithmStats,
}

impl StatsAccumulatorInner {
fn new<'a>(registry: Registry, algorithms: impl IntoIterator<Item = &'a str>) -> Self {
Self {
by_model: BTreeMap::new(),
total_requests: 0,
total_errors: 0,
routing_overhead: LatencyHistogram::default(),
routing_fallbacks: RoutingFallbackStats::default(),
by_classifier: BTreeMap::new(),
classifier_requests: 0,
classifier_errors: 0,
algorithm_stats: AlgorithmStats::new(registry, algorithms),
}
}

fn model_stats_mut(&mut self, model: String) -> &mut ModelStats {
self.by_model.entry(model).or_default()
}
Expand All @@ -165,8 +196,21 @@ impl StatsAccumulatorInner {
routing_overhead: self.routing_overhead.snapshot(),
routing_fallbacks: self.routing_fallbacks,
classifier,
algorithm_stats: self.algorithm_stats.snapshot(),
}
}

fn reset(&mut self) {
self.by_model.clear();
self.total_requests = 0;
self.total_errors = 0;
self.routing_overhead = LatencyHistogram::default();
self.routing_fallbacks = RoutingFallbackStats::default();
self.by_classifier.clear();
self.classifier_requests = 0;
self.classifier_errors = 0;
self.algorithm_stats.reset();
}
}

#[derive(Clone, Debug, Default)]
Expand Down Expand Up @@ -275,6 +319,7 @@ pub(crate) struct StatsSnapshot {
pub routing_overhead: LatencyHistogramSnapshot,
pub routing_fallbacks: RoutingFallbackStats,
pub classifier: ClassifierStatsSnapshot,
pub algorithm_stats: AlgorithmStatsSnapshot,
}

/// Legacy fallback counters retained in the stats response shape.
Expand Down
64 changes: 64 additions & 0 deletions crates/switchyard-server/src/stats/algorithms.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

//! Server-owned projections of algorithm OpenTelemetry metrics.

mod stage_router;

use std::collections::HashSet;

use prometheus::Registry;
use serde::Serialize;

use stage_router::{StageRouterCumulative, StageRouterStatsSnapshot};

const STAGE_ROUTER: &str = "stage_router";

/// Owns algorithm metric baselines behind the generic server stats interface.
pub(super) struct AlgorithmStats {
registry: Registry,
stage_router_baseline: Option<StageRouterCumulative>,
}

/// Curated algorithm-specific data included in the JSON stats response.
#[derive(Clone, Debug, Default, PartialEq, Serialize)]
pub(crate) struct AlgorithmStatsSnapshot {
#[serde(skip_serializing_if = "Option::is_none")]
pub stage_router: Option<StageRouterStatsSnapshot>,
}

impl AlgorithmStats {
pub(super) fn new<'a>(
registry: Registry,
algorithms: impl IntoIterator<Item = &'a str>,
) -> Self {
let algorithms: HashSet<_> = algorithms.into_iter().collect();
let families = registry.gather();
Self {
stage_router_baseline: algorithms
.contains(STAGE_ROUTER)
.then(|| StageRouterCumulative::collect(&families)),
registry,
}
}

pub(super) fn snapshot(&self) -> AlgorithmStatsSnapshot {
let families = self.registry.gather();
AlgorithmStatsSnapshot {
stage_router: self
.stage_router_baseline
.as_ref()
.map(|baseline| StageRouterCumulative::collect(&families).delta(baseline)),
}
}

pub(super) fn reset(&mut self) {
if let Some(baseline) = &mut self.stage_router_baseline {
*baseline = StageRouterCumulative::collect(&self.registry.gather());
}
}
}

pub(crate) fn histogram_buckets(metric: &str) -> Option<&'static [f64]> {
stage_router::histogram_buckets(metric)
}
Loading
Loading