Skip to content
29 changes: 21 additions & 8 deletions crates/libsy/src/algorithms/util/stage.rs
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,11 @@ const EXPLORING_METRIC: &str = "switchyard.stage_router.exploring";
/// Distribution of production-oriented tool activity.
const PRODUCTION_INTENSITY_METRIC: &str = "switchyard.stage_router.production_intensity";

// Histogram boundaries live with the instruments so every host exports the
// same stage-router distributions without duplicating algorithm knowledge.
const SCORE_BUCKETS: &[f64] = &[-1.0, -0.75, -0.5, -0.25, 0.0, 0.25, 0.5, 0.75, 1.0];
const UNIT_BUCKETS: &[f64] = &[0.0, 0.1, 0.25, 0.5, 0.75, 0.9, 1.0];

/// The two tiers a turn can route to.
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum Tier {
Expand Down Expand Up @@ -288,15 +293,23 @@ fn record_score_metrics(signal: &ToolSignals, outcome: &PickOutcome) {
};
let dimensions = dimensions_from_signal(signal);
let meter = meter();
for (name, value) in [
(SCORE_METRIC, score),
(CONFIDENCE_METRIC, confidence),
(SEVERITY_METRIC, dimensions.severity),
(SPINNING_METRIC, dimensions.spinning),
(EXPLORING_METRIC, dimensions.exploring),
(PRODUCTION_INTENSITY_METRIC, dimensions.production_intensity),
for (name, value, boundaries) in [
(SCORE_METRIC, score, SCORE_BUCKETS),
(CONFIDENCE_METRIC, confidence, UNIT_BUCKETS),
(SEVERITY_METRIC, dimensions.severity, UNIT_BUCKETS),
(SPINNING_METRIC, dimensions.spinning, UNIT_BUCKETS),
(EXPLORING_METRIC, dimensions.exploring, UNIT_BUCKETS),
(
PRODUCTION_INTENSITY_METRIC,
dimensions.production_intensity,
UNIT_BUCKETS,
),
] {
meter.f64_histogram(name).build().record(value, &[]);
meter
.f64_histogram(name)
.with_boundaries(boundaries.to_vec())
.build()
.record(value, &[]);
}
}

Expand Down
6 changes: 5 additions & 1 deletion crates/switchyard-server/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,7 @@ documented in [Stage-Router Routing](../../docs/routing_algorithms/stage_router_
| `POST` | `/v1/responses` | OpenAI Responses |
| `POST` | `/v1/messages/count_tokens` | Token count from a route's Anthropic target |
| `GET` | `/v1/models` | Routes served by this deployment |
| `GET` | `/v1/stats` | Per-model request, token, and cost totals |
| `GET` | `/v1/stats` | Per-model usage plus curated algorithm stats |
| `POST` | `/v1/stats/reset` | Clear accumulated stats |
| `GET` | `/metrics` | Prometheus text, see [Metrics](#metrics) |
| `GET` | `/health` | Liveness |
Expand All @@ -131,6 +131,10 @@ Requests name a route by its `id`, so `POST /v1/chat/completions` with `"model":
routes through the `[routes.general]` entry above. Any of the three request formats can address any
route, and the server translates between them.

For `stage_router`, `algorithm_stats.stage_router` groups routing decisions by source and semantic
target and summarizes its score, confidence, and input-dimension histograms. These values reset
with `/v1/stats/reset`; the process-lifetime counters on `/metrics` remain cumulative.

Token counting selects an Anthropic-format completion target, preferring target names or model IDs
containing `opus`, `sonnet`, then `haiku`. Other ties preserve the route's target order.

Expand Down
6 changes: 5 additions & 1 deletion crates/switchyard-server/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -219,10 +219,14 @@ impl ServerState {
return Err(ServerError::new("at least one algorithm route is required"));
}
let metrics = metrics::registry().map_err(ServerError::new)?;
let stats = StatsAccumulator::new(
metrics.clone(),
entries.values().map(|entry| entry.algorithm.name()),
);
Ok(Self {
routes: Arc::new(entries),
metrics,
stats: StatsAccumulator::default(),
stats,
routing_log: None,
track_cache_eligibility: tracking_enabled_from_env(),
})
Expand Down
1 change: 1 addition & 0 deletions crates/switchyard-server/src/stats.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
//! Process-local JSON stats accounting for the Rust server.

mod accumulator;
mod algorithms;
mod cache_eligibility;

pub(crate) use accumulator::{StatsAccumulator, StatsSnapshot, TokenUsage};
Expand Down
55 changes: 50 additions & 5 deletions crates/switchyard-server/src/stats/accumulator.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,10 @@ use std::collections::{BTreeMap, HashSet};
use std::sync::Arc;

use parking_lot::{Mutex, MutexGuard};
use prometheus::Registry;
use serde::Serialize;

use super::algorithms::{AlgorithmStats, AlgorithmStatsSnapshot};
use super::cache_eligibility::PrefixProbe;

const MAX_LATENCY_SAMPLES: usize = 10_000;
Expand All @@ -25,12 +27,28 @@ pub(crate) struct TokenUsage {
}

/// Thread-safe process-local stats store.
#[derive(Clone, Debug, Default)]
#[derive(Clone)]
pub(crate) struct StatsAccumulator {
inner: Arc<Mutex<StatsAccumulatorInner>>,
}

impl Default for StatsAccumulator {
fn default() -> Self {
Self::new(Registry::new(), std::iter::empty())
}
}

impl StatsAccumulator {
/// Creates a stats store for the supplied algorithm names.
pub(crate) fn new<'a>(
registry: Registry,
algorithms: impl IntoIterator<Item = &'a str>,
) -> Self {
Self {
inner: Arc::new(Mutex::new(StatsAccumulatorInner::new(registry, algorithms))),
}
}

/// Records one successful routed backend call.
pub(crate) fn record_success(&self, model: impl Into<String>, backend_latency_ms: f64) {
let mut inner = self.lock();
Expand Down Expand Up @@ -115,21 +133,19 @@ impl StatsAccumulator {

/// Returns a serializable point-in-time snapshot.
pub(crate) fn snapshot(&self) -> StatsSnapshot {
let inner = self.lock().clone();
inner.snapshot()
self.lock().snapshot()
}

/// Clears all accumulated stats.
pub(crate) fn reset(&self) {
*self.lock() = StatsAccumulatorInner::default();
self.lock().reset();
}

fn lock(&self) -> MutexGuard<'_, StatsAccumulatorInner> {
self.inner.lock()
}
}

#[derive(Clone, Debug, Default)]
struct StatsAccumulatorInner {
by_model: BTreeMap<String, ModelStats>,
total_requests: u64,
Expand All @@ -139,9 +155,24 @@ struct StatsAccumulatorInner {
by_classifier: BTreeMap<String, ModelStats>,
classifier_requests: u64,
classifier_errors: u64,
algorithm_stats: AlgorithmStats,
}

impl StatsAccumulatorInner {
fn new<'a>(registry: Registry, algorithms: impl IntoIterator<Item = &'a str>) -> Self {
Self {
by_model: BTreeMap::new(),
total_requests: 0,
total_errors: 0,
routing_overhead: LatencyHistogram::default(),
routing_fallbacks: RoutingFallbackStats::default(),
by_classifier: BTreeMap::new(),
classifier_requests: 0,
classifier_errors: 0,
algorithm_stats: AlgorithmStats::new(registry, algorithms),
}
}

fn model_stats_mut(&mut self, model: String) -> &mut ModelStats {
self.by_model.entry(model).or_default()
}
Expand All @@ -165,8 +196,21 @@ impl StatsAccumulatorInner {
routing_overhead: self.routing_overhead.snapshot(),
routing_fallbacks: self.routing_fallbacks,
classifier,
algorithm_stats: self.algorithm_stats.snapshot(),
}
}

fn reset(&mut self) {
self.by_model.clear();
self.total_requests = 0;
self.total_errors = 0;
self.routing_overhead = LatencyHistogram::default();
self.routing_fallbacks = RoutingFallbackStats::default();
self.by_classifier.clear();
self.classifier_requests = 0;
self.classifier_errors = 0;
self.algorithm_stats.reset();
}
}

#[derive(Clone, Debug, Default)]
Expand Down Expand Up @@ -275,6 +319,7 @@ pub(crate) struct StatsSnapshot {
pub routing_overhead: LatencyHistogramSnapshot,
pub routing_fallbacks: RoutingFallbackStats,
pub classifier: ClassifierStatsSnapshot,
pub algorithm_stats: AlgorithmStatsSnapshot,
}

/// Legacy fallback counters retained in the stats response shape.
Expand Down
60 changes: 60 additions & 0 deletions crates/switchyard-server/src/stats/algorithms.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

//! Server-owned projections of algorithm OpenTelemetry metrics.

mod stage_router;

use std::collections::HashSet;

use prometheus::Registry;
use serde::Serialize;

use stage_router::{StageRouterCumulative, StageRouterStatsSnapshot};

const STAGE_ROUTER: &str = "stage_router";

/// Owns algorithm metric baselines behind the generic server stats interface.
pub(super) struct AlgorithmStats {
registry: Registry,
stage_router_baseline: Option<StageRouterCumulative>,
}

/// Curated algorithm-specific data included in the JSON stats response.
#[derive(Clone, Debug, Default, PartialEq, Serialize)]
pub(crate) struct AlgorithmStatsSnapshot {
#[serde(skip_serializing_if = "Option::is_none")]
pub stage_router: Option<StageRouterStatsSnapshot>,
}

impl AlgorithmStats {
pub(super) fn new<'a>(
registry: Registry,
algorithms: impl IntoIterator<Item = &'a str>,
) -> Self {
let algorithms: HashSet<_> = algorithms.into_iter().collect();
let families = registry.gather();
Self {
stage_router_baseline: algorithms
.contains(STAGE_ROUTER)
.then(|| StageRouterCumulative::collect(&families)),
registry,
}
}

pub(super) fn snapshot(&self) -> AlgorithmStatsSnapshot {
let families = self.registry.gather();
AlgorithmStatsSnapshot {
stage_router: self
.stage_router_baseline
.as_ref()
.map(|baseline| StageRouterCumulative::collect(&families).delta(baseline)),
}
}

pub(super) fn reset(&mut self) {
if let Some(baseline) = &mut self.stage_router_baseline {
*baseline = StageRouterCumulative::collect(&self.registry.gather());
}
}
}
Loading
Loading