Skip to content
Closed
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 20 additions & 1 deletion crates/libsy/src/algorithms/stage.rs
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,8 @@ use super::llm_class::{LlmClassifierConfig, LlmTaskClassifier, TaskClassifierCon
use super::util::prompts::{SystemPromptProcessor, TargetPrompts};
use super::util::stage::{
DecisionSource, HandoffNoteConfig, PickerMode, StageClassifier, StageTargets,
record_decision_source, record_routing_decision,
max_efficient_confidence, record_decision_source, record_routing_decision,
scorer_cannot_leave_default,
};
use super::util::tool_signals::{DEFAULT_RECENT_WINDOW, ToolSignalProcessor};
use crate::core::algorithm::{Algorithm, Driver};
Expand Down Expand Up @@ -158,6 +159,24 @@ fn build_route(
),
});
}
// A threshold above the efficient ceiling silently disables the scorer for
// capable_first: every turn reads as "signals were weak" when in fact that
// branch cannot fire at all. Warn rather than reject, since the judge and the
// hard de-escalation shortcut still work and the setting stays a policy choice.
if scorer_cannot_leave_default(config.mode, config.confidence_threshold) {
tracing::warn!(
confidence_threshold = config.confidence_threshold,
efficient_ceiling = max_efficient_confidence(),
"stage_router picker \"capable_first\" with confidence_threshold {} cannot de-escalate: \
production_intensity is the only signal on the efficient side, so the scorer tops out \
at {:.4}. Every turn will fall through to the judge, or to the capable tier when no \
judge is configured. Set confidence_threshold at or below {:.4} for the scorer to \
reach the efficient tier.",
config.confidence_threshold,
max_efficient_confidence(),
max_efficient_confidence(),
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
);
}
// The tiers are a fixed pair; their targets are whatever the deployment calls
// them, and the classifier scores onto those names.
let targets = StageTargets::new(capable.clone(), efficient.clone());
Expand Down
100 changes: 100 additions & 0 deletions crates/libsy/src/algorithms/util/stage.rs
Original file line number Diff line number Diff line change
Expand Up @@ -334,6 +334,28 @@ pub fn score_signal(signal: &ToolSignals) -> ScoreResult {
}
}

/// Highest confidence the scorer can reach toward the efficient tier.
///
/// `production_intensity` is the only dimension on the efficient side and
/// [`ratio`] bounds it to `1.0`, so the efficient direction has exactly one
/// signal unit to work with — there is nothing for it to corroborate with. The
/// `tanh` squash then pulls that unit down from `SCORE_GAIN * SIGNAL_UNIT` to
/// roughly `0.4621`, which lands just under the commonly used `0.5` threshold.
pub fn max_efficient_confidence() -> f64 {
(SCORE_GAIN * SIGNAL_UNIT).tanh()
}

/// True when `confidence_threshold` puts every scorer outcome out of reach of
/// `mode`'s non-default tier, so the scorer can never change a turn's tier.
///
/// Only [`PickerMode::CapableFirst`] can hit this: leaving its default means
/// picking efficient, and that side caps at [`max_efficient_confidence`].
/// Escalation stacks several dimensions, so `efficient_first` always has some
/// reachable band and is never reported here.
pub fn scorer_cannot_leave_default(mode: PickerMode, confidence_threshold: f64) -> bool {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

all of these don't have to be pub, they can be pub(crate).

in fact, since they are only used in logs, they can be in-line or function within functions inside scorer_cannot_leave_default block.

or it can be just one function doing it all.

matches!(mode, PickerMode::CapableFirst) && confidence_threshold > max_efficient_confidence()
}
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved

/// Hard **escalate** — force the capable tier no matter what the scorer would
/// say. Fires on a critical error or a compacted context.
fn should_escalate(signal: &ToolSignals) -> bool {
Expand Down Expand Up @@ -649,6 +671,84 @@ mod tests {
);
}

#[test]
fn production_is_the_only_signal_pushing_toward_efficient() {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

these are too many tests for this small fix, pls change an existing test for this or have only one

// The de-escalation ceiling is a property of the dimension set: only
// `production_intensity` is negative, and `ratio` bounds it to 1.0. A
// turn that is pure production therefore scores the strongest efficient
// confidence the scorer can ever produce.
let mut signal = signal_from(json!([{"role": "user", "content": "hi"}]));
signal.recent_write_count = 3;
signal.recent_edit_count = 1;
let scored = score_signal(&signal);
assert!(scored.score < 0.0, "expected an efficient lean: {scored:?}");
assert!(
(scored.confidence - max_efficient_confidence()).abs() < 1e-12,
"pure production should reach the ceiling: {scored:?}"
);
}

#[test]
fn the_efficient_ceiling_sits_below_a_half() {
// tanh compresses one full signal unit (5.0 × 0.10) to ~0.4621, so the
// widely used 0.5 threshold sits above everything the efficient side can
// score. This is the constant the capable_first guard is built on.
assert!((max_efficient_confidence() - 0.462_117_157_260_009_7).abs() < 1e-12);
assert!(max_efficient_confidence() < 0.5);
}

#[test]
fn capable_first_reports_an_unreachable_efficient_tier() {
// Above the ceiling the scorer cannot leave the capable default, so the
// combination is inert rather than merely selective.
assert!(scorer_cannot_leave_default(PickerMode::CapableFirst, 0.5));
assert!(scorer_cannot_leave_default(PickerMode::CapableFirst, 0.47));
assert!(!scorer_cannot_leave_default(PickerMode::CapableFirst, 0.45));
// The gate is inclusive, so the ceiling itself still fires.
assert!(!scorer_cannot_leave_default(
PickerMode::CapableFirst,
max_efficient_confidence()
));
}

#[test]
fn efficient_first_is_never_reported_as_inert() {
// Escalation has more signals to stack, and this guard only speaks to the
// efficient direction.
assert!(!scorer_cannot_leave_default(
PickerMode::EfficientFirst,
0.5
));
assert!(!scorer_cannot_leave_default(
PickerMode::EfficientFirst,
1.0
));
}

#[test]
fn capable_first_above_the_ceiling_never_picks_efficient() {
// End-to-end on `pick_tier`: the strongest possible efficient signal is
// still handed on rather than resolved.
let mut signal = signal_from(json!([{"role": "user", "content": "hi"}]));
signal.recent_write_count = 3;
signal.recent_edit_count = 1;
assert!(matches!(
pick_tier(&signal, PickerMode::CapableFirst, 0.5),
PickOutcome::ConsultClassifier {
default_tier: Tier::Capable,
..
}
));
assert!(matches!(
pick_tier(&signal, PickerMode::CapableFirst, 0.45),
PickOutcome::Resolved {
tier: Tier::Efficient,
source: DecisionSource::Dimensions,
..
}
));
}

#[test]
fn the_picker_mode_names_the_tier_an_undecided_turn_falls_back_to() {
assert_eq!(PickerMode::CapableFirst.default_tier(), Tier::Capable);
Expand Down
2 changes: 1 addition & 1 deletion crates/libsy/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ pub use algorithms::util::tool_signals::{DEFAULT_RECENT_WINDOW, ToolSignals};
pub use algorithms::util::stage::{
CodingAgentDimensions, DECISION_SOURCE_KEY, DecisionSource, HandoffNoteConfig, PickOutcome,
PickerMode, ScoreResult, StageClassifier, StageTargets, Tier, dimensions_from_signal,
pick_tier, score_signal,
max_efficient_confidence, pick_tier, score_signal, scorer_cannot_leave_default,
};

mod observability;
Expand Down