diff --git a/harness/tests/e2e/README.md b/harness/tests/e2e/README.md index 2979787323..8a07cf91ec 100644 --- a/harness/tests/e2e/README.md +++ b/harness/tests/e2e/README.md @@ -364,6 +364,10 @@ rules: - start `ScenarioSpec::version` at `1`; increment it when prompts, hard gates, criteria, thresholds, execution policy, setup, or evaluation behavior changes, while structural refactors that preserve the behavioral contract keep it; +- for new code-defined objective evaluators, declare each check once as a static + `AssessmentSpec`: use `required` for conditions that must emit a hard gate and + a binary award, and `signal` for non-blocking quality awards; existing + evaluators can migrate to this pattern incrementally; - prompts describe user intent and never prescribe function ids; - declare a scenario-sized execution policy; - objective effects are hard gates, not judge opinions; diff --git a/harness/tests/e2e/src/scenarios/assessment.rs b/harness/tests/e2e/src/scenarios/assessment.rs new file mode 100644 index 0000000000..f26e0206c3 --- /dev/null +++ b/harness/tests/e2e/src/scenarios/assessment.rs @@ -0,0 +1,209 @@ +use anyhow::{bail, Result}; + +use crate::report::HardGateReport; + +use super::{CriterionAward, CriterionSpec, ObjectiveEvaluation}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum AssessmentKind { + Required, + Signal, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(super) struct AssessmentSpec { + id: &'static str, + weight: u8, + description: &'static str, + kind: AssessmentKind, +} + +impl AssessmentSpec { + pub(super) const fn required(id: &'static str, weight: u8, description: &'static str) -> Self { + Self { + id, + weight, + description, + kind: AssessmentKind::Required, + } + } + + pub(super) const fn signal(id: &'static str, weight: u8, description: &'static str) -> Self { + Self { + id, + weight, + description, + kind: AssessmentKind::Signal, + } + } + + pub(super) const fn weight(self) -> u8 { + self.weight + } + + pub(super) fn binary(self, passed: bool, reason: impl Into) -> AssessmentResult { + AssessmentResult { + spec: self, + awarded: if passed { self.weight } else { 0 }, + gate_passed: matches!(self.kind, AssessmentKind::Required).then_some(passed), + reason: reason.into(), + } + } + + pub(super) fn points(self, awarded: u8, reason: impl Into) -> Result { + if self.kind == AssessmentKind::Required { + bail!( + "required assessment {} must be evaluated as pass or fail", + self.id + ); + } + if awarded > self.weight { + bail!( + "assessment {} awarded {awarded} points, exceeding its weight {}", + self.id, + self.weight + ); + } + Ok(AssessmentResult { + spec: self, + awarded, + gate_passed: None, + reason: reason.into(), + }) + } + + fn criterion(self) -> CriterionSpec { + CriterionSpec { + id: self.id, + weight: self.weight, + description: self.description, + } + } +} + +#[derive(Debug)] +pub(super) struct AssessmentResult { + spec: AssessmentSpec, + awarded: u8, + gate_passed: Option, + reason: String, +} + +pub(super) fn criteria(specs: &[AssessmentSpec]) -> Vec { + specs + .iter() + .copied() + .map(AssessmentSpec::criterion) + .collect() +} + +pub(super) fn objective( + results: impl IntoIterator, +) -> ObjectiveEvaluation { + let mut hard_gates = Vec::new(); + let mut awards = Vec::new(); + + for result in results { + if let Some(passed) = result.gate_passed { + hard_gates.push(HardGateReport { + id: result.spec.id.to_string(), + passed, + reason: result.reason.clone(), + }); + } + awards.push(CriterionAward { + id: result.spec.id.to_string(), + awarded: result.awarded, + reason: result.reason, + }); + } + + ObjectiveEvaluation { hard_gates, awards } +} + +#[cfg(test)] +mod tests { + use super::*; + + const REQUIRED: AssessmentSpec = + AssessmentSpec::required("required", 70, "A required outcome."); + const SIGNAL: AssessmentSpec = AssessmentSpec::signal("signal", 30, "A quality signal."); + + #[test] + fn criteria_preserve_assessment_definitions() { + let criteria = criteria(&[REQUIRED, SIGNAL]); + + assert_eq!(criteria.len(), 2); + assert_eq!(criteria[0].id, "required"); + assert_eq!(criteria[0].weight, 70); + assert_eq!(criteria[0].description, "A required outcome."); + assert_eq!(criteria[1].id, "signal"); + assert_eq!(criteria[1].weight, 30); + assert_eq!(criteria[1].description, "A quality signal."); + assert_eq!( + criteria + .iter() + .map(|criterion| u16::from(criterion.weight)) + .sum::(), + 100 + ); + } + + #[test] + fn required_pass_produces_a_passing_gate_and_full_award() { + let evaluation = objective([REQUIRED.binary(true, "satisfied")]); + + assert_eq!(evaluation.hard_gates.len(), 1); + assert_eq!(evaluation.hard_gates[0].id, "required"); + assert!(evaluation.hard_gates[0].passed); + assert_eq!(evaluation.hard_gates[0].reason, "satisfied"); + assert_eq!(evaluation.awards.len(), 1); + assert_eq!(evaluation.awards[0].id, "required"); + assert_eq!(evaluation.awards[0].awarded, 70); + assert_eq!(evaluation.awards[0].reason, "satisfied"); + } + + #[test] + fn required_failure_produces_a_failed_gate_and_zero_award() { + let evaluation = objective([REQUIRED.binary(false, "missing")]); + + assert!(!evaluation.hard_gates[0].passed); + assert_eq!(evaluation.awards[0].awarded, 0); + } + + #[test] + fn signal_produces_an_award_without_a_gate() { + let passed = objective([SIGNAL.binary(true, "observed")]); + let failed = objective([SIGNAL.binary(false, "missing")]); + + assert!(passed.hard_gates.is_empty()); + assert_eq!(passed.awards[0].id, "signal"); + assert_eq!(passed.awards[0].awarded, 30); + assert!(failed.hard_gates.is_empty()); + assert_eq!(failed.awards[0].awarded, 0); + } + + #[test] + fn signal_accepts_partial_points_within_its_weight() { + let evaluation = objective([SIGNAL.points(12, "partial").unwrap()]); + + assert!(evaluation.hard_gates.is_empty()); + assert_eq!(evaluation.awards[0].awarded, 12); + } + + #[test] + fn signal_rejects_points_above_its_weight() { + assert_eq!( + SIGNAL.points(31, "too many").unwrap_err().to_string(), + "assessment signal awarded 31 points, exceeding its weight 30" + ); + } + + #[test] + fn required_assessments_reject_partial_points() { + assert_eq!( + REQUIRED.points(35, "partial").unwrap_err().to_string(), + "required assessment required must be evaluated as pass or fail" + ); + } +} diff --git a/harness/tests/e2e/src/scenarios/mod.rs b/harness/tests/e2e/src/scenarios/mod.rs index 3d8198f34a..b6dc6ee85c 100644 --- a/harness/tests/e2e/src/scenarios/mod.rs +++ b/harness/tests/e2e/src/scenarios/mod.rs @@ -12,6 +12,7 @@ use serde_json::Value; use crate::context::E2eContext; use crate::report::HardGateReport; +mod assessment; pub mod common; pub mod custom_validator; pub mod design_tradeoff; diff --git a/harness/tests/e2e/src/scenarios/persistent_state.rs b/harness/tests/e2e/src/scenarios/persistent_state.rs index 73b81a7bd3..52bfdd9d99 100644 --- a/harness/tests/e2e/src/scenarios/persistent_state.rs +++ b/harness/tests/e2e/src/scenarios/persistent_state.rs @@ -2,21 +2,38 @@ use serde_json::{json, Value}; use crate::context::E2eContext; +use super::assessment::{self, AssessmentSpec}; use super::common; use super::{ - CleanupFuture, CriterionSpec, EvaluationFuture, ExecutionPolicy, ObjectiveEvaluation, - ScenarioObservation, ScenarioSpec, + CleanupFuture, EvaluationFuture, ExecutionPolicy, ObjectiveEvaluation, ScenarioObservation, + ScenarioSpec, }; pub const ID: &str = "persistent_state"; const KEY: &str = "persistent_state"; +const DURABLE_RESULT: AssessmentSpec = AssessmentSpec::required( + "durable_result", + 60, + "The exact requested JSON is present at the requested state key.", +); +const FUNCTION_DISCIPLINE: AssessmentSpec = AssessmentSpec::required( + "function_discipline", + 30, + "Exactly one successful write targets the requested scope and key.", +); +const CONFIRMATION: AssessmentSpec = AssessmentSpec::signal( + "confirmation", + 10, + "The final response briefly confirms completion.", +); +const ASSESSMENTS: &[AssessmentSpec] = &[DURABLE_RESULT, FUNCTION_DISCIPLINE, CONFIRMATION]; pub fn scenario(run_id: &str) -> ScenarioSpec { let scope = scope(run_id); let expected = expected(run_id); ScenarioSpec { id: ID, - version: 1, + version: 2, prompt: format!( "Store this JSON value for later use in scope `{scope}` under key `{KEY}`: {}. Confirm briefly after it has been stored.", serde_json::to_string(&expected).expect("serialize static scenario value") @@ -30,23 +47,7 @@ pub fn scenario(run_id: &str) -> ScenarioSpec { }, denied_functions: &[], threshold: 90, - criteria: vec![ - CriterionSpec { - id: "durable_result", - weight: 60, - description: "The exact requested JSON is present at the requested state key.", - }, - CriterionSpec { - id: "function_discipline", - weight: 30, - description: "Exactly one successful write targets the requested scope and key.", - }, - CriterionSpec { - id: "confirmation", - weight: 10, - description: "The final response briefly confirms completion.", - }, - ], + criteria: assessment::criteria(ASSESSMENTS), judge_reference: None, setup: None, evaluate, @@ -74,53 +75,56 @@ fn evaluate<'a>( .collect(); let exact_write = writes.len() == 1 && writes[0].arguments == json!({ "scope": scope, "key": KEY, "value": expected }); - let state_matches = observed == expected; - let no_errors = observation.metrics.totals.function_call_errors == 0; - let response = observation.response.as_str(); - let concise_confirmation = !response.trim().is_empty() && response.chars().count() <= 240; - - Ok(ObjectiveEvaluation { - hard_gates: vec![ - common::gate( - "state_persisted", - state_matches, - format!("expected {expected}, observed {observed}"), - ), - common::gate( - "single_exact_write", - exact_write, - format!("observed {} state::set call(s)", writes.len()), - ), - common::gate( - "no_function_errors", - no_errors, - format!( - "observed {} function-call error(s)", - observation.metrics.totals.function_call_errors - ), - ), - ], - awards: vec![ - common::award( - "durable_result", - if state_matches { 60 } else { 0 }, - "awarded when the durable value exactly matches", - ), - common::award( - "function_discipline", - if exact_write && no_errors { 30 } else { 0 }, - "awarded for one exact, successful state write", - ), - common::award( - "confirmation", - if concise_confirmation { 10 } else { 0 }, - "awarded for a non-empty confirmation under 240 characters", - ), - ], - }) + assess( + &expected, + &observed, + exact_write, + writes.len(), + observation.metrics.totals.function_call_errors, + observation.response.as_str(), + ) }) } +fn assess( + expected: &Value, + observed: &Value, + exact_write: bool, + state_set_calls: usize, + function_call_errors: u64, + response: &str, +) -> anyhow::Result { + let state_matches = observed == expected; + let function_discipline = exact_write && function_call_errors == 0; + let response_present = !response.trim().is_empty(); + let response_chars = response.chars().count(); + let concise_confirmation = response_present && response_chars <= 240; + let confirmation_points = if concise_confirmation { + CONFIRMATION.weight() + } else { + 0 + }; + + Ok(assessment::objective([ + DURABLE_RESULT.binary( + state_matches, + format!("expected {expected}, observed {observed}"), + ), + FUNCTION_DISCIPLINE.binary( + function_discipline, + format!( + "exact_write={exact_write}; observed {state_set_calls} state::set call(s); observed {function_call_errors} function-call error(s)" + ), + ), + CONFIRMATION.points( + confirmation_points, + format!( + "response_present={response_present}; observed {response_chars} character(s); limit 240" + ), + )?, + ])) +} + fn cleanup<'a>(context: &'a E2eContext, run_id: &'a str) -> CleanupFuture<'a> { Box::pin(async move { let scope = scope(run_id); @@ -142,3 +146,136 @@ fn expected(run_id: &str) -> Value { "status": "stored" }) } + +#[cfg(test)] +mod tests { + use super::*; + + fn award(evaluation: &ObjectiveEvaluation, id: &str) -> u8 { + evaluation + .awards + .iter() + .find(|award| award.id == id) + .unwrap_or_else(|| panic!("missing award {id}")) + .awarded + } + + fn gate<'a>( + evaluation: &'a ObjectiveEvaluation, + id: &str, + ) -> &'a crate::report::HardGateReport { + evaluation + .hard_gates + .iter() + .find(|gate| gate.id == id) + .unwrap_or_else(|| panic!("missing gate {id}")) + } + + fn total(evaluation: &ObjectiveEvaluation) -> u16 { + evaluation + .awards + .iter() + .map(|award| u16::from(award.awarded)) + .sum() + } + + #[test] + fn scenario_uses_the_canonical_assessment_contract() { + let spec = scenario("run"); + + spec.validate().unwrap(); + assert_eq!(spec.version, 2); + assert_eq!(spec.threshold, 90); + assert_eq!( + spec.criteria + .iter() + .map(|criterion| (criterion.id, criterion.weight)) + .collect::>(), + vec![ + ("durable_result", 60), + ("function_discipline", 30), + ("confirmation", 10), + ] + ); + } + + #[test] + fn all_satisfied_assessments_score_one_hundred() { + let value = json!({ "stored": true }); + let evaluation = assess(&value, &value, true, 1, 0, "Stored.").unwrap(); + + assert_eq!(evaluation.hard_gates.len(), 2); + assert_eq!( + evaluation + .hard_gates + .iter() + .map(|gate| gate.id.as_str()) + .collect::>(), + vec!["durable_result", "function_discipline"] + ); + assert_eq!( + evaluation + .awards + .iter() + .map(|award| award.id.as_str()) + .collect::>(), + vec!["durable_result", "function_discipline", "confirmation"] + ); + assert!(gate(&evaluation, "durable_result").passed); + assert!(gate(&evaluation, "function_discipline").passed); + assert_eq!(award(&evaluation, "durable_result"), 60); + assert_eq!(award(&evaluation, "function_discipline"), 30); + assert_eq!(award(&evaluation, "confirmation"), 10); + assert_eq!(total(&evaluation), 100); + } + + #[test] + fn missing_or_overlong_confirmation_is_non_blocking() { + let value = json!({ "stored": true }); + + for response in [String::new(), "x".repeat(241)] { + let evaluation = assess(&value, &value, true, 1, 0, &response).unwrap(); + + assert!(evaluation.hard_gates.iter().all(|gate| gate.passed)); + assert_eq!(award(&evaluation, "confirmation"), 0); + assert_eq!(total(&evaluation), 90); + } + } + + #[test] + fn inexact_write_or_function_error_fails_function_discipline() { + let value = json!({ "stored": true }); + + for (exact_write, state_set_calls, function_call_errors) in [(false, 2, 0), (true, 1, 1)] { + let evaluation = assess( + &value, + &value, + exact_write, + state_set_calls, + function_call_errors, + "Stored.", + ) + .unwrap(); + + let discipline_gate = gate(&evaluation, "function_discipline"); + assert!(!discipline_gate.passed); + assert!(discipline_gate.reason.contains(&format!( + "observed {function_call_errors} function-call error(s)" + ))); + assert_eq!(award(&evaluation, "function_discipline"), 0); + assert_eq!(total(&evaluation), 70); + } + } + + #[test] + fn state_mismatch_fails_durable_result() { + let expected = json!({ "stored": true }); + let observed = json!({ "stored": false }); + let evaluation = assess(&expected, &observed, true, 1, 0, "Stored.").unwrap(); + + assert!(!gate(&evaluation, "durable_result").passed); + assert!(gate(&evaluation, "function_discipline").passed); + assert_eq!(award(&evaluation, "durable_result"), 0); + assert_eq!(total(&evaluation), 40); + } +}