diff --git a/harness/tests/e2e/src/scenarios/mechanical_reaction.rs b/harness/tests/e2e/src/scenarios/mechanical_reaction.rs index 10b9ace38..3d35d48f9 100644 --- a/harness/tests/e2e/src/scenarios/mechanical_reaction.rs +++ b/harness/tests/e2e/src/scenarios/mechanical_reaction.rs @@ -2,22 +2,48 @@ use serde_json::{json, Value}; use crate::context::E2eContext; +use super::assessment::{self, AssessmentSpec}; use super::{ - common, CleanupFuture, CriterionSpec, EvaluationFuture, ExecutionPolicy, ObjectiveEvaluation, - ScenarioObservation, ScenarioSpec, + common, CleanupFuture, EvaluationFuture, ExecutionPolicy, ScenarioObservation, ScenarioSpec, }; pub const ID: &str = "mechanical_reaction"; const SOURCE_KEY: &str = "source"; const MIRROR_KEY: &str = "mirror"; +const REACTIONS_ARMED: AssessmentSpec = AssessmentSpec::required( + "reactions_armed", + 30, + "The wake and mechanical call are registered before the source write.", +); +const MECHANICAL_MIRROR: AssessmentSpec = AssessmentSpec::required( + "mechanical_mirror", + 35, + "The call binding mirrors the complete source event without a root write.", +); +const PARENT_WOKEN: AssessmentSpec = AssessmentSpec::required( + "parent_woken", + 20, + "The mirror state event wakes only the original session.", +); +const CLEAN_COMPLETION: AssessmentSpec = AssessmentSpec::required( + "clean_completion", + 15, + "The run finishes without children, errors, or surviving bindings.", +); +const ASSESSMENTS: &[AssessmentSpec] = &[ + REACTIONS_ARMED, + MECHANICAL_MIRROR, + PARENT_WOKEN, + CLEAN_COMPLETION, +]; pub fn scenario(run_id: &str) -> ScenarioSpec { let names = Names::new(run_id); let source = source_value(run_id); ScenarioSpec { id: ID, - version: 1, + version: 2, prompt: format!( r#"Test a zero-token mechanical reaction in isolated state scope `{scope}`. @@ -51,29 +77,7 @@ binding armed."#, }, denied_functions: &[], threshold: 90, - criteria: vec![ - CriterionSpec { - id: "reactions_armed", - weight: 30, - description: "The wake and mechanical call are registered before the source write.", - }, - CriterionSpec { - id: "mechanical_mirror", - weight: 35, - description: - "The call binding mirrors the complete source event without a root write.", - }, - CriterionSpec { - id: "parent_woken", - weight: 20, - description: "The mirror state event wakes only the original session.", - }, - CriterionSpec { - id: "clean_completion", - weight: 15, - description: "The run finishes without children, errors, or surviving bindings.", - }, - ], + criteria: assessment::criteria(ASSESSMENTS), judge_reference: None, setup: None, evaluate, @@ -157,68 +161,40 @@ fn evaluate<'a>( exact_source_write && mirror_valid && call_delivered && call_payload_recorded; let clean_completion = active_bindings == 0 && no_errors && confirmed; - Ok(ObjectiveEvaluation { - hard_gates: vec![ - common::gate( - "reactions_preceded_source_write", - reactions_armed, - format!( - "registrations={}, wakes={}, call_bindings={}, writes={}", - registrations.len(), - wakes.len(), - mirrors.len(), - writes.len() - ), + Ok(assessment::objective([ + REACTIONS_ARMED.binary( + reactions_armed, + format!( + "registrations={}, wakes={}, call_bindings={}, writes={}", + registrations.len(), + wakes.len(), + mirrors.len(), + writes.len() ), - common::gate( - "source_event_was_mirrored_mechanically", - mechanical_mirror, - format!( - "exact_source_write={exact_source_write}, mirror_valid={mirror_valid}, \ + ), + MECHANICAL_MIRROR.binary( + mechanical_mirror, + format!( + "exact_source_write={exact_source_write}, mirror_valid={mirror_valid}, \ call_delivered={call_delivered}, call_payload_recorded={call_payload_recorded}" - ), ), - common::gate( - "mirror_woke_original_session", - parent_woken, - format!( - "wake_records={}, sessions={}", - wake_records.len(), - observation.metrics.totals.sessions - ), + ), + PARENT_WOKEN.binary( + parent_woken, + format!( + "wake_records={}, sessions={}", + wake_records.len(), + observation.metrics.totals.sessions ), - common::gate( - "mechanical_run_completed_cleanly", - clean_completion, - format!( - "active_bindings={active_bindings}, function_errors={}, confirmed={confirmed}", - observation.metrics.totals.function_call_errors - ), + ), + CLEAN_COMPLETION.binary( + clean_completion, + format!( + "active_bindings={active_bindings}, function_errors={}, confirmed={confirmed}", + observation.metrics.totals.function_call_errors ), - ], - awards: vec![ - common::award( - "reactions_armed", - if reactions_armed { 30 } else { 0 }, - "awarded when both reactions precede the only root state write", - ), - common::award( - "mechanical_mirror", - if mechanical_mirror { 35 } else { 0 }, - "awarded when the call binding persists the complete source event", - ), - common::award( - "parent_woken", - if parent_woken { 20 } else { 0 }, - "awarded when the mirror event wakes only the root session", - ), - common::award( - "clean_completion", - if clean_completion { 15 } else { 0 }, - "awarded for a confirmed result with no errors or live bindings", - ), - ], - }) + ), + ])) }) } @@ -398,6 +374,20 @@ mod tests { &names, &source )); - scenario("run").validate().unwrap(); + let spec = scenario("run"); + spec.validate().unwrap(); + assert_eq!(spec.version, 2); + assert_eq!( + spec.criteria + .iter() + .map(|criterion| (criterion.id, criterion.weight)) + .collect::>(), + vec![ + ("reactions_armed", 30), + ("mechanical_mirror", 35), + ("parent_woken", 20), + ("clean_completion", 15), + ] + ); } } diff --git a/harness/tests/e2e/src/scenarios/research_pipeline.rs b/harness/tests/e2e/src/scenarios/research_pipeline.rs index 5121692fa..eb23fce3e 100644 --- a/harness/tests/e2e/src/scenarios/research_pipeline.rs +++ b/harness/tests/e2e/src/scenarios/research_pipeline.rs @@ -4,9 +4,9 @@ use serde_json::{json, Value}; use crate::context::E2eContext; +use super::assessment::{self, AssessmentSpec}; use super::{ - common, CleanupFuture, CriterionSpec, EvaluationFuture, ExecutionPolicy, ObjectiveEvaluation, - ScenarioObservation, ScenarioSpec, + common, CleanupFuture, EvaluationFuture, ExecutionPolicy, ScenarioObservation, ScenarioSpec, }; pub const ID: &str = "research_pipeline"; @@ -17,12 +17,38 @@ const SUMMARY_KEY: &str = "summary"; const FACTS_KEY: &str = "facts"; const MIN_ARTICLE_CHARS: usize = 5_000; const MAX_ARTICLE_CHARS: usize = 6_500; +const SOURCE_CAPTURE: AssessmentSpec = AssessmentSpec::required( + "source_capture", + 25, + "All wakes are armed before the Wikipedia article is fetched and saved.", +); +const PARALLEL_ANALYSIS: AssessmentSpec = AssessmentSpec::required( + "parallel_analysis", + 30, + "The article wake causes two analysts to be spawned directly and in parallel.", +); +const BARRIER_FAN_IN: AssessmentSpec = AssessmentSpec::required( + "barrier_fan_in", + 25, + "The analysts persist valid outputs and the named barrier retires after both arrive.", +); +const RESEARCH_BRIEF: AssessmentSpec = AssessmentSpec::required( + "research_brief", + 20, + "The coordinator returns a merged brief in its barrier-woken turn and leaves no binding armed.", +); +const ASSESSMENTS: &[AssessmentSpec] = &[ + SOURCE_CAPTURE, + PARALLEL_ANALYSIS, + BARRIER_FAN_IN, + RESEARCH_BRIEF, +]; pub fn scenario(run_id: &str) -> ScenarioSpec { let names = Names::new(run_id); ScenarioSpec { id: ID, - version: 1, + version: 2, prompt: prompt(&names), filesystem_root: None, execution: ExecutionPolicy { @@ -33,28 +59,7 @@ pub fn scenario(run_id: &str) -> ScenarioSpec { }, denied_functions: &[], threshold: 90, - criteria: vec![ - CriterionSpec { - id: "source_capture", - weight: 25, - description: "All wakes are armed before the Wikipedia article is fetched and saved.", - }, - CriterionSpec { - id: "parallel_analysis", - weight: 30, - description: "The article wake causes two analysts to be spawned directly and in parallel.", - }, - CriterionSpec { - id: "barrier_fan_in", - weight: 25, - description: "The analysts persist valid outputs and the named barrier retires after both arrive.", - }, - CriterionSpec { - id: "research_brief", - weight: 20, - description: "The coordinator returns a merged brief in its barrier-woken turn and leaves no binding armed.", - }, - ], + criteria: assessment::criteria(ASSESSMENTS), judge_reference: None, setup: None, evaluate, @@ -280,68 +285,39 @@ fn evaluate<'a>( && barrier_woke; let report_complete = report_merged && active_bindings == 0 && no_errors; - Ok(ObjectiveEvaluation { - hard_gates: vec![ - common::gate( - "source_captured_after_watches", - source_captured, - format!( - "armed_before_fetch={armed_before_fetch}, source_order={source_order}, \ - article_valid={article_valid}, exact_write={exact_article_write}" - ), + Ok(assessment::objective([ + SOURCE_CAPTURE.binary( + source_captured, + format!( + "armed_before_fetch={armed_before_fetch}, source_order={source_order}, \ + article_valid={article_valid}, exact_write={exact_article_write}" ), - common::gate( - "analysts_spawned_directly_in_parallel", - direct_parallel_analysis, - format!( - "spawns={}, parallel_calls={parallel_calls}, \ - overlapping_sessions={overlapping_sessions}, \ - direct_sessions={sessions_direct}", - spawns.len() - ), + ), + PARALLEL_ANALYSIS.binary( + direct_parallel_analysis, + format!( + "spawns={}, parallel_calls={parallel_calls}, \ + overlapping_sessions={overlapping_sessions}, direct_sessions={sessions_direct}", + spawns.len() ), - common::gate( - "analyst_outputs_joined_by_barrier", - fan_in_complete, - format!( - "summary_valid={summary_valid}, facts_valid={facts_valid}, \ - analyst_writes={analyst_writes}, analyst_discipline={analyst_discipline}, \ - article_woke={article_woke}, barrier_woke={barrier_woke}" - ), + ), + BARRIER_FAN_IN.binary( + fan_in_complete, + format!( + "summary_valid={summary_valid}, facts_valid={facts_valid}, \ + analyst_writes={analyst_writes}, analyst_discipline={analyst_discipline}, \ + article_woke={article_woke}, barrier_woke={barrier_woke}" ), - common::gate( - "brief_returned_and_bindings_clean", - report_complete, - format!( - "report_merged={report_merged}, active_bindings={active_bindings}, \ - function_errors={}", - observation.metrics.totals.function_call_errors - ), + ), + RESEARCH_BRIEF.binary( + report_complete, + format!( + "report_merged={report_merged}, active_bindings={active_bindings}, \ + function_errors={}", + observation.metrics.totals.function_call_errors ), - ], - awards: vec![ - common::award( - "source_capture", - if source_captured { 25 } else { 0 }, - "awarded when all wakes precede one valid markdown fetch and article write", - ), - common::award( - "parallel_analysis", - if direct_parallel_analysis { 30 } else { 0 }, - "awarded for two direct, parallel analyst sessions", - ), - common::award( - "barrier_fan_in", - if fan_in_complete { 25 } else { 0 }, - "awarded when both durable analyst outputs retire the named barrier", - ), - common::award( - "research_brief", - if report_complete { 20 } else { 0 }, - "awarded for a merged root-session brief and complete binding cleanup", - ), - ], - }) + ), + ])) }) } @@ -674,6 +650,21 @@ mod tests { scenario.validate().unwrap(); assert!(!scenario.needs_judge()); assert!(!scenario.prompt.contains("harness::react")); + assert_eq!(scenario.version, 2); + assert_eq!(scenario.threshold, 90); + assert_eq!( + scenario + .criteria + .iter() + .map(|criterion| (criterion.id, criterion.weight)) + .collect::>(), + vec![ + ("source_capture", 25), + ("parallel_analysis", 30), + ("barrier_fan_in", 25), + ("research_brief", 20), + ] + ); } #[test] diff --git a/harness/tests/e2e/src/scenarios/timer_wake.rs b/harness/tests/e2e/src/scenarios/timer_wake.rs index 772ab14b3..b171709b6 100644 --- a/harness/tests/e2e/src/scenarios/timer_wake.rs +++ b/harness/tests/e2e/src/scenarios/timer_wake.rs @@ -2,20 +2,41 @@ use serde_json::{json, Value}; use crate::context::E2eContext; +use super::assessment::{self, AssessmentSpec}; use super::{ - common, CleanupFuture, CriterionSpec, EvaluationFuture, ExecutionPolicy, ObjectiveEvaluation, - ScenarioObservation, ScenarioSpec, + common, CleanupFuture, EvaluationFuture, ExecutionPolicy, ScenarioObservation, ScenarioSpec, }; pub const ID: &str = "timer_wake"; const RESULT_KEY: &str = "result"; +const TIMER_ARMED: AssessmentSpec = AssessmentSpec::required( + "timer_armed", + 30, + "One wake-only relative timer is armed before any result write.", +); +const PARENT_WOKEN: AssessmentSpec = AssessmentSpec::required( + "parent_woken", + 30, + "The timer retires after waking the original session exactly once.", +); +const WAKE_ACTION: AssessmentSpec = AssessmentSpec::required( + "wake_action", + 25, + "The timer-woken turn persists the requested result.", +); +const CLEAN_COMPLETION: AssessmentSpec = AssessmentSpec::required( + "clean_completion", + 15, + "The root completes without children, errors, or surviving bindings.", +); +const ASSESSMENTS: &[AssessmentSpec] = &[TIMER_ARMED, PARENT_WOKEN, WAKE_ACTION, CLEAN_COMPLETION]; pub fn scenario(run_id: &str) -> ScenarioSpec { let names = Names::new(run_id); ScenarioSpec { id: ID, - version: 1, + version: 2, prompt: format!( r#"Test the parent-owned timer control plane in isolated state scope `{scope}`. @@ -43,28 +64,7 @@ fired. Leave no binding armed."#, }, denied_functions: &[], threshold: 90, - criteria: vec![ - CriterionSpec { - id: "timer_armed", - weight: 30, - description: "One wake-only relative timer is armed before any result write.", - }, - CriterionSpec { - id: "parent_woken", - weight: 30, - description: "The timer retires after waking the original session exactly once.", - }, - CriterionSpec { - id: "wake_action", - weight: 25, - description: "The timer-woken turn persists the requested result.", - }, - CriterionSpec { - id: "clean_completion", - weight: 15, - description: "The root completes without children, errors, or surviving bindings.", - }, - ], + criteria: assessment::criteria(ASSESSMENTS), judge_reference: None, setup: None, evaluate, @@ -129,60 +129,32 @@ fn evaluate<'a>( let wake_action = exact_write && observed == expected; let clean_completion = active_bindings == 0 && no_errors && confirmed; - Ok(ObjectiveEvaluation { - hard_gates: vec![ - common::gate( - "relative_timer_armed_before_result", - timer_armed, - format!( - "registrations={}, timers={}, writes={}", - registrations.len(), - timers.len(), - writes.len() - ), + Ok(assessment::objective([ + TIMER_ARMED.binary( + timer_armed, + format!( + "registrations={}, timers={}, writes={}", + registrations.len(), + timers.len(), + writes.len() ), - common::gate( - "timer_woke_original_session", - parent_woken, - format!("timer_fired={timer_fired}, root_only={root_only}"), + ), + PARENT_WOKEN.binary( + parent_woken, + format!("timer_fired={timer_fired}, root_only={root_only}"), + ), + WAKE_ACTION.binary( + wake_action, + format!("exact_write={exact_write}, observed={observed}"), + ), + CLEAN_COMPLETION.binary( + clean_completion, + format!( + "active_bindings={active_bindings}, function_errors={}, confirmed={confirmed}", + observation.metrics.totals.function_call_errors ), - common::gate( - "woken_turn_persisted_result", - wake_action, - format!("exact_write={exact_write}, observed={observed}"), - ), - common::gate( - "timer_run_completed_cleanly", - clean_completion, - format!( - "active_bindings={active_bindings}, function_errors={}, confirmed={confirmed}", - observation.metrics.totals.function_call_errors - ), - ), - ], - awards: vec![ - common::award( - "timer_armed", - if timer_armed { 30 } else { 0 }, - "awarded for one wake-only relative timer before the result write", - ), - common::award( - "parent_woken", - if parent_woken { 30 } else { 0 }, - "awarded when the timer retires after waking only the root session", - ), - common::award( - "wake_action", - if wake_action { 25 } else { 0 }, - "awarded when the woken turn writes the exact result", - ), - common::award( - "clean_completion", - if clean_completion { 15 } else { 0 }, - "awarded for a confirmed result with no errors or live bindings", - ), - ], - }) + ), + ])) }) } @@ -268,6 +240,21 @@ mod tests { }; assert!(is_timer_registration(&call)); - scenario("run").validate().unwrap(); + + let spec = scenario("run"); + spec.validate().unwrap(); + assert_eq!(spec.version, 2); + assert_eq!( + spec.criteria + .iter() + .map(|criterion| (criterion.id, criterion.weight)) + .collect::>(), + vec![ + ("timer_armed", 30), + ("parent_woken", 30), + ("wake_action", 25), + ("clean_completion", 15), + ] + ); } }