diff --git a/.gitignore b/.gitignore index e0ffbb96234..39c1b977459 100644 --- a/.gitignore +++ b/.gitignore @@ -127,3 +127,6 @@ mutants.out.old/ # Skill bundles staged into a workspace at activation. Build output, never source. .skills/ + +# Added by cleanup skill: campaign state, local-only. +.cleanup/ diff --git a/crates/app/ironclaw_architecture_tests/tests/reborn_dependency_boundaries.rs b/crates/app/ironclaw_architecture_tests/tests/reborn_dependency_boundaries.rs index a96e8b5c628..f8cf1233f61 100644 --- a/crates/app/ironclaw_architecture_tests/tests/reborn_dependency_boundaries.rs +++ b/crates/app/ironclaw_architecture_tests/tests/reborn_dependency_boundaries.rs @@ -875,7 +875,14 @@ fn reborn_contracts_crates_carry_a_checked_size_ceiling() { // belong beside the turn contract consumed across loop families. // 20_156 -> 20_334 (2026-08-19, #7686 restack): capability dispatch-result // declarations retained beside main's provider-neutral output contracts. - ("ironclaw_host_api", 20_334), + // 20_334 -> 20_369 (2026-08-19, #7752 merge): +14 for `turn::ActivationProvenance`, + // the enum tagging why a run was created (Human / ParentAgent / System). + // It is turn vocabulary, and `ironclaw_turns` may not own turn + // vocabulary — a new turn type goes to `host_api` by that crate's own + // charter — so there is no lower crate to move it to. Pure DTO: three + // unit variants and a serde derive, no behavior. Value re-captured from + // this test's own report on the merged tree, never counted by eye. + ("ironclaw_host_api", 20_369), // 14_479 -> 13_949 (2026-08-07, #7157): downward re-capture after the // delivery-heuristic vocabulary (stored trigger delivery targets and // their run-profile plumbing) left this crate with the two-lane diff --git a/crates/app/ironclaw_composition/src/automation/conversation_turn_submitter.rs b/crates/app/ironclaw_composition/src/automation/conversation_turn_submitter.rs index c2c009ea37a..610e5e88d56 100644 --- a/crates/app/ironclaw_composition/src/automation/conversation_turn_submitter.rs +++ b/crates/app/ironclaw_composition/src/automation/conversation_turn_submitter.rs @@ -75,6 +75,7 @@ fn coordinator_submit_request(submission: ConversationTurnSubmission) -> SubmitT product_context.execution_policy = submission.execution_policy; } SubmitTurnRequest { + subagent_activation_provenance: None, requested_model: None, scope: submission.scope, actor: submission.actor, @@ -139,6 +140,13 @@ pub(crate) fn turn_submission_error(error: TurnError) -> TurnSubmissionError { TurnSubmissionErrorCategory::InvalidRequest, TurnSubmissionRetry::Permanent, ), + // A thread that spent its autonomous-wake budget is parked pending + // human attention, not permanently refused: the trigger poller may + // retry, and a human activation restores the budget. + AdmissionRejectionReason::SystemWakeStreak => ( + TurnSubmissionErrorCategory::AdmissionRejected, + TurnSubmissionRetry::RetryableAfterKeyRotation, + ), AdmissionRejectionReason::Policy | AdmissionRejectionReason::Unauthorized => ( TurnSubmissionErrorCategory::Unauthorized, TurnSubmissionRetry::Permanent, @@ -232,6 +240,13 @@ mod tests { TurnSubmissionErrorCategory::InvalidRequest, TurnSubmissionRetry::Permanent, ), + ( + TurnError::AdmissionRejected(AdmissionRejection::new( + AdmissionRejectionReason::SystemWakeStreak, + )), + TurnSubmissionErrorCategory::AdmissionRejected, + TurnSubmissionRetry::RetryableAfterKeyRotation, + ), ( TurnError::AdmissionRejected(AdmissionRejection::new( AdmissionRejectionReason::Policy, @@ -351,7 +366,7 @@ mod tests { distinct.len(), 12, "TurnError has 12 variants; the class table must name every one \ - (AdmissionRejected appears four times, once per rejection reason)" + (AdmissionRejected appears six times, once per rejection reason)" ); } diff --git a/crates/app/ironclaw_composition/src/factory/auth_tests.rs b/crates/app/ironclaw_composition/src/factory/auth_tests.rs index 31eb6279220..cbe9ea2accf 100644 --- a/crates/app/ironclaw_composition/src/factory/auth_tests.rs +++ b/crates/app/ironclaw_composition/src/factory/auth_tests.rs @@ -192,6 +192,7 @@ async fn standalone_oauth_turn_gate_callback_resumes_default_turn_coordinator() let actor = TurnActor::new(UserId::new("alice").unwrap()); let submit = turn_coordinator .submit_turn(SubmitTurnRequest { + subagent_activation_provenance: None, requested_model: None, output_contract: None, scope: scope.clone(), @@ -824,6 +825,7 @@ async fn submit_and_block_provider_auth_run( ) -> TurnRunId { let submit = turn_coordinator .submit_turn(SubmitTurnRequest { + subagent_activation_provenance: None, requested_model: None, output_contract: None, scope: scope.clone(), @@ -968,6 +970,7 @@ async fn submit_and_block_auth_run( ) -> ironclaw_host_api::turn::TurnRunId { let submit = turn_coordinator .submit_turn(SubmitTurnRequest { + subagent_activation_provenance: None, requested_model: None, output_contract: None, scope: scope.clone(), diff --git a/crates/app/ironclaw_composition/src/factory/tests.rs b/crates/app/ironclaw_composition/src/factory/tests.rs index 6a5eb67703f..c8017ff555c 100644 --- a/crates/app/ironclaw_composition/src/factory/tests.rs +++ b/crates/app/ironclaw_composition/src/factory/tests.rs @@ -2040,6 +2040,7 @@ async fn production_libsql_turn_state_uses_configured_runtime_identity() { Some(owner.clone()), ); let submit = ironclaw_turns::SubmitTurnRequest { + subagent_activation_provenance: None, requested_model: None, output_contract: None, scope, @@ -2129,6 +2130,7 @@ async fn production_libsql_turn_state_uses_default_runtime_identity_when_unconfi Some(owner.clone()), ); let submit = ironclaw_turns::SubmitTurnRequest { + subagent_activation_provenance: None, requested_model: None, output_contract: None, scope, diff --git a/crates/app/ironclaw_composition/src/runtime.rs b/crates/app/ironclaw_composition/src/runtime.rs index 1f65b64c861..48715e39227 100644 --- a/crates/app/ironclaw_composition/src/runtime.rs +++ b/crates/app/ironclaw_composition/src/runtime.rs @@ -2328,6 +2328,7 @@ impl RebornRuntime { let response = match self .turn_coordinator .submit_turn(SubmitTurnRequest { + subagent_activation_provenance: None, requested_model: accepted.replay_metadata.resolved_model.clone(), scope: scope.clone(), actor: TurnActor::new(self.actor_user_id.clone()), diff --git a/crates/app/ironclaw_composition/src/runtime/tests/core.rs b/crates/app/ironclaw_composition/src/runtime/tests/core.rs index 010a3f75984..f0f32a10b05 100644 --- a/crates/app/ironclaw_composition/src/runtime/tests/core.rs +++ b/crates/app/ironclaw_composition/src/runtime/tests/core.rs @@ -4257,6 +4257,7 @@ async fn cancel_run_propagates_to_children_when_event_sink_is_unavailable() { let parent = runtime .turn_coordinator .submit_turn(SubmitTurnRequest { + subagent_activation_provenance: None, requested_model: None, output_contract: None, scope: parent_scope.clone(), @@ -7195,6 +7196,7 @@ async fn deferred_busy_message_not_auto_submitted_after_run_cancellation() { let submitted_a = runtime .turn_coordinator .submit_turn(SubmitTurnRequest { + subagent_activation_provenance: None, requested_model: None, output_contract: None, scope: scope.clone(), diff --git a/crates/app/ironclaw_composition/tests/service_factory.rs b/crates/app/ironclaw_composition/tests/service_factory.rs index d75d271d451..3db56a8170e 100644 --- a/crates/app/ironclaw_composition/tests/service_factory.rs +++ b/crates/app/ironclaw_composition/tests/service_factory.rs @@ -1584,6 +1584,7 @@ async fn production_postgres_process_journal_pool_writes_rows_the_data_plane_rea ironclaw_turns::TurnCoordinator::submit_turn( services.turn_coordinator_for_test().as_ref(), ironclaw_turns::SubmitTurnRequest { + subagent_activation_provenance: None, requested_model: None, output_contract: None, scope, diff --git a/crates/app/ironclaw_composition/tests/webui_v2_serve.rs b/crates/app/ironclaw_composition/tests/webui_v2_serve.rs index 1ed9e6e044e..86b5ba16398 100644 --- a/crates/app/ironclaw_composition/tests/webui_v2_serve.rs +++ b/crates/app/ironclaw_composition/tests/webui_v2_serve.rs @@ -664,6 +664,7 @@ mod openai_compat_mount_tests { let response = self .coordinator .submit_turn(SubmitTurnRequest { + subagent_activation_provenance: None, scope, actor: caller.actor(), accepted_message_ref: accepted_message_ref.clone(), diff --git a/crates/contracts/ironclaw_host_api/src/turn.rs b/crates/contracts/ironclaw_host_api/src/turn.rs index e4c13c8b08b..ed4158586ae 100644 --- a/crates/contracts/ironclaw_host_api/src/turn.rs +++ b/crates/contracts/ironclaw_host_api/src/turn.rs @@ -859,6 +859,21 @@ impl From for String { } } +/// Why a run was created on its thread. Set once at run creation and immutable +/// thereafter — the derived activation-streak caps read bounded windows of this +/// field instead of maintaining a stored counter. +/// +/// `Human` is the ordinary case and resets both streaks. `ParentAgent` tags a +/// parent re-activating one of its own children. `System` tags a background +/// subagent completion waking its parent. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ActivationProvenance { + Human, + ParentAgent, + System, +} + /// The lifecycle position of one turn run. Terminal, gate-parked, and in-flight /// statuses are distinguished by the predicates below rather than by scattered /// match tables at the call sites. @@ -1185,6 +1200,26 @@ pub enum SubmitTurnResponse { mod tests { use super::*; + #[test] + fn activation_provenance_wire_strings_are_snake_case() { + assert_eq!( + serde_json::to_value(ActivationProvenance::Human).expect("serialize"), + serde_json::json!("human") + ); + assert_eq!( + serde_json::to_value(ActivationProvenance::ParentAgent).expect("serialize"), + serde_json::json!("parent_agent") + ); + assert_eq!( + serde_json::to_value(ActivationProvenance::System).expect("serialize"), + serde_json::json!("system") + ); + + let round_tripped: ActivationProvenance = + serde_json::from_value(serde_json::json!("parent_agent")).expect("deserialize"); + assert_eq!(round_tripped, ActivationProvenance::ParentAgent); + } + #[test] fn blocked_external_tool_status_is_non_terminal_and_keeps_lock() { assert!(!TurnStatus::BlockedExternalTool.is_terminal()); diff --git a/crates/domains/ironclaw_conversations/src/inbound.rs b/crates/domains/ironclaw_conversations/src/inbound.rs index 4294568f5d4..3e11a3de4ae 100644 --- a/crates/domains/ironclaw_conversations/src/inbound.rs +++ b/crates/domains/ironclaw_conversations/src/inbound.rs @@ -1381,6 +1381,7 @@ mod tests { product_context.execution_policy = submission.execution_policy; } SubmitTurnRequest { + subagent_activation_provenance: None, requested_model: None, scope: submission.scope, actor: submission.actor, diff --git a/crates/domains/ironclaw_conversations/tests/inbound_contract.rs b/crates/domains/ironclaw_conversations/tests/inbound_contract.rs index 6228f08e0b2..35bc50a7b89 100644 --- a/crates/domains/ironclaw_conversations/tests/inbound_contract.rs +++ b/crates/domains/ironclaw_conversations/tests/inbound_contract.rs @@ -4110,6 +4110,7 @@ fn submit_turn_request(submission: ConversationTurnSubmission) -> SubmitTurnRequ product_context.execution_policy = submission.execution_policy; } SubmitTurnRequest { + subagent_activation_provenance: None, requested_model: None, output_contract: None, scope: submission.scope, diff --git a/crates/kernel/ironclaw_host_runtime/tests/support/host_runtime_harness.rs b/crates/kernel/ironclaw_host_runtime/tests/support/host_runtime_harness.rs index 4e9f22ce4cb..e0a345d4345 100644 --- a/crates/kernel/ironclaw_host_runtime/tests/support/host_runtime_harness.rs +++ b/crates/kernel/ironclaw_host_runtime/tests/support/host_runtime_harness.rs @@ -2257,6 +2257,7 @@ pub(crate) fn http_without_body_then_operation_failed_wat() -> String { pub(crate) fn submit_turn_request(thread: &str, idempotency_key: &str) -> SubmitTurnRequest { SubmitTurnRequest { + subagent_activation_provenance: None, requested_model: None, output_contract: None, scope: TurnScope::new( diff --git a/crates/kernel/ironclaw_processes/src/journal.rs b/crates/kernel/ironclaw_processes/src/journal.rs index 18358df4718..e25d2f3f03e 100644 --- a/crates/kernel/ironclaw_processes/src/journal.rs +++ b/crates/kernel/ironclaw_processes/src/journal.rs @@ -951,6 +951,16 @@ pub trait ProcessSnapshotSource: Send + Sync { &self, scope: &ResourceScope, ) -> Result, Self::Error>; + + /// Newest-first, bounded read of one scope's agent-turn processes. + /// + /// Unlike [`Self::process_snapshots`] this is explicitly bounded: callers + /// that need a fixed recent window must not enumerate a whole scope. + async fn recent_agent_turn_snapshots( + &self, + scope: &ResourceScope, + limit: u32, + ) -> Result, Self::Error>; } #[async_trait] diff --git a/crates/kernel/ironclaw_processes/src/journal_store.rs b/crates/kernel/ironclaw_processes/src/journal_store.rs index 6906002e844..06e3e5cc325 100644 --- a/crates/kernel/ironclaw_processes/src/journal_store.rs +++ b/crates/kernel/ironclaw_processes/src/journal_store.rs @@ -733,6 +733,24 @@ where snapshots.sort_by_key(|snapshot| snapshot.process_id.as_uuid()); Ok(snapshots) } + + async fn recent_agent_turn_snapshots( + &self, + scope: &ResourceScope, + limit: u32, + ) -> Result, Self::Error> { + self.ensure_materialized().await?; + // `is_system()`, not `== ResourceScope::system()`: the constructor + // mints a fresh `invocation_id` on every call, so an equality check + // against it can never match and the guard would be dead. + if scope.is_system() { + return Err(ProcessJournalStoreError::InvalidRequest( + "system-wide process snapshot reads are unbounded; use paged process journal reads" + .to_string(), + )); + } + rows::recent_agent_turn_processes_for_scope(self.filesystem.as_ref(), scope, limit).await + } } #[async_trait] diff --git a/crates/kernel/ironclaw_processes/src/journal_store/rows.rs b/crates/kernel/ironclaw_processes/src/journal_store/rows.rs index a39c6194274..3775d9d192a 100644 --- a/crates/kernel/ironclaw_processes/src/journal_store/rows.rs +++ b/crates/kernel/ironclaw_processes/src/journal_store/rows.rs @@ -710,6 +710,80 @@ where decode_process_rows(&rows) } +/// Newest-first, bounded read of one scope's `AgentTurn` processes. +/// +/// `process_scope_v3` is keyed on `(scope_key, created_at, process_id)` and +/// deliberately not on `process_kind`, while a thread's scope also holds +/// capability-invocation processes. A flat `LIMIT` could therefore be +/// satisfied entirely by non-`AgentTurn` rows, so this walks the descending +/// keyset a page at a time and filters by kind in memory until it has `limit` +/// agent-turn rows. +/// +/// Bounded by `MAX_RECENT_PROCESS_PAGES` rather than walking a scope's whole +/// history: a thread whose newest ~8 pages hold no agent-turn run returns a +/// short window. The only caller is the derived activation-streak cap, which +/// treats a short window as "streak not established" — the same disposition it +/// gives a genuinely young thread. +pub(super) async fn recent_agent_turn_processes_for_scope( + filesystem: &ScopedFilesystem, + scope: &ResourceScope, + limit: u32, +) -> Result, ProcessJournalStoreError> +where + F: RootFilesystem, +{ + const MAX_RECENT_PROCESS_PAGES: u32 = 8; + + if limit == 0 { + return Ok(Vec::new()); + } + let prefix = scoped_path(&format!("{MATERIALIZED_PREFIX}/process"))?; + let filter = eq_value("scope_key", IndexValue::Text(scope_owner_key(scope)?))?; + // Over-fetch per page so a scope interleaving capability-invocation + // processes with its runs still makes progress on every round trip. + let page_limit = limit.saturating_mul(4).clamp(1, Page::MAX_LIMIT); + + let mut collected: Vec = Vec::new(); + let mut cursor: Option = None; + for _ in 0..MAX_RECENT_PROCESS_PAGES { + let mut page = ironclaw_filesystem::OrderedPage::new( + index_name("process_scope_v3")?, + index_key("created_at")?, + index_key("process_id")?, + SortDirection::Descending, + page_limit, + ); + if let Some(cursor) = cursor.clone() { + page = page.after(cursor); + } + let rows = filesystem + .query_ordered(&ResourceScope::system(), &prefix, &filter, &page) + .await?; + let exhausted = (rows.len() as u32) < page_limit; + let snapshots = decode_process_rows(&rows)?; + let Some(last) = snapshots.last() else { + // Every row in this page decoded away (tombstones); without a + // decodable row there is no cursor to advance past, so stop rather + // than re-reading the same page. + break; + }; + cursor = Some(ironclaw_filesystem::OrderedQueryCursor { + value: IndexValue::I64(last.created_at.timestamp_micros()), + tie_breaker: IndexValue::Text(last.process_id.as_uuid().to_string()), + }); + collected.extend( + snapshots + .into_iter() + .filter(|snapshot| snapshot.process_kind == ProcessKind::AgentTurn), + ); + if collected.len() >= limit as usize || exhausted { + break; + } + } + collected.truncate(limit as usize); + Ok(collected) +} + pub(super) async fn child_processes( filesystem: &ScopedFilesystem, parent_process_id: ProcessId, diff --git a/crates/kernel/ironclaw_processes/tests/process_journal_store_contract.rs b/crates/kernel/ironclaw_processes/tests/process_journal_store_contract.rs index b899a53ad0e..c238067bda2 100644 --- a/crates/kernel/ironclaw_processes/tests/process_journal_store_contract.rs +++ b/crates/kernel/ironclaw_processes/tests/process_journal_store_contract.rs @@ -4100,10 +4100,11 @@ fn process_input_payload_is_bounded_and_redacted() { assert!(!debug.contains("private-goal")); } -async fn submit_internal_process( +async fn submit_internal_process_at( store: &ProcessJournalStore, scope: &ResourceScope, process_id: ProcessId, + created_at: chrono::DateTime, ) -> ironclaw_processes::JournaledProcessSnapshot where F: ironclaw_filesystem::RootFilesystem + Send + Sync + 'static, @@ -4123,13 +4124,24 @@ where dependency: None, checkpoint_ref: None, input: None, - created_at: Utc::now(), + created_at, metadata: serde_json::Value::Null, }) .await .expect("submit internal process") } +async fn submit_internal_process( + store: &ProcessJournalStore, + scope: &ResourceScope, + process_id: ProcessId, +) -> ironclaw_processes::JournaledProcessSnapshot +where + F: ironclaw_filesystem::RootFilesystem + Send + Sync + 'static, +{ + submit_internal_process_at(store, scope, process_id, Utc::now()).await +} + fn scope() -> ResourceScope { ResourceScope { tenant_id: TenantId::new("tenant-journal").expect("tenant"), @@ -4154,3 +4166,272 @@ fn in_memory_backed_processes_filesystem() -> std::sync::Arc = recent.iter().map(|snapshot| snapshot.process_id).collect(); + let expected: Vec<_> = agent_turn_ids.iter().rev().take(3).copied().collect(); + assert_eq!( + returned, expected, + "must return the newest agent-turn processes, newest first" + ); +} + +/// A limit larger than the history returns everything without error — the +/// young-thread case the streak cap reads as "streak not established". +#[tokio::test] +async fn recent_agent_turn_snapshots_returns_a_short_window_for_a_young_thread() { + let store = ProcessJournalStore::new(in_memory_backed_processes_filesystem()); + let scope = scope(); + + submit_agent_turn_process(&store, &scope, ProcessId::new()).await; + submit_agent_turn_process(&store, &scope, ProcessId::new()).await; + + let recent = store + .recent_agent_turn_snapshots(&scope, 16) + .await + .expect("bounded recent read"); + + assert_eq!(recent.len(), 2); +} + +async fn submit_agent_turn_process_at( + store: &ProcessJournalStore, + scope: &ResourceScope, + process_id: ProcessId, + created_at: chrono::DateTime, +) -> ironclaw_processes::JournaledProcessSnapshot +where + F: ironclaw_filesystem::RootFilesystem + Send + Sync + 'static, +{ + store + .submit_process(SubmitProcessRequest { + process_id, + process_kind: ProcessKind::AgentTurn, + scope: scope.clone(), + exclusive_within_scope: false, + operation_id: None, + owner_user_id: Some(scope.user_id.clone()), + concurrency_class: None, + parent_process_id: None, + root_process_id: None, + spawn_tree_descendant_cap: None, + dependency: None, + checkpoint_ref: None, + input: None, + created_at, + metadata: serde_json::Value::Null, + }) + .await + .expect("agent-turn process submits") +} + +/// The window walk sorts on `(created_at micros, process_id text)` descending, +/// and `ProcessId` is a random UUID — so any two rows sharing a microsecond +/// resolve by UUID, which is not the order a test can predict. Ordering tests +/// seed strictly increasing stamps through this helper so the tie-breaker is +/// never reached. +async fn submit_agent_turn_process( + store: &ProcessJournalStore, + scope: &ResourceScope, + process_id: ProcessId, +) -> ironclaw_processes::JournaledProcessSnapshot +where + F: ironclaw_filesystem::RootFilesystem + Send + Sync + 'static, +{ + submit_agent_turn_process_at(store, scope, process_id, Utc::now()).await +} + +/// The keyset walk pages. With limit=2 the page size is 8, so seeding more +/// than 8 non-agent-turn processes ahead of the agent-turn rows forces the +/// walk to advance its cursor and fetch again — the multi-page path, which a +/// single-page test can never reach. +#[tokio::test] +async fn recent_agent_turn_snapshots_walks_past_a_full_page_of_other_kinds() { + let store = ProcessJournalStore::new(in_memory_backed_processes_filesystem()); + let scope = scope(); + + let mut agent_turn_ids = Vec::new(); + for _ in 0..2 { + agent_turn_ids.push(ProcessId::new()); + submit_agent_turn_process(&store, &scope, *agent_turn_ids.last().expect("just pushed")) + .await; + } + // Newer than every agent-turn row, and more than one page of them. + for _ in 0..12 { + submit_internal_process(&store, &scope, ProcessId::new()).await; + } + + let recent = store + .recent_agent_turn_snapshots(&scope, 2) + .await + .expect("bounded recent read"); + + let returned: Vec<_> = recent.iter().map(|snapshot| snapshot.process_id).collect(); + let expected: Vec<_> = agent_turn_ids.iter().rev().copied().collect(); + assert_eq!( + returned, expected, + "the walk must page past a full page of non-agent-turn rows to fill the window" + ); +} + +/// The bounded read carries its own system-scope rejection, distinct from the +/// unbounded `process_snapshots` it deliberately restricts. +#[tokio::test] +async fn recent_agent_turn_snapshots_rejects_a_system_wide_scope() { + let store = ProcessJournalStore::new(in_memory_backed_processes_filesystem()); + + let error = store + .recent_agent_turn_snapshots(&ResourceScope::system(), 4) + .await + .expect_err("system-wide reads are unbounded and must be refused"); + + assert!( + matches!(error, ProcessJournalStoreError::InvalidRequest(_)), + "expected InvalidRequest, got {error:?}" + ); +} + +/// Backend parity for the bounded window read. +/// +/// This is the first production descending keyset walk with a hand-built +/// cursor, and it is the only one binding an I64 sort value against a Text +/// tie-breaker across pages. The two SQL backends compare that cursor very +/// differently — jsonb operators on PostgreSQL, untyped columns on libSQL — so +/// `.claude/rules/database.md` "Backend parity" wants the adversarial case in a +/// shared body rather than an in-memory-only assertion. +async fn assert_recent_agent_turn_window_parity(store: ProcessJournalStore) +where + F: ironclaw_filesystem::RootFilesystem + Send + Sync + 'static, +{ + let scope = scope(); + + let base = Utc::now(); + let mut agent_turn_ids = Vec::new(); + for index in 0..3 { + let id = ProcessId::new(); + submit_agent_turn_process_at(&store, &scope, id, base + chrono::Duration::seconds(index)) + .await; + agent_turn_ids.push(id); + } + // Newer than every agent-turn row and more than one page, so the walk must + // advance its cursor across pages to fill the window. + for index in 0..12 { + submit_internal_process_at( + &store, + &scope, + ProcessId::new(), + base + chrono::Duration::seconds(100 + index), + ) + .await; + } + + let recent = store + .recent_agent_turn_snapshots(&scope, 2) + .await + .expect("bounded recent read"); + + assert_eq!(recent.len(), 2, "limit must bound agent-turn rows"); + assert!( + recent + .iter() + .all(|snapshot| snapshot.process_kind == ProcessKind::AgentTurn), + "non-agent-turn processes must never be returned" + ); + let returned: Vec<_> = recent.iter().map(|snapshot| snapshot.process_id).collect(); + let expected: Vec<_> = agent_turn_ids.iter().rev().take(2).copied().collect(); + assert_eq!( + returned, expected, + "must return the newest agent-turn processes, newest first, across pages" + ); +} + +#[tokio::test] +async fn recent_agent_turn_snapshots_hold_on_libsql() { + let storage = tempfile::tempdir().expect("temporary process journal database"); + let database = Arc::new( + libsql::Builder::new_local(storage.path().join("recent-window.db")) + .build() + .await + .expect("build libsql database"), + ); + let backend = Arc::new(LibSqlRootFilesystem::new(database).expect("libSQL filesystem runtime")); + backend + .run_migrations() + .await + .expect("migrate libsql filesystem"); + let filesystem = Arc::new(ScopedFilesystem::with_fixed_view( + backend, + MountView::new(vec![MountGrant::new( + MountAlias::new("/processes").expect("mount alias"), + VirtualPath::new("/engine/processes").expect("virtual path"), + MountPermissions::read_write_list_delete(), + )]) + .expect("mount view"), + )); + assert_recent_agent_turn_window_parity(ProcessJournalStore::new(filesystem)).await; +} + +#[tokio::test] +async fn recent_agent_turn_snapshots_hold_on_postgres() { + let Some(backend) = postgres_backend().await else { + return; + }; + let filesystem = Arc::new(ScopedFilesystem::with_fixed_view( + Arc::new(backend), + MountView::new(vec![MountGrant::new( + MountAlias::new("/processes").expect("mount alias"), + VirtualPath::new("/engine/processes").expect("virtual path"), + MountPermissions::read_write_list_delete(), + )]) + .expect("mount view"), + )); + assert_recent_agent_turn_window_parity(ProcessJournalStore::new(filesystem)).await; +} diff --git a/crates/kernel/ironclaw_turns/src/activation_streak.rs b/crates/kernel/ironclaw_turns/src/activation_streak.rs new file mode 100644 index 00000000000..16343839a23 --- /dev/null +++ b/crates/kernel/ironclaw_turns/src/activation_streak.rs @@ -0,0 +1,166 @@ +//! Derived activation-streak caps. +//! +//! Deliberately no stored counter and no new component: each cap is a +//! predicate over a bounded, newest-first window of the thread's own run +//! records, fetched with the complementary provenance excluded. + +use crate::{ActivationProvenance, TurnRunRecord}; + +/// Consecutive `System`-provenance activations allowed on one thread before +/// the reactive wake is refused. +/// +/// Independently named on purpose. It coincides numerically with the +/// 16-descendant spawn-tree cap and the SUBAGENT loop family's 16-iteration +/// limit, and those three budgets must never be merged by a refactor — they +/// bound unrelated things and would drift apart the moment one is tuned. +pub const SYSTEM_WAKE_STREAK_CAP: u32 = 16; + +/// How many raw run records to fetch per cap-sized window so that, after +/// `ParentAgent` runs are dropped, `SYSTEM_WAKE_STREAK_CAP` records remain. +/// +/// The window query is provenance-blind, so the design's "excluded from the +/// fetch" rule is approximated by over-fetching and truncating. A thread whose +/// recent history is more than `(OVERFETCH - 1) / OVERFETCH` `ParentAgent` +/// runs still yields a short window, which admits — a fail-open residual, not +/// a guarantee. Once the extend cap bounds consecutive `ParentAgent` +/// activations at 8, raising this to 9 makes the exclusion exact. +pub const SYSTEM_WAKE_WINDOW_OVERFETCH: u32 = 4; + +/// Whether a pending `System` activation may be admitted, given the thread's +/// newest-first window of `Human`/`System` runs (`ParentAgent` runs are +/// excluded by the caller's fetch, so they neither count nor reset). +/// +/// Refusing costs nothing durable: a settled await-edge stays settled and +/// drains via the run-start sweep or the boot pass. This gates the reactive +/// wake only, never delivery itself. +/// +/// An untagged run (`None`) is an ordinary human-initiated submission and +/// resets the streak, same as an explicit `Human`. +pub fn system_wake_admitted(recent: &[TurnRunRecord]) -> bool { + if recent.len() < SYSTEM_WAKE_STREAK_CAP as usize { + return true; + } + recent + .iter() + .take(SYSTEM_WAKE_STREAK_CAP as usize) + .any(|record| record.subagent_activation_provenance != Some(ActivationProvenance::System)) +} + +#[cfg(test)] +mod tests { + use super::{SYSTEM_WAKE_STREAK_CAP, system_wake_admitted}; + use crate::{ActivationProvenance, TurnRunRecord}; + + fn record_with(provenance: Option) -> TurnRunRecord { + use crate::{AcceptedMessageRef, EventCursor, TurnRunId, TurnScope, TurnStatus}; + use ironclaw_host_api::ids::{AgentId, ProjectId, TenantId, ThreadId}; + + let profile: crate::TurnRunProfile = serde_json::from_value(serde_json::json!({ + "id": "default", + "version": 1, + "allow_steering": false, + "auto_queue_followups": false, + })) + .expect("profile deserialization"); + TurnRunRecord { + run_id: TurnRunId::new(), + turn_id: crate::TurnId::new(), + scope: TurnScope::new( + TenantId::new("tenant-streak-test").expect("tenant"), + Some(AgentId::new("agent-streak-test").expect("agent")), + Some(ProjectId::new("project-streak-test").expect("project")), + ThreadId::new("thread-streak-test").expect("thread"), + ), + accepted_message_ref: AcceptedMessageRef::new("accepted-streak-test") + .expect("accepted"), + status: TurnStatus::Completed, + profile, + output_contract: Default::default(), + resolved_model_route: None, + model_usage: None, + execution_outcome: None, + checkpoint_id: None, + gate_ref: None, + blocked_activity_id: None, + credential_requirements: Vec::new(), + failure: None, + event_cursor: EventCursor(1), + runner_id: None, + lease_token: None, + lease_expires_at: None, + last_heartbeat_at: None, + claim_count: 0, + received_at: chrono::Utc::now(), + parent_run_id: None, + subagent_depth: 0, + spawn_tree_root_run_id: None, + subagent_activation_provenance: provenance, + product_context: None, + resume_disposition: None, + } + } + + /// `provenances[0]` is the newest run. + fn window(provenances: &[ActivationProvenance]) -> Vec { + provenances + .iter() + .map(|provenance| record_with(Some(*provenance))) + .collect() + } + + #[test] + fn under_cap_consecutive_system_wakes_are_admitted() { + let recent = window(&[ActivationProvenance::System; 15]); + assert!( + system_wake_admitted(&recent), + "15 consecutive System wakes is under the cap of {SYSTEM_WAKE_STREAK_CAP}" + ); + } + + #[test] + fn a_full_window_of_system_wakes_refuses_the_next_one() { + let recent = window(&[ActivationProvenance::System; SYSTEM_WAKE_STREAK_CAP as usize]); + assert!( + !system_wake_admitted(&recent), + "a full window of System runs means the pending wake would be the 17th consecutive one" + ); + } + + #[test] + fn a_human_activation_anywhere_in_the_window_resets_the_streak() { + let mut provenances = [ActivationProvenance::System; SYSTEM_WAKE_STREAK_CAP as usize]; + provenances[SYSTEM_WAKE_STREAK_CAP as usize - 1] = ActivationProvenance::Human; + assert!( + system_wake_admitted(&window(&provenances)), + "a Human run anywhere in the window resets the streak" + ); + } + + #[test] + fn a_short_history_is_admitted() { + let recent = window(&[ActivationProvenance::System; 3]); + assert!( + system_wake_admitted(&recent), + "a young thread with fewer than {SYSTEM_WAKE_STREAK_CAP} records must be admitted" + ); + } + + #[test] + fn untagged_legacy_runs_count_as_human_and_reset_the_streak() { + let mut recent = + window(&[ActivationProvenance::System; SYSTEM_WAKE_STREAK_CAP as usize - 1]); + recent.push(record_with(None)); + assert!( + system_wake_admitted(&recent), + "an untagged run is an ordinary human-initiated run and must reset the streak" + ); + } + + /// The three 16-valued budgets in this subsystem bound unrelated things. + /// This pins the wake cap's own identity so a future refactor cannot + /// quietly alias it to the descendant cap or the iteration limit. + #[test] + fn the_wake_streak_cap_is_its_own_named_budget() { + assert_eq!(SYSTEM_WAKE_STREAK_CAP, 16); + } +} diff --git a/crates/kernel/ironclaw_turns/src/agent_turn_runtime.rs b/crates/kernel/ironclaw_turns/src/agent_turn_runtime.rs index 3e2bd372841..e4f14d458f9 100644 --- a/crates/kernel/ironclaw_turns/src/agent_turn_runtime.rs +++ b/crates/kernel/ironclaw_turns/src/agent_turn_runtime.rs @@ -41,6 +41,27 @@ pub trait AgentTurnRuntimePort: Send + Sync { /// [`TurnError::ScopeNotFound`]. This keeps scoped lookups non-enumerating /// and gives higher-level helpers one canonical missing-run shape. async fn get_run_state(&self, request: GetRunStateRequest) -> Result; + + /// The newest `limit` agent-turn runs on this exact thread scope, newest + /// first. + /// + /// Bounded by construction: the derived activation-streak caps read a + /// fixed window of recent runs instead of keeping a stored counter, so + /// this must never enumerate a thread's whole history. + /// + /// Defaults to a refusal rather than an empty window. An empty window + /// reads as "streak not established" and would admit every autonomous + /// wake, so a runtime that cannot answer this question must not be able + /// to silently disable the cap that depends on it. + async fn recent_runs_for_thread( + &self, + _scope: &TurnScope, + _limit: u32, + ) -> Result, TurnError> { + Err(TurnError::InvalidRequest { + reason: "this turn runtime cannot read a thread's recent run window".to_string(), + }) + } } /// Classify an active run reference through the shared turn-state lookup. @@ -183,6 +204,9 @@ pub struct TurnRunRecord { pub subagent_depth: u32, #[serde(default, skip_serializing_if = "Option::is_none")] pub spawn_tree_root_run_id: Option, + /// Why this run was activated. Immutable after creation. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub subagent_activation_provenance: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub product_context: Option, #[serde( @@ -234,6 +258,7 @@ mod tests { })) .expect("profile deserialization"); TurnRunRecord { + subagent_activation_provenance: None, run_id: TurnRunId::new(), turn_id: crate::TurnId::new(), scope, diff --git a/crates/kernel/ironclaw_turns/src/coordinator.rs b/crates/kernel/ironclaw_turns/src/coordinator.rs index 767d918e06f..b4ce913fa1a 100644 --- a/crates/kernel/ironclaw_turns/src/coordinator.rs +++ b/crates/kernel/ironclaw_turns/src/coordinator.rs @@ -56,12 +56,13 @@ fn trace_coordinator_latency_error( } use crate::{ - AdmissionRejection, AdmissionRejectionReason, AgentTurnRuntimePort, - AgentTurnSpawnTreeRuntimePort, CancelRunRequest, CancelRunResponse, EventCursor, - GetRunStateRequest, ResumeTurnRequest, ResumeTurnResponse, RetryTurnRequest, RetryTurnResponse, - RunProfileId, RunProfileRequest, SubmitChildRunRequest, SubmitTurnRequest, SubmitTurnResponse, + ActivateThreadRequest, ActivationProvenance, AdmissionRejection, AdmissionRejectionReason, + AgentTurnRuntimePort, AgentTurnSpawnTreeRuntimePort, CancelRunRequest, CancelRunResponse, + EventCursor, GetRunStateRequest, ResumeTurnRequest, ResumeTurnResponse, RetryTurnRequest, + RetryTurnResponse, RunProfileId, RunProfileRequest, SYSTEM_WAKE_STREAK_CAP, + SYSTEM_WAKE_WINDOW_OVERFETCH, SubmitChildRunRequest, SubmitTurnRequest, SubmitTurnResponse, TurnCapacityResource, TurnError, TurnOriginKind, TurnRunId, TurnRunState, TurnScope, - TurnStatus, process_projection::AgentTurnProcessRuntime, + TurnStatus, process_projection::AgentTurnProcessRuntime, system_wake_admitted, }; use ironclaw_host_api::prepared_context::{ PreparedContextSource, PreparedTurnDeclarations, TurnLimits, @@ -139,6 +140,23 @@ pub trait TurnCoordinator: Send + Sync { request: SubmitTurnRequest, ) -> Result; + /// Re-activate an existing thread with a provenance tag. + /// + /// Defaults to a refusal rather than an untagged `submit_turn` + /// fallthrough: a coordinator that has not opted into activation semantics + /// must not silently create runs whose provenance the streak caps then + /// cannot see. `DefaultTurnCoordinator` provides the real implementation; + /// this default exists so the many test doubles of this trait need not + /// each restate it (the same reason `abort_prepared_turn` carries one). + async fn activate( + &self, + _request: ActivateThreadRequest, + ) -> Result { + Err(TurnError::InvalidRequest { + reason: "this coordinator does not support thread activation".to_string(), + }) + } + async fn resume_turn( &self, request: ResumeTurnRequest, @@ -584,6 +602,94 @@ where Ok(response) } + async fn activate( + &self, + request: ActivateThreadRequest, + ) -> Result { + // Autonomous-wake containment. Nothing else bounds the cumulative + // spawn -> settle -> wake -> spawn cycle: a parent that spawns a fresh + // child on every background completion would otherwise loop + // indefinitely under every existing cap, with no human in it. + // + // Refusing costs nothing durable. A settled await-edge stays settled + // and drains via the run-start sweep or the boot pass, so this gates + // the reactive wake only, never delivery. + if request.provenance == ActivationProvenance::System { + // ParentAgent runs are outside this window entirely: they neither + // count toward the System streak nor reset it, so that any + // human-free sequence — however interleaved — stays bounded by + // both caps. + // + // They must be excluded from the FETCH, not filtered after it. + // Filtering a K-sized fetch returns fewer than K records the + // moment ParentAgent runs are interleaved, and a short window + // reads as "streak not established" and admits — which silently + // disabled this cap on exactly the interleaved human-free + // sequences it exists to bound. The window query is + // provenance-blind, so the exclusion is done by over-fetching and + // truncating to the cap. + let fetch_limit = SYSTEM_WAKE_STREAK_CAP.saturating_mul(SYSTEM_WAKE_WINDOW_OVERFETCH); + let raw = self + .store + .recent_runs_for_thread(&request.scope, fetch_limit) + .await?; + // A retry of an already-accepted activation must reach + // `submit_turn`'s durable idempotency replay, not be refused by + // the run it created on its first attempt. Excluding the caller's + // own accepted message keeps `activate()`'s advertised + // "ordinary submission idempotency" true at the cap boundary. + let recent = raw + .iter() + .filter(|record| record.accepted_message_ref != request.accepted_message_ref) + .filter(|record| { + record.subagent_activation_provenance != Some(ActivationProvenance::ParentAgent) + }) + .take(SYSTEM_WAKE_STREAK_CAP as usize) + .cloned() + .collect::>(); + // Fail closed when the window could not be established. The store + // returns fewer rows than asked only when history ran out, so a + // *full* fetch that still cannot yield a cap-sized non-ParentAgent + // window means the streak is unknown, not absent — admitting there + // is the fail-open hole the over-fetch alone left behind. A + // genuinely short fetch is a young thread and still admits. + let window_crowded_out = u32::try_from(raw.len()).unwrap_or(u32::MAX) >= fetch_limit + && (recent.len() as u32) < SYSTEM_WAKE_STREAK_CAP; + if window_crowded_out || !system_wake_admitted(&recent) { + debug!( + thread_id = %request.scope.thread_id, + cap = SYSTEM_WAKE_STREAK_CAP, + "refusing System activation: consecutive-wake streak cap reached" + ); + return Err(TurnError::AdmissionRejected(AdmissionRejection::new( + AdmissionRejectionReason::SystemWakeStreak, + ))); + } + } + + // Routed through this coordinator's own `submit_turn` on purpose: + // admission, idempotency replay, profile resolution, and the wake + // notification are shared with every other submission. The only + // difference an activation makes is the provenance stamp. + self.submit_turn(SubmitTurnRequest { + scope: request.scope, + actor: request.actor, + accepted_message_ref: request.accepted_message_ref, + requested_run_profile: request.requested_run_profile, + output_contract: None, + requested_model: None, + idempotency_key: request.idempotency_key, + received_at: request.received_at, + requested_run_id: None, + parent_run_id: None, + subagent_depth: 0, + spawn_tree_root_run_id: None, + product_context: None, + subagent_activation_provenance: Some(request.provenance), + }) + .await + } + async fn resume_turn( &self, request: ResumeTurnRequest, @@ -787,6 +893,13 @@ where self.as_ref().submit_turn(request).await } + async fn activate( + &self, + request: ActivateThreadRequest, + ) -> Result { + self.as_ref().activate(request).await + } + async fn resume_turn( &self, request: ResumeTurnRequest, diff --git a/crates/kernel/ironclaw_turns/src/lib.rs b/crates/kernel/ironclaw_turns/src/lib.rs index e0e9c27ffe1..76a9f25d8dd 100644 --- a/crates/kernel/ironclaw_turns/src/lib.rs +++ b/crates/kernel/ironclaw_turns/src/lib.rs @@ -6,6 +6,7 @@ //! transition APIs are intentionally not re-exported from this crate prelude. #![warn(unreachable_pub)] +mod activation_streak; mod admission; mod agent_turn_runtime; mod checkpoint_state; @@ -23,6 +24,9 @@ mod status; #[cfg(any(test, feature = "test-support"))] pub mod test_support; +pub use activation_streak::{ + SYSTEM_WAKE_STREAK_CAP, SYSTEM_WAKE_WINDOW_OVERFETCH, system_wake_admitted, +}; pub use admission::{ AllowAllTurnAdmissionLimitProvider, StaticTurnAdmissionLimitProvider, TurnAdmissionAxisKind, TurnAdmissionBucket, TurnAdmissionBucketKind, TurnAdmissionBucketScope, @@ -62,13 +66,13 @@ pub use external_tool_catalog::{ // `ironclaw_host_api` directly — see this crate's CLAUDE.md. pub use host_managed_ports::{HostManagedLoopModelPort, HostManagedLoopPromptPort}; pub use ironclaw_host_api::turn::{ - AcceptedMessageRef, BlockedReason, CapabilityActivityId, EventCursor, GateKind, - GateResumeDisposition, IdempotencyKey, LoopExitId, LoopGateRef, LoopMessageRef, LoopResultRef, - ModelInvalidOutputDetailReason, ProductTurnContext, ReplyTargetBindingRef, RunOriginAdapter, - RunProfileId, RunProfileRequest, RunProfileVersion, SanitizedCancelReason, SanitizedFailure, - SourceBindingRef, SubmitTurnResponse, TurnActor, TurnCheckpointId, TurnExecutionOutcome, - TurnGateRef, TurnId, TurnLeaseToken, TurnOriginKind, TurnOwner, TurnRunId, TurnRunnerId, - TurnScope, TurnStatus, TurnSurfaceType, + AcceptedMessageRef, ActivationProvenance, BlockedReason, CapabilityActivityId, EventCursor, + GateKind, GateResumeDisposition, IdempotencyKey, LoopExitId, LoopGateRef, LoopMessageRef, + LoopResultRef, ModelInvalidOutputDetailReason, ProductTurnContext, ReplyTargetBindingRef, + RunOriginAdapter, RunProfileId, RunProfileRequest, RunProfileVersion, SanitizedCancelReason, + SanitizedFailure, SourceBindingRef, SubmitTurnResponse, TurnActor, TurnCheckpointId, + TurnExecutionOutcome, TurnGateRef, TurnId, TurnLeaseToken, TurnOriginKind, TurnOwner, + TurnRunId, TurnRunnerId, TurnScope, TurnStatus, TurnSurfaceType, }; pub use loop_exit::{ BlockedEvidenceRequest, CompletionEvidenceRequest, FailureEvidenceRequest, @@ -82,8 +86,8 @@ pub use process_projection::{ claimed_turn_run_from_process_claim, turn_run_state_from_process_snapshot, }; pub use request::{ - CancelRunRequest, GetRunStateRequest, ResumeTurnPrecondition, ResumeTurnRequest, - RetryTurnRequest, SubmitChildRunRequest, SubmitTurnRequest, TurnTimestamp, + ActivateThreadRequest, CancelRunRequest, GetRunStateRequest, ResumeTurnPrecondition, + ResumeTurnRequest, RetryTurnRequest, SubmitChildRunRequest, SubmitTurnRequest, TurnTimestamp, }; pub use response::{CancelRunResponse, ResumeTurnResponse, RetryTurnResponse, ThreadBusy}; pub use status::{ diff --git a/crates/kernel/ironclaw_turns/src/process_projection/metadata.rs b/crates/kernel/ironclaw_turns/src/process_projection/metadata.rs index 23752337962..3088b1d1b1f 100644 --- a/crates/kernel/ironclaw_turns/src/process_projection/metadata.rs +++ b/crates/kernel/ironclaw_turns/src/process_projection/metadata.rs @@ -5,8 +5,9 @@ use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; use crate::{ - AcceptedMessageRef, GateResumeDisposition, ProductTurnContext, RunProfileId, RunProfileVersion, - TurnActor, TurnRunRecord, TurnRunState, runner::ClaimedTurnRun, + AcceptedMessageRef, ActivationProvenance, GateResumeDisposition, ProductTurnContext, + RunProfileId, RunProfileVersion, TurnActor, TurnRunRecord, TurnRunState, + runner::ClaimedTurnRun, }; use ironclaw_host_api::turn::TurnExecutionOutcome; use ironclaw_loop_contracts::{LoopModelRouteSnapshot, LoopModelUsage, ResolvedRunProfile}; @@ -99,6 +100,11 @@ pub struct AgentTurnProcessStateMetadata { pub execution_outcome: Option, #[serde(default)] pub subagent_depth: u32, + /// Why this run was activated on its thread. Set once at run creation, + /// never mutated. Absent on rows written before the field existed, and on + /// every ordinary human-initiated submission. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub subagent_activation_provenance: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub spawn_tree_descendant_cap: Option, #[serde(default, skip_serializing_if = "Option::is_none")] @@ -132,6 +138,10 @@ impl AgentTurnProcessStateMetadata { model_usage: state.model_usage, execution_outcome: state.execution_outcome, subagent_depth: 0, + // State-derived rewrites do not carry lineage (see subagent_depth + // above); the durable provenance stays on the originally journaled + // metadata. + subagent_activation_provenance: None, spawn_tree_descendant_cap: None, product_context: state.product_context.clone(), resume_disposition: state.resume_disposition.clone(), @@ -145,6 +155,7 @@ impl AgentTurnProcessStateMetadata { resolved_run_profile: Some(claimed.resolved_run_profile.clone()), subagent_depth: claimed.subagent_depth, spawn_tree_descendant_cap: claimed.spawn_tree_descendant_cap, + subagent_activation_provenance: claimed.subagent_activation_provenance, ..Self::from_state(&claimed.state) } } diff --git a/crates/kernel/ironclaw_turns/src/process_projection/runtime.rs b/crates/kernel/ironclaw_turns/src/process_projection/runtime.rs index 52af351ad25..6f6ed70b3a1 100644 --- a/crates/kernel/ironclaw_turns/src/process_projection/runtime.rs +++ b/crates/kernel/ironclaw_turns/src/process_projection/runtime.rs @@ -212,6 +212,7 @@ impl AgentTurnProcessRuntime { model_usage: None, execution_outcome: None, subagent_depth: 0, + subagent_activation_provenance: request.subagent_activation_provenance, spawn_tree_descendant_cap: None, product_context: request.product_context, resume_disposition: None, @@ -291,6 +292,9 @@ impl AgentTurnProcessRuntime { subagent_depth, spawn_tree_root_run_id: Some(turn_run_id_from_process_id(root_process_id)), product_context: parent_metadata.product_context.clone(), + // A fresh child run is a spawn, not a re-activation of an existing + // thread, so it carries no activation provenance. + subagent_activation_provenance: None, }; admission_policy .check_submit(&submit_template) @@ -317,6 +321,11 @@ impl AgentTurnProcessRuntime { model_usage: None, execution_outcome: None, subagent_depth, + // A fresh child run is a spawn, not a re-activation of an existing + // thread, so it carries no activation provenance. `ParentAgent` is + // reserved for `subagent_extend` re-activating an already-terminal + // child. + subagent_activation_provenance: None, spawn_tree_descendant_cap: Some(request.spawn_tree_descendant_cap), product_context: parent_metadata.product_context, resume_disposition: None, @@ -628,6 +637,19 @@ fn failure_prohibits_retry(failure: &SanitizedFailure) -> bool { #[async_trait] impl crate::AgentTurnRuntimePort for AgentTurnProcessRuntime { + async fn recent_runs_for_thread( + &self, + scope: &TurnScope, + limit: u32, + ) -> Result, TurnError> { + self.snapshots + .recent_agent_turn_snapshots(&scope.to_resource_scope(), limit) + .await? + .into_iter() + .map(turn_run_record_from_process_snapshot) + .collect() + } + async fn submit_turn( &self, request: SubmitTurnRequest, @@ -1128,6 +1150,7 @@ fn turn_run_record_from_process_snapshot( received_at: state.received_at, parent_run_id: snapshot.parent_process_id.map(turn_run_id_from_process_id), subagent_depth: metadata.subagent_depth, + subagent_activation_provenance: metadata.subagent_activation_provenance, spawn_tree_root_run_id: snapshot.root_process_id.map(turn_run_id_from_process_id), product_context: state.product_context, resume_disposition: state.resume_disposition, @@ -1206,6 +1229,7 @@ pub fn claimed_turn_run_from_process_claim( resolved_run_profile, subagent_depth: metadata.subagent_depth, spawn_tree_descendant_cap: metadata.spawn_tree_descendant_cap, + subagent_activation_provenance: metadata.subagent_activation_provenance, runner_id: turn_runner_id_from_worker(&claimed.worker_id)?, lease_token: turn_lease_token_from_process(&claimed.lease_token)?, }) diff --git a/crates/kernel/ironclaw_turns/src/process_projection/store_adapter.rs b/crates/kernel/ironclaw_turns/src/process_projection/store_adapter.rs index 0b114b348dc..3b94e5ac34c 100644 --- a/crates/kernel/ironclaw_turns/src/process_projection/store_adapter.rs +++ b/crates/kernel/ironclaw_turns/src/process_projection/store_adapter.rs @@ -44,6 +44,17 @@ impl ProcessSnapshotSource for ProcessJournalStoreTurnAdapter { .await .map_err(turn_error_from_process_journal_store_error) } + + async fn recent_agent_turn_snapshots( + &self, + scope: &ResourceScope, + limit: u32, + ) -> Result, Self::Error> { + self.runtime + .recent_agent_turn_snapshots(scope, limit) + .await + .map_err(turn_error_from_process_journal_store_error) + } } #[async_trait] diff --git a/crates/kernel/ironclaw_turns/src/process_projection/tests.rs b/crates/kernel/ironclaw_turns/src/process_projection/tests.rs index 4e60738319c..bee3ac9a8ad 100644 --- a/crates/kernel/ironclaw_turns/src/process_projection/tests.rs +++ b/crates/kernel/ironclaw_turns/src/process_projection/tests.rs @@ -10,8 +10,8 @@ use std::sync::Arc; use super::*; use crate::TurnEventProjectionFromProcessJournal; use crate::{ - AcceptedMessageRef, AllowAllTurnAdmissionPolicy, CapabilityActivityId, EventCursor, - IdempotencyKey, RunProfileId, RunProfileVersion, TurnActor, TurnGateRef, TurnId, + AcceptedMessageRef, ActivationProvenance, AllowAllTurnAdmissionPolicy, CapabilityActivityId, + EventCursor, IdempotencyKey, RunProfileId, RunProfileVersion, TurnActor, TurnGateRef, TurnId, TurnRunProfile, TurnScope, events::TurnEventProjectionSource, }; use ironclaw_loop_contracts::InMemoryRunProfileResolver; @@ -37,6 +37,7 @@ fn profile() -> TurnRunProfile { fn record_with_status(status: TurnStatus) -> TurnRunRecord { TurnRunRecord { + subagent_activation_provenance: None, run_id: TurnRunId::new(), turn_id: TurnId::new(), scope: scope(), @@ -76,6 +77,7 @@ fn agent_turn_metadata( ) -> AgentTurnProcessStateMetadata { let run_profile = profile(); AgentTurnProcessStateMetadata { + subagent_activation_provenance: None, turn_id, actor: Some(actor), accepted_message_ref: AcceptedMessageRef::new("accepted-runtime-test") @@ -96,6 +98,126 @@ fn agent_turn_metadata( } } +/// Regression: a terminal transition must not erase activation provenance. +/// +/// `loop_exit.rs` writes agent-turn metadata on every terminal transition via +/// `agent_turn_metadata_from_claimed`, which rebuilds the envelope from +/// `from_state` and then restores the lineage fields. Provenance was not among +/// the restored fields, so every completed run's provenance read back as +/// `None` — and since the streak window treats a non-`System` record as a +/// reset, the autonomous-wake cap could never fire in production. +/// +/// Driven through the exact function the loop-exit path calls, not through a +/// test helper that transitions the process directly: the helper never runs +/// this rewrite, which is why the original crate tests missed this. +#[test] +fn terminal_metadata_rewrite_preserves_activation_provenance() { + let state = crate::TurnRunState { + scope: scope(), + actor: Some(TurnActor::new( + UserId::new("terminal-provenance-user").expect("user"), + )), + turn_id: TurnId::new(), + run_id: TurnRunId::new(), + status: TurnStatus::Running, + accepted_message_ref: AcceptedMessageRef::new("accepted-terminal-provenance") + .expect("accepted"), + resolved_run_profile_id: RunProfileId::default_profile(), + resolved_run_profile_version: RunProfileVersion::new(1), + output_contract: Default::default(), + allow_steering: true, + resolved_model_route: None, + model_usage: None, + execution_outcome: None, + received_at: Utc::now(), + checkpoint_id: None, + gate_ref: None, + blocked_activity_id: None, + credential_requirements: Vec::new(), + failure: None, + event_cursor: EventCursor(11), + product_context: None, + resume_disposition: None, + }; + let claimed = ClaimedTurnRun { + state, + resolved_run_profile: profile().resolved, + subagent_depth: 1, + spawn_tree_descendant_cap: Some(16), + subagent_activation_provenance: Some(ActivationProvenance::System), + runner_id: TurnRunnerId::new(), + lease_token: crate::TurnLeaseToken::new(), + }; + + let envelope = + crate::process_projection::agent_turn_metadata_from_claimed(&claimed, None, None); + + assert_eq!( + envelope["agent_turn"]["subagent_activation_provenance"], + serde_json::json!("system"), + "a terminal metadata rewrite must carry provenance forward, or the \ + streak cap silently stops firing once runs complete" + ); + assert_eq!( + envelope["agent_turn"]["subagent_depth"], + json!(1), + "the sibling lineage field must keep surviving too" + ); +} + +#[test] +fn activation_provenance_survives_the_agent_turn_metadata_round_trip() { + let mut metadata = agent_turn_metadata( + TurnActor::new(UserId::new("activation-provenance-user").expect("user")), + TurnId::new(), + 0, + ); + metadata.subagent_activation_provenance = Some(ActivationProvenance::System); + + let wire = serde_json::to_value(&metadata).expect("serialize metadata"); + assert_eq!( + wire["subagent_activation_provenance"], + serde_json::json!("system"), + "provenance must reach the durable wire shape" + ); + + let decoded: AgentTurnProcessStateMetadata = + serde_json::from_value(wire).expect("deserialize metadata"); + assert_eq!( + decoded.subagent_activation_provenance, + Some(ActivationProvenance::System), + "a System-tagged submission must round-trip through durable metadata" + ); +} + +#[test] +fn legacy_agent_turn_metadata_without_activation_provenance_defaults_to_none() { + let mut metadata = agent_turn_metadata( + TurnActor::new(UserId::new("activation-provenance-user").expect("user")), + TurnId::new(), + 0, + ); + // Set a non-default value before removing the field so this exercises the + // serde default rather than merely round-tripping an absent one. + metadata.subagent_activation_provenance = Some(ActivationProvenance::ParentAgent); + + let mut wire = serde_json::to_value(&metadata).expect("serialize metadata"); + assert!( + wire.as_object_mut() + .expect("metadata object") + .remove("subagent_activation_provenance") + .is_some(), + "current wire shape must serialize a present provenance" + ); + + let decoded: AgentTurnProcessStateMetadata = + serde_json::from_value(wire).expect("legacy metadata deserializes"); + assert_eq!( + decoded.subagent_activation_provenance, None, + "rows written before this field existed must stay readable as None" + ); +} + #[test] fn legacy_agent_turn_process_metadata_without_output_contract_defaults_to_assistant_message() { let metadata = AgentTurnProcessMetadata { @@ -882,6 +1004,7 @@ async fn retry_rebinds_checkpoint_through_the_real_process_store() { let state_ref = ProcessCheckpointRef::from_trusted("source-state"); let run_profile = profile(); let metadata = AgentTurnProcessStateMetadata { + subagent_activation_provenance: None, turn_id: TurnId::new(), actor: Some(actor.clone()), accepted_message_ref: AcceptedMessageRef::new("accepted-retry").expect("accepted"), @@ -1197,6 +1320,7 @@ fn claimed_turn_run_projects_to_process_claim() { resume_disposition: None, }; let claimed = ClaimedTurnRun { + subagent_activation_provenance: None, state: state.clone(), resolved_run_profile: profile().resolved, subagent_depth: 3, @@ -1254,6 +1378,7 @@ fn claimed_process_round_trips_to_turn_executor_view() { resume_disposition: None, }; let claimed = ClaimedTurnRun { + subagent_activation_provenance: None, state: state.clone(), resolved_run_profile: profile().resolved, subagent_depth: 4, @@ -1310,6 +1435,7 @@ fn ownerless_scope_round_trips_through_the_process_claim() { resume_disposition: None, }; let claimed = ClaimedTurnRun { + subagent_activation_provenance: None, state: state.clone(), resolved_run_profile: profile().resolved, subagent_depth: 0, diff --git a/crates/kernel/ironclaw_turns/src/request.rs b/crates/kernel/ironclaw_turns/src/request.rs index 19b4232318b..1a22ee7a9b6 100644 --- a/crates/kernel/ironclaw_turns/src/request.rs +++ b/crates/kernel/ironclaw_turns/src/request.rs @@ -3,9 +3,9 @@ use ironclaw_host_api::output::OutputContract; use serde::{Deserialize, Serialize}; use crate::{ - AcceptedMessageRef, GateKind, GateResumeDisposition, IdempotencyKey, ProductTurnContext, - RunProfileRequest, SanitizedCancelReason, TurnActor, TurnGateRef, TurnRunId, TurnScope, - TurnStatus, + AcceptedMessageRef, ActivationProvenance, GateKind, GateResumeDisposition, IdempotencyKey, + ProductTurnContext, RunProfileRequest, SanitizedCancelReason, TurnActor, TurnGateRef, + TurnRunId, TurnScope, TurnStatus, }; pub type TurnTimestamp = DateTime; @@ -78,6 +78,31 @@ pub struct SubmitTurnRequest { pub spawn_tree_root_run_id: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub product_context: Option, + /// Why this submission is activating the thread. `None` — the default for + /// every ordinary caller — is an untagged, human-initiated submission. + /// Only the coordinator's `activate()` entry point sets this. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub subagent_activation_provenance: Option, +} + +/// Re-activate an existing thread with an explicit provenance tag — the single +/// re-activation primitive. +/// +/// Deliberately *not* a second admission path: `activate` builds an ordinary +/// [`SubmitTurnRequest`], so one-active-run exclusivity, idempotency replay, +/// and busy rejection behave exactly as they do for any other submission. The +/// only thing activation adds is the provenance stamp the derived streak caps +/// read. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ActivateThreadRequest { + pub scope: TurnScope, + pub actor: TurnActor, + pub accepted_message_ref: AcceptedMessageRef, + pub provenance: ActivationProvenance, + pub idempotency_key: IdempotencyKey, + pub received_at: TurnTimestamp, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub requested_run_profile: Option, } /// Request shape for callers that are creating a child run from an existing diff --git a/crates/kernel/ironclaw_turns/src/runner.rs b/crates/kernel/ironclaw_turns/src/runner.rs index 145129b3ebd..42a39f49d97 100644 --- a/crates/kernel/ironclaw_turns/src/runner.rs +++ b/crates/kernel/ironclaw_turns/src/runner.rs @@ -14,6 +14,13 @@ pub struct ClaimedTurnRun { pub subagent_depth: u32, #[serde(default, skip_serializing_if = "Option::is_none")] pub spawn_tree_descendant_cap: Option, + /// Carried so a terminal metadata rewrite can restore it. `from_state` + /// rebuilds the metadata envelope without lineage, and the loop-exit path + /// writes that envelope on every terminal transition — without this the + /// provenance is erased the moment a run completes, and the derived + /// activation-streak caps read every historical run as untagged. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub subagent_activation_provenance: Option, pub runner_id: TurnRunnerId, pub lease_token: TurnLeaseToken, } diff --git a/crates/kernel/ironclaw_turns/src/status.rs b/crates/kernel/ironclaw_turns/src/status.rs index 56c2dc6fb84..4407512215f 100644 --- a/crates/kernel/ironclaw_turns/src/status.rs +++ b/crates/kernel/ironclaw_turns/src/status.rs @@ -124,6 +124,11 @@ pub enum AdmissionRejectionReason { Policy, Unauthorized, Unavailable, + /// The thread's consecutive autonomous-wake budget is exhausted. Distinct + /// from the other reasons because nothing is wrong with the request or the + /// caller — the thread is simply parked pending human attention, and a + /// caller may retry after a human activates it. + SystemWakeStreak, } impl AdmissionRejectionReason { @@ -134,6 +139,7 @@ impl AdmissionRejectionReason { Self::Policy => "policy", Self::Unauthorized => "unauthorized", Self::Unavailable => "unavailable", + Self::SystemWakeStreak => "system_wake_streak", } } } @@ -309,6 +315,11 @@ impl TurnError { TurnErrorCategory::Unauthorized } AdmissionRejectionReason::Unavailable => TurnErrorCategory::Unavailable, + // Capacity-shaped, not caller error: the thread's autonomous + // budget is spent and a human activation restores it, which is + // the same "retry later, nothing is malformed" shape the + // tenant-limit rejection carries. + AdmissionRejectionReason::SystemWakeStreak => TurnErrorCategory::AdmissionRejected, }, Self::ScopeNotFound => TurnErrorCategory::ScopeNotFound, Self::Unauthorized => TurnErrorCategory::Unauthorized, diff --git a/crates/kernel/ironclaw_turns/tests/activation_contract.rs b/crates/kernel/ironclaw_turns/tests/activation_contract.rs new file mode 100644 index 00000000000..018b4a99809 --- /dev/null +++ b/crates/kernel/ironclaw_turns/tests/activation_contract.rs @@ -0,0 +1,576 @@ +//! Behavior pins for `TurnCoordinator::activate` — the single re-activation +//! primitive. +//! +//! `activate` is deliberately not a second admission path: it builds an +//! ordinary `SubmitTurnRequest` so one-active-run exclusivity, idempotency +//! replay, and busy rejection all behave exactly as they do for any other +//! submission. The only thing it adds is the provenance stamp, and these tests +//! assert that stamp at the durable-record seam rather than at the call. + +use async_trait::async_trait; +use chrono::Utc; +use ironclaw_host_api::ids::{AgentId, ProjectId, TenantId, ThreadId, UserId}; +use ironclaw_processes::{ + ClaimProcessesRequest, ProcessKind, ProcessLeaseRequest, ProcessStateTransitionRequest, + ProcessWorkerId, +}; +use ironclaw_turns::{ + AcceptedMessageRef, ActivateThreadRequest, ActivationProvenance, AdmissionRejection, + AdmissionRejectionReason, AgentTurnRuntimePort, AgentTurnSpawnTreeRuntimePort, + CancelRunRequest, CancelRunResponse, DefaultTurnCoordinator, GetRunStateRequest, + IdempotencyKey, ResumeTurnRequest, ResumeTurnResponse, RetryTurnRequest, RetryTurnResponse, + SYSTEM_WAKE_STREAK_CAP, SYSTEM_WAKE_WINDOW_OVERFETCH, SubmitTurnRequest, SubmitTurnResponse, + TurnActor, TurnCoordinator, TurnError, TurnRunId, TurnRunState, TurnScope, + test_support::{InMemoryAgentTurnProcessSystem, in_memory_agent_turn_process_system}, +}; +use std::sync::Arc; + +/// Drive the thread's single active run to a terminal state so the next +/// activation is admitted on an idle thread rather than rejected as busy. +async fn complete_active_run(system: &InMemoryAgentTurnProcessSystem, scope: &TurnScope) { + let transitions = system.transitions(); + let claimed = transitions + .claim_next_processes(ClaimProcessesRequest { + worker_id: ProcessWorkerId::from_trusted("activation-contract-worker"), + scope_filter: Some(scope.to_resource_scope()), + process_id_filter: None, + process_kind_filter: Some(ProcessKind::AgentTurn), + max_processes: 1, + }) + .await + .expect("claim succeeds") + .pop() + .expect("a run is claimable"); + transitions + .complete_process(ProcessStateTransitionRequest { + lease: ProcessLeaseRequest { + process_id: claimed.state.process_id, + worker_id: claimed.worker_id.clone(), + lease_token: claimed.lease_token.clone(), + }, + metadata: None, + }) + .await + .expect("run completes"); +} + +fn scope(thread: &str) -> TurnScope { + TurnScope::new( + TenantId::new("tenant-activation").expect("tenant"), + Some(AgentId::new("agent-activation").expect("agent")), + Some(ProjectId::new("project-activation").expect("project")), + ThreadId::new(thread).expect("thread"), + ) +} + +fn actor() -> TurnActor { + TurnActor::new(UserId::new("user-activation").expect("user")) +} + +fn activate_request( + scope: TurnScope, + provenance: ActivationProvenance, + key: &str, +) -> ActivateThreadRequest { + ActivateThreadRequest { + scope, + actor: actor(), + // Derived from the key so distinct activations carry distinct accepted + // messages (as production does — each background wake references the + // settled child result that triggered it), while a deliberate retry + // reusing the key also reuses the message, which is what makes it a + // retry rather than a new activation. + accepted_message_ref: AcceptedMessageRef::new(format!("accepted-activation-{key}")) + .expect("accepted"), + provenance, + idempotency_key: IdempotencyKey::new(key).expect("idempotency key"), + received_at: Utc::now(), + requested_run_profile: None, + } +} + +fn submit_request(scope: TurnScope, key: &str) -> SubmitTurnRequest { + SubmitTurnRequest { + scope, + actor: actor(), + accepted_message_ref: AcceptedMessageRef::new("accepted-activation").expect("accepted"), + requested_run_profile: None, + output_contract: None, + requested_model: None, + idempotency_key: IdempotencyKey::new(key).expect("idempotency key"), + received_at: Utc::now(), + requested_run_id: None, + parent_run_id: None, + subagent_depth: 0, + spawn_tree_root_run_id: None, + product_context: None, + subagent_activation_provenance: None, + } +} + +/// `activate` must reach the ordinary admission path and stamp the provenance +/// onto the run it creates, so the derived streak caps can later see it. +#[tokio::test] +async fn activate_stamps_provenance_on_the_created_run_record() { + let system = in_memory_agent_turn_process_system(); + let runtime = Arc::new(system.runtime()); + let coordinator = DefaultTurnCoordinator::new(runtime.clone()); + let scope = scope("thread-activate-system"); + + let SubmitTurnResponse::Accepted { run_id, .. } = coordinator + .activate(activate_request( + scope.clone(), + ActivationProvenance::System, + "activate-system-key", + )) + .await + .expect("activate succeeds on an idle thread"); + + let record = runtime + .get_run_record(&scope, run_id) + .await + .expect("run record read") + .expect("run record exists"); + + assert_eq!( + record.subagent_activation_provenance, + Some(ActivationProvenance::System), + "activate must stamp the provenance onto the durable run record" + ); +} + +/// The control: an ordinary submission records no provenance, so an untagged +/// run can never be mistaken for an autonomous wake by the streak caps. +#[tokio::test] +async fn ordinary_submit_turn_records_no_activation_provenance() { + let system = in_memory_agent_turn_process_system(); + let runtime = Arc::new(system.runtime()); + let coordinator = DefaultTurnCoordinator::new(runtime.clone()); + let scope = scope("thread-activate-plain"); + + let SubmitTurnResponse::Accepted { run_id, .. } = coordinator + .submit_turn(submit_request(scope.clone(), "activate-plain-key")) + .await + .expect("ordinary submit succeeds"); + + let record = runtime + .get_run_record(&scope, run_id) + .await + .expect("run record read") + .expect("run record exists"); + + assert_eq!( + record.subagent_activation_provenance, None, + "an ordinary submission must stay untagged" + ); +} + +/// `ParentAgent` is a distinct tag from `System` and must survive the same way +/// — the two caps read disjoint windows, so a mixed-up tag would silently +/// change which budget a run consumes. +#[tokio::test] +async fn activate_distinguishes_parent_agent_from_system_provenance() { + let system = in_memory_agent_turn_process_system(); + let runtime = Arc::new(system.runtime()); + let coordinator = DefaultTurnCoordinator::new(runtime.clone()); + let scope = scope("thread-activate-parent"); + + let SubmitTurnResponse::Accepted { run_id, .. } = coordinator + .activate(activate_request( + scope.clone(), + ActivationProvenance::ParentAgent, + "activate-parent-key", + )) + .await + .expect("activate succeeds"); + + let record = runtime + .get_run_record(&scope, run_id) + .await + .expect("run record read") + .expect("run record exists"); + + assert_eq!( + record.subagent_activation_provenance, + Some(ActivationProvenance::ParentAgent) + ); +} + +/// A coordinator that has not opted into activation must refuse rather than +/// silently falling through to an untagged submission — an untagged autonomous +/// wake is invisible to the streak cap that exists to bound it. +#[tokio::test] +async fn default_activate_impl_refuses_rather_than_submitting_untagged() { + struct NonActivatingCoordinator; + + #[async_trait] + impl TurnCoordinator for NonActivatingCoordinator { + async fn prepare_turn(&self, _scope: TurnScope) -> Result { + unreachable!("not exercised by this test") + } + async fn submit_turn( + &self, + _request: SubmitTurnRequest, + ) -> Result { + panic!("the default activate() must not fall through to submit_turn"); + } + async fn resume_turn( + &self, + _request: ResumeTurnRequest, + ) -> Result { + unreachable!("not exercised by this test") + } + async fn retry_turn( + &self, + _request: RetryTurnRequest, + ) -> Result { + unreachable!("not exercised by this test") + } + async fn cancel_run( + &self, + _request: CancelRunRequest, + ) -> Result { + unreachable!("not exercised by this test") + } + async fn get_run_state( + &self, + _request: GetRunStateRequest, + ) -> Result { + unreachable!("not exercised by this test") + } + } + + let error = NonActivatingCoordinator + .activate(activate_request( + scope("thread-activate-default"), + ActivationProvenance::System, + "activate-default-key", + )) + .await + .expect_err("a coordinator without activation support must refuse"); + + assert!( + matches!(error, TurnError::InvalidRequest { .. }), + "the default activate() must fail closed, got {error:?}" + ); +} + +/// The cap must be enforced at the caller, not only in the predicate, and a +/// refusal must not create a run. Sixteen consecutive System activations +/// saturate the streak; the seventeenth is refused. +#[tokio::test] +async fn activate_refuses_a_system_wake_past_the_streak_cap() { + let system = in_memory_agent_turn_process_system(); + let runtime = Arc::new(system.runtime()); + let coordinator = DefaultTurnCoordinator::new(runtime.clone()); + let scope = scope("thread-streak-saturated"); + + for index in 0..SYSTEM_WAKE_STREAK_CAP { + let SubmitTurnResponse::Accepted { .. } = coordinator + .activate(activate_request( + scope.clone(), + ActivationProvenance::System, + &format!("streak-key-{index}"), + )) + .await + .unwrap_or_else(|error| panic!("wake {index} must be admitted, got {error:?}")); + // Drive each run terminal so the thread is idle for the next wake; + // otherwise the second activation would be refused as busy and this + // test would pass for the wrong reason. + complete_active_run(&system, &scope).await; + } + + let error = coordinator + .activate(activate_request( + scope.clone(), + ActivationProvenance::System, + "streak-key-over", + )) + .await + .expect_err("a saturated System streak must refuse the next wake"); + + assert!( + matches!( + error, + TurnError::AdmissionRejected(AdmissionRejection { + reason: AdmissionRejectionReason::SystemWakeStreak, + .. + }) + ), + "the cap refusal must carry its own reason, distinguishable from \ + 'activation unsupported' and 'window unreadable'; got {error:?}" + ); + // The refusal must also create no run. Asserting only the error would let a + // regression that admits the run and *then* errors still pass. + let after = AgentTurnRuntimePort::recent_runs_for_thread( + runtime.as_ref(), + &scope, + SYSTEM_WAKE_STREAK_CAP.saturating_mul(2), + ) + .await + .expect("window read"); + assert_eq!( + after.len() as u32, + SYSTEM_WAKE_STREAK_CAP, + "a refused wake must not have created a run" + ); +} + +/// A full raw fetch that cannot yield a cap-sized non-ParentAgent window means +/// the streak could not be established — unknown, not absent. Admitting there +/// is the fail-open hole the over-fetch alone left; this pins the fail-closed +/// behavior. A genuinely short fetch is a young thread and still admits. +#[tokio::test] +async fn a_window_crowded_out_by_parent_agent_runs_fails_closed() { + let system = in_memory_agent_turn_process_system(); + let runtime = Arc::new(system.runtime()); + let coordinator = DefaultTurnCoordinator::new(runtime.clone()); + let scope = scope("thread-streak-crowded"); + + // Fill the entire raw fetch window with ParentAgent runs, so no + // non-ParentAgent record survives the filter. + let fetch_limit = SYSTEM_WAKE_STREAK_CAP.saturating_mul(SYSTEM_WAKE_WINDOW_OVERFETCH); + for index in 0..fetch_limit { + coordinator + .activate(activate_request( + scope.clone(), + ActivationProvenance::ParentAgent, + &format!("crowded-parent-{index}"), + )) + .await + .unwrap_or_else(|error| panic!("parent activation {index} admitted, got {error:?}")); + complete_active_run(&system, &scope).await; + } + + let error = coordinator + .activate(activate_request( + scope.clone(), + ActivationProvenance::System, + "crowded-system", + )) + .await + .expect_err("an unestablished window must fail closed, not admit"); + + assert!( + matches!( + error, + TurnError::AdmissionRejected(AdmissionRejection { + reason: AdmissionRejectionReason::SystemWakeStreak, + .. + }) + ), + "expected the streak refusal, got {error:?}" + ); +} + +/// `activate()` advertises ordinary submission idempotency. Retrying an +/// already-accepted activation at the cap boundary must replay that +/// submission, not be refused by the run its own first attempt created. +#[tokio::test] +async fn replaying_an_accepted_activation_at_the_cap_is_not_refused() { + let system = in_memory_agent_turn_process_system(); + let runtime = Arc::new(system.runtime()); + let coordinator = DefaultTurnCoordinator::new(runtime.clone()); + let scope = scope("thread-streak-replay"); + + let mut last_run = None; + for index in 0..SYSTEM_WAKE_STREAK_CAP { + let SubmitTurnResponse::Accepted { run_id, .. } = coordinator + .activate(activate_request( + scope.clone(), + ActivationProvenance::System, + &format!("replay-key-{index}"), + )) + .await + .unwrap_or_else(|error| panic!("wake {index} admitted, got {error:?}")); + last_run = Some(run_id); + complete_active_run(&system, &scope).await; + } + + // Same idempotency key AND same accepted message as the last accepted + // activation: this is a retry, not a 17th wake. + let replayed = coordinator + .activate(activate_request( + scope.clone(), + ActivationProvenance::System, + &format!("replay-key-{}", SYSTEM_WAKE_STREAK_CAP - 1), + )) + .await + .expect("a retry of an accepted activation must replay, not hit the cap"); + + let SubmitTurnResponse::Accepted { run_id, .. } = replayed; + assert_eq!( + Some(run_id), + last_run, + "the replay must return the originally accepted run" + ); +} + +/// A `Human` activation resets the streak, so a thread that a person has come +/// back to can autonomously wake again. +#[tokio::test] +async fn a_human_activation_lets_system_wakes_resume_after_saturation() { + let system = in_memory_agent_turn_process_system(); + let runtime = Arc::new(system.runtime()); + let coordinator = DefaultTurnCoordinator::new(runtime.clone()); + let scope = scope("thread-streak-reset"); + + for index in 0..SYSTEM_WAKE_STREAK_CAP { + let SubmitTurnResponse::Accepted { .. } = coordinator + .activate(activate_request( + scope.clone(), + ActivationProvenance::System, + &format!("reset-key-{index}"), + )) + .await + .expect("wake admitted"); + complete_active_run(&system, &scope).await; + } + + let SubmitTurnResponse::Accepted { .. } = coordinator + .activate(activate_request( + scope.clone(), + ActivationProvenance::Human, + "reset-key-human", + )) + .await + .expect("a Human activation is never capped"); + complete_active_run(&system, &scope).await; + + coordinator + .activate(activate_request( + scope.clone(), + ActivationProvenance::System, + "reset-key-after-human", + )) + .await + .expect("a Human activation must reset the System streak"); +} + +/// A runtime that cannot answer the recent-window question must refuse a +/// System activation rather than admit it. An empty window reads as "streak +/// not established", so defaulting to one would silently disable the cap. +#[tokio::test] +async fn a_runtime_without_a_recent_window_refuses_system_activation() { + let runtime = Arc::new(WindowlessRuntime( + in_memory_agent_turn_process_system().runtime(), + )); + let coordinator = DefaultTurnCoordinator::new(runtime); + + let error = coordinator + .activate(activate_request( + scope("thread-no-window"), + ActivationProvenance::System, + "no-window-key", + )) + .await + .expect_err("a runtime that cannot read the window must refuse"); + + assert!( + matches!(error, TurnError::InvalidRequest { .. }), + "expected a fail-closed refusal, got {error:?}" + ); +} + +/// Wraps a real runtime but declines to answer the recent-window query, taking +/// the trait's fail-closed default. +struct WindowlessRuntime(ironclaw_turns::process_projection::AgentTurnProcessRuntime); + +#[async_trait] +impl ironclaw_turns::AgentTurnRuntimePort for WindowlessRuntime { + async fn submit_turn( + &self, + request: SubmitTurnRequest, + admission_policy: &dyn ironclaw_turns::TurnAdmissionPolicy, + run_profile_resolver: &dyn ironclaw_loop_contracts::RunProfileResolver, + ) -> Result { + self.0 + .submit_turn(request, admission_policy, run_profile_resolver) + .await + } + + async fn resume_turn( + &self, + request: ResumeTurnRequest, + ) -> Result { + ironclaw_turns::AgentTurnRuntimePort::resume_turn(&self.0, request).await + } + + async fn retry_turn(&self, request: RetryTurnRequest) -> Result { + ironclaw_turns::AgentTurnRuntimePort::retry_turn(&self.0, request).await + } + + async fn request_cancel( + &self, + request: CancelRunRequest, + ) -> Result { + ironclaw_turns::AgentTurnRuntimePort::request_cancel(&self.0, request).await + } + + async fn get_run_state(&self, request: GetRunStateRequest) -> Result { + ironclaw_turns::AgentTurnRuntimePort::get_run_state(&self.0, request).await + } +} + +/// Design section 8.3 requires ParentAgent runs to be excluded from the FETCH, +/// not filtered afterwards. Filtering a K-sized fetch yields fewer than K +/// records whenever ParentAgent runs are interleaved, and a short window reads +/// as "streak not established" — which silently disables the cap on exactly +/// the human-free interleaved sequences it exists to bound. +/// +/// This drives the design's required assertion (c): an interleaved ParentAgent +/// activation neither resets nor counts toward the System streak. +#[tokio::test] +async fn interleaved_parent_agent_runs_do_not_disable_the_system_streak_cap() { + let system = in_memory_agent_turn_process_system(); + let runtime = Arc::new(system.runtime()); + let coordinator = DefaultTurnCoordinator::new(runtime.clone()); + let scope = scope("thread-streak-interleaved"); + + // Alternate System / ParentAgent. No Human ever touches this thread, so the + // System streak must still saturate at SYSTEM_WAKE_STREAK_CAP. + for index in 0..SYSTEM_WAKE_STREAK_CAP { + coordinator + .activate(activate_request( + scope.clone(), + ActivationProvenance::System, + &format!("interleaved-system-{index}"), + )) + .await + .unwrap_or_else(|error| panic!("system wake {index} admitted, got {error:?}")); + complete_active_run(&system, &scope).await; + + coordinator + .activate(activate_request( + scope.clone(), + ActivationProvenance::ParentAgent, + &format!("interleaved-parent-{index}"), + )) + .await + .unwrap_or_else(|error| panic!("parent extend {index} admitted, got {error:?}")); + complete_active_run(&system, &scope).await; + } + + let error = coordinator + .activate(activate_request( + scope.clone(), + ActivationProvenance::System, + "interleaved-system-over", + )) + .await + .expect_err( + "ParentAgent runs must be excluded from the window, so the System streak \ + still saturates and the next wake is refused", + ); + + assert!( + matches!( + error, + TurnError::AdmissionRejected(AdmissionRejection { + reason: AdmissionRejectionReason::SystemWakeStreak, + .. + }) + ), + "expected the streak-cap refusal, got {error:?}" + ); +} diff --git a/crates/kernel/ironclaw_turns/tests/agent_loop_host_contract.rs b/crates/kernel/ironclaw_turns/tests/agent_loop_host_contract.rs index e77463e6054..38bdffd85f5 100644 --- a/crates/kernel/ironclaw_turns/tests/agent_loop_host_contract.rs +++ b/crates/kernel/ironclaw_turns/tests/agent_loop_host_contract.rs @@ -3760,6 +3760,7 @@ async fn claimed_run_context() -> LoopRunContext { let coordinator = DefaultTurnCoordinator::new(store.clone()); let response = coordinator .submit_turn(SubmitTurnRequest { + subagent_activation_provenance: None, requested_model: None, scope: scope.clone(), actor: TurnActor::new(UserId::new("user-loop").unwrap()), diff --git a/crates/kernel/ironclaw_turns/tests/coordinator_prepared_run_contract.rs b/crates/kernel/ironclaw_turns/tests/coordinator_prepared_run_contract.rs index 4a8defcc7ae..6eb8a96a219 100644 --- a/crates/kernel/ironclaw_turns/tests/coordinator_prepared_run_contract.rs +++ b/crates/kernel/ironclaw_turns/tests/coordinator_prepared_run_contract.rs @@ -85,6 +85,7 @@ fn submit_request( idempotency_key: &str, ) -> SubmitTurnRequest { SubmitTurnRequest { + subagent_activation_provenance: None, scope, actor: TurnActor::new(UserId::new("user-prepared-run").expect("user")), accepted_message_ref: AcceptedMessageRef::new("accepted-prepared-run").expect("accepted"), diff --git a/crates/loop/ironclaw_loop_host/src/subagent_spawn_port/tests.rs b/crates/loop/ironclaw_loop_host/src/subagent_spawn_port/tests.rs index 2e72097cc77..a6460b3452f 100644 --- a/crates/loop/ironclaw_loop_host/src/subagent_spawn_port/tests.rs +++ b/crates/loop/ironclaw_loop_host/src/subagent_spawn_port/tests.rs @@ -1142,6 +1142,7 @@ fn subagent_definition(allow_nesting: bool) -> SubagentDefinition { fn turn_record(run_context: &LoopRunContext, subagent_depth: u32) -> TurnRunRecord { let lineage_root = (subagent_depth > 0).then(TurnRunId::new); TurnRunRecord { + subagent_activation_provenance: None, run_id: run_context.run_id, turn_id: run_context.turn_id, scope: run_context.scope.clone(), diff --git a/crates/loop/ironclaw_loop_host/tests/run_lease_fence_memo.rs b/crates/loop/ironclaw_loop_host/tests/run_lease_fence_memo.rs index bbbe3e54569..42a45baeac6 100644 --- a/crates/loop/ironclaw_loop_host/tests/run_lease_fence_memo.rs +++ b/crates/loop/ironclaw_loop_host/tests/run_lease_fence_memo.rs @@ -225,6 +225,7 @@ impl Fixture { expires_at: Option>, ) -> TurnRunRecord { TurnRunRecord { + subagent_activation_provenance: None, run_id: self.run_context.run_id, turn_id: self.run_context.turn_id, scope: self.run_context.scope.clone(), diff --git a/crates/loop/ironclaw_turn_runner/src/loop_driver_host/run_lease_fence_tests.rs b/crates/loop/ironclaw_turn_runner/src/loop_driver_host/run_lease_fence_tests.rs index 6a0ee089846..6b3c56e0f98 100644 --- a/crates/loop/ironclaw_turn_runner/src/loop_driver_host/run_lease_fence_tests.rs +++ b/crates/loop/ironclaw_turn_runner/src/loop_driver_host/run_lease_fence_tests.rs @@ -154,6 +154,7 @@ fn claimed_run_matching( use ironclaw_turns::{AcceptedMessageRef, TurnRunnerId, TurnStatus}; ironclaw_turns::runner::ClaimedTurnRun { + subagent_activation_provenance: None, state: ironclaw_turns::TurnRunState { scope: scope.clone(), actor: None, diff --git a/crates/loop/ironclaw_turn_runner/src/loop_exit_applier/tests/support.rs b/crates/loop/ironclaw_turn_runner/src/loop_exit_applier/tests/support.rs index d4a32d0651c..0b88365b3a2 100644 --- a/crates/loop/ironclaw_turn_runner/src/loop_exit_applier/tests/support.rs +++ b/crates/loop/ironclaw_turn_runner/src/loop_exit_applier/tests/support.rs @@ -412,6 +412,7 @@ pub(super) fn claimed_run() -> ClaimedTurnRun { profile.checkpoint_policy.require_final_checkpoint = false; profile.checkpoint_policy.allow_no_reply_completion = false; ClaimedTurnRun { + subagent_activation_provenance: None, state: TurnRunState { scope, actor: None, diff --git a/crates/loop/ironclaw_turn_runner/src/steering_reconcile.rs b/crates/loop/ironclaw_turn_runner/src/steering_reconcile.rs index ede72deba8b..36aee334bff 100644 --- a/crates/loop/ironclaw_turn_runner/src/steering_reconcile.rs +++ b/crates/loop/ironclaw_turn_runner/src/steering_reconcile.rs @@ -40,9 +40,9 @@ use ironclaw_processes::{ RecoverExpiredProcessLeasesRequest, RecoverExpiredProcessLeasesResponse, SuspendProcessRequest, }; use ironclaw_turns::{ - CancelRunRequest, CancelRunResponse, GetRunStateRequest, ResumeTurnRequest, ResumeTurnResponse, - RetryTurnRequest, RetryTurnResponse, SubmitTurnRequest, SubmitTurnResponse, TurnCoordinator, - TurnError, TurnRunId, TurnRunState, TurnScope, + ActivateThreadRequest, CancelRunRequest, CancelRunResponse, GetRunStateRequest, + ResumeTurnRequest, ResumeTurnResponse, RetryTurnRequest, RetryTurnResponse, SubmitTurnRequest, + SubmitTurnResponse, TurnCoordinator, TurnError, TurnRunId, TurnRunState, TurnScope, }; use tracing::debug; @@ -79,6 +79,13 @@ impl TurnCoordinator for CancelReconcilingTurnCoordinator { self.inner.submit_turn(request).await } + async fn activate( + &self, + request: ActivateThreadRequest, + ) -> Result { + self.inner.activate(request).await + } + async fn resume_turn( &self, request: ResumeTurnRequest, @@ -341,11 +348,114 @@ mod tests { use ironclaw_loop_host::HostInputQueueError; use ironclaw_threads::ThreadMessageId; use ironclaw_turns::{ - EventCursor, IdempotencyKey, SanitizedCancelReason, TurnActor, TurnStatus, + AcceptedMessageRef, ActivateThreadRequest, ActivationProvenance, EventCursor, + IdempotencyKey, SanitizedCancelReason, TurnActor, TurnStatus, }; use super::*; + /// The decorator's contract is "every other method forwards". A method it + /// forgets silently inherits the trait's fail-closed default, and because + /// this is the ONE coordinator production composes, that turns into "the + /// feature is simply off in production" with no compile error to catch it. + #[tokio::test] + async fn activate_forwards_to_the_inner_coordinator() { + struct ActivateRecordingCoordinator { + seen: StdMutex>, + } + + #[async_trait] + impl TurnCoordinator for ActivateRecordingCoordinator { + async fn prepare_turn(&self, _scope: TurnScope) -> Result { + panic!("prepare_turn is not used by this test") + } + async fn submit_turn( + &self, + _request: SubmitTurnRequest, + ) -> Result { + panic!("submit_turn is not used by this test") + } + async fn activate( + &self, + request: ActivateThreadRequest, + ) -> Result { + self.seen + .lock() + .expect("activation recorder poisoned") + .push(request.provenance); + Err(TurnError::Unavailable { + reason: "inner reached".to_string(), + }) + } + async fn resume_turn( + &self, + _request: ResumeTurnRequest, + ) -> Result { + panic!("resume_turn is not used by this test") + } + async fn retry_turn( + &self, + _request: RetryTurnRequest, + ) -> Result { + panic!("retry_turn is not used by this test") + } + async fn cancel_run( + &self, + _request: CancelRunRequest, + ) -> Result { + panic!("cancel_run is not used by this test") + } + async fn get_run_state( + &self, + _request: GetRunStateRequest, + ) -> Result { + panic!("get_run_state is not used by this test") + } + } + + let inner = Arc::new(ActivateRecordingCoordinator { + seen: StdMutex::new(Vec::new()), + }); + let decorated = CancelReconcilingTurnCoordinator::new( + Arc::clone(&inner) as Arc, + Arc::new(RecordingQueue::default()), + ); + + let error = decorated + .activate(ActivateThreadRequest { + scope: TurnScope::new( + TenantId::new("tenant-reconcile").expect("tenant"), + Some(AgentId::new("agent-reconcile").expect("agent")), + Some(ProjectId::new("project-reconcile").expect("project")), + ThreadId::new("thread-reconcile").expect("thread"), + ), + actor: TurnActor::new(UserId::new("user-forward").expect("user")), + accepted_message_ref: AcceptedMessageRef::new("accepted-forward") + .expect("accepted"), + provenance: ActivationProvenance::System, + idempotency_key: IdempotencyKey::new("forward-key").expect("key"), + received_at: chrono::Utc::now(), + requested_run_profile: None, + }) + .await + .expect_err("the recording inner coordinator always errors"); + + assert!( + matches!(error, TurnError::Unavailable { .. }), + "the decorator must surface the INNER coordinator's outcome, not the \ + trait's fail-closed default; got {error:?}" + ); + assert_eq!( + inner + .seen + .lock() + .expect("activation recorder poisoned") + .as_slice(), + &[ActivationProvenance::System], + "activate must reach the inner coordinator exactly once" + ); + } + struct ScriptedCancelCoordinator { cancel_result: StdMutex>>, } diff --git a/crates/loop/ironclaw_turn_runner/src/structured_finalization/tests.rs b/crates/loop/ironclaw_turn_runner/src/structured_finalization/tests.rs index 2e3c6b30e49..c51a1bac618 100644 --- a/crates/loop/ironclaw_turn_runner/src/structured_finalization/tests.rs +++ b/crates/loop/ironclaw_turn_runner/src/structured_finalization/tests.rs @@ -114,6 +114,7 @@ struct LeaseRuntime { impl LeaseRuntime { fn record(&self) -> TurnRunRecord { TurnRunRecord { + subagent_activation_provenance: None, run_id: self.run_id, turn_id: TurnId::new(), scope: self.scope.clone(), diff --git a/crates/loop/ironclaw_turn_runner/src/subagent/await_edge/resolver.rs b/crates/loop/ironclaw_turn_runner/src/subagent/await_edge/resolver.rs index b7a363bddb1..8363be90382 100644 --- a/crates/loop/ironclaw_turn_runner/src/subagent/await_edge/resolver.rs +++ b/crates/loop/ironclaw_turn_runner/src/subagent/await_edge/resolver.rs @@ -917,6 +917,7 @@ mod tests { resolved_run_profile: ironclaw_loop_contracts::ResolvedRunProfile, ) -> TurnRunRecord { TurnRunRecord { + subagent_activation_provenance: None, run_id: child_run_id, turn_id: ironclaw_host_api::turn::TurnId::new(), scope: TurnScope::new( diff --git a/crates/loop/ironclaw_turn_runner/src/turn_scheduler.rs b/crates/loop/ironclaw_turn_runner/src/turn_scheduler.rs index 2bf5e8f3aff..8257f772ebc 100644 --- a/crates/loop/ironclaw_turn_runner/src/turn_scheduler.rs +++ b/crates/loop/ironclaw_turn_runner/src/turn_scheduler.rs @@ -497,6 +497,7 @@ mod tests { ThreadId::new("thread-scheduler-metadata").expect("thread"), ); let claimed = ClaimedTurnRun { + subagent_activation_provenance: None, state: TurnRunState { scope, actor: None, diff --git a/crates/loop/ironclaw_turn_runner/tests/turn_run_executor.rs b/crates/loop/ironclaw_turn_runner/tests/turn_run_executor.rs index ceba1b076fb..92c06478df1 100644 --- a/crates/loop/ironclaw_turn_runner/tests/turn_run_executor.rs +++ b/crates/loop/ironclaw_turn_runner/tests/turn_run_executor.rs @@ -259,6 +259,7 @@ fn completed_exit() -> LoopExit { fn claimed_run(context: &LoopRunContext) -> ClaimedTurnRun { ClaimedTurnRun { + subagent_activation_provenance: None, state: TurnRunState { scope: context.scope.clone(), actor: None, diff --git a/crates/product/ironclaw_assistant/src/auth_continuation.rs b/crates/product/ironclaw_assistant/src/auth_continuation.rs index 8e4db282563..fbb46a1af72 100644 --- a/crates/product/ironclaw_assistant/src/auth_continuation.rs +++ b/crates/product/ironclaw_assistant/src/auth_continuation.rs @@ -920,6 +920,7 @@ mod tests { let actor = TurnActor::new(UserId::new("alice").unwrap()); let submit = coordinator .submit_turn(SubmitTurnRequest { + subagent_activation_provenance: None, scope: scope.clone(), requested_model: None, actor: actor.clone(), diff --git a/crates/product/ironclaw_assistant/src/inbound_turn.rs b/crates/product/ironclaw_assistant/src/inbound_turn.rs index 96c3f3c14cf..0f8880daaf4 100644 --- a/crates/product/ironclaw_assistant/src/inbound_turn.rs +++ b/crates/product/ironclaw_assistant/src/inbound_turn.rs @@ -1537,6 +1537,7 @@ impl AcceptedProductInboundTurn { } }; let request = SubmitTurnRequest { + subagent_activation_provenance: None, scope: turn_scope.clone(), actor, accepted_message_ref: accepted_message_ref.clone(), diff --git a/crates/product/ironclaw_assistant/src/unbound_turn.rs b/crates/product/ironclaw_assistant/src/unbound_turn.rs index b84af829513..f91af06784f 100644 --- a/crates/product/ironclaw_assistant/src/unbound_turn.rs +++ b/crates/product/ironclaw_assistant/src/unbound_turn.rs @@ -193,6 +193,7 @@ impl UnboundTurnService { let response = self .coordinator .submit_turn(SubmitTurnRequest { + subagent_activation_provenance: None, scope: self.resolved_turn_scope(&thread_id, &submission.caller), actor: TurnActor::new(submission.caller.user_id), accepted_message_ref: accepted.accepted_message_ref, diff --git a/docs/internal/reborn/subagent-spawn/pr2-pr6-shape.md b/docs/internal/reborn/subagent-spawn/pr2-pr6-shape.md new file mode 100644 index 00000000000..d7c2af6f929 --- /dev/null +++ b/docs/internal/reborn/subagent-spawn/pr2-pr6-shape.md @@ -0,0 +1,226 @@ +# Shape: complete subagent support (PR2–PR6), enable last + +Decision record (2026-08-19, /shape gate). Companion recon: +`research-background-enable.md`. Canonical design (tiebreaker for every +mechanism below): `thread-harness-design.md` — this file only fixes slice +order, file placements, and signatures so the implementation plan is +transcription plus judgment. + +## Decision + +- **Scope:** ship the design's PR2 through PR6 — background delivery, + inspect, extend, WebUI child-tree, cancel. +- **Extend-vs-fork verdict:** extend the landed PR1 path everywhere + (`SubagentSpawnCapabilityPort`, `AwaitEdgeResolver`, + `subagent/await_edge/`). Nothing forks; no new crate, no cargo feature, + no stored counters (design standing rulings). +- **Deviation from the design's staging, with reason:** the design + prod-enables (clears `builtin.spawn_subagent` from + `disabled_capability_ids`) after PR2. We enable **after PR6 instead** — + subagents are complex enough that the observe/steer/cancel surfaces should + exist before real users get the tool. Strictly more conservative; costs + one line to change our mind. Everything else follows the design's staging + verbatim, including both hard prod-enable gates (drain safety scan, gate + escalation walk) landing with PR2. +- **Testing is not deferred to the end:** the five `#[ignore]`d e2e tests + come alive at PR2 via the harness's own capability enablement + (`tests/support/reborn_parity_qa/binary_e2e.rs:861` clears + `disabled_capability_ids`), independent of production enablement. Only + the final slice touches production defaults. +- **What gets deleted:** `background_subagents_disabled()` and its codec + rejections; the `drain_settled` stub body + dead `LoopBackgroundChildPort` + doc reference; the hard-coded `SpawnSubagentMode::Blocking` in + `finish_spawn`; at the end, the deny-filter entry and the + "capability stays off" test assertions. +- **Deletion-first check:** ran — the change is mostly *unblocking* landed + code; the only large additions the design allows are the ones it names + (activate primitive, escalation walk, WebUI tree). + +## Slices + +Each slice = one reviewable PR. Labels: [B] behavioral, [S] structural. +Design task IDs (P2.x…) in parentheses; the design section is the spec for +each. + +### Slice 1 [B] — activation provenance + `activate()` primitive (P2.4 prereq) + +- `ActivationProvenance { Human, ParentAgent, System }` — new enum in + `crates/contracts/ironclaw_host_api/src/turn.rs` (turn vocabulary is + host_api-owned per `crates/kernel/ironclaw_turns/AGENTS.md`). +- `TurnRunRecord.subagent_activation_provenance: Option` + — additive field, set once at run creation, immutable + (`crates/kernel/ironclaw_turns/src/agent_turn_runtime.rs`, beside + `parent_run_id`/`subagent_depth` at `:181-185`; mirror on `request.rs`). +- `activate(thread, input, provenance)` — the single re-activation + primitive (design §1). Home: `crates/kernel/ironclaw_turns/src/coordinator.rs` + beside `submit_turn` (TODO exact signature: takes typed thread id + + durable input submission + provenance; returns `ThreadBusy` when a run is + live — reuse `TurnError::ThreadBusy`, `status.rs:269`). It must NOT mint + `TrustedInboundTurnRequest` or touch trusted trigger submitters + (root `AGENTS.md:49`; extend the scan roots of + `reborn_dependency_boundaries.rs:1657` if any submitter-shaped type is + added). +- Streak-cap admission inside `activate()` (design §8.3, pulled into PR2): + `SYSTEM_WAKE_STREAK_CAP = 16`, derived `LIMIT K` query over run records + with `ParentAgent` excluded from the fetch; `Human` resets. Constant + independently named (never merged with the descendant cap 16 or iteration + limit 16). §6's ParentAgent cap of 8 ships later (Slice 7) on the same + field. +- Tests: crate-tier windowing rows (§8.3's four assertions a–d). + +### Slice 2 [B] — background mode accepted + delivery (P2.4) + +- Codec: delete both rejections in `TryFrom` + (`subagent_spawn_port.rs:218-227`) and `background_subagents_disabled()` + (`:1500`); advertise `mode` in `build_spawn_subagent_parameters_schema` + (`:62`, enum `["blocking","background"]`, default blocking); thread + `args.mode` through `finish_spawn` instead of the hard-coded `Blocking` + (`:908`); background spawns return an immediate spawn-result payload + (`spawn_result.rs` `SubagentSpawnMode::Background`) instead of + `await_dependent_run`. +- Update `prompts/spawn_subagent_description.md` (currently blocking-only + wording). +- Delivery, live parent: implement `PostCapabilityStage::drain_settled` + (`crates/loop/ironclaw_agent_loop/src/executor/post_capability.rs:34-39`) + wired to `AwaitEdgeResolver` — TODO seam: a small trait in + `ironclaw_agent_loop` (or reuse the existing loop-host port surface), + implemented on the turn_runner side, same dependency-inversion category + as `AwaitEdgeWriter`/`AwaitEdgeSettler` (`await_edge_port.rs`). No + `LoopBackgroundChildPort`. +- Delivery, parked/completed parent: resolver's settle path calls + `activate(parent_thread, input, System)`; `ThreadBusy` benign no-op; + one attempt per settled edge (the `settled` state is the dedupe). +- Batched drain (§8): multi-edge drain = one thread-snapshot read + one CAS + write across all settled `(result_ref, safe_summary)` pairs — extend + `drain_settled_group`'s tail (`resolver.rs:673`, the `:715` P2.4 comment + marks the spot); O(E+M). +- Three-trigger retry set (§8.2): settle-time activate, run-start sweep + (drain_settled runs on every `Continue`), boot pass (drains at resolver + layer, no activate, never streak-capped). +- Background edges: parent-run completion with edge open is normal + delivery, never abandonment (§2 mode-scoping). +- Tests (integration-tier, `tests/integration/`): + `settled_edge_threadbusy_is_healed_by_run_start_and_boot_pass` (both + scenarios + parent-completed precondition, §8.2); drain idempotency + crash-replay (§8.1); batched-drain write-count seam (§8 required test). + +### Slice 3 [B] — drain safety scan (prod-enable gate 1, P2.4/P2.5) + +- One synchronous scan call at the single drain write site + `update_parent_result_reference` (`resolver.rs:449`) before commit: + `SafetyLayer::sanitize_tool_output` + (`crates/substrates/ironclaw_safety/src/lib.rs:98`) or + `scan_inbound_for_secrets` (`:193`) — whichever the platform wiring + settles on; dependency already present. Covers blocking and background + (mode-agnostic call site). +- Test (integration-tier): crafted child drain content tripping the + leak-detector/injection patterns is redacted/rejected, not passed + verbatim (§7 required test). + +### Slice 4 [B] — gate escalation walk (prod-enable gate 2, P2.5/P2.6) + +- Any descendant `BlockedApproval`/`BlockedAuth` at any depth bubbles to + the **tree root's** originating surface via + `ironclaw_outbound::resolve_run_notification_context` (no second + fallback) — design §9. +- Resolution accepted from any owner-authenticated **human** surface, never + any LLM; surface layer checks `caller.user_id == root.owner_user_id` + (§9.2). +- Fix the named integration gap: `deliver_triggered_run` + (`crates/app/ironclaw_composition/src/slack_delivery.rs:2033`) watches + only the root run's own status — extend to descendant gates. +- Tests (integration-tier): gate walk end-to-end + non-owner rejection. + +### Slice 5 [B] — PR2 wrap-up: counters, operator command, e2e revival + +- `ResolveReport` counters + `ironclaw subagent edges [--scope …]` operator + command (§5.4; host-level trusted, cross-tenant like the rest of the CLI; + reports open-reservation counts beside edge counts). +- Round-5 boot-recovery fairness (bounded pending queue + per-tenant + in-flight cap ≤2 of 4) lands here with the `run_boot_recovery` process- + start wiring (§4.3 round-8 staging split) + its P1.9-extension test. +- Un-ignore all of `tests/reborn_subagent_spawn_e2e.rs`; rewrite + `background_spawn_is_rejected_before_child_run_or_auth_invocation` (:149) + into the background-accept + delivery scenario; suite runs with + harness-side capability enablement. Update `tests/CLAUDE.md` rows + (subagent group scenario, proactive/background gap #6369). + +### Slice 6 [B] — `subagent_inspect` + per-flavor config (PR3: P3.2, P3.3) + +- New capability ids under one multi-verb `FirstPartyCapabilityHandler`, + template `first_party_tools/trigger_management.rs:40-88,151-166,242` + (metadata only: status, gate state, byte counts — never raw transcript; + §7 PR3 scope note). +- Per-flavor budget plumbing (§10c, iteration limit per flavor) and + per-flavor model override (§10d) through + `flavors.rs`/`material_for_run`; crate-tier tests. + +### Slice 7 [B] — `subagent_extend` + human priority (PR4: P4.2, P4.3, P4.4) + +- `subagent_extend` = `activate(child_thread, input, ParentAgent)` + + consent-to-wake (own direct live child only) + §6 budget (8 consecutive + `ParentAgent`, derived `LIMIT 8` query, `System` excluded) — windowing + logic only; the field landed in Slice 1. +- Reservation re-claim at admission (extend on a full tree → the existing + capacity error; the one re-claim path, §5.1). +- `human_waiting` reservation marker (§6a): owner-gated CAS'd marker file + `{ owner: UserId, expires_at }`, 15-min lease + (`HUMAN_RESERVATION_LEASE_TTL`), lazy expiry, no reaper; new + `ThreadReserved` admission outcome treated like `ThreadBusy`. +- Tests: §6a's three integration cases; P4.2 crate rows; P4.4 full-tree + rejection (integration). + +### Slice 8 [B] — WebUI child tree (PR5a + PR5b) + +- `GET /api/webchat/v2/threads/{thread_id}/children` — lineage projection + over `TurnRunRecord.{parent_run_id, spawn_tree_root_run_id, + subagent_depth}`, no new store; ride the `nested_dispatch` re-parenting + shape (`runtime_projection.rs:197`). Route in `webui_v2/router.rs`. +- `ThreadTree` sidebar in `frontend/` + raw-vs-framed display rule (§11: + raw child transcript is human-only) + interrupt & take over (P5.4 — + console compose of run-cancel + `activate(Human)`, no new state). +- Tests: integration-tier endpoint test; frontend per its own tier. + +### Slice 9 [B] — `subagent_cancel` (PR6) — **needs security review** + +- Model-facing tool wrapping the run-cancel mechanism; drives child to + terminal → slot release via the §5.5 tri-state; explicit tree teardown is + the one legitimate open→abandoned path besides rollback (§2b). + +### Slice 10 [B] — production enable (the deviation point) + +- Remove the `builtin.spawn_subagent` entry from + `default_disabled_capability_ids()` + (`crates/loop/ironclaw_turn_runner/src/runtime.rs:277-282`) and the + `TEMP(disable-spawn-subagents)` composition note (`:801-806`, + `composition/src/runtime.rs:3892`). +- Flip the "capability stays off" assertions: + `tests/integration/tool_call.rs:756,797`, + `tests/integration/tool_disclosure.rs:168`, + `crates/app/ironclaw_composition/tests/service_factory.rs:386,413`. +- Update `crates/Architecture.md:873-879` status prose and any guidance + citing the deny-filter as current. +- Precondition checklist (design §7 + our deviation): Slices 1–9 merged, + both backends green, `cargo test -p ironclaw_architecture_tests` green, + `scripts/reborn-e2e-rust.sh` green, e2e suite un-ignored and green, + scan + escalation tests green, PR6 security review done. + +## Standing constraints for every slice (from recon — plan must carry these) + +- CAS discipline: single-record RMW via the shared bounded CAS path; + fixed close order state → release → prune → `delete_if_version` with the + token from the caller's own last successful CAS; dual-backend parity + suites (Postgres + libSQL + in-memory). +- Edge deletion is the sanctioned carve-out from "LLM data is never + deleted" — keep §2's reasoning in an implementation comment at the + delete site. +- Traits in `ironclaw_loop_host`/`ironclaw_agent_loop`, impls in + `ironclaw_turn_runner`; never append to `completion_observer.rs`; new + files < 800 lines; `subagent_spawn_port.rs` test-support ratchet frozen + at 3. +- Child authority: empty grant/lease set; allowlist is a ceiling; child + re-acquires leases via its own gates. +- No `info!`/`warn!` from background tasks (REPL rule) — `debug!` + + counters per §5.4. +- Run `cargo test -p ironclaw_architecture_tests` whenever edges, layer + keys, or pinned guidance files move. diff --git a/docs/internal/reborn/subagent-spawn/research-background-enable.md b/docs/internal/reborn/subagent-spawn/research-background-enable.md new file mode 100644 index 00000000000..23e2786ecfe --- /dev/null +++ b/docs/internal/reborn/subagent-spawn/research-background-enable.md @@ -0,0 +1,200 @@ +# Recon: completing subagent support with background subagents enabled + +/shape recon output (2026-08-19, branch `subagent`). Three read-only lanes +(trace / patterns / constraints); all file:line claims spot-checked against +live code. The canonical design this work executes is +`thread-harness-design.md` in this directory — this file is a map of what has +landed vs. what remains, not a new design. + +## Map + +### What has landed (PR1 of the design's 6-PR staging — blocking mode, switched off) + +- Spawn entry: `builtin.spawn_subagent` manifest in + `crates/kernel/ironclaw_host_runtime/src/first_party_tools/spawn_subagent.rs:4` + (registered `first_party_tools/mod.rs:248`); real behavior lives in the + loop-host decorator `SubagentSpawnCapabilityPort` + (`crates/loop/ironclaw_loop_host/src/subagent_spawn_port.rs:381`, ~1600 + lines): schema `:62` (only `subagent_type`/`task`/`handoff` — no mode field + advertised), admission caps (fanout ≤4 `:669`, depth ≤1 `:44`, descendant + cap, actor/scope checks `:727-774`), `finish_spawn` `:889` (placeholder + result → child thread with `SubagentThreadMetadata` → durable goal input → + `AwaitedChildSetRecord` → `submit_child_run` with process dependency → + parent parks on an await-dependent-run gate `:1101`), compensation rollback + `:1112`. Model-facing description: + `crates/loop/ironclaw_loop_host/prompts/spawn_subagent_description.md` + (blocking-only wording). +- Flavors: four static kinds (general/explorer/coder/planner), all + `allow_nesting: false` — + `crates/loop/ironclaw_turn_runner/src/subagent/flavors.rs:106-134`; + per-flavor directions in `subagent/directions/*.md`. +- Result return (blocking): `AwaitEdgeResolver` + (`crates/loop/ironclaw_turn_runner/src/subagent/await_edge/resolver.rs`) is + a `TurnCommittedEventObserver` (`:1881`); child terminal → + `settle_and_maybe_drain` `:603` → `drain_settled_group` `:673` (waits for + all siblings under the shared gate ref) → `update_parent_result_reference` + `:449` (framed, byte-capped, untrusted-wrapped via + `subagent/untrusted_text.rs`) → `resume_parent` `:489` + (`ResumeTurnPrecondition::BlockedDependentRunGate`) exactly once → close + edges. Edge store is a CAS'd projection over the kernel process-dependency + journal (`await_edge/store.rs`), states Open → Settled → Drained/Abandoned + (`await_edge/mod.rs:18`). Boot/lazy recovery: + `await_edge/boot_recovery.rs` (`ScopeRecoveryDriver`), edge reconstruction + from child-thread metadata `resolver.rs:285`. +- Production wiring: `crates/app/ironclaw_composition/src/runtime.rs:104-106, + 268-269, 3881`; DI seam (loop_host cannot depend on turn_runner): + `crates/loop/ironclaw_loop_host/src/await_edge_port.rs`. + +### The two off-switches + +1. `default_disabled_capability_ids()` + (`crates/loop/ironclaw_turn_runner/src/runtime.rs:277-282`, applied + `:801-806`, `TEMP(disable-spawn-subagents)`) deny-filters + `builtin.spawn_subagent` from every shipped surface. This is the sole + on/off gate by standing ruling (thread-harness-design.md §Terminology — + no `subagent.v2_enabled` flag, no cargo feature). +2. `background_subagents_disabled()` (`subagent_spawn_port.rs:1500`), + enforced in `TryFrom` `:222-227` for both + `mode: "background"` and legacy `run_in_background: true` — and + `finish_spawn` hard-codes `SpawnSubagentMode::Blocking` at `:908`. + `SpawnSubagentMode::Background` exists as a type (`:183`) and is carried + durably on `AwaitedChildSetRecord`/`SubagentThreadMetadata`, but is never + constructed in production. + +### What is missing (= the design's PR2, plus staged follow-ons) + +- **Background completion delivery.** Blocking works because the parent is + parked; background has no consumer. `PostCapabilityStage::drain_settled()` + (`crates/loop/ironclaw_agent_loop/src/executor/post_capability.rs:34-39`) + is a stub returning `Vec::new()`; the `LoopBackgroundChildPort` it names + was never built and the design supersedes it — delivery is + `activate(parent_thread, input, provenance=System)` with three healing + triggers (settle-time activate, run-start sweep on every Continue, boot + pass; §8.2, incl. required test + `settled_edge_threadbusy_is_healed_by_run_start_and_boot_pass`). +- **`ActivationProvenance` / `subagent_activation_provenance`** on + `TurnRunRecord` — zero hits in the tree. Type belongs in + `ironclaw_host_api::turn` per `ironclaw_turns/AGENTS.md:20`. +- **System-wake streak cap** (16 consecutive System activations, derived by + `LIMIT K` query, no stored counter; §8.3) — not present. +- **Batched multi-edge drain** (one snapshot read + one CAS write, O(E+M)) — + the shipped per-member loop is documented as blocking-only-adequate + (`resolver.rs:709-717`). +- **Gate-propagation escalation walk** (§9, pulled into PR2 as a prod-enable + gate): descendant `BlockedApproval`/`BlockedAuth` must bubble to the tree + root's originating surface via + `ironclaw_outbound::resolve_run_notification_context`; resolution accepted + only from an owner-authenticated human surface (`caller.user_id == + root.owner_user_id`), never any LLM. Named integration gap: + `deliver_triggered_run` (`ironclaw_composition/src/slack_delivery.rs:2033`) + watches only the root run's status. +- **Safety scan on the drain write** (second hard prod-enable gate, + design §7 round-5): one synchronous `SafetyLayer::sanitize_tool_output` + (`crates/substrates/ironclaw_safety/src/lib.rs:98`) or + `scan_inbound_for_secrets` (`:193`) call at + `update_parent_result_reference` before commit; `ironclaw_turn_runner` + already depends on `ironclaw_safety`. One seam covers both modes. +- **Observe/control surfaces** (staged after prod-enable): PR3 + `subagent_inspect` (metadata-only), PR4 `subagent_extend` (+ ParentAgent + streak cap of 8, `ThreadReserved`/human-priority reservation), PR5 WebUI + (`GET .../threads/{id}/children` + `ThreadTree` sidebar + raw-vs-framed + display rule), PR6 `subagent_cancel` (security review gate). WebUI has + zero subagent awareness today (i18n strings only); the one existing + "child work under parent" projection rule to ride is `nested_dispatch` + re-parenting in + `crates/events/ironclaw_event_projections/src/runtime_projection.rs:197`. +- **No cascade teardown**: `abandon_awaited_child` is rollback-only; for + background edges, parent completion with the edge open is the normal + delivery case, not abandonment (§2). + +### Patterns to copy (nearest existing implementations) + +- New-turn-from-background-worker (the shape `activate()` generalizes): + `crates/app/ironclaw_composition/src/automation/trigger_poller_trusted_submit.rs:148` + + `conversation_turn_submitter.rs:28,63` (incl. `ThreadBusy` → retryable + classification at `:118-121`). `TurnCoordinator::submit_turn` + + `TurnError::ThreadBusy` (`crates/kernel/ironclaw_turns/src/status.rs:269`) + is the real seam; the design's `activate()` does not exist yet. +- Wake-notifier (secondary nudge only, cannot create work): + `TurnRunWakeNotifier` (`crates/kernel/ironclaw_turns/src/coordinator.rs:102`, + best-effort notify `:475`; scheduler impl + `crates/loop/ironclaw_turn_runner/src/turn_scheduler.rs:383-388`). +- In-flight injection into a running parent (distinct third mechanism — + use at most one of the three per job): `HostInputQueue` / + `HostInputEnqueuePort` + (`crates/loop/ironclaw_loop_host/src/input_queue.rs:45,144`). +- Multi-verb built-in tool family (template for inspect/extend/cancel): + `crates/kernel/ironclaw_host_runtime/src/first_party_tools/trigger_management.rs:40-88,151-166,242,282,340`. + +### Binding constraints (beyond the design doc itself) + +- Root `AGENTS.md:47` — spawn creates/wires child runs only; everything else + goes through the existing runner/driver/executor path. `AGENTS.md:49` + + arch test `reborn_dependency_boundaries.rs:1400,1657` — background wake + must not mint `TrustedInboundTurnRequest` or touch trusted trigger + submitters (note: the `:1657` scan roots do not currently cover + `subagent_spawn_port.rs` — extend the scan-root list if a submitter-shaped + path is added). +- No new crate, no cargo feature, no stored counters, no new tables + (design §12 non-goals + `.claude/rules/cargo-features.md`). +- Closed await-edges are deleted — the one carved-out exception to + "LLM data is never deleted" (§2; preserve the carve-out reasoning in an + implementation comment). Fixed close order: state CAS → reservation + release → prune → `delete_if_version` with the token from the Released CAS. +- Dual-backend parity (Postgres + libSQL) via shared conformance suites; + every RMW through the shared bounded CAS helper + (`.claude/rules/database.md:54-82`). +- Traits in `ironclaw_loop_host`, impls in `ironclaw_turn_runner` + (dependency-inversion category of `type-placement.md`); never append to + the 4,758-line `completion_observer.rs`; new files aim <800 lines. +- `subagent_spawn_port.rs` ratchets: frozen at exactly 3 `test-support` + methods (`reborn_struct_test_support_ratchet.rs:378`); on the + provider-name allowlist (`reborn_dependency_boundaries.rs:2179`). +- Child authority: empty grant/lease set at start; surface allowlist is a + ceiling, not authority; child re-acquires leases via its own approval gate. + +### Tests + +- Today: one integration file + (`tests/integration/subagent_await_edge.rs:22,106`); five `#[ignore]`d e2e + cases (`tests/reborn_subagent_spawn_e2e.rs:25,90,149,203,313` — one of + which pins the background *rejection* and flips meaning); + active tests asserting the capability stays off + (`tests/integration/tool_call.rs:756,797`, `tool_disclosure.rs:168`, + `crates/app/ironclaw_composition/tests/service_factory.rs:386,413`); + zero Python E2E. Declared gaps at `tests/CLAUDE.md:519` (no subagent group + scenario) and the proactive/background row (#6369). +- Prod enable = clear the deny filter after PR2 + un-ignore e2e + matrix + green + the two hard gates (escalation walk, drain safety scan). Design + names integration-tier for drain idempotency/batching, ThreadBusy healing, + the scan gate, the gate walk + non-owner rejection; crate-tier only for + the streak-cap windowing, release idempotency/ordering, flavor overrides, + `delete_if_version` parity. Always run + `cargo test -p ironclaw_architecture_tests` and `scripts/reborn-e2e-rust.sh`. + +## Briefing + +A subagent here is not a separate engine: it is an ordinary child turn-run on +its own thread, spawned by the built-in `spawn_subagent` tool. The spawn +path, the durable parent↔child "await edge" bookkeeping, child-output +framing, crash recovery, and blocking-mode delivery (parent parks on a gate, +resolver wakes it when all children finish) are all fully built — and then +deliberately switched off in two places: the tool is deny-filtered out of +every production surface, and any request for background mode is rejected at +the argument decoder. + +"Complete support with background fully enabled" is therefore not a design +problem — an accepted, round-8-hardened design (`thread-harness-design.md`) +already specifies it, and PR1 of its six-PR staging has landed on this +branch. The remaining work is the design's PR2: let a child run *without* +parking the parent, and when the child finishes, durably hand the result to +a parent that may be mid-run, idle, or finished — by writing the framed +result into the parent transcript and "activating" the parent thread as a +System-provenance turn, with a run-start sweep and a boot pass healing any +missed wake, and a derived 16-wake streak cap stopping a parent from looping +autonomously forever. Two safety gates must land before the deny filter is +cleared: a blocked child's approval must escalate to the tree root's human +(otherwise a background child stuck on approval is invisible), and the +child→parent result write must pass the safety scan layer. After PR2 the +design stages observe/control surfaces: an inspect tool, an extend-runtime +tool, a WebUI child-thread tree, and cancel. diff --git a/docs/internal/superpowers/plans/2026-08-19-subagent-background-slices-1-2.md b/docs/internal/superpowers/plans/2026-08-19-subagent-background-slices-1-2.md new file mode 100644 index 00000000000..4bebb42860e --- /dev/null +++ b/docs/internal/superpowers/plans/2026-08-19-subagent-background-slices-1-2.md @@ -0,0 +1,1834 @@ +# Subagent Background Delivery (Slices 1–2) Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Let a subagent child run in the background — the parent keeps working instead of parking — and durably deliver the child's result back to the parent whether it is mid-run, idle, or already finished. + +**Architecture:** Blocking-mode subagents are already built end-to-end (spawn port → durable await-edge → resolver → parent resume) and switched off in production by a deny-filter. This plan does not fork any of it. It adds the one missing half: a provenance-tagged `activate()` re-activation primitive on the existing `TurnCoordinator`, and background delivery wired into the existing `AwaitEdgeResolver` drain path plus the existing (currently stubbed) `PostCapabilityStage::drain_settled` seam. Production stays deny-filtered; the e2e suite exercises the new path through the harness's own capability enablement. + +**Tech Stack:** Rust (async/tokio, `async_trait`), serde, the IronClaw Reborn workspace under `crates/`. No new crate, no cargo feature, no new dependency. + +**Spec:** `docs/internal/reborn/subagent-spawn/thread-harness-design.md` (canonical; §2, §5, §6, §8, §8.1, §8.2, §8.3 are the sections this plan implements). Shape decisions and slice order: `docs/internal/reborn/subagent-spawn/pr2-pr6-shape.md`. Recon map: `docs/internal/reborn/subagent-spawn/research-background-enable.md`. + +## Global Constraints + +Copied verbatim from the spec and the repo contract. Every task's requirements implicitly include this section. + +- **No feature flag, no new crate, no stored counters, no new tables.** `disabled_capability_ids` is the sole on/off gate for the capability (design "Standing ruling"; §12 non-goals; `.claude/rules/cargo-features.md`). +- **Production stays deny-filtered in this plan.** `default_disabled_capability_ids()` in `crates/loop/ironclaw_turn_runner/src/runtime.rs:277-282` is NOT touched by slices 1–2. Do not remove `builtin.spawn_subagent` from it. Do not edit the tests that assert the capability is off (`tests/integration/tool_call.rs:756,797`, `tests/integration/tool_disclosure.rs:168`, `crates/app/ironclaw_composition/tests/service_factory.rs:386,413`). +- **Spawn creates and wires child runs only.** Planning, execution, capability calls, checkpointing, gates, retries, and completion continue through the existing runner/driver/executor path (root `AGENTS.md:47`). +- **Never mint `TrustedInboundTurnRequest`** and never call a trusted trigger submitter factory from any code this plan touches (root `AGENTS.md:49`; pinned by `crates/app/ironclaw_architecture_tests/tests/reborn_dependency_boundaries.rs:1400,1657`). A background wake is an ordinary provenance-tagged submission, not trusted ingress. +- **Three independently named budget constants**, never merged by a refactor: `DEFAULT_SUBAGENT_MAX_TREE_DESCENDANTS = 16` (existing), the SUBAGENT family `iteration_limit` 16 (existing), and `SYSTEM_WAKE_STREAK_CAP = 16` (new in Task 4). They coincide numerically by accident (design §8.3). +- **No `.unwrap()` / `.expect()` in production code** (tests are fine). Propagate with cause: `.map_err(|e| SomeError::Variant { reason: e.to_string() })?`. `.map_err(|_| …)` is banned outright and is not `silent-ok`-exemptible (`.claude/rules/error-handling.md`). +- **Typed identities only.** Never re-derive a run/thread id from a display string. New fixed-set values are enums with explicit serde (`.claude/rules/types.md`). +- **Additive serde on every persisted struct:** `#[serde(default, skip_serializing_if = "Option::is_none")]` so existing durable rows keep deserializing. +- **`ironclaw_loop_host` may not depend on `ironclaw_turn_runner`.** Traits go in `loop_host`/`agent_loop`, impls in `turn_runner` (design §4.1; `.claude/rules/type-placement.md` dependency-inversion category). +- **Never append to `crates/loop/ironclaw_turn_runner/src/subagent/completion_observer.rs`** (4,758 lines, over budget — design §4.6). New files aim under 800 lines (`.claude/rules/architecture.md`). +- **`crates/loop/ironclaw_loop_host/src/subagent_spawn_port.rs` is frozen at exactly 3 `test-support` methods** by `crates/app/ironclaw_architecture_tests/tests/reborn_struct_test_support_ratchet.rs:378`. Adding a 4th fails the ratchet. +- **Prompt text lives in `prompts/*.md`**, loaded with `include_str!()`, never inline in Rust (root `AGENTS.md`; pinned by `reborn_composition_boundaries`). +- **No `info!`/`warn!` from background tasks** — they corrupt the REPL TUI. Use `debug!` plus counters (repo `CLAUDE.md`; design §5.4). +- **Test-first, always.** Write the failing test, run it, watch it fail for the right reason, then implement. Never weaken an assertion to go green. +- **Run after any task that moves dependency edges, layer keys, or pinned guidance:** `cargo test -p ironclaw_architecture_tests`. + +## Status (updated 2026-08-19) + +**Slice 1 is complete and committed** — Tasks 1–5, each test-first, each with +its verification command run and green. Commits: + +| Task | Commit | Evidence | +|---|---|---| +| 1 | `2ee01e5a2` | wire-string test red → green | +| 2 | `aea2c81e3` | metadata round-trip + legacy-default tests; workspace check clean across 26 changed files | +| 3 | `4a953a5d9` | 4 tests, incl. the fail-closed default | +| 4 | `8ae9832f7` | 2 tests, **mutation-verified** (flipping the sort and dropping the kind filter each kill the test) | +| 5 | `fc0a2f8f8` | 6 predicate tests + 3 caller tests, **mutation-verified** (disabling the cap kills 2) | + +Gates run at slice close: `cargo test -p ironclaw_turns -p ironclaw_processes` +(12 suites green), `cargo test -p ironclaw_architecture_tests` (all green after +re-pinning the `host_api` contracts size ceiling, which `ActivationProvenance` +tipped over), `cargo check --workspace --all-targets` clean, `cargo fmt`. +Both standing invariants re-verified: the production deny-filter is untouched +and the diff adds no trusted-ingress minting. + +**Two gates could not be run in this environment, and are outstanding:** +- `cargo clippy` — the only Rust toolchain on this box has no `clippy` + component and no `rustup` to add one. +- Anything requiring the WebUI build — its build script shells out to + `pnpm` via a corepack that is broken here (`ERR_VM_DYNAMIC_IMPORT_CALLBACK_MISSING`), + unrelated to these changes. Everything else was verified with + `SKIP_FRONTEND_BUILD=1`, the build script's own sanctioned opt-out. + +**Slice 2 (Tasks 6–10) has not been started.** Task 6 is the next action. + +--- + +### Task 1 ✅ DONE: `ActivationProvenance` vocabulary + +Adds the enum that tags *why* a run was created. Nothing reads it yet — Task 2 persists it, Tasks 3/4 act on it. + +**Files:** +- Modify: `crates/contracts/ironclaw_host_api/src/turn.rs` (add beside `TurnStatus`, which begins at line 866) +- Test: same file, in its existing `#[cfg(test)] mod tests` + +**Interfaces:** +- Consumes: nothing. +- Produces: `ironclaw_host_api::turn::ActivationProvenance` with variants `Human`, `ParentAgent`, `System`; wire strings `"human"`, `"parent_agent"`, `"system"`. Re-exported through `ironclaw_turns`' prelude (which already re-exports `TurnStatus` and friends), so downstream crates write `use ironclaw_turns::ActivationProvenance;`. + +- [x] **Step 1: Write the failing test** + +Add to the `#[cfg(test)] mod tests` block in `crates/contracts/ironclaw_host_api/src/turn.rs`: + +```rust +#[test] +fn activation_provenance_wire_strings_are_snake_case() { + use super::ActivationProvenance; + + assert_eq!( + serde_json::to_value(ActivationProvenance::Human).expect("serialize"), + serde_json::json!("human") + ); + assert_eq!( + serde_json::to_value(ActivationProvenance::ParentAgent).expect("serialize"), + serde_json::json!("parent_agent") + ); + assert_eq!( + serde_json::to_value(ActivationProvenance::System).expect("serialize"), + serde_json::json!("system") + ); + + let round_tripped: ActivationProvenance = + serde_json::from_value(serde_json::json!("parent_agent")).expect("deserialize"); + assert_eq!(round_tripped, ActivationProvenance::ParentAgent); +} +``` + +- [x] **Step 2: Run test to verify it fails** + +Run: `cargo test -p ironclaw_host_api activation_provenance_wire_strings_are_snake_case` +Expected: FAIL to compile — `cannot find type ActivationProvenance in this scope`. + +- [x] **Step 3: Write minimal implementation** + +Add to `crates/contracts/ironclaw_host_api/src/turn.rs`, immediately before the `TurnStatus` definition: + +```rust +/// Why a run was created on its thread. Set once at run creation and +/// immutable thereafter — the derived streak caps (design §6, §8.3) read +/// windows of this field instead of maintaining a stored counter. +/// +/// `Human` is the ordinary case and resets both streaks. `ParentAgent` tags a +/// parent re-activating one of its own children (`subagent_extend`). +/// `System` tags a background subagent completion waking its parent. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ActivationProvenance { + Human, + ParentAgent, + System, +} +``` + +- [x] **Step 4: Run test to verify it passes** + +Run: `cargo test -p ironclaw_host_api activation_provenance_wire_strings_are_snake_case` +Expected: PASS. + +- [x] **Step 5: Re-export from the turns prelude** + +Find the prelude re-export list in `crates/kernel/ironclaw_turns/src/lib.rs` (the one that already names `TurnStatus`) and add `ActivationProvenance` to it, alphabetically within its group. Confirm with: + +Run: `cargo build -p ironclaw_turns` +Expected: builds clean. + +- [x] **Step 6: Commit** + +```bash +git add crates/contracts/ironclaw_host_api/src/turn.rs crates/kernel/ironclaw_turns/src/lib.rs +git commit -m "feat(turns): add ActivationProvenance vocabulary for subagent activation tagging" +``` + +--- + +### Task 2 ✅ DONE: Persist provenance on the run record + +Threads the provenance from a submission into durable process metadata and back out onto `TurnRunRecord`. This is the field Tasks 3 and 4 read. + +**Files:** +- Modify: `crates/kernel/ironclaw_turns/src/request.rs` (`SubmitTurnRequest`, struct begins line 50; lineage fields at 73-77 are the pattern to copy) +- Modify: `crates/kernel/ironclaw_turns/src/agent_turn_runtime.rs` (`TurnRunRecord`, lineage fields around lines 181-185) +- Modify: `crates/kernel/ironclaw_turns/src/process_projection/metadata.rs` (`AgentTurnProcessStateMetadata`, `subagent_depth` at line 101 is the sibling to copy) +- Modify: `crates/kernel/ironclaw_turns/src/process_projection/runtime.rs` (write side: wherever `subagent_depth` is written into the metadata at submit; read side: `turn_run_record_from_process_snapshot`, line 1083, which reads `metadata.subagent_depth` at line 1130) +- Modify: all `SubmitTurnRequest { … }` literal sites (46 across `crates/` and `tests/` — see Step 5) +- Test: `crates/kernel/ironclaw_turns/tests/` (add to the existing process-projection round-trip suite; if none exists, create `crates/kernel/ironclaw_turns/tests/activation_provenance.rs`) + +**Interfaces:** +- Consumes: `ActivationProvenance` from Task 1. +- Produces: + - `SubmitTurnRequest.subagent_activation_provenance: Option` (serde-defaulted; `None` means an ordinary human-initiated submission). + - `TurnRunRecord.subagent_activation_provenance: Option` — set once at run creation, never mutated. + - `AgentTurnProcessStateMetadata.subagent_activation_provenance: Option` — the durable carrier. + +- [x] **Step 1: Write the failing test** + +Create `crates/kernel/ironclaw_turns/tests/activation_provenance.rs`. Follow the construction helpers used by the crate's existing projection tests — open `crates/kernel/ironclaw_turns/src/process_projection/runtime.rs`'s own `#[cfg(test)] mod tests` and reuse its snapshot fixture builder rather than inventing one: + +```rust +//! Provenance must survive the process-metadata round trip: a submission +//! tagged `System` must project back onto `TurnRunRecord` as `System`, and an +//! untagged legacy row must project as `None` (not a default variant). + +use ironclaw_turns::ActivationProvenance; + +#[test] +fn provenance_round_trips_through_agent_turn_process_metadata() { + let metadata_json = serde_json::json!({ + "turn_id": "turn-1", + "accepted_message_ref": "msg-1", + "resolved_run_profile_id": "default", + "resolved_run_profile_version": 1, + "subagent_activation_provenance": "system", + }); + + let metadata: ironclaw_turns::process_projection::AgentTurnProcessStateMetadata = + serde_json::from_value(metadata_json).expect("metadata deserializes"); + + assert_eq!( + metadata.subagent_activation_provenance, + Some(ActivationProvenance::System), + "a System-tagged submission must round-trip through durable metadata" + ); +} + +#[test] +fn legacy_metadata_without_provenance_projects_as_none() { + let metadata_json = serde_json::json!({ + "turn_id": "turn-1", + "accepted_message_ref": "msg-1", + "resolved_run_profile_id": "default", + "resolved_run_profile_version": 1, + }); + + let metadata: ironclaw_turns::process_projection::AgentTurnProcessStateMetadata = + serde_json::from_value(metadata_json).expect("legacy metadata deserializes"); + + assert_eq!( + metadata.subagent_activation_provenance, None, + "rows written before this field existed must stay readable as None" + ); +} +``` + +If `AgentTurnProcessStateMetadata` is not public from the crate root, place these two tests inside `crates/kernel/ironclaw_turns/src/process_projection/metadata.rs`'s own `#[cfg(test)] mod tests` instead and drop the module path prefix. Do not add a `pub` export just to make an external test compile. + +- [x] **Step 2: Run test to verify it fails** + +Run: `cargo test -p ironclaw_turns provenance_round_trips_through_agent_turn_process_metadata` +Expected: FAIL — `no field subagent_activation_provenance on type AgentTurnProcessStateMetadata`. + +- [x] **Step 3: Add the field to the durable metadata carrier** + +In `crates/kernel/ironclaw_turns/src/process_projection/metadata.rs`, immediately after the `subagent_depth` field (line 101): + +```rust + /// Why this run was activated on its thread (design §6/§8.3). Set once at + /// run creation, never mutated. Absent on rows written before the field + /// existed, and on every ordinary human-initiated submission. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub subagent_activation_provenance: Option, +``` + +Add `ActivationProvenance` to that file's `use` list. + +- [x] **Step 4: Run test to verify it passes** + +Run: `cargo test -p ironclaw_turns provenance_round_trips_through_agent_turn_process_metadata legacy_metadata_without_provenance_projects_as_none` +Expected: PASS (both). + +- [x] **Step 5: Add the field to the request and record, and fix every construction site** + +In `crates/kernel/ironclaw_turns/src/request.rs`, at the end of `SubmitTurnRequest` (after `product_context`): + +```rust + /// Why this submission is activating the thread (design §6/§8.3). + /// `None` — the default for every ordinary caller — is an untagged, + /// human-initiated submission. Only the coordinator's `activate()` entry + /// point sets this to `ParentAgent` or `System`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub subagent_activation_provenance: Option, +``` + +In `crates/kernel/ironclaw_turns/src/agent_turn_runtime.rs`, at the end of `TurnRunRecord` (after `resume_disposition`), the identical field with this doc comment: + +```rust + /// Why this run was activated (design §6/§8.3). Immutable after creation. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub subagent_activation_provenance: Option, +``` + +Then wire the projection in `crates/kernel/ironclaw_turns/src/process_projection/runtime.rs`: +- Read side, in `turn_run_record_from_process_snapshot` (line 1083), beside `subagent_depth: metadata.subagent_depth,` at line 1130, add: + ```rust + subagent_activation_provenance: metadata.subagent_activation_provenance, + ``` +- Write side: find where the submission builds `AgentTurnProcessStateMetadata` (grep `subagent_depth:` in this file — the write site sets it from the request) and copy the provenance across the same way: + ```rust + subagent_activation_provenance: request.subagent_activation_provenance, + ``` + +Now fix the construction sites. Both structs are built with explicit field lists (no `..Default::default()` spread anywhere), so the compiler will name every one: + +```bash +cargo build --workspace --all-targets 2>&1 | grep -c "missing field .subagent_activation_provenance" +``` + +Add `subagent_activation_provenance: None,` to each reported site. There are 46 `SubmitTurnRequest { … }` literals plus the `TurnRunRecord { … }` literals. Every one of them is an ordinary submission or a test fixture — `None` is correct for all of them. Do not set a non-`None` value anywhere in this task; Task 3 owns the only caller that does. + +- [x] **Step 6: Run the full crate suite plus a workspace build** + +Run: `cargo test -p ironclaw_turns` +Expected: PASS. + +Run: `cargo build --workspace --all-targets` +Expected: builds clean, zero `missing field` errors. + +- [x] **Step 7: Commit** + +```bash +git add -A +git commit -m "feat(turns): persist subagent activation provenance on the run record" +``` + +--- + +### Task 3 ✅ DONE: The `activate()` re-activation primitive + +The single primitive for re-activating an existing thread with a provenance tag. Background delivery (Task 7) calls it; `subagent_extend` (a later slice) will too. + +**Files:** +- Modify: `crates/kernel/ironclaw_turns/src/request.rs` (add `ActivateThreadRequest`) +- Modify: `crates/kernel/ironclaw_turns/src/coordinator.rs` (`TurnCoordinator` trait at line 125; `DefaultTurnCoordinator` impl at line 484; the `Arc` blanket impl at line 771) +- Test: `crates/kernel/ironclaw_turns/src/coordinator.rs`'s own `#[cfg(test)] mod tests` (it already has one — the declared-limits test lives at line 829) + +**Interfaces:** +- Consumes: `ActivationProvenance` (Task 1); `SubmitTurnRequest.subagent_activation_provenance` (Task 2). +- Produces: + ```rust + pub struct ActivateThreadRequest { + pub scope: TurnScope, + pub actor: TurnActor, + pub accepted_message_ref: AcceptedMessageRef, + pub provenance: ActivationProvenance, + pub idempotency_key: IdempotencyKey, + pub received_at: TurnTimestamp, + pub requested_run_profile: Option, + } + ``` + and `TurnCoordinator::activate(&self, request: ActivateThreadRequest) -> Result`, with a fail-closed default impl. `TurnError::ThreadBusy`-shaped rejection when a run is already live on the thread comes from the existing admission path unchanged — `activate` adds no new busy handling. + +- [x] **Step 1: Write the failing test** + +Add to the `#[cfg(test)] mod tests` block in `crates/kernel/ironclaw_turns/src/coordinator.rs`. Reuse the module's existing fixture helpers for building a coordinator — read the `declared_limits_narrow_profile_ceilings_and_never_widen_them` test at line 829 first and copy its setup shape rather than inventing new fixtures: + +```rust +/// `activate()` is `submit_turn` plus a provenance tag — it must reach the +/// same admission path and stamp the request so the run record carries the +/// tag. Anything else (a second submission path, a bypass of admission) would +/// violate the design's "spawn creates and wires child runs only" rule. +#[tokio::test] +async fn activate_submits_through_admission_and_stamps_provenance() { + let (coordinator, recorder) = recording_coordinator_fixture(); + + let response = coordinator + .activate(ActivateThreadRequest { + scope: test_scope(), + actor: test_actor(), + accepted_message_ref: test_message_ref(), + provenance: ActivationProvenance::System, + idempotency_key: IdempotencyKey::new("activate-test-1").expect("valid key"), + received_at: chrono::Utc::now(), + requested_run_profile: None, + }) + .await + .expect("activate succeeds on an idle thread"); + + assert!(matches!(response, SubmitTurnResponse::Accepted { .. })); + + let submitted = recorder.submitted_requests(); + assert_eq!(submitted.len(), 1, "activate must submit exactly one turn"); + assert_eq!( + submitted[0].subagent_activation_provenance, + Some(ActivationProvenance::System), + "activate must stamp the provenance onto the submission it builds" + ); +} + +/// A coordinator that has not opted into activation must refuse, not silently +/// fall through to an untagged submission (fail-closed default). +#[tokio::test] +async fn default_activate_impl_refuses_rather_than_submitting_untagged() { + struct NonActivatingCoordinator; + + #[async_trait] + impl TurnCoordinator for NonActivatingCoordinator { + async fn prepare_turn(&self, _scope: TurnScope) -> Result { + unreachable!("not exercised by this test") + } + async fn submit_turn( + &self, + _request: SubmitTurnRequest, + ) -> Result { + panic!("default activate() must not reach submit_turn"); + } + async fn resume_turn( + &self, + _request: ResumeTurnRequest, + ) -> Result { + unreachable!("not exercised by this test") + } + async fn retry_turn( + &self, + _request: RetryTurnRequest, + ) -> Result { + unreachable!("not exercised by this test") + } + async fn cancel_run( + &self, + _request: CancelRunRequest, + ) -> Result { + unreachable!("not exercised by this test") + } + async fn get_run_state( + &self, + _request: GetRunStateRequest, + ) -> Result { + unreachable!("not exercised by this test") + } + } + + let error = NonActivatingCoordinator + .activate(ActivateThreadRequest { + scope: test_scope(), + actor: test_actor(), + accepted_message_ref: test_message_ref(), + provenance: ActivationProvenance::System, + idempotency_key: IdempotencyKey::new("activate-test-2").expect("valid key"), + received_at: chrono::Utc::now(), + requested_run_profile: None, + }) + .await + .expect_err("default impl must refuse"); + + assert!( + matches!(error, TurnError::InvalidRequest { .. }), + "default activate() must fail closed, got {error:?}" + ); +} +``` + +`recording_coordinator_fixture()`, `test_scope()`, `test_actor()`, and `test_message_ref()` are helpers you add alongside these tests if the module does not already provide equivalents. The recorder must capture the full `SubmitTurnRequest` the coordinator built (not just a count) — per `.claude/rules/testing.md`, doubles capture every argument the production caller passes. + +- [x] **Step 2: Run test to verify it fails** + +Run: `cargo test -p ironclaw_turns activate_submits_through_admission_and_stamps_provenance default_activate_impl_refuses_rather_than_submitting_untagged` +Expected: FAIL to compile — `no method named activate`, `cannot find struct ActivateThreadRequest`. + +- [x] **Step 3: Add the request type** + +In `crates/kernel/ironclaw_turns/src/request.rs`, after `SubmitChildRunRequest`: + +```rust +/// Re-activate an existing thread with an explicit provenance tag — the single +/// re-activation primitive (design §1). This is deliberately *not* a second +/// admission path: `activate` builds an ordinary [`SubmitTurnRequest`], so +/// one-active-run exclusivity, idempotency replay, and busy rejection all +/// behave exactly as they do for any other submission. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ActivateThreadRequest { + pub scope: TurnScope, + pub actor: TurnActor, + pub accepted_message_ref: AcceptedMessageRef, + pub provenance: ActivationProvenance, + pub idempotency_key: IdempotencyKey, + pub received_at: TurnTimestamp, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub requested_run_profile: Option, +} +``` + +- [x] **Step 4: Add the trait method with a fail-closed default** + +In `crates/kernel/ironclaw_turns/src/coordinator.rs`, inside `pub trait TurnCoordinator` (line 125), after `submit_turn`: + +```rust + /// Re-activate an existing thread with a provenance tag (design §1, §8). + /// + /// Defaults to a refusal rather than an untagged `submit_turn` fallthrough: + /// a coordinator that has not opted into activation semantics must not + /// silently create runs whose provenance the streak caps then cannot see. + /// `DefaultTurnCoordinator` provides the real implementation; this default + /// exists so the many test doubles of this trait need not each restate it + /// (same reason `abort_prepared_turn` above carries one). + async fn activate( + &self, + _request: ActivateThreadRequest, + ) -> Result { + Err(TurnError::InvalidRequest { + reason: "this coordinator does not support thread activation".to_string(), + }) + } +``` + +- [x] **Step 5: Implement it on `DefaultTurnCoordinator`** + +In the `impl TurnCoordinator for DefaultTurnCoordinator` block (line 484), after `submit_turn`: + +```rust + async fn activate( + &self, + request: ActivateThreadRequest, + ) -> Result { + // Deliberately routed through this coordinator's own `submit_turn`: + // admission, idempotency replay, profile resolution, and the wake + // notification are shared with every other submission. The only + // difference an activation makes is the provenance stamp. + self.submit_turn(SubmitTurnRequest { + scope: request.scope, + actor: request.actor, + accepted_message_ref: request.accepted_message_ref, + requested_run_profile: request.requested_run_profile, + output_contract: None, + requested_model: None, + idempotency_key: request.idempotency_key, + received_at: request.received_at, + requested_run_id: None, + parent_run_id: None, + subagent_depth: 0, + spawn_tree_root_run_id: None, + product_context: None, + subagent_activation_provenance: Some(request.provenance), + }) + .await + } +``` + +If the compiler reports fields of `SubmitTurnRequest` this literal is missing, add them with the same neutral values the crate's other non-child submissions use — do not invent new semantics here. + +- [x] **Step 6: Forward it on the `Arc` blanket impl** + +In `impl TurnCoordinator for Arc` (line 771), add the forwarding method beside the others: + +```rust + async fn activate( + &self, + request: ActivateThreadRequest, + ) -> Result { + (**self).activate(request).await + } +``` + +Without this, `Arc` callers would silently get the trait's refusing default instead of the real implementation. + +- [x] **Step 7: Run tests to verify they pass** + +Run: `cargo test -p ironclaw_turns activate_submits_through_admission_and_stamps_provenance default_activate_impl_refuses_rather_than_submitting_untagged` +Expected: PASS (both). + +Run: `cargo test -p ironclaw_turns` +Expected: PASS. + +- [x] **Step 8: Commit** + +```bash +git add crates/kernel/ironclaw_turns/src/request.rs crates/kernel/ironclaw_turns/src/coordinator.rs +git commit -m "feat(turns): add provenance-tagged activate() re-activation primitive" +``` + +--- + +### Task 4 ✅ DONE: Bounded newest-first run query for a thread + +The streak caps (Task 5, and `subagent_extend` in a later slice) are *derived* — no stored counter — so they need to read a bounded window of a thread's most recent runs. Nothing today returns newest-first bounded run records for a thread; `children_of` is parent-keyed and unbounded. + +**Files:** +- Modify: `crates/kernel/ironclaw_processes/src/journal_store/rows.rs` (add `recent_processes_for_scope` beside `processes_for_scope` at line 691; `ordered_process_query` at line 1104 hardcodes `SortDirection::Ascending` at line 1131) +- Modify: `crates/kernel/ironclaw_processes/src/journal.rs` (`ProcessSnapshotSource`, line 946-954) +- Modify: `crates/kernel/ironclaw_processes/src/journal_store.rs` (the `impl ProcessSnapshotSource for ProcessJournalStore` block at line 715-736) +- Modify: `crates/kernel/ironclaw_turns/src/process_projection/store_adapter.rs:35-47` +- Modify: `crates/kernel/ironclaw_turns/src/agent_turn_runtime.rs` (`AgentTurnSpawnTreeRuntimePort`, line 73) +- Modify: `crates/kernel/ironclaw_turns/src/process_projection/runtime.rs` (impl block at line 668; mirror `children_of` at 684-698) +- Modify (stubs): `crates/loop/ironclaw_loop_host/src/subagent_spawn_port/tests.rs:947`, `crates/loop/ironclaw_turn_runner/src/structured_finalization/tests.rs:189` +- Test: `crates/kernel/ironclaw_processes/tests/process_journal_store_contract.rs` (existing `process_snapshots` cases at lines 823 and 2536 are the pattern) + +**Interfaces:** +- Consumes: nothing from earlier tasks. +- Produces: `AgentTurnSpawnTreeRuntimePort::recent_runs_for_thread(&self, scope: &TurnScope, limit: u32) -> Result, TurnError>` — newest-first, at most `limit` `AgentTurn` records for that exact thread scope. + +**Why this shape (do not redesign it):** the index this needs already exists — `ordered_index("process_scope_v3", &["scope_key", "created_at", "process_id"])` at `rows.rs:214-217`, where `scope_key` (`rows/keys.rs:59-69`) already includes `thread_id`. `SortDirection::Descending` is already implemented and tested in all three backends (`postgres.rs:689-699`, `libsql.rs:970-980`, `in_memory.rs:460,496`). So there is **no new index, no migration, and no per-backend SQL work** — index declaration never backfills (`ironclaw_filesystem/CONTRACT.md:168-169`), which is exactly why reusing `process_scope_v3` matters. + +**Two traps, both load-bearing:** +1. `process_scope_v3` is **not** keyed on `process_kind`, and a thread's scope also holds `CapabilityInvocation`/`CapabilityInvocationState` processes (`crates/kernel/ironclaw_processes/src/invocation_state.rs:695`). A raw `LIMIT 16` can therefore come back entirely non-`AgentTurn` and yield zero runs. **Over-fetch**: page the descending keyset until `limit` `AgentTurn` records are collected or a hard page budget is spent. Do not declare a new kind-keyed index — that would need the offline `migrate_row_native_indexes` step (`journal_store.rs:404-444`). +2. The architecture gate `crates/app/ironclaw_architecture_tests/tests/reborn_process_storage_scan_gate.rs:30-41` forbids `.query(` and `.tail_bounded(` inside `journal_store.rs` outside two named migration/startup functions. It scans **only** `journal_store.rs`, not `rows.rs`, and its own self-test at lines 210-223 confirms `.query_ordered(` is deliberately not scanned. So: put the enumeration in `rows.rs` and reach storage through `.query_ordered(`. Do not add a `.query(` call to `journal_store.rs`. + +- [x] **Step 1: Write the failing test** + +In `crates/kernel/ironclaw_processes/tests/process_journal_store_contract.rs`, beside the existing `process_snapshots` coverage. Use the file's existing backend-parametrized harness so the case runs on every backend — read the test at line 823 first and copy its setup: + +```rust +/// The streak caps read a bounded newest-first window of a thread's runs. +/// Two properties matter and both are load-bearing: ordering must be +/// newest-first (an ascending read returns the wrong end of history), and the +/// limit must be honoured against AgentTurn rows specifically — a thread's +/// scope also holds capability-invocation processes, so a naive LIMIT can be +/// filled entirely by non-AgentTurn rows. +#[tokio::test] +async fn recent_process_snapshots_returns_newest_first_bounded_by_limit() { + let harness = contract_harness().await; + let scope = harness.thread_scope(); + + // Seed 5 AgentTurn processes, oldest first, interleaved with a + // capability-invocation process that must not consume the limit. + let mut agent_turn_ids = Vec::new(); + for index in 0..5 { + agent_turn_ids.push(harness.seed_agent_turn_process(&scope, index).await); + harness.seed_capability_invocation_process(&scope, index).await; + } + + let recent = harness + .store() + .recent_process_snapshots(&scope, 3) + .await + .expect("recent snapshots"); + + assert_eq!(recent.len(), 3, "limit must bound the AgentTurn result count"); + let returned: Vec<_> = recent.iter().map(|s| s.process_id).collect(); + let expected: Vec<_> = agent_turn_ids.iter().rev().take(3).copied().collect(); + assert_eq!( + returned, expected, + "must return the newest 3 AgentTurn processes, newest first" + ); +} +``` + +`seed_capability_invocation_process` may not exist in the harness — if not, add it alongside the existing agent-turn seeder rather than dropping the interleaving, because the interleaving is the whole point of trap 1 above. + +- [x] **Step 2: Run test to verify it fails** + +Run: `cargo test -p ironclaw_processes recent_process_snapshots_returns_newest_first_bounded_by_limit` +Expected: FAIL to compile — `no method named recent_process_snapshots`. + +- [x] **Step 3: Add the descending bounded row query** + +In `crates/kernel/ironclaw_processes/src/journal_store/rows.rs`, give `ordered_process_query` (line 1104) a direction parameter, replacing the hardcoded `SortDirection::Ascending` at line 1131, and pass `SortDirection::Ascending` explicitly at its four existing call sites (`query_claim_candidates` line 930, `query_active_conflict` line 972, `query_running_quota_rows` line 995, `query_expired_processes` line 1046) so their behavior is unchanged. + +Then add, beside `processes_for_scope` (line 691): + +```rust +/// Newest-first, bounded read of one scope's `AgentTurn` processes. +/// +/// `process_scope_v3` is keyed on `(scope_key, created_at, process_id)` and +/// deliberately not on `process_kind`, while a thread's scope also holds +/// capability-invocation processes. A flat `LIMIT` would therefore be +/// satisfiable entirely by non-`AgentTurn` rows, so this pages the descending +/// keyset until it has `limit` agent-turn rows or spends its page budget. +pub(super) async fn recent_agent_turn_processes_for_scope( + filesystem: &F, + scope: &ResourceScope, + limit: u32, +) -> Result, ProcessJournalStoreError> +where + F: RootFilesystem + ?Sized, +{ + const MAX_PAGES: u32 = 8; + + let mut collected: Vec = Vec::new(); + let mut page = 0; + while collected.len() < limit as usize && page < MAX_PAGES { + let batch = ordered_process_query( + filesystem, + "process_scope_v3", + scope_key_filters(scope)?, + /* sort_key */ "created_at", + SortDirection::Descending, + /* limit */ limit * 2, + /* offset_page */ page, + ) + .await?; + let exhausted = batch.is_empty(); + collected.extend( + batch + .into_iter() + .filter(|snapshot| snapshot.process_kind == ProcessKind::AgentTurn), + ); + if exhausted { + break; + } + page += 1; + } + collected.truncate(limit as usize); + Ok(collected) +} +``` + +Match `ordered_process_query`'s real signature — read it at line 1104 and adapt the call above to whatever its filter/paging parameters actually are. Do not change its paging contract; only add the direction. + +- [x] **Step 4: Add the port method and thread it through** + +`crates/kernel/ironclaw_processes/src/journal.rs`, on `ProcessSnapshotSource` (line 946): + +```rust + /// Newest-first, bounded read of one scope's agent-turn processes. Unlike + /// [`Self::process_snapshots`] this is explicitly bounded — callers that + /// need a fixed recent window must not enumerate the whole scope. + async fn recent_process_snapshots( + &self, + scope: &ResourceScope, + limit: u32, + ) -> Result, Self::Error>; +``` + +Implement it in `journal_store.rs`'s `impl ProcessSnapshotSource for ProcessJournalStore` (line 715), mirroring `process_snapshots` at 721-735: call `ensure_materialized()`, keep the existing `ResourceScope::system()` rejection at 726-731, then delegate to `rows::recent_agent_turn_processes_for_scope`. **Introduce no `.query(` or `.tail_bounded(` call in this file.** + +Forward it through `crates/kernel/ironclaw_turns/src/process_projection/store_adapter.rs:35-47`. + +- [x] **Step 5: Run the storage test and the architecture gate** + +Run: `cargo test -p ironclaw_processes recent_process_snapshots_returns_newest_first_bounded_by_limit` +Expected: PASS on every backend the harness parametrizes. + +Run: `cargo test -p ironclaw_architecture_tests --test reborn_process_storage_scan_gate` +Expected: PASS. + +- [x] **Step 6: Expose it as `recent_runs_for_thread` on the turn port** + +In `crates/kernel/ironclaw_turns/src/agent_turn_runtime.rs`, on `AgentTurnSpawnTreeRuntimePort` (line 73) — **not** on `AgentTurnRuntimePort`, which has six test doubles this would break: + +```rust + /// The newest `limit` runs on this exact thread scope, newest first. + /// Bounded by construction: the derived activation-streak caps + /// (design §6, §8.3) read a fixed window instead of keeping a counter. + async fn recent_runs_for_thread( + &self, + scope: &TurnScope, + limit: u32, + ) -> Result, TurnError>; +``` + +Implement it in `crates/kernel/ironclaw_turns/src/process_projection/runtime.rs`'s impl block (line 668), mirroring `children_of` (684-698): call `self.snapshots.recent_process_snapshots(&scope.to_resource_scope(), limit)`, then map through `turn_run_record_from_process_snapshot` (line 1083). + +Add stub implementations to the two test doubles — `crates/loop/ironclaw_loop_host/src/subagent_spawn_port/tests.rs:947` and `crates/loop/ironclaw_turn_runner/src/structured_finalization/tests.rs:189` — returning `Ok(Vec::new())`. + +- [x] **Step 7: Verify the whole workspace still builds and the projection test passes** + +Run: `cargo test -p ironclaw_turns` +Expected: PASS. + +Run: `cargo build --workspace --all-targets` +Expected: clean. + +- [x] **Step 8: Commit** + +```bash +git add -A +git commit -m "feat(processes): add bounded newest-first agent-turn query for a thread scope" +``` + +--- + +### Task 5 ✅ DONE: System-wake streak cap in `activate()` + +Bounds the autonomous spawn → settle → wake → spawn cycle. Without it a parent that spawns a fresh child on every background completion loops forever with no human in it, under every existing cap. + +**Files:** +- Create: `crates/kernel/ironclaw_turns/src/activation_streak.rs` +- Modify: `crates/kernel/ironclaw_turns/src/lib.rs` (declare the module) +- Modify: `crates/kernel/ironclaw_turns/src/coordinator.rs` (`DefaultTurnCoordinator::activate` from Task 3) +- Test: `crates/kernel/ironclaw_turns/src/activation_streak.rs`'s own `#[cfg(test)] mod tests` + +**Interfaces:** +- Consumes: `ActivationProvenance` (Task 1); `TurnRunRecord.subagent_activation_provenance` (Task 2); `TurnCoordinator::activate` (Task 3); `recent_runs_for_thread` (Task 4). +- Produces: `pub const SYSTEM_WAKE_STREAK_CAP: u32 = 16;` and `pub fn system_wake_admitted(recent: &[TurnRunRecord]) -> bool` — pure windowing logic over an already-fetched window, so it is unit-testable without a store. + +**Spec rules that must hold exactly (design §8.3):** +- Fetch at most `K = SYSTEM_WAKE_STREAK_CAP = 16` records of `Human`/`System` provenance. `ParentAgent` runs are **excluded from the fetch**, not filtered after — they neither reset nor count. +- All `K` are `System` with no `Human` → refuse the pending `System` activation. +- A `Human` anywhere in the window, **or** fewer than `K` records in history → admit. +- A refusal loses nothing durable: the settled edge stays `Settled` and is drained by the run-start sweep or the boot pass. The cap gates the *reactive wake* only, never delivery. +- `SYSTEM_WAKE_STREAK_CAP` stays its own named constant. It must not be merged with `DEFAULT_SUBAGENT_MAX_TREE_DESCENDANTS` or the SUBAGENT family `iteration_limit`, which also happen to be 16. + +- [x] **Step 1: Write the failing test** + +Create `crates/kernel/ironclaw_turns/src/activation_streak.rs` with the tests first (implementation stub returning `unimplemented!()` so the file compiles): + +```rust +#[cfg(test)] +mod tests { + use super::{SYSTEM_WAKE_STREAK_CAP, system_wake_admitted}; + use crate::ActivationProvenance; + + /// Build a newest-first window of run records carrying only the field the + /// cap reads. `provenances[0]` is the newest run. + fn window(provenances: &[ActivationProvenance]) -> Vec { + provenances + .iter() + .map(|provenance| test_run_record_with_provenance(Some(*provenance))) + .collect() + } + + #[test] + fn under_cap_consecutive_system_wakes_are_admitted() { + let recent = window(&[ActivationProvenance::System; 15]); + assert!( + system_wake_admitted(&recent), + "15 consecutive System wakes is under the cap of {SYSTEM_WAKE_STREAK_CAP}" + ); + } + + #[test] + fn cap_plus_one_consecutive_system_wakes_is_refused() { + let recent = window(&[ActivationProvenance::System; 16]); + assert!( + !system_wake_admitted(&recent), + "a full window of System runs means the pending wake is the 17th consecutive one" + ); + } + + #[test] + fn a_human_activation_anywhere_in_the_window_resets_the_streak() { + let mut provenances = [ActivationProvenance::System; 16]; + provenances[15] = ActivationProvenance::Human; + assert!( + system_wake_admitted(&window(&provenances)), + "a Human run anywhere in the window resets the streak" + ); + } + + #[test] + fn a_short_history_is_admitted() { + let recent = window(&[ActivationProvenance::System; 3]); + assert!( + system_wake_admitted(&recent), + "a young thread with fewer than {SYSTEM_WAKE_STREAK_CAP} records must be admitted" + ); + } + + #[test] + fn untagged_legacy_runs_count_as_human_and_reset_the_streak() { + let mut recent = window(&[ActivationProvenance::System; 15]); + recent.push(test_run_record_with_provenance(None)); + assert!( + system_wake_admitted(&recent), + "an untagged run is an ordinary human-initiated run and must reset the streak" + ); + } +} +``` + +`test_run_record_with_provenance(Option) -> TurnRunRecord` is a fixture you add in the same `mod tests` — build it from whatever minimal `TurnRunRecord` constructor the crate's existing tests use (`crates/kernel/ironclaw_turns/src/agent_turn_runtime.rs` has a `#[cfg(test)] mod tests` with record fixtures — reuse it). + +Note the `ParentAgent` exclusion is **not** tested here: it is enforced by the *fetch*, not this function, and is covered in Step 4. + +- [x] **Step 2: Run test to verify it fails** + +Run: `cargo test -p ironclaw_turns activation_streak` +Expected: FAIL — `not yet implemented` panic from the stub (or a compile error if the stub is absent). + +- [x] **Step 3: Write minimal implementation** + +In the same file, above the test module: + +```rust +//! Derived activation-streak caps (design §6, §8.3). +//! +//! Deliberately no stored counter and no new component: each cap is a +//! predicate over a bounded, newest-first window of the thread's own run +//! records, fetched with the complementary provenance excluded. + +use crate::{ActivationProvenance, TurnRunRecord}; + +/// Consecutive `System`-provenance activations allowed on one thread before +/// the reactive wake is refused (design §8.3). +/// +/// Independently named on purpose. It coincides numerically with the +/// 16-descendant spawn-tree cap and the SUBAGENT family's 16-iteration limit, +/// and those three budgets must never be merged by a refactor. +pub const SYSTEM_WAKE_STREAK_CAP: u32 = 16; + +/// Whether a pending `System` activation may be admitted, given the thread's +/// newest-first window of `Human`/`System` runs (`ParentAgent` runs are +/// excluded by the caller's fetch, so they neither count nor reset). +/// +/// Refusing costs nothing durable: the settled await-edge stays `Settled` and +/// drains via the run-start sweep or the boot pass. This gates the reactive +/// wake only, never delivery itself. +pub fn system_wake_admitted(recent: &[TurnRunRecord]) -> bool { + if recent.len() < SYSTEM_WAKE_STREAK_CAP as usize { + return true; + } + recent + .iter() + .take(SYSTEM_WAKE_STREAK_CAP as usize) + .any(|record| { + record.subagent_activation_provenance != Some(ActivationProvenance::System) + }) +} +``` + +Declare `pub mod activation_streak;` in `crates/kernel/ironclaw_turns/src/lib.rs`. + +- [x] **Step 4: Run tests to verify they pass** + +Run: `cargo test -p ironclaw_turns activation_streak` +Expected: PASS (all five). + +- [x] **Step 5: Enforce the cap inside `activate()`** + +In `DefaultTurnCoordinator::activate` (Task 3), before building the `SubmitTurnRequest`, add the guard. It applies to `System` provenance only: + +```rust + if request.provenance == ActivationProvenance::System { + // ParentAgent runs are excluded from the window entirely, and must + // be excluded from the FETCH, not filtered after it: filtering a + // K-sized fetch returns fewer than K records whenever ParentAgent + // runs are interleaved, and a short window reads as "streak not + // established" and admits. + let fetch_limit = SYSTEM_WAKE_STREAK_CAP.saturating_mul(SYSTEM_WAKE_WINDOW_OVERFETCH); + let raw = self + .store + .recent_runs_for_thread(&request.scope, fetch_limit) + .await?; + // A retry of an already-accepted activation must reach submit_turn's + // durable idempotency replay, not be refused by the run it created. + let recent = raw + .iter() + .filter(|record| record.accepted_message_ref != request.accepted_message_ref) + .filter(|record| { + record.subagent_activation_provenance != Some(ActivationProvenance::ParentAgent) + }) + .take(SYSTEM_WAKE_STREAK_CAP as usize) + .cloned() + .collect::>(); + // Fail closed when the window could not be established: a FULL fetch + // that still cannot yield a cap-sized non-ParentAgent window means + // the streak is unknown, not absent. A short fetch is a young + // thread and still admits. + let window_crowded_out = u32::try_from(raw.len()).unwrap_or(u32::MAX) >= fetch_limit + && (recent.len() as u32) < SYSTEM_WAKE_STREAK_CAP; + if window_crowded_out || !system_wake_admitted(&recent) { + return Err(TurnError::AdmissionRejected(AdmissionRejection::new( + AdmissionRejectionReason::SystemWakeStreak, + ))); + } + } +``` + +`spawn_tree_runtime()` is the accessor for the `AgentTurnSpawnTreeRuntimePort`. `DefaultTurnCoordinator` holds `process_runtime`/`store` as `AgentTurnRuntimePort` — if neither is typed as the spawn-tree port, thread the spawn-tree port into `DefaultTurnCoordinator` as a required (**not** `Option>` — see `.claude/rules/architecture.md` smell 2) constructor dependency, and update its construction sites. If that turns out to widen the blast radius beyond this task, **stop and report** rather than reaching for an `Option`. + +Note the filter above is a belt-and-braces second pass: the *fetch* is what the spec requires to exclude `ParentAgent`, and `recent_runs_for_thread` returns all kinds. Keeping the filter here (rather than pushing a provenance predicate into the storage query) keeps Task 4's query general and the exclusion visible at the policy site. + +- [x] **Step 6: Add the caller-level test** + +Add to `crates/kernel/ironclaw_turns/src/coordinator.rs`'s test module, beside Task 3's tests: + +```rust +/// The cap must be enforced at the caller, not just in the predicate — and a +/// refusal must not submit a turn. +#[tokio::test] +async fn activate_refuses_system_wake_past_the_streak_cap_without_submitting() { + let (coordinator, recorder) = + recording_coordinator_with_recent_runs(vec![ + ActivationProvenance::System; + SYSTEM_WAKE_STREAK_CAP as usize + ]); + + let error = coordinator + .activate(ActivateThreadRequest { + scope: test_scope(), + actor: test_actor(), + accepted_message_ref: test_message_ref(), + provenance: ActivationProvenance::System, + idempotency_key: IdempotencyKey::new("streak-cap-1").expect("valid key"), + received_at: chrono::Utc::now(), + requested_run_profile: None, + }) + .await + .expect_err("a saturated System streak must refuse"); + + assert!(matches!(error, TurnError::InvalidRequest { .. })); + assert!( + recorder.submitted_requests().is_empty(), + "a refused wake must not submit a turn" + ); +} + +/// ParentAgent runs are excluded from the System window: a thread whose recent +/// history is all ParentAgent must still admit a System wake. +#[tokio::test] +async fn parent_agent_runs_do_not_saturate_the_system_streak() { + let (coordinator, _recorder) = + recording_coordinator_with_recent_runs(vec![ + ActivationProvenance::ParentAgent; + SYSTEM_WAKE_STREAK_CAP as usize + ]); + + coordinator + .activate(ActivateThreadRequest { + scope: test_scope(), + actor: test_actor(), + accepted_message_ref: test_message_ref(), + provenance: ActivationProvenance::System, + idempotency_key: IdempotencyKey::new("streak-cap-2").expect("valid key"), + received_at: chrono::Utc::now(), + requested_run_profile: None, + }) + .await + .expect("ParentAgent history must not block a System wake"); +} +``` + +- [x] **Step 7: Run tests to verify they pass** + +Run: `cargo test -p ironclaw_turns` +Expected: PASS. + +Run: `cargo clippy -p ironclaw_turns --all-targets --all-features -- -D warnings` +Expected: zero warnings. + +- [x] **Step 8: Commit** + +```bash +git add -A +git commit -m "feat(turns): bound autonomous System activations with a derived streak cap" +``` + +**Slice 1 is complete here.** Before starting Task 6, run the slice gate: + +```bash +cargo test -p ironclaw_turns -p ironclaw_processes +cargo test -p ironclaw_architecture_tests +cargo clippy --workspace --all-targets --all-features -- -D warnings +``` + +--- + +### Task 6: Accept `mode: "background"` at the spawn codec + +Deletes the rejection. This is pure unblocking — `SpawnSubagentMode::{Blocking, Background}` already exists (`subagent_spawn_port.rs:181-186`) and already rides durably on `AwaitedChildSetRecord.mode` (line 273) and `SubagentThreadMetadata.mode` (line 294). Nothing downstream constructs `Background` yet; Task 7 does that. + +**Files:** +- Modify: `crates/loop/ironclaw_loop_host/src/subagent_spawn_port.rs` (`SpawnSubagentArgs` line 188; `TryFrom` lines 218-242; `background_subagents_disabled()` lines 1500-1503; `build_spawn_subagent_parameters_schema` lines 62-107) +- Modify: `crates/loop/ironclaw_loop_host/prompts/spawn_subagent_description.md` +- Test: `crates/loop/ironclaw_loop_host/src/subagent_spawn_port/tests.rs` + +**Interfaces:** +- Consumes: nothing from slice 1. +- Produces: `SpawnSubagentArgs.mode: SpawnSubagentMode` (defaulting to `Blocking`), populated from either the `mode` wire field or the legacy `run_in_background` boolean. + +- [ ] **Step 1: Write the failing test** + +In `crates/loop/ironclaw_loop_host/src/subagent_spawn_port/tests.rs`. Several existing tests in this file assert the string `"background subagents are disabled"` — find them with `rg -n "background subagents are disabled" crates/loop/ironclaw_loop_host/src/subagent_spawn_port/tests.rs` and **replace** those assertions with the ones below rather than adding duplicates beside them. + +```rust +/// Background mode is accepted at the codec now that delivery exists. +/// Both spellings must land on the same typed mode: the explicit `mode` field +/// and the legacy `run_in_background` boolean. +#[test] +fn background_mode_is_accepted_from_both_wire_spellings() { + let explicit: SpawnSubagentArgs = serde_json::from_value::( + serde_json::json!({ + "subagent_type": "general", + "task": "research the competitor set", + "mode": "background", + }), + ) + .expect("wire args parse") + .try_into() + .expect("background mode is accepted"); + assert_eq!(explicit.mode, SpawnSubagentMode::Background); + + let legacy: SpawnSubagentArgs = serde_json::from_value::( + serde_json::json!({ + "subagent_type": "general", + "task": "research the competitor set", + "run_in_background": true, + }), + ) + .expect("wire args parse") + .try_into() + .expect("legacy run_in_background is accepted"); + assert_eq!(legacy.mode, SpawnSubagentMode::Background); +} + +/// Omitting the mode must stay blocking — the historical default, and the one +/// every existing caller and stored payload relies on. +#[test] +fn omitted_mode_defaults_to_blocking() { + let args: SpawnSubagentArgs = serde_json::from_value::( + serde_json::json!({ + "subagent_type": "general", + "task": "summarize this file", + }), + ) + .expect("wire args parse") + .try_into() + .expect("args convert"); + assert_eq!(args.mode, SpawnSubagentMode::Blocking); +} + +/// The model cannot ask for background mode it cannot see: the advertised +/// schema must carry the mode enum, and must keep rejecting unknown fields. +#[test] +fn advertised_schema_exposes_the_mode_enum() { + let schema = build_spawn_subagent_parameters_schema(&[]); + let mode = &schema["properties"]["mode"]; + assert_eq!(mode["enum"], serde_json::json!(["blocking", "background"])); + assert_eq!( + schema["additionalProperties"], + serde_json::json!(false), + "the schema must keep rejecting unknown fields" + ); + assert_eq!( + schema["required"], + serde_json::json!(["subagent_type", "task"]), + "mode stays optional — omitting it means blocking" + ); +} +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `cargo test -p ironclaw_loop_host background_mode_is_accepted_from_both_wire_spellings omitted_mode_defaults_to_blocking advertised_schema_exposes_the_mode_enum` +Expected: FAIL — `no field mode on SpawnSubagentArgs`, and the schema assertion fails because `properties.mode` is absent. + +- [ ] **Step 3: Add `mode` to the typed args and delete the rejection** + +In `crates/loop/ironclaw_loop_host/src/subagent_spawn_port.rs`, add to `SpawnSubagentArgs` (line 188): + +```rust + #[serde(default = "blocking_mode_default")] + pub mode: SpawnSubagentMode, +``` + +with + +```rust +fn blocking_mode_default() -> SpawnSubagentMode { + SpawnSubagentMode::Blocking +} +``` + +Replace the body of `TryFrom` (lines 221-241) — delete both rejection branches at 222-227 and map the two spellings onto the typed mode instead: + +```rust + fn try_from(value: SpawnSubagentWireArgs) -> Result { + // Two accepted spellings, one typed mode. `run_in_background: true` is + // the legacy boolean; `mode` is the current field. When both are + // present they must agree — silently preferring one would let a + // caller believe it asked for the other. + let mode = match (value.mode, value.run_in_background) { + (Some(SpawnSubagentWireMode::Background), _) | (None, true) => { + SpawnSubagentMode::Background + } + (Some(SpawnSubagentWireMode::Blocking), false) | (None, false) => { + SpawnSubagentMode::Blocking + } + (Some(SpawnSubagentWireMode::Blocking), true) => { + return Err(AgentLoopHostError::new( + AgentLoopHostErrorKind::InvalidInvocation, + "conflicting spawn mode: mode is \"blocking\" but run_in_background is true", + )); + } + }; + if value.task.len() > DEFAULT_SUBAGENT_GOAL_MAX_BYTES { + return Err(spawn_goal_field_too_large("task", value.task.len())); + } + if let Some(handoff) = value.handoff.as_deref() + && handoff.len() > DEFAULT_SUBAGENT_GOAL_MAX_BYTES + { + return Err(spawn_goal_field_too_large("handoff", handoff.len())); + } + Ok(Self { + subagent_kind: value.subagent_kind, + task: value.task, + handoff: value.handoff, + mode, + }) + } +``` + +Delete `background_subagents_disabled()` (lines 1500-1503) entirely. The compiler will flag any remaining caller. + +- [ ] **Step 4: Advertise `mode` in the schema** + +In `build_spawn_subagent_parameters_schema` (line 62), add to the `properties` object beside `handoff`: + +```rust + "mode": { + "type": "string", + "enum": ["blocking", "background"], + "description": "How to wait for the child. \"blocking\" (the default) pauses this turn until the child finishes and returns its result inline. \"background\" returns immediately and delivers the child's result later, letting you keep working meanwhile." + } +``` + +Leave `required` as `["subagent_type", "task"]` and `additionalProperties` as `false`. + +- [ ] **Step 5: Update the model-facing description** + +`crates/loop/ironclaw_loop_host/prompts/spawn_subagent_description.md` currently states the child "runs to completion and returns its final result" — blocking-only wording with no background affordance. Rewrite it to describe both modes and when to pick each: blocking when the answer is needed to continue this turn, background when the work is independent and you have other work to do meanwhile. Keep the file's existing tone and length; the prompt text stays in this file and is never inlined into Rust. + +- [ ] **Step 6: Run tests to verify they pass** + +Run: `cargo test -p ironclaw_loop_host` +Expected: PASS, including the rewritten assertions that previously expected a rejection. + +- [ ] **Step 7: Commit** + +```bash +git add -A +git commit -m "feat(subagent): accept background spawn mode at the codec and advertise it in the schema" +``` + +--- + +### Task 7: Background spawn returns immediately instead of parking + +`finish_spawn` hard-codes `SpawnSubagentMode::Blocking` at line 908 regardless of what the caller asked for, and always returns `resolution::await_dependent_run` (line 1101), which parks the parent on a gate. Background mode must thread the requested mode through and return the non-suspending channel instead. + +**Files:** +- Modify: `crates/loop/ironclaw_loop_host/src/subagent_spawn_port.rs` (`finish_spawn` lines 889-1109) +- Test: `crates/loop/ironclaw_loop_host/src/subagent_spawn_port/tests.rs` + +**Interfaces:** +- Consumes: `SpawnSubagentArgs.mode` (Task 6). +- Produces: a background spawn resolves to `resolution::spawned_child_run(...)` (`crates/contracts/ironclaw_loop_contracts/src/resolution.rs:213`) — the existing `Done`/`ChildSpawned` channel, documented there as "a NON-suspending child run whose result the executor appends before continuing". Blocking spawns keep returning `resolution::await_dependent_run` unchanged. + +- [ ] **Step 1: Write the failing test** + +In `crates/loop/ironclaw_loop_host/src/subagent_spawn_port/tests.rs`, beside the existing spawn tests (reuse their harness — the file already builds a port with `StaticAgentTurnRuntime` and `StaticCoordinator`): + +```rust +/// A background spawn must not park the parent. The durable records must also +/// carry Background, because recovery reads the mode off them to decide +/// whether a settled edge resumes a gate or activates a thread. +#[tokio::test] +async fn background_spawn_returns_without_parking_and_records_background_mode() { + let harness = spawn_port_harness().await; + + let resolution = harness + .invoke_spawn(serde_json::json!({ + "subagent_type": "general", + "task": "research the competitor set", + "mode": "background", + })) + .await + .expect("background spawn succeeds"); + + assert!( + matches!(resolution, Resolution::Done(_)), + "background spawn must resolve on the non-suspending channel, got {resolution:?}" + ); + + let metadata = harness.recorded_child_thread_metadata(); + assert_eq!( + metadata.mode, + SpawnSubagentMode::Background, + "child thread metadata must record Background so recovery routes delivery correctly" + ); + let awaited = harness.recorded_awaited_child_set_record(); + assert_eq!(awaited.mode, SpawnSubagentMode::Background); +} + +/// Blocking stays exactly as it was — this task must not change the shipped +/// blocking behavior in any way. +#[tokio::test] +async fn blocking_spawn_still_parks_the_parent_on_the_dependent_run_gate() { + let harness = spawn_port_harness().await; + + let resolution = harness + .invoke_spawn(serde_json::json!({ + "subagent_type": "general", + "task": "summarize this file", + })) + .await + .expect("blocking spawn succeeds"); + + assert!( + matches!(resolution, Resolution::Suspended(_)), + "blocking spawn must still park on the dependent-run gate, got {resolution:?}" + ); + assert_eq!( + harness.recorded_child_thread_metadata().mode, + SpawnSubagentMode::Blocking + ); +} +``` + +`spawn_port_harness()`, `invoke_spawn`, `recorded_child_thread_metadata`, and `recorded_awaited_child_set_record` are helpers — the file already has equivalents for the existing spawn tests. Reuse them; do not add a fourth `test-support` method to `SubagentSpawnCapabilityPort` itself, which is frozen at exactly 3 by `reborn_struct_test_support_ratchet.rs:378`. + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `cargo test -p ironclaw_loop_host background_spawn_returns_without_parking_and_records_background_mode blocking_spawn_still_parks_the_parent_on_the_dependent_run_gate` +Expected: the background test FAILS (resolution is `Suspended`, metadata mode is `Blocking`); the blocking test PASSES already. + +- [ ] **Step 3: Thread the mode through `finish_spawn`** + +In `finish_spawn`, replace line 908: + +```rust + let mode = SpawnSubagentMode::Blocking; +``` + +with + +```rust + let mode = args.mode; +``` + +Everything downstream already takes `mode` as a parameter — the `spawn_result_payload` call at line 917 and the `SubagentThreadMetadata` at line 947 both pass it through, so no other write site changes. + +- [ ] **Step 4: Return the non-suspending resolution for background** + +Replace the tail of `finish_spawn` (the `Ok(resolution::await_dependent_run(...))` at lines 1101-1109) with a branch on mode: + +```rust + match mode { + // Blocking: park this turn on the dependent-run gate. The + // resolver resumes it once every sibling in the gate group has + // settled (design §1). + SpawnSubagentMode::Blocking => { + let loop_gate_ref = + LoopGateRef::new(gate_ref.as_str()).map_err(invalid_static_ref)?; + Ok(resolution::await_dependent_run( + loop_gate_ref, + result_ref, + safe_summary("subagent spawned; waiting for completion"), + write_result.byte_len, + write_result.model_observation, + ) + .resolution) + } + // Background: do not park. The placeholder result is already in + // the transcript; the resolver overwrites it in place and wakes + // this thread when the child settles (design §8). The edge stays + // `open` across this parent run's own terminal transition — for + // background that is the normal delivery case, never abandonment + // (design §2). + SpawnSubagentMode::Background => Ok(resolution::spawned_child_run( + child_run_id, + result_ref, + safe_summary("subagent spawned in the background; result will arrive later"), + write_result.byte_len, + write_result.model_observation, + )), + } +``` + +- [ ] **Step 5: Run tests to verify they pass** + +Run: `cargo test -p ironclaw_loop_host` +Expected: PASS (both new tests and the whole existing suite — the blocking path must be untouched). + +- [ ] **Step 6: Commit** + +```bash +git add -A +git commit -m "feat(subagent): background spawns resolve without parking the parent turn" +``` + +--- + +### Task 8: Wake a background parent when its child settles + +The delivery half. In blocking mode the parent sits on a gate and `resume_parent` (`resolver.rs:489`) wakes it. A background parent is not parked — it may be mid-run, idle, or already finished — so it is woken by a `System`-provenance `activate()` instead. + +**Files:** +- Modify: `crates/loop/ironclaw_turn_runner/src/subagent/await_edge/resolver.rs` (`drain_settled_group` lines 673-746; the `resume_parent` call is at line 737) +- Test: `crates/loop/ironclaw_turn_runner/src/subagent/await_edge/resolver.rs`'s own `#[cfg(test)] mod tests` (the mixed-status group test at line 1431 is the pattern; the module already has a `StaticCoordinator` double at line 1375) + +**Interfaces:** +- Consumes: `TurnCoordinator::activate` + `ActivateThreadRequest` (Task 3); `AwaitEdge.mode` (already durable). +- Produces: no new public API. `drain_settled_group` gains a private mode branch. + +**Spec rules (design §8, §8.2 trigger 1):** +- `ThreadBusy` from the activation is a **benign no-op** — leave the edge `Settled` and let the run-start sweep or boot pass drain it. Do not retry in place, do not fail the drain, do not abandon the edge. +- Exactly **one** activation attempt per settled child; the edge's `Settled` state is the dedupe. +- The transcript write happens *before* the wake, exactly as it already does for blocking — a woken parent must find its result already in place. + +- [ ] **Step 1: Write the failing test** + +Add to the resolver's test module: + +```rust +/// A background child's completion must wake its parent thread with a +/// System-provenance activation rather than resuming a gate the parent is not +/// sitting on. +#[tokio::test] +async fn background_edge_activates_the_parent_instead_of_resuming_a_gate() { + let harness = resolver_harness_with_mode(SpawnSubagentMode::Background).await; + + harness.settle_child_as_completed().await; + + assert!( + harness.coordinator().resumes().is_empty(), + "a background parent is not parked, so nothing may call resume_turn" + ); + let activations = harness.coordinator().activations(); + assert_eq!(activations.len(), 1, "exactly one wake per settled child"); + assert_eq!(activations[0].provenance, ActivationProvenance::System); + assert_eq!(activations[0].scope.thread_id, harness.parent_thread_id()); + + assert_eq!( + harness.transcript_updates().len(), + 1, + "the child's framed result must be written into the parent transcript" + ); +} + +/// ThreadBusy is benign: the parent is mid-run, the edge stays settled, and +/// the run-start sweep or boot pass drains it later. Losing the result here +/// would be silent data loss. +#[tokio::test] +async fn background_wake_losing_to_thread_busy_leaves_the_edge_settled() { + let harness = resolver_harness_with_mode(SpawnSubagentMode::Background).await; + harness.coordinator().fail_next_activation_with(TurnError::ThreadBusy); + + let outcome = harness + .settle_child_as_completed() + .await + .expect("a busy parent must not fail the drain"); + + assert_eq!(outcome, ResolveOutcome::Drained); + assert_eq!( + harness.edge_state().await, + Some(AwaitEdgeState::Settled), + "the edge must stay settled so a later trigger can drain it" + ); +} + +/// Blocking is untouched by this task. +#[tokio::test] +async fn blocking_edge_still_resumes_the_gate_and_never_activates() { + let harness = resolver_harness_with_mode(SpawnSubagentMode::Blocking).await; + + harness.settle_child_as_completed().await; + + assert_eq!(harness.coordinator().resumes().len(), 1); + assert!( + harness.coordinator().activations().is_empty(), + "a blocking parent is resumed through its gate, never activated" + ); +} +``` + +Extend the module's existing `StaticCoordinator` double (line 1375) with an `activate` override that records `ActivateThreadRequest`s and can be primed to fail — per `.claude/rules/testing.md`, capture the whole request, not just a count. + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `cargo test -p ironclaw_turn_runner background_edge_activates_the_parent_instead_of_resuming_a_gate background_wake_losing_to_thread_busy_leaves_the_edge_settled` +Expected: FAIL — the background case currently calls `resume_turn`, so `activations()` is empty and `resumes()` is not. + +- [ ] **Step 3: Branch the drain on edge mode** + +In `drain_settled_group` (line 673), replace the unconditional `self.resume_parent(...)` at line 737: + +```rust + match edge.mode { + // Blocking: the parent is parked on the dependent-run gate. + SpawnSubagentMode::Blocking => { + self.resume_parent(&edge, parent_run_id, driving_child_run_id) + .await?; + } + // Background: the parent is not parked — it may be mid-run, idle, + // or already terminal. Wake the thread with a System activation + // (design §8). The framed results are already written above, so a + // woken parent finds them in place. + SpawnSubagentMode::Background => { + match self.activate_parent(&edge, parent_run_id).await { + Ok(()) => {} + // Benign: the parent is running right now. Leave the edge + // settled — §8.2's trigger 2 (run-start sweep) or trigger + // 3 (boot pass) drains it. Retrying here would just race + // the same live run. + Err(TurnError::ThreadBusy) => { + debug!( + parent_run_id = %parent_run_id, + "background subagent wake lost to a live parent run; \ + edge stays settled for the next drain trigger" + ); + return Ok(ResolveOutcome::Drained); + } + Err(error) => return Err(error), + } + } + } +``` + +The `ThreadBusy` early return must come **before** the `close_edge` loop at lines 740-743 — closing the edges would delete the very records the later triggers need. + +Add the private helper beside `resume_parent`: + +```rust + /// Wake a background parent's thread with a `System`-provenance + /// activation. Mirrors `resume_parent`'s use of the actor and scope cached + /// on `edge.parent_run_context` at open/reconstruct time — never a live + /// lookup, which deadlocks from inside the child's own commit-observer + /// callback (see `parent_run_context`'s doc comment). + async fn activate_parent( + &self, + edge: &AwaitEdge, + parent_run_id: TurnRunId, + ) -> Result<(), TurnError> { + let actor = edge + .parent_run_context + .actor + .clone() + .ok_or_else(|| TurnError::InvalidRequest { + reason: "subagent parent run context missing actor for activation".to_string(), + })?; + let coordinator = self + .coordinator + .get() + .ok_or_else(|| TurnError::Unavailable { + reason: "await-edge resolver coordinator is not bound".to_string(), + })?; + coordinator + .activate(ActivateThreadRequest { + scope: edge.parent_run_context.scope.clone(), + actor, + accepted_message_ref: background_wake_message_ref(edge)?, + provenance: ActivationProvenance::System, + idempotency_key: IdempotencyKey::new(format!( + "subagent-wake:{parent_run_id}:{}", + edge.child_thread_id + )) + .map_err(|reason| TurnError::InvalidRequest { reason })?, + received_at: chrono::Utc::now(), + requested_run_profile: None, + }) + .await + .map(|_| ()) + } +``` + +`background_wake_message_ref(edge)` builds the `AcceptedMessageRef` for the wake. **Verify its correct construction against the live type before writing it** — the framed result is already in the transcript under `edge.result_ref`, so the wake references that existing message rather than accepting new inbound content. If `AcceptedMessageRef` cannot be derived from an existing result ref, stop and report rather than inventing a synthetic inbound message: fabricating inbound content would put un-ingressed text on the parent's thread. + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `cargo test -p ironclaw_turn_runner` +Expected: PASS, including the untouched blocking tests (`mixed_status_group_updates_each_result_resumes_once_and_consumes_every_edge` at line 1431 must stay green). + +- [ ] **Step 5: Commit** + +```bash +git add -A +git commit -m "feat(subagent): wake background parents with a System activation on child settle" +``` + +--- + +### Task 9: Batch the multi-edge drain + +The shipped per-member loop (`drain_settled_group` lines 718-735) calls `update_parent_result_reference` once per settled edge, and each call rescans the parent transcript — the in-code comment at lines 714-717 marks this as deliberately adequate for blocking's tiny groups and explicitly reserves the batched form for background (P2.4). Background sweeps and boot passes can find many settled edges at once, where the per-edge loop degrades to O(E×M) over E edges and M messages. + +**Files:** +- Modify: `crates/domains/ironclaw_threads/src/filesystem_service.rs` (`update_tool_result_reference`, lines 1965-2013 — add a batch entry point beside it) +- Modify: `crates/loop/ironclaw_turn_runner/src/subagent/await_edge/resolver.rs` (`update_parent_result_reference` line 449; the drain loop lines 718-735) +- Test: `tests/integration/subagent_await_edge.rs` (the file already owns this seam — extend it, do not add a new file) + +**Interfaces:** +- Consumes: the drain path from Task 8. +- Produces: a batch update that takes the full set of `(result_ref, provider_call_id, safe_summary)` triples for one parent thread and applies them in **one** snapshot read plus **one** CAS write. + +- [ ] **Step 1: Write the failing test** + +In `tests/integration/subagent_await_edge.rs`: + +```rust +/// A multi-edge drain must not rescan the parent transcript per edge. +/// The design's bound is O(E+M), not O(E×M): one snapshot read and one CAS +/// write for the whole batch, however many children settled together. +#[tokio::test] +async fn draining_multiple_settled_edges_performs_one_snapshot_read_and_one_cas_write() { + let harness = await_edge_harness().await; + let counting = harness.counting_thread_service(); + + harness.spawn_background_children(3).await; + harness.settle_all_children().await; + harness.run_drain_pass().await; + + assert_eq!( + counting.snapshot_reads(), + 1, + "one snapshot read for the whole batch, not one per edge" + ); + assert_eq!( + counting.cas_writes(), + 1, + "one CAS write for the whole batch, not one per edge" + ); + assert_eq!( + harness.parent_transcript_results().len(), + 3, + "every child's framed result must still land in the parent transcript" + ); +} +``` + +The counting wrapper around the thread service's snapshot-read and CAS-write calls is the write-count seam the design names. Build it as a decorator in the integration support tree (`tests/integration/support/doubles/`), following `recording_test_capability_port.rs` as the shape. + +- [ ] **Step 2: Run test to verify it fails** + +Run: `cargo test --test reborn_integration_subagent_await_edge draining_multiple_settled_edges_performs_one_snapshot_read_and_one_cas_write` +(Confirm the exact target name from `tests/integration/CLAUDE.md` before running.) +Expected: FAIL — 3 snapshot reads and 3 CAS writes, one per edge. + +- [ ] **Step 3: Add the batch primitive to the thread service** + +In `crates/domains/ironclaw_threads/src/filesystem_service.rs`, beside `update_tool_result_reference` (line 1965), add a batch form taking a `Vec` of the per-result triples. It performs the existing `matches_tool_result_reference` rescan **once**, rewrites every matched message's content in the same pass, and commits through a single `apply_message_update` CAS-retry closure. Keep the single-result method — it stays the right primitive for one edge — and implement it in terms of the batch form with a one-element vector, so there is one code path, not two. + +The batch write is idempotent for the same reason the single write already is (design §8.1): it is a CAS-guarded in-place field update on an already-existing message, not an append, so replaying it reproduces identical content. + +- [ ] **Step 4: Pin drain-replay idempotency (design §8.1's required test)** + +The claim above is "verified, not asserted" in the design — but the *batch* form is new code, so it needs its own regression test. The dangerous window is a crash after the transcript write but before the edge's CAS to `drained`: recovery replays the write, and an append-shaped bug would duplicate the result instead of overwriting it. + +Add to `tests/integration/subagent_await_edge.rs`: + +```rust +/// Design §8.1: the drain's transcript write is an in-place CAS'd field +/// update, not an append, so a crash between the write and the edge's CAS to +/// `drained` is safe to replay. An append-shaped regression here would show up +/// as duplicated child results in the parent's transcript. +#[tokio::test] +async fn replayed_drain_write_leaves_exactly_one_result_message_unchanged() { + let harness = await_edge_harness().await; + harness.spawn_background_child().await; + + // Crash the drain after the transcript write, before the edge CAS. + harness.settle_child_with_crash_after_transcript_write().await; + + let after_crash = harness.parent_transcript_results(); + assert_eq!(after_crash.len(), 1, "the first write landed"); + assert_eq!( + harness.edge_state().await, + Some(AwaitEdgeState::Settled), + "the edge never reached drained, so recovery must replay" + ); + + harness.run_boot_recovery_pass().await; + + let after_replay = harness.parent_transcript_results(); + assert_eq!( + after_replay.len(), + 1, + "replay must overwrite in place, never append a second result" + ); + assert_eq!( + after_replay[0], after_crash[0], + "replayed content must be byte-identical" + ); +} +``` + +Run: `cargo test --test reborn_integration_subagent_await_edge replayed_drain_write_leaves_exactly_one_result_message_unchanged` +Expected: PASS once the batch form is in place. If it fails with two messages, the batch implementation appended instead of rewriting — fix the implementation, never the assertion. + +- [ ] **Step 5: Use the batch form from the drain** + +In `drain_settled_group`, replace the per-member loop at lines 718-735: accumulate every settled member's `(result_ref, provider_call_id, safe_summary)` into one vector first — still deriving each member's status and reason from **its own** edge, never the driving member's (the external-review fix the current comment at lines 709-713 records) — then issue a single batch call. + +- [ ] **Step 6: Run tests to verify they pass** + +Run: `cargo test --test reborn_integration_subagent_await_edge` +Expected: PASS. + +Run: `cargo test -p ironclaw_threads -p ironclaw_turn_runner` +Expected: PASS. + +- [ ] **Step 7: Commit** + +```bash +git add -A +git commit -m "perf(subagent): batch multi-edge drain into one transcript read and write" +``` + +--- + +### Task 10: Run-start sweep and the three-trigger healing test + +Trigger 2 of the design's retry set. `PostCapabilityStage::drain_settled` (`crates/loop/ironclaw_agent_loop/src/executor/post_capability.rs:34-39`) returns `Vec::new()` unconditionally and names `LoopBackgroundChildPort`, a type that was never built and that the design supersedes. This task implements it and pins the invariant that no settled edge can go permanently undrained. + +**Files:** +- Modify: `crates/contracts/ironclaw_loop_contracts/src/` (the port that carries the drain seam — see the boundary note below) +- Modify: `crates/loop/ironclaw_agent_loop/src/executor/post_capability.rs` +- Modify: `crates/loop/ironclaw_loop_host/src/subagent_spawn_port.rs` (the implementing decorator) and `crates/loop/ironclaw_loop_host/src/await_edge_port.rs` (the seam it delegates through) +- Modify: `crates/loop/ironclaw_turn_runner/src/subagent/await_edge/resolver.rs` (implement the drain-for-parent entry point) +- Test: `tests/integration/subagent_await_edge.rs` + +**Hard boundary constraint — read before designing this:** `ironclaw_agent_loop` is **contracts-tier only**. Its `BoundaryRule` in `crates/app/ironclaw_architecture_tests/tests/reborn_dependency_boundaries.rs` permits `ironclaw_common`, `ironclaw_host_api`, and `ironclaw_loop_contracts` and nothing else — it may not depend on `ironclaw_turns` or `ironclaw_turn_runner`. So the stage cannot call the resolver directly; the seam must be a port defined in `ironclaw_loop_contracts`, implemented up-stack. Note also that `AgentLoopDriverHost` is a single blanket impl over a bundle of `Loop*Port`s (`crates/contracts/ironclaw_loop_contracts/src/host/progress.rs:329`), so adding a *required* method to that bundle is not a drop-in. + +**Recommended shape (confirm against live types before writing code):** add the drain method to the existing `LoopCapabilityPort` with a **default implementation returning `Ok(0)`**, and implement it on `SubagentSpawnCapabilityPort` — the decorator that already owns spawn and already holds an await-edge seam (`deps.await_edge_writer`). That keeps the drain in the one component that already knows about await edges, adds no new port to the declared decorator chain, and leaves every other `LoopCapabilityPort` implementation untouched. **If this turns out to require a fourth `test-support` method on `SubagentSpawnCapabilityPort`, stop** — that struct is frozen at exactly 3 by `reborn_struct_test_support_ratchet.rs:378`. + +**Interfaces:** +- Consumes: Task 8's activation path and Task 9's batch drain. +- Produces: `drain_settled` returns the count of edges drained this iteration (replacing `Vec<()>`), and the resolver gains a `drain_settled_for_parent(parent_scope, parent_run_id)` entry point that the port calls. + +- [ ] **Step 1: Write the failing test** + +The design names this test explicitly. In `tests/integration/subagent_await_edge.rs`: + +```rust +/// Design §8.2's invariant, both halves in one test: a settle-time wake that +/// loses to ThreadBusy is always healed — by the run-start sweep if the thread +/// runs again, or by the boot pass if it does not. A settled edge can never go +/// permanently undrained. +/// +/// The parent-completed precondition is asserted too (design §2): a background +/// edge stays `open` across the parent run's own terminal transition — it is +/// never abandoned — and the child's later settle still delivers. +#[tokio::test] +async fn settled_edge_threadbusy_is_healed_by_run_start_and_boot_pass() { + // (a) run-start sweep heals it. + { + let harness = await_edge_harness().await; + harness.spawn_background_child().await; + harness.settle_child_while_parent_run_is_live().await; + + assert_eq!( + harness.edge_state().await, + Some(AwaitEdgeState::Settled), + "the settle-time wake lost to ThreadBusy, so the edge stays settled" + ); + assert_eq!( + harness.system_activation_attempts(), + 1, + "exactly one wake attempt per settled child — the settled state is the dedupe" + ); + + harness.advance_parent_loop_one_iteration().await; + + assert_eq!( + harness.edge_state().await, + None, + "the run-start sweep must drain and close the edge" + ); + assert_eq!(harness.parent_transcript_results().len(), 1); + } + + // (b) the boot pass heals it when the thread never runs again. + { + let harness = await_edge_harness().await; + harness.spawn_background_child().await; + harness.settle_child_while_parent_run_is_live().await; + + harness.run_boot_recovery_pass().await; + + assert_eq!( + harness.edge_state().await, + None, + "the boot pass must drain the edge at the resolver layer, with no activation" + ); + assert_eq!(harness.parent_transcript_results().len(), 1); + } +} + +/// A background edge is not abandoned when the parent run that opened it goes +/// terminal — for background that is the normal delivery case (design §2). +#[tokio::test] +async fn background_edge_survives_its_parent_runs_terminal_transition() { + let harness = await_edge_harness().await; + harness.spawn_background_child().await; + harness.complete_parent_run().await; + + assert_eq!( + harness.edge_state().await, + Some(AwaitEdgeState::Open), + "a background edge stays open across the parent run's terminal transition" + ); + + harness.settle_child_as_completed().await; + + assert_eq!(harness.parent_transcript_results().len(), 1); + assert_eq!(harness.system_activation_attempts(), 1); +} +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `cargo test --test reborn_integration_subagent_await_edge settled_edge_threadbusy_is_healed_by_run_start_and_boot_pass background_edge_survives_its_parent_runs_terminal_transition` +Expected: FAIL — case (a) leaves the edge `Settled` after the loop iteration because `drain_settled` is a stub. Case (b) may already pass via the existing boot recovery; if so, keep it — it pins behavior this task must not break. + +- [ ] **Step 3: Define the drain seam and implement `drain_settled`** + +Follow the boundary constraint above. Then replace the stub in `post_capability.rs`: + +```rust + /// R2 — drain settled background-mode subagent results (design §8.2, + /// trigger 2). `process` runs on every `TurnCompletedStep::Continue`, + /// including a freshly-activated run's first iteration, so any settled + /// edge whose settle-time wake lost to `ThreadBusy` is picked up the next + /// time this thread runs for any reason. + async fn drain_settled(&self, ctx: StageContext<'_>) -> u64 { +``` + +and call it from `process` where `let _drained = self.drain_settled();` sits today (line 71), replacing that discard. A drain failure must not fail the turn — log at `debug!` and continue; the boot pass is the backstop. Do **not** use `warn!`/`info!` here: this runs on a background path and would corrupt the REPL TUI. + +Update the stage's doc comment (lines 19-24) — it still describes R2 as a no-op awaiting `LoopBackgroundChildPort`, which this task retires. + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `cargo test --test reborn_integration_subagent_await_edge` +Expected: PASS. + +Run: `cargo test -p ironclaw_agent_loop -p ironclaw_loop_host -p ironclaw_turn_runner` +Expected: PASS. + +- [ ] **Step 5: Run the boundary gate** + +Run: `cargo test -p ironclaw_architecture_tests` +Expected: PASS — in particular `reborn_dependency_boundaries` (the agent-loop contracts-only rule), `reborn_loop_port_location_scan`, and `reborn_struct_test_support_ratchet`. + +- [ ] **Step 6: Commit** + +```bash +git add -A +git commit -m "feat(subagent): drain settled background children on every loop iteration" +``` + +--- + +## Slice 2 completion gate + +Run all of these before calling slices 1–2 done. Do not pipe test output through `head`/`tail` — a partial view under-counts failures. + +```bash +cargo fmt +cargo clippy --workspace --all-targets --all-features -- -D warnings +cargo test -p ironclaw_turns -p ironclaw_processes -p ironclaw_threads --no-fail-fast +cargo test -p ironclaw_agent_loop -p ironclaw_loop_host -p ironclaw_turn_runner --no-fail-fast +cargo test --test reborn_integration_subagent_await_edge --no-fail-fast +cargo test -p ironclaw_architecture_tests +bash scripts/reborn-e2e-rust.sh +scripts/pre-commit-safety.sh +``` + +Then confirm the two standing invariants this plan must not have broken: + +```bash +# Production is still deny-filtered — this must still return the capability id. +rg -n "spawn_subagent" crates/loop/ironclaw_turn_runner/src/runtime.rs + +# No trusted-ingress minting crept in. +rg -n "TrustedInboundTurnRequest|TrustedTriggerSubmitRequest" \ + crates/loop/ironclaw_turn_runner/src/subagent crates/loop/ironclaw_loop_host/src/subagent_spawn_port.rs +``` + +## What is deliberately NOT in this plan + +Named so a reader does not mistake their absence for an oversight — each is a later slice in `pr2-pr6-shape.md`: + +- **Clearing the deny-filter.** Production enablement is the last slice of the whole effort, after PR6, per the shape doc's deviation from the design's own staging. +- **The drain safety scan** (shape slice 3) and the **gate-propagation escalation walk** (shape slice 4). Both are hard prod-enable gates; neither is needed for the background mechanism to be correct, and both land before enablement. +- **`ResolveReport` counters, the `ironclaw subagent edges` operator command, and boot-recovery fairness** (shape slice 5). +- **`subagent_inspect` / `subagent_extend` / `subagent_cancel` and the WebUI child tree** (shape slices 6–9). +- **The §6 `ParentAgent` extend budget of 8.** Task 1 lands the `ParentAgent` variant and Task 2 persists it, but the 8-activation window is `subagent_extend`'s, and ships with it. +- **Un-ignoring `tests/reborn_subagent_spawn_e2e.rs`.** Its five cases — including the one that currently asserts background is *rejected* — flip in shape slice 5, alongside the counters and operator command. diff --git a/harness/latency/runner/src/workloads.rs b/harness/latency/runner/src/workloads.rs index 6d75b38ba3b..808b36687c1 100644 --- a/harness/latency/runner/src/workloads.rs +++ b/harness/latency/runner/src/workloads.rs @@ -433,6 +433,7 @@ fn turn_lifecycle_submit_request( let pad_len = payload_len.min(96); let pad = "x".repeat(pad_len); Ok(SubmitTurnRequest { +subagent_activation_provenance: None, scope, actor, accepted_message_ref: AcceptedMessageRef::new(format!("message-{lane}-{key}-{pad}"))?, diff --git a/tests/integration/unbound_turns.rs b/tests/integration/unbound_turns.rs index cc81415192b..d5d0bad9f57 100644 --- a/tests/integration/unbound_turns.rs +++ b/tests/integration/unbound_turns.rs @@ -121,6 +121,7 @@ fn submit_request( idempotency_key: &str, ) -> HarnessResult { Ok(SubmitTurnRequest { + subagent_activation_provenance: None, scope, actor: TurnActor::new(UserId::new(CALLER)?), accepted_message_ref, diff --git a/tools/ironclaw_stress/src/user_turn.rs b/tools/ironclaw_stress/src/user_turn.rs index 1ecee658aa2..85d7321017b 100644 --- a/tools/ironclaw_stress/src/user_turn.rs +++ b/tools/ironclaw_stress/src/user_turn.rs @@ -867,6 +867,7 @@ where } = time_stage( &mut stages.submit_turn, turn_coordinator.submit_turn(SubmitTurnRequest { + subagent_activation_provenance: None, scope: context.turn_scope.clone(), actor: TurnActor::new(context.user_id.clone()), accepted_message_ref: AcceptedMessageRef::new(accepted.message_id.to_string()) @@ -1060,6 +1061,7 @@ where let SubmitTurnResponse::Accepted { .. } = time_stage( &mut stages.submit_turn, turn_coordinator.submit_turn(SubmitTurnRequest { + subagent_activation_provenance: None, scope: context.turn_scope.clone(), actor: TurnActor::new(context.user_id.clone()), accepted_message_ref: AcceptedMessageRef::new(format!( @@ -1163,6 +1165,7 @@ where let submit_result = time_stage( &mut stages.submit_turn, turn_coordinator.submit_turn(SubmitTurnRequest { + subagent_activation_provenance: None, scope: context.turn_scope.clone(), actor: TurnActor::new(context.user_id.clone()), accepted_message_ref: AcceptedMessageRef::new(accepted.message_id.to_string())