Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
a5bba34
feat(acpmux): attest live turn ownership
teamleaderleo Oct 30, 2026
cebb8de
Issue turn attestations from enrolled server identity
teamleaderleo Oct 30, 2026
d42f881
Close turn liveness race and refresh lock inputs
teamleaderleo Oct 30, 2026
311eb28
Update acpmux lockfile for server identity dependency
teamleaderleo Oct 30, 2026
34381b2
Fix attestation test enrollment and protocol registry
teamleaderleo Oct 30, 2026
bfbfd4c
Regenerate acpmux lock and protocol schemas
teamleaderleo Oct 30, 2026
89439dd
Verify shared attestation signing vector
teamleaderleo Oct 30, 2026
982361f
Merge feat-cmux-next into turn attestation
teamleaderleo Oct 30, 2026
ec544af
Adapt attestation test to carried turn argument
teamleaderleo Oct 30, 2026
08245d8
fix: refresh protocol and state-root examples
teamleaderleo Oct 30, 2026
2bab246
Merge remote-tracking branch 'origin/feat-cmux-next' into lane/turn-a…
teamleaderleo Oct 30, 2026
0080692
fix: complete base merge after interrupted object fetch
teamleaderleo Oct 30, 2026
5f30651
Merge branch 'feat-cmux-next' into lane/turn-attestation-acpmux
teamleaderleo Oct 30, 2026
f25a69e
fix(acpmux): refuse attestations for reconstructed closed sessions
teamleaderleo Oct 30, 2026
e54f0ea
Merge remote-tracking branch 'origin/feat-cmux-next' into lane/turn-a…
teamleaderleo Oct 30, 2026
15d3dda
fix(acpmux): preserve retry-before-journal lock order at turn completion
teamleaderleo Oct 30, 2026
e28f774
test(acpmux): recover poisoned lock-order test signal mutexes
teamleaderleo Oct 30, 2026
0ebd491
fix(acpmux): revalidate and hold live turn through attestation issuance
teamleaderleo Oct 30, 2026
a1406f9
fix(acpmux): reject transfer-fenced instances during attestation admi…
teamleaderleo Oct 30, 2026
b3db459
Merge feat-cmux-next and retain attestation lifecycle guards
teamleaderleo Oct 30, 2026
1fabc97
Serialize replacement fencing with attestation admission
teamleaderleo Oct 30, 2026
ee66299
Fence retry completion with replacement
teamleaderleo Oct 30, 2026
7383553
Place retry replacement race hook at terminal check
teamleaderleo Oct 30, 2026
1364281
Make retry replacement race assert fence ordering
teamleaderleo Oct 30, 2026
3c7ca44
Avoid test thread panic after replacement race
teamleaderleo Oct 30, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions cmux-tui/Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions cmux-tui/crates/acpmux/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ cmux-tailnet = { workspace = true, optional = true }
tokio-rustls = { version = "0.26", default-features = false, features = ["ring", "tls12"], optional = true }

[dependencies]
cmux-server = { path = "../cmux-server" }
cmux-json.workspace = true
cmux-chat-index.workspace = true
cmux-coderouter.workspace = true
Expand Down Expand Up @@ -103,6 +104,7 @@ windows-sys = { workspace = true, features = [
] }

[dev-dependencies]
cmux-server-core = { path = "../cmux-server-core" }
tempfile = "3"
rcgen = "0.14"
# Paused time for the media handle sweeper's tests (`hub/media_handles_tests.rs`).
Expand Down
6 changes: 6 additions & 0 deletions cmux-tui/crates/acpmux/spec/acpmux-protocol.json
Original file line number Diff line number Diff line change
Expand Up @@ -1530,6 +1530,12 @@
"params": true,
"result": true
},
{
"kind": "request",
"name": "_acpmux/turn_attestation",
"params": true,
"result": true
},
{
"kind": "notification",
"name": "_acpmux/event",
Expand Down
62 changes: 59 additions & 3 deletions cmux-tui/crates/acpmux/src/hub/auto_resume.rs
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,26 @@ use tokio::sync::Notify;

use crate::protocol::session::{RetryReason, RetrySummary};

#[cfg(test)]
thread_local! {
static AFTER_FINISH_REPLACED_CHECK: std::cell::RefCell<Option<Box<dyn FnOnce()>>> = const {
std::cell::RefCell::new(None)
};
}

#[cfg(test)]
fn after_finish_replaced_check() {
let hook = AFTER_FINISH_REPLACED_CHECK.with(|slot| slot.borrow_mut().take());
if let Some(hook) = hook {
hook();
}
}

#[cfg(test)]
pub(super) fn set_after_finish_replaced_check(hook: Box<dyn FnOnce()>) {
AFTER_FINISH_REPLACED_CHECK.with(|slot| *slot.borrow_mut() = Some(hook));
}

/// `_acpmux/retry`: a failed turn retried on demand.
pub(crate) mod retry_turn;

Expand Down Expand Up @@ -210,14 +230,25 @@ enum Mark {
/// The retry state of one session instance; returning imports have their own mark.
pub(crate) struct Pending {
mark: Mutex<Option<Mark>>,
#[cfg(test)]
stop_locked: Mutex<Option<std::sync::mpsc::Sender<()>>>,
#[cfg(test)]
take_entered: Mutex<Option<std::sync::mpsc::Sender<()>>>,
/// Counts the times a continuation stopped being pending (`continuation_pending`): the
/// inbox's timer, put off while one was, looks again then (`inbox.rs`).
released: tokio::sync::watch::Sender<u64>,
}

impl Default for Pending {
fn default() -> Self {
Self { mark: Mutex::default(), released: tokio::sync::watch::channel(0).0 }
Self {
mark: Mutex::default(),
released: tokio::sync::watch::channel(0).0,
#[cfg(test)]
stop_locked: Mutex::default(),
#[cfg(test)]
take_entered: Mutex::default(),
}
}
}

Expand Down Expand Up @@ -369,10 +400,13 @@ impl TurnInfo {
}

impl Hub {
/// Mark an outgoing session before its durable store is staged. Retry tasks hold the same
/// per-instance lock around their replacement check and terminal journal effects.
/// Mark an outgoing session before its durable store is staged. The admission lock order is
/// `auto_resume -> append_lock -> turn` for completion and replacement; attestation takes
/// `append_lock -> turn` and never `auto_resume`. The replacement fence takes both locks in
/// that order, so it cannot pass a `replaced` check and then race a terminal retry append.
pub(super) fn seal_retry_replacement(&self, session: &Session) {
let _retry = session.auto_resume.lock();
let _append = session.append_lock.lock().unwrap_or_else(|e| e.into_inner());
session.replaced.store(true, std::sync::atomic::Ordering::SeqCst);
}

Expand All @@ -386,7 +420,17 @@ impl Hub {
turn_id: &str,
next: Option<super::TurnInfo>,
) -> Option<super::TurnInfo> {
#[cfg(test)]
if let Some(entered) =
session.auto_resume.take_entered.lock().unwrap_or_else(|e| e.into_inner()).take()
{
let _ = entered.send(());
}
let mut pending = session.auto_resume.lock();
// Stop and retry journal writes take auto_resume before append_lock. Keep that
// order here too: taking append_lock in finish_turn before this retry lock
// deadlocks against Stop. Attestation still shares the turn-clear boundary.
let _order = session.append_lock.lock().unwrap_or_else(|e| e.into_inner());
let turn = {
let mut running = session.turn.lock().unwrap_or_else(|e| e.into_inner());
let turn = running.take();
Expand Down Expand Up @@ -543,6 +587,12 @@ impl Hub {
/// ending now gets none, however it ends.
pub(super) fn stop_auto_resume(&self, session: &Session) {
let mut pending = session.auto_resume.lock();
#[cfg(test)]
if let Some(locked) =
session.auto_resume.stop_locked.lock().unwrap_or_else(|e| e.into_inner()).take()
{
let _ = locked.send(());
}
// A lazy startup may not have installed the saved retry's mark yet. Journal the
// cancellation boundary under the same lock as recovery: only schedules before
// this record are stopped, never a later turn after an earlier idle Stop.
Expand Down Expand Up @@ -886,6 +936,8 @@ impl Hub {
if session.replaced.load(std::sync::atomic::Ordering::SeqCst) {
return;
}
#[cfg(test)]
after_finish_replaced_check();
let skipped = |why: &str| json!({"promptId": root, "attempt": attempt, "turnId": cut_turn, "skipped": why});
match result {
Ok(v) if v.get("stopReason").and_then(Value::as_str) != Some("cancelled") => {
Expand Down Expand Up @@ -1048,3 +1100,7 @@ mod tests {
assert_eq!(staggered(u64::MAX, "s"), u64::MAX);
}
}

#[cfg(test)]
#[path = "auto_resume/turn_completion_tests.rs"]
mod turn_completion_tests;
51 changes: 51 additions & 0 deletions cmux-tui/crates/acpmux/src/hub/auto_resume/restore.rs
Original file line number Diff line number Diff line change
Expand Up @@ -737,6 +737,57 @@ mod tests {
assert!(session.turn().is_none());
}

#[test]
fn finish_auto_resume_and_replacement_share_the_lock_order() {
let (hub, session) = idle_chat();
let (finish_done_tx, finish_done_rx) = std::sync::mpsc::channel();
let (seal_handle_tx, seal_handle_rx) = std::sync::mpsc::channel();
let (sealed_before_append_tx, sealed_before_append_rx) = std::sync::mpsc::channel();
let finish_hub = hub.clone();
let finish_session = session.clone();
let finish = std::thread::spawn(move || {
let seal_hub = finish_hub.clone();
let seal_session = finish_session.clone();
super::set_after_finish_replaced_check(Box::new(move || {
let (seal_done_tx, seal_done_rx) = std::sync::mpsc::channel();
let seal = std::thread::spawn(move || {
seal_hub.seal_retry_replacement(&seal_session);
let _ = seal_done_tx.send(());
});
seal_handle_tx.send(seal).unwrap();
// An append-only fence can complete here, between the replaced check and
// the stale terminal append. The ordered fence must wait for finish.
sealed_before_append_tx
.send(seal_done_rx.recv_timeout(std::time::Duration::from_millis(100)).is_ok())
.unwrap();
}));
finish_hub.finish_auto_resume(
&finish_session,
&CutTurn {
from: None,
prompt_id: "finish-prompt",
turn_id: "finish-turn",
control: Control::Local,
trust_gate: false,
error: "stream disconnected",
},
1,
&[],
Err(RpcError::invalid_params("folder disappeared")),
false,
);
finish_done_tx.send(()).unwrap();
});
let seal = seal_handle_rx.recv_timeout(std::time::Duration::from_secs(5)).unwrap();
assert!(!sealed_before_append_rx.recv_timeout(std::time::Duration::from_secs(5)).unwrap());
finish_done_rx
.recv_timeout(std::time::Duration::from_secs(5))
.expect("replacement must not block terminal retry admission");
finish.join().unwrap();
seal.join().unwrap();
assert!(session.replaced.load(std::sync::atomic::Ordering::SeqCst));
}

#[tokio::test]
async fn retry_restore_does_not_apply_an_earlier_idle_cancel_to_a_later_failed_turn() {
let (hub, session) = idle_chat();
Expand Down
117 changes: 117 additions & 0 deletions cmux-tui/crates/acpmux/src/hub/auto_resume/turn_completion_tests.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,117 @@
//! Concurrency regression for Stop racing a completed turn.
//!
//! `finish_turn` and `stop_auto_resume` must acquire the retry and append state in
//! one direction. The test deliberately holds the live-turn lock so the old
//! append -> auto-resume -> turn and auto-resume -> turn -> append cycle is
//! observable without timing assumptions.

use crate::config::{Config, StoreMode};
use crate::hub::{Control, Hub, NewRequest, TurnInfo};
use serde_json::json;
use std::path::Path;
use std::sync::{Arc, mpsc};
use std::time::Duration;

fn fake_hub() -> Arc<Hub> {
let fake = concat!(env!("CARGO_MANIFEST_DIR"), "/tests/fake_agent.py");
let cfg = serde_json::from_value::<Config>(json!({
"harnesses": {"fake": {"argv": ["python3", fake]}},
"defaultHarness": "fake"
}))
.expect("synthetic fake harness config");
let mut cfg = cfg;
cfg.store.mode = StoreMode::Memory;
let store = crate::store::open(&cfg.store, Path::new("/nonexistent")).unwrap();
Hub::new(cfg, store)
}

/// Force Stop to hold the retry lock before completion reaches take_turn. Both must
/// finish within the bounded receives; a deadlocked worker is never joined on failure.
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn stop_and_completion_share_one_lock_order() {
let hub = fake_hub();
let session = hub
.new_session(NewRequest { cwd: Some(std::env::temp_dir()), ..NewRequest::default() })
.await
.expect("create synthetic session");
let child = hub.child_for(&session).await.expect("start synthetic child");
let mut turn =
TurnInfo::prompted("test", "work", "turn-lock-order", "prompt-lock-order", Control::Local);
turn.turn_seq =
hub.append(&session, "mux", "turn_started", json!({"turnId": turn.turn_id})).seq;
*session.turn.lock().unwrap_or_else(|e| e.into_inner()) = Some(turn.clone());
// Keep the live-turn lock held until both contenders have reached their lock boundary.
let turn_guard = session.turn.lock().unwrap_or_else(|e| e.into_inner());

// The stop thread takes auto_resume and then waits for the live-turn lock.
let (stop_locked_tx, stop_locked_rx) = mpsc::channel();
*session.auto_resume.stop_locked.lock().unwrap() = Some(stop_locked_tx);
let (stop_done_tx, stop_done_rx) = mpsc::channel();
let stop_hub = hub.clone();
let stop_session = session.clone();
let stop = std::thread::spawn(move || {
stop_hub.stop_auto_resume(&stop_session);
let _ = stop_done_tx.send(());
});
stop_locked_rx
.recv_timeout(Duration::from_secs(5))
.expect("Stop did not enter auto_resume state");

// Completion enters the real finish_turn path. On the old head it owns append_lock
// before take_turn waits for auto_resume, completing the cycle above.
let (take_entered_tx, take_entered_rx) = mpsc::channel();
*session.auto_resume.take_entered.lock().unwrap() = Some(take_entered_tx);
let complete_hub = hub.clone();
let complete_session = session.clone();
let complete_child = child.clone();
let (completion_done_tx, completion_done_rx) = mpsc::channel();
let completion = std::thread::spawn(move || {
let runtime = tokio::runtime::Builder::new_current_thread().enable_all().build().unwrap();
let result = runtime.block_on(async move {
complete_hub
.finish_turn(
&complete_session,
&complete_child,
Ok(json!({"stopReason":"end_turn"})),
&turn.prompt_id,
&turn.turn_id,
turn.turn_seq,
None,
None,
)
.await
});
let _ = completion_done_tx.send(result.is_ok());
result
});

// The hook is before take_turn's retry-lock acquisition. If finish_turn still takes
// append_lock first, Stop and completion are now on opposite sides of the cycle.
take_entered_rx
.recv_timeout(Duration::from_secs(5))
.expect("completion did not enter take_turn");
drop(turn_guard);

// A fixed order drains both operations promptly. The bounded receives make the old
// append -> retry -> turn cycle a test failure instead of an unbounded harness hang.
assert!(
stop_done_rx.recv_timeout(Duration::from_secs(5)).is_ok(),
"Stop/completion lock inversion left Stop blocked"
);
assert!(
completion_done_rx.recv_timeout(Duration::from_secs(5)).is_ok_and(|ok| ok),
"Stop/completion lock inversion left completion blocked"
);
stop.join().expect("Stop thread panicked");
let finished = completion.join().expect("completion thread panicked");
assert!(finished.is_ok(), "completion returned a synthetic error");
assert!(session.turn().is_none(), "completion left a live turn");
assert!(
hub.stopped_for_test(&session, "turn-lock-order"),
"completion lost Stop's retry fence"
);
let events = hub.events(&session.id, 0, 100).unwrap();
let stopped = events.iter().position(|event| event.kind == "turn_stop_requested").unwrap();
let finished = events.iter().position(|event| event.kind == "turn_result").unwrap();
assert!(stopped < finished, "completion overtook Stop's durable boundary");
}
1 change: 1 addition & 0 deletions cmux-tui/crates/acpmux/src/hub/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -197,6 +197,7 @@ mod transfer_in;
mod transfer_out;
mod transfer_watch;
mod trust_wait;
mod turn_attestation;
mod turn_git;
mod turn_usage;
mod usage_view;
Expand Down
22 changes: 14 additions & 8 deletions cmux-tui/crates/acpmux/src/hub/transfer_in.rs
Original file line number Diff line number Diff line change
Expand Up @@ -324,6 +324,19 @@ impl Hub {
let _ = std::fs::remove_file(path);
}

/// Fence the outgoing instance before staging the incoming durable store. Keep
/// this boundary shared by transfer_finish and its admission-race regression.
pub(super) async fn retire_transfer_instance(
&self,
old: &Arc<Session>,
) -> Result<(), RpcError> {
// Retry tasks hold the same lock across their replaced check and append.
self.seal_retry_replacement(old);
// Drain an earlier retention commit before replacing the journal.
drop(old.append_lock.lock().unwrap_or_else(|e| e.into_inner()));
self.kill(old, false).await
}

pub(super) async fn transfer_receive(&self, params: &Value) -> Result<Value, RpcError> {
let manifest = params.get("manifest").cloned().unwrap_or(Value::Null);
let guard = match params.get("guard") {
Expand Down Expand Up @@ -522,14 +535,7 @@ impl Hub {
let e = transfer_error("cwd", "the chat's folder is not here; pass cwd");
return Err(e.with_data(json!({"reason": "cwd_missing", "cwd": cwd})));
}
// Seal the old instance before staging its durable store. Retry tasks hold this
// lock across their replaced check and terminal append, so no late record can land
// in the replacement journal.
self.seal_retry_replacement(old);
// A log retention pass that took the chat's append lock before the mark finishes its
// commit first; one after it leaves the chat alone (`hub/retention.rs`).
drop(old.append_lock.lock().unwrap_or_else(|e| e.into_inner()));
self.kill(old, false).await?;
self.retire_transfer_instance(old).await?;
}
let source_store = if replacing_durable { self.store.session_dir(id) } else { None };
let source_store_backup = if let Some(source_store) = source_store.as_ref() {
Expand Down
Loading