From 26e4e71e3f934ca97273efc88bc8bbafef41d5ec Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 3 Nov 2026 13:46:00 +0000 Subject: [PATCH 01/52] cmux-journal: one append-only journal per daemon, with group commit and ack after durable (R-DURABILITY S1) A new crate, no consumer yet. It is stage 1 of the R-DURABILITY rewrite (handoff/rewrites/R-DURABILITY/DESIGN.md): every acknowledged change of the cmux-tui daemon and of acpmux goes into one journal of this format, and a reply goes out only once its record is synced. - Journal::append copies one record into memory (no I/O, no wait) and returns its LSN. One writer thread takes every pending record as one batch, writes it with one write, and syncs it once when it holds an acked record (group commit). Durable listeners then release the replies that wait for those LSNs. Stream records (terminal output, streamed tokens) are written in the next batch and synced with the next acked batch. - Sync levels: F_BARRIERFSYNC per acked batch on macOS (fdatasync on Linux), and F_FULLFSYNC at most 1 s after the first barrier sync, so a power loss takes back at most 1 s of acked batches, always a suffix. - Recovery is the one start path (clean exit, crash, kill -9, OS crash): it syncs the last segment, checks every record's CRC and LSN, and hands the records after the caller's compaction point to its replay. A segment is only ever appended to: a torn tail stays in its file and the journal goes on in a new segment whose header names the last good LSN. Damage that is not a torn tail moves the later segments to quarantine/ (never deleted), and no LSN is used twice. - A failed write is cut back and written again in the same segment; a failed sync is never retried on that segment (its pages may be clean and lost): the records after the durable LSN go into a new segment. - Storage is the file system seam. The `sim` feature's SimStorage keeps synced and unsynced bytes apart and crashes at any storage operation, then keeps what a process crash, an OS crash or a power loss keeps. Tests: unit tests of the format, group commit, ack after durable, listeners, sync levels, idle (no storage operation), backlog, seal, lock, rotation, failed writes and syncs at every operation, torn tails, quarantine and LSN reuse; the crash simulation (200 seeds per crash kind by default, CMUX_JOURNAL_CRASH_SEEDS for the acceptance run); and a round trip with a torn tail on the real file system. OWNERS.md: a row for the crate (R-DURABILITY rewrite owner). --- cmux-tui/Cargo.lock | 9 + cmux-tui/Cargo.toml | 3 + cmux-tui/crates/cmux-journal/Cargo.toml | 25 + .../cmux-journal/src/cmux_journal_tests.rs | 635 ++++++++++++++++++ cmux-tui/crates/cmux-journal/src/format.rs | 236 +++++++ cmux-tui/crates/cmux-journal/src/journal.rs | 363 ++++++++++ cmux-tui/crates/cmux-journal/src/lib.rs | 138 ++++ cmux-tui/crates/cmux-journal/src/recovery.rs | 475 +++++++++++++ cmux-tui/crates/cmux-journal/src/sim.rs | 576 ++++++++++++++++ cmux-tui/crates/cmux-journal/src/storage.rs | 251 +++++++ cmux-tui/crates/cmux-journal/src/writer.rs | 370 ++++++++++ plans/cmux-next/OWNERS.md | 1 + 12 files changed, 3082 insertions(+) create mode 100644 cmux-tui/crates/cmux-journal/Cargo.toml create mode 100644 cmux-tui/crates/cmux-journal/src/cmux_journal_tests.rs create mode 100644 cmux-tui/crates/cmux-journal/src/format.rs create mode 100644 cmux-tui/crates/cmux-journal/src/journal.rs create mode 100644 cmux-tui/crates/cmux-journal/src/lib.rs create mode 100644 cmux-tui/crates/cmux-journal/src/recovery.rs create mode 100644 cmux-tui/crates/cmux-journal/src/sim.rs create mode 100644 cmux-tui/crates/cmux-journal/src/storage.rs create mode 100644 cmux-tui/crates/cmux-journal/src/writer.rs diff --git a/cmux-tui/Cargo.lock b/cmux-tui/Cargo.lock index aa899b1cf103..7f9a2eccca8a 100644 --- a/cmux-tui/Cargo.lock +++ b/cmux-tui/Cargo.lock @@ -1278,6 +1278,15 @@ dependencies = [ "serde_json", ] +[[package]] +name = "cmux-journal" +version = "0.1.0" +dependencies = [ + "crc32fast", + "libc", + "tempfile", +] + [[package]] name = "cmux-json" version = "0.1.0" diff --git a/cmux-tui/Cargo.toml b/cmux-tui/Cargo.toml index 8883538ad3cc..64e236271a33 100644 --- a/cmux-tui/Cargo.toml +++ b/cmux-tui/Cargo.toml @@ -43,6 +43,7 @@ members = [ "crates/cmux-tui-hooks", "crates/cmux-tui-util", "crates/cmux-json", + "crates/cmux-journal", "crates/cmux-kitty-media", "crates/cmux-api-compat", "crates/cmux-tui-remote-access", @@ -123,6 +124,7 @@ default-members = [ "crates/cmux-tui-hooks", "crates/cmux-tui-util", "crates/cmux-json", + "crates/cmux-journal", "crates/cmux-api-compat", "crates/cmux-tui-remote-access", "crates/cmux-layout-reducer", @@ -237,6 +239,7 @@ cmux-tui-image-paste = { path = "crates/cmux-tui-image-paste" } cmux-tui-hooks = { path = "crates/cmux-tui-hooks" } cmux-tui-util = { path = "crates/cmux-tui-util" } cmux-json = { path = "crates/cmux-json" } +cmux-journal = { path = "crates/cmux-journal" } cmux-kitty-media = { path = "crates/cmux-kitty-media" } cmux-api-compat = { path = "crates/cmux-api-compat" } cmux-tui-remote-access = { path = "crates/cmux-tui-remote-access" } diff --git a/cmux-tui/crates/cmux-journal/Cargo.toml b/cmux-tui/crates/cmux-journal/Cargo.toml new file mode 100644 index 000000000000..4a9510af5dc7 --- /dev/null +++ b/cmux-tui/crates/cmux-journal/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "cmux-journal" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true +license.workspace = true +publish.workspace = true +description = "One append-only journal per daemon: group commit, ack after durable, one recovery path (R-DURABILITY)" + +[lints] +workspace = true + +[features] +# The simulated crash file system (`SimStorage`), for crash tests of the crates that use the +# journal. Never on in a daemon binary. +sim = [] + +[dependencies] +crc32fast = "1.5" + +[target.'cfg(unix)'.dependencies] +libc.workspace = true + +[dev-dependencies] +tempfile = "3" diff --git a/cmux-tui/crates/cmux-journal/src/cmux_journal_tests.rs b/cmux-tui/crates/cmux-journal/src/cmux_journal_tests.rs new file mode 100644 index 000000000000..a7067051f1ef --- /dev/null +++ b/cmux-tui/crates/cmux-journal/src/cmux_journal_tests.rs @@ -0,0 +1,635 @@ +//! Tests of the journal: format, group commit, ack after durable, recovery, failures, and the +//! crash simulation (every acked record survives a crash at any storage operation; nothing is +//! half applied; the lost tail is a suffix). +//! +//! The crash simulation runs `CMUX_JOURNAL_CRASH_SEEDS` seeds per crash kind (default 200; the +//! acceptance run in R-DURABILITY/DESIGN.md section 7 uses 100000 on the fleet). A failing seed +//! prints itself: rerun it alone with `CMUX_JOURNAL_CRASH_SEED=`. + +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use crate::format::{self, SEGMENT_HEADER_LEN}; +use crate::{ + AppendError, Class, Config, Crash, Damage, FsStorage, Journal, Lsn, OpenError, OpenReport, + SimRng, SimStorage, Storage, WaitError, +}; + +const WAIT: Duration = Duration::from_secs(10); + +/// What a replay saw: (lsn, kind, class, payload) per record. +type Seen = Vec<(Lsn, u16, Class, Vec)>; + +fn dir() -> PathBuf { + PathBuf::from("/sim/journal") +} + +fn quick() -> Config { + Config { + retry_min: Duration::from_millis(1), + retry_max: Duration::from_millis(5), + stream_sync_delay: Duration::from_millis(20), + ..Config::default() + } +} + +/// Opens the journal in `storage` from compaction point `from`. +fn open_at( + storage: &SimStorage, + config: Config, + from: Lsn, +) -> Result<(Journal, OpenReport, Seen), OpenError> { + let mut seen = Vec::new(); + let (journal, report) = + Journal::open(Arc::new(storage.clone()), &dir(), config, from, &mut |record| { + seen.push((record.lsn, record.kind, record.class, record.payload.to_vec())); + Ok(()) + })?; + Ok((journal, report, seen)) +} + +fn open(storage: &SimStorage) -> (Journal, OpenReport, Seen) { + open_at(storage, quick(), 0).unwrap_or_else(|error| panic!("open: {error}")) +} + +fn lsns(seen: &Seen) -> Vec { + seen.iter().map(|record| record.0).collect() +} + +fn payload(lsn: Lsn) -> Vec { + format!("record-{lsn}-{}", "x".repeat(usize::try_from(lsn % 97).unwrap_or(0))).into_bytes() +} + +/// Polls `check` for up to 5 s. +fn eventually(check: impl Fn() -> bool) -> bool { + (0..1000).any(|_| { + if check() { + return true; + } + std::thread::sleep(Duration::from_millis(5)); + false + }) +} + +#[test] +fn a_zero_record_header_never_checks_out() { + let header = [0u8; format::RECORD_HEADER_LEN]; + let decoded = format::RecordHeader::decode(&header).unwrap_or_else(|| panic!("decode")); + assert!(decoded.verify(&header, &[], 0).is_err(), "a zeroed header must fail its checksum"); +} + +#[test] +fn segment_names_sort_and_parse_for_every_number() { + for seq in [0, 1, 42, u64::MAX] { + assert_eq!(format::parse_segment_name(&format::segment_name(seq)), Some(seq)); + } + assert!(format::segment_name(9) < format::segment_name(10)); + assert_eq!(format::parse_segment_name("LOCK"), None); + assert_eq!(format::parse_segment_name("42.seg"), None); +} + +#[test] +fn records_round_trip_in_order_across_a_reopen() { + let storage = SimStorage::new(1); + let (journal, report, seen) = open(&storage); + assert!(seen.is_empty()); + assert_eq!(report.next_lsn, 1); + for lsn in 1..=50 { + let class = if lsn.is_multiple_of(3) { Class::Stream } else { Class::Ack }; + assert_eq!(journal.append(7, class, &payload(lsn)), Ok(lsn)); + } + journal.wait_durable(49, WAIT).unwrap_or_else(|error| panic!("wait: {error}")); + assert_eq!(journal.seal(WAIT), Ok(50)); + drop(journal); + let (journal, report, seen) = open(&storage.restart(Crash::PowerLoss)); + assert_eq!(report.replayed, 50); + assert_eq!(report.next_lsn, 51); + assert_eq!(report.damage, Vec::new()); + assert_eq!(lsns(&seen), (1..=50).collect::>()); + for (lsn, kind, class, bytes) in &seen { + assert_eq!(*kind, 7); + assert_eq!(*class, if lsn.is_multiple_of(3) { Class::Stream } else { Class::Ack }); + assert_eq!(*bytes, payload(*lsn)); + } + assert_eq!(journal.append(7, Class::Ack, b"next"), Ok(51)); +} + +#[test] +fn replay_starts_after_the_compaction_point() { + let storage = SimStorage::new(2); + let (journal, _, _) = open(&storage); + for lsn in 1..=20 { + assert_eq!(journal.append(1, Class::Ack, &payload(lsn)), Ok(lsn)); + } + assert_eq!(journal.seal(WAIT), Ok(20)); + drop(journal); + let (_, report, seen) = + open_at(&storage.restart(Crash::Process), quick(), 12).unwrap_or_else(|e| panic!("{e}")); + assert_eq!(lsns(&seen), (13..=20).collect::>()); + assert_eq!(report.replayed, 8); +} + +#[test] +fn an_ack_is_not_reported_before_its_sync_ends() { + let storage = SimStorage::new(3); + let (journal, _, _) = open(&storage); + storage.set_sync_delay(Duration::from_millis(300)); + let lsn = journal.append(1, Class::Ack, b"draft: hello").unwrap_or_else(|e| panic!("{e}")); + assert!(eventually(|| journal.stats().written_lsn >= lsn), "the record was never written"); + // Written, and its sync takes 300 ms: not durable, so no reply may say it is. + assert_eq!(journal.durable_lsn(), 0); + assert_eq!(journal.wait_durable(lsn, Duration::from_millis(50)), Err(WaitError::Timeout)); + journal.wait_durable(lsn, WAIT).unwrap_or_else(|e| panic!("{e}")); + assert!(journal.stats().barrier_syncs + journal.stats().full_syncs >= 1); +} + +#[test] +fn a_burst_shares_one_sync_per_batch() { + let storage = SimStorage::new(4); + let (journal, _, _) = open(&storage); + storage.set_sync_delay(Duration::from_millis(20)); + let journal = Arc::new(journal); + let threads: Vec<_> = (0..8u64) + .map(|thread| { + let journal = journal.clone(); + std::thread::spawn(move || { + for index in 0..50u64 { + let lsn = journal + .append(2, Class::Ack, &payload(thread * 1000 + index)) + .unwrap_or_else(|e| panic!("{e}")); + journal.wait_durable(lsn, WAIT).unwrap_or_else(|e| panic!("{e}")); + } + }) + }) + .collect(); + for thread in threads { + thread.join().unwrap_or_else(|_| panic!("a client thread panicked")); + } + let stats = journal.stats(); + assert_eq!(stats.records, 400); + let syncs = stats.barrier_syncs + stats.full_syncs; + // One sync per record would be 400; with 8 clients waiting at once, a batch holds several. + assert!(syncs < 200, "group commit: {syncs} syncs for 400 acked records ({stats:?})"); + assert!(stats.max_batch_records >= 4, "{stats:?}"); +} + +#[test] +fn durable_listeners_see_each_advance_in_order() { + let storage = SimStorage::new(5); + let (journal, _, _) = open(&storage); + let seen = Arc::new(Mutex::new(Vec::new())); + let sink = seen.clone(); + journal.on_durable(move |lsn| sink.lock().unwrap_or_else(|e| e.into_inner()).push(lsn)); + for lsn in 1..=30 { + assert_eq!(journal.append(1, Class::Ack, &payload(lsn)), Ok(lsn)); + } + let heard = || seen.lock().unwrap_or_else(|e| e.into_inner()).clone(); + assert!(eventually(|| heard().last() == Some(&30)), "{:?}", heard()); + let heard = heard(); + assert!(heard.windows(2).all(|pair| pair.first() < pair.get(1)), "{heard:?}"); +} + +#[test] +fn a_stream_record_is_written_at_once_and_synced_after_its_delay() { + let storage = SimStorage::new(6); + let config = Config { stream_sync_delay: Duration::from_millis(300), ..quick() }; + let (journal, _, _) = open_at(&storage, config, 0).unwrap_or_else(|e| panic!("{e}")); + let lsn = journal.append(9, Class::Stream, b"output").unwrap_or_else(|e| panic!("{e}")); + assert!(eventually(|| journal.stats().written_lsn >= lsn), "{:?}", journal.stats()); + assert_eq!(journal.durable_lsn(), 0, "a stream record alone is not synced at once"); + // Written, so a kill of every process now keeps it. + let (_, _, seen) = open(&storage.snapshot(Crash::Process)); + assert_eq!(seen.first().map(|record| record.3.clone()), Some(b"output".to_vec())); + journal.wait_durable(lsn, WAIT).unwrap_or_else(|e| panic!("{e}")); +} + +#[test] +fn an_idle_journal_touches_no_storage() { + let storage = SimStorage::new(7); + let config = Config { full_sync_interval: Duration::from_millis(50), ..quick() }; + let (journal, _, _) = open_at(&storage, config, 0).unwrap_or_else(|e| panic!("{e}")); + let lsn = journal.append(1, Class::Ack, b"x").unwrap_or_else(|e| panic!("{e}")); + journal.wait_durable(lsn, WAIT).unwrap_or_else(|e| panic!("{e}")); + assert!(eventually(|| journal.full_durable_lsn() >= lsn), "{:?}", journal.stats()); + let before = storage.ops(); + std::thread::sleep(Duration::from_millis(300)); + assert_eq!(storage.ops(), before, "an idle journal touched the storage"); +} + +#[test] +fn a_full_sync_follows_the_first_barrier_sync_within_the_interval() { + let storage = SimStorage::new(8); + let config = Config { full_sync_interval: Duration::from_millis(500), ..quick() }; + let (journal, _, _) = open_at(&storage, config, 0).unwrap_or_else(|e| panic!("{e}")); + let lsn = journal.append(1, Class::Ack, b"x").unwrap_or_else(|e| panic!("{e}")); + journal.wait_durable(lsn, WAIT).unwrap_or_else(|e| panic!("{e}")); + assert_eq!(journal.stats().full_syncs, 0, "the ack waited for a drive cache flush"); + assert!(eventually(|| journal.full_durable_lsn() >= lsn), "{:?}", journal.stats()); + // A power loss now keeps it. + let (_, _, seen) = open(&storage.snapshot(Crash::PowerLoss)); + assert_eq!(lsns(&seen), vec![lsn]); +} + +#[test] +fn appends_past_the_pending_cap_fail_without_waiting() { + let storage = SimStorage::new(9); + let config = Config { max_pending_bytes: 4096, ..quick() }; + let (journal, _, _) = open_at(&storage, config, 0).unwrap_or_else(|e| panic!("{e}")); + storage.set_sync_delay(Duration::from_millis(500)); + let big = vec![b'a'; 1000]; + let refused = (0..50).any(|_| match journal.append(1, Class::Ack, &big) { + Ok(_) => false, + Err(AppendError::Backlog) => true, + Err(other) => panic!("unexpected {other}"), + }); + assert!(refused, "the pending cap never refused an append"); + let too_large = vec![0u8; crate::MAX_PAYLOAD_LEN + 1]; + assert_eq!(journal.append(1, Class::Ack, &too_large), Err(AppendError::TooLarge)); +} + +#[test] +fn a_sealed_journal_refuses_appends() { + let storage = SimStorage::new(10); + let (journal, _, _) = open(&storage); + assert_eq!(journal.append(1, Class::Ack, b"a"), Ok(1)); + assert_eq!(journal.seal(WAIT), Ok(1)); + assert_eq!(journal.append(1, Class::Ack, b"b"), Err(AppendError::Closed)); + assert_eq!(journal.full_durable_lsn(), 1); +} + +#[test] +fn a_second_writer_is_refused_while_the_first_holds_the_lock() { + let storage = SimStorage::new(11); + let (first, _, _) = open(&storage); + assert!(matches!(open_at(&storage, quick(), 0), Err(OpenError::Locked))); + drop(first); + assert!(open_at(&storage, quick(), 0).is_ok()); +} + +#[test] +fn segments_rotate_and_recovery_reads_across_them() { + let storage = SimStorage::new(12); + let config = Config { segment_bytes: 600, ..quick() }; + let (journal, _, _) = open_at(&storage, config.clone(), 0).unwrap_or_else(|e| panic!("{e}")); + for lsn in 1..=40 { + let got = journal.append(3, Class::Ack, &payload(lsn)).unwrap_or_else(|e| panic!("{e}")); + journal.wait_durable(got, WAIT).unwrap_or_else(|e| panic!("{e}")); + } + assert!(journal.stats().segments_created >= 3, "{:?}", journal.stats()); + drop(journal); + let (_, report, seen) = + open_at(&storage.restart(Crash::PowerLoss), config, 0).unwrap_or_else(|e| panic!("{e}")); + assert_eq!(lsns(&seen), (1..=40).collect::>(), "{report:?}"); + assert!(report.segments >= 4); + // From a compaction point in a later segment, the earlier segments' records are not read. + let storage = storage.restart(Crash::Process); + let (_, _, seen) = open_at(&storage, quick(), 35).unwrap_or_else(|e| panic!("{e}")); + assert_eq!(lsns(&seen), (36..=40).collect::>()); + assert_eq!(storage.reads(&dir().join(format::segment_name(1))), 1, "only its header"); +} + +#[test] +fn a_failed_write_retries_in_the_same_segment() { + let storage = SimStorage::new(13); + let (journal, _, _) = open(&storage); + // The batch's write is the next storage operation; a full disk refuses it once. + storage.fail_after(1, std::io::ErrorKind::StorageFull); + let lsn = journal.append(1, Class::Ack, b"draft").unwrap_or_else(|e| panic!("{e}")); + journal.wait_durable(lsn, WAIT).unwrap_or_else(|e| panic!("{e}")); + let stats = journal.stats(); + assert_eq!((stats.failures, stats.segments_created, stats.failing), (1, 0, None)); + drop(journal); + let (_, report, seen) = open(&storage.restart(Crash::PowerLoss)); + assert_eq!(lsns(&seen), vec![1], "{report:?}"); +} + +#[test] +fn a_failed_sync_never_syncs_that_segment_again() { + let storage = SimStorage::new(14); + let (journal, _, _) = open(&storage); + assert_eq!(journal.append(1, Class::Ack, b"a"), Ok(1)); + journal.wait_durable(1, WAIT).unwrap_or_else(|e| panic!("{e}")); + // Settled: the full sync of the power-loss window ran, so the next operations are the batch's. + assert!(eventually(|| journal.full_durable_lsn() >= 1), "{:?}", journal.stats()); + // The next batch: its write (1), then its sync (2), which fails. + storage.fail_after(2, std::io::ErrorKind::Other); + assert_eq!(journal.append(1, Class::Ack, b"b"), Ok(2)); + journal.wait_durable(2, WAIT).unwrap_or_else(|e| panic!("{e}")); + let stats = journal.stats(); + assert_eq!((stats.failures, stats.segments_created), (1, 1), "{stats:?}"); + drop(journal); + let restarted = storage.restart(Crash::PowerLoss); + assert_eq!(restarted.names_in(&dir()).iter().filter(|name| name.ends_with(".seg")).count(), 2); + let (_, report, seen) = open(&restarted); + assert_eq!(lsns(&seen), vec![1, 2], "{report:?}"); + assert!(report.damage.is_empty(), "{report:?}"); +} + +#[test] +fn every_acked_record_survives_a_failure_at_any_operation() { + for at in 1..12 { + let storage = SimStorage::new(15 + at); + let (journal, _, _) = open(&storage); + assert_eq!(journal.append(1, Class::Ack, b"before"), Ok(1)); + journal.wait_durable(1, WAIT).unwrap_or_else(|e| panic!("{e}")); + storage.fail_after(at, std::io::ErrorKind::Other); + for index in 0..10 { + let lsn = + journal.append(1, Class::Ack, &payload(index)).unwrap_or_else(|e| panic!("{e}")); + journal.wait_durable(lsn, WAIT).unwrap_or_else(|e| panic!("failure at op {at}: {e}")); + } + assert!(journal.stats().failures <= 1); + drop(journal); + let (_, report, seen) = open(&storage.restart(Crash::PowerLoss)); + assert_eq!(lsns(&seen), (1..=11).collect::>(), "failure at op {at}: {report:?}"); + assert!(report.damage.is_empty(), "failure at op {at}: {report:?}"); + } +} + +#[test] +fn a_torn_tail_stays_in_its_file_and_the_journal_goes_on_after_it() { + let storage = SimStorage::new(40); + let (journal, _, _) = open(&storage); + assert_eq!(journal.append(1, Class::Ack, b"one"), Ok(1)); + assert_eq!(journal.seal(WAIT), Ok(1)); + drop(journal); + let first = dir().join(format::segment_name(1)); + // A crash in the middle of the next write: half a record. + let mut torn = storage.contents(&first).unwrap_or_default(); + torn.extend_from_slice(&[0x33; 9]); + let crashed = storage.restart(Crash::Process); + write_whole(&crashed, &first, &torn); + let (journal, report, seen) = open(&crashed); + assert_eq!((lsns(&seen), report.torn_tail_bytes), (vec![1], 9)); + assert_eq!(crashed.contents(&first), Some(torn), "recovery shortened a segment"); + assert_eq!(journal.append(1, Class::Ack, b"two"), Ok(2)); + assert_eq!(journal.seal(WAIT), Ok(2)); + drop(journal); + let (_, report, seen) = open(&crashed.restart(Crash::PowerLoss)); + assert_eq!(lsns(&seen), vec![1, 2]); + assert_eq!(report.torn_tail_bytes, 0); + assert!(report.damage.is_empty(), "{report:?}"); +} + +#[test] +fn a_damaged_segment_in_the_middle_is_quarantined_and_its_lsns_are_never_reused() { + let storage = SimStorage::new(31); + let config = Config { segment_bytes: 400, ..quick() }; + let (journal, _, _) = open_at(&storage, config.clone(), 0).unwrap_or_else(|e| panic!("{e}")); + for lsn in 1..=30 { + let got = journal.append(3, Class::Ack, &payload(lsn)).unwrap_or_else(|e| panic!("{e}")); + journal.wait_durable(got, WAIT).unwrap_or_else(|e| panic!("{e}")); + } + assert!(journal.stats().segments_created >= 3); + drop(journal); + // Media damage in the first record of segment 2. + storage.corrupt(&dir().join(format::segment_name(2)), SEGMENT_HEADER_LEN + 2); + let restarted = storage.restart(Crash::Process); + let (journal, report, seen) = open_at(&restarted, config, 0).unwrap_or_else(|e| panic!("{e}")); + let kept = Lsn::try_from(seen.len()).unwrap_or(0); + assert!(kept > 0 && kept < 30, "the valid prefix stays: {kept}"); + assert_eq!(lsns(&seen), (1..=kept).collect::>()); + match report.damage.first() { + Some(Damage::Corrupt { moved, lost_after, lost_through, .. }) => { + assert_eq!((*lost_after, *lost_through), (kept, 30), "{report:?}"); + assert!(!moved.is_empty()); + } + other => panic!("no damage report: {other:?}"), + } + assert!(!restarted.names_in(&dir().join("quarantine")).is_empty(), "kept for a salvage"); + assert_eq!(journal.append(3, Class::Ack, b"after"), Ok(31)); +} + +#[test] +fn a_compaction_point_past_the_end_never_reuses_an_lsn() { + let storage = SimStorage::new(32); + let (journal, _, _) = open(&storage); + assert_eq!(journal.append(1, Class::Ack, b"a"), Ok(1)); + assert_eq!(journal.seal(WAIT), Ok(1)); + drop(journal); + let (journal, report, _) = + open_at(&storage.restart(Crash::Process), quick(), 9).unwrap_or_else(|e| panic!("{e}")); + assert_eq!(report.damage, vec![Damage::Ahead { journal_end: 1, from: 9 }]); + assert_eq!(journal.append(1, Class::Ack, b"b"), Ok(10)); + assert_eq!(journal.seal(WAIT), Ok(10)); + drop(journal); + let (_, report, seen) = + open_at(&storage.restart(Crash::PowerLoss), quick(), 0).unwrap_or_else(|e| panic!("{e}")); + assert_eq!(lsns(&seen), vec![1, 10]); + assert!(report.damage.is_empty(), "{report:?}"); +} + +#[test] +fn a_replay_error_stops_the_open() { + let storage = SimStorage::new(33); + let (journal, _, _) = open(&storage); + assert_eq!(journal.append(1, Class::Ack, b"a"), Ok(1)); + assert_eq!(journal.seal(WAIT), Ok(1)); + drop(journal); + let failed = + Journal::open(Arc::new(storage.restart(Crash::Process)), &dir(), quick(), 0, &mut |_| { + Err("no".into()) + }); + assert!(matches!(failed, Err(OpenError::Replay { lsn: 1, .. }))); +} + +/// Replaces `path`'s bytes in `storage` (a test's own damage). +fn write_whole(storage: &SimStorage, path: &Path, bytes: &[u8]) { + let mut file = storage + .open(path, crate::OpenMode::Create) + .unwrap_or_else(|e| panic!("open {}: {e}", path.display())); + file.append(bytes).unwrap_or_else(|e| panic!("write: {e}")); + file.sync(crate::SyncLevel::Full).unwrap_or_else(|e| panic!("sync: {e}")); +} + +// ---------------------------------------------------------------- crash simulation + +/// Seeds per crash kind (module docs). +fn crash_seeds() -> Vec { + if let Some(seed) = std::env::var("CMUX_JOURNAL_CRASH_SEED").ok().and_then(|s| s.parse().ok()) { + return vec![seed]; + } + let count = std::env::var("CMUX_JOURNAL_CRASH_SEEDS") + .ok() + .and_then(|s| s.parse().ok()) + .unwrap_or(200u64); + (0..count).collect() +} + +/// Waits until `lsn` is durable; false once the storage crashed first. +fn acked(journal: &Journal, storage: &SimStorage, lsn: Lsn) -> bool { + loop { + match journal.wait_durable(lsn, Duration::from_millis(5)) { + Ok(()) => return true, + Err(WaitError::Timeout) if !storage.is_crashed() => {} + Err(_) => return false, + } + } +} + +/// One crash run: clients append and wait while the storage crashes at a random operation; then +/// recovery runs. Every recovered record must be exactly what was appended at its LSN, the +/// records must be 1..=n, and n must reach every record a client was told is durable (fully +/// durable, for a power loss), and every written record for a process crash. +fn crash_run(seed: u64, crash: Crash) -> Result<(), String> { + let mut rng = SimRng::new(seed); + let storage = SimStorage::new(rng.next_u64()); + let config = Config { + segment_bytes: 300 + rng.below(3000), + max_pending_bytes: 1 << 20, + full_sync_interval: Duration::from_millis(rng.below(4)), + stream_sync_delay: Duration::from_millis(rng.below(3)), + retry_min: Duration::from_millis(1), + retry_max: Duration::from_millis(2), + }; + let (journal, _, _) = open_at(&storage, config.clone(), 0).map_err(|e| e.to_string())?; + let journal = Arc::new(journal); + let appended: Arc)>>> = Arc::default(); + let durable = Arc::new(AtomicU64::new(0)); + let full_durable = Arc::new(AtomicU64::new(0)); + storage.crash_after(1 + rng.below(80)); + let clients = 1 + rng.below(4); + let threads: Vec<_> = (0..clients) + .map(|client| { + let journal = journal.clone(); + let storage = storage.clone(); + let appended = appended.clone(); + let durable = durable.clone(); + let full_durable = full_durable.clone(); + let mut rng = SimRng::new(seed ^ (client + 1).wrapping_mul(0x9E37)); + std::thread::spawn(move || { + for index in 0..30u64 { + let class = if rng.below(4) == 0 { Class::Stream } else { Class::Ack }; + let body = format!("{seed}-{client}-{index}-{}", "y".repeat(rng.index(64))); + let lsn = { + // The append and the model's copy are one step, so the model never misses + // an LSN the journal took. + let mut model = appended.lock().unwrap_or_else(|e| e.into_inner()); + let Ok(lsn) = journal.append(4, class, body.as_bytes()) else { break }; + model.insert(lsn, (class, body.into_bytes())); + lsn + }; + if class == Class::Ack { + if !acked(&journal, &storage, lsn) { + break; + } + durable.fetch_max(lsn, Ordering::SeqCst); + if journal.full_durable_lsn() >= lsn { + full_durable.fetch_max(lsn, Ordering::SeqCst); + } + } + } + }) + }) + .collect(); + for thread in threads { + thread.join().map_err(|_| "a client thread panicked".to_owned())?; + } + let stats = journal.stats(); + durable.fetch_max(stats.durable_lsn, Ordering::SeqCst); + full_durable.fetch_max(stats.full_durable_lsn, Ordering::SeqCst); + drop(journal); + let restarted = storage.restart(crash); + let (_, report, seen) = open_at(&restarted, config, 0).map_err(|e| e.to_string())?; + let appended = appended.lock().unwrap_or_else(|e| e.into_inner()); + for (index, (lsn, kind, class, bytes)) in seen.iter().enumerate() { + let position = Lsn::try_from(index).unwrap_or(0) + 1; + if *lsn != position { + return Err(format!("not a prefix: record {lsn} at position {position}")); + } + match appended.get(lsn) { + Some((want_class, want)) if want_class == class && want == bytes && *kind == 4 => {} + other => return Err(format!("record {lsn} is not what was appended: {other:?}")), + } + } + let last = Lsn::try_from(seen.len()).unwrap_or(0); + let must = match crash { + Crash::Process => durable.load(Ordering::SeqCst).max(stats.written_lsn), + Crash::Os => durable.load(Ordering::SeqCst), + Crash::PowerLoss => full_durable.load(Ordering::SeqCst), + }; + if last < must { + return Err(format!("record {must} lost: recovered 1..={last} ({crash:?}, {report:?})")); + } + if !report.damage.is_empty() { + return Err(format!("a crash is not damage: {report:?}")); + } + Ok(()) +} + +fn crash_runs(crash: Crash) { + let mut failures = Vec::new(); + for seed in crash_seeds() { + if let Err(error) = crash_run(seed, crash) { + failures.push(format!("seed {seed}: {error}")); + } + } + assert!( + failures.is_empty(), + "{} {crash:?} runs failed:\n{}", + failures.len(), + failures.join("\n") + ); +} + +#[test] +fn a_process_crash_at_any_storage_operation_keeps_every_written_record() { + crash_runs(Crash::Process); +} + +#[test] +fn an_os_crash_at_any_storage_operation_keeps_every_acked_record() { + crash_runs(Crash::Os); +} + +#[test] +fn a_power_loss_keeps_every_fully_synced_record_and_a_prefix() { + crash_runs(Crash::PowerLoss); +} + +// ---------------------------------------------------------------- real file system + +#[test] +fn the_real_file_system_round_trips_and_keeps_a_torn_tail_out() { + let root = tempfile::tempdir().unwrap_or_else(|e| panic!("tempdir: {e}")); + let dir = root.path().join("journal"); + let storage: Arc = Arc::new(FsStorage); + let open_fs = |seen: &mut Vec| { + Journal::open(storage.clone(), &dir, Config::default(), 0, &mut |record| { + seen.push(record.lsn); + Ok(()) + }) + .unwrap_or_else(|e| panic!("open: {e}")) + }; + let (journal, _) = open_fs(&mut Vec::new()); + for lsn in 1..=10 { + assert_eq!(journal.append(5, Class::Ack, &payload(lsn)), Ok(lsn)); + } + journal.wait_durable(10, WAIT).unwrap_or_else(|e| panic!("{e}")); + drop(journal); + // A torn write after the last record: half a header. + append_raw(&dir.join(format::segment_name(1)), &[0x11; 7]); + let mut seen = Vec::new(); + let (journal, report) = open_fs(&mut seen); + assert_eq!(seen, (1..=10).collect::>()); + assert_eq!(report.torn_tail_bytes, 7); + assert_eq!(journal.append(5, Class::Ack, b"after"), Ok(11)); + assert_eq!(journal.seal(WAIT), Ok(11)); + drop(journal); + let mut seen = Vec::new(); + let (_, report) = open_fs(&mut seen); + assert_eq!(seen, (1..=11).collect::>()); + assert!(report.damage.is_empty(), "{report:?}"); +} + +fn append_raw(path: &Path, bytes: &[u8]) { + use std::io::Write; + let mut file = std::fs::OpenOptions::new() + .append(true) + .open(path) + .unwrap_or_else(|e| panic!("open {}: {e}", path.display())); + file.write_all(bytes).unwrap_or_else(|e| panic!("append: {e}")); +} diff --git a/cmux-tui/crates/cmux-journal/src/format.rs b/cmux-tui/crates/cmux-journal/src/format.rs new file mode 100644 index 000000000000..294bfe874fea --- /dev/null +++ b/cmux-tui/crates/cmux-journal/src/format.rs @@ -0,0 +1,236 @@ +//! The on-disk format: a segment header, then records back to back. All integers are little +//! endian. +//! +//! Segment header (`SEGMENT_HEADER_LEN` bytes): +//! +//! | bytes | field | +//! | --- | --- | +//! | 0..8 | magic `CMXJRNL\0` | +//! | 8..12 | format version (1) | +//! | 12..16 | header length (56) | +//! | 16..24 | journal id: random, the same in every segment of one journal | +//! | 24..32 | segment number: +1 for each new segment, also in the file name | +//! | 32..40 | first LSN: the LSN of this segment's first record | +//! | 40..48 | previous last LSN: the journal's last LSN before this segment. Recovery reads the earlier segments only through it, so bytes a failed segment holds after it are not part of the journal (a writer starts a new segment after any failed write or sync) | +//! | 48..52 | reserved, 0 | +//! | 52..56 | CRC-32 of bytes 0..52 | +//! +//! Record (`RECORD_HEADER_LEN` bytes, then the payload): +//! +//! | bytes | field | +//! | --- | --- | +//! | 0..4 | CRC-32 of bytes 4..20 and the payload | +//! | 4..8 | payload length | +//! | 8..16 | LSN | +//! | 16..18 | kind (the daemon's) | +//! | 18 | class: 1 ack, 2 stream | +//! | 19 | flags, 0 | +//! +//! A run of zero bytes never reads as a record: the CRC of a zero header is not zero. + +use crate::{Class, Lsn}; + +pub(crate) const MAGIC: [u8; 8] = *b"CMXJRNL\0"; +pub(crate) const FORMAT_VERSION: u32 = 1; +pub(crate) const SEGMENT_HEADER_LEN: usize = 56; +/// The fixed part of a record, before its payload. +pub const RECORD_HEADER_LEN: usize = 20; +/// The largest payload one record may carry. A length field above it is damage. +pub const MAX_PAYLOAD_LEN: usize = 64 * 1024 * 1024; + +/// One segment's header, as written and as read back. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct SegmentHeader { + pub(crate) journal_id: u64, + pub(crate) seq: u64, + pub(crate) first_lsn: Lsn, + pub(crate) prev_last: Lsn, +} + +impl SegmentHeader { + pub(crate) fn encode(&self) -> [u8; SEGMENT_HEADER_LEN] { + let mut out = [0u8; SEGMENT_HEADER_LEN]; + let mut at = Cursor::new(&mut out); + at.put(&MAGIC); + at.put(&FORMAT_VERSION.to_le_bytes()); + at.put(&u32_len(SEGMENT_HEADER_LEN).to_le_bytes()); + at.put(&self.journal_id.to_le_bytes()); + at.put(&self.seq.to_le_bytes()); + at.put(&self.first_lsn.to_le_bytes()); + at.put(&self.prev_last.to_le_bytes()); + at.put(&0u32.to_le_bytes()); + let crc = out.get(..SEGMENT_HEADER_LEN - 4).map_or(0, crc32fast::hash); + if let Some(tail) = out.get_mut(SEGMENT_HEADER_LEN - 4..) { + tail.copy_from_slice(&crc.to_le_bytes()); + } + out + } + + /// The header in `bytes`, or why it is not one. + pub(crate) fn decode(bytes: &[u8]) -> Result { + let head = bytes.get(..SEGMENT_HEADER_LEN).ok_or("short segment header")?; + let body = head.get(..SEGMENT_HEADER_LEN - 4).ok_or("short segment header")?; + if crc32fast::hash(body) != read_u32(head, SEGMENT_HEADER_LEN - 4) { + return Err("segment header checksum"); + } + if head.get(..8) != Some(MAGIC.as_slice()) { + return Err("not a journal segment"); + } + if read_u32(head, 8) != FORMAT_VERSION { + return Err("unknown journal format version"); + } + if read_u32(head, 12) != u32_len(SEGMENT_HEADER_LEN) || read_u32(head, 48) != 0 { + return Err("unknown segment header layout"); + } + Ok(Self { + journal_id: read_u64(head, 16), + seq: read_u64(head, 24), + first_lsn: read_u64(head, 32), + prev_last: read_u64(head, 40), + }) + } +} + +/// A record header read back from a segment. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct RecordHeader { + pub(crate) crc: u32, + pub(crate) len: usize, + pub(crate) lsn: Lsn, + pub(crate) kind: u16, + pub(crate) class_code: u8, + pub(crate) flags: u8, +} + +impl RecordHeader { + /// Decodes the fixed part. `None` when `bytes` is short. + pub(crate) fn decode(bytes: &[u8]) -> Option { + let head = bytes.get(..RECORD_HEADER_LEN)?; + Some(Self { + crc: read_u32(head, 0), + len: usize::try_from(read_u32(head, 4)).ok()?, + lsn: read_u64(head, 8), + kind: u16::from_le_bytes([*head.get(16)?, *head.get(17)?]), + class_code: *head.get(18)?, + flags: *head.get(19)?, + }) + } + + /// The record's class when the header and `payload` are whole and the LSN is `expected`; + /// else why not. + pub(crate) fn verify( + &self, + header: &[u8], + payload: &[u8], + expected: Lsn, + ) -> Result { + let mut crc = crc32fast::Hasher::new(); + crc.update(header.get(4..RECORD_HEADER_LEN).ok_or("short record header")?); + crc.update(payload); + if crc.finalize() != self.crc { + return Err("record checksum"); + } + if self.lsn != expected { + return Err("record out of sequence"); + } + if self.flags != 0 { + return Err("unknown record flags"); + } + Class::from_code(self.class_code).ok_or("unknown record class") + } +} + +/// Appends one encoded record to `out`. `payload_crc` is the CRC-32 state of the payload alone, +/// computed by the caller off any lock ([`payload_crc`]). +pub(crate) fn encode_record( + out: &mut Vec, + lsn: Lsn, + kind: u16, + class: Class, + payload: &[u8], + payload_crc: &crc32fast::Hasher, +) { + let mut head = [0u8; RECORD_HEADER_LEN]; + let mut at = Cursor::new(&mut head); + at.put(&0u32.to_le_bytes()); + at.put(&u32_len(payload.len()).to_le_bytes()); + at.put(&lsn.to_le_bytes()); + at.put(&kind.to_le_bytes()); + at.put(&[class.code(), 0]); + let mut crc = crc32fast::Hasher::new(); + crc.update(head.get(4..).unwrap_or_default()); + crc.combine(payload_crc); + if let Some(field) = head.get_mut(..4) { + field.copy_from_slice(&crc.finalize().to_le_bytes()); + } + out.extend_from_slice(&head); + out.extend_from_slice(payload); +} + +/// The CRC-32 state of `payload`, for [`encode_record`]. +pub(crate) fn payload_crc(payload: &[u8]) -> crc32fast::Hasher { + let mut crc = crc32fast::Hasher::new(); + crc.update(payload); + crc +} + +/// The encoded size of a record with `payload_len` bytes of payload. +pub(crate) const fn record_len(payload_len: usize) -> usize { + RECORD_HEADER_LEN.saturating_add(payload_len) +} + +/// `00000000000000000042.seg` for segment 42 (every `u64` fits in 20 digits). +pub(crate) fn segment_name(seq: u64) -> String { + format!("{seq:020}.seg") +} + +/// The segment number of a file name `segment_name` made, or `None`. +pub(crate) fn parse_segment_name(name: &str) -> Option { + let digits = name.strip_suffix(".seg")?; + if digits.len() != 20 || !digits.bytes().all(|b| b.is_ascii_digit()) { + return None; + } + digits.parse().ok() +} + +/// A length that fits the format's 32-bit fields: payloads are capped at [`MAX_PAYLOAD_LEN`] and +/// headers are constants, so the saturation never happens. +fn u32_len(len: usize) -> u32 { + u32::try_from(len).unwrap_or(u32::MAX) +} + +fn read_u32(bytes: &[u8], at: usize) -> u32 { + let mut word = [0u8; 4]; + if let Some(src) = bytes.get(at..at.saturating_add(4)) { + word.copy_from_slice(src); + } + u32::from_le_bytes(word) +} + +fn read_u64(bytes: &[u8], at: usize) -> u64 { + let mut word = [0u8; 8]; + if let Some(src) = bytes.get(at..at.saturating_add(8)) { + word.copy_from_slice(src); + } + u64::from_le_bytes(word) +} + +/// Writes fields into a fixed buffer in order. +struct Cursor<'a> { + out: &'a mut [u8], + at: usize, +} + +impl<'a> Cursor<'a> { + fn new(out: &'a mut [u8]) -> Self { + Self { out, at: 0 } + } + + fn put(&mut self, bytes: &[u8]) { + let end = self.at.saturating_add(bytes.len()); + if let Some(dst) = self.out.get_mut(self.at..end) { + dst.copy_from_slice(bytes); + } + self.at = end; + } +} diff --git a/cmux-tui/crates/cmux-journal/src/journal.rs b/cmux-tui/crates/cmux-journal/src/journal.rs new file mode 100644 index 000000000000..d32acce25324 --- /dev/null +++ b/cmux-tui/crates/cmux-journal/src/journal.rs @@ -0,0 +1,363 @@ +//! The public handle ([`Journal`]): append, durable LSNs, waits, listeners, seal. + +use std::path::Path; +use std::sync::{Arc, Condvar, Mutex, MutexGuard, PoisonError}; +use std::time::{Duration, Instant}; + +use crate::format::{self, MAX_PAYLOAD_LEN}; +use crate::recovery::{self, OpenReport}; +use crate::storage::{Storage, StorageLock}; +use crate::writer::Writer; +use crate::{Class, Config, Lsn, Record}; + +/// Called with the new durable LSN each time it moves, on the writer thread. It must return +/// quickly and never wait (it wakes the replies that wait for the change). +pub(crate) type Listener = Arc; + +/// Counters since open (`Journal::stats`). +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct JournalStats { + /// The next LSN an append takes, and the last LSN written, synced, and fully synced. + pub next_lsn: Lsn, + pub written_lsn: Lsn, + pub durable_lsn: Lsn, + pub full_durable_lsn: Lsn, + /// Batches written, records in them, and their encoded bytes. + pub batches: u64, + pub records: u64, + pub bytes: u64, + /// The most records one batch held. + pub max_batch_records: u64, + pub barrier_syncs: u64, + pub full_syncs: u64, + /// Segments the writer made since open (rotation, or after a failed sync); a segment that + /// recovery made is not counted. + pub segments_created: u64, + /// Failed writes, syncs and segment creations. After each, the writer tries again after a + /// backoff: a failed write in the same segment, anything else in a new segment. + pub failures: u64, + /// The last failure, while the writer has not recovered from it. + pub failing: Option, +} + +pub(crate) struct State { + pub(crate) pending: Vec, + pub(crate) pending_last: Lsn, + pub(crate) pending_records: u64, + pub(crate) pending_ack: bool, + /// No appends any more (a seal, or the journal is being dropped). + pub(crate) closed: bool, + /// The writer stops after one last attempt (drop). + pub(crate) stop: bool, + /// The writer thread has ended. + pub(crate) ended: bool, + pub(crate) stats: JournalStats, +} + +pub(crate) struct Shared { + pub(crate) config: Config, + pub(crate) state: Mutex, + /// Wakes the writer: records came, a seal or a stop. + pub(crate) wake: Condvar, + /// Wakes waiters: the durable LSN moved, or the writer ended. + pub(crate) durable: Condvar, + pub(crate) listeners: Mutex>, +} + +impl Shared { + pub(crate) fn lock(&self) -> MutexGuard<'_, State> { + self.state.lock().unwrap_or_else(PoisonError::into_inner) + } +} + +/// Why an append was refused. None of them waits for the disk. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AppendError { + /// The journal is sealed or being dropped. + Closed, + /// The payload is larger than [`MAX_PAYLOAD_LEN`]. + TooLarge, + /// [`Config::max_pending_bytes`] wait for the writer already (a slow or failing disk). + Backlog, +} + +impl std::fmt::Display for AppendError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { + Self::Closed => "the journal is closed", + Self::TooLarge => "the journal record is too large", + Self::Backlog => "the journal writer is behind (a slow or failing disk)", + }) + } +} + +impl std::error::Error for AppendError {} + +/// Why a wait for durability ended without it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum WaitError { + Timeout, + /// The LSN was never appended. + NotAppended, + /// The writer ended (the journal is being dropped). + WriterEnded, +} + +impl std::fmt::Display for WaitError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { + Self::Timeout => "the journal record is not durable yet", + Self::NotAppended => "no journal record has that LSN", + Self::WriterEnded => "the journal writer ended", + }) + } +} + +impl std::error::Error for WaitError {} + +/// Why a seal did not finish: the last LSN, what was durable, and the writer's failure. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SealError { + pub last_lsn: Lsn, + pub full_durable_lsn: Lsn, + pub failing: Option, +} + +impl std::fmt::Display for SealError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "journal seal: {} of {} records fully durable", + self.full_durable_lsn, self.last_lsn + )?; + if let Some(failing) = &self.failing { + write!(f, " ({failing})")?; + } + Ok(()) + } +} + +impl std::error::Error for SealError {} + +/// Why open failed. +#[derive(Debug)] +pub enum OpenError { + /// Another writer holds the journal's lock. + Locked, + Io { + what: String, + error: std::io::Error, + }, + /// The caller's replay refused a record. + Replay { + lsn: Lsn, + error: String, + }, +} + +impl std::fmt::Display for OpenError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Locked => f.write_str("another process holds the journal"), + Self::Io { what, error } => write!(f, "journal {what}: {error}"), + Self::Replay { lsn, error } => write!(f, "journal replay of record {lsn}: {error}"), + } + } +} + +impl std::error::Error for OpenError {} + +/// One append-only journal (crate docs). `Send` and `Sync`: every daemon thread appends through +/// one shared handle. +pub struct Journal { + shared: Arc, + writer: Option>, + /// Held as long as the writer may write; released after the writer is joined. + _lock: Box, +} + +impl Journal { + /// Recovers the journal in `dir` (made when missing) and starts its writer. `replay` gets + /// every record after `from` (the caller's compaction point), in LSN order, before this + /// returns; a replay error aborts the open. Every record handed to `replay` is fully synced + /// first, so it is as durable as an acknowledged one. + pub fn open( + storage: Arc, + dir: &Path, + config: Config, + from: Lsn, + replay: &mut dyn FnMut(Record<'_>) -> Result<(), String>, + ) -> Result<(Self, OpenReport), OpenError> { + let recovered = recovery::recover(storage.as_ref(), dir, from, replay)?; + let start = recovered.next_lsn.saturating_sub(1); + let shared = Arc::new(Shared { + config, + state: Mutex::new(State { + pending: Vec::new(), + pending_last: 0, + pending_records: 0, + pending_ack: false, + closed: false, + stop: false, + ended: false, + stats: JournalStats { + next_lsn: recovered.next_lsn, + written_lsn: start, + durable_lsn: start, + full_durable_lsn: start, + ..JournalStats::default() + }, + }), + wake: Condvar::new(), + durable: Condvar::new(), + listeners: Mutex::new(Vec::new()), + }); + let report = recovered.report; + let lock = recovered.lock; + let writer = Writer::new(shared.clone(), storage, dir.to_path_buf(), recovered.writer); + let handle = std::thread::Builder::new() + .name("cmux-journal-writer".into()) + .spawn(move || writer.run()) + .map_err(|error| OpenError::Io { what: "writer thread".into(), error })?; + Ok((Self { shared, writer: Some(handle), _lock: lock }, report)) + } + + /// Appends one record and returns its LSN. A memory copy: no I/O, no wait. The record is + /// durable once [`Journal::durable_lsn`] reaches the LSN. + pub fn append(&self, kind: u16, class: Class, payload: &[u8]) -> Result { + if payload.len() > MAX_PAYLOAD_LEN { + return Err(AppendError::TooLarge); + } + // The payload's checksum is computed before the lock; the lock covers the LSN and copy. + let crc = format::payload_crc(payload); + let mut state = self.shared.lock(); + if state.closed { + return Err(AppendError::Closed); + } + let size = format::record_len(payload.len()); + if !state.pending.is_empty() + && state.pending.len().saturating_add(size) > self.shared.config.max_pending_bytes + { + return Err(AppendError::Backlog); + } + let lsn = state.stats.next_lsn; + state.stats.next_lsn = lsn.saturating_add(1); + let was_empty = state.pending.is_empty(); + format::encode_record(&mut state.pending, lsn, kind, class, payload, &crc); + state.pending_last = lsn; + state.pending_records = state.pending_records.saturating_add(1); + state.pending_ack |= class == Class::Ack; + drop(state); + if was_empty { + self.shared.wake.notify_one(); + } + Ok(lsn) + } + + /// Every record up to this LSN is synced (survives a crash of any process and a kernel + /// panic). + pub fn durable_lsn(&self) -> Lsn { + self.shared.lock().stats.durable_lsn + } + + /// Every record up to this LSN is fully synced (survives a power loss). + pub fn full_durable_lsn(&self) -> Lsn { + self.shared.lock().stats.full_durable_lsn + } + + /// The LSN of the last record appended (0 for none). + pub fn last_lsn(&self) -> Lsn { + self.shared.lock().stats.next_lsn.saturating_sub(1) + } + + pub fn stats(&self) -> JournalStats { + self.shared.lock().stats.clone() + } + + /// Calls `listener` with the new durable LSN each time it moves, on the writer thread. It + /// must return quickly and never wait: it only releases the replies that wait for the LSN. + pub fn on_durable(&self, listener: impl Fn(Lsn) + Send + Sync + 'static) { + self.shared + .listeners + .lock() + .unwrap_or_else(PoisonError::into_inner) + .push(Arc::new(listener)); + } + + /// Blocks until record `lsn` is durable. For tests and tools: a daemon's request path + /// never blocks on the disk; it holds its reply until a listener says the LSN is durable. + pub fn wait_durable(&self, lsn: Lsn, timeout: Duration) -> Result<(), WaitError> { + let deadline = Instant::now().checked_add(timeout); + let mut state = self.shared.lock(); + loop { + if state.stats.durable_lsn >= lsn { + return Ok(()); + } + if lsn >= state.stats.next_lsn { + return Err(WaitError::NotAppended); + } + if state.ended { + return Err(WaitError::WriterEnded); + } + let now = Instant::now(); + let left = + deadline.map_or(Duration::MAX, |deadline| deadline.saturating_duration_since(now)); + if left.is_zero() { + return Err(WaitError::Timeout); + } + state = self + .shared + .durable + .wait_timeout(state, left) + .unwrap_or_else(PoisonError::into_inner) + .0; + } + } + + /// Stops appends and waits until every record is fully synced (a clean shutdown and a + /// handoff: the next process reads all of it). Returns the last LSN. + pub fn seal(&self, timeout: Duration) -> Result { + let deadline = Instant::now().checked_add(timeout); + let mut state = self.shared.lock(); + state.closed = true; + self.shared.wake.notify_one(); + loop { + let last = state.stats.next_lsn.saturating_sub(1); + if state.stats.full_durable_lsn >= last { + return Ok(last); + } + let now = Instant::now(); + let left = + deadline.map_or(Duration::MAX, |deadline| deadline.saturating_duration_since(now)); + if left.is_zero() || state.ended { + return Err(SealError { + last_lsn: last, + full_durable_lsn: state.stats.full_durable_lsn, + failing: state.stats.failing.clone(), + }); + } + state = self + .shared + .durable + .wait_timeout(state, left) + .unwrap_or_else(PoisonError::into_inner) + .0; + } + } +} + +impl Drop for Journal { + /// Closes the journal: the writer makes one last attempt to write and fully sync what is + /// pending, then ends. Call [`Journal::seal`] first for a shutdown that must keep everything. + fn drop(&mut self) { + { + let mut state = self.shared.lock(); + state.closed = true; + state.stop = true; + } + self.shared.wake.notify_one(); + if let Some(handle) = self.writer.take() { + let _ = handle.join(); + } + } +} diff --git a/cmux-tui/crates/cmux-journal/src/lib.rs b/cmux-tui/crates/cmux-journal/src/lib.rs new file mode 100644 index 000000000000..a6ce20bb4f80 --- /dev/null +++ b/cmux-tui/crates/cmux-journal/src/lib.rs @@ -0,0 +1,138 @@ +//! One append-only journal per daemon, with group commit and ack after durable +//! (R-DURABILITY, `handoff/rewrites/R-DURABILITY/DESIGN.md`). +//! +//! A daemon appends one record per change ([`Journal::append`]: a memory copy, no I/O, no wait). +//! One writer thread takes every pending record as one batch, writes the batch with one `write`, +//! and, when the batch holds an [`Class::Ack`] record, syncs it once. Then it moves +//! [`Journal::durable_lsn`] and calls the daemon's durable listeners, which release the replies +//! that wait for those records. So a burst of N changes costs one write and one sync per batch, +//! not N, and no reply leaves before its change is on disk. +//! +//! The contract (the rules every user of this crate keeps): +//! +//! - **Ack after durable.** A reply, event or read that tells a client about a change goes out +//! only when `durable_lsn >= ` the change's LSN. +//! - **One change, one record.** Recovery applies a whole record or none of it. +//! - **Prefix order.** Recovery restores the records in LSN order, and a lost tail is a suffix +//! (only possible after an OS crash or a power loss), never a hole. +//! - **Streams.** A [`Class::Stream`] record (terminal output, streamed agent tokens) is written in +//! the next batch, so a kill of every process keeps it, and synced with the next acked batch, or +//! [`Config::stream_sync_delay`] after its write when no acked batch comes. +//! +//! Sync levels ([`SyncLevel`]): an acked batch is synced with `F_BARRIERFSYNC` on macOS +//! (`fdatasync` on Linux, `FlushFileBuffers` on Windows), so an acknowledged change survives a +//! kill of any process, a daemon crash and a kernel panic. When the last full sync is older than +//! [`Config::full_sync_interval`] (1 s), the batch is synced with `F_FULLFSYNC` instead, so a power +//! loss on macOS takes back at most that interval of acked batches, always a suffix. Rotation, +//! seal and recovery always sync fully. +//! +//! Files (the journal directory): `LOCK` (an exclusive lock held by the one writer), the segments +//! (`.seg`: a header, then records back to back, see `format.rs`), and +//! `quarantine/` for segments recovery could not read past damage that is not a torn tail. A +//! segment is only ever appended to; nothing here deletes or shortens one that holds a record. +//! +//! Recovery ([`Journal::open`]) is the one start path for a clean exit, a crash, a kill -9 and an +//! OS crash: it syncs the last segment, reads the segments in LSN order, checks each record's +//! checksum and LSN, and hands every record after the caller's compaction point to the caller's +//! replay. A torn tail of the last segment (a write that never completed, so never acked) stays in +//! the file, and the journal goes on in a new segment whose header says where the old one ends. +//! +//! [`Storage`] is the file system seam. [`FsStorage`] is the real one. With the `sim` feature, +//! `SimStorage` keeps the synced and unsynced bytes of each file apart and can crash at any +//! storage operation, then lose what a process crash, an OS crash or a power loss would lose. + +mod format; +mod journal; +mod recovery; +mod storage; +mod writer; + +#[cfg(any(test, feature = "sim"))] +mod sim; + +pub use format::{MAX_PAYLOAD_LEN, RECORD_HEADER_LEN}; +pub use journal::{AppendError, Journal, JournalStats, OpenError, SealError, WaitError}; +pub use recovery::{Damage, OpenReport}; +#[cfg(any(test, feature = "sim"))] +pub use sim::{Crash, SimRng, SimStorage}; +pub use storage::{FsStorage, OpenMode, Storage, StorageFile, StorageLock, SyncLevel}; + +use std::time::Duration; + +/// A record's position in the journal: 1 for the first record ever, +1 for each record after +/// it. 0 means "before every record". +pub type Lsn = u64; + +/// What a record's durability serves. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub enum Class { + /// A change some client is told about: its batch is synced before the writer reports it + /// durable. + Ack, + /// Stream bytes a client already has (terminal output, streamed tokens): written in the next + /// batch, synced with the next acked batch or after [`Config::stream_sync_delay`]. + Stream, +} + +impl Class { + pub(crate) const fn code(self) -> u8 { + match self { + Self::Ack => 1, + Self::Stream => 2, + } + } + + pub(crate) const fn from_code(code: u8) -> Option { + match code { + 1 => Some(Self::Ack), + 2 => Some(Self::Stream), + _ => None, + } + } +} + +/// One record, as recovery hands it to the caller's replay. The payload is borrowed from a +/// buffer that the next record reuses. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Record<'a> { + pub lsn: Lsn, + /// The daemon's record kind (its own payload schema; this crate never reads the payload). + pub kind: u16, + pub class: Class, + pub payload: &'a [u8], +} + +/// Writer and recovery limits. +#[derive(Clone, Debug)] +pub struct Config { + /// A segment past this size is closed (fully synced) and the next one starts. A segment can + /// pass it by one batch. + pub segment_bytes: u64, + /// The most encoded bytes that may wait for the writer. Past it, [`Journal::append`] fails with + /// [`AppendError::Backlog`]: the caller fails its request (an `Ack` change) or asks for a + /// covering snapshot (a `Stream`), and never waits for the disk. + pub max_pending_bytes: usize, + /// The longest time between two full syncs while acked batches come (the power-loss window). + pub full_sync_interval: Duration, + /// How long written `Stream` records may stay unsynced when no acked batch comes. + pub stream_sync_delay: Duration, + /// After a failed write or sync, the first and the longest wait before the writer tries again. + pub retry_min: Duration, + pub retry_max: Duration, +} + +impl Default for Config { + fn default() -> Self { + Self { + segment_bytes: 64 * 1024 * 1024, + max_pending_bytes: 64 * 1024 * 1024, + full_sync_interval: Duration::from_secs(1), + stream_sync_delay: Duration::from_secs(1), + retry_min: Duration::from_millis(100), + retry_max: Duration::from_secs(5), + } + } +} + +#[cfg(test)] +mod cmux_journal_tests; diff --git a/cmux-tui/crates/cmux-journal/src/recovery.rs b/cmux-tui/crates/cmux-journal/src/recovery.rs new file mode 100644 index 000000000000..10fd29bd5fa5 --- /dev/null +++ b/cmux-tui/crates/cmux-journal/src/recovery.rs @@ -0,0 +1,475 @@ +//! Recovery: the one start path, for a clean exit, a crash, a kill -9 and an OS crash. +//! +//! 1. Take the lock; list the segments in number order; read and check each header (its +//! journal id, number, and LSN range against the segment before it). +//! 2. Fully sync the last segment: the last process may have died after a write and before its +//! sync. So every record handed to the replay is durable before the daemon serves. +//! 3. Read the records after the caller's compaction point (`from`) and hand them to the replay. +//! An earlier segment is read only through the next segment's "previous last LSN": bytes after +//! it (left by a write or sync that failed, after which the writer started a new segment) are +//! not part of the journal. A segment whose records all lie at or before `from` is not read. +//! 4. The last segment is read up to its first record that does not check out: a torn write that +//! never completed, so it was never acknowledged. Those bytes stay in the file; the journal +//! goes on in a new segment whose header names the last good LSN. +//! 5. Damage that is not a torn tail (a bad header, or records missing before the next segment) +//! keeps the valid prefix. That segment's successors move to `quarantine/` (never deleted), the +//! report names them and the LSNs they may hold, and new records take LSNs after all of them, +//! so no LSN is ever used twice. +//! +//! Recovery writes nothing but a sync and, when the journal does not go on in its last segment, a +//! new segment. It never truncates or deletes a segment that holds a record. + +use std::path::Path; + +use crate::format::{ + self, MAX_PAYLOAD_LEN, RECORD_HEADER_LEN, RecordHeader, SEGMENT_HEADER_LEN, SegmentHeader, +}; +use crate::journal::OpenError; +use crate::storage::{OpenMode, Storage, StorageFile, StorageLock, SyncLevel}; +use crate::{Lsn, Record}; + +pub(crate) const LOCK_NAME: &str = "LOCK"; +pub(crate) const QUARANTINE_DIR: &str = "quarantine"; +/// Segment bytes read at once. +const READ_CHUNK: usize = 1024 * 1024; + +/// What recovery found ([`crate::Journal::open`]). +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct OpenReport { + /// Segments the journal holds after recovery, a new one included. + pub segments: usize, + /// The last LSN the journal holds or skipped over, and the LSN the next append takes. + pub last_lsn: Lsn, + pub next_lsn: Lsn, + /// Records handed to the replay. + pub replayed: u64, + /// Bytes at the end of the last segment that hold no whole record: a write that never + /// completed (never acknowledged). They stay in the file; the journal goes on after them in a + /// new segment. + pub torn_tail_bytes: u64, + pub damage: Vec, +} + +/// What recovery could not read, beyond a torn tail. The daemon reports it to the person; moved +/// segments stay in `quarantine/` for a salvage. +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum Damage { + /// `segment` could not be read past `offset` (media damage or a bug, never a crash). The + /// segments in `moved` (every later one, and this one too when its header was bad) are in + /// `quarantine/` now. The records after `lost_after` through `lost_through` are not in the + /// journal; new records take LSNs after `lost_through`. + Corrupt { + segment: String, + offset: u64, + reason: String, + moved: Vec, + lost_after: Lsn, + lost_through: Lsn, + }, + /// The records after `after` through `through` are not in the journal: the caller's + /// compaction point is older than the oldest record kept. + Missing { after: Lsn, through: Lsn }, + /// The caller's compaction point is past the journal's end: the caller has records the + /// journal lost. New records take LSNs after the compaction point, so none is used twice. + Ahead { journal_end: Lsn, from: Lsn }, +} + +/// The segment the writer appends to first. +pub(crate) struct Tail { + pub(crate) file: Box, + pub(crate) seq: u64, + pub(crate) size: u64, +} + +pub(crate) struct WriterStart { + pub(crate) journal_id: u64, + pub(crate) tail: Tail, +} + +pub(crate) struct Recovered { + pub(crate) lock: Box, + pub(crate) next_lsn: Lsn, + pub(crate) report: OpenReport, + pub(crate) writer: WriterStart, +} + +struct Segment { + seq: u64, + name: String, + file: Box, + size: u64, + header: SegmentHeader, +} + +/// Where the valid journal ends early (step 5). +struct Bad { + /// The first listed segment to move to quarantine. + from_seq: u64, + segment: String, + offset: u64, + reason: &'static str, + /// The highest LSN the journal is known to have held before the damage. + lost_through: Lsn, +} + +fn io(what: &'static str) -> impl FnOnce(std::io::Error) -> OpenError { + move |error| OpenError::Io { what: what.into(), error } +} + +pub(crate) fn len64(n: usize) -> u64 { + u64::try_from(n).unwrap_or(u64::MAX) +} + +pub(crate) fn recover( + storage: &dyn Storage, + dir: &Path, + from: Lsn, + replay: &mut dyn FnMut(Record<'_>) -> Result<(), String>, +) -> Result { + storage.create_dir_all(dir).map_err(io("make the folder"))?; + let lock = + storage.try_lock(&dir.join(LOCK_NAME)).map_err(io("lock"))?.ok_or(OpenError::Locked)?; + let mut listed: Vec<(u64, String)> = storage + .list(dir) + .map_err(io("list the folder"))? + .into_iter() + .filter_map(|name| Some((format::parse_segment_name(&name)?, name))) + .collect(); + listed.sort(); + let max_seq = listed.last().map_or(0, |(seq, _)| *seq); + + // 1 and 2. + let Headers { mut segments, reuse, mut bad } = read_headers(storage, dir, &listed)?; + if let Some(last) = segments.last_mut() { + let mut file = storage + .open(&dir.join(&last.name), OpenMode::Write) + .map_err(io("open the last segment"))?; + file.sync(SyncLevel::Full).map_err(io("sync the last segment"))?; + last.file = file; + } + + // 3 and 4. + let mut report = OpenReport::default(); + let mut journal_end = 0; + let mut tail_cut = None; + let mut index = 0; + while let Some(segment) = segments.get(index) { + let next = segments.get(index.saturating_add(1)); + let trusted_end = next.map(|next| next.header.prev_last); + if trusted_end.is_some_and(|end| end <= from) { + index = index.saturating_add(1); + continue; + } + let read = read_records(segment, trusted_end, from, replay, &mut report)?; + journal_end = read.end; + if let (Some(end), Some(next)) = (trusted_end, next) + && read.end < end + { + bad = Some(Bad { + from_seq: next.seq, + segment: segment.name.clone(), + offset: read.offset, + reason: "records are missing before the next segment", + lost_through: end, + }); + segments.truncate(index.saturating_add(1)); + break; + } + if trusted_end.is_none() && read.offset < segment.size { + tail_cut = Some(segment.size.saturating_sub(read.offset)); + } + index = index.saturating_add(1); + } + + // 5. + let damaged = bad.is_some(); + let mut lost_through = journal_end; + if let Some(bad) = bad { + let moved = quarantine(storage, dir, &listed, bad.from_seq)?; + lost_through = lost_through.max(bad.lost_through).max(moved.highest); + report.damage.push(Damage::Corrupt { + segment: bad.segment, + offset: bad.offset, + reason: bad.reason.to_owned(), + moved: moved.names, + lost_after: journal_end, + lost_through, + }); + } + if let Some(first) = segments.first() + && from < first.header.prev_last + { + report.damage.push(Damage::Missing { after: from, through: first.header.prev_last }); + } + if from > journal_end { + report.damage.push(Damage::Ahead { journal_end, from }); + } + let next_lsn = journal_end.max(lost_through).max(from).saturating_add(1); + + // The writer goes on in the last segment only when it ends in a whole record, nothing is + // damaged and no LSN is skipped; else a new segment starts now, before any append. + let journal_id = segments.first().map_or_else(new_journal_id, |first| first.header.journal_id); + let go_on = !damaged + && reuse.is_none() + && tail_cut.is_none() + && next_lsn == journal_end.saturating_add(1); + let kept = segments.len(); + let tail = match segments.pop() { + Some(last) if go_on => Tail { file: last.file, seq: last.seq, size: last.size }, + _ => { + let header = SegmentHeader { + journal_id, + seq: reuse.unwrap_or_else(|| max_seq.saturating_add(1)), + first_lsn: next_lsn, + prev_last: journal_end, + }; + let file = make_segment(storage, dir, &header).map_err(io("make a segment"))?; + report.segments = 1; + Tail { file, seq: header.seq, size: len64(SEGMENT_HEADER_LEN) } + } + }; + report.segments = report.segments.saturating_add(kept); + report.last_lsn = next_lsn.saturating_sub(1); + report.next_lsn = next_lsn; + report.torn_tail_bytes = tail_cut.unwrap_or(0); + Ok(Recovered { lock, next_lsn, report, writer: WriterStart { journal_id, tail } }) +} + +/// Makes segment `header.seq` (a file that holds a torn header is made again): its header, +/// fully synced, and its folder entry, synced. +pub(crate) fn make_segment( + storage: &dyn Storage, + dir: &Path, + header: &SegmentHeader, +) -> std::io::Result> { + let mut file = storage.open(&dir.join(format::segment_name(header.seq)), OpenMode::Create)?; + file.append(&header.encode())?; + file.sync(SyncLevel::Full)?; + storage.sync_dir(dir)?; + Ok(file) +} + +struct Headers { + segments: Vec, + /// The last segment's number when its header never got whole (its creation did not + /// complete): the writer makes it again. + reuse: Option, + bad: Option, +} + +fn read_headers( + storage: &dyn Storage, + dir: &Path, + listed: &[(u64, String)], +) -> Result { + let mut segments: Vec = Vec::new(); + let mut reuse = None; + let mut bad = None; + for (index, (seq, name)) in listed.iter().enumerate() { + let file = storage.open(&dir.join(name), OpenMode::Read).map_err(io("open a segment"))?; + let size = file.size().map_err(io("read a segment's size"))?; + let last = index.saturating_add(1) == listed.len(); + match read_header(file.as_ref(), size)?.and_then(|header| follows(&segments, header, *seq)) + { + Ok(header) => { + segments.push(Segment { seq: *seq, name: name.clone(), file, size, header }); + } + Err(Unreadable::Header(_)) if last && size <= len64(SEGMENT_HEADER_LEN) => { + reuse = Some(*seq); + } + Err(reason) => { + let lost_through = + segments.last().map_or(0, |prev| prev.header.first_lsn.saturating_sub(1)); + bad = Some(Bad { + from_seq: *seq, + segment: name.clone(), + offset: 0, + reason: reason.text(), + lost_through, + }); + break; + } + } + } + Ok(Headers { segments, reuse, bad }) +} + +/// Why a header does not start the next segment of this journal. +enum Unreadable { + Header(&'static str), + Chain(&'static str), +} + +impl Unreadable { + fn text(&self) -> &'static str { + match self { + Self::Header(text) | Self::Chain(text) => text, + } + } +} + +fn read_header( + file: &dyn StorageFile, + size: u64, +) -> Result, OpenError> { + if size < len64(SEGMENT_HEADER_LEN) { + return Ok(Err(Unreadable::Header("short segment header"))); + } + let mut bytes = [0u8; SEGMENT_HEADER_LEN]; + file.read_at(0, &mut bytes).map_err(io("read a segment header"))?; + Ok(SegmentHeader::decode(&bytes).map_err(Unreadable::Header)) +} + +/// `header` when it is segment `seq` of the journal `segments` holds, right after its last one. +fn follows( + segments: &[Segment], + header: SegmentHeader, + seq: u64, +) -> Result { + if header.seq != seq { + return Err(Unreadable::Chain("the segment number does not match its file name")); + } + if header.first_lsn <= header.prev_last { + return Err(Unreadable::Chain("the segment's LSN range is empty")); + } + if let Some(first) = segments.first() + && first.header.journal_id != header.journal_id + { + return Err(Unreadable::Chain("the segment belongs to another journal")); + } + if let Some(prev) = segments.last() + && header.prev_last < prev.header.first_lsn.saturating_sub(1) + { + return Err(Unreadable::Chain("the segment does not follow the one before it")); + } + Ok(header) +} + +/// How far one segment's records read: the file offset after the last good record, and the +/// LSN of that record (the LSN before the segment's first when it has none). +struct Read { + offset: u64, + end: Lsn, +} + +fn read_records( + segment: &Segment, + trusted_end: Option, + from: Lsn, + replay: &mut dyn FnMut(Record<'_>) -> Result<(), String>, + report: &mut OpenReport, +) -> Result { + let mut reader = + Reader { file: segment.file.as_ref(), size: segment.size, buf: Vec::new(), at: 0 }; + let mut offset = len64(SEGMENT_HEADER_LEN); + let mut expected = segment.header.first_lsn; + loop { + if trusted_end.is_some_and(|end| expected > end) { + break; + } + let Some(head) = reader.bytes(offset, RECORD_HEADER_LEN)? else { break }; + let Ok(head) = <[u8; RECORD_HEADER_LEN]>::try_from(head) else { break }; + let Some(header) = RecordHeader::decode(&head) else { break }; + if header.len > MAX_PAYLOAD_LEN { + break; + } + let payload_at = offset.saturating_add(len64(RECORD_HEADER_LEN)); + let Some(payload) = reader.bytes(payload_at, header.len)? else { break }; + let Ok(class) = header.verify(&head, payload, expected) else { break }; + if expected > from { + replay(Record { lsn: expected, kind: header.kind, class, payload }) + .map_err(|error| OpenError::Replay { lsn: expected, error })?; + report.replayed = report.replayed.saturating_add(1); + } + offset = payload_at.saturating_add(len64(header.len)); + expected = expected.saturating_add(1); + } + Ok(Read { offset, end: expected.saturating_sub(1) }) +} + +/// Reads a segment in chunks. +struct Reader<'a> { + file: &'a dyn StorageFile, + size: u64, + buf: Vec, + /// The file offset of `buf`'s first byte. + at: u64, +} + +impl Reader<'_> { + /// `len` bytes at `offset`, or `None` when the file ends first. + fn bytes(&mut self, offset: u64, len: usize) -> Result, OpenError> { + let Some(end) = offset.checked_add(len64(len)) else { return Ok(None) }; + if end > self.size { + return Ok(None); + } + let buffered_end = self.at.saturating_add(len64(self.buf.len())); + if offset < self.at || end > buffered_end { + let left = usize::try_from(self.size.saturating_sub(offset)).unwrap_or(usize::MAX); + self.buf.resize(len.max(READ_CHUNK).min(left), 0); + self.file.read_at(offset, &mut self.buf).map_err(io("read a segment"))?; + self.at = offset; + } + let start = usize::try_from(offset.saturating_sub(self.at)).unwrap_or(usize::MAX); + Ok(self.buf.get(start..start.saturating_add(len))) + } +} + +/// The segments moved to quarantine, and the highest LSN they hold. +struct Moved { + names: Vec, + highest: Lsn, +} + +/// Moves every listed segment from number `from_seq` on into `quarantine/`, under a name no +/// earlier recovery used. +fn quarantine( + storage: &dyn Storage, + dir: &Path, + listed: &[(u64, String)], + from_seq: u64, +) -> Result { + let target = dir.join(QUARANTINE_DIR); + storage.create_dir_all(&target).map_err(io("make the quarantine folder"))?; + let taken = storage.list(&target).map_err(io("list the quarantine folder"))?; + let mut moved = Moved { names: Vec::new(), highest: 0 }; + for (_, name) in listed.iter().filter(|(seq, _)| *seq >= from_seq) { + let path = dir.join(name); + moved.highest = moved.highest.max(highest_lsn(storage, &path)?); + let mut attempt = 0u32; + let new_name = loop { + let candidate = format!("{name}.{attempt}"); + if !taken.contains(&candidate) { + break candidate; + } + attempt = attempt.saturating_add(1); + }; + storage.rename(&path, &target.join(&new_name)).map_err(io("move a damaged segment"))?; + moved.names.push(new_name); + } + storage.sync_dir(&target).map_err(io("sync the quarantine folder"))?; + storage.sync_dir(dir).map_err(io("sync the folder"))?; + Ok(moved) +} + +/// The highest LSN a segment's readable records and header name (0 when its header is bad). +fn highest_lsn(storage: &dyn Storage, path: &Path) -> Result { + let file = storage.open(path, OpenMode::Read).map_err(io("open a damaged segment"))?; + let size = file.size().map_err(io("read a damaged segment's size"))?; + let Ok(header) = read_header(file.as_ref(), size)? else { return Ok(0) }; + let segment = Segment { seq: header.seq, name: String::new(), file, size, header }; + let read = read_records(&segment, None, Lsn::MAX, &mut |_| Ok(()), &mut OpenReport::default())?; + Ok(read.end.max(header.prev_last)) +} + +/// A random journal id (never 0), from a per-process random hasher key and the clock. +fn new_journal_id() -> u64 { + use std::hash::{BuildHasher, Hasher}; + let mut hasher = std::collections::hash_map::RandomState::new().build_hasher(); + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |elapsed| elapsed.as_nanos()); + hasher.write_u128(now); + hasher.write_u32(std::process::id()); + hasher.finish() | 1 +} diff --git a/cmux-tui/crates/cmux-journal/src/sim.rs b/cmux-tui/crates/cmux-journal/src/sim.rs new file mode 100644 index 000000000000..099bb8bb5302 --- /dev/null +++ b/cmux-tui/crates/cmux-journal/src/sim.rs @@ -0,0 +1,576 @@ +//! A simulated file system for crash tests (`sim` feature; always in this crate's tests). +//! +//! Each file keeps its current bytes (what a read sees, and what a process crash keeps) apart +//! from the bytes of each sync. Each directory keeps its current entries apart from its synced +//! entries. [`SimStorage::crash_after`] makes the n-th storage operation from now the crash +//! point: an append there writes a random prefix of its bytes (a torn write), any other +//! operation does nothing, and every later operation fails. [`SimStorage::restart`] then builds +//! the storage that the next process sees: +//! +//! - [`Crash::Process`] (kill -9 of any or all processes): every written byte and entry stays. +//! - [`Crash::Os`] (a kernel panic): each file keeps its content at its last sync, plus a random +//! prefix of what it got since (some of it zeroed or flipped: pages written back out of +//! order); each directory keeps its synced entries or, at random, its current ones. +//! - [`Crash::PowerLoss`]: as `Os`, but a barrier sync is power-safe only once a later full sync +//! ran (on any file: a drive cache flush covers every earlier write), so the content of each +//! file goes back to a random common cut between the last full sync and the crash. +//! +//! A run is deterministic for one seed and one sequence of operations. + +use std::collections::{BTreeMap, BTreeSet}; +use std::io; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex, PoisonError}; +use std::time::Duration; + +use crate::storage::{OpenMode, Storage, StorageFile, StorageLock, SyncLevel}; + +/// What a simulated crash takes away ([`SimStorage::restart`]). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Crash { + Process, + Os, + PowerLoss, +} + +/// SplitMix64: small, seeded, the same on every host. +#[derive(Clone, Debug)] +pub struct SimRng(u64); + +impl SimRng { + pub fn new(seed: u64) -> Self { + Self(seed) + } + + pub fn next_u64(&mut self) -> u64 { + self.0 = self.0.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = self.0; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) + } + + /// A value in `0..n` (0 when `n` is 0). + pub fn below(&mut self, n: u64) -> u64 { + self.next_u64().checked_rem(n).unwrap_or(0) + } + + /// `below` for lengths. + pub fn index(&mut self, n: usize) -> usize { + usize::try_from(self.below(u64::try_from(n).unwrap_or(u64::MAX))).unwrap_or(0) + } +} + +/// The simulated file system ([module docs](self)). Clones share one file system. +#[derive(Clone)] +pub struct SimStorage { + fs: Arc>, +} + +struct SimFile { + data: Vec, + /// (global sync number, content) at each sync, oldest first. + synced: Vec<(u64, Vec)>, +} + +struct Fs { + rng: SimRng, + files: BTreeMap, + next_inode: u64, + /// Current and synced directory entries: path to inode. + names: BTreeMap, + durable_names: BTreeMap, + dirs: BTreeSet, + locks: BTreeSet, + /// Storage operations so far, and the one that crashes. + ops: u64, + crash_at: Option, + /// One operation that fails with this error, once. + fail_at: Option<(u64, io::ErrorKind)>, + crashed: bool, + /// Syncs so far (file and directory), and the number of the last full one. + syncs: u64, + last_full: u64, + /// Each sync sleeps this long (a slow disk, so batches form). + sync_delay: Duration, + /// Read operations per path. + reads: BTreeMap, +} + +enum Step { + Run, + Crash, +} + +impl Fs { + /// Counts one operation and says whether it runs, crashes or fails. + fn begin(&mut self) -> io::Result { + if self.crashed { + return Err(io::Error::other("simulated crash: the process is gone")); + } + self.ops = self.ops.saturating_add(1); + if self.crash_at == Some(self.ops) { + self.crashed = true; + return Ok(Step::Crash); + } + if let Some((at, kind)) = self.fail_at + && at == self.ops + { + self.fail_at = None; + return Err(io::Error::new(kind, "simulated I/O failure")); + } + Ok(Step::Run) + } + + fn file(&mut self, inode: u64) -> io::Result<&mut SimFile> { + self.files.get_mut(&inode).ok_or_else(|| io::Error::other("simulated file is gone")) + } + + fn sync_number(&mut self, level: SyncLevel) -> u64 { + self.syncs = self.syncs.saturating_add(1); + if level == SyncLevel::Full { + self.last_full = self.syncs; + } + self.syncs + } +} + +fn crashed() -> io::Error { + io::Error::other("simulated crash") +} + +fn lock(fs: &Mutex) -> std::sync::MutexGuard<'_, Fs> { + fs.lock().unwrap_or_else(PoisonError::into_inner) +} + +impl SimStorage { + pub fn new(seed: u64) -> Self { + Self { + fs: Arc::new(Mutex::new(Fs { + rng: SimRng::new(seed), + files: BTreeMap::new(), + next_inode: 1, + names: BTreeMap::new(), + durable_names: BTreeMap::new(), + dirs: BTreeSet::new(), + locks: BTreeSet::new(), + ops: 0, + crash_at: None, + fail_at: None, + crashed: false, + syncs: 0, + last_full: 0, + sync_delay: Duration::ZERO, + reads: BTreeMap::new(), + })), + } + } + + /// The `n`-th storage operation from now (1 = the next one) is the crash point. + pub fn crash_after(&self, n: u64) { + let mut fs = lock(&self.fs); + fs.crash_at = Some(fs.ops.saturating_add(n.max(1))); + } + + /// The `n`-th storage operation from now fails once with `kind` (ENOSPC, EIO). + pub fn fail_after(&self, n: u64, kind: io::ErrorKind) { + let mut fs = lock(&self.fs); + fs.fail_at = Some((fs.ops.saturating_add(n.max(1)), kind)); + } + + /// Each later sync sleeps `delay` first. + pub fn set_sync_delay(&self, delay: Duration) { + lock(&self.fs).sync_delay = delay; + } + + pub fn is_crashed(&self) -> bool { + lock(&self.fs).crashed + } + + /// Storage operations so far. + pub fn ops(&self) -> u64 { + lock(&self.fs).ops + } + + /// Read operations on `path` so far. + pub fn reads(&self, path: &Path) -> u64 { + lock(&self.fs).reads.get(path).copied().unwrap_or(0) + } + + /// The current bytes of `path`, if it exists. + pub fn contents(&self, path: &Path) -> Option> { + let fs = lock(&self.fs); + let inode = fs.names.get(path)?; + fs.files.get(inode).map(|file| file.data.clone()) + } + + /// The names in `dir`, sorted. + pub fn names_in(&self, dir: &Path) -> Vec { + let fs = lock(&self.fs); + fs.names + .keys() + .filter(|path| path.parent() == Some(dir)) + .filter_map(|path| path.file_name()?.to_str().map(str::to_owned)) + .collect() + } + + /// Flips one durable and current byte of `path` (media damage), outside any crash. + pub fn corrupt(&self, path: &Path, offset: usize) { + let mut fs = lock(&self.fs); + let Some(inode) = fs.names.get(path).copied() else { return }; + if let Some(file) = fs.files.get_mut(&inode) { + if let Some(byte) = file.data.get_mut(offset) { + *byte ^= 0x5A; + } + for (_, content) in &mut file.synced { + if let Some(byte) = content.get_mut(offset) { + *byte ^= 0x5A; + } + } + } + } + + /// Removes `path` and makes the removal durable (a person deleted the file). + pub fn remove(&self, path: &Path) { + let mut fs = lock(&self.fs); + if let Some(inode) = fs.names.remove(path) { + fs.durable_names.remove(path); + fs.files.remove(&inode); + } + } + + /// The storage the next process sees after `crash` ([module docs](self)). This one is + /// crashed from now on: every later operation fails. + pub fn restart(&self, crash: Crash) -> Self { + lock(&self.fs).crashed = true; + self.snapshot(crash) + } + + /// The storage a process would see after `crash` now, while this one goes on. + pub fn snapshot(&self, crash: Crash) -> Self { + let mut old = lock(&self.fs); + let seed = old.rng.next_u64(); + let mut rng = SimRng::new(seed); + let cut = match crash { + Crash::Process | Crash::Os => None, + Crash::PowerLoss => { + let span = old.syncs.saturating_sub(old.last_full); + Some(old.last_full.saturating_add(rng.below(span.saturating_add(1)))) + } + }; + let names = match crash { + Crash::Process => old.names.clone(), + Crash::Os | Crash::PowerLoss => surviving_names(&old, &mut rng), + }; + let mut files = BTreeMap::new(); + for inode in names.values() { + if files.contains_key(inode) { + continue; + } + let Some(file) = old.files.get(inode) else { continue }; + let data = match crash { + Crash::Process => file.data.clone(), + Crash::Os | Crash::PowerLoss => surviving_bytes(file, cut, &mut rng), + }; + files.insert(*inode, SimFile { synced: vec![(0, data.clone())], data }); + } + let next = Fs { + rng, + files, + next_inode: old.next_inode, + durable_names: names.clone(), + names, + dirs: old.dirs.clone(), + locks: BTreeSet::new(), + ops: 0, + crash_at: None, + fail_at: None, + crashed: false, + syncs: 0, + last_full: 0, + sync_delay: old.sync_delay, + reads: BTreeMap::new(), + }; + Self { fs: Arc::new(Mutex::new(next)) } + } +} + +/// Each directory's synced entries, or at random its current ones (its rename or create may +/// have reached the disk). +fn surviving_names(fs: &Fs, rng: &mut SimRng) -> BTreeMap { + let mut parents = BTreeSet::new(); + for path in fs.names.keys().chain(fs.durable_names.keys()) { + if let Some(parent) = path.parent() { + parents.insert(parent.to_path_buf()); + } + } + let mut out = BTreeMap::new(); + for parent in parents { + let in_dir = |path: &&PathBuf| path.parent() == Some(parent.as_path()); + let source = if rng.below(2) == 0 { &fs.durable_names } else { &fs.names }; + for (path, inode) in source.iter().filter(|(path, _)| in_dir(path)) { + out.insert(path.clone(), *inode); + } + } + out +} + +/// The content at the last sync (at or before `cut`, for a power loss), plus a random prefix of +/// the bytes appended since, of which a random tail may be zeroed or one byte flipped. +fn surviving_bytes(file: &SimFile, cut: Option, rng: &mut SimRng) -> Vec { + let base = file + .synced + .iter() + .rev() + .find(|(number, _)| cut.is_none_or(|cut| *number <= cut)) + .map(|(_, content)| content.clone()) + .unwrap_or_default(); + let Some(extra) = file.data.strip_prefix(base.as_slice()) else { + // A truncation since the sync: it may not have reached the disk. + return base; + }; + let keep = rng.index(extra.len().saturating_add(1)); + let mut out = base.clone(); + out.extend_from_slice(extra.get(..keep).unwrap_or_default()); + if keep > 0 { + match rng.below(3) { + 0 => { + let from = base.len().saturating_add(rng.index(keep)); + for byte in out.get_mut(from..).unwrap_or_default() { + *byte = 0; + } + } + 1 => { + let at = base.len().saturating_add(rng.index(keep)); + if let Some(byte) = out.get_mut(at) { + *byte ^= 0xA5; + } + } + _ => {} + } + } + out +} + +impl Storage for SimStorage { + fn create_dir_all(&self, dir: &Path) -> io::Result<()> { + let mut fs = lock(&self.fs); + if let Step::Crash = fs.begin()? { + return Err(crashed()); + } + for ancestor in dir.ancestors() { + fs.dirs.insert(ancestor.to_path_buf()); + } + Ok(()) + } + + fn list(&self, dir: &Path) -> io::Result> { + let mut fs = lock(&self.fs); + if let Step::Crash = fs.begin()? { + return Err(crashed()); + } + if !fs.dirs.contains(dir) { + return Err(io::ErrorKind::NotFound.into()); + } + let mut out = Vec::new(); + for path in fs.names.keys() { + if path.parent() == Some(dir) + && let Some(name) = path.file_name().and_then(|name| name.to_str()) + { + out.push(name.to_owned()); + } + } + for path in &fs.dirs { + if path.parent() == Some(dir) + && let Some(name) = path.file_name().and_then(|name| name.to_str()) + { + out.push(name.to_owned()); + } + } + Ok(out) + } + + fn open(&self, path: &Path, mode: OpenMode) -> io::Result> { + let mut fs = lock(&self.fs); + if let Step::Crash = fs.begin()? { + return Err(crashed()); + } + let parent = path.parent().map(Path::to_path_buf).unwrap_or_default(); + if !fs.dirs.contains(&parent) { + return Err(io::ErrorKind::NotFound.into()); + } + let inode = match (fs.names.get(path).copied(), mode) { + (Some(inode), OpenMode::Create) => { + fs.file(inode)?.data.clear(); + inode + } + (Some(inode), OpenMode::Read | OpenMode::Write) => inode, + (None, OpenMode::Create) => { + let inode = fs.next_inode; + fs.next_inode = inode.saturating_add(1); + fs.files.insert(inode, SimFile { data: Vec::new(), synced: Vec::new() }); + fs.names.insert(path.to_path_buf(), inode); + inode + } + (None, OpenMode::Read | OpenMode::Write) => return Err(io::ErrorKind::NotFound.into()), + }; + Ok(Box::new(SimHandle { + fs: self.fs.clone(), + inode, + path: path.to_path_buf(), + writable: mode != OpenMode::Read, + })) + } + + fn rename(&self, from: &Path, to: &Path) -> io::Result<()> { + let mut fs = lock(&self.fs); + if let Step::Crash = fs.begin()? { + return Err(crashed()); + } + let parent = to.parent().map(Path::to_path_buf).unwrap_or_default(); + if !fs.dirs.contains(&parent) { + return Err(io::ErrorKind::NotFound.into()); + } + let inode = fs.names.remove(from).ok_or(io::ErrorKind::NotFound)?; + fs.names.insert(to.to_path_buf(), inode); + Ok(()) + } + + fn sync_dir(&self, dir: &Path) -> io::Result<()> { + let mut fs = lock(&self.fs); + if let Step::Crash = fs.begin()? { + return Err(crashed()); + } + let delay = fs.sync_delay; + if !delay.is_zero() { + drop(fs); + std::thread::sleep(delay); + fs = lock(&self.fs); + if fs.crashed { + return Err(crashed()); + } + } + fs.durable_names.retain(|path, _| path.parent() != Some(dir)); + let current: Vec<(PathBuf, u64)> = fs + .names + .iter() + .filter(|(path, _)| path.parent() == Some(dir)) + .map(|(path, inode)| (path.clone(), *inode)) + .collect(); + fs.durable_names.extend(current); + // A directory sync flushes the drive like a full sync (FsStorage uses F_FULLFSYNC). + fs.sync_number(SyncLevel::Full); + Ok(()) + } + + fn try_lock(&self, path: &Path) -> io::Result>> { + let mut fs = lock(&self.fs); + if let Step::Crash = fs.begin()? { + return Err(crashed()); + } + if !fs.locks.insert(path.to_path_buf()) { + return Ok(None); + } + Ok(Some(Box::new(SimLock { fs: self.fs.clone(), path: path.to_path_buf() }))) + } +} + +struct SimLock { + fs: Arc>, + path: PathBuf, +} + +impl StorageLock for SimLock {} + +impl Drop for SimLock { + fn drop(&mut self) { + lock(&self.fs).locks.remove(&self.path); + } +} + +struct SimHandle { + fs: Arc>, + inode: u64, + path: PathBuf, + writable: bool, +} + +impl SimHandle { + fn check_writable(&self) -> io::Result<()> { + if self.writable { Ok(()) } else { Err(io::ErrorKind::PermissionDenied.into()) } + } +} + +impl StorageFile for SimHandle { + fn size(&self) -> io::Result { + let mut fs = lock(&self.fs); + if let Step::Crash = fs.begin()? { + return Err(crashed()); + } + Ok(u64::try_from(fs.file(self.inode)?.data.len()).unwrap_or(u64::MAX)) + } + + fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> { + let mut fs = lock(&self.fs); + if let Step::Crash = fs.begin()? { + return Err(crashed()); + } + let count = fs.reads.entry(self.path.clone()).or_default(); + *count = count.saturating_add(1); + let file = fs.file(self.inode)?; + let start = usize::try_from(offset).map_err(|_| io::ErrorKind::UnexpectedEof)?; + let end = start.checked_add(buf.len()).ok_or(io::ErrorKind::UnexpectedEof)?; + let src = file.data.get(start..end).ok_or(io::ErrorKind::UnexpectedEof)?; + buf.copy_from_slice(src); + Ok(()) + } + + fn append(&mut self, bytes: &[u8]) -> io::Result<()> { + self.check_writable()?; + let mut fs = lock(&self.fs); + match fs.begin()? { + Step::Run => { + fs.file(self.inode)?.data.extend_from_slice(bytes); + Ok(()) + } + Step::Crash => { + let keep = fs.rng.index(bytes.len().saturating_add(1)); + let torn = bytes.get(..keep).unwrap_or_default().to_vec(); + fs.file(self.inode)?.data.extend_from_slice(&torn); + Err(crashed()) + } + } + } + + fn truncate(&mut self, size: u64) -> io::Result<()> { + self.check_writable()?; + let mut fs = lock(&self.fs); + if let Step::Crash = fs.begin()? { + return Err(crashed()); + } + let size = usize::try_from(size).map_err(|_| io::ErrorKind::InvalidInput)?; + fs.file(self.inode)?.data.truncate(size); + Ok(()) + } + + fn sync(&mut self, level: SyncLevel) -> io::Result<()> { + let mut fs = lock(&self.fs); + if let Step::Crash = fs.begin()? { + return Err(crashed()); + } + let delay = fs.sync_delay; + if !delay.is_zero() { + drop(fs); + std::thread::sleep(delay); + fs = lock(&self.fs); + if fs.crashed { + return Err(crashed()); + } + } + let number = fs.sync_number(level); + let file = fs.file(self.inode)?; + let content = file.data.clone(); + file.synced.push((number, content)); + Ok(()) + } +} diff --git a/cmux-tui/crates/cmux-journal/src/storage.rs b/cmux-tui/crates/cmux-journal/src/storage.rs new file mode 100644 index 000000000000..69dc1b45e070 --- /dev/null +++ b/cmux-tui/crates/cmux-journal/src/storage.rs @@ -0,0 +1,251 @@ +//! The file system seam. [`FsStorage`] is the real one; the `sim` feature adds `SimStorage`, which +//! can crash at any operation and then keep what a process crash, an OS crash or a power loss +//! keeps. + +use std::io; +use std::path::Path; + +/// How far a sync pushes a file's data toward the disk. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum SyncLevel { + /// macOS `F_BARRIERFSYNC` (an fsync, then a drive barrier: the data reaches the disk in write + /// order), Linux `fdatasync`, Windows `FlushFileBuffers`. The data survives a crash of any + /// process and a kernel panic. A power loss on macOS can take back a suffix written since + /// the last [`SyncLevel::Full`]. + Barrier, + /// macOS `F_FULLFSYNC` (the drive also flushes its cache), else `fsync`. The data survives a + /// power loss. + Full, +} + +/// How [`Storage::open`] opens a file. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum OpenMode { + /// An existing file, read only. + Read, + /// An existing file, to read and append. + Write, + /// A new or emptied file (owner-only), to read and append. + Create, +} + +/// The file system operations the journal uses. +pub trait Storage: Send + Sync { + /// Makes `dir` and its parents (owner-only). + fn create_dir_all(&self, dir: &Path) -> io::Result<()>; + /// The names of the entries in `dir`. + fn list(&self, dir: &Path) -> io::Result>; + fn open(&self, path: &Path, mode: OpenMode) -> io::Result>; + fn rename(&self, from: &Path, to: &Path) -> io::Result<()>; + /// Makes the entries of `dir` (a create or a rename in it) durable. + fn sync_dir(&self, dir: &Path) -> io::Result<()>; + /// An exclusive lock of `path`, held until the returned value is dropped, or `None` while + /// another holder has it. + fn try_lock(&self, path: &Path) -> io::Result>>; +} + +/// An open file. +pub trait StorageFile: Send { + fn size(&self) -> io::Result; + /// Fills `buf` from `offset`; a short file is an `UnexpectedEof` error. + fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()>; + /// Writes `bytes` at the end of the file. + fn append(&mut self, bytes: &[u8]) -> io::Result<()>; + fn truncate(&mut self, size: u64) -> io::Result<()>; + fn sync(&mut self, level: SyncLevel) -> io::Result<()>; +} + +/// A held lock ([`Storage::try_lock`]). +pub trait StorageLock: Send + Sync {} + +/// The real file system. +#[derive(Clone, Copy, Debug, Default)] +pub struct FsStorage; + +impl Storage for FsStorage { + fn create_dir_all(&self, dir: &Path) -> io::Result<()> { + let mut builder = std::fs::DirBuilder::new(); + builder.recursive(true); + #[cfg(unix)] + std::os::unix::fs::DirBuilderExt::mode(&mut builder, 0o700); + builder.create(dir) + } + + fn list(&self, dir: &Path) -> io::Result> { + let mut names = Vec::new(); + for entry in std::fs::read_dir(dir)? { + if let Ok(name) = entry?.file_name().into_string() { + names.push(name); + } + } + Ok(names) + } + + fn open(&self, path: &Path, mode: OpenMode) -> io::Result> { + let mut options = std::fs::OpenOptions::new(); + options.read(true); + match mode { + OpenMode::Read => {} + OpenMode::Write => { + options.write(true); + } + OpenMode::Create => { + options.write(true).create(true).truncate(true); + #[cfg(unix)] + std::os::unix::fs::OpenOptionsExt::mode(&mut options, 0o600); + } + } + let file = options.open(path)?; + let size = file.metadata()?.len(); + Ok(Box::new(FsFile { file, size })) + } + + fn rename(&self, from: &Path, to: &Path) -> io::Result<()> { + std::fs::rename(from, to) + } + + fn sync_dir(&self, dir: &Path) -> io::Result<()> { + sync_directory(dir) + } + + fn try_lock(&self, path: &Path) -> io::Result>> { + let mut options = std::fs::OpenOptions::new(); + options.read(true).write(true).create(true).truncate(false); + #[cfg(unix)] + std::os::unix::fs::OpenOptionsExt::mode(&mut options, 0o600); + let file = options.open(path)?; + match file.try_lock() { + Ok(()) => Ok(Some(Box::new(FsLock { _file: file }))), + Err(std::fs::TryLockError::WouldBlock) => Ok(None), + Err(std::fs::TryLockError::Error(error)) => Err(error), + } + } +} + +/// The lock file, locked; closing it releases the lock. +struct FsLock { + _file: std::fs::File, +} + +impl StorageLock for FsLock {} + +/// A file and its size, so an append writes at a known offset with no seek. +struct FsFile { + file: std::fs::File, + size: u64, +} + +impl StorageFile for FsFile { + fn size(&self) -> io::Result { + Ok(self.size) + } + + fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> { + read_exact_at(&self.file, offset, buf) + } + + fn append(&mut self, bytes: &[u8]) -> io::Result<()> { + write_all_at(&self.file, self.size, bytes)?; + self.size = self.size.saturating_add(u64::try_from(bytes.len()).unwrap_or(u64::MAX)); + Ok(()) + } + + fn truncate(&mut self, size: u64) -> io::Result<()> { + self.file.set_len(size)?; + self.size = size; + Ok(()) + } + + fn sync(&mut self, level: SyncLevel) -> io::Result<()> { + sync_file(&self.file, level) + } +} + +#[cfg(unix)] +fn read_exact_at(file: &std::fs::File, offset: u64, buf: &mut [u8]) -> io::Result<()> { + std::os::unix::fs::FileExt::read_exact_at(file, buf, offset) +} + +#[cfg(unix)] +fn write_all_at(file: &std::fs::File, offset: u64, bytes: &[u8]) -> io::Result<()> { + std::os::unix::fs::FileExt::write_all_at(file, bytes, offset) +} + +#[cfg(windows)] +fn read_exact_at(file: &std::fs::File, mut offset: u64, mut buf: &mut [u8]) -> io::Result<()> { + while !buf.is_empty() { + match std::os::windows::fs::FileExt::seek_read(file, buf, offset) { + Ok(0) => return Err(io::ErrorKind::UnexpectedEof.into()), + Ok(n) => { + buf = std::mem::take(&mut buf).get_mut(n..).unwrap_or_default(); + offset = offset.saturating_add(u64::try_from(n).unwrap_or(u64::MAX)); + } + Err(error) if error.kind() == io::ErrorKind::Interrupted => {} + Err(error) => return Err(error), + } + } + Ok(()) +} + +#[cfg(windows)] +fn write_all_at(file: &std::fs::File, mut offset: u64, mut bytes: &[u8]) -> io::Result<()> { + while !bytes.is_empty() { + match std::os::windows::fs::FileExt::seek_write(file, bytes, offset) { + Ok(0) => return Err(io::ErrorKind::WriteZero.into()), + Ok(n) => { + bytes = bytes.get(n..).unwrap_or_default(); + offset = offset.saturating_add(u64::try_from(n).unwrap_or(u64::MAX)); + } + Err(error) if error.kind() == io::ErrorKind::Interrupted => {} + Err(error) => return Err(error), + } + } + Ok(()) +} + +/// [`SyncLevel`] on macOS: `F_BARRIERFSYNC` or `F_FULLFSYNC`, and a plain `fsync` where the file +/// system refuses them (a network or FAT volume). std's `sync_data` and `sync_all` are both +/// `F_FULLFSYNC` there, so they are not used. +#[cfg(target_os = "macos")] +fn sync_file(file: &std::fs::File, level: SyncLevel) -> io::Result<()> { + use std::os::fd::AsRawFd; + let command = match level { + SyncLevel::Barrier => libc::F_BARRIERFSYNC, + SyncLevel::Full => libc::F_FULLFSYNC, + }; + // SAFETY: fcntl and fsync on a descriptor that `file` owns for the whole call. + if unsafe { libc::fcntl(file.as_raw_fd(), command) } != -1 { + return Ok(()); + } + // SAFETY: as above. + if unsafe { libc::fsync(file.as_raw_fd()) } == 0 { + return Ok(()); + } + Err(io::Error::last_os_error()) +} + +/// [`SyncLevel`] elsewhere: `fdatasync` (Linux; it also flushes a volatile drive cache) or `fsync`; +/// `FlushFileBuffers` on Windows for both. +#[cfg(not(target_os = "macos"))] +fn sync_file(file: &std::fs::File, level: SyncLevel) -> io::Result<()> { + match level { + SyncLevel::Barrier => file.sync_data(), + SyncLevel::Full => file.sync_all(), + } +} + +#[cfg(unix)] +fn sync_directory(dir: &Path) -> io::Result<()> { + use std::os::unix::fs::OpenOptionsExt; + let file = std::fs::OpenOptions::new() + .read(true) + .custom_flags(libc::O_DIRECTORY | libc::O_NOFOLLOW) + .open(dir)?; + sync_file(&file, SyncLevel::Full) +} + +/// Windows has no directory sync: NTFS journals its directory changes. +#[cfg(not(unix))] +fn sync_directory(_dir: &Path) -> io::Result<()> { + Ok(()) +} diff --git a/cmux-tui/crates/cmux-journal/src/writer.rs b/cmux-tui/crates/cmux-journal/src/writer.rs new file mode 100644 index 000000000000..2cedc4ac81ab --- /dev/null +++ b/cmux-tui/crates/cmux-journal/src/writer.rs @@ -0,0 +1,370 @@ +//! The writer thread: it takes every pending record as one batch, writes the batch with one +//! write, syncs it when it holds an acked record (group commit), rotates full segments, and +//! recovers from failed writes and syncs. +//! +//! Sync levels: an acked batch is synced with a barrier sync. The first barrier sync after a full +//! one starts the power-loss window; `Config::full_sync_interval` later the next batch's sync is a +//! full one, or, when no batch comes, a full sync runs alone. So an ack waits for a drive cache +//! flush at most once per interval, and never after an idle time. +//! +//! Failures: +//! - A failed write (a full disk): the segment is cut back to its last whole record, and the +//! writer writes the same bytes again after a backoff. If the cut fails too, as below. +//! - A failed sync, or a failed cut: the writer never writes or syncs that segment again. A failed +//! sync can leave the kernel's copy of its pages clean and lost, and a second sync would then +//! report success for data that never reached the disk. It keeps every record after the durable +//! LSN in memory (`retained`) and, after a backoff, writes them into a new segment whose header +//! names the durable LSN as its "previous last LSN", so recovery reads the failed segment only +//! through it. +//! - A failed segment creation: made again from the start after a backoff. +//! +//! While it backs off, the writer takes no new records; appends end in `AppendError::Backlog` +//! once `Config::max_pending_bytes` wait. Nothing waits for the disk except this thread. +//! +//! The thread sleeps on a condition variable while nothing is owed. It sets a timer only while +//! written stream records wait for their sync, synced records for their full sync, or a failure +//! for its retry. + +use std::io; +use std::path::PathBuf; +use std::sync::{Arc, MutexGuard, PoisonError}; +use std::time::{Duration, Instant}; + +use crate::Lsn; +use crate::format::{SEGMENT_HEADER_LEN, SegmentHeader}; +use crate::journal::{Shared, State}; +use crate::recovery::{WriterStart, len64, make_segment}; +use crate::storage::{Storage, StorageFile, SyncLevel}; + +/// Written and unsynced bytes past which a batch is synced at once, acked or not. +const MAX_UNSYNCED_BYTES: usize = 8 * 1024 * 1024; + +struct Active { + file: Box, + seq: u64, + size: u64, +} + +/// What the loop does next. +struct Job { + /// A seal or a stop: sync fully whatever is owed. + close: bool, + /// A stop: end after this job. + last: bool, +} + +/// A failed storage operation ([module docs](self)). +enum Failure { + /// A write; the segment is cut back to its last whole record and stays in use. + Write(io::Error), + /// A sync, a cut or a creation; the segment (if any) is not used again. + Segment(io::Error), +} + +pub(crate) struct Writer { + shared: Arc, + storage: Arc, + dir: PathBuf, + journal_id: u64, + active: Option, + /// The number the next new segment takes. + next_seq: u64, + /// Every record taken from `pending` and not yet durable, encoded, in LSN order. + retained: Vec, + /// The LSN of the last record in `retained`. + retained_last: Lsn, + retained_ack: bool, + /// Bytes of `retained` written to `active`. + written: usize, + /// When the oldest written and unsynced byte was written. + unsynced_since: Option, + durable: Lsn, + full_durable: Lsn, + /// When a barrier sync first made records durable that no full sync covers yet. + unfull_since: Option, + retry_at: Option, + backoff: Duration, +} + +impl Writer { + pub(crate) fn new( + shared: Arc, + storage: Arc, + dir: PathBuf, + start: WriterStart, + ) -> Self { + let durable = shared.lock().stats.durable_lsn; + let backoff = shared.config.retry_min; + let tail = start.tail; + Self { + shared, + storage, + dir, + journal_id: start.journal_id, + next_seq: tail.seq.saturating_add(1), + active: Some(Active { file: tail.file, seq: tail.seq, size: tail.size }), + retained: Vec::new(), + retained_last: durable, + retained_ack: false, + written: 0, + unsynced_since: None, + durable, + full_durable: durable, + unfull_since: None, + retry_at: None, + backoff, + } + } + + pub(crate) fn run(mut self) { + loop { + let job = self.next_job(); + if let Err(failure) = self.work(job.close) { + self.fail(failure); + } + if job.last { + break; + } + } + self.shared.lock().ended = true; + self.shared.durable.notify_all(); + } + + /// Waits until something is owed and says what. + fn next_job(&mut self) -> Job { + // The guard borrows this clone, so `self` stays free for `take`. + let shared = Arc::clone(&self.shared); + let mut state = shared.lock(); + loop { + let now = Instant::now(); + if state.stop { + // One last attempt at what is pending, then the thread ends. + self.take(&mut state); + return Job { close: true, last: true }; + } + if let Some(at) = self.retry_at { + if now < at { + state = self.wait(state, Some(at)); + continue; + } + self.retry_at = None; + return Job { close: state.closed, last: false }; + } + let took = self.take(&mut state); + let owed = !self.retained.is_empty() || self.durable > self.full_durable; + if took || (state.closed && owed) || self.sync_due(now) || self.full_due(now) { + return Job { close: state.closed, last: false }; + } + let deadline = self.deadline(); + state = self.wait(state, deadline); + } + } + + fn wait<'a>( + &self, + state: MutexGuard<'a, State>, + until: Option, + ) -> MutexGuard<'a, State> { + match until { + None => self.shared.wake.wait(state).unwrap_or_else(PoisonError::into_inner), + Some(at) => { + let left = at.saturating_duration_since(Instant::now()); + self.shared.wake.wait_timeout(state, left).unwrap_or_else(PoisonError::into_inner).0 + } + } + } + + /// Moves the pending records into `retained`: one batch. + fn take(&mut self, state: &mut State) -> bool { + if state.pending.is_empty() { + return false; + } + let stats = &mut state.stats; + stats.batches = stats.batches.saturating_add(1); + stats.records = stats.records.saturating_add(state.pending_records); + stats.bytes = stats.bytes.saturating_add(len64(state.pending.len())); + stats.max_batch_records = stats.max_batch_records.max(state.pending_records); + if self.retained.is_empty() { + // The emptied buffer goes back as the next pending one. + std::mem::swap(&mut self.retained, &mut state.pending); + } else { + self.retained.extend_from_slice(&state.pending); + state.pending.clear(); + } + self.retained_last = state.pending_last; + self.retained_ack |= state.pending_ack; + state.pending_last = 0; + state.pending_records = 0; + state.pending_ack = false; + true + } + + /// Written stream records waited out their delay. + fn sync_due(&self, now: Instant) -> bool { + !self.retained.is_empty() + && self.written == self.retained.len() + && self + .unsynced_since + .is_some_and(|since| passed(since, self.shared.config.stream_sync_delay, now)) + } + + /// Synced records waited out the full-sync interval. + fn full_due(&self, now: Instant) -> bool { + self.unfull_since + .is_some_and(|since| passed(since, self.shared.config.full_sync_interval, now)) + } + + fn deadline(&self) -> Option { + let config = &self.shared.config; + let stream = (!self.retained.is_empty() && self.written == self.retained.len()) + .then_some(self.unsynced_since) + .flatten() + .and_then(|since| since.checked_add(config.stream_sync_delay)); + let full = self.unfull_since.and_then(|since| since.checked_add(config.full_sync_interval)); + match (stream, full) { + (Some(a), Some(b)) => Some(a.min(b)), + (a, b) => a.or(b), + } + } + + fn work(&mut self, close: bool) -> Result<(), Failure> { + if self.active.is_none() { + self.create_segment().map_err(Failure::Segment)?; + } + let now = Instant::now(); + if self.written < self.retained.len() { + let Some(active) = self.active.as_mut() else { + return Err(Failure::Segment(no_segment())); + }; + let unwritten = self.retained.get(self.written..).unwrap_or_default(); + if let Err(error) = active.file.append(unwritten) { + return Err(match active.file.truncate(active.size) { + Ok(()) => Failure::Write(error), + Err(_) => Failure::Segment(error), + }); + } + active.size = active.size.saturating_add(len64(unwritten.len())); + self.written = self.retained.len(); + if self.unsynced_since.is_none() { + self.unsynced_since = Some(now); + } + let last = self.retained_last; + self.publish(|stats| stats.written_lsn = last); + } + let segment_bytes = self.shared.config.segment_bytes; + let full_segment = self.active.as_ref().is_some_and(|active| active.size >= segment_bytes); + // A full sync: at a seal, at a rotation, or when synced records waited out the interval. + let full = close || full_segment || self.full_due(now); + let held = !self.retained.is_empty(); + let stream_delay = self.shared.config.stream_sync_delay; + let due = held + && (self.retained_ack + || self.retained.len() >= MAX_UNSYNCED_BYTES + || self.unsynced_since.is_some_and(|since| passed(since, stream_delay, now))); + let owed = held || self.durable > self.full_durable; + if due || (full && owed) { + let level = if full { SyncLevel::Full } else { SyncLevel::Barrier }; + self.sync(level).map_err(Failure::Segment)?; + } + if full_segment && self.retained.is_empty() { + // Fully synced: the next write makes the next segment. + self.active = None; + } + Ok(()) + } + + fn sync(&mut self, level: SyncLevel) -> io::Result<()> { + let Some(active) = self.active.as_mut() else { return Err(no_segment()) }; + active.file.sync(level)?; + let now = Instant::now(); + if !self.retained.is_empty() { + self.durable = self.retained_last; + } + match level { + SyncLevel::Full => { + self.full_durable = self.durable; + self.unfull_since = None; + } + SyncLevel::Barrier => { + if self.durable > self.full_durable && self.unfull_since.is_none() { + self.unfull_since = Some(now); + } + } + } + self.retained.clear(); + self.retained_ack = false; + self.written = 0; + self.unsynced_since = None; + self.backoff = self.shared.config.retry_min; + let (durable, full_durable) = (self.durable, self.full_durable); + let advanced = self.publish(|stats| { + let advanced = durable > stats.durable_lsn; + stats.durable_lsn = durable; + stats.full_durable_lsn = full_durable; + match level { + SyncLevel::Barrier => stats.barrier_syncs = stats.barrier_syncs.saturating_add(1), + SyncLevel::Full => stats.full_syncs = stats.full_syncs.saturating_add(1), + } + stats.failing = None; + advanced + }); + self.shared.durable.notify_all(); + if advanced { + let listeners = + self.shared.listeners.lock().unwrap_or_else(PoisonError::into_inner).clone(); + for listener in listeners { + listener(durable); + } + } + Ok(()) + } + + fn create_segment(&mut self) -> io::Result<()> { + let header = SegmentHeader { + journal_id: self.journal_id, + seq: self.next_seq, + first_lsn: self.durable.saturating_add(1), + prev_last: self.durable, + }; + let file = make_segment(self.storage.as_ref(), &self.dir, &header)?; + self.active = Some(Active { file, seq: header.seq, size: len64(SEGMENT_HEADER_LEN) }); + self.next_seq = header.seq.saturating_add(1); + self.written = 0; + self.publish(|stats| stats.segments_created = stats.segments_created.saturating_add(1)); + Ok(()) + } + + /// A failed write, sync or segment creation ([module docs](self)). + fn fail(&mut self, failure: Failure) { + let error = match failure { + Failure::Write(error) => error, + Failure::Segment(error) => { + // Nothing in this segment after `durable` is trusted from now on. + self.active = None; + self.written = 0; + self.unsynced_since = None; + error + } + }; + self.retry_at = Instant::now().checked_add(self.backoff); + self.backoff = self.backoff.saturating_mul(2).min(self.shared.config.retry_max); + let text = error.to_string(); + self.publish(|stats| { + stats.failures = stats.failures.saturating_add(1); + stats.failing = Some(text); + }); + self.shared.durable.notify_all(); + } + + fn publish(&self, change: impl FnOnce(&mut crate::JournalStats) -> T) -> T { + change(&mut self.shared.lock().stats) + } +} + +fn passed(since: Instant, delay: Duration, now: Instant) -> bool { + since.checked_add(delay).is_none_or(|due| now >= due) +} + +fn no_segment() -> io::Error { + io::Error::other("the journal has no open segment") +} diff --git a/plans/cmux-next/OWNERS.md b/plans/cmux-next/OWNERS.md index 5a75980307a3..8cf910f9e525 100644 --- a/plans/cmux-next/OWNERS.md +++ b/plans/cmux-next/OWNERS.md @@ -29,6 +29,7 @@ same commit that changes ownership, and name who decided it. | Terminal surface and Ghostty integration | `Packages/macOS/CmuxNext/Sources/CmuxNextTerminal/`, `Packages/macOS/CmuxNext/Sources/CmuxNextTerminalGeometry/`, `Packages/macOS/CmuxNext/Sources/CmuxNextApp/**/Ghostty*.swift`, `Packages/macOS/CmuxNext/Sources/CmuxNextApp/TerminalHost*.swift`, `ghostty`, `docs/ghostty-fork.md` | PL-ghostty (session cmux-b6) | [ghostty-integration-DESIGN.md](ghostty-integration-DESIGN.md) | | cmux-tui daemon (request speed, journal, store, PTY) | `cmux-tui/crates/cmux-tui-core/`, `cmux-tui/crates/cmux-server/`, `cmux-tui/crates/cmux-server-core/`, `cmux-tui/crates/cmux-pty/` | PL-daemon (session cmux-b6): request speed, journal, store, PTY (lead decision cmux-b6 2026-10-24) | [perf-lead.md](perf-lead.md) "Daemon and acpmux targets" | | Daemon child processes (spawn, descriptors, the no-raw-spawn guard) | `cmux-tui/crates/cmux-process/` | PL-holder (session cmux-1c); PL-daemon migrates its own crates' call sites (lead decision cmux-b6 2026-10-24) | [pty-holder.md](pty-holder.md) | +| Durable journal (one append-only journal per daemon: record format, group commit, ack after durable, recovery) | `cmux-tui/crates/cmux-journal/` | R-DURABILITY rewrite owner (REWRITES program, lead 2026-11-03) | the crate docs, `cmux-tui/crates/cmux-journal/src/lib.rs` | | Flow traces | `cmux-tui/crates/cmux-flow-trace/` | PL-trace (session cmux-1c; lead decision cmux-b6 2026-10-24) | [trace.md](trace.md) | | Rust daemon client (attach, byte attachments, its fakes and tests) | `cmux-tui/bindings/rust-daemon-client/` | lead: cmux-ad (no workstream owns it; PL-tipwarden added the row on the lead's request 2026-10-25) | [app-daemon-connection-DESIGN.md](app-daemon-connection-DESIGN.md) | | PTY holder (custody, ring, attach) | `cmux-tui/crates/cmux-holder/`, `cmux-tui/crates/cmux-holder-v3/`, `cmux-tui/crates/cmux-tui-core/src/holder_runtime.rs`, `cmux-tui/crates/cmux-tui-core/src/holder_runtime/` | PL-daemon holder (session cmux-4f) | [pty-holder.md](pty-holder.md) | From 36f09c87b8d580dca9b9105749c7310ae97e6a70 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 3 Nov 2026 14:19:29 +0000 Subject: [PATCH 02/52] cmux-journal: type a test range; drop an unread segment field (fleet clippy) Fleet job 1a494d9372ec2bb7d71e8e1e at 26e4e71e3f9 refused two lines: E0689 (is_multiple_of on an untyped range in a test) and dead_code (Active::seq in the writer: the next segment's number is next_seq). The acpmux notify::Watcher import error in the same run is a base red. --- cmux-tui/crates/cmux-journal/src/cmux_journal_tests.rs | 2 +- cmux-tui/crates/cmux-journal/src/writer.rs | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/cmux-tui/crates/cmux-journal/src/cmux_journal_tests.rs b/cmux-tui/crates/cmux-journal/src/cmux_journal_tests.rs index a7067051f1ef..e45c2025a7cb 100644 --- a/cmux-tui/crates/cmux-journal/src/cmux_journal_tests.rs +++ b/cmux-tui/crates/cmux-journal/src/cmux_journal_tests.rs @@ -97,7 +97,7 @@ fn records_round_trip_in_order_across_a_reopen() { let (journal, report, seen) = open(&storage); assert!(seen.is_empty()); assert_eq!(report.next_lsn, 1); - for lsn in 1..=50 { + for lsn in 1..=50u64 { let class = if lsn.is_multiple_of(3) { Class::Stream } else { Class::Ack }; assert_eq!(journal.append(7, class, &payload(lsn)), Ok(lsn)); } diff --git a/cmux-tui/crates/cmux-journal/src/writer.rs b/cmux-tui/crates/cmux-journal/src/writer.rs index 2cedc4ac81ab..246837639e2c 100644 --- a/cmux-tui/crates/cmux-journal/src/writer.rs +++ b/cmux-tui/crates/cmux-journal/src/writer.rs @@ -41,7 +41,6 @@ const MAX_UNSYNCED_BYTES: usize = 8 * 1024 * 1024; struct Active { file: Box, - seq: u64, size: u64, } @@ -102,7 +101,7 @@ impl Writer { dir, journal_id: start.journal_id, next_seq: tail.seq.saturating_add(1), - active: Some(Active { file: tail.file, seq: tail.seq, size: tail.size }), + active: Some(Active { file: tail.file, size: tail.size }), retained: Vec::new(), retained_last: durable, retained_ack: false, @@ -327,7 +326,7 @@ impl Writer { prev_last: self.durable, }; let file = make_segment(self.storage.as_ref(), &self.dir, &header)?; - self.active = Some(Active { file, seq: header.seq, size: len64(SEGMENT_HEADER_LEN) }); + self.active = Some(Active { file, size: len64(SEGMENT_HEADER_LEN) }); self.next_seq = header.seq.saturating_add(1); self.written = 0; self.publish(|stats| stats.segments_created = stats.segments_created.saturating_add(1)); From f55f8c5c53e243ef2e93d482e5f37883ecc0e444 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 3 Nov 2026 14:39:54 +0000 Subject: [PATCH 03/52] cmux-journal: a failed F_FULLFSYNC is an error, not a plain fsync; no segment while nothing is owed - macOS sync_file fell back to fsync on ANY fcntl error. An EIO from F_FULLFSYNC or F_BARRIERFSYNC then became a success of a weaker sync, and the writer counted the records durable. Now only "not supported" (ENOTSUP, EOPNOTSUPP, EINVAL, ENOTTY: a network or FAT volume) falls back; any other error reaches the writer, which starts a new segment. - After a rotation the writer made the next segment at its next pass even with nothing to write (the shutdown pass did it too). A segment is now made only when records or a full sync are owed. - Test-only and lint-safety rewrites: a slice parameter in place of &Vec, a type alias for the crash model, a range check, one split match scrutinee. --- .../cmux-journal/src/cmux_journal_tests.rs | 13 +++++++----- cmux-tui/crates/cmux-journal/src/recovery.rs | 4 ++-- cmux-tui/crates/cmux-journal/src/sim.rs | 7 +++---- cmux-tui/crates/cmux-journal/src/storage.rs | 18 +++++++++++++---- cmux-tui/crates/cmux-journal/src/writer.rs | 20 ++++++++++++++----- 5 files changed, 42 insertions(+), 20 deletions(-) diff --git a/cmux-tui/crates/cmux-journal/src/cmux_journal_tests.rs b/cmux-tui/crates/cmux-journal/src/cmux_journal_tests.rs index e45c2025a7cb..593c21e506c1 100644 --- a/cmux-tui/crates/cmux-journal/src/cmux_journal_tests.rs +++ b/cmux-tui/crates/cmux-journal/src/cmux_journal_tests.rs @@ -20,8 +20,11 @@ use crate::{ const WAIT: Duration = Duration::from_secs(10); -/// What a replay saw: (lsn, kind, class, payload) per record. -type Seen = Vec<(Lsn, u16, Class, Vec)>; +/// One record as a replay saw it: (lsn, kind, class, payload). +type SeenRecord = (Lsn, u16, Class, Vec); +type Seen = Vec; +/// What the crash runs appended: LSN to (class, payload). +type Model = Arc)>>>; fn dir() -> PathBuf { PathBuf::from("/sim/journal") @@ -55,7 +58,7 @@ fn open(storage: &SimStorage) -> (Journal, OpenReport, Seen) { open_at(storage, quick(), 0).unwrap_or_else(|error| panic!("open: {error}")) } -fn lsns(seen: &Seen) -> Vec { +fn lsns(seen: &[SeenRecord]) -> Vec { seen.iter().map(|record| record.0).collect() } @@ -390,7 +393,7 @@ fn a_damaged_segment_in_the_middle_is_quarantined_and_its_lsns_are_never_reused( let restarted = storage.restart(Crash::Process); let (journal, report, seen) = open_at(&restarted, config, 0).unwrap_or_else(|e| panic!("{e}")); let kept = Lsn::try_from(seen.len()).unwrap_or(0); - assert!(kept > 0 && kept < 30, "the valid prefix stays: {kept}"); + assert!((1..30).contains(&kept), "the valid prefix stays: {kept}"); assert_eq!(lsns(&seen), (1..=kept).collect::>()); match report.damage.first() { Some(Damage::Corrupt { moved, lost_after, lost_through, .. }) => { @@ -487,7 +490,7 @@ fn crash_run(seed: u64, crash: Crash) -> Result<(), String> { }; let (journal, _, _) = open_at(&storage, config.clone(), 0).map_err(|e| e.to_string())?; let journal = Arc::new(journal); - let appended: Arc)>>> = Arc::default(); + let appended: Model = Arc::default(); let durable = Arc::new(AtomicU64::new(0)); let full_durable = Arc::new(AtomicU64::new(0)); storage.crash_after(1 + rng.below(80)); diff --git a/cmux-tui/crates/cmux-journal/src/recovery.rs b/cmux-tui/crates/cmux-journal/src/recovery.rs index 10fd29bd5fa5..63f2392baba7 100644 --- a/cmux-tui/crates/cmux-journal/src/recovery.rs +++ b/cmux-tui/crates/cmux-journal/src/recovery.rs @@ -269,8 +269,8 @@ fn read_headers( let file = storage.open(&dir.join(name), OpenMode::Read).map_err(io("open a segment"))?; let size = file.size().map_err(io("read a segment's size"))?; let last = index.saturating_add(1) == listed.len(); - match read_header(file.as_ref(), size)?.and_then(|header| follows(&segments, header, *seq)) - { + let decoded = read_header(file.as_ref(), size)?; + match decoded.and_then(|header| follows(&segments, header, *seq)) { Ok(header) => { segments.push(Segment { seq: *seq, name: name.clone(), file, size, header }); } diff --git a/cmux-tui/crates/cmux-journal/src/sim.rs b/cmux-tui/crates/cmux-journal/src/sim.rs index 099bb8bb5302..09d34a6b0819 100644 --- a/cmux-tui/crates/cmux-journal/src/sim.rs +++ b/cmux-tui/crates/cmux-journal/src/sim.rs @@ -336,8 +336,8 @@ fn surviving_bytes(file: &SimFile, cut: Option, rng: &mut SimRng) -> Vec { let from = base.len().saturating_add(rng.index(keep)); - for byte in out.get_mut(from..).unwrap_or_default() { - *byte = 0; + if let Some(tail) = out.get_mut(from..) { + tail.fill(0); } } 1 => { @@ -535,8 +535,7 @@ impl StorageFile for SimHandle { } Step::Crash => { let keep = fs.rng.index(bytes.len().saturating_add(1)); - let torn = bytes.get(..keep).unwrap_or_default().to_vec(); - fs.file(self.inode)?.data.extend_from_slice(&torn); + fs.file(self.inode)?.data.extend_from_slice(bytes.get(..keep).unwrap_or_default()); Err(crashed()) } } diff --git a/cmux-tui/crates/cmux-journal/src/storage.rs b/cmux-tui/crates/cmux-journal/src/storage.rs index 69dc1b45e070..6632e935f3b8 100644 --- a/cmux-tui/crates/cmux-journal/src/storage.rs +++ b/cmux-tui/crates/cmux-journal/src/storage.rs @@ -203,9 +203,11 @@ fn write_all_at(file: &std::fs::File, mut offset: u64, mut bytes: &[u8]) -> io:: Ok(()) } -/// [`SyncLevel`] on macOS: `F_BARRIERFSYNC` or `F_FULLFSYNC`, and a plain `fsync` where the file -/// system refuses them (a network or FAT volume). std's `sync_data` and `sync_all` are both -/// `F_FULLFSYNC` there, so they are not used. +/// [`SyncLevel`] on macOS: `F_BARRIERFSYNC` or `F_FULLFSYNC`. A plain `fsync` is used only where +/// the file system does not support the call (a network, FAT or other volume that answers +/// ENOTSUP, EOPNOTSUPP, EINVAL or ENOTTY); there no stronger sync exists. Any other error (EIO) is +/// returned: the data is not known to be on the disk, and the caller must not count it as +/// durable. std's `sync_data` and `sync_all` are both `F_FULLFSYNC` here, so they are not used. #[cfg(target_os = "macos")] fn sync_file(file: &std::fs::File, level: SyncLevel) -> io::Result<()> { use std::os::fd::AsRawFd; @@ -213,10 +215,18 @@ fn sync_file(file: &std::fs::File, level: SyncLevel) -> io::Result<()> { SyncLevel::Barrier => libc::F_BARRIERFSYNC, SyncLevel::Full => libc::F_FULLFSYNC, }; - // SAFETY: fcntl and fsync on a descriptor that `file` owns for the whole call. + // SAFETY: fcntl on a descriptor that `file` owns for the whole call. if unsafe { libc::fcntl(file.as_raw_fd(), command) } != -1 { return Ok(()); } + let error = io::Error::last_os_error(); + let unsupported = matches!( + error.raw_os_error(), + Some(libc::ENOTSUP | libc::EOPNOTSUPP | libc::EINVAL | libc::ENOTTY) + ); + if !unsupported { + return Err(error); + } // SAFETY: as above. if unsafe { libc::fsync(file.as_raw_fd()) } == 0 { return Ok(()); diff --git a/cmux-tui/crates/cmux-journal/src/writer.rs b/cmux-tui/crates/cmux-journal/src/writer.rs index 246837639e2c..eac6b778979c 100644 --- a/cmux-tui/crates/cmux-journal/src/writer.rs +++ b/cmux-tui/crates/cmux-journal/src/writer.rs @@ -215,10 +215,12 @@ impl Writer { fn deadline(&self) -> Option { let config = &self.shared.config; - let stream = (!self.retained.is_empty() && self.written == self.retained.len()) - .then_some(self.unsynced_since) - .flatten() - .and_then(|since| since.checked_add(config.stream_sync_delay)); + let written = !self.retained.is_empty() && self.written == self.retained.len(); + let stream = if written { + self.unsynced_since.and_then(|since| since.checked_add(config.stream_sync_delay)) + } else { + None + }; let full = self.unfull_since.and_then(|since| since.checked_add(config.full_sync_interval)); match (stream, full) { (Some(a), Some(b)) => Some(a.min(b)), @@ -227,7 +229,16 @@ impl Writer { } fn work(&mut self, close: bool) -> Result<(), Failure> { + // Records to write or sync, or synced records a full sync still owes. A rotation closes a + // fully synced segment, so it owes nothing and the next segment waits for the next record. + let owed = !self.retained.is_empty() || self.durable > self.full_durable; if self.active.is_none() { + if !owed { + return Ok(()); + } + // After a failed sync or full sync, the records after `durable` (in `retained`) go into + // the new segment, and its full syncs flush the drive cache for every file, so the + // records the failed segment holds through `durable` become fully durable too. self.create_segment().map_err(Failure::Segment)?; } let now = Instant::now(); @@ -260,7 +271,6 @@ impl Writer { && (self.retained_ack || self.retained.len() >= MAX_UNSYNCED_BYTES || self.unsynced_since.is_some_and(|since| passed(since, stream_delay, now))); - let owed = held || self.durable > self.full_durable; if due || (full && owed) { let level = if full { SyncLevel::Full } else { SyncLevel::Barrier }; self.sync(level).map_err(Failure::Segment)?; From 002739125579e750ad585529ebbce82c421fd3f4 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 3 Nov 2026 14:47:06 +0000 Subject: [PATCH 04/52] cmux-journal: the past-the-end test reads back the storage it wrote to Fleet job 7f6bfd5afb225cd1cee9d0f1 at 36f09c87b8d: 24 of 25 crate tests pass (the three 200-seed crash simulations included). The one red, a_compaction_point_past_the_end_never_reuses_an_lsn (left [1], right [1, 10]), was a test fault: it appended LSN 10 in the restarted storage and then read back the first one. The clippy type_complexity hit of the same run is the crash model type, fixed in f55f8c5c53e. --- cmux-tui/crates/cmux-journal/src/cmux_journal_tests.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/cmux-tui/crates/cmux-journal/src/cmux_journal_tests.rs b/cmux-tui/crates/cmux-journal/src/cmux_journal_tests.rs index 593c21e506c1..eb287aeb2b7e 100644 --- a/cmux-tui/crates/cmux-journal/src/cmux_journal_tests.rs +++ b/cmux-tui/crates/cmux-journal/src/cmux_journal_tests.rs @@ -413,14 +413,15 @@ fn a_compaction_point_past_the_end_never_reuses_an_lsn() { assert_eq!(journal.append(1, Class::Ack, b"a"), Ok(1)); assert_eq!(journal.seal(WAIT), Ok(1)); drop(journal); - let (journal, report, _) = - open_at(&storage.restart(Crash::Process), quick(), 9).unwrap_or_else(|e| panic!("{e}")); + // The second process: its compaction point (9) is past the journal's end (1). + let second = storage.restart(Crash::Process); + let (journal, report, _) = open_at(&second, quick(), 9).unwrap_or_else(|e| panic!("{e}")); assert_eq!(report.damage, vec![Damage::Ahead { journal_end: 1, from: 9 }]); assert_eq!(journal.append(1, Class::Ack, b"b"), Ok(10)); assert_eq!(journal.seal(WAIT), Ok(10)); drop(journal); let (_, report, seen) = - open_at(&storage.restart(Crash::PowerLoss), quick(), 0).unwrap_or_else(|e| panic!("{e}")); + open_at(&second.restart(Crash::PowerLoss), quick(), 0).unwrap_or_else(|e| panic!("{e}")); assert_eq!(lsns(&seen), vec![1, 10]); assert!(report.damage.is_empty(), "{report:?}"); } From a93684835c4d708b05b5788afc5275e2b9fe30a7 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 3 Nov 2026 15:36:11 +0000 Subject: [PATCH 05/52] cmux-journal: a sync a signal interrupts is made again (macOS) fcntl(F_BARRIERFSYNC / F_FULLFSYNC) and fsync can return EINTR. The writer counted that as a failed sync and moved to a new segment. Now the call is made again, as std's sync_data and sync_all do (cvt_r). --- cmux-tui/crates/cmux-journal/src/storage.rs | 29 +++++++++++++++------ 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/cmux-tui/crates/cmux-journal/src/storage.rs b/cmux-tui/crates/cmux-journal/src/storage.rs index 6632e935f3b8..ef9f0c31a104 100644 --- a/cmux-tui/crates/cmux-journal/src/storage.rs +++ b/cmux-tui/crates/cmux-journal/src/storage.rs @@ -207,19 +207,21 @@ fn write_all_at(file: &std::fs::File, mut offset: u64, mut bytes: &[u8]) -> io:: /// the file system does not support the call (a network, FAT or other volume that answers /// ENOTSUP, EOPNOTSUPP, EINVAL or ENOTTY); there no stronger sync exists. Any other error (EIO) is /// returned: the data is not known to be on the disk, and the caller must not count it as -/// durable. std's `sync_data` and `sync_all` are both `F_FULLFSYNC` here, so they are not used. +/// durable. A call a signal interrupts (EINTR) is made again, as std does. std's `sync_data` and +/// `sync_all` are both `F_FULLFSYNC` here, so they are not used. #[cfg(target_os = "macos")] fn sync_file(file: &std::fs::File, level: SyncLevel) -> io::Result<()> { use std::os::fd::AsRawFd; + let fd = file.as_raw_fd(); let command = match level { SyncLevel::Barrier => libc::F_BARRIERFSYNC, SyncLevel::Full => libc::F_FULLFSYNC, }; // SAFETY: fcntl on a descriptor that `file` owns for the whole call. - if unsafe { libc::fcntl(file.as_raw_fd(), command) } != -1 { - return Ok(()); - } - let error = io::Error::last_os_error(); + let error = match retry_interrupted(|| unsafe { libc::fcntl(fd, command) }) { + Ok(()) => return Ok(()), + Err(error) => error, + }; let unsupported = matches!( error.raw_os_error(), Some(libc::ENOTSUP | libc::EOPNOTSUPP | libc::EINVAL | libc::ENOTTY) @@ -228,10 +230,21 @@ fn sync_file(file: &std::fs::File, level: SyncLevel) -> io::Result<()> { return Err(error); } // SAFETY: as above. - if unsafe { libc::fsync(file.as_raw_fd()) } == 0 { - return Ok(()); + retry_interrupted(|| unsafe { libc::fsync(fd) }) +} + +/// Calls `call` until it returns something other than -1 with EINTR. +#[cfg(target_os = "macos")] +fn retry_interrupted(call: impl Fn() -> libc::c_int) -> io::Result<()> { + loop { + if call() != -1 { + return Ok(()); + } + let error = io::Error::last_os_error(); + if error.kind() != io::ErrorKind::Interrupted { + return Err(error); + } } - Err(io::Error::last_os_error()) } /// [`SyncLevel`] elsewhere: `fdatasync` (Linux; it also flushes a volatile drive cache) or `fsync`; From b6c6030c838fbf5515b6ead81b70ae15b32413f7 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 3 Nov 2026 19:24:39 +0000 Subject: [PATCH 06/52] cmux-journal: crash sim of write-back errors and crash in recovery; S1 review regressions (red) The S1 review (FIX at 89293fb15f39) found that the crash simulation could not find its F1 and F4: it never crashed inside recovery, a failed operation had no partial effect, and a failed sync never lost a page. This commit changes only the simulation and the tests; the next commit fixes the journal. The simulated machine (sim.rs) now has: - a page cache over a disk, per 4 KiB page; a sync writes the dirty pages; a sync that fails writes some of them and drops the rest as clean, so a later sync reports success without them (Linux write-back error behavior); - a write that fails writes a random prefix first; - one ordered log of names (journaling file system): a folder sync makes durable every change up to the last one in that folder; a rename between folders is one change; an unsynced folder can be lost; - a process crash keeps the machine whole, so a later OS crash still loses what it would have lost; - an OS crash writes each dirty page whole, not at all, or a prefix of its sectors; a power loss also cuts back to a common point after the last full sync; - counters of open files and of injected failures. The crash runs (crash_sim_tests.rs) now run several epochs per seed: appends, a crash at a random operation (sometimes after a failed write or sync), and an open that sometimes crashes inside recovery and is tried again. After each open the oracle checks every record byte for byte, that every record an earlier open replayed is still there, that the epoch's records are a prefix of its appends, and that every acked record (fully durable after a power loss; every written one when only processes crashed) survives. Four runs: process, OS, power loss, and mixed. Regression tests for the review findings, red on the code before the fix: - F1 a_crash_inside_the_quarantine_step_never_reuses_a_quarantined_lsn - F2 a_damaged_header_in_the_last_segment_never_reuses_its_lsns - F3 a_panic_on_the_writer_thread_closes_the_journal - F4 a_restart_after_a_failed_sync_keeps_every_later_ack - F6 recovery_holds_at_most_two_files_open --- .../cmux-journal/src/cmux_journal_tests.rs | 279 +++--- .../cmux-journal/src/crash_sim_tests.rs | 379 ++++++++ cmux-tui/crates/cmux-journal/src/lib.rs | 4 +- cmux-tui/crates/cmux-journal/src/sim.rs | 848 ++++++++++++------ 4 files changed, 1102 insertions(+), 408 deletions(-) create mode 100644 cmux-tui/crates/cmux-journal/src/crash_sim_tests.rs diff --git a/cmux-tui/crates/cmux-journal/src/cmux_journal_tests.rs b/cmux-tui/crates/cmux-journal/src/cmux_journal_tests.rs index eb287aeb2b7e..c8652102c377 100644 --- a/cmux-tui/crates/cmux-journal/src/cmux_journal_tests.rs +++ b/cmux-tui/crates/cmux-journal/src/cmux_journal_tests.rs @@ -1,21 +1,17 @@ //! Tests of the journal: format, group commit, ack after durable, recovery, failures, and the -//! crash simulation (every acked record survives a crash at any storage operation; nothing is -//! half applied; the lost tail is a suffix). -//! -//! The crash simulation runs `CMUX_JOURNAL_CRASH_SEEDS` seeds per crash kind (default 200; the -//! acceptance run in R-DURABILITY/DESIGN.md section 7 uses 100000 on the fleet). A failing seed -//! prints itself: rerun it alone with `CMUX_JOURNAL_CRASH_SEED=`. - -use std::collections::BTreeMap; +//! regressions of the S1 review (a crash inside recovery's quarantine step, a damaged last +//! header, a panic on the writer thread, a restart after a failed sync, open files at recovery). +//! The crash simulation is in `crash_sim_tests.rs`. + use std::path::{Path, PathBuf}; -use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::mpsc::RecvTimeoutError; use std::sync::{Arc, Mutex}; use std::time::Duration; use crate::format::{self, SEGMENT_HEADER_LEN}; use crate::{ AppendError, Class, Config, Crash, Damage, FsStorage, Journal, Lsn, OpenError, OpenReport, - SimRng, SimStorage, Storage, WaitError, + SIM_PAGE, SimStorage, Storage, WaitError, }; const WAIT: Duration = Duration::from_secs(10); @@ -23,8 +19,6 @@ const WAIT: Duration = Duration::from_secs(10); /// One record as a replay saw it: (lsn, kind, class, payload). type SeenRecord = (Lsn, u16, Class, Vec); type Seen = Vec; -/// What the crash runs appended: LSN to (class, payload). -type Model = Arc)>>>; fn dir() -> PathBuf { PathBuf::from("/sim/journal") @@ -449,149 +443,166 @@ fn write_whole(storage: &SimStorage, path: &Path, bytes: &[u8]) { file.sync(crate::SyncLevel::Full).unwrap_or_else(|e| panic!("sync: {e}")); } -// ---------------------------------------------------------------- crash simulation +// ---------------------------------------------------------------- S1 review regressions -/// Seeds per crash kind (module docs). -fn crash_seeds() -> Vec { - if let Some(seed) = std::env::var("CMUX_JOURNAL_CRASH_SEED").ok().and_then(|s| s.parse().ok()) { - return vec![seed]; - } - let count = std::env::var("CMUX_JOURNAL_CRASH_SEEDS") - .ok() - .and_then(|s| s.parse().ok()) - .unwrap_or(200u64); - (0..count).collect() -} - -/// Waits until `lsn` is durable; false once the storage crashed first. -fn acked(journal: &Journal, storage: &SimStorage, lsn: Lsn) -> bool { - loop { - match journal.wait_durable(lsn, Duration::from_millis(5)) { - Ok(()) => return true, - Err(WaitError::Timeout) if !storage.is_crashed() => {} - Err(_) => return false, - } +/// 40 acked records over several segments, with media damage in the first record of segment 2 +/// (so every later segment goes to quarantine at the next open); and its config. +fn damaged_in_the_middle(seed: u64) -> (SimStorage, Config) { + let storage = SimStorage::new(seed); + let config = Config { segment_bytes: 400, ..quick() }; + let (journal, _, _) = open_at(&storage, config.clone(), 0).unwrap_or_else(|e| panic!("{e}")); + for lsn in 1..=40 { + let got = journal.append(3, Class::Ack, &payload(lsn)).unwrap_or_else(|e| panic!("{e}")); + journal.wait_durable(got, WAIT).unwrap_or_else(|e| panic!("{e}")); } + assert!(journal.stats().segments_created >= 3, "{:?}", journal.stats()); + drop(journal); + storage.corrupt(&dir().join(format::segment_name(2)), SEGMENT_HEADER_LEN + 2); + (storage.restart(Crash::Process), config) } -/// One crash run: clients append and wait while the storage crashes at a random operation; then -/// recovery runs. Every recovered record must be exactly what was appended at its LSN, the -/// records must be 1..=n, and n must reach every record a client was told is durable (fully -/// durable, for a power loss), and every written record for a process crash. -fn crash_run(seed: u64, crash: Crash) -> Result<(), String> { - let mut rng = SimRng::new(seed); - let storage = SimStorage::new(rng.next_u64()); - let config = Config { - segment_bytes: 300 + rng.below(3000), - max_pending_bytes: 1 << 20, - full_sync_interval: Duration::from_millis(rng.below(4)), - stream_sync_delay: Duration::from_millis(rng.below(3)), - retry_min: Duration::from_millis(1), - retry_max: Duration::from_millis(2), - }; - let (journal, _, _) = open_at(&storage, config.clone(), 0).map_err(|e| e.to_string())?; - let journal = Arc::new(journal); - let appended: Model = Arc::default(); - let durable = Arc::new(AtomicU64::new(0)); - let full_durable = Arc::new(AtomicU64::new(0)); - storage.crash_after(1 + rng.below(80)); - let clients = 1 + rng.below(4); - let threads: Vec<_> = (0..clients) - .map(|client| { - let journal = journal.clone(); - let storage = storage.clone(); - let appended = appended.clone(); - let durable = durable.clone(); - let full_durable = full_durable.clone(); - let mut rng = SimRng::new(seed ^ (client + 1).wrapping_mul(0x9E37)); - std::thread::spawn(move || { - for index in 0..30u64 { - let class = if rng.below(4) == 0 { Class::Stream } else { Class::Ack }; - let body = format!("{seed}-{client}-{index}-{}", "y".repeat(rng.index(64))); - let lsn = { - // The append and the model's copy are one step, so the model never misses - // an LSN the journal took. - let mut model = appended.lock().unwrap_or_else(|e| e.into_inner()); - let Ok(lsn) = journal.append(4, class, body.as_bytes()) else { break }; - model.insert(lsn, (class, body.into_bytes())); - lsn - }; - if class == Class::Ack { - if !acked(&journal, &storage, lsn) { - break; - } - durable.fetch_max(lsn, Ordering::SeqCst); - if journal.full_durable_lsn() >= lsn { - full_durable.fetch_max(lsn, Ordering::SeqCst); - } +/// F1: the next open after a crash in the middle of the quarantine step (after a segment moved, +/// before the segment that skips its LSNs is durable) never gives a quarantined LSN to a new record. +#[test] +fn a_crash_inside_the_quarantine_step_never_reuses_a_quarantined_lsn() { + let (damaged, config) = damaged_in_the_middle(50); + let mut crashed_after_a_move = 0; + for crash in [Crash::Process, Crash::Os] { + for at in 1..=90 { + let storage = damaged.snapshot(Crash::Process); + storage.crash_after(at); + let (journal, report) = match open_at(&storage, config.clone(), 0) { + Ok((journal, report, _)) => (journal, report), + Err(_) => { + let restarted = storage.restart(crash); + if !restarted.names_in(&dir().join("quarantine")).is_empty() { + crashed_after_a_move += 1; } + let (journal, report, _) = open_at(&restarted, config.clone(), 0) + .unwrap_or_else(|e| panic!("{crash:?} at op {at}, then open: {e}")); + (journal, report) } - }) - }) - .collect(); - for thread in threads { - thread.join().map_err(|_| "a client thread panicked".to_owned())?; - } - let stats = journal.stats(); - durable.fetch_max(stats.durable_lsn, Ordering::SeqCst); - full_durable.fetch_max(stats.full_durable_lsn, Ordering::SeqCst); - drop(journal); - let restarted = storage.restart(crash); - let (_, report, seen) = open_at(&restarted, config, 0).map_err(|e| e.to_string())?; - let appended = appended.lock().unwrap_or_else(|e| e.into_inner()); - for (index, (lsn, kind, class, bytes)) in seen.iter().enumerate() { - let position = Lsn::try_from(index).unwrap_or(0) + 1; - if *lsn != position { - return Err(format!("not a prefix: record {lsn} at position {position}")); - } - match appended.get(lsn) { - Some((want_class, want)) if want_class == class && want == bytes && *kind == 4 => {} - other => return Err(format!("record {lsn} is not what was appended: {other:?}")), + }; + assert_eq!( + journal.append(3, Class::Ack, b"after"), + Ok(41), + "{crash:?} at op {at}: {report:?}" + ); } } - let last = Lsn::try_from(seen.len()).unwrap_or(0); - let must = match crash { - Crash::Process => durable.load(Ordering::SeqCst).max(stats.written_lsn), - Crash::Os => durable.load(Ordering::SeqCst), - Crash::PowerLoss => full_durable.load(Ordering::SeqCst), - }; - if last < must { - return Err(format!("record {must} lost: recovered 1..={last} ({crash:?}, {report:?})")); - } - if !report.damage.is_empty() { - return Err(format!("a crash is not damage: {report:?}")); - } - Ok(()) + assert!(crashed_after_a_move > 0, "no crash point fell after a move"); } -fn crash_runs(crash: Crash) { - let mut failures = Vec::new(); - for seed in crash_seeds() { - if let Err(error) = crash_run(seed, crash) { - failures.push(format!("seed {seed}: {error}")); +/// F2: a damaged header in the last segment: its records are read for their LSNs, so none is +/// given to a new record. +#[test] +fn a_damaged_header_in_the_last_segment_never_reuses_its_lsns() { + let storage = SimStorage::new(51); + let mut next = 1; + for _ in 0..2 { + let (journal, _, _) = open(&storage); + for _ in 0..10 { + assert_eq!(journal.append(3, Class::Ack, &payload(next)), Ok(next)); + next += 1; } + assert_eq!(journal.seal(WAIT), Ok(next - 1)); } - assert!( - failures.is_empty(), - "{} {crash:?} runs failed:\n{}", - failures.len(), - failures.join("\n") - ); + let last = storage + .names_in(&dir()) + .into_iter() + .filter(|name| name.ends_with(".seg")) + .max() + .unwrap_or_else(|| panic!("no segment")); + // One byte of the journal id in its header (media damage). + storage.corrupt(&dir().join(&last), 20); + let (journal, report, seen) = open(&storage.restart(Crash::Process)); + match report.damage.first() { + Some(Damage::Corrupt { lost_through, .. }) => assert_eq!(*lost_through, 20, "{report:?}"), + other => panic!("no damage report: {other:?} ({report:?})"), + } + assert!(seen.len() < 20, "{report:?}"); + assert_eq!(journal.append(3, Class::Ack, b"after"), Ok(21)); } +/// F3: a panic on the writer thread (here a durable listener's) closes the journal: waits end, +/// appends fail, and the listeners are dropped, so the channel a held reply waits on closes. #[test] -fn a_process_crash_at_any_storage_operation_keeps_every_written_record() { - crash_runs(Crash::Process); +fn a_panic_on_the_writer_thread_closes_the_journal() { + let storage = SimStorage::new(52); + let (journal, _, _) = open(&storage); + let (sender, receiver) = std::sync::mpsc::channel(); + journal.on_durable(move |lsn| { + let _ = sender.send(lsn); + }); + journal.on_durable(|_| panic!("a durable listener panicked")); + storage.set_sync_delay(Duration::from_millis(200)); + assert_eq!(journal.append(1, Class::Ack, b"a"), Ok(1)); + assert!(eventually(|| journal.stats().written_lsn >= 1), "{:?}", journal.stats()); + // Record 2 waits while record 1 syncs; then the listener's panic ends the writer thread. + assert_eq!(journal.append(1, Class::Ack, b"b"), Ok(2)); + assert_eq!(receiver.recv_timeout(WAIT), Ok(1)); + assert_eq!(journal.wait_durable(2, Duration::from_secs(2)), Err(WaitError::WriterEnded)); + assert_eq!(journal.append(1, Class::Ack, b"c"), Err(AppendError::Closed)); + assert_eq!(receiver.recv_timeout(Duration::from_secs(2)), Err(RecvTimeoutError::Disconnected)); + assert!(journal.stats().failing.is_some(), "{:?}", journal.stats()); } +/// F4: a sync fails, so its pages never reach the disk, though a later sync on another descriptor +/// reports success; the process is killed before the writer tries again. Every record acked after +/// the restart survives a later OS crash. #[test] -fn an_os_crash_at_any_storage_operation_keeps_every_acked_record() { - crash_runs(Crash::Os); +fn a_restart_after_a_failed_sync_keeps_every_later_ack() { + let slow_retry = Config { + retry_min: Duration::from_secs(60), + retry_max: Duration::from_secs(60), + ..quick() + }; + for seed in 0..8 { + let storage = SimStorage::new(53 + seed); + let (journal, _, _) = + open_at(&storage, slow_retry.clone(), 0).unwrap_or_else(|e| panic!("{e}")); + assert_eq!(journal.append(1, Class::Ack, b"a"), Ok(1)); + journal.wait_durable(1, WAIT).unwrap_or_else(|e| panic!("{e}")); + assert!(eventually(|| journal.full_durable_lsn() >= 1), "{:?}", journal.stats()); + // Record 2 spans whole pages: its write, then its sync, which fails. + storage.fail_after(2, std::io::ErrorKind::Other); + assert_eq!(journal.append(1, Class::Ack, &vec![b'b'; 3 * SIM_PAGE]), Ok(2)); + assert!(eventually(|| journal.stats().failures == 1), "{:?}", journal.stats()); + assert_eq!(journal.durable_lsn(), 1); + // A kill -9 while the writer waits to try again. + let restarted = storage.restart(Crash::Process); + drop(journal); + let (journal, _, seen) = open(&restarted); + assert_eq!(lsns(&seen), vec![1, 2], "seed {seed}"); + assert_eq!(journal.append(1, Class::Ack, b"c"), Ok(3)); + journal.wait_durable(3, WAIT).unwrap_or_else(|e| panic!("{e}")); + drop(journal); + let (_, report, seen) = open(&restarted.restart(Crash::Os)); + assert_eq!(lsns(&seen), vec![1, 2, 3], "seed {seed}: {report:?}"); + } } +/// F6: recovery opens one segment at a time, so a journal of many segments never reaches the +/// descriptor limit at open. #[test] -fn a_power_loss_keeps_every_fully_synced_record_and_a_prefix() { - crash_runs(Crash::PowerLoss); +fn recovery_holds_at_most_two_files_open() { + let storage = SimStorage::new(54); + let config = Config { segment_bytes: 200, ..quick() }; + let (journal, _, _) = open_at(&storage, config.clone(), 0).unwrap_or_else(|e| panic!("{e}")); + for lsn in 1..=60 { + let got = journal.append(3, Class::Ack, &payload(lsn)).unwrap_or_else(|e| panic!("{e}")); + journal.wait_durable(got, WAIT).unwrap_or_else(|e| panic!("{e}")); + } + assert!(journal.stats().segments_created >= 15, "{:?}", journal.stats()); + drop(journal); + let restarted = storage.restart(Crash::Process); + let (_journal, report, seen) = open_at(&restarted, config, 0).unwrap_or_else(|e| panic!("{e}")); + assert_eq!(seen.len(), 60, "{report:?}"); + assert!( + restarted.peak_open_files() <= 2, + "{} files open at once ({report:?})", + restarted.peak_open_files() + ); } // ---------------------------------------------------------------- real file system diff --git a/cmux-tui/crates/cmux-journal/src/crash_sim_tests.rs b/cmux-tui/crates/cmux-journal/src/crash_sim_tests.rs new file mode 100644 index 000000000000..c01b1aa00620 --- /dev/null +++ b/cmux-tui/crates/cmux-journal/src/crash_sim_tests.rs @@ -0,0 +1,379 @@ +//! The crash simulation (R-DURABILITY acceptance A1). In each epoch, clients append and wait for +//! acks while the storage crashes at a random operation, and sometimes fails one write or sync +//! before it. Then the next process opens the journal, and sometimes crashes inside that recovery +//! too, before a new epoch of appends. After each open that succeeds: +//! +//! - every record is exactly what was appended at its LSN (nothing half applied); +//! - every record an earlier open replayed comes back, first and in order; +//! - after them come the last epoch's records, as a prefix of its appends (a lost tail is a +//! suffix, never a hole); +//! - that prefix reaches every record a client was told is durable (fully durable, when a power +//! loss came since), and every written one when only processes crashed and no write or sync +//! failed; +//! - a crash is never reported as damage. +//! +//! `CMUX_JOURNAL_CRASH_SEEDS` seeds per test (default 200; the acceptance run in +//! R-DURABILITY/DESIGN.md section 7 uses 100000 on the fleet). A failing seed prints itself: +//! rerun it alone with `CMUX_JOURNAL_CRASH_SEED=`. +//! +//! The last tests check the simulated machine itself: the write-back error behavior and the +//! ordered log of names that the crash runs depend on. + +use std::collections::BTreeMap; +use std::io::ErrorKind; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +use crate::{ + Class, Config, Crash, Journal, Lsn, OpenError, OpenMode, OpenReport, SIM_PAGE, SimRng, + SimStorage, Storage, SyncLevel, WaitError, +}; + +/// LSN to (class, payload): what was appended, in every epoch. +type Model = Arc)>>>; +/// One record as a replay saw it: (lsn, kind, class, payload). +type Seen = Vec<(Lsn, u16, Class, Vec)>; + +const KIND: u16 = 4; + +fn dir() -> PathBuf { + PathBuf::from("/sim/journal") +} + +fn open(storage: &SimStorage, config: &Config) -> Result<(Journal, OpenReport, Seen), OpenError> { + let mut seen = Vec::new(); + let (journal, report) = + Journal::open(Arc::new(storage.clone()), &dir(), config.clone(), 0, &mut |record| { + seen.push((record.lsn, record.kind, record.class, record.payload.to_vec())); + Ok(()) + })?; + Ok((journal, report, seen)) +} + +/// What one epoch of appends leaves to check. +struct Epoch { + /// The LSN the epoch's first append took. + start: Lsn, + /// The highest acked, fully durable and written LSN, as the clients and the writer saw them. + durable: Lsn, + full_durable: Lsn, + written: Lsn, + /// A write or sync failed in the epoch (not by the crash). + failed: bool, +} + +fn rank(crash: Crash) -> u8 { + match crash { + Crash::Process => 0, + Crash::Os => 1, + Crash::PowerLoss => 2, + } +} + +/// The crash of the two that loses more. +fn worse(a: Crash, b: Crash) -> Crash { + if rank(b) > rank(a) { b } else { a } +} + +fn pick(kinds: &[Crash], rng: &mut SimRng) -> Crash { + kinds.get(rng.index(kinds.len())).copied().unwrap_or(Crash::Process) +} + +fn seeds() -> Vec { + if let Some(seed) = std::env::var("CMUX_JOURNAL_CRASH_SEED").ok().and_then(|s| s.parse().ok()) { + return vec![seed]; + } + let count = std::env::var("CMUX_JOURNAL_CRASH_SEEDS") + .ok() + .and_then(|s| s.parse().ok()) + .unwrap_or(200u64); + (0..count).collect() +} + +fn crash_run(seed: u64, kinds: &[Crash]) -> Result<(), String> { + let mut rng = SimRng::new(seed); + let mut storage = SimStorage::new(rng.next_u64()); + let config = Config { + segment_bytes: 300 + rng.below(3000), + max_pending_bytes: 1 << 20, + full_sync_interval: Duration::from_millis(rng.below(4)), + stream_sync_delay: Duration::from_millis(rng.below(3)), + retry_min: Duration::from_millis(1), + retry_max: Duration::from_millis(2), + }; + let model: Model = Arc::default(); + let mut kept: Vec = Vec::new(); + let mut last: Option = None; + let mut worst = Crash::Process; + let epochs = 1 + rng.below(3); + for epoch in 0..=epochs { + // The open, which may crash inside recovery; the next process tries again. + let mut attempts = 0; + let (journal, report, seen) = loop { + attempts += 1; + if attempts < 4 && rng.below(2) == 0 { + storage.crash_after(1 + rng.below(40)); + } + match open(&storage, &config) { + Ok(opened) => break opened, + Err(_) if storage.is_crashed() => { + let crash = pick(kinds, &mut rng); + worst = worse(worst, crash); + storage = storage.restart(crash); + } + Err(error) => return Err(format!("epoch {epoch}: open: {error}")), + } + }; + check(&seen, &report, &kept, last.as_ref(), worst, &model) + .map_err(|error| format!("epoch {epoch}: {error}"))?; + kept = seen.iter().map(|record| record.0).collect(); + if epoch == epochs { + return Ok(()); + } + // This epoch's crash point (it replaces one the open did not reach), and sometimes a + // failed write or sync before it. + storage.crash_after(1 + rng.below(80)); + if rng.below(2) == 0 { + let kind = if rng.below(2) == 0 { ErrorKind::StorageFull } else { ErrorKind::Other }; + storage.fail_after(1 + rng.below(40), kind); + } + let start = report.next_lsn; + let (durable, full_durable) = appends(seed ^ epoch, &journal, &storage, &model, &mut rng); + let stats = journal.stats(); + last = Some(Epoch { + start, + durable: durable.max(stats.durable_lsn), + full_durable: full_durable.max(stats.full_durable_lsn), + written: stats.written_lsn, + failed: storage.failures() > 0, + }); + drop(journal); + let crash = pick(kinds, &mut rng); + worst = crash; + storage = storage.restart(crash); + } + Ok(()) +} + +/// Clients append and wait for their acks until the storage crashes. Returns the highest acked +/// and the highest fully durable LSN a client saw. +fn appends( + seed: u64, + journal: &Journal, + storage: &SimStorage, + model: &Model, + rng: &mut SimRng, +) -> (Lsn, Lsn) { + let durable = AtomicU64::new(0); + let full_durable = AtomicU64::new(0); + let clients = 1 + rng.below(4); + std::thread::scope(|scope| { + for client in 0..clients { + let mut rng = SimRng::new(seed ^ (client + 1).wrapping_mul(0x9E37)); + let (durable, full_durable) = (&durable, &full_durable); + scope.spawn(move || { + for index in 0..30u64 { + let class = if rng.below(4) == 0 { Class::Stream } else { Class::Ack }; + // Now and then a record over several pages. + let size = + if rng.below(10) == 0 { 2000 + rng.index(8000) } else { rng.index(64) }; + let body = format!("{seed}-{client}-{index}-{}", "y".repeat(size)); + let lsn = { + // The append and the model's copy are one step, so the model never misses + // an LSN the journal took. + let mut model = model.lock().unwrap_or_else(|e| e.into_inner()); + let Ok(lsn) = journal.append(KIND, class, body.as_bytes()) else { break }; + model.insert(lsn, (class, body.into_bytes())); + lsn + }; + if class == Class::Ack { + if !acked(journal, storage, lsn) { + break; + } + durable.fetch_max(lsn, Ordering::SeqCst); + if journal.full_durable_lsn() >= lsn { + full_durable.fetch_max(lsn, Ordering::SeqCst); + } + } + } + }); + } + }); + (durable.load(Ordering::SeqCst), full_durable.load(Ordering::SeqCst)) +} + +/// Waits until `lsn` is durable; false once the storage crashed first. +fn acked(journal: &Journal, storage: &SimStorage, lsn: Lsn) -> bool { + let deadline = Instant::now() + Duration::from_secs(20); + loop { + match journal.wait_durable(lsn, Duration::from_millis(5)) { + Ok(()) => return true, + Err(WaitError::Timeout) if !storage.is_crashed() => { + assert!(Instant::now() < deadline, "the writer stalled: {:?}", journal.stats()); + } + Err(_) => return false, + } + } +} + +fn check( + seen: &Seen, + report: &OpenReport, + kept: &[Lsn], + last: Option<&Epoch>, + worst: Crash, + model: &Model, +) -> Result<(), String> { + if !report.damage.is_empty() { + return Err(format!("a crash is not damage: {report:?}")); + } + let model = model.lock().unwrap_or_else(|e| e.into_inner()); + for (lsn, kind, class, bytes) in seen { + match model.get(lsn) { + Some((want_class, want)) if want_class == class && want == bytes && *kind == KIND => {} + other => return Err(format!("record {lsn} is not what was appended: {other:?}")), + } + } + let got: Vec = seen.iter().map(|record| record.0).collect(); + if got.get(..kept.len()) != Some(kept) { + return Err(format!("records an earlier open replayed are gone: {kept:?}, then {got:?}")); + } + if report.next_lsn <= got.last().copied().unwrap_or(0) { + return Err(format!("the next LSN reuses a recovered one: {report:?}")); + } + let new = got.get(kept.len()..).unwrap_or_default(); + let Some(epoch) = last else { + return if new.is_empty() { Ok(()) } else { Err(format!("records from nowhere: {new:?}")) }; + }; + for (lsn, want) in new.iter().zip(epoch.start..) { + if *lsn != want { + return Err(format!("not a prefix of the appends from {}: {new:?}", epoch.start)); + } + } + let through = epoch.start.saturating_add(new.len() as u64).saturating_sub(1); + let must = match worst { + Crash::Process if !epoch.failed => epoch.durable.max(epoch.written), + Crash::Process | Crash::Os => epoch.durable, + Crash::PowerLoss => epoch.full_durable, + }; + if must >= epoch.start && through < must { + return Err(format!( + "record {must} lost: the appends from {} came back through {through} ({worst:?}, {report:?})", + epoch.start + )); + } + Ok(()) +} + +fn crash_runs(kinds: &[Crash]) { + let failures: Vec = seeds() + .into_iter() + .filter_map(|seed| { + crash_run(seed, kinds).err().map(|error| format!("seed {seed}: {error}")) + }) + .collect(); + assert!( + failures.is_empty(), + "{} {kinds:?} runs failed:\n{}", + failures.len(), + failures.join("\n") + ); +} + +#[test] +fn process_crashes_at_any_operation_keep_every_written_record() { + crash_runs(&[Crash::Process]); +} + +#[test] +fn os_crashes_at_any_operation_keep_every_acked_record() { + crash_runs(&[Crash::Os]); +} + +#[test] +fn power_losses_keep_every_fully_synced_record_and_a_prefix() { + crash_runs(&[Crash::PowerLoss]); +} + +#[test] +fn mixed_crashes_keep_what_the_worst_of_them_keeps() { + crash_runs(&[Crash::Process, Crash::Os, Crash::PowerLoss]); +} + +// ---------------------------------------------------------------- the simulated machine + +/// A folder with one empty file in it, all durable. +fn machine_with_file(seed: u64, path: &Path) -> (SimStorage, Box) { + let storage = SimStorage::new(seed); + let folder = path.parent().unwrap_or(Path::new("/")); + storage.create_dir_all(folder).unwrap_or_else(|e| panic!("mkdir: {e}")); + let file = storage.open(path, OpenMode::Create).unwrap_or_else(|e| panic!("create: {e}")); + for dir in folder.ancestors() { + storage.sync_dir(dir).unwrap_or_else(|e| panic!("sync {}: {e}", dir.display())); + } + (storage, file) +} + +#[test] +fn the_sim_never_writes_the_pages_a_failed_sync_dropped() { + let mut dropped = 0; + for seed in 0..64 { + let path = dir().join("file"); + let (storage, mut file) = machine_with_file(seed, &path); + let page = |byte: u8| vec![byte; SIM_PAGE]; + file.append(&page(1)).unwrap_or_else(|e| panic!("{e}")); + file.sync(SyncLevel::Full).unwrap_or_else(|e| panic!("{e}")); + file.append(&[page(2), page(2)].concat()).unwrap_or_else(|e| panic!("{e}")); + storage.fail_after(1, ErrorKind::Other); + assert!(file.sync(SyncLevel::Full).is_err()); + // A later sync reports success, and reads still see every byte. + file.sync(SyncLevel::Full).unwrap_or_else(|e| panic!("{e}")); + assert_eq!(storage.contents(&path).map(|bytes| bytes.len()), Some(3 * SIM_PAGE)); + let after = storage.restart(Crash::Os).contents(&path).unwrap_or_default(); + assert_eq!(after.get(..SIM_PAGE), Some(page(1).as_slice()), "seed {seed}"); + for chunk in after.get(SIM_PAGE..).unwrap_or_default().chunks(SIM_PAGE) { + // A page the failed sync wrote, or one it dropped, which never reached the disk. + assert!(chunk == page(2).as_slice() || chunk == page(0).as_slice(), "seed {seed}"); + dropped += usize::from(chunk == page(0).as_slice()); + } + } + assert!(dropped > 0, "no failed sync dropped a page"); +} + +#[test] +fn the_sim_keeps_a_rename_between_folders_whole() { + let (mut moved, mut stayed) = (0, 0); + for seed in 0..64 { + let from = dir().join("a").join("file"); + let to = dir().join("b").join("file"); + let (storage, _file) = machine_with_file(seed, &from); + storage.create_dir_all(&dir().join("b")).unwrap_or_else(|e| panic!("{e}")); + storage.sync_dir(&dir()).unwrap_or_else(|e| panic!("{e}")); + storage.rename(&from, &to).unwrap_or_else(|e| panic!("{e}")); + // No folder sync: an OS crash keeps the rename or not, never half of it. + let after = storage.restart(Crash::Os); + let a = after.names_in(&dir().join("a")).len(); + let b = after.names_in(&dir().join("b")).len(); + assert_eq!(a + b, 1, "seed {seed}: the file is in {a} + {b} folders"); + moved += b; + stayed += a; + } + assert!(moved > 0 && stayed > 0, "{moved} moved, {stayed} stayed"); +} + +#[test] +fn the_sim_keeps_a_rename_its_folder_sync_made_durable() { + for seed in 0..32 { + let from = dir().join("file.new"); + let to = dir().join("file"); + let (storage, _file) = machine_with_file(seed, &from); + storage.rename(&from, &to).unwrap_or_else(|e| panic!("{e}")); + storage.sync_dir(&dir()).unwrap_or_else(|e| panic!("{e}")); + for crash in [Crash::Os, Crash::PowerLoss] { + let after = storage.snapshot(crash); + assert_eq!(after.names_in(&dir()), vec!["file".to_owned()], "seed {seed}, {crash:?}"); + } + } +} diff --git a/cmux-tui/crates/cmux-journal/src/lib.rs b/cmux-tui/crates/cmux-journal/src/lib.rs index a6ce20bb4f80..388e1d687384 100644 --- a/cmux-tui/crates/cmux-journal/src/lib.rs +++ b/cmux-tui/crates/cmux-journal/src/lib.rs @@ -54,7 +54,7 @@ pub use format::{MAX_PAYLOAD_LEN, RECORD_HEADER_LEN}; pub use journal::{AppendError, Journal, JournalStats, OpenError, SealError, WaitError}; pub use recovery::{Damage, OpenReport}; #[cfg(any(test, feature = "sim"))] -pub use sim::{Crash, SimRng, SimStorage}; +pub use sim::{Crash, SIM_PAGE, SimRng, SimStorage}; pub use storage::{FsStorage, OpenMode, Storage, StorageFile, StorageLock, SyncLevel}; use std::time::Duration; @@ -136,3 +136,5 @@ impl Default for Config { #[cfg(test)] mod cmux_journal_tests; +#[cfg(test)] +mod crash_sim_tests; diff --git a/cmux-tui/crates/cmux-journal/src/sim.rs b/cmux-tui/crates/cmux-journal/src/sim.rs index 09d34a6b0819..93658eb7259c 100644 --- a/cmux-tui/crates/cmux-journal/src/sim.rs +++ b/cmux-tui/crates/cmux-journal/src/sim.rs @@ -1,30 +1,50 @@ -//! A simulated file system for crash tests (`sim` feature; always in this crate's tests). +//! A simulated machine for crash tests (`sim` feature; always in this crate's tests): a page +//! cache over a disk, and the one process that uses them. //! -//! Each file keeps its current bytes (what a read sees, and what a process crash keeps) apart -//! from the bytes of each sync. Each directory keeps its current entries apart from its synced -//! entries. [`SimStorage::crash_after`] makes the n-th storage operation from now the crash -//! point: an append there writes a random prefix of its bytes (a torn write), any other -//! operation does nothing, and every later operation fails. [`SimStorage::restart`] then builds -//! the storage that the next process sees: +//! Files: each file keeps what reads see (its page cache) apart from what its disk holds, per +//! [`SIM_PAGE`] page. A write dirties the pages it touches. A sync writes the dirty pages and the +//! length to the disk. A sync that fails ([`SimStorage::fail_after`]) writes a random part of the +//! dirty pages and drops the rest: those pages stay clean in the cache and never reach the disk, +//! so a later sync reports success without them (Linux's write-back error behavior). A write that +//! fails writes a random prefix of its bytes first. //! -//! - [`Crash::Process`] (kill -9 of any or all processes): every written byte and entry stays. -//! - [`Crash::Os`] (a kernel panic): each file keeps its content at its last sync, plus a random -//! prefix of what it got since (some of it zeroed or flipped: pages written back out of -//! order); each directory keeps its synced entries or, at random, its current ones. +//! Names: folder and file creations and renames go to one ordered log, as a journaling file +//! system commits them. A folder sync makes durable every logged change up to the last one that +//! changed an entry of that folder, and so every change before it. A rename between two folders +//! is one change: it never comes back twice or not at all. +//! +//! [`SimStorage::crash_after`] makes the n-th storage operation from now the crash point: an +//! append there writes a random prefix of its bytes, a create, rename, truncate or sync there +//! takes effect or not at random, and every later operation fails. [`SimStorage::restart`] then +//! builds the machine that the next process sees: +//! +//! - [`Crash::Process`] (kill -9 of any or all processes): the machine is unchanged: the cache, +//! the disk and the unsynced names stay as they are. Only the process and its locks are gone, +//! so a later OS crash or power loss still loses what it would have lost before. +//! - [`Crash::Os`] (a kernel panic): each file keeps its disk pages, and each dirty page is +//! written whole, not at all, or a prefix of its 512-byte sectors, at random; its length is +//! any length between its disk length and its cache length. The names keep a random prefix of +//! the log that holds every durable change. //! - [`Crash::PowerLoss`]: as `Os`, but a barrier sync is power-safe only once a later full sync -//! ran (on any file: a drive cache flush covers every earlier write), so the content of each -//! file goes back to a random common cut between the last full sync and the crash. +//! ran (on any file: a drive cache flush covers every earlier write). So each file goes back to +//! its disk content at a random common cut between the last full sync and the crash, and the +//! first sync after the cut is kept in part. //! //! A run is deterministic for one seed and one sequence of operations. use std::collections::{BTreeMap, BTreeSet}; use std::io; use std::path::{Path, PathBuf}; -use std::sync::{Arc, Mutex, PoisonError}; +use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; use std::time::Duration; use crate::storage::{OpenMode, Storage, StorageFile, StorageLock, SyncLevel}; +/// The page size of the simulated cache. +pub const SIM_PAGE: usize = 4096; +/// The unit a page write tears at. +const SECTOR: usize = 512; + /// What a simulated crash takes away ([`SimStorage::restart`]). #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum Crash { @@ -59,51 +79,348 @@ impl SimRng { pub fn index(&mut self, n: usize) -> usize { usize::try_from(self.below(u64::try_from(n).unwrap_or(u64::MAX))).unwrap_or(0) } + + fn coin(&mut self) -> bool { + self.below(2) == 0 + } } -/// The simulated file system ([module docs](self)). Clones share one file system. +/// The simulated storage of one process ([module docs](self)). Clones share it. #[derive(Clone)] pub struct SimStorage { fs: Arc>, } +#[derive(Clone)] struct SimFile { + /// What reads see: the page cache. data: Vec, - /// (global sync number, content) at each sync, oldest first. + /// What the disk holds. + disk: Vec, + /// Pages of `data` written since the disk got them. + dirty: BTreeSet, + /// The disk content after each good sync, with its machine-wide sync number (for a power + /// loss). A file the last crash kept starts with its content as sync 0. synced: Vec<(u64, Vec)>, } -struct Fs { +impl SimFile { + fn empty() -> Self { + Self { data: Vec::new(), disk: Vec::new(), dirty: BTreeSet::new(), synced: Vec::new() } + } + + fn kept(content: Vec) -> Self { + Self { + data: content.clone(), + disk: content.clone(), + dirty: BTreeSet::new(), + synced: vec![(0, content)], + } + } + + /// Marks the pages of bytes `start..end` dirty. + fn touch(&mut self, start: usize, end: usize) { + if start < end { + self.dirty.extend(start / SIM_PAGE..end.div_ceil(SIM_PAGE)); + } + } + + fn append(&mut self, bytes: &[u8]) { + let start = self.data.len(); + self.data.extend_from_slice(bytes); + self.touch(start, self.data.len()); + } + + /// A shortening or an extension with zeros. The new length reaches the disk with the next + /// sync (`write_back`); the bytes an extension adds are dirty. + fn set_len(&mut self, len: usize) { + let old = self.data.len(); + self.data.resize(len, 0); + self.dirty.retain(|page| page.saturating_mul(SIM_PAGE) < len); + self.touch(old, len); + } + + /// Writes `pages` of the cache and its length to the disk. + fn write_back(&mut self, pages: &[usize]) { + self.disk.resize(self.data.len(), 0); + for page in pages { + let start = page.saturating_mul(SIM_PAGE).min(self.data.len()); + let end = start.saturating_add(SIM_PAGE).min(self.data.len()); + if let (Some(dst), Some(src)) = + (self.disk.get_mut(start..end), self.data.get(start..end)) + { + dst.copy_from_slice(src); + } + } + } + + /// A sync that completed: every dirty page and the length are on the disk. + fn good_sync(&mut self, number: u64, last_full: u64) { + let pages: Vec = self.dirty.iter().copied().collect(); + self.write_back(&pages); + self.dirty.clear(); + // Only the last content at or before the last full sync can come back after a crash. + if let Some(keep) = self.synced.iter().rposition(|(sync, _)| *sync <= last_full) { + self.synced.drain(..keep); + } + self.synced.push((number, self.disk.clone())); + } + + /// A sync that failed: a random part of the dirty pages reached the disk; the cache dropped + /// the rest as clean, so no later sync writes them. + fn failed_sync(&mut self, rng: &mut SimRng) { + let pages: Vec = self.dirty.iter().copied().filter(|_| rng.coin()).collect(); + self.write_back(&pages); + self.dirty.clear(); + } + + /// What an OS crash could leave: the cache's dirty pages, and the disk's pages elsewhere + /// (a page a failed sync dropped never reached the disk). + fn os_view(&self) -> Vec { + let mut out = self.data.clone(); + for (page, chunk) in out.chunks_mut(SIM_PAGE).enumerate() { + if self.dirty.contains(&page) { + continue; + } + let start = page.saturating_mul(SIM_PAGE); + for (offset, byte) in chunk.iter_mut().enumerate() { + *byte = self.disk.get(start.saturating_add(offset)).copied().unwrap_or(0); + } + } + out + } + + /// The content after a crash: `cut` is the power-loss cut (none for an OS crash). + fn after_crash(&self, cut: Option, rng: &mut SimRng) -> Vec { + match cut { + None => mix(&self.disk, &self.os_view(), rng), + Some(cut) => { + let base = self + .synced + .iter() + .rev() + .find(|(sync, _)| *sync <= cut) + .map(|(_, content)| content.clone()) + .unwrap_or_default(); + let next = match self.synced.iter().find(|(sync, _)| *sync > cut) { + Some((_, content)) => content.clone(), + None => self.os_view(), + }; + mix(&base, &next, rng) + } + } + } +} + +/// `old` with some of `new`'s writes: a length between both lengths, and each page that differs +/// whole from `old`, whole from `new`, or `new`'s first sectors over `old`. Bytes past `new`'s end +/// (a shortening that did not reach the disk) stay `old`'s; bytes past both read as zeros. +fn mix(old: &[u8], new: &[u8], rng: &mut SimRng) -> Vec { + let (short, long) = (old.len().min(new.len()), old.len().max(new.len())); + let len = short.saturating_add(rng.index(long.saturating_sub(short).saturating_add(1))); + let old_at = |at: usize| old.get(at).copied().unwrap_or(0); + let new_at = |at: usize| new.get(at).copied().unwrap_or_else(|| old_at(at)); + let mut out = Vec::with_capacity(len); + let mut start = 0; + while start < len { + let end = start.saturating_add(SIM_PAGE).min(len); + let differs = (start..end).any(|at| old_at(at) != new_at(at)); + let new_until = match (differs, rng.below(3)) { + (false, _) | (true, 0) => start, + (true, 1) => end, + (true, _) => start.saturating_add(rng.index(SIM_PAGE / SECTOR).saturating_mul(SECTOR)), + }; + out.extend((start..end).map(|at| if at < new_until { new_at(at) } else { old_at(at) })); + start = end; + } + out +} + +/// A change of names, in the order the file system logs it. +#[derive(Clone, Debug)] +enum Change { + Folder(PathBuf), + Create(PathBuf, u64), + Rename(PathBuf, PathBuf), + Remove(PathBuf), +} + +impl Change { + /// Whether the change made or removed an entry of `dir`. + fn in_folder(&self, dir: &Path) -> bool { + let entry_of = |path: &PathBuf| path.parent() == Some(dir); + match self { + Self::Folder(path) | Self::Create(path, _) | Self::Remove(path) => entry_of(path), + Self::Rename(from, to) => entry_of(from) || entry_of(to), + } + } +} + +#[derive(Clone, Default)] +struct Names { + files: BTreeMap, + dirs: BTreeSet, +} + +impl Names { + fn apply(&mut self, change: &Change) { + match change { + Change::Folder(path) => { + self.dirs.insert(path.clone()); + } + Change::Create(path, inode) => { + self.files.insert(path.clone(), *inode); + } + Change::Rename(from, to) => { + if let Some(inode) = self.files.remove(from) { + self.files.insert(to.clone(), inode); + } + } + Change::Remove(path) => { + self.files.remove(path); + } + } + } +} + +/// What a process crash keeps: the cache, the disk and the names. +#[derive(Clone)] +struct Machine { rng: SimRng, files: BTreeMap, next_inode: u64, - /// Current and synced directory entries: path to inode. - names: BTreeMap, - durable_names: BTreeMap, - dirs: BTreeSet, + /// The names at the last crash (all durable), the changes since, how many of them are + /// durable, and the names now. + base: Names, + log: Vec, + durable_changes: usize, + names: Names, + /// Syncs so far (file and folder), and the number of the last full one. + syncs: u64, + last_full: u64, +} + +impl Machine { + fn new(seed: u64) -> Self { + Self { + rng: SimRng::new(seed), + files: BTreeMap::new(), + next_inode: 1, + base: Names::default(), + log: Vec::new(), + durable_changes: 0, + names: Names::default(), + syncs: 0, + last_full: 0, + } + } + + fn change(&mut self, change: Change) { + self.names.apply(&change); + self.log.push(change); + } + + fn sync_number(&mut self, level: SyncLevel) -> u64 { + self.syncs = self.syncs.saturating_add(1); + if level == SyncLevel::Full { + self.last_full = self.syncs; + } + self.syncs + } + + fn sync_folder(&mut self, dir: &Path) { + if let Some(last) = self.log.iter().rposition(|change| change.in_folder(dir)) { + self.durable_changes = self.durable_changes.max(last.saturating_add(1)); + } + // A folder sync flushes the drive like a full sync (FsStorage uses F_FULLFSYNC). + self.sync_number(SyncLevel::Full); + } + + fn file(&mut self, inode: u64) -> io::Result<&mut SimFile> { + self.files.get_mut(&inode).ok_or_else(|| io::Error::other("simulated file is gone")) + } + + /// The machine after an OS crash or a power loss ([module docs](self)). + fn after_crash(&mut self, crash: Crash) -> Self { + let mut rng = SimRng::new(self.rng.next_u64()); + let unsure = self.log.len().saturating_sub(self.durable_changes); + let kept = self.durable_changes.saturating_add(rng.index(unsure.saturating_add(1))); + let mut names = self.base.clone(); + for change in self.log.iter().take(kept) { + names.apply(change); + } + let cut = match crash { + Crash::Process | Crash::Os => None, + Crash::PowerLoss => { + let span = self.syncs.saturating_sub(self.last_full); + Some(self.last_full.saturating_add(rng.below(span.saturating_add(1)))) + } + }; + let mut files = BTreeMap::new(); + for inode in names.files.values() { + if let (false, Some(file)) = (files.contains_key(inode), self.files.get(inode)) { + files.insert(*inode, SimFile::kept(file.after_crash(cut, &mut rng))); + } + } + Self { + rng, + files, + next_inode: self.next_inode, + base: names.clone(), + log: Vec::new(), + durable_changes: 0, + names, + syncs: 0, + last_full: 0, + } + } +} + +/// One process: its view of the machine. +struct Fs { + machine: Machine, locks: BTreeSet, - /// Storage operations so far, and the one that crashes. + /// Storage operations so far, the one that crashes, and one that fails once. ops: u64, crash_at: Option, - /// One operation that fails with this error, once. fail_at: Option<(u64, io::ErrorKind)>, crashed: bool, - /// Syncs so far (file and directory), and the number of the last full one. - syncs: u64, - last_full: u64, /// Each sync sleeps this long (a slow disk, so batches form). sync_delay: Duration, /// Read operations per path. reads: BTreeMap, + /// Open files now, and the most at once. + open_files: u64, + peak_open_files: u64, + /// Failures `fail_after` caused. + failures: u64, } +/// What an operation does: run, fail once, or crash the process. enum Step { Run, + Fail(io::ErrorKind), Crash, } impl Fs { - /// Counts one operation and says whether it runs, crashes or fails. + fn new(machine: Machine, sync_delay: Duration) -> Self { + Self { + machine, + locks: BTreeSet::new(), + ops: 0, + crash_at: None, + fail_at: None, + crashed: false, + sync_delay, + reads: BTreeMap::new(), + open_files: 0, + peak_open_files: 0, + failures: 0, + } + } + + /// Counts one operation and says what it does. fn begin(&mut self) -> io::Result { if self.crashed { return Err(io::Error::other("simulated crash: the process is gone")); @@ -117,53 +434,59 @@ impl Fs { && at == self.ops { self.fail_at = None; - return Err(io::Error::new(kind, "simulated I/O failure")); + self.failures = self.failures.saturating_add(1); + return Ok(Step::Fail(kind)); } Ok(Step::Run) } - fn file(&mut self, inode: u64) -> io::Result<&mut SimFile> { - self.files.get_mut(&inode).ok_or_else(|| io::Error::other("simulated file is gone")) + /// A change of names: done, failed with no effect, or at a crash done or not at random. + fn change(&mut self, step: Step, change: Change) -> io::Result<()> { + match step { + Step::Run => { + self.machine.change(change); + Ok(()) + } + Step::Fail(kind) => Err(failed(kind)), + Step::Crash => { + if self.machine.rng.coin() { + self.machine.change(change); + } + Err(crashed()) + } + } } +} - fn sync_number(&mut self, level: SyncLevel) -> u64 { - self.syncs = self.syncs.saturating_add(1); - if level == SyncLevel::Full { - self.last_full = self.syncs; +/// Sleeps the sync delay with the lock released; `None` when the process crashed meanwhile. +fn slow_sync<'a>(mut guard: MutexGuard<'a, Fs>, fs: &'a Mutex) -> Option> { + let delay = guard.sync_delay; + if !delay.is_zero() { + drop(guard); + std::thread::sleep(delay); + guard = lock(fs); + if guard.crashed { + return None; } - self.syncs } + Some(guard) } fn crashed() -> io::Error { io::Error::other("simulated crash") } -fn lock(fs: &Mutex) -> std::sync::MutexGuard<'_, Fs> { +fn failed(kind: io::ErrorKind) -> io::Error { + io::Error::new(kind, "simulated I/O failure") +} + +fn lock(fs: &Mutex) -> MutexGuard<'_, Fs> { fs.lock().unwrap_or_else(PoisonError::into_inner) } impl SimStorage { pub fn new(seed: u64) -> Self { - Self { - fs: Arc::new(Mutex::new(Fs { - rng: SimRng::new(seed), - files: BTreeMap::new(), - next_inode: 1, - names: BTreeMap::new(), - durable_names: BTreeMap::new(), - dirs: BTreeSet::new(), - locks: BTreeSet::new(), - ops: 0, - crash_at: None, - fail_at: None, - crashed: false, - syncs: 0, - last_full: 0, - sync_delay: Duration::ZERO, - reads: BTreeMap::new(), - })), - } + Self { fs: Arc::new(Mutex::new(Fs::new(Machine::new(seed), Duration::ZERO))) } } /// The `n`-th storage operation from now (1 = the next one) is the crash point. @@ -172,7 +495,8 @@ impl SimStorage { fs.crash_at = Some(fs.ops.saturating_add(n.max(1))); } - /// The `n`-th storage operation from now fails once with `kind` (ENOSPC, EIO). + /// The `n`-th storage operation from now fails once with `kind` (ENOSPC, EIO), with the + /// partial effect the [module docs](self) name. pub fn fail_after(&self, n: u64, kind: io::ErrorKind) { let mut fs = lock(&self.fs); fs.fail_at = Some((fs.ops.saturating_add(n.max(1)), kind)); @@ -197,32 +521,45 @@ impl SimStorage { lock(&self.fs).reads.get(path).copied().unwrap_or(0) } - /// The current bytes of `path`, if it exists. + /// Operations that `fail_after` made fail. + pub fn failures(&self) -> u64 { + lock(&self.fs).failures + } + + /// The most files this process held open at once. + pub fn peak_open_files(&self) -> u64 { + lock(&self.fs).peak_open_files + } + + /// What a read of `path` sees now, if it exists. pub fn contents(&self, path: &Path) -> Option> { let fs = lock(&self.fs); - let inode = fs.names.get(path)?; - fs.files.get(inode).map(|file| file.data.clone()) + let inode = fs.machine.names.files.get(path)?; + fs.machine.files.get(inode).map(|file| file.data.clone()) } - /// The names in `dir`, sorted. + /// The file names in `dir`, sorted. pub fn names_in(&self, dir: &Path) -> Vec { let fs = lock(&self.fs); - fs.names + fs.machine + .names + .files .keys() .filter(|path| path.parent() == Some(dir)) .filter_map(|path| path.file_name()?.to_str().map(str::to_owned)) .collect() } - /// Flips one durable and current byte of `path` (media damage), outside any crash. + /// Flips one byte of `path` in the cache, on the disk and in every synced copy (media + /// damage), outside any crash. pub fn corrupt(&self, path: &Path, offset: usize) { let mut fs = lock(&self.fs); - let Some(inode) = fs.names.get(path).copied() else { return }; - if let Some(file) = fs.files.get_mut(&inode) { - if let Some(byte) = file.data.get_mut(offset) { - *byte ^= 0x5A; - } - for (_, content) in &mut file.synced { + let Some(inode) = fs.machine.names.files.get(path).copied() else { return }; + if let Some(file) = fs.machine.files.get_mut(&inode) { + let copies = std::iter::once(&mut file.data) + .chain(std::iter::once(&mut file.disk)) + .chain(file.synced.iter_mut().map(|(_, content)| content)); + for content in copies { if let Some(byte) = content.get_mut(offset) { *byte ^= 0x5A; } @@ -233,13 +570,14 @@ impl SimStorage { /// Removes `path` and makes the removal durable (a person deleted the file). pub fn remove(&self, path: &Path) { let mut fs = lock(&self.fs); - if let Some(inode) = fs.names.remove(path) { - fs.durable_names.remove(path); - fs.files.remove(&inode); + if let Some(inode) = fs.machine.names.files.get(path).copied() { + fs.machine.change(Change::Remove(path.to_path_buf())); + fs.machine.durable_changes = fs.machine.log.len(); + fs.machine.files.remove(&inode); } } - /// The storage the next process sees after `crash` ([module docs](self)). This one is + /// The storage the next process sees after `crash` ([module docs](self)). This process is /// crashed from now on: every later operation fails. pub fn restart(&self, crash: Crash) -> Self { lock(&self.fs).crashed = true; @@ -248,172 +586,120 @@ impl SimStorage { /// The storage a process would see after `crash` now, while this one goes on. pub fn snapshot(&self, crash: Crash) -> Self { - let mut old = lock(&self.fs); - let seed = old.rng.next_u64(); - let mut rng = SimRng::new(seed); - let cut = match crash { - Crash::Process | Crash::Os => None, - Crash::PowerLoss => { - let span = old.syncs.saturating_sub(old.last_full); - Some(old.last_full.saturating_add(rng.below(span.saturating_add(1)))) + let mut fs = lock(&self.fs); + let delay = fs.sync_delay; + let machine = match crash { + Crash::Process => { + let mut machine = fs.machine.clone(); + // The two processes draw apart from here. + machine.rng = SimRng::new(fs.machine.rng.next_u64()); + machine } + Crash::Os | Crash::PowerLoss => fs.machine.after_crash(crash), }; - let names = match crash { - Crash::Process => old.names.clone(), - Crash::Os | Crash::PowerLoss => surviving_names(&old, &mut rng), - }; - let mut files = BTreeMap::new(); - for inode in names.values() { - if files.contains_key(inode) { - continue; - } - let Some(file) = old.files.get(inode) else { continue }; - let data = match crash { - Crash::Process => file.data.clone(), - Crash::Os | Crash::PowerLoss => surviving_bytes(file, cut, &mut rng), - }; - files.insert(*inode, SimFile { synced: vec![(0, data.clone())], data }); - } - let next = Fs { - rng, - files, - next_inode: old.next_inode, - durable_names: names.clone(), - names, - dirs: old.dirs.clone(), - locks: BTreeSet::new(), - ops: 0, - crash_at: None, - fail_at: None, - crashed: false, - syncs: 0, - last_full: 0, - sync_delay: old.sync_delay, - reads: BTreeMap::new(), - }; - Self { fs: Arc::new(Mutex::new(next)) } - } -} - -/// Each directory's synced entries, or at random its current ones (its rename or create may -/// have reached the disk). -fn surviving_names(fs: &Fs, rng: &mut SimRng) -> BTreeMap { - let mut parents = BTreeSet::new(); - for path in fs.names.keys().chain(fs.durable_names.keys()) { - if let Some(parent) = path.parent() { - parents.insert(parent.to_path_buf()); - } - } - let mut out = BTreeMap::new(); - for parent in parents { - let in_dir = |path: &&PathBuf| path.parent() == Some(parent.as_path()); - let source = if rng.below(2) == 0 { &fs.durable_names } else { &fs.names }; - for (path, inode) in source.iter().filter(|(path, _)| in_dir(path)) { - out.insert(path.clone(), *inode); - } + Self { fs: Arc::new(Mutex::new(Fs::new(machine, delay))) } } - out } -/// The content at the last sync (at or before `cut`, for a power loss), plus a random prefix of -/// the bytes appended since, of which a random tail may be zeroed or one byte flipped. -fn surviving_bytes(file: &SimFile, cut: Option, rng: &mut SimRng) -> Vec { - let base = file - .synced - .iter() - .rev() - .find(|(number, _)| cut.is_none_or(|cut| *number <= cut)) - .map(|(_, content)| content.clone()) - .unwrap_or_default(); - let Some(extra) = file.data.strip_prefix(base.as_slice()) else { - // A truncation since the sync: it may not have reached the disk. - return base; - }; - let keep = rng.index(extra.len().saturating_add(1)); - let mut out = base.clone(); - out.extend_from_slice(extra.get(..keep).unwrap_or_default()); - if keep > 0 { - match rng.below(3) { - 0 => { - let from = base.len().saturating_add(rng.index(keep)); - if let Some(tail) = out.get_mut(from..) { - tail.fill(0); +impl Storage for SimStorage { + fn create_dir_all(&self, dir: &Path) -> io::Result<()> { + let mut fs = lock(&self.fs); + let step = fs.begin()?; + let mut missing: Vec = dir + .ancestors() + .filter(|path| !fs.machine.names.dirs.contains(*path)) + .map(Path::to_path_buf) + .collect(); + missing.reverse(); + match step { + Step::Run => { + for path in missing { + fs.machine.change(Change::Folder(path)); } + Ok(()) } - 1 => { - let at = base.len().saturating_add(rng.index(keep)); - if let Some(byte) = out.get_mut(at) { - *byte ^= 0xA5; + Step::Fail(kind) => Err(failed(kind)), + Step::Crash => { + let made = fs.machine.rng.index(missing.len().saturating_add(1)); + for path in missing.into_iter().take(made) { + fs.machine.change(Change::Folder(path)); } + Err(crashed()) } - _ => {} - } - } - out -} - -impl Storage for SimStorage { - fn create_dir_all(&self, dir: &Path) -> io::Result<()> { - let mut fs = lock(&self.fs); - if let Step::Crash = fs.begin()? { - return Err(crashed()); - } - for ancestor in dir.ancestors() { - fs.dirs.insert(ancestor.to_path_buf()); } - Ok(()) } fn list(&self, dir: &Path) -> io::Result> { let mut fs = lock(&self.fs); - if let Step::Crash = fs.begin()? { - return Err(crashed()); + match fs.begin()? { + Step::Run => {} + Step::Fail(kind) => return Err(failed(kind)), + Step::Crash => return Err(crashed()), } - if !fs.dirs.contains(dir) { + let names = &fs.machine.names; + if !names.dirs.contains(dir) { return Err(io::ErrorKind::NotFound.into()); } - let mut out = Vec::new(); - for path in fs.names.keys() { - if path.parent() == Some(dir) - && let Some(name) = path.file_name().and_then(|name| name.to_str()) - { - out.push(name.to_owned()); - } - } - for path in &fs.dirs { - if path.parent() == Some(dir) - && let Some(name) = path.file_name().and_then(|name| name.to_str()) - { - out.push(name.to_owned()); - } - } - Ok(out) + let in_dir = |path: &&PathBuf| path.parent() == Some(dir); + Ok(names + .files + .keys() + .filter(in_dir) + .chain(names.dirs.iter().filter(in_dir)) + .filter_map(|path| path.file_name()?.to_str().map(str::to_owned)) + .collect()) } fn open(&self, path: &Path, mode: OpenMode) -> io::Result> { let mut fs = lock(&self.fs); - if let Step::Crash = fs.begin()? { - return Err(crashed()); - } + let step = fs.begin()?; let parent = path.parent().map(Path::to_path_buf).unwrap_or_default(); - if !fs.dirs.contains(&parent) { - return Err(io::ErrorKind::NotFound.into()); + if !fs.machine.names.dirs.contains(&parent) { + return match step { + Step::Crash => Err(crashed()), + Step::Run | Step::Fail(_) => Err(io::ErrorKind::NotFound.into()), + }; } - let inode = match (fs.names.get(path).copied(), mode) { + let existing = fs.machine.names.files.get(path).copied(); + let inode = match (existing, mode) { + (None, OpenMode::Read | OpenMode::Write) => { + return match step { + Step::Crash => Err(crashed()), + Step::Run | Step::Fail(_) => Err(io::ErrorKind::NotFound.into()), + }; + } + (Some(inode), OpenMode::Read | OpenMode::Write) => { + match step { + Step::Run => {} + Step::Fail(kind) => return Err(failed(kind)), + Step::Crash => return Err(crashed()), + } + inode + } (Some(inode), OpenMode::Create) => { - fs.file(inode)?.data.clear(); + let empty = match step { + Step::Run => true, + Step::Fail(kind) => return Err(failed(kind)), + Step::Crash => fs.machine.rng.coin(), + }; + if empty { + fs.machine.file(inode)?.set_len(0); + } + if let Step::Crash = step { + return Err(crashed()); + } inode } - (Some(inode), OpenMode::Read | OpenMode::Write) => inode, (None, OpenMode::Create) => { - let inode = fs.next_inode; - fs.next_inode = inode.saturating_add(1); - fs.files.insert(inode, SimFile { data: Vec::new(), synced: Vec::new() }); - fs.names.insert(path.to_path_buf(), inode); + let inode = fs.machine.next_inode; + fs.machine.next_inode = inode.saturating_add(1); + fs.machine.files.insert(inode, SimFile::empty()); + fs.change(step, Change::Create(path.to_path_buf(), inode))?; inode } - (None, OpenMode::Read | OpenMode::Write) => return Err(io::ErrorKind::NotFound.into()), }; + fs.open_files = fs.open_files.saturating_add(1); + fs.peak_open_files = fs.peak_open_files.max(fs.open_files); Ok(Box::new(SimHandle { fs: self.fs.clone(), inode, @@ -424,49 +710,41 @@ impl Storage for SimStorage { fn rename(&self, from: &Path, to: &Path) -> io::Result<()> { let mut fs = lock(&self.fs); - if let Step::Crash = fs.begin()? { - return Err(crashed()); - } + let step = fs.begin()?; let parent = to.parent().map(Path::to_path_buf).unwrap_or_default(); - if !fs.dirs.contains(&parent) { - return Err(io::ErrorKind::NotFound.into()); + if !fs.machine.names.dirs.contains(&parent) || !fs.machine.names.files.contains_key(from) { + return match step { + Step::Crash => Err(crashed()), + Step::Run | Step::Fail(_) => Err(io::ErrorKind::NotFound.into()), + }; } - let inode = fs.names.remove(from).ok_or(io::ErrorKind::NotFound)?; - fs.names.insert(to.to_path_buf(), inode); - Ok(()) + fs.change(step, Change::Rename(from.to_path_buf(), to.to_path_buf())) } fn sync_dir(&self, dir: &Path) -> io::Result<()> { let mut fs = lock(&self.fs); - if let Step::Crash = fs.begin()? { - return Err(crashed()); - } - let delay = fs.sync_delay; - if !delay.is_zero() { - drop(fs); - std::thread::sleep(delay); - fs = lock(&self.fs); - if fs.crashed { - return Err(crashed()); + let done = match fs.begin()? { + Step::Run => true, + Step::Fail(kind) => return Err(failed(kind)), + Step::Crash => fs.machine.rng.coin(), + }; + if fs.crashed { + if done { + fs.machine.sync_folder(dir); } + return Err(crashed()); } - fs.durable_names.retain(|path, _| path.parent() != Some(dir)); - let current: Vec<(PathBuf, u64)> = fs - .names - .iter() - .filter(|(path, _)| path.parent() == Some(dir)) - .map(|(path, inode)| (path.clone(), *inode)) - .collect(); - fs.durable_names.extend(current); - // A directory sync flushes the drive like a full sync (FsStorage uses F_FULLFSYNC). - fs.sync_number(SyncLevel::Full); + let Some(mut fs) = slow_sync(fs, &self.fs) else { return Err(crashed()) }; + fs.machine.sync_folder(dir); Ok(()) } fn try_lock(&self, path: &Path) -> io::Result>> { let mut fs = lock(&self.fs); - if let Step::Crash = fs.begin()? { - return Err(crashed()); + match fs.begin()? { + Step::Run => {} + Step::Fail(kind) => return Err(failed(kind)), + Step::Crash => return Err(crashed()), } if !fs.locks.insert(path.to_path_buf()) { return Ok(None); @@ -501,23 +779,34 @@ impl SimHandle { } } +impl Drop for SimHandle { + fn drop(&mut self) { + let mut fs = lock(&self.fs); + fs.open_files = fs.open_files.saturating_sub(1); + } +} + impl StorageFile for SimHandle { fn size(&self) -> io::Result { let mut fs = lock(&self.fs); - if let Step::Crash = fs.begin()? { - return Err(crashed()); + match fs.begin()? { + Step::Run => {} + Step::Fail(kind) => return Err(failed(kind)), + Step::Crash => return Err(crashed()), } - Ok(u64::try_from(fs.file(self.inode)?.data.len()).unwrap_or(u64::MAX)) + Ok(u64::try_from(fs.machine.file(self.inode)?.data.len()).unwrap_or(u64::MAX)) } fn read_at(&self, offset: u64, buf: &mut [u8]) -> io::Result<()> { let mut fs = lock(&self.fs); - if let Step::Crash = fs.begin()? { - return Err(crashed()); + match fs.begin()? { + Step::Run => {} + Step::Fail(kind) => return Err(failed(kind)), + Step::Crash => return Err(crashed()), } let count = fs.reads.entry(self.path.clone()).or_default(); *count = count.saturating_add(1); - let file = fs.file(self.inode)?; + let file = fs.machine.file(self.inode)?; let start = usize::try_from(offset).map_err(|_| io::ErrorKind::UnexpectedEof)?; let end = start.checked_add(buf.len()).ok_or(io::ErrorKind::UnexpectedEof)?; let src = file.data.get(start..end).ok_or(io::ErrorKind::UnexpectedEof)?; @@ -528,48 +817,61 @@ impl StorageFile for SimHandle { fn append(&mut self, bytes: &[u8]) -> io::Result<()> { self.check_writable()?; let mut fs = lock(&self.fs); - match fs.begin()? { - Step::Run => { - fs.file(self.inode)?.data.extend_from_slice(bytes); - Ok(()) - } - Step::Crash => { - let keep = fs.rng.index(bytes.len().saturating_add(1)); - fs.file(self.inode)?.data.extend_from_slice(bytes.get(..keep).unwrap_or_default()); - Err(crashed()) + let step = fs.begin()?; + let written = match step { + Step::Run => bytes, + Step::Fail(_) | Step::Crash => { + let keep = fs.machine.rng.index(bytes.len().saturating_add(1)); + bytes.get(..keep).unwrap_or_default() } + }; + fs.machine.file(self.inode)?.append(written); + match step { + Step::Run => Ok(()), + Step::Fail(kind) => Err(failed(kind)), + Step::Crash => Err(crashed()), } } fn truncate(&mut self, size: u64) -> io::Result<()> { self.check_writable()?; let mut fs = lock(&self.fs); - if let Step::Crash = fs.begin()? { - return Err(crashed()); - } + let done = match fs.begin()? { + Step::Run => true, + Step::Fail(kind) => return Err(failed(kind)), + Step::Crash => fs.machine.rng.coin(), + }; let size = usize::try_from(size).map_err(|_| io::ErrorKind::InvalidInput)?; - fs.file(self.inode)?.data.truncate(size); - Ok(()) + if done { + fs.machine.file(self.inode)?.set_len(size); + } + if fs.crashed { Err(crashed()) } else { Ok(()) } } fn sync(&mut self, level: SyncLevel) -> io::Result<()> { let mut fs = lock(&self.fs); - if let Step::Crash = fs.begin()? { - return Err(crashed()); - } - let delay = fs.sync_delay; - if !delay.is_zero() { - drop(fs); - std::thread::sleep(delay); - fs = lock(&self.fs); - if fs.crashed { - return Err(crashed()); + let done = match fs.begin()? { + Step::Run => true, + Step::Fail(kind) => { + let Fs { machine, .. } = &mut *fs; + let file = machine + .files + .get_mut(&self.inode) + .ok_or_else(|| failed(io::ErrorKind::Other))?; + file.failed_sync(&mut machine.rng); + return Err(failed(kind)); } + Step::Crash => fs.machine.rng.coin(), + }; + if !fs.crashed { + let Some(guard) = slow_sync(fs, &self.fs) else { return Err(crashed()) }; + fs = guard; } - let number = fs.sync_number(level); - let file = fs.file(self.inode)?; - let content = file.data.clone(); - file.synced.push((number, content)); - Ok(()) + if done { + let number = fs.machine.sync_number(level); + let last_full = fs.machine.last_full; + fs.machine.file(self.inode)?.good_sync(number, last_full); + } + if fs.crashed { Err(crashed()) } else { Ok(()) } } } From 263da86f2235a01bb4c28e1c47eafa2ba4cfdf4f Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 3 Nov 2026 19:55:43 +0000 Subject: [PATCH 07/52] cmux-journal: the F2 regression damages the header of the last of several segments (red) The reviewer's F2 scenario has two segments: S1 holds 1..10, S2 (the last) holds 11..20, and S2's header is damaged. The test now rotates segments (300-byte segments, 20 acked records) and damages the last one's header, so the report must say lost_through 20 and the next append must take 21. On the code before the fix the moved segment's LSNs read as 0, so the next append reuses one. --- .../cmux-journal/src/cmux_journal_tests.rs | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/cmux-tui/crates/cmux-journal/src/cmux_journal_tests.rs b/cmux-tui/crates/cmux-journal/src/cmux_journal_tests.rs index c8652102c377..4f13d15537ee 100644 --- a/cmux-tui/crates/cmux-journal/src/cmux_journal_tests.rs +++ b/cmux-tui/crates/cmux-journal/src/cmux_journal_tests.rs @@ -498,15 +498,14 @@ fn a_crash_inside_the_quarantine_step_never_reuses_a_quarantined_lsn() { #[test] fn a_damaged_header_in_the_last_segment_never_reuses_its_lsns() { let storage = SimStorage::new(51); - let mut next = 1; - for _ in 0..2 { - let (journal, _, _) = open(&storage); - for _ in 0..10 { - assert_eq!(journal.append(3, Class::Ack, &payload(next)), Ok(next)); - next += 1; - } - assert_eq!(journal.seal(WAIT), Ok(next - 1)); + let config = Config { segment_bytes: 300, ..quick() }; + let (journal, _, _) = open_at(&storage, config.clone(), 0).unwrap_or_else(|e| panic!("{e}")); + for lsn in 1..=20 { + let got = journal.append(3, Class::Ack, &payload(lsn)).unwrap_or_else(|e| panic!("{e}")); + journal.wait_durable(got, WAIT).unwrap_or_else(|e| panic!("{e}")); } + assert!(journal.stats().segments_created >= 2, "{:?}", journal.stats()); + drop(journal); let last = storage .names_in(&dir()) .into_iter() @@ -515,12 +514,13 @@ fn a_damaged_header_in_the_last_segment_never_reuses_its_lsns() { .unwrap_or_else(|| panic!("no segment")); // One byte of the journal id in its header (media damage). storage.corrupt(&dir().join(&last), 20); - let (journal, report, seen) = open(&storage.restart(Crash::Process)); + let (journal, report, seen) = + open_at(&storage.restart(Crash::Process), config, 0).unwrap_or_else(|e| panic!("{e}")); match report.damage.first() { Some(Damage::Corrupt { lost_through, .. }) => assert_eq!(*lost_through, 20, "{report:?}"), other => panic!("no damage report: {other:?} ({report:?})"), } - assert!(seen.len() < 20, "{report:?}"); + assert!(!seen.is_empty() && seen.len() < 20, "{report:?}"); assert_eq!(journal.append(3, Class::Ack, b"after"), Ok(21)); } From 4dfdffc0d58a0da6fcf73c8ab0e6254be1bc23d0 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 4 Nov 2026 00:24:46 +0000 Subject: [PATCH 08/52] cmux-journal: the crash sim forces the F4 order (failed sync, kill, OS crash) (red) On the code before the S1 fix, the crash-sim step was green (fleet 184129d8 at 263da86f223): 6 of 7 runs passed, and the mixed run failed only seed 140 in the full run and passed when it ran alone. The random faults reached the order the review found (F4) in 1 of 800 runs. So these tests were no red/green proof. The order: the sync of a write that spans pages fails and the disk gets none of them (Linux keeps them clean in the cache, so a later sync reports success); the process is killed in the writer's backoff; the next open reads the failed segment from the cache, replays its records and goes on; an OS crash or a power loss then takes those pages, and records an open already replayed are gone. - sim.rs: `fail_next_sync(pages, kind)` fails the next sync that has at least that many dirty pages, and the disk gets none of them. `crash_after_failure(n)` makes the n-th operation after the next injected failure the crash point. - crash_sim_tests.rs: a new run forces the order on every seed (a_kill_after_a_failed_sync_then_os_crashes_keep_every_acked_record): epoch 0 aims the fault, the kill is a process crash at the writer's retry, the open after it does not crash, epoch 1 ends in an OS crash, and a third open checks that the records the second open replayed are all there. In the random runs, one epoch in four aims the fault, with the crash 1 to 3 operations after the failed sync and a crash of any kind after it. - A sim test checks the aimed fault itself (the_sim_fails_the_aimed_sync_and_crashes_in_the_backoff). Expected on this commit (old journal code): the forced run fails every seed; the mixed run fails most runs. The fix commit follows. --- .../cmux-journal/src/crash_sim_tests.rs | 129 +++++++++++++++--- cmux-tui/crates/cmux-journal/src/sim.rs | 65 +++++++-- 2 files changed, 169 insertions(+), 25 deletions(-) diff --git a/cmux-tui/crates/cmux-journal/src/crash_sim_tests.rs b/cmux-tui/crates/cmux-journal/src/crash_sim_tests.rs index c01b1aa00620..b3511b4503c5 100644 --- a/cmux-tui/crates/cmux-journal/src/crash_sim_tests.rs +++ b/cmux-tui/crates/cmux-journal/src/crash_sim_tests.rs @@ -14,10 +14,20 @@ //! //! `CMUX_JOURNAL_CRASH_SEEDS` seeds per test (default 200; the acceptance run in //! R-DURABILITY/DESIGN.md section 7 uses 100000 on the fleet). A failing seed prints itself: -//! rerun it alone with `CMUX_JOURNAL_CRASH_SEED=`. +//! rerun it with `CMUX_JOURNAL_CRASH_SEED=`. The clients run on threads, so a seed fixes the +//! faults and the record sizes but not where batches end: a rerun can take another path and pass. +//! The random runs are the acceptance property. The forced run below is the red/green proof: it +//! fails every seed on the code before the S1 fix. //! -//! The last tests check the simulated machine itself: the write-back error behavior and the -//! ordered log of names that the crash runs depend on. +//! The S1 review found an order (F4) that the random faults reached in 1 of 800 runs: a sync +//! fails and drops its pages, the process is killed in the writer's backoff, the next open goes +//! on from the page cache, and an OS crash or a power loss follows. So one epoch in four of the +//! random runs aims the failed sync and the crash at it (only the mixed run has both a process +//! crash and an OS crash, so only it can fail on that order), and one more run forces the order on +//! every seed. +//! +//! The last tests check the simulated machine itself: the write-back error behavior, the aimed +//! faults, and the ordered log of names that the crash runs depend on. use std::collections::BTreeMap; use std::io::ErrorKind; @@ -92,7 +102,26 @@ fn seeds() -> Vec { (0..count).collect() } +/// What goes wrong in each epoch of a crash run. +#[derive(Clone, Copy, PartialEq, Eq)] +enum Faults { + /// In each epoch, at random: a crash at a random operation, sometimes after a failed write or + /// sync; or (one epoch in four) the aimed fault below, with the crash at one of the three + /// operations after the failed sync and a crash of any kind after it. + Random, + /// In the first epoch, the first sync of a write that spans pages fails and the disk gets none + /// of them; the process is killed at the next storage operation (the writer's retry, after + /// its backoff). The open after the kill does not crash, so it reads the failed segment from + /// the page cache. The next epoch ends in an OS crash, which loses every page the failed sync + /// dropped, and a third open checks what is left. The code before the S1 fix fails every seed. + FailedSyncThenKill, +} + fn crash_run(seed: u64, kinds: &[Crash]) -> Result<(), String> { + crash_run_with(seed, kinds, Faults::Random) +} + +fn crash_run_with(seed: u64, kinds: &[Crash], faults: Faults) -> Result<(), String> { let mut rng = SimRng::new(seed); let mut storage = SimStorage::new(rng.next_u64()); let config = Config { @@ -107,13 +136,16 @@ fn crash_run(seed: u64, kinds: &[Crash]) -> Result<(), String> { let mut kept: Vec = Vec::new(); let mut last: Option = None; let mut worst = Crash::Process; - let epochs = 1 + rng.below(3); + let epochs = match faults { + Faults::FailedSyncThenKill => 2, + Faults::Random => 1 + rng.below(3), + }; for epoch in 0..=epochs { // The open, which may crash inside recovery; the next process tries again. let mut attempts = 0; let (journal, report, seen) = loop { attempts += 1; - if attempts < 4 && rng.below(2) == 0 { + if faults == Faults::Random && attempts < 4 && rng.below(2) == 0 { storage.crash_after(1 + rng.below(40)); } match open(&storage, &config) { @@ -132,15 +164,27 @@ fn crash_run(seed: u64, kinds: &[Crash]) -> Result<(), String> { if epoch == epochs { return Ok(()); } - // This epoch's crash point (it replaces one the open did not reach), and sometimes a - // failed write or sync before it. - storage.crash_after(1 + rng.below(80)); - if rng.below(2) == 0 { - let kind = if rng.below(2) == 0 { ErrorKind::StorageFull } else { ErrorKind::Other }; - storage.fail_after(1 + rng.below(40), kind); + // This epoch's faults. Each replaces a crash point the open did not reach. + let aimed = match faults { + Faults::FailedSyncThenKill => epoch == 0, + Faults::Random => rng.below(4) == 0, + }; + if aimed { + // The first sync of a batch: every record spans two pages (`appends`), so it has two. + storage.fail_next_sync(2, ErrorKind::Other); + let after = if faults == Faults::Random { 1 + rng.below(3) } else { 1 }; + storage.crash_after_failure(after); + } else { + storage.crash_after(1 + rng.below(80)); + if rng.below(2) == 0 { + let kind = + if rng.below(2) == 0 { ErrorKind::StorageFull } else { ErrorKind::Other }; + storage.fail_after(1 + rng.below(40), kind); + } } let start = report.next_lsn; - let (durable, full_durable) = appends(seed ^ epoch, &journal, &storage, &model, &mut rng); + let (durable, full_durable) = + appends(seed ^ epoch, &journal, &storage, &model, &mut rng, aimed); let stats = journal.stats(); last = Some(Epoch { start, @@ -150,7 +194,17 @@ fn crash_run(seed: u64, kinds: &[Crash]) -> Result<(), String> { failed: storage.failures() > 0, }); drop(journal); - let crash = pick(kinds, &mut rng); + if faults == Faults::FailedSyncThenKill + && aimed + && (storage.failures() == 0 || !storage.is_crashed()) + { + return Err(format!("epoch {epoch}: the aimed failed sync and kill did not happen")); + } + let crash = match faults { + Faults::FailedSyncThenKill if epoch == 0 => Crash::Process, + Faults::FailedSyncThenKill => Crash::Os, + Faults::Random => pick(kinds, &mut rng), + }; worst = crash; storage = storage.restart(crash); } @@ -165,6 +219,7 @@ fn appends( storage: &SimStorage, model: &Model, rng: &mut SimRng, + large: bool, ) -> (Lsn, Lsn) { let durable = AtomicU64::new(0); let full_durable = AtomicU64::new(0); @@ -176,9 +231,15 @@ fn appends( scope.spawn(move || { for index in 0..30u64 { let class = if rng.below(4) == 0 { Class::Stream } else { Class::Ack }; - // Now and then a record over several pages. - let size = - if rng.below(10) == 0 { 2000 + rng.index(8000) } else { rng.index(64) }; + // Now and then a record over several pages; when `large`, every record spans + // two pages, so the first batch's sync has two dirty pages. + let size = if large { + 5000 + rng.index(5000) + } else if rng.below(10) == 0 { + 2000 + rng.index(8000) + } else { + rng.index(64) + }; let body = format!("{seed}-{client}-{index}-{}", "y".repeat(size)); let lsn = { // The append and the model's copy are one step, so the model never misses @@ -302,6 +363,23 @@ fn mixed_crashes_keep_what_the_worst_of_them_keeps() { crash_runs(&[Crash::Process, Crash::Os, Crash::PowerLoss]); } +/// F4 of the S1 review: a sync fails and drops its pages, the process is killed before the writer +/// tries again, and later an OS crash comes. Every record the open after the kill replayed, and +/// every record acked after it, must survive. With random faults this order is rare (the random +/// runs found it once in 800 runs on the code before the fix), so each seed here forces it. +#[test] +fn a_kill_after_a_failed_sync_then_os_crashes_keep_every_acked_record() { + let failures: Vec = seeds() + .into_iter() + .filter_map(|seed| { + crash_run_with(seed, &[Crash::Os], Faults::FailedSyncThenKill) + .err() + .map(|error| format!("seed {seed}: {error}")) + }) + .collect(); + assert!(failures.is_empty(), "{} runs failed:\n{}", failures.len(), failures.join("\n")); +} + // ---------------------------------------------------------------- the simulated machine /// A folder with one empty file in it, all durable. @@ -342,6 +420,25 @@ fn the_sim_never_writes_the_pages_a_failed_sync_dropped() { assert!(dropped > 0, "no failed sync dropped a page"); } +#[test] +fn the_sim_fails_the_aimed_sync_and_crashes_in_the_backoff() { + let path = dir().join("file"); + let (storage, mut file) = machine_with_file(9, &path); + storage.fail_next_sync(2, ErrorKind::Other); + storage.crash_after_failure(2); + // One dirty page: the aimed failure waits for a sync of more. + file.append(&[1; 100]).unwrap_or_else(|e| panic!("{e}")); + file.sync(SyncLevel::Barrier).unwrap_or_else(|e| panic!("{e}")); + file.append(&vec![2; 2 * SIM_PAGE]).unwrap_or_else(|e| panic!("{e}")); + assert!(file.sync(SyncLevel::Barrier).is_err(), "the aimed sync did not fail"); + assert_eq!(storage.failures(), 1); + assert!(file.size().is_ok(), "the first operation after the failure must run"); + assert!(file.size().is_err() && storage.is_crashed(), "the second must be the crash"); + // None of the failed sync's pages reached the disk. + let after = storage.restart(Crash::Os).contents(&path).unwrap_or_default(); + assert!(after.iter().skip(100).all(|byte| *byte != 2), "a dropped page reached the disk"); +} + #[test] fn the_sim_keeps_a_rename_between_folders_whole() { let (mut moved, mut stayed) = (0, 0); diff --git a/cmux-tui/crates/cmux-journal/src/sim.rs b/cmux-tui/crates/cmux-journal/src/sim.rs index 93658eb7259c..68b3e6541532 100644 --- a/cmux-tui/crates/cmux-journal/src/sim.rs +++ b/cmux-tui/crates/cmux-journal/src/sim.rs @@ -13,6 +13,11 @@ //! changed an entry of that folder, and so every change before it. A rename between two folders //! is one change: it never comes back twice or not at all. //! +//! [`SimStorage::fail_next_sync`] fails the next sync of a write that spans pages, and the disk +//! gets none of them (the worst case). [`SimStorage::crash_after_failure`] puts the crash point +//! right after an injected failure: in the writer's backoff, at its first operation when it tries +//! again. +//! //! [`SimStorage::crash_after`] makes the n-th storage operation from now the crash point: an //! append there writes a random prefix of its bytes, a create, rename, truncate or sync there //! takes effect or not at random, and every later operation fails. [`SimStorage::restart`] then @@ -166,10 +171,13 @@ impl SimFile { self.synced.push((number, self.disk.clone())); } - /// A sync that failed: a random part of the dirty pages reached the disk; the cache dropped - /// the rest as clean, so no later sync writes them. - fn failed_sync(&mut self, rng: &mut SimRng) { - let pages: Vec = self.dirty.iter().copied().filter(|_| rng.coin()).collect(); + /// A sync that failed: a random part of the dirty pages reached the disk (none, without + /// `rng`); the cache dropped the rest as clean, so no later sync writes them. + fn failed_sync(&mut self, rng: Option<&mut SimRng>) { + let pages: Vec = match rng { + Some(rng) => self.dirty.iter().copied().filter(|_| rng.coin()).collect(), + None => Vec::new(), + }; self.write_back(&pages); self.dirty.clear(); } @@ -392,8 +400,12 @@ struct Fs { /// Open files now, and the most at once. open_files: u64, peak_open_files: u64, - /// Failures `fail_after` caused. + /// Failures `fail_after` and `fail_next_sync` caused. failures: u64, + /// The next file sync with at least this many dirty pages fails once with this error. + fail_sync: Option<(usize, io::ErrorKind)>, + /// Once an injected failure happened, the operation this many after it crashes. + crash_after_failure: Option, } /// What an operation does: run, fail once, or crash the process. @@ -417,6 +429,16 @@ impl Fs { open_files: 0, peak_open_files: 0, failures: 0, + fail_sync: None, + crash_after_failure: None, + } + } + + /// An injected failure happened: count it, and arm the crash that waits for it. + fn note_failure(&mut self) { + self.failures = self.failures.saturating_add(1); + if let Some(n) = self.crash_after_failure.take() { + self.crash_at = Some(self.ops.saturating_add(n.max(1))); } } @@ -434,7 +456,7 @@ impl Fs { && at == self.ops { self.fail_at = None; - self.failures = self.failures.saturating_add(1); + self.note_failure(); return Ok(Step::Fail(kind)); } Ok(Step::Run) @@ -521,7 +543,22 @@ impl SimStorage { lock(&self.fs).reads.get(path).copied().unwrap_or(0) } - /// Operations that `fail_after` made fail. + /// The next file sync that has at least `pages` dirty pages fails once with `kind`, and the + /// disk gets none of them: the worst case of a write-back error ([module docs](self)). + pub fn fail_next_sync(&self, pages: usize, kind: io::ErrorKind) { + lock(&self.fs).fail_sync = Some((pages, kind)); + } + + /// Once the next injected failure happened, the `n`-th operation after it (1 = the next one) + /// is the crash point: a crash in the writer's backoff, at its retry. It replaces a crash point + /// set before. + pub fn crash_after_failure(&self, n: u64) { + let mut fs = lock(&self.fs); + fs.crash_at = None; + fs.crash_after_failure = Some(n); + } + + /// Operations that `fail_after` and `fail_next_sync` made fail. pub fn failures(&self) -> u64 { lock(&self.fs).failures } @@ -850,7 +887,17 @@ impl StorageFile for SimHandle { fn sync(&mut self, level: SyncLevel) -> io::Result<()> { let mut fs = lock(&self.fs); - let done = match fs.begin()? { + let step = fs.begin()?; + if let (Step::Run, Some((pages, kind))) = (&step, fs.fail_sync) { + let file = fs.machine.file(self.inode)?; + if file.dirty.len() >= pages { + file.failed_sync(None); + fs.fail_sync = None; + fs.note_failure(); + return Err(failed(kind)); + } + } + let done = match step { Step::Run => true, Step::Fail(kind) => { let Fs { machine, .. } = &mut *fs; @@ -858,7 +905,7 @@ impl StorageFile for SimHandle { .files .get_mut(&self.inode) .ok_or_else(|| failed(io::ErrorKind::Other))?; - file.failed_sync(&mut machine.rng); + file.failed_sync(Some(&mut machine.rng)); return Err(failed(kind)); } Step::Crash => fs.machine.rng.coin(), From 3498975b5267b8a20d07d8ede8e14e54ebfabaf9 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Tue, 3 Nov 2026 20:40:26 +0000 Subject: [PATCH 09/52] cmux-journal: fix the S1 review findings F1-F7 (markers, recovery copy, LSN floor, writer end) Review FIX at 89293fb15f39 (PR #33919 comment 6610468202). The regression tests are the two commits before this one. - Sync markers (format.rs): an entry with flags 1 and no payload, whose LSN field says "every record of this segment before me is synced". The writer appends one after each successful sync that covered records (after the durable listeners, so no reply waits for it), and syncs it fully at a seal and at a rotation. - F4, recovery (step 4 and 6): the last segment is trusted only up to its last marker. Its records after the marker may sit in pages a failed sync dropped (a later sync on another descriptor reports success without them), so recovery writes them again into a new segment: header, records and a marker, fully synced under `.seg.new`, renamed into place, folder synced. The journal always goes on in a segment this open made, never in one an earlier process wrote. A restart in the writer's backoff after a failed sync is then safe. - F1, quarantine: every LSN a quarantined segment holds counts in the LSN floor at every open, and a quarantine that no new segment followed (a crash in the middle of an earlier recovery) is reported as damage again. The highest LSN of every moved segment is read before the first rename. - F2, highest LSN: a segment whose header is damaged is read from its first record (each record's checksum covers its LSN), then the bytes after the first bad entry are scanned for records and markers that still check out. The same scan runs after a cut in the last segment: a marker there is media damage (reported), records there are a torn write (`torn_lsns`); their LSNs are never used again. - F3, writer end: a guard closes the journal when the writer thread ends, also by a panic: appends fail with Closed, waits end with WriterEnded, the listeners are dropped (so a channel a held reply waits on closes), and stats.failing says why. The crate does not abort the process; the daemon owns that policy. - F6: recovery closes each segment after its header is read and opens one again only to read it. (Retirement of compacted segments is an S2 API, before the first consumer.) - F5: recovery syncs the parent folder after it makes the journal folder. - F7: format.rs and lib.rs now say what the code does: a failed write is cut back and retried in the same segment; the cut removes only bytes never synced and never acknowledged; nothing shortens a segment past a synced record. Docs: journal.rs says when open's replayed records are durable (on Ok) and what a stopped writer does. --- .../cmux-journal/src/cmux_journal_tests.rs | 10 +- cmux-tui/crates/cmux-journal/src/format.rs | 57 +- cmux-tui/crates/cmux-journal/src/journal.rs | 15 +- cmux-tui/crates/cmux-journal/src/lib.rs | 24 +- cmux-tui/crates/cmux-journal/src/recovery.rs | 600 ++++++++++++++---- cmux-tui/crates/cmux-journal/src/writer.rs | 117 +++- 6 files changed, 663 insertions(+), 160 deletions(-) diff --git a/cmux-tui/crates/cmux-journal/src/cmux_journal_tests.rs b/cmux-tui/crates/cmux-journal/src/cmux_journal_tests.rs index 4f13d15537ee..ab83c25d0d21 100644 --- a/cmux-tui/crates/cmux-journal/src/cmux_journal_tests.rs +++ b/cmux-tui/crates/cmux-journal/src/cmux_journal_tests.rs @@ -75,7 +75,7 @@ fn eventually(check: impl Fn() -> bool) -> bool { fn a_zero_record_header_never_checks_out() { let header = [0u8; format::RECORD_HEADER_LEN]; let decoded = format::RecordHeader::decode(&header).unwrap_or_else(|| panic!("decode")); - assert!(decoded.verify(&header, &[], 0).is_err(), "a zeroed header must fail its checksum"); + assert!(decoded.check(&header, &[]).is_err(), "a zeroed header must fail its checksum"); } #[test] @@ -466,9 +466,15 @@ fn damaged_in_the_middle(seed: u64) -> (SimStorage, Config) { #[test] fn a_crash_inside_the_quarantine_step_never_reuses_a_quarantined_lsn() { let (damaged, config) = damaged_in_the_middle(50); + // Every storage operation of the open is a crash point once. + let probe = damaged.snapshot(Crash::Process); + let (journal, report, _) = open_at(&probe, config.clone(), 0).unwrap_or_else(|e| panic!("{e}")); + assert!(matches!(report.damage.first(), Some(Damage::Corrupt { .. })), "{report:?}"); + let operations = probe.ops(); + drop(journal); let mut crashed_after_a_move = 0; for crash in [Crash::Process, Crash::Os] { - for at in 1..=90 { + for at in 1..=operations { let storage = damaged.snapshot(Crash::Process); storage.crash_after(at); let (journal, report) = match open_at(&storage, config.clone(), 0) { diff --git a/cmux-tui/crates/cmux-journal/src/format.rs b/cmux-tui/crates/cmux-journal/src/format.rs index 294bfe874fea..c29faac789c5 100644 --- a/cmux-tui/crates/cmux-journal/src/format.rs +++ b/cmux-tui/crates/cmux-journal/src/format.rs @@ -11,7 +11,7 @@ //! | 16..24 | journal id: random, the same in every segment of one journal | //! | 24..32 | segment number: +1 for each new segment, also in the file name | //! | 32..40 | first LSN: the LSN of this segment's first record | -//! | 40..48 | previous last LSN: the journal's last LSN before this segment. Recovery reads the earlier segments only through it, so bytes a failed segment holds after it are not part of the journal (a writer starts a new segment after any failed write or sync) | +//! | 40..48 | previous last LSN: the journal's last LSN before this segment. Recovery reads the earlier segments only through it, so bytes an earlier segment holds after it are not part of the journal (the writer starts a new segment after a failed sync, and recovery at every open) | //! | 48..52 | reserved, 0 | //! | 52..56 | CRC-32 of bytes 0..52 | //! @@ -24,9 +24,14 @@ //! | 8..16 | LSN | //! | 16..18 | kind (the daemon's) | //! | 18 | class: 1 ack, 2 stream | -//! | 19 | flags, 0 | +//! | 19 | flags: 0 for a record, 1 for a sync marker | //! -//! A run of zero bytes never reads as a record: the CRC of a zero header is not zero. +//! A sync marker (flags 1, kind 0, class 0, no payload) says that every record of its segment +//! before it is synced; its LSN field is the LSN of the last of them, and it takes no LSN of its +//! own. The writer writes one after each successful sync, before anything else it writes there. +//! Recovery re-writes a segment's records after its last marker (`recovery.rs`, step 4). +//! +//! A run of zero bytes never reads as an entry: the CRC of a zero header is not zero. use crate::{Class, Lsn}; @@ -37,6 +42,8 @@ pub(crate) const SEGMENT_HEADER_LEN: usize = 56; pub const RECORD_HEADER_LEN: usize = 20; /// The largest payload one record may carry. A length field above it is damage. pub const MAX_PAYLOAD_LEN: usize = 64 * 1024 * 1024; +/// The `flags` of a sync marker. +pub(crate) const MARKER: u8 = 1; /// One segment's header, as written and as read back. #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -116,30 +123,46 @@ impl RecordHeader { }) } - /// The record's class when the header and `payload` are whole and the LSN is `expected`; - /// else why not. - pub(crate) fn verify( - &self, - header: &[u8], - payload: &[u8], - expected: Lsn, - ) -> Result { + /// What the entry is when its header and `payload` are whole, whatever its LSN; else why not. + pub(crate) fn check(&self, header: &[u8], payload: &[u8]) -> Result { let mut crc = crc32fast::Hasher::new(); crc.update(header.get(4..RECORD_HEADER_LEN).ok_or("short record header")?); crc.update(payload); if crc.finalize() != self.crc { return Err("record checksum"); } - if self.lsn != expected { - return Err("record out of sequence"); - } - if self.flags != 0 { - return Err("unknown record flags"); + match self.flags { + 0 => Class::from_code(self.class_code).map(Entry::Record).ok_or("unknown record class"), + MARKER if self.len == 0 && self.kind == 0 && self.class_code == 0 => Ok(Entry::Marker), + _ => Err("unknown record flags"), } - Class::from_code(self.class_code).ok_or("unknown record class") } } +/// What one checked entry of a segment is. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Entry { + Record(Class), + /// A sync marker (module docs). + Marker, +} + +/// Appends a sync marker for `synced_through` (module docs) to `out`. +pub(crate) fn encode_marker(out: &mut Vec, synced_through: Lsn) { + let mut head = [0u8; RECORD_HEADER_LEN]; + let mut at = Cursor::new(&mut head); + at.put(&0u32.to_le_bytes()); + at.put(&0u32.to_le_bytes()); + at.put(&synced_through.to_le_bytes()); + at.put(&0u16.to_le_bytes()); + at.put(&[0, MARKER]); + let crc = head.get(4..).map_or(0, crc32fast::hash); + if let Some(field) = head.get_mut(..4) { + field.copy_from_slice(&crc.to_le_bytes()); + } + out.extend_from_slice(&head); +} + /// Appends one encoded record to `out`. `payload_crc` is the CRC-32 state of the payload alone, /// computed by the caller off any lock ([`payload_crc`]). pub(crate) fn encode_record( diff --git a/cmux-tui/crates/cmux-journal/src/journal.rs b/cmux-tui/crates/cmux-journal/src/journal.rs index d32acce25324..dd7dce0ee2de 100644 --- a/cmux-tui/crates/cmux-journal/src/journal.rs +++ b/cmux-tui/crates/cmux-journal/src/journal.rs @@ -36,7 +36,7 @@ pub struct JournalStats { /// Failed writes, syncs and segment creations. After each, the writer tries again after a /// backoff: a failed write in the same segment, anything else in a new segment. pub failures: u64, - /// The last failure, while the writer has not recovered from it. + /// The last failure, while the writer has not recovered from it, or why the writer stopped. pub failing: Option, } @@ -45,7 +45,7 @@ pub(crate) struct State { pub(crate) pending_last: Lsn, pub(crate) pending_records: u64, pub(crate) pending_ack: bool, - /// No appends any more (a seal, or the journal is being dropped). + /// No appends any more (a seal, a drop, or the writer stopped). pub(crate) closed: bool, /// The writer stops after one last attempt (drop). pub(crate) stop: bool, @@ -73,7 +73,7 @@ impl Shared { /// Why an append was refused. None of them waits for the disk. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum AppendError { - /// The journal is sealed or being dropped. + /// The journal is sealed or being dropped, or its writer stopped (a panic on its thread). Closed, /// The payload is larger than [`MAX_PAYLOAD_LEN`]. TooLarge, @@ -99,7 +99,7 @@ pub enum WaitError { Timeout, /// The LSN was never appended. NotAppended, - /// The writer ended (the journal is being dropped). + /// The writer ended: the journal is being dropped, or a panic on its thread stopped it. WriterEnded, } @@ -179,8 +179,10 @@ pub struct Journal { impl Journal { /// Recovers the journal in `dir` (made when missing) and starts its writer. `replay` gets /// every record after `from` (the caller's compaction point), in LSN order, before this - /// returns; a replay error aborts the open. Every record handed to `replay` is fully synced - /// first, so it is as durable as an acknowledged one. + /// returns; a replay error aborts the open. When this returns `Ok`, every record handed to + /// `replay` is fully durable (fully synced, or written again into a fully synced segment), so + /// it is as durable as an acknowledged one. When it returns an error, the caller drops what + /// the replay built: some of those records may not be on the disk. pub fn open( storage: Arc, dir: &Path, @@ -276,6 +278,7 @@ impl Journal { /// Calls `listener` with the new durable LSN each time it moves, on the writer thread. It /// must return quickly and never wait: it only releases the replies that wait for the LSN. + /// When the writer stops, the listeners are dropped, so a channel a listener sends on closes. pub fn on_durable(&self, listener: impl Fn(Lsn) + Send + Sync + 'static) { self.shared .listeners diff --git a/cmux-tui/crates/cmux-journal/src/lib.rs b/cmux-tui/crates/cmux-journal/src/lib.rs index 388e1d687384..b2692af9843c 100644 --- a/cmux-tui/crates/cmux-journal/src/lib.rs +++ b/cmux-tui/crates/cmux-journal/src/lib.rs @@ -27,19 +27,29 @@ //! seal and recovery always sync fully. //! //! Files (the journal directory): `LOCK` (an exclusive lock held by the one writer), the segments -//! (`.seg`: a header, then records back to back, see `format.rs`), and -//! `quarantine/` for segments recovery could not read past damage that is not a torn tail. A -//! segment is only ever appended to; nothing here deletes or shortens one that holds a record. +//! (`.seg`: a header, then records and sync markers back to back, see +//! `format.rs`), and `quarantine/` for segments recovery could not read past damage that is not a +//! torn tail. A segment is only ever appended to. The one exception is the writer's own failed +//! write: it cuts the segment back to its size before that write, so it removes only bytes that +//! were never synced and never acknowledged. Nothing here deletes a segment or shortens one past +//! a synced record. //! //! Recovery ([`Journal::open`]) is the one start path for a clean exit, a crash, a kill -9 and an //! OS crash: it syncs the last segment, reads the segments in LSN order, checks each record's //! checksum and LSN, and hands every record after the caller's compaction point to the caller's -//! replay. A torn tail of the last segment (a write that never completed, so never acked) stays in -//! the file, and the journal goes on in a new segment whose header says where the old one ends. +//! replay. The records of the last segment after its last sync marker are not known to be on the +//! disk (a sync of them may have failed), so recovery writes them again into a new segment, fully +//! synced before open returns. A torn tail (a write that never completed, so never acked) stays in +//! its file, and the journal goes on in a new segment whose header says where the old one ends. +//! +//! A panic on the writer thread (a durable listener's) closes the journal: appends fail with +//! [`AppendError::Closed`], waits end with [`WaitError::WriterEnded`], and the listeners are +//! dropped. The daemon then stops; recovery runs at its next start. //! //! [`Storage`] is the file system seam. [`FsStorage`] is the real one. With the `sim` feature, -//! `SimStorage` keeps the synced and unsynced bytes of each file apart and can crash at any -//! storage operation, then lose what a process crash, an OS crash or a power loss would lose. +//! `SimStorage` simulates a page cache over a disk and an ordered log of names, can fail or crash +//! at any storage operation, and then keeps what a process crash, an OS crash or a power loss +//! would keep. mod format; mod journal; diff --git a/cmux-tui/crates/cmux-journal/src/recovery.rs b/cmux-tui/crates/cmux-journal/src/recovery.rs index 63f2392baba7..04d601be893c 100644 --- a/cmux-tui/crates/cmux-journal/src/recovery.rs +++ b/cmux-tui/crates/cmux-journal/src/recovery.rs @@ -1,28 +1,36 @@ //! Recovery: the one start path, for a clean exit, a crash, a kill -9 and an OS crash. //! -//! 1. Take the lock; list the segments in number order; read and check each header (its -//! journal id, number, and LSN range against the segment before it). -//! 2. Fully sync the last segment: the last process may have died after a write and before its -//! sync. So every record handed to the replay is durable before the daemon serves. +//! 1. Take the lock; list the segments (and the ones in `quarantine/`); read and check each +//! live header (journal id, number, LSN chain). No file stays open after its header is read. +//! 2. Fully sync the last segment. The records the replay gets are then as durable as acked ones. //! 3. Read the records after the caller's compaction point (`from`) and hand them to the replay. //! An earlier segment is read only through the next segment's "previous last LSN": bytes after -//! it (left by a write or sync that failed, after which the writer started a new segment) are -//! not part of the journal. A segment whose records all lie at or before `from` is not read. -//! 4. The last segment is read up to its first record that does not check out: a torn write that -//! never completed, so it was never acknowledged. Those bytes stay in the file; the journal -//! goes on in a new segment whose header names the last good LSN. +//! it are not part of the journal. A segment whose records all lie at or before `from` is read +//! for its header only. +//! 4. The last segment is read up to its first entry that does not check out. Its records after +//! its last sync marker (`format.rs`) are not known to be on the disk: a sync may have failed +//! for them, and a later sync on another descriptor can report success for pages the kernel +//! dropped. So recovery writes them again into a new segment, made whole and synced under a +//! staging name and only then renamed into place, and the journal goes on there. A torn tail +//! stays in its file. The bytes after the first bad entry are scanned for entries that still +//! check out: a sync marker among them means synced records are damaged (media damage, never a +//! crash), and their LSNs are never used again. //! 5. Damage that is not a torn tail (a bad header, or records missing before the next segment) -//! keeps the valid prefix. That segment's successors move to `quarantine/` (never deleted), the -//! report names them and the LSNs they may hold, and new records take LSNs after all of them, -//! so no LSN is ever used twice. +//! keeps the valid prefix; the later segments move to `quarantine/` (never deleted). New +//! records take LSNs after every LSN that any segment, live or quarantined, holds, so no LSN +//! is ever used twice, also after a crash in the middle of an earlier recovery. +//! 6. The journal goes on in a segment this open makes (the copy of step 4, or an empty one), never +//! in a segment an earlier process wrote: that one may hold a page its cache had and its disk +//! never got. //! -//! Recovery writes nothing but a sync and, when the journal does not go on in its last segment, a -//! new segment. It never truncates or deletes a segment that holds a record. +//! Recovery writes nothing but syncs, new segments and the renames into `quarantine/`. It never +//! shortens or deletes a segment. use std::path::Path; use crate::format::{ - self, MAX_PAYLOAD_LEN, RECORD_HEADER_LEN, RecordHeader, SEGMENT_HEADER_LEN, SegmentHeader, + self, Entry, MAX_PAYLOAD_LEN, RECORD_HEADER_LEN, RecordHeader, SEGMENT_HEADER_LEN, + SegmentHeader, }; use crate::journal::OpenError; use crate::storage::{OpenMode, Storage, StorageFile, StorageLock, SyncLevel}; @@ -30,34 +38,40 @@ use crate::{Lsn, Record}; pub(crate) const LOCK_NAME: &str = "LOCK"; pub(crate) const QUARANTINE_DIR: &str = "quarantine"; +/// A segment recovery writes is first written whole under its name plus this, then renamed. +const STAGING_SUFFIX: &str = ".new"; /// Segment bytes read at once. const READ_CHUNK: usize = 1024 * 1024; /// What recovery found ([`crate::Journal::open`]). #[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct OpenReport { - /// Segments the journal holds after recovery, a new one included. + /// Live segments after recovery, the ones it made included. pub segments: usize, /// The last LSN the journal holds or skipped over, and the LSN the next append takes. pub last_lsn: Lsn, pub next_lsn: Lsn, /// Records handed to the replay. pub replayed: u64, - /// Bytes at the end of the last segment that hold no whole record: a write that never - /// completed (never acknowledged). They stay in the file; the journal goes on after them in a - /// new segment. + /// Records of the last segment after its last sync marker, written again into a new segment. + pub rewritten: u64, + /// Bytes at the end of the last segment that hold no whole entry: a write that never + /// completed. They stay in the file; the journal goes on in a new segment. pub torn_tail_bytes: u64, + /// Records found after such a tear that no sync marker vouches for (an OS crash wrote some + /// pages of an unsynced write and not others), as (first, last) LSN. They are not kept, and + /// their LSNs are not used again. + pub torn_lsns: Option<(Lsn, Lsn)>, pub damage: Vec, } -/// What recovery could not read, beyond a torn tail. The daemon reports it to the person; moved -/// segments stay in `quarantine/` for a salvage. +/// What recovery could not read, beyond a torn tail: media damage or a bug, never a crash. The +/// daemon reports it to the person; moved segments stay in `quarantine/` for a salvage. #[derive(Clone, Debug, PartialEq, Eq)] pub enum Damage { - /// `segment` could not be read past `offset` (media damage or a bug, never a crash). The - /// segments in `moved` (every later one, and this one too when its header was bad) are in - /// `quarantine/` now. The records after `lost_after` through `lost_through` are not in the - /// journal; new records take LSNs after `lost_through`. + /// `segment` could not be read past `offset`. The segments in `moved` are in `quarantine/` + /// now (named `.`). The records after `lost_after` through `lost_through` are not + /// in the journal; new records take LSNs after `lost_through`. Corrupt { segment: String, offset: u64, @@ -93,10 +107,10 @@ pub(crate) struct Recovered { pub(crate) writer: WriterStart, } +/// A live segment whose header checked out. Its file is opened again to read its records. struct Segment { seq: u64, name: String, - file: Box, size: u64, header: SegmentHeader, } @@ -108,7 +122,7 @@ struct Bad { segment: String, offset: u64, reason: &'static str, - /// The highest LSN the journal is known to have held before the damage. + /// The highest LSN the journal is known to have held past the damage. lost_through: Lsn, } @@ -120,6 +134,10 @@ pub(crate) fn len64(n: usize) -> u64 { u64::try_from(n).unwrap_or(u64::MAX) } +fn usize_of(n: u64) -> usize { + usize::try_from(n).unwrap_or(usize::MAX) +} + pub(crate) fn recover( storage: &dyn Storage, dir: &Path, @@ -127,31 +145,33 @@ pub(crate) fn recover( replay: &mut dyn FnMut(Record<'_>) -> Result<(), String>, ) -> Result { storage.create_dir_all(dir).map_err(io("make the folder"))?; + // The folder's own entry, durable before anything inside it counts. + if let Some(parent) = dir.parent().filter(|parent| !parent.as_os_str().is_empty()) { + storage.sync_dir(parent).map_err(io("sync the parent folder"))?; + } let lock = storage.try_lock(&dir.join(LOCK_NAME)).map_err(io("lock"))?.ok_or(OpenError::Locked)?; - let mut listed: Vec<(u64, String)> = storage - .list(dir) - .map_err(io("list the folder"))? - .into_iter() - .filter_map(|name| Some((format::parse_segment_name(&name)?, name))) - .collect(); - listed.sort(); - let max_seq = listed.last().map_or(0, |(seq, _)| *seq); + let listed = list_segments(storage, dir)?; + let quarantined = list_quarantined(storage, dir)?; + let live_max_seq = listed.last().map_or(0, |(seq, _)| *seq); + let q_max_seq = quarantined.iter().map(|(seq, _)| *seq).max().unwrap_or(0); // 1 and 2. let Headers { mut segments, reuse, mut bad } = read_headers(storage, dir, &listed)?; - if let Some(last) = segments.last_mut() { - let mut file = storage + if let Some(last) = segments.last() { + storage .open(&dir.join(&last.name), OpenMode::Write) - .map_err(io("open the last segment"))?; - file.sync(SyncLevel::Full).map_err(io("sync the last segment"))?; - last.file = file; + .and_then(|mut file| file.sync(SyncLevel::Full)) + .map_err(io("sync the last segment"))?; } // 3 and 4. let mut report = OpenReport::default(); let mut journal_end = 0; - let mut tail_cut = None; + // The last segment's read (the last one is never skipped: it has no next segment), and + // whether a hole made an earlier segment the last one. + let mut last_read = None; + let mut hole = false; let mut index = 0; while let Some(segment) = segments.get(index) { let next = segments.get(index.saturating_add(1)); @@ -160,7 +180,7 @@ pub(crate) fn recover( index = index.saturating_add(1); continue; } - let read = read_records(segment, trusted_end, from, replay, &mut report)?; + let read = read_segment(storage, dir, segment, trusted_end, from, replay, &mut report)?; journal_end = read.end; if let (Some(end), Some(next)) = (trusted_end, next) && read.end < end @@ -172,21 +192,82 @@ pub(crate) fn recover( reason: "records are missing before the next segment", lost_through: end, }); + // The next segment's header vouched for this one up to the damage: no tail to copy. + last_read = Some(Read { vouched: read.end, ..read }); + hole = true; segments.truncate(index.saturating_add(1)); break; } - if trusted_end.is_none() && read.offset < segment.size { - tail_cut = Some(segment.size.saturating_sub(read.offset)); + if next.is_none() { + last_read = Some(read); } index = index.saturating_add(1); } + // The bytes after the last segment's first bad entry. After a hole they are the damage that + // step 5 reports; they still raise the floor. + let mut floor = journal_end.max(from); + if let (Some(last), Some(read)) = (segments.last(), last_read.as_ref()) + && read.offset < last.size + { + let beyond = scan_beyond_path(storage, &dir.join(&last.name), read.offset, read.end)?; + floor = floor.max(beyond.max_lsn); + if !hole { + if beyond.marker { + report.damage.push(Damage::Corrupt { + segment: last.name.clone(), + offset: read.offset, + reason: "a synced record is damaged".to_owned(), + moved: Vec::new(), + lost_after: read.end, + lost_through: beyond.max_lsn, + }); + } else { + report.torn_tail_bytes = last.size.saturating_sub(read.offset); + if beyond.max_lsn > read.end { + report.torn_lsns = Some((read.end.saturating_add(1), beyond.max_lsn)); + } + } + } + } - // 5. - let damaged = bad.is_some(); - let mut lost_through = journal_end; + // 5: every LSN a quarantined segment holds counts, from this open's moves and earlier ones. + let q_floor = quarantined_floor(storage, dir, &quarantined, journal_end)?; + floor = floor.max(q_floor); + if q_max_seq > live_max_seq && bad.is_none() { + // An earlier recovery moved segments and stopped before its next segment was made. + report.damage.push(Damage::Corrupt { + segment: format::segment_name(q_max_seq), + offset: 0, + reason: "an earlier recovery moved these segments to quarantine/ and stopped before \ + it made the next segment" + .to_owned(), + moved: quarantined.iter().map(|(_, name)| name.clone()).collect(), + lost_after: journal_end, + lost_through: q_floor.max(journal_end), + }); + } + let mut max_seq = live_max_seq.max(q_max_seq); + let mut reuse = reuse; + if let Some(seq) = reuse + && seq <= q_max_seq + { + // Not reachable (a torn header is the newest creation): moved, so no live segment lies + // below a quarantined one. + if bad.is_none() { + bad = Some(Bad { + from_seq: seq, + segment: format::segment_name(seq), + offset: 0, + reason: "a segment with no header lies below quarantined segments", + lost_through: journal_end, + }); + } + reuse = None; + } if let Some(bad) = bad { - let moved = quarantine(storage, dir, &listed, bad.from_seq)?; - lost_through = lost_through.max(bad.lost_through).max(moved.highest); + let moved = quarantine(storage, dir, &listed, bad.from_seq, journal_end)?; + let lost_through = floor.max(bad.lost_through).max(moved.highest); + floor = lost_through; report.damage.push(Damage::Corrupt { segment: bad.segment, offset: bad.offset, @@ -204,39 +285,131 @@ pub(crate) fn recover( if from > journal_end { report.damage.push(Damage::Ahead { journal_end, from }); } - let next_lsn = journal_end.max(lost_through).max(from).saturating_add(1); - - // The writer goes on in the last segment only when it ends in a whole record, nothing is - // damaged and no LSN is skipped; else a new segment starts now, before any append. - let journal_id = segments.first().map_or_else(new_journal_id, |first| first.header.journal_id); - let go_on = !damaged - && reuse.is_none() - && tail_cut.is_none() - && next_lsn == journal_end.saturating_add(1); + let next_lsn = floor.saturating_add(1); + + // Where the writer starts. + let journal_id = match segments.first() { + Some(first) => first.header.journal_id, + None => quarantined_journal_id(storage, dir, &quarantined)?.unwrap_or_else(new_journal_id), + }; + let copy = match (segments.last(), last_read.as_ref()) { + (Some(last), Some(read)) if read.end > read.vouched => Some((last, read)), + _ => None, + }; + // The writer always starts in a segment this open makes: an earlier process's segment may + // hold a page that its cache had and its disk never got (a failed sync), so nothing is + // appended to it again. It is read only through the next segment's "previous last LSN". let kept = segments.len(); - let tail = match segments.pop() { - Some(last) if go_on => Tail { file: last.file, seq: last.seq, size: last.size }, + let mut made = 0usize; + let mut next_seq = || { + let seq = reuse.take().unwrap_or_else(|| max_seq.saturating_add(1)); + max_seq = max_seq.max(seq); + seq + }; + let mut tail = None; + let mut through = journal_end; + if let Some((last, read)) = copy { + // Step 4: the unvouched records, again, in a segment of their own. + let body = read_range(storage, &dir.join(&last.name), read.vouched_offset, read.offset)?; + let header = SegmentHeader { + journal_id, + seq: next_seq(), + first_lsn: read.vouched.saturating_add(1), + prev_last: read.vouched, + }; + let file = stage_segment(storage, dir, &header, &body, Some(read.end)) + .map_err(io("write the unsynced records again"))?; + let size = len64(SEGMENT_HEADER_LEN) + .saturating_add(len64(body.len())) + .saturating_add(len64(RECORD_HEADER_LEN)); + report.rewritten = read.end.saturating_sub(read.vouched); + made = made.saturating_add(1); + through = read.end; + tail = Some(Tail { file, seq: header.seq, size }); + } + let tail = match tail { + Some(tail) if next_lsn == through.saturating_add(1) => tail, _ => { let header = SegmentHeader { journal_id, - seq: reuse.unwrap_or_else(|| max_seq.saturating_add(1)), + seq: next_seq(), first_lsn: next_lsn, - prev_last: journal_end, + prev_last: through, }; - let file = make_segment(storage, dir, &header).map_err(io("make a segment"))?; - report.segments = 1; + let file = + stage_segment(storage, dir, &header, &[], None).map_err(io("make a segment"))?; + made = made.saturating_add(1); Tail { file, seq: header.seq, size: len64(SEGMENT_HEADER_LEN) } } }; - report.segments = report.segments.saturating_add(kept); + report.segments = kept.saturating_add(made); report.last_lsn = next_lsn.saturating_sub(1); report.next_lsn = next_lsn; - report.torn_tail_bytes = tail_cut.unwrap_or(0); Ok(Recovered { lock, next_lsn, report, writer: WriterStart { journal_id, tail } }) } -/// Makes segment `header.seq` (a file that holds a torn header is made again): its header, -/// fully synced, and its folder entry, synced. +/// The live segment files, by number. +fn list_segments(storage: &dyn Storage, dir: &Path) -> Result, OpenError> { + let mut listed: Vec<(u64, String)> = storage + .list(dir) + .map_err(io("list the folder"))? + .into_iter() + .filter_map(|name| Some((format::parse_segment_name(&name)?, name))) + .collect(); + listed.sort(); + Ok(listed) +} + +/// The files in `quarantine/` (named `.`), by the segment number in their name. +fn list_quarantined(storage: &dyn Storage, dir: &Path) -> Result, OpenError> { + let names = match storage.list(&dir.join(QUARANTINE_DIR)) { + Ok(names) => names, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()), + Err(error) => { + return Err(OpenError::Io { what: "list the quarantine folder".into(), error }); + } + }; + let mut out: Vec<(u64, String)> = names + .into_iter() + .filter_map(|name| { + let (segment, _) = name.rsplit_once('.')?; + Some((format::parse_segment_name(segment)?, name)) + }) + .collect(); + out.sort(); + Ok(out) +} + +/// Makes segment `header.seq` through a staging file: the header, `body` (whole records), and, +/// when `end_marker` is set, a sync marker for them; then a full sync, the rename into place (over +/// a torn file of the same name) and a sync of the folder. The marker needs no sync of its own +/// before it: no open reads the file before the rename, and the file is whole on the disk by then. +/// A crash before the rename leaves only the staging file, which no listing reads. +fn stage_segment( + storage: &dyn Storage, + dir: &Path, + header: &SegmentHeader, + body: &[u8], + end_marker: Option, +) -> std::io::Result> { + let name = format::segment_name(header.seq); + let staging = dir.join(format!("{name}{STAGING_SUFFIX}")); + let mut file = storage.open(&staging, OpenMode::Create)?; + let mut bytes = header.encode().to_vec(); + bytes.extend_from_slice(body); + if let Some(lsn) = end_marker { + format::encode_marker(&mut bytes, lsn); + } + file.append(&bytes)?; + file.sync(SyncLevel::Full)?; + storage.rename(&staging, &dir.join(&name))?; + storage.sync_dir(dir)?; + Ok(file) +} + +/// Makes segment `header.seq` in place (the writer's rotation and its new segment after a failed +/// sync: no record in it is acknowledged yet): its header, fully synced, and its folder entry, +/// synced. A torn file of the same name is made again. pub(crate) fn make_segment( storage: &dyn Storage, dir: &Path, @@ -252,7 +425,7 @@ pub(crate) fn make_segment( struct Headers { segments: Vec, /// The last segment's number when its header never got whole (its creation did not - /// complete): the writer makes it again. + /// complete): the new segment takes its number. reuse: Option, bad: Option, } @@ -266,14 +439,15 @@ fn read_headers( let mut reuse = None; let mut bad = None; for (index, (seq, name)) in listed.iter().enumerate() { - let file = storage.open(&dir.join(name), OpenMode::Read).map_err(io("open a segment"))?; - let size = file.size().map_err(io("read a segment's size"))?; + let (size, decoded) = { + let file = + storage.open(&dir.join(name), OpenMode::Read).map_err(io("open a segment"))?; + let size = file.size().map_err(io("read a segment's size"))?; + (size, read_header(file.as_ref(), size)?) + }; let last = index.saturating_add(1) == listed.len(); - let decoded = read_header(file.as_ref(), size)?; match decoded.and_then(|header| follows(&segments, header, *seq)) { - Ok(header) => { - segments.push(Segment { seq: *seq, name: name.clone(), file, size, header }); - } + Ok(header) => segments.push(Segment { seq: *seq, name: name.clone(), size, header }), Err(Unreadable::Header(_)) if last && size <= len64(SEGMENT_HEADER_LEN) => { reuse = Some(*seq); } @@ -345,46 +519,68 @@ fn follows( Ok(header) } -/// How far one segment's records read: the file offset after the last good record, and the -/// LSN of that record (the LSN before the segment's first when it has none). +/// How far one segment's entries read. struct Read { + /// The file offset after the last good entry. offset: u64, + /// The LSN of the last good record (the LSN before the segment's first when it has none). end: Lsn, + /// The LSN its last sync marker vouches for (the LSN before its first record when it has + /// none), and the offset after that marker. + vouched: Lsn, + vouched_offset: u64, } -fn read_records( +fn read_segment( + storage: &dyn Storage, + dir: &Path, segment: &Segment, trusted_end: Option, from: Lsn, replay: &mut dyn FnMut(Record<'_>) -> Result<(), String>, report: &mut OpenReport, ) -> Result { - let mut reader = - Reader { file: segment.file.as_ref(), size: segment.size, buf: Vec::new(), at: 0 }; - let mut offset = len64(SEGMENT_HEADER_LEN); - let mut expected = segment.header.first_lsn; + let file = + storage.open(&dir.join(&segment.name), OpenMode::Read).map_err(io("open a segment"))?; + let mut reader = Reader::new(file.as_ref(), segment.size); + let start = len64(SEGMENT_HEADER_LEN); + let before = segment.header.first_lsn.saturating_sub(1); + let mut read = Read { offset: start, end: before, vouched: before, vouched_offset: start }; loop { + let expected = read.end.saturating_add(1); if trusted_end.is_some_and(|end| expected > end) { break; } - let Some(head) = reader.bytes(offset, RECORD_HEADER_LEN)? else { break }; - let Ok(head) = <[u8; RECORD_HEADER_LEN]>::try_from(head) else { break }; - let Some(header) = RecordHeader::decode(&head) else { break }; - if header.len > MAX_PAYLOAD_LEN { - break; - } - let payload_at = offset.saturating_add(len64(RECORD_HEADER_LEN)); - let Some(payload) = reader.bytes(payload_at, header.len)? else { break }; - let Ok(class) = header.verify(&head, payload, expected) else { break }; - if expected > from { - replay(Record { lsn: expected, kind: header.kind, class, payload }) - .map_err(|error| OpenError::Replay { lsn: expected, error })?; - report.replayed = report.replayed.saturating_add(1); + let Some(entry) = reader.entry(read.offset)? else { break }; + match entry.kind { + Entry::Record(class) if entry.lsn == expected => { + if expected > from { + let payload = reader.payload(read.offset, entry.len)?; + replay(Record { lsn: expected, kind: entry.record_kind, class, payload }) + .map_err(|error| OpenError::Replay { lsn: expected, error })?; + report.replayed = report.replayed.saturating_add(1); + } + read.end = expected; + } + Entry::Marker if entry.lsn == read.end => { + read.vouched = read.end; + read.vouched_offset = read.offset.saturating_add(entry.bytes); + } + _ => break, } - offset = payload_at.saturating_add(len64(header.len)); - expected = expected.saturating_add(1); + read.offset = read.offset.saturating_add(entry.bytes); } - Ok(Read { offset, end: expected.saturating_sub(1) }) + Ok(read) +} + +/// One entry that checked out. +struct Checked { + kind: Entry, + lsn: Lsn, + record_kind: u16, + /// The payload length and the entry's whole length. + len: usize, + bytes: u64, } /// Reads a segment in chunks. @@ -396,7 +592,11 @@ struct Reader<'a> { at: u64, } -impl Reader<'_> { +impl<'a> Reader<'a> { + fn new(file: &'a dyn StorageFile, size: u64) -> Self { + Self { file, size, buf: Vec::new(), at: 0 } + } + /// `len` bytes at `offset`, or `None` when the file ends first. fn bytes(&mut self, offset: u64, len: usize) -> Result, OpenError> { let Some(end) = offset.checked_add(len64(len)) else { return Ok(None) }; @@ -405,14 +605,184 @@ impl Reader<'_> { } let buffered_end = self.at.saturating_add(len64(self.buf.len())); if offset < self.at || end > buffered_end { - let left = usize::try_from(self.size.saturating_sub(offset)).unwrap_or(usize::MAX); + let left = usize_of(self.size.saturating_sub(offset)); self.buf.resize(len.max(READ_CHUNK).min(left), 0); self.file.read_at(offset, &mut self.buf).map_err(io("read a segment"))?; self.at = offset; } - let start = usize::try_from(offset.saturating_sub(self.at)).unwrap_or(usize::MAX); + let start = usize_of(offset.saturating_sub(self.at)); Ok(self.buf.get(start..start.saturating_add(len))) } + + /// The entry at `offset` when it checks out (its checksum, flags and size), whatever its LSN. + fn entry(&mut self, offset: u64) -> Result, OpenError> { + self.entry_where(offset, &|_, _| true) + } + + /// The entry at `offset` when it checks out and `fits(kind, lsn)`: the cheap test runs before + /// the payload is read and its checksum computed. + fn entry_where( + &mut self, + offset: u64, + fits: &dyn Fn(Entry, Lsn) -> bool, + ) -> Result, OpenError> { + let Some(head) = self.bytes(offset, RECORD_HEADER_LEN)? else { return Ok(None) }; + let Ok(head) = <[u8; RECORD_HEADER_LEN]>::try_from(head) else { return Ok(None) }; + let Some(header) = RecordHeader::decode(&head) else { return Ok(None) }; + if header.len > MAX_PAYLOAD_LEN { + return Ok(None); + } + let guess = match header.flags { + format::MARKER => Entry::Marker, + _ => Entry::Record(crate::Class::Ack), + }; + if !fits(guess, header.lsn) { + return Ok(None); + } + let payload_at = offset.saturating_add(len64(RECORD_HEADER_LEN)); + let Some(payload) = self.bytes(payload_at, header.len)? else { return Ok(None) }; + let Ok(kind) = header.check(&head, payload) else { return Ok(None) }; + Ok(Some(Checked { + kind, + lsn: header.lsn, + record_kind: header.kind, + len: header.len, + bytes: len64(RECORD_HEADER_LEN).saturating_add(len64(header.len)), + })) + } + + /// The payload of the entry at `offset` (already checked by `entry`). + fn payload(&mut self, offset: u64, len: usize) -> Result<&[u8], OpenError> { + let at = offset.saturating_add(len64(RECORD_HEADER_LEN)); + Ok(self.bytes(at, len)?.unwrap_or_default()) + } +} + +/// What the bytes after a bad entry hold. +struct Beyond { + /// The highest LSN of an entry that checks out there (the base when none does). + max_lsn: Lsn, + /// A sync marker checks out there: the bad entry was synced. + marker: bool, +} + +fn scan_beyond_path( + storage: &dyn Storage, + path: &Path, + start: u64, + base: Lsn, +) -> Result { + let file = storage.open(path, OpenMode::Read).map_err(io("open a segment"))?; + let size = file.size().map_err(io("read a segment's size"))?; + scan_beyond(&mut Reader::new(file.as_ref(), size), start, base) +} + +/// Scans from `start` to the end, one byte at a time, for entries that check out and continue +/// past `base`: a record with an LSN above the last one found, or a marker at or above it. Such +/// LSNs are bounded by the number of entries the bytes can hold, so random bytes almost never +/// pass the LSN test, and their checksum is never computed. +fn scan_beyond(reader: &mut Reader<'_>, start: u64, base: Lsn) -> Result { + let span = reader.size.saturating_sub(start) / len64(RECORD_HEADER_LEN); + let high = base.saturating_add(span).saturating_add(1); + let mut beyond = Beyond { max_lsn: base, marker: false }; + let mut pos = start; + while pos.saturating_add(len64(RECORD_HEADER_LEN)) <= reader.size { + let low = beyond.max_lsn; + let fits = |kind: Entry, lsn: Lsn| match kind { + Entry::Record(_) => lsn > low && lsn <= high, + Entry::Marker => lsn >= low && lsn <= high, + }; + match reader.entry_where(pos, &fits)? { + Some(entry) => { + beyond.max_lsn = beyond.max_lsn.max(entry.lsn); + beyond.marker |= entry.kind == Entry::Marker; + pos = pos.saturating_add(entry.bytes); + } + None => pos = pos.saturating_add(1), + } + } + Ok(beyond) +} + +/// The bytes `[start, end)` of the segment at `path`. +fn read_range( + storage: &dyn Storage, + path: &Path, + start: u64, + end: u64, +) -> Result, OpenError> { + let file = storage.open(path, OpenMode::Read).map_err(io("open a segment"))?; + let mut bytes = vec![0u8; usize_of(end.saturating_sub(start))]; + file.read_at(start, &mut bytes).map_err(io("read a segment"))?; + Ok(bytes) +} + +/// The highest LSN a segment file holds. Its records are read in order from its first LSN, then +/// the bytes after the first bad entry are scanned. The first LSN comes from the header or, when +/// the header is damaged, from the record right after it (each record's checksum covers its +/// LSN). When neither reads, the scan takes records above `base`. +fn highest_lsn(storage: &dyn Storage, path: &Path, base: Lsn) -> Result { + let file = storage.open(path, OpenMode::Read).map_err(io("open a quarantined segment"))?; + let size = file.size().map_err(io("read a segment's size"))?; + let header = read_header(file.as_ref(), size)?.ok(); + let mut reader = Reader::new(file.as_ref(), size); + let start = len64(SEGMENT_HEADER_LEN); + let before = match header { + Some(header) => Some(header.first_lsn.saturating_sub(1)), + None => match reader.entry(start)? { + Some(Checked { kind: Entry::Record(_), lsn, .. }) => Some(lsn.saturating_sub(1)), + _ => None, + }, + }; + let (offset, from_lsn) = match before { + Some(before) => { + let mut end = before; + let mut offset = start; + while let Some(entry) = reader.entry(offset)? { + match entry.kind { + Entry::Record(_) if entry.lsn == end.saturating_add(1) => end = entry.lsn, + Entry::Marker if entry.lsn == end => {} + _ => break, + } + offset = offset.saturating_add(entry.bytes); + } + (offset, end.max(header.map_or(0, |header| header.prev_last))) + } + None => (start, base), + }; + Ok(scan_beyond(&mut reader, offset, from_lsn)?.max_lsn.max(from_lsn)) +} + +/// The highest LSN of the quarantined segments that lies above `base`. +fn quarantined_floor( + storage: &dyn Storage, + dir: &Path, + quarantined: &[(u64, String)], + base: Lsn, +) -> Result { + let mut floor = base; + for (_, name) in quarantined { + floor = floor.max(highest_lsn(storage, &dir.join(QUARANTINE_DIR).join(name), base)?); + } + Ok(floor) +} + +/// The journal id of a quarantined segment whose header reads, when no live one is left. +fn quarantined_journal_id( + storage: &dyn Storage, + dir: &Path, + quarantined: &[(u64, String)], +) -> Result, OpenError> { + for (_, name) in quarantined { + let file = storage + .open(&dir.join(QUARANTINE_DIR).join(name), OpenMode::Read) + .map_err(io("open a quarantined segment"))?; + let size = file.size().map_err(io("read a segment's size"))?; + if let Ok(header) = read_header(file.as_ref(), size)? { + return Ok(Some(header.journal_id)); + } + } + Ok(None) } /// The segments moved to quarantine, and the highest LSN they hold. @@ -422,20 +792,26 @@ struct Moved { } /// Moves every listed segment from number `from_seq` on into `quarantine/`, under a name no -/// earlier recovery used. +/// earlier recovery used. Their highest LSN is read first, then all are moved, then both folders +/// are synced. A crash in between is safe: the next open counts the LSNs of every quarantined +/// segment, and reports a quarantine that no new segment followed. fn quarantine( storage: &dyn Storage, dir: &Path, listed: &[(u64, String)], from_seq: u64, + base: Lsn, ) -> Result { let target = dir.join(QUARANTINE_DIR); storage.create_dir_all(&target).map_err(io("make the quarantine folder"))?; let taken = storage.list(&target).map_err(io("list the quarantine folder"))?; - let mut moved = Moved { names: Vec::new(), highest: 0 }; - for (_, name) in listed.iter().filter(|(seq, _)| *seq >= from_seq) { - let path = dir.join(name); - moved.highest = moved.highest.max(highest_lsn(storage, &path)?); + let moving: Vec<&String> = + listed.iter().filter(|(seq, _)| *seq >= from_seq).map(|(_, name)| name).collect(); + let mut moved = Moved { names: Vec::new(), highest: base }; + for name in &moving { + moved.highest = moved.highest.max(highest_lsn(storage, &dir.join(name), base)?); + } + for name in moving { let mut attempt = 0u32; let new_name = loop { let candidate = format!("{name}.{attempt}"); @@ -444,7 +820,7 @@ fn quarantine( } attempt = attempt.saturating_add(1); }; - storage.rename(&path, &target.join(&new_name)).map_err(io("move a damaged segment"))?; + storage.rename(&dir.join(name), &target.join(&new_name)).map_err(io("move a segment"))?; moved.names.push(new_name); } storage.sync_dir(&target).map_err(io("sync the quarantine folder"))?; @@ -452,16 +828,6 @@ fn quarantine( Ok(moved) } -/// The highest LSN a segment's readable records and header name (0 when its header is bad). -fn highest_lsn(storage: &dyn Storage, path: &Path) -> Result { - let file = storage.open(path, OpenMode::Read).map_err(io("open a damaged segment"))?; - let size = file.size().map_err(io("read a damaged segment's size"))?; - let Ok(header) = read_header(file.as_ref(), size)? else { return Ok(0) }; - let segment = Segment { seq: header.seq, name: String::new(), file, size, header }; - let read = read_records(&segment, None, Lsn::MAX, &mut |_| Ok(()), &mut OpenReport::default())?; - Ok(read.end.max(header.prev_last)) -} - /// A random journal id (never 0), from a per-process random hasher key and the clock. fn new_journal_id() -> u64 { use std::hash::{BuildHasher, Hasher}; diff --git a/cmux-tui/crates/cmux-journal/src/writer.rs b/cmux-tui/crates/cmux-journal/src/writer.rs index eac6b778979c..bcff22d771be 100644 --- a/cmux-tui/crates/cmux-journal/src/writer.rs +++ b/cmux-tui/crates/cmux-journal/src/writer.rs @@ -7,16 +7,29 @@ //! full one, or, when no batch comes, a full sync runs alone. So an ack waits for a drive cache //! flush at most once per interval, and never after an idle time. //! +//! Sync markers (`format.rs`): after each successful sync that covered records, the writer appends +//! a marker that vouches for them, before anything else it writes to that segment, after the +//! durable listeners ran (so no reply waits for it). The next sync covers the marker; at a seal and +//! at a rotation one more full sync covers it, so a sealed or full segment ends vouched for on +//! the disk. Recovery trusts the last segment only up to its last marker and writes the records +//! after it again into a new segment. +//! //! Failures: -//! - A failed write (a full disk): the segment is cut back to its last whole record, and the +//! - A failed write (a full disk): the segment is cut back to its size before the write, and the //! writer writes the same bytes again after a backoff. If the cut fails too, as below. //! - A failed sync, or a failed cut: the writer never writes or syncs that segment again. A failed //! sync can leave the kernel's copy of its pages clean and lost, and a second sync would then //! report success for data that never reached the disk. It keeps every record after the durable //! LSN in memory (`retained`) and, after a backoff, writes them into a new segment whose header //! names the durable LSN as its "previous last LSN", so recovery reads the failed segment only -//! through it. +//! through it. A restart in the backoff is safe too: the failed segment's records after its last +//! marker are written again by recovery. //! - A failed segment creation: made again from the start after a backoff. +//! - A panic on this thread (a listener's): the thread ends and the journal is closed: every +//! append fails with `Closed`, every wait ends with `WriterEnded`, the listeners are dropped and +//! `JournalStats::failing` says why. The daemon then stops (it never closed the journal itself), +//! and recovery runs at its next start. The crate does not abort the process: the daemon owns +//! that policy. //! //! While it backs off, the writer takes no new records; appends end in `AppendError::Backlog` //! once `Config::max_pending_bytes` wait. Nothing waits for the disk except this thread. @@ -31,7 +44,7 @@ use std::sync::{Arc, MutexGuard, PoisonError}; use std::time::{Duration, Instant}; use crate::Lsn; -use crate::format::{SEGMENT_HEADER_LEN, SegmentHeader}; +use crate::format::{self, RECORD_HEADER_LEN, SEGMENT_HEADER_LEN, SegmentHeader}; use crate::journal::{Shared, State}; use crate::recovery::{WriterStart, len64, make_segment}; use crate::storage::{Storage, StorageFile, SyncLevel}; @@ -54,12 +67,34 @@ struct Job { /// A failed storage operation ([module docs](self)). enum Failure { - /// A write; the segment is cut back to its last whole record and stays in use. + /// A write; the segment is cut back to its size before it and stays in use. Write(io::Error), /// A sync, a cut or a creation; the segment (if any) is not used again. Segment(io::Error), } +/// Closes the journal when the writer thread ends, also by a panic ([module docs](self)). The +/// listeners are dropped first, so whatever a held reply waits on (a channel) closes. +struct Ended(Arc); + +impl Drop for Ended { + fn drop(&mut self) { + let panicked = std::thread::panicking(); + self.0.listeners.lock().unwrap_or_else(PoisonError::into_inner).clear(); + { + let mut state = self.0.lock(); + state.ended = true; + state.closed = true; + if panicked { + state.stats.failing = + Some("the journal writer stopped: a panic on its thread".into()); + } + } + self.0.durable.notify_all(); + self.0.wake.notify_all(); + } +} + pub(crate) struct Writer { shared: Arc, storage: Arc, @@ -77,6 +112,12 @@ pub(crate) struct Writer { written: usize, /// When the oldest written and unsynced byte was written. unsynced_since: Option, + /// Records were written to `active` after its last marker. + unmarked: bool, + /// A marker `active` owes, for this LSN. + marker: Option, + /// A marker was written to `active` after its last sync. + marker_unsynced: bool, durable: Lsn, full_durable: Lsn, /// When a barrier sync first made records durable that no full sync covers yet. @@ -107,6 +148,9 @@ impl Writer { retained_ack: false, written: 0, unsynced_since: None, + unmarked: false, + marker: None, + marker_unsynced: false, durable, full_durable: durable, unfull_since: None, @@ -116,6 +160,7 @@ impl Writer { } pub(crate) fn run(mut self) { + let _ended = Ended(Arc::clone(&self.shared)); loop { let job = self.next_job(); if let Err(failure) = self.work(job.close) { @@ -125,8 +170,6 @@ impl Writer { break; } } - self.shared.lock().ended = true; - self.shared.durable.notify_all(); } /// Waits until something is owed and says what. @@ -241,6 +284,8 @@ impl Writer { // records the failed segment holds through `durable` become fully durable too. self.create_segment().map_err(Failure::Segment)?; } + // A marker that a failed write left owed goes first. + self.write_marker()?; let now = Instant::now(); if self.written < self.retained.len() { let Some(active) = self.active.as_mut() else { @@ -248,13 +293,11 @@ impl Writer { }; let unwritten = self.retained.get(self.written..).unwrap_or_default(); if let Err(error) = active.file.append(unwritten) { - return Err(match active.file.truncate(active.size) { - Ok(()) => Failure::Write(error), - Err(_) => Failure::Segment(error), - }); + return Err(cut_back(active, error)); } active.size = active.size.saturating_add(len64(unwritten.len())); self.written = self.retained.len(); + self.unmarked = true; if self.unsynced_since.is_none() { self.unsynced_since = Some(now); } @@ -274,9 +317,15 @@ impl Writer { if due || (full && owed) { let level = if full { SyncLevel::Full } else { SyncLevel::Barrier }; self.sync(level).map_err(Failure::Segment)?; + self.write_marker()?; + } + if (close || full_segment) && self.marker_unsynced { + // A sealed or full segment ends with its marker on the disk, so the next open finds + // every record vouched for and writes none again. + self.sync_marker().map_err(Failure::Segment)?; } if full_segment && self.retained.is_empty() { - // Fully synced: the next write makes the next segment. + // Fully synced: the next record makes the next segment. self.active = None; } Ok(()) @@ -285,10 +334,15 @@ impl Writer { fn sync(&mut self, level: SyncLevel) -> io::Result<()> { let Some(active) = self.active.as_mut() else { return Err(no_segment()) }; active.file.sync(level)?; + self.marker_unsynced = false; let now = Instant::now(); if !self.retained.is_empty() { self.durable = self.retained_last; } + if self.unmarked { + // Every record in the segment is on the disk now: the next write vouches for them. + self.marker = Some(self.durable); + } match level { SyncLevel::Full => { self.full_durable = self.durable; @@ -328,6 +382,32 @@ impl Writer { Ok(()) } + /// Appends the marker `active` owes ([module docs](self)). It is not synced here: the next + /// sync covers it, and recovery syncs the last segment before it reads it. + fn write_marker(&mut self) -> Result<(), Failure> { + let Some(lsn) = self.marker else { return Ok(()) }; + let Some(active) = self.active.as_mut() else { return Ok(()) }; + let mut bytes = Vec::with_capacity(RECORD_HEADER_LEN); + format::encode_marker(&mut bytes, lsn); + if let Err(error) = active.file.append(&bytes) { + return Err(cut_back(active, error)); + } + active.size = active.size.saturating_add(len64(bytes.len())); + self.marker = None; + self.unmarked = false; + self.marker_unsynced = true; + Ok(()) + } + + /// A full sync of `active` for its last marker alone (a seal or a rotation). + fn sync_marker(&mut self) -> io::Result<()> { + let Some(active) = self.active.as_mut() else { return Err(no_segment()) }; + active.file.sync(SyncLevel::Full)?; + self.marker_unsynced = false; + self.publish(|stats| stats.full_syncs = stats.full_syncs.saturating_add(1)); + Ok(()) + } + fn create_segment(&mut self) -> io::Result<()> { let header = SegmentHeader { journal_id: self.journal_id, @@ -339,6 +419,9 @@ impl Writer { self.active = Some(Active { file, size: len64(SEGMENT_HEADER_LEN) }); self.next_seq = header.seq.saturating_add(1); self.written = 0; + self.unmarked = false; + self.marker = None; + self.marker_unsynced = false; self.publish(|stats| stats.segments_created = stats.segments_created.saturating_add(1)); Ok(()) } @@ -352,6 +435,9 @@ impl Writer { self.active = None; self.written = 0; self.unsynced_since = None; + self.unmarked = false; + self.marker = None; + self.marker_unsynced = false; error } }; @@ -370,6 +456,15 @@ impl Writer { } } +/// A failed append: the segment is cut back to its size before it. A failed cut ends the +/// segment's use. +fn cut_back(active: &mut Active, error: io::Error) -> Failure { + match active.file.truncate(active.size) { + Ok(()) => Failure::Write(error), + Err(_) => Failure::Segment(error), + } +} + fn passed(since: Instant, delay: Duration, now: Instant) -> bool { since.checked_add(delay).is_none_or(|due| now >= due) } From 45f6486c427d7be14d41ab6c7acc349b9b9af04a Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 4 Nov 2026 01:49:13 +0000 Subject: [PATCH 10/52] cmux-journal: split the crash sim's machine model into sim/machine.rs (review G5) sim.rs had 924 lines and 63 functions, over the 60-function limit for a new Rust file (the god-file check warned "not in baseline: split it"). The machine model (each file's page cache and disk, the ordered log of names, and what an OS crash or a power loss leaves) moves to sim/machine.rs (297 lines, 19 functions); the process view and the Storage implementation stay in sim.rs (639 lines, 30 functions). No behavior change: the items move as they are, made visible to the sim module. --- cmux-tui/crates/cmux-journal/src/sim.rs | 293 +---------------- .../crates/cmux-journal/src/sim/machine.rs | 297 ++++++++++++++++++ 2 files changed, 301 insertions(+), 289 deletions(-) create mode 100644 cmux-tui/crates/cmux-journal/src/sim/machine.rs diff --git a/cmux-tui/crates/cmux-journal/src/sim.rs b/cmux-tui/crates/cmux-journal/src/sim.rs index 68b3e6541532..97291d3cedc3 100644 --- a/cmux-tui/crates/cmux-journal/src/sim.rs +++ b/cmux-tui/crates/cmux-journal/src/sim.rs @@ -45,10 +45,13 @@ use std::time::Duration; use crate::storage::{OpenMode, Storage, StorageFile, StorageLock, SyncLevel}; +mod machine; +use machine::{Change, Machine, SimFile}; + /// The page size of the simulated cache. pub const SIM_PAGE: usize = 4096; /// The unit a page write tears at. -const SECTOR: usize = 512; +pub(super) const SECTOR: usize = 512; /// What a simulated crash takes away ([`SimStorage::restart`]). #[derive(Clone, Copy, Debug, PartialEq, Eq)] @@ -96,294 +99,6 @@ pub struct SimStorage { fs: Arc>, } -#[derive(Clone)] -struct SimFile { - /// What reads see: the page cache. - data: Vec, - /// What the disk holds. - disk: Vec, - /// Pages of `data` written since the disk got them. - dirty: BTreeSet, - /// The disk content after each good sync, with its machine-wide sync number (for a power - /// loss). A file the last crash kept starts with its content as sync 0. - synced: Vec<(u64, Vec)>, -} - -impl SimFile { - fn empty() -> Self { - Self { data: Vec::new(), disk: Vec::new(), dirty: BTreeSet::new(), synced: Vec::new() } - } - - fn kept(content: Vec) -> Self { - Self { - data: content.clone(), - disk: content.clone(), - dirty: BTreeSet::new(), - synced: vec![(0, content)], - } - } - - /// Marks the pages of bytes `start..end` dirty. - fn touch(&mut self, start: usize, end: usize) { - if start < end { - self.dirty.extend(start / SIM_PAGE..end.div_ceil(SIM_PAGE)); - } - } - - fn append(&mut self, bytes: &[u8]) { - let start = self.data.len(); - self.data.extend_from_slice(bytes); - self.touch(start, self.data.len()); - } - - /// A shortening or an extension with zeros. The new length reaches the disk with the next - /// sync (`write_back`); the bytes an extension adds are dirty. - fn set_len(&mut self, len: usize) { - let old = self.data.len(); - self.data.resize(len, 0); - self.dirty.retain(|page| page.saturating_mul(SIM_PAGE) < len); - self.touch(old, len); - } - - /// Writes `pages` of the cache and its length to the disk. - fn write_back(&mut self, pages: &[usize]) { - self.disk.resize(self.data.len(), 0); - for page in pages { - let start = page.saturating_mul(SIM_PAGE).min(self.data.len()); - let end = start.saturating_add(SIM_PAGE).min(self.data.len()); - if let (Some(dst), Some(src)) = - (self.disk.get_mut(start..end), self.data.get(start..end)) - { - dst.copy_from_slice(src); - } - } - } - - /// A sync that completed: every dirty page and the length are on the disk. - fn good_sync(&mut self, number: u64, last_full: u64) { - let pages: Vec = self.dirty.iter().copied().collect(); - self.write_back(&pages); - self.dirty.clear(); - // Only the last content at or before the last full sync can come back after a crash. - if let Some(keep) = self.synced.iter().rposition(|(sync, _)| *sync <= last_full) { - self.synced.drain(..keep); - } - self.synced.push((number, self.disk.clone())); - } - - /// A sync that failed: a random part of the dirty pages reached the disk (none, without - /// `rng`); the cache dropped the rest as clean, so no later sync writes them. - fn failed_sync(&mut self, rng: Option<&mut SimRng>) { - let pages: Vec = match rng { - Some(rng) => self.dirty.iter().copied().filter(|_| rng.coin()).collect(), - None => Vec::new(), - }; - self.write_back(&pages); - self.dirty.clear(); - } - - /// What an OS crash could leave: the cache's dirty pages, and the disk's pages elsewhere - /// (a page a failed sync dropped never reached the disk). - fn os_view(&self) -> Vec { - let mut out = self.data.clone(); - for (page, chunk) in out.chunks_mut(SIM_PAGE).enumerate() { - if self.dirty.contains(&page) { - continue; - } - let start = page.saturating_mul(SIM_PAGE); - for (offset, byte) in chunk.iter_mut().enumerate() { - *byte = self.disk.get(start.saturating_add(offset)).copied().unwrap_or(0); - } - } - out - } - - /// The content after a crash: `cut` is the power-loss cut (none for an OS crash). - fn after_crash(&self, cut: Option, rng: &mut SimRng) -> Vec { - match cut { - None => mix(&self.disk, &self.os_view(), rng), - Some(cut) => { - let base = self - .synced - .iter() - .rev() - .find(|(sync, _)| *sync <= cut) - .map(|(_, content)| content.clone()) - .unwrap_or_default(); - let next = match self.synced.iter().find(|(sync, _)| *sync > cut) { - Some((_, content)) => content.clone(), - None => self.os_view(), - }; - mix(&base, &next, rng) - } - } - } -} - -/// `old` with some of `new`'s writes: a length between both lengths, and each page that differs -/// whole from `old`, whole from `new`, or `new`'s first sectors over `old`. Bytes past `new`'s end -/// (a shortening that did not reach the disk) stay `old`'s; bytes past both read as zeros. -fn mix(old: &[u8], new: &[u8], rng: &mut SimRng) -> Vec { - let (short, long) = (old.len().min(new.len()), old.len().max(new.len())); - let len = short.saturating_add(rng.index(long.saturating_sub(short).saturating_add(1))); - let old_at = |at: usize| old.get(at).copied().unwrap_or(0); - let new_at = |at: usize| new.get(at).copied().unwrap_or_else(|| old_at(at)); - let mut out = Vec::with_capacity(len); - let mut start = 0; - while start < len { - let end = start.saturating_add(SIM_PAGE).min(len); - let differs = (start..end).any(|at| old_at(at) != new_at(at)); - let new_until = match (differs, rng.below(3)) { - (false, _) | (true, 0) => start, - (true, 1) => end, - (true, _) => start.saturating_add(rng.index(SIM_PAGE / SECTOR).saturating_mul(SECTOR)), - }; - out.extend((start..end).map(|at| if at < new_until { new_at(at) } else { old_at(at) })); - start = end; - } - out -} - -/// A change of names, in the order the file system logs it. -#[derive(Clone, Debug)] -enum Change { - Folder(PathBuf), - Create(PathBuf, u64), - Rename(PathBuf, PathBuf), - Remove(PathBuf), -} - -impl Change { - /// Whether the change made or removed an entry of `dir`. - fn in_folder(&self, dir: &Path) -> bool { - let entry_of = |path: &PathBuf| path.parent() == Some(dir); - match self { - Self::Folder(path) | Self::Create(path, _) | Self::Remove(path) => entry_of(path), - Self::Rename(from, to) => entry_of(from) || entry_of(to), - } - } -} - -#[derive(Clone, Default)] -struct Names { - files: BTreeMap, - dirs: BTreeSet, -} - -impl Names { - fn apply(&mut self, change: &Change) { - match change { - Change::Folder(path) => { - self.dirs.insert(path.clone()); - } - Change::Create(path, inode) => { - self.files.insert(path.clone(), *inode); - } - Change::Rename(from, to) => { - if let Some(inode) = self.files.remove(from) { - self.files.insert(to.clone(), inode); - } - } - Change::Remove(path) => { - self.files.remove(path); - } - } - } -} - -/// What a process crash keeps: the cache, the disk and the names. -#[derive(Clone)] -struct Machine { - rng: SimRng, - files: BTreeMap, - next_inode: u64, - /// The names at the last crash (all durable), the changes since, how many of them are - /// durable, and the names now. - base: Names, - log: Vec, - durable_changes: usize, - names: Names, - /// Syncs so far (file and folder), and the number of the last full one. - syncs: u64, - last_full: u64, -} - -impl Machine { - fn new(seed: u64) -> Self { - Self { - rng: SimRng::new(seed), - files: BTreeMap::new(), - next_inode: 1, - base: Names::default(), - log: Vec::new(), - durable_changes: 0, - names: Names::default(), - syncs: 0, - last_full: 0, - } - } - - fn change(&mut self, change: Change) { - self.names.apply(&change); - self.log.push(change); - } - - fn sync_number(&mut self, level: SyncLevel) -> u64 { - self.syncs = self.syncs.saturating_add(1); - if level == SyncLevel::Full { - self.last_full = self.syncs; - } - self.syncs - } - - fn sync_folder(&mut self, dir: &Path) { - if let Some(last) = self.log.iter().rposition(|change| change.in_folder(dir)) { - self.durable_changes = self.durable_changes.max(last.saturating_add(1)); - } - // A folder sync flushes the drive like a full sync (FsStorage uses F_FULLFSYNC). - self.sync_number(SyncLevel::Full); - } - - fn file(&mut self, inode: u64) -> io::Result<&mut SimFile> { - self.files.get_mut(&inode).ok_or_else(|| io::Error::other("simulated file is gone")) - } - - /// The machine after an OS crash or a power loss ([module docs](self)). - fn after_crash(&mut self, crash: Crash) -> Self { - let mut rng = SimRng::new(self.rng.next_u64()); - let unsure = self.log.len().saturating_sub(self.durable_changes); - let kept = self.durable_changes.saturating_add(rng.index(unsure.saturating_add(1))); - let mut names = self.base.clone(); - for change in self.log.iter().take(kept) { - names.apply(change); - } - let cut = match crash { - Crash::Process | Crash::Os => None, - Crash::PowerLoss => { - let span = self.syncs.saturating_sub(self.last_full); - Some(self.last_full.saturating_add(rng.below(span.saturating_add(1)))) - } - }; - let mut files = BTreeMap::new(); - for inode in names.files.values() { - if let (false, Some(file)) = (files.contains_key(inode), self.files.get(inode)) { - files.insert(*inode, SimFile::kept(file.after_crash(cut, &mut rng))); - } - } - Self { - rng, - files, - next_inode: self.next_inode, - base: names.clone(), - log: Vec::new(), - durable_changes: 0, - names, - syncs: 0, - last_full: 0, - } - } -} - /// One process: its view of the machine. struct Fs { machine: Machine, diff --git a/cmux-tui/crates/cmux-journal/src/sim/machine.rs b/cmux-tui/crates/cmux-journal/src/sim/machine.rs new file mode 100644 index 000000000000..55ef8b6c13ca --- /dev/null +++ b/cmux-tui/crates/cmux-journal/src/sim/machine.rs @@ -0,0 +1,297 @@ +//! The simulated machine of `SimStorage` (see `sim.rs`): each file's page cache and disk, the +//! ordered log of names, and what an OS crash or a power loss leaves of them. + +use std::collections::{BTreeMap, BTreeSet}; +use std::io; +use std::path::{Path, PathBuf}; + +use super::{Crash, SECTOR, SIM_PAGE, SimRng}; +use crate::storage::SyncLevel; + +#[derive(Clone)] +pub(super) struct SimFile { + /// What reads see: the page cache. + pub(super) data: Vec, + /// What the disk holds. + pub(super) disk: Vec, + /// Pages of `data` written since the disk got them. + pub(super) dirty: BTreeSet, + /// The disk content after each good sync, with its machine-wide sync number (for a power + /// loss). A file the last crash kept starts with its content as sync 0. + pub(super) synced: Vec<(u64, Vec)>, +} + +impl SimFile { + pub(super) fn empty() -> Self { + Self { data: Vec::new(), disk: Vec::new(), dirty: BTreeSet::new(), synced: Vec::new() } + } + + pub(super) fn kept(content: Vec) -> Self { + Self { + data: content.clone(), + disk: content.clone(), + dirty: BTreeSet::new(), + synced: vec![(0, content)], + } + } + + /// Marks the pages of bytes `start..end` dirty. + pub(super) fn touch(&mut self, start: usize, end: usize) { + if start < end { + self.dirty.extend(start / SIM_PAGE..end.div_ceil(SIM_PAGE)); + } + } + + pub(super) fn append(&mut self, bytes: &[u8]) { + let start = self.data.len(); + self.data.extend_from_slice(bytes); + self.touch(start, self.data.len()); + } + + /// A shortening or an extension with zeros. The new length reaches the disk with the next + /// sync (`write_back`); the bytes an extension adds are dirty. + pub(super) fn set_len(&mut self, len: usize) { + let old = self.data.len(); + self.data.resize(len, 0); + self.dirty.retain(|page| page.saturating_mul(SIM_PAGE) < len); + self.touch(old, len); + } + + /// Writes `pages` of the cache and its length to the disk. + pub(super) fn write_back(&mut self, pages: &[usize]) { + self.disk.resize(self.data.len(), 0); + for page in pages { + let start = page.saturating_mul(SIM_PAGE).min(self.data.len()); + let end = start.saturating_add(SIM_PAGE).min(self.data.len()); + if let (Some(dst), Some(src)) = + (self.disk.get_mut(start..end), self.data.get(start..end)) + { + dst.copy_from_slice(src); + } + } + } + + /// A sync that completed: every dirty page and the length are on the disk. + pub(super) fn good_sync(&mut self, number: u64, last_full: u64) { + let pages: Vec = self.dirty.iter().copied().collect(); + self.write_back(&pages); + self.dirty.clear(); + // Only the last content at or before the last full sync can come back after a crash. + if let Some(keep) = self.synced.iter().rposition(|(sync, _)| *sync <= last_full) { + self.synced.drain(..keep); + } + self.synced.push((number, self.disk.clone())); + } + + /// A sync that failed: a random part of the dirty pages reached the disk (none, without + /// `rng`); the cache dropped the rest as clean, so no later sync writes them. + pub(super) fn failed_sync(&mut self, rng: Option<&mut SimRng>) { + let pages: Vec = match rng { + Some(rng) => self.dirty.iter().copied().filter(|_| rng.coin()).collect(), + None => Vec::new(), + }; + self.write_back(&pages); + self.dirty.clear(); + } + + /// What an OS crash could leave: the cache's dirty pages, and the disk's pages elsewhere + /// (a page a failed sync dropped never reached the disk). + pub(super) fn os_view(&self) -> Vec { + let mut out = self.data.clone(); + for (page, chunk) in out.chunks_mut(SIM_PAGE).enumerate() { + if self.dirty.contains(&page) { + continue; + } + let start = page.saturating_mul(SIM_PAGE); + for (offset, byte) in chunk.iter_mut().enumerate() { + *byte = self.disk.get(start.saturating_add(offset)).copied().unwrap_or(0); + } + } + out + } + + /// The content after a crash: `cut` is the power-loss cut (none for an OS crash). + pub(super) fn after_crash(&self, cut: Option, rng: &mut SimRng) -> Vec { + match cut { + None => mix(&self.disk, &self.os_view(), rng), + Some(cut) => { + let base = self + .synced + .iter() + .rev() + .find(|(sync, _)| *sync <= cut) + .map(|(_, content)| content.clone()) + .unwrap_or_default(); + let next = match self.synced.iter().find(|(sync, _)| *sync > cut) { + Some((_, content)) => content.clone(), + None => self.os_view(), + }; + mix(&base, &next, rng) + } + } + } +} + +/// `old` with some of `new`'s writes: a length between both lengths, and each page that differs +/// whole from `old`, whole from `new`, or `new`'s first sectors over `old`. Bytes past `new`'s end +/// (a shortening that did not reach the disk) stay `old`'s; bytes past both read as zeros. +pub(super) fn mix(old: &[u8], new: &[u8], rng: &mut SimRng) -> Vec { + let (short, long) = (old.len().min(new.len()), old.len().max(new.len())); + let len = short.saturating_add(rng.index(long.saturating_sub(short).saturating_add(1))); + let old_at = |at: usize| old.get(at).copied().unwrap_or(0); + let new_at = |at: usize| new.get(at).copied().unwrap_or_else(|| old_at(at)); + let mut out = Vec::with_capacity(len); + let mut start = 0; + while start < len { + let end = start.saturating_add(SIM_PAGE).min(len); + let differs = (start..end).any(|at| old_at(at) != new_at(at)); + let new_until = match (differs, rng.below(3)) { + (false, _) | (true, 0) => start, + (true, 1) => end, + (true, _) => start.saturating_add(rng.index(SIM_PAGE / SECTOR).saturating_mul(SECTOR)), + }; + out.extend((start..end).map(|at| if at < new_until { new_at(at) } else { old_at(at) })); + start = end; + } + out +} + +/// A change of names, in the order the file system logs it. +#[derive(Clone, Debug)] +pub(super) enum Change { + Folder(PathBuf), + Create(PathBuf, u64), + Rename(PathBuf, PathBuf), + Remove(PathBuf), +} + +impl Change { + /// Whether the change made or removed an entry of `dir`. + pub(super) fn in_folder(&self, dir: &Path) -> bool { + let entry_of = |path: &PathBuf| path.parent() == Some(dir); + match self { + Self::Folder(path) | Self::Create(path, _) | Self::Remove(path) => entry_of(path), + Self::Rename(from, to) => entry_of(from) || entry_of(to), + } + } +} + +#[derive(Clone, Default)] +pub(super) struct Names { + pub(super) files: BTreeMap, + pub(super) dirs: BTreeSet, +} + +impl Names { + pub(super) fn apply(&mut self, change: &Change) { + match change { + Change::Folder(path) => { + self.dirs.insert(path.clone()); + } + Change::Create(path, inode) => { + self.files.insert(path.clone(), *inode); + } + Change::Rename(from, to) => { + if let Some(inode) = self.files.remove(from) { + self.files.insert(to.clone(), inode); + } + } + Change::Remove(path) => { + self.files.remove(path); + } + } + } +} + +/// What a process crash keeps: the cache, the disk and the names. +#[derive(Clone)] +pub(super) struct Machine { + pub(super) rng: SimRng, + pub(super) files: BTreeMap, + pub(super) next_inode: u64, + /// The names at the last crash (all durable), the changes since, how many of them are + /// durable, and the names now. + pub(super) base: Names, + pub(super) log: Vec, + pub(super) durable_changes: usize, + pub(super) names: Names, + /// Syncs so far (file and folder), and the number of the last full one. + pub(super) syncs: u64, + pub(super) last_full: u64, +} + +impl Machine { + pub(super) fn new(seed: u64) -> Self { + Self { + rng: SimRng::new(seed), + files: BTreeMap::new(), + next_inode: 1, + base: Names::default(), + log: Vec::new(), + durable_changes: 0, + names: Names::default(), + syncs: 0, + last_full: 0, + } + } + + pub(super) fn change(&mut self, change: Change) { + self.names.apply(&change); + self.log.push(change); + } + + pub(super) fn sync_number(&mut self, level: SyncLevel) -> u64 { + self.syncs = self.syncs.saturating_add(1); + if level == SyncLevel::Full { + self.last_full = self.syncs; + } + self.syncs + } + + pub(super) fn sync_folder(&mut self, dir: &Path) { + if let Some(last) = self.log.iter().rposition(|change| change.in_folder(dir)) { + self.durable_changes = self.durable_changes.max(last.saturating_add(1)); + } + // A folder sync flushes the drive like a full sync (FsStorage uses F_FULLFSYNC). + self.sync_number(SyncLevel::Full); + } + + pub(super) fn file(&mut self, inode: u64) -> io::Result<&mut SimFile> { + self.files.get_mut(&inode).ok_or_else(|| io::Error::other("simulated file is gone")) + } + + /// The machine after an OS crash or a power loss ([module docs](self)). + pub(super) fn after_crash(&mut self, crash: Crash) -> Self { + let mut rng = SimRng::new(self.rng.next_u64()); + let unsure = self.log.len().saturating_sub(self.durable_changes); + let kept = self.durable_changes.saturating_add(rng.index(unsure.saturating_add(1))); + let mut names = self.base.clone(); + for change in self.log.iter().take(kept) { + names.apply(change); + } + let cut = match crash { + Crash::Process | Crash::Os => None, + Crash::PowerLoss => { + let span = self.syncs.saturating_sub(self.last_full); + Some(self.last_full.saturating_add(rng.below(span.saturating_add(1)))) + } + }; + let mut files = BTreeMap::new(); + for inode in names.files.values() { + if let (false, Some(file)) = (files.contains_key(inode), self.files.get(inode)) { + files.insert(*inode, SimFile::kept(file.after_crash(cut, &mut rng))); + } + } + Self { + rng, + files, + next_inode: self.next_inode, + base: names.clone(), + log: Vec::new(), + durable_changes: 0, + names, + syncs: 0, + last_full: 0, + } + } +} From 644e77a4c52e242115fe0bb5e968500ebcb60b47 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 4 Nov 2026 02:08:42 +0000 Subject: [PATCH 11/52] cmux-journal: re-review regressions G1-G4 and page eviction in the crash sim (red) Re-review FIX at 3498975b5267 (PR #33919 comment 6620113993). Tests and sim only; the fix follows. - sim: the pages a failed sync dropped stay in the cache until an eviction; `evict_dropped_pages_after_read(true)` makes each read evict the dropped pages it read, so a second read of them sees the disk's bytes. - G1 recovery_copies_the_records_it_checked_when_the_cache_drops_them: a sync fails and drops record 2's pages, a kill in the backoff, the next open reads record 2 from the cache (replayed) and the pages are evicted after that read; then an OS crash. Record 2 and the record 3 acked after the restart must survive. On this code recovery's second, unchecked read stages zeros under a vouching marker, and the last open keeps only record 1 and reports damage. - G2: the F1 regression asserts that every open after the damage reports the loss (lost_through 40). On this code a crash at the open's last step (the folder sync after the new segment's rename) loses the report for good. - G3 a_journal_in_a_symlinked_folder_opens_and_round_trips (real file system): a journal in a symlinked state folder, and a journal folder that is a symlink. On this code the parent folder's sync opens with O_NOFOLLOW (ELOOP). - G4 an_open_after_a_finished_quarantine_reads_no_quarantined_segment: on this code every open reads every quarantined segment in full. --- .../cmux-journal/src/cmux_journal_tests.rs | 101 ++++++++++++++++++ cmux-tui/crates/cmux-journal/src/sim.rs | 18 +++- .../crates/cmux-journal/src/sim/machine.rs | 44 +++++++- 3 files changed, 160 insertions(+), 3 deletions(-) diff --git a/cmux-tui/crates/cmux-journal/src/cmux_journal_tests.rs b/cmux-tui/crates/cmux-journal/src/cmux_journal_tests.rs index ab83c25d0d21..e5fc7dcd5e01 100644 --- a/cmux-tui/crates/cmux-journal/src/cmux_journal_tests.rs +++ b/cmux-tui/crates/cmux-journal/src/cmux_journal_tests.rs @@ -466,6 +466,14 @@ fn damaged_in_the_middle(seed: u64) -> (SimStorage, Config) { #[test] fn a_crash_inside_the_quarantine_step_never_reuses_a_quarantined_lsn() { let (damaged, config) = damaged_in_the_middle(50); + // Every open after the damage reports it (G2 of the re-review), also after a crash at the + // open's last step, which no later open can see the damage from. + let reports_the_loss = |report: &OpenReport| { + report + .damage + .iter() + .any(|damage| matches!(damage, Damage::Corrupt { lost_through: 40, .. })) + }; // Every storage operation of the open is a crash point once. let probe = damaged.snapshot(Crash::Process); let (journal, report, _) = open_at(&probe, config.clone(), 0).unwrap_or_else(|e| panic!("{e}")); @@ -494,6 +502,7 @@ fn a_crash_inside_the_quarantine_step_never_reuses_a_quarantined_lsn() { Ok(41), "{crash:?} at op {at}: {report:?}" ); + assert!(reports_the_loss(&report), "{crash:?} at op {at}: {report:?}"); } } assert!(crashed_after_a_move > 0, "no crash point fell after a move"); @@ -611,6 +620,63 @@ fn recovery_holds_at_most_two_files_open() { ); } +/// G1 of the re-review: a sync fails and drops its pages, and the process is killed before the +/// writer tries again. The next open reads those pages from the cache and replays their records; +/// then the kernel evicts the pages, and a second read would see the disk's zeros. The copy that +/// recovery writes must hold the records the replay got, so a later OS crash keeps them. +#[test] +fn recovery_copies_the_records_it_checked_when_the_cache_drops_them() { + let slow_retry = Config { + retry_min: Duration::from_secs(60), + retry_max: Duration::from_secs(60), + ..quick() + }; + let storage = SimStorage::new(64); + let (journal, _, _) = open_at(&storage, slow_retry, 0).unwrap_or_else(|e| panic!("{e}")); + // Record 1 fills page 0 and part of page 1, and is fully synced, its marker too. + assert_eq!(journal.append(1, Class::Ack, &vec![b'a'; SIM_PAGE + 1000]), Ok(1)); + journal.wait_durable(1, WAIT).unwrap_or_else(|e| panic!("{e}")); + assert!(eventually(|| journal.full_durable_lsn() >= 1), "{:?}", journal.stats()); + // Record 2's sync fails, and the disk gets none of its pages (page 1 on). + storage.fail_next_sync(2, std::io::ErrorKind::Other); + assert_eq!(journal.append(1, Class::Ack, &vec![b'b'; 2 * SIM_PAGE]), Ok(2)); + assert!(eventually(|| journal.stats().failures == 1), "{:?}", journal.stats()); + // A kill -9 while the writer waits to try again. + let restarted = storage.restart(Crash::Process); + drop(journal); + // Each read of the next open sees the cache, then evicts the dropped pages it read. + restarted.evict_dropped_pages_after_read(true); + let (journal, report, seen) = open(&restarted); + restarted.evict_dropped_pages_after_read(false); + assert_eq!(lsns(&seen), vec![1, 2], "{report:?}"); + assert_eq!(journal.append(1, Class::Ack, b"c"), Ok(3)); + journal.wait_durable(3, WAIT).unwrap_or_else(|e| panic!("{e}")); + drop(journal); + let (_, report, seen) = open(&restarted.restart(Crash::Os)); + assert_eq!(lsns(&seen), vec![1, 2, 3], "{report:?}"); + assert!(report.damage.is_empty(), "{report:?}"); +} + +/// G4 of the re-review: once a segment that an open made after its quarantine is live, the +/// quarantined segments hold no LSN above it, so later opens read none of them. +#[test] +fn an_open_after_a_finished_quarantine_reads_no_quarantined_segment() { + let (damaged, config) = damaged_in_the_middle(55); + let (journal, report, _) = + open_at(&damaged, config.clone(), 0).unwrap_or_else(|e| panic!("{e}")); + assert!(matches!(report.damage.first(), Some(Damage::Corrupt { .. })), "{report:?}"); + drop(journal); + let restarted = damaged.restart(Crash::Process); + let quarantine = dir().join("quarantine"); + let moved = restarted.names_in(&quarantine); + assert!(!moved.is_empty()); + let (journal, _, _) = open_at(&restarted, config, 0).unwrap_or_else(|e| panic!("{e}")); + for name in &moved { + assert_eq!(restarted.reads(&quarantine.join(name)), 0, "{name} was read"); + } + assert_eq!(journal.append(3, Class::Ack, b"after"), Ok(41)); +} + // ---------------------------------------------------------------- real file system #[test] @@ -654,3 +720,38 @@ fn append_raw(path: &Path, bytes: &[u8]) { .unwrap_or_else(|e| panic!("open {}: {e}", path.display())); file.write_all(bytes).unwrap_or_else(|e| panic!("append: {e}")); } + +/// G3 of the re-review: the daemon's state folder, or the journal folder itself, may be a +/// symlink (GNU stow, state kept on another volume). The journal opens there and round-trips. +#[cfg(unix)] +#[test] +fn a_journal_in_a_symlinked_folder_opens_and_round_trips() { + let root = tempfile::tempdir().unwrap_or_else(|e| panic!("tempdir: {e}")); + let real = root.path().join("real"); + let journal_real = root.path().join("journal-real"); + for folder in [&real, &journal_real] { + std::fs::create_dir(folder).unwrap_or_else(|e| panic!("mkdir: {e}")); + } + let state = root.path().join("state"); + std::os::unix::fs::symlink(&real, &state).unwrap_or_else(|e| panic!("symlink: {e}")); + let journal_link = real.join("journal"); + std::os::unix::fs::symlink(&journal_real, &journal_link) + .unwrap_or_else(|e| panic!("symlink: {e}")); + let storage: Arc = Arc::new(FsStorage); + // A journal folder in a symlinked state folder; a journal folder that is a symlink. + for dir in [state.join("chats-journal"), journal_link] { + let (journal, _) = + Journal::open(storage.clone(), &dir, Config::default(), 0, &mut |_| Ok(())) + .unwrap_or_else(|e| panic!("open {}: {e}", dir.display())); + assert_eq!(journal.append(5, Class::Ack, b"x"), Ok(1)); + assert_eq!(journal.seal(WAIT), Ok(1)); + drop(journal); + let mut seen = Vec::new(); + Journal::open(storage.clone(), &dir, Config::default(), 0, &mut |record| { + seen.push(record.lsn); + Ok(()) + }) + .unwrap_or_else(|e| panic!("reopen {}: {e}", dir.display())); + assert_eq!(seen, vec![1], "{}", dir.display()); + } +} diff --git a/cmux-tui/crates/cmux-journal/src/sim.rs b/cmux-tui/crates/cmux-journal/src/sim.rs index 97291d3cedc3..0150785be2f7 100644 --- a/cmux-tui/crates/cmux-journal/src/sim.rs +++ b/cmux-tui/crates/cmux-journal/src/sim.rs @@ -14,7 +14,8 @@ //! is one change: it never comes back twice or not at all. //! //! [`SimStorage::fail_next_sync`] fails the next sync of a write that spans pages, and the disk -//! gets none of them (the worst case). [`SimStorage::crash_after_failure`] puts the crash point +//! gets none of them (the worst case). The cache keeps the dropped pages until an eviction, which +//! [`SimStorage::evict_dropped_pages_after_read`] makes happen right after each read of them. [`SimStorage::crash_after_failure`] puts the crash point //! right after an injected failure: in the writer's backoff, at its first operation when it tries //! again. //! @@ -121,6 +122,8 @@ struct Fs { fail_sync: Option<(usize, io::ErrorKind)>, /// Once an injected failure happened, the operation this many after it crashes. crash_after_failure: Option, + /// Each read evicts the dropped pages it read (`SimStorage::evict_dropped_pages_after_read`). + evict_after_read: bool, } /// What an operation does: run, fail once, or crash the process. @@ -146,6 +149,7 @@ impl Fs { failures: 0, fail_sync: None, crash_after_failure: None, + evict_after_read: false, } } @@ -273,6 +277,14 @@ impl SimStorage { fs.crash_after_failure = Some(n); } + /// From now on, in this process, each read returns the bytes the cache holds and then evicts + /// the pages it read that a failed sync dropped: a second read of them sees the disk's bytes. + /// The kernel may evict a clean page at any time; this is the worst time for a reader that + /// reads twice. + pub fn evict_dropped_pages_after_read(&self, on: bool) { + lock(&self.fs).evict_after_read = on; + } + /// Operations that `fail_after` and `fail_next_sync` made fail. pub fn failures(&self) -> u64 { lock(&self.fs).failures @@ -558,11 +570,15 @@ impl StorageFile for SimHandle { } let count = fs.reads.entry(self.path.clone()).or_default(); *count = count.saturating_add(1); + let evict = fs.evict_after_read; let file = fs.machine.file(self.inode)?; let start = usize::try_from(offset).map_err(|_| io::ErrorKind::UnexpectedEof)?; let end = start.checked_add(buf.len()).ok_or(io::ErrorKind::UnexpectedEof)?; let src = file.data.get(start..end).ok_or(io::ErrorKind::UnexpectedEof)?; buf.copy_from_slice(src); + if evict { + file.evict(start, end); + } Ok(()) } diff --git a/cmux-tui/crates/cmux-journal/src/sim/machine.rs b/cmux-tui/crates/cmux-journal/src/sim/machine.rs index 55ef8b6c13ca..1a8a30727ca3 100644 --- a/cmux-tui/crates/cmux-journal/src/sim/machine.rs +++ b/cmux-tui/crates/cmux-journal/src/sim/machine.rs @@ -19,11 +19,20 @@ pub(super) struct SimFile { /// The disk content after each good sync, with its machine-wide sync number (for a power /// loss). A file the last crash kept starts with its content as sync 0. pub(super) synced: Vec<(u64, Vec)>, + /// Pages a failed sync dropped: the cache has their bytes, the disk does not, and no later + /// sync writes them (they are not dirty). An eviction makes reads see the disk's bytes. + pub(super) dropped: BTreeSet, } impl SimFile { pub(super) fn empty() -> Self { - Self { data: Vec::new(), disk: Vec::new(), dirty: BTreeSet::new(), synced: Vec::new() } + Self { + data: Vec::new(), + disk: Vec::new(), + dirty: BTreeSet::new(), + synced: Vec::new(), + dropped: BTreeSet::new(), + } } pub(super) fn kept(content: Vec) -> Self { @@ -32,13 +41,18 @@ impl SimFile { disk: content.clone(), dirty: BTreeSet::new(), synced: vec![(0, content)], + dropped: BTreeSet::new(), } } /// Marks the pages of bytes `start..end` dirty. pub(super) fn touch(&mut self, start: usize, end: usize) { if start < end { - self.dirty.extend(start / SIM_PAGE..end.div_ceil(SIM_PAGE)); + for page in start / SIM_PAGE..end.div_ceil(SIM_PAGE) { + // A dirty page goes to the disk whole at the next sync, dropped bytes included. + self.dropped.remove(&page); + self.dirty.insert(page); + } } } @@ -54,6 +68,7 @@ impl SimFile { let old = self.data.len(); self.data.resize(len, 0); self.dirty.retain(|page| page.saturating_mul(SIM_PAGE) < len); + self.dropped.retain(|page| page.saturating_mul(SIM_PAGE) < len); self.touch(old, len); } @@ -61,6 +76,7 @@ impl SimFile { pub(super) fn write_back(&mut self, pages: &[usize]) { self.disk.resize(self.data.len(), 0); for page in pages { + self.dropped.remove(page); let start = page.saturating_mul(SIM_PAGE).min(self.data.len()); let end = start.saturating_add(SIM_PAGE).min(self.data.len()); if let (Some(dst), Some(src)) = @@ -91,9 +107,33 @@ impl SimFile { None => Vec::new(), }; self.write_back(&pages); + let lost: Vec = + self.dirty.iter().copied().filter(|page| !pages.contains(page)).collect(); + self.dropped.extend(lost); self.dirty.clear(); } + /// The kernel evicts the dropped pages that bytes `start..end` touch: later reads see what the + /// disk holds there (zeros past its end). + pub(super) fn evict(&mut self, start: usize, end: usize) { + if start >= end { + return; + } + for page in start / SIM_PAGE..end.div_ceil(SIM_PAGE) { + if !self.dropped.remove(&page) { + continue; + } + let from = page.saturating_mul(SIM_PAGE).min(self.data.len()); + let to = from.saturating_add(SIM_PAGE).min(self.data.len()); + for at in from..to { + let disk = self.disk.get(at).copied().unwrap_or(0); + if let Some(byte) = self.data.get_mut(at) { + *byte = disk; + } + } + } + } + /// What an OS crash could leave: the cache's dirty pages, and the disk's pages elsewhere /// (a page a failed sync dropped never reached the disk). pub(super) fn os_view(&self) -> Vec { From cb893c6b64cd3d66c0ac083dbce95701ad6fb3ac Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 4 Nov 2026 02:39:19 +0000 Subject: [PATCH 12/52] cmux-journal: fix the re-review findings G1-G4 and G6 Re-review FIX at 3498975b5267 (PR #33919 comment 6620113993); the regression tests are the commit before this one (red: fleet de35f6d0). - G1: recovery writes again exactly the bytes its checked read returned. The last segment's entries after its last marker are kept from the checked read (header and payload) and staged from memory; the second, unchecked read of the file is gone. A page that a failed sync dropped can be evicted between two reads, and the second read then returned the disk's zeros, which the copy kept under a vouching marker. - G2: the damage an open finds is written to `DAMAGE` (whole, fully synced, renamed into place, folder synced) before the new segment that hides it from later opens. Every open reports the recorded damage first, until the daemon calls the new `Journal::clear_damage`, which moves the record into `quarantine/` (kept, never deleted). - G3: a folder sync follows a symlink in its path (no O_NOFOLLOW): a state folder, or the journal folder itself, may be a symlink. The sync is for the folder that holds the entries. - G4: the quarantined segments are read for their LSNs only while no live segment is numbered above them (an unfinished quarantine); once one is, its first LSN already carries their floor. - G6: `checked_div` in the scan of the bytes after a cut (the crate's one tracked arithmetic hit). - G5 (the sim split) is the commit before the regressions. --- cmux-tui/crates/cmux-journal/src/journal.rs | 19 +- cmux-tui/crates/cmux-journal/src/lib.rs | 5 +- cmux-tui/crates/cmux-journal/src/recovery.rs | 206 ++++++++++++++++--- cmux-tui/crates/cmux-journal/src/storage.rs | 8 +- 4 files changed, 199 insertions(+), 39 deletions(-) diff --git a/cmux-tui/crates/cmux-journal/src/journal.rs b/cmux-tui/crates/cmux-journal/src/journal.rs index dd7dce0ee2de..e578c10339a3 100644 --- a/cmux-tui/crates/cmux-journal/src/journal.rs +++ b/cmux-tui/crates/cmux-journal/src/journal.rs @@ -1,6 +1,6 @@ //! The public handle ([`Journal`]): append, durable LSNs, waits, listeners, seal. -use std::path::Path; +use std::path::{Path, PathBuf}; use std::sync::{Arc, Condvar, Mutex, MutexGuard, PoisonError}; use std::time::{Duration, Instant}; @@ -171,6 +171,8 @@ impl std::error::Error for OpenError {} /// one shared handle. pub struct Journal { shared: Arc, + storage: Arc, + dir: PathBuf, writer: Option>, /// Held as long as the writer may write; released after the writer is joined. _lock: Box, @@ -216,12 +218,15 @@ impl Journal { }); let report = recovered.report; let lock = recovered.lock; - let writer = Writer::new(shared.clone(), storage, dir.to_path_buf(), recovered.writer); + let writer = + Writer::new(shared.clone(), Arc::clone(&storage), dir.to_path_buf(), recovered.writer); let handle = std::thread::Builder::new() .name("cmux-journal-writer".into()) .spawn(move || writer.run()) .map_err(|error| OpenError::Io { what: "writer thread".into(), error })?; - Ok((Self { shared, writer: Some(handle), _lock: lock }, report)) + let journal = + Self { shared, storage, dir: dir.to_path_buf(), writer: Some(handle), _lock: lock }; + Ok((journal, report)) } /// Appends one record and returns its LSN. A memory copy: no I/O, no wait. The record is @@ -287,6 +292,14 @@ impl Journal { .push(Arc::new(listener)); } + /// Ends the reports of the damage that earlier opens found: every open reports it again + /// ([`OpenReport::damage`]) until the daemon has told the person and calls this. The record + /// moves into `quarantine/` (kept, never deleted). It renames and syncs folders on the + /// caller's thread: call it off any request path. + pub fn clear_damage(&self) -> std::io::Result<()> { + recovery::clear_damage(self.storage.as_ref(), &self.dir) + } + /// Blocks until record `lsn` is durable. For tests and tools: a daemon's request path /// never blocks on the disk; it holds its reply until a listener says the LSN is durable. pub fn wait_durable(&self, lsn: Lsn, timeout: Duration) -> Result<(), WaitError> { diff --git a/cmux-tui/crates/cmux-journal/src/lib.rs b/cmux-tui/crates/cmux-journal/src/lib.rs index b2692af9843c..4d440e8403cf 100644 --- a/cmux-tui/crates/cmux-journal/src/lib.rs +++ b/cmux-tui/crates/cmux-journal/src/lib.rs @@ -28,8 +28,9 @@ //! //! Files (the journal directory): `LOCK` (an exclusive lock held by the one writer), the segments //! (`.seg`: a header, then records and sync markers back to back, see -//! `format.rs`), and `quarantine/` for segments recovery could not read past damage that is not a -//! torn tail. A segment is only ever appended to. The one exception is the writer's own failed +//! `format.rs`), `quarantine/` for segments recovery could not read past damage that is not a +//! torn tail, and `DAMAGE`, the damage earlier opens found, reported at every open until +//! [`Journal::clear_damage`] moves it into `quarantine/`. A segment is only ever appended to. The one exception is the writer's own failed //! write: it cuts the segment back to its size before that write, so it removes only bytes that //! were never synced and never acknowledged. Nothing here deletes a segment or shortens one past //! a synced record. diff --git a/cmux-tui/crates/cmux-journal/src/recovery.rs b/cmux-tui/crates/cmux-journal/src/recovery.rs index 04d601be893c..84deae2752e0 100644 --- a/cmux-tui/crates/cmux-journal/src/recovery.rs +++ b/cmux-tui/crates/cmux-journal/src/recovery.rs @@ -18,7 +18,9 @@ //! 5. Damage that is not a torn tail (a bad header, or records missing before the next segment) //! keeps the valid prefix; the later segments move to `quarantine/` (never deleted). New //! records take LSNs after every LSN that any segment, live or quarantined, holds, so no LSN -//! is ever used twice, also after a crash in the middle of an earlier recovery. +//! is ever used twice, also after a crash in the middle of an earlier recovery. The damage is +//! written to `DAMAGE` (whole, synced) before the new segment of step 6 hides it from later +//! opens, and every open reports it again until `Journal::clear_damage`. //! 6. The journal goes on in a segment this open makes (the copy of step 4, or an empty one), never //! in a segment an earlier process wrote: that one may hold a page its cache had and its disk //! never got. @@ -38,6 +40,10 @@ use crate::{Lsn, Record}; pub(crate) const LOCK_NAME: &str = "LOCK"; pub(crate) const QUARANTINE_DIR: &str = "quarantine"; +/// The damage earlier opens found ([`Damage::Corrupt`]), kept until `Journal::clear_damage`. +pub(crate) const DAMAGE_NAME: &str = "DAMAGE"; +/// The name `Journal::clear_damage` gives a cleared damage record in `quarantine/`. +pub(crate) const CLEARED_DAMAGE: &str = "damage-report"; /// A segment recovery writes is first written whole under its name plus this, then renamed. const STAGING_SUFFIX: &str = ".new"; /// Segment bytes read at once. @@ -66,7 +72,9 @@ pub struct OpenReport { } /// What recovery could not read, beyond a torn tail: media damage or a bug, never a crash. The -/// daemon reports it to the person; moved segments stay in `quarantine/` for a salvage. +/// daemon reports it to the person; moved segments stay in `quarantine/` for a salvage. Every +/// open reports the [`Damage::Corrupt`] that earlier opens found, first, until the daemon calls +/// `Journal::clear_damage`. #[derive(Clone, Debug, PartialEq, Eq)] pub enum Damage { /// `segment` could not be read past `offset`. The segments in `moved` are in `quarantine/` @@ -151,6 +159,7 @@ pub(crate) fn recover( } let lock = storage.try_lock(&dir.join(LOCK_NAME)).map_err(io("lock"))?.ok_or(OpenError::Locked)?; + let earlier = read_damage(storage, dir)?; let listed = list_segments(storage, dir)?; let quarantined = list_quarantined(storage, dir)?; let live_max_seq = listed.last().map_or(0, |(seq, _)| *seq); @@ -231,7 +240,14 @@ pub(crate) fn recover( } // 5: every LSN a quarantined segment holds counts, from this open's moves and earlier ones. - let q_floor = quarantined_floor(storage, dir, &quarantined, journal_end)?; + // Once a live segment is numbered above every quarantined one, it was made after their + // quarantine with a first LSN above all of theirs, and `journal_end` is at least that: the + // quarantined files are not read again (review G4). + let q_floor = if q_max_seq > live_max_seq { + quarantined_floor(storage, dir, &quarantined, journal_end)? + } else { + journal_end + }; floor = floor.max(q_floor); if q_max_seq > live_max_seq && bad.is_none() { // An earlier recovery moved segments and stopped before its next segment was made. @@ -292,10 +308,23 @@ pub(crate) fn recover( Some(first) => first.header.journal_id, None => quarantined_journal_id(storage, dir, &quarantined)?.unwrap_or_else(new_journal_id), }; - let copy = match (segments.last(), last_read.as_ref()) { - (Some(last), Some(read)) if read.end > read.vouched => Some((last, read)), - _ => None, - }; + let copy = last_read.as_ref().filter(|read| read.end > read.vouched); + // The damage found here, durable before the new segment hides it from every later open + // (review G2), with the damage earlier opens recorded first. + let found: Vec = report + .damage + .iter() + .filter(|damage| matches!(damage, Damage::Corrupt { .. })) + .cloned() + .collect(); + if !found.is_empty() { + let mut all = earlier.clone(); + all.extend(found); + write_damage(storage, dir, &all).map_err(io("record the damage"))?; + } + let mut damage = earlier; + damage.append(&mut report.damage); + report.damage = damage; // The writer always starts in a segment this open makes: an earlier process's segment may // hold a page that its cache had and its disk never got (a failed sync), so nothing is // appended to it again. It is read only through the next segment's "previous last LSN". @@ -308,16 +337,17 @@ pub(crate) fn recover( }; let mut tail = None; let mut through = journal_end; - if let Some((last, read)) = copy { - // Step 4: the unvouched records, again, in a segment of their own. - let body = read_range(storage, &dir.join(&last.name), read.vouched_offset, read.offset)?; + if let Some(read) = copy { + // Step 4: the unvouched records, again, in a segment of their own: the bytes the + // checked read returned, never a second read of the file. + let body = &read.unvouched; let header = SegmentHeader { journal_id, seq: next_seq(), first_lsn: read.vouched.saturating_add(1), prev_last: read.vouched, }; - let file = stage_segment(storage, dir, &header, &body, Some(read.end)) + let file = stage_segment(storage, dir, &header, body, Some(read.end)) .map_err(io("write the unsynced records again"))?; let size = len64(SEGMENT_HEADER_LEN) .saturating_add(len64(body.len())) @@ -348,6 +378,111 @@ pub(crate) fn recover( Ok(Recovered { lock, next_lsn, report, writer: WriterStart { journal_id, tail } }) } +/// The damage that earlier opens recorded (`DAMAGE`, one line each), none when there is no file. +/// A line that does not read is reported as damage itself, so no report is lost silently. +fn read_damage(storage: &dyn Storage, dir: &Path) -> Result, OpenError> { + let file = match storage.open(&dir.join(DAMAGE_NAME), OpenMode::Read) { + Ok(file) => file, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()), + Err(error) => return Err(OpenError::Io { what: "open the damage record".into(), error }), + }; + let size = file.size().map_err(io("read the damage record's size"))?; + let mut bytes = vec![0u8; usize_of(size)]; + file.read_at(0, &mut bytes).map_err(io("read the damage record"))?; + let text = String::from_utf8_lossy(&bytes); + Ok(text.lines().filter(|line| !line.is_empty()).map(decode_damage).collect()) +} + +/// Writes every [`Damage::Corrupt`] of `damage` to `DAMAGE`: whole under a staging name, fully +/// synced, renamed into place, folder synced. +fn write_damage(storage: &dyn Storage, dir: &Path, damage: &[Damage]) -> std::io::Result<()> { + let mut text = String::new(); + for entry in damage { + if let Damage::Corrupt { segment, offset, reason, moved, lost_after, lost_through } = entry + { + let clean = |field: &str| field.replace(['\t', '\n', '\r'], " "); + let moved: Vec = moved.iter().map(|name| clean(name.as_str())).collect(); + text.push_str(&format!( + "corrupt\t{}\t{offset}\t{lost_after}\t{lost_through}\t{}\t{}\n", + clean(segment.as_str()), + moved.join(","), + clean(reason.as_str()), + )); + } + } + let staging = dir.join(format!("{DAMAGE_NAME}{STAGING_SUFFIX}")); + let mut file = storage.open(&staging, OpenMode::Create)?; + file.append(text.as_bytes())?; + file.sync(SyncLevel::Full)?; + storage.rename(&staging, &dir.join(DAMAGE_NAME))?; + storage.sync_dir(dir) +} + +/// One line of `DAMAGE`; a line that does not read is damage itself. +fn decode_damage(line: &str) -> Damage { + let fields: Vec<&str> = line.splitn(7, '\t').collect(); + let number = |index: usize| fields.get(index).and_then(|field| field.parse::().ok()); + match ( + fields.first(), + fields.get(1), + number(2), + number(3), + number(4), + fields.get(5), + fields.get(6), + ) { + ( + Some(&"corrupt"), + Some(segment), + Some(offset), + Some(lost_after), + Some(lost_through), + Some(moved), + Some(reason), + ) => Damage::Corrupt { + segment: (*segment).to_owned(), + offset, + reason: (*reason).to_owned(), + moved: moved.split(',').filter(|name| !name.is_empty()).map(str::to_owned).collect(), + lost_after, + lost_through, + }, + _ => Damage::Corrupt { + segment: DAMAGE_NAME.to_owned(), + offset: 0, + reason: format!("a line of the damage record does not read: {line}"), + moved: Vec::new(), + lost_after: 0, + lost_through: 0, + }, + } +} + +/// Moves `DAMAGE` into `quarantine/` (kept, never deleted), so later opens report no damage that +/// the daemon already told the person about. No file: nothing to do. +pub(crate) fn clear_damage(storage: &dyn Storage, dir: &Path) -> std::io::Result<()> { + let from = dir.join(DAMAGE_NAME); + match storage.open(&from, OpenMode::Read) { + Ok(_) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()), + Err(error) => return Err(error), + } + let target = dir.join(QUARANTINE_DIR); + storage.create_dir_all(&target)?; + let taken = storage.list(&target)?; + let mut attempt = 0u32; + let name = loop { + let candidate = format!("{CLEARED_DAMAGE}.{attempt}"); + if !taken.contains(&candidate) { + break candidate; + } + attempt = attempt.saturating_add(1); + }; + storage.rename(&from, &target.join(name))?; + storage.sync_dir(&target)?; + storage.sync_dir(dir) +} + /// The live segment files, by number. fn list_segments(storage: &dyn Storage, dir: &Path) -> Result, OpenError> { let mut listed: Vec<(u64, String)> = storage @@ -529,6 +664,11 @@ struct Read { /// none), and the offset after that marker. vouched: Lsn, vouched_offset: u64, + /// The last segment's entries after its last marker, as the checked read returned them + /// (header and payload): what recovery writes again (step 4). Never read from the file a + /// second time: a page that a failed sync dropped can be evicted between two reads, and the + /// second read then returns the disk's bytes (review G1). + unvouched: Vec, } fn read_segment( @@ -545,7 +685,15 @@ fn read_segment( let mut reader = Reader::new(file.as_ref(), segment.size); let start = len64(SEGMENT_HEADER_LEN); let before = segment.header.first_lsn.saturating_sub(1); - let mut read = Read { offset: start, end: before, vouched: before, vouched_offset: start }; + let mut read = Read { + offset: start, + end: before, + vouched: before, + vouched_offset: start, + unvouched: Vec::new(), + }; + // Only the last segment (no next segment bounds it) can be written again. + let keep_tail = trusted_end.is_none(); loop { let expected = read.end.saturating_add(1); if trusted_end.is_some_and(|end| expected > end) { @@ -554,17 +702,25 @@ fn read_segment( let Some(entry) = reader.entry(read.offset)? else { break }; match entry.kind { Entry::Record(class) if entry.lsn == expected => { - if expected > from { + if expected > from || keep_tail { + // The bytes the check covered: `entry` read them into the buffer. let payload = reader.payload(read.offset, entry.len)?; - replay(Record { lsn: expected, kind: entry.record_kind, class, payload }) - .map_err(|error| OpenError::Replay { lsn: expected, error })?; - report.replayed = report.replayed.saturating_add(1); + if expected > from { + replay(Record { lsn: expected, kind: entry.record_kind, class, payload }) + .map_err(|error| OpenError::Replay { lsn: expected, error })?; + report.replayed = report.replayed.saturating_add(1); + } + if keep_tail { + read.unvouched.extend_from_slice(&entry.head); + read.unvouched.extend_from_slice(payload); + } } read.end = expected; } Entry::Marker if entry.lsn == read.end => { read.vouched = read.end; read.vouched_offset = read.offset.saturating_add(entry.bytes); + read.unvouched.clear(); } _ => break, } @@ -576,6 +732,8 @@ fn read_segment( /// One entry that checked out. struct Checked { kind: Entry, + /// The header bytes the check covered. + head: [u8; RECORD_HEADER_LEN], lsn: Lsn, record_kind: u16, /// The payload length and the entry's whole length. @@ -644,6 +802,7 @@ impl<'a> Reader<'a> { let Ok(kind) = header.check(&head, payload) else { return Ok(None) }; Ok(Some(Checked { kind, + head, lsn: header.lsn, record_kind: header.kind, len: header.len, @@ -682,7 +841,7 @@ fn scan_beyond_path( /// LSNs are bounded by the number of entries the bytes can hold, so random bytes almost never /// pass the LSN test, and their checksum is never computed. fn scan_beyond(reader: &mut Reader<'_>, start: u64, base: Lsn) -> Result { - let span = reader.size.saturating_sub(start) / len64(RECORD_HEADER_LEN); + let span = reader.size.saturating_sub(start).checked_div(len64(RECORD_HEADER_LEN)).unwrap_or(0); let high = base.saturating_add(span).saturating_add(1); let mut beyond = Beyond { max_lsn: base, marker: false }; let mut pos = start; @@ -704,19 +863,6 @@ fn scan_beyond(reader: &mut Reader<'_>, start: u64, base: Lsn) -> Result Result, OpenError> { - let file = storage.open(path, OpenMode::Read).map_err(io("open a segment"))?; - let mut bytes = vec![0u8; usize_of(end.saturating_sub(start))]; - file.read_at(start, &mut bytes).map_err(io("read a segment"))?; - Ok(bytes) -} - /// The highest LSN a segment file holds. Its records are read in order from its first LSN, then /// the bytes after the first bad entry are scanned. The first LSN comes from the header or, when /// the header is damaged, from the record right after it (each record's checksum covers its diff --git a/cmux-tui/crates/cmux-journal/src/storage.rs b/cmux-tui/crates/cmux-journal/src/storage.rs index ef9f0c31a104..1704406fd62b 100644 --- a/cmux-tui/crates/cmux-journal/src/storage.rs +++ b/cmux-tui/crates/cmux-journal/src/storage.rs @@ -257,13 +257,13 @@ fn sync_file(file: &std::fs::File, level: SyncLevel) -> io::Result<()> { } } +/// Opens `dir` as a folder (`O_DIRECTORY`) and syncs it. A symlink in the path is followed: +/// the sync is for the folder that holds the entries, and a daemon's state folder, or the journal +/// folder itself, may be a symlink (GNU stow, state on another volume; review G3). #[cfg(unix)] fn sync_directory(dir: &Path) -> io::Result<()> { use std::os::unix::fs::OpenOptionsExt; - let file = std::fs::OpenOptions::new() - .read(true) - .custom_flags(libc::O_DIRECTORY | libc::O_NOFOLLOW) - .open(dir)?; + let file = std::fs::OpenOptions::new().read(true).custom_flags(libc::O_DIRECTORY).open(dir)?; sync_file(&file, SyncLevel::Full) } From 936b8ace4288ce77140fa75cd10f4d3fd00ad8fb Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 4 Nov 2026 03:30:18 +0000 Subject: [PATCH 13/52] cmux-journal: third-review regressions G7 (A, B) and N1 (red) Third review FIX at cb893c6b64cd (PR #33919 comment 6621681018). Tests only. - G7 A a_damaged_header_after_a_quarantine_never_reuses_a_quarantined_lsn: after a finished quarantine, one byte of the empty new segment's header changes. On this code the next open skips the quarantine scan (the listing counts the damaged segment) and the next append reuses a quarantined LSN. - G7 B a_damaged_newest_segment_after_a_quarantine_reports_its_records: the newest segment holds acked 41..43, and its header and its first record are damaged. On this code the moved segment is scanned with a base far below 41, so the loss is not reported through 43 and the next append is not 44. - N1: the F1 regression also asserts that an open reports each loss once (no reported LSN range lies inside another). On this code a crash after the quarantine's renames makes the next open report the same loss twice. --- .../cmux-journal/src/cmux_journal_tests.rs | 86 +++++++++++++++++++ 1 file changed, 86 insertions(+) diff --git a/cmux-tui/crates/cmux-journal/src/cmux_journal_tests.rs b/cmux-tui/crates/cmux-journal/src/cmux_journal_tests.rs index e5fc7dcd5e01..f9636163e15b 100644 --- a/cmux-tui/crates/cmux-journal/src/cmux_journal_tests.rs +++ b/cmux-tui/crates/cmux-journal/src/cmux_journal_tests.rs @@ -474,6 +474,24 @@ fn a_crash_inside_the_quarantine_step_never_reuses_a_quarantined_lsn() { .iter() .any(|damage| matches!(damage, Damage::Corrupt { lost_through: 40, .. })) }; + // And tells it once (N1 of the third review): no reported loss lies inside another one. + let reports_it_once = |report: &OpenReport| { + let ranges: Vec<(Lsn, Lsn)> = report + .damage + .iter() + .filter_map(|damage| match damage { + Damage::Corrupt { lost_after, lost_through, .. } => { + Some((*lost_after, *lost_through)) + } + _ => None, + }) + .collect(); + ranges.iter().enumerate().all(|(i, (after, through))| { + ranges.iter().enumerate().all(|(j, (other_after, other_through))| { + i == j || !(other_after <= after && through <= other_through) + }) + }) + }; // Every storage operation of the open is a crash point once. let probe = damaged.snapshot(Crash::Process); let (journal, report, _) = open_at(&probe, config.clone(), 0).unwrap_or_else(|e| panic!("{e}")); @@ -503,6 +521,7 @@ fn a_crash_inside_the_quarantine_step_never_reuses_a_quarantined_lsn() { "{crash:?} at op {at}: {report:?}" ); assert!(reports_the_loss(&report), "{crash:?} at op {at}: {report:?}"); + assert!(reports_it_once(&report), "{crash:?} at op {at}: {report:?}"); } } assert!(crashed_after_a_move > 0, "no crash point fell after a move"); @@ -677,6 +696,73 @@ fn an_open_after_a_finished_quarantine_reads_no_quarantined_segment() { assert_eq!(journal.append(3, Class::Ack, b"after"), Ok(41)); } +/// The newest segment after a finished quarantine, and the storage once that open has ended. +fn after_a_finished_quarantine(seed: u64) -> (SimStorage, Config, String) { + let (damaged, config) = damaged_in_the_middle(seed); + let (journal, report, _) = + open_at(&damaged, config.clone(), 0).unwrap_or_else(|e| panic!("{e}")); + assert!(matches!(report.damage.first(), Some(Damage::Corrupt { .. })), "{report:?}"); + drop(journal); + let storage = damaged.restart(Crash::Process); + let newest = storage + .names_in(&dir()) + .into_iter() + .filter(|name| name.ends_with(".seg")) + .max() + .unwrap_or_else(|| panic!("no segment")); + (storage, config, newest) +} + +/// G7 of the third review, scenario A: after a finished quarantine, the empty segment the open +/// made gets one damaged header byte. The next open must still count the quarantined LSNs. +#[test] +fn a_damaged_header_after_a_quarantine_never_reuses_a_quarantined_lsn() { + let (storage, config, newest) = after_a_finished_quarantine(56); + // One byte of the journal id in the header of the empty newest segment. + storage.corrupt(&dir().join(&newest), 20); + let (journal, report, _) = + open_at(&storage.restart(Crash::Process), config, 0).unwrap_or_else(|e| panic!("{e}")); + assert_eq!(journal.append(3, Class::Ack, b"after"), Ok(41), "{report:?}"); +} + +/// G7 scenario B: the newest segment holds acked records 41..43, and its header and its first +/// record are damaged. The loss must be reported through 43, and no LSN of it given out again. +#[test] +fn a_damaged_newest_segment_after_a_quarantine_reports_its_records() { + let (storage, config, newest) = after_a_finished_quarantine(57); + let (journal, _, _) = open_at(&storage, config.clone(), 0).unwrap_or_else(|e| panic!("{e}")); + for lsn in 41..=43 { + assert_eq!(journal.append(3, Class::Ack, &payload(lsn)), Ok(lsn)); + } + assert_eq!(journal.seal(WAIT), Ok(43)); + drop(journal); + let storage = storage.restart(Crash::Process); + let newest_now = storage + .names_in(&dir()) + .into_iter() + .filter(|name| name.ends_with(".seg")) + .max() + .unwrap_or_else(|| panic!("no segment")); + // The segment that holds 41..43 (the open made a new, empty one after it: damage both). + for name in [&newest, &newest_now] { + storage.corrupt(&dir().join(name), 20); + storage.corrupt(&dir().join(name), SEGMENT_HEADER_LEN + 2); + } + let (journal, report, _) = + open_at(&storage.restart(Crash::Process), config, 0).unwrap_or_else(|e| panic!("{e}")); + let lost_through = report + .damage + .iter() + .filter_map(|damage| match damage { + Damage::Corrupt { lost_through, .. } => Some(*lost_through), + _ => None, + }) + .max() + .unwrap_or(0); + assert!(lost_through >= 43, "the loss of 41..43 is not reported: {report:?}"); + assert_eq!(journal.append(3, Class::Ack, b"after"), Ok(44), "{report:?}"); +} + // ---------------------------------------------------------------- real file system #[test] From db9eb7bcda552ae31969d673f8fabd42ae27725f Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 4 Nov 2026 03:41:42 +0000 Subject: [PATCH 14/52] cmux-journal: fix the third-review findings G7, N1, N2; split recovery.rs Third review FIX at cb893c6b64cd (PR #33919 comment 6621681018); the regression tests are the commit before this one. - G7: the quarantine scan is skipped only when a segment whose header checked out is numbered above every quarantined segment (the folder listing also counted a newest segment with a damaged header, which carries no floor). The scans start from the running floor: `quarantined_floor` and `quarantine` pass the highest LSN counted so far as the base of the next file's scan, so a quarantined file with no readable header and no readable first record is scanned for the LSNs right above what is counted. - The "unfinished quarantine" report is made only when the quarantine holds LSNs nothing else counted (an old quarantine behind a segment that a crash tore at its creation holds none, and a crash is not damage). Its reason now also names a damaged newest header. - N1: a loss whose LSN range lies inside a recorded one is the same loss found again after a crash: it is neither recorded nor reported twice. - N2: recovery.rs's docs name `DAMAGE`; two doc lines over 100 columns are wrapped (lib.rs, sim.rs). - recovery.rs was at 987 of its 1000 lines: the damage record moves to recovery/damage.rs, the entry reader and the LSN scans to recovery/scan.rs (moved as they are). --- cmux-tui/crates/cmux-journal/src/lib.rs | 8 +- cmux-tui/crates/cmux-journal/src/recovery.rs | 427 ++---------------- .../cmux-journal/src/recovery/damage.rs | 118 +++++ .../crates/cmux-journal/src/recovery/scan.rs | 261 +++++++++++ cmux-tui/crates/cmux-journal/src/sim.rs | 6 +- 5 files changed, 434 insertions(+), 386 deletions(-) create mode 100644 cmux-tui/crates/cmux-journal/src/recovery/damage.rs create mode 100644 cmux-tui/crates/cmux-journal/src/recovery/scan.rs diff --git a/cmux-tui/crates/cmux-journal/src/lib.rs b/cmux-tui/crates/cmux-journal/src/lib.rs index 4d440e8403cf..bf4c54e9c1d8 100644 --- a/cmux-tui/crates/cmux-journal/src/lib.rs +++ b/cmux-tui/crates/cmux-journal/src/lib.rs @@ -30,10 +30,10 @@ //! (`.seg`: a header, then records and sync markers back to back, see //! `format.rs`), `quarantine/` for segments recovery could not read past damage that is not a //! torn tail, and `DAMAGE`, the damage earlier opens found, reported at every open until -//! [`Journal::clear_damage`] moves it into `quarantine/`. A segment is only ever appended to. The one exception is the writer's own failed -//! write: it cuts the segment back to its size before that write, so it removes only bytes that -//! were never synced and never acknowledged. Nothing here deletes a segment or shortens one past -//! a synced record. +//! [`Journal::clear_damage`] moves it into `quarantine/`. A segment is only ever appended to. The +//! one exception is the writer's own failed write: it cuts the segment back to its size before +//! that write, so it removes only bytes that were never synced and never acknowledged. Nothing +//! here deletes a segment or shortens one past a synced record. //! //! Recovery ([`Journal::open`]) is the one start path for a clean exit, a crash, a kill -9 and an //! OS crash: it syncs the last segment, reads the segments in LSN order, checks each record's diff --git a/cmux-tui/crates/cmux-journal/src/recovery.rs b/cmux-tui/crates/cmux-journal/src/recovery.rs index 84deae2752e0..25b246c819e6 100644 --- a/cmux-tui/crates/cmux-journal/src/recovery.rs +++ b/cmux-tui/crates/cmux-journal/src/recovery.rs @@ -25,19 +25,22 @@ //! in a segment an earlier process wrote: that one may hold a page its cache had and its disk //! never got. //! -//! Recovery writes nothing but syncs, new segments and the renames into `quarantine/`. It never -//! shortens or deletes a segment. +//! Recovery writes nothing but syncs, new segments, the renames into `quarantine/` and the damage +//! record `DAMAGE`. It never shortens or deletes a segment. use std::path::Path; -use crate::format::{ - self, Entry, MAX_PAYLOAD_LEN, RECORD_HEADER_LEN, RecordHeader, SEGMENT_HEADER_LEN, - SegmentHeader, -}; +use crate::format::{self, Entry, RECORD_HEADER_LEN, SEGMENT_HEADER_LEN, SegmentHeader}; use crate::journal::OpenError; use crate::storage::{OpenMode, Storage, StorageFile, StorageLock, SyncLevel}; use crate::{Lsn, Record}; +mod damage; +mod scan; +pub(crate) use damage::clear_damage; +use damage::{read_damage, write_damage}; +use scan::{Reader, quarantine, quarantined_floor, quarantined_journal_id, scan_beyond_path}; + pub(crate) const LOCK_NAME: &str = "LOCK"; pub(crate) const QUARANTINE_DIR: &str = "quarantine"; /// The damage earlier opens found ([`Damage::Corrupt`]), kept until `Journal::clear_damage`. @@ -134,6 +137,17 @@ struct Bad { lost_through: Lsn, } +/// `known` is a recorded loss and `found` a loss inside its LSN range. +fn covers(known: &Damage, found: &Damage) -> bool { + match (known, found) { + ( + Damage::Corrupt { lost_after: after, lost_through: through, .. }, + Damage::Corrupt { lost_after: found_after, lost_through: found_through, .. }, + ) => after <= found_after && found_through <= through, + _ => false, + } +} + fn io(what: &'static str) -> impl FnOnce(std::io::Error) -> OpenError { move |error| OpenError::Io { what: what.into(), error } } @@ -167,6 +181,11 @@ pub(crate) fn recover( // 1 and 2. let Headers { mut segments, reuse, mut bad } = read_headers(storage, dir, &listed)?; + // The newest segment whose header checked out. Only such a header carries the LSN floor of + // the quarantine before it (a damaged one says nothing: review G7). + let checked_max_seq = segments.last().map_or(0, |segment| segment.seq); + // The quarantine is unfinished, or its floor is not known from a live header. + let quarantine_open = q_max_seq > checked_max_seq; if let Some(last) = segments.last() { storage .open(&dir.join(&last.name), OpenMode::Write) @@ -240,26 +259,30 @@ pub(crate) fn recover( } // 5: every LSN a quarantined segment holds counts, from this open's moves and earlier ones. - // Once a live segment is numbered above every quarantined one, it was made after their - // quarantine with a first LSN above all of theirs, and `journal_end` is at least that: the - // quarantined files are not read again (review G4). - let q_floor = if q_max_seq > live_max_seq { - quarantined_floor(storage, dir, &quarantined, journal_end)? - } else { - journal_end - }; + // Once a segment whose header checked out is numbered above every quarantined one, it was + // made after their quarantine with a first LSN above all of theirs, and `journal_end` is at + // least that: the quarantined files are not read again (review G4). The scan starts from the + // running floor, so a quarantined file with no readable header is scanned for LSNs above + // what is already counted (review G7). + let before_quarantine = floor; + let q_floor = + if quarantine_open { quarantined_floor(storage, dir, &quarantined, floor)? } else { floor }; floor = floor.max(q_floor); - if q_max_seq > live_max_seq && bad.is_none() { - // An earlier recovery moved segments and stopped before its next segment was made. + // Reported only when the quarantine holds LSNs that nothing else counted: an old quarantine + // behind a segment torn by a crash at its creation holds none, and a crash is not damage. + if quarantine_open && bad.is_none() && q_floor > before_quarantine { + // An earlier recovery moved segments and stopped before its next segment was made, or + // that segment's header is damaged. report.damage.push(Damage::Corrupt { segment: format::segment_name(q_max_seq), offset: 0, - reason: "an earlier recovery moved these segments to quarantine/ and stopped before \ - it made the next segment" + reason: "segments in quarantine/ hold LSNs after the newest segment that reads: an \ + earlier recovery stopped before it made the next segment, or its header is \ + damaged" .to_owned(), moved: quarantined.iter().map(|(_, name)| name.clone()).collect(), lost_after: journal_end, - lost_through: q_floor.max(journal_end), + lost_through: q_floor, }); } let mut max_seq = live_max_seq.max(q_max_seq); @@ -281,7 +304,7 @@ pub(crate) fn recover( reuse = None; } if let Some(bad) = bad { - let moved = quarantine(storage, dir, &listed, bad.from_seq, journal_end)?; + let moved = quarantine(storage, dir, &listed, bad.from_seq, floor)?; let lost_through = floor.max(bad.lost_through).max(moved.highest); floor = lost_through; report.damage.push(Damage::Corrupt { @@ -310,16 +333,12 @@ pub(crate) fn recover( }; let copy = last_read.as_ref().filter(|read| read.end > read.vouched); // The damage found here, durable before the new segment hides it from every later open - // (review G2), with the damage earlier opens recorded first. - let found: Vec = report - .damage - .iter() - .filter(|damage| matches!(damage, Damage::Corrupt { .. })) - .cloned() - .collect(); - if !found.is_empty() { + // (review G2), after the damage earlier opens recorded. A loss that lies inside one already + // recorded is the same loss, found again after a crash: it is not recorded twice (review N1). + report.damage.retain(|damage| !earlier.iter().any(|known| covers(known, damage))); + if report.damage.iter().any(|damage| matches!(damage, Damage::Corrupt { .. })) { let mut all = earlier.clone(); - all.extend(found); + all.extend(report.damage.iter().cloned()); write_damage(storage, dir, &all).map_err(io("record the damage"))?; } let mut damage = earlier; @@ -378,111 +397,6 @@ pub(crate) fn recover( Ok(Recovered { lock, next_lsn, report, writer: WriterStart { journal_id, tail } }) } -/// The damage that earlier opens recorded (`DAMAGE`, one line each), none when there is no file. -/// A line that does not read is reported as damage itself, so no report is lost silently. -fn read_damage(storage: &dyn Storage, dir: &Path) -> Result, OpenError> { - let file = match storage.open(&dir.join(DAMAGE_NAME), OpenMode::Read) { - Ok(file) => file, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()), - Err(error) => return Err(OpenError::Io { what: "open the damage record".into(), error }), - }; - let size = file.size().map_err(io("read the damage record's size"))?; - let mut bytes = vec![0u8; usize_of(size)]; - file.read_at(0, &mut bytes).map_err(io("read the damage record"))?; - let text = String::from_utf8_lossy(&bytes); - Ok(text.lines().filter(|line| !line.is_empty()).map(decode_damage).collect()) -} - -/// Writes every [`Damage::Corrupt`] of `damage` to `DAMAGE`: whole under a staging name, fully -/// synced, renamed into place, folder synced. -fn write_damage(storage: &dyn Storage, dir: &Path, damage: &[Damage]) -> std::io::Result<()> { - let mut text = String::new(); - for entry in damage { - if let Damage::Corrupt { segment, offset, reason, moved, lost_after, lost_through } = entry - { - let clean = |field: &str| field.replace(['\t', '\n', '\r'], " "); - let moved: Vec = moved.iter().map(|name| clean(name.as_str())).collect(); - text.push_str(&format!( - "corrupt\t{}\t{offset}\t{lost_after}\t{lost_through}\t{}\t{}\n", - clean(segment.as_str()), - moved.join(","), - clean(reason.as_str()), - )); - } - } - let staging = dir.join(format!("{DAMAGE_NAME}{STAGING_SUFFIX}")); - let mut file = storage.open(&staging, OpenMode::Create)?; - file.append(text.as_bytes())?; - file.sync(SyncLevel::Full)?; - storage.rename(&staging, &dir.join(DAMAGE_NAME))?; - storage.sync_dir(dir) -} - -/// One line of `DAMAGE`; a line that does not read is damage itself. -fn decode_damage(line: &str) -> Damage { - let fields: Vec<&str> = line.splitn(7, '\t').collect(); - let number = |index: usize| fields.get(index).and_then(|field| field.parse::().ok()); - match ( - fields.first(), - fields.get(1), - number(2), - number(3), - number(4), - fields.get(5), - fields.get(6), - ) { - ( - Some(&"corrupt"), - Some(segment), - Some(offset), - Some(lost_after), - Some(lost_through), - Some(moved), - Some(reason), - ) => Damage::Corrupt { - segment: (*segment).to_owned(), - offset, - reason: (*reason).to_owned(), - moved: moved.split(',').filter(|name| !name.is_empty()).map(str::to_owned).collect(), - lost_after, - lost_through, - }, - _ => Damage::Corrupt { - segment: DAMAGE_NAME.to_owned(), - offset: 0, - reason: format!("a line of the damage record does not read: {line}"), - moved: Vec::new(), - lost_after: 0, - lost_through: 0, - }, - } -} - -/// Moves `DAMAGE` into `quarantine/` (kept, never deleted), so later opens report no damage that -/// the daemon already told the person about. No file: nothing to do. -pub(crate) fn clear_damage(storage: &dyn Storage, dir: &Path) -> std::io::Result<()> { - let from = dir.join(DAMAGE_NAME); - match storage.open(&from, OpenMode::Read) { - Ok(_) => {} - Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()), - Err(error) => return Err(error), - } - let target = dir.join(QUARANTINE_DIR); - storage.create_dir_all(&target)?; - let taken = storage.list(&target)?; - let mut attempt = 0u32; - let name = loop { - let candidate = format!("{CLEARED_DAMAGE}.{attempt}"); - if !taken.contains(&candidate) { - break candidate; - } - attempt = attempt.saturating_add(1); - }; - storage.rename(&from, &target.join(name))?; - storage.sync_dir(&target)?; - storage.sync_dir(dir) -} - /// The live segment files, by number. fn list_segments(storage: &dyn Storage, dir: &Path) -> Result, OpenError> { let mut listed: Vec<(u64, String)> = storage @@ -729,251 +643,6 @@ fn read_segment( Ok(read) } -/// One entry that checked out. -struct Checked { - kind: Entry, - /// The header bytes the check covered. - head: [u8; RECORD_HEADER_LEN], - lsn: Lsn, - record_kind: u16, - /// The payload length and the entry's whole length. - len: usize, - bytes: u64, -} - -/// Reads a segment in chunks. -struct Reader<'a> { - file: &'a dyn StorageFile, - size: u64, - buf: Vec, - /// The file offset of `buf`'s first byte. - at: u64, -} - -impl<'a> Reader<'a> { - fn new(file: &'a dyn StorageFile, size: u64) -> Self { - Self { file, size, buf: Vec::new(), at: 0 } - } - - /// `len` bytes at `offset`, or `None` when the file ends first. - fn bytes(&mut self, offset: u64, len: usize) -> Result, OpenError> { - let Some(end) = offset.checked_add(len64(len)) else { return Ok(None) }; - if end > self.size { - return Ok(None); - } - let buffered_end = self.at.saturating_add(len64(self.buf.len())); - if offset < self.at || end > buffered_end { - let left = usize_of(self.size.saturating_sub(offset)); - self.buf.resize(len.max(READ_CHUNK).min(left), 0); - self.file.read_at(offset, &mut self.buf).map_err(io("read a segment"))?; - self.at = offset; - } - let start = usize_of(offset.saturating_sub(self.at)); - Ok(self.buf.get(start..start.saturating_add(len))) - } - - /// The entry at `offset` when it checks out (its checksum, flags and size), whatever its LSN. - fn entry(&mut self, offset: u64) -> Result, OpenError> { - self.entry_where(offset, &|_, _| true) - } - - /// The entry at `offset` when it checks out and `fits(kind, lsn)`: the cheap test runs before - /// the payload is read and its checksum computed. - fn entry_where( - &mut self, - offset: u64, - fits: &dyn Fn(Entry, Lsn) -> bool, - ) -> Result, OpenError> { - let Some(head) = self.bytes(offset, RECORD_HEADER_LEN)? else { return Ok(None) }; - let Ok(head) = <[u8; RECORD_HEADER_LEN]>::try_from(head) else { return Ok(None) }; - let Some(header) = RecordHeader::decode(&head) else { return Ok(None) }; - if header.len > MAX_PAYLOAD_LEN { - return Ok(None); - } - let guess = match header.flags { - format::MARKER => Entry::Marker, - _ => Entry::Record(crate::Class::Ack), - }; - if !fits(guess, header.lsn) { - return Ok(None); - } - let payload_at = offset.saturating_add(len64(RECORD_HEADER_LEN)); - let Some(payload) = self.bytes(payload_at, header.len)? else { return Ok(None) }; - let Ok(kind) = header.check(&head, payload) else { return Ok(None) }; - Ok(Some(Checked { - kind, - head, - lsn: header.lsn, - record_kind: header.kind, - len: header.len, - bytes: len64(RECORD_HEADER_LEN).saturating_add(len64(header.len)), - })) - } - - /// The payload of the entry at `offset` (already checked by `entry`). - fn payload(&mut self, offset: u64, len: usize) -> Result<&[u8], OpenError> { - let at = offset.saturating_add(len64(RECORD_HEADER_LEN)); - Ok(self.bytes(at, len)?.unwrap_or_default()) - } -} - -/// What the bytes after a bad entry hold. -struct Beyond { - /// The highest LSN of an entry that checks out there (the base when none does). - max_lsn: Lsn, - /// A sync marker checks out there: the bad entry was synced. - marker: bool, -} - -fn scan_beyond_path( - storage: &dyn Storage, - path: &Path, - start: u64, - base: Lsn, -) -> Result { - let file = storage.open(path, OpenMode::Read).map_err(io("open a segment"))?; - let size = file.size().map_err(io("read a segment's size"))?; - scan_beyond(&mut Reader::new(file.as_ref(), size), start, base) -} - -/// Scans from `start` to the end, one byte at a time, for entries that check out and continue -/// past `base`: a record with an LSN above the last one found, or a marker at or above it. Such -/// LSNs are bounded by the number of entries the bytes can hold, so random bytes almost never -/// pass the LSN test, and their checksum is never computed. -fn scan_beyond(reader: &mut Reader<'_>, start: u64, base: Lsn) -> Result { - let span = reader.size.saturating_sub(start).checked_div(len64(RECORD_HEADER_LEN)).unwrap_or(0); - let high = base.saturating_add(span).saturating_add(1); - let mut beyond = Beyond { max_lsn: base, marker: false }; - let mut pos = start; - while pos.saturating_add(len64(RECORD_HEADER_LEN)) <= reader.size { - let low = beyond.max_lsn; - let fits = |kind: Entry, lsn: Lsn| match kind { - Entry::Record(_) => lsn > low && lsn <= high, - Entry::Marker => lsn >= low && lsn <= high, - }; - match reader.entry_where(pos, &fits)? { - Some(entry) => { - beyond.max_lsn = beyond.max_lsn.max(entry.lsn); - beyond.marker |= entry.kind == Entry::Marker; - pos = pos.saturating_add(entry.bytes); - } - None => pos = pos.saturating_add(1), - } - } - Ok(beyond) -} - -/// The highest LSN a segment file holds. Its records are read in order from its first LSN, then -/// the bytes after the first bad entry are scanned. The first LSN comes from the header or, when -/// the header is damaged, from the record right after it (each record's checksum covers its -/// LSN). When neither reads, the scan takes records above `base`. -fn highest_lsn(storage: &dyn Storage, path: &Path, base: Lsn) -> Result { - let file = storage.open(path, OpenMode::Read).map_err(io("open a quarantined segment"))?; - let size = file.size().map_err(io("read a segment's size"))?; - let header = read_header(file.as_ref(), size)?.ok(); - let mut reader = Reader::new(file.as_ref(), size); - let start = len64(SEGMENT_HEADER_LEN); - let before = match header { - Some(header) => Some(header.first_lsn.saturating_sub(1)), - None => match reader.entry(start)? { - Some(Checked { kind: Entry::Record(_), lsn, .. }) => Some(lsn.saturating_sub(1)), - _ => None, - }, - }; - let (offset, from_lsn) = match before { - Some(before) => { - let mut end = before; - let mut offset = start; - while let Some(entry) = reader.entry(offset)? { - match entry.kind { - Entry::Record(_) if entry.lsn == end.saturating_add(1) => end = entry.lsn, - Entry::Marker if entry.lsn == end => {} - _ => break, - } - offset = offset.saturating_add(entry.bytes); - } - (offset, end.max(header.map_or(0, |header| header.prev_last))) - } - None => (start, base), - }; - Ok(scan_beyond(&mut reader, offset, from_lsn)?.max_lsn.max(from_lsn)) -} - -/// The highest LSN of the quarantined segments that lies above `base`. -fn quarantined_floor( - storage: &dyn Storage, - dir: &Path, - quarantined: &[(u64, String)], - base: Lsn, -) -> Result { - let mut floor = base; - for (_, name) in quarantined { - floor = floor.max(highest_lsn(storage, &dir.join(QUARANTINE_DIR).join(name), base)?); - } - Ok(floor) -} - -/// The journal id of a quarantined segment whose header reads, when no live one is left. -fn quarantined_journal_id( - storage: &dyn Storage, - dir: &Path, - quarantined: &[(u64, String)], -) -> Result, OpenError> { - for (_, name) in quarantined { - let file = storage - .open(&dir.join(QUARANTINE_DIR).join(name), OpenMode::Read) - .map_err(io("open a quarantined segment"))?; - let size = file.size().map_err(io("read a segment's size"))?; - if let Ok(header) = read_header(file.as_ref(), size)? { - return Ok(Some(header.journal_id)); - } - } - Ok(None) -} - -/// The segments moved to quarantine, and the highest LSN they hold. -struct Moved { - names: Vec, - highest: Lsn, -} - -/// Moves every listed segment from number `from_seq` on into `quarantine/`, under a name no -/// earlier recovery used. Their highest LSN is read first, then all are moved, then both folders -/// are synced. A crash in between is safe: the next open counts the LSNs of every quarantined -/// segment, and reports a quarantine that no new segment followed. -fn quarantine( - storage: &dyn Storage, - dir: &Path, - listed: &[(u64, String)], - from_seq: u64, - base: Lsn, -) -> Result { - let target = dir.join(QUARANTINE_DIR); - storage.create_dir_all(&target).map_err(io("make the quarantine folder"))?; - let taken = storage.list(&target).map_err(io("list the quarantine folder"))?; - let moving: Vec<&String> = - listed.iter().filter(|(seq, _)| *seq >= from_seq).map(|(_, name)| name).collect(); - let mut moved = Moved { names: Vec::new(), highest: base }; - for name in &moving { - moved.highest = moved.highest.max(highest_lsn(storage, &dir.join(name), base)?); - } - for name in moving { - let mut attempt = 0u32; - let new_name = loop { - let candidate = format!("{name}.{attempt}"); - if !taken.contains(&candidate) { - break candidate; - } - attempt = attempt.saturating_add(1); - }; - storage.rename(&dir.join(name), &target.join(&new_name)).map_err(io("move a segment"))?; - moved.names.push(new_name); - } - storage.sync_dir(&target).map_err(io("sync the quarantine folder"))?; - storage.sync_dir(dir).map_err(io("sync the folder"))?; - Ok(moved) -} - /// A random journal id (never 0), from a per-process random hasher key and the clock. fn new_journal_id() -> u64 { use std::hash::{BuildHasher, Hasher}; diff --git a/cmux-tui/crates/cmux-journal/src/recovery/damage.rs b/cmux-tui/crates/cmux-journal/src/recovery/damage.rs new file mode 100644 index 000000000000..a8537363c6b6 --- /dev/null +++ b/cmux-tui/crates/cmux-journal/src/recovery/damage.rs @@ -0,0 +1,118 @@ +//! The damage record (`DAMAGE`): what earlier opens found, kept and reported at every open until +//! `Journal::clear_damage` moves it into `quarantine/` (step 5 of `recovery.rs`). + +use std::path::Path; + +use super::{CLEARED_DAMAGE, DAMAGE_NAME, QUARANTINE_DIR, STAGING_SUFFIX, io, usize_of}; +use crate::journal::OpenError; +use crate::recovery::Damage; +use crate::storage::{OpenMode, Storage, SyncLevel}; + +/// The damage that earlier opens recorded (`DAMAGE`, one line each), none when there is no file. +/// A line that does not read is reported as damage itself, so no report is lost silently. +pub(super) fn read_damage(storage: &dyn Storage, dir: &Path) -> Result, OpenError> { + let file = match storage.open(&dir.join(DAMAGE_NAME), OpenMode::Read) { + Ok(file) => file, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()), + Err(error) => return Err(OpenError::Io { what: "open the damage record".into(), error }), + }; + let size = file.size().map_err(io("read the damage record's size"))?; + let mut bytes = vec![0u8; usize_of(size)]; + file.read_at(0, &mut bytes).map_err(io("read the damage record"))?; + let text = String::from_utf8_lossy(&bytes); + Ok(text.lines().filter(|line| !line.is_empty()).map(decode_damage).collect()) +} + +/// Writes every [`Damage::Corrupt`] of `damage` to `DAMAGE`: whole under a staging name, fully +/// synced, renamed into place, folder synced. +pub(super) fn write_damage( + storage: &dyn Storage, + dir: &Path, + damage: &[Damage], +) -> std::io::Result<()> { + let mut text = String::new(); + for entry in damage { + if let Damage::Corrupt { segment, offset, reason, moved, lost_after, lost_through } = entry + { + let clean = |field: &str| field.replace(['\t', '\n', '\r'], " "); + let moved: Vec = moved.iter().map(|name| clean(name.as_str())).collect(); + text.push_str(&format!( + "corrupt\t{}\t{offset}\t{lost_after}\t{lost_through}\t{}\t{}\n", + clean(segment.as_str()), + moved.join(","), + clean(reason.as_str()), + )); + } + } + let staging = dir.join(format!("{DAMAGE_NAME}{STAGING_SUFFIX}")); + let mut file = storage.open(&staging, OpenMode::Create)?; + file.append(text.as_bytes())?; + file.sync(SyncLevel::Full)?; + storage.rename(&staging, &dir.join(DAMAGE_NAME))?; + storage.sync_dir(dir) +} + +/// One line of `DAMAGE`; a line that does not read is damage itself. +fn decode_damage(line: &str) -> Damage { + let fields: Vec<&str> = line.splitn(7, '\t').collect(); + let number = |index: usize| fields.get(index).and_then(|field| field.parse::().ok()); + match ( + fields.first(), + fields.get(1), + number(2), + number(3), + number(4), + fields.get(5), + fields.get(6), + ) { + ( + Some(&"corrupt"), + Some(segment), + Some(offset), + Some(lost_after), + Some(lost_through), + Some(moved), + Some(reason), + ) => Damage::Corrupt { + segment: (*segment).to_owned(), + offset, + reason: (*reason).to_owned(), + moved: moved.split(',').filter(|name| !name.is_empty()).map(str::to_owned).collect(), + lost_after, + lost_through, + }, + _ => Damage::Corrupt { + segment: DAMAGE_NAME.to_owned(), + offset: 0, + reason: format!("a line of the damage record does not read: {line}"), + moved: Vec::new(), + lost_after: 0, + lost_through: 0, + }, + } +} + +/// Moves `DAMAGE` into `quarantine/` (kept, never deleted), so later opens report no damage that +/// the daemon already told the person about. No file: nothing to do. +pub(crate) fn clear_damage(storage: &dyn Storage, dir: &Path) -> std::io::Result<()> { + let from = dir.join(DAMAGE_NAME); + match storage.open(&from, OpenMode::Read) { + Ok(_) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()), + Err(error) => return Err(error), + } + let target = dir.join(QUARANTINE_DIR); + storage.create_dir_all(&target)?; + let taken = storage.list(&target)?; + let mut attempt = 0u32; + let name = loop { + let candidate = format!("{CLEARED_DAMAGE}.{attempt}"); + if !taken.contains(&candidate) { + break candidate; + } + attempt = attempt.saturating_add(1); + }; + storage.rename(&from, &target.join(name))?; + storage.sync_dir(&target)?; + storage.sync_dir(dir) +} diff --git a/cmux-tui/crates/cmux-journal/src/recovery/scan.rs b/cmux-tui/crates/cmux-journal/src/recovery/scan.rs new file mode 100644 index 000000000000..f28896ccec1e --- /dev/null +++ b/cmux-tui/crates/cmux-journal/src/recovery/scan.rs @@ -0,0 +1,261 @@ +//! Reading segments entry by entry, and the scans that find every LSN a damaged or quarantined +//! segment holds (steps 3 to 5 of `recovery.rs`). + +use std::path::Path; + +use super::{QUARANTINE_DIR, READ_CHUNK, io, len64, read_header, usize_of}; +use crate::Lsn; +use crate::format::{ + self, Entry, MAX_PAYLOAD_LEN, RECORD_HEADER_LEN, RecordHeader, SEGMENT_HEADER_LEN, +}; +use crate::journal::OpenError; +use crate::storage::{OpenMode, Storage, StorageFile}; + +/// One entry that checked out. +pub(super) struct Checked { + pub(super) kind: Entry, + /// The header bytes the check covered. + pub(super) head: [u8; RECORD_HEADER_LEN], + pub(super) lsn: Lsn, + pub(super) record_kind: u16, + /// The payload length and the entry's whole length. + pub(super) len: usize, + pub(super) bytes: u64, +} + +/// Reads a segment in chunks. +pub(super) struct Reader<'a> { + file: &'a dyn StorageFile, + size: u64, + buf: Vec, + /// The file offset of `buf`'s first byte. + at: u64, +} + +impl<'a> Reader<'a> { + pub(super) fn new(file: &'a dyn StorageFile, size: u64) -> Self { + Self { file, size, buf: Vec::new(), at: 0 } + } + + /// `len` bytes at `offset`, or `None` when the file ends first. + pub(super) fn bytes(&mut self, offset: u64, len: usize) -> Result, OpenError> { + let Some(end) = offset.checked_add(len64(len)) else { return Ok(None) }; + if end > self.size { + return Ok(None); + } + let buffered_end = self.at.saturating_add(len64(self.buf.len())); + if offset < self.at || end > buffered_end { + let left = usize_of(self.size.saturating_sub(offset)); + self.buf.resize(len.max(READ_CHUNK).min(left), 0); + self.file.read_at(offset, &mut self.buf).map_err(io("read a segment"))?; + self.at = offset; + } + let start = usize_of(offset.saturating_sub(self.at)); + Ok(self.buf.get(start..start.saturating_add(len))) + } + + /// The entry at `offset` when it checks out (its checksum, flags and size), whatever its LSN. + pub(super) fn entry(&mut self, offset: u64) -> Result, OpenError> { + self.entry_where(offset, &|_, _| true) + } + + /// The entry at `offset` when it checks out and `fits(kind, lsn)`: the cheap test runs before + /// the payload is read and its checksum computed. + pub(super) fn entry_where( + &mut self, + offset: u64, + fits: &dyn Fn(Entry, Lsn) -> bool, + ) -> Result, OpenError> { + let Some(head) = self.bytes(offset, RECORD_HEADER_LEN)? else { return Ok(None) }; + let Ok(head) = <[u8; RECORD_HEADER_LEN]>::try_from(head) else { return Ok(None) }; + let Some(header) = RecordHeader::decode(&head) else { return Ok(None) }; + if header.len > MAX_PAYLOAD_LEN { + return Ok(None); + } + let guess = match header.flags { + format::MARKER => Entry::Marker, + _ => Entry::Record(crate::Class::Ack), + }; + if !fits(guess, header.lsn) { + return Ok(None); + } + let payload_at = offset.saturating_add(len64(RECORD_HEADER_LEN)); + let Some(payload) = self.bytes(payload_at, header.len)? else { return Ok(None) }; + let Ok(kind) = header.check(&head, payload) else { return Ok(None) }; + Ok(Some(Checked { + kind, + head, + lsn: header.lsn, + record_kind: header.kind, + len: header.len, + bytes: len64(RECORD_HEADER_LEN).saturating_add(len64(header.len)), + })) + } + + /// The payload of the entry at `offset` (already checked by `entry`). + pub(super) fn payload(&mut self, offset: u64, len: usize) -> Result<&[u8], OpenError> { + let at = offset.saturating_add(len64(RECORD_HEADER_LEN)); + Ok(self.bytes(at, len)?.unwrap_or_default()) + } +} + +/// What the bytes after a bad entry hold. +pub(super) struct Beyond { + /// The highest LSN of an entry that checks out there (the base when none does). + pub(super) max_lsn: Lsn, + /// A sync marker checks out there: the bad entry was synced. + pub(super) marker: bool, +} + +pub(super) fn scan_beyond_path( + storage: &dyn Storage, + path: &Path, + start: u64, + base: Lsn, +) -> Result { + let file = storage.open(path, OpenMode::Read).map_err(io("open a segment"))?; + let size = file.size().map_err(io("read a segment's size"))?; + scan_beyond(&mut Reader::new(file.as_ref(), size), start, base) +} + +/// Scans from `start` to the end, one byte at a time, for entries that check out and continue +/// past `base`: a record with an LSN above the last one found, or a marker at or above it. Such +/// LSNs are bounded by the number of entries the bytes can hold, so random bytes almost never +/// pass the LSN test, and their checksum is never computed. +pub(super) fn scan_beyond( + reader: &mut Reader<'_>, + start: u64, + base: Lsn, +) -> Result { + let span = reader.size.saturating_sub(start).checked_div(len64(RECORD_HEADER_LEN)).unwrap_or(0); + let high = base.saturating_add(span).saturating_add(1); + let mut beyond = Beyond { max_lsn: base, marker: false }; + let mut pos = start; + while pos.saturating_add(len64(RECORD_HEADER_LEN)) <= reader.size { + let low = beyond.max_lsn; + let fits = |kind: Entry, lsn: Lsn| match kind { + Entry::Record(_) => lsn > low && lsn <= high, + Entry::Marker => lsn >= low && lsn <= high, + }; + match reader.entry_where(pos, &fits)? { + Some(entry) => { + beyond.max_lsn = beyond.max_lsn.max(entry.lsn); + beyond.marker |= entry.kind == Entry::Marker; + pos = pos.saturating_add(entry.bytes); + } + None => pos = pos.saturating_add(1), + } + } + Ok(beyond) +} + +/// The highest LSN a segment file holds. Its records are read in order from its first LSN, then +/// the bytes after the first bad entry are scanned. The first LSN comes from the header or, when +/// the header is damaged, from the record right after it (each record's checksum covers its +/// LSN). When neither reads, the scan takes records above `base`. +pub(super) fn highest_lsn(storage: &dyn Storage, path: &Path, base: Lsn) -> Result { + let file = storage.open(path, OpenMode::Read).map_err(io("open a quarantined segment"))?; + let size = file.size().map_err(io("read a segment's size"))?; + let header = read_header(file.as_ref(), size)?.ok(); + let mut reader = Reader::new(file.as_ref(), size); + let start = len64(SEGMENT_HEADER_LEN); + let before = match header { + Some(header) => Some(header.first_lsn.saturating_sub(1)), + None => match reader.entry(start)? { + Some(Checked { kind: Entry::Record(_), lsn, .. }) => Some(lsn.saturating_sub(1)), + _ => None, + }, + }; + let (offset, from_lsn) = match before { + Some(before) => { + let mut end = before; + let mut offset = start; + while let Some(entry) = reader.entry(offset)? { + match entry.kind { + Entry::Record(_) if entry.lsn == end.saturating_add(1) => end = entry.lsn, + Entry::Marker if entry.lsn == end => {} + _ => break, + } + offset = offset.saturating_add(entry.bytes); + } + (offset, end.max(header.map_or(0, |header| header.prev_last))) + } + None => (start, base), + }; + Ok(scan_beyond(&mut reader, offset, from_lsn)?.max_lsn.max(from_lsn)) +} + +/// The highest LSN of the quarantined segments that lies above `base`. +pub(super) fn quarantined_floor( + storage: &dyn Storage, + dir: &Path, + quarantined: &[(u64, String)], + base: Lsn, +) -> Result { + let mut floor = base; + for (_, name) in quarantined { + floor = floor.max(highest_lsn(storage, &dir.join(QUARANTINE_DIR).join(name), floor)?); + } + Ok(floor) +} + +/// The journal id of a quarantined segment whose header reads, when no live one is left. +pub(super) fn quarantined_journal_id( + storage: &dyn Storage, + dir: &Path, + quarantined: &[(u64, String)], +) -> Result, OpenError> { + for (_, name) in quarantined { + let file = storage + .open(&dir.join(QUARANTINE_DIR).join(name), OpenMode::Read) + .map_err(io("open a quarantined segment"))?; + let size = file.size().map_err(io("read a segment's size"))?; + if let Ok(header) = read_header(file.as_ref(), size)? { + return Ok(Some(header.journal_id)); + } + } + Ok(None) +} + +/// The segments moved to quarantine, and the highest LSN they hold. +pub(super) struct Moved { + pub(super) names: Vec, + pub(super) highest: Lsn, +} + +/// Moves every listed segment from number `from_seq` on into `quarantine/`, under a name no +/// earlier recovery used. Their highest LSN is read first, then all are moved, then both folders +/// are synced. A crash in between is safe: the next open counts the LSNs of every quarantined +/// segment, and reports a quarantine that no new segment followed. +pub(super) fn quarantine( + storage: &dyn Storage, + dir: &Path, + listed: &[(u64, String)], + from_seq: u64, + base: Lsn, +) -> Result { + let target = dir.join(QUARANTINE_DIR); + storage.create_dir_all(&target).map_err(io("make the quarantine folder"))?; + let taken = storage.list(&target).map_err(io("list the quarantine folder"))?; + let moving: Vec<&String> = + listed.iter().filter(|(seq, _)| *seq >= from_seq).map(|(_, name)| name).collect(); + let mut moved = Moved { names: Vec::new(), highest: base }; + for name in &moving { + moved.highest = moved.highest.max(highest_lsn(storage, &dir.join(name), moved.highest)?); + } + for name in moving { + let mut attempt = 0u32; + let new_name = loop { + let candidate = format!("{name}.{attempt}"); + if !taken.contains(&candidate) { + break candidate; + } + attempt = attempt.saturating_add(1); + }; + storage.rename(&dir.join(name), &target.join(&new_name)).map_err(io("move a segment"))?; + moved.names.push(new_name); + } + storage.sync_dir(&target).map_err(io("sync the quarantine folder"))?; + storage.sync_dir(dir).map_err(io("sync the folder"))?; + Ok(moved) +} diff --git a/cmux-tui/crates/cmux-journal/src/sim.rs b/cmux-tui/crates/cmux-journal/src/sim.rs index 0150785be2f7..c4eac20bf19f 100644 --- a/cmux-tui/crates/cmux-journal/src/sim.rs +++ b/cmux-tui/crates/cmux-journal/src/sim.rs @@ -15,9 +15,9 @@ //! //! [`SimStorage::fail_next_sync`] fails the next sync of a write that spans pages, and the disk //! gets none of them (the worst case). The cache keeps the dropped pages until an eviction, which -//! [`SimStorage::evict_dropped_pages_after_read`] makes happen right after each read of them. [`SimStorage::crash_after_failure`] puts the crash point -//! right after an injected failure: in the writer's backoff, at its first operation when it tries -//! again. +//! [`SimStorage::evict_dropped_pages_after_read`] makes happen right after each read of them. +//! [`SimStorage::crash_after_failure`] puts the crash point right after an injected failure: in +//! the writer's backoff, at its first operation when it tries again. //! //! [`SimStorage::crash_after`] makes the n-th storage operation from now the crash point: an //! append there writes a random prefix of its bytes, a create, rename, truncate or sync there From 215c8f0485ea000b719aad0fbf29dde9e82703c8 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 4 Nov 2026 03:49:16 +0000 Subject: [PATCH 15/52] cmux-journal: report each loss once, also when one open finds it twice (N1) The red run of the N1 regression (fleet 27d011ca, op 64 of the F1 test) shows the case the previous commit did not cover: a crash after the quarantine's renames and before DAMAGE is written; the next open finds both the damaged synced record of the last segment (7..14) and the unfinished quarantine (7..40), and reported both. Now every loss inside another (recorded or found by this open) is dropped, the larger range kept; DAMAGE is rewritten only when its set of losses changed. --- cmux-tui/crates/cmux-journal/src/recovery.rs | 60 +++++++++++++------- 1 file changed, 39 insertions(+), 21 deletions(-) diff --git a/cmux-tui/crates/cmux-journal/src/recovery.rs b/cmux-tui/crates/cmux-journal/src/recovery.rs index 25b246c819e6..d277c106770f 100644 --- a/cmux-tui/crates/cmux-journal/src/recovery.rs +++ b/cmux-tui/crates/cmux-journal/src/recovery.rs @@ -76,8 +76,8 @@ pub struct OpenReport { /// What recovery could not read, beyond a torn tail: media damage or a bug, never a crash. The /// daemon reports it to the person; moved segments stay in `quarantine/` for a salvage. Every -/// open reports the [`Damage::Corrupt`] that earlier opens found, first, until the daemon calls -/// `Journal::clear_damage`. +/// open reports the losses ([`Damage::Corrupt`]) that earlier opens found, until the daemon calls +/// `Journal::clear_damage`; a loss that lies inside another is reported once, as the larger range. #[derive(Clone, Debug, PartialEq, Eq)] pub enum Damage { /// `segment` could not be read past `offset`. The segments in `moved` are in `quarantine/` @@ -137,15 +137,32 @@ struct Bad { lost_through: Lsn, } -/// `known` is a recorded loss and `found` a loss inside its LSN range. -fn covers(known: &Damage, found: &Damage) -> bool { - match (known, found) { - ( - Damage::Corrupt { lost_after: after, lost_through: through, .. }, - Damage::Corrupt { lost_after: found_after, lost_through: found_through, .. }, - ) => after <= found_after && found_through <= through, - _ => false, - } +/// `damage` without the losses ([`Damage::Corrupt`]) whose LSN range lies inside another's; of +/// two equal ranges the first stays. Other damage stays as it is. +fn distinct_losses(damage: Vec) -> Vec { + let ranges: Vec> = damage + .iter() + .map(|entry| match entry { + Damage::Corrupt { lost_after, lost_through, .. } => Some((*lost_after, *lost_through)), + _ => None, + }) + .collect(); + let inside_another = |index: usize| { + let Some(Some((after, through))) = ranges.get(index).copied() else { return false }; + ranges.iter().enumerate().any(|(other, range)| { + let Some((other_after, other_through)) = *range else { return false }; + other != index + && other_after <= after + && through <= other_through + && ((other_after, other_through) != (after, through) || other < index) + }) + }; + damage + .into_iter() + .enumerate() + .filter(|(index, _)| !inside_another(*index)) + .map(|(_, entry)| entry) + .collect() } fn io(what: &'static str) -> impl FnOnce(std::io::Error) -> OpenError { @@ -333,17 +350,18 @@ pub(crate) fn recover( }; let copy = last_read.as_ref().filter(|read| read.end > read.vouched); // The damage found here, durable before the new segment hides it from every later open - // (review G2), after the damage earlier opens recorded. A loss that lies inside one already - // recorded is the same loss, found again after a crash: it is not recorded twice (review N1). - report.damage.retain(|damage| !earlier.iter().any(|known| covers(known, damage))); - if report.damage.iter().any(|damage| matches!(damage, Damage::Corrupt { .. })) { - let mut all = earlier.clone(); - all.extend(report.damage.iter().cloned()); - write_damage(storage, dir, &all).map_err(io("record the damage"))?; + // (review G2), after the damage earlier opens recorded. A loss that lies inside another is + // the same loss found again (after a crash, or by two steps of this open): it is recorded and + // reported once, as the larger range (review N1). + let mut all = earlier.clone(); + all.append(&mut report.damage); + let all = distinct_losses(all); + let recorded: Vec = + all.iter().filter(|damage| matches!(damage, Damage::Corrupt { .. })).cloned().collect(); + if recorded != earlier { + write_damage(storage, dir, &recorded).map_err(io("record the damage"))?; } - let mut damage = earlier; - damage.append(&mut report.damage); - report.damage = damage; + report.damage = all; // The writer always starts in a segment this open makes: an earlier process's segment may // hold a page that its cache had and its disk never got (a failed sync), so nothing is // appended to it again. It is read only through the next segment's "previous last LSN". From 4da46785b3948e66055a04a75737b889e2cf2a10 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 4 Nov 2026 04:51:35 +0000 Subject: [PATCH 16/52] cmux-journal: fourth-review regressions G8, N3, N4 (red) Fourth review FIX at 215c8f0485ea (PR #33919 comment 6622740919). Tests only. - G8 a_recovery_copy_above_a_quarantine_never_reuses_a_quarantined_lsn (the reviewer's three steps): records 2..5 unsynced and a kill -9; open #1 copies them into C1 and goes on there with 6..10; C1's header is damaged, open #2 quarantines C1, copies 2..5 again into C2 and makes the empty E (first LSN 11); E's header is damaged. On this code open #3 sees C2 numbered above the quarantine, skips its scan and gives 6, an LSN the quarantine holds. - N3: G7 B and the F1 test assert that every file in quarantine/ is named by a reported loss. On this code the absorbed loss's names are dropped (G7 B). - N4: the F1 test asserts that the loss through 40 names the segment where the journal ends (00000000000000000002.seg). On this code a crash after the moves into quarantine/ and before DAMAGE turns it into the vaguer "unfinished quarantine" report. --- .../cmux-journal/src/cmux_journal_tests.rs | 80 +++++++++++++++++-- 1 file changed, 73 insertions(+), 7 deletions(-) diff --git a/cmux-tui/crates/cmux-journal/src/cmux_journal_tests.rs b/cmux-tui/crates/cmux-journal/src/cmux_journal_tests.rs index f9636163e15b..e9be79ac353d 100644 --- a/cmux-tui/crates/cmux-journal/src/cmux_journal_tests.rs +++ b/cmux-tui/crates/cmux-journal/src/cmux_journal_tests.rs @@ -468,11 +468,13 @@ fn a_crash_inside_the_quarantine_step_never_reuses_a_quarantined_lsn() { let (damaged, config) = damaged_in_the_middle(50); // Every open after the damage reports it (G2 of the re-review), also after a crash at the // open's last step, which no later open can see the damage from. + // At the segment where the journal ends (N4 of the fourth review: the record is made before + // the moves, so a crash after them never turns it into a vaguer one). let reports_the_loss = |report: &OpenReport| { - report - .damage - .iter() - .any(|damage| matches!(damage, Damage::Corrupt { lost_through: 40, .. })) + report.damage.iter().any(|damage| { + matches!(damage, Damage::Corrupt { lost_through: 40, segment, .. } + if segment.as_str() == "00000000000000000002.seg") + }) }; // And tells it once (N1 of the third review): no reported loss lies inside another one. let reports_it_once = |report: &OpenReport| { @@ -503,8 +505,8 @@ fn a_crash_inside_the_quarantine_step_never_reuses_a_quarantined_lsn() { for at in 1..=operations { let storage = damaged.snapshot(Crash::Process); storage.crash_after(at); - let (journal, report) = match open_at(&storage, config.clone(), 0) { - Ok((journal, report, _)) => (journal, report), + let (journal, report, opened) = match open_at(&storage, config.clone(), 0) { + Ok((journal, report, _)) => (journal, report, storage), Err(_) => { let restarted = storage.restart(crash); if !restarted.names_in(&dir().join("quarantine")).is_empty() { @@ -512,7 +514,7 @@ fn a_crash_inside_the_quarantine_step_never_reuses_a_quarantined_lsn() { } let (journal, report, _) = open_at(&restarted, config.clone(), 0) .unwrap_or_else(|e| panic!("{crash:?} at op {at}, then open: {e}")); - (journal, report) + (journal, report, restarted) } }; assert_eq!( @@ -522,6 +524,7 @@ fn a_crash_inside_the_quarantine_step_never_reuses_a_quarantined_lsn() { ); assert!(reports_the_loss(&report), "{crash:?} at op {at}: {report:?}"); assert!(reports_it_once(&report), "{crash:?} at op {at}: {report:?}"); + assert_names_every_quarantined_file(&opened, &report); } } assert!(crashed_after_a_move > 0, "no crash point fell after a move"); @@ -761,6 +764,69 @@ fn a_damaged_newest_segment_after_a_quarantine_reports_its_records() { .unwrap_or(0); assert!(lost_through >= 43, "the loss of 41..43 is not reported: {report:?}"); assert_eq!(journal.append(3, Class::Ack, b"after"), Ok(44), "{report:?}"); + // N3 of the fourth review: the absorbed loss's files stay named, for a salvage. + assert_names_every_quarantined_file(&storage, &report); +} + +/// Every segment file in `quarantine/` is named by a reported loss (a salvage finds its records). +fn assert_names_every_quarantined_file(storage: &SimStorage, report: &OpenReport) { + let named: std::collections::BTreeSet<&str> = report + .damage + .iter() + .flat_map(|damage| match damage { + Damage::Corrupt { moved, .. } => moved.iter().map(String::as_str).collect(), + _ => Vec::new(), + }) + .collect(); + for name in storage.names_in(&dir().join("quarantine")) { + if !name.starts_with("damage-report") { + assert!(named.contains(name.as_str()), "{name} is not named: {report:?}"); + } + } +} + +/// G8 of the fourth review: a recovery copy is numbered above the quarantine, but its records +/// start below the quarantined LSNs. Records 2..5 are written, not synced, and a kill -9 comes: +/// open #1 copies them into segment 2 (C1), and the writer goes on there with 6..10. C1's header +/// is damaged: open #2 quarantines C1, copies 2..5 again into segment 3 (C2) and makes the empty +/// segment 4 (E, first LSN 11). E's header is damaged too. Open #3 must still give 11. +#[test] +fn a_recovery_copy_above_a_quarantine_never_reuses_a_quarantined_lsn() { + let storage = SimStorage::new(58); + let config = Config { stream_sync_delay: Duration::from_secs(60), ..quick() }; + let (journal, _, _) = open_at(&storage, config.clone(), 0).unwrap_or_else(|e| panic!("{e}")); + assert_eq!(journal.append(1, Class::Ack, b"one"), Ok(1)); + journal.wait_durable(1, WAIT).unwrap_or_else(|e| panic!("{e}")); + assert!(eventually(|| journal.full_durable_lsn() >= 1), "{:?}", journal.stats()); + for lsn in 2..=5 { + assert_eq!(journal.append(1, Class::Stream, &payload(lsn)), Ok(lsn)); + } + assert!(eventually(|| journal.stats().written_lsn >= 5), "{:?}", journal.stats()); + assert_eq!(journal.durable_lsn(), 1, "the stream records must not be synced yet"); + let killed = storage.restart(Crash::Process); + drop(journal); + // Open #1: the copy C1 (segment 2), and 6..10 in it. + let (journal, report, seen) = + open_at(&killed, config.clone(), 0).unwrap_or_else(|e| panic!("{e}")); + assert_eq!((lsns(&seen), report.rewritten), ((1..=5).collect::>(), 4), "{report:?}"); + for lsn in 6..=10 { + assert_eq!(journal.append(1, Class::Ack, &payload(lsn)), Ok(lsn)); + } + assert_eq!(journal.seal(WAIT), Ok(10)); + drop(journal); + let killed = killed.restart(Crash::Process); + killed.corrupt(&dir().join(format::segment_name(2)), 20); + // Open #2: C1 to quarantine, C2 (segment 3) and the empty E (segment 4); no append. + let (journal, report, _) = + open_at(&killed, config.clone(), 0).unwrap_or_else(|e| panic!("{e}")); + assert_eq!(report.next_lsn, 11, "{report:?}"); + drop(journal); + let killed = killed.restart(Crash::Process); + killed.corrupt(&dir().join(format::segment_name(4)), 20); + // Open #3. + let (journal, report, _) = open_at(&killed, config, 0).unwrap_or_else(|e| panic!("{e}")); + assert_eq!(journal.append(1, Class::Ack, b"after"), Ok(11), "{report:?}"); + assert_names_every_quarantined_file(&killed, &report); } // ---------------------------------------------------------------- real file system From b589044b852914143563178340846c89c2f5d6a0 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 4 Nov 2026 05:01:14 +0000 Subject: [PATCH 17/52] cmux-journal: the quarantine's floor in its file names; DAMAGE before the moves (G8, N3, N4) Fourth review FIX at 215c8f0485ea (PR #33919 comment 6622740919); the regression tests are the commit before this one. - G8 (the G7 class): the skip of the quarantine scan trusted a live segment numbered above the quarantine to carry its floor, which a recovery copy does not (its records start at vouched + 1). Now each moved segment is named `..`, with the highest LSN counted when it moved (that file's and every one before it, scanned from the open's floor), and every open takes the quarantine's floor from the names alone: no quarantined file is read (G4 stays), and no header can hide the floor. `quarantined_floor` and the `checked_max_seq` skip are gone. The "unfinished quarantine" report is made when the names' floor is above everything else counted. - N4: `quarantine()` is split into `plan_quarantine` (measure and name every file, no change on disk) and `move_quarantined`; the loss, with the planned names, goes to DAMAGE before the moves. A crash after the moves keeps the precise report (the damaged segment and reason). - N3: when one loss absorbs another, the absorbed loss's files and reason go to the kept one, so DAMAGE names every quarantined file. - Docs: recovery step 5 and lib.rs name the quarantine file format. --- cmux-tui/crates/cmux-journal/src/lib.rs | 3 +- cmux-tui/crates/cmux-journal/src/recovery.rs | 134 +++++++++++------- .../crates/cmux-journal/src/recovery/scan.rs | 107 ++++++++------ 3 files changed, 147 insertions(+), 97 deletions(-) diff --git a/cmux-tui/crates/cmux-journal/src/lib.rs b/cmux-tui/crates/cmux-journal/src/lib.rs index bf4c54e9c1d8..50234b02b575 100644 --- a/cmux-tui/crates/cmux-journal/src/lib.rs +++ b/cmux-tui/crates/cmux-journal/src/lib.rs @@ -29,7 +29,8 @@ //! Files (the journal directory): `LOCK` (an exclusive lock held by the one writer), the segments //! (`.seg`: a header, then records and sync markers back to back, see //! `format.rs`), `quarantine/` for segments recovery could not read past damage that is not a -//! torn tail, and `DAMAGE`, the damage earlier opens found, reported at every open until +//! torn tail (named `..`: no LSN at or below `floor` is used again), and +//! `DAMAGE`, the damage earlier opens found, reported at every open until //! [`Journal::clear_damage`] moves it into `quarantine/`. A segment is only ever appended to. The //! one exception is the writer's own failed write: it cuts the segment back to its size before //! that write, so it removes only bytes that were never synced and never acknowledged. Nothing diff --git a/cmux-tui/crates/cmux-journal/src/recovery.rs b/cmux-tui/crates/cmux-journal/src/recovery.rs index d277c106770f..437024199f7e 100644 --- a/cmux-tui/crates/cmux-journal/src/recovery.rs +++ b/cmux-tui/crates/cmux-journal/src/recovery.rs @@ -16,10 +16,12 @@ //! check out: a sync marker among them means synced records are damaged (media damage, never a //! crash), and their LSNs are never used again. //! 5. Damage that is not a torn tail (a bad header, or records missing before the next segment) -//! keeps the valid prefix; the later segments move to `quarantine/` (never deleted). New -//! records take LSNs after every LSN that any segment, live or quarantined, holds, so no LSN -//! is ever used twice, also after a crash in the middle of an earlier recovery. The damage is -//! written to `DAMAGE` (whole, synced) before the new segment of step 6 hides it from later +//! keeps the valid prefix; the later segments move to `quarantine/` (never deleted), each +//! named `..` with the highest LSN counted when it moved. New records take +//! LSNs after every LSN that any segment, live or quarantined, holds (the quarantine's from +//! the names: no file is read and no header can hide them), so no LSN is ever used twice, also +//! after a crash in the middle of an earlier recovery. The damage is written to `DAMAGE` +//! (whole, synced) before the moves and before the new segment of step 6 hides it from later //! opens, and every open reports it again until `Journal::clear_damage`. //! 6. The journal goes on in a segment this open makes (the copy of step 4, or an empty one), never //! in a segment an earlier process wrote: that one may hold a page its cache had and its disk @@ -39,7 +41,10 @@ mod damage; mod scan; pub(crate) use damage::clear_damage; use damage::{read_damage, write_damage}; -use scan::{Reader, quarantine, quarantined_floor, quarantined_journal_id, scan_beyond_path}; +use scan::{ + Quarantined, Reader, move_quarantined, plan_quarantine, quarantined_journal_id, + scan_beyond_path, +}; pub(crate) const LOCK_NAME: &str = "LOCK"; pub(crate) const QUARANTINE_DIR: &str = "quarantine"; @@ -138,8 +143,10 @@ struct Bad { } /// `damage` without the losses ([`Damage::Corrupt`]) whose LSN range lies inside another's; of -/// two equal ranges the first stays. Other damage stays as it is. -fn distinct_losses(damage: Vec) -> Vec { +/// two equal ranges the first stays. A dropped loss's files (`moved`) and reason go to a kept +/// loss whose range holds it, so the record still names every quarantined file (review N3). +/// Other damage stays as it is. +fn distinct_losses(mut damage: Vec) -> Vec { let ranges: Vec> = damage .iter() .map(|entry| match entry { @@ -147,20 +154,50 @@ fn distinct_losses(damage: Vec) -> Vec { _ => None, }) .collect(); - let inside_another = |index: usize| { - let Some(Some((after, through))) = ranges.get(index).copied() else { return false }; - ranges.iter().enumerate().any(|(other, range)| { - let Some((other_after, other_through)) = *range else { return false }; - other != index - && other_after <= after - && through <= other_through - && ((other_after, other_through) != (after, through) || other < index) - }) + let holds = |outer: usize, inner: usize| match ( + ranges.get(outer).copied().flatten(), + ranges.get(inner).copied().flatten(), + ) { + (Some((outer_after, outer_through)), Some((after, through))) => { + outer != inner && outer_after <= after && through <= outer_through + } + _ => false, }; + let dropped: Vec = (0..damage.len()) + .map(|inner| { + (0..damage.len()).any(|outer| { + holds(outer, inner) && (ranges.get(outer) != ranges.get(inner) || outer < inner) + }) + }) + .collect(); + let is_dropped = |index: usize| dropped.get(index).copied().unwrap_or(false); + for inner in (0..damage.len()).filter(|index| is_dropped(*index)) { + let Some(outer) = + (0..damage.len()).find(|outer| !is_dropped(*outer) && holds(*outer, inner)) + else { + continue; + }; + let Some(Damage::Corrupt { moved, reason, .. }) = damage.get(inner).cloned() else { + continue; + }; + if let Some(Damage::Corrupt { moved: kept_moved, reason: kept_reason, .. }) = + damage.get_mut(outer) + { + for name in moved { + if !kept_moved.contains(&name) { + kept_moved.push(name); + } + } + if !kept_reason.contains(reason.as_str()) { + kept_reason.push_str("; "); + kept_reason.push_str(&reason); + } + } + } damage .into_iter() .enumerate() - .filter(|(index, _)| !inside_another(*index)) + .filter(|(index, _)| !is_dropped(*index)) .map(|(_, entry)| entry) .collect() } @@ -194,15 +231,13 @@ pub(crate) fn recover( let listed = list_segments(storage, dir)?; let quarantined = list_quarantined(storage, dir)?; let live_max_seq = listed.last().map_or(0, |(seq, _)| *seq); - let q_max_seq = quarantined.iter().map(|(seq, _)| *seq).max().unwrap_or(0); + let q_max_seq = quarantined.iter().map(|entry| entry.seq).max().unwrap_or(0); + // The quarantine's floor, from the names alone: no file is read, and no header can hide it + // (reviews G4, G7, G8). + let q_floor = quarantined.iter().map(|entry| entry.floor).max().unwrap_or(0); // 1 and 2. let Headers { mut segments, reuse, mut bad } = read_headers(storage, dir, &listed)?; - // The newest segment whose header checked out. Only such a header carries the LSN floor of - // the quarantine before it (a damaged one says nothing: review G7). - let checked_max_seq = segments.last().map_or(0, |segment| segment.seq); - // The quarantine is unfinished, or its floor is not known from a live header. - let quarantine_open = q_max_seq > checked_max_seq; if let Some(last) = segments.last() { storage .open(&dir.join(&last.name), OpenMode::Write) @@ -275,19 +310,13 @@ pub(crate) fn recover( } } - // 5: every LSN a quarantined segment holds counts, from this open's moves and earlier ones. - // Once a segment whose header checked out is numbered above every quarantined one, it was - // made after their quarantine with a first LSN above all of theirs, and `journal_end` is at - // least that: the quarantined files are not read again (review G4). The scan starts from the - // running floor, so a quarantined file with no readable header is scanned for LSNs above - // what is already counted (review G7). + // 5: every LSN a quarantined segment holds counts, from earlier opens (`q_floor`, their + // names) and from this one's moves (planned below, from this floor). let before_quarantine = floor; - let q_floor = - if quarantine_open { quarantined_floor(storage, dir, &quarantined, floor)? } else { floor }; floor = floor.max(q_floor); // Reported only when the quarantine holds LSNs that nothing else counted: an old quarantine // behind a segment torn by a crash at its creation holds none, and a crash is not damage. - if quarantine_open && bad.is_none() && q_floor > before_quarantine { + if bad.is_none() && q_floor > before_quarantine { // An earlier recovery moved segments and stopped before its next segment was made, or // that segment's header is damaged. report.damage.push(Damage::Corrupt { @@ -297,7 +326,7 @@ pub(crate) fn recover( earlier recovery stopped before it made the next segment, or its header is \ damaged" .to_owned(), - moved: quarantined.iter().map(|(_, name)| name.clone()).collect(), + moved: quarantined.iter().map(|entry| entry.name.clone()).collect(), lost_after: journal_end, lost_through: q_floor, }); @@ -320,15 +349,20 @@ pub(crate) fn recover( } reuse = None; } - if let Some(bad) = bad { - let moved = quarantine(storage, dir, &listed, bad.from_seq, floor)?; - let lost_through = floor.max(bad.lost_through).max(moved.highest); + let plan = match &bad { + Some(bad) => { + Some(plan_quarantine(storage, dir, &listed, &quarantined, bad.from_seq, floor)?) + } + None => None, + }; + if let (Some(bad), Some(plan)) = (bad, plan.as_ref()) { + let lost_through = floor.max(bad.lost_through).max(plan.highest); floor = lost_through; report.damage.push(Damage::Corrupt { segment: bad.segment, offset: bad.offset, reason: bad.reason.to_owned(), - moved: moved.names, + moved: plan.names.clone(), lost_after: journal_end, lost_through, }); @@ -349,10 +383,11 @@ pub(crate) fn recover( None => quarantined_journal_id(storage, dir, &quarantined)?.unwrap_or_else(new_journal_id), }; let copy = last_read.as_ref().filter(|read| read.end > read.vouched); - // The damage found here, durable before the new segment hides it from every later open - // (review G2), after the damage earlier opens recorded. A loss that lies inside another is - // the same loss found again (after a crash, or by two steps of this open): it is recorded and - // reported once, as the larger range (review N1). + // The damage found here, durable before the moves into `quarantine/` (review N4) and before + // the new segment hides it from every later open (review G2), after the damage earlier opens + // recorded. A loss that lies inside another is the same loss found again (after a crash, or + // by two steps of this open): it is recorded and reported once, as the larger range, with + // the files and reasons of both (reviews N1, N3). let mut all = earlier.clone(); all.append(&mut report.damage); let all = distinct_losses(all); @@ -362,6 +397,9 @@ pub(crate) fn recover( write_damage(storage, dir, &recorded).map_err(io("record the damage"))?; } report.damage = all; + if let Some(plan) = &plan { + move_quarantined(storage, dir, plan)?; + } // The writer always starts in a segment this open makes: an earlier process's segment may // hold a page that its cache had and its disk never got (a failed sync), so nothing is // appended to it again. It is read only through the next segment's "previous last LSN". @@ -427,8 +465,8 @@ fn list_segments(storage: &dyn Storage, dir: &Path) -> Result Ok(listed) } -/// The files in `quarantine/` (named `.`), by the segment number in their name. -fn list_quarantined(storage: &dyn Storage, dir: &Path) -> Result, OpenError> { +/// The segment files in `quarantine/` ([`Quarantined`]), by segment number and attempt. +fn list_quarantined(storage: &dyn Storage, dir: &Path) -> Result, OpenError> { let names = match storage.list(&dir.join(QUARANTINE_DIR)) { Ok(names) => names, Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()), @@ -436,14 +474,8 @@ fn list_quarantined(storage: &dyn Storage, dir: &Path) -> Result = names - .into_iter() - .filter_map(|name| { - let (segment, _) = name.rsplit_once('.')?; - Some((format::parse_segment_name(segment)?, name)) - }) - .collect(); - out.sort(); + let mut out: Vec = names.into_iter().filter_map(Quarantined::parse).collect(); + out.sort_by(|a, b| (a.seq, a.attempt).cmp(&(b.seq, b.attempt))); Ok(out) } diff --git a/cmux-tui/crates/cmux-journal/src/recovery/scan.rs b/cmux-tui/crates/cmux-journal/src/recovery/scan.rs index f28896ccec1e..77e4c167e063 100644 --- a/cmux-tui/crates/cmux-journal/src/recovery/scan.rs +++ b/cmux-tui/crates/cmux-journal/src/recovery/scan.rs @@ -185,29 +185,15 @@ pub(super) fn highest_lsn(storage: &dyn Storage, path: &Path, base: Lsn) -> Resu Ok(scan_beyond(&mut reader, offset, from_lsn)?.max_lsn.max(from_lsn)) } -/// The highest LSN of the quarantined segments that lies above `base`. -pub(super) fn quarantined_floor( - storage: &dyn Storage, - dir: &Path, - quarantined: &[(u64, String)], - base: Lsn, -) -> Result { - let mut floor = base; - for (_, name) in quarantined { - floor = floor.max(highest_lsn(storage, &dir.join(QUARANTINE_DIR).join(name), floor)?); - } - Ok(floor) -} - /// The journal id of a quarantined segment whose header reads, when no live one is left. pub(super) fn quarantined_journal_id( storage: &dyn Storage, dir: &Path, - quarantined: &[(u64, String)], + quarantined: &[Quarantined], ) -> Result, OpenError> { - for (_, name) in quarantined { + for entry in quarantined { let file = storage - .open(&dir.join(QUARANTINE_DIR).join(name), OpenMode::Read) + .open(&dir.join(QUARANTINE_DIR).join(&entry.name), OpenMode::Read) .map_err(io("open a quarantined segment"))?; let size = file.size().map_err(io("read a segment's size"))?; if let Ok(header) = read_header(file.as_ref(), size)? { @@ -217,45 +203,76 @@ pub(super) fn quarantined_journal_id( Ok(None) } -/// The segments moved to quarantine, and the highest LSN they hold. -pub(super) struct Moved { +/// A segment file in `quarantine/`, named `..`: no LSN at or below +/// `floor` is ever used again (it is the highest LSN this segment and every one moved with it +/// before it hold, or the floor of the open that moved them). The floor is in the name, so an +/// open counts it without reading the file, whatever happened to any header (review G8). +pub(super) struct Quarantined { + pub(super) seq: u64, + pub(super) attempt: u32, + pub(super) floor: Lsn, + pub(super) name: String, +} + +impl Quarantined { + /// The name's parts; `None` for a name this crate did not make. + pub(super) fn parse(name: String) -> Option { + let (digits, rest) = name.split_once(".seg.")?; + let seq = format::parse_segment_name(&format!("{digits}.seg"))?; + let (attempt, floor) = rest.split_once('.')?; + let (attempt, floor) = (attempt.parse().ok()?, floor.parse().ok()?); + Some(Self { seq, attempt, floor, name }) + } +} + +/// The segments to move into `quarantine/` (step 5), measured before anything moves: each one's +/// new name, and the highest LSN of them all. +pub(super) struct Plan { + moves: Vec<(String, String)>, pub(super) names: Vec, pub(super) highest: Lsn, } -/// Moves every listed segment from number `from_seq` on into `quarantine/`, under a name no -/// earlier recovery used. Their highest LSN is read first, then all are moved, then both folders -/// are synced. A crash in between is safe: the next open counts the LSNs of every quarantined -/// segment, and reports a quarantine that no new segment followed. -pub(super) fn quarantine( +/// Measures every listed segment from number `from_seq` on and names it for `quarantine/`. Each +/// file is scanned from the floor counted so far (`base`, then the files before it), so a file +/// with no readable header is read for the LSNs above what is counted, and its name carries the +/// floor after it. Nothing moves yet: the caller records the loss durably first (review N4). +pub(super) fn plan_quarantine( storage: &dyn Storage, dir: &Path, listed: &[(u64, String)], + quarantined: &[Quarantined], from_seq: u64, base: Lsn, -) -> Result { +) -> Result { + let mut used: std::collections::BTreeSet<(u64, u32)> = + quarantined.iter().map(|entry| (entry.seq, entry.attempt)).collect(); + let mut plan = Plan { moves: Vec::new(), names: Vec::new(), highest: base }; + for (seq, name) in listed.iter().filter(|(seq, _)| *seq >= from_seq) { + plan.highest = plan.highest.max(highest_lsn(storage, &dir.join(name), plan.highest)?); + let attempt = + (0u32..).find(|attempt| !used.contains(&(*seq, *attempt))).unwrap_or(u32::MAX); + used.insert((*seq, attempt)); + let new_name = format!("{name}.{attempt}.{}", plan.highest); + plan.moves.push((name.clone(), new_name.clone())); + plan.names.push(new_name); + } + Ok(plan) +} + +/// Moves the planned segments into `quarantine/`, then syncs both folders. A crash in between is +/// safe: each moved file's name carries its floor, and a segment not moved yet is found damaged +/// again and planned the same way. +pub(super) fn move_quarantined( + storage: &dyn Storage, + dir: &Path, + plan: &Plan, +) -> Result<(), OpenError> { let target = dir.join(QUARANTINE_DIR); storage.create_dir_all(&target).map_err(io("make the quarantine folder"))?; - let taken = storage.list(&target).map_err(io("list the quarantine folder"))?; - let moving: Vec<&String> = - listed.iter().filter(|(seq, _)| *seq >= from_seq).map(|(_, name)| name).collect(); - let mut moved = Moved { names: Vec::new(), highest: base }; - for name in &moving { - moved.highest = moved.highest.max(highest_lsn(storage, &dir.join(name), moved.highest)?); - } - for name in moving { - let mut attempt = 0u32; - let new_name = loop { - let candidate = format!("{name}.{attempt}"); - if !taken.contains(&candidate) { - break candidate; - } - attempt = attempt.saturating_add(1); - }; - storage.rename(&dir.join(name), &target.join(&new_name)).map_err(io("move a segment"))?; - moved.names.push(new_name); + for (from, to) in &plan.moves { + storage.rename(&dir.join(from), &target.join(to)).map_err(io("move a segment"))?; } storage.sync_dir(&target).map_err(io("sync the quarantine folder"))?; - storage.sync_dir(dir).map_err(io("sync the folder"))?; - Ok(moved) + storage.sync_dir(dir).map_err(io("sync the folder")) } From 9c47594e1c048dd623b70897b3272c7c3bbb5c26 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 4 Nov 2026 05:20:08 +0000 Subject: [PATCH 18/52] cmux-journal: sort the quarantine list with sort_by_key (clippy unnecessary_sort_by) Fleet d9f22bed at b589044b852: the 33 journal tests pass, and clippy -D warnings fails at recovery.rs:478 (`consider using sort_by_key`). --- cmux-tui/crates/cmux-journal/src/recovery.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmux-tui/crates/cmux-journal/src/recovery.rs b/cmux-tui/crates/cmux-journal/src/recovery.rs index 437024199f7e..81f659f3148c 100644 --- a/cmux-tui/crates/cmux-journal/src/recovery.rs +++ b/cmux-tui/crates/cmux-journal/src/recovery.rs @@ -475,7 +475,7 @@ fn list_quarantined(storage: &dyn Storage, dir: &Path) -> Result = names.into_iter().filter_map(Quarantined::parse).collect(); - out.sort_by(|a, b| (a.seq, a.attempt).cmp(&(b.seq, b.attempt))); + out.sort_by_key(|entry| (entry.seq, entry.attempt)); Ok(out) } From b4369cffe687c7289e876d3e735c65fa82b34278 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 4 Nov 2026 00:40:43 +0000 Subject: [PATCH 19/52] cmux-journal: Journal::on_written, a listener per written batch (for S2's projector) R-DURABILITY S2 (acpmux event logs through the journal) projects each chat's log line into its ndjson file once the journal has WRITTEN the batch that holds it, so readers see stream lines at once. The listener runs on the writer thread after the write and before the sync, once per batch that moved the written LSN: a batch written again into a new segment after a failed sync is not heard twice. When the writer stops, these listeners are dropped too. Test: heard while the batch's sync still runs, once. Port of 145faa6faa8 (branch rewrite/r-durability-s2-acpmux-logs) onto the S1 review fixes. --- .../cmux-journal/src/cmux_journal_tests.rs | 17 +++++++++++++ cmux-tui/crates/cmux-journal/src/journal.rs | 17 +++++++++++++ cmux-tui/crates/cmux-journal/src/writer.rs | 24 ++++++++++++++----- 3 files changed, 52 insertions(+), 6 deletions(-) diff --git a/cmux-tui/crates/cmux-journal/src/cmux_journal_tests.rs b/cmux-tui/crates/cmux-journal/src/cmux_journal_tests.rs index e9be79ac353d..5e376a33d64a 100644 --- a/cmux-tui/crates/cmux-journal/src/cmux_journal_tests.rs +++ b/cmux-tui/crates/cmux-journal/src/cmux_journal_tests.rs @@ -189,6 +189,23 @@ fn durable_listeners_see_each_advance_in_order() { assert!(heard.windows(2).all(|pair| pair.first() < pair.get(1)), "{heard:?}"); } +#[test] +fn written_listeners_hear_each_batch_before_its_sync() { + let storage = SimStorage::new(41); + let (journal, _, _) = open(&storage); + storage.set_sync_delay(Duration::from_millis(300)); + let heard = Arc::new(Mutex::new(Vec::new())); + let sink = heard.clone(); + journal.on_written(move |lsn| sink.lock().unwrap_or_else(|e| e.into_inner()).push(lsn)); + let lsn = journal.append(1, Class::Ack, b"x").unwrap_or_else(|e| panic!("{e}")); + let written = || heard.lock().unwrap_or_else(|e| e.into_inner()).clone(); + assert!(eventually(|| written().last() == Some(&lsn)), "{:?}", written()); + // Heard while its 300 ms sync still runs: the record is written, not durable. + assert_eq!(journal.durable_lsn(), 0); + journal.wait_durable(lsn, WAIT).unwrap_or_else(|e| panic!("{e}")); + assert_eq!(written(), vec![lsn], "one call per batch that moved the written LSN"); +} + #[test] fn a_stream_record_is_written_at_once_and_synced_after_its_delay() { let storage = SimStorage::new(6); diff --git a/cmux-tui/crates/cmux-journal/src/journal.rs b/cmux-tui/crates/cmux-journal/src/journal.rs index e578c10339a3..e34f171d1aab 100644 --- a/cmux-tui/crates/cmux-journal/src/journal.rs +++ b/cmux-tui/crates/cmux-journal/src/journal.rs @@ -62,6 +62,8 @@ pub(crate) struct Shared { /// Wakes waiters: the durable LSN moved, or the writer ended. pub(crate) durable: Condvar, pub(crate) listeners: Mutex>, + /// Called with the new written LSN after each batch's write, before its sync. + pub(crate) written_listeners: Mutex>, } impl Shared { @@ -215,6 +217,7 @@ impl Journal { wake: Condvar::new(), durable: Condvar::new(), listeners: Mutex::new(Vec::new()), + written_listeners: Mutex::new(Vec::new()), }); let report = recovered.report; let lock = recovered.lock; @@ -300,6 +303,20 @@ impl Journal { recovery::clear_damage(self.storage.as_ref(), &self.dir) } + /// Calls `listener` with the new written LSN after each batch is written, before it is synced, + /// on the writer thread. A written record survives a kill of any or all processes (it is in + /// the page cache), not an OS crash. A daemon's projector writes its compaction output from + /// here, so it never holds a record the journal never held. It must return quickly and never + /// wait. A batch written again into a new segment (after a failed sync) is not heard twice. + /// When the writer stops, the listeners are dropped. + pub fn on_written(&self, listener: impl Fn(Lsn) + Send + Sync + 'static) { + self.shared + .written_listeners + .lock() + .unwrap_or_else(PoisonError::into_inner) + .push(Arc::new(listener)); + } + /// Blocks until record `lsn` is durable. For tests and tools: a daemon's request path /// never blocks on the disk; it holds its reply until a listener says the LSN is durable. pub fn wait_durable(&self, lsn: Lsn, timeout: Duration) -> Result<(), WaitError> { diff --git a/cmux-tui/crates/cmux-journal/src/writer.rs b/cmux-tui/crates/cmux-journal/src/writer.rs index bcff22d771be..ab4bca3d626b 100644 --- a/cmux-tui/crates/cmux-journal/src/writer.rs +++ b/cmux-tui/crates/cmux-journal/src/writer.rs @@ -81,6 +81,7 @@ impl Drop for Ended { fn drop(&mut self) { let panicked = std::thread::panicking(); self.0.listeners.lock().unwrap_or_else(PoisonError::into_inner).clear(); + self.0.written_listeners.lock().unwrap_or_else(PoisonError::into_inner).clear(); { let mut state = self.0.lock(); state.ended = true; @@ -302,7 +303,14 @@ impl Writer { self.unsynced_since = Some(now); } let last = self.retained_last; - self.publish(|stats| stats.written_lsn = last); + let advanced = self.publish(|stats| { + let advanced = last > stats.written_lsn; + stats.written_lsn = stats.written_lsn.max(last); + advanced + }); + if advanced { + self.call(&self.shared.written_listeners, last); + } } let segment_bytes = self.shared.config.segment_bytes; let full_segment = self.active.as_ref().is_some_and(|active| active.size >= segment_bytes); @@ -373,15 +381,19 @@ impl Writer { }); self.shared.durable.notify_all(); if advanced { - let listeners = - self.shared.listeners.lock().unwrap_or_else(PoisonError::into_inner).clone(); - for listener in listeners { - listener(durable); - } + self.call(&self.shared.listeners, durable); } Ok(()) } + /// Calls each listener in `list` with `lsn`, with no lock held. + fn call(&self, list: &std::sync::Mutex>, lsn: Lsn) { + let listeners = list.lock().unwrap_or_else(PoisonError::into_inner).clone(); + for listener in listeners { + listener(lsn); + } + } + /// Appends the marker `active` owes ([module docs](self)). It is not synced here: the next /// sync covers it, and recovery syncs the last segment before it reads it. fn write_marker(&mut self) -> Result<(), Failure> { From 0c10b5e755275662e388c5f13c69673d979f7ae9 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 4 Nov 2026 01:06:59 +0000 Subject: [PATCH 20/52] cmux-journal: Journal::retire_through, the one delete (segments a compaction point covers) R-DURABILITY S2: a daemon's projector compacts the journal into its own files (acpmux: the chat ndjson logs, then a synced checkpoint). Without a delete the segment count only grows (review finding F6). `retire_through(compacted)` deletes the segments that hold no record after the caller's compaction point: a segment goes when the next segment's "previous last LSN" is at or before it, so the newest segment never goes. The point is capped at the durable LSN. The deletes and a folder sync run on the caller's thread (the projector, never a request path). A delete that fails leaves its segment live for a later call. Why a crash during a retirement is safe: the caller reopens from a point at or after `compacted`, and recovery reads nothing from a segment the point covers (it skips it on its next segment's header), so any subset of the deletes may reach the disk. A point older than the retired segments is reported as Damage::Missing. Quarantined segments are never deleted. - Storage::remove_file (FsStorage: std::fs::remove_file; SimStorage: one more change in the ordered log of names, durable at the folder sync). - The live segment list: recovery's, then each segment the writer makes. - JournalStats::segments_retired. - Tests: only covered segments go, and a second call deletes none; an OS crash after it keeps every record after the point and an older point is reported; a crash at every operation of a retirement (process, OS crash, power loss) keeps every record after the point and never reuses an LSN; the newest segment stays; a point in the middle of what this writer wrote. This adds a delete of journal data, so its PR is risk:high. --- .../cmux-journal/src/cmux_journal_tests.rs | 109 ++++++++++++++++++ cmux-tui/crates/cmux-journal/src/journal.rs | 62 +++++++++- cmux-tui/crates/cmux-journal/src/lib.rs | 5 +- cmux-tui/crates/cmux-journal/src/recovery.rs | 18 ++- cmux-tui/crates/cmux-journal/src/sim.rs | 14 +++ cmux-tui/crates/cmux-journal/src/storage.rs | 8 +- cmux-tui/crates/cmux-journal/src/writer.rs | 7 +- 7 files changed, 218 insertions(+), 5 deletions(-) diff --git a/cmux-tui/crates/cmux-journal/src/cmux_journal_tests.rs b/cmux-tui/crates/cmux-journal/src/cmux_journal_tests.rs index 5e376a33d64a..cb424a26fe06 100644 --- a/cmux-tui/crates/cmux-journal/src/cmux_journal_tests.rs +++ b/cmux-tui/crates/cmux-journal/src/cmux_journal_tests.rs @@ -846,6 +846,115 @@ fn a_recovery_copy_above_a_quarantine_never_reuses_a_quarantined_lsn() { assert_names_every_quarantined_file(&killed, &report); } +// ---------------------------------------------------------------- retirement (S2) + +/// 60 acked records over many 200-byte segments, sealed; and the config. +fn many_segments(seed: u64) -> (SimStorage, Config) { + let storage = SimStorage::new(seed); + let config = Config { segment_bytes: 200, ..quick() }; + let (journal, _, _) = open_at(&storage, config.clone(), 0).unwrap_or_else(|e| panic!("{e}")); + for lsn in 1..=60 { + let got = journal.append(3, Class::Ack, &payload(lsn)).unwrap_or_else(|e| panic!("{e}")); + journal.wait_durable(got, WAIT).unwrap_or_else(|e| panic!("{e}")); + } + assert_eq!(journal.seal(WAIT), Ok(60)); + drop(journal); + (storage.restart(Crash::Process), config) +} + +fn segment_count(storage: &SimStorage) -> usize { + storage.names_in(&dir()).iter().filter(|name| name.ends_with(".seg")).count() +} + +#[test] +fn retirement_deletes_only_the_segments_a_compaction_point_covers() { + let (storage, config) = many_segments(60); + let (journal, _, _) = open_at(&storage, config.clone(), 40).unwrap_or_else(|e| panic!("{e}")); + let before = segment_count(&storage); + let retired = journal.retire_through(40).unwrap_or_else(|e| panic!("{e}")); + assert!(retired >= 5, "{retired} of {before} segments retired"); + assert_eq!(segment_count(&storage), before - retired); + assert_eq!(journal.stats().segments_retired, u64::try_from(retired).unwrap_or(0)); + assert_eq!(journal.retire_through(40).unwrap_or_else(|e| panic!("{e}")), 0, "covered once"); + drop(journal); + // The deletes were made durable: an OS crash keeps them, and every record after 40 is kept. + let restarted = storage.restart(Crash::Os); + assert_eq!(segment_count(&restarted), before - retired); + let (journal, report, seen) = + open_at(&restarted, config.clone(), 40).unwrap_or_else(|e| panic!("{e}")); + assert_eq!(segment_count(&restarted), before - retired + 1, "the open made one segment"); + assert_eq!(lsns(&seen), (41..=60).collect::>(), "{report:?}"); + assert!(report.damage.is_empty(), "{report:?}"); + assert_eq!(journal.append(3, Class::Ack, b"after"), Ok(61)); + drop(journal); + // A compaction point older than the retired segments is told, not hidden. + let (_, report, _) = + open_at(&restarted.restart(Crash::Process), config, 0).unwrap_or_else(|e| panic!("{e}")); + assert!(matches!(report.damage.first(), Some(Damage::Missing { after: 0, .. })), "{report:?}"); +} + +#[test] +fn a_crash_at_any_point_of_a_retirement_keeps_every_record_after_it() { + let (base, config) = many_segments(61); + // Every storage operation of one retirement is a crash point once. + let probe = base.snapshot(Crash::Process); + let (journal, _, _) = open_at(&probe, config.clone(), 40).unwrap_or_else(|e| panic!("{e}")); + let before = probe.ops(); + assert!(journal.retire_through(40).unwrap_or_else(|e| panic!("{e}")) >= 5); + let operations = probe.ops() - before; + drop(journal); + for crash in [Crash::Process, Crash::Os, Crash::PowerLoss] { + for at in 1..=operations { + let storage = base.snapshot(Crash::Process); + let (journal, _, _) = + open_at(&storage, config.clone(), 40).unwrap_or_else(|e| panic!("{e}")); + storage.crash_after(at); + assert!(journal.retire_through(40).is_err(), "{crash:?} at op {at}: no crash"); + drop(journal); + let restarted = storage.restart(crash); + let (journal, report, seen) = open_at(&restarted, config.clone(), 40) + .unwrap_or_else(|e| panic!("{crash:?} at op {at}: {e}")); + assert_eq!(lsns(&seen), (41..=60).collect::>(), "{crash:?} at {at}: {report:?}"); + assert!(report.damage.is_empty(), "{crash:?} at op {at}: {report:?}"); + assert_eq!(journal.append(3, Class::Ack, b"after"), Ok(61), "{crash:?} at op {at}"); + } + } +} + +#[test] +fn retirement_never_deletes_the_segment_the_writer_appends_to() { + let (storage, config) = many_segments(62); + let (journal, _, _) = open_at(&storage, config.clone(), 60).unwrap_or_else(|e| panic!("{e}")); + // Past the durable LSN: capped at it. + journal.retire_through(Lsn::MAX).unwrap_or_else(|e| panic!("{e}")); + assert_eq!(segment_count(&storage), 1, "only the newest segment is left"); + assert_eq!(journal.append(3, Class::Ack, b"after"), Ok(61)); + journal.wait_durable(61, WAIT).unwrap_or_else(|e| panic!("{e}")); + drop(journal); + let (_, report, seen) = + open_at(&storage.restart(Crash::Os), config, 60).unwrap_or_else(|e| panic!("{e}")); + assert_eq!(lsns(&seen), vec![61], "{report:?}"); + assert!(report.damage.is_empty(), "{report:?}"); +} + +#[test] +fn retirement_keeps_segments_the_writer_made_after_the_point() { + let storage = SimStorage::new(63); + let config = Config { segment_bytes: 200, ..quick() }; + let (journal, _, _) = open_at(&storage, config.clone(), 0).unwrap_or_else(|e| panic!("{e}")); + for lsn in 1..=30 { + let got = journal.append(3, Class::Ack, &payload(lsn)).unwrap_or_else(|e| panic!("{e}")); + journal.wait_durable(got, WAIT).unwrap_or_else(|e| panic!("{e}")); + } + // The point is in the middle of what this writer wrote. + journal.retire_through(12).unwrap_or_else(|e| panic!("{e}")); + drop(journal); + let (_, report, seen) = + open_at(&storage.restart(Crash::Os), config, 12).unwrap_or_else(|e| panic!("{e}")); + assert_eq!(lsns(&seen), (13..=30).collect::>(), "{report:?}"); + assert!(report.damage.is_empty(), "{report:?}"); +} + // ---------------------------------------------------------------- real file system #[test] diff --git a/cmux-tui/crates/cmux-journal/src/journal.rs b/cmux-tui/crates/cmux-journal/src/journal.rs index e34f171d1aab..50e9ab9ff321 100644 --- a/cmux-tui/crates/cmux-journal/src/journal.rs +++ b/cmux-tui/crates/cmux-journal/src/journal.rs @@ -5,7 +5,7 @@ use std::sync::{Arc, Condvar, Mutex, MutexGuard, PoisonError}; use std::time::{Duration, Instant}; use crate::format::{self, MAX_PAYLOAD_LEN}; -use crate::recovery::{self, OpenReport}; +use crate::recovery::{self, LiveSegment, OpenReport}; use crate::storage::{Storage, StorageLock}; use crate::writer::Writer; use crate::{Class, Config, Lsn, Record}; @@ -33,6 +33,8 @@ pub struct JournalStats { /// Segments the writer made since open (rotation, or after a failed sync); a segment that /// recovery made is not counted. pub segments_created: u64, + /// Segments `Journal::retire_through` deleted since open. + pub segments_retired: u64, /// Failed writes, syncs and segment creations. After each, the writer tries again after a /// backoff: a failed write in the same segment, anything else in a new segment. pub failures: u64, @@ -64,6 +66,8 @@ pub(crate) struct Shared { pub(crate) listeners: Mutex>, /// Called with the new written LSN after each batch's write, before its sync. pub(crate) written_listeners: Mutex>, + /// The live segments, oldest first: recovery's, then each one the writer makes. + pub(crate) segments: Mutex>, } impl Shared { @@ -218,6 +222,7 @@ impl Journal { durable: Condvar::new(), listeners: Mutex::new(Vec::new()), written_listeners: Mutex::new(Vec::new()), + segments: Mutex::new(recovered.live), }); let report = recovered.report; let lock = recovered.lock; @@ -317,6 +322,61 @@ impl Journal { .push(Arc::new(listener)); } + /// Deletes the segments that hold no record after `compacted`, the caller's compaction point: + /// every record at or before it is in the caller's own durable output (its projection, + /// synced, and the checkpoint that names `compacted`, synced too). A segment is deleted when + /// the next segment's "previous last LSN" is at or before `compacted` (recovery reads nothing + /// past that from it), so the newest segment is never deleted. `compacted` is capped at the + /// durable LSN. Returns how many segments it deleted. + /// + /// The only delete in this crate. The caller must open the journal later with a compaction + /// point at or after `compacted`: recovery skips a segment the point covers, so a crash in + /// the middle of a retirement, or a delete that a crash takes back, is harmless; an older + /// compaction point is reported as `Damage::Missing`. It deletes and syncs the folder on the + /// caller's thread: call it from a background thread, never on a request path. A segment + /// whose delete fails stays live, and a later call deletes it. + pub fn retire_through(&self, compacted: Lsn) -> std::io::Result { + let compacted = compacted.min(self.durable_lsn()); + let retiring: Vec = { + let mut live = self.shared.segments.lock().unwrap_or_else(PoisonError::into_inner); + let count = live + .windows(2) + .take_while(|pair| pair.get(1).is_some_and(|next| next.prev_last <= compacted)) + .count(); + live.drain(..count).collect() + }; + if retiring.is_empty() { + return Ok(0); + } + for (index, segment) in retiring.iter().enumerate() { + let path = self.dir.join(format::segment_name(segment.seq)); + match self.storage.remove_file(&path) { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => { + // This one and the rest stay live, ahead of the segments made since. + let mut live = + self.shared.segments.lock().unwrap_or_else(PoisonError::into_inner); + let mut restored = retiring.get(index..).unwrap_or_default().to_vec(); + restored.append(&mut live); + *live = restored; + drop(live); + self.count_retired(index); + return Err(error); + } + } + } + self.count_retired(retiring.len()); + self.storage.sync_dir(&self.dir)?; + Ok(retiring.len()) + } + + fn count_retired(&self, count: usize) { + let mut state = self.shared.lock(); + let retired = u64::try_from(count).unwrap_or(u64::MAX); + state.stats.segments_retired = state.stats.segments_retired.saturating_add(retired); + } + /// Blocks until record `lsn` is durable. For tests and tools: a daemon's request path /// never blocks on the disk; it holds its reply until a listener says the LSN is durable. pub fn wait_durable(&self, lsn: Lsn, timeout: Duration) -> Result<(), WaitError> { diff --git a/cmux-tui/crates/cmux-journal/src/lib.rs b/cmux-tui/crates/cmux-journal/src/lib.rs index 50234b02b575..bc1daa1904d9 100644 --- a/cmux-tui/crates/cmux-journal/src/lib.rs +++ b/cmux-tui/crates/cmux-journal/src/lib.rs @@ -34,7 +34,10 @@ //! [`Journal::clear_damage`] moves it into `quarantine/`. A segment is only ever appended to. The //! one exception is the writer's own failed write: it cuts the segment back to its size before //! that write, so it removes only bytes that were never synced and never acknowledged. Nothing -//! here deletes a segment or shortens one past a synced record. +//! here deletes a segment or shortens one past a synced record, except one: +//! [`Journal::retire_through`] removes the segments that hold no record after the caller's +//! compaction point, and the caller opens the journal from that point later (quarantined segments +//! are never deleted). //! //! Recovery ([`Journal::open`]) is the one start path for a clean exit, a crash, a kill -9 and an //! OS crash: it syncs the last segment, reads the segments in LSN order, checks each record's diff --git a/cmux-tui/crates/cmux-journal/src/recovery.rs b/cmux-tui/crates/cmux-journal/src/recovery.rs index 81f659f3148c..98ff9bc15cd6 100644 --- a/cmux-tui/crates/cmux-journal/src/recovery.rs +++ b/cmux-tui/crates/cmux-journal/src/recovery.rs @@ -121,6 +121,16 @@ pub(crate) struct Recovered { pub(crate) next_lsn: Lsn, pub(crate) report: OpenReport, pub(crate) writer: WriterStart, + /// The live segments after recovery, oldest first (`Journal::retire_through`). + pub(crate) live: Vec, +} + +/// A live segment: its number and its header's "previous last LSN". Every record recovery reads +/// from a segment lies at or before the next segment's `prev_last`. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) struct LiveSegment { + pub(crate) seq: u64, + pub(crate) prev_last: Lsn, } /// A live segment whose header checked out. Its file is opened again to read its records. @@ -404,6 +414,10 @@ pub(crate) fn recover( // hold a page that its cache had and its disk never got (a failed sync), so nothing is // appended to it again. It is read only through the next segment's "previous last LSN". let kept = segments.len(); + let mut live: Vec = segments + .iter() + .map(|segment| LiveSegment { seq: segment.seq, prev_last: segment.header.prev_last }) + .collect(); let mut made = 0usize; let mut next_seq = || { let seq = reuse.take().unwrap_or_else(|| max_seq.saturating_add(1)); @@ -430,6 +444,7 @@ pub(crate) fn recover( report.rewritten = read.end.saturating_sub(read.vouched); made = made.saturating_add(1); through = read.end; + live.push(LiveSegment { seq: header.seq, prev_last: header.prev_last }); tail = Some(Tail { file, seq: header.seq, size }); } let tail = match tail { @@ -444,13 +459,14 @@ pub(crate) fn recover( let file = stage_segment(storage, dir, &header, &[], None).map_err(io("make a segment"))?; made = made.saturating_add(1); + live.push(LiveSegment { seq: header.seq, prev_last: header.prev_last }); Tail { file, seq: header.seq, size: len64(SEGMENT_HEADER_LEN) } } }; report.segments = kept.saturating_add(made); report.last_lsn = next_lsn.saturating_sub(1); report.next_lsn = next_lsn; - Ok(Recovered { lock, next_lsn, report, writer: WriterStart { journal_id, tail } }) + Ok(Recovered { lock, next_lsn, report, writer: WriterStart { journal_id, tail }, live }) } /// The live segment files, by number. diff --git a/cmux-tui/crates/cmux-journal/src/sim.rs b/cmux-tui/crates/cmux-journal/src/sim.rs index c4eac20bf19f..6d09b396983c 100644 --- a/cmux-tui/crates/cmux-journal/src/sim.rs +++ b/cmux-tui/crates/cmux-journal/src/sim.rs @@ -485,6 +485,20 @@ impl Storage for SimStorage { fs.change(step, Change::Rename(from.to_path_buf(), to.to_path_buf())) } + fn remove_file(&self, path: &Path) -> io::Result<()> { + let mut fs = lock(&self.fs); + let step = fs.begin()?; + if !fs.machine.names.files.contains_key(path) { + return match step { + Step::Crash => Err(crashed()), + Step::Run | Step::Fail(_) => Err(io::ErrorKind::NotFound.into()), + }; + } + // The file's bytes stay while a name or an open handle may still need them; a crash + // keeps only the files its surviving names point to. + fs.change(step, Change::Remove(path.to_path_buf())) + } + fn sync_dir(&self, dir: &Path) -> io::Result<()> { let mut fs = lock(&self.fs); let done = match fs.begin()? { diff --git a/cmux-tui/crates/cmux-journal/src/storage.rs b/cmux-tui/crates/cmux-journal/src/storage.rs index 1704406fd62b..df3fd381ec1b 100644 --- a/cmux-tui/crates/cmux-journal/src/storage.rs +++ b/cmux-tui/crates/cmux-journal/src/storage.rs @@ -37,7 +37,9 @@ pub trait Storage: Send + Sync { fn list(&self, dir: &Path) -> io::Result>; fn open(&self, path: &Path, mode: OpenMode) -> io::Result>; fn rename(&self, from: &Path, to: &Path) -> io::Result<()>; - /// Makes the entries of `dir` (a create or a rename in it) durable. + /// Removes the file at `path` (`Journal::retire_through`, the journal's only delete). + fn remove_file(&self, path: &Path) -> io::Result<()>; + /// Makes the entries of `dir` (a create, rename or remove in it) durable. fn sync_dir(&self, dir: &Path) -> io::Result<()>; /// An exclusive lock of `path`, held until the returned value is dropped, or `None` while /// another holder has it. @@ -104,6 +106,10 @@ impl Storage for FsStorage { std::fs::rename(from, to) } + fn remove_file(&self, path: &Path) -> io::Result<()> { + std::fs::remove_file(path) + } + fn sync_dir(&self, dir: &Path) -> io::Result<()> { sync_directory(dir) } diff --git a/cmux-tui/crates/cmux-journal/src/writer.rs b/cmux-tui/crates/cmux-journal/src/writer.rs index ab4bca3d626b..7aa44d7003e5 100644 --- a/cmux-tui/crates/cmux-journal/src/writer.rs +++ b/cmux-tui/crates/cmux-journal/src/writer.rs @@ -46,7 +46,7 @@ use std::time::{Duration, Instant}; use crate::Lsn; use crate::format::{self, RECORD_HEADER_LEN, SEGMENT_HEADER_LEN, SegmentHeader}; use crate::journal::{Shared, State}; -use crate::recovery::{WriterStart, len64, make_segment}; +use crate::recovery::{LiveSegment, WriterStart, len64, make_segment}; use crate::storage::{Storage, StorageFile, SyncLevel}; /// Written and unsynced bytes past which a batch is synced at once, acked or not. @@ -428,6 +428,11 @@ impl Writer { prev_last: self.durable, }; let file = make_segment(self.storage.as_ref(), &self.dir, &header)?; + self.shared + .segments + .lock() + .unwrap_or_else(PoisonError::into_inner) + .push(LiveSegment { seq: header.seq, prev_last: header.prev_last }); self.active = Some(Active { file, size: len64(SEGMENT_HEADER_LEN) }); self.next_seq = header.seq.saturating_add(1); self.written = 0; From 26d685bfe1fde05bd87a2a830f64fe7b724ec060 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 4 Nov 2026 03:19:07 +0000 Subject: [PATCH 21/52] acpmux: move MemoryStore and the store settings into their own files store.rs (2139 lines, 120 fns) and config.rs (1909 lines, 97 fns) are over the god-file limit, and the journal store (R-DURABILITY S2) must not grow them. MemoryStore moves to store/memory.rs and StoreMode, StoreConfig and its defaults to config/store_config.rs; both stay exported where they were (crate::store::MemoryStore, crate::config::{StoreConfig, StoreMode}). No behavior change: the code moves as it is. --- cmux-tui/crates/acpmux/src/config.rs | 50 +---------- .../crates/acpmux/src/config/store_config.rs | 52 +++++++++++ cmux-tui/crates/acpmux/src/store.rs | 89 +------------------ cmux-tui/crates/acpmux/src/store/memory.rs | 89 +++++++++++++++++++ 4 files changed, 145 insertions(+), 135 deletions(-) create mode 100644 cmux-tui/crates/acpmux/src/config/store_config.rs create mode 100644 cmux-tui/crates/acpmux/src/store/memory.rs diff --git a/cmux-tui/crates/acpmux/src/config.rs b/cmux-tui/crates/acpmux/src/config.rs index 7f424fd00d2f..3af2e6cd25ef 100644 --- a/cmux-tui/crates/acpmux/src/config.rs +++ b/cmux-tui/crates/acpmux/src/config.rs @@ -824,54 +824,6 @@ pub fn turn_cut(name: &str, profile: &HarnessProfile) -> Option { } } -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(rename_all = "lowercase")] -#[derive(Default)] -pub enum StoreMode { - Memory, - #[default] - Local, -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -#[serde(rename_all = "camelCase")] -pub struct StoreConfig { - #[serde(default)] - pub mode: StoreMode, - /// Segment roll size in bytes for the event log. - #[serde(default = "default_segment_bytes")] - pub segment_bytes: u64, - /// Days a closed chat's event log is kept whole (`hub/retention.rs`): after them, the - /// harness's raw lines and long tool output are removed from it, the chat's messages kept. - /// Default 14; 0 keeps every log whole. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub closed_log_days: Option, -} - -impl StoreConfig { - /// How long a closed chat's log is kept whole; None: always. - pub fn closed_log_retention(&self) -> Option { - match self.closed_log_days.unwrap_or(14) { - 0 => None, - days => Some(std::time::Duration::from_secs(u64::from(days) * 86_400)), - } - } -} - -fn default_segment_bytes() -> u64 { - 8 * 1024 * 1024 -} - -impl Default for StoreConfig { - fn default() -> Self { - Self { - mode: StoreMode::Local, - segment_bytes: default_segment_bytes(), - closed_log_days: None, - } - } -} - /// Permission policy applied when no attached client answers. #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, Default)] #[serde(rename_all = "kebab-case")] @@ -1879,6 +1831,8 @@ pub use discover::{ }; mod peer; pub use peer::PeerConfig; +mod store_config; +pub use store_config::{StoreConfig, StoreMode}; mod pool; pub use pool::PoolConfig; mod btw; diff --git a/cmux-tui/crates/acpmux/src/config/store_config.rs b/cmux-tui/crates/acpmux/src/config/store_config.rs new file mode 100644 index 000000000000..b707ed4522ec --- /dev/null +++ b/cmux-tui/crates/acpmux/src/config/store_config.rs @@ -0,0 +1,52 @@ +//! The event store's settings (`store` in the config file): where sessions and their logs live, +//! how a log is cut into segments, and how long a closed chat's log is kept whole. + +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "lowercase")] +#[derive(Default)] +pub enum StoreMode { + Memory, + #[default] + Local, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct StoreConfig { + #[serde(default)] + pub mode: StoreMode, + /// Segment roll size in bytes for the event log. + #[serde(default = "default_segment_bytes")] + pub segment_bytes: u64, + /// Days a closed chat's event log is kept whole (`hub/retention.rs`): after them, the + /// harness's raw lines and long tool output are removed from it, the chat's messages kept. + /// Default 14; 0 keeps every log whole. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub closed_log_days: Option, +} + +impl StoreConfig { + /// How long a closed chat's log is kept whole; None: always. + pub fn closed_log_retention(&self) -> Option { + match self.closed_log_days.unwrap_or(14) { + 0 => None, + days => Some(std::time::Duration::from_secs(u64::from(days) * 86_400)), + } + } +} + +fn default_segment_bytes() -> u64 { + 8 * 1024 * 1024 +} + +impl Default for StoreConfig { + fn default() -> Self { + Self { + mode: StoreMode::Local, + segment_bytes: default_segment_bytes(), + closed_log_days: None, + } + } +} diff --git a/cmux-tui/crates/acpmux/src/store.rs b/cmux-tui/crates/acpmux/src/store.rs index dfb87198dbbd..71859028c087 100644 --- a/cmux-tui/crates/acpmux/src/store.rs +++ b/cmux-tui/crates/acpmux/src/store.rs @@ -981,93 +981,8 @@ pub fn open(config: &StoreConfig, root: &Path) -> Result> { }) } -// ---------------------------------------------------------------- memory - -#[derive(Default)] -pub struct MemoryStore { - inner: Mutex)>>, -} - -impl Store for MemoryStore { - fn list(&self) -> Result> { - Ok(self.inner.lock_or_recover().values().map(|(m, _)| m.clone()).collect()) - } - fn load(&self, id: &str) -> Result> { - Ok(self.inner.lock_or_recover().get(id).map(|(m, _)| m.clone())) - } - fn save(&self, meta: &SessionMeta) -> Result<()> { - let mut g = self.inner.lock_or_recover(); - g.entry(meta.id.clone()) - .and_modify(|(m, _)| *m = meta.clone()) - .or_insert_with(|| (meta.clone(), Vec::new())); - Ok(()) - } - fn append(&self, id: &str, record: &EventRecord) -> Result<()> { - let mut g = self.inner.lock_or_recover(); - if let Some((_, events)) = g.get_mut(id) { - events.push(record.clone()); - // Bound memory: keep the newest 20k events. - if events.len() > 20_000 { - let drop = events.len() - 20_000; - events.drain(..drop); - } - } - Ok(()) - } - fn events(&self, id: &str, after: u64, limit: usize) -> Result> { - let g = self.inner.lock_or_recover(); - Ok(g.get(id) - .map(|(_, ev)| ev.iter().filter(|e| e.seq > after).take(limit).cloned().collect()) - .unwrap_or_default()) - } - /// Newest first, a bounded window at a time: a visitor that stops early clones only the - /// windows it read, and it runs off the lock. - fn scan_back( - &self, - id: &str, - after: u64, - before: u64, - skip: &dyn Fn(&[u8], &[u8]) -> bool, - visit: &mut dyn FnMut(EventRecord) -> bool, - ) -> Result<()> { - const WINDOW: usize = 256; - let mut before = before; - loop { - let (window, lowest) = { - let g = self.inner.lock_or_recover(); - let Some((_, events)) = g.get(id) else { return Ok(()) }; - let end = events.partition_point(|e| e.seq < before); - let mut window = Vec::new(); - let mut lowest = None; - for e in events - .get(..end) - .unwrap_or_default() - .iter() - .rev() - .take_while(|e| e.seq > after) - .take(WINDOW) - { - lowest = Some(e.seq); - if !skip(e.dir.as_bytes(), e.kind.as_bytes()) { - window.push(e.clone()); - } - } - (window, lowest) - }; - let Some(lowest) = lowest else { return Ok(()) }; - for record in window { - if !visit(record) { - return Ok(()); - } - } - before = lowest; - } - } - fn delete(&self, id: &str) -> Result<()> { - self.inner.lock_or_recover().remove(id); - Ok(()) - } -} +mod memory; +pub use memory::MemoryStore; // ----------------------------------------------------------------- local diff --git a/cmux-tui/crates/acpmux/src/store/memory.rs b/cmux-tui/crates/acpmux/src/store/memory.rs new file mode 100644 index 000000000000..42b5a7e571ae --- /dev/null +++ b/cmux-tui/crates/acpmux/src/store/memory.rs @@ -0,0 +1,89 @@ +//! The store that keeps sessions and their events in memory (`store.mode` `memory`; tests). + +use super::*; + +#[derive(Default)] +pub struct MemoryStore { + inner: Mutex)>>, +} + +impl Store for MemoryStore { + fn list(&self) -> Result> { + Ok(self.inner.lock_or_recover().values().map(|(m, _)| m.clone()).collect()) + } + fn load(&self, id: &str) -> Result> { + Ok(self.inner.lock_or_recover().get(id).map(|(m, _)| m.clone())) + } + fn save(&self, meta: &SessionMeta) -> Result<()> { + let mut g = self.inner.lock_or_recover(); + g.entry(meta.id.clone()) + .and_modify(|(m, _)| *m = meta.clone()) + .or_insert_with(|| (meta.clone(), Vec::new())); + Ok(()) + } + fn append(&self, id: &str, record: &EventRecord) -> Result<()> { + let mut g = self.inner.lock_or_recover(); + if let Some((_, events)) = g.get_mut(id) { + events.push(record.clone()); + // Bound memory: keep the newest 20k events. + if events.len() > 20_000 { + let drop = events.len() - 20_000; + events.drain(..drop); + } + } + Ok(()) + } + fn events(&self, id: &str, after: u64, limit: usize) -> Result> { + let g = self.inner.lock_or_recover(); + Ok(g.get(id) + .map(|(_, ev)| ev.iter().filter(|e| e.seq > after).take(limit).cloned().collect()) + .unwrap_or_default()) + } + /// Newest first, a bounded window at a time: a visitor that stops early clones only the + /// windows it read, and it runs off the lock. + fn scan_back( + &self, + id: &str, + after: u64, + before: u64, + skip: &dyn Fn(&[u8], &[u8]) -> bool, + visit: &mut dyn FnMut(EventRecord) -> bool, + ) -> Result<()> { + const WINDOW: usize = 256; + let mut before = before; + loop { + let (window, lowest) = { + let g = self.inner.lock_or_recover(); + let Some((_, events)) = g.get(id) else { return Ok(()) }; + let end = events.partition_point(|e| e.seq < before); + let mut window = Vec::new(); + let mut lowest = None; + for e in events + .get(..end) + .unwrap_or_default() + .iter() + .rev() + .take_while(|e| e.seq > after) + .take(WINDOW) + { + lowest = Some(e.seq); + if !skip(e.dir.as_bytes(), e.kind.as_bytes()) { + window.push(e.clone()); + } + } + (window, lowest) + }; + let Some(lowest) = lowest else { return Ok(()) }; + for record in window { + if !visit(record) { + return Ok(()); + } + } + before = lowest; + } + } + fn delete(&self, id: &str) -> Result<()> { + self.inner.lock_or_recover().remove(id); + Ok(()) + } +} From 53e933431e06edc2b8573ed2dc00ed8220ec17f5 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 4 Nov 2026 03:22:51 +0000 Subject: [PATCH 22/52] acpmux: the event log's journal switch and its crash tests (red) R-DURABILITY S2 (DESIGN.md 3.3b). Today no event-log append is synced: a line a client was told about is on the disk only after the next flush (shutdown), so an OS crash or a power loss takes back every line since (E12). This commit adds only the switch, a seam and the tests; the journal store follows. - `store.journal` (StoreConfig::journal, default false). - `Store::wait_durable(timeout)`: tests and tools wait until every appended record is durable. The default returns at once (the Local and Memory stores never sync an append), which is today's behavior. - `store::open_with_journal` (tests): the store with the journal on a given `cmux_journal::Storage`. Here it ignores the journal and opens LocalStore. - acpmux depends on cmux-journal (dev: with the `sim` feature). - store/journal_store_tests.rs, red on this commit: - a_line_the_store_says_is_durable_survives_an_os_crash (lines 2..20 are lost: nothing synced them); - a_deleted_chat_stays_deleted_after_a_crash and a_line_the_log_refused_is_never_written_later pass here, and must keep passing with the journal store (no replay may bring them back); - a_kill_keeps_every_line_once passes here and must keep passing (a replay after a kill must write no line twice). --- cmux-tui/Cargo.lock | 1 + cmux-tui/crates/acpmux/Cargo.toml | 4 + .../crates/acpmux/src/config/store_config.rs | 7 + cmux-tui/crates/acpmux/src/store.rs | 24 +++ .../acpmux/src/store/journal_store_tests.rs | 164 ++++++++++++++++++ 5 files changed, 200 insertions(+) create mode 100644 cmux-tui/crates/acpmux/src/store/journal_store_tests.rs diff --git a/cmux-tui/Cargo.lock b/cmux-tui/Cargo.lock index d57b9f2bed1b..a652974a2ad6 100644 --- a/cmux-tui/Cargo.lock +++ b/cmux-tui/Cargo.lock @@ -20,6 +20,7 @@ dependencies = [ "cmux-git", "cmux-host", "cmux-host-budget", + "cmux-journal", "cmux-json", "cmux-link", "cmux-local-auth", diff --git a/cmux-tui/crates/acpmux/Cargo.toml b/cmux-tui/crates/acpmux/Cargo.toml index daad8a8946b2..1d3828277cdb 100644 --- a/cmux-tui/crates/acpmux/Cargo.toml +++ b/cmux-tui/crates/acpmux/Cargo.toml @@ -34,6 +34,8 @@ chrono.workspace = true chrono-tz.workspace = true cmux-server = { path = "../cmux-server" } cmux-json.workspace = true +# The event log's journal (R-DURABILITY S2, `store.journal`). +cmux-journal.workspace = true cmux-chat-index.workspace = true cmux-palette-ranker.workspace = true cmux-coderouter.workspace = true @@ -130,6 +132,8 @@ windows-sys = { workspace = true, features = [ ] } [dev-dependencies] +# The simulated crash file system, for the journal store's crash tests. +cmux-journal = { workspace = true, features = ["sim"] } cmux-server-core = { path = "../cmux-server-core" } rcgen = "0.14" # Paused time for the media handle sweeper's tests (`hub/media_handles_tests.rs`). diff --git a/cmux-tui/crates/acpmux/src/config/store_config.rs b/cmux-tui/crates/acpmux/src/config/store_config.rs index b707ed4522ec..15859e756fe4 100644 --- a/cmux-tui/crates/acpmux/src/config/store_config.rs +++ b/cmux-tui/crates/acpmux/src/config/store_config.rs @@ -25,6 +25,12 @@ pub struct StoreConfig { /// Default 14; 0 keeps every log whole. #[serde(default, skip_serializing_if = "Option::is_none")] pub closed_log_days: Option, + /// Every event-log line goes through one append-only journal with group commit + /// (R-DURABILITY S2, `store/journal_store.rs`): a line is durable within one sync, and a + /// restart after a crash writes again every line the journal holds past its checkpoint. + /// Default off. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub journal: bool, } impl StoreConfig { @@ -47,6 +53,7 @@ impl Default for StoreConfig { mode: StoreMode::Local, segment_bytes: default_segment_bytes(), closed_log_days: None, + journal: false, } } } diff --git a/cmux-tui/crates/acpmux/src/store.rs b/cmux-tui/crates/acpmux/src/store.rs index 71859028c087..044e6b29d36a 100644 --- a/cmux-tui/crates/acpmux/src/store.rs +++ b/cmux-tui/crates/acpmux/src/store.rs @@ -838,6 +838,14 @@ pub trait Store: Send + Sync { fn flush(&self) -> Result<()> { Ok(()) } + /// Waits up to `timeout` until every record appended so far is durable; false when the time + /// ran out. For tests and tools: a request never blocks on the disk. The Local and Memory + /// stores do not sync an append, so they return true at once (their appends are as durable + /// as they will be until `flush`). + fn wait_durable(&self, timeout: std::time::Duration) -> Result { + let _ = timeout; + Ok(true) + } /// The daemon's shutdown is done: from now on every write is refused. It lets go of /// `daemon.lock` before its process exits, so the next daemon may already read this store; /// what it reads is the old daemon's last word (`Hub::shutdown_all`). Waits for writes under @@ -981,6 +989,19 @@ pub fn open(config: &StoreConfig, root: &Path) -> Result> { }) } +/// [`open`] with the journal on `storage` (a simulated file system in the crash tests) and its +/// own limits, for `store.journal` (R-DURABILITY S2). +#[cfg(test)] +pub(crate) fn open_with_journal( + config: &StoreConfig, + root: &Path, + storage: std::sync::Arc, + journal: cmux_journal::Config, +) -> Result> { + let _ = (storage, journal); + open(config, root) +} + mod memory; pub use memory::MemoryStore; @@ -1997,6 +2018,9 @@ fn is_zero(v: &u64) -> bool { #[cfg(test)] mod head_tests; +#[cfg(all(test, unix))] +mod journal_store_tests; + impl LocalStore { /// `Store::scan`, and with `strict` `Store::scan_strict`. fn scan_lines( diff --git a/cmux-tui/crates/acpmux/src/store/journal_store_tests.rs b/cmux-tui/crates/acpmux/src/store/journal_store_tests.rs new file mode 100644 index 000000000000..13428e092f15 --- /dev/null +++ b/cmux-tui/crates/acpmux/src/store/journal_store_tests.rs @@ -0,0 +1,164 @@ +//! The event log through the journal (`store.journal`, R-DURABILITY S2; DESIGN.md 3.3b): a line +//! the store says is durable survives an OS crash, a kill keeps every line once, and a chat that +//! was deleted stays deleted. +//! +//! The journal runs on a simulated file system (`cmux_journal::SimStorage`), so a crash keeps +//! exactly what a real one would. The chat logs (`sessions//events/*.ndjson`) are real files +//! in a temporary folder; an OS crash of them is modelled as the worst case: every log file goes +//! back to its length at the last `flush` (the kernel lost every write since). + +use super::*; +use cmux_journal::{Crash, SimStorage}; +use serde_json::json; +use std::sync::Arc; +use std::time::Duration; + +const CHAT: &str = "chat-a"; +const WAIT: Duration = Duration::from_secs(10); + +fn config() -> StoreConfig { + StoreConfig { journal: true, ..StoreConfig::default() } +} + +fn journal_config() -> cmux_journal::Config { + cmux_journal::Config { + retry_min: Duration::from_millis(1), + retry_max: Duration::from_millis(5), + ..cmux_journal::Config::default() + } +} + +fn open(home: &Path, journal: &SimStorage) -> Box { + open_with_journal(&config(), home, Arc::new(journal.clone()), journal_config()) + .unwrap_or_else(|e| panic!("open the store: {e:#}")) +} + +fn record(seq: u64, kind: &str) -> EventRecord { + EventRecord { + seq, + at: 1_000 + seq, + dir: "mux".into(), + kind: kind.into(), + msg: json!({ "n": seq }), + host_seq: None, + } +} + +fn seqs(store: &dyn Store, id: &str) -> Vec { + let events = store.events(id, 0, 10_000).unwrap_or_else(|e| panic!("read {id}: {e:#}")); + events.iter().map(|record| record.seq).collect() +} + +/// Every event-log segment under `home/sessions`, with its length. +fn log_files(home: &Path) -> BTreeMap { + let mut out = BTreeMap::new(); + let Ok(chats) = std::fs::read_dir(home.join("sessions")) else { return out }; + for chat in chats.flatten() { + let Ok(segments) = std::fs::read_dir(chat.path().join("events")) else { continue }; + for segment in segments.flatten() { + let path = segment.path(); + if path.extension().is_some_and(|ext| ext == "ndjson") { + let len = segment.metadata().map(|meta| meta.len()).unwrap_or(0); + out.insert(path, len); + } + } + } + out +} + +/// An OS crash of the log files: each goes back to its length in `synced`, or to empty. +fn lose_unsynced_log_writes(home: &Path, synced: &BTreeMap) { + for path in log_files(home).into_keys() { + let len = synced.get(&path).copied().unwrap_or(0); + let file = std::fs::OpenOptions::new() + .write(true) + .open(&path) + .unwrap_or_else(|e| panic!("open {}: {e}", path.display())); + file.set_len(len).unwrap_or_else(|e| panic!("cut {}: {e}", path.display())); + } +} + +#[test] +fn a_line_the_store_says_is_durable_survives_an_os_crash() { + let home = tempfile::tempdir().unwrap_or_else(|e| panic!("tempdir: {e}")); + let journal = SimStorage::new(7); + let store = open(home.path(), &journal); + store.append(CHAT, &record(1, "created")).unwrap_or_else(|e| panic!("{e:#}")); + store.flush().unwrap_or_else(|e| panic!("{e:#}")); + let synced = log_files(home.path()); + for seq in 2..=20 { + store.append(CHAT, &record(seq, "queued")).unwrap_or_else(|e| panic!("{e:#}")); + } + assert!(store.wait_durable(WAIT).unwrap_or_else(|e| panic!("{e:#}")), "never durable"); + // An OS crash now: the journal keeps what it synced, the logs lose every write since the + // flush. + let crashed = journal.restart(Crash::Os); + drop(store); + lose_unsynced_log_writes(home.path(), &synced); + let store = open(home.path(), &crashed); + assert_eq!(seqs(store.as_ref(), CHAT), (1..=20).collect::>()); +} + +#[test] +fn a_kill_keeps_every_line_once() { + let home = tempfile::tempdir().unwrap_or_else(|e| panic!("tempdir: {e}")); + let journal = SimStorage::new(8); + let store = open(home.path(), &journal); + store.append(CHAT, &record(1, "created")).unwrap_or_else(|e| panic!("{e:#}")); + store.flush().unwrap_or_else(|e| panic!("{e:#}")); + for seq in 2..=30 { + store.append(CHAT, &record(seq, "queued")).unwrap_or_else(|e| panic!("{e:#}")); + } + store.append("chat-b", &record(1, "created")).unwrap_or_else(|e| panic!("{e:#}")); + assert!(store.wait_durable(WAIT).unwrap_or_else(|e| panic!("{e:#}")), "never durable"); + // A kill -9: the logs keep every write, the journal every write it got. + let killed = journal.restart(Crash::Process); + drop(store); + let store = open(home.path(), &killed); + assert_eq!(seqs(store.as_ref(), CHAT), (1..=30).collect::>(), "each line once"); + assert_eq!(seqs(store.as_ref(), "chat-b"), vec![1]); + // And again: a second restart writes nothing twice. + let again = killed.restart(Crash::Process); + drop(store); + let store = open(home.path(), &again); + assert_eq!(seqs(store.as_ref(), CHAT), (1..=30).collect::>(), "each line once"); +} + +#[test] +fn a_deleted_chat_stays_deleted_after_a_crash() { + let home = tempfile::tempdir().unwrap_or_else(|e| panic!("tempdir: {e}")); + let journal = SimStorage::new(9); + let store = open(home.path(), &journal); + for seq in 1..=5 { + store.append(CHAT, &record(seq, "queued")).unwrap_or_else(|e| panic!("{e:#}")); + } + store.append("chat-b", &record(1, "created")).unwrap_or_else(|e| panic!("{e:#}")); + store.delete(CHAT).unwrap_or_else(|e| panic!("{e:#}")); + assert!(store.wait_durable(WAIT).unwrap_or_else(|e| panic!("{e:#}")), "never durable"); + let crashed = journal.restart(Crash::Os); + drop(store); + let store = open(home.path(), &crashed); + assert_eq!(seqs(store.as_ref(), CHAT), Vec::::new(), "the deleted chat came back"); + assert!(!home.path().join("sessions").join(CHAT).exists()); + assert_eq!(seqs(store.as_ref(), "chat-b"), vec![1]); +} + +#[test] +fn a_line_the_log_refused_is_never_written_later() { + let home = tempfile::tempdir().unwrap_or_else(|e| panic!("tempdir: {e}")); + let journal = SimStorage::new(10); + let store = open(home.path(), &journal); + store.append(CHAT, &record(1, "created")).unwrap_or_else(|e| panic!("{e:#}")); + // A file where chat-b's folder would be: its log cannot be written. + let sessions = home.path().join("sessions"); + std::fs::create_dir_all(&sessions).unwrap_or_else(|e| panic!("{e}")); + std::fs::write(sessions.join("chat-b"), b"not a folder").unwrap_or_else(|e| panic!("{e}")); + assert!(store.append("chat-b", &record(1, "created")).is_err(), "the append was refused"); + assert!(store.wait_durable(WAIT).unwrap_or_else(|e| panic!("{e:#}")), "never durable"); + let crashed = journal.restart(Crash::Os); + drop(store); + std::fs::remove_file(sessions.join("chat-b")).unwrap_or_else(|e| panic!("{e}")); + let store = open(home.path(), &crashed); + assert_eq!(seqs(store.as_ref(), "chat-b"), Vec::::new(), "a refused line came back"); + assert_eq!(seqs(store.as_ref(), CHAT), vec![1]); +} From 2d66751a1fdccbacc4b77748a8f79daf8e2895f9 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 4 Nov 2026 04:26:26 +0000 Subject: [PATCH 23/52] acpmux: the event log through one journal, behind `store.journal` (R-DURABILITY S2) DESIGN.md 3.3b. With `store.journal` on, each event-log append is one record in `/journal/` (cmux-journal: group commit, one sync per batch; a power loss takes back at most the last second, always a suffix), then the same line goes to the chat's ndjson as before. Readers, segments and file formats do not change. Default off. - store/journal_store.rs (JournalStore over LocalStore): - append: under one lock, the journal record (a memory copy, no I/O), then the log line; a line the log refuses gets a `cancel` record. Agent stream lines (`in` session/update, claude.*, codex.*) are Class::Stream, every other line Class::Ack. - checkpoint (`/journal-projected.json`): every 30 s while the journal grew, past 64 MiB, at flush (shutdown) and before a retention commit. It syncs each log written since (LocalStore::sync_log, with no lock held while the disk works) and the sessions folder, writes the LSN (temp file, sync, rename, folder sync, every error checked), then retires the journal before it. A log close that failed its sync freezes the checkpoint until a restart (LocalStore::sync_failed). - recovery: the records after the checkpoint; for each chat, the log is cut right after its last readable line below the first journaled seq (later segments removed, the boundary one shortened, their kind-index copies dropped), and the journaled lines are written again in order. `forget` deletes a chat again; `cancel` keeps a refused line out. - delete: `forget` record, folder removal, then a bounded wait until the record is durable. - seal: the logs refuse writes and the journal syncs fully. - LocalStore: `append` is serialize + `append_line`; `sync_log`; a failed sync at a log's close is noted (`sync_failed`). - store::open opens the JournalStore when `store.journal` is on. Not yet (S2c): replies and events do not wait for durability; the daemon does not stop when the journal closes; the writer thread keeps the process's Utility I/O policy. --- cmux-tui/crates/acpmux/src/store.rs | 130 ++-- cmux-tui/crates/acpmux/src/store/compact.rs | 2 +- .../crates/acpmux/src/store/journal_store.rs | 620 ++++++++++++++++++ 3 files changed, 706 insertions(+), 46 deletions(-) create mode 100644 cmux-tui/crates/acpmux/src/store/journal_store.rs diff --git a/cmux-tui/crates/acpmux/src/store.rs b/cmux-tui/crates/acpmux/src/store.rs index 044e6b29d36a..50b5387617b8 100644 --- a/cmux-tui/crates/acpmux/src/store.rs +++ b/cmux-tui/crates/acpmux/src/store.rs @@ -985,6 +985,12 @@ fn parse_record(line: &[u8]) -> serde_json::Result { pub fn open(config: &StoreConfig, root: &Path) -> Result> { Ok(match config.mode { StoreMode::Memory => Box::new(MemoryStore::default()), + StoreMode::Local if config.journal => Box::new(journal_store::JournalStore::open( + config, + root, + std::sync::Arc::new(cmux_journal::FsStorage), + cmux_journal::Config::default(), + )?), StoreMode::Local => Box::new(LocalStore::new(root.join("sessions"), config.segment_bytes)?), }) } @@ -998,10 +1004,11 @@ pub(crate) fn open_with_journal( storage: std::sync::Arc, journal: cmux_journal::Config, ) -> Result> { - let _ = (storage, journal); - open(config, root) + Ok(Box::new(journal_store::JournalStore::open(config, root, storage, journal)?)) } +mod journal_store; + mod memory; pub use memory::MemoryStore; @@ -1014,6 +1021,9 @@ pub struct LocalStore { /// Read-held by every write, true once `seal` ran: a write either lands before the seal /// returns or not at all. sealed: std::sync::RwLock, + /// A log's sync on its close failed (`SegmentWriter::close`): its lines may be lost on the + /// disk. The journal store then keeps every line in its journal (`store/journal_store.rs`). + sync_failed: std::sync::atomic::AtomicBool, } /// The logs held open, at most [`OPEN_LOGS`]: the least recently written one @@ -1039,11 +1049,12 @@ struct SegmentWriter { impl SegmentWriter { /// Closes the log, durable first as `flush` would have made it. Called under the /// writers' lock, so the cache never holds more than [`OPEN_LOGS`] and a session's - /// records stay in order. A failed sync is logged: its records were written (an append is - /// never synced on its own), so only a machine crash before the kernel writes them back - /// loses them, as it would any append since the last `flush`. - fn close(self, id: &str) { + /// records stay in order. A failed sync is logged and noted in `failed`: its records were + /// written (an append is never synced on its own), so only a machine crash before the kernel + /// writes them back loses them, as it would any append since the last `flush`. + fn close(self, id: &str, failed: &std::sync::atomic::AtomicBool) { if let Err(e) = self.file.sync_data() { + failed.store(true, std::sync::atomic::Ordering::SeqCst); tracing::warn!(session = %id, "sync event log on close: {e}"); } } @@ -1058,6 +1069,7 @@ impl LocalStore { segment_bytes: segment_bytes.max(64 * 1024), writers: Mutex::new(Writers::default()), sealed: Default::default(), + sync_failed: Default::default(), }) } @@ -1379,6 +1391,71 @@ fn first_seq(path: &Path) -> Result> { } } +impl LocalStore { + /// Syncs `id`'s open log segment and its folders, without holding the writers' lock while + /// the disk works (the journal store's checkpoint, `store/journal_store.rs`). A segment that + /// is not open was synced at its close (`SegmentWriter::close`, which notes a failure). + pub(super) fn sync_log(&self, id: &str) -> Result<()> { + let open = { + let writers = self.writers.lock_or_recover(); + match writers.open.get(id) { + Some(writer) => Some(writer.file.try_clone()?), + None => None, + } + }; + if let Some(file) = open { + file.sync_all()?; + } + let session = self.dir(id); + sync_directory(&session.join("events"))?; + sync_directory(&session) + } + + /// One whole serialized record (`line` ends with its newline) at the end of `id`'s log: the + /// body of [`Store::append`], and the journal store's write of a journaled line + /// (`store/journal_store.rs`). Refused once the store is sealed. + pub(super) fn append_line(&self, id: &str, line: &[u8]) -> Result<()> { + // One lock hold from the eviction to the write: the least recently written log is + // closed before this one opens, so no more than `OPEN_LOGS` are ever open, and no other + // append comes between (a session's records stay in order). Its sync runs under the lock, + // once per eviction: only while more than `OPEN_LOGS` sessions are written. + let _open = self.writable()?; + let mut writers = self.writers.lock_or_recover(); + if !writers.open.contains_key(id) + && writers.open.len() >= OPEN_LOGS + && let Some(oldest) = + writers.open.iter().min_by_key(|(_, w)| w.used).map(|(k, _)| k.clone()) + && let Some(w) = writers.open.remove(&oldest) + { + w.close(&oldest, &self.sync_failed); + } + if !writers.open.contains_key(id) { + let writer = self.open_writer(id)?; + writers.open.insert(id.to_owned(), writer); + } + writers.clock += 1; + let clock = writers.clock; + let full = writers.open.get(id).is_some_and(|w| w.written >= self.segment_bytes); + if full && let Some(old) = writers.open.remove(id) { + // The full segment closes before the next one opens (rare: once per segment). + let index = old.index + 1; + old.close(id, &self.sync_failed); + let path = self.dir(id).join("events").join(format!("{index:06}.ndjson")); + let file = open_private_append(&path)?; + writers + .open + .insert(id.to_owned(), SegmentWriter { index, file, written: 0, used: clock }); + } + let Some(w) = writers.open.get_mut(id) else { + anyhow::bail!("no segment writer for session {id}"); + }; + w.used = clock; + w.file.write_all(line)?; + w.written += line.len() as u64; + Ok(()) + } +} + impl Store for LocalStore { fn list(&self) -> Result> { let Ok(rd) = std::fs::read_dir(&self.root) else { @@ -1460,46 +1537,9 @@ impl Store for LocalStore { } fn append(&self, id: &str, record: &EventRecord) -> Result<()> { - // One lock hold from the eviction to the write: the least recently written log is - // closed before this one opens, so no more than `OPEN_LOGS` are ever open, and no other - // append comes between (a session's records stay in order). Its sync runs under the lock, - // once per eviction: only while more than `OPEN_LOGS` sessions are written. - let _open = self.writable()?; - let mut writers = self.writers.lock_or_recover(); - if !writers.open.contains_key(id) - && writers.open.len() >= OPEN_LOGS - && let Some(oldest) = - writers.open.iter().min_by_key(|(_, w)| w.used).map(|(k, _)| k.clone()) - && let Some(w) = writers.open.remove(&oldest) - { - w.close(&oldest); - } - if !writers.open.contains_key(id) { - let writer = self.open_writer(id)?; - writers.open.insert(id.to_owned(), writer); - } - writers.clock += 1; - let clock = writers.clock; - let full = writers.open.get(id).is_some_and(|w| w.written >= self.segment_bytes); - if full && let Some(old) = writers.open.remove(id) { - // The full segment closes before the next one opens (rare: once per segment). - let index = old.index + 1; - old.close(id); - let path = self.dir(id).join("events").join(format!("{index:06}.ndjson")); - let file = open_private_append(&path)?; - writers - .open - .insert(id.to_owned(), SegmentWriter { index, file, written: 0, used: clock }); - } - let Some(w) = writers.open.get_mut(id) else { - anyhow::bail!("no segment writer for session {id}"); - }; - w.used = clock; let mut line = serde_json::to_string(record)?; line.push('\n'); - w.file.write_all(line.as_bytes())?; - w.written += line.len() as u64; - Ok(()) + self.append_line(id, line.as_bytes()) } /// Reads only past `after`: a segment whose successor starts at or before `after + 1` @@ -1665,7 +1705,7 @@ impl Store for LocalStore { fn close_log(&self, id: &str) { let mut writers = self.writers.lock_or_recover(); if let Some(w) = writers.open.remove(id) { - w.close(id); + w.close(id, &self.sync_failed); } } diff --git a/cmux-tui/crates/acpmux/src/store/compact.rs b/cmux-tui/crates/acpmux/src/store/compact.rs index 99bb9fad372e..e9140b2ff6f8 100644 --- a/cmux-tui/crates/acpmux/src/store/compact.rs +++ b/cmux-tui/crates/acpmux/src/store/compact.rs @@ -225,7 +225,7 @@ impl LocalStore { write_atomic(&self.marker(id), &serde_json::to_vec_pretty(&marker)?)?; // Its next append opens the compacted segment, not the old one's descriptor. if let Some(w) = writers.open.remove(id) { - w.close(id); + w.close(id, &self.sync_failed); } // Every segment stays, an empty one too: a reader that listed the segments a moment // ago opens each by its name. diff --git a/cmux-tui/crates/acpmux/src/store/journal_store.rs b/cmux-tui/crates/acpmux/src/store/journal_store.rs new file mode 100644 index 000000000000..4590dceee689 --- /dev/null +++ b/cmux-tui/crates/acpmux/src/store/journal_store.rs @@ -0,0 +1,620 @@ +//! The event log through one append-only journal (`store.journal`, R-DURABILITY S2; +//! `handoff/rewrites/R-DURABILITY/DESIGN.md` 3.3b). +//! +//! Without it, no event-log append is synced: a line a client was told about reaches the disk at +//! the next `flush` (shutdown), so an OS crash or a power loss takes back every line since. With +//! it, each append is one record in `/journal/` (`cmux-journal`: group commit, one sync per +//! batch, a power loss takes back at most the last second, always a suffix), and then the same +//! line goes to the chat's `events/*.ndjson`, as before. Readers, segments and every file format +//! stay as they are: the logs are the journal's projection, read as before. +//! +//! - Order: one lock holds from the journal append to the log write, so for each chat the log's +//! lines are in LSN order, and a log line always has its record. A line the log refuses gets a +//! `cancel` record, so no replay writes it. +//! - Checkpoint (`/journal-projected.json`, the journal's compaction point): every 30 s while +//! the journal grew, past 64 MiB of it, at `flush` (shutdown) and before a retention rewrite. It +//! syncs every log written since the last one and the sessions folder, then writes the LSN +//! that every record up to has its log effect, durably, then retires the journal segments +//! before it. A log sync that failed on its own (`LocalStore::sync_failed`) freezes the +//! checkpoint: the journal then keeps every line until a restart writes them again. +//! - Recovery (open): the records after the checkpoint are read. For each chat with lines among +//! them, its log is cut right after its last readable line below the first such line's `seq` +//! (lines before it were synced by the checkpoint; after it are torn writes, lines an OS crash +//! dropped, or lines the journal holds), and the journal's lines are written again, in order. +//! A chat with a `forget` record (a delete) is deleted again. Exact and idempotent: a second +//! crash in the middle is repaired by the next open the same way. +//! - Delete: a `forget` record, then the folder's removal, then a wait (bounded) until the record +//! is durable, so a crash never brings a deleted chat back. +//! +//! Not yet (S2c): replies and events do not wait for their records to be durable, and the daemon +//! does not stop when the journal closes (a panic on its writer). Session metadata is S3. + +use super::*; +use cmux_journal::{AppendError, Class, Journal, Lsn, Record}; +use std::sync::atomic::Ordering as AtomicOrdering; +use std::sync::{Arc, Condvar}; +use std::time::Duration; + +/// The journal record kinds. +const KIND_LOG: u16 = 1; +const KIND_FORGET: u16 = 2; +const KIND_CANCEL: u16 = 3; + +const CHECKPOINT_NAME: &str = "journal-projected.json"; +const CHECKPOINT_SCHEMA: &str = "acpmux.journal-projected.v1"; +const CHECKPOINT_EVERY: Duration = Duration::from_secs(30); +/// Journal bytes past the last checkpoint that wake the checkpointer at once. +const CHECKPOINT_BYTES: u64 = 64 * 1024 * 1024; +/// The longest a seal waits for the journal's last full sync, and a delete for its record. +const SEAL_WAIT: Duration = Duration::from_secs(5); +const FORGET_WAIT: Duration = Duration::from_secs(5); + +pub struct JournalStore { + inner: Arc, + checkpointer: Mutex>>, +} + +struct Inner { + local: LocalStore, + journal: Journal, + home: PathBuf, + order: Mutex, + /// One checkpoint at a time. + checkpointing: Mutex<()>, + timer: Mutex, + wake: Condvar, +} + +#[derive(Default)] +struct Order { + /// Every journal record up to this LSN has its log effect done (a line written, a chat + /// deleted) or none to do (a cancel). + applied: Lsn, + /// The LSN of the last checkpoint. + checkpointed: Lsn, + /// Chats whose log was written since the last checkpoint. + dirty: BTreeSet, + /// Journal payload bytes since the last checkpoint. + bytes: u64, + /// A log sync failed: the checkpoint stays until a restart writes the logs again. + frozen: bool, +} + +#[derive(Default)] +struct Timer { + stop: bool, + /// A checkpoint is wanted now (the journal grew past `CHECKPOINT_BYTES`). + wanted: bool, +} + +impl JournalStore { + pub(super) fn open( + config: &StoreConfig, + home: &Path, + storage: Arc, + journal_config: cmux_journal::Config, + ) -> Result { + let local = LocalStore::new(home.join("sessions"), config.segment_bytes)?; + let from = read_checkpoint(home); + let mut replay = Replay::default(); + let (journal, report) = + Journal::open(storage, &home.join("journal"), journal_config, from, &mut |record| { + replay.take(record) + }) + .map_err(|error| anyhow::anyhow!("open the event journal: {error}"))?; + for damage in &report.damage { + tracing::error!(?damage, "the event journal could not read some records"); + } + let inner = Arc::new(Inner { + local, + journal, + home: home.to_path_buf(), + order: Mutex::new(Order { checkpointed: from, ..Order::default() }), + checkpointing: Mutex::new(()), + timer: Mutex::new(Timer::default()), + wake: Condvar::new(), + }); + inner.apply(replay)?; + if inner.journal.last_lsn() > from + && let Err(error) = inner.checkpoint() + { + tracing::warn!("event journal checkpoint after the replay: {error:#}"); + } + let worker = Arc::clone(&inner); + let checkpointer = std::thread::Builder::new() + .name("acpmux-journal-checkpoint".into()) + .spawn(move || worker.run_checkpointer()) + .context("start the event journal's checkpointer")?; + Ok(Self { inner, checkpointer: Mutex::new(Some(checkpointer)) }) + } +} + +impl Drop for JournalStore { + /// Stops the checkpointer. No checkpoint here: a drop is not a shutdown (`flush` is). + fn drop(&mut self) { + self.inner.timer.lock_or_recover().stop = true; + self.inner.wake.notify_all(); + if let Some(handle) = self.checkpointer.lock_or_recover().take() { + let _ = handle.join(); + } + } +} + +// ---------------------------------------------------------------- records + +fn encode_log(id: &str, line: &[u8]) -> Vec { + let len = u32::try_from(id.len()).unwrap_or(u32::MAX); + let mut payload = + Vec::with_capacity(4usize.saturating_add(id.len()).saturating_add(line.len())); + payload.extend_from_slice(&len.to_le_bytes()); + payload.extend_from_slice(id.as_bytes()); + payload.extend_from_slice(line); + payload +} + +fn decode_log(payload: &[u8]) -> Option<(String, &[u8])> { + let len = usize::try_from(u32::from_le_bytes(payload.get(..4)?.try_into().ok()?)).ok()?; + let end = 4usize.checked_add(len)?; + let id = std::str::from_utf8(payload.get(4..end)?).ok()?; + Some((id.to_owned(), payload.get(end..)?)) +} + +/// A streamed agent line: synced with the next acked batch or a second later (`Class::Stream`). +fn streamed(record: &EventRecord) -> bool { + record.dir == "in" + && (record.kind == crate::rpc::method::SESSION_UPDATE + || record.kind.starts_with("claude.") + || record.kind.starts_with("codex.")) +} + +/// The `seq` of a whole log line, or `None` when it does not read. +fn line_seq(line: &[u8]) -> Option { + record_seq(line).or_else(|| parse_record(line).ok().map(|record| record.seq)) +} + +// ---------------------------------------------------------------- the checkpoint file + +#[derive(Serialize, Deserialize)] +struct Checkpoint { + schema: String, + lsn: Lsn, +} + +/// The checkpoint's LSN; 0 when there is none or it does not read (the replay then starts at the +/// journal's oldest record, which rewrites lines the logs already hold, the same bytes). +fn read_checkpoint(home: &Path) -> Lsn { + let path = home.join(CHECKPOINT_NAME); + let bytes = match std::fs::read(&path) { + Ok(bytes) => bytes, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return 0, + Err(error) => { + tracing::warn!("read {}: {error}", path.display()); + return 0; + } + }; + match serde_json::from_slice::(&bytes) { + Ok(checkpoint) if checkpoint.schema == CHECKPOINT_SCHEMA => checkpoint.lsn, + _ => { + tracing::warn!( + "{} does not read; the replay starts at the oldest record", + path.display() + ); + 0 + } + } +} + +/// Writes the checkpoint whole under a temporary name, syncs it, renames it into place and syncs +/// the home folder; an error at any step (the folder's sync too) is returned. +fn write_checkpoint(home: &Path, lsn: Lsn) -> Result<()> { + let path = home.join(CHECKPOINT_NAME); + crate::config::refuse_lost_home(&path)?; + let tmp = home.join(format!("{CHECKPOINT_NAME}.tmp-{}", std::process::id())); + let bytes = serde_json::to_vec(&Checkpoint { schema: CHECKPOINT_SCHEMA.into(), lsn })?; + let mut options = OpenOptions::new(); + options.write(true).create(true).truncate(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600).custom_flags(libc::O_NOFOLLOW); + } + let mut file = options.open(&tmp).with_context(|| format!("write {}", tmp.display()))?; + file.write_all(&bytes)?; + file.sync_all()?; + drop(file); + std::fs::rename(&tmp, &path).with_context(|| format!("rename to {}", path.display()))?; + sync_directory(home) +} + +// ---------------------------------------------------------------- recovery + +/// The records after the checkpoint, by chat. +#[derive(Default)] +struct Replay { + chats: BTreeMap, + cancelled: BTreeSet, +} + +#[derive(Default)] +struct ChatReplay { + /// Its lines after its last `forget`, in LSN order. + lines: Vec<(Lsn, Vec)>, + forgotten: bool, +} + +impl Replay { + fn take(&mut self, record: Record<'_>) -> std::result::Result<(), String> { + match record.kind { + KIND_LOG => { + let (chat, line) = decode_log(record.payload) + .ok_or_else(|| format!("log record {} does not read", record.lsn))?; + self.chats.entry(chat).or_default().lines.push((record.lsn, line.to_vec())); + } + KIND_FORGET => { + let chat = std::str::from_utf8(record.payload) + .map_err(|_| format!("forget record {} does not read", record.lsn))?; + let entry = self.chats.entry(chat.to_owned()).or_default(); + entry.lines.clear(); + entry.forgotten = true; + } + KIND_CANCEL => { + let lsn = record + .payload + .try_into() + .map(u64::from_le_bytes) + .map_err(|_| format!("cancel record {} does not read", record.lsn))?; + self.cancelled.insert(lsn); + } + other => return Err(format!("unknown event journal record kind {other}")), + } + Ok(()) + } +} + +impl Inner { + /// Writes the replayed records' effects to the logs ([module docs](self)). + fn apply(&self, replay: Replay) -> Result<()> { + let Replay { chats, cancelled } = replay; + for (chat, mut entry) in chats { + if entry.forgotten && self.local.dir(&chat).exists() { + Store::delete(&self.local, &chat) + .with_context(|| format!("delete {chat} again (its forget record)"))?; + } + entry.lines.retain(|(lsn, _)| !cancelled.contains(lsn)); + let Some(from_seq) = entry.lines.first().and_then(|(_, line)| line_seq(line)) else { + if !entry.lines.is_empty() { + tracing::error!(session = %chat, "a journaled line has no seq; not written again"); + } + continue; + }; + self.cut_from(&chat, from_seq) + .with_context(|| format!("cut the log of {chat} for its replay"))?; + for (_, line) in &entry.lines { + self.local.append_line(&chat, line)?; + } + self.order.lock_or_recover().dirty.insert(chat); + } + let mut order = self.order.lock_or_recover(); + order.applied = self.journal.last_lsn(); + Ok(()) + } + + /// Cuts `id`'s log right after its last readable line with a `seq` below `from_seq`: later + /// segments go, and the boundary segment is shortened. Each such segment's kind-index copy goes + /// too (a cache, made again by the next scan). + fn cut_from(&self, id: &str, from_seq: u64) -> Result<()> { + let segments = self.local.segments(id)?; + for (index, path) in segments.iter().rev() { + let starts_before = first_seq(path)?.is_some_and(|seq| seq < from_seq); + if !starts_before { + std::fs::remove_file(path).with_context(|| format!("remove {}", path.display()))?; + self.forget_index_copy(id, *index); + continue; + } + let keep = end_of_last_line_before(path, from_seq)?; + let len = std::fs::metadata(path)?.len(); + if keep < len { + truncate_private(path, keep)?; + self.forget_index_copy(id, *index); + } + break; + } + Ok(()) + } + + fn forget_index_copy(&self, id: &str, index: u32) { + let copy = self.local.dir(id).join(kind_index::DIR).join(format!("{index:06}.ndjson")); + match std::fs::remove_file(©) { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => tracing::warn!("remove {}: {error}", copy.display()), + } + } + + // ------------------------------------------------------------ checkpoints + + fn checkpoint(&self) -> Result<()> { + let _one = self.checkpointing.lock_or_recover(); + let (lsn, dirty) = { + let mut order = self.order.lock_or_recover(); + if order.frozen || (order.applied <= order.checkpointed && order.dirty.is_empty()) { + return Ok(()); + } + // A failed checkpoint waits for the timer or another `CHECKPOINT_BYTES`. + order.bytes = 0; + (order.applied, std::mem::take(&mut order.dirty)) + }; + if let Err(error) = self.sync_logs(&dirty) { + self.order.lock_or_recover().dirty.extend(dirty); + return Err(error); + } + if self.local.sync_failed.load(AtomicOrdering::SeqCst) { + self.order.lock_or_recover().frozen = true; + anyhow::bail!( + "an event log failed to sync: the journal keeps every line until a restart" + ); + } + write_checkpoint(&self.home, lsn)?; + self.order.lock_or_recover().checkpointed = lsn; + if let Err(error) = self.journal.retire_through(lsn) { + tracing::warn!("retire the event journal before {lsn}: {error}"); + } + Ok(()) + } + + /// Syncs the logs of `dirty` (their open segment and folders) and the sessions folder (a + /// delete's removal, a new chat's folder). + fn sync_logs(&self, dirty: &BTreeSet) -> Result<()> { + for chat in dirty { + if self.local.dir(chat).exists() { + self.local.sync_log(chat).with_context(|| format!("sync the log of {chat}"))?; + } + } + sync_directory(&self.local.root) + } + + fn run_checkpointer(&self) { + let mut timer = self.timer.lock_or_recover(); + loop { + if timer.stop { + return; + } + if !timer.wanted { + timer = self + .wake + .wait_timeout(timer, CHECKPOINT_EVERY) + .unwrap_or_else(std::sync::PoisonError::into_inner) + .0; + if timer.stop { + return; + } + } + timer.wanted = false; + drop(timer); + if let Err(error) = self.checkpoint() { + tracing::warn!("event journal checkpoint: {error:#}"); + } + timer = self.timer.lock_or_recover(); + } + } + + fn want_checkpoint(&self) { + let mut timer = self.timer.lock_or_recover(); + if !timer.wanted { + timer.wanted = true; + drop(timer); + self.wake.notify_all(); + } + } +} + +/// The offset right after the last whole line of `path` whose `seq` reads and is below +/// `from_seq`; 0 when there is none. +fn end_of_last_line_before(path: &Path, from_seq: u64) -> Result { + let mut reader = BufReader::new(crate::config::private_state_file(path)?); + let mut line = Vec::new(); + let (mut offset, mut keep) = (0u64, 0u64); + loop { + line.clear(); + let read = reader.read_until(b'\n', &mut line)?; + if read == 0 { + return Ok(keep); + } + offset = offset.saturating_add(u64::try_from(read).unwrap_or(u64::MAX)); + if line.last() != Some(&b'\n') { + return Ok(keep); + } + if line_seq(&line).is_some_and(|seq| seq < from_seq) { + keep = offset; + } + } +} + +/// Shortens the log segment at `path` (owner-only, never through a symlink) and syncs it. +fn truncate_private(path: &Path, len: u64) -> Result<()> { + crate::config::refuse_lost_home(path)?; + let mut options = OpenOptions::new(); + options.write(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.custom_flags(libc::O_NOFOLLOW); + } + let file = options.open(path).with_context(|| format!("open {}", path.display()))?; + file.set_len(len).with_context(|| format!("shorten {}", path.display()))?; + file.sync_all()?; + Ok(()) +} + +// ---------------------------------------------------------------- the store + +impl JournalStore { + fn journal_error(error: AppendError) -> anyhow::Error { + if error == AppendError::Closed { + tracing::error!("the event journal is closed: no event is stored"); + } + anyhow::anyhow!("the event journal: {error}") + } +} + +impl Store for JournalStore { + fn list(&self) -> Result> { + Store::list(&self.inner.local) + } + fn load(&self, id: &str) -> Result> { + Store::load(&self.inner.local, id) + } + fn load_startup(&self, id: &str) -> Result> { + Store::load_startup(&self.inner.local, id) + } + fn save(&self, meta: &SessionMeta) -> Result<()> { + Store::save(&self.inner.local, meta) + } + fn append(&self, id: &str, record: &EventRecord) -> Result<()> { + let mut line = serde_json::to_string(record)?; + line.push('\n'); + let class = if streamed(record) { Class::Stream } else { Class::Ack }; + let payload = encode_log(id, line.as_bytes()); + let inner = &self.inner; + let mut order = inner.order.lock_or_recover(); + let lsn = + inner.journal.append(KIND_LOG, class, &payload).map_err(JournalStore::journal_error)?; + let written = inner.local.append_line(id, line.as_bytes()); + if written.is_err() + && let Err(error) = inner.journal.append(KIND_CANCEL, Class::Ack, &lsn.to_le_bytes()) + { + tracing::error!(session = %id, "the event journal could not cancel record {lsn}: {error}"); + } + order.applied = inner.journal.last_lsn(); + if written.is_ok() && !order.dirty.contains(id) { + order.dirty.insert(id.to_owned()); + } + order.bytes = order.bytes.saturating_add(u64::try_from(payload.len()).unwrap_or(u64::MAX)); + let full = order.bytes > CHECKPOINT_BYTES && !order.frozen; + drop(order); + if full { + inner.want_checkpoint(); + } + written + } + fn flush_session(&self, id: &str) -> Result<()> { + Store::flush_session(&self.inner.local, id) + } + fn events(&self, id: &str, after: u64, limit: usize) -> Result> { + Store::events(&self.inner.local, id, after, limit) + } + fn scan(&self, id: &str, after: u64, visit: &mut dyn FnMut(EventRecord) -> bool) -> Result<()> { + Store::scan(&self.inner.local, id, after, visit) + } + fn scan_strict( + &self, + id: &str, + after: u64, + visit: &mut dyn FnMut(EventRecord) -> bool, + ) -> Result<()> { + Store::scan_strict(&self.inner.local, id, after, visit) + } + fn scan_back( + &self, + id: &str, + after: u64, + before: u64, + skip: &dyn Fn(&[u8], &[u8]) -> bool, + visit: &mut dyn FnMut(EventRecord) -> bool, + ) -> Result<()> { + Store::scan_back(&self.inner.local, id, after, before, skip, visit) + } + fn tail_after(&self, id: &str, after: u64) -> Result<(u64, u64)> { + Store::tail_after(&self.inner.local, id, after) + } + fn scan_kinds( + &self, + id: &str, + kinds: &[&str], + visit: &mut dyn FnMut(EventRecord) -> bool, + ) -> Result<()> { + Store::scan_kinds(&self.inner.local, id, kinds, visit) + } + fn scan_kinds_strict( + &self, + id: &str, + kinds: &[&str], + visit: &mut dyn FnMut(EventRecord) -> bool, + ) -> Result<()> { + Store::scan_kinds_strict(&self.inner.local, id, kinds, visit) + } + fn scan_heads(&self, id: &str, visit: &mut dyn FnMut(&RecordHead<'_>) -> bool) -> Result<()> { + Store::scan_heads(&self.inner.local, id, visit) + } + /// A checkpoint: every log written so far is synced and the journal before it retired. + fn flush(&self) -> Result<()> { + self.inner.checkpoint() + } + fn wait_durable(&self, timeout: Duration) -> Result { + let lsn = self.inner.journal.last_lsn(); + if lsn == 0 { + return Ok(true); + } + match self.inner.journal.wait_durable(lsn, timeout) { + Ok(()) => Ok(true), + Err(cmux_journal::WaitError::Timeout) => Ok(false), + Err(error) => Err(anyhow::anyhow!("the event journal: {error}")), + } + } + /// The logs refuse writes, and the journal fully syncs every record (the shutdown's `flush` + /// then checkpoints). + fn seal(&self) { + Store::seal(&self.inner.local); + if let Err(error) = self.inner.journal.seal(SEAL_WAIT) { + tracing::error!("seal the event journal: {error}"); + } + } + fn close_log(&self, id: &str) { + Store::close_log(&self.inner.local, id); + } + fn compacted_seq(&self, id: &str) -> Option { + Store::compacted_seq(&self.inner.local, id) + } + fn log_written_at(&self, id: &str) -> Option { + Store::log_written_at(&self.inner.local, id) + } + fn stage_compaction(&self, id: &str, mode: CompactMode) -> Result> { + Store::stage_compaction(&self.inner.local, id, mode) + } + /// A checkpoint first, so no journal record of the chat is past it: a replay never writes + /// the lines a retention rewrite cut back into its log. + fn commit_compaction(&self, id: &str, staged: Staged) -> Result { + self.inner.checkpoint()?; + Store::commit_compaction(&self.inner.local, id, staged) + } + fn delete(&self, id: &str) -> Result<()> { + let inner = &self.inner; + let forget = { + let mut order = inner.order.lock_or_recover(); + let lsn = inner + .journal + .append(KIND_FORGET, Class::Ack, id.as_bytes()) + .map_err(JournalStore::journal_error)?; + let removed = Store::delete(&inner.local, id); + order.applied = inner.journal.last_lsn(); + order.dirty.remove(id); + removed.map(|()| lsn) + }?; + if let Err(error) = inner.journal.wait_durable(forget, FORGET_WAIT) { + tracing::warn!(session = %id, "the delete's journal record is not durable yet: {error}"); + } + Ok(()) + } + fn session_dir(&self, id: &str) -> Option { + Store::session_dir(&self.inner.local, id) + } + fn handoff_dir(&self) -> Option { + Store::handoff_dir(&self.inner.local) + } + fn peer_cache_dir(&self) -> Option { + Store::peer_cache_dir(&self.inner.local) + } + fn briefs_path(&self) -> Option { + Store::briefs_path(&self.inner.local) + } +} From 00512a295deafdbb9da89ad1396f7d3bd0e978bb Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 4 Nov 2026 05:35:27 +0000 Subject: [PATCH 24/52] acpmux: journal store tests for the seal and a chat made again after its delete (red) - a_chat_made_again_after_its_delete_keeps_its_files_through_a_replay: a chat is deleted (its forget record), made again under the same id (a transfer back: session.json and new lines), then a kill -9. On this code the replay deletes the folder again and its session.json is gone. - a_line_after_the_seal_is_refused_and_never_written_later: coverage of the seal (it passes here; the fix also closes a race the test cannot force). --- .../acpmux/src/store/journal_store_tests.rs | 51 +++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/cmux-tui/crates/acpmux/src/store/journal_store_tests.rs b/cmux-tui/crates/acpmux/src/store/journal_store_tests.rs index 13428e092f15..5260248e9050 100644 --- a/cmux-tui/crates/acpmux/src/store/journal_store_tests.rs +++ b/cmux-tui/crates/acpmux/src/store/journal_store_tests.rs @@ -162,3 +162,54 @@ fn a_line_the_log_refused_is_never_written_later() { assert_eq!(seqs(store.as_ref(), "chat-b"), Vec::::new(), "a refused line came back"); assert_eq!(seqs(store.as_ref(), CHAT), vec![1]); } + +#[test] +fn a_line_after_the_seal_is_refused_and_never_written_later() { + let home = tempfile::tempdir().unwrap_or_else(|e| panic!("tempdir: {e}")); + let journal = SimStorage::new(11); + let store = open(home.path(), &journal); + store.append(CHAT, &record(1, "created")).unwrap_or_else(|e| panic!("{e:#}")); + store.seal(); + assert!(store.append(CHAT, &record(2, "queued")).is_err(), "an append after the seal"); + store.flush().unwrap_or_else(|e| panic!("{e:#}")); + let killed = journal.restart(Crash::Process); + drop(store); + let store = open(home.path(), &killed); + assert_eq!(seqs(store.as_ref(), CHAT), vec![1]); +} + +/// A chat deleted and then made again under the same id (a transfer back) before a crash: the +/// replay writes its new lines and keeps its own files (its `session.json`). +#[test] +fn a_chat_made_again_after_its_delete_keeps_its_files_through_a_replay() { + let home = tempfile::tempdir().unwrap_or_else(|e| panic!("tempdir: {e}")); + let journal = SimStorage::new(12); + let store = open(home.path(), &journal); + for seq in 1..=3 { + store.append(CHAT, &record(seq, "queued")).unwrap_or_else(|e| panic!("{e:#}")); + } + store.delete(CHAT).unwrap_or_else(|e| panic!("{e:#}")); + let meta: SessionMeta = serde_json::from_value(json!({ + "schema": META_SCHEMA, + "id": CHAT, + "name": "again", + "harness": "claude", + "family": "claude", + "cwd": "/tmp", + "status": "idle", + "createdAt": 1, + "updatedAt": 1, + })) + .unwrap_or_else(|e| panic!("meta: {e}")); + store.save(&meta).unwrap_or_else(|e| panic!("{e:#}")); + for seq in 1..=2 { + store.append(CHAT, &record(seq, "created")).unwrap_or_else(|e| panic!("{e:#}")); + } + assert!(store.wait_durable(WAIT).unwrap_or_else(|e| panic!("{e:#}")), "never durable"); + let killed = journal.restart(Crash::Process); + drop(store); + let store = open(home.path(), &killed); + assert_eq!(seqs(store.as_ref(), CHAT), vec![1, 2]); + let loaded = store.load(CHAT).unwrap_or_else(|e| panic!("{e:#}")); + assert_eq!(loaded.map(|meta| meta.name), Some("again".to_owned()), "its session.json is gone"); +} From ab0e9535d9da3e21db59d7e4438b5d63f2e67d1a Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 4 Nov 2026 05:35:29 +0000 Subject: [PATCH 25/52] acpmux: journal store: the seal under the appends' lock; a replay keeps a chat made again - seal takes the appends' lock: an append lands in both the journal and the logs, or in neither. Before, an append that raced the seal could leave a journal record whose line the sealed logs refused (its cancel then failed on the closed journal), and the next start would write that line. - append and delete check the logs' seal first, under the same lock, and do not touch the journal after it (no error per append at shutdown). - replay deletes a forgotten chat again only when no line follows its forget record: otherwise the chat was made again under the same id, and its folder holds that chat's own files. --- cmux-tui/crates/acpmux/src/store/journal_store.rs | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/cmux-tui/crates/acpmux/src/store/journal_store.rs b/cmux-tui/crates/acpmux/src/store/journal_store.rs index 4590dceee689..04d0982147cb 100644 --- a/cmux-tui/crates/acpmux/src/store/journal_store.rs +++ b/cmux-tui/crates/acpmux/src/store/journal_store.rs @@ -276,7 +276,9 @@ impl Inner { fn apply(&self, replay: Replay) -> Result<()> { let Replay { chats, cancelled } = replay; for (chat, mut entry) in chats { - if entry.forgotten && self.local.dir(&chat).exists() { + // Deleted again, unless lines follow its forget: then the chat was made again under + // the same id (a transfer back), and its folder holds that chat's own files. + if entry.forgotten && entry.lines.is_empty() && self.local.dir(&chat).exists() { Store::delete(&self.local, &chat) .with_context(|| format!("delete {chat} again (its forget record)"))?; } @@ -477,6 +479,9 @@ impl Store for JournalStore { let payload = encode_log(id, line.as_bytes()); let inner = &self.inner; let mut order = inner.order.lock_or_recover(); + // After the seal (under the same lock, so exact): refused as the logs refuse it, and the + // journal is not touched. + drop(inner.local.writable()?); let lsn = inner.journal.append(KIND_LOG, class, &payload).map_err(JournalStore::journal_error)?; let written = inner.local.append_line(id, line.as_bytes()); @@ -564,6 +569,10 @@ impl Store for JournalStore { /// The logs refuse writes, and the journal fully syncs every record (the shutdown's `flush` /// then checkpoints). fn seal(&self) { + // Under the appends' lock: an append either lands in the journal and the logs, or in + // neither (never a record whose line the sealed logs refused, which a later replay + // would write). + let _order = self.inner.order.lock_or_recover(); Store::seal(&self.inner.local); if let Err(error) = self.inner.journal.seal(SEAL_WAIT) { tracing::error!("seal the event journal: {error}"); @@ -591,6 +600,7 @@ impl Store for JournalStore { let inner = &self.inner; let forget = { let mut order = inner.order.lock_or_recover(); + drop(inner.local.writable()?); let lsn = inner .journal .append(KIND_FORGET, Class::Ack, id.as_bytes()) From 65cead07fed088899bb998a21c1a1f7dea489171 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 4 Nov 2026 06:02:41 +0000 Subject: [PATCH 26/52] cmux-journal: fifth-review regressions G9 (red) Fifth review FIX at 9c47594e1c04 (PR #33919 comment 6623794646). Tests only. - a_skip_hidden_behind_a_recovery_copy_is_never_given_out_again (the reviewer's route): record 1 synced, 2..10 Stream and 11 Ack synced as one run with marker 11, media damage in record 5. Open #1 reports (4, 11], copies 2..4 into segment 2 and makes the empty segment 3 from 12. Segment 3's header is damaged. On this code open #2 reads segment 1 only through the copy's previous last LSN, never scans the damage again, and gives 5. - torn_lsns_hidden_behind_a_recovery_copy_are_never_given_out_again: the same for torn records 4..5 (a tear, then a whole record that no marker vouches for) behind the copy of 2..3: on this code the next append gets 4. --- .../cmux-journal/src/cmux_journal_tests.rs | 80 +++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/cmux-tui/crates/cmux-journal/src/cmux_journal_tests.rs b/cmux-tui/crates/cmux-journal/src/cmux_journal_tests.rs index e9be79ac353d..feac6c088040 100644 --- a/cmux-tui/crates/cmux-journal/src/cmux_journal_tests.rs +++ b/cmux-tui/crates/cmux-journal/src/cmux_journal_tests.rs @@ -829,6 +829,86 @@ fn a_recovery_copy_above_a_quarantine_never_reuses_a_quarantined_lsn() { assert_names_every_quarantined_file(&killed, &report); } +/// G9 of the fifth review: an open skips LSNs (damaged synced records in the last segment) and +/// copies the records before the damage into a new segment; the empty segment after the copy +/// carried the skip in its header alone. With that header damaged, the next open must still skip. +#[test] +fn a_skip_hidden_behind_a_recovery_copy_is_never_given_out_again() { + let storage = SimStorage::new(59); + let config = Config { stream_sync_delay: Duration::from_secs(60), ..quick() }; + let (journal, _, _) = open_at(&storage, config.clone(), 0).unwrap_or_else(|e| panic!("{e}")); + assert_eq!(journal.append(1, Class::Ack, &payload(1)), Ok(1)); + journal.wait_durable(1, WAIT).unwrap_or_else(|e| panic!("{e}")); + assert!(eventually(|| journal.full_durable_lsn() >= 1), "{:?}", journal.stats()); + // 2..10 written, not synced; 11's sync covers them all: one synced run, marker 11 after it. + for lsn in 2..=10 { + assert_eq!(journal.append(1, Class::Stream, &payload(lsn)), Ok(lsn)); + } + assert_eq!(journal.append(1, Class::Ack, &payload(11)), Ok(11)); + assert_eq!(journal.seal(WAIT), Ok(11)); + drop(journal); + // Media damage in a payload byte of record 5. + let record = |lsn: Lsn| format::RECORD_HEADER_LEN + payload(lsn).len(); + let at = SEGMENT_HEADER_LEN + + record(1) + + format::RECORD_HEADER_LEN + + (2..=4).map(record).sum::() + + format::RECORD_HEADER_LEN + + 1; + let storage = storage.restart(Crash::Process); + storage.corrupt(&dir().join(format::segment_name(1)), at); + // Open #1: (4, 11] reported, 2..4 copied into segment 2, the empty segment 3 from 12. + let (journal, report, _) = + open_at(&storage, config.clone(), 0).unwrap_or_else(|e| panic!("{e}")); + let reported = report + .damage + .iter() + .any(|damage| matches!(damage, Damage::Corrupt { lost_after: 4, lost_through: 11, .. })); + assert!(reported, "{report:?}"); + assert_eq!((report.next_lsn, report.rewritten), (12, 3), "{report:?}"); + drop(journal); + let storage = storage.restart(Crash::Process); + storage.corrupt(&dir().join(format::segment_name(3)), 20); + // Open #2. + let (journal, report, _) = open_at(&storage, config, 0).unwrap_or_else(|e| panic!("{e}")); + assert_eq!(journal.append(1, Class::Ack, b"after"), Ok(12), "{report:?}"); +} + +/// The same for torn records (an OS crash wrote some pages of an unsynced write): their LSNs are +/// never given out again, also when a copy hides the segment that holds them. +#[test] +fn torn_lsns_hidden_behind_a_recovery_copy_are_never_given_out_again() { + let storage = SimStorage::new(60); + let config = Config { stream_sync_delay: Duration::from_secs(60), ..quick() }; + let (journal, _, _) = open_at(&storage, config.clone(), 0).unwrap_or_else(|e| panic!("{e}")); + assert_eq!(journal.append(1, Class::Ack, &payload(1)), Ok(1)); + journal.wait_durable(1, WAIT).unwrap_or_else(|e| panic!("{e}")); + assert!(eventually(|| journal.full_durable_lsn() >= 1), "{:?}", journal.stats()); + for lsn in 2..=3 { + assert_eq!(journal.append(1, Class::Stream, &payload(lsn)), Ok(lsn)); + } + assert!(eventually(|| journal.stats().written_lsn >= 3), "{:?}", journal.stats()); + let storage = storage.restart(Crash::Process); + drop(journal); + // A torn write after them: bytes that are not an entry, then record 5 whole. + let path = dir().join(format::segment_name(1)); + let mut bytes = storage.contents(&path).unwrap_or_default(); + bytes.extend_from_slice(&[0x5A; 13]); + let torn = payload(5); + format::encode_record(&mut bytes, 5, 1, Class::Stream, &torn, &format::payload_crc(&torn)); + write_whole(&storage, &path, &bytes); + // Open #1: torn LSNs 4..5, 2..3 copied into segment 2, the empty segment 3 from 6. + let (journal, report, _) = + open_at(&storage, config.clone(), 0).unwrap_or_else(|e| panic!("{e}")); + assert_eq!((report.torn_lsns, report.next_lsn), (Some((4, 5)), 6), "{report:?}"); + drop(journal); + let storage = storage.restart(Crash::Process); + storage.corrupt(&dir().join(format::segment_name(3)), 20); + // Open #2. + let (journal, report, _) = open_at(&storage, config, 0).unwrap_or_else(|e| panic!("{e}")); + assert_eq!(journal.append(1, Class::Ack, b"after"), Ok(6), "{report:?}"); +} + // ---------------------------------------------------------------- real file system #[test] From 51cfdee876b2a4dafe4de970e211be9d130e3e26 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 4 Nov 2026 06:06:10 +0000 Subject: [PATCH 27/52] cmux-journal: name a skipped LSN floor in a FLOOR. file (G9) Fifth review FIX at 9c47594e1c04 (PR #33919 comment 6623794646); the regression tests are the commit before this one. When an open gives out LSNs above everything its kept and new segments say (damaged synced records or torn records in the last segment, which a copy then hides from every later open), the only record of that floor was the header of the empty segment it makes; a damaged header lost it, and DAMAGE is cleared by `clear_damage` (torn records are not damage at all). Now such an open makes the empty file `FLOOR.` and syncs its folder before any new segment, and every open takes the highest floor from the folder listing (no file read), as it does for the quarantine names. The caller's compaction point, the quarantine's names and this open's moves say their floors already: no file is made for them. --- cmux-tui/crates/cmux-journal/src/lib.rs | 5 +- cmux-tui/crates/cmux-journal/src/recovery.rs | 52 ++++++++++++++++---- 2 files changed, 45 insertions(+), 12 deletions(-) diff --git a/cmux-tui/crates/cmux-journal/src/lib.rs b/cmux-tui/crates/cmux-journal/src/lib.rs index 50234b02b575..a246ac9223e5 100644 --- a/cmux-tui/crates/cmux-journal/src/lib.rs +++ b/cmux-tui/crates/cmux-journal/src/lib.rs @@ -29,8 +29,9 @@ //! Files (the journal directory): `LOCK` (an exclusive lock held by the one writer), the segments //! (`.seg`: a header, then records and sync markers back to back, see //! `format.rs`), `quarantine/` for segments recovery could not read past damage that is not a -//! torn tail (named `..`: no LSN at or below `floor` is used again), and -//! `DAMAGE`, the damage earlier opens found, reported at every open until +//! torn tail (named `..`: no LSN at or below `floor` is used again), +//! `FLOOR.` (empty: an LSN floor that no segment header carries alone), and `DAMAGE`, the +//! damage earlier opens found, reported at every open until //! [`Journal::clear_damage`] moves it into `quarantine/`. A segment is only ever appended to. The //! one exception is the writer's own failed write: it cuts the segment back to its size before //! that write, so it removes only bytes that were never synced and never acknowledged. Nothing diff --git a/cmux-tui/crates/cmux-journal/src/recovery.rs b/cmux-tui/crates/cmux-journal/src/recovery.rs index 81f659f3148c..dc8ad0080eec 100644 --- a/cmux-tui/crates/cmux-journal/src/recovery.rs +++ b/cmux-tui/crates/cmux-journal/src/recovery.rs @@ -25,10 +25,12 @@ //! opens, and every open reports it again until `Journal::clear_damage`. //! 6. The journal goes on in a segment this open makes (the copy of step 4, or an empty one), never //! in a segment an earlier process wrote: that one may hold a page its cache had and its disk -//! never got. +//! never got. When the next LSN is above everything the kept and new segments say (damaged or +//! torn records behind the copy), an empty file `FLOOR.` names the floor first, and every +//! open takes the highest one from the listing. //! -//! Recovery writes nothing but syncs, new segments, the renames into `quarantine/` and the damage -//! record `DAMAGE`. It never shortens or deletes a segment. +//! Recovery writes nothing but syncs, new segments, the renames into `quarantine/`, the damage +//! record `DAMAGE` and `FLOOR.` files. It never shortens or deletes a segment. use std::path::Path; @@ -50,6 +52,8 @@ pub(crate) const LOCK_NAME: &str = "LOCK"; pub(crate) const QUARANTINE_DIR: &str = "quarantine"; /// The damage earlier opens found ([`Damage::Corrupt`]), kept until `Journal::clear_damage`. pub(crate) const DAMAGE_NAME: &str = "DAMAGE"; +/// The prefix of an empty file that names an LSN floor (`FLOOR.`, step 6). +const FLOOR_PREFIX: &str = "FLOOR."; /// The name `Journal::clear_damage` gives a cleared damage record in `quarantine/`. pub(crate) const CLEARED_DAMAGE: &str = "damage-report"; /// A segment recovery writes is first written whole under its name plus this, then renamed. @@ -228,7 +232,7 @@ pub(crate) fn recover( let lock = storage.try_lock(&dir.join(LOCK_NAME)).map_err(io("lock"))?.ok_or(OpenError::Locked)?; let earlier = read_damage(storage, dir)?; - let listed = list_segments(storage, dir)?; + let (listed, named_floor) = list_segments(storage, dir)?; let quarantined = list_quarantined(storage, dir)?; let live_max_seq = listed.last().map_or(0, |(seq, _)| *seq); let q_max_seq = quarantined.iter().map(|entry| entry.seq).max().unwrap_or(0); @@ -312,6 +316,8 @@ pub(crate) fn recover( // 5: every LSN a quarantined segment holds counts, from earlier opens (`q_floor`, their // names) and from this one's moves (planned below, from this floor). + // The floors earlier opens named (`FLOOR.`, below): from the listing, no file read. + floor = floor.max(named_floor); let before_quarantine = floor; floor = floor.max(q_floor); // Reported only when the quarantine holds LSNs that nothing else counted: an old quarantine @@ -400,6 +406,18 @@ pub(crate) fn recover( if let Some(plan) = &plan { move_quarantined(storage, dir, plan)?; } + // An open that gives out LSNs above everything its segments will say (damaged or torn + // records, behind a copy that hides them from later opens) names that floor in a file of its + // own, durable before any new segment: a header can be damaged later, and `DAMAGE` is cleared + // (review G9). The caller's compaction point and the quarantine's names say theirs already. + let said = journal_end + .max(from) + .max(named_floor) + .max(q_floor) + .max(plan.as_ref().map_or(0, |plan| plan.highest)); + if floor > said { + name_floor(storage, dir, floor).map_err(io("name the LSN floor"))?; + } // The writer always starts in a segment this open makes: an earlier process's segment may // hold a page that its cache had and its disk never got (a failed sync), so nothing is // appended to it again. It is read only through the next segment's "previous last LSN". @@ -453,16 +471,30 @@ pub(crate) fn recover( Ok(Recovered { lock, next_lsn, report, writer: WriterStart { journal_id, tail } }) } -/// The live segment files, by number. -fn list_segments(storage: &dyn Storage, dir: &Path) -> Result, OpenError> { - let mut listed: Vec<(u64, String)> = storage - .list(dir) - .map_err(io("list the folder"))? +/// The live segment files, by number, and the highest floor a `FLOOR.` file names. +fn list_segments( + storage: &dyn Storage, + dir: &Path, +) -> Result<(Vec<(u64, String)>, Lsn), OpenError> { + let names = storage.list(dir).map_err(io("list the folder"))?; + let floor = names + .iter() + .filter_map(|name| name.strip_prefix(FLOOR_PREFIX)?.parse::().ok()) + .max() + .unwrap_or(0); + let mut listed: Vec<(u64, String)> = names .into_iter() .filter_map(|name| Some((format::parse_segment_name(&name)?, name))) .collect(); listed.sort(); - Ok(listed) + Ok((listed, floor)) +} + +/// Makes the empty file `FLOOR.` and its folder entry durable: no LSN at or below `lsn` is +/// given out again. +fn name_floor(storage: &dyn Storage, dir: &Path, lsn: Lsn) -> std::io::Result<()> { + let _made = storage.open(&dir.join(format!("{FLOOR_PREFIX}{lsn}")), OpenMode::Create)?; + storage.sync_dir(dir) } /// The segment files in `quarantine/` ([`Quarantined`]), by segment number and attempt. From e030f4fe402e747ef967882850599e92c248d668 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 4 Nov 2026 07:15:24 +0000 Subject: [PATCH 28/52] acpmux: journal store regressions from the S2 review (red) Review FIX at ab0e9535d9da (PR #34901 comment 6624439824). Each test fails on this commit and passes after the fix that follows it: - 1a lines_written_with_the_switch_off_survive_turning_it_on_again - 1b a_kill_while_the_journal_syncs_keeps_the_lines_the_log_got - 1c a_line_flush_session_made_durable_survives_losing_its_record - 2 a_folder_put_back_after_a_delete_is_not_deleted_by_the_replay - 3 a_kill_in_a_delete_leaves_the_chat_whole_or_gone - 4 a_delete_does_not_wait_for_the_journal_sync - 5 a_failed_log_sync_freezes_the_checkpoint, a_checkpoint_that_keeps_failing_asks_the_daemon_to_stop - 6 one_chat_that_cannot_be_repaired_does_not_stop_the_open - 7 the_journal_storage_refuses_a_lost_home - the repair keeps a log's lines after a hole, also through a crash in it: an_os_crash_hole_is_repaired_and_the_lines_after_it_kept, a_crash_inside_a_repair_loses_no_line - nits: an_unreadable_checkpoint_fails_the_open, the_open_waits_for_the_previous_daemon_to_let_go_of_the_journal, a_crashed_checkpoint_temporary_is_removed_at_open, a_crash_right_after_a_checkpoint_reports_no_damage - the catch-up with the switch off: switching_off_after_a_crash_keeps_every_acknowledged_line - cmux-journal: check_append_says_what_append_would_do So they compile here, this commit adds the seams they use, with no behavior: `Store::stop_reason` (default None), `Journal::check_append` (always Ok), `FencedStorage` (never asks its fence), the open report and a test stop point after a repair's cut. --- cmux-tui/crates/acpmux/src/store.rs | 11 +- .../crates/acpmux/src/store/journal_store.rs | 73 ++- .../acpmux/src/store/journal_store_tests.rs | 459 ++++++++++++++++++ .../cmux-journal/src/cmux_journal_tests.rs | 26 + cmux-tui/crates/cmux-journal/src/journal.rs | 9 + 5 files changed, 576 insertions(+), 2 deletions(-) diff --git a/cmux-tui/crates/acpmux/src/store.rs b/cmux-tui/crates/acpmux/src/store.rs index 50b5387617b8..c4113a653a1f 100644 --- a/cmux-tui/crates/acpmux/src/store.rs +++ b/cmux-tui/crates/acpmux/src/store.rs @@ -846,6 +846,12 @@ pub trait Store: Send + Sync { let _ = timeout; Ok(true) } + /// Why the store asks the daemon to stop with a restart, as it changes: `Some(reason)` once + /// the store cannot go on safely (the journal store's frozen checkpoint, whose logs only a + /// restart's replay writes again). None: a store that never asks. + fn stop_reason(&self) -> Option>> { + None + } /// The daemon's shutdown is done: from now on every write is refused. It lets go of /// `daemon.lock` before its process exits, so the next daemon may already read this store; /// what it reads is the old daemon's last word (`Hub::shutdown_all`). Waits for writes under @@ -988,7 +994,10 @@ pub fn open(config: &StoreConfig, root: &Path) -> Result> { StoreMode::Local if config.journal => Box::new(journal_store::JournalStore::open( config, root, - std::sync::Arc::new(cmux_journal::FsStorage), + std::sync::Arc::new(journal_store::FencedStorage::new( + std::sync::Arc::new(cmux_journal::FsStorage), + Box::new(crate::config::refuse_lost_home), + )), cmux_journal::Config::default(), )?), StoreMode::Local => Box::new(LocalStore::new(root.join("sessions"), config.segment_bytes)?), diff --git a/cmux-tui/crates/acpmux/src/store/journal_store.rs b/cmux-tui/crates/acpmux/src/store/journal_store.rs index 04d0982147cb..7ce477196677 100644 --- a/cmux-tui/crates/acpmux/src/store/journal_store.rs +++ b/cmux-tui/crates/acpmux/src/store/journal_store.rs @@ -52,6 +52,62 @@ const FORGET_WAIT: Duration = Duration::from_secs(5); pub struct JournalStore { inner: Arc, checkpointer: Mutex>>, + /// What the journal's open reported (the tests read it). + #[cfg(test)] + opened: cmux_journal::OpenReport, +} + +/// The journal's storage, refusing every write under a lost home: `fence` (in the daemon, +/// `config::refuse_lost_home`) is asked before each operation that makes, writes, renames or +/// removes a file or folder. +pub(super) struct FencedStorage { + inner: Arc, + fence: Box Result<()> + Send + Sync>, +} + +impl FencedStorage { + pub(super) fn new( + inner: Arc, + fence: Box Result<()> + Send + Sync>, + ) -> Self { + Self { inner, fence } + } +} + +impl cmux_journal::Storage for FencedStorage { + fn create_dir_all(&self, dir: &Path) -> std::io::Result<()> { + let _ = &self.fence; + self.inner.create_dir_all(dir) + } + fn list(&self, dir: &Path) -> std::io::Result> { + self.inner.list(dir) + } + fn open( + &self, + path: &Path, + mode: cmux_journal::OpenMode, + ) -> std::io::Result> { + self.inner.open(path, mode) + } + fn rename(&self, from: &Path, to: &Path) -> std::io::Result<()> { + self.inner.rename(from, to) + } + fn remove_file(&self, path: &Path) -> std::io::Result<()> { + self.inner.remove_file(path) + } + fn sync_dir(&self, dir: &Path) -> std::io::Result<()> { + self.inner.sync_dir(dir) + } + fn try_lock(&self, path: &Path) -> std::io::Result>> { + self.inner.try_lock(path) + } +} + +#[cfg(test)] +thread_local! { + /// Tests: a repair on this thread stops with an error right after it cut a log, as a crash + /// there would. + pub(super) static STOP_AFTER_CUT: std::cell::Cell = const { std::cell::Cell::new(false) }; } struct Inner { @@ -125,7 +181,18 @@ impl JournalStore { .name("acpmux-journal-checkpoint".into()) .spawn(move || worker.run_checkpointer()) .context("start the event journal's checkpointer")?; - Ok(Self { inner, checkpointer: Mutex::new(Some(checkpointer)) }) + Ok(Self { + inner, + checkpointer: Mutex::new(Some(checkpointer)), + #[cfg(test)] + opened: report, + }) + } + + /// What the journal's open reported. + #[cfg(test)] + pub(super) fn opened(&self) -> &cmux_journal::OpenReport { + &self.opened } } @@ -319,6 +386,10 @@ impl Inner { truncate_private(path, keep)?; self.forget_index_copy(id, *index); } + #[cfg(test)] + if STOP_AFTER_CUT.with(std::cell::Cell::get) { + anyhow::bail!("stopped after the cut (a test)"); + } break; } Ok(()) diff --git a/cmux-tui/crates/acpmux/src/store/journal_store_tests.rs b/cmux-tui/crates/acpmux/src/store/journal_store_tests.rs index 5260248e9050..645b61d56930 100644 --- a/cmux-tui/crates/acpmux/src/store/journal_store_tests.rs +++ b/cmux-tui/crates/acpmux/src/store/journal_store_tests.rs @@ -7,6 +7,7 @@ //! in a temporary folder; an OS crash of them is modelled as the worst case: every log file goes //! back to its length at the last `flush` (the kernel lost every write since). +use super::journal_store::{FencedStorage, JournalStore, STOP_AFTER_CUT}; use super::*; use cmux_journal::{Crash, SimStorage}; use serde_json::json; @@ -213,3 +214,461 @@ fn a_chat_made_again_after_its_delete_keeps_its_files_through_a_replay() { let loaded = store.load(CHAT).unwrap_or_else(|e| panic!("{e:#}")); assert_eq!(loaded.map(|meta| meta.name), Some("again".to_owned()), "its session.json is gone"); } + +// ------------------------------------------------- review of #34901 (FIX at ab0e9535d9da) + +/// A store with the switch off on `home` (the logs alone). +fn open_off(home: &Path) -> Box { + super::open(&StoreConfig::default(), home).unwrap_or_else(|e| panic!("open the store: {e:#}")) +} + +/// A line written straight into the newest log segment of `id`, as a log line whose journal +/// record was lost. +fn write_log_line(home: &Path, id: &str, record: &EventRecord) { + let events = home.join("sessions").join(id).join("events"); + let last = std::fs::read_dir(&events) + .unwrap_or_else(|e| panic!("read {}: {e}", events.display())) + .flatten() + .map(|entry| entry.path()) + .filter(|path| path.extension().is_some_and(|ext| ext == "ndjson")) + .max() + .unwrap_or_else(|| panic!("no segment in {}", events.display())); + let mut line = serde_json::to_vec(record).unwrap_or_else(|e| panic!("{e}")); + line.push(b'\n'); + let mut file = std::fs::OpenOptions::new() + .append(true) + .open(&last) + .unwrap_or_else(|e| panic!("open {}: {e}", last.display())); + file.write_all(&line).unwrap_or_else(|e| panic!("write {}: {e}", last.display())); +} + +/// Overwrites, in `id`'s log, the lines of `seqs` with zero bytes (their newlines too): a hole an +/// OS crash leaves where the kernel never wrote a page. +fn zero_log_lines(home: &Path, id: &str, seqs: &[u64]) { + let events = home.join("sessions").join(id).join("events"); + for entry in std::fs::read_dir(&events).unwrap_or_else(|e| panic!("{e}")).flatten() { + let path = entry.path(); + let mut bytes = std::fs::read(&path).unwrap_or_else(|e| panic!("{e}")); + let mut offset = 0; + let mut changed = false; + while offset < bytes.len() { + let end = bytes[offset..] + .iter() + .position(|b| *b == b'\n') + .map_or(bytes.len(), |at| offset + at + 1); + if record_seq(&bytes[offset..end]).is_some_and(|seq| seqs.contains(&seq)) { + bytes[offset..end].fill(0); + changed = true; + } + offset = end; + } + if changed { + std::fs::write(path, bytes).unwrap_or_else(|e| panic!("{e}")); + } + } +} + +/// Finding 1a: a kill with the switch on, a run with it off, then the switch on again. +#[test] +fn lines_written_with_the_switch_off_survive_turning_it_on_again() { + let home = tempfile::tempdir().unwrap_or_else(|e| panic!("tempdir: {e}")); + let journal = SimStorage::new(20); + let store = open(home.path(), &journal); + store.append(CHAT, &record(1, "created")).unwrap_or_else(|e| panic!("{e:#}")); + store.flush().unwrap_or_else(|e| panic!("{e:#}")); + for seq in 2..=5 { + store.append(CHAT, &record(seq, "queued")).unwrap_or_else(|e| panic!("{e:#}")); + } + assert!(store.wait_durable(WAIT).unwrap_or_else(|e| panic!("{e:#}")), "never durable"); + let killed = journal.restart(Crash::Process); + drop(store); + let off = open_off(home.path()); + for seq in 6..=9 { + off.append(CHAT, &record(seq, "queued")).unwrap_or_else(|e| panic!("{e:#}")); + } + off.flush().unwrap_or_else(|e| panic!("{e:#}")); + drop(off); + let store = open(home.path(), &killed); + assert_eq!( + seqs(store.as_ref(), CHAT), + (1..=9).collect::>(), + "the lines written with the switch off are gone" + ); +} + +/// Finding 1b: a kill -9 while the journal writer syncs. Record 4 is only in memory; its line is +/// in the log's page cache, which a kill keeps. +#[test] +fn a_kill_while_the_journal_syncs_keeps_the_lines_the_log_got() { + let home = tempfile::tempdir().unwrap_or_else(|e| panic!("tempdir: {e}")); + let journal = SimStorage::new(21); + let store = open(home.path(), &journal); + store.append(CHAT, &record(1, "created")).unwrap_or_else(|e| panic!("{e:#}")); + store.flush().unwrap_or_else(|e| panic!("{e:#}")); + store.append(CHAT, &record(2, "queued")).unwrap_or_else(|e| panic!("{e:#}")); + assert!(store.wait_durable(WAIT).unwrap_or_else(|e| panic!("{e:#}")), "never durable"); + journal.set_sync_delay(Duration::from_secs(2)); + store.append(CHAT, &record(3, "queued")).unwrap_or_else(|e| panic!("{e:#}")); + // The writer wrote 3 and is in its 2 s sync. + std::thread::sleep(Duration::from_millis(300)); + store.append(CHAT, &record(4, "queued")).unwrap_or_else(|e| panic!("{e:#}")); + let killed = journal.restart(Crash::Process); + killed.set_sync_delay(Duration::ZERO); + drop(store); + let store = open(home.path(), &killed); + assert_eq!(seqs(store.as_ref(), CHAT), vec![1, 2, 3, 4], "a kill lost a line the log had"); +} + +/// Finding 1c: a line `flush_session` made durable in the log, whose journal record an OS crash +/// took back (it was still in memory: the writer was in another batch's sync). +#[test] +fn a_line_flush_session_made_durable_survives_losing_its_record() { + let home = tempfile::tempdir().unwrap_or_else(|e| panic!("tempdir: {e}")); + let journal = SimStorage::new(24); + let store = open(home.path(), &journal); + store.append(CHAT, &record(1, "created")).unwrap_or_else(|e| panic!("{e:#}")); + store.flush().unwrap_or_else(|e| panic!("{e:#}")); + store.append(CHAT, &record(2, "queued")).unwrap_or_else(|e| panic!("{e:#}")); + assert!(store.wait_durable(WAIT).unwrap_or_else(|e| panic!("{e:#}")), "never durable"); + journal.set_sync_delay(Duration::from_secs(2)); + store.append("chat-b", &record(1, "created")).unwrap_or_else(|e| panic!("{e:#}")); + // The writer is in chat-b's 2 s sync: record 3 waits in memory. + std::thread::sleep(Duration::from_millis(300)); + store.append(CHAT, &record(3, "queued")).unwrap_or_else(|e| panic!("{e:#}")); + store.flush_session(CHAT).unwrap_or_else(|e| panic!("{e:#}")); + let synced = log_files(home.path()); + let crashed = journal.restart(Crash::Os); + crashed.set_sync_delay(Duration::ZERO); + drop(store); + lose_unsynced_log_writes(home.path(), &synced); + let store = open(home.path(), &crashed); + assert_eq!(seqs(store.as_ref(), CHAT), vec![1, 2, 3], "a line flush_session synced is gone"); +} + +/// Finding 2: a folder put back by a rename after its delete (a replacing transfer's rollback, +/// `hub/transfer_in.rs`). +#[test] +fn a_folder_put_back_after_a_delete_is_not_deleted_by_the_replay() { + let home = tempfile::tempdir().unwrap_or_else(|e| panic!("tempdir: {e}")); + let journal = SimStorage::new(22); + let store = open(home.path(), &journal); + store.append(CHAT, &record(1, "created")).unwrap_or_else(|e| panic!("{e:#}")); + store.flush().unwrap_or_else(|e| panic!("{e:#}")); + store.close_log(CHAT); + let sessions = home.path().join("sessions"); + let backup = sessions.join(".chat-a.replacement-backup"); + std::fs::rename(sessions.join(CHAT), &backup).unwrap_or_else(|e| panic!("{e}")); + store.append(CHAT, &record(1, "imported")).unwrap_or_else(|e| panic!("{e:#}")); + store.delete(CHAT).unwrap_or_else(|e| panic!("{e:#}")); + std::fs::rename(&backup, sessions.join(CHAT)).unwrap_or_else(|e| panic!("{e}")); + assert!(store.wait_durable(WAIT).unwrap_or_else(|e| panic!("{e:#}")), "never durable"); + let killed = journal.restart(Crash::Process); + drop(store); + let store = open(home.path(), &killed); + assert_eq!(seqs(store.as_ref(), CHAT), vec![1], "the chat put back was deleted"); +} + +/// Finding 3: a kill -9 while a delete's forget record waits in memory. After the restart the +/// chat is whole or gone, never a new folder with only its last lines. +#[test] +fn a_kill_in_a_delete_leaves_the_chat_whole_or_gone() { + let home = tempfile::tempdir().unwrap_or_else(|e| panic!("tempdir: {e}")); + let journal = SimStorage::new(23); + let store: Arc = Arc::from(open(home.path(), &journal)); + store.append(CHAT, &record(1, "created")).unwrap_or_else(|e| panic!("{e:#}")); + store.flush().unwrap_or_else(|e| panic!("{e:#}")); + store.append(CHAT, &record(2, "queued")).unwrap_or_else(|e| panic!("{e:#}")); + assert!(store.wait_durable(WAIT).unwrap_or_else(|e| panic!("{e:#}")), "never durable"); + journal.set_sync_delay(Duration::from_secs(3)); + store.append("chat-b", &record(1, "created")).unwrap_or_else(|e| panic!("{e:#}")); + // The writer is in chat-b's 3 s sync. + std::thread::sleep(Duration::from_millis(300)); + let deleting = { + let store = Arc::clone(&store); + std::thread::spawn(move || store.delete(CHAT)) + }; + // The folder is gone; the forget record waits in memory. + std::thread::sleep(Duration::from_millis(500)); + let killed = journal.restart(Crash::Process); + killed.set_sync_delay(Duration::ZERO); + let _ = deleting.join(); + drop(store); + let store = open(home.path(), &killed); + let left = seqs(store.as_ref(), CHAT); + assert!(left.is_empty() || left == vec![1, 2], "a deleted chat came back in part: {left:?}"); +} + +/// Finding 4: a delete does not wait for the journal's sync (it held every append of every chat +/// and a Tokio worker for up to 5 s). +#[test] +fn a_delete_does_not_wait_for_the_journal_sync() { + let home = tempfile::tempdir().unwrap_or_else(|e| panic!("tempdir: {e}")); + let journal = SimStorage::new(25); + let store = open(home.path(), &journal); + store.append(CHAT, &record(1, "created")).unwrap_or_else(|e| panic!("{e:#}")); + assert!(store.wait_durable(WAIT).unwrap_or_else(|e| panic!("{e:#}")), "never durable"); + journal.set_sync_delay(Duration::from_secs(3)); + store.append("chat-b", &record(1, "created")).unwrap_or_else(|e| panic!("{e:#}")); + std::thread::sleep(Duration::from_millis(100)); + let started = std::time::Instant::now(); + store.delete(CHAT).unwrap_or_else(|e| panic!("{e:#}")); + let took = started.elapsed(); + journal.set_sync_delay(Duration::ZERO); + assert!(took < Duration::from_secs(1), "the delete waited {took:?} for the journal"); + assert!(!home.path().join("sessions").join(CHAT).exists(), "the chat is still there"); +} + +/// Finding 5: a log sync that fails inside a checkpoint freezes it (a later sync may report +/// success for pages the kernel dropped), and the store asks the daemon to stop. +#[test] +fn a_failed_log_sync_freezes_the_checkpoint() { + use std::os::unix::fs::PermissionsExt; + // SAFETY: geteuid has no preconditions. + if unsafe { libc::geteuid() } == 0 { + eprintln!("skipped: root opens a folder of mode 000"); + return; + } + let home = tempfile::tempdir().unwrap_or_else(|e| panic!("tempdir: {e}")); + let journal = SimStorage::new(26); + let store = open(home.path(), &journal); + store.append(CHAT, &record(1, "created")).unwrap_or_else(|e| panic!("{e:#}")); + store.flush().unwrap_or_else(|e| panic!("{e:#}")); + store.append(CHAT, &record(2, "queued")).unwrap_or_else(|e| panic!("{e:#}")); + let events = home.path().join("sessions").join(CHAT).join("events"); + let set_mode = |mode: u32| { + std::fs::set_permissions(&events, std::fs::Permissions::from_mode(mode)) + .unwrap_or_else(|e| panic!("chmod {}: {e}", events.display())); + }; + set_mode(0o000); + let failed = store.flush(); + set_mode(0o700); + assert!(failed.is_err(), "the checkpoint synced a folder it cannot open"); + assert!(store.flush().is_err(), "a checkpoint after a failed sync went ahead"); + let reason = store.stop_reason().and_then(|reason| reason.borrow().clone()); + assert!(reason.is_some(), "the store did not ask the daemon to stop"); +} + +/// Finding 5, the other way: a checkpoint that keeps failing for another reason (its file cannot +/// be put in place) asks the daemon to stop after a bounded number of tries, so the journal past +/// it does not grow without end. +#[test] +fn a_checkpoint_that_keeps_failing_asks_the_daemon_to_stop() { + let home = tempfile::tempdir().unwrap_or_else(|e| panic!("tempdir: {e}")); + let journal = SimStorage::new(27); + let store = open(home.path(), &journal); + // A non-empty folder where the checkpoint file goes: its rename fails. + let blocker = home.path().join("journal-projected.json"); + std::fs::create_dir_all(blocker.join("in-the-way")).unwrap_or_else(|e| panic!("{e}")); + for seq in 1..=40 { + store.append(CHAT, &record(seq, "queued")).unwrap_or_else(|e| panic!("{e:#}")); + let _ = store.flush(); + } + let reason = store.stop_reason().and_then(|reason| reason.borrow().clone()); + assert!(reason.is_some(), "40 failed checkpoints and the store goes on"); +} + +/// Finding 6: one chat whose log cannot be repaired (a segment that is a link) does not stop the +/// open; its journaled lines are kept beside the logs. +#[test] +fn one_chat_that_cannot_be_repaired_does_not_stop_the_open() { + let home = tempfile::tempdir().unwrap_or_else(|e| panic!("tempdir: {e}")); + let journal = SimStorage::new(28); + let store = open(home.path(), &journal); + store.append(CHAT, &record(1, "created")).unwrap_or_else(|e| panic!("{e:#}")); + store.append("chat-b", &record(1, "created")).unwrap_or_else(|e| panic!("{e:#}")); + store.flush().unwrap_or_else(|e| panic!("{e:#}")); + for seq in 2..=3 { + store.append(CHAT, &record(seq, "queued")).unwrap_or_else(|e| panic!("{e:#}")); + store.append("chat-b", &record(seq, "queued")).unwrap_or_else(|e| panic!("{e:#}")); + } + assert!(store.wait_durable(WAIT).unwrap_or_else(|e| panic!("{e:#}")), "never durable"); + let killed = journal.restart(Crash::Process); + drop(store); + // chat-b's segment becomes a link to a copy of itself: the store never follows one. + let events = home.path().join("sessions").join("chat-b").join("events"); + let segment = events.join("000001.ndjson"); + let copy = home.path().join("elsewhere.ndjson"); + std::fs::copy(&segment, ©).unwrap_or_else(|e| panic!("{e}")); + std::fs::remove_file(&segment).unwrap_or_else(|e| panic!("{e}")); + std::os::unix::fs::symlink(©, &segment).unwrap_or_else(|e| panic!("{e}")); + let store = open(home.path(), &killed); + assert_eq!(seqs(store.as_ref(), CHAT), vec![1, 2, 3]); + let kept: Vec = std::fs::read_dir(home.path().join("journal-unapplied")) + .unwrap_or_else(|e| panic!("journal-unapplied: {e}")) + .flatten() + .flat_map(|entry| std::fs::read(entry.path()).unwrap_or_default()) + .collect(); + let kept_seqs: Vec = kept.split(|b| *b == b'\n').filter_map(record_seq).collect(); + assert_eq!(kept_seqs, vec![2, 3], "chat-b's journaled lines were not kept"); +} + +/// Finding 7: the journal's storage refuses every write under a lost home. +#[test] +fn the_journal_storage_refuses_a_lost_home() { + use cmux_journal::{OpenMode, Storage as _}; + let home = tempfile::tempdir().unwrap_or_else(|e| panic!("tempdir: {e}")); + let lost = home.path().join("journal"); + let fence_root = lost.clone(); + let storage = FencedStorage::new( + Arc::new(SimStorage::new(29)), + Box::new(move |path: &Path| { + anyhow::ensure!( + !path.starts_with(&fence_root), + "{} is under a lost home", + path.display() + ); + Ok(()) + }), + ); + assert!(storage.create_dir_all(&lost).is_err(), "made a folder under a lost home"); + assert!(storage.open(&lost.join("a.seg"), OpenMode::Create).is_err(), "made a file"); + assert!(storage.open(&lost.join("a.seg"), OpenMode::Write).is_err(), "opened a file to write"); + assert!(storage.rename(&lost.join("a"), &lost.join("b")).is_err(), "renamed a file"); + assert!(storage.remove_file(&lost.join("a.seg")).is_err(), "removed a file"); + // Reads and syncs are not writes. + let elsewhere = home.path().join("other"); + assert!(storage.create_dir_all(&elsewhere).is_ok(), "refused outside the lost home"); +} + +/// An OS crash leaves a hole of zeros over two journaled lines, and the log has a line after them +/// that the journal never held: the repair writes the two lines back and keeps the last one. +#[test] +fn an_os_crash_hole_is_repaired_and_the_lines_after_it_kept() { + let home = tempfile::tempdir().unwrap_or_else(|e| panic!("tempdir: {e}")); + let journal = SimStorage::new(30); + let store = open(home.path(), &journal); + for seq in 1..=5 { + store.append(CHAT, &record(seq, "queued")).unwrap_or_else(|e| panic!("{e:#}")); + } + store.flush().unwrap_or_else(|e| panic!("{e:#}")); + for seq in 6..=9 { + store.append(CHAT, &record(seq, "queued")).unwrap_or_else(|e| panic!("{e:#}")); + } + assert!(store.wait_durable(WAIT).unwrap_or_else(|e| panic!("{e:#}")), "never durable"); + let crashed = journal.restart(Crash::Process); + drop(store); + zero_log_lines(home.path(), CHAT, &[6, 7]); + write_log_line(home.path(), CHAT, &record(10, "queued")); + let store = open(home.path(), &crashed); + assert_eq!(seqs(store.as_ref(), CHAT), (1..=10).collect::>()); +} + +/// A crash inside that repair, right after the log was cut: the next open still has every line, +/// the one only the log held too. +#[test] +fn a_crash_inside_a_repair_loses_no_line() { + let home = tempfile::tempdir().unwrap_or_else(|e| panic!("tempdir: {e}")); + let journal = SimStorage::new(31); + let store = open(home.path(), &journal); + for seq in 1..=5 { + store.append(CHAT, &record(seq, "queued")).unwrap_or_else(|e| panic!("{e:#}")); + } + store.flush().unwrap_or_else(|e| panic!("{e:#}")); + for seq in 6..=9 { + store.append(CHAT, &record(seq, "queued")).unwrap_or_else(|e| panic!("{e:#}")); + } + assert!(store.wait_durable(WAIT).unwrap_or_else(|e| panic!("{e:#}")), "never durable"); + let crashed = journal.restart(Crash::Process); + drop(store); + zero_log_lines(home.path(), CHAT, &[6, 7]); + write_log_line(home.path(), CHAT, &record(10, "queued")); + STOP_AFTER_CUT.with(|stop| stop.set(true)); + let stopped = + open_with_journal(&config(), home.path(), Arc::new(crashed.clone()), journal_config()); + STOP_AFTER_CUT.with(|stop| stop.set(false)); + assert!(stopped.is_err(), "the repair did not stop where the test asked"); + drop(stopped); + let again = crashed.restart(Crash::Process); + let store = open(home.path(), &again); + assert_eq!(seqs(store.as_ref(), CHAT), (1..=10).collect::>()); +} + +/// Nit: an unreadable checkpoint fails the open (a replay from the oldest record could undo a +/// retention rewrite). +#[test] +fn an_unreadable_checkpoint_fails_the_open() { + let home = tempfile::tempdir().unwrap_or_else(|e| panic!("tempdir: {e}")); + let journal = SimStorage::new(32); + let store = open(home.path(), &journal); + store.append(CHAT, &record(1, "created")).unwrap_or_else(|e| panic!("{e:#}")); + store.flush().unwrap_or_else(|e| panic!("{e:#}")); + drop(store); + std::fs::write(home.path().join("journal-projected.json"), b"{not json") + .unwrap_or_else(|e| panic!("{e}")); + let reopened = + open_with_journal(&config(), home.path(), Arc::new(journal.clone()), journal_config()); + assert!(reopened.is_err(), "an unreadable checkpoint was read as LSN 0"); +} + +/// Nit: a restart may overlap the old daemon's exit; the open waits a bounded time for the +/// journal's lock. +#[test] +fn the_open_waits_for_the_previous_daemon_to_let_go_of_the_journal() { + let home = tempfile::tempdir().unwrap_or_else(|e| panic!("tempdir: {e}")); + let journal = SimStorage::new(33); + let first = open(home.path(), &journal); + let releaser = std::thread::spawn(move || { + std::thread::sleep(Duration::from_millis(300)); + drop(first); + }); + let second = + open_with_journal(&config(), home.path(), Arc::new(journal.clone()), journal_config()); + let _ = releaser.join(); + assert!(second.is_ok(), "the open did not wait for the lock: {:?}", second.err()); +} + +/// Nit: a checkpoint's temporary file that a crash left is removed at the next open. +#[test] +fn a_crashed_checkpoint_temporary_is_removed_at_open() { + let home = tempfile::tempdir().unwrap_or_else(|e| panic!("tempdir: {e}")); + let journal = SimStorage::new(34); + let stale = home.path().join("journal-projected.json.tmp-99999"); + std::fs::write(&stale, b"{}").unwrap_or_else(|e| panic!("{e}")); + let _store = open(home.path(), &journal); + assert!(!stale.exists(), "a crashed checkpoint's temporary file was left"); +} + +/// Nit: a crash right after a checkpoint is a normal crash: the open reports no damage (the +/// checkpoint was past the journal's durable records, which read as "the journal lost records"). +#[test] +fn a_crash_right_after_a_checkpoint_reports_no_damage() { + let home = tempfile::tempdir().unwrap_or_else(|e| panic!("tempdir: {e}")); + let journal = SimStorage::new(35); + let store = open(home.path(), &journal); + store.append(CHAT, &record(1, "created")).unwrap_or_else(|e| panic!("{e:#}")); + assert!(store.wait_durable(WAIT).unwrap_or_else(|e| panic!("{e:#}")), "never durable"); + journal.set_sync_delay(Duration::from_secs(2)); + store.append("chat-b", &record(1, "created")).unwrap_or_else(|e| panic!("{e:#}")); + // The writer is in chat-b's 2 s sync: record 2 waits in memory through the checkpoint. + std::thread::sleep(Duration::from_millis(300)); + store.append(CHAT, &record(2, "queued")).unwrap_or_else(|e| panic!("{e:#}")); + store.flush().unwrap_or_else(|e| panic!("{e:#}")); + let crashed = journal.restart(Crash::Os); + crashed.set_sync_delay(Duration::ZERO); + drop(store); + let reopened = JournalStore::open(&config(), home.path(), Arc::new(crashed), journal_config()) + .unwrap_or_else(|e| panic!("{e:#}")); + assert!(reopened.opened().damage.is_empty(), "{:?}", reopened.opened()); + assert_eq!(seqs(&reopened, CHAT), vec![1, 2]); +} + +/// Switch off after a crash with it on: the start with the switch off writes the journal's lines +/// that the logs lost back first, so every acknowledged line is there (the catch-up of DESIGN.md +/// 3.6). +#[test] +fn switching_off_after_a_crash_keeps_every_acknowledged_line() { + let home = tempfile::tempdir().unwrap_or_else(|e| panic!("tempdir: {e}")); + let store = super::open(&config(), home.path()).unwrap_or_else(|e| panic!("{e:#}")); + store.append(CHAT, &record(1, "created")).unwrap_or_else(|e| panic!("{e:#}")); + store.flush().unwrap_or_else(|e| panic!("{e:#}")); + let synced = log_files(home.path()); + for seq in 2..=5 { + store.append(CHAT, &record(seq, "queued")).unwrap_or_else(|e| panic!("{e:#}")); + } + assert!(store.wait_durable(WAIT).unwrap_or_else(|e| panic!("{e:#}")), "never durable"); + drop(store); + // An OS crash: the logs lose their writes since the flush; the journal synced its records. + lose_unsynced_log_writes(home.path(), &synced); + let off = open_off(home.path()); + assert_eq!(seqs(off.as_ref(), CHAT), (1..=5).collect::>()); +} diff --git a/cmux-tui/crates/cmux-journal/src/cmux_journal_tests.rs b/cmux-tui/crates/cmux-journal/src/cmux_journal_tests.rs index 532e024d0428..aa44256fca3e 100644 --- a/cmux-tui/crates/cmux-journal/src/cmux_journal_tests.rs +++ b/cmux-tui/crates/cmux-journal/src/cmux_journal_tests.rs @@ -264,6 +264,32 @@ fn appends_past_the_pending_cap_fail_without_waiting() { assert_eq!(journal.append(1, Class::Ack, &too_large), Err(AppendError::TooLarge)); } +/// `check_append` says what `append` would do: a caller that checks under its own lock, writes +/// elsewhere, then appends, is never refused after it wrote (acpmux's journal store). +#[test] +fn check_append_says_what_append_would_do() { + let storage = SimStorage::new(36); + let config = Config { max_pending_bytes: 4096, ..quick() }; + let (journal, _, _) = open_at(&storage, config, 0).unwrap_or_else(|e| panic!("{e}")); + storage.set_sync_delay(Duration::from_millis(500)); + let big = vec![b'a'; 1000]; + let mut refused = false; + for _ in 0..50 { + let checked = journal.check_append(big.len()); + let appended = journal.append(1, Class::Ack, &big).map(|_| ()); + assert_eq!(checked, appended, "check_append and append disagree"); + if appended == Err(AppendError::Backlog) { + refused = true; + break; + } + } + assert!(refused, "the pending cap never refused an append"); + assert_eq!(journal.check_append(crate::MAX_PAYLOAD_LEN + 1), Err(AppendError::TooLarge)); + storage.set_sync_delay(Duration::ZERO); + assert!(journal.seal(WAIT).is_ok()); + assert_eq!(journal.check_append(1), Err(AppendError::Closed)); +} + #[test] fn a_sealed_journal_refuses_appends() { let storage = SimStorage::new(10); diff --git a/cmux-tui/crates/cmux-journal/src/journal.rs b/cmux-tui/crates/cmux-journal/src/journal.rs index 50e9ab9ff321..9295e766e57d 100644 --- a/cmux-tui/crates/cmux-journal/src/journal.rs +++ b/cmux-tui/crates/cmux-journal/src/journal.rs @@ -269,6 +269,15 @@ impl Journal { Ok(lsn) } + /// Whether [`Journal::append`] of a `payload_len`-byte payload would be refused now, and why. + /// A caller that runs this check and its append under one lock of its own, with every append + /// under that lock, is sure of the append unless the writer stops in between + /// ([`AppendError::Closed`]): only appends fill the queue, and only the writer empties it. + pub fn check_append(&self, payload_len: usize) -> Result<(), AppendError> { + let _ = payload_len; + Ok(()) + } + /// Every record up to this LSN is synced (survives a crash of any process and a kernel /// panic). pub fn durable_lsn(&self) -> Lsn { From 8b71b4f7a6909092575afd582f18603c2eb6f54e Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 4 Nov 2026 07:36:36 +0000 Subject: [PATCH 29/52] acpmux: the checkpoint freeze test fails a log sync, not a folder open (red) The freeze guards against a failed fsync (a later fsync of the same file may report success for pages the kernel dropped), not against a folder the store cannot open: a chmod 000 folder is an open error, which the fix retries instead of freezing. The test now fails the log sync itself through a test seam, `FAIL_NEXT_LOG_SYNC`. Red on the code before the fix, like the other review regressions (fleet d8e7037d, 1c22175e, 63604b26 at e030f4fe402: 17 of 17 new acpmux tests and the crate's check_append test fail). --- cmux-tui/crates/acpmux/src/store.rs | 10 ++++++++++ .../acpmux/src/store/journal_store_tests.rs | 17 +++-------------- 2 files changed, 13 insertions(+), 14 deletions(-) diff --git a/cmux-tui/crates/acpmux/src/store.rs b/cmux-tui/crates/acpmux/src/store.rs index c4113a653a1f..651ebad4b5b8 100644 --- a/cmux-tui/crates/acpmux/src/store.rs +++ b/cmux-tui/crates/acpmux/src/store.rs @@ -1400,11 +1400,21 @@ fn first_seq(path: &Path) -> Result> { } } +#[cfg(test)] +thread_local! { + /// Tests: the next `LocalStore::sync_log` on this thread fails, as a failed fsync would. + pub(crate) static FAIL_NEXT_LOG_SYNC: std::cell::Cell = const { std::cell::Cell::new(false) }; +} + impl LocalStore { /// Syncs `id`'s open log segment and its folders, without holding the writers' lock while /// the disk works (the journal store's checkpoint, `store/journal_store.rs`). A segment that /// is not open was synced at its close (`SegmentWriter::close`, which notes a failure). pub(super) fn sync_log(&self, id: &str) -> Result<()> { + #[cfg(test)] + if FAIL_NEXT_LOG_SYNC.with(|fail| fail.replace(false)) { + return Err(std::io::Error::other("a failed sync (a test)").into()); + } let open = { let writers = self.writers.lock_or_recover(); match writers.open.get(id) { diff --git a/cmux-tui/crates/acpmux/src/store/journal_store_tests.rs b/cmux-tui/crates/acpmux/src/store/journal_store_tests.rs index 645b61d56930..754dc351a8c9 100644 --- a/cmux-tui/crates/acpmux/src/store/journal_store_tests.rs +++ b/cmux-tui/crates/acpmux/src/store/journal_store_tests.rs @@ -422,27 +422,16 @@ fn a_delete_does_not_wait_for_the_journal_sync() { /// success for pages the kernel dropped), and the store asks the daemon to stop. #[test] fn a_failed_log_sync_freezes_the_checkpoint() { - use std::os::unix::fs::PermissionsExt; - // SAFETY: geteuid has no preconditions. - if unsafe { libc::geteuid() } == 0 { - eprintln!("skipped: root opens a folder of mode 000"); - return; - } let home = tempfile::tempdir().unwrap_or_else(|e| panic!("tempdir: {e}")); let journal = SimStorage::new(26); let store = open(home.path(), &journal); store.append(CHAT, &record(1, "created")).unwrap_or_else(|e| panic!("{e:#}")); store.flush().unwrap_or_else(|e| panic!("{e:#}")); store.append(CHAT, &record(2, "queued")).unwrap_or_else(|e| panic!("{e:#}")); - let events = home.path().join("sessions").join(CHAT).join("events"); - let set_mode = |mode: u32| { - std::fs::set_permissions(&events, std::fs::Permissions::from_mode(mode)) - .unwrap_or_else(|e| panic!("chmod {}: {e}", events.display())); - }; - set_mode(0o000); + FAIL_NEXT_LOG_SYNC.with(|fail| fail.set(true)); let failed = store.flush(); - set_mode(0o700); - assert!(failed.is_err(), "the checkpoint synced a folder it cannot open"); + FAIL_NEXT_LOG_SYNC.with(|fail| fail.set(false)); + assert!(failed.is_err(), "the checkpoint went on past a failed log sync"); assert!(store.flush().is_err(), "a checkpoint after a failed sync went ahead"); let reason = store.stop_reason().and_then(|reason| reason.borrow().clone()); assert!(reason.is_some(), "the store did not ask the daemon to stop"); From 03fccdef69c57e297eac7dedf1fabf987fdeae0d Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 4 Nov 2026 07:42:35 +0000 Subject: [PATCH 30/52] acpmux: the journal store repairs logs, never rebuilds them (S2 review) Review FIX at ab0e9535d9da (PR #34901 comment 6624439824); the regression tests are the two commits before this one. DESIGN.md 3.3c. - 1a/1b/1c: recovery no longer cuts a chat's log at its first journaled line and writes back only the journal's lines. It merges the log's lines and the journal's by seq: a line in both must be equal, a line only the log has stays, a line only the journal has goes in its place. Only bytes that do not read (a torn tail, a hole of zeros) are dropped. When the merge must change the log before its end, the lines only the log holds from there are journaled and made durable first, so a crash inside the repair loses no line. A log that moved on without the journal (a line in both that differs, lines out of order) is left as it is, and the journal's lines go to journal-unapplied/. - Append writes the log first, then the record, under one lock, after `Journal::check_append` (new in cmux-journal: the append's refusals, asked before any write). So a record always has its log line, and the `cancel` record is gone. - 2/3/4: the replay never deletes a folder and never makes a missing one again (a forget only drops the chat's earlier journaled lines). Delete appends its forget under the lock and removes the folder off it, with no wait for the record. - 5: a failed fsync in a checkpoint freezes it (a folder that does not open is retried); a frozen checkpoint returns an error, and the store asks the daemon to stop for a restart (`Store::stop_reason`, wired in daemon.rs), whose replay writes the logs again. Ten failed checkpoints in a row ask the same, so the journal past the checkpoint stays bounded. - 6: an error in one chat keeps its journaled lines in journal-unapplied/ and the open goes on. - 7: the journal's storage refuses writes under a lost home (`FencedStorage`, `config::refuse_lost_home`). - With store.journal off, a journal left by a run with it on is replayed, checkpointed and retired first (`catch_up`). - Nits: the checkpoint is never past the journal's durable end (no `Ahead` after a normal crash); damage is cleared after the next checkpoint; a kind-index copy goes before its segment is cut; an unreadable checkpoint fails the open; the open waits up to 5 s for the journal lock; crashed checkpoint temporaries are removed; `on_written` leaves cmux-journal (no user); README lists store.journal. --- cmux-tui/crates/acpmux/README.md | 8 + cmux-tui/crates/acpmux/src/daemon.rs | 22 + cmux-tui/crates/acpmux/src/store.rs | 37 +- .../crates/acpmux/src/store/journal_store.rs | 706 ++++++++++++++---- .../cmux-journal/src/cmux_journal_tests.rs | 17 - cmux-tui/crates/cmux-journal/src/journal.rs | 49 +- cmux-tui/crates/cmux-journal/src/writer.rs | 24 +- 7 files changed, 632 insertions(+), 231 deletions(-) diff --git a/cmux-tui/crates/acpmux/README.md b/cmux-tui/crates/acpmux/README.md index 6bd8512def36..8edb60931a5a 100644 --- a/cmux-tui/crates/acpmux/README.md +++ b/cmux-tui/crates/acpmux/README.md @@ -1465,6 +1465,14 @@ When no config exists, harnesses are imported from `~/.acpx/config.json` (its `a would free; `--archive DIR` keeps the old segments under `DIR///events/`; `--older-than-days N` and `--only CHAT` narrow it. Read when the daemon starts. +- `store.journal`: `true` sends every event-log line through one append-only journal in + `journal/` as well (default `false`, a dogfood switch). A line is then durable within one group + commit (an OS crash or a power loss keeps it), and the next start writes back into a chat's log + every line it lost. `journal-projected.json` names how far the logs are synced. Lines the start + could not write into a log that changed without the journal (a run with the switch off after a + crash, or an older build) go to `journal-unapplied/`. Turning it off is safe: the next start + writes the journal's lines into the logs first. A failed log sync stops the daemon for a restart, + which writes the logs again. Read when the daemon starts. - `websocket`: optional. The same protocol over WebSocket text frames, for remote clients such as a dashboard or a Durable Object. Put it behind a tunnel; the token is a bearer token. diff --git a/cmux-tui/crates/acpmux/src/daemon.rs b/cmux-tui/crates/acpmux/src/daemon.rs index 498589eebb00..98aab5f8b55d 100644 --- a/cmux-tui/crates/acpmux/src/daemon.rs +++ b/cmux-tui/crates/acpmux/src/daemon.rs @@ -698,6 +698,9 @@ pub async fn run(opts: DaemonOptions) -> Result<()> { #[cfg(all(unix, feature = "tailnet"))] let tailnet = crate::tailnet::embedded_service::Service::spawn(hub.clone()); + // The store asks for a restart when it cannot go on safely (`Store::stop_reason`): a frozen + // event journal checkpoint, whose logs only the next start's replay writes again. + let mut store_stop = std::pin::pin!(store_asks_to_stop(hub.store.stop_reason())); let shutdown = async { // Every stop names its cause in the log (`app_orphan` and `unused` name their own). #[cfg(unix)] @@ -714,6 +717,10 @@ pub async fn run(opts: DaemonOptions) -> Result<()> { }, _ = next_signal(int.as_mut()) => tracing::warn!("stopping: SIGINT"), _ = hub.shutdown.notified() => tracing::warn!("stopping: a client asked it to"), + // A restart handoff: the next start repairs what the store could not. + () = &mut store_stop => { + hub.restart_requested.store(true, std::sync::atomic::Ordering::SeqCst); + }, // No build of its app is left to adopt its agents, so they end with it. An app // that only quit or was updated keeps them, as its Quit does // (`daemon/app_orphan.rs`). @@ -767,6 +774,9 @@ pub async fn run(opts: DaemonOptions) -> Result<()> { tokio::select! { _ = ctrl_c => {}, _ = hub.shutdown.notified() => {}, + () = &mut store_stop => { + hub.restart_requested.store(true, std::sync::atomic::Ordering::SeqCst); + }, // No build of its app is left to adopt its agents, so they end with it. An app // that only quit or was updated keeps them, as its Quit does // (`daemon/app_orphan.rs`). @@ -920,6 +930,18 @@ impl Drop for AbortOnDrop { } } +/// Resolves, with the reason logged, once the store asks the daemon to stop +/// (`Store::stop_reason`); never for a store that does not ask, or once the store is gone. +async fn store_asks_to_stop(reason: Option>>) { + let Some(mut reason) = reason else { return std::future::pending().await }; + match reason.wait_for(Option::is_some).await { + Ok(why) => { + tracing::error!("stopping for a restart: {}", why.as_deref().unwrap_or_default()); + } + Err(_) => std::future::pending().await, + } +} + /// The next delivery of a registered signal; never, if registration failed. #[cfg(unix)] async fn next_signal(stream: Option<&mut tokio::signal::unix::Signal>) { diff --git a/cmux-tui/crates/acpmux/src/store.rs b/cmux-tui/crates/acpmux/src/store.rs index 651ebad4b5b8..8f8882bded0f 100644 --- a/cmux-tui/crates/acpmux/src/store.rs +++ b/cmux-tui/crates/acpmux/src/store.rs @@ -994,13 +994,16 @@ pub fn open(config: &StoreConfig, root: &Path) -> Result> { StoreMode::Local if config.journal => Box::new(journal_store::JournalStore::open( config, root, - std::sync::Arc::new(journal_store::FencedStorage::new( - std::sync::Arc::new(cmux_journal::FsStorage), - Box::new(crate::config::refuse_lost_home), - )), + std::sync::Arc::new(journal_store::FencedStorage::daemon()), cmux_journal::Config::default(), )?), - StoreMode::Local => Box::new(LocalStore::new(root.join("sessions"), config.segment_bytes)?), + StoreMode::Local => { + // A journal a run with `store.journal` on left: its lines go to the logs first. + if let Err(error) = journal_store::catch_up(config, root) { + tracing::error!("the event journal's catch-up with store.journal off: {error:#}"); + } + Box::new(LocalStore::new(root.join("sessions"), config.segment_bytes)?) + } }) } @@ -1413,7 +1416,7 @@ impl LocalStore { pub(super) fn sync_log(&self, id: &str) -> Result<()> { #[cfg(test)] if FAIL_NEXT_LOG_SYNC.with(|fail| fail.replace(false)) { - return Err(std::io::Error::other("a failed sync (a test)").into()); + return Err(SyncFailed(std::io::Error::other("a failed sync (a test)")).into()); } let open = { let writers = self.writers.lock_or_recover(); @@ -1423,7 +1426,7 @@ impl LocalStore { } }; if let Some(file) = open { - file.sync_all()?; + file.sync_all().map_err(SyncFailed)?; } let session = self.dir(id); sync_directory(&session.join("events"))?; @@ -1793,6 +1796,24 @@ impl Store for LocalStore { } } +/// A sync call (`fsync`) that failed, as opposed to an error before it (a folder that does not +/// open): what it was to make durable may be lost, and a later sync of the same file may report +/// success without it (the journal store then keeps every line until a restart). +#[derive(Debug)] +pub(crate) struct SyncFailed(pub(crate) std::io::Error); + +impl std::fmt::Display for SyncFailed { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "sync failed: {}", self.0) + } +} + +impl std::error::Error for SyncFailed { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + Some(&self.0) + } +} + /// Sync a directory entry after a file or subdirectory was created or replaced. #[cfg(unix)] fn sync_directory(path: &Path) -> Result<()> { @@ -1801,7 +1822,7 @@ fn sync_directory(path: &Path) -> Result<()> { .read(true) .custom_flags(libc::O_DIRECTORY | libc::O_NOFOLLOW) .open(path)?; - dir.sync_all()?; + dir.sync_all().map_err(SyncFailed)?; Ok(()) } diff --git a/cmux-tui/crates/acpmux/src/store/journal_store.rs b/cmux-tui/crates/acpmux/src/store/journal_store.rs index 7ce477196677..0c7b0597a901 100644 --- a/cmux-tui/crates/acpmux/src/store/journal_store.rs +++ b/cmux-tui/crates/acpmux/src/store/journal_store.rs @@ -1,53 +1,69 @@ //! The event log through one append-only journal (`store.journal`, R-DURABILITY S2; -//! `handoff/rewrites/R-DURABILITY/DESIGN.md` 3.3b). +//! `handoff/rewrites/R-DURABILITY/DESIGN.md` 3.3b and 3.3c). //! //! Without it, no event-log append is synced: a line a client was told about reaches the disk at //! the next `flush` (shutdown), so an OS crash or a power loss takes back every line since. With -//! it, each append is one record in `/journal/` (`cmux-journal`: group commit, one sync per -//! batch, a power loss takes back at most the last second, always a suffix), and then the same -//! line goes to the chat's `events/*.ndjson`, as before. Readers, segments and every file format +//! it, each line goes to the chat's `events/*.ndjson` as before, and then one record with the same +//! line goes to `/journal/` (`cmux-journal`: group commit, one sync per batch, a power loss +//! takes back at most the last second, always a suffix). Readers, segments and every file format //! stay as they are: the logs are the journal's projection, read as before. //! -//! - Order: one lock holds from the journal append to the log write, so for each chat the log's -//! lines are in LSN order, and a log line always has its record. A line the log refuses gets a -//! `cancel` record, so no replay writes it. +//! - Order: one lock holds from the journal's check to its append, with the log write between, +//! so for each chat the log's lines are in LSN order. The journal's refusals (closed, behind) +//! come before the log write and a log's refusal comes before the record, so every record's +//! line went to the log. A log line may lack its record: a crash between the two, or a record +//! still in memory at a kill. //! - Checkpoint (`/journal-projected.json`, the journal's compaction point): every 30 s while //! the journal grew, past 64 MiB of it, at `flush` (shutdown) and before a retention rewrite. It -//! syncs every log written since the last one and the sessions folder, then writes the LSN -//! that every record up to has its log effect, durably, then retires the journal segments -//! before it. A log sync that failed on its own (`LocalStore::sync_failed`) freezes the -//! checkpoint: the journal then keeps every line until a restart writes them again. -//! - Recovery (open): the records after the checkpoint are read. For each chat with lines among -//! them, its log is cut right after its last readable line below the first such line's `seq` -//! (lines before it were synced by the checkpoint; after it are torn writes, lines an OS crash -//! dropped, or lines the journal holds), and the journal's lines are written again, in order. -//! A chat with a `forget` record (a delete) is deleted again. Exact and idempotent: a second -//! crash in the middle is repaired by the next open the same way. -//! - Delete: a `forget` record, then the folder's removal, then a wait (bounded) until the record -//! is durable, so a crash never brings a deleted chat back. +//! syncs every log written since the last one and the sessions folder, then writes, durably, an +//! LSN at or before the journal's durable end that every record up to has its log effect, then +//! retires the journal segments before it. A failed log sync (in it, or at a log's close) +//! freezes it: a later sync may report success for pages the kernel dropped, so the journal +//! keeps every line and the store asks the daemon to stop ([`Store::stop_reason`]); the next +//! start's replay writes the logs again. Ten failed checkpoints in a row ask the same. +//! - Recovery (open) repairs each log and never deletes a whole line or a folder: see +//! [`Inner::repair`]. A chat whose folder is gone is not written again. A log that moved on +//! without the journal (a run with the switch off, or an older build, after a crash) is left as +//! it is, and the journal's lines for it go to `/journal-unapplied/`, as do the lines of a +//! chat whose log cannot be read or written. A crash in the middle is repaired by the next open +//! the same way. +//! - Delete: a `forget` record (the replay then drops the chat's earlier journaled lines, so a +//! chat made again under the same id gets only its own), then the folder's removal, off the +//! appends' lock and without waiting for the record. +//! - With the switch off, [`catch_up`] runs this recovery once on a journal left by a run with it +//! on, so every acknowledged line is in the logs before they are written alone. //! //! Not yet (S2c): replies and events do not wait for their records to be durable, and the daemon //! does not stop when the journal closes (a panic on its writer). Session metadata is S3. use super::*; -use cmux_journal::{AppendError, Class, Journal, Lsn, Record}; +use cmux_journal::{AppendError, Class, Damage, Journal, Lsn, OpenError, Record}; use std::sync::atomic::Ordering as AtomicOrdering; use std::sync::{Arc, Condvar}; -use std::time::Duration; +use std::time::{Duration, Instant}; -/// The journal record kinds. +/// The journal record kinds. (Kind 3, a `cancel` of a line the log refused, was used before the +/// review of #34901 and never shipped: a refused line is never journaled now.) const KIND_LOG: u16 = 1; const KIND_FORGET: u16 = 2; -const KIND_CANCEL: u16 = 3; +const JOURNAL_DIR: &str = "journal"; const CHECKPOINT_NAME: &str = "journal-projected.json"; const CHECKPOINT_SCHEMA: &str = "acpmux.journal-projected.v1"; +/// Where the journaled lines of a log that recovery leaves as it is go. +const UNAPPLIED_DIR: &str = "journal-unapplied"; const CHECKPOINT_EVERY: Duration = Duration::from_secs(30); /// Journal bytes past the last checkpoint that wake the checkpointer at once. const CHECKPOINT_BYTES: u64 = 64 * 1024 * 1024; -/// The longest a seal waits for the journal's last full sync, and a delete for its record. +/// Checkpoints in a row that fail before the store asks the daemon to stop: the journal past the +/// last one (what the next open holds in memory) must not grow without end. +const CHECKPOINT_FAILURES_TO_STOP: u32 = 10; +/// The longest a seal waits for the journal's last full sync. const SEAL_WAIT: Duration = Duration::from_secs(5); -const FORGET_WAIT: Duration = Duration::from_secs(5); +/// The longest an open waits for the journal's lock: a restart overlaps the old daemon's exit. +const LOCK_WAIT: Duration = Duration::from_secs(5); +/// The longest a repair waits for the lines it journals before it cuts a log. +const REPAIR_WAIT: Duration = Duration::from_secs(60); pub struct JournalStore { inner: Arc, @@ -59,7 +75,7 @@ pub struct JournalStore { /// The journal's storage, refusing every write under a lost home: `fence` (in the daemon, /// `config::refuse_lost_home`) is asked before each operation that makes, writes, renames or -/// removes a file or folder. +/// removes a file or folder. Reads, syncs and the lock are not writes. pub(super) struct FencedStorage { inner: Arc, fence: Box Result<()> + Send + Sync>, @@ -72,11 +88,20 @@ impl FencedStorage { ) -> Self { Self { inner, fence } } + + /// The daemon's: the real file system behind `config::refuse_lost_home`. + pub(super) fn daemon() -> Self { + Self::new(Arc::new(cmux_journal::FsStorage), Box::new(crate::config::refuse_lost_home)) + } + + fn check(&self, path: &Path) -> std::io::Result<()> { + (self.fence)(path).map_err(|error| std::io::Error::other(format!("{error:#}"))) + } } impl cmux_journal::Storage for FencedStorage { fn create_dir_all(&self, dir: &Path) -> std::io::Result<()> { - let _ = &self.fence; + self.check(dir)?; self.inner.create_dir_all(dir) } fn list(&self, dir: &Path) -> std::io::Result> { @@ -87,12 +112,18 @@ impl cmux_journal::Storage for FencedStorage { path: &Path, mode: cmux_journal::OpenMode, ) -> std::io::Result> { + if mode != cmux_journal::OpenMode::Read { + self.check(path)?; + } self.inner.open(path, mode) } fn rename(&self, from: &Path, to: &Path) -> std::io::Result<()> { + self.check(from)?; + self.check(to)?; self.inner.rename(from, to) } fn remove_file(&self, path: &Path) -> std::io::Result<()> { + self.check(path)?; self.inner.remove_file(path) } fn sync_dir(&self, dir: &Path) -> std::io::Result<()> { @@ -105,8 +136,8 @@ impl cmux_journal::Storage for FencedStorage { #[cfg(test)] thread_local! { - /// Tests: a repair on this thread stops with an error right after it cut a log, as a crash - /// there would. + /// Tests: a repair on this thread stops the open with an error right after it cut a log, as a + /// crash there would. pub(super) static STOP_AFTER_CUT: std::cell::Cell = const { std::cell::Cell::new(false) }; } @@ -119,12 +150,14 @@ struct Inner { checkpointing: Mutex<()>, timer: Mutex, wake: Condvar, + /// Why the daemon must stop (`Store::stop_reason`). + stop: tokio::sync::watch::Sender>, } #[derive(Default)] struct Order { /// Every journal record up to this LSN has its log effect done (a line written, a chat - /// deleted) or none to do (a cancel). + /// deleted). applied: Lsn, /// The LSN of the last checkpoint. checkpointed: Lsn, @@ -134,6 +167,10 @@ struct Order { bytes: u64, /// A log sync failed: the checkpoint stays until a restart writes the logs again. frozen: bool, + /// Checkpoints in a row that failed. + failures: u32, + /// The open reported damage: cleared (`Journal::clear_damage`) after the next checkpoint. + damage: bool, } #[derive(Default)] @@ -150,30 +187,38 @@ impl JournalStore { storage: Arc, journal_config: cmux_journal::Config, ) -> Result { + remove_checkpoint_temporaries(home); let local = LocalStore::new(home.join("sessions"), config.segment_bytes)?; - let from = read_checkpoint(home); - let mut replay = Replay::default(); - let (journal, report) = - Journal::open(storage, &home.join("journal"), journal_config, from, &mut |record| { - replay.take(record) - }) - .map_err(|error| anyhow::anyhow!("open the event journal: {error}"))?; + let from = read_checkpoint(home)?; + let (journal, report, replay) = + open_journal(storage, &home.join(JOURNAL_DIR), journal_config, from)?; for damage in &report.damage { - tracing::error!(?damage, "the event journal could not read some records"); + match damage { + // The checkpoint is never past the journal's durable end, so only a journal that + // lost synced records or a checkpoint file put back from elsewhere says this. + Damage::Ahead { .. } => { + tracing::warn!(?damage, "the event journal ends before its checkpoint"); + } + _ => tracing::error!(?damage, "the event journal could not read some records"), + } } + let (stop, _) = tokio::sync::watch::channel(None); let inner = Arc::new(Inner { local, journal, home: home.to_path_buf(), - order: Mutex::new(Order { checkpointed: from, ..Order::default() }), + order: Mutex::new(Order { + checkpointed: from, + damage: !report.damage.is_empty(), + ..Order::default() + }), checkpointing: Mutex::new(()), timer: Mutex::new(Timer::default()), wake: Condvar::new(), + stop, }); inner.apply(replay)?; - if inner.journal.last_lsn() > from - && let Err(error) = inner.checkpoint() - { + if let Err(error) = inner.checkpoint() { tracing::warn!("event journal checkpoint after the replay: {error:#}"); } let worker = Arc::clone(&inner); @@ -196,17 +241,50 @@ impl JournalStore { } } -impl Drop for JournalStore { - /// Stops the checkpointer. No checkpoint here: a drop is not a shutdown (`flush` is). - fn drop(&mut self) { - self.inner.timer.lock_or_recover().stop = true; - self.inner.wake.notify_all(); - if let Some(handle) = self.checkpointer.lock_or_recover().take() { - let _ = handle.join(); +/// Opens the journal, replaying the records after `from`; waits up to [`LOCK_WAIT`] while another +/// process holds it (the daemon before a restart, still exiting). +fn open_journal( + storage: Arc, + dir: &Path, + config: cmux_journal::Config, + from: Lsn, +) -> Result<(Journal, cmux_journal::OpenReport, Replay)> { + let deadline = Instant::now() + LOCK_WAIT; + loop { + let mut replay = Replay::default(); + let opened = + Journal::open(Arc::clone(&storage), dir, config.clone(), from, &mut |record| { + replay.take(record) + }); + match opened { + Ok((journal, report)) => return Ok((journal, report, replay)), + Err(OpenError::Locked) if Instant::now() < deadline => { + std::thread::sleep(Duration::from_millis(50)); + } + Err(error) => anyhow::bail!("open the event journal: {error}"), } } } +/// With `store.journal` off: a journal that a run with it on left behind is replayed into the logs, +/// checkpointed and retired first, so a crash with the switch on and a start with it off keep +/// every acknowledged line, and turning it on again replays nothing old (DESIGN.md 3.3c, the +/// catch-up of 3.6). Nothing to do when there is no journal. +pub(super) fn catch_up(config: &StoreConfig, home: &Path) -> Result<()> { + if std::fs::symlink_metadata(home.join(JOURNAL_DIR)).is_err() { + return Ok(()); + } + let store = JournalStore::open( + config, + home, + Arc::new(FencedStorage::daemon()), + cmux_journal::Config::default(), + )?; + store.inner.checkpoint()?; + Store::seal(&store); + Ok(()) +} + // ---------------------------------------------------------------- records fn encode_log(id: &str, line: &[u8]) -> Vec { @@ -247,26 +325,39 @@ struct Checkpoint { lsn: Lsn, } -/// The checkpoint's LSN; 0 when there is none or it does not read (the replay then starts at the -/// journal's oldest record, which rewrites lines the logs already hold, the same bytes). -fn read_checkpoint(home: &Path) -> Lsn { +/// The checkpoint's LSN; 0 when there is none. One that does not read fails the open: a replay +/// from the journal's oldest record could undo a retention rewrite. +fn read_checkpoint(home: &Path) -> Result { let path = home.join(CHECKPOINT_NAME); let bytes = match std::fs::read(&path) { Ok(bytes) => bytes, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => return 0, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(0), Err(error) => { - tracing::warn!("read {}: {error}", path.display()); - return 0; + anyhow::bail!("read the event journal's checkpoint {}: {error}", path.display()) } }; match serde_json::from_slice::(&bytes) { - Ok(checkpoint) if checkpoint.schema == CHECKPOINT_SCHEMA => checkpoint.lsn, - _ => { - tracing::warn!( - "{} does not read; the replay starts at the oldest record", - path.display() - ); - 0 + Ok(checkpoint) if checkpoint.schema == CHECKPOINT_SCHEMA => Ok(checkpoint.lsn), + _ => anyhow::bail!( + "the event journal's checkpoint {} does not read; move it aside to replay the whole journal", + path.display() + ), + } +} + +/// Removes the temporary files of checkpoints a crash interrupted (`write_checkpoint`). +fn remove_checkpoint_temporaries(home: &Path) { + let Ok(entries) = std::fs::read_dir(home) else { return }; + let prefix = format!("{CHECKPOINT_NAME}.tmp-"); + for entry in entries.flatten() { + if !entry.file_name().to_string_lossy().starts_with(&prefix) { + continue; + } + let path = entry.path(); + if let Err(error) = crate::config::refuse_lost_home(&path) + .and_then(|()| std::fs::remove_file(&path).map_err(anyhow::Error::from)) + { + tracing::warn!("remove {}: {error:#}", path.display()); } } } @@ -298,15 +389,7 @@ fn write_checkpoint(home: &Path, lsn: Lsn) -> Result<()> { /// The records after the checkpoint, by chat. #[derive(Default)] struct Replay { - chats: BTreeMap, - cancelled: BTreeSet, -} - -#[derive(Default)] -struct ChatReplay { - /// Its lines after its last `forget`, in LSN order. - lines: Vec<(Lsn, Vec)>, - forgotten: bool, + chats: BTreeMap)>>, } impl Replay { @@ -315,22 +398,14 @@ impl Replay { KIND_LOG => { let (chat, line) = decode_log(record.payload) .ok_or_else(|| format!("log record {} does not read", record.lsn))?; - self.chats.entry(chat).or_default().lines.push((record.lsn, line.to_vec())); + self.chats.entry(chat).or_default().push((record.lsn, line.to_vec())); } + // The chat's earlier lines are not its own any more: a chat made again under the same + // id gets only the lines after this. KIND_FORGET => { let chat = std::str::from_utf8(record.payload) .map_err(|_| format!("forget record {} does not read", record.lsn))?; - let entry = self.chats.entry(chat.to_owned()).or_default(); - entry.lines.clear(); - entry.forgotten = true; - } - KIND_CANCEL => { - let lsn = record - .payload - .try_into() - .map(u64::from_le_bytes) - .map_err(|_| format!("cancel record {} does not read", record.lsn))?; - self.cancelled.insert(lsn); + self.chats.entry(chat.to_owned()).or_default().clear(); } other => return Err(format!("unknown event journal record kind {other}")), } @@ -338,112 +413,368 @@ impl Replay { } } +/// What a repair did with one chat's log. +enum Repaired { + /// It holds every journaled line now (nothing was missing, or they were written). + Done, + /// Its folder is gone: the chat was deleted, or lost with its `session.json`. Not written. + Gone, + /// The log moved on without the journal: left as it is. + MovedOn(String), +} + +/// Why a repair stopped. +enum RepairError { + /// The chat's log could not be read or written; `keep` are the lines that only the journal + /// may hold now. + Chat { error: anyhow::Error, keep: Vec> }, + /// Tests: stopped right after its cut, as a crash would. + #[cfg(test)] + Stopped, +} + +/// One piece of a log read back from a repair's start: a whole line with its `seq`, or bytes +/// that do not read (a torn tail, a hole of zeros, a line that does not parse). +struct Piece { + segment: u32, + offset: u64, + line: Option<(u64, Vec)>, +} + impl Inner { - /// Writes the replayed records' effects to the logs ([module docs](self)). + /// Writes the replayed records' effects to the logs ([module docs](self)). One chat's failure + /// keeps its lines in `journal-unapplied/` and the open goes on; only a failure to keep them + /// fails it. fn apply(&self, replay: Replay) -> Result<()> { - let Replay { chats, cancelled } = replay; - for (chat, mut entry) in chats { - // Deleted again, unless lines follow its forget: then the chat was made again under - // the same id (a transfer back), and its folder holds that chat's own files. - if entry.forgotten && entry.lines.is_empty() && self.local.dir(&chat).exists() { - Store::delete(&self.local, &chat) - .with_context(|| format!("delete {chat} again (its forget record)"))?; - } - entry.lines.retain(|(lsn, _)| !cancelled.contains(lsn)); - let Some(from_seq) = entry.lines.first().and_then(|(_, line)| line_seq(line)) else { - if !entry.lines.is_empty() { - tracing::error!(session = %chat, "a journaled line has no seq; not written again"); + for (chat, journaled) in replay.chats { + let Some(first_lsn) = journaled.first().map(|(lsn, _)| *lsn) else { continue }; + match self.repair(&chat, &journaled) { + Ok(Repaired::Done) => { + self.order.lock_or_recover().dirty.insert(chat); } - continue; - }; - self.cut_from(&chat, from_seq) - .with_context(|| format!("cut the log of {chat} for its replay"))?; - for (_, line) in &entry.lines { - self.local.append_line(&chat, line)?; + Ok(Repaired::Gone) => { + tracing::info!(session = %chat, "journaled lines of a chat whose folder is gone: not written again"); + } + Ok(Repaired::MovedOn(why)) => { + let lines: Vec> = journaled.into_iter().map(|(_, line)| line).collect(); + let kept = self.keep_aside(&chat, first_lsn, &lines)?; + tracing::error!(session = %chat, kept = %kept.display(), "the chat's log moved on without the event journal ({why}); left as it is"); + } + Err(RepairError::Chat { error, keep }) => { + let kept = self.keep_aside(&chat, first_lsn, &keep)?; + tracing::error!(session = %chat, kept = %kept.display(), "the chat's log could not be repaired: {error:#}"); + } + #[cfg(test)] + Err(RepairError::Stopped) => anyhow::bail!("stopped after the cut (a test)"), } - self.order.lock_or_recover().dirty.insert(chat); } let mut order = self.order.lock_or_recover(); order.applied = self.journal.last_lsn(); Ok(()) } - /// Cuts `id`'s log right after its last readable line with a `seq` below `from_seq`: later - /// segments go, and the boundary segment is shortened. Each such segment's kind-index copy goes - /// too (a cache, made again by the next scan). - fn cut_from(&self, id: &str, from_seq: u64) -> Result<()> { - let segments = self.local.segments(id)?; - for (index, path) in segments.iter().rev() { - let starts_before = first_seq(path)?.is_some_and(|seq| seq < from_seq); - if !starts_before { - std::fs::remove_file(path).with_context(|| format!("remove {}", path.display()))?; - self.forget_index_copy(id, *index); - continue; + /// Repairs `id`'s log with its journaled lines (`journaled`, in LSN order): it never deletes a + /// whole line or a folder. + /// + /// The log is read from just after its last line with a `seq` below the first journaled one. + /// Its lines and the journal's merge by `seq`: a line in both must be the same bytes; a line + /// only the log has stays; a line only the journal has goes in its place. If that only adds + /// lines after the log's last one, they are appended. Otherwise (bytes that do not read, or a + /// line to put before a later one), the lines that only the log has from that point are + /// journaled and made durable first; then the log is cut there and the merged lines written, + /// so a crash at any step loses no line: the next open repairs the same way. + fn repair( + &self, + id: &str, + journaled: &[(Lsn, Vec)], + ) -> std::result::Result { + let keep_journaled = || journaled.iter().map(|(_, line)| line.clone()).collect::>(); + let chat_error = |error: anyhow::Error| RepairError::Chat { error, keep: keep_journaled() }; + match std::fs::symlink_metadata(self.local.dir(id)) { + Ok(_) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + return Ok(Repaired::Gone); } - let keep = end_of_last_line_before(path, from_seq)?; - let len = std::fs::metadata(path)?.len(); - if keep < len { - truncate_private(path, keep)?; - self.forget_index_copy(id, *index); + Err(error) => return Err(chat_error(error.into())), + } + let mut wanted: BTreeMap = BTreeMap::new(); + for (_, line) in journaled { + let Some(seq) = line_seq(line) else { + return Ok(Repaired::MovedOn("a journaled line has no seq".into())); + }; + if wanted.insert(seq, line).is_some_and(|other| other != line.as_slice()) { + return Ok(Repaired::MovedOn(format!("two journaled lines with seq {seq}"))); + } + } + let Some(&first) = wanted.keys().next() else { return Ok(Repaired::Done) }; + let pieces = self.read_from(id, first).map_err(chat_error)?; + let mut merged = wanted.clone(); + let mut last = None; + for (seq, bytes) in pieces.iter().filter_map(|piece| piece.line.as_ref()) { + if last.is_some_and(|last| *seq <= last) { + return Ok(Repaired::MovedOn(format!("its line {seq} is out of seq order"))); } + last = Some(*seq); + if wanted.get(seq).is_some_and(|held| *held != bytes.as_slice()) { + return Ok(Repaired::MovedOn(format!("its line {seq} differs from the journal's"))); + } + merged.insert(*seq, bytes); + } + let merged: Vec<(u64, &[u8])> = merged.into_iter().collect(); + // How far the log already is the merged lines, and the first piece that is not. + let mut same = 0usize; + let mut cut = None; + for piece in &pieces { + let next = merged.get(same).copied(); + match (&piece.line, next) { + (Some((seq, bytes)), Some((want, line))) + if *seq == want && bytes.as_slice() == line => + { + same = same.saturating_add(1); + } + _ => { + cut = Some((piece.segment, piece.offset)); + break; + } + } + } + let rest = merged.get(same..).unwrap_or_default(); + if let Some((segment, offset)) = cut { + // The lines only the log holds from the cut on: into the journal first. + let only_logged: Vec<&[u8]> = rest + .iter() + .filter(|(seq, _)| !wanted.contains_key(seq)) + .map(|(_, line)| *line) + .collect(); + self.journal_lines(id, &only_logged).map_err(chat_error)?; + let keep_rest = || rest.iter().map(|(_, line)| line.to_vec()).collect::>(); + self.cut_at(id, segment, offset) + .map_err(|error| RepairError::Chat { error, keep: keep_rest() })?; #[cfg(test)] if STOP_AFTER_CUT.with(std::cell::Cell::get) { - anyhow::bail!("stopped after the cut (a test)"); + return Err(RepairError::Stopped); + } + for (_, line) in rest { + self.local + .append_line(id, line) + .map_err(|error| RepairError::Chat { error, keep: keep_rest() })?; } - break; + } else { + for (_, line) in rest { + self.local.append_line(id, line).map_err(chat_error)?; + } + } + Ok(Repaired::Done) + } + + /// Appends `lines` of chat `id` to the journal and waits until they are durable. + fn journal_lines(&self, id: &str, lines: &[&[u8]]) -> Result<()> { + let mut last = None; + for line in lines { + let lsn = self + .journal + .append(KIND_LOG, Class::Ack, &encode_log(id, line)) + .map_err(|error| anyhow::anyhow!("journal a line only the log held: {error}"))?; + last = Some(lsn); + } + if let Some(lsn) = last { + self.journal + .wait_durable(lsn, REPAIR_WAIT) + .map_err(|error| anyhow::anyhow!("journal the lines only the log held: {error}"))?; } Ok(()) } - fn forget_index_copy(&self, id: &str, index: u32) { + /// The pieces of `id`'s log from just after its last whole line with a `seq` below `first`. + fn read_from(&self, id: &str, first: u64) -> Result> { + let segments = self.local.segments(id)?; + // The last segment whose first readable line is below `first`; else the first segment. + let mut start = 0usize; + for (at, (_, path)) in segments.iter().enumerate().rev() { + if first_seq(path)?.is_some_and(|seq| seq < first) { + start = at; + break; + } + } + let mut pieces = Vec::new(); + for (at, (index, path)) in segments.iter().enumerate().skip(start) { + let from = if at == start { end_of_last_line_before(path, first)? } else { 0 }; + read_pieces(path, *index, from, &mut pieces)?; + } + Ok(pieces) + } + + /// Cuts `id`'s log at `offset` of segment `segment`: later segments go, and that one is + /// shortened. The kind-index copy of each segment that changes goes first: a crash between a + /// cut and its copy's removal would leave a copy that a log written back to the same length + /// passes for current. + fn cut_at(&self, id: &str, segment: u32, offset: u64) -> Result<()> { + let segments = self.local.segments(id)?; + for (index, _) in segments.iter().filter(|(index, _)| *index >= segment) { + self.forget_index_copy(id, *index)?; + } + for (index, path) in segments.iter().rev() { + if *index > segment { + crate::config::refuse_lost_home(path)?; + std::fs::remove_file(path).with_context(|| format!("remove {}", path.display()))?; + } else if *index == segment { + truncate_private(path, offset)?; + } + } + Ok(()) + } + + fn forget_index_copy(&self, id: &str, index: u32) -> Result<()> { let copy = self.local.dir(id).join(kind_index::DIR).join(format!("{index:06}.ndjson")); + crate::config::refuse_lost_home(©)?; match std::fs::remove_file(©) { - Ok(()) => {} - Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} - Err(error) => tracing::warn!("remove {}: {error}", copy.display()), + Ok(()) => Ok(()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => { + Err(anyhow::Error::from(error).context(format!("remove {}", copy.display()))) + } } } + /// Keeps `lines` of chat `id` (its journaled lines from LSN `first_lsn` on) in + /// `/journal-unapplied/..ndjson`, owner-only and synced, and returns + /// that path. + fn keep_aside(&self, id: &str, first_lsn: Lsn, lines: &[Vec]) -> Result { + let dir = self.home.join(UNAPPLIED_DIR); + crate::config::refuse_lost_home(&dir)?; + crate::config::private_state_dir(&dir)?; + let name: String = id + .chars() + .map(|c| if c.is_ascii_alphanumeric() || matches!(c, '-' | '_') { c } else { '_' }) + .collect(); + let path = dir.join(format!("{name}.{first_lsn}.ndjson")); + let mut options = OpenOptions::new(); + options.write(true).create(true).truncate(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600).custom_flags(libc::O_NOFOLLOW); + } + let mut file = options.open(&path).with_context(|| format!("write {}", path.display()))?; + for line in lines { + file.write_all(line)?; + } + file.sync_all()?; + drop(file); + sync_directory(&dir)?; + Ok(path) + } + // ------------------------------------------------------------ checkpoints + /// One checkpoint ([module docs](self)); ten failures in a row ask the daemon to stop. fn checkpoint(&self) -> Result<()> { let _one = self.checkpointing.lock_or_recover(); + let result = self.checkpoint_once(); + let mut order = self.order.lock_or_recover(); + match &result { + Ok(()) => order.failures = 0, + Err(error) if !order.frozen => { + order.failures = order.failures.saturating_add(1); + if order.failures >= CHECKPOINT_FAILURES_TO_STOP { + drop(order); + self.ask_to_stop(format!( + "{CHECKPOINT_FAILURES_TO_STOP} event journal checkpoints failed in a row, the last: {error:#}" + )); + } + } + Err(_) => {} + } + result + } + + fn checkpoint_once(&self) -> Result<()> { let (lsn, dirty) = { let mut order = self.order.lock_or_recover(); - if order.frozen || (order.applied <= order.checkpointed && order.dirty.is_empty()) { + anyhow::ensure!( + !order.frozen, + "the event journal's checkpoint is frozen (a log sync failed): a restart writes the logs again" + ); + if order.applied <= order.checkpointed && order.dirty.is_empty() { + let checkpointed = order.checkpointed; + drop(order); + self.retire(checkpointed); return Ok(()); } // A failed checkpoint waits for the timer or another `CHECKPOINT_BYTES`. order.bytes = 0; - (order.applied, std::mem::take(&mut order.dirty)) + // Never past the journal's durable end: a crash right after is a normal one. + (order.applied.min(self.journal.durable_lsn()), std::mem::take(&mut order.dirty)) }; if let Err(error) = self.sync_logs(&dirty) { self.order.lock_or_recover().dirty.extend(dirty); return Err(error); } if self.local.sync_failed.load(AtomicOrdering::SeqCst) { - self.order.lock_or_recover().frozen = true; + self.freeze("an event log failed to sync at its close".into()); anyhow::bail!( "an event log failed to sync: the journal keeps every line until a restart" ); } write_checkpoint(&self.home, lsn)?; - self.order.lock_or_recover().checkpointed = lsn; + let damage = { + let mut order = self.order.lock_or_recover(); + order.checkpointed = order.checkpointed.max(lsn); + std::mem::take(&mut order.damage) + }; + if damage && let Err(error) = self.journal.clear_damage() { + tracing::warn!("clear the event journal's damage report: {error}"); + } + self.retire(lsn); + Ok(()) + } + + fn retire(&self, lsn: Lsn) { if let Err(error) = self.journal.retire_through(lsn) { tracing::warn!("retire the event journal before {lsn}: {error}"); } - Ok(()) } /// Syncs the logs of `dirty` (their open segment and folders) and the sessions folder (a - /// delete's removal, a new chat's folder). + /// delete's removal, a new chat's folder). A chat deleted meanwhile is passed. A failed sync + /// call freezes the checkpoint; another error (a folder that does not open) fails only this + /// checkpoint, and the next one tries again. fn sync_logs(&self, dirty: &BTreeSet) -> Result<()> { + let failed = |error: anyhow::Error| { + if error.chain().any(|cause| cause.is::()) { + self.freeze(format!("an event log failed to sync: {error:#}")); + } + error + }; for chat in dirty { - if self.local.dir(chat).exists() { - self.local.sync_log(chat).with_context(|| format!("sync the log of {chat}"))?; + match self.local.sync_log(chat) { + Ok(()) => {} + Err(error) if is_not_found(&error) => {} + Err(error) => { + return Err(failed(error.context(format!("sync the log of {chat}")))); + } } } - sync_directory(&self.local.root) + sync_directory(&self.local.root).map_err(failed) + } + + /// A log sync failed: no checkpoint until a restart, and the daemon is asked to stop. + fn freeze(&self, why: String) { + self.order.lock_or_recover().frozen = true; + self.ask_to_stop(why); + } + + fn ask_to_stop(&self, why: String) { + tracing::error!( + "{why}: the event journal keeps every line; the daemon stops so that its restart writes the logs again" + ); + self.stop.send_if_modified(|reason| { + if reason.is_some() { + return false; + } + *reason = Some(why); + true + }); } fn run_checkpointer(&self) { @@ -481,6 +812,51 @@ impl Inner { } } +/// Whether `error` comes from a file or folder that is not there. +fn is_not_found(error: &anyhow::Error) -> bool { + error.chain().any(|cause| { + cause + .downcast_ref::() + .is_some_and(|io| io.kind() == std::io::ErrorKind::NotFound) + }) +} + +/// Reads `path` (segment `segment`) from `from` into `out`, piece by piece. +fn read_pieces(path: &Path, segment: u32, from: u64, out: &mut Vec) -> Result<()> { + use std::io::{Seek, SeekFrom}; + let mut file = crate::config::private_state_file(path)?; + file.seek(SeekFrom::Start(from))?; + let mut reader = BufReader::new(file); + let mut line = Vec::new(); + let mut offset = from; + loop { + line.clear(); + let read = reader.read_until(b'\n', &mut line)?; + if read == 0 { + return Ok(()); + } + let len = u64::try_from(read).unwrap_or(u64::MAX); + let whole = line.last() == Some(&b'\n'); + // A hole of zeros an OS crash left runs into the next line: the zeros are one piece, the + // line after them another. + let zeros = line.iter().take_while(|byte| **byte == 0).count(); + let after_zeros = line.get(zeros..).unwrap_or_default(); + if whole && let Some(seq) = line_seq(&line) { + out.push(Piece { segment, offset, line: Some((seq, line.clone())) }); + } else if whole + && zeros > 0 + && let Some(seq) = line_seq(after_zeros) + { + out.push(Piece { segment, offset, line: None }); + let at = offset.saturating_add(u64::try_from(zeros).unwrap_or(u64::MAX)); + out.push(Piece { segment, offset: at, line: Some((seq, after_zeros.to_vec())) }); + } else { + out.push(Piece { segment, offset, line: None }); + } + offset = offset.saturating_add(len); + } +} + /// The offset right after the last whole line of `path` whose `seq` reads and is below /// `from_seq`; 0 when there is none. fn end_of_last_line_before(path: &Path, from_seq: u64) -> Result { @@ -543,6 +919,7 @@ impl Store for JournalStore { fn save(&self, meta: &SessionMeta) -> Result<()> { Store::save(&self.inner.local, meta) } + /// The line to the log first, then its record ([module docs](self)). fn append(&self, id: &str, record: &EventRecord) -> Result<()> { let mut line = serde_json::to_string(record)?; line.push('\n'); @@ -550,28 +927,26 @@ impl Store for JournalStore { let payload = encode_log(id, line.as_bytes()); let inner = &self.inner; let mut order = inner.order.lock_or_recover(); - // After the seal (under the same lock, so exact): refused as the logs refuse it, and the - // journal is not touched. - drop(inner.local.writable()?); - let lsn = - inner.journal.append(KIND_LOG, class, &payload).map_err(JournalStore::journal_error)?; - let written = inner.local.append_line(id, line.as_bytes()); - if written.is_err() - && let Err(error) = inner.journal.append(KIND_CANCEL, Class::Ack, &lsn.to_le_bytes()) - { - tracing::error!(session = %id, "the event journal could not cancel record {lsn}: {error}"); - } - order.applied = inner.journal.last_lsn(); - if written.is_ok() && !order.dirty.contains(id) { + // Every append is under this lock, and only the writer empties the journal's queue, so + // the append below is taken unless the writer stops in between. + inner.journal.check_append(payload.len()).map_err(JournalStore::journal_error)?; + // Refused once the store is sealed (under the same lock, so exact). + inner.local.append_line(id, line.as_bytes())?; + if !order.dirty.contains(id) { order.dirty.insert(id.to_owned()); } + // Refused only if the writer stopped since the check: the line is in the log without its + // record, as after a crash between the two. + let lsn = + inner.journal.append(KIND_LOG, class, &payload).map_err(JournalStore::journal_error)?; + order.applied = lsn; order.bytes = order.bytes.saturating_add(u64::try_from(payload.len()).unwrap_or(u64::MAX)); let full = order.bytes > CHECKPOINT_BYTES && !order.frozen; drop(order); if full { inner.want_checkpoint(); } - written + Ok(()) } fn flush_session(&self, id: &str) -> Result<()> { Store::flush_session(&self.inner.local, id) @@ -637,12 +1012,13 @@ impl Store for JournalStore { Err(error) => Err(anyhow::anyhow!("the event journal: {error}")), } } + fn stop_reason(&self) -> Option>> { + Some(self.inner.stop.subscribe()) + } /// The logs refuse writes, and the journal fully syncs every record (the shutdown's `flush` /// then checkpoints). fn seal(&self) { - // Under the appends' lock: an append either lands in the journal and the logs, or in - // neither (never a record whose line the sealed logs refused, which a later replay - // would write). + // Under the appends' lock: an append lands in the log and the journal, or in neither. let _order = self.inner.order.lock_or_recover(); Store::seal(&self.inner.local); if let Err(error) = self.inner.journal.seal(SEAL_WAIT) { @@ -662,29 +1038,26 @@ impl Store for JournalStore { Store::stage_compaction(&self.inner.local, id, mode) } /// A checkpoint first, so no journal record of the chat is past it: a replay never writes - /// the lines a retention rewrite cut back into its log. + /// the lines a retention rewrite cut back into its log. A frozen or failed checkpoint stops + /// the rewrite. fn commit_compaction(&self, id: &str, staged: Staged) -> Result { self.inner.checkpoint()?; Store::commit_compaction(&self.inner.local, id, staged) } + /// The `forget` record under the appends' lock, then the folder's removal off it, without a + /// wait for the record: the replay never makes a folder that is gone again. fn delete(&self, id: &str) -> Result<()> { let inner = &self.inner; - let forget = { + { let mut order = inner.order.lock_or_recover(); drop(inner.local.writable()?); let lsn = inner .journal .append(KIND_FORGET, Class::Ack, id.as_bytes()) .map_err(JournalStore::journal_error)?; - let removed = Store::delete(&inner.local, id); - order.applied = inner.journal.last_lsn(); - order.dirty.remove(id); - removed.map(|()| lsn) - }?; - if let Err(error) = inner.journal.wait_durable(forget, FORGET_WAIT) { - tracing::warn!(session = %id, "the delete's journal record is not durable yet: {error}"); + order.applied = lsn; } - Ok(()) + Store::delete(&inner.local, id) } fn session_dir(&self, id: &str) -> Option { Store::session_dir(&self.inner.local, id) @@ -699,3 +1072,14 @@ impl Store for JournalStore { Store::briefs_path(&self.inner.local) } } + +impl Drop for JournalStore { + /// Stops the checkpointer. No checkpoint here: a drop is not a shutdown (`flush` is). + fn drop(&mut self) { + self.inner.timer.lock_or_recover().stop = true; + self.inner.wake.notify_all(); + if let Some(handle) = self.checkpointer.lock_or_recover().take() { + let _ = handle.join(); + } + } +} diff --git a/cmux-tui/crates/cmux-journal/src/cmux_journal_tests.rs b/cmux-tui/crates/cmux-journal/src/cmux_journal_tests.rs index aa44256fca3e..9bc989f5220b 100644 --- a/cmux-tui/crates/cmux-journal/src/cmux_journal_tests.rs +++ b/cmux-tui/crates/cmux-journal/src/cmux_journal_tests.rs @@ -189,23 +189,6 @@ fn durable_listeners_see_each_advance_in_order() { assert!(heard.windows(2).all(|pair| pair.first() < pair.get(1)), "{heard:?}"); } -#[test] -fn written_listeners_hear_each_batch_before_its_sync() { - let storage = SimStorage::new(41); - let (journal, _, _) = open(&storage); - storage.set_sync_delay(Duration::from_millis(300)); - let heard = Arc::new(Mutex::new(Vec::new())); - let sink = heard.clone(); - journal.on_written(move |lsn| sink.lock().unwrap_or_else(|e| e.into_inner()).push(lsn)); - let lsn = journal.append(1, Class::Ack, b"x").unwrap_or_else(|e| panic!("{e}")); - let written = || heard.lock().unwrap_or_else(|e| e.into_inner()).clone(); - assert!(eventually(|| written().last() == Some(&lsn)), "{:?}", written()); - // Heard while its 300 ms sync still runs: the record is written, not durable. - assert_eq!(journal.durable_lsn(), 0); - journal.wait_durable(lsn, WAIT).unwrap_or_else(|e| panic!("{e}")); - assert_eq!(written(), vec![lsn], "one call per batch that moved the written LSN"); -} - #[test] fn a_stream_record_is_written_at_once_and_synced_after_its_delay() { let storage = SimStorage::new(6); diff --git a/cmux-tui/crates/cmux-journal/src/journal.rs b/cmux-tui/crates/cmux-journal/src/journal.rs index 9295e766e57d..f4bb3057e88b 100644 --- a/cmux-tui/crates/cmux-journal/src/journal.rs +++ b/cmux-tui/crates/cmux-journal/src/journal.rs @@ -64,8 +64,6 @@ pub(crate) struct Shared { /// Wakes waiters: the durable LSN moved, or the writer ended. pub(crate) durable: Condvar, pub(crate) listeners: Mutex>, - /// Called with the new written LSN after each batch's write, before its sync. - pub(crate) written_listeners: Mutex>, /// The live segments, oldest first: recovery's, then each one the writer makes. pub(crate) segments: Mutex>, } @@ -76,6 +74,23 @@ impl Shared { } } +impl State { + /// Whether a record of `size` bytes (header included) may join the queue now: refused when + /// the journal is closed, or when the queue holds records and would grow past + /// [`Config::max_pending_bytes`] (one record is always taken, however large). + fn admits(&self, config: &Config, size: usize) -> Result<(), AppendError> { + if self.closed { + return Err(AppendError::Closed); + } + if !self.pending.is_empty() + && self.pending.len().saturating_add(size) > config.max_pending_bytes + { + return Err(AppendError::Backlog); + } + Ok(()) + } +} + /// Why an append was refused. None of them waits for the disk. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum AppendError { @@ -221,7 +236,6 @@ impl Journal { wake: Condvar::new(), durable: Condvar::new(), listeners: Mutex::new(Vec::new()), - written_listeners: Mutex::new(Vec::new()), segments: Mutex::new(recovered.live), }); let report = recovered.report; @@ -246,15 +260,8 @@ impl Journal { // The payload's checksum is computed before the lock; the lock covers the LSN and copy. let crc = format::payload_crc(payload); let mut state = self.shared.lock(); - if state.closed { - return Err(AppendError::Closed); - } let size = format::record_len(payload.len()); - if !state.pending.is_empty() - && state.pending.len().saturating_add(size) > self.shared.config.max_pending_bytes - { - return Err(AppendError::Backlog); - } + state.admits(&self.shared.config, size)?; let lsn = state.stats.next_lsn; state.stats.next_lsn = lsn.saturating_add(1); let was_empty = state.pending.is_empty(); @@ -274,8 +281,10 @@ impl Journal { /// under that lock, is sure of the append unless the writer stops in between /// ([`AppendError::Closed`]): only appends fill the queue, and only the writer empties it. pub fn check_append(&self, payload_len: usize) -> Result<(), AppendError> { - let _ = payload_len; - Ok(()) + if payload_len > MAX_PAYLOAD_LEN { + return Err(AppendError::TooLarge); + } + self.shared.lock().admits(&self.shared.config, format::record_len(payload_len)) } /// Every record up to this LSN is synced (survives a crash of any process and a kernel @@ -317,20 +326,6 @@ impl Journal { recovery::clear_damage(self.storage.as_ref(), &self.dir) } - /// Calls `listener` with the new written LSN after each batch is written, before it is synced, - /// on the writer thread. A written record survives a kill of any or all processes (it is in - /// the page cache), not an OS crash. A daemon's projector writes its compaction output from - /// here, so it never holds a record the journal never held. It must return quickly and never - /// wait. A batch written again into a new segment (after a failed sync) is not heard twice. - /// When the writer stops, the listeners are dropped. - pub fn on_written(&self, listener: impl Fn(Lsn) + Send + Sync + 'static) { - self.shared - .written_listeners - .lock() - .unwrap_or_else(PoisonError::into_inner) - .push(Arc::new(listener)); - } - /// Deletes the segments that hold no record after `compacted`, the caller's compaction point: /// every record at or before it is in the caller's own durable output (its projection, /// synced, and the checkpoint that names `compacted`, synced too). A segment is deleted when diff --git a/cmux-tui/crates/cmux-journal/src/writer.rs b/cmux-tui/crates/cmux-journal/src/writer.rs index 7aa44d7003e5..6340ffb5f841 100644 --- a/cmux-tui/crates/cmux-journal/src/writer.rs +++ b/cmux-tui/crates/cmux-journal/src/writer.rs @@ -81,7 +81,6 @@ impl Drop for Ended { fn drop(&mut self) { let panicked = std::thread::panicking(); self.0.listeners.lock().unwrap_or_else(PoisonError::into_inner).clear(); - self.0.written_listeners.lock().unwrap_or_else(PoisonError::into_inner).clear(); { let mut state = self.0.lock(); state.ended = true; @@ -303,14 +302,7 @@ impl Writer { self.unsynced_since = Some(now); } let last = self.retained_last; - let advanced = self.publish(|stats| { - let advanced = last > stats.written_lsn; - stats.written_lsn = stats.written_lsn.max(last); - advanced - }); - if advanced { - self.call(&self.shared.written_listeners, last); - } + self.publish(|stats| stats.written_lsn = last); } let segment_bytes = self.shared.config.segment_bytes; let full_segment = self.active.as_ref().is_some_and(|active| active.size >= segment_bytes); @@ -381,19 +373,15 @@ impl Writer { }); self.shared.durable.notify_all(); if advanced { - self.call(&self.shared.listeners, durable); + let listeners = + self.shared.listeners.lock().unwrap_or_else(PoisonError::into_inner).clone(); + for listener in listeners { + listener(durable); + } } Ok(()) } - /// Calls each listener in `list` with `lsn`, with no lock held. - fn call(&self, list: &std::sync::Mutex>, lsn: Lsn) { - let listeners = list.lock().unwrap_or_else(PoisonError::into_inner).clone(); - for listener in listeners { - listener(lsn); - } - } - /// Appends the marker `active` owes ([module docs](self)). It is not synced here: the next /// sync covers it, and recovery syncs the last segment before it reads it. fn write_marker(&mut self) -> Result<(), Failure> { From 92b52bfe6f0bb0986ae251d405bd2ef71831ddfb Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 4 Nov 2026 07:58:46 +0000 Subject: [PATCH 31/52] acpmux: split the journal store under its size budget (a move) journal_store.rs grew to 1085 lines and 71 functions with the review fix (budget 1000 and 60), and the fix grew store.rs and daemon.rs, which are over budget on the base. A pure move, no behavior change: - store/journal_store/{checkpoint,fenced,repair}.rs: the checkpoint and its thread, `FencedStorage`, the replay and the repair. - store/log_sync.rs: `LocalStore::sync_log`, `SyncFailed` and the test seam `FAIL_NEXT_LOG_SYNC`, out of store.rs (2139 lines on the base, 2113 now). - daemon/stop_causes.rs: `AbortOnDrop`, `next_signal` and `store_asks_to_stop`, out of daemon.rs (2052 on the base, 2046 now). check-no-godfiles.sh --base 51cfdee876b on every changed file: no file over budget grew, and no new file is over budget. --- cmux-tui/crates/acpmux/src/daemon.rs | 36 +- .../crates/acpmux/src/daemon/stop_causes.rs | 36 + cmux-tui/crates/acpmux/src/store.rs | 49 +- .../crates/acpmux/src/store/journal_store.rs | 659 +----------------- .../src/store/journal_store/checkpoint.rs | 225 ++++++ .../acpmux/src/store/journal_store/fenced.rs | 64 ++ .../acpmux/src/store/journal_store/repair.rs | 366 ++++++++++ .../acpmux/src/store/journal_store_tests.rs | 1 + cmux-tui/crates/acpmux/src/store/log_sync.rs | 53 ++ 9 files changed, 762 insertions(+), 727 deletions(-) create mode 100644 cmux-tui/crates/acpmux/src/daemon/stop_causes.rs create mode 100644 cmux-tui/crates/acpmux/src/store/journal_store/checkpoint.rs create mode 100644 cmux-tui/crates/acpmux/src/store/journal_store/fenced.rs create mode 100644 cmux-tui/crates/acpmux/src/store/journal_store/repair.rs create mode 100644 cmux-tui/crates/acpmux/src/store/log_sync.rs diff --git a/cmux-tui/crates/acpmux/src/daemon.rs b/cmux-tui/crates/acpmux/src/daemon.rs index 98aab5f8b55d..58fd8f2a06c0 100644 --- a/cmux-tui/crates/acpmux/src/daemon.rs +++ b/cmux-tui/crates/acpmux/src/daemon.rs @@ -50,6 +50,10 @@ pub(crate) mod owner_exit; mod persist_retry; #[cfg(unix)] mod socket_rebind; +mod stop_causes; +#[cfg(unix)] +use stop_causes::next_signal; +use stop_causes::{AbortOnDrop, store_asks_to_stop}; pub(crate) mod unused_exit; #[cfg(windows)] mod windows; @@ -921,38 +925,6 @@ fn remove_bound_socket(bound: Option<&home_watch::Owned>) { } } -/// Aborts a task when dropped. -struct AbortOnDrop(tokio::task::AbortHandle); - -impl Drop for AbortOnDrop { - fn drop(&mut self) { - self.0.abort(); - } -} - -/// Resolves, with the reason logged, once the store asks the daemon to stop -/// (`Store::stop_reason`); never for a store that does not ask, or once the store is gone. -async fn store_asks_to_stop(reason: Option>>) { - let Some(mut reason) = reason else { return std::future::pending().await }; - match reason.wait_for(Option::is_some).await { - Ok(why) => { - tracing::error!("stopping for a restart: {}", why.as_deref().unwrap_or_default()); - } - Err(_) => std::future::pending().await, - } -} - -/// The next delivery of a registered signal; never, if registration failed. -#[cfg(unix)] -async fn next_signal(stream: Option<&mut tokio::signal::unix::Signal>) { - match stream { - Some(stream) => { - stream.recv().await; - } - None => std::future::pending::<()>().await, - } -} - /// Read the person key (one line, at most 128 bytes) from an inherited /// descriptor and close it. The key is never logged. #[cfg(unix)] diff --git a/cmux-tui/crates/acpmux/src/daemon/stop_causes.rs b/cmux-tui/crates/acpmux/src/daemon/stop_causes.rs new file mode 100644 index 000000000000..1e0a049ca0b2 --- /dev/null +++ b/cmux-tui/crates/acpmux/src/daemon/stop_causes.rs @@ -0,0 +1,36 @@ +//! What stops the daemon besides a client: a signal, and the store asking for a restart (moved +//! out of `daemon.rs`, which is over its size budget). + +/// Aborts a task when dropped. +pub(super) struct AbortOnDrop(pub(super) tokio::task::AbortHandle); + +impl Drop for AbortOnDrop { + fn drop(&mut self) { + self.0.abort(); + } +} + +/// Resolves, with the reason logged, once the store asks the daemon to stop +/// (`Store::stop_reason`); never for a store that does not ask, or once the store is gone. +pub(super) async fn store_asks_to_stop( + reason: Option>>, +) { + let Some(mut reason) = reason else { return std::future::pending().await }; + match reason.wait_for(Option::is_some).await { + Ok(why) => { + tracing::error!("stopping for a restart: {}", why.as_deref().unwrap_or_default()); + } + Err(_) => std::future::pending().await, + } +} + +/// The next delivery of a registered signal; never, if registration failed. +#[cfg(unix)] +pub(super) async fn next_signal(stream: Option<&mut tokio::signal::unix::Signal>) { + match stream { + Some(stream) => { + stream.recv().await; + } + None => std::future::pending::<()>().await, + } +} diff --git a/cmux-tui/crates/acpmux/src/store.rs b/cmux-tui/crates/acpmux/src/store.rs index 8f8882bded0f..f9c7ea3e91c7 100644 --- a/cmux-tui/crates/acpmux/src/store.rs +++ b/cmux-tui/crates/acpmux/src/store.rs @@ -1020,6 +1020,8 @@ pub(crate) fn open_with_journal( } mod journal_store; +mod log_sync; +pub(crate) use log_sync::SyncFailed; mod memory; pub use memory::MemoryStore; @@ -1403,36 +1405,7 @@ fn first_seq(path: &Path) -> Result> { } } -#[cfg(test)] -thread_local! { - /// Tests: the next `LocalStore::sync_log` on this thread fails, as a failed fsync would. - pub(crate) static FAIL_NEXT_LOG_SYNC: std::cell::Cell = const { std::cell::Cell::new(false) }; -} - impl LocalStore { - /// Syncs `id`'s open log segment and its folders, without holding the writers' lock while - /// the disk works (the journal store's checkpoint, `store/journal_store.rs`). A segment that - /// is not open was synced at its close (`SegmentWriter::close`, which notes a failure). - pub(super) fn sync_log(&self, id: &str) -> Result<()> { - #[cfg(test)] - if FAIL_NEXT_LOG_SYNC.with(|fail| fail.replace(false)) { - return Err(SyncFailed(std::io::Error::other("a failed sync (a test)")).into()); - } - let open = { - let writers = self.writers.lock_or_recover(); - match writers.open.get(id) { - Some(writer) => Some(writer.file.try_clone()?), - None => None, - } - }; - if let Some(file) = open { - file.sync_all().map_err(SyncFailed)?; - } - let session = self.dir(id); - sync_directory(&session.join("events"))?; - sync_directory(&session) - } - /// One whole serialized record (`line` ends with its newline) at the end of `id`'s log: the /// body of [`Store::append`], and the journal store's write of a journaled line /// (`store/journal_store.rs`). Refused once the store is sealed. @@ -1796,24 +1769,6 @@ impl Store for LocalStore { } } -/// A sync call (`fsync`) that failed, as opposed to an error before it (a folder that does not -/// open): what it was to make durable may be lost, and a later sync of the same file may report -/// success without it (the journal store then keeps every line until a restart). -#[derive(Debug)] -pub(crate) struct SyncFailed(pub(crate) std::io::Error); - -impl std::fmt::Display for SyncFailed { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "sync failed: {}", self.0) - } -} - -impl std::error::Error for SyncFailed { - fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { - Some(&self.0) - } -} - /// Sync a directory entry after a file or subdirectory was created or replaced. #[cfg(unix)] fn sync_directory(path: &Path) -> Result<()> { diff --git a/cmux-tui/crates/acpmux/src/store/journal_store.rs b/cmux-tui/crates/acpmux/src/store/journal_store.rs index 0c7b0597a901..2d13eed5f523 100644 --- a/cmux-tui/crates/acpmux/src/store/journal_store.rs +++ b/cmux-tui/crates/acpmux/src/store/journal_store.rs @@ -37,11 +37,20 @@ //! does not stop when the journal closes (a panic on its writer). Session metadata is S3. use super::*; -use cmux_journal::{AppendError, Class, Damage, Journal, Lsn, OpenError, Record}; -use std::sync::atomic::Ordering as AtomicOrdering; +use cmux_journal::{AppendError, Class, Damage, Journal, Lsn, OpenError}; use std::sync::{Arc, Condvar}; use std::time::{Duration, Instant}; +mod checkpoint; +mod fenced; +mod repair; + +use checkpoint::{read_checkpoint, remove_checkpoint_temporaries}; +pub(super) use fenced::FencedStorage; +use repair::Replay; +#[cfg(test)] +pub(super) use repair::STOP_AFTER_CUT; + /// The journal record kinds. (Kind 3, a `cancel` of a line the log refused, was used before the /// review of #34901 and never shipped: a refused line is never journaled now.) const KIND_LOG: u16 = 1; @@ -73,74 +82,6 @@ pub struct JournalStore { opened: cmux_journal::OpenReport, } -/// The journal's storage, refusing every write under a lost home: `fence` (in the daemon, -/// `config::refuse_lost_home`) is asked before each operation that makes, writes, renames or -/// removes a file or folder. Reads, syncs and the lock are not writes. -pub(super) struct FencedStorage { - inner: Arc, - fence: Box Result<()> + Send + Sync>, -} - -impl FencedStorage { - pub(super) fn new( - inner: Arc, - fence: Box Result<()> + Send + Sync>, - ) -> Self { - Self { inner, fence } - } - - /// The daemon's: the real file system behind `config::refuse_lost_home`. - pub(super) fn daemon() -> Self { - Self::new(Arc::new(cmux_journal::FsStorage), Box::new(crate::config::refuse_lost_home)) - } - - fn check(&self, path: &Path) -> std::io::Result<()> { - (self.fence)(path).map_err(|error| std::io::Error::other(format!("{error:#}"))) - } -} - -impl cmux_journal::Storage for FencedStorage { - fn create_dir_all(&self, dir: &Path) -> std::io::Result<()> { - self.check(dir)?; - self.inner.create_dir_all(dir) - } - fn list(&self, dir: &Path) -> std::io::Result> { - self.inner.list(dir) - } - fn open( - &self, - path: &Path, - mode: cmux_journal::OpenMode, - ) -> std::io::Result> { - if mode != cmux_journal::OpenMode::Read { - self.check(path)?; - } - self.inner.open(path, mode) - } - fn rename(&self, from: &Path, to: &Path) -> std::io::Result<()> { - self.check(from)?; - self.check(to)?; - self.inner.rename(from, to) - } - fn remove_file(&self, path: &Path) -> std::io::Result<()> { - self.check(path)?; - self.inner.remove_file(path) - } - fn sync_dir(&self, dir: &Path) -> std::io::Result<()> { - self.inner.sync_dir(dir) - } - fn try_lock(&self, path: &Path) -> std::io::Result>> { - self.inner.try_lock(path) - } -} - -#[cfg(test)] -thread_local! { - /// Tests: a repair on this thread stops the open with an error right after it cut a log, as a - /// crash there would. - pub(super) static STOP_AFTER_CUT: std::cell::Cell = const { std::cell::Cell::new(false) }; -} - struct Inner { local: LocalStore, journal: Journal, @@ -317,584 +258,6 @@ fn line_seq(line: &[u8]) -> Option { record_seq(line).or_else(|| parse_record(line).ok().map(|record| record.seq)) } -// ---------------------------------------------------------------- the checkpoint file - -#[derive(Serialize, Deserialize)] -struct Checkpoint { - schema: String, - lsn: Lsn, -} - -/// The checkpoint's LSN; 0 when there is none. One that does not read fails the open: a replay -/// from the journal's oldest record could undo a retention rewrite. -fn read_checkpoint(home: &Path) -> Result { - let path = home.join(CHECKPOINT_NAME); - let bytes = match std::fs::read(&path) { - Ok(bytes) => bytes, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(0), - Err(error) => { - anyhow::bail!("read the event journal's checkpoint {}: {error}", path.display()) - } - }; - match serde_json::from_slice::(&bytes) { - Ok(checkpoint) if checkpoint.schema == CHECKPOINT_SCHEMA => Ok(checkpoint.lsn), - _ => anyhow::bail!( - "the event journal's checkpoint {} does not read; move it aside to replay the whole journal", - path.display() - ), - } -} - -/// Removes the temporary files of checkpoints a crash interrupted (`write_checkpoint`). -fn remove_checkpoint_temporaries(home: &Path) { - let Ok(entries) = std::fs::read_dir(home) else { return }; - let prefix = format!("{CHECKPOINT_NAME}.tmp-"); - for entry in entries.flatten() { - if !entry.file_name().to_string_lossy().starts_with(&prefix) { - continue; - } - let path = entry.path(); - if let Err(error) = crate::config::refuse_lost_home(&path) - .and_then(|()| std::fs::remove_file(&path).map_err(anyhow::Error::from)) - { - tracing::warn!("remove {}: {error:#}", path.display()); - } - } -} - -/// Writes the checkpoint whole under a temporary name, syncs it, renames it into place and syncs -/// the home folder; an error at any step (the folder's sync too) is returned. -fn write_checkpoint(home: &Path, lsn: Lsn) -> Result<()> { - let path = home.join(CHECKPOINT_NAME); - crate::config::refuse_lost_home(&path)?; - let tmp = home.join(format!("{CHECKPOINT_NAME}.tmp-{}", std::process::id())); - let bytes = serde_json::to_vec(&Checkpoint { schema: CHECKPOINT_SCHEMA.into(), lsn })?; - let mut options = OpenOptions::new(); - options.write(true).create(true).truncate(true); - #[cfg(unix)] - { - use std::os::unix::fs::OpenOptionsExt; - options.mode(0o600).custom_flags(libc::O_NOFOLLOW); - } - let mut file = options.open(&tmp).with_context(|| format!("write {}", tmp.display()))?; - file.write_all(&bytes)?; - file.sync_all()?; - drop(file); - std::fs::rename(&tmp, &path).with_context(|| format!("rename to {}", path.display()))?; - sync_directory(home) -} - -// ---------------------------------------------------------------- recovery - -/// The records after the checkpoint, by chat. -#[derive(Default)] -struct Replay { - chats: BTreeMap)>>, -} - -impl Replay { - fn take(&mut self, record: Record<'_>) -> std::result::Result<(), String> { - match record.kind { - KIND_LOG => { - let (chat, line) = decode_log(record.payload) - .ok_or_else(|| format!("log record {} does not read", record.lsn))?; - self.chats.entry(chat).or_default().push((record.lsn, line.to_vec())); - } - // The chat's earlier lines are not its own any more: a chat made again under the same - // id gets only the lines after this. - KIND_FORGET => { - let chat = std::str::from_utf8(record.payload) - .map_err(|_| format!("forget record {} does not read", record.lsn))?; - self.chats.entry(chat.to_owned()).or_default().clear(); - } - other => return Err(format!("unknown event journal record kind {other}")), - } - Ok(()) - } -} - -/// What a repair did with one chat's log. -enum Repaired { - /// It holds every journaled line now (nothing was missing, or they were written). - Done, - /// Its folder is gone: the chat was deleted, or lost with its `session.json`. Not written. - Gone, - /// The log moved on without the journal: left as it is. - MovedOn(String), -} - -/// Why a repair stopped. -enum RepairError { - /// The chat's log could not be read or written; `keep` are the lines that only the journal - /// may hold now. - Chat { error: anyhow::Error, keep: Vec> }, - /// Tests: stopped right after its cut, as a crash would. - #[cfg(test)] - Stopped, -} - -/// One piece of a log read back from a repair's start: a whole line with its `seq`, or bytes -/// that do not read (a torn tail, a hole of zeros, a line that does not parse). -struct Piece { - segment: u32, - offset: u64, - line: Option<(u64, Vec)>, -} - -impl Inner { - /// Writes the replayed records' effects to the logs ([module docs](self)). One chat's failure - /// keeps its lines in `journal-unapplied/` and the open goes on; only a failure to keep them - /// fails it. - fn apply(&self, replay: Replay) -> Result<()> { - for (chat, journaled) in replay.chats { - let Some(first_lsn) = journaled.first().map(|(lsn, _)| *lsn) else { continue }; - match self.repair(&chat, &journaled) { - Ok(Repaired::Done) => { - self.order.lock_or_recover().dirty.insert(chat); - } - Ok(Repaired::Gone) => { - tracing::info!(session = %chat, "journaled lines of a chat whose folder is gone: not written again"); - } - Ok(Repaired::MovedOn(why)) => { - let lines: Vec> = journaled.into_iter().map(|(_, line)| line).collect(); - let kept = self.keep_aside(&chat, first_lsn, &lines)?; - tracing::error!(session = %chat, kept = %kept.display(), "the chat's log moved on without the event journal ({why}); left as it is"); - } - Err(RepairError::Chat { error, keep }) => { - let kept = self.keep_aside(&chat, first_lsn, &keep)?; - tracing::error!(session = %chat, kept = %kept.display(), "the chat's log could not be repaired: {error:#}"); - } - #[cfg(test)] - Err(RepairError::Stopped) => anyhow::bail!("stopped after the cut (a test)"), - } - } - let mut order = self.order.lock_or_recover(); - order.applied = self.journal.last_lsn(); - Ok(()) - } - - /// Repairs `id`'s log with its journaled lines (`journaled`, in LSN order): it never deletes a - /// whole line or a folder. - /// - /// The log is read from just after its last line with a `seq` below the first journaled one. - /// Its lines and the journal's merge by `seq`: a line in both must be the same bytes; a line - /// only the log has stays; a line only the journal has goes in its place. If that only adds - /// lines after the log's last one, they are appended. Otherwise (bytes that do not read, or a - /// line to put before a later one), the lines that only the log has from that point are - /// journaled and made durable first; then the log is cut there and the merged lines written, - /// so a crash at any step loses no line: the next open repairs the same way. - fn repair( - &self, - id: &str, - journaled: &[(Lsn, Vec)], - ) -> std::result::Result { - let keep_journaled = || journaled.iter().map(|(_, line)| line.clone()).collect::>(); - let chat_error = |error: anyhow::Error| RepairError::Chat { error, keep: keep_journaled() }; - match std::fs::symlink_metadata(self.local.dir(id)) { - Ok(_) => {} - Err(error) if error.kind() == std::io::ErrorKind::NotFound => { - return Ok(Repaired::Gone); - } - Err(error) => return Err(chat_error(error.into())), - } - let mut wanted: BTreeMap = BTreeMap::new(); - for (_, line) in journaled { - let Some(seq) = line_seq(line) else { - return Ok(Repaired::MovedOn("a journaled line has no seq".into())); - }; - if wanted.insert(seq, line).is_some_and(|other| other != line.as_slice()) { - return Ok(Repaired::MovedOn(format!("two journaled lines with seq {seq}"))); - } - } - let Some(&first) = wanted.keys().next() else { return Ok(Repaired::Done) }; - let pieces = self.read_from(id, first).map_err(chat_error)?; - let mut merged = wanted.clone(); - let mut last = None; - for (seq, bytes) in pieces.iter().filter_map(|piece| piece.line.as_ref()) { - if last.is_some_and(|last| *seq <= last) { - return Ok(Repaired::MovedOn(format!("its line {seq} is out of seq order"))); - } - last = Some(*seq); - if wanted.get(seq).is_some_and(|held| *held != bytes.as_slice()) { - return Ok(Repaired::MovedOn(format!("its line {seq} differs from the journal's"))); - } - merged.insert(*seq, bytes); - } - let merged: Vec<(u64, &[u8])> = merged.into_iter().collect(); - // How far the log already is the merged lines, and the first piece that is not. - let mut same = 0usize; - let mut cut = None; - for piece in &pieces { - let next = merged.get(same).copied(); - match (&piece.line, next) { - (Some((seq, bytes)), Some((want, line))) - if *seq == want && bytes.as_slice() == line => - { - same = same.saturating_add(1); - } - _ => { - cut = Some((piece.segment, piece.offset)); - break; - } - } - } - let rest = merged.get(same..).unwrap_or_default(); - if let Some((segment, offset)) = cut { - // The lines only the log holds from the cut on: into the journal first. - let only_logged: Vec<&[u8]> = rest - .iter() - .filter(|(seq, _)| !wanted.contains_key(seq)) - .map(|(_, line)| *line) - .collect(); - self.journal_lines(id, &only_logged).map_err(chat_error)?; - let keep_rest = || rest.iter().map(|(_, line)| line.to_vec()).collect::>(); - self.cut_at(id, segment, offset) - .map_err(|error| RepairError::Chat { error, keep: keep_rest() })?; - #[cfg(test)] - if STOP_AFTER_CUT.with(std::cell::Cell::get) { - return Err(RepairError::Stopped); - } - for (_, line) in rest { - self.local - .append_line(id, line) - .map_err(|error| RepairError::Chat { error, keep: keep_rest() })?; - } - } else { - for (_, line) in rest { - self.local.append_line(id, line).map_err(chat_error)?; - } - } - Ok(Repaired::Done) - } - - /// Appends `lines` of chat `id` to the journal and waits until they are durable. - fn journal_lines(&self, id: &str, lines: &[&[u8]]) -> Result<()> { - let mut last = None; - for line in lines { - let lsn = self - .journal - .append(KIND_LOG, Class::Ack, &encode_log(id, line)) - .map_err(|error| anyhow::anyhow!("journal a line only the log held: {error}"))?; - last = Some(lsn); - } - if let Some(lsn) = last { - self.journal - .wait_durable(lsn, REPAIR_WAIT) - .map_err(|error| anyhow::anyhow!("journal the lines only the log held: {error}"))?; - } - Ok(()) - } - - /// The pieces of `id`'s log from just after its last whole line with a `seq` below `first`. - fn read_from(&self, id: &str, first: u64) -> Result> { - let segments = self.local.segments(id)?; - // The last segment whose first readable line is below `first`; else the first segment. - let mut start = 0usize; - for (at, (_, path)) in segments.iter().enumerate().rev() { - if first_seq(path)?.is_some_and(|seq| seq < first) { - start = at; - break; - } - } - let mut pieces = Vec::new(); - for (at, (index, path)) in segments.iter().enumerate().skip(start) { - let from = if at == start { end_of_last_line_before(path, first)? } else { 0 }; - read_pieces(path, *index, from, &mut pieces)?; - } - Ok(pieces) - } - - /// Cuts `id`'s log at `offset` of segment `segment`: later segments go, and that one is - /// shortened. The kind-index copy of each segment that changes goes first: a crash between a - /// cut and its copy's removal would leave a copy that a log written back to the same length - /// passes for current. - fn cut_at(&self, id: &str, segment: u32, offset: u64) -> Result<()> { - let segments = self.local.segments(id)?; - for (index, _) in segments.iter().filter(|(index, _)| *index >= segment) { - self.forget_index_copy(id, *index)?; - } - for (index, path) in segments.iter().rev() { - if *index > segment { - crate::config::refuse_lost_home(path)?; - std::fs::remove_file(path).with_context(|| format!("remove {}", path.display()))?; - } else if *index == segment { - truncate_private(path, offset)?; - } - } - Ok(()) - } - - fn forget_index_copy(&self, id: &str, index: u32) -> Result<()> { - let copy = self.local.dir(id).join(kind_index::DIR).join(format!("{index:06}.ndjson")); - crate::config::refuse_lost_home(©)?; - match std::fs::remove_file(©) { - Ok(()) => Ok(()), - Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), - Err(error) => { - Err(anyhow::Error::from(error).context(format!("remove {}", copy.display()))) - } - } - } - - /// Keeps `lines` of chat `id` (its journaled lines from LSN `first_lsn` on) in - /// `/journal-unapplied/..ndjson`, owner-only and synced, and returns - /// that path. - fn keep_aside(&self, id: &str, first_lsn: Lsn, lines: &[Vec]) -> Result { - let dir = self.home.join(UNAPPLIED_DIR); - crate::config::refuse_lost_home(&dir)?; - crate::config::private_state_dir(&dir)?; - let name: String = id - .chars() - .map(|c| if c.is_ascii_alphanumeric() || matches!(c, '-' | '_') { c } else { '_' }) - .collect(); - let path = dir.join(format!("{name}.{first_lsn}.ndjson")); - let mut options = OpenOptions::new(); - options.write(true).create(true).truncate(true); - #[cfg(unix)] - { - use std::os::unix::fs::OpenOptionsExt; - options.mode(0o600).custom_flags(libc::O_NOFOLLOW); - } - let mut file = options.open(&path).with_context(|| format!("write {}", path.display()))?; - for line in lines { - file.write_all(line)?; - } - file.sync_all()?; - drop(file); - sync_directory(&dir)?; - Ok(path) - } - - // ------------------------------------------------------------ checkpoints - - /// One checkpoint ([module docs](self)); ten failures in a row ask the daemon to stop. - fn checkpoint(&self) -> Result<()> { - let _one = self.checkpointing.lock_or_recover(); - let result = self.checkpoint_once(); - let mut order = self.order.lock_or_recover(); - match &result { - Ok(()) => order.failures = 0, - Err(error) if !order.frozen => { - order.failures = order.failures.saturating_add(1); - if order.failures >= CHECKPOINT_FAILURES_TO_STOP { - drop(order); - self.ask_to_stop(format!( - "{CHECKPOINT_FAILURES_TO_STOP} event journal checkpoints failed in a row, the last: {error:#}" - )); - } - } - Err(_) => {} - } - result - } - - fn checkpoint_once(&self) -> Result<()> { - let (lsn, dirty) = { - let mut order = self.order.lock_or_recover(); - anyhow::ensure!( - !order.frozen, - "the event journal's checkpoint is frozen (a log sync failed): a restart writes the logs again" - ); - if order.applied <= order.checkpointed && order.dirty.is_empty() { - let checkpointed = order.checkpointed; - drop(order); - self.retire(checkpointed); - return Ok(()); - } - // A failed checkpoint waits for the timer or another `CHECKPOINT_BYTES`. - order.bytes = 0; - // Never past the journal's durable end: a crash right after is a normal one. - (order.applied.min(self.journal.durable_lsn()), std::mem::take(&mut order.dirty)) - }; - if let Err(error) = self.sync_logs(&dirty) { - self.order.lock_or_recover().dirty.extend(dirty); - return Err(error); - } - if self.local.sync_failed.load(AtomicOrdering::SeqCst) { - self.freeze("an event log failed to sync at its close".into()); - anyhow::bail!( - "an event log failed to sync: the journal keeps every line until a restart" - ); - } - write_checkpoint(&self.home, lsn)?; - let damage = { - let mut order = self.order.lock_or_recover(); - order.checkpointed = order.checkpointed.max(lsn); - std::mem::take(&mut order.damage) - }; - if damage && let Err(error) = self.journal.clear_damage() { - tracing::warn!("clear the event journal's damage report: {error}"); - } - self.retire(lsn); - Ok(()) - } - - fn retire(&self, lsn: Lsn) { - if let Err(error) = self.journal.retire_through(lsn) { - tracing::warn!("retire the event journal before {lsn}: {error}"); - } - } - - /// Syncs the logs of `dirty` (their open segment and folders) and the sessions folder (a - /// delete's removal, a new chat's folder). A chat deleted meanwhile is passed. A failed sync - /// call freezes the checkpoint; another error (a folder that does not open) fails only this - /// checkpoint, and the next one tries again. - fn sync_logs(&self, dirty: &BTreeSet) -> Result<()> { - let failed = |error: anyhow::Error| { - if error.chain().any(|cause| cause.is::()) { - self.freeze(format!("an event log failed to sync: {error:#}")); - } - error - }; - for chat in dirty { - match self.local.sync_log(chat) { - Ok(()) => {} - Err(error) if is_not_found(&error) => {} - Err(error) => { - return Err(failed(error.context(format!("sync the log of {chat}")))); - } - } - } - sync_directory(&self.local.root).map_err(failed) - } - - /// A log sync failed: no checkpoint until a restart, and the daemon is asked to stop. - fn freeze(&self, why: String) { - self.order.lock_or_recover().frozen = true; - self.ask_to_stop(why); - } - - fn ask_to_stop(&self, why: String) { - tracing::error!( - "{why}: the event journal keeps every line; the daemon stops so that its restart writes the logs again" - ); - self.stop.send_if_modified(|reason| { - if reason.is_some() { - return false; - } - *reason = Some(why); - true - }); - } - - fn run_checkpointer(&self) { - let mut timer = self.timer.lock_or_recover(); - loop { - if timer.stop { - return; - } - if !timer.wanted { - timer = self - .wake - .wait_timeout(timer, CHECKPOINT_EVERY) - .unwrap_or_else(std::sync::PoisonError::into_inner) - .0; - if timer.stop { - return; - } - } - timer.wanted = false; - drop(timer); - if let Err(error) = self.checkpoint() { - tracing::warn!("event journal checkpoint: {error:#}"); - } - timer = self.timer.lock_or_recover(); - } - } - - fn want_checkpoint(&self) { - let mut timer = self.timer.lock_or_recover(); - if !timer.wanted { - timer.wanted = true; - drop(timer); - self.wake.notify_all(); - } - } -} - -/// Whether `error` comes from a file or folder that is not there. -fn is_not_found(error: &anyhow::Error) -> bool { - error.chain().any(|cause| { - cause - .downcast_ref::() - .is_some_and(|io| io.kind() == std::io::ErrorKind::NotFound) - }) -} - -/// Reads `path` (segment `segment`) from `from` into `out`, piece by piece. -fn read_pieces(path: &Path, segment: u32, from: u64, out: &mut Vec) -> Result<()> { - use std::io::{Seek, SeekFrom}; - let mut file = crate::config::private_state_file(path)?; - file.seek(SeekFrom::Start(from))?; - let mut reader = BufReader::new(file); - let mut line = Vec::new(); - let mut offset = from; - loop { - line.clear(); - let read = reader.read_until(b'\n', &mut line)?; - if read == 0 { - return Ok(()); - } - let len = u64::try_from(read).unwrap_or(u64::MAX); - let whole = line.last() == Some(&b'\n'); - // A hole of zeros an OS crash left runs into the next line: the zeros are one piece, the - // line after them another. - let zeros = line.iter().take_while(|byte| **byte == 0).count(); - let after_zeros = line.get(zeros..).unwrap_or_default(); - if whole && let Some(seq) = line_seq(&line) { - out.push(Piece { segment, offset, line: Some((seq, line.clone())) }); - } else if whole - && zeros > 0 - && let Some(seq) = line_seq(after_zeros) - { - out.push(Piece { segment, offset, line: None }); - let at = offset.saturating_add(u64::try_from(zeros).unwrap_or(u64::MAX)); - out.push(Piece { segment, offset: at, line: Some((seq, after_zeros.to_vec())) }); - } else { - out.push(Piece { segment, offset, line: None }); - } - offset = offset.saturating_add(len); - } -} - -/// The offset right after the last whole line of `path` whose `seq` reads and is below -/// `from_seq`; 0 when there is none. -fn end_of_last_line_before(path: &Path, from_seq: u64) -> Result { - let mut reader = BufReader::new(crate::config::private_state_file(path)?); - let mut line = Vec::new(); - let (mut offset, mut keep) = (0u64, 0u64); - loop { - line.clear(); - let read = reader.read_until(b'\n', &mut line)?; - if read == 0 { - return Ok(keep); - } - offset = offset.saturating_add(u64::try_from(read).unwrap_or(u64::MAX)); - if line.last() != Some(&b'\n') { - return Ok(keep); - } - if line_seq(&line).is_some_and(|seq| seq < from_seq) { - keep = offset; - } - } -} - -/// Shortens the log segment at `path` (owner-only, never through a symlink) and syncs it. -fn truncate_private(path: &Path, len: u64) -> Result<()> { - crate::config::refuse_lost_home(path)?; - let mut options = OpenOptions::new(); - options.write(true); - #[cfg(unix)] - { - use std::os::unix::fs::OpenOptionsExt; - options.custom_flags(libc::O_NOFOLLOW); - } - let file = options.open(path).with_context(|| format!("open {}", path.display()))?; - file.set_len(len).with_context(|| format!("shorten {}", path.display()))?; - file.sync_all()?; - Ok(()) -} - // ---------------------------------------------------------------- the store impl JournalStore { diff --git a/cmux-tui/crates/acpmux/src/store/journal_store/checkpoint.rs b/cmux-tui/crates/acpmux/src/store/journal_store/checkpoint.rs new file mode 100644 index 000000000000..a8ab9092c51a --- /dev/null +++ b/cmux-tui/crates/acpmux/src/store/journal_store/checkpoint.rs @@ -0,0 +1,225 @@ +//! The checkpoint (`/journal-projected.json`, the journal's compaction point) and its +//! thread ([module docs](super)). + +use super::*; +use std::sync::atomic::Ordering as AtomicOrdering; + +#[derive(Serialize, Deserialize)] +struct Checkpoint { + schema: String, + lsn: Lsn, +} + +/// The checkpoint's LSN; 0 when there is none. One that does not read fails the open: a replay +/// from the journal's oldest record could undo a retention rewrite. +pub(super) fn read_checkpoint(home: &Path) -> Result { + let path = home.join(CHECKPOINT_NAME); + let bytes = match std::fs::read(&path) { + Ok(bytes) => bytes, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(0), + Err(error) => { + anyhow::bail!("read the event journal's checkpoint {}: {error}", path.display()) + } + }; + match serde_json::from_slice::(&bytes) { + Ok(checkpoint) if checkpoint.schema == CHECKPOINT_SCHEMA => Ok(checkpoint.lsn), + _ => anyhow::bail!( + "the event journal's checkpoint {} does not read; move it aside to replay the whole journal", + path.display() + ), + } +} + +/// Removes the temporary files of checkpoints a crash interrupted (`write_checkpoint`). +pub(super) fn remove_checkpoint_temporaries(home: &Path) { + let Ok(entries) = std::fs::read_dir(home) else { return }; + let prefix = format!("{CHECKPOINT_NAME}.tmp-"); + for entry in entries.flatten() { + if !entry.file_name().to_string_lossy().starts_with(&prefix) { + continue; + } + let path = entry.path(); + if let Err(error) = crate::config::refuse_lost_home(&path) + .and_then(|()| std::fs::remove_file(&path).map_err(anyhow::Error::from)) + { + tracing::warn!("remove {}: {error:#}", path.display()); + } + } +} + +/// Writes the checkpoint whole under a temporary name, syncs it, renames it into place and syncs +/// the home folder; an error at any step (the folder's sync too) is returned. +fn write_checkpoint(home: &Path, lsn: Lsn) -> Result<()> { + let path = home.join(CHECKPOINT_NAME); + crate::config::refuse_lost_home(&path)?; + let tmp = home.join(format!("{CHECKPOINT_NAME}.tmp-{}", std::process::id())); + let bytes = serde_json::to_vec(&Checkpoint { schema: CHECKPOINT_SCHEMA.into(), lsn })?; + let mut options = OpenOptions::new(); + options.write(true).create(true).truncate(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600).custom_flags(libc::O_NOFOLLOW); + } + let mut file = options.open(&tmp).with_context(|| format!("write {}", tmp.display()))?; + file.write_all(&bytes)?; + file.sync_all()?; + drop(file); + std::fs::rename(&tmp, &path).with_context(|| format!("rename to {}", path.display()))?; + sync_directory(home) +} + +impl Inner { + /// One checkpoint ([module docs](self)); ten failures in a row ask the daemon to stop. + pub(super) fn checkpoint(&self) -> Result<()> { + let _one = self.checkpointing.lock_or_recover(); + let result = self.checkpoint_once(); + let mut order = self.order.lock_or_recover(); + match &result { + Ok(()) => order.failures = 0, + Err(error) if !order.frozen => { + order.failures = order.failures.saturating_add(1); + if order.failures >= CHECKPOINT_FAILURES_TO_STOP { + drop(order); + self.ask_to_stop(format!( + "{CHECKPOINT_FAILURES_TO_STOP} event journal checkpoints failed in a row, the last: {error:#}" + )); + } + } + Err(_) => {} + } + result + } + + fn checkpoint_once(&self) -> Result<()> { + let (lsn, dirty) = { + let mut order = self.order.lock_or_recover(); + anyhow::ensure!( + !order.frozen, + "the event journal's checkpoint is frozen (a log sync failed): a restart writes the logs again" + ); + if order.applied <= order.checkpointed && order.dirty.is_empty() { + let checkpointed = order.checkpointed; + drop(order); + self.retire(checkpointed); + return Ok(()); + } + // A failed checkpoint waits for the timer or another `CHECKPOINT_BYTES`. + order.bytes = 0; + // Never past the journal's durable end: a crash right after is a normal one. + (order.applied.min(self.journal.durable_lsn()), std::mem::take(&mut order.dirty)) + }; + if let Err(error) = self.sync_logs(&dirty) { + self.order.lock_or_recover().dirty.extend(dirty); + return Err(error); + } + if self.local.sync_failed.load(AtomicOrdering::SeqCst) { + self.freeze("an event log failed to sync at its close".into()); + anyhow::bail!( + "an event log failed to sync: the journal keeps every line until a restart" + ); + } + write_checkpoint(&self.home, lsn)?; + let damage = { + let mut order = self.order.lock_or_recover(); + order.checkpointed = order.checkpointed.max(lsn); + std::mem::take(&mut order.damage) + }; + if damage && let Err(error) = self.journal.clear_damage() { + tracing::warn!("clear the event journal's damage report: {error}"); + } + self.retire(lsn); + Ok(()) + } + + fn retire(&self, lsn: Lsn) { + if let Err(error) = self.journal.retire_through(lsn) { + tracing::warn!("retire the event journal before {lsn}: {error}"); + } + } + + /// Syncs the logs of `dirty` (their open segment and folders) and the sessions folder (a + /// delete's removal, a new chat's folder). A chat deleted meanwhile is passed. A failed sync + /// call freezes the checkpoint; another error (a folder that does not open) fails only this + /// checkpoint, and the next one tries again. + fn sync_logs(&self, dirty: &BTreeSet) -> Result<()> { + let failed = |error: anyhow::Error| { + if error.chain().any(|cause| cause.is::()) { + self.freeze(format!("an event log failed to sync: {error:#}")); + } + error + }; + for chat in dirty { + match self.local.sync_log(chat) { + Ok(()) => {} + Err(error) if is_not_found(&error) => {} + Err(error) => { + return Err(failed(error.context(format!("sync the log of {chat}")))); + } + } + } + sync_directory(&self.local.root).map_err(failed) + } + + /// A log sync failed: no checkpoint until a restart, and the daemon is asked to stop. + fn freeze(&self, why: String) { + self.order.lock_or_recover().frozen = true; + self.ask_to_stop(why); + } + + fn ask_to_stop(&self, why: String) { + tracing::error!( + "{why}: the event journal keeps every line; the daemon stops so that its restart writes the logs again" + ); + self.stop.send_if_modified(|reason| { + if reason.is_some() { + return false; + } + *reason = Some(why); + true + }); + } + + pub(super) fn run_checkpointer(&self) { + let mut timer = self.timer.lock_or_recover(); + loop { + if timer.stop { + return; + } + if !timer.wanted { + timer = self + .wake + .wait_timeout(timer, CHECKPOINT_EVERY) + .unwrap_or_else(std::sync::PoisonError::into_inner) + .0; + if timer.stop { + return; + } + } + timer.wanted = false; + drop(timer); + if let Err(error) = self.checkpoint() { + tracing::warn!("event journal checkpoint: {error:#}"); + } + timer = self.timer.lock_or_recover(); + } + } + + pub(super) fn want_checkpoint(&self) { + let mut timer = self.timer.lock_or_recover(); + if !timer.wanted { + timer.wanted = true; + drop(timer); + self.wake.notify_all(); + } + } +} + +/// Whether `error` comes from a file or folder that is not there. +fn is_not_found(error: &anyhow::Error) -> bool { + error.chain().any(|cause| { + cause + .downcast_ref::() + .is_some_and(|io| io.kind() == std::io::ErrorKind::NotFound) + }) +} diff --git a/cmux-tui/crates/acpmux/src/store/journal_store/fenced.rs b/cmux-tui/crates/acpmux/src/store/journal_store/fenced.rs new file mode 100644 index 000000000000..ac50d78b4bc4 --- /dev/null +++ b/cmux-tui/crates/acpmux/src/store/journal_store/fenced.rs @@ -0,0 +1,64 @@ +//! The journal's storage behind the lost-home fence (`config::refuse_lost_home`). + +use super::*; + +/// The journal's storage, refusing every write under a lost home: `fence` (in the daemon, +/// `config::refuse_lost_home`) is asked before each operation that makes, writes, renames or +/// removes a file or folder. Reads, syncs and the lock are not writes. +pub(crate) struct FencedStorage { + inner: Arc, + fence: Box Result<()> + Send + Sync>, +} + +impl FencedStorage { + pub(crate) fn new( + inner: Arc, + fence: Box Result<()> + Send + Sync>, + ) -> Self { + Self { inner, fence } + } + + /// The daemon's: the real file system behind `config::refuse_lost_home`. + pub(crate) fn daemon() -> Self { + Self::new(Arc::new(cmux_journal::FsStorage), Box::new(crate::config::refuse_lost_home)) + } + + fn check(&self, path: &Path) -> std::io::Result<()> { + (self.fence)(path).map_err(|error| std::io::Error::other(format!("{error:#}"))) + } +} + +impl cmux_journal::Storage for FencedStorage { + fn create_dir_all(&self, dir: &Path) -> std::io::Result<()> { + self.check(dir)?; + self.inner.create_dir_all(dir) + } + fn list(&self, dir: &Path) -> std::io::Result> { + self.inner.list(dir) + } + fn open( + &self, + path: &Path, + mode: cmux_journal::OpenMode, + ) -> std::io::Result> { + if mode != cmux_journal::OpenMode::Read { + self.check(path)?; + } + self.inner.open(path, mode) + } + fn rename(&self, from: &Path, to: &Path) -> std::io::Result<()> { + self.check(from)?; + self.check(to)?; + self.inner.rename(from, to) + } + fn remove_file(&self, path: &Path) -> std::io::Result<()> { + self.check(path)?; + self.inner.remove_file(path) + } + fn sync_dir(&self, dir: &Path) -> std::io::Result<()> { + self.inner.sync_dir(dir) + } + fn try_lock(&self, path: &Path) -> std::io::Result>> { + self.inner.try_lock(path) + } +} diff --git a/cmux-tui/crates/acpmux/src/store/journal_store/repair.rs b/cmux-tui/crates/acpmux/src/store/journal_store/repair.rs new file mode 100644 index 000000000000..0a82713769ea --- /dev/null +++ b/cmux-tui/crates/acpmux/src/store/journal_store/repair.rs @@ -0,0 +1,366 @@ +//! Recovery: the replay of the records after the checkpoint and the repair of each chat's log +//! ([module docs](super), DESIGN.md 3.3c). + +use super::*; +use cmux_journal::Record; + +#[cfg(test)] +thread_local! { + /// Tests: a repair on this thread stops the open with an error right after it cut a log, as a + /// crash there would. + pub(crate) static STOP_AFTER_CUT: std::cell::Cell = const { std::cell::Cell::new(false) }; +} + +/// The records after the checkpoint, by chat. +#[derive(Default)] +pub(super) struct Replay { + chats: BTreeMap)>>, +} + +impl Replay { + pub(super) fn take(&mut self, record: Record<'_>) -> std::result::Result<(), String> { + match record.kind { + KIND_LOG => { + let (chat, line) = decode_log(record.payload) + .ok_or_else(|| format!("log record {} does not read", record.lsn))?; + self.chats.entry(chat).or_default().push((record.lsn, line.to_vec())); + } + // The chat's earlier lines are not its own any more: a chat made again under the same + // id gets only the lines after this. + KIND_FORGET => { + let chat = std::str::from_utf8(record.payload) + .map_err(|_| format!("forget record {} does not read", record.lsn))?; + self.chats.entry(chat.to_owned()).or_default().clear(); + } + other => return Err(format!("unknown event journal record kind {other}")), + } + Ok(()) + } +} + +/// What a repair did with one chat's log. +enum Repaired { + /// It holds every journaled line now (nothing was missing, or they were written). + Done, + /// Its folder is gone: the chat was deleted, or lost with its `session.json`. Not written. + Gone, + /// The log moved on without the journal: left as it is. + MovedOn(String), +} + +/// Why a repair stopped. +enum RepairError { + /// The chat's log could not be read or written; `keep` are the lines that only the journal + /// may hold now. + Chat { error: anyhow::Error, keep: Vec> }, + /// Tests: stopped right after its cut, as a crash would. + #[cfg(test)] + Stopped, +} + +/// One piece of a log read back from a repair's start: a whole line with its `seq`, or bytes +/// that do not read (a torn tail, a hole of zeros, a line that does not parse). +struct Piece { + segment: u32, + offset: u64, + line: Option<(u64, Vec)>, +} + +impl Inner { + /// Writes the replayed records' effects to the logs ([module docs](self)). One chat's failure + /// keeps its lines in `journal-unapplied/` and the open goes on; only a failure to keep them + /// fails it. + pub(super) fn apply(&self, replay: Replay) -> Result<()> { + for (chat, journaled) in replay.chats { + let Some(first_lsn) = journaled.first().map(|(lsn, _)| *lsn) else { continue }; + match self.repair(&chat, &journaled) { + Ok(Repaired::Done) => { + self.order.lock_or_recover().dirty.insert(chat); + } + Ok(Repaired::Gone) => { + tracing::info!(session = %chat, "journaled lines of a chat whose folder is gone: not written again"); + } + Ok(Repaired::MovedOn(why)) => { + let lines: Vec> = journaled.into_iter().map(|(_, line)| line).collect(); + let kept = self.keep_aside(&chat, first_lsn, &lines)?; + tracing::error!(session = %chat, kept = %kept.display(), "the chat's log moved on without the event journal ({why}); left as it is"); + } + Err(RepairError::Chat { error, keep }) => { + let kept = self.keep_aside(&chat, first_lsn, &keep)?; + tracing::error!(session = %chat, kept = %kept.display(), "the chat's log could not be repaired: {error:#}"); + } + #[cfg(test)] + Err(RepairError::Stopped) => anyhow::bail!("stopped after the cut (a test)"), + } + } + let mut order = self.order.lock_or_recover(); + order.applied = self.journal.last_lsn(); + Ok(()) + } + + /// Repairs `id`'s log with its journaled lines (`journaled`, in LSN order): it never deletes a + /// whole line or a folder. + /// + /// The log is read from just after its last line with a `seq` below the first journaled one. + /// Its lines and the journal's merge by `seq`: a line in both must be the same bytes; a line + /// only the log has stays; a line only the journal has goes in its place. If that only adds + /// lines after the log's last one, they are appended. Otherwise (bytes that do not read, or a + /// line to put before a later one), the lines that only the log has from that point are + /// journaled and made durable first; then the log is cut there and the merged lines written, + /// so a crash at any step loses no line: the next open repairs the same way. + fn repair( + &self, + id: &str, + journaled: &[(Lsn, Vec)], + ) -> std::result::Result { + let keep_journaled = || journaled.iter().map(|(_, line)| line.clone()).collect::>(); + let chat_error = |error: anyhow::Error| RepairError::Chat { error, keep: keep_journaled() }; + match std::fs::symlink_metadata(self.local.dir(id)) { + Ok(_) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + return Ok(Repaired::Gone); + } + Err(error) => return Err(chat_error(error.into())), + } + let mut wanted: BTreeMap = BTreeMap::new(); + for (_, line) in journaled { + let Some(seq) = line_seq(line) else { + return Ok(Repaired::MovedOn("a journaled line has no seq".into())); + }; + if wanted.insert(seq, line).is_some_and(|other| other != line.as_slice()) { + return Ok(Repaired::MovedOn(format!("two journaled lines with seq {seq}"))); + } + } + let Some(&first) = wanted.keys().next() else { return Ok(Repaired::Done) }; + let pieces = self.read_from(id, first).map_err(chat_error)?; + let mut merged = wanted.clone(); + let mut last = None; + for (seq, bytes) in pieces.iter().filter_map(|piece| piece.line.as_ref()) { + if last.is_some_and(|last| *seq <= last) { + return Ok(Repaired::MovedOn(format!("its line {seq} is out of seq order"))); + } + last = Some(*seq); + if wanted.get(seq).is_some_and(|held| *held != bytes.as_slice()) { + return Ok(Repaired::MovedOn(format!("its line {seq} differs from the journal's"))); + } + merged.insert(*seq, bytes); + } + let merged: Vec<(u64, &[u8])> = merged.into_iter().collect(); + // How far the log already is the merged lines, and the first piece that is not. + let mut same = 0usize; + let mut cut = None; + for piece in &pieces { + let next = merged.get(same).copied(); + match (&piece.line, next) { + (Some((seq, bytes)), Some((want, line))) + if *seq == want && bytes.as_slice() == line => + { + same = same.saturating_add(1); + } + _ => { + cut = Some((piece.segment, piece.offset)); + break; + } + } + } + let rest = merged.get(same..).unwrap_or_default(); + if let Some((segment, offset)) = cut { + // The lines only the log holds from the cut on: into the journal first. + let only_logged: Vec<&[u8]> = rest + .iter() + .filter(|(seq, _)| !wanted.contains_key(seq)) + .map(|(_, line)| *line) + .collect(); + self.journal_lines(id, &only_logged).map_err(chat_error)?; + let keep_rest = || rest.iter().map(|(_, line)| line.to_vec()).collect::>(); + self.cut_at(id, segment, offset) + .map_err(|error| RepairError::Chat { error, keep: keep_rest() })?; + #[cfg(test)] + if STOP_AFTER_CUT.with(std::cell::Cell::get) { + return Err(RepairError::Stopped); + } + for (_, line) in rest { + self.local + .append_line(id, line) + .map_err(|error| RepairError::Chat { error, keep: keep_rest() })?; + } + } else { + for (_, line) in rest { + self.local.append_line(id, line).map_err(chat_error)?; + } + } + Ok(Repaired::Done) + } + + /// Appends `lines` of chat `id` to the journal and waits until they are durable. + fn journal_lines(&self, id: &str, lines: &[&[u8]]) -> Result<()> { + let mut last = None; + for line in lines { + let lsn = self + .journal + .append(KIND_LOG, Class::Ack, &encode_log(id, line)) + .map_err(|error| anyhow::anyhow!("journal a line only the log held: {error}"))?; + last = Some(lsn); + } + if let Some(lsn) = last { + self.journal + .wait_durable(lsn, REPAIR_WAIT) + .map_err(|error| anyhow::anyhow!("journal the lines only the log held: {error}"))?; + } + Ok(()) + } + + /// The pieces of `id`'s log from just after its last whole line with a `seq` below `first`. + fn read_from(&self, id: &str, first: u64) -> Result> { + let segments = self.local.segments(id)?; + // The last segment whose first readable line is below `first`; else the first segment. + let mut start = 0usize; + for (at, (_, path)) in segments.iter().enumerate().rev() { + if first_seq(path)?.is_some_and(|seq| seq < first) { + start = at; + break; + } + } + let mut pieces = Vec::new(); + for (at, (index, path)) in segments.iter().enumerate().skip(start) { + let from = if at == start { end_of_last_line_before(path, first)? } else { 0 }; + read_pieces(path, *index, from, &mut pieces)?; + } + Ok(pieces) + } + + /// Cuts `id`'s log at `offset` of segment `segment`: later segments go, and that one is + /// shortened. The kind-index copy of each segment that changes goes first: a crash between a + /// cut and its copy's removal would leave a copy that a log written back to the same length + /// passes for current. + fn cut_at(&self, id: &str, segment: u32, offset: u64) -> Result<()> { + let segments = self.local.segments(id)?; + for (index, _) in segments.iter().filter(|(index, _)| *index >= segment) { + self.forget_index_copy(id, *index)?; + } + for (index, path) in segments.iter().rev() { + if *index > segment { + crate::config::refuse_lost_home(path)?; + std::fs::remove_file(path).with_context(|| format!("remove {}", path.display()))?; + } else if *index == segment { + truncate_private(path, offset)?; + } + } + Ok(()) + } + + fn forget_index_copy(&self, id: &str, index: u32) -> Result<()> { + let copy = self.local.dir(id).join(kind_index::DIR).join(format!("{index:06}.ndjson")); + crate::config::refuse_lost_home(©)?; + match std::fs::remove_file(©) { + Ok(()) => Ok(()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => { + Err(anyhow::Error::from(error).context(format!("remove {}", copy.display()))) + } + } + } + + /// Keeps `lines` of chat `id` (its journaled lines from LSN `first_lsn` on) in + /// `/journal-unapplied/..ndjson`, owner-only and synced, and returns + /// that path. + fn keep_aside(&self, id: &str, first_lsn: Lsn, lines: &[Vec]) -> Result { + let dir = self.home.join(UNAPPLIED_DIR); + crate::config::refuse_lost_home(&dir)?; + crate::config::private_state_dir(&dir)?; + let name: String = id + .chars() + .map(|c| if c.is_ascii_alphanumeric() || matches!(c, '-' | '_') { c } else { '_' }) + .collect(); + let path = dir.join(format!("{name}.{first_lsn}.ndjson")); + let mut options = OpenOptions::new(); + options.write(true).create(true).truncate(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600).custom_flags(libc::O_NOFOLLOW); + } + let mut file = options.open(&path).with_context(|| format!("write {}", path.display()))?; + for line in lines { + file.write_all(line)?; + } + file.sync_all()?; + drop(file); + sync_directory(&dir)?; + Ok(path) + } +} + +/// Reads `path` (segment `segment`) from `from` into `out`, piece by piece. +fn read_pieces(path: &Path, segment: u32, from: u64, out: &mut Vec) -> Result<()> { + use std::io::{Seek, SeekFrom}; + let mut file = crate::config::private_state_file(path)?; + file.seek(SeekFrom::Start(from))?; + let mut reader = BufReader::new(file); + let mut line = Vec::new(); + let mut offset = from; + loop { + line.clear(); + let read = reader.read_until(b'\n', &mut line)?; + if read == 0 { + return Ok(()); + } + let len = u64::try_from(read).unwrap_or(u64::MAX); + let whole = line.last() == Some(&b'\n'); + // A hole of zeros an OS crash left runs into the next line: the zeros are one piece, the + // line after them another. + let zeros = line.iter().take_while(|byte| **byte == 0).count(); + let after_zeros = line.get(zeros..).unwrap_or_default(); + if whole && let Some(seq) = line_seq(&line) { + out.push(Piece { segment, offset, line: Some((seq, line.clone())) }); + } else if whole + && zeros > 0 + && let Some(seq) = line_seq(after_zeros) + { + out.push(Piece { segment, offset, line: None }); + let at = offset.saturating_add(u64::try_from(zeros).unwrap_or(u64::MAX)); + out.push(Piece { segment, offset: at, line: Some((seq, after_zeros.to_vec())) }); + } else { + out.push(Piece { segment, offset, line: None }); + } + offset = offset.saturating_add(len); + } +} + +/// The offset right after the last whole line of `path` whose `seq` reads and is below +/// `from_seq`; 0 when there is none. +fn end_of_last_line_before(path: &Path, from_seq: u64) -> Result { + let mut reader = BufReader::new(crate::config::private_state_file(path)?); + let mut line = Vec::new(); + let (mut offset, mut keep) = (0u64, 0u64); + loop { + line.clear(); + let read = reader.read_until(b'\n', &mut line)?; + if read == 0 { + return Ok(keep); + } + offset = offset.saturating_add(u64::try_from(read).unwrap_or(u64::MAX)); + if line.last() != Some(&b'\n') { + return Ok(keep); + } + if line_seq(&line).is_some_and(|seq| seq < from_seq) { + keep = offset; + } + } +} + +/// Shortens the log segment at `path` (owner-only, never through a symlink) and syncs it. +fn truncate_private(path: &Path, len: u64) -> Result<()> { + crate::config::refuse_lost_home(path)?; + let mut options = OpenOptions::new(); + options.write(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.custom_flags(libc::O_NOFOLLOW); + } + let file = options.open(path).with_context(|| format!("open {}", path.display()))?; + file.set_len(len).with_context(|| format!("shorten {}", path.display()))?; + file.sync_all()?; + Ok(()) +} diff --git a/cmux-tui/crates/acpmux/src/store/journal_store_tests.rs b/cmux-tui/crates/acpmux/src/store/journal_store_tests.rs index 754dc351a8c9..1b1946335ba3 100644 --- a/cmux-tui/crates/acpmux/src/store/journal_store_tests.rs +++ b/cmux-tui/crates/acpmux/src/store/journal_store_tests.rs @@ -8,6 +8,7 @@ //! back to its length at the last `flush` (the kernel lost every write since). use super::journal_store::{FencedStorage, JournalStore, STOP_AFTER_CUT}; +use super::log_sync::FAIL_NEXT_LOG_SYNC; use super::*; use cmux_journal::{Crash, SimStorage}; use serde_json::json; diff --git a/cmux-tui/crates/acpmux/src/store/log_sync.rs b/cmux-tui/crates/acpmux/src/store/log_sync.rs new file mode 100644 index 000000000000..c5f11ecce3a1 --- /dev/null +++ b/cmux-tui/crates/acpmux/src/store/log_sync.rs @@ -0,0 +1,53 @@ +//! A log's sync for the journal store's checkpoint, and the error of a failed `fsync` (moved out +//! of `store.rs`, which is over its size budget). + +use super::*; + +#[cfg(test)] +thread_local! { + /// Tests: the next `LocalStore::sync_log` on this thread fails, as a failed fsync would. + pub(crate) static FAIL_NEXT_LOG_SYNC: std::cell::Cell = const { std::cell::Cell::new(false) }; +} + +/// A sync call (`fsync`) that failed, as opposed to an error before it (a folder that does not +/// open): what it was to make durable may be lost, and a later sync of the same file may report +/// success without it (the journal store then keeps every line until a restart). +#[derive(Debug)] +pub(crate) struct SyncFailed(pub(crate) std::io::Error); + +impl std::fmt::Display for SyncFailed { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "sync failed: {}", self.0) + } +} + +impl std::error::Error for SyncFailed { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + Some(&self.0) + } +} + +impl LocalStore { + /// Syncs `id`'s open log segment and its folders, without holding the writers' lock while + /// the disk works (the journal store's checkpoint, `store/journal_store.rs`). A segment that + /// is not open was synced at its close (`SegmentWriter::close`, which notes a failure). + pub(super) fn sync_log(&self, id: &str) -> Result<()> { + #[cfg(test)] + if FAIL_NEXT_LOG_SYNC.with(|fail| fail.replace(false)) { + return Err(SyncFailed(std::io::Error::other("a failed sync (a test)")).into()); + } + let open = { + let writers = self.writers.lock_or_recover(); + match writers.open.get(id) { + Some(writer) => Some(writer.file.try_clone()?), + None => None, + } + }; + if let Some(file) = open { + file.sync_all().map_err(SyncFailed)?; + } + let session = self.dir(id); + sync_directory(&session.join("events"))?; + sync_directory(&session) + } +} From 41d51e841c54a757cf8c3168cae6ab8faf71bcf3 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 4 Nov 2026 08:03:03 +0000 Subject: [PATCH 32/52] acpmux: the unrepairable-chat test waits for line 1 to be durable before its checkpoint At 03fccdef69c (fleet 32df3c72 on aws-m4pro-9, 144d4ccc on Linux) the test kept chat-b's lines [1, 2, 3] aside, not [2, 3]: it flushed right after appending line 1, and the checkpoint is now never past the journal's durable end (the review's Ahead nit), so line 1 was still past it. The fix's behavior is right; the test now waits until line 1 is durable, so its checkpoint covers it. On the code before the fix the test still fails (the open stops at chat-b's link). --- cmux-tui/crates/acpmux/src/store/journal_store_tests.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cmux-tui/crates/acpmux/src/store/journal_store_tests.rs b/cmux-tui/crates/acpmux/src/store/journal_store_tests.rs index 1b1946335ba3..0aab40f1ac97 100644 --- a/cmux-tui/crates/acpmux/src/store/journal_store_tests.rs +++ b/cmux-tui/crates/acpmux/src/store/journal_store_tests.rs @@ -466,6 +466,9 @@ fn one_chat_that_cannot_be_repaired_does_not_stop_the_open() { let store = open(home.path(), &journal); store.append(CHAT, &record(1, "created")).unwrap_or_else(|e| panic!("{e:#}")); store.append("chat-b", &record(1, "created")).unwrap_or_else(|e| panic!("{e:#}")); + // Durable first: a checkpoint never passes the journal's durable end, and this one must + // cover line 1. + assert!(store.wait_durable(WAIT).unwrap_or_else(|e| panic!("{e:#}")), "never durable"); store.flush().unwrap_or_else(|e| panic!("{e:#}")); for seq in 2..=3 { store.append(CHAT, &record(seq, "queued")).unwrap_or_else(|e| panic!("{e:#}")); From 2abbb1a0a4aadc15ce7965990e53f15f89788e5c Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 4 Nov 2026 08:06:30 +0000 Subject: [PATCH 33/52] acpmux: replies and events wait for durability, tests (red, S2c) DESIGN.md 3.3d. With the event journal on and every journal sync taking 400 ms (`SimStorage::set_sync_delay`): - an_acked_change_is_answered_once_its_record_is_durable: a rename's reply must come no sooner than one sync after its `renamed` record. Red here: the reply goes at once. - agent_output_is_not_held_behind_another_chats_acked_change: agent output to a connection attached to its chat goes at once while another chat's acknowledged change waits for its sync. Green here (nothing waits); it guards the fix against holding agent output behind changes the connection does not hear. --- .../acpmux/src/server/durable_gate_tests.rs | 143 ++++++++++++++++++ cmux-tui/crates/acpmux/src/server/fanout.rs | 4 + 2 files changed, 147 insertions(+) create mode 100644 cmux-tui/crates/acpmux/src/server/durable_gate_tests.rs diff --git a/cmux-tui/crates/acpmux/src/server/durable_gate_tests.rs b/cmux-tui/crates/acpmux/src/server/durable_gate_tests.rs new file mode 100644 index 000000000000..c3e195d55297 --- /dev/null +++ b/cmux-tui/crates/acpmux/src/server/durable_gate_tests.rs @@ -0,0 +1,143 @@ +//! Replies and events wait for durability (R-DURABILITY S2c, DESIGN.md 3.3d): with the event +//! journal on, an acknowledged change is answered only once its record is durable, and agent +//! output to a connection that heard no acknowledged change is not held behind one. +//! +//! The journal runs on the simulated file system (`cmux_journal::SimStorage`) with every sync +//! taking [`SYNC`], so "durable" takes a time a test can see. + +use crate::config::{Config, StoreMode}; +use crate::rpc::{Message, method}; +use crate::server::{Origin, serve_connection_with}; +use cmux_journal::SimStorage; +use serde_json::{Value, json}; +use std::path::Path; +use std::sync::Arc; +use std::time::{Duration, Instant}; +use tokio::sync::mpsc; + +const CHAT: &str = "chat-a"; +const OTHER: &str = "chat-b"; +/// How long every journal sync takes once a test sets it. +const SYNC: Duration = Duration::from_millis(400); + +/// A hub on a journal store with two chats. +fn hub(home: &Path, journal: &SimStorage) -> Arc { + let mut config = Config::default(); + config.store.mode = StoreMode::Local; + config.store.journal = true; + let journal_config = cmux_journal::Config { + retry_min: Duration::from_millis(1), + retry_max: Duration::from_millis(5), + ..cmux_journal::Config::default() + }; + let store = crate::store::open_with_journal( + &config.store, + home, + Arc::new(journal.clone()), + journal_config, + ) + .unwrap_or_else(|e| panic!("open the store: {e:#}")); + for id in [CHAT, OTHER] { + let meta = serde_json::from_value(json!({ + "schema": crate::store::META_SCHEMA, "id": id, "name": id, + "harness": "fake", "cwd": "/tmp", "status": "idle", "createdAt": 1, "updatedAt": 1 + })) + .unwrap_or_else(|e| panic!("meta: {e}")); + store.save(&meta).unwrap_or_else(|e| panic!("{e:#}")); + } + crate::hub::Hub::new(config, store) +} + +/// A local connection, initialized, and its outgoing lines. +async fn connect(hub: &Arc) -> (mpsc::Sender, mpsc::Receiver) { + let (input, incoming) = mpsc::channel(64); + let (output, mut outgoing) = mpsc::channel(256); + tokio::spawn(serve_connection_with(hub.clone(), incoming, output, Origin::Local)); + let hello = json!({"protocolVersion": 1, "clientCapabilities": {}, + "clientInfo": {"name": "test", "version": "0"}}); + send(&input, Message::request(1, method::INITIALIZE, hello)).await; + let reply = until(&mut outgoing, |line| line["id"] == 1).await; + assert!(reply.get("error").is_none(), "initialize: {reply}"); + (input, outgoing) +} + +async fn send(input: &mpsc::Sender, message: Message) { + input.send(message.to_line()).await.unwrap_or_else(|e| panic!("send: {e}")); +} + +/// The next line `outgoing` sends that `stop` takes, within 10 s. +async fn until(outgoing: &mut mpsc::Receiver, stop: impl Fn(&Value) -> bool) -> Value { + tokio::time::timeout(Duration::from_secs(10), async { + loop { + let line = outgoing.recv().await.unwrap_or_else(|| panic!("the connection ended")); + let line: Value = + serde_json::from_str(&line).unwrap_or_else(|e| panic!("not JSON ({e}): {line}")); + if stop(&line) { + return line; + } + } + }) + .await + .unwrap_or_else(|_| panic!("no such line within 10 s")) +} + +/// A rename appends its `renamed` record (an acknowledged change): the reply waits until that +/// record is durable, one sync after it was appended at the earliest. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn an_acked_change_is_answered_once_its_record_is_durable() { + let home = tempfile::tempdir().unwrap_or_else(|e| panic!("tempdir: {e}")); + let journal = SimStorage::new(60); + let hub = hub(home.path(), &journal); + let (input, mut outgoing) = connect(&hub).await; + journal.set_sync_delay(SYNC); + let started = Instant::now(); + let rename = json!({"sessionId": CHAT, "newName": "renamed"}); + send(&input, Message::request(2, method::MUX_RENAME, rename)).await; + let reply = until(&mut outgoing, |line| line["id"] == 2).await; + let took = started.elapsed(); + journal.set_sync_delay(Duration::ZERO); + assert!(reply.get("error").is_none(), "rename: {reply}"); + assert!( + took >= SYNC.saturating_sub(Duration::from_millis(50)), + "the rename was answered after {took:?}, before its record could be durable" + ); +} + +/// Agent output (a `Stream` record) to a connection attached to its chat goes at once, while +/// another chat's acknowledged change, which this connection does not hear, waits for its sync. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn agent_output_is_not_held_behind_another_chats_acked_change() { + let home = tempfile::tempdir().unwrap_or_else(|e| panic!("tempdir: {e}")); + let journal = SimStorage::new(61); + let hub = hub(home.path(), &journal); + let (viewer, mut viewer_out) = connect(&hub).await; + let attach = json!({"sessionId": CHAT, "visible": false, "waitLogs": false}); + send(&viewer, Message::request(2, method::MUX_ATTACH, attach)).await; + let reply = until(&mut viewer_out, |line| line["id"] == 2).await; + assert!(reply.get("error").is_none(), "attach: {reply}"); + let (other, mut other_out) = connect(&hub).await; + journal.set_sync_delay(SYNC); + let rename = json!({"sessionId": OTHER, "newName": "renamed"}); + send(&other, Message::request(2, method::MUX_RENAME, rename)).await; + // The rename's record is appended; its reply waits for its sync. + tokio::time::sleep(Duration::from_millis(100)).await; + let session = hub.resolve(CHAT).unwrap_or_else(|e| panic!("{e:?}")); + let started = Instant::now(); + let _ = hub.append( + &session, + "in", + method::SESSION_UPDATE, + json!({"jsonrpc": "2.0", "method": method::SESSION_UPDATE, "params": { + "sessionId": CHAT, + "update": {"sessionUpdate": "agent_message_chunk", + "content": {"type": "text", "text": "hi"}} + }}), + ); + let update = until(&mut viewer_out, |line| line["method"] == method::SESSION_UPDATE).await; + let took = started.elapsed(); + let renamed = until(&mut other_out, |line| line["id"] == 2).await; + journal.set_sync_delay(Duration::ZERO); + assert!(renamed.get("error").is_none(), "rename: {renamed}"); + assert_eq!(update["params"]["update"]["content"]["text"], "hi", "{update}"); + assert!(took < SYNC / 2, "agent output waited {took:?} behind another chat's change"); +} diff --git a/cmux-tui/crates/acpmux/src/server/fanout.rs b/cmux-tui/crates/acpmux/src/server/fanout.rs index f8a96d472de1..d10b4bc57239 100644 --- a/cmux-tui/crates/acpmux/src/server/fanout.rs +++ b/cmux-tui/crates/acpmux/src/server/fanout.rs @@ -104,3 +104,7 @@ fn send_presence(conn: &Conn, note: &serde_json::Value) { conn.send(&Message::notification(crate::rpc::method::MUX_PRESENCE, note.clone())); } } + +#[cfg(all(test, unix))] +#[path = "durable_gate_tests.rs"] +mod durable_gate_tests; From d68f79440799e8db726d281382791c689bb0c44e Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 4 Nov 2026 08:17:35 +0000 Subject: [PATCH 34/52] acpmux: replies and events wait until what they show is durable (S2c) DESIGN.md 3.3d; the tests are the commit before this one. With `store.journal` on, each connection gets an outbound gate (server/durable_gate.rs) between it and its transport: - Every line a connection queues raises the gate's `needs` to the last acknowledged LSN (`Store::durability().acked`, set in the journal store's append before the hub publishes anything of the record), except the lines of a `Stream` record's event (agent output, DESIGN C4): the fan-out delivers those in a thread-local stream scope. - The gate's task takes every queued line at once, reads `needs`, waits (async) until the journal's durable LSN reaches it, then sends the batch in order: one wait per batch, as the journal's group commit. - The durable LSN is a `tokio::sync::watch` fed by `Journal::on_durable`. The listener owns a guard: a writer that ends while the store is not being dropped (a panic on its thread) asks the daemon to stop (`Store::stop_reason`, decision 9.8), and the watch closes, so each gate drops what it holds and ends, which closes its connection. - Without the journal, `Store::durability()` is None: no gate, and `Conn::raise_gate` does nothing. Moves, so server/mod.rs and store.rs stay under their size on the base: the outbound queue helpers to server/outbound.rs, `open_with_journal` to the journal store, and `streamed` to store/durability.rs. --- .../crates/acpmux/src/server/durable_gate.rs | 124 ++++++++++++++ cmux-tui/crates/acpmux/src/server/fanout.rs | 13 +- cmux-tui/crates/acpmux/src/server/mod.rs | 154 +++--------------- cmux-tui/crates/acpmux/src/server/outbound.rs | 138 ++++++++++++++++ cmux-tui/crates/acpmux/src/store.rs | 22 ++- .../crates/acpmux/src/store/durability.rs | 26 +++ .../crates/acpmux/src/store/journal_store.rs | 86 ++++++++-- 7 files changed, 402 insertions(+), 161 deletions(-) create mode 100644 cmux-tui/crates/acpmux/src/server/durable_gate.rs create mode 100644 cmux-tui/crates/acpmux/src/server/outbound.rs create mode 100644 cmux-tui/crates/acpmux/src/store/durability.rs diff --git a/cmux-tui/crates/acpmux/src/server/durable_gate.rs b/cmux-tui/crates/acpmux/src/server/durable_gate.rs new file mode 100644 index 000000000000..bb7fcb316b53 --- /dev/null +++ b/cmux-tui/crates/acpmux/src/server/durable_gate.rs @@ -0,0 +1,124 @@ +//! A connection's outbound gate (R-DURABILITY S2c, DESIGN.md 3.3d): with the event journal on, no +//! line goes to a client before every acknowledged change it may show is durable. +//! +//! Each line a connection queues raises the gate's `needs` to the store's last acknowledged LSN +//! ([`Durability::acked`]) before it is queued, except the lines of a `Stream` record's event +//! (agent output, shown at once: DESIGN.md C4), sent inside a [`stream_scope`]. The gate's task +//! takes every queued line at once, reads `needs` after that, waits (async, no thread blocked) +//! until the journal's durable LSN reaches it, then sends the batch in order. So a reply goes +//! out only once the changes it may show are durable, an acknowledged event once its record is, +//! and a connection that heard nothing acknowledged never waits; a burst costs one wait per +//! batch, as the journal's group commit costs one sync. When the journal's writer ended (the +//! durable watch closed), the gate drops what it holds and ends, which closes the connection; +//! the store asks the daemon to stop. +//! +//! Without the journal ([`crate::store::Store::durability`] is None) there is no gate: lines go +//! straight to the transport, as before. + +use crate::store::Durability; +use std::cell::Cell; +use std::sync::Arc; +use std::sync::atomic::{AtomicU64, Ordering}; +use tokio::sync::{mpsc, watch}; + +/// What a connection's lines wait for. +pub(super) struct Gate { + /// The highest acknowledged LSN a line queued so far may show. + needs: AtomicU64, + acked: Arc, +} + +impl Gate { + /// Called right before a line is queued: it may show every change acknowledged so far. On + /// every line a connection sends: one thread-local read and two atomics, no lock. + pub(super) fn raise(&self) { + if STREAM.with(Cell::get) { + return; + } + self.needs.fetch_max(self.acked.load(Ordering::SeqCst), Ordering::SeqCst); + } +} + +thread_local! { + /// A `Stream` record's event is being delivered on this thread ([`stream_scope`]). + static STREAM: Cell = const { Cell::new(false) }; +} + +/// While it lives, the lines this thread queues show agent output only: they raise no gate. +pub(super) struct StreamScope { + was: bool, +} + +impl Drop for StreamScope { + fn drop(&mut self) { + STREAM.with(|stream| stream.set(self.was)); + } +} + +/// A scope for delivering the event of `record`: a stream scope for a `Stream` record's. +pub(super) fn stream_scope(record: &crate::store::EventRecord) -> Option { + if !crate::store::streamed(record) { + return None; + } + Some(StreamScope { was: STREAM.with(|stream| stream.replace(true)) }) +} + +/// Puts a gate between a connection and its transport `out` when the store journals its appends: +/// the sender the connection queues its lines on, and the gate it raises. Without the journal, +/// `out` itself and no gate. +pub(super) fn install( + durability: Option, + out: mpsc::Sender, +) -> (mpsc::Sender, Option>) { + let Some(Durability { acked, durable }) = durability else { return (out, None) }; + let gate = Arc::new(Gate { needs: AtomicU64::new(0), acked }); + let (lines_tx, lines) = mpsc::channel(out.max_capacity()); + tokio::spawn(run(lines, out, Arc::clone(&gate), durable)); + (lines_tx, Some(gate)) +} + +/// The gate's task ([module docs](self)). Ends when the connection's senders are gone, the +/// transport closed, or the journal's writer ended. +async fn run( + mut lines: mpsc::Receiver, + out: mpsc::Sender, + gate: Arc, + mut durable: watch::Receiver, +) { + let limit = lines.max_capacity(); + let mut batch = Vec::with_capacity(limit.min(64)); + loop { + tokio::select! { + biased; + () = out.closed() => return, + taken = lines.recv_many(&mut batch, limit) => { + if taken == 0 { + return; + } + } + } + // After the lines were taken: every line among them raised the gate before it was queued. + let needs = gate.needs.load(Ordering::SeqCst); + if *durable.borrow() < needs { + tokio::select! { + biased; + () = out.closed() => return, + reached = durable.wait_for(|lsn| *lsn >= needs) => { + if reached.is_err() { + tracing::error!("the event journal's writer ended: this connection closes"); + return; + } + } + } + } + for line in batch.drain(..) { + if out.send(line).await.is_err() { + return; + } + } + } +} + +#[cfg(all(test, unix))] +#[path = "durable_gate_tests.rs"] +mod durable_gate_tests; diff --git a/cmux-tui/crates/acpmux/src/server/fanout.rs b/cmux-tui/crates/acpmux/src/server/fanout.rs index d10b4bc57239..429ce4982c86 100644 --- a/cmux-tui/crates/acpmux/src/server/fanout.rs +++ b/cmux-tui/crates/acpmux/src/server/fanout.rs @@ -2,7 +2,8 @@ //! stream ends. Cached inventory has a separate opt-in and sanitized feed. use super::{ - Conn, Hub, Message, Ordering, broadcast, deliver, harness_admin, inventory_watch, json, + Conn, Hub, Message, Ordering, broadcast, deliver, durable_gate, harness_admin, inventory_watch, + json, }; use std::sync::Arc; @@ -79,7 +80,11 @@ pub(super) fn start(hub: Arc, conn: Arc) -> tokio::task::JoinHandle<( } }; match ev { - Ok(ev) => deliver(&hub, &conn, ev), + Ok(ev) => { + // Agent output shows nothing acknowledged: it raises no gate (`durable_gate.rs`). + let _stream = durable_gate::stream_scope(&ev.record); + deliver(&hub, &conn, ev); + } Err(broadcast::error::RecvError::Lagged(n)) => { conn.send(&Message::notification("_acpmux/lagged", json!({"dropped": n}))); } @@ -104,7 +109,3 @@ fn send_presence(conn: &Conn, note: &serde_json::Value) { conn.send(&Message::notification(crate::rpc::method::MUX_PRESENCE, note.clone())); } } - -#[cfg(all(test, unix))] -#[path = "durable_gate_tests.rs"] -mod durable_gate_tests; diff --git a/cmux-tui/crates/acpmux/src/server/mod.rs b/cmux-tui/crates/acpmux/src/server/mod.rs index ad633ba81d68..3a6a4c51ea07 100644 --- a/cmux-tui/crates/acpmux/src/server/mod.rs +++ b/cmux-tui/crates/acpmux/src/server/mod.rs @@ -159,6 +159,9 @@ pub struct Conn { raw_text: bool, name: StdMutex, out: mpsc::Sender, + /// Holds its lines until what they may show is durable (`durable_gate.rs`); None without the + /// event journal. + gate: Option>, /// True once the client stopped sending (its read side ended): it exited or hung up. ended: tokio::sync::watch::Sender, /// Per-session preparation cancellation; no capability or owner token is retained. @@ -468,9 +471,17 @@ impl Conn { self.send_replying(msg, None); } + /// The next line may show every change acknowledged so far (`durable_gate.rs`). + fn raise_gate(&self) { + if let Some(gate) = &self.gate { + gate.raise(); + } + } + /// [`Self::send`] of the reply to a request whose method is `reply_to`: the size watch names /// it (`wire_size.rs`). fn send_replying(&self, msg: &Message, reply_to: Option<&str>) { + self.raise_gate(); // A request to the client waits on its answer, so it is never dropped as lossy. let control = matches!(msg, Message::Response { .. } | Message::Request { .. }); if msg.method() == Some(method::MUX_SESSION_CHANGED) { @@ -559,6 +570,7 @@ impl Conn { let lag = json!({"watch": true, "dropped": 0, "reason": "list_changed"}); Message::notification("_acpmux/lagged", lag).to_line() }; + self.raise_gate(); let _guard = self.outbound_lock.lock().unwrap_or_else(|e| e.into_inner()); let stale = self.notices.load(Ordering::SeqCst) != seen; if stale && !anyway { @@ -591,6 +603,7 @@ impl Conn { /// answer that opened the Mac's side (`hub/github_owner.rs`). Refused, with nothing sent, /// while one of those still waits for room, or when it is larger than a frame. fn send_request_in_order(&self, msg: &Message) -> Result<(), RpcError> { + self.raise_gate(); let masked = (!self.raw_text).then(|| peer_text::masked(msg)).flatten(); let msg = masked.as_ref().unwrap_or(msg); let line = msg.bounded_line().map_err(RpcError::too_large)?; @@ -664,139 +677,6 @@ impl Conn { } } -/// Keep two queue slots available for replies and one for the gap marker that -/// tells a reader to replay from its last cursor. -const OUTBOUND_CONTROL_RESERVE: usize = 3; - -fn enqueue_outbound( - out: &mpsc::Sender, - line: String, - control: bool, - dropped: &Arc, - gap_pending: &Arc, - outbound_lock: &Arc>, - pending_controls: &Arc, -) { - let _guard = outbound_lock.lock().unwrap_or_else(|e| e.into_inner()); - enqueue_outbound_locked( - out, - line, - control, - dropped, - gap_pending, - outbound_lock, - pending_controls, - ); -} - -/// [`enqueue_outbound`] of a control line that must go out after every control queued before -/// it: queued only if none of those still waits for room in a full queue (each such waits in a -/// task of its own, which a line queued now would pass). False, with nothing queued, if one does -/// or the queue's reader is gone. -fn enqueue_after_controls( - out: &mpsc::Sender, - line: String, - dropped: &Arc, - gap_pending: &Arc, - outbound_lock: &Arc>, - pending_controls: &Arc, -) -> bool { - let _guard = outbound_lock.lock().unwrap_or_else(|e| e.into_inner()); - if out.is_closed() || pending_controls.load(Ordering::Acquire) != 0 { - return false; - } - enqueue_outbound_locked(out, line, true, dropped, gap_pending, outbound_lock, pending_controls); - true -} - -/// [`enqueue_outbound`] with `outbound_lock` held. -fn enqueue_outbound_locked( - out: &mpsc::Sender, - line: String, - control: bool, - dropped: &Arc, - gap_pending: &Arc, - outbound_lock: &Arc>, - pending_controls: &Arc, -) { - // Notifications are lossy by design when the reader is too slow, but the - // The final three permits are reserved for two response/control messages - // and the recovery marker. - let reserved = OUTBOUND_CONTROL_RESERVE + pending_controls.load(Ordering::Acquire); - if !control && out.capacity() <= reserved { - report_outbound_loss_locked(out, dropped, gap_pending, outbound_lock); - return; - } - match out.try_send(line) { - Ok(()) => { - // Let a later queue-pressure episode produce a fresh marker once - // the previous marker and backlog have drained. - if gap_pending.load(Ordering::Acquire) - && pending_controls.load(Ordering::Acquire) == 0 - && out.capacity() > OUTBOUND_CONTROL_RESERVE + 1 - { - gap_pending.store(false, Ordering::Release); - } - } - Err(mpsc::error::TrySendError::Full(line)) if control => { - let out = out.clone(); - pending_controls.fetch_add(1, Ordering::AcqRel); - let pending_controls = Arc::clone(pending_controls); - tokio::spawn(async move { - let _ = out.send(line).await; - pending_controls.fetch_sub(1, Ordering::AcqRel); - }); - } - Err(mpsc::error::TrySendError::Full(_)) => { - report_outbound_loss_locked(out, dropped, gap_pending, outbound_lock) - } - Err(mpsc::error::TrySendError::Closed(_)) => {} - } -} - -fn report_outbound_loss_locked( - out: &mpsc::Sender, - dropped: &Arc, - gap_pending: &Arc, - outbound_lock: &Arc>, -) { - let total = dropped.fetch_add(1, Ordering::Relaxed) + 1; - if gap_pending.swap(true, Ordering::AcqRel) { - return; - } - let marker = Message::notification( - "_acpmux/lagged", - json!({"dropped": total, "reason": "output_queue_full"}), - ) - .to_line(); - if out.try_send(marker).is_err() { - // Keep the marker pending even when the queue is currently full. The - // waiter owns the next available slot, so a quiet connection still - // receives the replay signal after its reader catches up. - let out = out.clone(); - let dropped = Arc::clone(dropped); - let gap_pending = Arc::clone(gap_pending); - let outbound_lock = Arc::clone(outbound_lock); - tokio::spawn(async move { - let permit = out.reserve().await; - let Ok(permit) = permit else { - gap_pending.store(false, Ordering::Release); - return; - }; - let _guard = outbound_lock.lock().unwrap_or_else(|e| e.into_inner()); - let total = dropped.load(Ordering::Relaxed); - permit.send( - Message::notification( - "_acpmux/lagged", - json!({"dropped": total, "reason": "output_queue_full"}), - ) - .to_line(), - ); - gap_pending.store(false, Ordering::Release); - }); - } -} - // ----------------------------------------------------------------- listen pub async fn listen_unix(hub: Arc, path: PathBuf) -> Result<()> { @@ -1587,6 +1467,7 @@ async fn serve_connection_from( // Counted while it is served: a daemon with none stops once unused (`daemon/unused_exit.rs`). let _client = crate::daemon::unused_exit::client(); let raw_text = !peer_text::masks(origin); + let (out, gate) = durable_gate::install(hub.store.durability(), out); let conn = Arc::new(Conn { id: uuid::Uuid::now_v7().to_string(), origin, @@ -1595,6 +1476,7 @@ async fn serve_connection_from( raw_text, name: StdMutex::new(String::new()), out, + gate, outbound_lock: Arc::new(StdMutex::new(())), outbound_dropped: Arc::new(AtomicU64::new(0)), outbound_gap_pending: Arc::new(AtomicBool::new(false)), @@ -2051,6 +1933,7 @@ mod chat_windows; pub use chat_windows::live_tasks as live_chat_window_tasks; mod chats; pub mod conn_auth; +mod durable_gate; mod ephemeral; mod fanout; mod forum; @@ -2075,6 +1958,11 @@ pub mod local_app; pub(crate) mod machine_health; mod models_fanout; mod new_worktree; +mod outbound; +use outbound::{ + OUTBOUND_CONTROL_RESERVE, enqueue_after_controls, enqueue_outbound, enqueue_outbound_locked, + report_outbound_loss_locked, +}; pub(crate) mod outbox; mod pairing; pub mod peer_auth; diff --git a/cmux-tui/crates/acpmux/src/server/outbound.rs b/cmux-tui/crates/acpmux/src/server/outbound.rs new file mode 100644 index 000000000000..8f340d47101b --- /dev/null +++ b/cmux-tui/crates/acpmux/src/server/outbound.rs @@ -0,0 +1,138 @@ +//! A connection's outbound queue: replies and requests to the client are never dropped, a +//! notification is when its reader falls behind, and a gap marker tells the reader to read again +//! (moved out of `server/mod.rs`, which is over its size budget). + +use super::*; + +/// Keep two queue slots available for replies and one for the gap marker that +/// tells a reader to replay from its last cursor. +pub(super) const OUTBOUND_CONTROL_RESERVE: usize = 3; + +pub(super) fn enqueue_outbound( + out: &mpsc::Sender, + line: String, + control: bool, + dropped: &Arc, + gap_pending: &Arc, + outbound_lock: &Arc>, + pending_controls: &Arc, +) { + let _guard = outbound_lock.lock().unwrap_or_else(|e| e.into_inner()); + enqueue_outbound_locked( + out, + line, + control, + dropped, + gap_pending, + outbound_lock, + pending_controls, + ); +} + +/// [`enqueue_outbound`] of a control line that must go out after every control queued before +/// it: queued only if none of those still waits for room in a full queue (each such waits in a +/// task of its own, which a line queued now would pass). False, with nothing queued, if one does +/// or the queue's reader is gone. +pub(super) fn enqueue_after_controls( + out: &mpsc::Sender, + line: String, + dropped: &Arc, + gap_pending: &Arc, + outbound_lock: &Arc>, + pending_controls: &Arc, +) -> bool { + let _guard = outbound_lock.lock().unwrap_or_else(|e| e.into_inner()); + if out.is_closed() || pending_controls.load(Ordering::Acquire) != 0 { + return false; + } + enqueue_outbound_locked(out, line, true, dropped, gap_pending, outbound_lock, pending_controls); + true +} + +/// [`enqueue_outbound`] with `outbound_lock` held. +pub(super) fn enqueue_outbound_locked( + out: &mpsc::Sender, + line: String, + control: bool, + dropped: &Arc, + gap_pending: &Arc, + outbound_lock: &Arc>, + pending_controls: &Arc, +) { + // Notifications are lossy by design when the reader is too slow, but the + // The final three permits are reserved for two response/control messages + // and the recovery marker. + let reserved = OUTBOUND_CONTROL_RESERVE + pending_controls.load(Ordering::Acquire); + if !control && out.capacity() <= reserved { + report_outbound_loss_locked(out, dropped, gap_pending, outbound_lock); + return; + } + match out.try_send(line) { + Ok(()) => { + // Let a later queue-pressure episode produce a fresh marker once + // the previous marker and backlog have drained. + if gap_pending.load(Ordering::Acquire) + && pending_controls.load(Ordering::Acquire) == 0 + && out.capacity() > OUTBOUND_CONTROL_RESERVE + 1 + { + gap_pending.store(false, Ordering::Release); + } + } + Err(mpsc::error::TrySendError::Full(line)) if control => { + let out = out.clone(); + pending_controls.fetch_add(1, Ordering::AcqRel); + let pending_controls = Arc::clone(pending_controls); + tokio::spawn(async move { + let _ = out.send(line).await; + pending_controls.fetch_sub(1, Ordering::AcqRel); + }); + } + Err(mpsc::error::TrySendError::Full(_)) => { + report_outbound_loss_locked(out, dropped, gap_pending, outbound_lock) + } + Err(mpsc::error::TrySendError::Closed(_)) => {} + } +} + +pub(super) fn report_outbound_loss_locked( + out: &mpsc::Sender, + dropped: &Arc, + gap_pending: &Arc, + outbound_lock: &Arc>, +) { + let total = dropped.fetch_add(1, Ordering::Relaxed) + 1; + if gap_pending.swap(true, Ordering::AcqRel) { + return; + } + let marker = Message::notification( + "_acpmux/lagged", + json!({"dropped": total, "reason": "output_queue_full"}), + ) + .to_line(); + if out.try_send(marker).is_err() { + // Keep the marker pending even when the queue is currently full. The + // waiter owns the next available slot, so a quiet connection still + // receives the replay signal after its reader catches up. + let out = out.clone(); + let dropped = Arc::clone(dropped); + let gap_pending = Arc::clone(gap_pending); + let outbound_lock = Arc::clone(outbound_lock); + tokio::spawn(async move { + let permit = out.reserve().await; + let Ok(permit) = permit else { + gap_pending.store(false, Ordering::Release); + return; + }; + let _guard = outbound_lock.lock().unwrap_or_else(|e| e.into_inner()); + let total = dropped.load(Ordering::Relaxed); + permit.send( + Message::notification( + "_acpmux/lagged", + json!({"dropped": total, "reason": "output_queue_full"}), + ) + .to_line(), + ); + gap_pending.store(false, Ordering::Release); + }); + } +} diff --git a/cmux-tui/crates/acpmux/src/store.rs b/cmux-tui/crates/acpmux/src/store.rs index f9c7ea3e91c7..605f2ff6be50 100644 --- a/cmux-tui/crates/acpmux/src/store.rs +++ b/cmux-tui/crates/acpmux/src/store.rs @@ -846,6 +846,11 @@ pub trait Store: Send + Sync { let _ = timeout; Ok(true) } + /// How durable its appends are (`server/durable_gate.rs`). None: appends are not journaled, + /// and no connection waits for them. + fn durability(&self) -> Option { + None + } /// Why the store asks the daemon to stop with a restart, as it changes: `Some(reason)` once /// the store cannot go on safely (the journal store's frozen checkpoint, whose logs only a /// restart's replay writes again). None: a store that never asks. @@ -1007,19 +1012,12 @@ pub fn open(config: &StoreConfig, root: &Path) -> Result> { }) } -/// [`open`] with the journal on `storage` (a simulated file system in the crash tests) and its -/// own limits, for `store.journal` (R-DURABILITY S2). -#[cfg(test)] -pub(crate) fn open_with_journal( - config: &StoreConfig, - root: &Path, - storage: std::sync::Arc, - journal: cmux_journal::Config, -) -> Result> { - Ok(Box::new(journal_store::JournalStore::open(config, root, storage, journal)?)) -} - +mod durability; +pub use durability::Durability; +pub(crate) use durability::streamed; mod journal_store; +#[cfg(test)] +pub(crate) use journal_store::open_with_journal; mod log_sync; pub(crate) use log_sync::SyncFailed; diff --git a/cmux-tui/crates/acpmux/src/store/durability.rs b/cmux-tui/crates/acpmux/src/store/durability.rs new file mode 100644 index 000000000000..a407c44045d5 --- /dev/null +++ b/cmux-tui/crates/acpmux/src/store/durability.rs @@ -0,0 +1,26 @@ +//! How durable the store's appends are, for the connections that must not show an acknowledged +//! change before it is (R-DURABILITY S2c, DESIGN.md 3.3d; `server/durable_gate.rs`). + +use super::EventRecord; +use std::sync::Arc; +use std::sync::atomic::AtomicU64; + +/// The journal store's durability, as a connection's outbound gate reads it. +#[derive(Clone)] +pub struct Durability { + /// The LSN of the last acknowledged (`Ack`) record appended: set in the append, before the + /// hub publishes anything of it. + pub acked: Arc, + /// The journal's durable LSN. Closed when the journal's writer ended: nothing becomes + /// durable any more. + pub durable: tokio::sync::watch::Receiver, +} + +/// A streamed agent line: a `Stream` record, synced with the next acked batch or a second later, +/// and shown at once (DESIGN.md C4). Every other line is acknowledged (`Ack`). +pub(crate) fn streamed(record: &EventRecord) -> bool { + record.dir == "in" + && (record.kind == crate::rpc::method::SESSION_UPDATE + || record.kind.starts_with("claude.") + || record.kind.starts_with("codex.")) +} diff --git a/cmux-tui/crates/acpmux/src/store/journal_store.rs b/cmux-tui/crates/acpmux/src/store/journal_store.rs index 2d13eed5f523..54b9b1213a60 100644 --- a/cmux-tui/crates/acpmux/src/store/journal_store.rs +++ b/cmux-tui/crates/acpmux/src/store/journal_store.rs @@ -33,11 +33,16 @@ //! - With the switch off, [`catch_up`] runs this recovery once on a journal left by a run with it //! on, so every acknowledged line is in the logs before they are written alone. //! -//! Not yet (S2c): replies and events do not wait for their records to be durable, and the daemon -//! does not stop when the journal closes (a panic on its writer). Session metadata is S3. +//! - Durability (S2c): [`Store::durability`] gives the last acknowledged LSN and the journal's +//! durable LSN, for which each connection's outbound gate waits (`server/durable_gate.rs`). A +//! writer that ends while the store is not dropped (a panic on its thread) closes that watch +//! and asks the daemon to stop (DESIGN.md decision 9.8). +//! +//! Session metadata is S3. use super::*; use cmux_journal::{AppendError, Class, Damage, Journal, Lsn, OpenError}; +use std::sync::atomic::{AtomicBool, AtomicU64}; use std::sync::{Arc, Condvar}; use std::time::{Duration, Instant}; @@ -93,6 +98,36 @@ struct Inner { wake: Condvar, /// Why the daemon must stop (`Store::stop_reason`). stop: tokio::sync::watch::Sender>, + /// The LSN of the last `Ack` record appended (`Store::durability`). + acked: Arc, + /// The journal's durable LSN, fed by its writer; closed when the writer ends. + durable: tokio::sync::watch::Receiver, + /// The store is being dropped: its writer's end is not a failure. + dropping: Arc, +} + +/// Owned by the journal's durable listener, which the writer drops when it ends: a writer that +/// ended while the store was not being dropped (a panic on its thread) asks the daemon to stop +/// (DESIGN.md decision 9.8), and the durable watch it fed closes. +struct WriterEnded { + stop: tokio::sync::watch::Sender>, + dropping: Arc, +} + +impl Drop for WriterEnded { + fn drop(&mut self) { + if self.dropping.load(std::sync::atomic::Ordering::SeqCst) { + return; + } + tracing::error!("the event journal's writer ended: no change can be acknowledged"); + self.stop.send_if_modified(|reason| { + let unset = reason.is_none(); + if unset { + *reason = Some("the event journal's writer ended".into()); + } + unset + }); + } } #[derive(Default)] @@ -144,6 +179,19 @@ impl JournalStore { } } let (stop, _) = tokio::sync::watch::channel(None); + let dropping = Arc::new(AtomicBool::new(false)); + let (durable_tx, durable) = tokio::sync::watch::channel(journal.durable_lsn()); + let ended = WriterEnded { stop: stop.clone(), dropping: Arc::clone(&dropping) }; + journal.on_durable(move |lsn| { + let _owned = &ended; + durable_tx.send_if_modified(|was| { + let moved = lsn > *was; + if moved { + *was = lsn; + } + moved + }); + }); let inner = Arc::new(Inner { local, journal, @@ -157,6 +205,9 @@ impl JournalStore { timer: Mutex::new(Timer::default()), wake: Condvar::new(), stop, + acked: Arc::new(AtomicU64::new(0)), + durable, + dropping, }); inner.apply(replay)?; if let Err(error) = inner.checkpoint() { @@ -182,6 +233,18 @@ impl JournalStore { } } +/// [`super::open`] with the journal on `storage` (a simulated file system in the crash tests) and +/// its own limits, for `store.journal`. +#[cfg(test)] +pub(crate) fn open_with_journal( + config: &StoreConfig, + root: &Path, + storage: Arc, + journal: cmux_journal::Config, +) -> Result> { + Ok(Box::new(JournalStore::open(config, root, storage, journal)?)) +} + /// Opens the journal, replaying the records after `from`; waits up to [`LOCK_WAIT`] while another /// process holds it (the daemon before a restart, still exiting). fn open_journal( @@ -245,14 +308,6 @@ fn decode_log(payload: &[u8]) -> Option<(String, &[u8])> { Some((id.to_owned(), payload.get(end..)?)) } -/// A streamed agent line: synced with the next acked batch or a second later (`Class::Stream`). -fn streamed(record: &EventRecord) -> bool { - record.dir == "in" - && (record.kind == crate::rpc::method::SESSION_UPDATE - || record.kind.starts_with("claude.") - || record.kind.starts_with("codex.")) -} - /// The `seq` of a whole log line, or `None` when it does not read. fn line_seq(line: &[u8]) -> Option { record_seq(line).or_else(|| parse_record(line).ok().map(|record| record.seq)) @@ -302,6 +357,9 @@ impl Store for JournalStore { // record, as after a crash between the two. let lsn = inner.journal.append(KIND_LOG, class, &payload).map_err(JournalStore::journal_error)?; + if class == Class::Ack { + inner.acked.store(lsn, std::sync::atomic::Ordering::SeqCst); + } order.applied = lsn; order.bytes = order.bytes.saturating_add(u64::try_from(payload.len()).unwrap_or(u64::MAX)); let full = order.bytes > CHECKPOINT_BYTES && !order.frozen; @@ -378,6 +436,12 @@ impl Store for JournalStore { fn stop_reason(&self) -> Option>> { Some(self.inner.stop.subscribe()) } + fn durability(&self) -> Option { + Some(Durability { + acked: Arc::clone(&self.inner.acked), + durable: self.inner.durable.clone(), + }) + } /// The logs refuse writes, and the journal fully syncs every record (the shutdown's `flush` /// then checkpoints). fn seal(&self) { @@ -418,6 +482,7 @@ impl Store for JournalStore { .journal .append(KIND_FORGET, Class::Ack, id.as_bytes()) .map_err(JournalStore::journal_error)?; + inner.acked.store(lsn, std::sync::atomic::Ordering::SeqCst); order.applied = lsn; } Store::delete(&inner.local, id) @@ -439,6 +504,7 @@ impl Store for JournalStore { impl Drop for JournalStore { /// Stops the checkpointer. No checkpoint here: a drop is not a shutdown (`flush` is). fn drop(&mut self) { + self.inner.dropping.store(true, std::sync::atomic::Ordering::SeqCst); self.inner.timer.lock_or_recover().stop = true; self.inner.wake.notify_all(); if let Some(handle) = self.checkpointer.lock_or_recover().take() { From 7a8bbdce62ca611aa2cc973185b69e39fb5f321a Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 4 Nov 2026 08:20:23 +0000 Subject: [PATCH 35/52] acpmux: the journal lock wait's deadline without an unchecked Instant addition `Instant + Duration` can panic on overflow (clippy::arithmetic_side_effects, tracked by the panic ratchet). The wait now has no deadline only in the case that cannot happen (an Instant 5 s ahead overflowing), where it stops waiting at once. --- cmux-tui/crates/acpmux/src/store/journal_store.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cmux-tui/crates/acpmux/src/store/journal_store.rs b/cmux-tui/crates/acpmux/src/store/journal_store.rs index 2d13eed5f523..82e4a89fba3d 100644 --- a/cmux-tui/crates/acpmux/src/store/journal_store.rs +++ b/cmux-tui/crates/acpmux/src/store/journal_store.rs @@ -190,7 +190,7 @@ fn open_journal( config: cmux_journal::Config, from: Lsn, ) -> Result<(Journal, cmux_journal::OpenReport, Replay)> { - let deadline = Instant::now() + LOCK_WAIT; + let deadline = Instant::now().checked_add(LOCK_WAIT); loop { let mut replay = Replay::default(); let opened = @@ -199,7 +199,9 @@ fn open_journal( }); match opened { Ok((journal, report)) => return Ok((journal, report, replay)), - Err(OpenError::Locked) if Instant::now() < deadline => { + Err(OpenError::Locked) + if deadline.is_some_and(|deadline| Instant::now() < deadline) => + { std::thread::sleep(Duration::from_millis(50)); } Err(error) => anyhow::bail!("open the event journal: {error}"), From 80c5fb2b7f8dbcebc66820a7dd304f574114195b Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 4 Nov 2026 09:05:18 +0000 Subject: [PATCH 36/52] acpmux: a failed log sync stops the real daemon for a restart, test (red) Lead rule 2026-11-04 (3): a daemon-level test of the store's "stop for a restart" before S2c merges, red first. With store.journal on and every log sync failing (a debug build's ACPMUX_TEST_FAIL_LOG_SYNC; the checkpoint every 200 ms through ACPMUX_TEST_JOURNAL_CHECKPOINT_MS), a rename of a chat on disk makes its log dirty, the checkpoint's sync fails and freezes it, and the store asks the daemon to stop. The test asserts: the WebSocket client gets the restart close (1012), the socket client's pending `_acpmux/wait` ends with the connection (no hang), and the daemon exits with status 74. Red here: the daemon stops and sends 1012, but exits 0 (no exit status says that it stopped to be repaired). This commit adds only the two debug-build seams (release builds have none) and the test. --- .../src/store/journal_store/checkpoint.rs | 15 +- cmux-tui/crates/acpmux/src/store/log_sync.rs | 10 ++ .../acpmux/tests/daemon_restart_close.rs | 169 ++++++++++++++++++ 3 files changed, 193 insertions(+), 1 deletion(-) diff --git a/cmux-tui/crates/acpmux/src/store/journal_store/checkpoint.rs b/cmux-tui/crates/acpmux/src/store/journal_store/checkpoint.rs index a8ab9092c51a..0ce6bc124e63 100644 --- a/cmux-tui/crates/acpmux/src/store/journal_store/checkpoint.rs +++ b/cmux-tui/crates/acpmux/src/store/journal_store/checkpoint.rs @@ -189,7 +189,7 @@ impl Inner { if !timer.wanted { timer = self .wake - .wait_timeout(timer, CHECKPOINT_EVERY) + .wait_timeout(timer, checkpoint_every()) .unwrap_or_else(std::sync::PoisonError::into_inner) .0; if timer.stop { @@ -215,6 +215,19 @@ impl Inner { } } +/// How long the checkpointer waits between two checkpoints. Debug builds only: +/// `ACPMUX_TEST_JOURNAL_CHECKPOINT_MS` (10 ms to 60 s) shortens it for the daemon-level tests. +fn checkpoint_every() -> Duration { + #[cfg(debug_assertions)] + if let Some(ms) = std::env::var("ACPMUX_TEST_JOURNAL_CHECKPOINT_MS") + .ok() + .and_then(|value| value.parse::().ok()) + { + return Duration::from_millis(ms.clamp(10, 60_000)); + } + CHECKPOINT_EVERY +} + /// Whether `error` comes from a file or folder that is not there. fn is_not_found(error: &anyhow::Error) -> bool { error.chain().any(|cause| { diff --git a/cmux-tui/crates/acpmux/src/store/log_sync.rs b/cmux-tui/crates/acpmux/src/store/log_sync.rs index c5f11ecce3a1..39f5b1527b36 100644 --- a/cmux-tui/crates/acpmux/src/store/log_sync.rs +++ b/cmux-tui/crates/acpmux/src/store/log_sync.rs @@ -36,6 +36,16 @@ impl LocalStore { if FAIL_NEXT_LOG_SYNC.with(|fail| fail.replace(false)) { return Err(SyncFailed(std::io::Error::other("a failed sync (a test)")).into()); } + // Debug builds only: `ACPMUX_TEST_FAIL_LOG_SYNC` fails every log sync, as a disk whose + // fsync fails would, so the daemon-level test (`tests/daemon_restart_close.rs`) can see + // the stop for a restart. Release builds have no seam. + #[cfg(debug_assertions)] + if std::env::var_os("ACPMUX_TEST_FAIL_LOG_SYNC").is_some() { + return Err(SyncFailed(std::io::Error::other( + "a failed sync (ACPMUX_TEST_FAIL_LOG_SYNC)", + )) + .into()); + } let open = { let writers = self.writers.lock_or_recover(); match writers.open.get(id) { diff --git a/cmux-tui/crates/acpmux/tests/daemon_restart_close.rs b/cmux-tui/crates/acpmux/tests/daemon_restart_close.rs index 989a91ab9329..7c9abc475411 100644 --- a/cmux-tui/crates/acpmux/tests/daemon_restart_close.rs +++ b/cmux-tui/crates/acpmux/tests/daemon_restart_close.rs @@ -119,3 +119,172 @@ async fn daemon_update_and_sigterm_send_1012_but_explicit_stop_is_normal() { } } } + +/// The daemon's exit status when its store asked it to stop for a restart (`daemon::EXIT_REPAIR`): +/// EX_IOERR. A supervisor starts it again (launchd `KeepAlive`, systemd `Restart=always`; 75 is +/// the only status the units prevent). +const EXIT_REPAIR: i32 = 74; + +/// One JSON-RPC request on a socket connection. +async fn send(wr: &mut tokio::net::unix::OwnedWriteHalf, id: u64, method: &str, params: Value) { + use tokio::io::AsyncWriteExt; + let line = json!({"jsonrpc": "2.0", "id": id, "method": method, "params": params}); + wr.write_all(format!("{line}\n").as_bytes()).await.unwrap(); +} + +/// The next line `lines` gives that `stop` takes, or what ended the wait (EOF or a read error) +/// as Err; panics after `budget`. +async fn line_until( + lines: &mut tokio::io::Lines>, + budget: Duration, + stop: impl Fn(&Value) -> bool, +) -> Result { + tokio::time::timeout(budget, async { + loop { + match lines.next_line().await { + Ok(Some(line)) => { + let value: Value = serde_json::from_str(&line).unwrap_or(Value::Null); + if stop(&value) { + return Ok(value); + } + } + Ok(None) => return Err("end of stream".to_owned()), + Err(error) => return Err(error.to_string()), + } + } + }) + .await + .unwrap_or_else(|_| panic!("no line within {budget:?}: the client hangs")) +} + +/// R-DURABILITY S2c, lead rule 2026-11-04 (3). With `store.journal` on, a log sync that fails (a +/// debug build's `ACPMUX_TEST_FAIL_LOG_SYNC`) freezes the event journal's checkpoint, and the store +/// asks the daemon to stop for a restart, whose replay writes the logs again. The daemon exits with +/// [`EXIT_REPAIR`] within its exit budget; a WebSocket client gets the restart close (1012), and a +/// socket client waiting for a reply sees its connection end, typed for the CLI as +/// `daemon_closed` (retryable), never a hang. +#[tokio::test] +async fn a_failed_log_sync_stops_the_daemon_for_a_restart() { + use std::os::unix::fs::PermissionsExt; + let _ = HERMETIC; + let dir = cmux_unix_socket::short_test_dir("amx-repair"); + let home = dir.path(); + let private = |path: &std::path::Path, mode: u32| { + std::fs::set_permissions(path, std::fs::Permissions::from_mode(mode)).unwrap(); + }; + private(home, 0o700); + let socket = home.join("s.sock"); + std::fs::write(home.join("config.json"), r#"{"store":{"journal":true}}"#).unwrap(); + private(&home.join("config.json"), 0o600); + // A chat on disk: a rename of it appends an acknowledged record without starting an agent. + let id = "01a15903-0000-7000-8000-000000000a01"; + let chat = home.join("sessions").join(&id); + std::fs::create_dir_all(&chat).unwrap(); + private(&home.join("sessions"), 0o700); + private(&chat, 0o700); + let meta = json!({"schema": "acpmux.session.v1", "id": id, "name": "repair", "harness": "claude", + "cwd": "/tmp", "status": "idle", "createdAt": 1, "updatedAt": 1}); + std::fs::write(chat.join("session.json"), meta.to_string()).unwrap(); + private(&chat.join("session.json"), 0o600); + + let mut child = daemon_env::tokio_command(home, daemon_env::Agents::Deny) + .env("ACPMUX_HOME", home) + .env("ACPMUX_SOCKET", &socket) + .env("ACPMUX_TEST_FAIL_LOG_SYNC", "1") + .env("ACPMUX_TEST_JOURNAL_CHECKPOINT_MS", "200") + .args([ + "daemon", + "run", + "--listen", + "127.0.0.1:0", + "--token", + "test-token", + "--ready-fd", + "1", + ]) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::inherit()) + .kill_on_drop(true) + .spawn() + .unwrap(); + let stdout = child.stdout.take().unwrap(); + let ready = tokio::time::timeout(Duration::from_secs(30), async { + let mut lines = BufReader::new(stdout).lines(); + while let Some(line) = lines.next_line().await.unwrap() { + if let Ok(value) = serde_json::from_str::(&line) + && value["ready"] == true + { + return value; + } + } + panic!("daemon ended before ready"); + }) + .await + .unwrap(); + let mut request = + format!("ws://{}", ready["listen"].as_str().unwrap()).into_client_request().unwrap(); + request.headers_mut().insert("authorization", "Bearer test-token".parse().unwrap()); + let (mut ws, _) = tokio_tungstenite::connect_async(request).await.unwrap(); + ws.send(Message::Text( + json!({"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":1}}) + .to_string() + .into(), + )) + .await + .unwrap(); + + let stream = tokio::net::UnixStream::connect(&socket).await.unwrap(); + let (rd, mut wr) = stream.into_split(); + let mut lines = BufReader::new(rd).lines(); + // Requests run concurrently on one connection: each waits for the one before. Anything that + // writes the chat's log may stop the daemon first, so a reply may also be the end of the stream. + let steps = [ + ("initialize", json!({"protocolVersion": 1})), + // The session index loads after the ready line: `_acpmux/info` waits for it. + ("_acpmux/info", json!({"sessionId": id})), + // The rename's `renamed` line makes the chat's log dirty; the next checkpoint (200 ms) + // syncs it, the sync fails, the checkpoint freezes, and the store asks for a restart. + ("_acpmux/rename", json!({"sessionId": id, "newName": "renamed"})), + // Waits until the chat closes: no result comes from a daemon that stops. + ("_acpmux/wait", json!({"sessions": [id], "until": ["closed"]})), + ]; + let mut ended = None; + for (n, (method, params)) in steps.into_iter().enumerate() { + let n = u64::try_from(n).unwrap() + 1; + send(&mut wr, n, method, params).await; + match line_until(&mut lines, Duration::from_secs(60), |v| v["id"] == n).await { + Ok(reply) if method == "_acpmux/wait" => { + assert!(reply.get("error").is_some(), "the wait got a result: {reply}"); + } + Ok(reply) => assert!(reply.get("error").is_none(), "{method}: {reply}"), + Err(why) => { + ended = Some(why); + break; + } + } + } + if ended.is_none() { + ended = line_until(&mut lines, Duration::from_secs(60), |_| false).await.err(); + } + assert!(ended.is_some(), "the socket connection did not end"); + + let frame = tokio::time::timeout(Duration::from_secs(30), async { + loop { + match ws.next().await { + Some(Ok(Message::Close(close))) => return close.map(|frame| u16::from(frame.code)), + Some(Ok(_)) => {} + Some(Err(error)) => panic!("websocket: {error}"), + None => panic!("the websocket ended without a close frame"), + } + } + }) + .await + .unwrap_or_else(|_| panic!("no close frame within 30 s: the client hangs")); + assert_eq!(frame, Some(1012), "a restart close"); + let status = tokio::time::timeout(Duration::from_secs(30), child.wait()) + .await + .unwrap_or_else(|_| panic!("the daemon did not exit within 30 s")) + .unwrap(); + assert_eq!(status.code(), Some(EXIT_REPAIR), "the daemon's exit status: {status:?}"); +} From 41f3969efde39b3e7bd85f9c13416d62f5f62d90 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 4 Nov 2026 09:08:53 +0000 Subject: [PATCH 37/52] acpmux: a daemon its store stopped for a restart exits with status 74 The test is the commit before this one. `daemon::run` returned Ok and the process exited 0 for every stop, so a supervisor or a person could not tell a stop to be repaired from a normal one; with systemd `Restart=on-failure` it would not even start again. Now the store's stop (`store_asks_to_stop`: a failed log sync froze the event journal's checkpoint, or the journal's writer ended) marks itself, sets the restart handoff (WebSocket clients get 1012, as before), and the process exits with `daemon::EXIT_REPAIR` (74, EX_IOERR). The units prevent only 75, so launchd KeepAlive and systemd Restart=always start it again; the app does not read the daemon's exit status. README: store.journal names the status. --- cmux-tui/crates/acpmux/README.md | 4 ++- cmux-tui/crates/acpmux/src/cli/entry.rs | 5 +-- cmux-tui/crates/acpmux/src/daemon.rs | 11 +++---- .../crates/acpmux/src/daemon/stop_causes.rs | 33 +++++++++++++++---- 4 files changed, 36 insertions(+), 17 deletions(-) diff --git a/cmux-tui/crates/acpmux/README.md b/cmux-tui/crates/acpmux/README.md index d59cc371df2d..76663b0c2fde 100644 --- a/cmux-tui/crates/acpmux/README.md +++ b/cmux-tui/crates/acpmux/README.md @@ -1482,7 +1482,9 @@ When no config exists, harnesses are imported from `~/.acpx/config.json` (its `a could not write into a log that changed without the journal (a run with the switch off after a crash, or an older build) go to `journal-unapplied/`. Turning it off is safe: the next start writes the journal's lines into the logs first. A failed log sync stops the daemon for a restart, - which writes the logs again. Read when the daemon starts. + which writes the logs again: it exits with status 74, and its supervisor (or the next client) + starts it again. A reply and an acknowledged event wait until what they show is durable. Read + when the daemon starts. - `websocket`: optional. The same protocol over WebSocket text frames, for remote clients such as a dashboard or a Durable Object. Put it behind a tunnel; the token is a bearer token. diff --git a/cmux-tui/crates/acpmux/src/cli/entry.rs b/cmux-tui/crates/acpmux/src/cli/entry.rs index 35681dee78e8..811395d2f838 100644 --- a/cmux-tui/crates/acpmux/src/cli/entry.rs +++ b/cmux-tui/crates/acpmux/src/cli/entry.rs @@ -263,8 +263,9 @@ async fn async_main(args: Vec, invocation: Invocation) -> Result<()> { } // The daemon has stopped its agents and synced its store. Exit // now rather than wait for the runtime to drain blocking tasks - // (a launcher `--version` check can take 20 s). - std::process::exit(0) + // (a launcher `--version` check can take 20 s); 74 when its store + // asked for a restart (`daemon::EXIT_REPAIR`). + std::process::exit(crate::daemon::exit_status()) } Some(Command::CliAgent { last_block, diff --git a/cmux-tui/crates/acpmux/src/daemon.rs b/cmux-tui/crates/acpmux/src/daemon.rs index e6017ff02669..6a7839bd2fc8 100644 --- a/cmux-tui/crates/acpmux/src/daemon.rs +++ b/cmux-tui/crates/acpmux/src/daemon.rs @@ -54,6 +54,7 @@ mod stop_causes; #[cfg(unix)] use stop_causes::next_signal; use stop_causes::{AbortOnDrop, store_asks_to_stop}; +pub use stop_causes::{EXIT_REPAIR, exit_status}; pub(crate) mod unused_exit; #[cfg(windows)] mod windows; @@ -709,7 +710,7 @@ pub async fn run(opts: DaemonOptions) -> Result<()> { let tailnet = crate::tailnet::embedded_service::Service::spawn(hub.clone()); // The store asks for a restart when it cannot go on safely (`Store::stop_reason`): a frozen // event journal checkpoint, whose logs only the next start's replay writes again. - let mut store_stop = std::pin::pin!(store_asks_to_stop(hub.store.stop_reason())); + let mut store_stop = std::pin::pin!(store_asks_to_stop(hub.clone())); let shutdown = async { // Every stop names its cause in the log (`app_orphan` and `unused` name their own). #[cfg(unix)] @@ -727,9 +728,7 @@ pub async fn run(opts: DaemonOptions) -> Result<()> { _ = next_signal(int.as_mut()) => tracing::warn!("stopping: SIGINT"), _ = hub.shutdown.notified() => tracing::warn!("stopping: a client asked it to"), // A restart handoff: the next start repairs what the store could not. - () = &mut store_stop => { - hub.restart_requested.store(true, std::sync::atomic::Ordering::SeqCst); - }, + () = &mut store_stop => {}, // No build of its app is left to adopt its agents, so they end with it. An app // that only quit or was updated keeps them, as its Quit does // (`daemon/app_orphan.rs`). @@ -783,9 +782,7 @@ pub async fn run(opts: DaemonOptions) -> Result<()> { tokio::select! { _ = ctrl_c => {}, _ = hub.shutdown.notified() => {}, - () = &mut store_stop => { - hub.restart_requested.store(true, std::sync::atomic::Ordering::SeqCst); - }, + () = &mut store_stop => {}, // No build of its app is left to adopt its agents, so they end with it. An app // that only quit or was updated keeps them, as its Quit does // (`daemon/app_orphan.rs`). diff --git a/cmux-tui/crates/acpmux/src/daemon/stop_causes.rs b/cmux-tui/crates/acpmux/src/daemon/stop_causes.rs index 1e0a049ca0b2..e83f240bc3c7 100644 --- a/cmux-tui/crates/acpmux/src/daemon/stop_causes.rs +++ b/cmux-tui/crates/acpmux/src/daemon/stop_causes.rs @@ -1,6 +1,24 @@ //! What stops the daemon besides a client: a signal, and the store asking for a restart (moved //! out of `daemon.rs`, which is over its size budget). +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; + +/// The daemon's exit status after its store asked it to stop for a restart ([`store_asks_to_stop`]: +/// a failed log sync froze the event journal's checkpoint, or the journal's writer ended): EX_IOERR. +/// The next start's replay repairs the logs; a supervisor starts it again (launchd `KeepAlive`, +/// systemd `Restart=always`: the units prevent only 75), and otherwise the next client does. +pub const EXIT_REPAIR: i32 = 74; + +/// The store's stop won the daemon's stop select. +static STOPPED_FOR_REPAIR: AtomicBool = AtomicBool::new(false); + +/// The process's exit status after `daemon::run` returned: [`EXIT_REPAIR`] when the store asked +/// for the stop, else 0. +pub fn exit_status() -> i32 { + if STOPPED_FOR_REPAIR.load(Ordering::SeqCst) { EXIT_REPAIR } else { 0 } +} + /// Aborts a task when dropped. pub(super) struct AbortOnDrop(pub(super) tokio::task::AbortHandle); @@ -10,18 +28,19 @@ impl Drop for AbortOnDrop { } } -/// Resolves, with the reason logged, once the store asks the daemon to stop -/// (`Store::stop_reason`); never for a store that does not ask, or once the store is gone. -pub(super) async fn store_asks_to_stop( - reason: Option>>, -) { - let Some(mut reason) = reason else { return std::future::pending().await }; +/// Resolves once the store asks the daemon to stop (`Store::stop_reason`), with the reason +/// logged, the stop marked a restart handoff (WebSocket clients get 1012) and the exit status set +/// to [`EXIT_REPAIR`]; never for a store that does not ask, or once the store is gone. +pub(super) async fn store_asks_to_stop(hub: Arc) { + let Some(mut reason) = hub.store.stop_reason() else { return std::future::pending().await }; match reason.wait_for(Option::is_some).await { Ok(why) => { tracing::error!("stopping for a restart: {}", why.as_deref().unwrap_or_default()); } - Err(_) => std::future::pending().await, + Err(_) => return std::future::pending().await, } + hub.restart_requested.store(true, Ordering::SeqCst); + STOPPED_FOR_REPAIR.store(true, Ordering::SeqCst); } /// The next delivery of a registered signal; never, if registration failed. From 62b4f68ef8ba38c350806811299707e8c8c62dcc Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 4 Nov 2026 09:16:37 +0000 Subject: [PATCH 38/52] acpmux: the restart test's request in flight goes through the acpmux client Lead rule (3): the client sees a typed error, not a hang. The socket client is now the acpmux client library (acpmux::client::Client, as the CLI uses it), and the test asserts what the CLI makes of the request that the stop ends: `daemon_closed`, retryable (cli::errors::classify). The first red (fleet 41f2d037, hq run 41804640229, at 80c5fb2b7f8) failed only on the exit status (0, wanted 74), with 1012 and the end of the connection passing. --- .../acpmux/tests/daemon_restart_close.rs | 95 ++++++------------- 1 file changed, 27 insertions(+), 68 deletions(-) diff --git a/cmux-tui/crates/acpmux/tests/daemon_restart_close.rs b/cmux-tui/crates/acpmux/tests/daemon_restart_close.rs index 7c9abc475411..13e35806c6fb 100644 --- a/cmux-tui/crates/acpmux/tests/daemon_restart_close.rs +++ b/cmux-tui/crates/acpmux/tests/daemon_restart_close.rs @@ -125,44 +125,11 @@ async fn daemon_update_and_sigterm_send_1012_but_explicit_stop_is_normal() { /// the only status the units prevent). const EXIT_REPAIR: i32 = 74; -/// One JSON-RPC request on a socket connection. -async fn send(wr: &mut tokio::net::unix::OwnedWriteHalf, id: u64, method: &str, params: Value) { - use tokio::io::AsyncWriteExt; - let line = json!({"jsonrpc": "2.0", "id": id, "method": method, "params": params}); - wr.write_all(format!("{line}\n").as_bytes()).await.unwrap(); -} - -/// The next line `lines` gives that `stop` takes, or what ended the wait (EOF or a read error) -/// as Err; panics after `budget`. -async fn line_until( - lines: &mut tokio::io::Lines>, - budget: Duration, - stop: impl Fn(&Value) -> bool, -) -> Result { - tokio::time::timeout(budget, async { - loop { - match lines.next_line().await { - Ok(Some(line)) => { - let value: Value = serde_json::from_str(&line).unwrap_or(Value::Null); - if stop(&value) { - return Ok(value); - } - } - Ok(None) => return Err("end of stream".to_owned()), - Err(error) => return Err(error.to_string()), - } - } - }) - .await - .unwrap_or_else(|_| panic!("no line within {budget:?}: the client hangs")) -} - /// R-DURABILITY S2c, lead rule 2026-11-04 (3). With `store.journal` on, a log sync that fails (a /// debug build's `ACPMUX_TEST_FAIL_LOG_SYNC`) freezes the event journal's checkpoint, and the store /// asks the daemon to stop for a restart, whose replay writes the logs again. The daemon exits with -/// [`EXIT_REPAIR`] within its exit budget; a WebSocket client gets the restart close (1012), and a -/// socket client waiting for a reply sees its connection end, typed for the CLI as -/// `daemon_closed` (retryable), never a hang. +/// [`EXIT_REPAIR`]; a WebSocket client gets the restart close (1012), and the acpmux client's +/// request in flight ends with the CLI's typed, retryable `daemon_closed`, never a hang. #[tokio::test] async fn a_failed_log_sync_stops_the_daemon_for_a_restart() { use std::os::unix::fs::PermissionsExt; @@ -234,40 +201,32 @@ async fn a_failed_log_sync_stops_the_daemon_for_a_restart() { .await .unwrap(); - let stream = tokio::net::UnixStream::connect(&socket).await.unwrap(); - let (rd, mut wr) = stream.into_split(); - let mut lines = BufReader::new(rd).lines(); - // Requests run concurrently on one connection: each waits for the one before. Anything that - // writes the chat's log may stop the daemon first, so a reply may also be the end of the stream. - let steps = [ - ("initialize", json!({"protocolVersion": 1})), - // The session index loads after the ready line: `_acpmux/info` waits for it. - ("_acpmux/info", json!({"sessionId": id})), - // The rename's `renamed` line makes the chat's log dirty; the next checkpoint (200 ms) - // syncs it, the sync fails, the checkpoint freezes, and the store asks for a restart. - ("_acpmux/rename", json!({"sessionId": id, "newName": "renamed"})), - // Waits until the chat closes: no result comes from a daemon that stops. - ("_acpmux/wait", json!({"sessions": [id], "until": ["closed"]})), - ]; - let mut ended = None; - for (n, (method, params)) in steps.into_iter().enumerate() { - let n = u64::try_from(n).unwrap() + 1; - send(&mut wr, n, method, params).await; - match line_until(&mut lines, Duration::from_secs(60), |v| v["id"] == n).await { - Ok(reply) if method == "_acpmux/wait" => { - assert!(reply.get("error").is_some(), "the wait got a result: {reply}"); - } - Ok(reply) => assert!(reply.get("error").is_none(), "{method}: {reply}"), - Err(why) => { - ended = Some(why); - break; - } + // The acpmux client library, as the CLI and the app's helpers use it. + let client = acpmux::client::Client::connect(&socket).await.unwrap(); + // The session index loads after the ready line: `_acpmux/info` waits for it. + let info = client.request("_acpmux/info", json!({"sessionId": id})).await; + assert!(info.is_ok(), "info: {info:?}"); + // The rename's `renamed` line makes the chat's log dirty; the next checkpoint (200 ms) syncs + // it, the sync fails, the checkpoint freezes, and the store asks for a restart. A rename that + // loses the race to the stop ends like the wait below. + let renamed = + client.request("_acpmux/rename", json!({"sessionId": id, "newName": "renamed"})).await; + // Waits until the chat closes: no result comes from a daemon that stops. + let waited = match renamed { + Ok(_) => { + let wait = + client.request("_acpmux/wait", json!({"sessions": [id], "until": ["closed"]})); + tokio::time::timeout(Duration::from_secs(60), wait) + .await + .unwrap_or_else(|_| panic!("the wait got no answer within 60 s: the client hangs")) } - } - if ended.is_none() { - ended = line_until(&mut lines, Duration::from_secs(60), |_| false).await.err(); - } - assert!(ended.is_some(), "the socket connection did not end"); + Err(error) => Err(error), + }; + let error = waited.expect_err("a result from a daemon that must stop"); + // What the CLI makes of it: a typed, retryable error, never a hang. + let typed = acpmux::cli::errors::classify(&error); + assert_eq!(typed.detail, "daemon_closed", "{error:#}"); + assert!(typed.retryable, "{error:#}"); let frame = tokio::time::timeout(Duration::from_secs(30), async { loop { From 994f74119fbdabe6aca00e7c6e0d328cb0d2d791 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 4 Nov 2026 09:34:35 +0000 Subject: [PATCH 39/52] acpmux: the daemon's disk tier with the journal on, tests (red) Lead rule 2026-11-04 (2): a reply the person waits for never waits on a throttled disk sync. The fleet A/B (STATUS 11554) found that at IOPOL_UTILITY an acknowledged journal append waits about 16 ms p50 under another process's disk stress (0.7 to 1.1 ms at the default tier). - the_daemon_with_the_journal_keeps_the_default_disk_tier: with store.journal on, the daemon process keeps the default tier; without it, utility as before (each case in a child of the test binary). - a_background_thread_yields_its_disk_work: a dedicated background thread (the checkpointer) works at utility; a thread beside it does not. The API they use is stubbed here (`qos::enter_daemon` ignores the journal; `qos::background_thread` does nothing), so both fail on macOS. --- cmux-tui/crates/acpmux/src/qos.rs | 17 ++++ .../crates/acpmux/src/qos_journal_tests.rs | 79 +++++++++++++++++++ 2 files changed, 96 insertions(+) create mode 100644 cmux-tui/crates/acpmux/src/qos_journal_tests.rs diff --git a/cmux-tui/crates/acpmux/src/qos.rs b/cmux-tui/crates/acpmux/src/qos.rs index 4c204b3cb62c..ae3b0cc6a0ac 100644 --- a/cmux-tui/crates/acpmux/src/qos.rs +++ b/cmux-tui/crates/acpmux/src/qos.rs @@ -102,6 +102,19 @@ fn agent_background() -> bool { std::env::var_os(ENV).is_some_and(|v| v == "background") } +/// The daemon's tier once its config is read: with `store.journal` on, acknowledged replies wait +/// on the journal's sync, so the daemon keeps the default disk tier (lead rule 2026-11-04: a reply +/// the person waits for never waits on a throttled sync) and only its background threads yield +/// ([`background_thread`]). Without the journal, [`Tier::Serving`] as before. +pub fn enter_daemon(journal: bool) { + let _ = journal; + enter(Role::Daemon); +} + +/// The calling thread, dedicated to background work for its whole life (the event journal's +/// checkpointer): its disk I/O at the utility tier. Only for a thread that does nothing else. +pub fn background_thread() {} + /// Put this process in `role`'s tier. Call it first thing in `main`, before /// any thread starts: for [`Tier::Agent`] on macOS it re-executes the process /// in place and returns only when that is not done (already clamped, turned @@ -417,6 +430,10 @@ mod imp { } } +#[cfg(test)] +#[path = "qos_journal_tests.rs"] +mod journal_tests; + #[cfg(test)] mod tests { use super::*; diff --git a/cmux-tui/crates/acpmux/src/qos_journal_tests.rs b/cmux-tui/crates/acpmux/src/qos_journal_tests.rs new file mode 100644 index 000000000000..8b21939d3b89 --- /dev/null +++ b/cmux-tui/crates/acpmux/src/qos_journal_tests.rs @@ -0,0 +1,79 @@ +//! The daemon's disk tier with the event journal on (R-DURABILITY S2c; lead rule 2026-11-04: a +//! reply the person waits for never waits on a throttled disk sync). Fleet A/B (STATUS 11554): +//! at `IOPOL_UTILITY` an acknowledged append waited about 16 ms p50 under another process's disk +//! stress, against 0.7 to 1.1 ms at the default tier. + +use super::*; + +/// The process's disk I/O policy (macOS `getiopolicy_np`, process scope). +#[cfg(target_os = "macos")] +fn process_io() -> libc::c_int { + unsafe extern "C" { + fn getiopolicy_np(iotype: libc::c_int, scope: libc::c_int) -> libc::c_int; + } + // SAFETY: a plain syscall with constant arguments. + unsafe { getiopolicy_np(0, 0) } +} + +/// The calling thread's own disk I/O policy (0: it follows the process's). +#[cfg(target_os = "macos")] +fn thread_io() -> libc::c_int { + unsafe extern "C" { + fn getiopolicy_np(iotype: libc::c_int, scope: libc::c_int) -> libc::c_int; + } + // SAFETY: as above, thread scope. + unsafe { getiopolicy_np(0, 1) } +} + +/// With the journal on, the daemon's process keeps the default disk tier; without it, utility as +/// before. A test process's own policy, so each case runs in a child of this test binary. +#[cfg(target_os = "macos")] +#[test] +fn the_daemon_with_the_journal_keeps_the_default_disk_tier() { + if let Ok(case) = std::env::var("ACPMUX_QOS_JOURNAL_CASE") { + enter_daemon(case == "journal"); + println!("QOS-CASE io={}", process_io()); + return; + } + let exe = std::env::current_exe().unwrap(); + let mut seen = Vec::new(); + for case in ["journal", "nojournal"] { + let out = std::process::Command::new(&exe) + .args([ + "--exact", + "qos::journal_tests::the_daemon_with_the_journal_keeps_the_default_disk_tier", + "--nocapture", + ]) + .env("ACPMUX_QOS_JOURNAL_CASE", case) + .env_remove(ENV) + .output() + .unwrap(); + let text = String::from_utf8_lossy(&out.stdout).into_owned(); + let io = text + .lines() + .find_map(|line| line.strip_prefix("QOS-CASE io=")) + .and_then(|io| io.trim().parse::().ok()) + .unwrap_or_else(|| panic!("case {case}: no QOS-CASE line in {text}")); + seen.push((case, io)); + } + // 4 is IOPOL_UTILITY; the default reads 0 (or 1, IOPOL_IMPORTANT, the same tier). + assert!(matches!(seen[0], ("journal", 0 | 1)), "the daemon with the journal: {seen:?}"); + assert_eq!(seen[1], ("nojournal", 4), "the daemon without the journal: {seen:?}"); +} + +/// A dedicated background thread (the checkpointer) does its disk work at the utility tier, and +/// a thread beside it keeps the process's. +#[cfg(target_os = "macos")] +#[test] +fn a_background_thread_yields_its_disk_work() { + let (own, other) = std::thread::scope(|scope| { + let own = scope.spawn(|| { + background_thread(); + thread_io() + }); + let other = scope.spawn(thread_io); + (own.join().unwrap(), other.join().unwrap()) + }); + assert_eq!(own, 4, "the background thread's disk tier"); + assert_eq!(other, 0, "a thread beside it"); +} From f27e942e44d590620113dd3bec00cb31ec47e6a2 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 4 Nov 2026 09:36:38 +0000 Subject: [PATCH 40/52] acpmux: the restart test borrows no &str twice (clippy needless_borrows_for_generic_args) Found by fleet 7f8afd9d (hq run: the S2c server step at 62b4f68ef8b); the step's other clippy red is the base red in link_push_bench.rs. --- cmux-tui/crates/acpmux/tests/daemon_restart_close.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmux-tui/crates/acpmux/tests/daemon_restart_close.rs b/cmux-tui/crates/acpmux/tests/daemon_restart_close.rs index 13e35806c6fb..3d1969bd69fa 100644 --- a/cmux-tui/crates/acpmux/tests/daemon_restart_close.rs +++ b/cmux-tui/crates/acpmux/tests/daemon_restart_close.rs @@ -145,7 +145,7 @@ async fn a_failed_log_sync_stops_the_daemon_for_a_restart() { private(&home.join("config.json"), 0o600); // A chat on disk: a rename of it appends an acknowledged record without starting an agent. let id = "01a15903-0000-7000-8000-000000000a01"; - let chat = home.join("sessions").join(&id); + let chat = home.join("sessions").join(id); std::fs::create_dir_all(&chat).unwrap(); private(&home.join("sessions"), 0o700); private(&chat, 0o700); From 350a9d7a7acd00fbe3e3f2b9a177953bd4bee246 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 4 Nov 2026 10:34:56 +0000 Subject: [PATCH 41/52] acpmux: with the journal on, the daemon keeps the default disk tier (S2c) The tests are 994f74119fb. Lead rule 2026-11-04: a reply the person waits for never waits on a throttled disk sync. Measured on two owned Macs (STATUS 11554): one acknowledged append to durable at IOPOL_UTILITY under another app's disk stress took 16.3-16.7 ms p50 and 37-45 ms p95, against 0.7-1.1 ms p50 and 1.5-1.8 ms p95 at the default tier. - qos::enter_daemon(journal): with store.journal on, the daemon process keeps the default disk tier; without it, utility as before. The switch is read alone from config.json (config::store_journal_enabled), before any thread starts, with no harness discovery. - qos::background_thread: the journal store's checkpointer lowers its own disk I/O to utility for its whole life (a thread that does nothing else: a thread-scope policy drops it out of QoS for good). --- cmux-tui/crates/acpmux/src/cli/entry.rs | 5 +++- cmux-tui/crates/acpmux/src/config.rs | 2 +- .../crates/acpmux/src/config/store_config.rs | 24 +++++++++++++++++ cmux-tui/crates/acpmux/src/qos.rs | 26 ++++++++++++++++--- .../crates/acpmux/src/store/journal_store.rs | 8 +++++- 5 files changed, 58 insertions(+), 7 deletions(-) diff --git a/cmux-tui/crates/acpmux/src/cli/entry.rs b/cmux-tui/crates/acpmux/src/cli/entry.rs index 811395d2f838..e9a18fb8c4d1 100644 --- a/cmux-tui/crates/acpmux/src/cli/entry.rs +++ b/cmux-tui/crates/acpmux/src/cli/entry.rs @@ -229,7 +229,10 @@ async fn async_main(args: Vec, invocation: Invocation) -> Result<()> { ) .with_target(false) .init(); - crate::qos::enter(crate::qos::Role::Daemon); + // With `store.journal` on, an acknowledged reply waits on the journal's sync, which a + // utility disk tier throttles 15-30x under another app's disk work (STATUS 11554): + // the daemon stays at the default tier, and only its background threads yield. + crate::qos::enter_daemon(crate::config::store_journal_enabled()); let result = crate::daemon::run(DaemonOptions { ws_listen: listen, ws_token: token, diff --git a/cmux-tui/crates/acpmux/src/config.rs b/cmux-tui/crates/acpmux/src/config.rs index b775ab63ab92..6d73f1010296 100644 --- a/cmux-tui/crates/acpmux/src/config.rs +++ b/cmux-tui/crates/acpmux/src/config.rs @@ -1918,7 +1918,7 @@ pub use discover::{ mod peer; pub use peer::PeerConfig; mod store_config; -pub use store_config::{StoreConfig, StoreMode}; +pub use store_config::{StoreConfig, StoreMode, store_journal_enabled}; mod pool; pub use pool::PoolConfig; mod btw; diff --git a/cmux-tui/crates/acpmux/src/config/store_config.rs b/cmux-tui/crates/acpmux/src/config/store_config.rs index 15859e756fe4..f046264c322a 100644 --- a/cmux-tui/crates/acpmux/src/config/store_config.rs +++ b/cmux-tui/crates/acpmux/src/config/store_config.rs @@ -57,3 +57,27 @@ impl Default for StoreConfig { } } } + +/// Whether `store.journal` is on in this home's config file, read without loading the rest of the +/// config (no harness discovery): the daemon's process tier is set before it starts any thread +/// (`qos::enter_daemon`). A file that is missing or does not read counts as off, the default; the +/// daemon's own config load then reports the error. +pub fn store_journal_enabled() -> bool { + #[derive(Deserialize)] + struct File { + #[serde(default)] + store: Option, + } + #[derive(Deserialize)] + struct Store { + #[serde(default)] + journal: bool, + } + let path = super::Config::path(); + let Ok(mut file) = super::private_state_file(&path) else { return false }; + let mut text = String::new(); + if std::io::Read::read_to_string(&mut file, &mut text).is_err() { + return false; + } + serde_json::from_str::(&text).ok().and_then(|file| file.store).is_some_and(|s| s.journal) +} diff --git a/cmux-tui/crates/acpmux/src/qos.rs b/cmux-tui/crates/acpmux/src/qos.rs index ae3b0cc6a0ac..e5e137e6d2d8 100644 --- a/cmux-tui/crates/acpmux/src/qos.rs +++ b/cmux-tui/crates/acpmux/src/qos.rs @@ -107,13 +107,22 @@ fn agent_background() -> bool { /// the person waits for never waits on a throttled sync) and only its background threads yield /// ([`background_thread`]). Without the journal, [`Tier::Serving`] as before. pub fn enter_daemon(journal: bool) { - let _ = journal; + if journal { + // The default disk tier: nothing to set (the CPU is unchanged for the daemon either way). + return; + } enter(Role::Daemon); } /// The calling thread, dedicated to background work for its whole life (the event journal's -/// checkpointer): its disk I/O at the utility tier. Only for a thread that does nothing else. -pub fn background_thread() {} +/// checkpointer): its disk I/O at the utility tier. Only for a thread that does nothing else: a +/// thread-scope I/O policy also drops the thread out of QoS for good (see `imp::ThreadOnScreen`), +/// so a pool thread that is reused must never take it. +pub fn background_thread() { + if enabled() { + imp::thread_disk_utility(); + } +} /// Put this process in `role`'s tier. Call it first thing in `main`, before /// any thread starts: for [`Tier::Agent`] on macOS it re-executes the process @@ -171,7 +180,6 @@ mod imp { const IOPOL_TYPE_DISK: libc::c_int = 0; const IOPOL_SCOPE_PROCESS: libc::c_int = 0; - #[cfg(test)] const IOPOL_SCOPE_THREAD: libc::c_int = 1; const IOPOL_UTILITY: libc::c_int = 4; const PRIO_DARWIN_BG: libc::c_int = 0x1000; @@ -200,6 +208,14 @@ mod imp { } } + /// The calling thread's disk I/O at utility (`background_thread`). + pub(super) fn thread_disk_utility() { + // SAFETY: plain syscall wrapper on the calling thread with constant arguments. + if unsafe { setiopolicy_np(IOPOL_TYPE_DISK, IOPOL_SCOPE_THREAD, IOPOL_UTILITY) } != 0 { + tracing::debug!("setiopolicy_np (thread): {}", std::io::Error::last_os_error()); + } + } + /// The clamp setter, looked up at run time: it is not in the public SDK /// headers, so a macOS without it starts unclamped instead of failing to load. fn clamp_fn() -> Option { @@ -375,6 +391,7 @@ mod imp { #[cfg(target_os = "linux")] mod imp { pub(super) fn disk_utility() {} + pub(super) fn thread_disk_utility() {} /// nice +5: inherited by everything the process starts. Not undone (an /// unprivileged process cannot lower its nice value), and not needed to be. @@ -410,6 +427,7 @@ mod imp { #[cfg(not(any(target_os = "macos", target_os = "linux")))] mod imp { pub(super) fn disk_utility() {} + pub(super) fn thread_disk_utility() {} pub(super) fn clamp_cpu_utility() {} pub(super) fn process_background() { clamp_cpu_utility(); diff --git a/cmux-tui/crates/acpmux/src/store/journal_store.rs b/cmux-tui/crates/acpmux/src/store/journal_store.rs index 2266f0522db6..c4ccc5a42b1c 100644 --- a/cmux-tui/crates/acpmux/src/store/journal_store.rs +++ b/cmux-tui/crates/acpmux/src/store/journal_store.rs @@ -216,7 +216,13 @@ impl JournalStore { let worker = Arc::clone(&inner); let checkpointer = std::thread::Builder::new() .name("acpmux-journal-checkpoint".into()) - .spawn(move || worker.run_checkpointer()) + .spawn(move || { + // Its syncs and retirements are background work: they yield the disk to the + // person's apps (`qos::background_thread`), while replies wait only on the + // journal writer, which keeps the daemon's tier. + crate::qos::background_thread(); + worker.run_checkpointer() + }) .context("start the event journal's checkpointer")?; Ok(Self { inner, From a43e979ae3910151528906a7e6a4e1380dd684d7 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 4 Nov 2026 11:01:39 +0000 Subject: [PATCH 42/52] acpmux: an agent chunk is a Stream record, test (red) The hub's tap logs an ACP session/update under its sessionUpdate kind (hub/tap.rs: `kind = su.to_owned()`), so a record's kind is `agent_message_chunk`, `tool_call`, ... and never `session/update`; the message's method is. The journal store's `streamed` checked the kind, so every agent chunk was journaled as Ack: a sync per batch at the token rate, and in S2c each token's event waited for that sync (the S2c I/O A/B: STRESS token to client 2.2 ms -> 20.3 ms p50 with the journal on). Red here: the chunk reads back as Ack. --- .../crates/acpmux/src/store/journal_store.rs | 6 ++ .../acpmux/src/store/journal_store_tests.rs | 61 +++++++++++++++++++ 2 files changed, 67 insertions(+) diff --git a/cmux-tui/crates/acpmux/src/store/journal_store.rs b/cmux-tui/crates/acpmux/src/store/journal_store.rs index 82e4a89fba3d..4a8664bca29e 100644 --- a/cmux-tui/crates/acpmux/src/store/journal_store.rs +++ b/cmux-tui/crates/acpmux/src/store/journal_store.rs @@ -240,6 +240,12 @@ fn encode_log(id: &str, line: &[u8]) -> Vec { payload } +/// [`decode_log`], for the tests that read the journal itself. +#[cfg(test)] +pub(super) fn decode_log_for_test(payload: &[u8]) -> Option<(String, &[u8])> { + decode_log(payload) +} + fn decode_log(payload: &[u8]) -> Option<(String, &[u8])> { let len = usize::try_from(u32::from_le_bytes(payload.get(..4)?.try_into().ok()?)).ok()?; let end = 4usize.checked_add(len)?; diff --git a/cmux-tui/crates/acpmux/src/store/journal_store_tests.rs b/cmux-tui/crates/acpmux/src/store/journal_store_tests.rs index 0aab40f1ac97..510c2553359d 100644 --- a/cmux-tui/crates/acpmux/src/store/journal_store_tests.rs +++ b/cmux-tui/crates/acpmux/src/store/journal_store_tests.rs @@ -665,3 +665,64 @@ fn switching_off_after_a_crash_keeps_every_acknowledged_line() { let off = open_off(home.path()); assert_eq!(seqs(off.as_ref(), CHAT), (1..=5).collect::>()); } + +/// An agent's streamed output is a `Stream` record, synced with the next acknowledged batch or a +/// second later (DESIGN.md 3.3b, C4); every other line is `Ack`. The hub's tap logs an ACP +/// `session/update` under its `sessionUpdate` kind (`agent_message_chunk`, `tool_call`, ...), so +/// the record's kind is never `session/update`: its message's method is. Found by the S2c I/O A/B +/// (STATUS 11723): every chunk was `Ack`, and its event waited for its sync. +#[test] +fn an_agent_chunk_is_a_stream_record() { + let home = tempfile::tempdir().unwrap_or_else(|e| panic!("tempdir: {e}")); + let journal = SimStorage::new(37); + let store = open(home.path(), &journal); + let chunk = EventRecord { + seq: 1, + at: 1_001, + dir: "in".into(), + kind: "agent_message_chunk".into(), + msg: json!({"jsonrpc": "2.0", "method": "session/update", "params": { + "sessionId": CHAT, + "update": {"sessionUpdate": "agent_message_chunk", + "content": {"type": "text", "text": "hi"}} + }}), + host_seq: None, + }; + let raw = EventRecord { seq: 2, kind: "claude.assistant".into(), ..chunk.clone() }; + store.append(CHAT, &chunk).unwrap_or_else(|e| panic!("{e:#}")); + store.append(CHAT, &raw).unwrap_or_else(|e| panic!("{e:#}")); + store.append(CHAT, &record(3, "queued")).unwrap_or_else(|e| panic!("{e:#}")); + assert!(store.wait_durable(WAIT).unwrap_or_else(|e| panic!("{e:#}")), "never durable"); + let restarted = journal.restart(Crash::Process); + drop(store); + // Each record's class, as the journal holds it. + let mut classes = Vec::new(); + let (reopened, _) = cmux_journal::Journal::open( + Arc::new(restarted), + &home.path().join("journal"), + journal_config(), + 0, + &mut |record| { + if let Some((_, line)) = super::journal_store::decode_log_for_test(record.payload) { + classes.push((line_seq(line), record.class)); + } + Ok(()) + }, + ) + .unwrap_or_else(|e| panic!("reopen the journal: {e}")); + drop(reopened); + assert_eq!( + classes, + vec![ + (Some(1), cmux_journal::Class::Stream), + (Some(2), cmux_journal::Class::Stream), + (Some(3), cmux_journal::Class::Ack), + ], + "an agent chunk and a raw harness line are Stream, a queue line Ack" + ); +} + +/// The `seq` of one journaled line (the test reads the journal itself). +fn line_seq(line: &[u8]) -> Option { + record_seq(line) +} From ce8dcba5458e931acfdf4648c5fbc8bdbe275151 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 4 Nov 2026 11:01:59 +0000 Subject: [PATCH 43/52] acpmux: an agent chunk is a Stream record (the record's message names session/update) The test is the commit before this one. `streamed` now reads the logged message's method: the hub's tap logs a session/update under its sessionUpdate kind (hub/tap.rs), so the kind never said it. Agent output is again Stream (synced with the next acknowledged batch or a second later), as DESIGN.md 3.3b says. --- cmux-tui/crates/acpmux/src/store/journal_store.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cmux-tui/crates/acpmux/src/store/journal_store.rs b/cmux-tui/crates/acpmux/src/store/journal_store.rs index 4a8664bca29e..0b6d0121657b 100644 --- a/cmux-tui/crates/acpmux/src/store/journal_store.rs +++ b/cmux-tui/crates/acpmux/src/store/journal_store.rs @@ -254,9 +254,13 @@ fn decode_log(payload: &[u8]) -> Option<(String, &[u8])> { } /// A streamed agent line: synced with the next acked batch or a second later (`Class::Stream`). +/// The hub's tap logs an ACP `session/update` under its `sessionUpdate` kind (`hub/tap.rs`), so +/// the message's method says it, not the kind; a history replay (`*.replay`) too. fn streamed(record: &EventRecord) -> bool { record.dir == "in" - && (record.kind == crate::rpc::method::SESSION_UPDATE + && (record.msg.get("method").and_then(Value::as_str) + == Some(crate::rpc::method::SESSION_UPDATE) + || record.kind == crate::rpc::method::SESSION_UPDATE || record.kind.starts_with("claude.") || record.kind.starts_with("codex.")) } From d2c63860ece2572779f1568b444d39d8b37a096c Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 4 Nov 2026 11:56:18 +0000 Subject: [PATCH 44/52] acpmux: review 2 of #34901, tests (red): a crash model that follows real log syncs The logs' OS crash model now follows the syncs that ran (store/sync_model.rs, tests only): a segment keeps its length at its last sync that counts; a failed sync loses what was written since the one before, and a later sync does not bring it back; a cut ends a loss it removes. Every log segment sync goes through log_sync::sync_segment (same calls as before), where the model and FAIL_NEXT_LOG_SYNC hook in. Red tests, one per finding: - 1a an_os_crash_after_a_restart_from_a_kill_keeps_every_acked_line: the restart's first checkpoint retires records whose log lines no sync made durable; expected [1..5], the head gives [1]. - 1b an_os_crash_after_a_restart_from_a_failed_sync_keeps_every_acked_line: the same after a frozen checkpoint and a restart; expected [1..5]. - 2 a_hole_that_starts_inside_a_line_is_repaired (the reviewer's test): a torn line is read as line 6 of a log that moved on; expected [1..9], the head gives [1, 2, 3, 4, 5, 9]. - 3 a_failed_sync_in_flush_session_freezes_the_checkpoint: a failed sync in flush_session does not freeze the checkpoint. --- cmux-tui/crates/acpmux/src/store.rs | 8 +- .../acpmux/src/store/journal_store/repair.rs | 4 +- .../acpmux/src/store/journal_store_tests.rs | 146 ++++++++++++++++++ cmux-tui/crates/acpmux/src/store/log_sync.rs | 24 ++- .../crates/acpmux/src/store/sync_model.rs | 119 ++++++++++++++ 5 files changed, 291 insertions(+), 10 deletions(-) create mode 100644 cmux-tui/crates/acpmux/src/store/sync_model.rs diff --git a/cmux-tui/crates/acpmux/src/store.rs b/cmux-tui/crates/acpmux/src/store.rs index 2a7129a894e5..52452b8ff7ac 100644 --- a/cmux-tui/crates/acpmux/src/store.rs +++ b/cmux-tui/crates/acpmux/src/store.rs @@ -1040,6 +1040,8 @@ pub(crate) fn open_with_journal( mod journal_store; mod log_sync; pub(crate) use log_sync::SyncFailed; +#[cfg(all(test, unix))] +mod sync_model; mod memory; pub use memory::MemoryStore; @@ -1085,7 +1087,7 @@ impl SegmentWriter { /// written (an append is never synced on its own), so only a machine crash before the kernel /// writes them back loses them, as it would any append since the last `flush`. fn close(self, id: &str, failed: &std::sync::atomic::AtomicBool) { - if let Err(e) = self.file.sync_data() { + if let Err(e) = log_sync::sync_segment(&self.file, false) { failed.store(true, std::sync::atomic::Ordering::SeqCst); tracing::warn!(session = %id, "sync event log on close: {e}"); } @@ -1540,7 +1542,7 @@ impl Store for LocalStore { fn flush_session(&self, id: &str) -> Result<()> { let writers = self.writers.lock_or_recover(); if let Some(writer) = writers.open.get(id) { - writer.file.sync_all()?; + log_sync::sync_segment(&writer.file, true)?; } let session = self.dir(id); sync_directory(&session.join("events"))?; @@ -1709,7 +1711,7 @@ impl Store for LocalStore { fn flush(&self) -> Result<()> { for w in self.writers.lock_or_recover().open.values() { - w.file.sync_data()?; + log_sync::sync_segment(&w.file, false)?; } Ok(()) } diff --git a/cmux-tui/crates/acpmux/src/store/journal_store/repair.rs b/cmux-tui/crates/acpmux/src/store/journal_store/repair.rs index 0a82713769ea..8fd720c04a2c 100644 --- a/cmux-tui/crates/acpmux/src/store/journal_store/repair.rs +++ b/cmux-tui/crates/acpmux/src/store/journal_store/repair.rs @@ -361,6 +361,8 @@ fn truncate_private(path: &Path, len: u64) -> Result<()> { } let file = options.open(path).with_context(|| format!("open {}", path.display()))?; file.set_len(len).with_context(|| format!("shorten {}", path.display()))?; - file.sync_all()?; + #[cfg(all(test, unix))] + sync_model::cut(&file, len); + log_sync::sync_segment(&file, true)?; Ok(()) } diff --git a/cmux-tui/crates/acpmux/src/store/journal_store_tests.rs b/cmux-tui/crates/acpmux/src/store/journal_store_tests.rs index 510c2553359d..d3c4eb496d24 100644 --- a/cmux-tui/crates/acpmux/src/store/journal_store_tests.rs +++ b/cmux-tui/crates/acpmux/src/store/journal_store_tests.rs @@ -9,6 +9,7 @@ use super::journal_store::{FencedStorage, JournalStore, STOP_AFTER_CUT}; use super::log_sync::FAIL_NEXT_LOG_SYNC; +use super::sync_model; use super::*; use cmux_journal::{Crash, SimStorage}; use serde_json::json; @@ -726,3 +727,148 @@ fn an_agent_chunk_is_a_stream_record() { fn line_seq(line: &[u8]) -> Option { record_seq(line) } + +// ---------------------------------------------------------------- review 2 of #34901 + +/// Overwrites `id`'s log with zeros from the middle of line `from` to the middle of line `to`: +/// the hole an OS crash leaves where a page in the middle never reached the disk (a page boundary +/// is almost never at a line start). +fn zero_log_mid_lines(home: &Path, id: &str, from: u64, to: u64) { + let events = home.join("sessions").join(id).join("events"); + for entry in std::fs::read_dir(&events).unwrap_or_else(|e| panic!("{e}")).flatten() { + let path = entry.path(); + if path.extension().is_none_or(|ext| ext != "ndjson") { + continue; + } + let mut bytes = std::fs::read(&path).unwrap_or_else(|e| panic!("{e}")); + let mut lines = BTreeMap::new(); + let mut offset = 0; + while offset < bytes.len() { + let end = bytes + .get(offset..) + .and_then(|rest| rest.iter().position(|b| *b == b'\n')) + .map_or(bytes.len(), |at| offset + at + 1); + if let Some(seq) = bytes.get(offset..end).and_then(record_seq) { + lines.insert(seq, (offset, end)); + } + offset = end; + } + if let (Some(&(a, a_end)), Some(&(b, b_end))) = (lines.get(&from), lines.get(&to)) { + if let Some(hole) = bytes.get_mut(a + (a_end - a) / 2..b + (b_end - b) / 2) { + hole.fill(0); + } + std::fs::write(&path, bytes).unwrap_or_else(|e| panic!("{e}")); + } + } +} + +/// Finding 2: an OS crash hole that starts inside a line. Line 6 keeps its `{"seq":6,` head and +/// runs through the zeros into the tail of line 8: it is torn bytes, not a line of a log that +/// moved on, so the journaled lines go back in. +#[test] +fn a_hole_that_starts_inside_a_line_is_repaired() { + let home = tempfile::tempdir().unwrap_or_else(|e| panic!("tempdir: {e}")); + let journal = SimStorage::new(40); + let store = open(home.path(), &journal); + for seq in 1..=5 { + store.append(CHAT, &record(seq, "queued")).unwrap_or_else(|e| panic!("{e:#}")); + } + assert!(store.wait_durable(WAIT).unwrap_or_else(|e| panic!("{e:#}")), "never durable"); + store.flush().unwrap_or_else(|e| panic!("{e:#}")); + for seq in 6..=9 { + store.append(CHAT, &record(seq, "queued")).unwrap_or_else(|e| panic!("{e:#}")); + } + assert!(store.wait_durable(WAIT).unwrap_or_else(|e| panic!("{e:#}")), "never durable"); + let crashed = journal.restart(Crash::Os); + drop(store); + zero_log_mid_lines(home.path(), CHAT, 6, 8); + let store = open(home.path(), &crashed); + assert_eq!( + seqs(store.as_ref(), CHAT), + (1..=9).collect::>(), + "a torn line was taken for a line of a log that moved on" + ); +} + +/// Finding 1a: after a kill, the dead daemon's log writes may be in the page cache only. The next +/// start's first checkpoint retires their records, so it must sync those bytes first. An OS crash +/// after that restart keeps every acknowledged line (the logs' crash model follows real syncs). +#[test] +fn an_os_crash_after_a_restart_from_a_kill_keeps_every_acked_line() { + let home = tempfile::tempdir().unwrap_or_else(|e| panic!("tempdir: {e}")); + let journal = SimStorage::new(41); + let store = open(home.path(), &journal); + store.append(CHAT, &record(1, "created")).unwrap_or_else(|e| panic!("{e:#}")); + assert!(store.wait_durable(WAIT).unwrap_or_else(|e| panic!("{e:#}")), "never durable"); + store.flush().unwrap_or_else(|e| panic!("{e:#}")); + for seq in 2..=5 { + store.append(CHAT, &record(seq, "queued")).unwrap_or_else(|e| panic!("{e:#}")); + } + assert!(store.wait_durable(WAIT).unwrap_or_else(|e| panic!("{e:#}")), "never durable"); + let killed = journal.restart(Crash::Process); + drop(store); + // The restart replays 2..5 and its first checkpoint runs. + let store = open(home.path(), &killed); + let crashed = killed.restart(Crash::Os); + drop(store); + sync_model::os_crash(&home.path().join("sessions")); + let store = open(home.path(), &crashed); + assert_eq!( + seqs(store.as_ref(), CHAT), + (1..=5).collect::>(), + "the restart's checkpoint retired records whose log lines no sync made durable" + ); +} + +/// Finding 1b: a failed log sync freezes the checkpoint and the daemon restarts. The kernel may +/// have marked the lost pages clean, so a plain sync after the restart reports success without +/// them: the restart must write those lines again before its checkpoint retires their records. +#[test] +fn an_os_crash_after_a_restart_from_a_failed_sync_keeps_every_acked_line() { + let home = tempfile::tempdir().unwrap_or_else(|e| panic!("tempdir: {e}")); + let journal = SimStorage::new(42); + let store = open(home.path(), &journal); + store.append(CHAT, &record(1, "created")).unwrap_or_else(|e| panic!("{e:#}")); + assert!(store.wait_durable(WAIT).unwrap_or_else(|e| panic!("{e:#}")), "never durable"); + store.flush().unwrap_or_else(|e| panic!("{e:#}")); + for seq in 2..=5 { + store.append(CHAT, &record(seq, "queued")).unwrap_or_else(|e| panic!("{e:#}")); + } + assert!(store.wait_durable(WAIT).unwrap_or_else(|e| panic!("{e:#}")), "never durable"); + FAIL_NEXT_LOG_SYNC.with(|fail| fail.set(true)); + let failed = store.flush(); + FAIL_NEXT_LOG_SYNC.with(|fail| fail.set(false)); + assert!(failed.is_err(), "the checkpoint went on past a failed log sync"); + // The daemon stops for a restart (no crash), and the restart's first checkpoint runs. + drop(store); + let store = open(home.path(), &journal); + let crashed = journal.restart(Crash::Os); + drop(store); + sync_model::os_crash(&home.path().join("sessions")); + let store = open(home.path(), &crashed); + assert_eq!( + seqs(store.as_ref(), CHAT), + (1..=5).collect::>(), + "the restart's checkpoint retired records whose lines a failed sync lost" + ); +} + +/// Finding 3: a failed sync in `flush_session` (transfer receipts, forks, auto-resume, team +/// messages) freezes the checkpoint, as a failed sync in a checkpoint or at a log's close does. +#[test] +fn a_failed_sync_in_flush_session_freezes_the_checkpoint() { + let home = tempfile::tempdir().unwrap_or_else(|e| panic!("tempdir: {e}")); + let journal = SimStorage::new(43); + let store = open(home.path(), &journal); + for seq in 1..=3 { + store.append(CHAT, &record(seq, "queued")).unwrap_or_else(|e| panic!("{e:#}")); + } + assert!(store.wait_durable(WAIT).unwrap_or_else(|e| panic!("{e:#}")), "never durable"); + FAIL_NEXT_LOG_SYNC.with(|fail| fail.set(true)); + let failed = store.flush_session(CHAT); + FAIL_NEXT_LOG_SYNC.with(|fail| fail.set(false)); + assert!(failed.is_err(), "flush_session's sync did not fail as the test asked"); + assert!(store.flush().is_err(), "a checkpoint after a failed sync in flush_session went ahead"); + let reason = store.stop_reason().and_then(|reason| reason.borrow().clone()); + assert!(reason.is_some(), "the store did not ask the daemon to stop"); +} diff --git a/cmux-tui/crates/acpmux/src/store/log_sync.rs b/cmux-tui/crates/acpmux/src/store/log_sync.rs index c5f11ecce3a1..fb92c521e3fe 100644 --- a/cmux-tui/crates/acpmux/src/store/log_sync.rs +++ b/cmux-tui/crates/acpmux/src/store/log_sync.rs @@ -5,10 +5,26 @@ use super::*; #[cfg(test)] thread_local! { - /// Tests: the next `LocalStore::sync_log` on this thread fails, as a failed fsync would. + /// Tests: the next sync of a log segment on this thread ([`sync_segment`]) fails, as a failed + /// fsync would. pub(crate) static FAIL_NEXT_LOG_SYNC: std::cell::Cell = const { std::cell::Cell::new(false) }; } +/// Syncs one log segment: its data, or with `all` its metadata too. The tests' crash model +/// follows each sync (`store/sync_model.rs`). +pub(super) fn sync_segment(file: &File, all: bool) -> std::io::Result<()> { + #[cfg(test)] + if FAIL_NEXT_LOG_SYNC.with(|fail| fail.replace(false)) { + #[cfg(unix)] + super::sync_model::synced(file, false); + return Err(std::io::Error::other("a failed sync (a test)")); + } + let result = if all { file.sync_all() } else { file.sync_data() }; + #[cfg(all(test, unix))] + super::sync_model::synced(file, result.is_ok()); + result +} + /// A sync call (`fsync`) that failed, as opposed to an error before it (a folder that does not /// open): what it was to make durable may be lost, and a later sync of the same file may report /// success without it (the journal store then keeps every line until a restart). @@ -32,10 +48,6 @@ impl LocalStore { /// the disk works (the journal store's checkpoint, `store/journal_store.rs`). A segment that /// is not open was synced at its close (`SegmentWriter::close`, which notes a failure). pub(super) fn sync_log(&self, id: &str) -> Result<()> { - #[cfg(test)] - if FAIL_NEXT_LOG_SYNC.with(|fail| fail.replace(false)) { - return Err(SyncFailed(std::io::Error::other("a failed sync (a test)")).into()); - } let open = { let writers = self.writers.lock_or_recover(); match writers.open.get(id) { @@ -44,7 +56,7 @@ impl LocalStore { } }; if let Some(file) = open { - file.sync_all().map_err(SyncFailed)?; + sync_segment(&file, true).map_err(SyncFailed)?; } let session = self.dir(id); sync_directory(&session.join("events"))?; diff --git a/cmux-tui/crates/acpmux/src/store/sync_model.rs b/cmux-tui/crates/acpmux/src/store/sync_model.rs new file mode 100644 index 000000000000..9655f772ffb4 --- /dev/null +++ b/cmux-tui/crates/acpmux/src/store/sync_model.rs @@ -0,0 +1,119 @@ +//! Tests: the event logs' crash model follows the syncs that ran (review 2 of #34901). +//! +//! At an OS crash a log segment keeps its length at its last sync that counts. A failed sync loses +//! what was written since the sync before it, and a later sync does not bring that back: the +//! kernel may mark those pages clean without writing them, and the next sync then reports success. +//! A cut at or before the lost bytes ends the loss (the bytes after it are written again). Each +//! segment is known by its real path and inode, so a test's files never meet another test's, and +//! a segment removed and made again starts with nothing kept. + +use std::collections::HashMap; +use std::fs::File; +use std::path::{Path, PathBuf}; +use std::sync::{LazyLock, Mutex}; + +#[derive(Clone, Copy, Default)] +struct Kept { + ino: u64, + /// The length the disk holds since the last sync that counts. + durable: u64, + /// A failed sync lost the bytes from here on, until a cut at or before it. + lost_from: Option, +} + +static FILES: LazyLock>> = LazyLock::new(Mutex::default); + +/// The real path of `file`'s descriptor. +#[cfg(target_os = "linux")] +fn path_of(file: &File) -> Option { + use std::os::fd::AsRawFd; + std::fs::read_link(format!("/proc/self/fd/{}", file.as_raw_fd())).ok() +} + +/// The real path of `file`'s descriptor. +#[cfg(target_os = "macos")] +fn path_of(file: &File) -> Option { + use std::os::fd::AsRawFd; + use std::os::unix::ffi::OsStrExt; + let mut buf = vec![0u8; usize::try_from(libc::PATH_MAX).ok()?]; + // SAFETY: F_GETPATH writes at most PATH_MAX bytes, a NUL-terminated path, into `buf`. + let got = unsafe { libc::fcntl(file.as_raw_fd(), libc::F_GETPATH, buf.as_mut_ptr()) }; + if got == -1 { + return None; + } + let len = buf.iter().position(|byte| *byte == 0)?; + Some(PathBuf::from(std::ffi::OsStr::from_bytes(buf.get(..len)?))) +} + +/// The real path of `file`'s descriptor (not known here: the model keeps nothing). +#[cfg(not(any(target_os = "linux", target_os = "macos")))] +fn path_of(_file: &File) -> Option { + None +} + +fn key(path: &Path) -> Option { + std::fs::canonicalize(path).ok() +} + +fn update(file: &File, change: impl FnOnce(&mut Kept, u64)) { + use std::os::unix::fs::MetadataExt; + let (Some(path), Ok(meta)) = (path_of(file).as_deref().and_then(key), file.metadata()) else { + return; + }; + let mut files = FILES.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + let kept = files.entry(path).or_default(); + if kept.ino != meta.ino() { + *kept = Kept { ino: meta.ino(), ..Kept::default() }; + } + change(kept, meta.len()); +} + +/// A sync of the log segment `file` ran; `ok` is its result. +pub(crate) fn synced(file: &File, ok: bool) { + update(file, |kept, len| { + if !ok { + kept.lost_from.get_or_insert(kept.durable); + } else if kept.lost_from.is_none() { + kept.durable = len; + } + }); +} + +/// The log segment `file` was cut to `len` (its sync follows). +pub(crate) fn cut(file: &File, len: u64) { + update(file, |kept, _| { + kept.durable = kept.durable.min(len); + if kept.lost_from.is_some_and(|from| from >= len) { + kept.lost_from = None; + } + }); +} + +/// An OS crash of the event logs under `sessions` (`/sessions`): each segment goes back to +/// what the disk kept of it, and a segment no sync counted for goes back to empty. +pub(crate) fn os_crash(sessions: &Path) { + use std::os::unix::fs::MetadataExt; + let Ok(chats) = std::fs::read_dir(sessions) else { return }; + let mut files = FILES.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + for chat in chats.flatten() { + let Ok(segments) = std::fs::read_dir(chat.path().join("events")) else { continue }; + for segment in segments.flatten() { + let path = segment.path(); + let (Some(real), Ok(meta)) = (key(&path), std::fs::metadata(&path)) else { continue }; + if !meta.is_file() || path.extension().is_none_or(|ext| ext != "ndjson") { + continue; + } + let kept = files.get(&real).copied().filter(|kept| kept.ino == meta.ino()); + let len = kept.map_or(0, |kept| { + kept.lost_from.map_or(kept.durable, |from| from.min(kept.durable)) + }); + let len = len.min(meta.len()); + let file = std::fs::OpenOptions::new() + .write(true) + .open(&path) + .unwrap_or_else(|e| panic!("open {}: {e}", path.display())); + file.set_len(len).unwrap_or_else(|e| panic!("cut {}: {e}", path.display())); + files.insert(real, Kept { ino: meta.ino(), durable: len, lost_from: None }); + } + } +} From 84f9c266975af5eec7216f751ea055e279ba7467 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 4 Nov 2026 12:02:53 +0000 Subject: [PATCH 45/52] acpmux: review 2 of #34901: a restart writes the replayed lines again; a torn line is not a line; flush_session freezes Finding 1 (high): the repair now writes every merged line again from the repair start, even when the log already has them all. A killed daemon's log writes may be in the page cache only, and after a failed sync the kernel may have dropped pages that still read back and that a later sync reports as written; the first checkpoint after the open then syncs bytes this process wrote before it retires their records. The lines only the log has from the repair start are journaled and made durable first, as before a cut. The work is bounded by the checkpoint interval. Finding 2: a piece of the log is a line only when it parses as one JSON value (piece_seq). A hole that starts inside a line keeps that line's {"seq":N, head and runs into a later line's tail; it was read as a line of a log that moved on, and the chat's journaled lines went aside. Now its bytes go and the lines are written again. Finding 3: a failed sync in flush_session freezes the checkpoint (its file sync is a SyncFailed now), as a failed sync in a checkpoint or at a log's close does. Nits: cut_at syncs events/ after its removals (and the kind-index folder after its copies go), so a removed segment cannot come back beside the lines written again; the README says that with the switch on an event line over 64 MiB is refused and its log does not get it. Tests: the four red tests of d2c63860ece. --- cmux-tui/crates/acpmux/README.md | 4 +- cmux-tui/crates/acpmux/src/store.rs | 2 +- .../crates/acpmux/src/store/journal_store.rs | 22 +++-- .../src/store/journal_store/checkpoint.rs | 2 +- .../acpmux/src/store/journal_store/repair.rs | 95 ++++++++++--------- 5 files changed, 70 insertions(+), 55 deletions(-) diff --git a/cmux-tui/crates/acpmux/README.md b/cmux-tui/crates/acpmux/README.md index d59cc371df2d..b441d8d362ac 100644 --- a/cmux-tui/crates/acpmux/README.md +++ b/cmux-tui/crates/acpmux/README.md @@ -1482,7 +1482,9 @@ When no config exists, harnesses are imported from `~/.acpx/config.json` (its `a could not write into a log that changed without the journal (a run with the switch off after a crash, or an older build) go to `journal-unapplied/`. Turning it off is safe: the next start writes the journal's lines into the logs first. A failed log sync stops the daemon for a restart, - which writes the logs again. Read when the daemon starts. + which writes the logs again. With it on, an event line over 64 MiB (the journal's record limit) + is refused, and its log does not get it either; with it off, the log takes it. Read when the + daemon starts. - `websocket`: optional. The same protocol over WebSocket text frames, for remote clients such as a dashboard or a Durable Object. Put it behind a tunnel; the token is a bearer token. diff --git a/cmux-tui/crates/acpmux/src/store.rs b/cmux-tui/crates/acpmux/src/store.rs index 52452b8ff7ac..27ddaad23b80 100644 --- a/cmux-tui/crates/acpmux/src/store.rs +++ b/cmux-tui/crates/acpmux/src/store.rs @@ -1542,7 +1542,7 @@ impl Store for LocalStore { fn flush_session(&self, id: &str) -> Result<()> { let writers = self.writers.lock_or_recover(); if let Some(writer) = writers.open.get(id) { - log_sync::sync_segment(&writer.file, true)?; + log_sync::sync_segment(&writer.file, true).map_err(SyncFailed)?; } let session = self.dir(id); sync_directory(&session.join("events"))?; diff --git a/cmux-tui/crates/acpmux/src/store/journal_store.rs b/cmux-tui/crates/acpmux/src/store/journal_store.rs index 0b6d0121657b..338a6b2b9d7a 100644 --- a/cmux-tui/crates/acpmux/src/store/journal_store.rs +++ b/cmux-tui/crates/acpmux/src/store/journal_store.rs @@ -17,12 +17,16 @@ //! the journal grew, past 64 MiB of it, at `flush` (shutdown) and before a retention rewrite. It //! syncs every log written since the last one and the sessions folder, then writes, durably, an //! LSN at or before the journal's durable end that every record up to has its log effect, then -//! retires the journal segments before it. A failed log sync (in it, or at a log's close) -//! freezes it: a later sync may report success for pages the kernel dropped, so the journal -//! keeps every line and the store asks the daemon to stop ([`Store::stop_reason`]); the next -//! start's replay writes the logs again. Ten failed checkpoints in a row ask the same. +//! retires the journal segments before it. A failed log sync (in it, at a log's close, or in +//! `flush_session`) freezes it: a later sync may report success for pages the kernel dropped, +//! so the journal keeps every line and the store asks the daemon to stop +//! ([`Store::stop_reason`]); the next start's replay writes the logs again. Ten failed +//! checkpoints in a row ask the same. //! - Recovery (open) repairs each log and never deletes a whole line or a folder: see -//! [`Inner::repair`]. A chat whose folder is gone is not written again. A log that moved on +//! [`Inner::repair`]. It writes every replayed chat's lines again from the point the journal +//! takes over, so the first checkpoint after it syncs bytes this process wrote (a killed +//! daemon's writes may be in the page cache only, and a failed sync's dropped pages still read +//! back). A chat whose folder is gone is not written again. A log that moved on //! without the journal (a run with the switch off, or an older build, after a crash) is left as //! it is, and the journal's lines for it go to `/journal-unapplied/`, as do the lines of a //! chat whose log cannot be read or written. A crash in the middle is repaired by the next open @@ -323,8 +327,14 @@ impl Store for JournalStore { } Ok(()) } + /// A failed file sync here freezes the checkpoint, as one in a checkpoint or at a log's + /// close does: a later sync may report success for the pages it dropped. fn flush_session(&self, id: &str) -> Result<()> { - Store::flush_session(&self.inner.local, id) + Store::flush_session(&self.inner.local, id).inspect_err(|error| { + if error.chain().any(|cause| cause.is::()) { + self.inner.freeze(format!("an event log failed to sync: {error:#}")); + } + }) } fn events(&self, id: &str, after: u64, limit: usize) -> Result> { Store::events(&self.inner.local, id, after, limit) diff --git a/cmux-tui/crates/acpmux/src/store/journal_store/checkpoint.rs b/cmux-tui/crates/acpmux/src/store/journal_store/checkpoint.rs index a8ab9092c51a..08fd0b7b77c8 100644 --- a/cmux-tui/crates/acpmux/src/store/journal_store/checkpoint.rs +++ b/cmux-tui/crates/acpmux/src/store/journal_store/checkpoint.rs @@ -162,7 +162,7 @@ impl Inner { } /// A log sync failed: no checkpoint until a restart, and the daemon is asked to stop. - fn freeze(&self, why: String) { + pub(super) fn freeze(&self, why: String) { self.order.lock_or_recover().frozen = true; self.ask_to_stop(why); } diff --git a/cmux-tui/crates/acpmux/src/store/journal_store/repair.rs b/cmux-tui/crates/acpmux/src/store/journal_store/repair.rs index 8fd720c04a2c..08ea0439b601 100644 --- a/cmux-tui/crates/acpmux/src/store/journal_store/repair.rs +++ b/cmux-tui/crates/acpmux/src/store/journal_store/repair.rs @@ -101,13 +101,19 @@ impl Inner { /// Repairs `id`'s log with its journaled lines (`journaled`, in LSN order): it never deletes a /// whole line or a folder. /// - /// The log is read from just after its last line with a `seq` below the first journaled one. - /// Its lines and the journal's merge by `seq`: a line in both must be the same bytes; a line - /// only the log has stays; a line only the journal has goes in its place. If that only adds - /// lines after the log's last one, they are appended. Otherwise (bytes that do not read, or a - /// line to put before a later one), the lines that only the log has from that point are - /// journaled and made durable first; then the log is cut there and the merged lines written, - /// so a crash at any step loses no line: the next open repairs the same way. + /// The log is read from just after its last line with a `seq` below the first journaled one + /// (the repair start). Its lines and the journal's merge by `seq`: a line in both must be the + /// same bytes; a line only the log has stays; a line only the journal has goes in its place. + /// A line is a whole JSON line: bytes that do not parse (a torn tail, a hole of zeros, a line + /// a hole ran into a later one) are not a line and go. + /// + /// Every merged line is written again from the repair start, even when the log has them all: + /// a killed daemon's log writes may be in the page cache only, and after a failed sync the + /// kernel may have dropped pages it still shows and that a later sync reports as written. So + /// the first checkpoint after this open syncs bytes this process wrote before it retires their + /// records (review 2 of #34901, finding 1). The lines that only the log has from the repair + /// start are journaled and made durable first; then the log is cut there and the merged lines + /// written, so a crash at any step loses no line: the next open repairs the same way. fn repair( &self, id: &str, @@ -146,48 +152,26 @@ impl Inner { merged.insert(*seq, bytes); } let merged: Vec<(u64, &[u8])> = merged.into_iter().collect(); - // How far the log already is the merged lines, and the first piece that is not. - let mut same = 0usize; - let mut cut = None; - for piece in &pieces { - let next = merged.get(same).copied(); - match (&piece.line, next) { - (Some((seq, bytes)), Some((want, line))) - if *seq == want && bytes.as_slice() == line => - { - same = same.saturating_add(1); - } - _ => { - cut = Some((piece.segment, piece.offset)); - break; - } - } - } - let rest = merged.get(same..).unwrap_or_default(); - if let Some((segment, offset)) = cut { - // The lines only the log holds from the cut on: into the journal first. - let only_logged: Vec<&[u8]> = rest + let keep_merged = || merged.iter().map(|(_, line)| line.to_vec()).collect::>(); + if let Some(start) = pieces.first() { + // The lines only the log holds from the repair start: into the journal first. + let only_logged: Vec<&[u8]> = merged .iter() .filter(|(seq, _)| !wanted.contains_key(seq)) .map(|(_, line)| *line) .collect(); self.journal_lines(id, &only_logged).map_err(chat_error)?; - let keep_rest = || rest.iter().map(|(_, line)| line.to_vec()).collect::>(); - self.cut_at(id, segment, offset) - .map_err(|error| RepairError::Chat { error, keep: keep_rest() })?; + self.cut_at(id, start.segment, start.offset) + .map_err(|error| RepairError::Chat { error, keep: keep_merged() })?; #[cfg(test)] if STOP_AFTER_CUT.with(std::cell::Cell::get) { return Err(RepairError::Stopped); } - for (_, line) in rest { - self.local - .append_line(id, line) - .map_err(|error| RepairError::Chat { error, keep: keep_rest() })?; - } - } else { - for (_, line) in rest { - self.local.append_line(id, line).map_err(chat_error)?; - } + } + for (_, line) in &merged { + self.local + .append_line(id, line) + .map_err(|error| RepairError::Chat { error, keep: keep_merged() })?; } Ok(Repaired::Done) } @@ -230,14 +214,16 @@ impl Inner { } /// Cuts `id`'s log at `offset` of segment `segment`: later segments go, and that one is - /// shortened. The kind-index copy of each segment that changes goes first: a crash between a - /// cut and its copy's removal would leave a copy that a log written back to the same length - /// passes for current. + /// shortened. The kind-index copy of each segment that changes goes first, durably: a crash + /// between a cut and its copy's removal would leave a copy that a log written back to the same + /// length passes for current. The removals are durable before the merged lines are written + /// again: else a crash could bring a removed segment back beside them. fn cut_at(&self, id: &str, segment: u32, offset: u64) -> Result<()> { let segments = self.local.segments(id)?; for (index, _) in segments.iter().filter(|(index, _)| *index >= segment) { self.forget_index_copy(id, *index)?; } + sync_if_there(&self.local.dir(id).join(kind_index::DIR))?; for (index, path) in segments.iter().rev() { if *index > segment { crate::config::refuse_lost_home(path)?; @@ -246,7 +232,7 @@ impl Inner { truncate_private(path, offset)?; } } - Ok(()) + sync_directory(&self.local.dir(id).join("events")) } fn forget_index_copy(&self, id: &str, index: u32) -> Result<()> { @@ -311,11 +297,11 @@ fn read_pieces(path: &Path, segment: u32, from: u64, out: &mut Vec) -> Re // line after them another. let zeros = line.iter().take_while(|byte| **byte == 0).count(); let after_zeros = line.get(zeros..).unwrap_or_default(); - if whole && let Some(seq) = line_seq(&line) { + if whole && let Some(seq) = piece_seq(&line) { out.push(Piece { segment, offset, line: Some((seq, line.clone())) }); } else if whole && zeros > 0 - && let Some(seq) = line_seq(after_zeros) + && let Some(seq) = piece_seq(after_zeros) { out.push(Piece { segment, offset, line: None }); let at = offset.saturating_add(u64::try_from(zeros).unwrap_or(u64::MAX)); @@ -327,6 +313,23 @@ fn read_pieces(path: &Path, segment: u32, from: u64, out: &mut Vec) -> Re } } +/// The `seq` of a piece that is one whole JSON line; `None` for bytes that are not. A hole an OS +/// crash left inside a line runs that line into a later one's tail: it keeps the first line's +/// `{"seq":N,` head but is not JSON (review 2 of #34901, finding 2). +fn piece_seq(line: &[u8]) -> Option { + serde_json::from_slice::(line).ok()?; + line_seq(line) +} + +/// Syncs the folder at `path`; nothing when it is not there. +fn sync_if_there(path: &Path) -> Result<()> { + match std::fs::symlink_metadata(path) { + Ok(_) => sync_directory(path), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(anyhow::Error::from(error).context(format!("stat {}", path.display()))), + } +} + /// The offset right after the last whole line of `path` whose `seq` reads and is below /// `from_seq`; 0 when there is none. fn end_of_last_line_before(path: &Path, from_seq: u64) -> Result { From db608285a7ecd021d5af433823c08454afeee8b9 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 4 Nov 2026 14:30:13 +0000 Subject: [PATCH 46/52] acpmux: the qos journal tests module is macOS-only (its Linux build held only an unused import: clippy red, review of #35400 F0) --- cmux-tui/crates/acpmux/src/qos.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmux-tui/crates/acpmux/src/qos.rs b/cmux-tui/crates/acpmux/src/qos.rs index e5e137e6d2d8..43bfd2264ad5 100644 --- a/cmux-tui/crates/acpmux/src/qos.rs +++ b/cmux-tui/crates/acpmux/src/qos.rs @@ -448,7 +448,8 @@ mod imp { } } -#[cfg(test)] +// Its tests read macOS disk policies only. +#[cfg(all(test, target_os = "macos"))] #[path = "qos_journal_tests.rs"] mod journal_tests; From 3916f84dd68b1750258001a9d568c103c0e151c9 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 4 Nov 2026 14:31:11 +0000 Subject: [PATCH 47/52] acpmux: review of #35400 F1 and F2, tests (red) - a_read_that_shows_a_line_covers_its_record (F1): a read sees a log line between its log write and its record (the appends' lock held there), and the reply built from it goes out with no wait for that record. - a_journal_that_syncs_nothing_asks_the_daemon_to_stop (F2): with a journal that makes nothing durable, every reply waits without a limit and nothing asks the daemon to stop. Test seams only: PAUSE_AFTER_LOG_WRITE, TEST_STALL_MS. --- .../crates/acpmux/src/server/durable_gate.rs | 4 ++ .../acpmux/src/server/durable_gate_tests.rs | 21 ++++++++ .../crates/acpmux/src/store/journal_store.rs | 19 ++++++++ .../acpmux/src/store/journal_store_tests.rs | 48 +++++++++++++++++++ 4 files changed, 92 insertions(+) diff --git a/cmux-tui/crates/acpmux/src/server/durable_gate.rs b/cmux-tui/crates/acpmux/src/server/durable_gate.rs index bb7fcb316b53..976f8c229d49 100644 --- a/cmux-tui/crates/acpmux/src/server/durable_gate.rs +++ b/cmux-tui/crates/acpmux/src/server/durable_gate.rs @@ -119,6 +119,10 @@ async fn run( } } +/// Tests: the stall limit in milliseconds when not 0. +#[cfg(test)] +pub(crate) static TEST_STALL_MS: AtomicU64 = AtomicU64::new(0); + #[cfg(all(test, unix))] #[path = "durable_gate_tests.rs"] mod durable_gate_tests; diff --git a/cmux-tui/crates/acpmux/src/server/durable_gate_tests.rs b/cmux-tui/crates/acpmux/src/server/durable_gate_tests.rs index c3e195d55297..93cbdc4fe38a 100644 --- a/cmux-tui/crates/acpmux/src/server/durable_gate_tests.rs +++ b/cmux-tui/crates/acpmux/src/server/durable_gate_tests.rs @@ -141,3 +141,24 @@ async fn agent_output_is_not_held_behind_another_chats_acked_change() { assert_eq!(update["params"]["update"]["content"]["text"], "hi", "{update}"); assert!(took < SYNC / 2, "agent output waited {took:?} behind another chat's change"); } + +/// F2: a journal that makes nothing durable (a hung or failing disk; the writer retries without +/// end) must not hold every reply without a limit: past the stall limit, the store asks the +/// daemon to stop for a restart. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_journal_that_syncs_nothing_asks_the_daemon_to_stop() { + use std::sync::atomic::Ordering; + super::TEST_STALL_MS.store(300, Ordering::SeqCst); + let home = tempfile::tempdir().unwrap_or_else(|e| panic!("tempdir: {e}")); + let journal = SimStorage::new(62); + let hub = hub(home.path(), &journal); + let (input, _outgoing) = connect(&hub).await; + let mut stop = hub.store.stop_reason().unwrap_or_else(|| panic!("no stop reason")); + journal.set_sync_delay(Duration::from_secs(3600)); + let rename = json!({"sessionId": CHAT, "newName": "renamed"}); + send(&input, Message::request(2, method::MUX_RENAME, rename)).await; + let asked = + tokio::time::timeout(Duration::from_secs(5), stop.wait_for(Option::is_some)).await.is_ok(); + journal.set_sync_delay(Duration::ZERO); + assert!(asked, "a reply waited past the stall limit and the store did not ask to stop"); +} diff --git a/cmux-tui/crates/acpmux/src/store/journal_store.rs b/cmux-tui/crates/acpmux/src/store/journal_store.rs index 1186b817967b..e0bbcb7358ca 100644 --- a/cmux-tui/crates/acpmux/src/store/journal_store.rs +++ b/cmux-tui/crates/acpmux/src/store/journal_store.rs @@ -83,6 +83,23 @@ const LOCK_WAIT: Duration = Duration::from_secs(5); /// The longest a repair waits for the lines it journals before it cuts a log. const REPAIR_WAIT: Duration = Duration::from_secs(60); +/// Tests: the next append stops right after its log write, says so on the first channel and +/// waits for the second (the appends' lock held, as at that point in a real append). +#[cfg(test)] +pub(crate) static PAUSE_AFTER_LOG_WRITE: Mutex< + Option<(std::sync::mpsc::Sender<()>, std::sync::mpsc::Receiver<()>)>, +> = Mutex::new(None); + +#[cfg(test)] +fn pause_after_log_write() { + let pause = + PAUSE_AFTER_LOG_WRITE.lock().unwrap_or_else(std::sync::PoisonError::into_inner).take(); + if let Some((arrived, release)) = pause { + let _ = arrived.send(()); + let _ = release.recv(); + } +} + pub struct JournalStore { inner: Arc, checkpointer: Mutex>>, @@ -368,6 +385,8 @@ impl Store for JournalStore { inner.journal.check_append(payload.len()).map_err(JournalStore::journal_error)?; // Refused once the store is sealed (under the same lock, so exact). inner.local.append_line(id, line.as_bytes())?; + #[cfg(test)] + pause_after_log_write(); if !order.dirty.contains(id) { order.dirty.insert(id.to_owned()); } diff --git a/cmux-tui/crates/acpmux/src/store/journal_store_tests.rs b/cmux-tui/crates/acpmux/src/store/journal_store_tests.rs index d3c4eb496d24..6cf2adac10a1 100644 --- a/cmux-tui/crates/acpmux/src/store/journal_store_tests.rs +++ b/cmux-tui/crates/acpmux/src/store/journal_store_tests.rs @@ -872,3 +872,51 @@ fn a_failed_sync_in_flush_session_freezes_the_checkpoint() { let reason = store.stop_reason().and_then(|reason| reason.borrow().clone()); assert!(reason.is_some(), "the store did not ask the daemon to stop"); } + +// ---------------------------------------------------------------- review of #35400 (S2c) + +/// F1: a read that shows a line returns only once that line's record is appended, so the reply +/// built from it raises its connection's gate to that record. Before, a read saw the log line +/// between the log write and the record, and the reply went out with no wait for it. +#[test] +fn a_read_that_shows_a_line_covers_its_record() { + use super::journal_store::PAUSE_AFTER_LOG_WRITE; + use std::sync::atomic::Ordering; + let home = tempfile::tempdir().unwrap_or_else(|e| panic!("tempdir: {e}")); + let journal = SimStorage::new(47); + let store: Arc = Arc::from(open(home.path(), &journal)); + let durability = store.durability().unwrap_or_else(|| panic!("no durability")); + store.append(CHAT, &record(1, "created")).unwrap_or_else(|e| panic!("{e:#}")); + assert!(store.wait_durable(WAIT).unwrap_or_else(|e| panic!("{e:#}")), "never durable"); + let (arrived_tx, arrived) = std::sync::mpsc::channel(); + let (release, release_rx) = std::sync::mpsc::channel(); + *PAUSE_AFTER_LOG_WRITE.lock().unwrap_or_else(|e| e.into_inner()) = + Some((arrived_tx, release_rx)); + let appender = { + let store = Arc::clone(&store); + std::thread::spawn(move || store.append(CHAT, &record(2, "queued"))) + }; + arrived.recv_timeout(WAIT).unwrap_or_else(|e| panic!("the append never paused: {e}")); + let reader = { + let store = Arc::clone(&store); + let acked = Arc::clone(&durability.acked); + std::thread::spawn(move || { + let seen = seqs(store.as_ref(), CHAT); + (seen, acked.load(Ordering::SeqCst)) + }) + }; + std::thread::sleep(Duration::from_millis(200)); + let _ = release.send(()); + appender + .join() + .unwrap_or_else(|_| panic!("append panicked")) + .unwrap_or_else(|e| panic!("{e:#}")); + let (seen, acked_then) = reader.join().unwrap_or_else(|_| panic!("read panicked")); + let acked_now = durability.acked.load(Ordering::SeqCst); + if seen.contains(&2) { + assert_eq!( + acked_then, acked_now, + "a read showed line 2 before its record was appended (acked {acked_then}, then {acked_now})" + ); + } +} From c553a19fd8b8ae39d5d9741205dea7d1d48aafb6 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 4 Nov 2026 14:36:41 +0000 Subject: [PATCH 48/52] acpmux: review of #35400 F1, the streaming read, test (red): session/load's replay sends a line before its record --- .../acpmux/src/server/durable_gate_tests.rs | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/cmux-tui/crates/acpmux/src/server/durable_gate_tests.rs b/cmux-tui/crates/acpmux/src/server/durable_gate_tests.rs index 93cbdc4fe38a..092765b708d0 100644 --- a/cmux-tui/crates/acpmux/src/server/durable_gate_tests.rs +++ b/cmux-tui/crates/acpmux/src/server/durable_gate_tests.rs @@ -162,3 +162,41 @@ async fn a_journal_that_syncs_nothing_asks_the_daemon_to_stop() { journal.set_sync_delay(Duration::ZERO); assert!(asked, "a reply waited past the stall limit and the store did not ask to stop"); } + +/// F1, the streaming read: `session/load`'s replay sends each record as it reads it +/// (`Hub::scan_events`). It must stop at the records whose journal record was appended when it +/// started: a line the log holds while its record is not appended is not sent. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_replay_never_sends_a_line_before_its_record() { + use crate::store::PAUSE_AFTER_LOG_WRITE; + let home = tempfile::tempdir().unwrap_or_else(|e| panic!("tempdir: {e}")); + let journal = SimStorage::new(63); + let hub = hub(home.path(), &journal); + let session = hub.resolve(CHAT).unwrap_or_else(|e| panic!("{e:?}")); + let first = hub.append(&session, "mux", "note", json!({"n": 1})); + let (arrived_tx, arrived) = std::sync::mpsc::channel(); + let (release, release_rx) = std::sync::mpsc::channel::<()>(); + *PAUSE_AFTER_LOG_WRITE.lock().unwrap_or_else(|e| e.into_inner()) = + Some((arrived_tx, release_rx)); + let appender = { + let (hub, session) = (hub.clone(), session.clone()); + std::thread::spawn(move || hub.append(&session, "mux", "note", json!({"n": 2})).seq) + }; + arrived + .recv_timeout(Duration::from_secs(10)) + .unwrap_or_else(|e| panic!("the append never paused: {e}")); + let releaser = std::thread::spawn(move || { + std::thread::sleep(Duration::from_millis(200)); + let _ = release.send(()); + }); + let mut seen = Vec::new(); + hub.scan_events(CHAT, 0, &mut |record| { + seen.push(record.seq); + true + }) + .unwrap_or_else(|e| panic!("{e:#}")); + releaser.join().unwrap_or_else(|_| panic!("releaser panicked")); + let second = appender.join().unwrap_or_else(|_| panic!("append panicked")); + assert!(seen.contains(&first.seq), "the replay missed line {}: {seen:?}", first.seq); + assert!(!seen.contains(&second), "the replay sent line {second} before its record: {seen:?}"); +} From 7ee04c03c92f1e7c0b0725ee73088c12ca6a2bc1 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 4 Nov 2026 14:37:43 +0000 Subject: [PATCH 49/52] acpmux: the test seam PAUSE_AFTER_LOG_WRITE is reachable from the server tests --- cmux-tui/crates/acpmux/src/store.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmux-tui/crates/acpmux/src/store.rs b/cmux-tui/crates/acpmux/src/store.rs index 8d92c9cfbd63..a1cd1dead8d8 100644 --- a/cmux-tui/crates/acpmux/src/store.rs +++ b/cmux-tui/crates/acpmux/src/store.rs @@ -1037,6 +1037,8 @@ mod journal_store; #[cfg(test)] pub(crate) use journal_store::open_with_journal; mod log_sync; +#[cfg(test)] +pub(crate) use journal_store::PAUSE_AFTER_LOG_WRITE; pub(crate) use log_sync::SyncFailed; #[cfg(all(test, unix))] mod sync_model; From d328b8a9689e866c010e2b658b08cd01f4a247f2 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 4 Nov 2026 14:40:38 +0000 Subject: [PATCH 50/52] acpmux: review of #35400: a read covers the records of the lines it shows; a stalled journal stops the daemon F1: the append writes a line to the log, then appends its record and sets the acknowledged LSN, all under the appends' lock; reads take no lock. A read could show a line before its record existed, and the reply built from it went out at once. Now every journal-store read of the logs ends with a fence on the appends' lock (`Inner::fenced`): a line the read saw has its record appended and its LSN acknowledged when the read returns, so the reply's gate covers it. The one read that sends as it reads (`Hub::scan_events`, session/load's replay) stops at the chat's last_seq read first. F2: a journal writer that keeps failing (it retries without end) held every reply on every connection with no limit. A connection's gate now waits at most 10 s (`STALL_LIMIT`); past it, `Durability::stalled` asks the daemon to stop for a restart, as a failed log sync does. Tests: the 3 of 3916f84dd68 and c553a19fd8b. --- cmux-tui/crates/acpmux/src/hub/mod.rs | 12 ++++- .../crates/acpmux/src/server/durable_gate.rs | 49 +++++++++++++++---- .../crates/acpmux/src/store/durability.rs | 3 ++ .../crates/acpmux/src/store/journal_store.rs | 37 +++++++++++--- .../src/store/journal_store/checkpoint.rs | 2 +- 5 files changed, 83 insertions(+), 20 deletions(-) diff --git a/cmux-tui/crates/acpmux/src/hub/mod.rs b/cmux-tui/crates/acpmux/src/hub/mod.rs index 638815f0d42c..47f530460cfd 100644 --- a/cmux-tui/crates/acpmux/src/hub/mod.rs +++ b/cmux-tui/crates/acpmux/src/hub/mod.rs @@ -2314,14 +2314,22 @@ impl Hub { self.store.events(id, after, limit) } - /// Visits the records past `after` in order, one at a time, until `visit` returns false. + /// Visits the records past `after` in order, one at a time, until `visit` returns false. A + /// caller sends each record as it reads it (`session/load`'s replay), so the visit stops at + /// the chat's `last_seq` read first: a line the log holds before its journal record is + /// appended (R-DURABILITY S2c) is never sent ahead of that record. A chat no session holds + /// takes no appends: read to the end. pub fn scan_events( &self, id: &str, after: u64, visit: &mut dyn FnMut(EventRecord) -> bool, ) -> Result<()> { - self.store.scan(id, after, visit) + let live = self.sessions.lock().unwrap_or_else(|e| e.into_inner()).get(id).cloned(); + let Some(last) = live.map(|session| session.meta().last_seq) else { + return self.store.scan(id, after, visit); + }; + self.store.scan(id, after, &mut |record| record.seq <= last && visit(record)) } pub fn session_dir(&self, id: &str) -> Option { diff --git a/cmux-tui/crates/acpmux/src/server/durable_gate.rs b/cmux-tui/crates/acpmux/src/server/durable_gate.rs index 976f8c229d49..f16641f0fa26 100644 --- a/cmux-tui/crates/acpmux/src/server/durable_gate.rs +++ b/cmux-tui/crates/acpmux/src/server/durable_gate.rs @@ -19,6 +19,7 @@ use crate::store::Durability; use std::cell::Cell; use std::sync::Arc; use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::Duration; use tokio::sync::{mpsc, watch}; /// What a connection's lines wait for. @@ -70,13 +71,27 @@ pub(super) fn install( durability: Option, out: mpsc::Sender, ) -> (mpsc::Sender, Option>) { - let Some(Durability { acked, durable }) = durability else { return (out, None) }; + let Some(Durability { acked, durable, stalled }) = durability else { return (out, None) }; let gate = Arc::new(Gate { needs: AtomicU64::new(0), acked }); let (lines_tx, lines) = mpsc::channel(out.max_capacity()); - tokio::spawn(run(lines, out, Arc::clone(&gate), durable)); + tokio::spawn(run(lines, out, Arc::clone(&gate), durable, stalled)); (lines_tx, Some(gate)) } +/// How long a connection's lines may wait for the journal before the store asks the daemon to +/// stop for a restart (review of #35400, F2): a sync takes milliseconds, a few seconds on a +/// disk under heavy load; this long only on a hung or failing disk, whose writer retries without +/// end and would hold every reply. +const STALL_LIMIT: Duration = Duration::from_secs(10); + +fn stall_limit() -> Duration { + #[cfg(test)] + if let ms @ 1.. = TEST_STALL_MS.load(Ordering::SeqCst) { + return Duration::from_millis(ms); + } + STALL_LIMIT +} + /// The gate's task ([module docs](self)). Ends when the connection's senders are gone, the /// transport closed, or the journal's writer ended. async fn run( @@ -84,6 +99,7 @@ async fn run( out: mpsc::Sender, gate: Arc, mut durable: watch::Receiver, + stalled: Arc, ) { let limit = lines.max_capacity(); let mut batch = Vec::with_capacity(limit.min(64)); @@ -100,13 +116,28 @@ async fn run( // After the lines were taken: every line among them raised the gate before it was queued. let needs = gate.needs.load(Ordering::SeqCst); if *durable.borrow() < needs { - tokio::select! { - biased; - () = out.closed() => return, - reached = durable.wait_for(|lsn| *lsn >= needs) => { - if reached.is_err() { - tracing::error!("the event journal's writer ended: this connection closes"); - return; + let reached = durable.wait_for(|lsn| *lsn >= needs); + tokio::pin!(reached); + let stall = tokio::time::sleep(stall_limit()); + tokio::pin!(stall); + let mut asked = false; + loop { + tokio::select! { + biased; + () = out.closed() => return, + reached = &mut reached => { + if reached.is_err() { + tracing::error!("the event journal's writer ended: this connection closes"); + return; + } + break; + } + () = &mut stall, if !asked => { + asked = true; + stalled(format!( + "a reply waited {:?} for the event journal, which made nothing durable", + stall_limit() + )); } } } diff --git a/cmux-tui/crates/acpmux/src/store/durability.rs b/cmux-tui/crates/acpmux/src/store/durability.rs index 5924bb2ee818..e69dbc640dd4 100644 --- a/cmux-tui/crates/acpmux/src/store/durability.rs +++ b/cmux-tui/crates/acpmux/src/store/durability.rs @@ -14,6 +14,9 @@ pub struct Durability { /// The journal's durable LSN. Closed when the journal's writer ended: nothing becomes /// durable any more. pub durable: tokio::sync::watch::Receiver, + /// Called when a line waited past the stall limit for the journal (a hung or failing disk: + /// its writer retries without end): the store asks the daemon to stop for a restart. + pub stalled: Arc, } /// A streamed agent line: a `Stream` record, synced with the next acked batch or a second later, diff --git a/cmux-tui/crates/acpmux/src/store/journal_store.rs b/cmux-tui/crates/acpmux/src/store/journal_store.rs index e0bbcb7358ca..57eb6a7b023d 100644 --- a/cmux-tui/crates/acpmux/src/store/journal_store.rs +++ b/cmux-tui/crates/acpmux/src/store/journal_store.rs @@ -90,6 +90,16 @@ pub(crate) static PAUSE_AFTER_LOG_WRITE: Mutex< Option<(std::sync::mpsc::Sender<()>, std::sync::mpsc::Receiver<()>)>, > = Mutex::new(None); +impl Inner { + /// Runs `read`, then waits for an append under way to end (`order` taken and let go): every + /// line `read` saw has its record appended by the time this returns. + fn fenced(&self, read: impl FnOnce() -> T) -> T { + let out = read(); + drop(self.order.lock_or_recover()); + out + } +} + #[cfg(test)] fn pause_after_log_write() { let pause = @@ -415,11 +425,16 @@ impl Store for JournalStore { } }) } + // Every read of the logs ends with a fence on the appends' lock (`Inner::fenced`): a line + // the read saw has its record appended, and its acknowledged LSN set, when the read returns, + // so a reply built from the read raises its connection's gate past that record (review of + // #35400, F1). A read that sends as it reads stops at the chat's `last_seq` instead + // (`Hub::scan_events`). fn events(&self, id: &str, after: u64, limit: usize) -> Result> { - Store::events(&self.inner.local, id, after, limit) + self.inner.fenced(|| Store::events(&self.inner.local, id, after, limit)) } fn scan(&self, id: &str, after: u64, visit: &mut dyn FnMut(EventRecord) -> bool) -> Result<()> { - Store::scan(&self.inner.local, id, after, visit) + self.inner.fenced(|| Store::scan(&self.inner.local, id, after, visit)) } fn scan_strict( &self, @@ -427,7 +442,7 @@ impl Store for JournalStore { after: u64, visit: &mut dyn FnMut(EventRecord) -> bool, ) -> Result<()> { - Store::scan_strict(&self.inner.local, id, after, visit) + self.inner.fenced(|| Store::scan_strict(&self.inner.local, id, after, visit)) } fn scan_back( &self, @@ -437,10 +452,10 @@ impl Store for JournalStore { skip: &dyn Fn(&[u8], &[u8]) -> bool, visit: &mut dyn FnMut(EventRecord) -> bool, ) -> Result<()> { - Store::scan_back(&self.inner.local, id, after, before, skip, visit) + self.inner.fenced(|| Store::scan_back(&self.inner.local, id, after, before, skip, visit)) } fn tail_after(&self, id: &str, after: u64) -> Result<(u64, u64)> { - Store::tail_after(&self.inner.local, id, after) + self.inner.fenced(|| Store::tail_after(&self.inner.local, id, after)) } fn scan_kinds( &self, @@ -448,7 +463,7 @@ impl Store for JournalStore { kinds: &[&str], visit: &mut dyn FnMut(EventRecord) -> bool, ) -> Result<()> { - Store::scan_kinds(&self.inner.local, id, kinds, visit) + self.inner.fenced(|| Store::scan_kinds(&self.inner.local, id, kinds, visit)) } fn scan_kinds_strict( &self, @@ -456,10 +471,10 @@ impl Store for JournalStore { kinds: &[&str], visit: &mut dyn FnMut(EventRecord) -> bool, ) -> Result<()> { - Store::scan_kinds_strict(&self.inner.local, id, kinds, visit) + self.inner.fenced(|| Store::scan_kinds_strict(&self.inner.local, id, kinds, visit)) } fn scan_heads(&self, id: &str, visit: &mut dyn FnMut(&RecordHead<'_>) -> bool) -> Result<()> { - Store::scan_heads(&self.inner.local, id, visit) + self.inner.fenced(|| Store::scan_heads(&self.inner.local, id, visit)) } /// A checkpoint: every log written so far is synced and the journal before it retired. fn flush(&self) -> Result<()> { @@ -480,9 +495,15 @@ impl Store for JournalStore { Some(self.inner.stop.subscribe()) } fn durability(&self) -> Option { + let inner = Arc::downgrade(&self.inner); Some(Durability { acked: Arc::clone(&self.inner.acked), durable: self.inner.durable.clone(), + stalled: Arc::new(move |why: String| { + if let Some(inner) = inner.upgrade() { + inner.ask_to_stop(why); + } + }), }) } /// The logs refuse writes, and the journal fully syncs every record (the shutdown's `flush` diff --git a/cmux-tui/crates/acpmux/src/store/journal_store/checkpoint.rs b/cmux-tui/crates/acpmux/src/store/journal_store/checkpoint.rs index fd430cfe9d8d..4db267e4faaf 100644 --- a/cmux-tui/crates/acpmux/src/store/journal_store/checkpoint.rs +++ b/cmux-tui/crates/acpmux/src/store/journal_store/checkpoint.rs @@ -167,7 +167,7 @@ impl Inner { self.ask_to_stop(why); } - fn ask_to_stop(&self, why: String) { + pub(super) fn ask_to_stop(&self, why: String) { tracing::error!( "{why}: the event journal keeps every line; the daemon stops so that its restart writes the logs again" ); From 39f9580eb7e889ef42e5befb6241ca028cee9920 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 4 Nov 2026 14:46:13 +0000 Subject: [PATCH 51/52] threading-model: the acpmux event journal's threads and bounds (writer, checkpointer, per-connection gate; review of #35400) --- plans/cmux-next/threading-model.md | 1 + 1 file changed, 1 insertion(+) diff --git a/plans/cmux-next/threading-model.md b/plans/cmux-next/threading-model.md index 45ff88680378..adcc8f1618a0 100644 --- a/plans/cmux-next/threading-model.md +++ b/plans/cmux-next/threading-model.md @@ -71,6 +71,7 @@ and changes both with the code). | **Per-stream lanes** | Not on feat-cmux-next yet (PL-conn, `pl-conn`) | | | Added here when it lands | | **Holder** (`cmux-holder`, one `poll` loop; daemon side `holder-link`) | PTYs that outlive the daemon | `poll` | not set | 1 MiB ring per terminal, 1 MiB link queue, 512 KiB credit window; daemon attached: the holder stops reading the PTY (the shell blocks); no daemon: the ring keeps the newest bytes | | **acpmux** (tokio multi-thread; `acpmux-socket-read` / `-write` bridges) | Agent sessions | Socket I/O on its runtime | not set | In 256 / out 4096 per connection, full outbox drops (`try_send`), oversized notices become `_acpmux/lagged`; hub broadcast 8192 with `lagged {dropped}`. Owner PL-acpmux (cmux-4f) | +| **acpmux event journal** (`store.journal`, default off; R-DURABILITY S2/S2c): the `cmux-journal` writer thread, `acpmux-journal-checkpoint` (one thread), one outbound gate task per connection (tokio) | Durable event-log records (group commit); the checkpoint (log syncs, retirement); holding a connection's lines until what they show is durable | The writer: disk syncs. The checkpointer: log syncs. A gate task: nothing (an async wait on the durable watch). Appends and reads take the appends' lock only around memory work and one buffered log write | With the journal on, the daemon keeps the default disk tier (a reply the person waits for never waits on a throttled sync); the checkpointer runs at utility (thread scope) | The writer's queue (a refused append fails the request); a gate waits at most 10 s for the journal, then the store asks the daemon to stop for a restart (exit 74); a checkpoint every 30 s while there is work, past 64 MiB of journal, at shutdown. Owner R-DURABILITY | | **Chromium** (in-process browser: UI thread = the app's main thread; Chromium's own IO and worker threads; renderer, GPU and utility helper processes; out-of-process host prototype `com.cmuxterm.chromium-host.read` / `.exit`) | Web pages | Chromium's IO thread blocks on its own sockets, never the UI thread | Chromium's | Child exits on `com.cmuxterm.app.next.cef-children`; host prototype: 256 KiB send buffer, 4096 frames, over: the connection ends | ## QoS and task priority (PL-qos, 2026-10-24) From eb24c40e1dfe23615f170567d9c46dddb7f71371 Mon Sep 17 00:00:00 2001 From: lawrencecchen <54008264+lawrencecchen@users.noreply.github.com> Date: Wed, 4 Nov 2026 14:57:45 +0000 Subject: [PATCH 52/52] acpmux: the stall test's journal syncs take 3 s, not an hour (a sync under way kept its delay and hung the test's end) --- .../crates/acpmux/src/server/durable_gate_tests.rs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/cmux-tui/crates/acpmux/src/server/durable_gate_tests.rs b/cmux-tui/crates/acpmux/src/server/durable_gate_tests.rs index 092765b708d0..3d20d2ddb660 100644 --- a/cmux-tui/crates/acpmux/src/server/durable_gate_tests.rs +++ b/cmux-tui/crates/acpmux/src/server/durable_gate_tests.rs @@ -152,14 +152,18 @@ async fn a_journal_that_syncs_nothing_asks_the_daemon_to_stop() { let home = tempfile::tempdir().unwrap_or_else(|e| panic!("tempdir: {e}")); let journal = SimStorage::new(62); let hub = hub(home.path(), &journal); - let (input, _outgoing) = connect(&hub).await; + let (input, mut outgoing) = connect(&hub).await; let mut stop = hub.store.stop_reason().unwrap_or_else(|| panic!("no stop reason")); - journal.set_sync_delay(Duration::from_secs(3600)); + // Each sync takes 3 s: past the 300 ms stall limit, and short enough that the journal ends + // (a sync under way keeps its delay, and the store's drop waits for it). + journal.set_sync_delay(Duration::from_secs(3)); let rename = json!({"sessionId": CHAT, "newName": "renamed"}); send(&input, Message::request(2, method::MUX_RENAME, rename)).await; - let asked = - tokio::time::timeout(Duration::from_secs(5), stop.wait_for(Option::is_some)).await.is_ok(); + let asked = tokio::time::timeout(Duration::from_millis(2500), stop.wait_for(Option::is_some)) + .await + .is_ok(); journal.set_sync_delay(Duration::ZERO); + let _ = until(&mut outgoing, |line| line["id"] == 2).await; assert!(asked, "a reply waited past the stall limit and the store did not ask to stop"); }