Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions crates/ironclaw_filesystem/src/catalog.rs
Original file line number Diff line number Diff line change
Expand Up @@ -361,6 +361,17 @@ impl RootFilesystem for CompositeRootFilesystem {
self.matching_mount(path)?.backend.delete(path).await
}

async fn delete_if_version(
&self,
path: &VirtualPath,
expected_version: RecordVersion,
) -> Result<(), FilesystemError> {
self.matching_mount(path)?
.backend
.delete_if_version(path, expected_version)
.await
}

async fn create_dir_all(&self, path: &VirtualPath) -> Result<(), FilesystemError> {
self.matching_mount(path)?
.backend
Expand Down
49 changes: 49 additions & 0 deletions crates/ironclaw_filesystem/src/hsm.rs
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,20 @@ impl RootFilesystem for HsmBackend {
async fn delete(&self, path: &VirtualPath) -> Result<(), FilesystemError> {
self.inner.delete(path).await
}

async fn delete_if_version(

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Medium — PR body still says HsmBackend is out of scope.

The current PR body lists HsmBackend as out of scope, but the diff implements HsmBackend::delete_if_version and adds HSM coverage. The review-discipline rule says a PR's title and body must match its diff, so the description is now stale after the HSM fix landed.

Fix: Update the PR body to include HsmBackend support, or remove the HsmBackend implementation and tests from this PR.

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed in 1792aeb: updated the PR body via gh pr edit. Scope now lists every backend that implements delete_if_version (in-memory, libSQL, Postgres, HsmBackend) and every forwarding wrapper (ScopedFilesystem, CompositeRootFilesystem, MountScopedRootFilesystem in ironclaw_host_runtime, SkillManagementRootFilesystem in ironclaw_skills), notes the narrowed RecordVersion -> i64 overflow guard (CorruptRecordVersion before any statement runs), and states the ABA (version-not-generation-stable) invariant. Footer (🤖 Generated with [Claude Code](https://claude.com/claude-code)) preserved.

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Fixed pre-session in round 4 (1792aeb's commit message notes the PR-body sync). Confirmed the current PR body correctly lists HsmBackend as delegating to its inner backend, not out of scope. Predates this session; leaving unresolved for the reviewer of record.

&self,
path: &VirtualPath,
expected_version: RecordVersion,
) -> Result<(), FilesystemError> {
// Review fix (PR #5749): the HSM declares `Delete` + `TxnCapability::Cas`
// (see `declared_capabilities` above), so it must actually serve
// `delete_if_version` rather than fall through to the trait default
// `Unsupported` — that combination would pass mount-time capability
// validation but fail at call time. Delegate to the inner backend,
// same as every other unified-entry-plane method on this wrapper.
self.inner.delete_if_version(path, expected_version).await
}
}

#[cfg(test)]
Expand Down Expand Up @@ -219,6 +233,41 @@ mod tests {
));
}

#[tokio::test]
async fn delete_if_version_delegates_to_inner_backend() {
// Review fix (PR #5749): HsmBackend declares Delete + TxnCapability::Cas,
// so delete_if_version must actually delegate to the inner backend
// instead of hitting the trait default `Unsupported`.
let hsm = HsmBackend::new();
let path = vpath("/secrets/account/api-key");

let version = hsm
.put(
&path,
Entry::bytes(b"ciphertext-blob".to_vec()),
CasExpectation::Absent,
)
.await
.unwrap();

// Wrong version is rejected with VersionMismatch, proving the call
// actually reached the inner backend's CAS logic rather than a stub.
let other_version = version.next();
let err = hsm
.delete_if_version(&path, other_version)
.await
.unwrap_err();
assert!(matches!(err, FilesystemError::VersionMismatch { .. }));

// Correct version deletes.
hsm.delete_if_version(&path, version).await.unwrap();
assert!(hsm.get(&path).await.unwrap().is_none());

// Now absent: NotFound, not Unsupported.
let err = hsm.delete_if_version(&path, version).await.unwrap_err();
assert!(matches!(err, FilesystemError::NotFound { .. }));
}

#[tokio::test]
async fn composite_rejects_overclaimed_hsm_descriptor() {
let mut composite = CompositeRootFilesystem::new();
Expand Down
128 changes: 128 additions & 0 deletions crates/ironclaw_filesystem/src/in_memory.rs
Original file line number Diff line number Diff line change
Expand Up @@ -167,6 +167,33 @@ impl RootFilesystem for InMemoryBackend {
Ok(())
}

async fn delete_if_version(
&self,
path: &VirtualPath,
expected_version: RecordVersion,
) -> Result<(), FilesystemError> {
// Deliberately not `check_cas`: its Version arm collapses an absent
// row into VersionMismatch{found: None}, but delete callers must see
// absent as NotFound (already gone) vs present-at-another-version as
// VersionMismatch (gone stale). Single-key: no subtree/log sweep.
let mut state = self.state.lock().await;
let Some(current) = state.entries.get(path).map(|stored| stored.version) else {
return Err(FilesystemError::NotFound {
path: path.clone(),
operation: FilesystemOperation::Delete,
});
};
if expected_version != current {
return Err(FilesystemError::VersionMismatch {
path: path.clone(),
expected: Some(expected_version),
found: Some(current),
});
}
state.entries.remove(path);
Ok(())
}

async fn list_dir(&self, path: &VirtualPath) -> Result<Vec<DirEntry>, FilesystemError> {
let state = self.state.lock().await;
let prefix = with_trailing_slash(path.as_str());
Expand Down Expand Up @@ -781,6 +808,107 @@ mod tests {
assert!(matches!(err, FilesystemError::VersionMismatch { .. }));
}

#[tokio::test]
async fn delete_if_version_distinguishes_missing_stale_and_current() {
let fs = InMemoryBackend::new();
let path = vpath("/secrets/leases/cas-delete");

// Missing path is NotFound (already gone), never VersionMismatch.
let err = fs
.delete_if_version(&path, RecordVersion::from_backend(1))
.await
.unwrap_err();
assert!(matches!(
err,
FilesystemError::NotFound {
operation: FilesystemOperation::Delete,
..
}
));

// Simulates the state a concurrent writer would leave behind (this
// is a sequential script, not a real race — see
// concurrent_cas_storm.rs for genuine parallel coverage): the
// version the deleter read (v1) is bumped by a put before the
// delete lands. The stale delete loses with the observed version
// and the entry survives at v2.
let v1 = fs
.put(&path, Entry::bytes(vec![1]), CasExpectation::Absent)
.await
.unwrap();
let log_seq = fs.append(&path, b"kept".to_vec()).await.unwrap();
let v2 = fs
.put(&path, Entry::bytes(vec![2]), CasExpectation::Version(v1))
.await
.unwrap();
let err = fs.delete_if_version(&path, v1).await.unwrap_err();
match err {
FilesystemError::VersionMismatch {
expected, found, ..
} => {
assert_eq!(expected, Some(v1));
assert_eq!(found, Some(v2));
}
other => panic!("expected VersionMismatch, got {other:?}"),
}
let got = fs.get(&path).await.unwrap().unwrap();
assert_eq!(got.version, v2);
assert_eq!(got.entry.body, vec![2]);

// Correct version deletes exactly the entry — single-key, so the
// event log at the same path survives (blind delete would sweep it).
fs.delete_if_version(&path, v2).await.unwrap();
assert!(fs.get(&path).await.unwrap().is_none());
let log = fs.tail(&path, SeqNo::ZERO).await.unwrap();
assert_eq!(log.len(), 1);
assert_eq!(log[0].seq, log_seq);
}

/// Pins the ABA hazard `delete_if_version`'s doc comment warns about:
/// version tokens are not generation-stable, so a version captured
/// before a delete+recreate cycle can match a *different* incarnation
/// of the same path. This is documented, known behavior — not a bug —
/// but was previously asserted only in prose. Round-A review finding
/// (PR #5749): pin it with a real assertion so a future change to this
/// contract (e.g. generation-stable versions) breaks a test loudly
/// instead of silently.
#[tokio::test]
async fn delete_if_version_is_vulnerable_to_aba_across_delete_recreate_cycles() {
let fs = InMemoryBackend::new();
let path = vpath("/secrets/leases/cas-delete-aba");

// First incarnation: created and fully deleted.
let v1_first = fs
.put(&path, Entry::bytes(vec![1]), CasExpectation::Absent)
.await
.unwrap();
fs.delete_if_version(&path, v1_first).await.unwrap();
assert!(fs.get(&path).await.unwrap().is_none());

// Second incarnation restarts the version counter at the same raw
// value as the first (version is not generation-stable).
let v1_second = fs
.put(&path, Entry::bytes(vec![2]), CasExpectation::Absent)
.await
.unwrap();
assert_eq!(
v1_first, v1_second,
"version must restart after a full delete, or this ABA hazard doesn't apply"
);

// The stale `v1_first` token — captured against the first
// incarnation — incorrectly authorizes deleting the second
// incarnation's live data. This is the documented hazard, not a
// regression: callers that recreate paths must pair every
// successful delete with an unconditional postcondition recheck,
// per the trait doc comment.
fs.delete_if_version(&path, v1_first).await.unwrap();
assert!(
fs.get(&path).await.unwrap().is_none(),
"stale version token wrongly matched and deleted the second incarnation"
);
}

#[tokio::test]
async fn query_filters_on_indexed_projection() {
let fs = InMemoryBackend::new();
Expand Down
145 changes: 145 additions & 0 deletions crates/ironclaw_filesystem/src/libsql.rs
Original file line number Diff line number Diff line change
Expand Up @@ -802,6 +802,52 @@ impl RootFilesystem for LibSqlRootFilesystem {
Ok(())
}

async fn delete_if_version(
&self,
path: &VirtualPath,
expected_version: RecordVersion,
) -> Result<(), FilesystemError> {
// Single-key CAS delete: unlike `delete`, no subtree/event/sequence
// sweep. `is_dir = 0` scopes it to the record plane, matching `put`'s
// Version arm and `current_version_libsql`.
//
// Review fix (PR #5749): the conditional DELETE and the zero-rows
// diagnosis read must be atomic w.r.t. a concurrent delete+recreate
// on the same path, or the diagnosis can observe a version written
// *after* our DELETE decided 0 rows matched, misclassifying the
// outcome. `BEGIN IMMEDIATE` takes the write lock up front (same
// idiom as `put`) so the DELETE and the follow-up SELECT run as one
// unit on one connection — this also keeps the call stack to a
// single checkout, matching the one-checkout-per-call-stack
// invariant the bounded pool (see `libsql_pool`, issue #5466) enforces
// (no nested `self.connect()`).
//
// Round-A review: validate `expected_version` before taking the
// pool checkout / write lock. An out-of-range version can never
// match a real row, so failing closed here avoids holding a
// contended connection (and SQLite's write lock) for a call
// destined to error — relevant under the concurrent CAS storms
// this pool exists to survive.
let expected_raw = record_version_to_i64(path, expected_version)?;
let conn = self.connect().await?;
conn.execute("BEGIN IMMEDIATE", ())
.await
.map_err(|error| libsql_db_error(path.clone(), FilesystemOperation::Delete, error))?;
let result =
delete_if_version_libsql_inner(&conn, path, expected_version, expected_raw).await;
match result {
Ok(()) => conn
.execute("COMMIT", ())
.await
.map(|_| ())
.map_err(|error| libsql_db_error(path.clone(), FilesystemOperation::Delete, error)),
Err(err) => {
let _ = conn.execute("ROLLBACK", ()).await;
Err(err)
}
}
}

async fn append(&self, path: &VirtualPath, payload: Vec<u8>) -> Result<SeqNo, FilesystemError> {
let conn = self.connect().await?;
// INTEGER PRIMARY KEY AUTOINCREMENT assigns a fresh monotonic id per
Expand Down Expand Up @@ -1314,6 +1360,43 @@ async fn current_version_libsql(
Ok(Some(record_version_from_i64(path, version)?))
}

/// Body of `delete_if_version` extracted so the outer caller can wrap the
/// conditional DELETE and the zero-rows diagnosis SELECT in one
/// BEGIN IMMEDIATE / COMMIT transaction, with a single ROLLBACK path.
/// Running both statements on the same connection inside the same
/// transaction is what makes the classification atomic: nothing else can
/// delete-then-recreate the row between the DELETE and the diagnosis read.
#[cfg(feature = "libsql")]
async fn delete_if_version_libsql_inner(
conn: &libsql::Connection,
path: &VirtualPath,
expected_version: RecordVersion,
expected_raw: i64,
) -> Result<(), FilesystemError> {
let deleted = conn
.execute(
"DELETE FROM root_filesystem_entries \
WHERE path = ?1 AND is_dir = 0 AND version = ?2",
libsql::params![path.as_str(), expected_raw],
)
.await
.map_err(|error| libsql_db_error(path.clone(), FilesystemOperation::Delete, error))?;
if deleted > 0 {
return Ok(());
}
// 0 rows: absent row → NotFound (already gone, benign); row present
// at another version → VersionMismatch (gone stale). Distinct from
// put's diagnosis, which collapses absent into VersionMismatch.
if let Some(found) = current_version_libsql(conn, path).await? {

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Low — libSQL CAS-delete diagnosis can report a read operation.

delete_if_version_libsql_inner reuses current_version_libsql for the zero-row diagnosis, but that helper maps query/row errors as FilesystemOperation::ReadFile. The sibling Postgres CAS-delete path maps the same delete-side query failure as FilesystemOperation::Delete, and the surrounding libSQL delete statements also use Delete, so a libSQL diagnosis failure from a delete call would surface with the wrong operation shape in errors and telemetry.

Fix: Either pass the caller operation into current_version_libsql or add a delete-specific current-version helper so the delete_if_version diagnosis maps backend errors with FilesystemOperation::Delete.

Also flagged by: tests/Low (libSQL corrupt stored version path is untested)

return Err(FilesystemError::VersionMismatch {
path: path.clone(),
expected: Some(expected_version),
found: Some(found),
});
}
Err(not_found(path.clone(), FilesystemOperation::Delete))
}

/// Body of `run_migrations` extracted so the outer caller can wrap the
/// whole sequence in BEGIN IMMEDIATE / COMMIT with one rollback path.
#[cfg(feature = "libsql")]
Expand Down Expand Up @@ -2255,4 +2338,66 @@ mod tests {
other => panic!("expected FilesystemError::BackendInfrastructure, got {other:?}"),
}
}

/// Deterministic, single-task regression pin for the atomicity fix
/// (commit 1792aebb2 / PR #5749 round 4): `delete_if_version`'s
/// zero-rows diagnosis must reuse the SAME connection the conditional
/// DELETE ran on, not check out a second one. Round-B review finding:
/// the concurrency storm test in `tests/concurrent_cas_storm.rs`
/// doesn't actually discriminate this — every racer shares one
/// pre-fetched version and nothing recreates the path mid-round, so
/// it passes with or without the fix. This test does discriminate it,
/// with no concurrency required: build a deliberately size-1 pool (via
/// `build_libsql_pool_with_config`), let `delete_if_version` check out
/// its only connection, and hit the stale-version (0-rows) branch. If
/// the diagnosis internally called `self.connect()` again — the
/// pre-fix pattern — that second checkout would deadlock against the
/// first (nothing else can return the only connection) and time out;
/// reusing the passed-in `conn` completes immediately.
#[tokio::test]
async fn delete_if_version_diagnosis_reuses_the_delete_connection_under_a_size_one_pool() {
let dir = tempfile::tempdir().unwrap();
let db_path = dir.path().join("delete-single-conn-test.db");
let db = std::sync::Arc::new(libsql::Builder::new_local(db_path).build().await.unwrap());
let fs = LibSqlRootFilesystem {
pool: crate::libsql_pool::build_libsql_pool_with_config(
db,
1,
std::time::Duration::from_millis(200),
),
};
fs.run_migrations().await.unwrap();

let path = VirtualPath::new("/secrets/single-conn").unwrap();
let v1 = fs
.put(&path, Entry::bytes(vec![1]), CasExpectation::Absent)
.await
.unwrap();

// Stale version drives the 0-rows branch, which must diagnose
// NotFound/VersionMismatch via `current_version_libsql(conn, ...)`
// on the connection already checked out above, not a second
// checkout — a second checkout would time out against the
// size-1 pool's only (self-held) connection.
let stale = RecordVersion::from_backend(v1.get() + 1);
let err = fs.delete_if_version(&path, stale).await.unwrap_err();
assert!(
matches!(err, FilesystemError::VersionMismatch { .. }),
"expected VersionMismatch (proves the diagnosis ran to \
completion without deadlocking on the size-1 pool), got: {err:?}"
);

// Round-C review: the assertion above only proves the diagnosis
// didn't deadlock: `ROLLBACK` itself could still fail to run (or
// fail and leave the connection mid-transaction) without failing
// that assertion. Prove the connection actually came back to the
// size-1 pool in a clean, reusable state by checking it out again
// for a real CAS delete — a still-open transaction from the
// VersionMismatch path would make this second call either hang
// against the size-1 pool or fail on a nested-transaction error.
fs.delete_if_version(&path, v1)
.await
.expect("connection must return to the size-1 pool clean after a VersionMismatch, not deadlock or error on a leftover transaction");
assert!(fs.get(&path).await.unwrap().is_none());
}
}
Loading
Loading