diff --git a/packages/rs-drive-abci/src/execution/platform_events/initialization/create_genesis_state/test/shielded.rs b/packages/rs-drive-abci/src/execution/platform_events/initialization/create_genesis_state/test/shielded.rs index 39a803611e2..76ec156b84f 100644 --- a/packages/rs-drive-abci/src/execution/platform_events/initialization/create_genesis_state/test/shielded.rs +++ b/packages/rs-drive-abci/src/execution/platform_events/initialization/create_genesis_state/test/shielded.rs @@ -1136,6 +1136,124 @@ mod platform_tests { ); } + /// Reproduces the production devnet InitChain failure: + /// `DRIVE_SHIELDED_SNAPSHOT apply failed: grovedb: ingest_subtree_sst: + /// ... ingest_subtree_sst(default, /tmp/drv-shielded-snapshot-*.sst) + /// failed: Invalid argument: Global seqno is required, but disabled` + /// + /// Root cause: `apply_shielded_snapshot`'s SST ingest commits DIRECTLY to + /// the underlying RocksDB (`ingest_external_file_cf`), bypassing the + /// InitChain GroveDB transaction — and nothing wipes the DB on InitChain. + /// The genesis transaction is committed only at the end of finalizing + /// block 1, so if InitChain never reaches that commit (any genesis/block-1 + /// failure, a drive-abci restart, or a Tenderdash InitChain retry) the + /// transaction rolls back but the **already-ingested SST keys persist** in + /// the committed `default` CF. The next InitChain attempt re-runs the + /// ingest over the now-committed keys; the SST's key range overlaps + /// existing committed keys, so RocksDB needs to assign a global sequence + /// number, which grovedb's `ingest_subtree_sst` forbids + /// (`allow_global_seqno=false`) → "Global seqno is required, but disabled", + /// and the devnet can never initialize. + /// + /// Applying a deterministic genesis snapshot must therefore be IDEMPOTENT: + /// after a failed attempt's transaction rolls back, the next attempt must + /// succeed even though the orphaned ingest survived the rollback. + /// + /// This test models the production scenario exactly: + /// 1. apply the snapshot WITH a transaction (`Some(tx1)`) — the ingest + /// writes straight to RocksDB, the parent-leaf patch goes into `tx1`; + /// 2. DROP `tx1` without committing — the InitChain failure path. This + /// rolls back the parent-leaf patch (and, in production, all genesis + /// writes) but NOT the ingest, which never went through `tx1`; + /// 3. apply again WITH a fresh transaction (`Some(tx2)`) — the retry. + /// + /// The key point (and the answer to "shouldn't rollback restore an empty + /// DB?"): rollback would restore an empty DB only if the ingest were + /// transactional. It is not — so after step 2 the `default` CF still holds + /// the orphaned SST keys, and step 3's re-ingest overlaps them. + /// + /// Would have caught the production failure in CI: + /// ✖ step 3 errors with "Global seqno is required, but disabled" before + /// the fix, + /// ✔ step 3 succeeds (safe no-op) after. + #[test] + fn snapshot_reapply_is_idempotent_under_initchain_retry() { + let platform_version = PlatformVersion::latest(); + + // --- A: seed a small pool and dump it to a snapshot file --- + let platform_a = build_regtest_platform(); + let cfg = ShieldedSeedConfig { + total_notes: 5_000, + rng_seed: 0xDEAD_BEEF, + }; + let tx_a = platform_a.drive.grove.start_transaction(); + platform_a + .seed_shielded_pool_with_config( + &cfg, + &BlockInfo::default(), + Some(&tx_a), + platform_version, + ) + .expect("seed A"); + tx_a.commit().expect("commit A"); + let anchor_a = read_current_anchor(&platform_a, None, platform_version); + assert_ne!(anchor_a, EMPTY_SINSEMILLA_ROOT); + + let dump_dir = tempfile::tempdir().expect("tempdir"); + let snapshot_path = dump_dir.path().join("shielded-pool.snap"); + crate::shielded_snapshot::dump_shielded_subtree( + &platform_a.drive.grove, + None, + &snapshot_path, + platform_version, + ) + .expect("dump"); + + // --- B, attempt 1: apply inside a transaction, then ROLL BACK --- + // Models an InitChain that ingests the snapshot but never reaches the + // block-1 commit (a later genesis/block failure, a restart, or a + // Tenderdash InitChain retry). + let platform_b = build_regtest_platform(); + { + let tx1 = platform_b.drive.grove.start_transaction(); + crate::shielded_snapshot::apply_shielded_snapshot( + &platform_b.drive.grove, + Some(&tx1), + &snapshot_path, + platform_version, + ) + .expect("first apply (inside tx1) must succeed"); + // Drop tx1 WITHOUT committing — the failure/rollback path. The + // parent-leaf patch (in tx1) is discarded; the ingest is NOT. + drop(tx1); + } + + // --- B, attempt 2: the retry, in a fresh transaction --- + // The orphaned ingest from attempt 1 is still committed in the + // `default` CF, so this re-ingest overlaps it — the exact production + // failure ("Global seqno is required, but disabled"). + let tx2 = platform_b.drive.grove.start_transaction(); + let retry = crate::shielded_snapshot::apply_shielded_snapshot( + &platform_b.drive.grove, + Some(&tx2), + &snapshot_path, + platform_version, + ); + assert!( + retry.is_ok(), + "re-applying a deterministic genesis snapshot after a rolled-back \ + attempt must be idempotent (InitChain retry path), but it failed: \ + {retry:?}" + ); + tx2.commit().expect("commit retry tx"); + + let anchor_b = read_current_anchor(&platform_b, None, platform_version); + assert_eq!( + anchor_b, anchor_a, + "anchor must match after an idempotent re-apply" + ); + } + // Real InitChain hook coverage happens via the dashmate devnet flow // (see docs/genesis-snapshot-design.md §13 e2e). An in-process equivalent // would need `std::env::set_var`, which this crate's diff --git a/packages/rs-drive-abci/src/shielded_snapshot/mod.rs b/packages/rs-drive-abci/src/shielded_snapshot/mod.rs index 342b7748283..e7b64a893b5 100644 --- a/packages/rs-drive-abci/src/shielded_snapshot/mod.rs +++ b/packages/rs-drive-abci/src/shielded_snapshot/mod.rs @@ -510,16 +510,40 @@ pub fn apply_shielded_snapshot( std::fs::write(&sst_tmp, sst_slice)?; let _cleanup = SstTmpGuard(sst_tmp.clone()); - // 4. Bulk-ingest. Bypasses any open transaction; OK at InitChain time - // (txn abort = wipe-and-restart, so orphan data is unreachable). - grove - .ingest_subtree_sst(SUBTREE_CF, &sst_tmp) - .map_err(|e| ShieldedSnapshotError::GroveDb(format!("ingest_subtree_sst: {e}")))?; - - // 5. Cross-validate: reload CommitmentTree from the just-ingested data - // and check recomputed combined_root matches header. Drift surfaces - // BEFORE we touch the parent Merk. + // 4. Bulk-ingest the SST — UNLESS a prior attempt already ingested it. + // + // The ingest writes straight to RocksDB (`ingest_external_file_cf`), + // bypassing `transaction`, and nothing wipes the DB on InitChain. The + // genesis transaction is committed only at the END of block 1, so if + // InitChain never reaches that commit (a later genesis/block-1 failure, + // a drive-abci restart, or a Tenderdash InitChain retry) the transaction + // rolls back but these directly-committed keys SURVIVE — the rollback + // restores an empty DB for everything except this non-transactional + // ingest. A naive re-ingest on the next attempt overlaps the orphaned + // keys, and RocksDB rejects the overlapping ingest with "Global seqno is + // required, but disabled" (grovedb ingests with `allow_global_seqno = + // false`), wedging the chain permanently. + // + // The snapshot is deterministic, so an already-populated subtree holds + // exactly the keys we'd write. Detect that committed state and skip the + // re-ingest; the combined_root cross-validation in step 5 still verifies + // the data (and fails loudly if a stale/foreign subtree is present). let subtree_segments = shielded_subtree_segments(); + if shielded_subtree_has_committed_keys(grove, &subtree_segments) { + tracing::info!( + "apply_shielded_snapshot: shielded subtree already populated (a prior \ + InitChain attempt's ingest survived a rolled-back transaction); \ + skipping re-ingest and re-validating the existing data" + ); + } else { + grove + .ingest_subtree_sst(SUBTREE_CF, &sst_tmp) + .map_err(|e| ShieldedSnapshotError::GroveDb(format!("ingest_subtree_sst: {e}")))?; + } + + // 5. Cross-validate: reload CommitmentTree from the ingested (or + // already-present) data and check recomputed combined_root matches + // header. Drift surfaces BEFORE we touch the parent Merk. let subtree_path = SubtreePath::from(subtree_segments.as_slice()); let local_tx; @@ -587,6 +611,28 @@ impl Drop for SstTmpGuard { } } +/// Returns `true` if the shielded commitment-tree subtree already has at least +/// one key committed to the underlying RocksDB. +/// +/// Read through a fresh, throwaway transaction (rolled back on drop), which +/// sees the latest committed state — exactly the state the SST ingest's +/// overlap check compares against. This is the idempotency probe for +/// [`apply_shielded_snapshot`]: the ingest bypasses the caller's transaction, +/// so a prior InitChain attempt's keys survive a rolled-back genesis and must +/// not be re-ingested (RocksDB would reject the overlapping ingest with +/// "Global seqno is required, but disabled"). +fn shielded_subtree_has_committed_keys(grove: &GroveDb, subtree_segments: &[Vec]) -> bool { + let probe_tx = grove.start_transaction(); + let path = SubtreePath::from(subtree_segments); + let ctx = grove + .raw_storage() + .get_transactional_storage_context(path, None, &probe_tx) + .unwrap(); + let mut iter = ctx.raw_iter(); + iter.seek_to_first().unwrap(); + iter.valid().unwrap() +} + #[cfg(test)] mod tests { use super::*;