fix(drive-abci): make shielded snapshot ingest idempotent across InitChain retries

packages/rs-drive-abci/src/execution/platform_events/initialization/create_genesis_state/test/shielded.rs

@@ -1136,6 +1136,124 @@ mod platform_tests {
         );
     }
 
+    /// Reproduces the production devnet InitChain failure:
+    ///   `DRIVE_SHIELDED_SNAPSHOT apply failed: grovedb: ingest_subtree_sst:
+    ///    ... ingest_subtree_sst(default, /tmp/drv-shielded-snapshot-*.sst)
+    ///    failed: Invalid argument: Global seqno is required, but disabled`
+    ///
+    /// Root cause: `apply_shielded_snapshot`'s SST ingest commits DIRECTLY to
+    /// the underlying RocksDB (`ingest_external_file_cf`), bypassing the
+    /// InitChain GroveDB transaction — and nothing wipes the DB on InitChain.
+    /// The genesis transaction is committed only at the end of finalizing
+    /// block 1, so if InitChain never reaches that commit (any genesis/block-1
+    /// failure, a drive-abci restart, or a Tenderdash InitChain retry) the
+    /// transaction rolls back but the **already-ingested SST keys persist** in
+    /// the committed `default` CF. The next InitChain attempt re-runs the
+    /// ingest over the now-committed keys; the SST's key range overlaps
+    /// existing committed keys, so RocksDB needs to assign a global sequence
+    /// number, which grovedb's `ingest_subtree_sst` forbids
+    /// (`allow_global_seqno=false`) → "Global seqno is required, but disabled",
+    /// and the devnet can never initialize.
+    ///
+    /// Applying a deterministic genesis snapshot must therefore be IDEMPOTENT:
+    /// after a failed attempt's transaction rolls back, the next attempt must
+    /// succeed even though the orphaned ingest survived the rollback.
+    ///
+    /// This test models the production scenario exactly:
+    ///   1. apply the snapshot WITH a transaction (`Some(tx1)`) — the ingest
+    ///      writes straight to RocksDB, the parent-leaf patch goes into `tx1`;
+    ///   2. DROP `tx1` without committing — the InitChain failure path. This
+    ///      rolls back the parent-leaf patch (and, in production, all genesis
+    ///      writes) but NOT the ingest, which never went through `tx1`;
+    ///   3. apply again WITH a fresh transaction (`Some(tx2)`) — the retry.
+    ///
+    /// The key point (and the answer to "shouldn't rollback restore an empty
+    /// DB?"): rollback would restore an empty DB only if the ingest were
+    /// transactional. It is not — so after step 2 the `default` CF still holds
+    /// the orphaned SST keys, and step 3's re-ingest overlaps them.
+    ///
+    /// Would have caught the production failure in CI:
+    ///   ✖ step 3 errors with "Global seqno is required, but disabled" before
+    ///     the fix,
+    ///   ✔ step 3 succeeds (safe no-op) after.
+    #[test]
+    fn snapshot_reapply_is_idempotent_under_initchain_retry() {
+        let platform_version = PlatformVersion::latest();
+
+        // --- A: seed a small pool and dump it to a snapshot file ---
+        let platform_a = build_regtest_platform();
+        let cfg = ShieldedSeedConfig {
+            total_notes: 5_000,
+            rng_seed: 0xDEAD_BEEF,
+        };
+        let tx_a = platform_a.drive.grove.start_transaction();
+        platform_a
+            .seed_shielded_pool_with_config(
+                &cfg,
+                &BlockInfo::default(),
+                Some(&tx_a),
+                platform_version,
+            )
+            .expect("seed A");
+        tx_a.commit().expect("commit A");
+        let anchor_a = read_current_anchor(&platform_a, None, platform_version);
+        assert_ne!(anchor_a, EMPTY_SINSEMILLA_ROOT);
+
+        let dump_dir = tempfile::tempdir().expect("tempdir");
+        let snapshot_path = dump_dir.path().join("shielded-pool.snap");
+        crate::shielded_snapshot::dump_shielded_subtree(
+            &platform_a.drive.grove,
+            None,
+            &snapshot_path,
+            platform_version,
+        )
+        .expect("dump");
+
+        // --- B, attempt 1: apply inside a transaction, then ROLL BACK ---
+        // Models an InitChain that ingests the snapshot but never reaches the
+        // block-1 commit (a later genesis/block failure, a restart, or a
+        // Tenderdash InitChain retry).
+        let platform_b = build_regtest_platform();
+        {
+            let tx1 = platform_b.drive.grove.start_transaction();
+            crate::shielded_snapshot::apply_shielded_snapshot(
+                &platform_b.drive.grove,
+                Some(&tx1),
+                &snapshot_path,
+                platform_version,
+            )
+            .expect("first apply (inside tx1) must succeed");
+            // Drop tx1 WITHOUT committing — the failure/rollback path. The
+            // parent-leaf patch (in tx1) is discarded; the ingest is NOT.
+            drop(tx1);
+        }
+
+        // --- B, attempt 2: the retry, in a fresh transaction ---
+        // The orphaned ingest from attempt 1 is still committed in the
+        // `default` CF, so this re-ingest overlaps it — the exact production
+        // failure ("Global seqno is required, but disabled").
+        let tx2 = platform_b.drive.grove.start_transaction();
+        let retry = crate::shielded_snapshot::apply_shielded_snapshot(
+            &platform_b.drive.grove,
+            Some(&tx2),
+            &snapshot_path,
+            platform_version,
+        );
+        assert!(
+            retry.is_ok(),
+            "re-applying a deterministic genesis snapshot after a rolled-back \
+             attempt must be idempotent (InitChain retry path), but it failed: \
+             {retry:?}"
+        );
+        tx2.commit().expect("commit retry tx");
+
+        let anchor_b = read_current_anchor(&platform_b, None, platform_version);
+        assert_eq!(
+            anchor_b, anchor_a,
+            "anchor must match after an idempotent re-apply"
+        );
+    }
+
     // Real InitChain hook coverage happens via the dashmate devnet flow
     // (see docs/genesis-snapshot-design.md §13 e2e). An in-process equivalent
     // would need `std::env::set_var`, which this crate's

packages/rs-drive-abci/src/shielded_snapshot/mod.rs

@@ -510,16 +510,40 @@ pub fn apply_shielded_snapshot(
     std::fs::write(&sst_tmp, sst_slice)?;
     let _cleanup = SstTmpGuard(sst_tmp.clone());
 
-    // 4. Bulk-ingest. Bypasses any open transaction; OK at InitChain time
-    //    (txn abort = wipe-and-restart, so orphan data is unreachable).
-    grove
-        .ingest_subtree_sst(SUBTREE_CF, &sst_tmp)
-        .map_err(|e| ShieldedSnapshotError::GroveDb(format!("ingest_subtree_sst: {e}")))?;
-
-    // 5. Cross-validate: reload CommitmentTree from the just-ingested data
-    //    and check recomputed combined_root matches header. Drift surfaces
-    //    BEFORE we touch the parent Merk.
+    // 4. Bulk-ingest the SST — UNLESS a prior attempt already ingested it.
+    //
+    //    The ingest writes straight to RocksDB (`ingest_external_file_cf`),
+    //    bypassing `transaction`, and nothing wipes the DB on InitChain. The
+    //    genesis transaction is committed only at the END of block 1, so if
+    //    InitChain never reaches that commit (a later genesis/block-1 failure,
+    //    a drive-abci restart, or a Tenderdash InitChain retry) the transaction
+    //    rolls back but these directly-committed keys SURVIVE — the rollback
+    //    restores an empty DB for everything except this non-transactional
+    //    ingest. A naive re-ingest on the next attempt overlaps the orphaned
+    //    keys, and RocksDB rejects the overlapping ingest with "Global seqno is
+    //    required, but disabled" (grovedb ingests with `allow_global_seqno =
+    //    false`), wedging the chain permanently.
+    //
+    //    The snapshot is deterministic, so an already-populated subtree holds
+    //    exactly the keys we'd write. Detect that committed state and skip the
+    //    re-ingest; the combined_root cross-validation in step 5 still verifies
+    //    the data (and fails loudly if a stale/foreign subtree is present).
     let subtree_segments = shielded_subtree_segments();
+    if shielded_subtree_has_committed_keys(grove, &subtree_segments) {
+        tracing::info!(
+            "apply_shielded_snapshot: shielded subtree already populated (a prior \
+             InitChain attempt's ingest survived a rolled-back transaction); \
+             skipping re-ingest and re-validating the existing data"
+        );
+    } else {
+        grove
+            .ingest_subtree_sst(SUBTREE_CF, &sst_tmp)
+            .map_err(|e| ShieldedSnapshotError::GroveDb(format!("ingest_subtree_sst: {e}")))?;
+    }
+
+    // 5. Cross-validate: reload CommitmentTree from the ingested (or
+    //    already-present) data and check recomputed combined_root matches
+    //    header. Drift surfaces BEFORE we touch the parent Merk.
     let subtree_path = SubtreePath::from(subtree_segments.as_slice());
 
     let local_tx;
@@ -587,6 +611,28 @@ impl Drop for SstTmpGuard {
     }
 }
 
+/// Returns `true` if the shielded commitment-tree subtree already has at least
+/// one key committed to the underlying RocksDB.
+///
+/// Read through a fresh, throwaway transaction (rolled back on drop), which
+/// sees the latest committed state — exactly the state the SST ingest's
+/// overlap check compares against. This is the idempotency probe for
+/// [`apply_shielded_snapshot`]: the ingest bypasses the caller's transaction,
+/// so a prior InitChain attempt's keys survive a rolled-back genesis and must
+/// not be re-ingested (RocksDB would reject the overlapping ingest with
+/// "Global seqno is required, but disabled").
+fn shielded_subtree_has_committed_keys(grove: &GroveDb, subtree_segments: &[Vec<u8>]) -> bool {
+    let probe_tx = grove.start_transaction();
+    let path = SubtreePath::from(subtree_segments);
+    let ctx = grove
+        .raw_storage()
+        .get_transactional_storage_context(path, None, &probe_tx)
+        .unwrap();
+    let mut iter = ctx.raw_iter();
+    iter.seek_to_first().unwrap();
+    iter.valid().unwrap()
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;