diff --git a/arkrpc/version.go b/arkrpc/version.go index ca9295ad5..d0881354a 100644 --- a/arkrpc/version.go +++ b/arkrpc/version.go @@ -7,9 +7,14 @@ package arkrpc // expectations. // // It is negotiated through the direct GetInfo bootstrap RPC and bound to a -// client runtime for its lifetime. Production currently supports only v1; a -// synthetic v2 may be configured in tests to exercise selection, binding, and -// rejection behavior, but no production default advertises a version beyond -// v1. This constant is deliberately separate from the mailbox transport -// version and the VTXO construction version, which evolve independently. +// client runtime for its lifetime. This constant is deliberately separate +// from the mailbox transport version and the VTXO construction version, which +// evolve independently. const ArkProtocolVersionV1 uint32 = 1 + +// ArkProtocolVersionV2 identifies the complete one-confirmation reorg-safety +// contract: durable lineage registration, fail-closed admission, reversible +// effects through the negotiated policy horizon, and recovery on both peers. +// Defining the compatibility boundary does not enable it; clients must not +// advertise v2 until every required gate and end-to-end proof is present. +const ArkProtocolVersionV2 uint32 = 2 diff --git a/chainsource/chainsource.go b/chainsource/chainsource.go index 0a1284d02..0b860d80c 100644 --- a/chainsource/chainsource.go +++ b/chainsource/chainsource.go @@ -20,13 +20,7 @@ const ( epochChannelSize = 10 // DefaultFinalityDepth is the conventional Bitcoin reorg-safety - // depth. After this many inclusive confirmations the ConfActor - // and SpendActor synthesize their Done events when the backend - // transport (notably lndclient over gRPC) does not surface one - // of its own. Six is the value the wider Lightning stack treats - // as final for the purposes of channel funding / settlement, so - // using it here keeps the unroll subsystem's finality threshold - // aligned with the rest of the daemon's chain assumptions. + // depth. Daemon policy may override it through ChainSourceConfig. DefaultFinalityDepth uint32 = 6 ) diff --git a/chainsource/conf_actor.go b/chainsource/conf_actor.go index 6482e6a28..97bbed7c2 100644 --- a/chainsource/conf_actor.go +++ b/chainsource/conf_actor.go @@ -34,7 +34,7 @@ type ConfActorConfig struct { // // The depth is counted inclusively (a tx confirmed at height H is // at depth 1; height-based finality fires once the actor observes - // a block at H + FinalityDepth - 1). The conventional choice is 6. + // a block at H + FinalityDepth - 1). FinalityDepth uint32 } diff --git a/db/boarding_sweep_store.go b/db/boarding_sweep_store.go index dad9f5b5f..5b40d48b0 100644 --- a/db/boarding_sweep_store.go +++ b/db/boarding_sweep_store.go @@ -174,12 +174,36 @@ func (b *BoardingWalletStore) MarkBoardingSweepFailed(ctx context.Context, now := b.clock.Now().Unix() return b.db.ExecTx(ctx, WriteTxOption(), func(q BoardingStore) error { + sweep, err := q.GetBoardingSweep(ctx, txid[:]) + if err != nil { + return fmt.Errorf("get sweep: %w", err) + } + if sweep.Status != sweepStatusPending && + sweep.Status != sweepStatusPublished { + + // Terminal evidence is monotonic. A duplicate or stale + // failure notification must never downgrade a + // confirmed/external sweep or release any of the value + // it consumed. + return nil + } + inputs, err := q.ListBoardingSweepInputs(ctx, txid[:]) if err != nil { return fmt.Errorf("list sweep inputs: %w", err) } for _, input := range inputs { + if input.Status != inputStatusPending && + input.Status != inputStatusPublished { + + // A terminal spend is objective chain evidence. + // A later local broadcaster failure may release + // the other inputs, but must never restore this + // intent to a spendable state. + continue + } + err = q.UpdateBoardingIntentStatus( ctx, sqlc.UpdateBoardingIntentStatusParams{ OutpointHash: input.OutpointHash, @@ -374,6 +398,22 @@ func (b *BoardingWalletStore) MarkBoardingSweepInputSpent(ctx context.Context, return sql.ErrNoRows } + // Finality for one input is sufficient to make that specific + // boarding intent permanently spent, even while sibling inputs + // are still unresolved. This prevents a later sweep-level + // failure from restoring an objectively consumed intent. + err = q.UpdateBoardingIntentStatus( + ctx, sqlc.UpdateBoardingIntentStatusParams{ + OutpointHash: outpoint.Hash[:], + OutpointIndex: int32(outpoint.Index), + Status: "swept", + LastUpdateTime: now, + }, + ) + if err != nil { + return fmt.Errorf("mark intent swept: %w", err) + } + count, err := q.CountUnresolvedBoardingSweepInputs( ctx, sweepTxid[:], ) @@ -389,24 +429,11 @@ func (b *BoardingWalletStore) MarkBoardingSweepInputSpent(ctx context.Context, if err != nil { return fmt.Errorf("list sweep inputs: %w", err) } - for _, input := range inputs { - err = q.UpdateBoardingIntentStatus( - ctx, sqlc.UpdateBoardingIntentStatusParams{ - OutpointHash: input.OutpointHash, - OutpointIndex: input.OutpointIndex, - Status: "swept", - LastUpdateTime: now, - }, - ) - if err != nil { - return fmt.Errorf("mark intent swept: %w", err) - } - } - sweepStatus := sweepStatusExternalResolved + sweepStatus := sweepStatusConfirmed for _, input := range inputs { - if input.Status == inputStatusSpent { - sweepStatus = sweepStatusConfirmed + if input.Status != inputStatusSpent { + sweepStatus = sweepStatusExternalResolved break } } diff --git a/db/boarding_wallet_test.go b/db/boarding_wallet_test.go index 3ec512bc1..2e8651b10 100644 --- a/db/boarding_wallet_test.go +++ b/db/boarding_wallet_test.go @@ -2,6 +2,7 @@ package db import ( "database/sql" + "errors" "testing" btcaddr "github.com/btcsuite/btcd/address/v2" @@ -822,6 +823,94 @@ func TestBoardingSweepFailedRestoresIntent(t *testing.T) { require.Equal(t, wallet.BoardingStatusConfirmed, updated.Status) } +// TestBoardingSweepFailedPreservesTerminalInput verifies that a local +// sweep-level failure restores only unresolved siblings. An input with a +// final external spend must remain unavailable forever. +func TestBoardingSweepFailedPreservesTerminalInput(t *testing.T) { + t.Parallel() + + ctx := t.Context() + store, _ := newBoardingStoreForTest(t) + terminal := createSweepStoreIntentWithSeed(t, store, 31) + unresolved := createSweepStoreIntentWithSeed(t, store, 32) + + sweepTx := wire.NewMsgTx(2) + sweepTx.AddTxIn(&wire.TxIn{ + PreviousOutPoint: terminal.Outpoint, + }) + sweepTx.AddTxIn(&wire.TxIn{ + PreviousOutPoint: unresolved.Outpoint, + }) + sweepTx.AddTxOut(&wire.TxOut{ + Value: 19_000, + PkScript: []byte{txscript.OP_TRUE}, + }) + sweepTxid := sweepTx.TxHash() + + err := store.CreatePendingBoardingSweep(ctx, wallet.NewBoardingSweep{ + Tx: sweepTx, + TotalAmount: 20_000, + FeeAmount: 1_000, + VBytes: 300, + Inputs: []wallet.NewBoardingSweepInput{ + { + Outpoint: terminal.Outpoint, + Amount: terminal.ChainInfo.Amount, + PreviousStatus: terminal.Status, + }, + { + Outpoint: unresolved.Outpoint, + Amount: unresolved.ChainInfo.Amount, + PreviousStatus: unresolved.Status, + }, + }, + }) + require.NoError(t, err) + + externalTxid := chainhash.Hash{0xfa} + resolved, err := store.MarkBoardingSweepInputSpent( + ctx, terminal.Outpoint, externalTxid, 333, + ) + require.NoError(t, err) + require.False(t, resolved) + + terminalIntent, err := store.GetIntent(ctx, terminal.Outpoint) + require.NoError(t, err) + require.Equal(t, wallet.BoardingStatusSwept, terminalIntent.Status) + + err = store.MarkBoardingSweepFailed( + ctx, sweepTxid, errors.New("conflicting input"), + ) + require.NoError(t, err) + + terminalIntent, err = store.GetIntent(ctx, terminal.Outpoint) + require.NoError(t, err) + require.Equal(t, wallet.BoardingStatusSwept, terminalIntent.Status) + + unresolvedIntent, err := store.GetIntent(ctx, unresolved.Outpoint) + require.NoError(t, err) + require.Equal( + t, wallet.BoardingStatusConfirmed, unresolvedIntent.Status, + ) + + record, err := store.GetBoardingSweep(ctx, sweepTxid) + require.NoError(t, err) + require.NotNil(t, record) + require.Equal(t, wallet.BoardingSweepStatusFailed, record.Status) + statusByOutpoint := make(map[wire.OutPoint]string, len(record.Inputs)) + for _, input := range record.Inputs { + statusByOutpoint[input.Outpoint] = input.Status + } + require.Equal( + t, wallet.BoardingSweepInputStatusExternalSpent, + statusByOutpoint[terminal.Outpoint], + ) + require.Equal( + t, wallet.BoardingSweepInputStatusFailed, + statusByOutpoint[unresolved.Outpoint], + ) +} + // TestBoardingSweepExternalSpendResolvesSeparately verifies an externally // spent sweep input does not make the aggregate look confirmed by our tx. func TestBoardingSweepExternalSpendResolvesSeparately(t *testing.T) { @@ -880,6 +969,26 @@ func TestBoardingSweepExternalSpendResolvesSeparately(t *testing.T) { ) require.NoError(t, err) require.Empty(t, confirmed) + + // A stale broadcaster failure cannot overwrite the objective terminal + // spend or restore the consumed boarding intent. + err = store.MarkBoardingSweepFailed( + ctx, sweepTxid, errors.New("late broadcaster failure"), + ) + require.NoError(t, err) + + record, err := store.GetBoardingSweep(ctx, sweepTxid) + require.NoError(t, err) + require.Equal( + t, wallet.BoardingSweepStatusExternalResolved, record.Status, + ) + require.Equal( + t, wallet.BoardingSweepInputStatusExternalSpent, + dbSweepInputStatus(t, record.Inputs), + ) + updatedIntent, err := store.GetIntent(ctx, intent.Outpoint) + require.NoError(t, err) + require.Equal(t, wallet.BoardingStatusSwept, updatedIntent.Status) } // TestActiveBoardingSweepInputUnique verifies the DB enforces that one diff --git a/sample-waved.conf b/sample-waved.conf index 41689968a..8c0f42e5c 100644 --- a/sample-waved.conf +++ b/sample-waved.conf @@ -40,6 +40,11 @@ # btcwallet, and unknown signers. Set one to force serial signing. # signingworkers=0 +# Deepest chain replacement for which provisional evidence remains +# recoverable. Terminal finality begins one confirmation later. The zero value +# uses the default of 30; the current cross-backend maximum is 144. +# reorgsafetydepth=30 + # Explicit opt-in for mainnet operation. # allow-mainnet=false @@ -223,6 +228,14 @@ # Maximum unroll fee rate in sat/vB. The zero value uses the unroll default. # unroll.maxfeeratesatpervbyte=0 +# Per-anchor restart-reconciliation probe timeout in seconds. The chainsource- +# backed ChainReconciler issues one probe per persisted anchor on Resume; a +# probe that exceeds this budget leaves the durable checkpoint unchanged and +# fails closed for a later retry. Operators running against a slow chain +# backend can raise the budget. The zero value uses the reconciler's internal +# default (10s). +# unroll.reconcileprobetimeoutsec=0 + # Swap server address for swapruntime builds. Empty uses the network+transport # default; see docs/signet.md for the public test-network endpoints. # swap.serveraddress= diff --git a/systest/boarding_sweep_reorg_test.go b/systest/boarding_sweep_reorg_test.go new file mode 100644 index 000000000..7c8fe56d5 --- /dev/null +++ b/systest/boarding_sweep_reorg_test.go @@ -0,0 +1,307 @@ +//go:build systest + +package systest + +import ( + "context" + "crypto/sha256" + "testing" + "time" + + btcaddr "github.com/btcsuite/btcd/address/v2" + "github.com/btcsuite/btcd/btcutil/v2" + "github.com/btcsuite/btcd/chaincfg/v2" + "github.com/btcsuite/btcd/txscript/v2" + "github.com/lightninglabs/wavelength/baselib/actor" + "github.com/lightninglabs/wavelength/txconfirm" + "github.com/lightninglabs/wavelength/wallet" + "github.com/stretchr/testify/require" +) + +// recordingBoardingSweepRef collects the BoardingSweepTxNotification +// values that txconfirm's lifecycle, fanned through the boarding +// sweep MapNotification, delivers to the actor's mailbox. The test +// uses this to assert that each chainsource event maps to the right +// BoardingSweepTxStatus rather than collapsing reorgs into a failure. +type recordingBoardingSweepRef struct { + id string + msgs chan wallet.BoardingSweepTxNotification +} + +func newRecordingBoardingSweepRef(id string) *recordingBoardingSweepRef { + return &recordingBoardingSweepRef{ + id: id, + msgs: make(chan wallet.BoardingSweepTxNotification, 16), + } +} + +// ID returns the subscriber identifier. +func (r *recordingBoardingSweepRef) ID() string { + return r.id +} + +// Tell records the inbound boarding-sweep notification on the channel +// for the test to consume. +func (r *recordingBoardingSweepRef) Tell(_ context.Context, + msg wallet.WalletMsg) error { + + notif, ok := msg.(wallet.BoardingSweepTxNotification) + if !ok { + + // Not a notification we care about (and there should be + // no others on this subscriber path). + return nil + } + + r.msgs <- notif + + return nil +} + +// TryTell mirrors Tell for the non-blocking TellOnlyRef contract; the +// recording ref never applies backpressure, so it simply delegates. +func (r *recordingBoardingSweepRef) TryTell(ctx context.Context, + msg wallet.WalletMsg) error { + + return r.Tell(ctx, msg) +} + +// await pulls the next boarding-sweep notification or fails the test +// on timeout. +func (r *recordingBoardingSweepRef) await( + t *testing.T) wallet.BoardingSweepTxNotification { + + t.Helper() + + select { + case msg := <-r.msgs: + return msg + + case <-time.After(txConfirmSystestEventTimeout): + t.Fatalf("timeout waiting for boarding-sweep notification (%s)", + txConfirmSystestEventTimeout) + + return wallet.BoardingSweepTxNotification{} + } +} + +// TestBoardingSweepReorgRoundTrip exercises the boarding sweep +// MapNotification under a real bitcoind reorg, end-to-end through the +// full txconfirm pipeline. It is the boarding-sweep complement of +// TestTxConfirmReorgRoundTrip: the same chain-event substrate, but +// asserting on the wallet-level lifecycle statuses that drive +// handleSweepTxNotification rather than the raw txconfirm +// notifications. +// +// Lifecycle asserted: +// +// BoardingSweepTxStatusConfirmed (first conf landed) +// BoardingSweepTxStatusReorged (conf block disconnected) +// BoardingSweepTxStatusConfirmed (re-confirmation on canonical chain) +// +// TxFinalized is exercised by TestTxConfirmReorgRoundTrip — that test +// proves the chainsource finality-depth synthesizer fires after the +// reorg-safety horizon, which is the same wire that would deliver +// BoardingSweepTxStatusFinalized to the wallet handler. Including a +// dedicated Finalized assertion here would require mining to the +// 31-confirmation terminal boundary without exercising any +// boarding-sweep-specific code +// path that the unit tests in +// wallet/boarding_sweep_actor_test.go::TestSweepTxNotificationFinalizedIsBenign +// do not already cover. +// +// What this test specifically pins: +// +// - The boarding-sweep MapNotification correctly classifies a real +// chainsource Confirmed event as +// BoardingSweepTxStatusConfirmed (not Failed). +// - The boarding-sweep MapNotification correctly classifies a real +// chainsource Reorged event as +// BoardingSweepTxStatusReorged (not Failed). This is the load- +// bearing regression the wallet refactor was designed to prevent. +// - The handler is multi-shot: txconfirm keeps the watch armed +// past the first TxConfirmed, so a re-confirmation on the +// canonical chain re-fires +// BoardingSweepTxStatusConfirmed. +// - BlockHeight is preserved across the wire on each Confirmed +// event. +func TestBoardingSweepReorgRoundTrip(t *testing.T) { + ParallelN(t) + + h := NewSysTestHarness(t) + ctx := h.Context() + + chainSource := h.NewChainSourceActor() + + // Spawn a real txconfirm actor over the real chainsource. Wallet + // is nil because the systest tx is non-anchor; CPFP fee-input + // selection is never triggered. + txconfBehavior := txconfirm.NewTxBroadcasterActor(txconfirm.Config{ + ChainSource: chainSource, + }) + txconfInstance := actor.NewActor(actor.ActorConfig[ + txconfirm.Msg, txconfirm.Resp, + ]{ + ID: "txconfirm-boarding-sweep-reorg", + Behavior: txconfBehavior, + MailboxSize: 64, + }) + txconfBehavior.SetSelfRef(txconfInstance.TellRef()) + txconfInstance.Start() + t.Cleanup(txconfInstance.Stop) + + // Synthetic watched pkScript — same trick as + // TestTxConfirmReorgRoundTrip. + pubKeyHash := sha256.Sum256([]byte(t.Name())) + addr, err := btcaddr.NewAddressWitnessPubKeyHash( + pubKeyHash[:20], &chaincfg.RegressionNetParams, + ) + require.NoError(t, err, "build synthetic P2WPKH address") + pkScript, err := txscript.PayToAddrScript(addr) + require.NoError(t, err, "derive pkScript for synthetic address") + + heightHint := h.Harness.BlockCount() + + signedTx := h.Harness.SignedV3Tx( + pkScript, btcutil.Amount(btcutil.SatoshiPerBitcoin/100), + ) + txidVal := signedTx.TxHash() + txid := &txidVal + t.Logf("constructed v3 tx: txid=%s", txid) + + // Build the subscriber chain that the production code uses: + // + // walletNotif (TellOnlyRef[BoardingSweepTxNotification], wrap + // via MapMessage so it can sit on a WalletMsg pipe) + // -> wallet.MapBoardingSweepNotification (the production + // classifier from boarding_sweep_actor.go) hosted on a + // txconfirm subscriber. + // + // The recording ref stands in for the wallet actor's mailbox; the + // MapNotification function is the same one the production + // submitSweepConfirmer wires up. + walletRecorder := newRecordingBoardingSweepRef( + "boarding-sweep-systest-sub", + ) + subscriber := wallet.NewBoardingSweepTxconfirmSubscriber( + walletRecorder, + ) + + // Register the EnsureConfirmedReq BEFORE mining so we exercise + // the live-detection path. + _, err = txconfInstance.Ref().Ask( + ctx, &txconfirm.EnsureConfirmedReq{ + Tx: signedTx, + ConfirmationPkScript: pkScript, + Label: "systest-boarding-sweep-reorg", + HeightHint: heightHint, + TargetConfs: 1, + Subscriber: subscriber, + }, + ).Await(ctx).Unpack() + require.NoError(t, err, "EnsureConfirmedReq failed") + + // 1. Mine the block that confirms the systest tx. + originalBlocks := h.Harness.Generate(1) + require.Len(t, originalBlocks, 1) + originalBlock := originalBlocks[0] + + // 2. Expect Status=Confirmed at the original block height with + // at least one confirmation. NumConfs being non-zero pins the + // classifier's assignment of ev.NumConfs through the + // txconfirm.TxConfirmed -> BoardingSweepTxNotification mapping; + // a regression that zeroed it (e.g. by accidentally pulling + // from the wrong field on the event) would otherwise be silently + // invisible to consumers that read NumConfs to gate on + // confirmation depth. + first := walletRecorder.await(t) + require.Equal( + t, wallet.BoardingSweepTxStatusConfirmed, first.Status, "fir"+ + "st notification must be Confirmed, got status=%d", + first.Status, + ) + require.Equal(t, *txid, first.Txid) + require.Equal( + t, int32(originalBlock.Height), first.BlockHeight, + "first Confirmed BlockHeight should match the mined block", + ) + require.GreaterOrEqual( + t, first.NumConfs, uint32(1), + "first Confirmed NumConfs must be at least the target (1)", + ) + t.Logf( + "first Confirmed: txid=%s height=%d num_confs=%d", first.Txid, + first.BlockHeight, first.NumConfs, + ) + + // 3. Reorg the conf block out, mine a strictly longer replacement + // branch, wait for lnd to chain-sync. + reorg := h.Harness.Reorg(1, 2) + require.Equal( + t, originalBlock.Hash, reorg.Disconnected[0].Hash, + "the reorg should have disconnected the conf block", + ) + require.Len(t, reorg.Connected, 2) + t.Logf( + "reorg: disconnected=%d connected=%d fork=%d", + len(reorg.Disconnected), len(reorg.Connected), + reorg.ForkPoint.Height, + ) + + // 4. Expect Status=Reorged. The load-bearing assertion: a real + // chainsource Reorged event must NOT classify as Failed (which + // would trigger MarkBoardingSweepFailed in the production + // handler). + second := walletRecorder.await(t) + require.Equal( + t, wallet.BoardingSweepTxStatusReorged, second.Status, "seco"+ + "nd notification must be Reorged (NOT Failed); got "+ + "status=%d. A Reorged-as-Failed regression is "+ + "exactly what this systest exists to catch", + second.Status, + ) + require.Equal(t, *txid, second.Txid) + require.Empty( + t, second.Reason, "Reorged must not carry a failure reason", + ) + t.Logf("Reorged: txid=%s", second.Txid) + + // 5. Expect a fresh Status=Confirmed on the canonical chain. The + // re-confirmation must land in one of the new connected blocks + // (NumConfs is also asserted to pin the field-through invariant + // under the re-confirmation path — the classifier's TxConfirmed + // arm is exercised twice in this lifecycle). Note: the + // re-confirmation can legitimately land at the SAME block height + // as the disconnected block — Reorg(1, 2) replaces 1 block with + // 2, so the first new block sits at the same height as the old + // one, just with a different hash. The notification doesn't + // carry BlockHash so we can't directly assert hash inequality; + // the canonical-height membership check below is the strongest + // available signal that this is a fresh confirmation on the new + // tip rather than a recycled stale event. + third := walletRecorder.await(t) + require.Equal( + t, wallet.BoardingSweepTxStatusConfirmed, third.Status, "thi"+ + "rd notification must be Confirmed "+ + "(re-confirmation), got status=%d", third.Status, + ) + require.Equal(t, *txid, third.Txid) + canonicalHeights := make(map[int32]struct{}, len(reorg.Connected)) + for _, blk := range reorg.Connected { + canonicalHeights[int32(blk.Height)] = struct{}{} + } + _, onCanonical := canonicalHeights[third.BlockHeight] + require.True( + t, onCanonical, "re-confirmation BlockHeight=%d must be one "+ + "of the new connected block heights %v", + third.BlockHeight, reorg.Connected, + ) + require.GreaterOrEqual( + t, third.NumConfs, uint32(1), + "re-confirmation NumConfs must be at least the target (1)", + ) + t.Logf( + "re-Confirmed: txid=%s height=%d num_confs=%d", third.Txid, + third.BlockHeight, third.NumConfs, + ) +} diff --git a/txconfirm/messages.go b/txconfirm/messages.go index 24ecbb7cc..ac23c781f 100644 --- a/txconfirm/messages.go +++ b/txconfirm/messages.go @@ -171,6 +171,12 @@ type EnsureConfirmedResp struct { // Created is true when the request created a new tracking entry and // false when it attached to existing state. Created bool + + // DefinitelyNotBroadcast is true only when the responder can prove + // that no broadcast attempt crossed the chain boundary. Callers may + // use this to distinguish a locally rejected request from an ambiguous + // broadcast failure. The conservative default is false. + DefinitelyNotBroadcast bool } // MessageType returns the stable message type identifier. diff --git a/unroll/AGENTS.md b/unroll/AGENTS.md index db008b220..95cbd2409 100644 --- a/unroll/AGENTS.md +++ b/unroll/AGENTS.md @@ -66,11 +66,17 @@ For field-level detail, use `go doc github.com/lightninglabs/wavelength/unroll.< terminal outcome is forwarded to the VTXO manager as a `vtxo.ExitOutcomeNotification` so VTXO lifecycle tracks the unroll's terminal on-chain result rather than the user's intent to exit - (wavelength#602): a clean failure (`!HadOnChainFootprint`) → + (wavelength#602): a failure with neither a locally recorded footprint nor + chain-boundary uncertainty (`!HadOnChainFootprint && !ReliveUnsafe`) → `ExitOutcomeRecoverable` (roll back to live), a completed exit → - `ExitOutcomeConfirmed` (retire to spent). `UnrollTerminatedMsg` carries - `HadOnChainFootprint`, computed by `jobHadOnChainFootprint` (any - confirmed/in-flight proof node or a non-pending sweep). It also carries + `ExitOutcomeConfirmed` (retire to spent). The live actor sets + `ReliveUnsafe` before every chain-boundary attempt; only objective + canonical-absence evidence or an explicit `DefinitelyNotBroadcast` + response may clear it. An ambiguous rejection, absence, or timeout + therefore leaves the VTXO held in unilateral exit. + `UnrollTerminatedMsg` carries `HadOnChainFootprint`, computed by + `jobHadOnChainFootprint` (any confirmed/in-flight proof node or a + non-pending sweep), and the durable `ReliveUnsafe` guard. It also carries the child's `ExitPolicyKind`, which the child stamps from its own durable exit policy (`exitPolicyKind`), so the terminal message is self-contained: it stays authoritative after the registry has evicted its in-memory @@ -99,8 +105,8 @@ For field-level detail, use `go doc github.com/lightninglabs/wavelength/unroll.< repeat after termination returns `Created=false` with the historical `ActorID`, never clobbering the sweep txid / failure reason. The one exception is a **recoverable** terminal failure - (`RecoverableFailure`): the prior exit failed cleanly with no on-chain - footprint and the VTXO was rolled back to live (wavelength#602), so + (`RecoverableFailure`): objective canonical-absence evidence allowed the + VTXO to roll back to live (wavelength#602), so a fresh `EnsureUnrollRequest` re-admits (spawns a new child, overwriting the stale record) instead of deduping — otherwise a recovered VTXO could never be unrolled again. Any existing unroll job @@ -264,8 +270,9 @@ For field-level detail, use `go doc github.com/lightninglabs/wavelength/unroll.< `r.active`, `r.pending`, AND `Store.GetRecord` before spawning so a repeat for an already-terminal outpoint returns the historical `ActorID` and never overwrites stored sweep txid / failure reason. - A **recoverable** terminal failure is the deliberate exception: the - VTXO was rolled back to live (wavelength#602), so `handleEnsure` + A **recoverable** terminal failure is the deliberate exception: objective + canonical-absence evidence allowed the VTXO to roll back to live + (wavelength#602), so `handleEnsure` falls through both the `r.pending` and `Store.GetRecord` arms to re-admit a fresh exit rather than strand the recovered coin. - **Fail-closed on restore gaps.** `handleEnsure` validates restorable diff --git a/unroll/CLAUDE.md b/unroll/CLAUDE.md index db008b220..95cbd2409 100644 --- a/unroll/CLAUDE.md +++ b/unroll/CLAUDE.md @@ -66,11 +66,17 @@ For field-level detail, use `go doc github.com/lightninglabs/wavelength/unroll.< terminal outcome is forwarded to the VTXO manager as a `vtxo.ExitOutcomeNotification` so VTXO lifecycle tracks the unroll's terminal on-chain result rather than the user's intent to exit - (wavelength#602): a clean failure (`!HadOnChainFootprint`) → + (wavelength#602): a failure with neither a locally recorded footprint nor + chain-boundary uncertainty (`!HadOnChainFootprint && !ReliveUnsafe`) → `ExitOutcomeRecoverable` (roll back to live), a completed exit → - `ExitOutcomeConfirmed` (retire to spent). `UnrollTerminatedMsg` carries - `HadOnChainFootprint`, computed by `jobHadOnChainFootprint` (any - confirmed/in-flight proof node or a non-pending sweep). It also carries + `ExitOutcomeConfirmed` (retire to spent). The live actor sets + `ReliveUnsafe` before every chain-boundary attempt; only objective + canonical-absence evidence or an explicit `DefinitelyNotBroadcast` + response may clear it. An ambiguous rejection, absence, or timeout + therefore leaves the VTXO held in unilateral exit. + `UnrollTerminatedMsg` carries `HadOnChainFootprint`, computed by + `jobHadOnChainFootprint` (any confirmed/in-flight proof node or a + non-pending sweep), and the durable `ReliveUnsafe` guard. It also carries the child's `ExitPolicyKind`, which the child stamps from its own durable exit policy (`exitPolicyKind`), so the terminal message is self-contained: it stays authoritative after the registry has evicted its in-memory @@ -99,8 +105,8 @@ For field-level detail, use `go doc github.com/lightninglabs/wavelength/unroll.< repeat after termination returns `Created=false` with the historical `ActorID`, never clobbering the sweep txid / failure reason. The one exception is a **recoverable** terminal failure - (`RecoverableFailure`): the prior exit failed cleanly with no on-chain - footprint and the VTXO was rolled back to live (wavelength#602), so + (`RecoverableFailure`): objective canonical-absence evidence allowed the + VTXO to roll back to live (wavelength#602), so a fresh `EnsureUnrollRequest` re-admits (spawns a new child, overwriting the stale record) instead of deduping — otherwise a recovered VTXO could never be unrolled again. Any existing unroll job @@ -264,8 +270,9 @@ For field-level detail, use `go doc github.com/lightninglabs/wavelength/unroll.< `r.active`, `r.pending`, AND `Store.GetRecord` before spawning so a repeat for an already-terminal outpoint returns the historical `ActorID` and never overwrites stored sweep txid / failure reason. - A **recoverable** terminal failure is the deliberate exception: the - VTXO was rolled back to live (wavelength#602), so `handleEnsure` + A **recoverable** terminal failure is the deliberate exception: objective + canonical-absence evidence allowed the VTXO to roll back to live + (wavelength#602), so `handleEnsure` falls through both the `r.pending` and `Store.GetRecord` arms to re-admit a fresh exit rather than strand the recovered coin. - **Fail-closed on restore gaps.** `handleEnsure` validates restorable diff --git a/unroll/actor.go b/unroll/actor.go index 13f6c30d4..59c63eaa3 100644 --- a/unroll/actor.go +++ b/unroll/actor.go @@ -75,6 +75,21 @@ type Config struct { // LedgerSink receives the confirmed on-chain exit fee once the // final sweep has confirmed. LedgerSink fn.Option[ledger.Sink] + + // ChainReconcilerFactory, when set, is invoked after the actor + // loads its proof to construct a ChainReconciler that reconciles + // the persisted checkpoint against the canonical chain at + // startup. Anchors that are no longer live on chain (typically + // reorged out while the daemon was offline) are pruned before + // the FSM session is built so the actor does not broadcast a + // sweep on top of stale planner state. + // + // The factory shape lets the production implementation + // (NewChainSourceReconciler) bind to the actor's proof for + // pkScript lookups while still letting unit tests inject a stub + // that ignores the proof. When None the actor skips + // reconciliation entirely. + ChainReconcilerFactory fn.Option[ChainReconcilerFactory] } // VTXOUnrollActor wraps one durable per-target unroll actor. @@ -178,6 +193,22 @@ type behavior struct { // so a policy-gated sweep is not reported as imminent. None until the // sweep phase resolves the policy. requiredLockTime fn.Option[uint32] + + // reconciled records that the restored checkpoint has been + // reconciled against the canonical chain via cfg.ChainReconciler. + // Reconciliation runs exactly once per actor lifetime, on the + // first ensureLoaded call, before the FSM session is bound. + reconciled bool + + // sweepFinalized latches true after a TxFinalizedMsg arrives for + // the txid currently recorded as the sweep in PlannerState. While + // false, PhaseCompleted is treated as PROVISIONAL: the registry is + // NOT told the actor has terminated, so a reorg of the sweep + // confirmation still has a live actor to deliver the rollback to. + // Once sweepFinalized is true the next notifyRegistryOfTerminal + // call fires UnrollTerminatedMsg and the registry evicts the + // child. + sweepFinalized bool } // unrollTx is the transaction-scoped store handed to the unroll behavior inside @@ -310,9 +341,40 @@ func (b *behavior) dispatch(ctx context.Context, ax actor.Exec[unrollTx], Reason: b.failureReasonForTx(m.Txid, m.Reason), }) + case *TxReorgedMsg: + return b.handleEvent(ctx, ax, &TxReorgedEvent{ + Txid: m.Txid, + }) + + case *TxFinalizedMsg: + // Latch sweepFinalized BEFORE handleEvent so the + // notifyRegistry call inside driveEvent sees the + // post-finalization view and fires UnrollTerminatedMsg + // instead of holding the actor in provisional Completed. + if err := b.ensureLoaded(ctx); err != nil { + return fn.Err[Resp](err) + } + b.maybeLatchSweepFinalized(m.Txid) + + return b.handleEvent(ctx, ax, &TxFinalizedEvent{ + Txid: m.Txid, + }) + case *SpendObservedMsg: return b.handleSpendObserved(ctx, ax, m) + case *SpendReorgedMsg: + return b.handleEvent(ctx, ax, &SpendReorgedEvent{}) + + case *SpendFinalizedMsg: + return b.handleEvent(ctx, ax, &SpendFinalizedEvent{}) + + // GetStateRequest is intentionally NOT handled here: Receive + // short-circuits it before it ever reaches dispatch (it is a read-only + // probe that drives no FSM transition and writes nothing). A + // GetStateRequest can only arrive at this switch via a future refactor + // that breaks that invariant, in which case the default arm's loud + // error is the signal we want -- not a silently duplicated read path. default: return fn.Err[Resp]( fmt.Errorf("unknown unroll message: %T", msg), @@ -940,6 +1002,8 @@ func (b *behavior) ensureNodeConfirmed(ctx context.Context, return b.driveEvent(ctx, ax, &TxFailedEvent{ Txid: txid, + DefinitelyNotBroadcast: ensureResp. + DefinitelyNotBroadcast, Reason: b.failureReasonForTx( txid, "txconfirm returned failed state", ), @@ -974,14 +1038,35 @@ func (b *behavior) watchDeferredCheckpoint(ctx context.Context, }, ) + // Deferred-checkpoint watches register directly against + // chainsource rather than txconfirm, but the rollback contract + // is identical: an operator-confirmed checkpoint that reorgs out + // while the actor is live must drop from ConfirmedTxids so the + // planner does not advance off a stale anchor, and a finality + // signal lets the chainsource sub-actor release its registration + // at the reorg-safety horizon. Wire both lifecycle refs through + // the same selfRef the positive event uses. + reorgRef := chainsource.MapConfReorgedEvent( + b.selfRef, func(event chainsource.ConfReorgedEvent) Msg { + return &TxReorgedMsg{Txid: event.Txid} + }, + ) + doneRef := chainsource.MapConfDoneEvent( + b.selfRef, func(event chainsource.ConfDoneEvent) Msg { + return &TxFinalizedMsg{Txid: event.Txid} + }, + ) + txidCopy := txid _, err = b.cfg.ChainSource.Ask(ctx, &chainsource.RegisterConfRequest{ - CallerID: b.deferredCheckpointCallerID(), - Txid: &txidCopy, - PkScript: append([]byte(nil), pkScript...), - TargetConfs: 1, - HeightHint: b.proofNodeConfHeightHint(ctx, txid), - NotifyActor: fn.Some(notifyRef), + CallerID: b.deferredCheckpointCallerID(), + Txid: &txidCopy, + PkScript: append([]byte(nil), pkScript...), + TargetConfs: 1, + HeightHint: b.proofNodeConfHeightHint(ctx, txid), + NotifyActor: fn.Some(notifyRef), + NotifyReorged: fn.Some(reorgRef), + NotifyDone: fn.Some(doneRef), }).Await(ctx).Unpack() if err != nil { return err @@ -1271,6 +1356,23 @@ func (b *behavior) ensureLoaded(ctx context.Context) error { b.planner = planner } + // Reconcile the restored checkpoint against the canonical chain + // before binding the FSM session. Any confirmed-tx anchor the + // reconciler reports as absent gets dropped (with its + // descendants), and a target / sweep confirmation that vanished + // while the daemon was offline downgrades the sweep state. This + // must run before stateFromCheckpoint so the FSM is built from + // the post-reconciliation snapshot, and before any side effects + // (block subscription, spend watch, ResumeEvent reissue) so a + // stale sweep is never re-broadcast on a now-unconfirmed target. + if !b.reconciled { + if err := b.reconcileOnRestart(ctx); err != nil { + return err + } + + b.reconciled = true + } + if b.session == nil { initialState := State(&Idle{}) if b.pending != nil && b.pending.Started { @@ -1308,11 +1410,9 @@ func (b *behavior) ensureLoaded(ctx context.Context) error { // // txconfirm delivers notifications in its own type space // ([txconfirm.Notification]) but our durable mailbox only accepts [Msg] -// variants so the delivery store can codec them. This helper threads a -// filtering adapter: every txconfirm notification is synchronously -// re-wrapped into the matching mailbox message and forwarded to our -// self-ref for durable enqueue, or dropped when the lifecycle event has -// no unroll-side meaning (see mapTxconfirmNotification). +// variants so the delivery store can codec them. This helper maps every +// reorg-aware lifecycle notification into the corresponding durable unroll +// message. Unknown variants still fail loudly. func (b *behavior) notificationRef() actor.TellOnlyRef[txconfirm.Notification] { return txconfirm.FilterMapNotification( b.selfRef, mapTxconfirmNotification, @@ -1321,27 +1421,10 @@ func (b *behavior) notificationRef() actor.TellOnlyRef[txconfirm.Notification] { // mapTxconfirmNotification re-wraps one txconfirm lifecycle event into // the unroll actor's durable mailbox message space. Subscribers receive -// the FULL reorg-aware lifecycle, so every variant needs an explicit -// disposition: -// -// - TxConfirmed marks the proof node (or sweep) confirmed. -// -// - TxFinalized re-plays the authoritative confirmation numbers once -// the tx is past the backend's reorg-safety depth. The FSM's -// confirmation handling is idempotent (appendUniqueSorted et al), -// so mapping it to a second TxConfirmedMsg is a safe no-op when the -// original TxConfirmed already landed and a recovery when it was -// dropped. Treating it as anything else is a live bug: it used to -// fall into the unknown-notification arm below and terminally -// failed the whole unroll job the moment a backend synthesized -// finality. -// -// - TxReorged is best-effort and superseded by the next reliable -// event (re-TxConfirmed / TxFinalized / TxFailed), and the planner -// has no un-confirm transition; drop it and let the follow-up event -// re-establish state. -// -// - TxFailed, and any unknown future variant, terminate loudly. +// the full reorg-aware lifecycle. Unlike the foundation version of the +// unroll planner, this FSM has explicit reversible reorg and finality +// transitions, so neither notification may be dropped or collapsed into a +// repeated confirmation. Unknown future variants still terminate loudly. func mapTxconfirmNotification(msg txconfirm.Notification) (Msg, bool) { switch m := msg.(type) { case *txconfirm.TxConfirmed: @@ -1352,14 +1435,14 @@ func mapTxconfirmNotification(msg txconfirm.Notification) (Msg, bool) { }, true case *txconfirm.TxFinalized: - return &TxConfirmedMsg{ - Txid: m.Txid, - Height: m.BlockHeight, - NumConfs: m.NumConfs, + return &TxFinalizedMsg{ + Txid: m.Txid, }, true case *txconfirm.TxReorged: - return nil, false + return &TxReorgedMsg{ + Txid: m.Txid, + }, true case *txconfirm.TxFailed: return &TxFailedMsg{ @@ -1405,10 +1488,40 @@ func (b *behavior) restoreCheckpoint(ctx context.Context) error { b.pending = decoded b.sweepTx = copyTx(decoded.SweepTx) + b.sweepFinalized = decoded.SweepFinalized return nil } +// reconcileOnRestart asks the configured ChainReconciler whether each +// anchor recorded in the restored checkpoint is still live on the +// canonical chain, and prunes the in-memory checkpoint accordingly. +// Called from ensureLoaded exactly once per actor lifetime, BEFORE the +// FSM session is bound. +// +// A missing reconciler is a no-op: backends that cannot answer +// historical confirmation queries fall back to the conservative +// behavior of treating the restored checkpoint as authoritative. +// Transport errors from the reconciler are surfaced upward; the actor +// must not proceed to broadcast off a checkpoint we could not verify. +func (b *behavior) reconcileOnRestart(ctx context.Context) error { + if b.cfg.ChainReconcilerFactory.IsNone() { + return nil + } + + if b.pending == nil || !b.pending.Started { + return nil + } + + factory := b.cfg.ChainReconcilerFactory.UnsafeFromSome() + reconciler := factory(b.cfg.TargetOutpoint, b.proof) + if reconciler == nil { + return nil + } + + return reconcileCheckpoint(ctx, reconciler, b.proof, b.pending) +} + // ensureBlockSubscription starts the actor's shared block epoch subscription on // first use so CSV waits advance in the live daemon. func (b *behavior) ensureBlockSubscription(ctx context.Context) error { @@ -1494,14 +1607,28 @@ func (b *behavior) ensureSpendWatch(ctx context.Context) error { } }, ) + reorgRef := chainsource.MapSpendReorgedEvent( + b.selfRef, + func(_ chainsource.SpendReorgedEvent) Msg { + return &SpendReorgedMsg{} + }, + ) + doneRef := chainsource.MapSpendDoneEvent( + b.selfRef, + func(_ chainsource.SpendDoneEvent) Msg { + return &SpendFinalizedMsg{} + }, + ) _, err = b.cfg.ChainSource.Ask( ctx, &chainsource.RegisterSpendRequest{ - CallerID: b.spendCallerID(), - Outpoint: &targetOutpoint, - PkScript: pkScript, - HeightHint: uint32(b.desc.CreatedHeight), - NotifyActor: fn.Some(notifyRef), + CallerID: b.spendCallerID(), + Outpoint: &targetOutpoint, + PkScript: pkScript, + HeightHint: uint32(b.desc.CreatedHeight), + NotifyActor: fn.Some(notifyRef), + NotifyReorged: fn.Some(reorgRef), + NotifyDone: fn.Some(doneRef), }, ).Await(ctx).Unpack() if err != nil { @@ -1757,18 +1884,36 @@ func (b *behavior) handleSpendObserved(ctx context.Context, } } - // Case 4: neither of the above. Someone else spent the watched output. - // This happens if the operator cooperatively claimed the VTXO, - // if a reorg replaced history, or in fraud scenarios. There is - // no way for this unroll to proceed, so terminate with a - // reason string that identifies the spender for operator - // triage. - spentOutpoint := b.cfg.TargetOutpoint - if msg.Outpoint != (wire.OutPoint{}) { - spentOutpoint = msg.Outpoint + // Case 4: neither of the above. Someone else spent the watched + // output. This can happen for two structurally different + // outputs, with different reversibility properties: + // + // a. The target outpoint was spent by an unknown party. This + // is the cooperative-claim / fraud scenario the external- + // spend reorg-safety work is built for: a reorg of the + // spending block can resurrect the recovery job, so the + // actor parks in AwaitingExternalSpendFinality. A + // subsequent SpendFinalizedMsg promotes the spend to a + // permanent FailReason; a SpendReorgedMsg clears the + // observation and resumes planning. + // + // b. A proof-node output was spent by a transaction that is + // not itself part of the proof graph (ackProofOutputSpend + // acked the parent confirmation already, but returned + // false for the spender lookup). The proof graph cannot + // complete through this fork, so the unroll job is + // terminally dead. Fail with a reason that identifies the + // spender for operator triage. + if msg.Outpoint == (wire.OutPoint{}) || + msg.Outpoint == b.cfg.TargetOutpoint { + return b.handleEvent(ctx, ax, &ExternalSpendObservedEvent{ + SpendingTxid: msg.SpendingTxid, + SpendingHeight: msg.SpendingHeight, + }) } + reason := fmt.Sprintf("watched outpoint %s spent externally by tx %s "+ - "at height %d", spentOutpoint, msg.SpendingTxid, + "at height %d", msg.Outpoint, msg.SpendingTxid, msg.SpendingHeight) return b.handleEvent(ctx, ax, &FailEvent{Reason: reason}) @@ -1841,6 +1986,14 @@ func (b *behavior) checkpointWrite() (*actorCheckpoint, } checkpoint := checkpointFromState(state, b.sweepTx) + + // Persist the sweep-finalized latch so it survives restart: commitAck + // re-persists this checkpoint atomically with the message ack, so a + // finalized sweep whose terminal handoff was deferred/failed is not + // lost — the actor rehydrates as terminal-eligible rather than stuck + // "provisional completed". + checkpoint.SweepFinalized = b.sweepFinalized + raw, err := encodeCheckpoint(checkpoint) if err != nil { return nil, nil, err @@ -2145,19 +2298,60 @@ func (b *behavior) failureReasonForTx(txid chainhash.Hash, return fmt.Sprintf("proof tx %s failed: %s", txid, reason) } +// maybeLatchSweepFinalized sets b.sweepFinalized when an incoming +// TxFinalizedMsg refers to the txid currently recorded as the sweep in +// PlannerState. The flag gates notifyRegistryIfTerminal so that a +// PhaseCompleted entry without a matching finalization stays +// provisional — the actor is kept alive by the registry until the +// sweep is past the backend's reorg-safety depth, so a reorg of the +// sweep confirmation has a live actor to receive the rollback. +// +// Late finalizations (e.g. for a proof tx, or for a sweep txid that has +// since been replaced by a re-broadcast) are ignored. +func (b *behavior) maybeLatchSweepFinalized(txid chainhash.Hash) { + state, err := b.currentState() + if err != nil { + return + } + + job := stateJob(state) + if job.PlannerState.Sweep.Txid.IsNone() { + return + } + if job.PlannerState.Sweep.Txid.UnsafeFromSome() != txid { + return + } + + b.sweepFinalized = true +} + // notifyRegistryIfTerminal forwards one UnrollTerminatedMsg to the -// registry when the FSM reaches Completed or Failed, at most once per -// actor lifetime. +// registry when the FSM reaches a TRULY terminal state, at most once +// per actor lifetime. +// +// PhaseFailed always means FailReason has been set (proof-tx terminal failure +// or sweep retry budget exhausted), so the actor is terminal immediately. // -// The registry uses this to move the outpoint out of its active map and -// mark the durable store terminal. If the FSM receives additional events -// after reaching terminal (e.g. a late TxConfirmed for a proof node that -// materialized before we failed for other reasons) terminalNotified -// keeps us from spamming the registry with repeats. +// PhaseCompleted is treated as PROVISIONAL until b.sweepFinalized is +// latched by a matching TxFinalizedMsg for the recorded sweep txid. +// While provisional, the registry keeps the child in its active map so +// a reorg of the sweep confirmation has a live actor to deliver the +// rollback to. The sweep's finality signal arrives via txconfirm's +// TxFinalized, which txconfirm emits off chainsource's height-based Done +// synthesis -- enabled for every backend (including lndclient over gRPC, +// whose native Done channel is allocated-but-never-written) by wiring +// FinalityDepth = configured policy horizon + 1 at waved/server.go. So the +// child IS evicted once the sweep confirmation buries past the +// reorg-safety depth, not retained indefinitely. // -// Failure to Tell is warned but not fatal — the registry will rediscover -// the terminal phase the next time it queries child state, and it holds -// its own persistence retry loop for the control-plane record. +// PhaseExternalSpendObserved is reversible by design and never reaches this +// function as terminal; SpendFinalizedMsg moves the actor to PhaseCompleted +// with ExternalSpendFinalized set so the consumed VTXO retires on chain. +// +// Failure to Tell is warned but not fatal — the registry will +// rediscover the terminal phase the next time it queries child state, +// and it holds its own persistence retry loop for the control-plane +// record. func (b *behavior) notifyRegistryIfTerminal(ctx context.Context) { state, err := b.currentState() if err != nil { @@ -2167,11 +2361,14 @@ func (b *behavior) notifyRegistryIfTerminal(ctx context.Context) { } phase := phaseFromState(state) - if phase != PhaseCompleted && phase != PhaseFailed { + job := stateJob(state) + trulyTerminal := phase == PhaseFailed || + (phase == PhaseCompleted && + (b.sweepFinalized || job.ExternalSpendFinalized)) + if !trulyTerminal { return } - job := stateJob(state) if !b.emitExitCostIfCompleted(ctx, phase, job) { // The exit-cost leg has not yet been durably handed to the @@ -2182,13 +2379,24 @@ func (b *behavior) notifyRegistryIfTerminal(ctx context.Context) { return } - // On a terminal failure, ask the wallet backend to drop the job's - // broadcast-but-unconfirmed transactions so a full-node wallet stops - // perpetually rebroadcasting a proof (or sweep) tx that can never - // confirm now that the exit has failed (wavelength#609). Gated by its - // own once-flag so it is independent of the registry handoff below and - // does not re-issue removals on every subsequent terminal tick. - if phase == PhaseFailed && !b.abandonedBroadcastsRemoved { + // On a terminal failure — or once an external spend that finalized the + // target out from under an in-flight exit is itself final — ask the + // wallet backend to drop the job's broadcast-but-unconfirmed + // transactions so a full-node wallet stops perpetually rebroadcasting a + // proof (or sweep) tx that can never confirm now that the exit is over + // (wavelength#609). Under reorg-safe exits an external target spend is + // a reversible completion rather than a failure, so we defer the + // cleanup to ExternalSpendFinalized: the still-in-flight proof tree is + // just as abandoned as on the failure path, but only once the spend can + // no longer reorg back out. A normal sweep completion is excluded — its + // transactions confirmed on chain and are never rebroadcast. Gated by + // its own once-flag so it is independent of the registry handoff below + // and does not re-issue removals on every subsequent terminal tick. + externalSpendFinal := phase == PhaseCompleted && + job.ExternalSpendFinalized + if (phase == PhaseFailed || externalSpendFinal) && + !b.abandonedBroadcastsRemoved { + b.removeAbandonedBroadcasts(ctx, job) b.abandonedBroadcastsRemoved = true } @@ -2203,6 +2411,7 @@ func (b *behavior) notifyRegistryIfTerminal(ctx context.Context) { Phase: phase, FailReason: job.FailReason, HadOnChainFootprint: jobHadOnChainFootprint(job), + ReliveUnsafe: job.ReliveUnsafe, ExitPolicyKind: b.exitPolicyKind(), } @@ -2372,7 +2581,8 @@ func (b *behavior) removeAbandonedBroadcasts(ctx context.Context, func (b *behavior) emitExitCostIfCompleted(ctx context.Context, phase Phase, job *JobState) bool { - if phase != PhaseCompleted || b.exitCostNotified || + if phase != PhaseCompleted || job.ExternalSpendFinalized || + b.exitCostNotified || b.cfg.LedgerSink.IsNone() { return true } diff --git a/unroll/actor_test.go b/unroll/actor_test.go index cd143c4d2..47a82da98 100644 --- a/unroll/actor_test.go +++ b/unroll/actor_test.go @@ -137,10 +137,11 @@ func (m *mockVTXOStore) DeleteVTXO(context.Context, wire.OutPoint) error { type fakeTxConfirmRef struct { mu sync.Mutex - requests []*txconfirm.EnsureConfirmedReq - responseStates map[chainhash.Hash]txconfirm.TxState - confirmHeights map[chainhash.Hash]int32 - failureReasons map[chainhash.Hash]string + requests []*txconfirm.EnsureConfirmedReq + responseStates map[chainhash.Hash]txconfirm.TxState + confirmHeights map[chainhash.Hash]int32 + failureReasons map[chainhash.Hash]string + definitelyNotBroadcast map[chainhash.Hash]bool // onAsk, when set, is invoked with each EnsureConfirmedReq as it is // recorded (outside the store lock). Tests use it to assert ordering @@ -188,6 +189,8 @@ func (f *fakeTxConfirmRef) Ask(_ context.Context, f.requests = append(f.requests, req) state := f.responseStates[req.Tx.TxHash()] height := f.confirmHeights[req.Tx.TxHash()] + definitelyNotBroadcast := + f.definitelyNotBroadcast[req.Tx.TxHash()] onAsk := f.onAsk f.mu.Unlock() @@ -226,9 +229,10 @@ func (f *fakeTxConfirmRef) Ask(_ context.Context, promise.Complete( fn.Ok[txconfirm.Resp]( &txconfirm.EnsureConfirmedResp{ - Txid: req.Tx.TxHash(), - State: state, - Created: true, + Txid: req.Tx.TxHash(), + State: state, + Created: true, + DefinitelyNotBroadcast: definitelyNotBroadcast, }, ), ) @@ -343,6 +347,23 @@ func (f *fakeTxConfirmRef) setImmediateFailed(txid chainhash.Hash, f.failureReasons[txid] = reason } +// setImmediateDefiniteNoBroadcast configures one txid to fail before any +// broadcast attempt crosses the chain boundary. +func (f *fakeTxConfirmRef) setImmediateDefiniteNoBroadcast(txid chainhash.Hash, + reason string) { + + f.setImmediateFailed(txid, reason) + + f.mu.Lock() + defer f.mu.Unlock() + + if f.definitelyNotBroadcast == nil { + f.definitelyNotBroadcast = make(map[chainhash.Hash]bool) + } + + f.definitelyNotBroadcast[txid] = true +} + // emitConfirmed delivers a txconfirm success notification to the subscriber. func (f *fakeTxConfirmRef) emitConfirmed(t *testing.T, index int, txid chainhash.Hash, height int32) { @@ -406,6 +427,42 @@ func (f *fakeTxConfirmRef) emitFailed(t *testing.T, index int, require.NoError(t, err) } +// emitReorged delivers a txconfirm reorg notification to the subscriber +// behind the request at index. +func (f *fakeTxConfirmRef) emitReorged(t *testing.T, index int, + txid chainhash.Hash) { + + t.Helper() + + f.mu.Lock() + require.Less(t, index, len(f.requests)) + subscriber := f.requests[index].Subscriber + f.mu.Unlock() + + err := subscriber.Tell(t.Context(), &txconfirm.TxReorged{ + Txid: txid, + }) + require.NoError(t, err) +} + +// emitFinalized delivers a txconfirm finalized notification to the +// subscriber behind the request at index. +func (f *fakeTxConfirmRef) emitFinalized(t *testing.T, index int, + txid chainhash.Hash) { + + t.Helper() + + f.mu.Lock() + require.Less(t, index, len(f.requests)) + subscriber := f.requests[index].Subscriber + f.mu.Unlock() + + err := subscriber.Tell(t.Context(), &txconfirm.TxFinalized{ + Txid: txid, + }) + require.NoError(t, err) +} + // confRef aliases the chainsource confirmation notification target. type confRef = actor.TellOnlyRef[chainsource.ConfirmationEvent] @@ -415,16 +472,18 @@ type confReq = chainsource.RegisterConfRequest // fakeChainSourceRef is a minimal chainsource actor ref for sweep fee // estimation tests. type fakeChainSourceRef struct { - mu sync.Mutex - bestHeight int32 - feeRate int64 - feeErr error - blockRef actor.TellOnlyRef[chainsource.BlockEpoch] - spendRefs map[wire.OutPoint]spendEventRef - spendRegs []wire.OutPoint - removedTxes []chainhash.Hash - confRefs map[chainhash.Hash]confRef - confReqs map[chainhash.Hash]*confReq + mu sync.Mutex + bestHeight int32 + feeRate int64 + feeErr error + blockRef actor.TellOnlyRef[chainsource.BlockEpoch] + spendRefs map[wire.OutPoint]spendEventRef + spendRegs []wire.OutPoint + spendReorgedRef actor.TellOnlyRef[chainsource.SpendReorgedEvent] + spendFinalizedRef actor.TellOnlyRef[chainsource.SpendDoneEvent] + removedTxes []chainhash.Hash + confRefs map[chainhash.Hash]confRef + confReqs map[chainhash.Hash]*confReq } // spendEventRef is the fake chain-source spend notification actor reference. @@ -542,6 +601,18 @@ func (f *fakeChainSourceRef) Ask(_ context.Context, } f.spendRefs[outpoint] = msg.NotifyActor.UnwrapOr(nil) f.spendRegs = append(f.spendRegs, outpoint) + + // Reorg/finalized refs are only wired by ensureSpendWatch + // (target outpoint). Proof-node spend watches from + // ensureProofSpendWatches leave these unset; capture them + // only when the caller actually provided them so a later + // proof-node registration cannot wipe the target's refs. + if msg.NotifyReorged.IsSome() { + f.spendReorgedRef = msg.NotifyReorged.UnwrapOr(nil) + } + if msg.NotifyDone.IsSome() { + f.spendFinalizedRef = msg.NotifyDone.UnwrapOr(nil) + } f.mu.Unlock() promise.Complete( fn.Ok[chainsource.ChainSourceResp]( @@ -681,6 +752,40 @@ func (f *fakeChainSourceRef) removedTxSnapshot() []chainhash.Hash { return append([]chainhash.Hash(nil), f.removedTxes...) } +// emitSpendReorged delivers a SpendReorgedEvent to the subscribed actor. +func (f *fakeChainSourceRef) emitSpendReorged(t *testing.T) { + t.Helper() + + f.mu.Lock() + ref := f.spendReorgedRef + f.mu.Unlock() + + require.NotNil(t, ref) + require.NoError( + t, + ref.Tell( + t.Context(), chainsource.SpendReorgedEvent{}, + ), + ) +} + +// emitSpendFinalized delivers a SpendDoneEvent to the subscribed actor. +func (f *fakeChainSourceRef) emitSpendFinalized(t *testing.T) { + t.Helper() + + f.mu.Lock() + ref := f.spendFinalizedRef + f.mu.Unlock() + + require.NotNil(t, ref) + require.NoError( + t, + ref.Tell( + t.Context(), chainsource.SpendDoneEvent{}, + ), + ) +} + // fakeSweepWallet is a minimal signer plus wallet-destination test double. type fakeSweepWallet struct{} @@ -2182,6 +2287,11 @@ func TestResumeReissuesDeferredCheckpointWatch(t *testing.T) { t.Cleanup(resumedActor.Stop) mustAsk(t, resumedActor.Ref(), &ResumeUnrollRequest{Height: 111}) + checkpoint := mustDecodeCheckpoint(t, store, "resume-deferred-test") + require.True( + t, checkpoint.ReliveUnsafe, "restart must persist the "+ + "fail-closed relive guard before reissue", + ) require.Eventually(t, func() bool { return chainRef.confWatchCount() == 1 }, testTimeout, 10*time.Millisecond) @@ -2509,19 +2619,20 @@ func TestTerminalFailureRemovesAbandonedBroadcasts(t *testing.T) { }) // The root proof tx is submitted (in-flight) on start. Wait for the - // target spend watch, then fail the job with an external spend of the - // target outpoint (case 4). A target spend, unlike a root-output spend, - // does not confirm the still-in-flight root. + // reversible spend watches to arm, then spend the target externally + // (case 4). Under reorg-safe exits this is a reversible completion — + // the VTXO left on chain via someone else's tx — not an immediate + // failure, so the job parks in PhaseExternalSpendObserved until the + // spend finalizes. A target spend, unlike a root-output spend, does not + // confirm the still-in-flight root. target := proof.TargetOutpoint() rootTxid := proof.RootTxids()[0] require.Eventually(t, func() bool { - for _, outpoint := range chainSource.spendRegistrations() { - if outpoint == target { - return true - } - } + chainSource.mu.Lock() + defer chainSource.mu.Unlock() - return false + return chainSource.spendReorgedRef != nil && + chainSource.spendFinalizedRef != nil }, testTimeout, 10*time.Millisecond) chainSource.emitSpendForOutpoint(t, target, chainhash.Hash{0xee}, 101) @@ -2532,11 +2643,24 @@ func TestTerminalFailureRemovesAbandonedBroadcasts(t *testing.T) { ).(*GetStateResp) require.True(t, ok) - return stateResp.Phase == PhaseFailed + return stateResp.Phase == PhaseExternalSpendObserved + }, testTimeout, 10*time.Millisecond) + + // Finalize the external spend past the reorg-safety depth. The job + // completes and the abandoned in-flight root — which can never confirm + // now that the target is spent — must be removed so a full-node wallet + // stops rebroadcasting it (wavelength#609). + chainSource.emitSpendFinalized(t) + + require.Eventually(t, func() bool { + stateResp, ok := mustAsk( + t, unrollActor.Ref(), &GetStateRequest{}, + ).(*GetStateResp) + require.True(t, ok) + + return stateResp.Phase == PhaseCompleted }, testTimeout, 10*time.Millisecond) - // The in-flight root tx must have been removed from the wallet so a - // full-node wallet stops rebroadcasting it. require.Eventually(t, func() bool { for _, txid := range chainSource.removedTxSnapshot() { if txid == rootTxid { @@ -2546,7 +2670,8 @@ func TestTerminalFailureRemovesAbandonedBroadcasts(t *testing.T) { return false }, testTimeout, 10*time.Millisecond, - "terminal failure must remove the in-flight proof tx (#609)") + "terminal external spend must remove the in-flight proof tx "+ + "(#609)") } // TestRejectedProofTxIsRemovedOnTerminalFailure covers wavelength#609's own @@ -2876,6 +3001,35 @@ func TestProofTxFailureTransitionsToFailed(t *testing.T) { " failed: txconfirm returned failed state", checkpoint.Fail, ) + require.True(t, checkpoint.ReliveUnsafe) +} + +// TestDefiniteNoBroadcastClearsReliveGuard proves that an explicit local +// rejection may recover the VTXO while an ambiguous TxStateFailed response +// remains fail-closed. +func TestDefiniteNoBroadcastClearsReliveGuard(t *testing.T) { + proof := buildLinearProof(t) + desc := testDescriptor(t, proof.TargetOutpoint(), proof.CSVDelay()) + rootTxid := proof.RootTxids()[0] + unrollActor, _, txconfirmRef, store := newActorHarness(t, proof, desc) + txconfirmRef.setImmediateDefiniteNoBroadcast(rootTxid, "rejected") + + mustAsk(t, unrollActor.Ref(), &StartUnrollRequest{ + Height: 100, + Trigger: TriggerManual, + }) + + require.Eventually(t, func() bool { + stateResp, ok := mustAsk( + t, unrollActor.Ref(), &GetStateRequest{}, + ).(*GetStateResp) + require.True(t, ok) + + return stateResp.Phase == PhaseFailed + }, testTimeout, 10*time.Millisecond) + + checkpoint := mustDecodeCheckpoint(t, store, "unroll-test") + require.False(t, checkpoint.ReliveUnsafe) } // TestResumeReissuesSweepConfirmation verifies that resume reattaches @@ -3127,6 +3281,12 @@ func TestSweepConfirmationCompletesActor(t *testing.T) { ) require.True(t, checkpoint.State.Sweep.ConfirmHeight.IsSome()) + // Reorg-safe completion treats PhaseCompleted as provisional on + // confirmation and defers the terminal handoff (and the ExitCostMsg + // emission) until the sweep finalizes past reorg-safety depth. Drive + // the finalize so the exit cost lands. + txconfirmRef.emitFinalized(t, 2, sweepTxid) + ledgerMsg, ok := ledgerSink.AwaitMessage(testTimeout) require.True(t, ok) exitCostMsg, ok := ledgerMsg.(*ledger.ExitCostMsg) @@ -3228,6 +3388,20 @@ func TestExitCostTellFailureDefersTerminalHandoff(t *testing.T) { return stateResp.Phase == PhaseCompleted }, testTimeout, 10*time.Millisecond) + // Finalize the sweep so PhaseCompleted is no longer provisional and + // the actor becomes terminal-eligible. The terminal handoff still + // requires a successful exit-cost emission, which fails here because + // the sink has no ledger actor behind it. + txconfirmRef.emitFinalized(t, 2, sweepTxid) + + // Synchronize with the actor before mutating its config below. The + // finalize above is delivered on the actor mailbox ahead of this Ask + // (FIFO), so by the time the Ask returns the deferred-handoff attempt + // it triggered -- including its read of cfg.LedgerSink -- has completed + // and happens-before the sink swap. Without this edge the race detector + // flags the concurrent field access against that read. + mustAsk(t, unrollActor.Ref(), &GetStateRequest{}) + // The failing ledger sink must have deferred the terminal handoff: // the registry sees no UnrollTerminatedMsg. _, ok := registryRef.AwaitMessage(50 * time.Millisecond) @@ -3591,12 +3765,19 @@ func TestSweepFailureRetriesThenFails(t *testing.T) { require.Equal(t, maxSweepAttempts, checkpoint.SweepAttempts) } -// TestExternalSpendTerminatesActor verifies that an external spend of the -// target VTXO (not our proof nodes or sweep) terminates the actor. +// TestExternalSpendTerminatesActor verifies that an external spend of +// the target VTXO is treated as a provisional block, and that +// finalization resolves it as a terminal on-chain completion. Without the +// finalization signal the actor stays in AwaitingExternalSpendFinality +// so a reorg of the spending block has a live actor to resume. func TestExternalSpendTerminatesActor(t *testing.T) { proof := buildLinearProof(t) desc := testDescriptor(t, proof.TargetOutpoint(), proof.CSVDelay()) - unrollActor, beh, _, _ := newActorHarness(t, proof, desc) + unrollActor, beh, _, store := newActorHarness(t, proof, desc) + registryRef := actor.NewChannelTellOnlyRef[RegistryMsg]( + "unroll-registry", 1, + ) + beh.cfg.RegistryRef = registryRef chainSource, ok := beh.cfg.ChainSource.(*fakeChainSourceRef) require.True(t, ok) @@ -3606,15 +3787,26 @@ func TestExternalSpendTerminatesActor(t *testing.T) { Trigger: TriggerManual, }) - // Ensure spend watch is registered. + // Ensure spend watch is registered for the target outpoint with + // reorg / done callback refs wired so the actor can be driven + // through the full reversible lifecycle. require.Eventually(t, func() bool { + var targetRegistered bool for _, outpoint := range chainSource.spendRegistrations() { if outpoint == proof.TargetOutpoint() { - return true + targetRegistered = true + break } } + if !targetRegistered { + return false + } - return false + chainSource.mu.Lock() + defer chainSource.mu.Unlock() + + return chainSource.spendReorgedRef != nil && + chainSource.spendFinalizedRef != nil }, testTimeout, 10*time.Millisecond) // Simulate an external party spending the target VTXO. @@ -3623,20 +3815,46 @@ func TestExternalSpendTerminatesActor(t *testing.T) { t, proof.TargetOutpoint(), externalTxid, 101, ) + // The actor must enter the reversible + // AwaitingExternalSpendFinality phase rather than terminating. require.Eventually(t, func() bool { stateResp, ok := mustAsk( t, unrollActor.Ref(), &GetStateRequest{}, ).(*GetStateResp) require.True(t, ok) - return stateResp.Phase == PhaseFailed + return stateResp.Phase == PhaseExternalSpendObserved + }, testTimeout, 10*time.Millisecond) + + // Finalize the spend. The actor resolves the target on chain and + // transitions to PhaseCompleted without fabricating a sweep. + chainSource.emitSpendFinalized(t) + + require.Eventually(t, func() bool { + stateResp, ok := mustAsk( + t, unrollActor.Ref(), &GetStateRequest{}, + ).(*GetStateResp) + require.True(t, ok) + + return stateResp.Phase == PhaseCompleted }, testTimeout, 10*time.Millisecond) stateResp, ok := mustAsk( t, unrollActor.Ref(), &GetStateRequest{}, ).(*GetStateResp) require.True(t, ok) - require.Contains(t, stateResp.FailReason, "spent externally") + require.Empty(t, stateResp.FailReason) + + terminalMsg, ok := registryRef.AwaitMessage(testTimeout) + require.True(t, ok, "final external spend must retire the target") + terminated, ok := terminalMsg.(*UnrollTerminatedMsg) + require.True(t, ok) + require.Equal(t, PhaseCompleted, terminated.Phase) + require.True(t, terminated.HadOnChainFootprint) + require.Nil(t, terminated.SweepTxid) + + checkpoint := mustDecodeCheckpoint(t, store, "unroll-test") + require.True(t, checkpoint.ExternalSpendFinalized) } // TestStartUnrollIsIdempotent verifies that reissuing start does not duplicate diff --git a/unroll/db_store.go b/unroll/db_store.go index fc1b8ef47..0acfeec8c 100644 --- a/unroll/db_store.go +++ b/unroll/db_store.go @@ -107,8 +107,9 @@ func (s *DBRegistryStore) ListNonTerminalRecords(ctx context.Context) ( // MarkTerminal marks one target terminal in the unilateral-exit job table. // recoverable selects UnilateralExitJobStatusFailedRecoverable over the -// plain Failed status for a no-footprint failure so boot-time reconciliation -// can roll the VTXO back to live (wavelength#602). +// plain Failed status for a no-footprint failure with objective +// canonical-absence evidence, allowing boot-time reconciliation to roll the +// VTXO back to live (wavelength#602). func (s *DBRegistryStore) MarkTerminal(ctx context.Context, target wire.OutPoint, phase Phase, recoverable bool, failReason string, sweepTxid *chainhash.Hash) error { @@ -194,8 +195,8 @@ func registryExitPolicy(record RegistryRecord, return exitPolicyKind(record.ExitPolicyKind), record.ExitPolicyRef } -// statusForRecord maps a registry record into the DB status enum, routing a -// recoverable (no-footprint) failure to the distinct FailedRecoverable status +// statusForRecord maps a registry record into the DB status enum, routing an +// authoritatively recoverable failure to the distinct FailedRecoverable status // so it round-trips back to RecoverableFailure=true on the next read. func statusForRecord(record RegistryRecord) db.UnilateralExitJobStatus { if record.Phase == PhaseFailed && record.RecoverableFailure { @@ -230,6 +231,20 @@ func statusForPhase(phase Phase) db.UnilateralExitJobStatus { case PhaseFailed: return db.UnilateralExitJobStatusFailed + case PhaseMaterializing: + return db.UnilateralExitJobStatusMaterializing + + // PhaseExternalSpendObserved is deliberately collapsed onto + // Materializing: it is a transient, reorg-reversible parked phase (an + // external spend was seen but has not finalized), the DB status enum + // has no dedicated value for it, and the true phase is always + // reconstructed from the durable checkpoint on restore -- so no state + // is lost. The collapse only affects the coarse operator-facing status + // during the parked window; add a dedicated enum value if that window + // needs to be independently observable. + case PhaseExternalSpendObserved: + return db.UnilateralExitJobStatusMaterializing + default: return db.UnilateralExitJobStatusMaterializing } diff --git a/unroll/fsm_logic.go b/unroll/fsm_logic.go index e9f0df9ac..e34df3ebf 100644 --- a/unroll/fsm_logic.go +++ b/unroll/fsm_logic.go @@ -77,6 +77,10 @@ func processEventWithJob(ctx context.Context, job *JobState, event Event, if e.Height > nextJob.Height { nextJob.Height = e.Height } + // A positive-only notifier cannot prove that no external spend + // landed while the daemon was offline. Persist the fail-closed + // relive guard before any reissued side effect. + nextJob.ReliveUnsafe = true reissue = true case *HeightUpdatedEvent: @@ -91,6 +95,41 @@ func processEventWithJob(ctx context.Context, job *JobState, event Event, case *TxFailedEvent: applyFailedEvent(nextJob, e) + case *TxReorgedEvent: + applyReorgedEvent(nextJob, e, env) + + case *TxFinalizedEvent: + // TxFinalized is informational at the unroll layer: the + // underlying chain anchor is no longer reversible, but the + // planner has already accounted for the confirmation. Nothing + // to mutate; we still run the planner below in case the + // finality changes any derived decision. + + case *ExternalSpendObservedEvent: + nextJob.ProvisionalExternalSpend = fn.Some(ExternalSpendAnchor{ + SpendingTxid: e.SpendingTxid, + SpendingHeight: e.SpendingHeight, + }) + if e.SpendingHeight > nextJob.Height { + nextJob.Height = e.SpendingHeight + } + + case *SpendReorgedEvent: + nextJob.ProvisionalExternalSpend = + fn.None[ExternalSpendAnchor]() + + case *SpendFinalizedEvent: + // Finalizing a provisional external spend resolves the target + // on chain. If no provisional anchor was held, SpendFinalized + // is informational and ignored. + nextJob.ProvisionalExternalSpend.WhenSome( + func(ExternalSpendAnchor) { + nextJob.ExternalSpendFinalized = true + nextJob.ProvisionalExternalSpend = + fn.None[ExternalSpendAnchor]() + }, + ) + case *SweepBroadcastedEvent: nextJob.PlannerState.Sweep.Status = unrollplan.SweepStatusBroadcasted @@ -194,10 +233,9 @@ func deriveStateTransition(ctx context.Context, job *JobState, env *Environment, return nil, err } - // Terminal short-circuit. applyFailedEvent populates FailReason - // for proof-tx terminal failures, and handleSpendObserved emits - // explicit FailEvents for external-spend detection; both land - // here before any planner work. + // Failure short-circuit. applyFailedEvent populates FailReason for + // proof-tx terminal failures; finalized external spends use the + // distinct consumed-success branch immediately below. if job.FailReason != "" { return &StateTransition{ NextState: &Failed{ @@ -206,6 +244,32 @@ func deriveStateTransition(ctx context.Context, job *JobState, env *Environment, }, nil } + // A finalized external spend is an objective terminal success from the + // VTXO lifecycle's perspective: the target is consumed and must retire. + if job.ExternalSpendFinalized { + return &StateTransition{ + NextState: &Completed{ + Job: job.Copy(), + }, + }, nil + } + + // Provisional external spend short-circuit. While the spend is + // observed but not finalized, the planner must not advance toward + // a sweep: on chain the target output no longer exists, and + // broadcasting a sweep on top would fail. A SpendReorgedEvent + // clears ProvisionalExternalSpend and the next derivation falls + // through to the normal planner phase decision; a + // SpendFinalizedEvent marks the target durably consumed and the + // completed branch above handles the terminal transition. + if job.ProvisionalExternalSpend.IsSome() { + return &StateTransition{ + NextState: &AwaitingExternalSpendFinality{ + Job: job.Copy(), + }, + }, nil + } + // Consult the pure planner. Plan() is stateless; it reads // PlannerState + the proof graph and returns a snapshot with Done // / NeedSweep / CSV / Ready fields. All phase decisions below @@ -437,10 +501,158 @@ func applyFailedEvent(job *JobState, event *TxFailedEvent) { job.DeferredCheckpoints = removeDeferredCheckpoint( job.DeferredCheckpoints, event.Txid, ) + if event.DefinitelyNotBroadcast { + job.ReliveUnsafe = false + } job.FailReason = event.Reason } +// applyReorgedEvent rolls back the chain anchor for a previously +// confirmed proof or sweep transaction. The semantics mirror +// applyConfirmedEvent in reverse: +// +// - If the reorged txid matches the recorded sweep txid, downgrade +// SweepStatus from Confirmed back to Broadcasted and clear the +// stored ConfirmHeight. The signed sweep bytes are still durable +// in the actor checkpoint and the txconfirm subscription is still +// live, so the actor naturally re-runs AwaitingSweepConfirmation +// until the sweep reconfirms. +// +// - Otherwise the reorged txid is a proof node: drop it from +// ConfirmedTxids so the planner stops treating it as ready. If +// the reorged tx is the target itself, clear TargetConfirmHeight +// so CSV maturity is recomputed when the target reconfirms, and +// downgrade any sweep that depended on the now-invalidated target +// confirmation. The proof node remains broadcastable; the next +// deriveStateTransition will route an EnsureReadyTransactions for +// it if the planner's frontier turns up the txid again. +// +// Height is never rolled back: chain height is a global property +// reported by the block subscription, not by the per-tx watch, so a +// reorged confirmation does not move best-height down even though it +// invalidates the per-tx anchor. +func applyReorgedEvent(job *JobState, event *TxReorgedEvent, env *Environment) { + if job == nil || event == nil { + return + } + + // Sweep reorg: downgrade SweepStatus so deriveStateTransition lands + // in AwaitingSweepConfirmation (the sweep tx is still on chain, + // pending re-confirmation) rather than Completed. + if job.PlannerState.Sweep.Txid.IsSome() && + job.PlannerState.Sweep.Txid.UnsafeFromSome() == event.Txid { + + if job.PlannerState.Sweep.Status == + unrollplan.SweepStatusConfirmed { + + job.PlannerState.Sweep.Status = + unrollplan.SweepStatusBroadcasted + } + job.PlannerState.Sweep.ConfirmHeight = fn.None[int32]() + + return + } + + // Proof-node reorg: clear the confirmed anchor for this txid AND + // every descendant that we had recorded as confirmed or in-flight. + // State.Validate enforces a topological invariant (every + // confirmed/in-flight node has confirmed parents) that would fail + // if we dropped only the immediate ancestor and left descendants + // in place. Pruning the entire reorged subtree keeps the planner + // state internally consistent; txconfirm's txid-keyed dedup + // absorbs the re-submits when the planner re-emits the same nodes + // on its ready frontier after the parent reconfirms. + reorgedSubtree := collectReorgedSubtree(env, event.Txid) + job.PlannerState.ConfirmedTxids = removeHashes( + job.PlannerState.ConfirmedTxids, reorgedSubtree, + ) + job.PlannerState.InFlightTxids = removeHashes( + job.PlannerState.InFlightTxids, reorgedSubtree, + ) + for txid := range reorgedSubtree { + job.DeferredCheckpoints = removeDeferredCheckpoint( + job.DeferredCheckpoints, txid, + ) + } + + // If this proof tx was the target, the CSV anchor is also gone. + // unrollplan's State.Validate enforces "broadcasted / confirmed + // sweep requires confirmed target" so any non-pending sweep must + // be reset to Pending when the target loses its anchor. The + // signed sweep bytes still live in the actor checkpoint + // (b.sweepTx); a re-confirmed target drives NeedSweep again and + // startSweep reuses those bytes rather than deriving a new wallet + // pkScript or producing a different sweep txid. + if env != nil && env.Proof != nil && + event.Txid == env.Proof.TargetOutpoint().Hash { + + job.PlannerState.TargetConfirmHeight = fn.None[int32]() + + if job.PlannerState.Sweep.Status != + unrollplan.SweepStatusPending { + + job.PlannerState.Sweep.Status = + unrollplan.SweepStatusPending + job.PlannerState.Sweep.Txid = fn.None[chainhash.Hash]() + job.PlannerState.Sweep.ConfirmHeight = fn.None[int32]() + } + } +} + +// collectReorgedSubtree returns the set of every txid transitively +// descended from root within the proof graph, inclusive of root itself. +// A nil environment or missing root yields a singleton set so the +// reducer can still drop the reorged txid from local bookkeeping even +// when the proof is not loaded. +func collectReorgedSubtree(env *Environment, + root chainhash.Hash) map[chainhash.Hash]struct{} { + + subtree := map[chainhash.Hash]struct{}{root: {}} + if env == nil || env.Proof == nil { + return subtree + } + + queue := []chainhash.Hash{root} + for len(queue) > 0 { + next := queue[0] + queue = queue[1:] + + children, err := env.Proof.ChildTxids(next) + if err != nil { + continue + } + for _, child := range children { + if _, ok := subtree[child]; ok { + continue + } + subtree[child] = struct{}{} + queue = append(queue, child) + } + } + + return subtree +} + +// removeHashes returns hashes with every entry in drop removed. +func removeHashes(hashes []chainhash.Hash, + drop map[chainhash.Hash]struct{}) []chainhash.Hash { + + if len(hashes) == 0 || len(drop) == 0 { + return hashes + } + + filtered := hashes[:0:0] + for _, h := range hashes { + if _, ok := drop[h]; ok { + continue + } + filtered = append(filtered, h) + } + + return filtered +} + // applySweepBuildFailed records a sweep build or broadcast failure and // decides whether to terminate or retry. // diff --git a/unroll/fsm_types.go b/unroll/fsm_types.go index bab30dedb..274a25382 100644 --- a/unroll/fsm_types.go +++ b/unroll/fsm_types.go @@ -103,6 +103,40 @@ type JobState struct { // SweepAttempts counts sweep build or broadcast failures so the actor // can retry up to maxSweepAttempts before giving up. SweepAttempts int + + // ProvisionalExternalSpend records an external spend of the target + // outpoint that has been observed but has not yet been finalized + // past the backend's reorg-safety depth. While set, the actor is + // parked: deriveStateTransition refuses to advance toward a sweep, + // since the on-chain state says the target output no longer exists. + // A SpendReorgedEvent clears the anchor and lets planning resume; + // a SpendFinalizedEvent resolves the target as permanently consumed. + // + // This field is persisted so a restart cannot forget a reversible + // spend and resume planning before canonical evidence is reconciled. + ProvisionalExternalSpend fn.Option[ExternalSpendAnchor] + + // ExternalSpendFinalized records that the provisional target spend + // crossed policy finality. It is a terminal on-chain resolution, not a + // recoverable failure: the VTXO manager must retire the consumed coin. + ExternalSpendFinalized bool + + // ReliveUnsafe is set before a job issues or reissues any chain side + // effect. Until authoritative canonical absence exists, a later + // clean-looking broadcast failure cannot prove the off-chain VTXO + // remained live. + ReliveUnsafe bool +} + +// ExternalSpendAnchor captures the identity of an external spend of the +// target outpoint that the actor has observed but has not yet treated +// as final. +type ExternalSpendAnchor struct { + // SpendingTxid is the txid that consumed the target outpoint. + SpendingTxid chainhash.Hash + + // SpendingHeight is the block height the spending tx confirmed at. + SpendingHeight int32 } // Copy returns a deep copy of the job state. @@ -113,14 +147,17 @@ func (j *JobState) Copy() *JobState { deferred := copyDeferredCheckpoints(j.DeferredCheckpoints) copyState := &JobState{ - Height: j.Height, - Trigger: j.Trigger, - ExitPolicyKind: exitPolicyKind(j.ExitPolicyKind), - ExitPolicyRef: j.ExitPolicyRef, - PlannerState: copyPlannerState(j.PlannerState), - DeferredCheckpoints: deferred, - FailReason: j.FailReason, - SweepAttempts: j.SweepAttempts, + Height: j.Height, + Trigger: j.Trigger, + ExitPolicyKind: exitPolicyKind(j.ExitPolicyKind), + ExitPolicyRef: j.ExitPolicyRef, + PlannerState: copyPlannerState(j.PlannerState), + DeferredCheckpoints: deferred, + FailReason: j.FailReason, + SweepAttempts: j.SweepAttempts, + ProvisionalExternalSpend: j.ProvisionalExternalSpend, + ExternalSpendFinalized: j.ExternalSpendFinalized, + ReliveUnsafe: j.ReliveUnsafe, } return copyState @@ -198,11 +235,67 @@ type TxFailedEvent struct { // Reason is the stable human-readable failure reason. Reason string + + // DefinitelyNotBroadcast proves that the failed transaction never + // crossed the chain boundary. Only this explicit evidence may clear + // the fail-closed relive guard. + DefinitelyNotBroadcast bool } // eventSealed marks TxFailedEvent as an FSM event. func (e *TxFailedEvent) eventSealed() {} +// TxReorgedEvent records that a previously confirmed proof or sweep +// transaction was reorged out of the canonical chain. +type TxReorgedEvent struct { + // Txid is the reorged transaction hash. + Txid chainhash.Hash +} + +// eventSealed marks TxReorgedEvent as an FSM event. +func (e *TxReorgedEvent) eventSealed() {} + +// TxFinalizedEvent records that a confirmation is past the backend's +// reorg-safety depth. +type TxFinalizedEvent struct { + // Txid is the finalized transaction hash. + Txid chainhash.Hash +} + +// eventSealed marks TxFinalizedEvent as an FSM event. +func (e *TxFinalizedEvent) eventSealed() {} + +// ExternalSpendObservedEvent records an external spend of the target +// outpoint that has not yet been finalized. The reducer parks the +// actor in AwaitingExternalSpendFinality. +type ExternalSpendObservedEvent struct { + // SpendingTxid is the txid that consumed the target outpoint. + SpendingTxid chainhash.Hash + + // SpendingHeight is the block height the spending tx confirmed at. + SpendingHeight int32 +} + +// eventSealed marks ExternalSpendObservedEvent as an FSM event. +func (e *ExternalSpendObservedEvent) eventSealed() {} + +// SpendReorgedEvent records that a previously observed spend of the +// target outpoint was reorged out of the canonical chain. The reducer +// clears any provisional external-spend block on JobState. +type SpendReorgedEvent struct{} + +// eventSealed marks SpendReorgedEvent as an FSM event. +func (e *SpendReorgedEvent) eventSealed() {} + +// SpendFinalizedEvent records that a previously observed external spend +// of the target outpoint is past the backend's reorg-safety depth. The +// reducer promotes any provisional external-spend block to terminal consumed +// state so the VTXO is retired rather than restored. +type SpendFinalizedEvent struct{} + +// eventSealed marks SpendFinalizedEvent as an FSM event. +func (e *SpendFinalizedEvent) eventSealed() {} + // SweepBroadcastedEvent records that the actor built the final sweep and // submitted it to txconfirm. type SweepBroadcastedEvent struct { @@ -315,6 +408,7 @@ func (s *Idle) ProcessEvent(ctx context.Context, event Event, Trigger: e.Trigger, ExitPolicyKind: exitPolicyKind(e.ExitPolicyKind), ExitPolicyRef: e.ExitPolicyRef, + ReliveUnsafe: true, } return deriveStateTransition( @@ -323,9 +417,10 @@ func (s *Idle) ProcessEvent(ctx context.Context, event Event, case *ResumeEvent: job := &JobState{ - Height: e.Height, - Trigger: TriggerRestart, - FailReason: "", + Height: e.Height, + Trigger: TriggerRestart, + FailReason: "", + ReliveUnsafe: true, } return deriveStateTransition( @@ -445,7 +540,46 @@ func (s *AwaitingSweepConfirmation) ProcessEvent(ctx context.Context, return processEventWithJob(ctx, s.Job, event, env) } -// Completed indicates the final sweep has confirmed. +// AwaitingExternalSpendFinality indicates the actor has observed an +// external spend of the target outpoint but is waiting for either a +// reorg (which clears the observation and resumes planning) or a +// finality signal (which retires the consumed target). +type AwaitingExternalSpendFinality struct { + // Job is the durable FSM state. + Job *JobState +} + +// String returns a human-readable state label. +func (s *AwaitingExternalSpendFinality) String() string { + return "AwaitingExternalSpendFinality" +} + +// IsTerminal returns false because the observation is reversible. +func (s *AwaitingExternalSpendFinality) IsTerminal() bool { + return false +} + +// stateSealed marks AwaitingExternalSpendFinality as implementing State. +func (s *AwaitingExternalSpendFinality) stateSealed() {} + +// ProcessEvent delegates to the shared reducer so every event kind is +// applied uniformly; the reducer's planner-gate keeps the actor parked +// in this state until SpendReorgedEvent or SpendFinalizedEvent clears +// or promotes the provisional anchor. +func (s *AwaitingExternalSpendFinality) ProcessEvent(ctx context.Context, + event Event, env *Environment) (*StateTransition, error) { + + return processEventWithJob(ctx, s.Job, event, env) +} + +// Completed indicates the target is durably resolved on chain, either by the +// final sweep or by a finalized external spend. +// +// Completed is reversible until finality: a TxReorgedEvent for the +// recorded sweep txid (or for the target tx that the sweep depended on) +// must be routed through the same reducer as the non-terminal states so +// the actor can roll back to AwaitingSweepConfirmation when the chain +// disagrees. Any other event is rejected as before. type Completed struct { // Job is the durable FSM state. Job *JobState @@ -456,19 +590,33 @@ func (s *Completed) String() string { return "Completed" } -// IsTerminal returns true because this state is terminal. +// IsTerminal reports whether the planner-derived state is still +// terminal. Completed remains the public phase label, but a reorg event +// can re-enter the FSM so it cannot be marked terminal at the protofsm +// level — otherwise the engine refuses to deliver the rollback event. func (s *Completed) IsTerminal() bool { - return true + return false } // stateSealed marks Completed as implementing State. func (s *Completed) stateSealed() {} -// ProcessEvent rejects further events in the terminal completed state. -func (s *Completed) ProcessEvent(context.Context, Event, *Environment) ( - *StateTransition, error) { +// ProcessEvent applies reorg / finality events in the completed state +// and absorbs every other event kind as an idempotent no-op. Late +// chain notifications (a TxConfirmedEvent or HeightUpdatedEvent that +// raced the terminal transition while the registry is draining the +// actor for cleanup) are already reflected in the terminal checkpoint +// and should not error. +func (s *Completed) ProcessEvent(ctx context.Context, event Event, + env *Environment) (*StateTransition, error) { - return nil, fmt.Errorf("completed state is terminal") + switch event.(type) { + case *TxReorgedEvent, *TxFinalizedEvent: + return processEventWithJob(ctx, s.Job, event, env) + + default: + return &StateTransition{NextState: s}, nil + } } // Failed indicates the actor reached terminal failure. diff --git a/unroll/messages.go b/unroll/messages.go index 09e3bd00d..3499716d9 100644 --- a/unroll/messages.go +++ b/unroll/messages.go @@ -42,6 +42,10 @@ const ( txFailedMsgTLVType tlv.Type = 0x7904 getStateRequestTLVType tlv.Type = 0x7905 spendObservedMsgTLVType tlv.Type = 0x7906 + txReorgedMsgTLVType tlv.Type = 0x7907 + txFinalizedMsgTLVType tlv.Type = 0x7908 + spendReorgedMsgTLVType tlv.Type = 0x7909 + spendFinalizedMsgTLVType tlv.Type = 0x790a ) // Durable mailbox priorities. Admission stays at the default priority so a @@ -77,6 +81,9 @@ const ( spendObservedHeightRecType tlv.Type = 3 spendObservedOutHashRecType tlv.Type = 5 spendObservedOutIndexRecType tlv.Type = 7 + + txReorgedTxidRecType tlv.Type = 1 + txFinalizedTxidRecType tlv.Type = 1 ) // StartTrigger identifies what caused the unroll actor to start. @@ -125,11 +132,20 @@ const ( // awaiting confirmation. PhaseSweepConfirmation Phase = "sweep_confirmation" - // PhaseCompleted indicates the sweep confirmed successfully. + // PhaseCompleted indicates the target resolved on chain through either + // the final sweep or a finalized external spend. PhaseCompleted Phase = "completed" // PhaseFailed indicates the actor reached terminal failure. PhaseFailed Phase = "failed" + + // PhaseExternalSpendObserved indicates the actor has observed an + // external spend of the target outpoint that has not yet been + // finalized past the backend's reorg-safety depth. The actor is + // parked: it does not advance toward a sweep, but it has not + // terminated either, since a reorg of the spending block can + // resurrect the recovery job. + PhaseExternalSpendObserved Phase = "external_spend_observed" ) // Msg is the durable mailbox surface accepted by the VTXO unroll actor. @@ -800,6 +816,258 @@ func newCodec() *actor.MessageCodec { spendObservedMsgTLVType, func() actor.TLVMessage { return &SpendObservedMsg{} }, ) + codec.MustRegister( + txReorgedMsgTLVType, + func() actor.TLVMessage { return &TxReorgedMsg{} }, + ) + codec.MustRegister( + txFinalizedMsgTLVType, + func() actor.TLVMessage { return &TxFinalizedMsg{} }, + ) + codec.MustRegister( + spendReorgedMsgTLVType, + func() actor.TLVMessage { return &SpendReorgedMsg{} }, + ) + codec.MustRegister( + spendFinalizedMsgTLVType, + func() actor.TLVMessage { return &SpendFinalizedMsg{} }, + ) return codec } + +// SpendReorgedMsg reports that a previously delivered SpendObservedMsg +// for the target outpoint has been reorged out of the canonical chain. +// The actor must roll back any provisional external-spend state it +// recorded for the prior observation; if a new spend on the new tip +// follows, it arrives as a fresh SpendObservedMsg on the same +// subscription. +// +// The payload is intentionally empty: each unroll actor watches +// exactly one target outpoint, and the chainsource sub-actor is +// already keyed on that outpoint, so no additional correlation +// metadata is needed. +type SpendReorgedMsg struct { + actor.BaseMessage +} + +// MessageType returns the stable message type identifier. +func (m *SpendReorgedMsg) MessageType() string { + return "SpendReorgedMsg" +} + +// TLVType returns the durable mailbox type ID. +func (m *SpendReorgedMsg) TLVType() tlv.Type { + return spendReorgedMsgTLVType +} + +// Priority returns the durable mailbox priority for spend reorg events. +func (m *SpendReorgedMsg) Priority() int { + return unrollProgressPriority +} + +// Encode serializes the message as a (currently empty) TLV stream. +func (m *SpendReorgedMsg) Encode(w io.Writer) error { + stream, err := tlv.NewStream() + if err != nil { + return fmt.Errorf("create stream: %w", err) + } + + return stream.Encode(w) +} + +// Decode deserializes the message from a TLV stream. +func (m *SpendReorgedMsg) Decode(r io.Reader) error { + stream, err := tlv.NewStream() + if err != nil { + return fmt.Errorf("create stream: %w", err) + } + + if err := stream.Decode(r); err != nil { + return fmt.Errorf("decode: %w", err) + } + + return nil +} + +// unrollMsgSealed seals SpendReorgedMsg into the message surface. +func (m *SpendReorgedMsg) unrollMsgSealed() {} + +// SpendFinalizedMsg reports that a previously observed external spend +// of the target outpoint is past the backend's reorg-safety depth. +// Receiving it converts any provisional external-spend block on the +// actor into a terminal on-chain completion. +type SpendFinalizedMsg struct { + actor.BaseMessage +} + +// MessageType returns the stable message type identifier. +func (m *SpendFinalizedMsg) MessageType() string { + return "SpendFinalizedMsg" +} + +// TLVType returns the durable mailbox type ID. +func (m *SpendFinalizedMsg) TLVType() tlv.Type { + return spendFinalizedMsgTLVType +} + +// Priority returns the durable mailbox priority for spend finality +// events. +func (m *SpendFinalizedMsg) Priority() int { + return unrollProgressPriority +} + +// Encode serializes the message as a (currently empty) TLV stream. +func (m *SpendFinalizedMsg) Encode(w io.Writer) error { + stream, err := tlv.NewStream() + if err != nil { + return fmt.Errorf("create stream: %w", err) + } + + return stream.Encode(w) +} + +// Decode deserializes the message from a TLV stream. +func (m *SpendFinalizedMsg) Decode(r io.Reader) error { + stream, err := tlv.NewStream() + if err != nil { + return fmt.Errorf("create stream: %w", err) + } + + if err := stream.Decode(r); err != nil { + return fmt.Errorf("decode: %w", err) + } + + return nil +} + +// unrollMsgSealed seals SpendFinalizedMsg into the message surface. +func (m *SpendFinalizedMsg) unrollMsgSealed() {} + +// TxReorgedMsg reports that a previously delivered TxConfirmedMsg has +// been reorged out of the canonical chain. Subscribers should treat the +// prior confirmation as no longer valid; if the transaction re-confirms, +// a fresh TxConfirmedMsg will follow on the same subscription. +type TxReorgedMsg struct { + actor.BaseMessage + + // Txid identifies the reorged transaction. + Txid chainhash.Hash +} + +// MessageType returns the stable message type identifier. +func (m *TxReorgedMsg) MessageType() string { + return "TxReorgedMsg" +} + +// TLVType returns the durable mailbox type ID. +func (m *TxReorgedMsg) TLVType() tlv.Type { + return txReorgedMsgTLVType +} + +// Priority returns the durable mailbox priority for reorg events. Reorg +// rollback must run before low-value reads and height ticks because +// downstream side effects (e.g. sweep gating) gate on its outcome. +func (m *TxReorgedMsg) Priority() int { + return unrollProgressPriority +} + +// Encode serializes the message as a TLV stream. +func (m *TxReorgedMsg) Encode(w io.Writer) error { + txid := [32]byte(m.Txid) + + stream, err := tlv.NewStream( + tlv.MakePrimitiveRecord(txReorgedTxidRecType, &txid), + ) + if err != nil { + return fmt.Errorf("create stream: %w", err) + } + + return stream.Encode(w) +} + +// Decode deserializes the message from a TLV stream. +func (m *TxReorgedMsg) Decode(r io.Reader) error { + var txid [32]byte + + stream, err := tlv.NewStream( + tlv.MakePrimitiveRecord(txReorgedTxidRecType, &txid), + ) + if err != nil { + return fmt.Errorf("create stream: %w", err) + } + + if err := stream.Decode(r); err != nil { + return fmt.Errorf("decode: %w", err) + } + + m.Txid = chainhash.Hash(txid) + + return nil +} + +// unrollMsgSealed seals TxReorgedMsg into the message surface. +func (m *TxReorgedMsg) unrollMsgSealed() {} + +// TxFinalizedMsg reports that a tracked transaction is past the +// backend's reorg-safety depth and will receive no further events. +// Consumers may use this signal to drop any reorg-recovery bookkeeping +// they were holding for the watch. +type TxFinalizedMsg struct { + actor.BaseMessage + + // Txid identifies the finalized transaction. + Txid chainhash.Hash +} + +// MessageType returns the stable message type identifier. +func (m *TxFinalizedMsg) MessageType() string { + return "TxFinalizedMsg" +} + +// TLVType returns the durable mailbox type ID. +func (m *TxFinalizedMsg) TLVType() tlv.Type { + return txFinalizedMsgTLVType +} + +// Priority returns the durable mailbox priority for finality events. +func (m *TxFinalizedMsg) Priority() int { + return unrollProgressPriority +} + +// Encode serializes the message as a TLV stream. +func (m *TxFinalizedMsg) Encode(w io.Writer) error { + txid := [32]byte(m.Txid) + + stream, err := tlv.NewStream( + tlv.MakePrimitiveRecord(txFinalizedTxidRecType, &txid), + ) + if err != nil { + return fmt.Errorf("create stream: %w", err) + } + + return stream.Encode(w) +} + +// Decode deserializes the message from a TLV stream. +func (m *TxFinalizedMsg) Decode(r io.Reader) error { + var txid [32]byte + + stream, err := tlv.NewStream( + tlv.MakePrimitiveRecord(txFinalizedTxidRecType, &txid), + ) + if err != nil { + return fmt.Errorf("create stream: %w", err) + } + + if err := stream.Decode(r); err != nil { + return fmt.Errorf("decode: %w", err) + } + + m.Txid = chainhash.Hash(txid) + + return nil +} + +// unrollMsgSealed seals TxFinalizedMsg into the message surface. +func (m *TxFinalizedMsg) unrollMsgSealed() {} diff --git a/unroll/messages_test.go b/unroll/messages_test.go index 097e39204..e4321bcb4 100644 --- a/unroll/messages_test.go +++ b/unroll/messages_test.go @@ -127,6 +127,56 @@ func TestDurableMessageTLVRoundTrip(t *testing.T) { require.Equal(t, orig.SpendingHeight, got.SpendingHeight) }) + t.Run("TxReorgedMsg", func(t *testing.T) { + t.Parallel() + + orig := &TxReorgedMsg{Txid: txid} + + var buf bytes.Buffer + require.NoError(t, orig.Encode(&buf)) + + got := &TxReorgedMsg{} + require.NoError(t, got.Decode(bytes.NewReader(buf.Bytes()))) + require.Equal(t, orig.Txid, got.Txid) + }) + + t.Run("SpendReorgedMsg", func(t *testing.T) { + t.Parallel() + + orig := &SpendReorgedMsg{} + + var buf bytes.Buffer + require.NoError(t, orig.Encode(&buf)) + + got := &SpendReorgedMsg{} + require.NoError(t, got.Decode(bytes.NewReader(buf.Bytes()))) + }) + + t.Run("SpendFinalizedMsg", func(t *testing.T) { + t.Parallel() + + orig := &SpendFinalizedMsg{} + + var buf bytes.Buffer + require.NoError(t, orig.Encode(&buf)) + + got := &SpendFinalizedMsg{} + require.NoError(t, got.Decode(bytes.NewReader(buf.Bytes()))) + }) + + t.Run("TxFinalizedMsg", func(t *testing.T) { + t.Parallel() + + orig := &TxFinalizedMsg{Txid: txid} + + var buf bytes.Buffer + require.NoError(t, orig.Encode(&buf)) + + got := &TxFinalizedMsg{} + require.NoError(t, got.Decode(bytes.NewReader(buf.Bytes()))) + require.Equal(t, orig.Txid, got.Txid) + }) + t.Run("GetStateRequest", func(t *testing.T) { t.Parallel() diff --git a/unroll/notification_map_test.go b/unroll/notification_map_test.go index fea01e23a..d93a75a0c 100644 --- a/unroll/notification_map_test.go +++ b/unroll/notification_map_test.go @@ -10,11 +10,8 @@ import ( // TestMapTxconfirmNotification pins the disposition of every txconfirm // lifecycle event at the unroll subscriber boundary. The load-bearing -// cases are TxFinalized and TxReorged: subscribers receive the FULL -// reorg-aware lifecycle, and before this mapping existed both fell into -// the unknown-notification arm and terminally failed the unroll job — -// finality (synthesized by every non-lnd backend once the confirmation -// passed the reorg-safety depth) killed otherwise-healthy exits. +// cases are TxFinalized and TxReorged: this planner has durable transitions +// for both, so the adapter must preserve rather than drop or collapse them. func TestMapTxconfirmNotification(t *testing.T) { t.Parallel() @@ -37,9 +34,7 @@ func TestMapTxconfirmNotification(t *testing.T) { require.EqualValues(t, 3, confirmed.NumConfs) }) - t.Run("finalized maps to confirmation, not failure", func( - t *testing.T) { - + t.Run("finalized maps to finality", func(t *testing.T) { t.Parallel() msg, ok := mapTxconfirmNotification(&txconfirm.TxFinalized{ @@ -49,28 +44,21 @@ func TestMapTxconfirmNotification(t *testing.T) { }) require.True(t, ok) - confirmed, isConfirmed := msg.(*TxConfirmedMsg) - require.True( - t, isConfirmed, "finality must replay a "+ - "confirmation; mapping it to a failure "+ - "terminally kills the unroll job", - ) - require.Equal(t, txid, confirmed.Txid) - require.EqualValues(t, 130, confirmed.Height) - require.EqualValues(t, 6, confirmed.NumConfs) + finalized, isFinalized := msg.(*TxFinalizedMsg) + require.True(t, isFinalized) + require.Equal(t, txid, finalized.Txid) }) - t.Run("reorged is dropped", func(t *testing.T) { + t.Run("reorged maps to reorg", func(t *testing.T) { t.Parallel() - _, ok := mapTxconfirmNotification(&txconfirm.TxReorged{ + msg, ok := mapTxconfirmNotification(&txconfirm.TxReorged{ Txid: txid, }) - require.False( - t, ok, "TxReorged is best-effort and superseded by "+ - "the next reliable event; it must not "+ - "become a failure", - ) + require.True(t, ok) + reorged, isReorged := msg.(*TxReorgedMsg) + require.True(t, isReorged) + require.Equal(t, txid, reorged.Txid) }) t.Run("failed maps to failure", func(t *testing.T) { diff --git a/unroll/reconcile.go b/unroll/reconcile.go new file mode 100644 index 000000000..1aec81387 --- /dev/null +++ b/unroll/reconcile.go @@ -0,0 +1,320 @@ +package unroll + +import ( + "context" + "fmt" + + "github.com/btcsuite/btcd/chainhash/v2" + "github.com/btcsuite/btcd/wire/v2" + "github.com/lightninglabs/wavelength/lib/recovery" + "github.com/lightninglabs/wavelength/unrollplan" + fn "github.com/lightningnetwork/lnd/fn/v2" +) + +// ConfirmedAnchor describes a tx confirmation that the backend reports +// is currently live on the canonical chain. +type ConfirmedAnchor struct { + // Txid is the confirmed transaction hash. + Txid chainhash.Hash + + // Height is the block height the transaction confirmed at. + Height int32 +} + +// SpendAnchor describes an outpoint spend that the backend reports is +// currently live on the canonical chain. +type SpendAnchor struct { + // Outpoint is the spent outpoint. + Outpoint wire.OutPoint + + // SpendingTxid is the transaction that consumed the outpoint. + SpendingTxid chainhash.Hash + + // SpendingHeight is the block height the spending tx confirmed at. + SpendingHeight int32 +} + +// ChainReconciler is the narrow interface the unroll actor uses on +// restart to verify that the chain anchors recorded in its checkpoint +// are still live on the canonical chain. +// +// The interface is intentionally small: it answers two questions, both +// keyed on data the actor already has in its checkpoint. Implementations +// query the backend (lnd's chainntnfs, neutrino, or a bitcoind RPC) and +// return fn.None for "not on chain anymore" so the unroll layer can +// stay backend-agnostic. +// +// Both methods return errors for transport-level failures the caller +// should retry; an "anchor is not on chain" answer is conveyed via +// fn.None, not via error. +type ChainReconciler interface { + // ConfirmedTx reports the current confirmation status of a tx on + // the canonical chain. fn.None means the tx is no longer + // confirmed (typically because the block was reorged out while + // the daemon was offline). + ConfirmedTx(ctx context.Context, + txid chainhash.Hash) (fn.Option[ConfirmedAnchor], error) + + // SpentOutpoint reports the current spend status of an outpoint + // on the canonical chain. fn.None means the outpoint is currently + // unspent. + SpentOutpoint(ctx context.Context, + outpoint wire.OutPoint) (fn.Option[SpendAnchor], error) +} + +// NegativeEvidenceReconciler optionally declares whether fn.None is backed by +// an authoritative completed query. Reconcilers that do not implement this +// interface retain the full ChainReconciler contract and are assumed capable +// of proving both positive and negative answers. +type NegativeEvidenceReconciler interface { + // HasAuthoritativeNegativeEvidence reports whether the implementation + // can prove absence without inferring it from silence or a timeout. + HasAuthoritativeNegativeEvidence() bool +} + +// ChainReconcilerFactory builds a ChainReconciler bound to a specific +// per-target unroll actor. Per-target actors invoke the factory after +// loading their proof; the target outpoint identifies the calling +// actor and lets the implementation namespace its chainsource probe +// caller IDs so two actors that share a proof-graph ancestor (the +// common case for sibling VTXOs) cannot collide on chainsource +// service keys when they reconcile concurrently. The proof argument +// supplies output scripts because lnd's tx index is not always +// available, so the (txid, pkScript) pair is the canonical way to +// identify a tx during a historical block scan. Test stubs that do +// not need either argument can ignore them. +type ChainReconcilerFactory func(target wire.OutPoint, + proof *recovery.Proof) ChainReconciler + +// reconcileCheckpoint walks the persisted checkpoint and prunes any +// chain anchor that the reconciler reports is no longer live on the +// canonical chain. The checkpoint is mutated in place. +// +// The reconciliation order mirrors applyReorgedEvent: +// +// - For every confirmed proof-graph txid, ask the reconciler whether +// it is still confirmed. Anchors that are not are dropped, and so +// are their descendants in the proof graph (State.Validate's +// topological invariant requires every confirmed/in-flight node to +// have confirmed parents). +// - If the target tx was confirmed but is now absent, clear +// TargetConfirmHeight and downgrade the sweep to Pending — same +// gate the live reorg reducer enforces. +// - If a Broadcasted / Confirmed sweep tx is no longer on chain, +// reset the sweep state so the actor re-broadcasts on its next +// planning step. +// - Reconcile the target outpoint spend status against any persisted +// ProvisionalExternalSpend anchor: set, refresh, or clear it so +// the restored FSM enters the same parked state a live +// SpendObservedEvent / SpendReorgedEvent sequence would have +// produced. +// +// On any reconciler error we surface it untouched: the actor must not +// proceed to broadcast off a checkpoint we could not verify. +func reconcileCheckpoint(ctx context.Context, reconciler ChainReconciler, + proof *recovery.Proof, checkpoint *actorCheckpoint) error { + + if reconciler == nil || checkpoint == nil || proof == nil { + return nil + } + + // Snapshot the input list so the loop iterates over a stable + // slice while we mutate the live ConfirmedTxids field. + confirmedIn := append( + []chainhash.Hash(nil), checkpoint.State.ConfirmedTxids..., + ) + + for _, txid := range confirmedIn { + anchor, err := reconciler.ConfirmedTx(ctx, txid) + if err != nil { + return fmt.Errorf("reconcile confirmed tx %s: %w", txid, + err) + } + if anchor.IsSome() { + // The tx is still confirmed. Refresh the + // target-derived height in case the tx was re-mined + // at a different block height while the daemon was + // offline; leaving the stale height in place would + // make CSV maturity look closer than it actually is + // and could drive a premature sweep on the next + // planning step. Also raise the checkpoint's + // best-height watermark to at least the live anchor + // height so derived state is internally consistent. + liveHeight := anchor.UnsafeFromSome().Height + if txid == proof.TargetOutpoint().Hash { + checkpoint.State.TargetConfirmHeight = fn.Some( + liveHeight, + ) + } + if liveHeight > checkpoint.Height { + checkpoint.Height = liveHeight + } + + continue + } + + pruneReorgedSubtree(proof, &checkpoint.State, txid) + + if txid == proof.TargetOutpoint().Hash { + downgradeSweepOnTargetLoss(&checkpoint.State) + } + } + + // Verify only a sweep checkpoint that already claims confirmation. + // A merely broadcast sweep may legitimately be unconfirmed, so notifier + // silence cannot distinguish its expected state from a reorg. + if checkpoint.State.Sweep.Txid.IsSome() && + checkpoint.State.Sweep.Status == + unrollplan.SweepStatusConfirmed { + + sweepTxid := checkpoint.State.Sweep.Txid.UnsafeFromSome() + anchor, err := reconciler.ConfirmedTx(ctx, sweepTxid) + if err != nil { + return fmt.Errorf("reconcile sweep tx %s: %w", + sweepTxid, err) + } + switch { + case anchor.IsNone(): + // Sweep is no longer on chain. Reset so the next + // derivation re-broadcasts using the cached bytes. + checkpoint.State.Sweep.Status = + unrollplan.SweepStatusPending + checkpoint.State.Sweep.Txid = + fn.None[chainhash.Hash]() + checkpoint.State.Sweep.ConfirmHeight = + fn.None[int32]() + + case checkpoint.State.Sweep.Status == + unrollplan.SweepStatusConfirmed: + + // Sweep is still on chain but at potentially a new + // height. Recompute ConfirmHeight from the live + // answer so a sweep that confirmed in a different + // block is not stuck reporting a stale height. + checkpoint.State.Sweep.ConfirmHeight = fn.Some( + anchor.UnsafeFromSome().Height, + ) + } + } + + if err := reconcileExternalSpend( + ctx, reconciler, proof, checkpoint, + ); err != nil { + return err + } + + return nil +} + +// reconcileExternalSpend cross-checks the target outpoint's spend +// status on the canonical chain against any ProvisionalExternalSpend +// anchor the checkpoint carries. Four cases land here: +// +// 1. Chain says target spent + checkpoint anchor matches the same +// spender txid: no-op (refresh the height in case it shifted to a +// re-organized block). +// 2. Chain says target spent + checkpoint anchor for a DIFFERENT +// spender: update the anchor to the live spender. +// 3. Chain says target spent + no checkpoint anchor: install one. +// This is the "external spend confirmed while the daemon was +// offline" path. +// 4. Chain says target unspent + checkpoint anchor present: clear +// the anchor. The spending block was reorged out during downtime. +// +// In cases 2/3, "spent by a proof-graph node" or "spent by our own +// sweep" are treated as benign: those are the same classifications +// the live spend-watch handler runs in handleSpendObserved. +func reconcileExternalSpend(ctx context.Context, reconciler ChainReconciler, + proof *recovery.Proof, checkpoint *actorCheckpoint) error { + + // With no persisted positive there is nothing a positive-only notifier + // can safely disprove. The live historical spend watch is re-armed + // before Resume routes work; later canonicality Snapshot/Ready supplies + // the authoritative negative barrier required by one-confirmation mode. + if checkpoint.ProvisionalExternalSpend.IsNone() { + capability, ok := reconciler.(NegativeEvidenceReconciler) + if ok && !capability.HasAuthoritativeNegativeEvidence() { + return nil + } + } + + targetOutpoint := proof.TargetOutpoint() + anchor, err := reconciler.SpentOutpoint(ctx, targetOutpoint) + if err != nil { + return fmt.Errorf("reconcile target spend %s: %w", + targetOutpoint, err) + } + + if anchor.IsNone() { + // Case 4: chain reports target unspent. Any persisted + // anchor was reorged out while we were offline; clear it + // so the FSM does NOT park in + // AwaitingExternalSpendFinality on restore. + checkpoint.ProvisionalExternalSpend = + fn.None[ExternalSpendAnchor]() + + return nil + } + + live := anchor.UnsafeFromSome() + + // Skip benign spenders that the live spend-watch path would also + // classify as "expected materialization traffic": a proof-graph + // node, or our own sweep. + if proof != nil { + if _, ok := proof.Node(live.SpendingTxid); ok { + checkpoint.ProvisionalExternalSpend = + fn.None[ExternalSpendAnchor]() + + return nil + } + } + if checkpoint.State.Sweep.Txid.IsSome() && + checkpoint.State.Sweep.Txid.UnsafeFromSome() == + live.SpendingTxid { + + checkpoint.ProvisionalExternalSpend = + fn.None[ExternalSpendAnchor]() + + return nil + } + + // Cases 1, 2, 3: install or refresh the anchor with the live + // spender so the restored FSM enters AwaitingExternalSpendFinality + // rather than blindly resuming planner-driven materialization. + checkpoint.ProvisionalExternalSpend = fn.Some(ExternalSpendAnchor{ + SpendingTxid: live.SpendingTxid, + SpendingHeight: live.SpendingHeight, + }) + + return nil +} + +// pruneReorgedSubtree drops a txid and every transitive descendant from +// both ConfirmedTxids and InFlightTxids, and clears any deferred +// checkpoints referencing the subtree. +func pruneReorgedSubtree(proof *recovery.Proof, state *unrollplan.State, + root chainhash.Hash) { + + subtree := collectReorgedSubtree( + &Environment{Proof: proof}, root, + ) + state.ConfirmedTxids = removeHashes(state.ConfirmedTxids, subtree) + state.InFlightTxids = removeHashes(state.InFlightTxids, subtree) +} + +// downgradeSweepOnTargetLoss resets the target-derived planner state +// when the target tx is no longer on the canonical chain. Mirrors the +// "target reorg" branch of applyReorgedEvent so restart reconciliation +// and live rollback converge on identical post-conditions. +func downgradeSweepOnTargetLoss(state *unrollplan.State) { + state.TargetConfirmHeight = fn.None[int32]() + + if state.Sweep.Status == unrollplan.SweepStatusPending { + return + } + + state.Sweep.Status = unrollplan.SweepStatusPending + state.Sweep.Txid = fn.None[chainhash.Hash]() + state.Sweep.ConfirmHeight = fn.None[int32]() +} diff --git a/unroll/reconcile_chainsource.go b/unroll/reconcile_chainsource.go new file mode 100644 index 000000000..366f48480 --- /dev/null +++ b/unroll/reconcile_chainsource.go @@ -0,0 +1,319 @@ +package unroll + +import ( + "context" + "errors" + "fmt" + "log/slog" + "time" + + "github.com/btcsuite/btcd/chainhash/v2" + "github.com/btcsuite/btcd/wire/v2" + "github.com/btcsuite/btclog/v2" + "github.com/lightninglabs/wavelength/baselib/actor" + "github.com/lightninglabs/wavelength/chainsource" + "github.com/lightninglabs/wavelength/lib/recovery" + fn "github.com/lightningnetwork/lnd/fn/v2" +) + +// defaultReconcileProbeTimeout bounds how long the chainsource-backed +// reconciler waits for a confirmation or spend event before reporting +// that canonical evidence is unavailable. lnd's chainntnfs dispatches +// historical confirmation events immediately when the tx is in chain +// and the height-hint is fresh; legitimate "still confirmed" answers +// return well under a second on a healthy node, so a 10-second +// budget leaves comfortable headroom for slow restarts while still +// failing closed when the backend cannot answer. +const defaultReconcileProbeTimeout = 10 * time.Second + +// ErrReconcileEvidenceUnavailable means restart reconciliation could not +// obtain objective chain evidence. Callers must retain the durable checkpoint +// and retry; absence or timeout is never proof of a reorg or an unspent output. +var ErrReconcileEvidenceUnavailable = errors.New("reconciliation chain " + + "evidence unavailable") + +// ChainSourceReconcilerConfig configures NewChainSourceReconciler. +type ChainSourceReconcilerConfig struct { + // ChainSource is the actor ref used to issue RegisterConf / + // RegisterSpend probes. + ChainSource actor.ActorRef[ + chainsource.ChainSourceMsg, chainsource.ChainSourceResp, + ] + + // Proof is the immutable recovery graph the reconciler queries + // pkScripts from. Required. + Proof *recovery.Proof + + // CallerID prefix used when registering probes; the reconciler + // appends per-probe suffixes so chainsource sees a unique key + // per (caller, txid / outpoint, confs) tuple. + // + // Callers that share the underlying ChainSource between multiple + // reconcilers MUST give each instance a unique prefix (typically + // by including the per-actor target identity) so concurrent + // probes of the same shared proof-graph txid do not collide on + // the chainsource service-key namespace. + CallerID string + + // ProbeTimeout bounds each individual confirmation / spend + // probe. Zero falls back to defaultReconcileProbeTimeout. + // Slow chainsource backends can legitimately need a higher + // budget here. A probe that times out leaves the durable checkpoint + // unchanged and causes restart recovery to fail closed for retry. + ProbeTimeout time.Duration + + // CleanupTimeout bounds the best-effort unregister send that + // runs when a probe is interrupted before its future fires. + // Zero falls back to 5 seconds. + CleanupTimeout time.Duration + + // Log is an optional logger. Probe timeouts are surfaced at warn level + // so operators can distinguish unavailable evidence from an actual + // negative canonicality result. Zero falls back to btclog.Disabled. + Log fn.Option[btclog.Logger] +} + +// chainSourceReconciler is a ChainReconciler that answers queries by +// probing the chainsource actor with short-timeout RegisterConf and +// RegisterSpend requests in future mode. +// +// The implementation is intentionally simple: it does NOT consume a +// dedicated "is this tx in chain right now" API (chainsource does not expose +// one today). Instead it leans on lnd's chainntnfs behavior of dispatching a +// historical positive immediately when the watched item is currently on +// chain. Silence cannot prove the negative, so a timed-out probe returns +// ErrReconcileEvidenceUnavailable and leaves the durable checkpoint intact. +// +// Probes that complete on their own self-clean (the chainsource +// sub-actor exits after delivering the single positive event in +// future mode); probes that time out enqueue a best-effort +// UnregisterConfRequest on a fresh background context so the +// long-lived chainsource sub-actor does not leak per restart. +type chainSourceReconciler struct { + chainSource actor.ActorRef[ + chainsource.ChainSourceMsg, chainsource.ChainSourceResp, + ] + proof *recovery.Proof + callerID string + probeTimeout time.Duration + cleanupTimeout time.Duration + log btclog.Logger +} + +// HasAuthoritativeNegativeEvidence reports false because chainsource's current +// subscription API emits positives but has no completed-scan absence result. +func (r *chainSourceReconciler) HasAuthoritativeNegativeEvidence() bool { + return false +} + +// NewChainSourceReconciler constructs a chainsource-backed +// ChainReconciler. The proof is required because chainsource probes +// rely on (txid, pkScript) pairs for historical scans. +func NewChainSourceReconciler(cfg ChainSourceReconcilerConfig) ChainReconciler { + if cfg.Proof == nil { + return nil + } + probeTimeout := cfg.ProbeTimeout + if probeTimeout <= 0 { + probeTimeout = defaultReconcileProbeTimeout + } + cleanupTimeout := cfg.CleanupTimeout + if cleanupTimeout <= 0 { + cleanupTimeout = 5 * time.Second + } + + return &chainSourceReconciler{ + chainSource: cfg.ChainSource, + proof: cfg.Proof, + callerID: cfg.CallerID, + probeTimeout: probeTimeout, + cleanupTimeout: cleanupTimeout, + log: cfg.Log.UnwrapOr(btclog.Disabled), + } +} + +// ConfirmedTx probes chainsource for the current confirmation status +// of txid. The probe registers a one-shot conf watch in future mode +// using the txid + first-output pkScript from the proof, then awaits +// the future with a bounded timeout. +func (r *chainSourceReconciler) ConfirmedTx(ctx context.Context, + txid chainhash.Hash) (fn.Option[ConfirmedAnchor], error) { + + node, ok := r.proof.Node(txid) + if !ok || node == nil || node.Tx == nil || + len(node.Tx.TxOut) == 0 { + return fn.None[ConfirmedAnchor](), fmt.Errorf("%w: tx %s has "+ + "no queryable output", ErrReconcileEvidenceUnavailable, + txid) + } + pkScript := append([]byte(nil), node.Tx.TxOut[0].PkScript...) + + callerID := fmt.Sprintf("%s-conf-%s", r.callerID, txid) + probeCtx, cancel := context.WithTimeout(ctx, r.probeTimeout) + defer cancel() + + resp, err := r.chainSource.Ask( + probeCtx, &chainsource.RegisterConfRequest{ + CallerID: callerID, + Txid: &txid, + PkScript: pkScript, + TargetConfs: 1, + }, + ).Await(probeCtx).Unpack() + if err != nil { + return fn.None[ConfirmedAnchor](), fmt.Errorf("%w: register "+ + "conf probe %s: %w", ErrReconcileEvidenceUnavailable, + txid, err) + } + + confResp, ok := resp.(*chainsource.RegisterConfResponse) + if !ok || confResp.Future == nil { + return fn.None[ConfirmedAnchor](), fmt.Errorf("register conf "+ + "probe %s: unexpected response %T", txid, resp) + } + + // Schedule a best-effort cleanup unregister in case the probe + // times out: the chainsource sub-actor would otherwise stay + // alive watching for a confirmation that, by assumption, is + // never coming. The cleanup runs on a fresh background context + // so it executes even when the probe context was cancelled. + probeTxid := txid + probePkScript := pkScript + //nolint:contextcheck // cleanup intentionally uses its own context + defer r.cleanupConfWatch(callerID, &probeTxid, probePkScript) + + event, err := confResp.Future.Await(probeCtx).Unpack() + if err != nil { + if errors.Is(err, context.DeadlineExceeded) || + errors.Is(err, context.Canceled) { + + r.log.WarnS(ctx, "Reconciler conf probe timed out; "+ + "leaving checkpoint unchanged", err, + slog.String("txid", txid.String()), + slog.Duration("timeout", r.probeTimeout), + ) + } + + return fn.None[ConfirmedAnchor](), fmt.Errorf("%w: await conf "+ + "probe %s: %w", ErrReconcileEvidenceUnavailable, txid, + err) + } + + return fn.Some(ConfirmedAnchor{ + Txid: event.Txid, + Height: event.BlockHeight, + }), nil +} + +// SpentOutpoint probes chainsource for the current spend status of +// outpoint. Symmetric to ConfirmedTx: registers a future-mode spend +// watch with the outpoint's pkScript and awaits the future with a +// bounded timeout, scheduling an UnregisterSpendRequest cleanup on +// the way out. +func (r *chainSourceReconciler) SpentOutpoint(ctx context.Context, + outpoint wire.OutPoint) (fn.Option[SpendAnchor], error) { + + node, ok := r.proof.Node(outpoint.Hash) + if !ok || node == nil || node.Tx == nil { + return fn.None[SpendAnchor](), fmt.Errorf("%w: outpoint %s "+ + "has no queryable transaction", + ErrReconcileEvidenceUnavailable, outpoint) + } + if int(outpoint.Index) >= len(node.Tx.TxOut) { + return fn.None[SpendAnchor](), fmt.Errorf("%w: outpoint %s "+ + "has no queryable output", + ErrReconcileEvidenceUnavailable, outpoint) + } + pkScript := append( + []byte(nil), node.Tx.TxOut[outpoint.Index].PkScript..., + ) + + callerID := fmt.Sprintf("%s-spend-%s", r.callerID, outpoint) + probeCtx, cancel := context.WithTimeout(ctx, r.probeTimeout) + defer cancel() + + probeOutpoint := outpoint + resp, err := r.chainSource.Ask( + probeCtx, &chainsource.RegisterSpendRequest{ + CallerID: callerID, + Outpoint: &probeOutpoint, + PkScript: pkScript, + }, + ).Await(probeCtx).Unpack() + if err != nil { + return fn.None[SpendAnchor](), fmt.Errorf("%w: register spend "+ + "probe %s: %w", ErrReconcileEvidenceUnavailable, + outpoint, err) + } + + spendResp, ok := resp.(*chainsource.RegisterSpendResponse) + if !ok || spendResp.Future == nil { + return fn.None[SpendAnchor](), fmt.Errorf("register spend "+ + "probe %s: unexpected response %T", outpoint, resp) + } + + //nolint:contextcheck // cleanup intentionally uses its own context + defer r.cleanupSpendWatch(callerID, &probeOutpoint) + + event, err := spendResp.Future.Await(probeCtx).Unpack() + if err != nil { + if errors.Is(err, context.DeadlineExceeded) || + errors.Is(err, context.Canceled) { + + r.log.WarnS(ctx, "Reconciler spend probe timed "+ + "out; leaving checkpoint unchanged", err, + slog.String("outpoint", outpoint.String()), + slog.Duration("timeout", r.probeTimeout), + ) + } + + return fn.None[SpendAnchor](), fmt.Errorf("%w: await spend "+ + "probe %s: %w", ErrReconcileEvidenceUnavailable, + outpoint, err) + } + + return fn.Some(SpendAnchor{ + Outpoint: event.Outpoint, + SpendingTxid: event.SpendingTxid, + SpendingHeight: event.SpendingHeight, + }), nil +} + +// cleanupConfWatch sends a best-effort UnregisterConfRequest on a +// fresh background context so a timed-out probe does not leak the +// underlying chainsource sub-actor. Errors are intentionally ignored +// because the cleanup is purely a hygiene operation. +func (r *chainSourceReconciler) cleanupConfWatch(callerID string, + txid *chainhash.Hash, pkScript []byte) { + + cleanupCtx, cancel := context.WithTimeout( + context.Background(), r.cleanupTimeout, + ) + defer cancel() + + _, _ = r.chainSource.Ask( + cleanupCtx, &chainsource.UnregisterConfRequest{ + CallerID: callerID, + Txid: txid, + PkScript: pkScript, + TargetConfs: 1, + }, + ).Await(cleanupCtx).Unpack() +} + +// cleanupSpendWatch is the spend-side analogue of cleanupConfWatch. +func (r *chainSourceReconciler) cleanupSpendWatch(callerID string, + outpoint *wire.OutPoint) { + + cleanupCtx, cancel := context.WithTimeout( + context.Background(), r.cleanupTimeout, + ) + defer cancel() + + _, _ = r.chainSource.Ask( + cleanupCtx, &chainsource.UnregisterSpendRequest{ + CallerID: callerID, + Outpoint: outpoint, + }, + ).Await(cleanupCtx).Unpack() +} diff --git a/unroll/reconcile_chainsource_test.go b/unroll/reconcile_chainsource_test.go new file mode 100644 index 000000000..dc4aea720 --- /dev/null +++ b/unroll/reconcile_chainsource_test.go @@ -0,0 +1,317 @@ +package unroll + +import ( + "context" + "fmt" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/btcsuite/btcd/btcutil/v2" + "github.com/btcsuite/btcd/chainhash/v2" + "github.com/btcsuite/btcd/wire/v2" + "github.com/lightninglabs/wavelength/baselib/actor" + "github.com/lightninglabs/wavelength/chainsource" + fn "github.com/lightningnetwork/lnd/fn/v2" + "github.com/stretchr/testify/require" +) + +// reconcileMockBackend is a minimal ChainBackend test double tailored +// for the reconciler concurrency test. It records every RegisterConf +// call so the test can assert per-probe caller-ID uniqueness, and +// pre-arms a positive confirmation reply that fires immediately when +// the txid matches the configured target. +type reconcileMockBackend struct { + confirmTxid chainhash.Hash + confirmHeight int32 + + // registers records (callerID-like txid, pkScript) pairs as seen + // from successive RegisterConf calls. The caller-ID is not visible + // at this layer (chainsource consumes it before spawning the sub- + // actor), so the test instead asserts that the sub-actor IDs the + // system spawned are distinct via System.Has on the service-key. + mu sync.Mutex + registers []struct { + Txid chainhash.Hash + PkScript []byte + } + + // bestHeight is reported by BestBlock. + bestHeight int32 + + // epochCh / epochCancel back RegisterBlocks. + epochCh chan *chainsource.BlockEpoch + epochCancel atomic.Int32 +} + +func newReconcileMockBackend() *reconcileMockBackend { + return &reconcileMockBackend{ + bestHeight: 200, + epochCh: make(chan *chainsource.BlockEpoch, 1), + } +} + +func (m *reconcileMockBackend) EstimateFee(context.Context, uint32) ( + btcutil.Amount, error) { + + return 1000, nil +} + +func (m *reconcileMockBackend) BestBlock(context.Context) (int32, + chainhash.Hash, error) { + + return m.bestHeight, chainhash.Hash{}, nil +} + +func (m *reconcileMockBackend) TestMempoolAccept(context.Context, + ...*wire.MsgTx) ([]chainsource.MempoolAcceptResult, error) { + + return nil, nil +} + +func (m *reconcileMockBackend) BroadcastTx(context.Context, *wire.MsgTx, + string) error { + + return nil +} + +func (m *reconcileMockBackend) SubmitPackage(context.Context, []*wire.MsgTx, + *wire.MsgTx) error { + + return nil +} + +// RegisterConf records the registration and arms a one-shot positive +// confirmation when the requested txid matches the configured target. +// Every call returns a fresh ConfRegistration with its own channels so +// concurrent registrants do not contend on a shared buffered channel. +func (m *reconcileMockBackend) RegisterConf(_ context.Context, + txid *chainhash.Hash, pkScript []byte, _, _ uint32, _ bool) ( + *chainsource.ConfRegistration, error) { + + m.mu.Lock() + m.registers = append(m.registers, struct { + Txid chainhash.Hash + PkScript []byte + }{ + Txid: *txid, + PkScript: append([]byte(nil), pkScript...), + }) + m.mu.Unlock() + + confCh := make(chan *chainsource.TxConfirmation, 1) + reorged := make(chan uint64, 1) + done := make(chan struct{}, 1) + + if txid != nil && *txid == m.confirmTxid { + blockTx := wire.NewMsgTx(2) + blockHash := chainhash.Hash{0x42} + confCh <- &chainsource.TxConfirmation{ + BlockHash: &blockHash, + BlockHeight: uint32(m.confirmHeight), + Tx: blockTx, + } + } + + return &chainsource.ConfRegistration{ + Confirmed: confCh, + Reorged: reorged, + Done: done, + Cancel: func() {}, + }, nil +} + +func (m *reconcileMockBackend) RegisterSpend(context.Context, *wire.OutPoint, + []byte, uint32) (*chainsource.SpendRegistration, error) { + + return &chainsource.SpendRegistration{ + Spend: make(chan *chainsource.SpendDetail, 1), + Reorged: make(chan uint64, 1), + Done: make(chan struct{}, 1), + Cancel: func() {}, + }, nil +} + +func (m *reconcileMockBackend) RegisterBlocks(context.Context) ( + *chainsource.BlockRegistration, error) { + + return &chainsource.BlockRegistration{ + Epochs: m.epochCh, + Cancel: func() { + m.epochCancel.Add(1) + }, + }, nil +} + +func (m *reconcileMockBackend) Start() error { return nil } +func (m *reconcileMockBackend) Stop() error { return nil } + +// TestChainSourceReconcilerConcurrentProbesDoNotCollide exercises the +// invariant that two reconcilers probing the same shared proof-graph +// txid against a single chainsource actor do NOT collide on chainsource +// service keys when each reconciler is built with a target-specific +// caller-ID prefix. +// +// Chainsource keys its per-probe sub-actors on +// (CallerID, txid/pkScript, TargetConfs) (see handleRegisterConf in +// chainsource/chainsource.go). Two reconcilers built with the same +// static prefix probing the same txid would map to identical service +// keys; the second Spawn would collide with the first sub-actor's +// registration and drop or merge the second probe. Production wiring +// in waved/server.go bakes the per-actor target outpoint into the +// caller-ID prefix exactly to avoid this; this test pins that +// invariant for the chainsource-backed reconciler. +func TestChainSourceReconcilerConcurrentProbesDoNotCollide(t *testing.T) { + t.Parallel() + + proof := buildLinearProof(t) + roots := proof.RootTxids() + require.Len(t, roots, 1, "linear proof should have one root") + rootTxid := roots[0] + + backend := newReconcileMockBackend() + backend.confirmTxid = rootTxid + backend.confirmHeight = 150 + + system := actor.NewActorSystem() + defer func() { + _ = system.Shutdown(t.Context()) + }() + + chainSource := chainsource.NewChainSourceActor( + chainsource.ChainSourceConfig{ + Backend: backend, + System: system, + }, + ) + chainRef := chainsource.ChainSourceKey.Spawn( + system, "chainsource-reconcile", chainSource, + ) + + target1 := proof.TargetOutpoint() + target2 := wire.OutPoint{ + Hash: chainhash.Hash{ + 0xaa, + }, + Index: 7, + } + + mkReconciler := func(target wire.OutPoint) ChainReconciler { + return NewChainSourceReconciler(ChainSourceReconcilerConfig{ + ChainSource: chainRef, + Proof: proof, + CallerID: fmt.Sprintf( + "unroll-reconcile-%s", target, + ), + ProbeTimeout: 5 * time.Second, + }) + } + + rec1 := mkReconciler(target1) + rec2 := mkReconciler(target2) + + type probeResult struct { + anchor fn.Option[ConfirmedAnchor] + err error + } + + results := make(chan probeResult, 2) + probe := func(r ChainReconciler) { + ctx, cancel := context.WithTimeout( + t.Context(), 10*time.Second, + ) + defer cancel() + + anchor, err := r.ConfirmedTx(ctx, rootTxid) + results <- probeResult{anchor: anchor, err: err} + } + + go probe(rec1) + go probe(rec2) + + for range 2 { + select { + case res := <-results: + require.NoError( + t, res.err, + "concurrent reconciler probe failed", + ) + require.True( + t, res.anchor.IsSome(), + "reconciler did not see confirmation; a "+ + "static caller-ID collision would "+ + "silently swallow the second probe", + ) + require.Equal( + t, backend.confirmHeight, + res.anchor.UnsafeFromSome().Height, + ) + + case <-time.After(15 * time.Second): + t.Fatal( + "timed out waiting for concurrent " + + "reconciler probes; static " + + "caller-ID would cause one probe " + + "to wait forever", + ) + } + } + + // Both reconcilers must have hit the backend with a RegisterConf + // for the shared txid — confirming no internal short-circuit. + backend.mu.Lock() + defer backend.mu.Unlock() + require.Len( + t, backend.registers, 2, + "expected both reconciler probes to reach the backend", + ) + for _, r := range backend.registers { + require.Equal(t, rootTxid, r.Txid) + } +} + +// TestChainSourceReconcilerTimeoutFailsClosed proves notifier silence is +// reported as unavailable evidence for both confirmation and spend probes. It +// must never be converted into an objective absent or unspent answer. +func TestChainSourceReconcilerTimeoutFailsClosed(t *testing.T) { + t.Parallel() + + proof := buildLinearProof(t) + backend := newReconcileMockBackend() + system := actor.NewActorSystem() + defer func() { + _ = system.Shutdown(t.Context()) + }() + + chainRef := chainsource.ChainSourceKey.Spawn( + system, "chainsource-reconcile-timeout", + chainsource.NewChainSourceActor( + chainsource.ChainSourceConfig{ + Backend: backend, + System: system, + }, + ), + ) + reconciler := NewChainSourceReconciler( + ChainSourceReconcilerConfig{ + ChainSource: chainRef, + Proof: proof, + CallerID: "unroll-reconcile-timeout", + ProbeTimeout: 25 * time.Millisecond, + CleanupTimeout: time.Second, + }, + ) + + rootTxids := proof.RootTxids() + require.NotEmpty(t, rootTxids) + confirmed, err := reconciler.ConfirmedTx(t.Context(), rootTxids[0]) + require.ErrorIs(t, err, ErrReconcileEvidenceUnavailable) + require.True(t, confirmed.IsNone()) + + spent, err := reconciler.SpentOutpoint( + t.Context(), proof.TargetOutpoint(), + ) + require.ErrorIs(t, err, ErrReconcileEvidenceUnavailable) + require.True(t, spent.IsNone()) +} diff --git a/unroll/reconcile_test.go b/unroll/reconcile_test.go new file mode 100644 index 000000000..4cd3b44e3 --- /dev/null +++ b/unroll/reconcile_test.go @@ -0,0 +1,614 @@ +package unroll + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/btcsuite/btcd/chainhash/v2" + "github.com/btcsuite/btcd/wire/v2" + "github.com/btcsuite/btclog/v2" + "github.com/lightninglabs/wavelength/baselib/actor" + "github.com/lightninglabs/wavelength/lib/recovery" + "github.com/lightninglabs/wavelength/unrollplan" + fn "github.com/lightningnetwork/lnd/fn/v2" + "github.com/stretchr/testify/require" +) + +// stubChainReconciler is a fully controllable ChainReconciler test +// double. The two maps drive the ConfirmedTx / SpentOutpoint answers +// and a non-nil err field lets a test inject a transport failure on +// either query. +type stubChainReconciler struct { + confirmed map[chainhash.Hash]ConfirmedAnchor + spent map[wire.OutPoint]SpendAnchor + err error +} + +// positiveOnlyReconciler models the current chainsource subscription surface: +// it can replay historical positives, but silence is not an authoritative +// unspent answer. +type positiveOnlyReconciler struct { + *stubChainReconciler + spendCalls int +} + +// HasAuthoritativeNegativeEvidence reports that notifier silence cannot prove +// absence. +func (p *positiveOnlyReconciler) HasAuthoritativeNegativeEvidence() bool { + return false +} + +// SpentOutpoint records any attempted unanchored negative probe. +func (p *positiveOnlyReconciler) SpentOutpoint(ctx context.Context, + outpoint wire.OutPoint) (fn.Option[SpendAnchor], error) { + + p.spendCalls++ + + return p.stubChainReconciler.SpentOutpoint(ctx, outpoint) +} + +// ConfirmedTx returns the configured anchor for txid, or fn.None. +func (s *stubChainReconciler) ConfirmedTx(_ context.Context, + txid chainhash.Hash) (fn.Option[ConfirmedAnchor], error) { + + if s.err != nil { + return fn.None[ConfirmedAnchor](), s.err + } + + anchor, ok := s.confirmed[txid] + if !ok { + return fn.None[ConfirmedAnchor](), nil + } + + return fn.Some(anchor), nil +} + +// SpentOutpoint returns the configured anchor for outpoint, or fn.None. +func (s *stubChainReconciler) SpentOutpoint(_ context.Context, + outpoint wire.OutPoint) (fn.Option[SpendAnchor], error) { + + if s.err != nil { + return fn.None[SpendAnchor](), s.err + } + + anchor, ok := s.spent[outpoint] + if !ok { + return fn.None[SpendAnchor](), nil + } + + return fn.Some(anchor), nil +} + +// TestPositiveOnlyReconcilerSkipsUnanchoredSpendProbe verifies that an ordinary +// restart does not wait for notifier silence to prove the target unspent. With +// no persisted spend anchor there is no positive fact to reconcile; the actor +// re-arms its live spend watch and the durable relive guard protects the +// offline uncertainty window. +func TestPositiveOnlyReconcilerSkipsUnanchoredSpendProbe(t *testing.T) { + proof := buildLinearProof(t) + rootTxid := proof.RootTxids()[0] + reconciler := &positiveOnlyReconciler{ + stubChainReconciler: &stubChainReconciler{ + confirmed: map[chainhash.Hash]ConfirmedAnchor{ + rootTxid: { + Txid: rootTxid, + Height: 101, + }, + }, + }, + } + checkpoint := &actorCheckpoint{ + Version: checkpointVersion, + Height: 101, + Started: true, + State: unrollplan.State{ + ConfirmedTxids: []chainhash.Hash{ + rootTxid, + }, + }, + } + + err := reconcileCheckpoint( + t.Context(), reconciler, proof, checkpoint, + ) + require.NoError(t, err) + require.Zero(t, reconciler.spendCalls) + require.Equal( + t, []chainhash.Hash{rootTxid}, checkpoint.State.ConfirmedTxids, + ) +} + +// restoreHarness boots a fresh unroll behavior from a fabricated +// checkpoint so the reconcile tests can inspect the post-reconciliation +// FSM state without running the rest of the actor's IO surface. +func restoreHarness(t *testing.T, proof *recovery.Proof, + checkpoint *actorCheckpoint, reconciler ChainReconciler) (*behavior, + *fakeTxConfirmRef, *fakeChainSourceRef) { + + t.Helper() + + desc := testDescriptor(t, proof.TargetOutpoint(), proof.CSVDelay()) + store := newMemCheckpointStore() + txconfirmRef := &fakeTxConfirmRef{} + chainRef := &fakeChainSourceRef{} + + raw, err := encodeCheckpoint(checkpoint) + require.NoError(t, err) + + err = store.SaveCheckpoint(t.Context(), actor.CheckpointParams{ + ActorID: "reconcile-test", + StateType: checkpointStateType, + StateData: raw, + Version: checkpointVersion, + }) + require.NoError(t, err) + + cfg := Config{ + TargetOutpoint: proof.TargetOutpoint(), + ActorID: "reconcile-test", + DeliveryStore: store, + ProofAssembler: &mockProofAssembler{ + proof: proof, + }, + VTXOStore: &mockVTXOStore{ + desc: desc, + }, + TxConfirmRef: txconfirmRef, + ChainSource: chainRef, + Wallet: &fakeSweepWallet{}, + Log: fn.Some(btclog.Disabled), + } + if reconciler != nil { + // Wrap the stub in a factory that ignores the proof, so + // the production code path (which always calls the + // factory) sees the test stub unchanged. + stub := reconciler + cfg.ChainReconcilerFactory = fn.Some( + ChainReconcilerFactory( + func(wire.OutPoint, + *recovery.Proof) ChainReconciler { + + return stub + }, + ), + ) + } + + beh := &behavior{ + cfg: cfg, + log: btclog.Disabled, + } + require.NoError(t, beh.restoreCheckpoint(t.Context())) + + return beh, txconfirmRef, chainRef +} + +// TestReconcileTargetReorgedOfflineResumesMaterialization restores a +// checkpoint whose target tx is recorded as confirmed but is no longer +// on the canonical chain. After reconciliation the actor must restart +// without the stale TargetConfirmHeight, in PhaseMaterializing, so it +// does not broadcast a sweep on a target that has been reorged out. +func TestReconcileTargetReorgedOfflineResumesMaterialization(t *testing.T) { + proof := buildLinearProof(t) + rootTxid := proof.RootTxids()[0] + targetTxid := proof.TargetOutpoint().Hash + + checkpoint := &actorCheckpoint{ + Version: checkpointVersion, + Height: 103, + Started: true, + Trigger: TriggerRestart, + State: unrollplan.State{ + ConfirmedTxids: []chainhash.Hash{ + rootTxid, targetTxid, + }, + TargetConfirmHeight: fn.Some[int32](102), + }, + } + + // Reconciler reports the root is still on chain at H=101 but the + // target tx is absent — its confirming block was reorged out + // while the daemon was offline. + reconciler := &stubChainReconciler{ + confirmed: map[chainhash.Hash]ConfirmedAnchor{ + rootTxid: { + Txid: rootTxid, + Height: 101, + }, + }, + } + + beh, _, _ := restoreHarness(t, proof, checkpoint, reconciler) + + actorInstance := actor.NewActor(actor.ActorConfig[Msg, Resp]{ + ID: "reconcile-test", + Behavior: &txExecAdapter{b: beh, ax: newMemExecFor(beh)}, + MailboxSize: 16, + }) + beh.selfRef = actorInstance.TellRef() + actorInstance.Start() + t.Cleanup(actorInstance.Stop) + + mustAsk(t, actorInstance.Ref(), &ResumeUnrollRequest{Height: 103}) + + require.Eventually(t, func() bool { + resp, ok := mustAsk( + t, actorInstance.Ref(), &GetStateRequest{}, + ).(*GetStateResp) + require.True(t, ok) + + // Target anchor must be gone, planner must be back in + // materialization, and no FailReason should have leaked + // through. + for _, h := range resp.PlannerState.ConfirmedTxids { + if h == targetTxid { + return false + } + } + + return resp.Phase == PhaseMaterializing && + resp.PlannerState.TargetConfirmHeight.IsNone() && + resp.FailReason == "" + }, testTimeout, 10*time.Millisecond, + "reconciliation never cleared the stale target anchor") +} + +// TestReconcileSweepReorgedOfflineDowngradesSweep restores a +// checkpoint with a Confirmed sweep that is no longer on chain. After +// reconciliation the sweep state must reset so the next planner pass +// re-emits NeedSweep and the cached sweep tx is re-broadcast. +func TestReconcileSweepReorgedOfflineDowngradesSweep(t *testing.T) { + proof := buildLinearProof(t) + rootTxid := proof.RootTxids()[0] + targetTxid := proof.TargetOutpoint().Hash + sweepTxid := chainhash.Hash{0xaa} + + checkpoint := &actorCheckpoint{ + Version: checkpointVersion, + Height: 110, + Started: true, + Trigger: TriggerRestart, + State: unrollplan.State{ + ConfirmedTxids: []chainhash.Hash{ + rootTxid, targetTxid, + }, + TargetConfirmHeight: fn.Some[int32](102), + Sweep: unrollplan.SweepState{ + Status: unrollplan.SweepStatusConfirmed, + Txid: fn.Some(sweepTxid), + ConfirmHeight: fn.Some[int32](108), + }, + }, + } + + // Reconciler: every proof anchor still confirmed, but the sweep + // is no longer on chain. + reconciler := &stubChainReconciler{ + confirmed: map[chainhash.Hash]ConfirmedAnchor{ + rootTxid: { + Txid: rootTxid, + Height: 101, + }, + targetTxid: { + Txid: targetTxid, + Height: 102, + }, + }, + } + + beh, _, _ := restoreHarness(t, proof, checkpoint, reconciler) + + actorInstance := actor.NewActor(actor.ActorConfig[Msg, Resp]{ + ID: "reconcile-test", + Behavior: &txExecAdapter{b: beh, ax: newMemExecFor(beh)}, + MailboxSize: 16, + }) + beh.selfRef = actorInstance.TellRef() + actorInstance.Start() + t.Cleanup(actorInstance.Stop) + + mustAsk(t, actorInstance.Ref(), &ResumeUnrollRequest{Height: 110}) + + require.Eventually(t, func() bool { + resp, ok := mustAsk( + t, actorInstance.Ref(), &GetStateRequest{}, + ).(*GetStateResp) + require.True(t, ok) + + // The target anchor must survive a sweep-only reorg — + // CSV maturity is still valid even though the sweep + // itself needs to be re-broadcast. The planner should + // move the actor back into AwaitingSweepBroadcast (or + // have already requested a fresh sweep build). + sweep := resp.PlannerState.Sweep + + return resp.PlannerState.TargetConfirmHeight.IsSome() && + (sweep.Status == unrollplan.SweepStatusPending || + sweep.Status == + unrollplan.SweepStatusBroadcasted) && + sweep.ConfirmHeight.IsNone() + }, testTimeout, 10*time.Millisecond, + "reconciliation never downgraded the stale sweep anchor") +} + +// TestReconcileRefreshesTargetHeightAfterOfflineReMine restores a +// checkpoint that recorded the target tx confirmed at one height, but +// the reconciler reports the target as still confirmed at a DIFFERENT +// (higher) height — i.e. the target tx was re-mined into a different +// block while the daemon was offline. The reconciler must refresh +// TargetConfirmHeight to the live value so CSV maturity is computed +// off the correct on-chain anchor, not the stale checkpoint height. +func TestReconcileRefreshesTargetHeightAfterOfflineReMine(t *testing.T) { + proof := buildLinearProof(t) + rootTxid := proof.RootTxids()[0] + targetTxid := proof.TargetOutpoint().Hash + + const ( + staleHeight = int32(102) + liveHeight = int32(105) + ) + + checkpoint := &actorCheckpoint{ + Version: checkpointVersion, + Height: staleHeight, + Started: true, + Trigger: TriggerRestart, + State: unrollplan.State{ + ConfirmedTxids: []chainhash.Hash{ + rootTxid, targetTxid, + }, + TargetConfirmHeight: fn.Some(staleHeight), + }, + } + + reconciler := &stubChainReconciler{ + confirmed: map[chainhash.Hash]ConfirmedAnchor{ + rootTxid: { + Txid: rootTxid, + Height: 101, + }, + targetTxid: { + Txid: targetTxid, + Height: liveHeight, + }, + }, + } + + err := reconcileCheckpoint(t.Context(), reconciler, proof, checkpoint) + require.NoError(t, err) + + require.True(t, checkpoint.State.TargetConfirmHeight.IsSome()) + require.Equal( + t, liveHeight, + checkpoint.State.TargetConfirmHeight.UnsafeFromSome(), + "TargetConfirmHeight must refresh to the live anchor height", + ) + require.GreaterOrEqual( + t, checkpoint.Height, liveHeight, + "checkpoint best-height should not lag the live anchor", + ) +} + +// TestReconcileTransportErrorSurfacesUpward asserts that a transport +// failure from the reconciler propagates out of ensureLoaded rather +// than silently letting the actor proceed off an unverified +// checkpoint. +func TestReconcileTransportErrorSurfacesUpward(t *testing.T) { + proof := buildLinearProof(t) + rootTxid := proof.RootTxids()[0] + + checkpoint := &actorCheckpoint{ + Version: checkpointVersion, + Height: 101, + Started: true, + Trigger: TriggerRestart, + State: unrollplan.State{ + ConfirmedTxids: []chainhash.Hash{ + rootTxid, + }, + }, + } + + reconciler := &stubChainReconciler{ + err: errors.New("backend unreachable"), + } + + beh, _, _ := restoreHarness(t, proof, checkpoint, reconciler) + + err := beh.ensureLoaded(t.Context()) + require.Error(t, err) + require.Contains(t, err.Error(), "backend unreachable") +} + +// TestReconcileExternalSpendStillLiveRestoresParkedActor restores a +// checkpoint that already carries a ProvisionalExternalSpend anchor +// and verifies that the reconciler keeps it set when the chain still +// reports the target as spent — the actor must enter +// AwaitingExternalSpendFinality on the next Resume rather than +// resuming planner-driven materialization. +func TestReconcileExternalSpendStillLiveRestoresParkedActor(t *testing.T) { + proof := buildLinearProof(t) + rootTxid := proof.RootTxids()[0] + externalTxid := chainhash.Hash{0xee} + + checkpoint := &actorCheckpoint{ + Version: checkpointVersion, + Height: 155, + Started: true, + Trigger: TriggerRestart, + State: unrollplan.State{ + ConfirmedTxids: []chainhash.Hash{ + rootTxid, + }, + }, + ProvisionalExternalSpend: fn.Some(ExternalSpendAnchor{ + SpendingTxid: externalTxid, + SpendingHeight: 155, + }), + } + + reconciler := &stubChainReconciler{ + confirmed: map[chainhash.Hash]ConfirmedAnchor{ + rootTxid: { + Txid: rootTxid, + Height: 101, + }, + }, + spent: map[wire.OutPoint]SpendAnchor{ + proof.TargetOutpoint(): { + Outpoint: proof.TargetOutpoint(), + SpendingTxid: externalTxid, + SpendingHeight: 155, + }, + }, + } + + beh, _, _ := restoreHarness(t, proof, checkpoint, reconciler) + + actorInstance := actor.NewActor(actor.ActorConfig[Msg, Resp]{ + ID: "reconcile-test", + Behavior: &txExecAdapter{b: beh, ax: newMemExecFor(beh)}, + MailboxSize: 16, + }) + beh.selfRef = actorInstance.TellRef() + actorInstance.Start() + t.Cleanup(actorInstance.Stop) + + mustAsk(t, actorInstance.Ref(), &ResumeUnrollRequest{Height: 155}) + + require.Eventually(t, func() bool { + resp, ok := mustAsk( + t, actorInstance.Ref(), &GetStateRequest{}, + ).(*GetStateResp) + require.True(t, ok) + + return resp.Phase == PhaseExternalSpendObserved && + resp.FailReason == "" + }, testTimeout, 10*time.Millisecond, + "actor did not restore into PhaseExternalSpendObserved") +} + +// TestReconcileExternalSpendReorgedOfflineResumesActor restores a +// checkpoint carrying a ProvisionalExternalSpend anchor but the +// reconciler reports the target outpoint as currently unspent: the +// spending block was reorged out while the daemon was offline. +// Reconciliation must clear the anchor so the actor resumes +// materialization instead of staying parked. +func TestReconcileExternalSpendReorgedOfflineResumesActor(t *testing.T) { + proof := buildLinearProof(t) + rootTxid := proof.RootTxids()[0] + externalTxid := chainhash.Hash{0xee} + + checkpoint := &actorCheckpoint{ + Version: checkpointVersion, + Height: 155, + Started: true, + Trigger: TriggerRestart, + State: unrollplan.State{ + InFlightTxids: []chainhash.Hash{ + rootTxid, + }, + }, + ProvisionalExternalSpend: fn.Some(ExternalSpendAnchor{ + SpendingTxid: externalTxid, + SpendingHeight: 155, + }), + } + + // Reconciler reports the target outpoint as unspent. The + // proof root is still in-flight; no confirmed anchors to verify + // here. + reconciler := &stubChainReconciler{} + + beh, _, _ := restoreHarness(t, proof, checkpoint, reconciler) + + actorInstance := actor.NewActor(actor.ActorConfig[Msg, Resp]{ + ID: "reconcile-test", + Behavior: &txExecAdapter{b: beh, ax: newMemExecFor(beh)}, + MailboxSize: 16, + }) + beh.selfRef = actorInstance.TellRef() + actorInstance.Start() + t.Cleanup(actorInstance.Stop) + + mustAsk(t, actorInstance.Ref(), &ResumeUnrollRequest{Height: 155}) + + require.Eventually(t, func() bool { + resp, ok := mustAsk( + t, actorInstance.Ref(), &GetStateRequest{}, + ).(*GetStateResp) + require.True(t, ok) + + return resp.Phase == PhaseMaterializing && + resp.FailReason == "" + }, testTimeout, 10*time.Millisecond, + "actor did not resume materialization after offline "+ + "external-spend reorg") +} + +// TestReconcileExternalSpendObservedOfflineParksActor restores a +// checkpoint that does NOT carry a ProvisionalExternalSpend anchor, +// but the reconciler reports the target outpoint as currently spent +// by a txid that is neither in the proof graph nor our sweep. The +// reconciler must install the anchor so the actor enters +// AwaitingExternalSpendFinality on the next Resume. +func TestReconcileExternalSpendObservedOfflineParksActor(t *testing.T) { + proof := buildLinearProof(t) + rootTxid := proof.RootTxids()[0] + externalTxid := chainhash.Hash{0xee} + + checkpoint := &actorCheckpoint{ + Version: checkpointVersion, + Height: 155, + Started: true, + Trigger: TriggerRestart, + State: unrollplan.State{ + ConfirmedTxids: []chainhash.Hash{ + rootTxid, + }, + }, + } + + reconciler := &stubChainReconciler{ + confirmed: map[chainhash.Hash]ConfirmedAnchor{ + rootTxid: { + Txid: rootTxid, + Height: 101, + }, + }, + spent: map[wire.OutPoint]SpendAnchor{ + proof.TargetOutpoint(): { + Outpoint: proof.TargetOutpoint(), + SpendingTxid: externalTxid, + SpendingHeight: 155, + }, + }, + } + + beh, _, _ := restoreHarness(t, proof, checkpoint, reconciler) + + actorInstance := actor.NewActor(actor.ActorConfig[Msg, Resp]{ + ID: "reconcile-test", + Behavior: &txExecAdapter{b: beh, ax: newMemExecFor(beh)}, + MailboxSize: 16, + }) + beh.selfRef = actorInstance.TellRef() + actorInstance.Start() + t.Cleanup(actorInstance.Stop) + + mustAsk(t, actorInstance.Ref(), &ResumeUnrollRequest{Height: 155}) + + require.Eventually(t, func() bool { + resp, ok := mustAsk( + t, actorInstance.Ref(), &GetStateRequest{}, + ).(*GetStateResp) + require.True(t, ok) + + return resp.Phase == PhaseExternalSpendObserved && + resp.FailReason == "" + }, testTimeout, 10*time.Millisecond, + "actor did not park in PhaseExternalSpendObserved after "+ + "offline external spend") +} diff --git a/unroll/registry.go b/unroll/registry.go index be41ec765..13361beba 100644 --- a/unroll/registry.go +++ b/unroll/registry.go @@ -103,8 +103,8 @@ type RegistryStore interface { ListNonTerminalRecords(ctx context.Context) ([]RegistryRecord, error) // MarkTerminal persists one terminal target state. recoverable marks a - // no-footprint failure that boot-time reconciliation may roll back to - // live. + // no-footprint failure with objective canonical-absence evidence that + // boot-time reconciliation may roll back to live. MarkTerminal(ctx context.Context, target wire.OutPoint, phase Phase, recoverable bool, failReason string, sweepTxid *chainhash.Hash) error @@ -166,14 +166,25 @@ type RegistryConfig struct { FraudCheckpointSafetyMargin int32 // VTXOExitObserver, when set, receives an ExitOutcomeNotification each - // time a child unroll job reaches a terminal phase: a clean failure - // (no on-chain footprint) asks the VTXO manager to roll the VTXO back - // to live, and a completed exit asks it to retire the VTXO to spent. - // This is the feedback edge that keeps VTXO lifecycle gated on the - // unroll job's terminal on-chain outcome rather than the user's intent - // to exit (wavelength#602). When None, terminal outcomes are not - // forwarded (used by tests that don't exercise the manager). + // time a child unroll job reaches a terminal phase: a failure with no + // footprint and objective canonical-absence evidence asks the VTXO + // manager to roll the VTXO back to live, and a completed exit asks it + // to retire the VTXO to spent. This is the feedback edge that keeps + // VTXO lifecycle gated on the unroll job's terminal on-chain outcome + // rather than the user's intent to exit (wavelength#602). When None, + // terminal outcomes are not forwarded (used by tests that don't + // exercise the manager). VTXOExitObserver fn.Option[actor.TellOnlyRef[vtxo.ManagerMsg]] + + // ChainReconcilerFactory, when set, is forwarded to each spawned + // per-target actor so it can verify its checkpoint anchors + // against the canonical chain on restart. The factory is invoked + // once per actor lifetime after the proof loads. When None + // children skip reconciliation; production wiring should pass a + // factory backed by NewChainSourceReconciler so an offline + // reorg window does not silently leave actors driving side + // effects off stale planner state. + ChainReconcilerFactory fn.Option[ChainReconcilerFactory] } // UnrollRegistryActor wraps the thin unroll registry actor. @@ -236,6 +247,9 @@ func NewUnrollRegistryActor(cfg RegistryConfig) *UnrollRegistryActor { active: make(map[wire.OutPoint]*VTXOUnrollActor), pending: make(map[wire.OutPoint]RegistryRecord), persisting: make(map[wire.OutPoint]RegistryRecord), + terminalPolicyKinds: make( + map[wire.OutPoint]ExitPolicyKind, + ), } registry := actor.NewActor(actor.ActorConfig[RegistryMsg, RegistryResp]{ @@ -264,6 +278,11 @@ type registryBehavior struct { pending map[wire.OutPoint]RegistryRecord persisting map[wire.OutPoint]RegistryRecord + // terminalPolicyKinds retains the message-authoritative policy kind + // while a failed terminal store write is being retried. It must not be + // stamped onto a record without its matching durable ref. + terminalPolicyKinds map[wire.OutPoint]ExitPolicyKind + spawnFunc func(context.Context, wire.OutPoint) (*VTXOUnrollActor, error) } @@ -464,9 +483,9 @@ func (r *registryBehavior) handleEnsure(ctx context.Context, // the durable store. Re-spawning a fresh actor on top of an existing // record would clobber the recorded sweep txid or fail reason. // - // A recoverable failure is the exception: the prior exit failed - // cleanly with no on-chain footprint and the VTXO was rolled back to - // live (wavelength#602), so a fresh exit must be admittable rather + // A recoverable failure is the exception: objective canonical-absence + // evidence allowed the VTXO to roll back to live (wavelength#602), so a + // fresh exit must be admittable rather // than deduped against the dead attempt. Fall through to the spawn // path, whose UpsertRecord overwrites the stale recoverable row. if record, ok := r.pending[req.Outpoint]; ok && @@ -544,14 +563,14 @@ func (r *registryBehavior) handleEnsure(ctx context.Context, }) } - // Terminal record. A recoverable failure (clean, no on-chain - // footprint) means the VTXO was rolled back to live, so a new - // exit is allowed: fall through to the spawn path below, whose - // UpsertRecord overwrites the stale record. Any other terminal - // record (Completed, or a footprint-bearing Failed) is a real - // end state — dedup against it and return the historical - // ActorID so the recorded sweep txid / failure reason are - // never clobbered. + // Terminal record. A recoverable failure backed by objective + // canonical-absence evidence means the VTXO was rolled back to + // live, so a new exit is allowed: fall through to the spawn + // path below, whose UpsertRecord overwrites the stale record. + // Any other terminal record (Completed, or a footprint-bearing + // Failed) is a real end state — dedup against it and return the + // historical ActorID so the recorded sweep txid / failure + // reason are never clobbered. if !existing.RecoverableFailure { return fn.Ok[RegistryResp](&EnsureUnrollResp{ ActorID: existing.ActorID, @@ -625,11 +644,11 @@ func (r *registryBehavior) handleEnsure(ctx context.Context, // notifyRegistryIfTerminal so the registry record // cannot stay PhasePending forever. // - // 2. Real start error — proof assembly, store, planner. - // Hiding it under a Created=true would silently strand - // the user's funds in unilateral_exit with no - // progress. Mark the durable row PhaseFailed so - // GetUnrollStatus surfaces a terminal status. + // 2. Real start error — proof assembly, store, planner, or + // outbox failure after submission. The registry cannot + // distinguish those positions, so it preserves the durable + // hold and surfaces the error. A fresh Ensure or restart + // retries restoration without reliving value. if isCancellationRace(err) { r.watchChildAdmissionResult( context.WithoutCancel(ctx), req.Outpoint, child, @@ -648,7 +667,7 @@ func (r *registryBehavior) handleEnsure(ctx context.Context, }) } - r.failAdmittedChild( + r.parkAdmittedChild( ctx, req.Outpoint, child, fmt.Errorf("start child: %w", err), ) @@ -659,7 +678,7 @@ func (r *registryBehavior) handleEnsure(ctx context.Context, state, err := r.childState(startCtx, child) if err != nil { if !isCancellationRace(err) { - r.failAdmittedChild( + r.parkAdmittedChild( ctx, req.Outpoint, child, fmt.Errorf("read child state: %w", err), ) @@ -741,9 +760,8 @@ func (r *registryBehavior) watchChildAdmissionResult(ctx context.Context, }() } -// handleChildAdmissionResult converts a late child start failure into the -// same terminal registry state that a synchronous start failure would have -// produced. +// handleChildAdmissionResult parks a late child start failure in the same +// fail-closed non-terminal state as a synchronous start failure. func (r *registryBehavior) handleChildAdmissionResult(ctx context.Context, req *childAdmissionResultMsg) fn.Result[RegistryResp] { @@ -752,7 +770,7 @@ func (r *registryBehavior) handleChildAdmissionResult(ctx context.Context, return fn.Ok[RegistryResp](&RegistryAckResp{}) } - r.failAdmittedChild( + r.parkAdmittedChild( ctx, req.Outpoint, child, fmt.Errorf("start child: %s", req.Err), ) @@ -760,63 +778,60 @@ func (r *registryBehavior) handleChildAdmissionResult(ctx context.Context, return fn.Ok[RegistryResp](&RegistryAckResp{}) } -// failAdmittedChild records a terminal failure for a child that already has -// a durable pending row. This keeps GetUnrollStatus from falling back to -// "not found" after VTXO ownership has moved to unilateral exit. -// -// These failures (start / proof-assembly / restore errors) all occur before -// the child broadcasts anything, so the VTXO has no on-chain footprint and is -// recoverable to live. The record is persisted as a recoverable failure and -// the VTXO manager is notified so the VTXO is rolled back rather than -// stranded in unilateral exit (wavelength#602). -func (r *registryBehavior) failAdmittedChild(ctx context.Context, +// parkAdmittedChild stops a child whose admission result is ambiguous while +// preserving its durable non-terminal hold. A child stages state before +// routing outbox work, and an external txconfirm submission can succeed before +// a later actor/store error reaches the registry. Therefore a start or status +// error is not proof of a clean no-footprint failure and must never relive the +// VTXO. A fresh Ensure or restart restores the pending record and retries. +func (r *registryBehavior) parkAdmittedChild(ctx context.Context, target wire.OutPoint, child *VTXOUnrollActor, err error) { child.Stop() delete(r.active, target) record := RegistryRecord{ - TargetOutpoint: target, - ActorID: child.Ref().ID(), - Phase: PhaseFailed, - FailReason: err.Error(), - RecoverableFailure: true, + TargetOutpoint: target, + ActorID: child.Ref().ID(), + Phase: PhasePending, + FailReason: err.Error(), } if pending, ok := r.pending[target]; ok { record = cloneRegistryRecord(pending) - record.Phase = PhaseFailed + if record.IsTerminal() { + + // A terminal child handoff won the mailbox race. + // Preserve its objective result; the late admission + // error carries no higher priority than that durable + // terminal state. + return + } record.FailReason = err.Error() - record.RecoverableFailure = true + record.RecoverableFailure = false } r.pending[target] = cloneRegistryRecord(record) - // Roll the VTXO back to live. The terminal record below is the durable - // backstop if this best-effort notification is lost. A recovery-only - // target is held in exit instead (see notifyVTXOExit / the manager), - // so its policy rides along. - r.notifyVTXOExit(context.WithoutCancel(ctx), &UnrollTerminatedMsg{ - Outpoint: target, - ActorID: record.ActorID, - Phase: PhaseFailed, - FailReason: err.Error(), - HadOnChainFootprint: false, - }, record.ExitPolicyKind) - - markErr := r.cfg.Store.MarkTerminal( - context.WithoutCancel(ctx), target, PhaseFailed, true, - err.Error(), nil, + markErr := r.cfg.Store.UpsertRecord( + context.WithoutCancel(ctx), cloneRegistryRecord(record), ) if markErr != nil { - r.log.WarnS(ctx, "Failed to mark admitted unroll child "+ - "terminal", markErr, + r.log.WarnS(ctx, "Failed to preserve parked unroll child", + markErr, slog.String("outpoint", target.String()), slog.String("actor_id", child.Ref().ID()), ) // Registry persistence retries are actor-owned follow-up work. //nolint:contextcheck r.requestPersist(target, 0) + + return } + + // The durable non-terminal row now owns retry. Dropping the cache lets + // a fresh Ensure reach the store-backed inline-restore branch instead + // of deduping forever against a child that was deliberately stopped. + delete(r.pending, target) } // isCancellationRace reports whether admission should preserve the pending @@ -836,8 +851,8 @@ func (r *registryBehavior) failAdmittedChild(ctx context.Context, // baselib/actor's durable Ask short-circuits and returns that error // BEFORE calling mailbox.Send when the actor's own ctx is already done, // so there is no persisted message to trust. Letting it fall through to -// failAdmittedChild surfaces a deterministic terminal record instead of -// pretending the job is in flight. +// parkAdmittedChild preserves the fail-closed durable hold instead of treating +// the process error as objective no-footprint evidence. func isCancellationRace(err error) bool { return errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) @@ -954,17 +969,19 @@ func (r *registryBehavior) handleGetStatus(ctx context.Context, // stopped only after a queued state probe has drained through it. That // keeps the registry from synchronously cancelling the child while the // child is still acking the terminal durable message that notified us. -// The snapshot is then enqueued for async persistence via requestPersist. -// Terminal writes intentionally stay on the async retry path — unlike -// admission, a failed terminal write does not orphan the job (the -// in-memory pending record keeps answering GetStatus), so there is no -// reason to block the registry goroutine on a flaky store. +// The terminal row is written synchronously before any VTXO outcome is +// forwarded. If that write fails, the in-memory record enters the existing +// async retry path and the outcome stays withheld until a retry succeeds. This +// durability barrier is load-bearing for recoverable failures: business state +// must never become live before its terminal invalidation is durable. func (r *registryBehavior) handleTerminated(ctx context.Context, req *UnrollTerminatedMsg) fn.Result[RegistryResp] { - // A terminal failure with no on-chain footprint is recoverable: the - // VTXO never left off-chain custody, so it can be rolled back to live. - recoverable := req.Phase == PhaseFailed && !req.HadOnChainFootprint + // A terminal failure is recoverable only when this actor recorded no + // footprint and objective canonical-absence evidence cleared the + // chain-boundary uncertainty guard. + recoverable := req.Phase == PhaseFailed && + !req.HadOnChainFootprint && !req.ReliveUnsafe record := RegistryRecord{ TargetOutpoint: req.Outpoint, @@ -998,9 +1015,6 @@ func (r *registryBehavior) handleTerminated(ctx context.Context, } r.pending[req.Outpoint] = cloneRegistryRecord(record) - // Registry persistence retries are actor-owned follow-up work. - //nolint:contextcheck - r.requestPersist(req.Outpoint, 0) // The child's terminal message carries its durable exit policy, which // outlives r.pending: a completed async persist can evict the cached @@ -1013,6 +1027,38 @@ func (r *registryBehavior) handleTerminated(ctx context.Context, if policyKind == "" { policyKind = record.ExitPolicyKind } + if r.terminalPolicyKinds == nil { + r.terminalPolicyKinds = make( + map[wire.OutPoint]ExitPolicyKind, + ) + } + r.terminalPolicyKinds[req.Outpoint] = policyKind + + // Persist terminal invalidation before forwarding any outcome that can + // restore business state. A crash after reliving the VTXO but before + // the terminal row landed would let restart restore the old + // non-terminal job and reissue on-chain work against a live coin. + persistCtx := context.WithoutCancel(ctx) + err := r.cfg.Store.MarkTerminal( + persistCtx, req.Outpoint, req.Phase, recoverable, + req.FailReason, req.SweepTxid, + ) + if err != nil { + r.log.WarnS( + ctx, + "Failed to persist terminal unroll state; holding VTXO", + err, + slog.String("outpoint", req.Outpoint.String()), + slog.String("phase", string(req.Phase)), + ) + // Registry persistence retries are actor-owned follow-up work. + // The VTXO outcome stays withheld until + // handlePersistRecordResult sees a successful durable write. + //nolint:contextcheck + r.requestPersist(req.Outpoint, 0) + + return fn.Ok[RegistryResp](&RegistryAckResp{}) + } // Forward the terminal outcome to the VTXO manager so the VTXO's // lifecycle tracks the unroll job's terminal on-chain result rather @@ -1021,17 +1067,18 @@ func (r *registryBehavior) handleTerminated(ctx context.Context, // policy rides along so the manager can hold a recovery-only target in // exit rather than relive it as a live coin. r.notifyVTXOExit(context.WithoutCancel(ctx), req, policyKind) + delete(r.pending, req.Outpoint) + delete(r.terminalPolicyKinds, req.Outpoint) return fn.Ok[RegistryResp](&RegistryAckResp{}) } // notifyVTXOExit forwards a child's terminal outcome to the VTXO manager. // -// - PhaseFailed with no on-chain footprint: the unroll never broadcast, -// so the VTXO is still live from the operator's perspective. Ask the -// manager to roll it back to live (ExitOutcomeRecoverable). -// - PhaseCompleted: the exit was swept and confirmed on-chain, so ask the -// manager to retire the VTXO to spent (ExitOutcomeConfirmed). +// - PhaseFailed with no footprint and objective canonical-absence evidence: +// roll the VTXO back live (ExitOutcomeRecoverable). +// - PhaseCompleted: the target was durably consumed by the final sweep or a +// finalized external spend, so retire it (ExitOutcomeConfirmed). // - PhaseFailed with an on-chain footprint: the exit has begun on-chain; // leave the VTXO in unilateral-exit (no notification). // @@ -1054,7 +1101,9 @@ func (r *registryBehavior) notifyVTXOExit(ctx context.Context, case req.Phase == PhaseCompleted: outcome = vtxo.ExitOutcomeConfirmed - case req.Phase == PhaseFailed && !req.HadOnChainFootprint: + case req.Phase == PhaseFailed && !req.HadOnChainFootprint && + !req.ReliveUnsafe: + outcome = vtxo.ExitOutcomeRecoverable default: @@ -1248,13 +1297,43 @@ func (r *registryBehavior) tryRestoreOne(ctx context.Context, return nil, fmt.Errorf("spawn failed on restore: %w", err) } - _, err = child.Ref().Ask(ctx, &ResumeUnrollRequest{ - Height: height, - }).Await(ctx).Unpack() + stateRaw, err := child.Ref().Ask( + ctx, &GetStateRequest{}, + ).Await(ctx).Unpack() + if err != nil { + child.Stop() + + return nil, fmt.Errorf("read checkpoint state on restore: %w", + err) + } + state, ok := stateRaw.(*GetStateResp) + if !ok { + child.Stop() + + return nil, fmt.Errorf("read checkpoint state on restore: "+ + "unexpected response %T", stateRaw) + } + + var restoreReq Msg = &ResumeUnrollRequest{Height: height} + if !state.Started { + // Admission may have failed before the first checkpoint Stage. + // Recreate the original start identity from the durable + // registry row instead of inventing a + // TriggerRestart/standard-policy job. + restoreReq = &StartUnrollRequest{ + Height: height, + Trigger: record.Trigger, + ExitPolicyKind: record.ExitPolicyKind, + ExitPolicyRef: record.ExitPolicyRef, + } + } + + _, err = child.Ref().Ask(ctx, restoreReq).Await(ctx).Unpack() if err != nil { child.Stop() - return nil, fmt.Errorf("resume failed on restore: %w", err) + return nil, fmt.Errorf("start/resume failed on restore: %w", + err) } return child, nil @@ -1341,6 +1420,32 @@ func (r *registryBehavior) handlePersistRecordResult(ctx context.Context, if req.Err == "" { record, ok := r.pending[req.Outpoint] if ok && sameRegistryRecord(record, req.Record) { + if record.IsTerminal() { + policyKind := record.ExitPolicyKind + outpoint := req.Outpoint + policyKinds := r.terminalPolicyKinds + pendingKind, exists := policyKinds[outpoint] + if exists { + policyKind = pendingKind + } + r.notifyVTXOExit( + context.WithoutCancel(ctx), + &UnrollTerminatedMsg{ + Outpoint: req.Outpoint, + ActorID: record.ActorID, + Phase: record.Phase, + FailReason: record.FailReason, + SweepTxid: copyHash( + record.SweepTxid, + ), + HadOnChainFootprint: !record. + RecoverableFailure, + ExitPolicyKind: policyKind, + }, + policyKind, + ) + delete(r.terminalPolicyKinds, req.Outpoint) + } delete(r.pending, req.Outpoint) } else if ok { // Registry persistence retries are actor-owned work. @@ -1489,6 +1594,7 @@ func (r *registryBehavior) childConfig(target wire.OutPoint) Config { ExitSpendPolicyResolver: r.cfg.ExitSpendPolicyResolver, FraudCheckpointSafetyMargin: r.cfg.FraudCheckpointSafetyMargin, RegistryRef: r.selfRef, + ChainReconcilerFactory: r.cfg.ChainReconcilerFactory, } } diff --git a/unroll/registry_exit_test.go b/unroll/registry_exit_test.go index 852c07d9a..bab2cc77d 100644 --- a/unroll/registry_exit_test.go +++ b/unroll/registry_exit_test.go @@ -12,6 +12,7 @@ import ( "github.com/btcsuite/btcd/wire/v2" "github.com/lightninglabs/wavelength/baselib/actor" "github.com/lightninglabs/wavelength/lib/actormsg" + "github.com/lightninglabs/wavelength/txconfirm" "github.com/lightninglabs/wavelength/vtxo" fn "github.com/lightningnetwork/lnd/fn/v2" "github.com/stretchr/testify/require" @@ -88,10 +89,15 @@ func newExitObserverRegistry(target wire.OutPoint) (*registryBehavior, return behavior, observer } -// TestRegistryForwardsCleanFailureAsRecoverable verifies a terminal failure -// with no on-chain footprint is forwarded to the VTXO manager as a -// recoverable exit so the VTXO is rolled back to live (wavelength#602). -func TestRegistryForwardsCleanFailureAsRecoverable(t *testing.T) { +// TestRegistryForwardsAuthoritativelyCleanFailureAsRecoverable verifies a +// terminal failure whose canonical-absence reconciliation cleared +// ReliveUnsafe is forwarded to the VTXO manager as a recoverable exit. The +// live actor keeps ReliveUnsafe set across every chain-boundary attempt; this +// message shape is reserved for objective negative evidence supplied by a +// stronger reconciler. +func TestRegistryForwardsAuthoritativelyCleanFailureAsRecoverable( + t *testing.T) { + target := wire.OutPoint{Hash: chainhash.Hash{1}, Index: 0} behavior, observer := newExitObserverRegistry(target) @@ -111,6 +117,25 @@ func TestRegistryForwardsCleanFailureAsRecoverable(t *testing.T) { require.Equal(t, "min relay fee not met", notes[0].Reason) } +// TestRegistryDoesNotReliveRestartUncertainty verifies a clean-looking failure +// after restart remains fail-closed when canonical absence was not proven. +func TestRegistryDoesNotReliveRestartUncertainty(t *testing.T) { + target := wire.OutPoint{Hash: chainhash.Hash{9}, Index: 0} + behavior, observer := newExitObserverRegistry(target) + + _, err := behavior.handleTerminated(t.Context(), &UnrollTerminatedMsg{ + Outpoint: target, + ActorID: actorIDForTarget(target), + Phase: PhaseFailed, + FailReason: "reissue rejected", + HadOnChainFootprint: false, + ReliveUnsafe: true, + }).Unpack() + require.NoError(t, err) + require.Empty(t, observer.notifications()) + require.False(t, behavior.pending[target].RecoverableFailure) +} + // TestRegistryForwardsExitPolicyFromTerminalMsg verifies the exit policy the // VTXO manager sees on a recoverable failure comes from the child's terminal // message, not the registry's in-memory pending cache. The pending record is @@ -158,8 +183,9 @@ func TestRegistryForwardsExitPolicyFromTerminalMsg(t *testing.T) { // stamped kind would overwrite the store's (kind, ref) identity with // (kind, ""). The no-cache record therefore leaves the policy empty, // letting registryExitPolicy preserve the durable admission identity. - persisted, ok := behavior.pending[target] - require.True(t, ok) + persisted, err := behavior.cfg.Store.GetRecord(t.Context(), target) + require.NoError(t, err) + require.NotNil(t, persisted) require.Empty( t, persisted.ExitPolicyKind, "the terminal record must not "+ "carry a ref-less kind that clobbers the store "+ @@ -209,29 +235,48 @@ func TestRegistryForwardsCompletionAsConfirmed(t *testing.T) { require.Equal(t, vtxo.ExitOutcomeConfirmed, notes[0].Outcome) } -// TestRegistryRecoversCleanFailureEndToEnd is the in-process integration test -// for the wavelength#602 recovery path. Unlike the unit tests above (which -// call handleTerminated directly with a synthetic UnrollTerminatedMsg), this -// drives a REAL child unroll actor through admission, proof submission, and a -// txconfirm broadcast rejection to terminal Failed, then asserts the registry -// computes HadOnChainFootprint=false from the real FSM/planner state and -// forwards ExitOutcomeRecoverable to the VTXO manager observer. +// TestRegistryDoesNotReliveBroadcastFailureEndToEnd is the in-process +// integration test for a failure racing a chain-boundary side effect. Unlike +// the unit tests above (which call handleTerminated directly with a synthetic +// UnrollTerminatedMsg), this drives a real child through admission, proof +// submission, and a txconfirm rejection to terminal Failed. // -// This is the seam the unit tests cannot cover: the bug was an emergent -// lifecycle gap across the child actor → registry → observer chain, and the -// footprint determination is the change's highest-risk assumption. Here it -// runs against the real machinery rather than a hand-built message. -func TestRegistryRecoversCleanFailureEndToEnd(t *testing.T) { +// A rejection is not objective evidence that the transaction was never +// accepted or confirmed: a historical confirmation callback may already be +// queued behind it. The real actor must therefore keep ReliveUnsafe set, and +// the registry must withhold a recovery notification even though its local +// planner recorded no confirmed or in-flight transaction. +func TestRegistryDoesNotReliveBroadcastFailureEndToEnd(t *testing.T) { proof := buildLinearProof(t) desc := testDescriptor(t, proof.TargetOutpoint(), proof.CSVDelay()) + store := newMemRegistryStore() observer := &captureExitObserver{} var observerRef actor.TellOnlyRef[vtxo.ManagerMsg] = observer + checkpoints := newMemCheckpointStore() + guardPersistedBeforeSubmit := make(chan bool, 1) txconfirmRef := &fakeTxConfirmRef{} + txconfirmRef.onAsk = func(_ *txconfirm.EnsureConfirmedReq) { + checkpoint, loadErr := checkpoints.LoadCheckpoint( + context.Background(), + actorIDForTarget( + proof.TargetOutpoint(), + ), + ) + if loadErr != nil || checkpoint == nil { + guardPersistedBeforeSubmit <- false + + return + } + + decoded, decodeErr := decodeCheckpoint(checkpoint.StateData) + guardPersistedBeforeSubmit <- decodeErr == nil && + decoded.ReliveUnsafe + } cfg := RegistryConfig{ - Store: newMemRegistryStore(), - DeliveryStore: newMemCheckpointStore(), + Store: store, + DeliveryStore: checkpoints, ProofAssembler: &mockProofAssembler{ proof: proof, }, @@ -249,10 +294,9 @@ func TestRegistryRecoversCleanFailureEndToEnd(t *testing.T) { registry := newRegistryHarnessWithSpawn(t, cfg) t.Cleanup(registry.Stop) - // Reject the first proof tx the child submits (the #602 trigger: e.g. a - // sub-dust proof tx that can't meet min relay fee). The child drives a - // terminal TxFailedEvent with nothing confirmed and nothing left - // in-flight, so the job is a clean failure with no on-chain footprint. + // Reject the first proof tx the child submits (for example, a sub-dust + // proof tx that cannot meet the relay fee). This proves only that this + // submission failed; it does not prove canonical absence. txconfirmRef.setImmediateFailed(proof.RootTxids()[0], "min relay fee") _, err := registry.Ref().Ask(t.Context(), &EnsureUnrollRequest{ @@ -260,33 +304,44 @@ func TestRegistryRecoversCleanFailureEndToEnd(t *testing.T) { Trigger: TriggerManual, }).Await(t.Context()).Unpack() require.NoError(t, err) + require.True( + t, <-guardPersistedBeforeSubmit, "fresh admission must "+ + "persist the fail-closed relive guard before "+ + "txconfirm submission", + ) require.Eventually(t, func() bool { - return len(observer.notifications()) == 1 + record, err := store.GetRecord( + t.Context(), proof.TargetOutpoint(), + ) + require.NoError(t, err) + + return record != nil && record.Phase == PhaseFailed }, testTimeout, 10*time.Millisecond, - "registry should forward a recovery notification") + "registry should durably record the terminal failure") - notes := observer.notifications() - require.Equal(t, proof.TargetOutpoint(), notes[0].Outpoint) - require.Equal( - t, vtxo.ExitOutcomeRecoverable, notes[0].Outcome, - "a clean no-broadcast failure must be reported as recoverable", + record, err := store.GetRecord(t.Context(), proof.TargetOutpoint()) + require.NoError(t, err) + require.False(t, record.RecoverableFailure) + require.Empty( + t, observer.notifications(), + "a broadcast rejection without canonical-absence evidence "+ + "must not relive the target", ) } -// TestRegistryFailedAdmissionNotifiesRecoverable verifies that a child whose -// start fails before any broadcast (failAdmittedChild) both persists a -// recoverable terminal record and notifies the VTXO manager to roll the VTXO -// back to live. These pre-broadcast failures move ownership to unilateral -// exit but leave no on-chain footprint, so they must be recoverable -// (wavelength#602). -func TestRegistryFailedAdmissionNotifiesRecoverable(t *testing.T) { +// TestRegistryFailedAdmissionDoesNotRelive verifies that a child start error +// is not accepted as objective no-footprint evidence. Outbox work may have +// reached txconfirm before the error surfaced, so the registry must retain a +// non-terminal hold and leave the VTXO in unilateral exit. +func TestRegistryFailedAdmissionDoesNotRelive(t *testing.T) { proof := buildLinearProof(t) desc := testDescriptor(t, proof.TargetOutpoint(), proof.CSVDelay()) store := newMemRegistryStore() observer := &captureExitObserver{} var observerRef actor.TellOnlyRef[vtxo.ManagerMsg] = observer + submissionCrossed := make(chan struct{}, 1) registry := newRegistryHarnessWithSpawn(t, RegistryConfig{ Store: store, @@ -300,13 +355,25 @@ func TestRegistryFailedAdmissionNotifiesRecoverable(t *testing.T) { }) t.Cleanup(registry.Stop) + var spawnCount int registry.behavior.spawnFunc = func(_ context.Context, target wire.OutPoint) (*VTXOUnrollActor, error) { + spawnCount++ + firstSpawn := spawnCount == 1 behavior := actor.NewFunctionBehavior( func(_ context.Context, msg Msg) fn.Result[Resp] { switch msg.(type) { case *StartUnrollRequest: + if !firstSpawn { + return fn.Ok[Resp](&AckResp{}) + } + + // Model an outbox submission succeeding + // before a later stage/commit error + // reaches the registry. + submissionCrossed <- struct{}{} + return fn.Err[Resp]( errors.New("start boom"), ) @@ -336,24 +403,36 @@ func TestRegistryFailedAdmissionNotifiesRecoverable(t *testing.T) { Trigger: TriggerManual, }).Await(t.Context()).Unpack() require.Error(t, err) + select { + case <-submissionCrossed: + case <-time.After(testTimeout): + t.Fatal("start error arrived before modeled submission") + } - // The pre-broadcast failure is reported to the manager as recoverable. - require.Eventually(t, func() bool { - return len(observer.notifications()) == 1 - }, testTimeout, 10*time.Millisecond, - "failed admission should notify the manager") - - notes := observer.notifications() - require.Equal(t, proof.TargetOutpoint(), notes[0].Outpoint) - require.Equal(t, vtxo.ExitOutcomeRecoverable, notes[0].Outcome) + require.Empty(t, observer.notifications()) - // And the durable record is the recoverable terminal variant, so boot - // reconciliation can recover it even if the notification is lost. + // The durable record stays non-terminal so a fresh Ensure or restart + // can restore it without making the ambiguous failure spendable. record, err := store.GetRecord(t.Context(), proof.TargetOutpoint()) require.NoError(t, err) require.NotNil(t, record) - require.Equal(t, PhaseFailed, record.Phase) - require.True(t, record.RecoverableFailure) + require.Equal(t, PhasePending, record.Phase) + require.False(t, record.RecoverableFailure) + require.Contains(t, record.FailReason, "start boom") + + // Once the transient error clears, a fresh caller restores the durable + // hold. It is not a new admission because ownership never returned + // live. + resp, err := registry.Ref().Ask(t.Context(), &EnsureUnrollRequest{ + Outpoint: proof.TargetOutpoint(), + Trigger: TriggerManual, + }).Await(t.Context()).Unpack() + require.NoError(t, err) + ensureResp, ok := resp.(*EnsureUnrollResp) + require.True(t, ok) + require.False(t, ensureResp.Created) + require.Equal(t, 2, spawnCount) + require.Empty(t, observer.notifications()) } // TestRegistryReadmitsTargetAfterRecoverableFailure verifies that once a VTXO @@ -363,92 +442,34 @@ func TestRegistryFailedAdmissionNotifiesRecoverable(t *testing.T) { // trade a guaranteed strand for a deferred one: the VTXO would recover to live // but could never be unrolled again, because the terminal record blocks every // subsequent admission. This exercises the in-memory pending-cache arm of the -// dedup gate (the record lingers in r.pending after failAdmittedChild). +// dedup gate after a real child terminal handoff has classified the failure. func TestRegistryReadmitsTargetAfterRecoverableFailure(t *testing.T) { proof := buildLinearProof(t) desc := testDescriptor(t, proof.TargetOutpoint(), proof.CSVDelay()) target := proof.TargetOutpoint() store := newMemRegistryStore() - observer := &captureExitObserver{} - var observerRef actor.TellOnlyRef[vtxo.ManagerMsg] = observer - registry := newRegistryHarnessWithSpawn(t, RegistryConfig{ - Store: store, - DeliveryStore: newMemCheckpointStore(), - ProofAssembler: &mockProofAssembler{proof: proof}, - VTXOStore: &mockVTXOStore{desc: desc}, - TxConfirmRef: &fakeTxConfirmRef{}, - ChainSource: &fakeRegistryChainSourceRef{height: 200}, - Wallet: &fakeSweepWallet{}, - VTXOExitObserver: fn.Some(observerRef), + Store: store, + DeliveryStore: newMemCheckpointStore(), + ProofAssembler: &mockProofAssembler{proof: proof}, + VTXOStore: &mockVTXOStore{desc: desc}, + TxConfirmRef: &fakeTxConfirmRef{}, + ChainSource: &fakeRegistryChainSourceRef{height: 200}, + Wallet: &fakeSweepWallet{}, }) t.Cleanup(registry.Stop) - // The first spawn fails its start (pre-broadcast admission failure, - // the #602 trigger); the second spawn is a healthy child. A counter - // distinguishes the two admissions deterministically. - var spawnCount int - registry.behavior.spawnFunc = func(_ context.Context, - spawnTarget wire.OutPoint) (*VTXOUnrollActor, error) { - - spawnCount++ - firstSpawn := spawnCount == 1 - childBehavior := actor.NewFunctionBehavior( - func(_ context.Context, msg Msg) fn.Result[Resp] { - switch msg.(type) { - case *StartUnrollRequest: - if firstSpawn { - return fn.Err[Resp]( - errors.New( - "start boom", - ), - ) - } - - return fn.Ok[Resp](&AckResp{}) - - case *GetStateRequest: - return fn.Ok[Resp](&GetStateResp{ - Started: !firstSpawn, - Phase: PhasePending, - }) - - default: - return fn.Err[Resp]( - fmt.Errorf("unexpected msg %T", - msg), - ) - } - }, - ) - - // Test children are owned by t.Cleanup after creation. - //nolint:contextcheck - return newTestUnrollChild(t, spawnTarget, childBehavior), nil + seeded := RegistryRecord{ + TargetOutpoint: target, + ActorID: "prior-clean-failure", + Trigger: TriggerManual, + Phase: PhaseFailed, + FailReason: "proof tx rejected before broadcast", + RecoverableFailure: true, } - - // First admission fails pre-broadcast: failAdmittedChild records a - // recoverable terminal row, leaves it in r.pending, and notifies the - // manager to roll the VTXO back to live. - _, err := registry.Ref().Ask(t.Context(), &EnsureUnrollRequest{ - Outpoint: target, - Trigger: TriggerManual, - }).Await(t.Context()).Unpack() - require.Error(t, err) - - require.Eventually(t, func() bool { - return len(observer.notifications()) == 1 - }, testTimeout, 10*time.Millisecond, - "failed admission should roll the VTXO back to live") - - pre, err := store.GetRecord(t.Context(), target) - require.NoError(t, err) - require.NotNil(t, pre) - require.True( - t, pre.RecoverableFailure, - "the failed admission must leave a recoverable terminal record", - ) + require.NoError(t, store.UpsertRecord(t.Context(), seeded)) + registry.behavior.pending[target] = cloneRegistryRecord(seeded) // Second admission for the SAME outpoint must re-admit (Created=true) // rather than dedup against the recoverable record. @@ -465,9 +486,6 @@ func TestRegistryReadmitsTargetAfterRecoverableFailure(t *testing.T) { "re-admittable, not deduped against the dead "+ "recoverable record", ) - require.Equal( - t, 2, spawnCount, "the second Ensure must spawn a fresh child", - ) // The stale recoverable record is overwritten by the fresh admission, // so the VTXO can progress through a new exit. @@ -478,7 +496,7 @@ func TestRegistryReadmitsTargetAfterRecoverableFailure(t *testing.T) { t, post.RecoverableFailure, "re-admission must overwrite the stale recoverable record", ) - require.Equal(t, PhasePending, post.Phase) + require.Equal(t, PhaseMaterializing, post.Phase) } // TestRegistryReadmitsTargetAfterRecoverableFailureAcrossRestart verifies the diff --git a/unroll/registry_messages.go b/unroll/registry_messages.go index e6dc0a538..089c4d906 100644 --- a/unroll/registry_messages.go +++ b/unroll/registry_messages.go @@ -155,13 +155,18 @@ type UnrollTerminatedMsg struct { SweepTxid *chainhash.Hash // HadOnChainFootprint reports whether the job ever published anything - // on-chain (a confirmed or in-flight proof node, or a broadcast - // sweep). It is false only for a clean failure that never broadcast, - // which is the sole case where the target VTXO is safe to roll back - // to live (the operator still considers it live). See - // wavelength#602. + // on-chain (a confirmed or in-flight proof node, or a broadcast sweep). + // False means only that the local planner recorded no footprint; + // ReliveUnsafe must also be false before the target VTXO may return to + // live. See wavelength#602. HadOnChainFootprint bool + // ReliveUnsafe reports that chain-boundary uncertainty prevents a + // terminal failure from proving the VTXO remained live, even if this + // process did not record its own on-chain footprint. It remains set + // until objective canonical-absence evidence clears it. + ReliveUnsafe bool + // ExitPolicyKind is the child's durable exit policy for this target. // The child sources it from its own persisted state, so it stays // authoritative even after the registry has evicted its in-memory diff --git a/unroll/registry_test.go b/unroll/registry_test.go index a0b61b499..32ec18cd8 100644 --- a/unroll/registry_test.go +++ b/unroll/registry_test.go @@ -179,6 +179,26 @@ func (s *terminalFlakyRegistryStore) UpsertRecord(ctx context.Context, return s.memRegistryStore.UpsertRecord(ctx, record) } +// MarkTerminal fails through the same counter as terminal UpsertRecord so the +// synchronous durability barrier falls back to the retry path under test. +func (s *terminalFlakyRegistryStore) MarkTerminal(ctx context.Context, + target wire.OutPoint, phase Phase, recoverable bool, failReason string, + sweepTxid *chainhash.Hash) error { + + s.mu.Lock() + if s.upsertErrors > 0 { + s.upsertErrors-- + s.mu.Unlock() + + return errors.New("injected terminal upsert failure") + } + s.mu.Unlock() + + return s.memRegistryStore.MarkTerminal( + ctx, target, phase, recoverable, failReason, sweepTxid, + ) +} + // alwaysFailUpsertRegistryStore rejects every terminal-phase upsert while // keeping the non-terminal initial write and all read paths backed by the // in-memory store. This matches the fail-closed admission contract that the @@ -210,6 +230,13 @@ func (s *alwaysFailUpsertRegistryStore) UpsertRecord(ctx context.Context, return errors.New("injected upsert failure") } +// MarkTerminal rejects the synchronous terminal durability barrier too. +func (s *alwaysFailUpsertRegistryStore) MarkTerminal(context.Context, + wire.OutPoint, Phase, bool, string, *chainhash.Hash) error { + + return errors.New("injected terminal failure") +} + // blockingRegistryStore holds terminal-phase upserts until released so tests // can verify that registry status remains available while persistence is // stalled. Non-terminal writes (including the fail-closed admission write @@ -255,6 +282,14 @@ func (s *blockingRegistryStore) UpsertRecord(ctx context.Context, return s.memRegistryStore.UpsertRecord(ctx, record) } +// MarkTerminal pushes this test store onto the asynchronous fallback so the +// blocking UpsertRecord can be observed without blocking the registry actor. +func (s *blockingRegistryStore) MarkTerminal(context.Context, wire.OutPoint, + Phase, bool, string, *chainhash.Hash) error { + + return errors.New("use asynchronous terminal persistence") +} + // cancelOnPendingRegistryStore cancels the caller context after the initial // pending admission row has been persisted. type cancelOnPendingRegistryStore struct { @@ -1158,9 +1193,10 @@ func TestRegistryEnsureStartsChildAfterCallerCancellation(t *testing.T) { require.NoError(t, <-startCtxErr) } -// TestRegistryEnsureMarksRealStartErrorFailed verifies that the pending-row -// safeguard does not hide non-cancellation child start failures. -func TestRegistryEnsureMarksRealStartErrorFailed(t *testing.T) { +// TestRegistryEnsureParksRealStartError verifies that a child start error +// remains retryable and fail-closed instead of being misclassified as proof +// that the VTXO is safe to relive. +func TestRegistryEnsureParksRealStartError(t *testing.T) { proof := buildLinearProof(t) desc := testDescriptor(t, proof.TargetOutpoint(), proof.CSVDelay()) store := newMemRegistryStore() @@ -1218,8 +1254,9 @@ func TestRegistryEnsureMarksRealStartErrorFailed(t *testing.T) { record, err := store.GetRecord(t.Context(), proof.TargetOutpoint()) require.NoError(t, err) require.NotNil(t, record) - require.Equal(t, PhaseFailed, record.Phase) + require.Equal(t, PhasePending, record.Phase) require.Contains(t, record.FailReason, "start boom") + require.False(t, record.RecoverableFailure) } // TestRegistryTerminalPersistRetriesUntilDurable verifies that a @@ -1356,15 +1393,18 @@ func TestRegistryTerminalStatusRemainsQueryableWhilePersistBlocked( store := newBlockingRegistryStore() checkpoints := newMemCheckpointStore() txconfirmRef := &fakeTxConfirmRef{} + observer := &captureExitObserver{} + var observerRef actor.TellOnlyRef[vtxo.ManagerMsg] = observer registry := newRegistryHarnessWithSpawn(t, RegistryConfig{ - Store: store, - DeliveryStore: checkpoints, - ProofAssembler: &mockProofAssembler{proof: proof}, - VTXOStore: &mockVTXOStore{desc: desc}, - TxConfirmRef: txconfirmRef, - ChainSource: &fakeRegistryChainSourceRef{height: 200}, - Wallet: &fakeSweepWallet{}, + Store: store, + DeliveryStore: checkpoints, + ProofAssembler: &mockProofAssembler{proof: proof}, + VTXOStore: &mockVTXOStore{desc: desc}, + TxConfirmRef: txconfirmRef, + ChainSource: &fakeRegistryChainSourceRef{height: 200}, + Wallet: &fakeSweepWallet{}, + VTXOExitObserver: fn.Some(observerRef), }) t.Cleanup(registry.Stop) @@ -1387,6 +1427,10 @@ func TestRegistryTerminalStatusRemainsQueryableWhilePersistBlocked( case <-time.After(testTimeout): t.Fatal("timed out waiting for blocked registry persist") } + require.Empty( + t, observer.notifications(), + "terminal outcome must wait for durable invalidation", + ) require.Eventually(t, func() bool { resp, err := registry.Ref().Ask( @@ -1412,6 +1456,11 @@ func TestRegistryTerminalStatusRemainsQueryableWhilePersistBlocked( return record != nil && record.Phase == PhaseFailed }, testTimeout, 10*time.Millisecond) + require.Empty( + t, observer.notifications(), + "a real broadcast failure remains held without objective "+ + "canonical-absence evidence", + ) } // TestRegistryRestoreFailureLeavesRecordRetryable verifies that a @@ -1550,12 +1599,11 @@ func TestRegistryRestoreFailureLeavesRecordRetryable(t *testing.T) { require.Equal(t, PhaseMaterializing, status.Phase) } -// TestRegistryLateAdmissionFailureMarksFailed verifies that when the +// TestRegistryLateAdmissionFailureStaysPending verifies that when the // registry's synchronous admission wait times out, the eventual child -// start result is still observed and a late deterministic start failure -// becomes a terminal registry record instead of leaving PhasePending -// forever. -func TestRegistryLateAdmissionFailureMarksFailed(t *testing.T) { +// start result is still observed while the durable record remains +// non-terminal and fail-closed for a later restore. +func TestRegistryLateAdmissionFailureStaysPending(t *testing.T) { proof := buildLinearProof(t) desc := testDescriptor(t, proof.TargetOutpoint(), proof.CSVDelay()) store := newMemRegistryStore() @@ -1601,11 +1649,81 @@ func TestRegistryLateAdmissionFailureMarksFailed(t *testing.T) { require.NoError(t, err) return record != nil && - record.Phase == PhaseFailed && + record.Phase == PhasePending && + !record.RecoverableFailure && strings.Contains(record.FailReason, "late start boom") }, testTimeout, 10*time.Millisecond) } +// TestRegistryRestoreWithoutCheckpointReplaysOriginalStart verifies that a +// parked admission which failed before its first checkpoint does not resume as +// an invented restart/standard-policy job. The durable registry identity must +// seed a fresh Start request verbatim. +func TestRegistryRestoreWithoutCheckpointReplaysOriginalStart(t *testing.T) { + target := wire.OutPoint{Hash: chainhash.Hash{0xe1}, Index: 4} + startRequests := make(chan *StartUnrollRequest, 1) + behavior := ®istryBehavior{} + behavior.spawnFunc = func(_ context.Context, + spawnTarget wire.OutPoint) (*VTXOUnrollActor, error) { + + require.Equal(t, target, spawnTarget) + childBehavior := actor.NewFunctionBehavior( + func(_ context.Context, msg Msg) fn.Result[Resp] { + switch request := msg.(type) { + case *GetStateRequest: + return fn.Ok[Resp](&GetStateResp{ + Started: false, + Phase: PhasePending, + }) + + case *StartUnrollRequest: + startRequests <- request + + return fn.Ok[Resp](&AckResp{}) + + case *ResumeUnrollRequest: + return fn.Err[Resp]( + errors.New("unexpected resume"), + ) + + default: + return fn.Err[Resp]( + fmt.Errorf("unexpected msg %T", + msg), + ) + } + }, + ) + + // Test children are owned by the returned handle below. + //nolint:contextcheck + return newTestUnrollChild(t, spawnTarget, childBehavior), nil + } + + record := RegistryRecord{ + TargetOutpoint: target, + ActorID: actorIDForTarget(target), + Trigger: TriggerFraudSpend, + ExitPolicyKind: ExitPolicyKind("custom-recovery-policy"), + ExitPolicyRef: "recovery-job-42", + Phase: PhasePending, + } + child, err := behavior.tryRestoreOne(t.Context(), record, 321) + require.NoError(t, err) + t.Cleanup(child.Stop) + + select { + case request := <-startRequests: + require.Equal(t, int32(321), request.Height) + require.Equal(t, record.Trigger, request.Trigger) + require.Equal(t, record.ExitPolicyKind, request.ExitPolicyKind) + require.Equal(t, record.ExitPolicyRef, request.ExitPolicyRef) + + case <-time.After(testTimeout): + t.Fatal("restore did not replay original start identity") + } +} + // TestRegistryEnsureRestoresFailedNonTerminalRecord verifies that when a // non-terminal record exists in the durable store but no child is // active (because a prior RestoreNonTerminal hit a transient error and diff --git a/unroll/reorg_safety_test.go b/unroll/reorg_safety_test.go new file mode 100644 index 000000000..d4ea9af59 --- /dev/null +++ b/unroll/reorg_safety_test.go @@ -0,0 +1,787 @@ +package unroll + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/btcsuite/btcd/chainhash/v2" + "github.com/btcsuite/btclog/v2" + "github.com/lightninglabs/wavelength/baselib/actor" + "github.com/lightninglabs/wavelength/txconfirm" + "github.com/lightninglabs/wavelength/unrollplan" + fn "github.com/lightningnetwork/lnd/fn/v2" + "github.com/stretchr/testify/require" +) + +// TestProofRootReorgBlocksDownstreamMaterialization drives a proof root +// through Confirmed -> Reorged -> Confirmed and asserts that the unroll +// actor does NOT submit the target transaction while the root anchor is +// torn down. Only once the root reconfirms does the actor advance the +// proof graph. +// +// This is the load-bearing reorg-safety case: before the rollback work +// the actor treated the first TxConfirmedMsg as a monotonic fact and +// would have submitted the target while the chain was still missing the +// root. After rollback, the target submission is gated on the root +// anchor being live. +func TestProofRootReorgBlocksDownstreamMaterialization(t *testing.T) { + proof := buildLinearProof(t) + desc := testDescriptor(t, proof.TargetOutpoint(), proof.CSVDelay()) + unrollActor, _, txconfirmRef, _ := newActorHarness(t, proof, desc) + + mustAsk(t, unrollActor.Ref(), &StartUnrollRequest{ + Height: 100, + Trigger: TriggerManual, + }) + + rootTxid := proof.RootTxids()[0] + targetTxid := proof.TargetOutpoint().Hash + + // Wait for the actor to register the root with txconfirm. + require.Eventually(t, func() bool { + return txconfirmRef.requestCountForTxid(rootTxid) == 1 + }, testTimeout, 10*time.Millisecond, "root never submitted") + + // 1. Root confirms; the actor should immediately submit the target. + txconfirmRef.emitConfirmed(t, 0, rootTxid, 101) + require.Eventually(t, func() bool { + return txconfirmRef.requestCountForTxid(targetTxid) == 1 + }, testTimeout, 10*time.Millisecond, "target never submitted") + + // 2. Root reorgs out. The actor must drop the root anchor AND the + // dependent target anchor from in-flight, since State.Validate + // requires every in-flight node to have confirmed parents in the + // proof graph. + targetSubmissionsBeforeReorg := txconfirmRef.requestCountForTxid( + targetTxid, + ) + txconfirmRef.emitReorged(t, 0, rootTxid) + + require.Eventually(t, func() bool { + resp, ok := mustAsk( + t, unrollActor.Ref(), &GetStateRequest{}, + ).(*GetStateResp) + require.True(t, ok) + for _, h := range resp.PlannerState.ConfirmedTxids { + if h == rootTxid { + return false + } + } + for _, h := range resp.PlannerState.InFlightTxids { + if h == targetTxid { + return false + } + } + + return true + }, testTimeout, 10*time.Millisecond, + "root + target anchors never cleared after reorg") + + // The actor naturally re-submits the root on its ready frontier + // after the reorg drops it from ConfirmedTxids — txconfirm dedup + // absorbs that re-submit. What matters for reorg safety is that the + // TARGET was NOT re-submitted while its parent anchor was missing: + // driving target broadcast off a non-anchored parent is exactly the + // unsafe behavior this work fixes. + time.Sleep(50 * time.Millisecond) + require.Equal( + t, targetSubmissionsBeforeReorg, + txconfirmRef.requestCountForTxid(targetTxid), + "target was re-submitted while its parent was reorged out", + ) + + // 3. Root reconfirms in a different block. The actor should keep + // progressing without resubmitting target (txconfirm dedup already + // holds the in-flight subscription, but the planner state must be + // coherent). + txconfirmRef.emitConfirmed(t, 0, rootTxid, 102) + require.Eventually(t, func() bool { + resp, ok := mustAsk( + t, unrollActor.Ref(), &GetStateRequest{}, + ).(*GetStateResp) + require.True(t, ok) + for _, h := range resp.PlannerState.ConfirmedTxids { + if h == rootTxid { + return true + } + } + + return false + }, testTimeout, 10*time.Millisecond, + "root anchor never re-recorded after reconfirm") +} + +// TestTargetReorgClearsCSVMaturity confirms the target tx, waits for the +// sweep to broadcast, then reorgs the target out. The actor must clear +// TargetConfirmHeight and downgrade the sweep so the planner stops +// reporting Done. +func TestTargetReorgClearsCSVMaturity(t *testing.T) { + proof := buildLinearProof(t) + desc := testDescriptor(t, proof.TargetOutpoint(), proof.CSVDelay()) + unrollActor, _, txconfirmRef, _ := newActorHarness(t, proof, desc) + + mustAsk(t, unrollActor.Ref(), &StartUnrollRequest{ + Height: 100, + Trigger: TriggerManual, + }) + + rootTxid := proof.RootTxids()[0] + targetTxid := proof.TargetOutpoint().Hash + + require.Eventually(t, func() bool { + return txconfirmRef.requestCountForTxid(rootTxid) == 1 + }, testTimeout, 10*time.Millisecond) + + txconfirmRef.emitConfirmed(t, 0, rootTxid, 101) + require.Eventually(t, func() bool { + return txconfirmRef.requestCountForTxid(targetTxid) == 1 + }, testTimeout, 10*time.Millisecond) + + txconfirmRef.emitConfirmed(t, 1, targetTxid, 102) + + // Advance height past CSV maturity so the sweep is built and + // broadcast. + mustAsk(t, unrollActor.Ref(), &HeightObservedMsg{ + Height: 102 + int32(proof.CSVDelay()) + 1, + }) + + require.Eventually(t, func() bool { + return txconfirmRef.requestCount() >= 3 + }, testTimeout, 10*time.Millisecond, + "sweep was never broadcast") + + sweepReq := txconfirmRef.lastRequest(t) + sweepTxid := sweepReq.Tx.TxHash() + + // Sweep confirms. + txconfirmRef.emitConfirmed(t, 2, sweepTxid, 110) + require.Eventually(t, func() bool { + resp, ok := mustAsk( + t, unrollActor.Ref(), &GetStateRequest{}, + ).(*GetStateResp) + require.True(t, ok) + + return resp.Phase == PhaseCompleted + }, testTimeout, 10*time.Millisecond, + "sweep never reached Completed") + + // 1. Target reorg invalidates the CSV anchor AND the sweep + // confirmation (which depended on it). + txconfirmRef.emitReorged(t, 1, targetTxid) + + require.Eventually(t, func() bool { + resp, ok := mustAsk( + t, unrollActor.Ref(), &GetStateRequest{}, + ).(*GetStateResp) + require.True(t, ok) + + csvCleared := resp.PlannerState.TargetConfirmHeight.IsNone() + targetCleared := true + for _, h := range resp.PlannerState.ConfirmedTxids { + if h == targetTxid { + targetCleared = false + } + } + + return csvCleared && targetCleared + }, testTimeout, 10*time.Millisecond, + "target anchor never cleared after reorg") +} + +// recordingRegistryRef is a stub RegistryRef that captures every +// message delivered by the per-target actor's notifyRegistryIfTerminal +// path so a test can assert "the registry has / has not been told the +// actor is terminal". +type recordingRegistryRef struct { + id string + mu sync.Mutex + msgs []RegistryMsg +} + +// ID returns the stub registry ref identifier. +func (r *recordingRegistryRef) ID() string { + return r.id +} + +// Tell records the inbound RegistryMsg and acks. +func (r *recordingRegistryRef) Tell(_ context.Context, msg RegistryMsg) error { + r.mu.Lock() + defer r.mu.Unlock() + r.msgs = append(r.msgs, msg) + + return nil +} + +func (r *recordingRegistryRef) TryTell(ctx context.Context, + msg RegistryMsg) error { + + return r.Tell(ctx, msg) +} + +// terminatedCount returns how many UnrollTerminatedMsg messages have +// been delivered so far. +func (r *recordingRegistryRef) terminatedCount() int { + r.mu.Lock() + defer r.mu.Unlock() + + count := 0 + for _, m := range r.msgs { + if _, ok := m.(*UnrollTerminatedMsg); ok { + count++ + } + } + + return count +} + +// terminatedMessages returns a copy of every terminal message captured by the +// registry stub. +func (r *recordingRegistryRef) terminatedMessages() []*UnrollTerminatedMsg { + r.mu.Lock() + defer r.mu.Unlock() + + var messages []*UnrollTerminatedMsg + for _, msg := range r.msgs { + terminated, ok := msg.(*UnrollTerminatedMsg) + if !ok { + continue + } + + messages = append(messages, terminated) + } + + return messages +} + +// TestRestartFailureCarriesDurableReliveGuard verifies that restart marks the +// job unsafe to relive before it reissues an in-flight transaction, and that a +// synchronous rejection carries the guard through the real actor terminal +// handoff. This closes the process-boundary gap where a conflicting offline +// spend could otherwise make the reissue fail "cleanly" and revive a spent +// VTXO. +func TestRestartFailureCarriesDurableReliveGuard(t *testing.T) { + proof := buildLinearProof(t) + desc := testDescriptor(t, proof.TargetOutpoint(), proof.CSVDelay()) + store := newMemCheckpointStore() + txconfirmRef := &fakeTxConfirmRef{} + registryRef := &recordingRegistryRef{id: "restart-reg-stub"} + rootTxid := proof.RootTxids()[0] + + raw, err := encodeCheckpoint(&actorCheckpoint{ + Version: checkpointVersion, + Height: 100, + Started: true, + Trigger: TriggerManual, + State: unrollplan.State{ + InFlightTxids: []chainhash.Hash{rootTxid}, + }, + }) + require.NoError(t, err) + require.NoError( + t, + store.SaveCheckpoint( + t.Context(), actor.CheckpointParams{ + ActorID: "restart-relive-guard-test", + StateType: checkpointStateType, + StateData: raw, + Version: checkpointVersion, + }, + ), + ) + + persistedBeforeReissue := make(chan bool, 1) + txconfirmRef.onAsk = func(_ *txconfirm.EnsureConfirmedReq) { + checkpoint, loadErr := store.LoadCheckpoint( + context.Background(), + "restart-relive-guard-test", + ) + if loadErr != nil || checkpoint == nil { + persistedBeforeReissue <- false + + return + } + + decoded, decodeErr := decodeCheckpoint(checkpoint.StateData) + persisted := decodeErr == nil && decoded.ReliveUnsafe + persistedBeforeReissue <- persisted + } + txconfirmRef.setImmediateFailed(rootTxid, "conflicting spend") + + cfg := Config{ + TargetOutpoint: proof.TargetOutpoint(), + ActorID: "restart-relive-guard-test", + DeliveryStore: store, + ProofAssembler: &mockProofAssembler{ + proof: proof, + }, + VTXOStore: &mockVTXOStore{ + desc: desc, + }, + TxConfirmRef: txconfirmRef, + ChainSource: &fakeChainSourceRef{}, + Wallet: &fakeSweepWallet{}, + Log: fn.Some(btclog.Disabled), + RegistryRef: registryRef, + } + resumeBehavior := &behavior{cfg: cfg, log: btclog.Disabled} + require.NoError(t, resumeBehavior.restoreCheckpoint(t.Context())) + + resumedActor := actor.NewActor(actor.ActorConfig[Msg, Resp]{ + ID: "restart-relive-guard-test", + Behavior: adaptTx(resumeBehavior), + MailboxSize: 64, + }) + resumeBehavior.selfRef = resumedActor.TellRef() + resumedActor.Start() + t.Cleanup(resumedActor.Stop) + + mustAsk(t, resumedActor.Ref(), &ResumeUnrollRequest{Height: 101}) + require.True( + t, <-persistedBeforeReissue, + "relive guard must be durable before txconfirm reissue", + ) + require.Eventually(t, func() bool { + return registryRef.terminatedCount() == 1 + }, testTimeout, 10*time.Millisecond) + + messages := registryRef.terminatedMessages() + require.Len(t, messages, 1) + require.Equal(t, PhaseFailed, messages[0].Phase) + require.False(t, messages[0].HadOnChainFootprint) + require.True(t, messages[0].ReliveUnsafe) +} + +// TestSweepCompletionStaysProvisionalUntilFinalized proves the Phase 7 +// invariant: a per-target actor that has reached PhaseCompleted does +// NOT notify the registry as terminal until a TxFinalizedMsg for the +// sweep txid arrives. Until then the actor stays alive so a reorg of +// the sweep confirmation has a live actor to deliver the rollback to. +func TestSweepCompletionStaysProvisionalUntilFinalized(t *testing.T) { + proof := buildLinearProof(t) + desc := testDescriptor(t, proof.TargetOutpoint(), proof.CSVDelay()) + + store := newMemCheckpointStore() + txconfirmRef := &fakeTxConfirmRef{} + registryRef := &recordingRegistryRef{id: "reg-stub"} + + cfg := Config{ + TargetOutpoint: proof.TargetOutpoint(), + ActorID: "unroll-finality-test", + DeliveryStore: store, + ProofAssembler: &mockProofAssembler{ + proof: proof, + }, + VTXOStore: &mockVTXOStore{ + desc: desc, + }, + TxConfirmRef: txconfirmRef, + ChainSource: &fakeChainSourceRef{}, + Wallet: &fakeSweepWallet{}, + Log: fn.Some(btclog.Disabled), + RegistryRef: registryRef, + } + beh := &behavior{cfg: cfg, log: btclog.Disabled} + require.NoError(t, beh.restoreCheckpoint(t.Context())) + + actorInstance := actor.NewActor(actor.ActorConfig[Msg, Resp]{ + ID: "unroll-finality-test", + Behavior: &txExecAdapter{b: beh, ax: newMemExecFor(beh)}, + MailboxSize: 32, + }) + beh.selfRef = actorInstance.TellRef() + actorInstance.Start() + t.Cleanup(actorInstance.Stop) + + mustAsk(t, actorInstance.Ref(), &StartUnrollRequest{ + Height: 100, + Trigger: TriggerManual, + }) + + rootTxid := proof.RootTxids()[0] + targetTxid := proof.TargetOutpoint().Hash + + require.Eventually(t, func() bool { + return txconfirmRef.requestCountForTxid(rootTxid) == 1 + }, testTimeout, 10*time.Millisecond) + + txconfirmRef.emitConfirmed(t, 0, rootTxid, 101) + require.Eventually(t, func() bool { + return txconfirmRef.requestCountForTxid(targetTxid) == 1 + }, testTimeout, 10*time.Millisecond) + + txconfirmRef.emitConfirmed(t, 1, targetTxid, 102) + mustAsk(t, actorInstance.Ref(), &HeightObservedMsg{ + Height: 102 + int32(proof.CSVDelay()) + 1, + }) + + require.Eventually(t, func() bool { + return txconfirmRef.requestCount() >= 3 + }, testTimeout, 10*time.Millisecond) + + sweepTxid := txconfirmRef.lastRequest(t).Tx.TxHash() + txconfirmRef.emitConfirmed(t, 2, sweepTxid, 110) + + // Wait for the actor to settle on PhaseCompleted. + require.Eventually(t, func() bool { + resp, ok := mustAsk( + t, actorInstance.Ref(), &GetStateRequest{}, + ).(*GetStateResp) + require.True(t, ok) + + return resp.Phase == PhaseCompleted + }, testTimeout, 10*time.Millisecond) + + // 1. PROVISIONAL: registry must NOT have been told the actor is + // terminal. The sweep can still reorg out. + time.Sleep(50 * time.Millisecond) + require.Equal( + t, 0, registryRef.terminatedCount(), + "registry was told terminal before TxFinalized arrived", + ) + + // 2. Sweep reorgs out; the live actor rolls back to + // AwaitingSweepConfirmation. Registry still has not been + // terminated. + txconfirmRef.emitReorged(t, 2, sweepTxid) + require.Eventually(t, func() bool { + resp, ok := mustAsk( + t, actorInstance.Ref(), &GetStateRequest{}, + ).(*GetStateResp) + require.True(t, ok) + + return resp.Phase == PhaseSweepConfirmation + }, testTimeout, 10*time.Millisecond, + "actor never rolled back to AwaitingSweepConfirmation") + require.Equal( + t, 0, registryRef.terminatedCount(), + "registry was told terminal during sweep reorg recovery", + ) + + // 3. Sweep re-confirms; actor returns to PhaseCompleted. + txconfirmRef.emitConfirmed(t, 2, sweepTxid, 111) + require.Eventually(t, func() bool { + resp, ok := mustAsk( + t, actorInstance.Ref(), &GetStateRequest{}, + ).(*GetStateResp) + require.True(t, ok) + + return resp.Phase == PhaseCompleted + }, testTimeout, 10*time.Millisecond) + require.Equal( + t, 0, registryRef.terminatedCount(), + "registry was told terminal after sweep re-confirm but "+ + "before TxFinalized", + ) + + // 4. Finality. TxFinalizedMsg matches the recorded sweep txid, + // the actor latches sweepFinalized, and the next driveEvent + // fires UnrollTerminatedMsg. + txconfirmRef.emitFinalized(t, 2, sweepTxid) + require.Eventually(t, func() bool { + return registryRef.terminatedCount() == 1 + }, testTimeout, 10*time.Millisecond, + "registry was never told terminal after TxFinalized") +} + +// TestExternalSpendReorgResumesActor observes an external spend of the +// target outpoint while materialization is in flight, then drives a +// reorg of the spending block. The actor must park in +// AwaitingExternalSpendFinality while the spend is provisional, then +// resume normal planning once the spend reorgs out. Without this +// behavior, a transient reorg-out replacement of the target would +// permanently terminate the recovery job. +func TestExternalSpendReorgResumesActor(t *testing.T) { + proof := buildLinearProof(t) + desc := testDescriptor(t, proof.TargetOutpoint(), proof.CSVDelay()) + unrollActor, beh, txconfirmRef, _ := newActorHarness(t, proof, desc) + + chainSource, ok := beh.cfg.ChainSource.(*fakeChainSourceRef) + require.True(t, ok) + + mustAsk(t, unrollActor.Ref(), &StartUnrollRequest{ + Height: 100, + Trigger: TriggerManual, + }) + + rootTxid := proof.RootTxids()[0] + require.Eventually(t, func() bool { + return txconfirmRef.requestCountForTxid(rootTxid) == 1 + }, testTimeout, 10*time.Millisecond, + "root never submitted") + + // Ensure spend watch on the target is registered with the + // reorg / done refs wired before driving any events. + require.Eventually(t, func() bool { + var targetRegistered bool + for _, op := range chainSource.spendRegistrations() { + if op == proof.TargetOutpoint() { + targetRegistered = true + break + } + } + if !targetRegistered { + return false + } + + chainSource.mu.Lock() + defer chainSource.mu.Unlock() + + return chainSource.spendReorgedRef != nil && + chainSource.spendFinalizedRef != nil + }, testTimeout, 10*time.Millisecond) + + // 1. An external party broadcasts a spend of the target outpoint. + externalTxid := chainhash.Hash{0xee} + chainSource.emitSpendForOutpoint( + t, proof.TargetOutpoint(), externalTxid, 110, + ) + + // The actor must park in AwaitingExternalSpendFinality rather + // than transitioning to Failed; the spend is provisional. + require.Eventually(t, func() bool { + resp, ok := mustAsk( + t, unrollActor.Ref(), &GetStateRequest{}, + ).(*GetStateResp) + require.True(t, ok) + + return resp.Phase == PhaseExternalSpendObserved && + resp.FailReason == "" + }, testTimeout, 10*time.Millisecond, + "actor never parked in PhaseExternalSpendObserved") + + // 2. The spending block is reorged out. The actor must clear the + // provisional anchor and resume normal planning. Because the + // proof root is still in-flight (no confirmation has arrived + // yet), the actor goes back to PhaseMaterializing. + chainSource.emitSpendReorged(t) + + require.Eventually(t, func() bool { + resp, ok := mustAsk( + t, unrollActor.Ref(), &GetStateRequest{}, + ).(*GetStateResp) + require.True(t, ok) + + return resp.Phase == PhaseMaterializing && + resp.FailReason == "" + }, testTimeout, 10*time.Millisecond, + "actor did not resume materialization after spend reorg") +} + +// TestSweepConfirmationReorgReversesCompleted drives the full happy path +// to Completed and then reorgs the sweep confirmation out. The actor +// must drop SweepStatusConfirmed back to Broadcasted and clear the +// stored ConfirmHeight so the planner stops reporting Done. +func TestSweepConfirmationReorgReversesCompleted(t *testing.T) { + proof := buildLinearProof(t) + desc := testDescriptor(t, proof.TargetOutpoint(), proof.CSVDelay()) + unrollActor, _, txconfirmRef, _ := newActorHarness(t, proof, desc) + + mustAsk(t, unrollActor.Ref(), &StartUnrollRequest{ + Height: 100, + Trigger: TriggerManual, + }) + + rootTxid := proof.RootTxids()[0] + targetTxid := proof.TargetOutpoint().Hash + + require.Eventually(t, func() bool { + return txconfirmRef.requestCountForTxid(rootTxid) == 1 + }, testTimeout, 10*time.Millisecond) + + txconfirmRef.emitConfirmed(t, 0, rootTxid, 101) + require.Eventually(t, func() bool { + return txconfirmRef.requestCountForTxid(targetTxid) == 1 + }, testTimeout, 10*time.Millisecond) + + txconfirmRef.emitConfirmed(t, 1, targetTxid, 102) + mustAsk(t, unrollActor.Ref(), &HeightObservedMsg{ + Height: 102 + int32(proof.CSVDelay()) + 1, + }) + + require.Eventually(t, func() bool { + return txconfirmRef.requestCount() >= 3 + }, testTimeout, 10*time.Millisecond) + + sweepReq := txconfirmRef.lastRequest(t) + sweepTxid := sweepReq.Tx.TxHash() + + txconfirmRef.emitConfirmed(t, 2, sweepTxid, 110) + require.Eventually(t, func() bool { + resp, ok := mustAsk( + t, unrollActor.Ref(), &GetStateRequest{}, + ).(*GetStateResp) + require.True(t, ok) + + return resp.Phase == PhaseCompleted + }, testTimeout, 10*time.Millisecond) + + // 1. Sweep confirmation reorgs out. The actor must roll back to + // AwaitingSweepConfirmation: the signed sweep is still durable, the + // txconfirm subscription is still live, but the planner stops + // reporting Done. + txconfirmRef.emitReorged(t, 2, sweepTxid) + + require.Eventually(t, func() bool { + resp, ok := mustAsk( + t, unrollActor.Ref(), &GetStateRequest{}, + ).(*GetStateResp) + require.True(t, ok) + + return resp.Phase == PhaseSweepConfirmation && + resp.PlannerState.Sweep.Status == + unrollplan.SweepStatusBroadcasted && + resp.PlannerState.Sweep.ConfirmHeight.IsNone() + }, testTimeout, 10*time.Millisecond, + "sweep reorg did not roll the actor back to "+ + "AwaitingSweepConfirmation") + + // 2. Sweep reconfirms in a different block. The actor must move + // back to Completed. + txconfirmRef.emitConfirmed(t, 2, sweepTxid, 111) + require.Eventually(t, func() bool { + resp, ok := mustAsk( + t, unrollActor.Ref(), &GetStateRequest{}, + ).(*GetStateResp) + require.True(t, ok) + + sweep := resp.PlannerState.Sweep + + return resp.Phase == PhaseCompleted && + sweep.ConfirmHeight.IsSome() && + sweep.ConfirmHeight.UnsafeFromSome() == 111 + }, testTimeout, 10*time.Millisecond, + "sweep did not re-Complete after reconfirm") +} + +// TestOfflineReorgRestartReconcilesBeforeSideEffects bundles the three +// offline-reorg scenarios (target reorged out, sweep reorged out, and +// external-spend reorged out) into one restart and asserts that the +// reconciler runs BEFORE the FSM session is built — i.e. before the +// actor can broadcast a stale sweep or park in +// AwaitingExternalSpendFinality on a vanished spender. +// +// The setup mimics a daemon that committed a checkpoint reflecting: +// +// - root + target proof nodes confirmed +// - a sweep tx broadcast AND confirmed at H=108 +// - an external spender provisionally observed at H=110 +// +// While the daemon was offline, the chain reorged: the target's +// confirming block was orphaned (so the sweep that spent it can no +// longer be confirmed either), and the external spender's block was +// also dropped. The reconciler the actor consults on restart returns +// "root still confirmed" and "everything else absent". +// +// After Resume the actor must end up in the post-reconcile state +// (sweep downgraded to Pending, TargetConfirmHeight cleared, target +// pruned from ConfirmedTxids, ProvisionalExternalSpend cleared) and +// must NOT have re-submitted the stale sweep tx to txconfirm before +// the planner re-derives broadcast intent from the post-reconcile +// PlannerState. The latter is the load-bearing safety invariant: a +// re-broadcast off a stale checkpoint would race a fresh wallet +// pkScript against any future re-mining of the target. +func TestOfflineReorgRestartReconcilesBeforeSideEffects(t *testing.T) { + proof := buildLinearProof(t) + rootTxid := proof.RootTxids()[0] + targetTxid := proof.TargetOutpoint().Hash + staleSweepTxid := chainhash.Hash{0xaa} + staleExternalTxid := chainhash.Hash{0xbb} + + checkpoint := &actorCheckpoint{ + Version: checkpointVersion, + Height: 112, + Started: true, + Trigger: TriggerRestart, + State: unrollplan.State{ + ConfirmedTxids: []chainhash.Hash{ + rootTxid, targetTxid, + }, + TargetConfirmHeight: fn.Some[int32](102), + Sweep: unrollplan.SweepState{ + Status: unrollplan.SweepStatusConfirmed, + Txid: fn.Some(staleSweepTxid), + ConfirmHeight: fn.Some[int32](108), + }, + }, + ProvisionalExternalSpend: fn.Some(ExternalSpendAnchor{ + SpendingTxid: staleExternalTxid, + SpendingHeight: 110, + }), + } + + // Reconciler reports: + // - root still confirmed (so the planner keeps the partial + // proof-graph progress) + // - target absent (offline reorg) + // - stale sweep absent (offline reorg) + // - target outpoint unspent (external spender reorged out) + reconciler := &stubChainReconciler{ + confirmed: map[chainhash.Hash]ConfirmedAnchor{ + rootTxid: { + Txid: rootTxid, + Height: 101, + }, + }, + } + + beh, txconfirmRef, _ := restoreHarness( + t, proof, checkpoint, reconciler, + ) + + actorInstance := actor.NewActor(actor.ActorConfig[Msg, Resp]{ + ID: "reconcile-test", + Behavior: &txExecAdapter{b: beh, ax: newMemExecFor(beh)}, + MailboxSize: 16, + }) + beh.selfRef = actorInstance.TellRef() + actorInstance.Start() + t.Cleanup(actorInstance.Stop) + + mustAsk(t, actorInstance.Ref(), &ResumeUnrollRequest{Height: 112}) + + // The actor must end up in the post-reconcile state: sweep + // downgraded, target-derived height cleared, target pruned from + // ConfirmedTxids, external spend cleared, root preserved. + require.Eventually(t, func() bool { + resp, ok := mustAsk( + t, actorInstance.Ref(), &GetStateRequest{}, + ).(*GetStateResp) + require.True(t, ok) + + for _, h := range resp.PlannerState.ConfirmedTxids { + if h == targetTxid { + return false + } + } + sweep := resp.PlannerState.Sweep + rootPresent := false + for _, h := range resp.PlannerState.ConfirmedTxids { + if h == rootTxid { + rootPresent = true + break + } + } + + return rootPresent && + sweep.Status == unrollplan.SweepStatusPending && + sweep.Txid.IsNone() && + sweep.ConfirmHeight.IsNone() && + resp.PlannerState.TargetConfirmHeight.IsNone() && + resp.Phase == PhaseMaterializing && + resp.FailReason == "" + }, testTimeout, 10*time.Millisecond, + "actor did not reach post-reconcile state on restart") + + // Load-bearing safety invariant: every txconfirm request issued + // after restart must target a tx the post-reconcile PlannerState + // authorizes. In particular, the orphaned sweep txid must never + // be re-submitted — that would replay a sweep against a target + // the chain says no longer exists. + require.Zero( + t, txconfirmRef.requestCountForTxid(staleSweepTxid), + "stale sweep tx was re-submitted to txconfirm before "+ + "reconciliation downgraded it", + ) +} diff --git a/unroll/snapshot.go b/unroll/snapshot.go index 9ba2733af..7fb6483c8 100644 --- a/unroll/snapshot.go +++ b/unroll/snapshot.go @@ -2,10 +2,13 @@ package unroll import ( "bytes" + "encoding/binary" "fmt" + "github.com/btcsuite/btcd/chainhash/v2" "github.com/btcsuite/btcd/wire/v2" "github.com/lightninglabs/wavelength/unrollplan" + fn "github.com/lightningnetwork/lnd/fn/v2" "github.com/lightningnetwork/lnd/tlv" ) @@ -93,21 +96,91 @@ const ( // checkpointExitPolicyRefRecordType carries the policy-specific // durable-state reference. checkpointExitPolicyRefRecordType tlv.Type = 21 + + // checkpointExternalSpendRecordType is optional; present only when + // the actor has observed an external spend of the target outpoint + // that has not yet been finalized. Payload is a fixed-layout + // 36-byte blob (32-byte SpendingTxid + 4-byte big-endian + // SpendingHeight) so a daemon restarting mid-spend-finality-window + // can rehydrate into AwaitingExternalSpendFinality instead of + // dropping the provisional anchor and broadcasting a sweep on a + // target the chain says no longer exists. + checkpointExternalSpendRecordType tlv.Type = 23 + + // checkpointSweepFinalizedRecordType is optional; present (value 1) + // only when the sweep has finalized past the backend's reorg-safety + // depth. It makes the in-memory sweep-finalized latch durable so a + // PhaseCompleted entry whose terminal handoff was deferred or failed + // does not restart as permanently "provisional completed" (the latch + // gates notifyRegistryIfTerminal). Omitted when false. + checkpointSweepFinalizedRecordType tlv.Type = 25 + + // checkpointExternalSpendFinalizedRecordType is optional; present + // (value 1) when a target spend crossed policy finality. This prevents + // restart from misclassifying an objectively consumed target as a + // recoverable failure. + checkpointExternalSpendFinalizedRecordType tlv.Type = 27 + + // checkpointReliveUnsafeRecordType is present for every checkpoint + // whose actor has started. A value of 1 means chain-boundary work may + // have escaped; 0 records an explicit clear by authoritative negative + // evidence. Started checkpoints written before this record existed + // omit it and decode fail-closed as unsafe. + checkpointReliveUnsafeRecordType tlv.Type = 29 ) +// externalSpendBlobSize is the canonical wire size of the persisted +// ProvisionalExternalSpend payload: 32-byte txid + 4-byte big-endian +// height. +const externalSpendBlobSize = chainhash.HashSize + 4 + // actorCheckpoint is the durable checkpoint shape for one VTXO unroll actor. type actorCheckpoint struct { - Version uint8 - Height int32 - Started bool - Trigger StartTrigger - State unrollplan.State - ExitPolicyKind ExitPolicyKind - ExitPolicyRef string - SweepTx *wire.MsgTx - Fail string - SweepAttempts int - DeferredCheckpoints []DeferredCheckpoint + Version uint8 + Height int32 + Started bool + Trigger StartTrigger + State unrollplan.State + ExitPolicyKind ExitPolicyKind + ExitPolicyRef string + SweepTx *wire.MsgTx + Fail string + SweepAttempts int + DeferredCheckpoints []DeferredCheckpoint + ProvisionalExternalSpend fn.Option[ExternalSpendAnchor] + SweepFinalized bool + ExternalSpendFinalized bool + ReliveUnsafe bool +} + +// encodeExternalSpendBlob serializes an ExternalSpendAnchor into the +// fixed-layout payload carried by checkpointExternalSpendRecordType. +func encodeExternalSpendBlob(anchor ExternalSpendAnchor) []byte { + blob := make([]byte, externalSpendBlobSize) + copy(blob[:chainhash.HashSize], anchor.SpendingTxid[:]) + binary.BigEndian.PutUint32( + blob[chainhash.HashSize:], uint32(anchor.SpendingHeight), + ) + + return blob +} + +// decodeExternalSpendBlob parses a fixed-layout external-spend payload +// back into an ExternalSpendAnchor. +func decodeExternalSpendBlob(blob []byte) (ExternalSpendAnchor, error) { + if len(blob) != externalSpendBlobSize { + return ExternalSpendAnchor{}, fmt.Errorf("external spend blob "+ + "has %d bytes, want %d", len(blob), + externalSpendBlobSize) + } + + var anchor ExternalSpendAnchor + copy(anchor.SpendingTxid[:], blob[:chainhash.HashSize]) + anchor.SpendingHeight = int32( + binary.BigEndian.Uint32(blob[chainhash.HashSize:]), + ) + + return anchor, nil } // encodeCheckpoint serializes one actor checkpoint into canonical TLV @@ -226,6 +299,49 @@ func encodeCheckpoint(value *actorCheckpoint) ([]byte, error) { ) } + value.ProvisionalExternalSpend.WhenSome( + func(anchor ExternalSpendAnchor) { + blob := encodeExternalSpendBlob(anchor) + records = append( + records, tlv.MakePrimitiveRecord( + checkpointExternalSpendRecordType, + &blob, + ), + ) + }, + ) + + if value.SweepFinalized { + finalized := uint8(1) + records = append( + records, tlv.MakePrimitiveRecord( + checkpointSweepFinalizedRecordType, &finalized, + ), + ) + } + + if value.ExternalSpendFinalized { + finalized := uint8(1) + records = append( + records, tlv.MakePrimitiveRecord( + checkpointExternalSpendFinalizedRecordType, + &finalized, + ), + ) + } + + if value.Started { + unsafe := uint8(0) + if value.ReliveUnsafe { + unsafe = 1 + } + records = append( + records, tlv.MakePrimitiveRecord( + checkpointReliveUnsafeRecordType, &unsafe, + ), + ) + } + stream, err := tlv.NewStream(records...) if err != nil { return nil, fmt.Errorf("create checkpoint stream: %w", err) @@ -255,17 +371,21 @@ func encodeCheckpoint(value *actorCheckpoint) ([]byte, error) { // truncated state. func decodeCheckpoint(raw []byte) (*actorCheckpoint, error) { var ( - version uint8 - height uint32 - started uint8 - trigger uint32 - stateBytes []byte - sweepBytes []byte - failBytes []byte - attempts uint32 - deferredBytes []byte - policyKind []byte - policyRef []byte + version uint8 + height uint32 + started uint8 + trigger uint32 + stateBytes []byte + sweepBytes []byte + failBytes []byte + attempts uint32 + deferredBytes []byte + policyKind []byte + policyRef []byte + extSpendBlob []byte + sweepFinalized uint8 + extFinalized uint8 + reliveUnsafe uint8 ) stream, err := tlv.NewStream( @@ -302,6 +422,19 @@ func decodeCheckpoint(raw []byte) (*actorCheckpoint, error) { tlv.MakePrimitiveRecord( checkpointExitPolicyRefRecordType, &policyRef, ), + tlv.MakePrimitiveRecord( + checkpointExternalSpendRecordType, &extSpendBlob, + ), + tlv.MakePrimitiveRecord( + checkpointSweepFinalizedRecordType, &sweepFinalized, + ), + tlv.MakePrimitiveRecord( + checkpointExternalSpendFinalizedRecordType, + &extFinalized, + ), + tlv.MakePrimitiveRecord( + checkpointReliveUnsafeRecordType, &reliveUnsafe, + ), ) if err != nil { return nil, fmt.Errorf("create checkpoint stream: %w", err) @@ -367,6 +500,32 @@ func decodeCheckpoint(raw []byte) (*actorCheckpoint, error) { checkpoint.DeferredCheckpoints = checkpoints } + if _, ok := parsed[checkpointExternalSpendRecordType]; ok { + anchor, err := decodeExternalSpendBlob(extSpendBlob) + if err != nil { + return nil, fmt.Errorf("decode external spend: %w", err) + } + checkpoint.ProvisionalExternalSpend = fn.Some(anchor) + } + + if _, ok := parsed[checkpointSweepFinalizedRecordType]; ok { + checkpoint.SweepFinalized = sweepFinalized != 0 + } + + if _, ok := parsed[checkpointExternalSpendFinalizedRecordType]; ok { + checkpoint.ExternalSpendFinalized = extFinalized != 0 + } + + if _, ok := parsed[checkpointReliveUnsafeRecordType]; ok { + checkpoint.ReliveUnsafe = reliveUnsafe != 0 + } else if checkpoint.Started { + // Checkpoints written before the guard was added have no + // record. A started job may already have crossed a chain + // boundary, so absence cannot be treated as evidence that + // reliving is safe. + checkpoint.ReliveUnsafe = true + } + return checkpoint, nil } diff --git a/unroll/snapshot_test.go b/unroll/snapshot_test.go index 33c40a62f..ab89ba75b 100644 --- a/unroll/snapshot_test.go +++ b/unroll/snapshot_test.go @@ -81,6 +81,28 @@ func TestCheckpointCodecRoundTripHandcrafted(t *testing.T) { SweepAttempts: 1, }, }, + { + name: "sweep_finalized", + checkpoint: &actorCheckpoint{ + Version: checkpointVersion, + Height: 210, + Started: true, + Trigger: TriggerManual, + State: unrollplan.State{ + ConfirmedTxids: []chainhash.Hash{ + targetTxid, + }, + Sweep: unrollplan.SweepState{ + Status: unrollplan. + SweepStatusBroadcasted, + Txid: fn.Some(sweepTxid), + }, + }, + SweepTx: sweepTx, + SweepAttempts: 1, + SweepFinalized: true, + }, + }, { name: "failed", checkpoint: &actorCheckpoint{ @@ -111,6 +133,51 @@ func TestCheckpointCodecRoundTripHandcrafted(t *testing.T) { }}, }, }, + { + name: "provisional_external_spend", + checkpoint: &actorCheckpoint{ + Version: checkpointVersion, + Height: 155, + Started: true, + Trigger: TriggerManual, + State: unrollplan.State{ + ConfirmedTxids: []chainhash.Hash{ + targetTxid, + }, + TargetConfirmHeight: fn.Some[int32]( + 154, + ), + }, + ProvisionalExternalSpend: fn.Some( + ExternalSpendAnchor{ + SpendingTxid: hashFromByteCk( + 0xee, + ), + SpendingHeight: 155, + }, + ), + }, + }, + { + name: "finalized_external_spend", + checkpoint: &actorCheckpoint{ + Version: checkpointVersion, + Height: 185, + Started: true, + Trigger: TriggerManual, + ExternalSpendFinalized: true, + }, + }, + { + name: "restart_relive_unsafe", + checkpoint: &actorCheckpoint{ + Version: checkpointVersion, + Height: 190, + Started: true, + Trigger: TriggerManual, + ReliveUnsafe: true, + }, + }, } for _, tc := range cases { @@ -155,6 +222,33 @@ func TestCheckpointCodecVersionMismatch(t *testing.T) { require.ErrorContains(t, err, "unsupported checkpoint version") } +// TestCheckpointCodecLegacyStartedDefaultsReliveUnsafe proves an in-flight +// checkpoint written before the relive guard existed cannot decode fail-open +// at the upgrade boundary. +func TestCheckpointCodecLegacyStartedDefaultsReliveUnsafe(t *testing.T) { + checkpoint := &actorCheckpoint{ + Version: checkpointVersion, + Started: true, + Trigger: TriggerManual, + } + raw, err := encodeCheckpoint(checkpoint) + require.NoError(t, err) + + // Record type 29 is the final canonical record and uses one-byte TLV + // type/length encodings. Removing it reproduces the pre-field wire + // shape while retaining Started=true. + reliveRecord := []byte{ + byte(checkpointReliveUnsafeRecordType), 1, 0, + } + require.True(t, bytes.HasSuffix(raw, reliveRecord)) + raw = raw[:len(raw)-len(reliveRecord)] + + decoded, err := decodeCheckpoint(raw) + require.NoError(t, err) + require.True(t, decoded.Started) + require.True(t, decoded.ReliveUnsafe) +} + // TestCheckpointCodecCorruptDataRejected asserts that malformed input (empty, // truncated, random garbage) is rejected with an error rather than panicking // or returning a zero-value checkpoint. @@ -573,6 +667,10 @@ func requireCheckpointEqual(t *testing.T, want, got *actorCheckpoint) { require.Equal(t, want.Trigger, got.Trigger) require.Equal(t, want.Fail, got.Fail) require.Equal(t, want.SweepAttempts, got.SweepAttempts) + require.Equal( + t, want.ExternalSpendFinalized, got.ExternalSpendFinalized, + ) + require.Equal(t, want.ReliveUnsafe, got.ReliveUnsafe) require.ElementsMatch( t, want.DeferredCheckpoints, got.DeferredCheckpoints, ) @@ -597,6 +695,16 @@ func requireCheckpointEqual(t *testing.T, want, got *actorCheckpoint) { t, txsEqualCk(want.SweepTx, got.SweepTx), "sweep transactions differ", ) + require.Equal( + t, want.ProvisionalExternalSpend.IsSome(), + got.ProvisionalExternalSpend.IsSome(), + ) + if want.ProvisionalExternalSpend.IsSome() { + require.Equal( + t, want.ProvisionalExternalSpend.UnsafeFromSome(), + got.ProvisionalExternalSpend.UnsafeFromSome(), + ) + } } // hashFromByteCk builds a chainhash.Hash with the first byte set to b, useful diff --git a/unroll/state_snapshot.go b/unroll/state_snapshot.go index d0693ba6c..b8e14aad7 100644 --- a/unroll/state_snapshot.go +++ b/unroll/state_snapshot.go @@ -40,20 +40,21 @@ func checkpointFromState(state State, sweepTx *wire.MsgTx) *actorCheckpoint { } checkpoint.Fail = job.FailReason checkpoint.SweepAttempts = job.SweepAttempts + checkpoint.ProvisionalExternalSpend = job.ProvisionalExternalSpend + checkpoint.ExternalSpendFinalized = job.ExternalSpendFinalized + checkpoint.ReliveUnsafe = job.ReliveUnsafe return checkpoint } -// jobHadOnChainFootprint reports whether the job ever published anything -// on-chain. A footprint exists if any proof node confirmed or is still -// in-flight (submitted to txconfirm, so potentially in the mempool), or if -// the sweep advanced past pending. It is false only for a clean failure -// that never broadcast, which is the sole case where the target VTXO is -// safe to roll back to live: any footprint means the unilateral exit has -// begun on-chain and the operator no longer treats the VTXO as live. See -// wavelength#602. +// jobHadOnChainFootprint reports whether the job published anything on-chain +// or observed a finalized external spend. A false result means only that the +// local planner recorded no footprint; the independent ReliveUnsafe guard must +// also be cleared by objective canonical-absence evidence before the VTXO may +// return to live. See wavelength#602. // -// This reflects THIS job's own footprint. An exit driven on-chain by a +// Except for ExternalSpendFinalized, this reflects THIS job's own footprint. +// An exit driven on-chain by a // third party (the operator, or a prior holder of a fraudulently re-spent // VTXO) does not reach the recoverable branch: our submission of the same // proof node is an ignorable "already known" broadcast rather than a hard @@ -66,9 +67,13 @@ func jobHadOnChainFootprint(job *JobState) bool { return false } + sweepPublished := job.PlannerState.Sweep.Status != + unrollplan.SweepStatusPending + return len(job.PlannerState.ConfirmedTxids) > 0 || len(job.PlannerState.InFlightTxids) > 0 || - job.PlannerState.Sweep.Status != unrollplan.SweepStatusPending + sweepPublished || + job.ExternalSpendFinalized } // effectiveSweepTxid returns the durable sweep txid from planner state when @@ -102,14 +107,19 @@ func stateFromCheckpoint(checkpoint *actorCheckpoint) State { deferred := copyDeferredCheckpoints(checkpoint.DeferredCheckpoints) job := &JobState{ - Height: checkpoint.Height, - Trigger: checkpoint.Trigger, - ExitPolicyKind: exitPolicyKind(checkpoint.ExitPolicyKind), - ExitPolicyRef: checkpoint.ExitPolicyRef, - PlannerState: copyPlannerState(checkpoint.State), - DeferredCheckpoints: deferred, - FailReason: checkpoint.Fail, - SweepAttempts: checkpoint.SweepAttempts, + Height: checkpoint.Height, + Trigger: checkpoint.Trigger, + ExitPolicyKind: exitPolicyKind( + checkpoint.ExitPolicyKind, + ), + ExitPolicyRef: checkpoint.ExitPolicyRef, + PlannerState: copyPlannerState(checkpoint.State), + DeferredCheckpoints: deferred, + FailReason: checkpoint.Fail, + SweepAttempts: checkpoint.SweepAttempts, + ProvisionalExternalSpend: checkpoint.ProvisionalExternalSpend, + ExternalSpendFinalized: checkpoint.ExternalSpendFinalized, + ReliveUnsafe: checkpoint.ReliveUnsafe, } switch phaseFromPlannerState(job) { @@ -119,6 +129,9 @@ func stateFromCheckpoint(checkpoint *actorCheckpoint) State { case PhaseFailed: return &Failed{Job: job} + case PhaseExternalSpendObserved: + return &AwaitingExternalSpendFinality{Job: job} + case PhaseSweepConfirmation: return &AwaitingSweepConfirmation{Job: job} @@ -152,6 +165,9 @@ func phaseFromState(state State) Phase { case *AwaitingSweepConfirmation: return PhaseSweepConfirmation + case *AwaitingExternalSpendFinality: + return PhaseExternalSpendObserved + case *Completed: return PhaseCompleted @@ -174,6 +190,19 @@ func phaseFromPlannerState(job *JobState) Phase { return PhaseFailed } + if job.ExternalSpendFinalized { + return PhaseCompleted + } + + // A persisted provisional external spend takes precedence over the + // sweep-based phase derivation: the actor was parked waiting for either + // a reorg (which clears the anchor) or finality (which resolves it as + // permanently consumed). Surfacing this phase keeps restart + // reconciliation and the live reducer aligned on the same state. + if job.ProvisionalExternalSpend.IsSome() { + return PhaseExternalSpendObserved + } + switch { case job.PlannerState.Sweep.Status == unrollplan.SweepStatusConfirmed: return PhaseCompleted @@ -207,6 +236,9 @@ func stateJob(state State) *JobState { case *AwaitingSweepConfirmation: return s.Job.Copy() + case *AwaitingExternalSpendFinality: + return s.Job.Copy() + case *Completed: return s.Job.Copy() diff --git a/unroll/state_snapshot_test.go b/unroll/state_snapshot_test.go index aad87a4f6..004681327 100644 --- a/unroll/state_snapshot_test.go +++ b/unroll/state_snapshot_test.go @@ -87,6 +87,17 @@ func TestCheckpointRoundTripByPhase(t *testing.T) { }, typ: &Completed{}, }, + { + name: "external_spend_completed", + state: &Completed{ + Job: &JobState{ + Height: 106, + Trigger: TriggerRestart, + ExternalSpendFinalized: true, + }, + }, + typ: &Completed{}, + }, { name: "failed", state: &Failed{ @@ -145,6 +156,14 @@ func TestCheckpointRoundTripByPhase(t *testing.T) { t, stateJob(tc.state).DeferredCheckpoints, stateJob(restored).DeferredCheckpoints, ) + require.Equal( + t, stateJob(tc.state).ExternalSpendFinalized, + stateJob(restored).ExternalSpendFinalized, + ) + require.Equal( + t, stateJob(tc.state).ReliveUnsafe, + stateJob(restored).ReliveUnsafe, + ) }) } } diff --git a/vhtlcrecovery/coordinator/service.go b/vhtlcrecovery/coordinator/service.go index 5560e41aa..91cd4502e 100644 --- a/vhtlcrecovery/coordinator/service.go +++ b/vhtlcrecovery/coordinator/service.go @@ -565,7 +565,13 @@ func (s *Service) reconcileLoaded(ctx context.Context, switch status.UnrollPhase { case unroll.PhasePending, unroll.PhaseMaterializing, unroll.PhaseCSVPending, unroll.PhaseSweepBroadcast, - unroll.PhaseSweepConfirmation: + unroll.PhaseSweepConfirmation, + unroll.PhaseExternalSpendObserved: + // PhaseExternalSpendObserved is a parked, non-terminal state: + // the unroll actor observed an unfinalized external spend of + // the target and is holding (a reorg can resurrect it), so the + // recovery job likewise holds its current state until the + // unroll resolves one way or the other. return status, nil case unroll.PhaseCompleted: diff --git a/wallet/boarding_sweep_actor.go b/wallet/boarding_sweep_actor.go index b8c36e340..77363398a 100644 --- a/wallet/boarding_sweep_actor.go +++ b/wallet/boarding_sweep_actor.go @@ -210,19 +210,63 @@ func (m *ReplayPendingIntentsResponse) MessageType() string { func (m *ReplayPendingIntentsResponse) walletRespSealed() {} -// BoardingSweepSpendNotification is a Tell carrying a chainsource spend -// event for a boarding-sweep input. Emitted by the chainsource subscription -// the wallet actor sets up via MapSpendEvent. +// BoardingSweepSpendStatus identifies which point in the chainsource +// spend-watch reorg-aware lifecycle drove this notification. Splitting +// the status out makes it impossible for the handler to confuse a +// reorg-out with a fresh spend: a reorged-out spending block should +// not advance the input's persistent state machine further than the +// canonical chain has. +type BoardingSweepSpendStatus int + +const ( + // BoardingSweepSpendStatusUnknown is the zero value; receiving it + // indicates a programmer error (MapSpendEvent fell through). + BoardingSweepSpendStatusUnknown BoardingSweepSpendStatus = iota + + // BoardingSweepSpendStatusSpent reports that the watched outpoint + // was observed spent on the canonical chain by SpendingTxid at + // SpendingHeight. The observation is provisional until + // BoardingSweepSpendStatusDone arrives — a reorg can roll it back + // via BoardingSweepSpendStatusReorged. + BoardingSweepSpendStatusSpent + + // BoardingSweepSpendStatusReorged reports that a previously + // delivered Spent observation was reorged out. The handler logs + // the divergence and leaves the watch armed; the chainsource + // sub-actor stays alive in reorg-aware mode so a subsequent + // re-spend on the new canonical chain re-fires + // BoardingSweepSpendStatusSpent. + BoardingSweepSpendStatusReorged + + // BoardingSweepSpendStatusDone reports that the spend observation + // is past the chainsource backend's reorg-safety depth and is no + // longer reversible. Synthesized by the spend-actor's + // FinalityDepth synthesizer when the backend (notably lndclient + // over gRPC) does not surface a native Done signal. + BoardingSweepSpendStatusDone +) + +// BoardingSweepSpendNotification is a Tell carrying one event from the +// chainsource spend-watch reorg-aware lifecycle for a boarding-sweep +// input. Emitted by the subscription the wallet actor sets up via +// MapSpendEvent / MapSpendReorgedEvent / MapSpendDoneEvent. type BoardingSweepSpendNotification struct { actor.BaseMessage - // Outpoint is the boarding UTXO that was spent. + // Status identifies which lifecycle event this notification + // carries. See BoardingSweepSpendStatus. + Status BoardingSweepSpendStatus + + // Outpoint is the boarding UTXO whose spend lifecycle changed. Outpoint wire.OutPoint - // SpendingTxid is the transaction that confirmed the spend. + // SpendingTxid is the transaction that spent the outpoint when + // Status=Spent; zero on Reorged / Done (the wire-level event + // carries no spender identity past the first Spent observation). SpendingTxid chainhash.Hash - // SpendingHeight is the block height of the spending transaction. + // SpendingHeight is the block height of the spending transaction + // when Status=Spent; zero on Reorged / Done. SpendingHeight int32 } @@ -257,8 +301,7 @@ const ( // delivered TxConfirmed for this sweep was reorged out. The // handler must NOT call MarkBoardingSweepFailed; instead it // leaves the spend watches and pending state armed so the next - // TxConfirmed on the new canonical chain (or an eventual - // TxFailed) drives the terminal decision. + // TxConfirmed on the new canonical chain can drive finality. BoardingSweepTxStatusReorged // BoardingSweepTxStatusFinalized reports that the sweep @@ -268,9 +311,9 @@ const ( BoardingSweepTxStatusFinalized // BoardingSweepTxStatusFailed reports a terminal failure from - // txconfirm (broadcast rejected, retry budget exhausted, etc.). - // The handler marks the sweep failed in the store and cleans up - // pending state. + // txconfirm. It is terminal for that broadcaster attempt, but is not + // objective evidence that every input is unspent. The wallet therefore + // keeps the sweep reserved for the normal tip-paced resume path. BoardingSweepTxStatusFailed ) @@ -298,8 +341,8 @@ func classifyTxconfirmNotificationForBoardingSweep( case *txconfirm.TxReorged: // Reorg-out: do NOT mark the sweep as failed in the // handler. Leave spend watches and pending state armed so - // the next TxConfirmed on the new canonical chain (or an - // eventual TxFailed) drives the terminal decision. + // the next TxConfirmed on the new canonical chain can drive + // finality. return BoardingSweepTxNotification{ Status: BoardingSweepTxStatusReorged, Txid: ev.Txid, @@ -416,6 +459,12 @@ type pendingSweepState struct { // chainsource spend-watch deregistration. inputs map[wire.OutPoint]string + // provisionalSpends keeps the spender identity that arrives with a + // Spent event until the matching Done event seals it. Reorged removes + // the observation, while restart reconstructs it from the still-pending + // durable row and historical spend watch. + provisionalSpends map[wire.OutPoint]provisionalSweepSpend + // totalAmount is the gross input value of the sweep. totalAmount btcutil.Amount @@ -437,6 +486,13 @@ type pendingSweepState struct { submitted bool } +// provisionalSweepSpend is the reversible positive observation retained until +// chainsource reports that the spend reached policy finality. +type provisionalSweepSpend struct { + txid chainhash.Hash + height int32 +} + // boardingSweepCallerID returns the deterministic chainsource caller-id // used to register or cancel a per-outpoint spend watch. func boardingSweepCallerID(op wire.OutPoint) string { @@ -838,22 +894,19 @@ func (a *Ark) publishBoardingSweep(ctx context.Context, ctx, args.signed.Tx, args.pkScript, uint32(args.bestHeight), ); err != nil { - failErr := a.sweepStore.MarkBoardingSweepFailed( - ctx, pending.txid, err, - ) - if failErr != nil { - log.WarnS(ctx, "Failed to roll back failed sweep", - failErr, slog.String( - "txid", pending.txid.String(), - )) - } - - delete(a.pendingSweeps, pending.txid) - a.cancelSweepSpendWatches(ctx, pending) + // A broadcaster failure proves only that this submission + // attempt failed. It cannot prove all inputs remained unspent, + // especially when a conflicting spend notification may already + // be queued. Keep the durable reservation and watches intact. + // The existing per-tip recovery pass retries at a bounded + // cadence; an immediate self-retry here could spin forever on a + // structurally invalid tx. + pending.submitted = false return fn.Ok[WalletResp]( failedSweepResponse( - fmt.Errorf("submit sweep to broadcaster: %w", + fmt.Errorf("submit sweep to broadcaster: %w; "+ + "inputs remain reserved for recovery", err), args.feeRate, args.confTarget, @@ -923,6 +976,7 @@ func (a *Ark) registerSweepSpendWatch(ctx context.Context, notify := chainsource.MapSpendEvent(a.selfRef, func(ev chainsource.SpendEvent) WalletMsg { return BoardingSweepSpendNotification{ + Status: BoardingSweepSpendStatusSpent, Outpoint: ev.Outpoint, SpendingTxid: ev.SpendingTxid, SpendingHeight: ev.SpendingHeight, @@ -930,16 +984,44 @@ func (a *Ark) registerSweepSpendWatch(ctx context.Context, }, ) + // Reorg-aware lifecycle refs. Without these, a reorged-out + // external spender would leave the input row marked + // external_spent in the persistent store with no rollback path — + // the chainsource SpendActor would either tear the registration + // down (legacy mode) or hold reorg events for nobody (reorg-aware + // mode missing the refs). Wiring them keeps the registration + // alive past the first Spent event, surfaces Reorged on rollback, + // and lets the FinalityDepth synthesizer eventually fire Done so + // the sub-actor exits cleanly. + reorgedNotify := chainsource.MapSpendReorgedEvent(a.selfRef, + func(ev chainsource.SpendReorgedEvent) WalletMsg { + return BoardingSweepSpendNotification{ + Status: BoardingSweepSpendStatusReorged, + Outpoint: ev.Outpoint, + } + }, + ) + doneNotify := chainsource.MapSpendDoneEvent(a.selfRef, + func(ev chainsource.SpendDoneEvent) WalletMsg { + return BoardingSweepSpendNotification{ + Status: BoardingSweepSpendStatusDone, + Outpoint: ev.Outpoint, + } + }, + ) + callerID := boardingSweepCallerID(op) notifyOpt := fn.Some[actor.TellOnlyRef[chainsource.SpendEvent]]( notify, ) req := &chainsource.RegisterSpendRequest{ - CallerID: callerID, - Outpoint: &op, - PkScript: out.PkScript, - HeightHint: heightHint, - NotifyActor: notifyOpt, + CallerID: callerID, + Outpoint: &op, + PkScript: out.PkScript, + HeightHint: heightHint, + NotifyActor: notifyOpt, + NotifyReorged: fn.Some(reorgedNotify), + NotifyDone: fn.Some(doneNotify), } future := a.chainSource.Ask(ctx, req) @@ -965,6 +1047,9 @@ func (a *Ark) cancelSweepSpendWatches(ctx context.Context, cancellations[op] = id } pending.inputs = make(map[wire.OutPoint]string) + pending.provisionalSpends = make( + map[wire.OutPoint]provisionalSweepSpend, + ) for op, callerID := range cancellations { op := op @@ -1069,8 +1154,9 @@ func (a *Ark) submitSweepToConfirm(ctx context.Context, tx *wire.MsgTx, return nil } -// handleSweepSpendNotification updates persistent state when an input of -// an in-flight aggregate sweep is observed spent on chain. +// handleSweepSpendNotification holds a positive spend observation in memory +// until Done seals it. Reorged discards that provisional observation, so the +// durable sweep row never advances on evidence that can still disappear. func (a *Ark) handleSweepSpendNotification(ctx context.Context, notif BoardingSweepSpendNotification) fn.Result[WalletResp] { @@ -1078,109 +1164,169 @@ func (a *Ark) handleSweepSpendNotification(ctx context.Context, return fn.Ok[WalletResp](&BoardingSweepNotificationAck{}) } - var ( - callerID string - pending *pendingSweepState - ) + var pending *pendingSweepState for _, candidate := range a.pendingSweeps { - id, ok := candidate.inputs[notif.Outpoint] + _, ok := candidate.inputs[notif.Outpoint] if !ok { continue } - callerID = id pending = candidate break } - // The sweep's own input-spend notification is only a provisional - // one-confirmation observation. txconfirm owns the reorg-aware watch - // for that transaction and will commit the store transition at - // Finalized. Deferring here prevents a Confirmed -> Reorged -> Failed - // lifecycle from leaving the store and ledger falsely - // terminal-successful. - if pending != nil && notif.SpendingTxid == pending.txid { - a.logger(ctx).DebugS( - ctx, - "Deferring own sweep spend until finality", - slog.String("outpoint", notif.Outpoint.String()), - slog.String("sweep_txid", pending.txid.String()), - ) + switch notif.Status { + case BoardingSweepSpendStatusSpent: + if pending == nil { + return fn.Ok[WalletResp]( + &BoardingSweepNotificationAck{}, + ) + } + if pending.provisionalSpends == nil { + pending.provisionalSpends = make( + map[wire.OutPoint]provisionalSweepSpend, + ) + } + + pending.provisionalSpends[notif.Outpoint] = + provisionalSweepSpend{ + txid: notif.SpendingTxid, + height: notif.SpendingHeight, + } return fn.Ok[WalletResp](&BoardingSweepNotificationAck{}) - } - resolved, err := a.sweepStore.MarkBoardingSweepInputSpent( - ctx, notif.Outpoint, notif.SpendingTxid, notif.SpendingHeight, - ) - switch { - // A duplicate spend event for an input row that has already been - // resolved (e.g. a re-org-then-respend, or a stale buffered event - // arriving after the sweep moved to a terminal state) returns - // ErrNoRows from the store. This is benign — the row is already in - // its target state — so suppress it to Debug rather than alerting on - // every duplicate. - case errors.Is(err, sql.ErrNoRows): - a.logger(ctx).DebugS(ctx, - "Boarding sweep input already resolved; "+ - "ignoring duplicate spend", + case BoardingSweepSpendStatusReorged: + if pending != nil { + delete(pending.provisionalSpends, notif.Outpoint) + } + + a.logger(ctx).WarnS(ctx, "Boarding-sweep input spend "+ + "reorged out; waiting for a canonical re-spend", + nil, slog.String("outpoint", notif.Outpoint.String()), - slog.String( - "spending_txid", notif.SpendingTxid.String(), - )) + ) return fn.Ok[WalletResp](&BoardingSweepNotificationAck{}) - case err != nil: - a.logger(ctx).WarnS(ctx, - "Failed to mark boarding sweep input spent", err, + case BoardingSweepSpendStatusDone: + if pending == nil { + return fn.Ok[WalletResp]( + &BoardingSweepNotificationAck{}, + ) + } + + spend, ok := pending.provisionalSpends[notif.Outpoint] + if !ok { + a.logger(ctx).WarnS(ctx, "Final boarding-sweep spend "+ + "arrived without its positive observation", + nil, + slog.String("outpoint", notif.Outpoint.String()), + ) + a.rearmSweepSpendWatch(ctx, pending, notif.Outpoint) + + return fn.Ok[WalletResp]( + &BoardingSweepNotificationAck{}, + ) + } + + resolved, err := a.sweepStore.MarkBoardingSweepInputSpent( + ctx, notif.Outpoint, spend.txid, spend.height, + ) + + return a.finishSweepSpend( + ctx, pending, notif.Outpoint, spend.txid, resolved, err, + ) + + default: + a.logger(ctx).WarnS(ctx, "Boarding-sweep spend notification "+ + "with unknown status", + fmt.Errorf("status=%d", notif.Status), slog.String("outpoint", notif.Outpoint.String()), - slog.String( - "spending_txid", notif.SpendingTxid.String(), - )) + ) return fn.Ok[WalletResp](&BoardingSweepNotificationAck{}) } +} - if pending != nil { - delete(pending.inputs, notif.Outpoint) - delete(a.pendingSweepInputs, notif.Outpoint) - if resolved { - delete(a.pendingSweeps, pending.txid) - } - } +// finishSweepSpend applies a terminal store result and keeps the durable row +// recoverable when persistence fails. +func (a *Ark) finishSweepSpend(ctx context.Context, pending *pendingSweepState, + outpoint wire.OutPoint, spendingTxid chainhash.Hash, resolved bool, + err error) fn.Result[WalletResp] { - if callerID != "" { - op := notif.Outpoint - if err := a.chainSource.Tell(ctx, - &chainsource.UnregisterSpendRequest{ - CallerID: callerID, - Outpoint: &op, - }); err != nil { + switch { + case errors.Is(err, sql.ErrNoRows): + a.logger(ctx).DebugS( + ctx, + "Boarding sweep input already resolved; ignoring "+ + "duplicate", + slog.String("outpoint", outpoint.String()), + slog.String("spending_txid", spendingTxid.String()), + ) - a.logger(ctx).DebugS( - ctx, - "Best-effort unregister spend failed", - err, - slog.String("outpoint", op.String()), - ) - } + case err != nil: + a.logger(ctx).WarnS( + ctx, + "Failed to finalize boarding sweep input spend", + err, + slog.String("outpoint", outpoint.String()), + slog.String("spending_txid", spendingTxid.String()), + ) + a.rearmSweepSpendWatch(ctx, pending, outpoint) + + return fn.Ok[WalletResp](&BoardingSweepNotificationAck{}) } - if resolved && pending != nil { - // Cancel any straggler spend watches for the same sweep so - // chainsource sub-actors do not leak. + delete(pending.provisionalSpends, outpoint) + delete(pending.inputs, outpoint) + delete(a.pendingSweepInputs, outpoint) + if resolved || len(pending.inputs) == 0 { + delete(a.pendingSweeps, pending.txid) a.cancelSweepSpendWatches(ctx, pending) } return fn.Ok[WalletResp](&BoardingSweepNotificationAck{}) } +// rearmSweepSpendWatch drops stale in-memory watch ownership and asks the +// normal resume path to replay the still-pending durable row. +func (a *Ark) rearmSweepSpendWatch(ctx context.Context, + pending *pendingSweepState, outpoint wire.OutPoint) { + + delete(pending.provisionalSpends, outpoint) + delete(pending.inputs, outpoint) + delete(a.pendingSweepInputs, outpoint) + + a.scheduleBoardingSweepResume(ctx) +} + +// scheduleBoardingSweepResume asks the actor-owned recovery path to reconstruct +// pending watches without inheriting notification-message cancellation. +func (a *Ark) scheduleBoardingSweepResume(ctx context.Context) { + if a.selfRef == nil { + return + } + + // The wallet actor, not the notification message, owns recovery. Keep + // trace values while detaching the re-arm from a caller context that + // may already be canceled as a chainsource sub-actor shuts down. + rearmCtx := context.WithoutCancel(ctx) + err := a.selfRef.Tell(rearmCtx, &ResumeBoardingSweepsRequest{}) + if err != nil { + a.logger(ctx).WarnS( + ctx, + "Failed to schedule boarding sweep spend re-arm", + err, + ) + } +} + // handleSweepTxNotification processes the reorg-aware txconfirm lifecycle for // a tracked aggregate sweep. Confirmed remains provisional. Finalized commits -// the store and ledger transitions and releases the in-memory watches, while -// Failed mirrors a terminal broadcaster failure into the store. +// the store and ledger transitions and releases the in-memory watches. Failed +// ends one broadcaster attempt but keeps the inputs reserved for safe replay. func (a *Ark) handleSweepTxNotification(ctx context.Context, notif BoardingSweepTxNotification) fn.Result[WalletResp] { @@ -1205,16 +1351,6 @@ func (a *Ark) handleSweepTxNotification(ctx context.Context, // alive, so a subsequent TxConfirmed on the new canonical // chain remains provisional and an eventual Finalized commits // the canonical result. - // - // Best-effort backstop only: the per-input chainsource - // spend watch fires handleSweepSpendNotification if some - // other party spends an input. That spend-watch path is - // not itself reorg-symmetric — MapSpendEvent collapses the - // chainsource SpendEvent lifecycle to a single - // BoardingSweepSpendNotification without surfacing - // Reorged / Done — so a reorged-out external spender will - // leave the input row marked external_spent until manual - // reconciliation. Closing that gap is tracked separately. a.logger(ctx).WarnS( ctx, "Boarding sweep confirmation reorged out; waiting "+ @@ -1234,7 +1370,21 @@ func (a *Ark) handleSweepTxNotification(ctx context.Context, slog.String("txid", notif.Txid.String()), slog.Int("block_height", int(notif.BlockHeight)), ) - a.reconcileSweepInputsOnFinalized(ctx, notif) + if !a.reconcileSweepInputsOnFinalized(ctx, notif) { + // The txconfirm lifecycle is terminal, but durable + // input state is not. Re-submit through the normal + // recovery path so historical confirmation replay can + // retry finalization. + pending := a.pendingSweeps[notif.Txid] + if pending != nil { + pending.submitted = false + } + a.scheduleBoardingSweepResume(ctx) + + return fn.Ok[WalletResp]( + &BoardingSweepNotificationAck{}, + ) + } a.emitSweepConfirmedLedger(ctx, notif) pending := a.pendingSweeps[notif.Txid] @@ -1252,21 +1402,31 @@ func (a *Ark) handleSweepTxNotification(ctx context.Context, // would diverge the store from the ledger. If the persisted // record shows the sweep already reached a terminal-success // status, ignore the failure rather than undo a booked sweep. - if rec, ok := a.lookupSweepRecord(ctx, notif.Txid); ok && - isTerminalSuccessSweepStatus(rec.Status) { - - a.logger(ctx).WarnS( - ctx, - "Ignoring boarding sweep failure for an "+ - "already-resolved sweep", - errors.New(notif.Reason), - slog.String("txid", notif.Txid.String()), - slog.String("status", rec.Status), - ) - - return fn.Ok[WalletResp]( - &BoardingSweepNotificationAck{}, - ) + if rec, ok := a.lookupSweepRecord(ctx, notif.Txid); ok { + switch { + case isTerminalSuccessSweepStatus(rec.Status): + a.logger(ctx).WarnS( + ctx, + "Ignoring resolved sweep failure", + errors.New(notif.Reason), + slog.String("txid", + notif.Txid.String()), + slog.String("status", rec.Status), + ) + + return fn.Ok[WalletResp]( + &BoardingSweepNotificationAck{}, + ) + + case rec.Status == BoardingSweepStatusFailed: + // A legacy/pre-submission path already + // completed the failure transition. Do not + // schedule a futile replay for a row the + // pending-sweep query will not return. + return fn.Ok[WalletResp]( + &BoardingSweepNotificationAck{}, + ) + } } a.logger(ctx).WarnS( @@ -1280,23 +1440,13 @@ func (a *Ark) handleSweepTxNotification(ctx context.Context, // operators can alert on a stuck boarding-sweep watcher. a.emitBackgroundTaskError(ctx, "boarding_sweep_watcher") - err := a.sweepStore.MarkBoardingSweepFailed( - ctx, notif.Txid, errors.New(notif.Reason), - ) - if err != nil { - a.logger(ctx).WarnS( - ctx, - "Failed to mark boarding sweep failed", - err, - slog.String("txid", notif.Txid.String()), - ) - } - - pending := a.pendingSweeps[notif.Txid] - delete(a.pendingSweeps, notif.Txid) - - if pending != nil { - a.cancelSweepSpendWatches(ctx, pending) + // The attempt failed, but that is not negative chain evidence + // for any input. Keep the durable reservation and every spend + // watch alive so a queued/conflicting spend can still reach + // Done. The tip-driven resume path creates a fresh txconfirm + // attempt at a bounded cadence until recovery succeeds. + if pending := a.pendingSweeps[notif.Txid]; pending != nil { + pending.submitted = false } default: @@ -1322,22 +1472,47 @@ func (a *Ark) handleSweepTxNotification(ctx context.Context, // The store's status guard rejects a redundant transition with sql.ErrNoRows; // that signals "input row already advanced past pending/published" and is // treated as a benign no-op. Other errors still log at warn because they -// indicate a real persistence problem. +// indicate a real persistence problem. The return value is true only when +// every durable input transition is known to be complete. func (a *Ark) reconcileSweepInputsOnFinalized(ctx context.Context, - notif BoardingSweepTxNotification) { + notif BoardingSweepTxNotification) bool { - pending, ok := a.pendingSweeps[notif.Txid] - if !ok || pending == nil { - return + record, found := a.lookupSweepRecord(ctx, notif.Txid) + if !found { + return false } + if record.Status == BoardingSweepStatusConfirmed { + return true + } + if record.Status != BoardingSweepStatusPending && + record.Status != BoardingSweepStatusPublished { - for op := range pending.inputs { - _, err := a.sweepStore.MarkBoardingSweepInputSpent( - ctx, op, notif.Txid, notif.BlockHeight, + a.logger(ctx).WarnS( + ctx, + "Finalized sweep conflicts with durable status", + nil, + slog.String("txid", notif.Txid.String()), + slog.String("status", record.Status), + ) + + return false + } + + durable := true + resolved := false + for _, input := range record.Inputs { + if input.Status != BoardingSweepInputStatusPending && + input.Status != BoardingSweepInputStatusPublished { + + continue + } + + inputResolved, err := a.sweepStore.MarkBoardingSweepInputSpent( + ctx, input.Outpoint, notif.Txid, notif.BlockHeight, ) switch { case err == nil: - // Success. + resolved = resolved || inputResolved case errors.Is(err, sql.ErrNoRows): // Row already past pending/published — likely @@ -1346,15 +1521,31 @@ func (a *Ark) reconcileSweepInputsOnFinalized(ctx context.Context, // Idempotent no-op. default: + durable = false a.logger(ctx).WarnS( ctx, "Failed to mark sweep input spent on confirm", err, - slog.String("outpoint", op.String()), + slog.String("outpoint", + input.Outpoint.String()), slog.String("txid", notif.Txid.String()), ) } } + + if !durable { + return false + } + if resolved { + return true + } + + // Every candidate raced to a terminal row between the initial read and + // its guarded update. Re-read the aggregate so a concurrent idempotent + // finalization is accepted only if the durable parent is now confirmed. + record, found = a.lookupSweepRecord(ctx, notif.Txid) + + return found && record.Status == BoardingSweepStatusConfirmed } // emitSweepConfirmedLedger emits the double-entry ledger and UTXO audit diff --git a/wallet/boarding_sweep_actor_test.go b/wallet/boarding_sweep_actor_test.go index 6f9d226ac..b76f67ddc 100644 --- a/wallet/boarding_sweep_actor_test.go +++ b/wallet/boarding_sweep_actor_test.go @@ -403,47 +403,55 @@ func TestSweepBoardingUTXOsPreviewBuildsTx(t *testing.T) { ) } -// TestSweepSpendNotificationMarksInputSpent verifies that a chainsource -// spend event for a tracked input is forwarded to the store and the -// in-memory tracking map is cleaned up when the sweep resolves. -func TestSweepSpendNotificationMarksInputSpent(t *testing.T) { +// TestSweepSpendNotificationKeepsPositiveProvisional verifies that a +// chainsource spend event remains reversible until the final Done event. +func TestSweepSpendNotificationKeepsPositiveProvisional(t *testing.T) { t.Parallel() op := wire.OutPoint{Hash: chainhash.Hash{0xab}, Index: 0} spendingTxid := chainhash.Hash{0xcd} store := &MockBoardingSweepStore{} - store.On( - "MarkBoardingSweepInputSpent", mock.Anything, op, - spendingTxid, int32(220), - ).Return(true, nil) a := newSweepTestArk(t, store, nil, 0, 0) - // Pre-populate pending sweep state so the handler clears it on - // resolution. + // Pre-populate pending sweep state. The positive observation must not + // clear it or touch durable state before the finality boundary. pendingTxid := chainhash.Hash{0xee} - a.pendingSweeps[pendingTxid] = &pendingSweepState{ + pending := &pendingSweepState{ txid: pendingTxid, inputs: map[wire.OutPoint]string{ op: boardingSweepCallerID(op), }, + provisionalSpends: map[wire.OutPoint]provisionalSweepSpend{ + op: { + txid: chainhash.Hash{ + 0xb3, + }, + height: 120, + }, + }, } + a.pendingSweeps[pendingTxid] = pending + a.pendingSweepInputs[op] = pendingTxid result := a.handleSweepSpendNotification( t.Context(), BoardingSweepSpendNotification{ + Status: BoardingSweepSpendStatusSpent, Outpoint: op, SpendingTxid: spendingTxid, SpendingHeight: 220, }, ) require.True(t, result.IsOk()) - require.Empty( - t, a.pendingSweeps, - "resolved sweep must be evicted from in-memory tracking", + require.Same(t, pending, a.pendingSweeps[pendingTxid]) + require.Equal(t, pendingTxid, a.pendingSweepInputs[op]) + require.Equal( + t, provisionalSweepSpend{ + txid: spendingTxid, height: 220, + }, pending.provisionalSpends[op], ) - - store.AssertExpectations(t) + store.AssertNotCalled(t, "MarkBoardingSweepInputSpent") } // TestSweepSpendNotificationDefersOwnSweepUntilFinality verifies that the @@ -468,6 +476,7 @@ func TestSweepSpendNotificationDefersOwnSweepUntilFinality(t *testing.T) { result := a.handleSweepSpendNotification( t.Context(), BoardingSweepSpendNotification{ + Status: BoardingSweepSpendStatusSpent, Outpoint: op, SpendingTxid: sweepTxid, SpendingHeight: 220, @@ -476,6 +485,11 @@ func TestSweepSpendNotificationDefersOwnSweepUntilFinality(t *testing.T) { require.True(t, result.IsOk()) require.Same(t, pending, a.pendingSweeps[sweepTxid]) require.Equal(t, sweepTxid, a.pendingSweepInputs[op]) + require.Equal( + t, provisionalSweepSpend{ + txid: sweepTxid, height: 220, + }, pending.provisionalSpends[op], + ) store.AssertNotCalled(t, "MarkBoardingSweepInputSpent") } @@ -983,30 +997,28 @@ func TestSweepTxNotificationMissingTxSkipsLegs(t *testing.T) { ) } -// TestSweepTxNotificationFailedMarksFailed verifies that a terminal -// txconfirm failure is mirrored into the persistent store and the -// in-memory pending state is dropped. -func TestSweepTxNotificationFailedMarksFailed(t *testing.T) { +// TestSweepTxNotificationFailedKeepsInputsReserved verifies that a terminal +// txconfirm attempt failure does not masquerade as proof that every input is +// unspent. Recovery retains the watches and schedules a fresh attempt. +func TestSweepTxNotificationFailedKeepsInputsReserved(t *testing.T) { t.Parallel() failedTxid := chainhash.Hash{0x99} store := &MockBoardingSweepStore{} - store.On( - "MarkBoardingSweepFailed", mock.Anything, failedTxid, - mock.Anything, - ).Return(nil) - // The Failed arm looks the sweep up first to ignore failures for an - // already-resolved sweep; absent record means the failure proceeds. + // The persisted lookup is absent in this actor-only test. In production + // the in-memory pending entry corresponds to the pending durable row. store.On( "GetBoardingSweep", mock.Anything, failedTxid, ).Return(nil, nil) a := newSweepTestArk(t, store, nil, 0, 0) - a.pendingSweeps[failedTxid] = &pendingSweepState{ - txid: failedTxid, - inputs: map[wire.OutPoint]string{}, + pending := &pendingSweepState{ + txid: failedTxid, + submitted: true, + inputs: map[wire.OutPoint]string{}, } + a.pendingSweeps[failedTxid] = pending result := a.handleSweepTxNotification( t.Context(), BoardingSweepTxNotification{ @@ -1016,8 +1028,9 @@ func TestSweepTxNotificationFailedMarksFailed(t *testing.T) { }, ) require.True(t, result.IsOk()) - require.Empty(t, a.pendingSweeps) - + require.Same(t, pending, a.pendingSweeps[failedTxid]) + require.False(t, pending.submitted) + store.AssertNotCalled(t, "MarkBoardingSweepFailed") store.AssertExpectations(t) } @@ -1079,6 +1092,17 @@ func TestSweepTxNotificationFinalizedCommitsAndCleansUp(t *testing.T) { "MarkBoardingSweepInputSpent", mock.Anything, op, finalizedTxid, int32(800_750), ).Return(true, nil) + store.On( + "GetBoardingSweep", mock.Anything, finalizedTxid, + ).Return(&BoardingSweepRecord{ + Txid: finalizedTxid, + Status: BoardingSweepStatusPublished, + Inputs: []BoardingSweepInputRecord{{ + Txid: finalizedTxid, + Outpoint: op, + Status: BoardingSweepInputStatusPublished, + }}, + }, nil) a := newSweepTestArk(t, store, nil, 0, 0) pending := &pendingSweepState{ @@ -1095,7 +1119,7 @@ func TestSweepTxNotificationFinalizedCommitsAndCleansUp(t *testing.T) { Status: BoardingSweepTxStatusFinalized, Txid: finalizedTxid, BlockHeight: 800_750, - NumConfs: 6, + NumConfs: 31, }, ) require.True(t, result.IsOk()) @@ -1106,6 +1130,136 @@ func TestSweepTxNotificationFinalizedCommitsAndCleansUp(t *testing.T) { store.AssertExpectations(t) } +// TestSweepTxNotificationFinalizedPersistenceFailureRearms verifies that a +// terminal chain signal cannot release watches or run later effects before all +// input transitions are durable. +func TestSweepTxNotificationFinalizedPersistenceFailureRearms(t *testing.T) { + t.Parallel() + + finalizedTxid := chainhash.Hash{0xa9} + op := wire.OutPoint{Hash: chainhash.Hash{0xb9}, Index: 0} + store := &MockBoardingSweepStore{} + store.On( + "MarkBoardingSweepInputSpent", mock.Anything, op, + finalizedTxid, int32(800_760), + ).Return(false, errors.New("database unavailable")) + store.On( + "GetBoardingSweep", mock.Anything, finalizedTxid, + ).Return(&BoardingSweepRecord{ + Txid: finalizedTxid, + Status: BoardingSweepStatusPublished, + Inputs: []BoardingSweepInputRecord{{ + Txid: finalizedTxid, + Outpoint: op, + Status: BoardingSweepInputStatusPublished, + }}, + }, nil) + + a := newSweepTestArk(t, store, nil, 0, 0) + selfRef := actor.NewChannelTellOnlyRef[WalletMsg]("wallet-self", 1) + a.selfRef = selfRef + sink, drain := newCapturingLedgerSink(t) + a.ledgerSink = fn.Some(sink) + pending := &pendingSweepState{ + txid: finalizedTxid, + submitted: true, + inputs: map[wire.OutPoint]string{ + op: boardingSweepCallerID(op), + }, + } + a.pendingSweeps[finalizedTxid] = pending + a.pendingSweepInputs[op] = finalizedTxid + + ctx, cancel := context.WithCancel(t.Context()) + cancel() + result := a.handleSweepTxNotification( + ctx, BoardingSweepTxNotification{ + Status: BoardingSweepTxStatusFinalized, + Txid: finalizedTxid, + BlockHeight: 800_760, + NumConfs: 31, + }, + ) + require.True(t, result.IsOk()) + require.Same(t, pending, a.pendingSweeps[finalizedTxid]) + require.Equal(t, finalizedTxid, a.pendingSweepInputs[op]) + require.False(t, pending.submitted) + + msg, ok := selfRef.AwaitMessage(time.Second) + require.True( + t, ok, "failed durable finalization must schedule recovery", + ) + _, ok = msg.(*ResumeBoardingSweepsRequest) + require.True(t, ok) + require.Empty( + t, drain(0), + "ledger must not run before all input transitions are durable", + ) + store.AssertExpectations(t) +} + +// TestSweepTxNotificationFinalizedCoversUnwatchedInputs verifies that the +// terminal backstop reads the complete durable input set instead of trusting +// the in-memory watch map. A registration failure must not leave one consumed +// boarding input available after the sweep itself reaches finality. +func TestSweepTxNotificationFinalizedCoversUnwatchedInputs(t *testing.T) { + t.Parallel() + + finalizedTxid := chainhash.Hash{0xaa} + armed := wire.OutPoint{Hash: chainhash.Hash{0xba}, Index: 0} + unwatched := wire.OutPoint{Hash: chainhash.Hash{0xbb}, Index: 1} + store := &MockBoardingSweepStore{} + store.On( + "GetBoardingSweep", mock.Anything, finalizedTxid, + ).Return(&BoardingSweepRecord{ + Txid: finalizedTxid, + Status: BoardingSweepStatusPublished, + Inputs: []BoardingSweepInputRecord{ + { + Txid: finalizedTxid, + Outpoint: armed, + Status: BoardingSweepInputStatusPublished, + }, + { + Txid: finalizedTxid, + Outpoint: unwatched, + Status: BoardingSweepInputStatusPublished, + }, + }, + }, nil) + store.On( + "MarkBoardingSweepInputSpent", mock.Anything, armed, + finalizedTxid, int32(800_770), + ).Return(false, nil) + store.On( + "MarkBoardingSweepInputSpent", mock.Anything, unwatched, + finalizedTxid, int32(800_770), + ).Return(true, nil) + + a := newSweepTestArk(t, store, nil, 0, 0) + pending := &pendingSweepState{ + txid: finalizedTxid, + inputs: map[wire.OutPoint]string{ + armed: boardingSweepCallerID(armed), + }, + } + a.pendingSweeps[finalizedTxid] = pending + a.pendingSweepInputs[armed] = finalizedTxid + + result := a.handleSweepTxNotification( + t.Context(), BoardingSweepTxNotification{ + Status: BoardingSweepTxStatusFinalized, + Txid: finalizedTxid, + BlockHeight: 800_770, + NumConfs: 31, + }, + ) + require.True(t, result.IsOk()) + require.Empty(t, a.pendingSweeps) + require.Empty(t, a.pendingSweepInputs) + store.AssertExpectations(t) +} + // TestSweepTxNotificationConfirmedRemainsProvisional verifies that the first // confirmation cannot mutate durable sweep success or release recovery state. func TestSweepTxNotificationConfirmedRemainsProvisional(t *testing.T) { @@ -1162,33 +1316,26 @@ func TestSweepTxNotificationReorgedAfterPendingCleared(t *testing.T) { store.AssertExpectations(t) } -// TestSweepTxNotificationFailedAfterReorgedStillTerminates verifies -// the Reorged arm does NOT suppress a subsequent terminal Failed -// notification: a sequence of Reorged then Failed must still call -// MarkBoardingSweepFailed and drop the pending entry, otherwise a -// reorg followed by a hard broadcast failure would be silently -// stranded. -func TestSweepTxNotificationFailedAfterReorgedStillTerminates(t *testing.T) { +// TestSweepTxNotificationFailedAfterReorgedStaysReserved verifies a failed +// re-confirmation attempt after a reorg cannot restore inputs whose canonical +// spend state is still uncertain. +func TestSweepTxNotificationFailedAfterReorgedStaysReserved(t *testing.T) { t.Parallel() txid := chainhash.Hash{0xa4} store := &MockBoardingSweepStore{} - store.On( - "MarkBoardingSweepFailed", mock.Anything, txid, - mock.Anything, - ).Return(nil) - // The Failed arm now looks the sweep up first to ignore a spurious - // failure for an already-resolved sweep. Here the record is absent - // (not terminal-success), so the failure path proceeds as before. store.On( "GetBoardingSweep", mock.Anything, txid, ).Return(nil, nil) a := newSweepTestArk(t, store, nil, 0, 0) + selfRef := actor.NewChannelTellOnlyRef[WalletMsg]("wallet-self", 1) + a.selfRef = selfRef pending := &pendingSweepState{ - txid: txid, - inputs: map[wire.OutPoint]string{}, + txid: txid, + submitted: true, + inputs: map[wire.OutPoint]string{}, } a.pendingSweeps[txid] = pending @@ -1203,8 +1350,8 @@ func TestSweepTxNotificationFailedAfterReorgedStillTerminates(t *testing.T) { require.True(t, reorgResult.IsOk()) require.Same(t, pending, a.pendingSweeps[txid]) - // Step 2: Failed — terminal path must still fire even though we - // passed through Reorged first. + // Step 2: Failed — the broadcaster attempt ends, but the durable + // reservation and spend-watch ownership remain fail-closed. failResult := a.handleSweepTxNotification( t.Context(), BoardingSweepTxNotification{ Status: BoardingSweepTxStatusFailed, @@ -1213,11 +1360,9 @@ func TestSweepTxNotificationFailedAfterReorgedStillTerminates(t *testing.T) { }, ) require.True(t, failResult.IsOk()) - require.Empty( - t, a.pendingSweeps, - "Failed after Reorged must still tear down pendingSweeps", - ) - + require.Same(t, pending, a.pendingSweeps[txid]) + require.False(t, pending.submitted) + store.AssertNotCalled(t, "MarkBoardingSweepFailed") store.AssertExpectations(t) } @@ -1291,3 +1436,238 @@ func TestSweepTxNotificationUnknownStatusIsBenign(t *testing.T) { store.AssertExpectations(t) } + +// TestSweepSpendNotificationReorgedDoesNotMarkSpent verifies that a +// chainsource SpendReorgedEvent on a watched boarding-sweep input is +// classified as BoardingSweepSpendStatusReorged and processed by the +// Reorged arm of handleSweepSpendNotification — specifically, the +// store's MarkBoardingSweepInputSpent must NOT be called (would +// double-spend the row's state machine) and pendingSweeps tracking +// must be left intact (the chainsource sub-actor stays alive in +// reorg-aware mode; a follow-up Spent on the canonical chain will +// drive the row through the existing happy path). +func TestSweepSpendNotificationReorgedDoesNotMarkSpent(t *testing.T) { + t.Parallel() + + op := wire.OutPoint{Hash: chainhash.Hash{0xb1}, Index: 0} + + store := &MockBoardingSweepStore{} + // CRITICAL: MarkBoardingSweepInputSpent must NOT be called on + // reorg. The testify/mock will fail the test if any unexpected + // method is invoked, so we simply do not register + // MarkBoardingSweepInputSpent here. + + a := newSweepTestArk(t, store, nil, 0, 0) + + pendingTxid := chainhash.Hash{0xb2} + pending := &pendingSweepState{ + txid: pendingTxid, + inputs: map[wire.OutPoint]string{ + op: boardingSweepCallerID(op), + }, + } + a.pendingSweeps[pendingTxid] = pending + a.pendingSweepInputs[op] = pendingTxid + + result := a.handleSweepSpendNotification( + t.Context(), BoardingSweepSpendNotification{ + Status: BoardingSweepSpendStatusReorged, + Outpoint: op, + }, + ) + require.True(t, result.IsOk()) + + // Reorg must leave both in-memory tracking maps alone — the + // sub-actor stays alive in reorg-aware mode and a follow-up + // Spent event on the canonical chain will re-drive the row. + require.Same( + t, pending, a.pendingSweeps[pendingTxid], + "reorg must not evict pending sweep tracking", + ) + require.Equal( + t, pendingTxid, a.pendingSweepInputs[op], + "reorg must not evict per-input back-pointer", + ) + _, provisional := pending.provisionalSpends[op] + require.False( + t, provisional, + "reorg must discard the non-canonical positive observation", + ) + + store.AssertExpectations(t) +} + +// TestSweepSpendNotificationDoneCommitsPositive verifies that Done seals the +// matching positive observation in the store and releases in-memory tracking. +func TestSweepSpendNotificationDoneCommitsPositive(t *testing.T) { + t.Parallel() + + op := wire.OutPoint{Hash: chainhash.Hash{0xc1}, Index: 0} + spendingTxid := chainhash.Hash{0xc3} + + store := &MockBoardingSweepStore{} + store.On( + "MarkBoardingSweepInputSpent", mock.Anything, op, + spendingTxid, int32(140), + ).Return(true, nil) + + a := newSweepTestArk(t, store, nil, 0, 0) + + pendingTxid := chainhash.Hash{0xc2} + pending := &pendingSweepState{ + txid: pendingTxid, + inputs: map[wire.OutPoint]string{ + op: boardingSweepCallerID(op), + }, + provisionalSpends: map[wire.OutPoint]provisionalSweepSpend{ + op: { + txid: spendingTxid, + height: 140, + }, + }, + } + a.pendingSweeps[pendingTxid] = pending + a.pendingSweepInputs[op] = pendingTxid + + result := a.handleSweepSpendNotification( + t.Context(), BoardingSweepSpendNotification{ + Status: BoardingSweepSpendStatusDone, + Outpoint: op, + }, + ) + require.True(t, result.IsOk()) + + require.Empty(t, a.pendingSweeps) + require.Empty(t, a.pendingSweepInputs) + require.Empty(t, pending.provisionalSpends) + + store.AssertExpectations(t) +} + +// TestSweepSpendNotificationPersistenceFailureRearms verifies that a final +// spend is never forgotten when the terminal store write fails. Recovery is +// owned by the wallet actor and therefore survives cancellation of the +// notification context. +func TestSweepSpendNotificationPersistenceFailureRearms(t *testing.T) { + t.Parallel() + + op := wire.OutPoint{Hash: chainhash.Hash{0xc4}, Index: 1} + spendingTxid := chainhash.Hash{0xc5} + pendingTxid := chainhash.Hash{0xc6} + store := &MockBoardingSweepStore{} + store.On( + "MarkBoardingSweepInputSpent", mock.Anything, op, + spendingTxid, int32(150), + ).Return(false, errors.New("database unavailable")) + + a := newSweepTestArk(t, store, nil, 0, 0) + selfRef := actor.NewChannelTellOnlyRef[WalletMsg]("wallet-self", 1) + a.selfRef = selfRef + pending := &pendingSweepState{ + txid: pendingTxid, + inputs: map[wire.OutPoint]string{ + op: boardingSweepCallerID(op), + }, + provisionalSpends: map[wire.OutPoint]provisionalSweepSpend{ + op: { + txid: spendingTxid, + height: 150, + }, + }, + } + a.pendingSweeps[pendingTxid] = pending + a.pendingSweepInputs[op] = pendingTxid + + ctx, cancel := context.WithCancel(t.Context()) + cancel() + result := a.handleSweepSpendNotification( + ctx, BoardingSweepSpendNotification{ + Status: BoardingSweepSpendStatusDone, + Outpoint: op, + }, + ) + require.True(t, result.IsOk()) + require.Same(t, pending, a.pendingSweeps[pendingTxid]) + require.NotContains(t, pending.inputs, op) + require.NotContains(t, a.pendingSweepInputs, op) + require.NotContains(t, pending.provisionalSpends, op) + + msg, ok := selfRef.AwaitMessage(time.Second) + require.True( + t, ok, "canceled notification must still schedule recovery", + ) + _, ok = msg.(*ResumeBoardingSweepsRequest) + require.True(t, ok) + store.AssertExpectations(t) +} + +// TestSweepSpendNotificationDoneWithoutPositiveRearms verifies that an +// impossible Done-first ordering fails closed and reconstructs the watch from +// the still-pending durable sweep rather than inventing spender evidence. +func TestSweepSpendNotificationDoneWithoutPositiveRearms(t *testing.T) { + t.Parallel() + + op := wire.OutPoint{Hash: chainhash.Hash{0xc7}, Index: 2} + pendingTxid := chainhash.Hash{0xc8} + store := &MockBoardingSweepStore{} + a := newSweepTestArk(t, store, nil, 0, 0) + selfRef := actor.NewChannelTellOnlyRef[WalletMsg]("wallet-self", 1) + a.selfRef = selfRef + pending := &pendingSweepState{ + txid: pendingTxid, + inputs: map[wire.OutPoint]string{ + op: boardingSweepCallerID(op), + }, + } + a.pendingSweeps[pendingTxid] = pending + a.pendingSweepInputs[op] = pendingTxid + + result := a.handleSweepSpendNotification( + t.Context(), BoardingSweepSpendNotification{ + Status: BoardingSweepSpendStatusDone, + Outpoint: op, + }, + ) + require.True(t, result.IsOk()) + store.AssertNotCalled(t, "MarkBoardingSweepInputSpent") + + msg, ok := selfRef.AwaitMessage(time.Second) + require.True(t, ok) + _, ok = msg.(*ResumeBoardingSweepsRequest) + require.True(t, ok) +} + +// TestSweepSpendNotificationUnknownStatusIsBenign verifies the default +// arm of handleSweepSpendNotification handles an unrecognised status +// without touching the store. Guards against a future chainsource +// lifecycle addition being delivered without a matching MapSpendEvent +// classification arm. +func TestSweepSpendNotificationUnknownStatusIsBenign(t *testing.T) { + t.Parallel() + + op := wire.OutPoint{Hash: chainhash.Hash{0xd1}, Index: 0} + store := &MockBoardingSweepStore{} + // No expectations — unknown must touch nothing. + + a := newSweepTestArk(t, store, nil, 0, 0) + pendingTxid := chainhash.Hash{0xd2} + pending := &pendingSweepState{ + txid: pendingTxid, + inputs: map[wire.OutPoint]string{ + op: boardingSweepCallerID(op), + }, + } + a.pendingSweeps[pendingTxid] = pending + a.pendingSweepInputs[op] = pendingTxid + + result := a.handleSweepSpendNotification( + t.Context(), BoardingSweepSpendNotification{ + Status: BoardingSweepSpendStatusUnknown, + Outpoint: op, + }, + ) + require.True(t, result.IsOk()) + require.Same(t, pending, a.pendingSweeps[pendingTxid]) + + store.AssertExpectations(t) +} diff --git a/waved/config.go b/waved/config.go index d98a2671e..a80d572c4 100644 --- a/waved/config.go +++ b/waved/config.go @@ -176,6 +176,16 @@ const ( // creating an excessive number of simultaneous cryptographic jobs. MaxSigningWorkers = 64 + // DefaultReorgSafetyDepth is the deepest chain replacement the daemon + // keeps provisional and promises to recover from once reorg-safe v2 is + // enabled. Terminal finality begins one confirmation later. + DefaultReorgSafetyDepth uint32 = 30 + + // MaxReorgSafetyDepth is the largest policy horizon supported by every + // current chain backend. LND's notifier retains reversible watches for + // at most 144 disconnected blocks. + MaxReorgSafetyDepth uint32 = 144 + // DefaultWalletType is the default wallet backend. The "lwwallet" // backend uses an in-process lightweight wallet backed by // btcwallet and Esplora, requiring no external lnd node. @@ -338,6 +348,13 @@ type Config struct { // original serial behavior. SigningWorkers int `mapstructure:"signingworkers"` + // ReorgSafetyDepth is the deepest chain replacement for which evidence + // remains provisional and recoverable. Terminal finality begins at + // ReorgSafetyDepth+1 confirmations. Zero uses + // DefaultReorgSafetyDepth. This policy is not advertised until the full + // v2 safety capability and depth negotiation are enabled. + ReorgSafetyDepth uint32 `mapstructure:"reorgsafetydepth"` + // RegistrationTimeout is the maximum wall-clock duration to // wait for the server's RoundJoined admission watermark after // sending a JoinRoundRequest. If zero, the round package @@ -524,6 +541,14 @@ type UnrollConfig struct { // MaxFeeRateSatPerVByte caps fee estimates to prevent runaway // fees. Zero uses the default of 100 sat/vB. MaxFeeRateSatPerVByte int64 `mapstructure:"maxfeeratesatpervbyte"` + + // ReconcileProbeTimeoutSec bounds each per-anchor restart- + // reconciliation probe issued by the chainsource-backed + // ChainReconciler (in seconds). A probe that times out leaves the + // durable checkpoint unchanged and fails closed for a later retry; + // operators running against a slow backend can raise this budget. + // Zero uses the reconciler's internal default (10s). + ReconcileProbeTimeoutSec int64 `mapstructure:"reconcileprobetimeoutsec"` } // FeeEstimationConfig groups optional external chain fee providers used by the @@ -1184,6 +1209,7 @@ func DefaultConfig() *Config { }, MaxOperatorFeeSat: DefaultMaxOperatorFeeSat, SigningWorkers: DefaultSigningWorkers, + ReorgSafetyDepth: DefaultReorgSafetyDepth, OOR: defaultOORConfig(), FeeEstimation: &FeeEstimationConfig{ MempoolSpace: &MempoolSpaceFeeConfig{}, @@ -1243,6 +1269,10 @@ func (c *Config) Validate() error { return fmt.Errorf("signingworkers exceeds maximum %d: got %d", MaxSigningWorkers, c.SigningWorkers) } + if c.ReorgSafetyDepth > MaxReorgSafetyDepth { + return fmt.Errorf("reorgsafetydepth exceeds maximum %d: got %d", + MaxReorgSafetyDepth, c.ReorgSafetyDepth) + } if c.OOR == nil { c.OOR = defaultOORConfig() @@ -1411,6 +1441,21 @@ func (c *Config) validateWalletConfig() error { return nil } +// reorgSafetyDepth resolves the daemon's configured recovery horizon. +func (c *Config) reorgSafetyDepth() uint32 { + if c.ReorgSafetyDepth == 0 { + return DefaultReorgSafetyDepth + } + + return c.ReorgSafetyDepth +} + +// chainFinalityDepth returns the first inclusive confirmation depth beyond +// the configured recovery horizon. +func (c *Config) chainFinalityDepth() uint32 { + return c.reorgSafetyDepth() + 1 +} + // transportEndpoints holds the gRPC and REST addresses for one service. type transportEndpoints struct { grpc string diff --git a/waved/config_reorg_safety_test.go b/waved/config_reorg_safety_test.go new file mode 100644 index 000000000..c78e196ff --- /dev/null +++ b/waved/config_reorg_safety_test.go @@ -0,0 +1,68 @@ +package waved + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +// TestReorgSafetyDepthResolution pins the inclusive policy boundary and its +// zero-value default. +func TestReorgSafetyDepthResolution(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + configured uint32 + wantHorizon uint32 + wantFinality uint32 + }{ + { + name: "zero uses default", + wantHorizon: DefaultReorgSafetyDepth, + wantFinality: DefaultReorgSafetyDepth + 1, + }, + { + name: "custom", + configured: 42, + wantHorizon: 42, + wantFinality: 43, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + cfg := &Config{ReorgSafetyDepth: tc.configured} + require.Equal(t, tc.wantHorizon, cfg.reorgSafetyDepth()) + require.Equal( + t, tc.wantFinality, cfg.chainFinalityDepth(), + ) + }) + } +} + +// TestConfigValidateRejectsUnsupportedReorgSafetyDepth verifies that policy +// cannot outlive the shortest current backend observation horizon. +func TestConfigValidateRejectsUnsupportedReorgSafetyDepth(t *testing.T) { + t.Parallel() + + cfg := DefaultConfig() + cfg.Network = "regtest" + cfg.Server.Host = "127.0.0.1:10010" + cfg.Wallet.EsploraURL = "http://127.0.0.1:3000" + cfg.ReorgSafetyDepth = MaxReorgSafetyDepth + 1 + + err := cfg.Validate() + require.ErrorContains(t, err, "reorgsafetydepth exceeds maximum") +} + +// TestDefaultConfigHasReorgSafetyDepth locks in a non-zero policy default. +func TestDefaultConfigHasReorgSafetyDepth(t *testing.T) { + t.Parallel() + + cfg := DefaultConfig() + require.Equal(t, DefaultReorgSafetyDepth, cfg.ReorgSafetyDepth) + require.Equal(t, DefaultReorgSafetyDepth+1, cfg.chainFinalityDepth()) +} diff --git a/waved/operator_negotiation.go b/waved/operator_negotiation.go index a623412c4..5430a7c9c 100644 --- a/waved/operator_negotiation.go +++ b/waved/operator_negotiation.go @@ -75,8 +75,8 @@ func operatorTermsFromResponse(resp *arkrpc.GetInfoResponse) ( } // clientSupportedArkVersions returns the Ark protocol versions this client -// advertises during bootstrap, ordered by preference. Production supports only -// v1; no production default advertises a higher version. +// advertises during bootstrap, ordered by preference. V2 remains deliberately +// absent until the complete one-confirmation safety stack is enabled. func clientSupportedArkVersions() []uint32 { return []uint32{arkrpc.ArkProtocolVersionV1} } diff --git a/waved/operator_negotiation_test.go b/waved/operator_negotiation_test.go index 9439fd4e7..ba71ba643 100644 --- a/waved/operator_negotiation_test.go +++ b/waved/operator_negotiation_test.go @@ -153,6 +153,16 @@ func TestOperatorTermsFreeRefreshWindow(t *testing.T) { require.Equal(t, uint32(72), terms.FreeRefreshWindowBlocks) } +// TestClientSupportedArkVersionsKeepsV2Disabled verifies that merely defining +// the reorg-safety compatibility boundary cannot negotiate it prematurely. +func TestClientSupportedArkVersionsKeepsV2Disabled(t *testing.T) { + t.Parallel() + + versions := clientSupportedArkVersions() + require.Equal(t, []uint32{arkrpc.ArkProtocolVersionV1}, versions) + require.NotContains(t, versions, arkrpc.ArkProtocolVersionV2) +} + // TestNegotiateArkBootstrapZeroSelection proves the client refuses to bootstrap // when the operator returns a zero selection (no common version, or a // pre-versioning server). There is no legacy fallback. diff --git a/waved/rpc_server.go b/waved/rpc_server.go index 5bae39408..59f517eb5 100644 --- a/waved/rpc_server.go +++ b/waved/rpc_server.go @@ -5610,6 +5610,19 @@ func unrollPhaseToProto(phase unroll.Phase) waverpc.UnrollJobStatus { case unroll.PhaseFailed: return waverpc.UnrollJobStatus_UNROLL_JOB_STATUS_FAILED + case unroll.PhaseMaterializing: + return waverpc.UnrollJobStatus_UNROLL_JOB_STATUS_MATERIALIZING + + // PhaseExternalSpendObserved is deliberately reported as + // MATERIALIZING: it is a transient, reorg-reversible parked phase (an + // external spend was seen but has not finalized) with no dedicated + // proto status, and the true phase is always reconstructed from the + // durable checkpoint on restore. The collapse only coarsens the + // operator-facing status during the parked window; add a dedicated + // proto value if that window needs to be independently observable. + case unroll.PhaseExternalSpendObserved: + return waverpc.UnrollJobStatus_UNROLL_JOB_STATUS_MATERIALIZING + default: return waverpc.UnrollJobStatus_UNROLL_JOB_STATUS_MATERIALIZING } diff --git a/waved/server.go b/waved/server.go index f1123770f..e02609fb8 100644 --- a/waved/server.go +++ b/waved/server.go @@ -2259,13 +2259,13 @@ func (s *Server) registerChainSourceActor( chainsource.ChainSourceConfig{ Backend: s.chainBackend, System: s.actorSystem, - // Enable height-based Done synthesis at the default - // safety depth. Without this, conf/spend sub-actors - // driven through lndclient (whose Done channel is - // allocated-but-never-written) would never receive a - // finality signal, and reorg-aware consumers like - // txconfirm would leak watch state forever. - FinalityDepth: chainsource.DefaultFinalityDepth, + // Enable height-based Done synthesis just beyond the + // configured recovery horizon. Without this, conf/spend + // sub-actors driven through lndclient (whose Done + // channel is allocated-but-never-written) would never + // receive a finality signal, and reorg-aware consumers + // like txconfirm would leak watch state forever. + FinalityDepth: s.cfg.chainFinalityDepth(), }, ) @@ -5671,6 +5671,35 @@ func (s *Server) initUnrollSubsystem(ctx context.Context, exitObserver = fn.Some[actor.TellOnlyRef[vtxo.ManagerMsg]](ref) }) + // Build a chainsource-backed reconciler factory so every per-target + // actor re-observes persisted positive anchors before restart. The + // current subscription API cannot prove absence, so silence leaves the + // checkpoint unchanged and fails closed; it is never treated as a + // reorg. See unroll/reconcile.go for the safety rationale. + // + // The factory bakes the target outpoint into the chainsource + // caller-ID prefix so two actors that reconcile the same + // shared proof-graph ancestor concurrently land on distinct + // service keys (chainsource keys on (CallerID, Txid, PkScript, + // TargetConfs)); a static prefix would collide. + reconcileLog := s.subLogger("UREC") + probeTimeout := s.unrollReconcileProbeTimeout() + reconcilerFactory := func(target wire.OutPoint, + proof *recovery.Proof) unroll.ChainReconciler { + + return unroll.NewChainSourceReconciler( + unroll.ChainSourceReconcilerConfig{ + ChainSource: chainSourceRef, + Proof: proof, + CallerID: fmt.Sprintf( + "unroll-reconcile-%s", target, + ), + ProbeTimeout: probeTimeout, + Log: fn.Some(reconcileLog), + }, + ) + } + registry := unroll.NewUnrollRegistryActor(unroll.RegistryConfig{ Store: &unroll.DBRegistryStore{ UEStore: ueStore, @@ -5691,6 +5720,9 @@ func (s *Server) initUnrollSubsystem(ctx context.Context, Preimage: preimages, }, VTXOExitObserver: exitObserver, + ChainReconcilerFactory: fn.Some( + unroll.ChainReconcilerFactory(reconcilerFactory), + ), }) s.unrollRegistry = registry s.unrollRegistryRef = fn.Some(registry.Ref()) @@ -6079,9 +6111,10 @@ func (f *faultyUnrollTxConfirm) Ask(ctx context.Context, promise.Complete( fn.Ok[txconfirm.Resp]( &txconfirm.EnsureConfirmedResp{ - Txid: req.Tx.TxHash(), - State: txconfirm.TxStateFailed, - Created: true, + Txid: req.Tx.TxHash(), + State: txconfirm.TxStateFailed, + Created: true, + DefinitelyNotBroadcast: true, }, ), ) @@ -6169,6 +6202,20 @@ func (s *Server) unrollMaxFeeRate() int64 { return 0 } +// unrollReconcileProbeTimeout returns the configured per-anchor probe +// timeout for the chainsource-backed restart reconciler, or zero so +// the reconciler falls back to defaultReconcileProbeTimeout. +func (s *Server) unrollReconcileProbeTimeout() time.Duration { + if s.cfg.Unroll != nil && + s.cfg.Unroll.ReconcileProbeTimeoutSec > 0 { + return time.Duration( + s.cfg.Unroll.ReconcileProbeTimeoutSec, + ) * time.Second + } + + return 0 +} + // unrollBumpAfterBlocks returns the configured fee-bump cadence (in // blocks) for the shared txconfirm actor used by the unroll subsystem, // or zero to let txconfirm fall back to DefaultFeeBumpIntervalBlocks.