fix(statesync): statesync is unstable and doesn't time out

.mockery.yaml

@@ -53,6 +53,7 @@ packages:
   github.com/dashpay/tenderdash/internal/statesync:
     interfaces:
       StateProvider:
+      ConsensusStateProvider:
   github.com/dashpay/tenderdash/libs/store:
     interfaces:
       Store:

config/config.go

@@ -1,7 +1,6 @@
 package config
 
 import (
-	"encoding/hex"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -1001,18 +1000,16 @@ type StateSyncConfig struct {
 	// with net.Dial, for example: "host.example.com:2125".
 	RPCServers []string `mapstructure:"rpc-servers"`
 
-	// The hash and height of a trusted block. Must be within the trust-period.
-	TrustHeight int64  `mapstructure:"trust-height"`
-	TrustHash   string `mapstructure:"trust-hash"`
-
-	// The trust period should be set so that Tendermint can detect and gossip
-	// misbehavior before it is considered expired. For chains based on the Cosmos SDK,
-	// one day less than the unbonding period should suffice.
-	TrustPeriod time.Duration `mapstructure:"trust-period"`
-
 	// Time to spend discovering snapshots before initiating a restore.
 	DiscoveryTime time.Duration `mapstructure:"discovery-time"`
 
+	// Number of times to retry state sync. When retries are exhausted, the node will
+	// fall back to the regular block sync. Set to 0 to disable retries. Default is 3.
+	//
+	// Note that in pessimistic case, it will take at least `discovery-time * retries` before
+	// falling back to block sync.
+	Retries int `mapstructure:"retries"`
+
 	// Temporary directory for state sync snapshot chunks, defaults to os.TempDir().
 	// The synchronizer will create a new, randomly named directory within this directory
 	// and remove it when the sync is complete.
@@ -1026,22 +1023,13 @@ type StateSyncConfig struct {
 	Fetchers int `mapstructure:"fetchers"`
 }
 
-func (cfg *StateSyncConfig) TrustHashBytes() []byte {
-	// validated in ValidateBasic, so we can safely panic here
-	bytes, err := hex.DecodeString(cfg.TrustHash)
-	if err != nil {
-		panic(err)
-	}
-	return bytes
-}
-
 // DefaultStateSyncConfig returns a default configuration for the state sync service
 func DefaultStateSyncConfig() *StateSyncConfig {
 	return &StateSyncConfig{
-		TrustPeriod:         168 * time.Hour,
 		DiscoveryTime:       15 * time.Second,
 		ChunkRequestTimeout: 15 * time.Second,
 		Fetchers:            4,
+		Retries:             3,
 	}
 }
 
@@ -1074,21 +1062,8 @@ func (cfg *StateSyncConfig) ValidateBasic() error {
 		return errors.New("discovery time must be 0s or greater than five seconds")
 	}
 
-	if cfg.TrustPeriod <= 0 {
-		return errors.New("trusted-period is required")
-	}
-
-	if cfg.TrustHeight <= 0 {
-		return errors.New("trusted-height is required")
-	}
-
-	if len(cfg.TrustHash) == 0 {
-		return errors.New("trusted-hash is required")
-	}
-
-	_, err := hex.DecodeString(cfg.TrustHash)
-	if err != nil {
-		return fmt.Errorf("invalid trusted-hash: %w", err)
+	if cfg.Retries < 0 {
+		return errors.New("retries must be greater than or equal to zero")
 	}
 
 	if cfg.ChunkRequestTimeout < 5*time.Second {

config/toml.go

@@ -495,18 +495,15 @@ use-p2p = {{ .StateSync.UseP2P }}
 # for example: "host.example.com:2125"
 rpc-servers = "{{ StringsJoin .StateSync.RPCServers "," }}"
 
-# The hash and height of a trusted block. Must be within the trust-period.
-trust-height = {{ .StateSync.TrustHeight }}
-trust-hash = "{{ .StateSync.TrustHash }}"
-
-# The trust period should be set so that Tendermint can detect and gossip misbehavior before
-# it is considered expired. For chains based on the Cosmos SDK, one day less than the unbonding
-# period should suffice.
-trust-period = "{{ .StateSync.TrustPeriod }}"
-
 # Time to spend discovering snapshots before initiating a restore.
 discovery-time = "{{ .StateSync.DiscoveryTime }}"
 
+# Number of times to retry state sync. When retries are exhausted, the node will
+# fall back to the regular block sync. Set to 0 to disable retries. Default is 3.
+# Note that in pessimistic case, it will take at least (discovery-time * retries) before
+# falling back to block sync.
+retries = {{ .StateSync.Retries }}
+
 # Temporary directory for state sync snapshot chunks, defaults to os.TempDir().
 # The synchronizer will create a new, randomly named directory within this directory
 # and remove it when the sync is complete.

go.mod

@@ -9,8 +9,8 @@ require (
 	github.com/btcsuite/btcutil v1.0.3-0.20201208143702-a53e38424cce
 	github.com/containerd/continuity v0.4.5 // indirect
 	github.com/dashpay/bls-signatures/go-bindings v0.0.0-20230207105415-06df92693ac8
-	github.com/dashpay/dashd-go v0.25.0
-	github.com/dashpay/dashd-go/btcec/v2 v2.1.0 // indirect
+	github.com/dashpay/dashd-go v0.26.1
+	github.com/dashpay/dashd-go/btcec/v2 v2.2.0 // indirect
 	github.com/fortytw2/leaktest v1.3.0
 	github.com/fxamacker/cbor/v2 v2.4.0
 	github.com/go-kit/kit v0.13.0
@@ -81,7 +81,7 @@ require (
 	github.com/alingse/nilnesserr v0.1.2 // indirect
 	github.com/antlr4-go/antlr/v4 v4.13.1 // indirect
 	github.com/bombsimon/wsl/v4 v4.5.0 // indirect
-	github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f // indirect
+	github.com/btcsuite/btclog v1.0.0 // indirect
 	github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd // indirect
 	github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792 // indirect
 	github.com/bufbuild/protocompile v0.14.1 // indirect
@@ -109,7 +109,7 @@ require (
 	github.com/containerd/ttrpc v1.2.7 // indirect
 	github.com/containerd/typeurl/v2 v2.2.3 // indirect
 	github.com/curioswitch/go-reassign v0.3.0 // indirect
-	github.com/dashpay/dashd-go/btcutil v1.2.0 // indirect
+	github.com/dashpay/dashd-go/btcutil v1.3.0 // indirect
 	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.3.0 // indirect
 	github.com/dgraph-io/badger/v4 v4.5.1 // indirect
 	github.com/dgraph-io/ristretto/v2 v2.1.0 // indirect

go.sum

@@ -108,8 +108,9 @@ github.com/btcsuite/btcd v0.22.1 h1:CnwP9LM/M9xuRrGSCGeMVs9iv09uMqwsVX7EeIpgV2c=
 github.com/btcsuite/btcd v0.22.1/go.mod h1:wqgTSL29+50LRkmOVknEdmt8ZojIzhuWvgu/iptuN7Y=
 github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 h1:q0rUy8C/TYNBQS1+CGKw68tLOFYSNEs0TFnxxnS9+4U=
 github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc=
-github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f h1:bAs4lUbRJpnnkd9VhRV3jjAVU7DJVjMaK+IsvSeZvFo=
 github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufoqS23FtqVCzL0ZqgP5MqXbb4fg/WgDys70nA=
+github.com/btcsuite/btclog v1.0.0 h1:sEkpKJMmfGiyZjADwEIgB1NSwMyfdD1FB8v6+w1T0Ns=
+github.com/btcsuite/btclog v1.0.0/go.mod h1:w7xnGOhwT3lmrS4H3b/D1XAXxvh+tbhUm8xeHN2y3TQ=
 github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg=
 github.com/btcsuite/btcutil v1.0.3-0.20201208143702-a53e38424cce h1:YtWJF7RHm2pYCvA5t0RPmAaLUhREsKuKd+SLhxFbFeQ=
 github.com/btcsuite/btcutil v1.0.3-0.20201208143702-a53e38424cce/go.mod h1:0DVlHczLPewLcPGEIeUEzfOJhqGPQ0mJJRDBtD307+o=
@@ -222,12 +223,12 @@ github.com/daixiang0/gci v0.13.5 h1:kThgmH1yBmZSBCh1EJVxQ7JsHpm5Oms0AMed/0LaH4c=
 github.com/daixiang0/gci v0.13.5/go.mod h1:12etP2OniiIdP4q+kjUGrC/rUagga7ODbqsom5Eo5Yk=
 github.com/dashpay/bls-signatures/go-bindings v0.0.0-20230207105415-06df92693ac8 h1:v4K3CiDoFY1gjcWL/scRcwzyjBwh8TVG3ek8cWolK1g=
 github.com/dashpay/bls-signatures/go-bindings v0.0.0-20230207105415-06df92693ac8/go.mod h1:auvGS60NBZ+a21aCCQh366PdsjDvHinsCvl28VrYPu4=
-github.com/dashpay/dashd-go v0.25.0 h1:tswVRmM2fLHC/JhpuAZ5Oa0TpOO6L+tqiE+QLTCvIQc=
-github.com/dashpay/dashd-go v0.25.0/go.mod h1:4yuk/laGME2RnQRTdqTbw87PhT+42hE1anLCnpkgls8=
-github.com/dashpay/dashd-go/btcec/v2 v2.1.0 h1:fXwlLf5H+TtgHxjGMU74NesKzk6NisjKMPF04pBcygk=
-github.com/dashpay/dashd-go/btcec/v2 v2.1.0/go.mod h1:1i8XtxdOmvK6mYEUCneVXTzFbrCUw3wq1u91j8gvsns=
-github.com/dashpay/dashd-go/btcutil v1.2.0 h1:YMq7L0V0au5bbphIhpsBBc+nfOZqU+gJ4pkgRZB7Eiw=
-github.com/dashpay/dashd-go/btcutil v1.2.0/go.mod h1:7UHoqUh3LY3OI4mEcogx0CnL3rtzDQyoqvsOCZZtvzE=
+github.com/dashpay/dashd-go v0.26.1 h1:/ZFgtPw1fPHpvoJgKfXo/v63ZXddjJm8KrHRpxcSpy0=
+github.com/dashpay/dashd-go v0.26.1/go.mod h1:7KKS2jSPkC1pTz9WLXpiXZ96wT5bUqKTRuk35AyRQ74=
+github.com/dashpay/dashd-go/btcec/v2 v2.2.0 h1:tk54BC++OvOUu0vcPoG8+45dGoJXKsmupYAawBO/1Vk=
+github.com/dashpay/dashd-go/btcec/v2 v2.2.0/go.mod h1:uOmCM/hVoJ1x6w+3SX+zQv+2LdrK3aO59RV41jNvTF4=
+github.com/dashpay/dashd-go/btcutil v1.3.0 h1:yDX8tz7C/KhFHbGlRXBpNN+zlkmAgwkICD9DlAv/Vsc=
+github.com/dashpay/dashd-go/btcutil v1.3.0/go.mod h1:sMWZ0iR8a/wmIA6b5+ccjOGUfq+iZvi5t6ECaLCW+kw=
 github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=

internal/blocksync/reactor.go

@@ -159,7 +159,7 @@ func (r *Reactor) OnStop() {
 
 // processPeerUpdate processes a PeerUpdate.
 func (r *Reactor) processPeerUpdate(ctx context.Context, peerUpdate p2p.PeerUpdate, client *client.Client) {
-	r.logger.Debug("received peer update", "peer", peerUpdate.NodeID, "status", peerUpdate.Status)
+	r.logger.Trace("received peer update", "peer", peerUpdate.NodeID, "status", peerUpdate.Status)
 
 	// XXX: Pool#RedoRequest can sometimes give us an empty peer.
 	if len(peerUpdate.NodeID) == 0 {

internal/consensus/state.go

@@ -307,6 +307,10 @@ func (cs *State) SetProposedAppVersion(ver uint64) {
 	cs.emitter.Emit(setProposedAppVersionEventName, ver)
 }
 
+func (cs *State) GetCurrentHeight() int64 {
+	return cs.stateDataStore.Get().Height
+}
+
 func (cs *State) updateStateFromStore() error {
 	state, err := cs.stateStore.Load()
 	if err != nil {

internal/consensus/state_data.go

@@ -119,7 +119,7 @@ func (s *StateDataStore) Subscribe(evsw *eventemitter.EventEmitter) {
 	})
 }
 
-// StateData is a copy of the current RoundState nad state.State stored in the store
+// StateData is a copy of the current RoundState and state.State stored in the store
 // Along with data, StateData provides some methods to check or update data inside
 type StateData struct {
 	config *config.ConsensusConfig

internal/evidence/reactor.go

@@ -201,7 +201,7 @@ func (r *Reactor) processEvidenceCh(ctx context.Context) {
 //
 // REF: https://github.com/tendermint/tendermint/issues/4727
 func (r *Reactor) processPeerUpdate(ctx context.Context, peerUpdate p2p.PeerUpdate) {
-	r.logger.Debug("received peer update", "peer", peerUpdate.NodeID, "status", peerUpdate.Status)
+	r.logger.Trace("received peer update", "peer", peerUpdate.NodeID, "status", peerUpdate.Status)
 
 	r.mtx.Lock()
 	defer r.mtx.Unlock()

internal/libs/sync/mutexguard.go

@@ -0,0 +1,54 @@
+package sync
+
+// UnlockFn is a function that unlocks a mutex.
+// It returns true if the mutex was unlocked, false if it was already unlocked.
+type UnlockFn func() bool
+
+// Mtx is a mutex interface.
+// Implemented by sync.Mutex and deadlock.Mutex.
+type Mtx interface {
+	Lock()
+	Unlock()
+}
+
+// RMtx is a mutex that can be locked for read.
+//
+// Implemented by sync.RwMutex and deadlock.RwMutex.
+type RMtx interface {
+	RLock()
+	RUnlock()
+}
+
+// LockGuard locks the mutex and returns a function that unlocks it.
+// The returned function must be called to release the lock.
+// The returned function may be called multiple times - only the first call will unlock the mutex, others will be no-ops.
+func LockGuard(mtx Mtx) UnlockFn {
+	mtx.Lock()
+	locked := true
+
+	return func() bool {
+		if locked {
+			locked = false
+			mtx.Unlock()
+			return true
+		}
+		return false
+	}
+}
+
+// RLockGuard locks the read-write mutex for reading and returns a function that unlocks it.
+// The returned function must be called to release the lock.
+// The returned function may be called multiple times - only the first call will unlock the mutex, others will be no-ops.
+func RLockGuard(mtx RMtx) UnlockFn {
+	mtx.RLock()
+	locked := true
+
+	return func() bool {
+		if locked {
+			locked = false
+			mtx.RUnlock()
+			return true
+		}
+		return false
+	}
+}