diff --git a/.golangci.yml b/.golangci.yml index 75452472d026..adb59f318f21 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -60,6 +60,10 @@ issues: - path: crypto/bn256/ linters: - revive + - path: cmd/utils/flags.go + text: "SA1019: cfg.TxLookupLimit is deprecated: use 'TransactionHistory' instead." + - path: cmd/utils/flags.go + text: "SA1019: ethconfig.Defaults.TxLookupLimit is deprecated: use 'TransactionHistory' instead." - path: internal/build/pgp.go text: 'SA1019: "golang.org/x/crypto/openpgp" is deprecated: this package is unmaintained except for security fixes.' - path: core/vm/contracts.go diff --git a/cmd/devp2p/internal/ethtest/suite.go b/cmd/devp2p/internal/ethtest/suite.go index b5cc27a2b590..5cb9fa0297d9 100644 --- a/cmd/devp2p/internal/ethtest/suite.go +++ b/cmd/devp2p/internal/ethtest/suite.go @@ -849,7 +849,16 @@ func (s *Suite) TestBlobViolations(t *utesting.T) { if code, _, err := conn.Read(); err != nil { t.Fatalf("expected disconnect on blob violation, got err: %v", err) } else if code != discMsg { - t.Fatalf("expected disconnect on blob violation, got msg code: %d", code) + if code == protoOffset(ethProto)+eth.NewPooledTransactionHashesMsg { + // sometimes we'll get a blob transaction hashes announcement before the disconnect + // because blob transactions are scheduled to be fetched right away. + if code, _, err = conn.Read(); err != nil { + t.Fatalf("expected disconnect on blob violation, got err on second read: %v", err) + } + } + if code != discMsg { + t.Fatalf("expected disconnect on blob violation, got msg code: %d", code) + } } conn.Close() } diff --git a/cmd/geth/config.go b/cmd/geth/config.go index 522e5e22f2a9..fd57ff40def4 100644 --- a/cmd/geth/config.go +++ b/cmd/geth/config.go @@ -75,8 +75,8 @@ var tomlSettings = toml.Config{ }, MissingField: func(rt reflect.Type, field string) error { id := fmt.Sprintf("%s.%s", rt.String(), field) - if deprecated(id) { - log.Warn("Config field is deprecated and won't have an effect", "name", id) + if deprecatedConfigFields[id] { + log.Warn(fmt.Sprintf("Config field '%s' is deprecated and won't have any effect.", id)) return nil } var link string @@ -87,6 +87,19 @@ var tomlSettings = toml.Config{ }, } +var deprecatedConfigFields = map[string]bool{ + "ethconfig.Config.EVMInterpreter": true, + "ethconfig.Config.EWASMInterpreter": true, + "ethconfig.Config.TrieCleanCacheJournal": true, + "ethconfig.Config.TrieCleanCacheRejournal": true, + "ethconfig.Config.LightServ": true, + "ethconfig.Config.LightIngress": true, + "ethconfig.Config.LightEgress": true, + "ethconfig.Config.LightPeers": true, + "ethconfig.Config.LightNoPrune": true, + "ethconfig.Config.LightNoSyncServe": true, +} + type ethstatsConfig struct { URL string `toml:",omitempty"` } @@ -314,21 +327,6 @@ func applyMetricConfig(ctx *cli.Context, cfg *gethConfig) { } } -func deprecated(field string) bool { - switch field { - case "ethconfig.Config.EVMInterpreter": - return true - case "ethconfig.Config.EWASMInterpreter": - return true - case "ethconfig.Config.TrieCleanCacheJournal": - return true - case "ethconfig.Config.TrieCleanCacheRejournal": - return true - default: - return false - } -} - func setAccountManagerBackends(conf *node.Config, am *accounts.Manager, keydir string) error { scryptN := keystore.StandardScryptN scryptP := keystore.StandardScryptP diff --git a/cmd/utils/flags_legacy.go b/cmd/utils/flags_legacy.go index 1dfd1a5f8934..f145f605d6aa 100644 --- a/cmd/utils/flags_legacy.go +++ b/cmd/utils/flags_legacy.go @@ -92,25 +92,21 @@ var ( LightServeFlag = &cli.IntFlag{ Name: "light.serve", Usage: "Maximum percentage of time allowed for serving LES requests (deprecated)", - Value: ethconfig.Defaults.LightServ, Category: flags.DeprecatedCategory, } LightIngressFlag = &cli.IntFlag{ Name: "light.ingress", Usage: "Incoming bandwidth limit for serving light clients (deprecated)", - Value: ethconfig.Defaults.LightIngress, Category: flags.DeprecatedCategory, } LightEgressFlag = &cli.IntFlag{ Name: "light.egress", Usage: "Outgoing bandwidth limit for serving light clients (deprecated)", - Value: ethconfig.Defaults.LightEgress, Category: flags.DeprecatedCategory, } LightMaxPeersFlag = &cli.IntFlag{ Name: "light.maxpeers", Usage: "Maximum number of light clients to serve, or light servers to attach to (deprecated)", - Value: ethconfig.Defaults.LightPeers, Category: flags.DeprecatedCategory, } LightNoPruneFlag = &cli.BoolFlag{ diff --git a/core/genesis.go b/core/genesis.go index 4ca24807fccd..9e213e5163b0 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -145,13 +145,12 @@ func hashAlloc(ga *types.GenesisAlloc, isVerkle bool) (common.Hash, error) { return statedb.Commit(0, false) } -// flushAlloc is very similar with hash, but the main difference is all the generated -// states will be persisted into the given database. Also, the genesis state -// specification will be flushed as well. -func flushAlloc(ga *types.GenesisAlloc, db ethdb.Database, triedb *triedb.Database, blockhash common.Hash) error { +// flushAlloc is very similar with hash, but the main difference is all the +// generated states will be persisted into the given database. +func flushAlloc(ga *types.GenesisAlloc, db ethdb.Database, triedb *triedb.Database) (common.Hash, error) { statedb, err := state.New(types.EmptyRootHash, state.NewDatabaseWithNodeDB(db, triedb), nil) if err != nil { - return err + return common.Hash{}, err } for addr, account := range *ga { if account.Balance != nil { @@ -167,21 +166,15 @@ func flushAlloc(ga *types.GenesisAlloc, db ethdb.Database, triedb *triedb.Databa } root, err := statedb.Commit(0, false) if err != nil { - return err + return common.Hash{}, err } // Commit newly generated states into disk if it's not empty. if root != types.EmptyRootHash { if err := triedb.Commit(root, true); err != nil { - return err + return common.Hash{}, err } } - // Marshal the genesis state specification and persist. - blob, err := json.Marshal(ga) - if err != nil { - return err - } - rawdb.WriteGenesisStateSpec(db, blockhash, blob) - return nil + return root, nil } func getGenesisState(db ethdb.Database, blockhash common.Hash) (alloc types.GenesisAlloc, err error) { @@ -426,6 +419,11 @@ func (g *Genesis) ToBlock() *types.Block { if err != nil { panic(err) } + return g.toBlockWithRoot(root) +} + +// toBlockWithRoot constructs the genesis block with the given genesis state root. +func (g *Genesis) toBlockWithRoot(root common.Hash) *types.Block { head := &types.Header{ Number: new(big.Int).SetUint64(g.Number), Nonce: types.EncodeNonce(g.Nonce), @@ -482,8 +480,7 @@ func (g *Genesis) ToBlock() *types.Block { // Commit writes the block and state of a genesis specification to the database. // The block is committed as the canonical head block. func (g *Genesis) Commit(db ethdb.Database, triedb *triedb.Database) (*types.Block, error) { - block := g.ToBlock() - if block.Number().Sign() != 0 { + if g.Number != 0 { return nil, errors.New("can't commit genesis block with number > 0") } config := g.Config @@ -493,15 +490,22 @@ func (g *Genesis) Commit(db ethdb.Database, triedb *triedb.Database) (*types.Blo if err := config.CheckConfigForkOrder(); err != nil { return nil, err } - if config.Clique != nil && len(block.Extra()) < 32+crypto.SignatureLength { + if config.Clique != nil && len(g.ExtraData) < 32+crypto.SignatureLength { return nil, errors.New("can't start clique chain without signers") } - // All the checks has passed, flushAlloc the states derived from the genesis - // specification as well as the specification itself into the provided - // database. - if err := flushAlloc(&g.Alloc, db, triedb, block.Hash()); err != nil { + // flush the data to disk and compute the state root + root, err := flushAlloc(&g.Alloc, db, triedb) + if err != nil { + return nil, err + } + block := g.toBlockWithRoot(root) + + // Marshal the genesis state specification and persist. + blob, err := json.Marshal(g.Alloc) + if err != nil { return nil, err } + rawdb.WriteGenesisStateSpec(db, block.Hash(), blob) rawdb.WriteTd(db, block.Hash(), block.NumberU64(), block.Difficulty()) rawdb.WriteBlock(db, block) rawdb.WriteReceipts(db, block.Hash(), block.NumberU64(), nil) diff --git a/core/txpool/blobpool/blobpool.go b/core/txpool/blobpool/blobpool.go index cb266fd10a07..d66a08aa17a4 100644 --- a/core/txpool/blobpool/blobpool.go +++ b/core/txpool/blobpool/blobpool.go @@ -1155,11 +1155,11 @@ func (p *BlobPool) validateTx(tx *types.Transaction) error { ) switch { case tx.GasFeeCapIntCmp(minGasFeeCap.ToBig()) < 0: - return fmt.Errorf("%w: new tx gas fee cap %v <= %v queued + %d%% replacement penalty", txpool.ErrReplaceUnderpriced, tx.GasFeeCap(), prev.execFeeCap, p.config.PriceBump) + return fmt.Errorf("%w: new tx gas fee cap %v < %v queued + %d%% replacement penalty", txpool.ErrReplaceUnderpriced, tx.GasFeeCap(), prev.execFeeCap, p.config.PriceBump) case tx.GasTipCapIntCmp(minGasTipCap.ToBig()) < 0: - return fmt.Errorf("%w: new tx gas tip cap %v <= %v queued + %d%% replacement penalty", txpool.ErrReplaceUnderpriced, tx.GasTipCap(), prev.execTipCap, p.config.PriceBump) + return fmt.Errorf("%w: new tx gas tip cap %v < %v queued + %d%% replacement penalty", txpool.ErrReplaceUnderpriced, tx.GasTipCap(), prev.execTipCap, p.config.PriceBump) case tx.BlobGasFeeCapIntCmp(minBlobGasFeeCap.ToBig()) < 0: - return fmt.Errorf("%w: new tx blob gas fee cap %v <= %v queued + %d%% replacement penalty", txpool.ErrReplaceUnderpriced, tx.BlobGasFeeCap(), prev.blobFeeCap, p.config.PriceBump) + return fmt.Errorf("%w: new tx blob gas fee cap %v < %v queued + %d%% replacement penalty", txpool.ErrReplaceUnderpriced, tx.BlobGasFeeCap(), prev.blobFeeCap, p.config.PriceBump) } } return nil diff --git a/eth/backend.go b/eth/backend.go index 8679018dabfc..442d408bb00d 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -377,16 +377,8 @@ func (s *Ethereum) Start() error { // Regularly update shutdown marker s.shutdownTracker.Start() - // Figure out a max peers count based on the server limits - maxPeers := s.p2pServer.MaxPeers - if s.config.LightServ > 0 { - if s.config.LightPeers >= s.p2pServer.MaxPeers { - return fmt.Errorf("invalid peer config: light peer count (%d) >= total peer count (%d)", s.config.LightPeers, s.p2pServer.MaxPeers) - } - maxPeers -= s.config.LightPeers - } - // Start the networking layer and the light server if requested - s.handler.Start(maxPeers) + // Start the networking layer + s.handler.Start(s.p2pServer.MaxPeers) return nil } diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index 7453fb1efdd3..5a955d9bbfc3 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -53,7 +53,6 @@ var Defaults = Config{ TxLookupLimit: 2350000, TransactionHistory: 2350000, StateHistory: params.FullImmutabilityThreshold, - LightPeers: 100, DatabaseCache: 512, TrieCleanCache: 154, TrieDirtyCache: 256, @@ -87,11 +86,13 @@ type Config struct { EthDiscoveryURLs []string SnapDiscoveryURLs []string + // State options. NoPruning bool // Whether to disable pruning and flush everything to disk NoPrefetch bool // Whether to disable prefetching and only load state on demand - // Deprecated, use 'TransactionHistory' instead. - TxLookupLimit uint64 `toml:",omitempty"` // The maximum number of blocks from head whose tx indices are reserved. + // Deprecated: use 'TransactionHistory' instead. + TxLookupLimit uint64 `toml:",omitempty"` // The maximum number of blocks from head whose tx indices are reserved. + TransactionHistory uint64 `toml:",omitempty"` // The maximum number of blocks from head whose tx indices are reserved. StateHistory uint64 `toml:",omitempty"` // The maximum number of blocks from head whose state histories are reserved. @@ -105,14 +106,6 @@ type Config struct { // presence of these blocks for every new peer connection. RequiredBlocks map[uint64]common.Hash `toml:"-"` - // Light client options - LightServ int `toml:",omitempty"` // Maximum percentage of time allowed for serving LES requests - LightIngress int `toml:",omitempty"` // Incoming bandwidth limit for light servers - LightEgress int `toml:",omitempty"` // Outgoing bandwidth limit for light servers - LightPeers int `toml:",omitempty"` // Maximum number of LES client peers - LightNoPrune bool `toml:",omitempty"` // Whether to disable light chain pruning - LightNoSyncServe bool `toml:",omitempty"` // Whether to serve light clients before syncing - // Database options SkipBcVersionCheck bool `toml:"-"` DatabaseHandles int `toml:"-"` diff --git a/eth/ethconfig/gen_config.go b/eth/ethconfig/gen_config.go index 147a5599848b..2854c870975c 100644 --- a/eth/ethconfig/gen_config.go +++ b/eth/ethconfig/gen_config.go @@ -29,12 +29,6 @@ func (c Config) MarshalTOML() (interface{}, error) { StateHistory uint64 `toml:",omitempty"` StateScheme string `toml:",omitempty"` RequiredBlocks map[uint64]common.Hash `toml:"-"` - LightServ int `toml:",omitempty"` - LightIngress int `toml:",omitempty"` - LightEgress int `toml:",omitempty"` - LightPeers int `toml:",omitempty"` - LightNoPrune bool `toml:",omitempty"` - LightNoSyncServe bool `toml:",omitempty"` SkipBcVersionCheck bool `toml:"-"` DatabaseHandles int `toml:"-"` DatabaseCache int @@ -73,12 +67,6 @@ func (c Config) MarshalTOML() (interface{}, error) { enc.StateHistory = c.StateHistory enc.StateScheme = c.StateScheme enc.RequiredBlocks = c.RequiredBlocks - enc.LightServ = c.LightServ - enc.LightIngress = c.LightIngress - enc.LightEgress = c.LightEgress - enc.LightPeers = c.LightPeers - enc.LightNoPrune = c.LightNoPrune - enc.LightNoSyncServe = c.LightNoSyncServe enc.SkipBcVersionCheck = c.SkipBcVersionCheck enc.DatabaseHandles = c.DatabaseHandles enc.DatabaseCache = c.DatabaseCache @@ -121,12 +109,6 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { StateHistory *uint64 `toml:",omitempty"` StateScheme *string `toml:",omitempty"` RequiredBlocks map[uint64]common.Hash `toml:"-"` - LightServ *int `toml:",omitempty"` - LightIngress *int `toml:",omitempty"` - LightEgress *int `toml:",omitempty"` - LightPeers *int `toml:",omitempty"` - LightNoPrune *bool `toml:",omitempty"` - LightNoSyncServe *bool `toml:",omitempty"` SkipBcVersionCheck *bool `toml:"-"` DatabaseHandles *int `toml:"-"` DatabaseCache *int @@ -192,24 +174,6 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.RequiredBlocks != nil { c.RequiredBlocks = dec.RequiredBlocks } - if dec.LightServ != nil { - c.LightServ = *dec.LightServ - } - if dec.LightIngress != nil { - c.LightIngress = *dec.LightIngress - } - if dec.LightEgress != nil { - c.LightEgress = *dec.LightEgress - } - if dec.LightPeers != nil { - c.LightPeers = *dec.LightPeers - } - if dec.LightNoPrune != nil { - c.LightNoPrune = *dec.LightNoPrune - } - if dec.LightNoSyncServe != nil { - c.LightNoSyncServe = *dec.LightNoSyncServe - } if dec.SkipBcVersionCheck != nil { c.SkipBcVersionCheck = *dec.SkipBcVersionCheck } diff --git a/eth/fetcher/tx_fetcher.go b/eth/fetcher/tx_fetcher.go index 18c5ff007a9f..7472f5fb1a22 100644 --- a/eth/fetcher/tx_fetcher.go +++ b/eth/fetcher/tx_fetcher.go @@ -17,12 +17,12 @@ package fetcher import ( - "bytes" "errors" "fmt" "math" mrand "math/rand" "sort" + "sync/atomic" "time" "github.com/ethereum/go-ethereum/common" @@ -35,7 +35,7 @@ import ( ) const ( - // maxTxAnnounces is the maximum number of unique transaction a peer + // maxTxAnnounces is the maximum number of unique transactions a peer // can announce in a short time. maxTxAnnounces = 4096 @@ -114,16 +114,23 @@ var errTerminated = errors.New("terminated") type txAnnounce struct { origin string // Identifier of the peer originating the notification hashes []common.Hash // Batch of transaction hashes being announced - metas []*txMetadata // Batch of metadatas associated with the hashes (nil before eth/68) + metas []txMetadata // Batch of metadata associated with the hashes } -// txMetadata is a set of extra data transmitted along the announcement for better -// fetch scheduling. +// txMetadata provides the extra data transmitted along with the announcement +// for better fetch scheduling. type txMetadata struct { kind byte // Transaction consensus type size uint32 // Transaction size in bytes } +// txMetadataWithSeq is a wrapper of transaction metadata with an extra field +// tracking the transaction sequence number. +type txMetadataWithSeq struct { + txMetadata + seq uint64 +} + // txRequest represents an in-flight transaction retrieval request destined to // a specific peers. type txRequest struct { @@ -137,7 +144,7 @@ type txRequest struct { type txDelivery struct { origin string // Identifier of the peer originating the notification hashes []common.Hash // Batch of transaction hashes having been delivered - metas []txMetadata // Batch of metadatas associated with the delivered hashes + metas []txMetadata // Batch of metadata associated with the delivered hashes direct bool // Whether this is a direct reply or a broadcast } @@ -159,7 +166,7 @@ type txDrop struct { // The invariants of the fetcher are: // - Each tracked transaction (hash) must only be present in one of the // three stages. This ensures that the fetcher operates akin to a finite -// state automata and there's do data leak. +// state automata and there's no data leak. // - Each peer that announced transactions may be scheduled retrievals, but // only ever one concurrently. This ensures we can immediately know what is // missing from a reply and reschedule it. @@ -169,18 +176,19 @@ type TxFetcher struct { drop chan *txDrop quit chan struct{} + txSeq atomic.Uint64 // Unique transaction sequence number underpriced *lru.Cache[common.Hash, time.Time] // Transactions discarded as too cheap (don't re-fetch) // Stage 1: Waiting lists for newly discovered transactions that might be // broadcast without needing explicit request/reply round trips. - waitlist map[common.Hash]map[string]struct{} // Transactions waiting for an potential broadcast - waittime map[common.Hash]mclock.AbsTime // Timestamps when transactions were added to the waitlist - waitslots map[string]map[common.Hash]*txMetadata // Waiting announcements grouped by peer (DoS protection) + waitlist map[common.Hash]map[string]struct{} // Transactions waiting for an potential broadcast + waittime map[common.Hash]mclock.AbsTime // Timestamps when transactions were added to the waitlist + waitslots map[string]map[common.Hash]*txMetadataWithSeq // Waiting announcements grouped by peer (DoS protection) // Stage 2: Queue of transactions that waiting to be allocated to some peer // to be retrieved directly. - announces map[string]map[common.Hash]*txMetadata // Set of announced transactions, grouped by origin peer - announced map[common.Hash]map[string]struct{} // Set of download locations, grouped by transaction hash + announces map[string]map[common.Hash]*txMetadataWithSeq // Set of announced transactions, grouped by origin peer + announced map[common.Hash]map[string]struct{} // Set of download locations, grouped by transaction hash // Stage 3: Set of transactions currently being retrieved, some which may be // fulfilled and some rescheduled. Note, this step shares 'announces' from the @@ -218,8 +226,8 @@ func NewTxFetcherForTests( quit: make(chan struct{}), waitlist: make(map[common.Hash]map[string]struct{}), waittime: make(map[common.Hash]mclock.AbsTime), - waitslots: make(map[string]map[common.Hash]*txMetadata), - announces: make(map[string]map[common.Hash]*txMetadata), + waitslots: make(map[string]map[common.Hash]*txMetadataWithSeq), + announces: make(map[string]map[common.Hash]*txMetadataWithSeq), announced: make(map[common.Hash]map[string]struct{}), fetching: make(map[common.Hash]string), requests: make(map[string]*txRequest), @@ -247,7 +255,7 @@ func (f *TxFetcher) Notify(peer string, types []byte, sizes []uint32, hashes []c // loop, so anything caught here is time saved internally. var ( unknownHashes = make([]common.Hash, 0, len(hashes)) - unknownMetas = make([]*txMetadata, 0, len(hashes)) + unknownMetas = make([]txMetadata, 0, len(hashes)) duplicate int64 underpriced int64 @@ -260,11 +268,11 @@ func (f *TxFetcher) Notify(peer string, types []byte, sizes []uint32, hashes []c underpriced++ default: unknownHashes = append(unknownHashes, hash) - if types == nil { - unknownMetas = append(unknownMetas, nil) - } else { - unknownMetas = append(unknownMetas, &txMetadata{kind: types[i], size: sizes[i]}) - } + + // Transaction metadata has been available since eth68, and all + // legacy eth protocols (prior to eth68) have been deprecated. + // Therefore, metadata is always expected in the announcement. + unknownMetas = append(unknownMetas, txMetadata{kind: types[i], size: sizes[i]}) } } txAnnounceKnownMeter.Mark(duplicate) @@ -431,9 +439,11 @@ func (f *TxFetcher) loop() { ann.metas = ann.metas[:want-maxTxAnnounces] } // All is well, schedule the remainder of the transactions - idleWait := len(f.waittime) == 0 - _, oldPeer := f.announces[ann.origin] - + var ( + idleWait = len(f.waittime) == 0 + _, oldPeer = f.announces[ann.origin] + hasBlob bool + ) for i, hash := range ann.hashes { // If the transaction is already downloading, add it to the list // of possible alternates (in case the current retrieval fails) and @@ -443,9 +453,17 @@ func (f *TxFetcher) loop() { // Stage 2 and 3 share the set of origins per tx if announces := f.announces[ann.origin]; announces != nil { - announces[hash] = ann.metas[i] + announces[hash] = &txMetadataWithSeq{ + txMetadata: ann.metas[i], + seq: f.txSeq.Add(1), + } } else { - f.announces[ann.origin] = map[common.Hash]*txMetadata{hash: ann.metas[i]} + f.announces[ann.origin] = map[common.Hash]*txMetadataWithSeq{ + hash: { + txMetadata: ann.metas[i], + seq: f.txSeq.Add(1), + }, + } } continue } @@ -456,9 +474,17 @@ func (f *TxFetcher) loop() { // Stage 2 and 3 share the set of origins per tx if announces := f.announces[ann.origin]; announces != nil { - announces[hash] = ann.metas[i] + announces[hash] = &txMetadataWithSeq{ + txMetadata: ann.metas[i], + seq: f.txSeq.Add(1), + } } else { - f.announces[ann.origin] = map[common.Hash]*txMetadata{hash: ann.metas[i]} + f.announces[ann.origin] = map[common.Hash]*txMetadataWithSeq{ + hash: { + txMetadata: ann.metas[i], + seq: f.txSeq.Add(1), + }, + } } continue } @@ -475,24 +501,47 @@ func (f *TxFetcher) loop() { f.waitlist[hash][ann.origin] = struct{}{} if waitslots := f.waitslots[ann.origin]; waitslots != nil { - waitslots[hash] = ann.metas[i] + waitslots[hash] = &txMetadataWithSeq{ + txMetadata: ann.metas[i], + seq: f.txSeq.Add(1), + } } else { - f.waitslots[ann.origin] = map[common.Hash]*txMetadata{hash: ann.metas[i]} + f.waitslots[ann.origin] = map[common.Hash]*txMetadataWithSeq{ + hash: { + txMetadata: ann.metas[i], + seq: f.txSeq.Add(1), + }, + } } continue } // Transaction unknown to the fetcher, insert it into the waiting list f.waitlist[hash] = map[string]struct{}{ann.origin: {}} - f.waittime[hash] = f.clock.Now() + // Assign the current timestamp as the wait time, but for blob transactions, + // skip the wait time since they are only announced. + if ann.metas[i].kind != types.BlobTxType { + f.waittime[hash] = f.clock.Now() + } else { + hasBlob = true + f.waittime[hash] = f.clock.Now() - mclock.AbsTime(txArriveTimeout) + } if waitslots := f.waitslots[ann.origin]; waitslots != nil { - waitslots[hash] = ann.metas[i] + waitslots[hash] = &txMetadataWithSeq{ + txMetadata: ann.metas[i], + seq: f.txSeq.Add(1), + } } else { - f.waitslots[ann.origin] = map[common.Hash]*txMetadata{hash: ann.metas[i]} + f.waitslots[ann.origin] = map[common.Hash]*txMetadataWithSeq{ + hash: { + txMetadata: ann.metas[i], + seq: f.txSeq.Add(1), + }, + } } } // If a new item was added to the waitlist, schedule it into the fetcher - if idleWait && len(f.waittime) > 0 { + if hasBlob || (idleWait && len(f.waittime) > 0) { f.rescheduleWait(waitTimer, waitTrigger) } // If this peer is new and announced something already queued, maybe @@ -516,7 +565,7 @@ func (f *TxFetcher) loop() { if announces := f.announces[peer]; announces != nil { announces[hash] = f.waitslots[peer][hash] } else { - f.announces[peer] = map[common.Hash]*txMetadata{hash: f.waitslots[peer][hash]} + f.announces[peer] = map[common.Hash]*txMetadataWithSeq{hash: f.waitslots[peer][hash]} } delete(f.waitslots[peer], hash) if len(f.waitslots[peer]) == 0 { @@ -873,7 +922,7 @@ func (f *TxFetcher) scheduleFetches(timer *mclock.Timer, timeout chan struct{}, hashes = make([]common.Hash, 0, maxTxRetrievals) bytes uint64 ) - f.forEachAnnounce(f.announces[peer], func(hash common.Hash, meta *txMetadata) bool { + f.forEachAnnounce(f.announces[peer], func(hash common.Hash, meta txMetadata) bool { // If the transaction is already fetching, skip to the next one if _, ok := f.fetching[hash]; ok { return true @@ -892,13 +941,8 @@ func (f *TxFetcher) scheduleFetches(timer *mclock.Timer, timeout chan struct{}, if len(hashes) >= maxTxRetrievals { return false // break in the for-each } - if meta != nil { // Only set eth/68 and upwards - bytes += uint64(meta.size) - if bytes >= maxTxRetrievalSize { - return false - } - } - return true // scheduled, try to add more + bytes += uint64(meta.size) + return bytes < maxTxRetrievalSize }) // If any hashes were allocated, request them from the peer if len(hashes) > 0 { @@ -943,28 +987,26 @@ func (f *TxFetcher) forEachPeer(peers map[string]struct{}, do func(peer string)) } } -// forEachAnnounce does a range loop over a map of announcements in production, -// but during testing it does a deterministic sorted random to allow reproducing -// issues. -func (f *TxFetcher) forEachAnnounce(announces map[common.Hash]*txMetadata, do func(hash common.Hash, meta *txMetadata) bool) { - // If we're running production, use whatever Go's map gives us - if f.rand == nil { - for hash, meta := range announces { - if !do(hash, meta) { - return - } - } - return +// forEachAnnounce loops over the given announcements in arrival order, invoking +// the do function for each until it returns false. We enforce an arrival +// ordering to minimize the chances of transaction nonce-gaps, which result in +// transactions being rejected by the txpool. +func (f *TxFetcher) forEachAnnounce(announces map[common.Hash]*txMetadataWithSeq, do func(hash common.Hash, meta txMetadata) bool) { + type announcement struct { + hash common.Hash + meta txMetadata + seq uint64 } - // We're running the test suite, make iteration deterministic - list := make([]common.Hash, 0, len(announces)) - for hash := range announces { - list = append(list, hash) + // process announcements by their arrival order + list := make([]announcement, 0, len(announces)) + for hash, entry := range announces { + list = append(list, announcement{hash: hash, meta: entry.txMetadata, seq: entry.seq}) } - sortHashes(list) - rotateHashes(list, f.rand.Intn(len(list))) - for _, hash := range list { - if !do(hash, announces[hash]) { + sort.Slice(list, func(i, j int) bool { + return list[i].seq < list[j].seq + }) + for i := range list { + if !do(list[i].hash, list[i].meta) { return } } @@ -980,26 +1022,3 @@ func rotateStrings(slice []string, n int) { slice[i] = orig[(i+n)%len(orig)] } } - -// sortHashes sorts a slice of hashes. This method is only used in tests in order -// to simulate random map iteration but keep it deterministic. -func sortHashes(slice []common.Hash) { - for i := 0; i < len(slice); i++ { - for j := i + 1; j < len(slice); j++ { - if bytes.Compare(slice[i][:], slice[j][:]) > 0 { - slice[i], slice[j] = slice[j], slice[i] - } - } - } -} - -// rotateHashes rotates the contents of a slice by n steps. This method is only -// used in tests to simulate random map iteration but keep it deterministic. -func rotateHashes(slice []common.Hash, n int) { - orig := make([]common.Hash, len(slice)) - copy(orig, slice) - - for i := 0; i < len(orig); i++ { - slice[i] = orig[(i+n)%len(orig)] - } -} diff --git a/eth/fetcher/tx_fetcher_test.go b/eth/fetcher/tx_fetcher_test.go index 3d3ef81edef6..f80b1d6096d3 100644 --- a/eth/fetcher/tx_fetcher_test.go +++ b/eth/fetcher/tx_fetcher_test.go @@ -45,13 +45,10 @@ var ( type announce struct { hash common.Hash - kind *byte - size *uint32 + kind byte + size uint32 } -func typeptr(t byte) *byte { return &t } -func sizeptr(n uint32) *uint32 { return &n } - type doTxNotify struct { peer string hashes []common.Hash @@ -70,16 +67,10 @@ type doWait struct { type doDrop string type doFunc func() -type isWaitingWithMeta map[string][]announce -type isWaiting map[string][]common.Hash +type isWaiting map[string][]announce -type isScheduledWithMeta struct { - tracking map[string][]announce - fetching map[string][]common.Hash - dangling map[string][]common.Hash -} type isScheduled struct { - tracking map[string][]common.Hash + tracking map[string][]announce fetching map[string][]common.Hash dangling map[string][]common.Hash } @@ -92,104 +83,13 @@ type txFetcherTest struct { steps []interface{} } -// Tests that transaction announcements are added to a waitlist, and none -// of them are scheduled for retrieval until the wait expires. -func TestTransactionFetcherWaiting(t *testing.T) { - testTransactionFetcherParallel(t, txFetcherTest{ - init: func() *TxFetcher { - return NewTxFetcher( - func(common.Hash) bool { return false }, - nil, - func(string, []common.Hash) error { return nil }, - nil, - ) - }, - steps: []interface{}{ - // Initial announcement to get something into the waitlist - doTxNotify{peer: "A", hashes: []common.Hash{{0x01}, {0x02}}}, - isWaiting(map[string][]common.Hash{ - "A": {{0x01}, {0x02}}, - }), - // Announce from a new peer to check that no overwrite happens - doTxNotify{peer: "B", hashes: []common.Hash{{0x03}, {0x04}}}, - isWaiting(map[string][]common.Hash{ - "A": {{0x01}, {0x02}}, - "B": {{0x03}, {0x04}}, - }), - // Announce clashing hashes but unique new peer - doTxNotify{peer: "C", hashes: []common.Hash{{0x01}, {0x04}}}, - isWaiting(map[string][]common.Hash{ - "A": {{0x01}, {0x02}}, - "B": {{0x03}, {0x04}}, - "C": {{0x01}, {0x04}}, - }), - // Announce existing and clashing hashes from existing peer - doTxNotify{peer: "A", hashes: []common.Hash{{0x01}, {0x03}, {0x05}}}, - isWaiting(map[string][]common.Hash{ - "A": {{0x01}, {0x02}, {0x03}, {0x05}}, - "B": {{0x03}, {0x04}}, - "C": {{0x01}, {0x04}}, - }), - isScheduled{tracking: nil, fetching: nil}, - - // Wait for the arrival timeout which should move all expired items - // from the wait list to the scheduler - doWait{time: txArriveTimeout, step: true}, - isWaiting(nil), - isScheduled{ - tracking: map[string][]common.Hash{ - "A": {{0x01}, {0x02}, {0x03}, {0x05}}, - "B": {{0x03}, {0x04}}, - "C": {{0x01}, {0x04}}, - }, - fetching: map[string][]common.Hash{ // Depends on deterministic test randomizer - "A": {{0x02}, {0x03}, {0x05}}, - "C": {{0x01}, {0x04}}, - }, - }, - // Queue up a non-fetchable transaction and then trigger it with a new - // peer (weird case to test 1 line in the fetcher) - doTxNotify{peer: "C", hashes: []common.Hash{{0x06}, {0x07}}}, - isWaiting(map[string][]common.Hash{ - "C": {{0x06}, {0x07}}, - }), - doWait{time: txArriveTimeout, step: true}, - isScheduled{ - tracking: map[string][]common.Hash{ - "A": {{0x01}, {0x02}, {0x03}, {0x05}}, - "B": {{0x03}, {0x04}}, - "C": {{0x01}, {0x04}, {0x06}, {0x07}}, - }, - fetching: map[string][]common.Hash{ - "A": {{0x02}, {0x03}, {0x05}}, - "C": {{0x01}, {0x04}}, - }, - }, - doTxNotify{peer: "D", hashes: []common.Hash{{0x06}, {0x07}}}, - isScheduled{ - tracking: map[string][]common.Hash{ - "A": {{0x01}, {0x02}, {0x03}, {0x05}}, - "B": {{0x03}, {0x04}}, - "C": {{0x01}, {0x04}, {0x06}, {0x07}}, - "D": {{0x06}, {0x07}}, - }, - fetching: map[string][]common.Hash{ - "A": {{0x02}, {0x03}, {0x05}}, - "C": {{0x01}, {0x04}}, - "D": {{0x06}, {0x07}}, - }, - }, - }, - }) -} - // Tests that transaction announcements with associated metadata are added to a // waitlist, and none of them are scheduled for retrieval until the wait expires. // // This test is an extended version of TestTransactionFetcherWaiting. It's mostly // to cover the metadata checks without bloating up the basic behavioral tests // with all the useless extra fields. -func TestTransactionFetcherWaitingWithMeta(t *testing.T) { +func TestTransactionFetcherWaiting(t *testing.T) { testTransactionFetcherParallel(t, txFetcherTest{ init: func() *TxFetcher { return NewTxFetcher( @@ -202,80 +102,80 @@ func TestTransactionFetcherWaitingWithMeta(t *testing.T) { steps: []interface{}{ // Initial announcement to get something into the waitlist doTxNotify{peer: "A", hashes: []common.Hash{{0x01}, {0x02}}, types: []byte{types.LegacyTxType, types.LegacyTxType}, sizes: []uint32{111, 222}}, - isWaitingWithMeta(map[string][]announce{ + isWaiting(map[string][]announce{ "A": { - {common.Hash{0x01}, typeptr(types.LegacyTxType), sizeptr(111)}, - {common.Hash{0x02}, typeptr(types.LegacyTxType), sizeptr(222)}, + {common.Hash{0x01}, types.LegacyTxType, 111}, + {common.Hash{0x02}, types.LegacyTxType, 222}, }, }), // Announce from a new peer to check that no overwrite happens doTxNotify{peer: "B", hashes: []common.Hash{{0x03}, {0x04}}, types: []byte{types.LegacyTxType, types.LegacyTxType}, sizes: []uint32{333, 444}}, - isWaitingWithMeta(map[string][]announce{ + isWaiting(map[string][]announce{ "A": { - {common.Hash{0x01}, typeptr(types.LegacyTxType), sizeptr(111)}, - {common.Hash{0x02}, typeptr(types.LegacyTxType), sizeptr(222)}, + {common.Hash{0x01}, types.LegacyTxType, 111}, + {common.Hash{0x02}, types.LegacyTxType, 222}, }, "B": { - {common.Hash{0x03}, typeptr(types.LegacyTxType), sizeptr(333)}, - {common.Hash{0x04}, typeptr(types.LegacyTxType), sizeptr(444)}, + {common.Hash{0x03}, types.LegacyTxType, 333}, + {common.Hash{0x04}, types.LegacyTxType, 444}, }, }), // Announce clashing hashes but unique new peer doTxNotify{peer: "C", hashes: []common.Hash{{0x01}, {0x04}}, types: []byte{types.LegacyTxType, types.LegacyTxType}, sizes: []uint32{111, 444}}, - isWaitingWithMeta(map[string][]announce{ + isWaiting(map[string][]announce{ "A": { - {common.Hash{0x01}, typeptr(types.LegacyTxType), sizeptr(111)}, - {common.Hash{0x02}, typeptr(types.LegacyTxType), sizeptr(222)}, + {common.Hash{0x01}, types.LegacyTxType, 111}, + {common.Hash{0x02}, types.LegacyTxType, 222}, }, "B": { - {common.Hash{0x03}, typeptr(types.LegacyTxType), sizeptr(333)}, - {common.Hash{0x04}, typeptr(types.LegacyTxType), sizeptr(444)}, + {common.Hash{0x03}, types.LegacyTxType, 333}, + {common.Hash{0x04}, types.LegacyTxType, 444}, }, "C": { - {common.Hash{0x01}, typeptr(types.LegacyTxType), sizeptr(111)}, - {common.Hash{0x04}, typeptr(types.LegacyTxType), sizeptr(444)}, + {common.Hash{0x01}, types.LegacyTxType, 111}, + {common.Hash{0x04}, types.LegacyTxType, 444}, }, }), // Announce existing and clashing hashes from existing peer. Clashes // should not overwrite previous announcements. doTxNotify{peer: "A", hashes: []common.Hash{{0x01}, {0x03}, {0x05}}, types: []byte{types.LegacyTxType, types.LegacyTxType, types.LegacyTxType}, sizes: []uint32{999, 333, 555}}, - isWaitingWithMeta(map[string][]announce{ + isWaiting(map[string][]announce{ "A": { - {common.Hash{0x01}, typeptr(types.LegacyTxType), sizeptr(111)}, - {common.Hash{0x02}, typeptr(types.LegacyTxType), sizeptr(222)}, - {common.Hash{0x03}, typeptr(types.LegacyTxType), sizeptr(333)}, - {common.Hash{0x05}, typeptr(types.LegacyTxType), sizeptr(555)}, + {common.Hash{0x01}, types.LegacyTxType, 111}, + {common.Hash{0x02}, types.LegacyTxType, 222}, + {common.Hash{0x03}, types.LegacyTxType, 333}, + {common.Hash{0x05}, types.LegacyTxType, 555}, }, "B": { - {common.Hash{0x03}, typeptr(types.LegacyTxType), sizeptr(333)}, - {common.Hash{0x04}, typeptr(types.LegacyTxType), sizeptr(444)}, + {common.Hash{0x03}, types.LegacyTxType, 333}, + {common.Hash{0x04}, types.LegacyTxType, 444}, }, "C": { - {common.Hash{0x01}, typeptr(types.LegacyTxType), sizeptr(111)}, - {common.Hash{0x04}, typeptr(types.LegacyTxType), sizeptr(444)}, + {common.Hash{0x01}, types.LegacyTxType, 111}, + {common.Hash{0x04}, types.LegacyTxType, 444}, }, }), // Announce clashing hashes with conflicting metadata. Somebody will // be in the wrong, but we don't know yet who. doTxNotify{peer: "D", hashes: []common.Hash{{0x01}, {0x02}}, types: []byte{types.LegacyTxType, types.BlobTxType}, sizes: []uint32{999, 222}}, - isWaitingWithMeta(map[string][]announce{ + isWaiting(map[string][]announce{ "A": { - {common.Hash{0x01}, typeptr(types.LegacyTxType), sizeptr(111)}, - {common.Hash{0x02}, typeptr(types.LegacyTxType), sizeptr(222)}, - {common.Hash{0x03}, typeptr(types.LegacyTxType), sizeptr(333)}, - {common.Hash{0x05}, typeptr(types.LegacyTxType), sizeptr(555)}, + {common.Hash{0x01}, types.LegacyTxType, 111}, + {common.Hash{0x02}, types.LegacyTxType, 222}, + {common.Hash{0x03}, types.LegacyTxType, 333}, + {common.Hash{0x05}, types.LegacyTxType, 555}, }, "B": { - {common.Hash{0x03}, typeptr(types.LegacyTxType), sizeptr(333)}, - {common.Hash{0x04}, typeptr(types.LegacyTxType), sizeptr(444)}, + {common.Hash{0x03}, types.LegacyTxType, 333}, + {common.Hash{0x04}, types.LegacyTxType, 444}, }, "C": { - {common.Hash{0x01}, typeptr(types.LegacyTxType), sizeptr(111)}, - {common.Hash{0x04}, typeptr(types.LegacyTxType), sizeptr(444)}, + {common.Hash{0x01}, types.LegacyTxType, 111}, + {common.Hash{0x04}, types.LegacyTxType, 444}, }, "D": { - {common.Hash{0x01}, typeptr(types.LegacyTxType), sizeptr(999)}, - {common.Hash{0x02}, typeptr(types.BlobTxType), sizeptr(222)}, + {common.Hash{0x01}, types.LegacyTxType, 999}, + {common.Hash{0x02}, types.BlobTxType, 222}, }, }), isScheduled{tracking: nil, fetching: nil}, @@ -284,25 +184,25 @@ func TestTransactionFetcherWaitingWithMeta(t *testing.T) { // from the wait list to the scheduler doWait{time: txArriveTimeout, step: true}, isWaiting(nil), - isScheduledWithMeta{ + isScheduled{ tracking: map[string][]announce{ "A": { - {common.Hash{0x01}, typeptr(types.LegacyTxType), sizeptr(111)}, - {common.Hash{0x02}, typeptr(types.LegacyTxType), sizeptr(222)}, - {common.Hash{0x03}, typeptr(types.LegacyTxType), sizeptr(333)}, - {common.Hash{0x05}, typeptr(types.LegacyTxType), sizeptr(555)}, + {common.Hash{0x01}, types.LegacyTxType, 111}, + {common.Hash{0x02}, types.LegacyTxType, 222}, + {common.Hash{0x03}, types.LegacyTxType, 333}, + {common.Hash{0x05}, types.LegacyTxType, 555}, }, "B": { - {common.Hash{0x03}, typeptr(types.LegacyTxType), sizeptr(333)}, - {common.Hash{0x04}, typeptr(types.LegacyTxType), sizeptr(444)}, + {common.Hash{0x03}, types.LegacyTxType, 333}, + {common.Hash{0x04}, types.LegacyTxType, 444}, }, "C": { - {common.Hash{0x01}, typeptr(types.LegacyTxType), sizeptr(111)}, - {common.Hash{0x04}, typeptr(types.LegacyTxType), sizeptr(444)}, + {common.Hash{0x01}, types.LegacyTxType, 111}, + {common.Hash{0x04}, types.LegacyTxType, 444}, }, "D": { - {common.Hash{0x01}, typeptr(types.LegacyTxType), sizeptr(999)}, - {common.Hash{0x02}, typeptr(types.BlobTxType), sizeptr(222)}, + {common.Hash{0x01}, types.LegacyTxType, 999}, + {common.Hash{0x02}, types.BlobTxType, 222}, }, }, fetching: map[string][]common.Hash{ // Depends on deterministic test randomizer @@ -314,34 +214,34 @@ func TestTransactionFetcherWaitingWithMeta(t *testing.T) { // Queue up a non-fetchable transaction and then trigger it with a new // peer (weird case to test 1 line in the fetcher) doTxNotify{peer: "C", hashes: []common.Hash{{0x06}, {0x07}}, types: []byte{types.LegacyTxType, types.LegacyTxType}, sizes: []uint32{666, 777}}, - isWaitingWithMeta(map[string][]announce{ + isWaiting(map[string][]announce{ "C": { - {common.Hash{0x06}, typeptr(types.LegacyTxType), sizeptr(666)}, - {common.Hash{0x07}, typeptr(types.LegacyTxType), sizeptr(777)}, + {common.Hash{0x06}, types.LegacyTxType, 666}, + {common.Hash{0x07}, types.LegacyTxType, 777}, }, }), doWait{time: txArriveTimeout, step: true}, - isScheduledWithMeta{ + isScheduled{ tracking: map[string][]announce{ "A": { - {common.Hash{0x01}, typeptr(types.LegacyTxType), sizeptr(111)}, - {common.Hash{0x02}, typeptr(types.LegacyTxType), sizeptr(222)}, - {common.Hash{0x03}, typeptr(types.LegacyTxType), sizeptr(333)}, - {common.Hash{0x05}, typeptr(types.LegacyTxType), sizeptr(555)}, + {common.Hash{0x01}, types.LegacyTxType, 111}, + {common.Hash{0x02}, types.LegacyTxType, 222}, + {common.Hash{0x03}, types.LegacyTxType, 333}, + {common.Hash{0x05}, types.LegacyTxType, 555}, }, "B": { - {common.Hash{0x03}, typeptr(types.LegacyTxType), sizeptr(333)}, - {common.Hash{0x04}, typeptr(types.LegacyTxType), sizeptr(444)}, + {common.Hash{0x03}, types.LegacyTxType, 333}, + {common.Hash{0x04}, types.LegacyTxType, 444}, }, "C": { - {common.Hash{0x01}, typeptr(types.LegacyTxType), sizeptr(111)}, - {common.Hash{0x04}, typeptr(types.LegacyTxType), sizeptr(444)}, - {common.Hash{0x06}, typeptr(types.LegacyTxType), sizeptr(666)}, - {common.Hash{0x07}, typeptr(types.LegacyTxType), sizeptr(777)}, + {common.Hash{0x01}, types.LegacyTxType, 111}, + {common.Hash{0x04}, types.LegacyTxType, 444}, + {common.Hash{0x06}, types.LegacyTxType, 666}, + {common.Hash{0x07}, types.LegacyTxType, 777}, }, "D": { - {common.Hash{0x01}, typeptr(types.LegacyTxType), sizeptr(999)}, - {common.Hash{0x02}, typeptr(types.BlobTxType), sizeptr(222)}, + {common.Hash{0x01}, types.LegacyTxType, 999}, + {common.Hash{0x02}, types.BlobTxType, 222}, }, }, fetching: map[string][]common.Hash{ @@ -351,31 +251,31 @@ func TestTransactionFetcherWaitingWithMeta(t *testing.T) { }, }, doTxNotify{peer: "E", hashes: []common.Hash{{0x06}, {0x07}}, types: []byte{types.LegacyTxType, types.LegacyTxType}, sizes: []uint32{666, 777}}, - isScheduledWithMeta{ + isScheduled{ tracking: map[string][]announce{ "A": { - {common.Hash{0x01}, typeptr(types.LegacyTxType), sizeptr(111)}, - {common.Hash{0x02}, typeptr(types.LegacyTxType), sizeptr(222)}, - {common.Hash{0x03}, typeptr(types.LegacyTxType), sizeptr(333)}, - {common.Hash{0x05}, typeptr(types.LegacyTxType), sizeptr(555)}, + {common.Hash{0x01}, types.LegacyTxType, 111}, + {common.Hash{0x02}, types.LegacyTxType, 222}, + {common.Hash{0x03}, types.LegacyTxType, 333}, + {common.Hash{0x05}, types.LegacyTxType, 555}, }, "B": { - {common.Hash{0x03}, typeptr(types.LegacyTxType), sizeptr(333)}, - {common.Hash{0x04}, typeptr(types.LegacyTxType), sizeptr(444)}, + {common.Hash{0x03}, types.LegacyTxType, 333}, + {common.Hash{0x04}, types.LegacyTxType, 444}, }, "C": { - {common.Hash{0x01}, typeptr(types.LegacyTxType), sizeptr(111)}, - {common.Hash{0x04}, typeptr(types.LegacyTxType), sizeptr(444)}, - {common.Hash{0x06}, typeptr(types.LegacyTxType), sizeptr(666)}, - {common.Hash{0x07}, typeptr(types.LegacyTxType), sizeptr(777)}, + {common.Hash{0x01}, types.LegacyTxType, 111}, + {common.Hash{0x04}, types.LegacyTxType, 444}, + {common.Hash{0x06}, types.LegacyTxType, 666}, + {common.Hash{0x07}, types.LegacyTxType, 777}, }, "D": { - {common.Hash{0x01}, typeptr(types.LegacyTxType), sizeptr(999)}, - {common.Hash{0x02}, typeptr(types.BlobTxType), sizeptr(222)}, + {common.Hash{0x01}, types.LegacyTxType, 999}, + {common.Hash{0x02}, types.BlobTxType, 222}, }, "E": { - {common.Hash{0x06}, typeptr(types.LegacyTxType), sizeptr(666)}, - {common.Hash{0x07}, typeptr(types.LegacyTxType), sizeptr(777)}, + {common.Hash{0x06}, types.LegacyTxType, 666}, + {common.Hash{0x07}, types.LegacyTxType, 777}, }, }, fetching: map[string][]common.Hash{ @@ -403,17 +303,28 @@ func TestTransactionFetcherSkipWaiting(t *testing.T) { }, steps: []interface{}{ // Push an initial announcement through to the scheduled stage - doTxNotify{peer: "A", hashes: []common.Hash{{0x01}, {0x02}}}, - isWaiting(map[string][]common.Hash{ - "A": {{0x01}, {0x02}}, + doTxNotify{ + peer: "A", + hashes: []common.Hash{{0x01}, {0x02}}, + types: []byte{types.LegacyTxType, types.LegacyTxType}, + sizes: []uint32{111, 222}, + }, + isWaiting(map[string][]announce{ + "A": { + {common.Hash{0x01}, types.LegacyTxType, 111}, + {common.Hash{0x02}, types.LegacyTxType, 222}, + }, }), isScheduled{tracking: nil, fetching: nil}, doWait{time: txArriveTimeout, step: true}, isWaiting(nil), isScheduled{ - tracking: map[string][]common.Hash{ - "A": {{0x01}, {0x02}}, + tracking: map[string][]announce{ + "A": { + {common.Hash{0x01}, types.LegacyTxType, 111}, + {common.Hash{0x02}, types.LegacyTxType, 222}, + }, }, fetching: map[string][]common.Hash{ "A": {{0x01}, {0x02}}, @@ -421,13 +332,18 @@ func TestTransactionFetcherSkipWaiting(t *testing.T) { }, // Announce overlaps from the same peer, ensure the new ones end up // in stage one, and clashing ones don't get double tracked - doTxNotify{peer: "A", hashes: []common.Hash{{0x02}, {0x03}}}, - isWaiting(map[string][]common.Hash{ - "A": {{0x03}}, + doTxNotify{peer: "A", hashes: []common.Hash{{0x02}, {0x03}}, types: []byte{types.LegacyTxType, types.LegacyTxType}, sizes: []uint32{222, 333}}, + isWaiting(map[string][]announce{ + "A": { + {common.Hash{0x03}, types.LegacyTxType, 333}, + }, }), isScheduled{ - tracking: map[string][]common.Hash{ - "A": {{0x01}, {0x02}}, + tracking: map[string][]announce{ + "A": { + {common.Hash{0x01}, types.LegacyTxType, 111}, + {common.Hash{0x02}, types.LegacyTxType, 222}, + }, }, fetching: map[string][]common.Hash{ "A": {{0x01}, {0x02}}, @@ -435,15 +351,25 @@ func TestTransactionFetcherSkipWaiting(t *testing.T) { }, // Announce overlaps from a new peer, ensure new transactions end up // in stage one and clashing ones get tracked for the new peer - doTxNotify{peer: "B", hashes: []common.Hash{{0x02}, {0x03}, {0x04}}}, - isWaiting(map[string][]common.Hash{ - "A": {{0x03}}, - "B": {{0x03}, {0x04}}, + doTxNotify{peer: "B", hashes: []common.Hash{{0x02}, {0x03}, {0x04}}, types: []byte{types.LegacyTxType, types.LegacyTxType, types.LegacyTxType}, sizes: []uint32{222, 333, 444}}, + isWaiting(map[string][]announce{ + "A": { + {common.Hash{0x03}, types.LegacyTxType, 333}, + }, + "B": { + {common.Hash{0x03}, types.LegacyTxType, 333}, + {common.Hash{0x04}, types.LegacyTxType, 444}, + }, }), isScheduled{ - tracking: map[string][]common.Hash{ - "A": {{0x01}, {0x02}}, - "B": {{0x02}}, + tracking: map[string][]announce{ + "A": { + {common.Hash{0x01}, types.LegacyTxType, 111}, + {common.Hash{0x02}, types.LegacyTxType, 222}, + }, + "B": { + {common.Hash{0x02}, types.LegacyTxType, 222}, + }, }, fetching: map[string][]common.Hash{ "A": {{0x01}, {0x02}}, @@ -467,17 +393,23 @@ func TestTransactionFetcherSingletonRequesting(t *testing.T) { }, steps: []interface{}{ // Push an initial announcement through to the scheduled stage - doTxNotify{peer: "A", hashes: []common.Hash{{0x01}, {0x02}}}, - isWaiting(map[string][]common.Hash{ - "A": {{0x01}, {0x02}}, + doTxNotify{peer: "A", hashes: []common.Hash{{0x01}, {0x02}}, types: []byte{types.LegacyTxType, types.LegacyTxType}, sizes: []uint32{111, 222}}, + isWaiting(map[string][]announce{ + "A": { + {common.Hash{0x01}, types.LegacyTxType, 111}, + {common.Hash{0x02}, types.LegacyTxType, 222}, + }, }), isScheduled{tracking: nil, fetching: nil}, doWait{time: txArriveTimeout, step: true}, isWaiting(nil), isScheduled{ - tracking: map[string][]common.Hash{ - "A": {{0x01}, {0x02}}, + tracking: map[string][]announce{ + "A": { + {common.Hash{0x01}, types.LegacyTxType, 111}, + {common.Hash{0x02}, types.LegacyTxType, 222}, + }, }, fetching: map[string][]common.Hash{ "A": {{0x01}, {0x02}}, @@ -485,13 +417,19 @@ func TestTransactionFetcherSingletonRequesting(t *testing.T) { }, // Announce a new set of transactions from the same peer and ensure // they do not start fetching since the peer is already busy - doTxNotify{peer: "A", hashes: []common.Hash{{0x03}, {0x04}}}, - isWaiting(map[string][]common.Hash{ - "A": {{0x03}, {0x04}}, + doTxNotify{peer: "A", hashes: []common.Hash{{0x03}, {0x04}}, types: []byte{types.LegacyTxType, types.LegacyTxType}, sizes: []uint32{333, 444}}, + isWaiting(map[string][]announce{ + "A": { + {common.Hash{0x03}, types.LegacyTxType, 333}, + {common.Hash{0x04}, types.LegacyTxType, 444}, + }, }), isScheduled{ - tracking: map[string][]common.Hash{ - "A": {{0x01}, {0x02}}, + tracking: map[string][]announce{ + "A": { + {common.Hash{0x01}, types.LegacyTxType, 111}, + {common.Hash{0x02}, types.LegacyTxType, 222}, + }, }, fetching: map[string][]common.Hash{ "A": {{0x01}, {0x02}}, @@ -500,8 +438,13 @@ func TestTransactionFetcherSingletonRequesting(t *testing.T) { doWait{time: txArriveTimeout, step: true}, isWaiting(nil), isScheduled{ - tracking: map[string][]common.Hash{ - "A": {{0x01}, {0x02}, {0x03}, {0x04}}, + tracking: map[string][]announce{ + "A": { + {common.Hash{0x01}, types.LegacyTxType, 111}, + {common.Hash{0x02}, types.LegacyTxType, 222}, + {common.Hash{0x03}, types.LegacyTxType, 333}, + {common.Hash{0x04}, types.LegacyTxType, 444}, + }, }, fetching: map[string][]common.Hash{ "A": {{0x01}, {0x02}}, @@ -509,14 +452,25 @@ func TestTransactionFetcherSingletonRequesting(t *testing.T) { }, // Announce a duplicate set of transactions from a new peer and ensure // uniquely new ones start downloading, even if clashing. - doTxNotify{peer: "B", hashes: []common.Hash{{0x02}, {0x03}, {0x05}, {0x06}}}, - isWaiting(map[string][]common.Hash{ - "B": {{0x05}, {0x06}}, + doTxNotify{peer: "B", hashes: []common.Hash{{0x02}, {0x03}, {0x05}, {0x06}}, types: []byte{types.LegacyTxType, types.LegacyTxType, types.LegacyTxType, types.LegacyTxType}, sizes: []uint32{222, 333, 555, 666}}, + isWaiting(map[string][]announce{ + "B": { + {common.Hash{0x05}, types.LegacyTxType, 555}, + {common.Hash{0x06}, types.LegacyTxType, 666}, + }, }), isScheduled{ - tracking: map[string][]common.Hash{ - "A": {{0x01}, {0x02}, {0x03}, {0x04}}, - "B": {{0x02}, {0x03}}, + tracking: map[string][]announce{ + "A": { + {common.Hash{0x01}, types.LegacyTxType, 111}, + {common.Hash{0x02}, types.LegacyTxType, 222}, + {common.Hash{0x03}, types.LegacyTxType, 333}, + {common.Hash{0x04}, types.LegacyTxType, 444}, + }, + "B": { + {common.Hash{0x02}, types.LegacyTxType, 222}, + {common.Hash{0x03}, types.LegacyTxType, 333}, + }, }, fetching: map[string][]common.Hash{ "A": {{0x01}, {0x02}}, @@ -547,17 +501,23 @@ func TestTransactionFetcherFailedRescheduling(t *testing.T) { }, steps: []interface{}{ // Push an initial announcement through to the scheduled stage - doTxNotify{peer: "A", hashes: []common.Hash{{0x01}, {0x02}}}, - isWaiting(map[string][]common.Hash{ - "A": {{0x01}, {0x02}}, + doTxNotify{peer: "A", hashes: []common.Hash{{0x01}, {0x02}}, types: []byte{types.LegacyTxType, types.LegacyTxType}, sizes: []uint32{111, 222}}, + isWaiting(map[string][]announce{ + "A": { + {common.Hash{0x01}, types.LegacyTxType, 111}, + {common.Hash{0x02}, types.LegacyTxType, 222}, + }, }), isScheduled{tracking: nil, fetching: nil}, doWait{time: txArriveTimeout, step: true}, isWaiting(nil), isScheduled{ - tracking: map[string][]common.Hash{ - "A": {{0x01}, {0x02}}, + tracking: map[string][]announce{ + "A": { + {common.Hash{0x01}, types.LegacyTxType, 111}, + {common.Hash{0x02}, types.LegacyTxType, 222}, + }, }, fetching: map[string][]common.Hash{ "A": {{0x01}, {0x02}}, @@ -565,12 +525,17 @@ func TestTransactionFetcherFailedRescheduling(t *testing.T) { }, // While the original peer is stuck in the request, push in an second // data source. - doTxNotify{peer: "B", hashes: []common.Hash{{0x02}}}, + doTxNotify{peer: "B", hashes: []common.Hash{{0x02}}, types: []byte{types.LegacyTxType}, sizes: []uint32{222}}, isWaiting(nil), isScheduled{ - tracking: map[string][]common.Hash{ - "A": {{0x01}, {0x02}}, - "B": {{0x02}}, + tracking: map[string][]announce{ + "A": { + {common.Hash{0x01}, types.LegacyTxType, 111}, + {common.Hash{0x02}, types.LegacyTxType, 222}, + }, + "B": { + {common.Hash{0x02}, types.LegacyTxType, 222}, + }, }, fetching: map[string][]common.Hash{ "A": {{0x01}, {0x02}}, @@ -584,8 +549,10 @@ func TestTransactionFetcherFailedRescheduling(t *testing.T) { doWait{time: 0, step: true}, isWaiting(nil), isScheduled{ - tracking: map[string][]common.Hash{ - "B": {{0x02}}, + tracking: map[string][]announce{ + "B": { + {common.Hash{0x02}, types.LegacyTxType, 222}, + }, }, fetching: map[string][]common.Hash{ "B": {{0x02}}, @@ -617,17 +584,21 @@ func TestTransactionFetcherCleanup(t *testing.T) { }, steps: []interface{}{ // Push an initial announcement through to the scheduled stage - doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0]}}, - isWaiting(map[string][]common.Hash{ - "A": {testTxsHashes[0]}, + doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0]}, types: []byte{testTxs[0].Type()}, sizes: []uint32{uint32(testTxs[0].Size())}}, + isWaiting(map[string][]announce{ + "A": { + {testTxsHashes[0], testTxs[0].Type(), uint32(testTxs[0].Size())}, + }, }), isScheduled{tracking: nil, fetching: nil}, doWait{time: txArriveTimeout, step: true}, isWaiting(nil), isScheduled{ - tracking: map[string][]common.Hash{ - "A": {testTxsHashes[0]}, + tracking: map[string][]announce{ + "A": { + {testTxsHashes[0], testTxs[0].Type(), uint32(testTxs[0].Size())}, + }, }, fetching: map[string][]common.Hash{ "A": {testTxsHashes[0]}, @@ -657,17 +628,21 @@ func TestTransactionFetcherCleanupEmpty(t *testing.T) { }, steps: []interface{}{ // Push an initial announcement through to the scheduled stage - doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0]}}, - isWaiting(map[string][]common.Hash{ - "A": {testTxsHashes[0]}, + doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0]}, types: []byte{testTxs[0].Type()}, sizes: []uint32{uint32(testTxs[0].Size())}}, + isWaiting(map[string][]announce{ + "A": { + {testTxsHashes[0], testTxs[0].Type(), uint32(testTxs[0].Size())}, + }, }), isScheduled{tracking: nil, fetching: nil}, doWait{time: txArriveTimeout, step: true}, isWaiting(nil), isScheduled{ - tracking: map[string][]common.Hash{ - "A": {testTxsHashes[0]}, + tracking: map[string][]announce{ + "A": { + {testTxsHashes[0], testTxs[0].Type(), uint32(testTxs[0].Size())}, + }, }, fetching: map[string][]common.Hash{ "A": {testTxsHashes[0]}, @@ -696,17 +671,29 @@ func TestTransactionFetcherMissingRescheduling(t *testing.T) { }, steps: []interface{}{ // Push an initial announcement through to the scheduled stage - doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0], testTxsHashes[1], testTxsHashes[2]}}, - isWaiting(map[string][]common.Hash{ - "A": {testTxsHashes[0], testTxsHashes[1], testTxsHashes[2]}, + doTxNotify{peer: "A", + hashes: []common.Hash{testTxsHashes[0], testTxsHashes[1], testTxsHashes[2]}, + types: []byte{testTxs[0].Type(), testTxs[1].Type(), testTxs[2].Type()}, + sizes: []uint32{uint32(testTxs[0].Size()), uint32(testTxs[1].Size()), uint32(testTxs[2].Size())}, + }, + isWaiting(map[string][]announce{ + "A": { + {testTxsHashes[0], testTxs[0].Type(), uint32(testTxs[0].Size())}, + {testTxsHashes[1], testTxs[1].Type(), uint32(testTxs[1].Size())}, + {testTxsHashes[2], testTxs[2].Type(), uint32(testTxs[2].Size())}, + }, }), isScheduled{tracking: nil, fetching: nil}, doWait{time: txArriveTimeout, step: true}, isWaiting(nil), isScheduled{ - tracking: map[string][]common.Hash{ - "A": {testTxsHashes[0], testTxsHashes[1], testTxsHashes[2]}, + tracking: map[string][]announce{ + "A": { + {testTxsHashes[0], testTxs[0].Type(), uint32(testTxs[0].Size())}, + {testTxsHashes[1], testTxs[1].Type(), uint32(testTxs[1].Size())}, + {testTxsHashes[2], testTxs[2].Type(), uint32(testTxs[2].Size())}, + }, }, fetching: map[string][]common.Hash{ "A": {testTxsHashes[0], testTxsHashes[1], testTxsHashes[2]}, @@ -714,10 +701,12 @@ func TestTransactionFetcherMissingRescheduling(t *testing.T) { }, // Deliver the middle transaction requested, the one before which // should be dropped and the one after re-requested. - doTxEnqueue{peer: "A", txs: []*types.Transaction{testTxs[0]}, direct: true}, // This depends on the deterministic random + doTxEnqueue{peer: "A", txs: []*types.Transaction{testTxs[1]}, direct: true}, isScheduled{ - tracking: map[string][]common.Hash{ - "A": {testTxsHashes[2]}, + tracking: map[string][]announce{ + "A": { + {testTxsHashes[2], testTxs[2].Type(), uint32(testTxs[2].Size())}, + }, }, fetching: map[string][]common.Hash{ "A": {testTxsHashes[2]}, @@ -743,17 +732,27 @@ func TestTransactionFetcherMissingCleanup(t *testing.T) { }, steps: []interface{}{ // Push an initial announcement through to the scheduled stage - doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0], testTxsHashes[1]}}, - isWaiting(map[string][]common.Hash{ - "A": {testTxsHashes[0], testTxsHashes[1]}, + doTxNotify{peer: "A", + hashes: []common.Hash{testTxsHashes[0], testTxsHashes[1]}, + types: []byte{testTxs[0].Type(), testTxs[1].Type()}, + sizes: []uint32{uint32(testTxs[0].Size()), uint32(testTxs[1].Size())}, + }, + isWaiting(map[string][]announce{ + "A": { + {testTxsHashes[0], testTxs[0].Type(), uint32(testTxs[0].Size())}, + {testTxsHashes[1], testTxs[1].Type(), uint32(testTxs[1].Size())}, + }, }), isScheduled{tracking: nil, fetching: nil}, doWait{time: txArriveTimeout, step: true}, isWaiting(nil), isScheduled{ - tracking: map[string][]common.Hash{ - "A": {testTxsHashes[0], testTxsHashes[1]}, + tracking: map[string][]announce{ + "A": { + {testTxsHashes[0], testTxs[0].Type(), uint32(testTxs[0].Size())}, + {testTxsHashes[1], testTxs[1].Type(), uint32(testTxs[1].Size())}, + }, }, fetching: map[string][]common.Hash{ "A": {testTxsHashes[0], testTxsHashes[1]}, @@ -782,18 +781,23 @@ func TestTransactionFetcherBroadcasts(t *testing.T) { }, steps: []interface{}{ // Set up three transactions to be in different stats, waiting, queued and fetching - doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0]}}, + doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0]}, types: []byte{testTxs[0].Type()}, sizes: []uint32{uint32(testTxs[0].Size())}}, doWait{time: txArriveTimeout, step: true}, - doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[1]}}, + doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[1]}, types: []byte{testTxs[1].Type()}, sizes: []uint32{uint32(testTxs[1].Size())}}, doWait{time: txArriveTimeout, step: true}, - doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[2]}}, + doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[2]}, types: []byte{testTxs[2].Type()}, sizes: []uint32{uint32(testTxs[2].Size())}}, - isWaiting(map[string][]common.Hash{ - "A": {testTxsHashes[2]}, + isWaiting(map[string][]announce{ + "A": { + {testTxsHashes[2], testTxs[2].Type(), uint32(testTxs[2].Size())}, + }, }), isScheduled{ - tracking: map[string][]common.Hash{ - "A": {testTxsHashes[0], testTxsHashes[1]}, + tracking: map[string][]announce{ + "A": { + {testTxsHashes[0], testTxs[0].Type(), uint32(testTxs[0].Size())}, + {testTxsHashes[1], testTxs[1].Type(), uint32(testTxs[1].Size())}, + }, }, fetching: map[string][]common.Hash{ "A": {testTxsHashes[0]}, @@ -830,29 +834,40 @@ func TestTransactionFetcherWaitTimerResets(t *testing.T) { ) }, steps: []interface{}{ - doTxNotify{peer: "A", hashes: []common.Hash{{0x01}}}, - isWaiting(map[string][]common.Hash{ - "A": {{0x01}}, + doTxNotify{peer: "A", hashes: []common.Hash{{0x01}}, types: []byte{types.LegacyTxType}, sizes: []uint32{111}}, + isWaiting(map[string][]announce{ + "A": { + {common.Hash{0x01}, types.LegacyTxType, 111}, + }, }), isScheduled{nil, nil, nil}, doWait{time: txArriveTimeout / 2, step: false}, - isWaiting(map[string][]common.Hash{ - "A": {{0x01}}, + isWaiting(map[string][]announce{ + "A": { + {common.Hash{0x01}, types.LegacyTxType, 111}, + }, }), isScheduled{nil, nil, nil}, - doTxNotify{peer: "A", hashes: []common.Hash{{0x02}}}, - isWaiting(map[string][]common.Hash{ - "A": {{0x01}, {0x02}}, + doTxNotify{peer: "A", hashes: []common.Hash{{0x02}}, types: []byte{types.LegacyTxType}, sizes: []uint32{222}}, + isWaiting(map[string][]announce{ + "A": { + {common.Hash{0x01}, types.LegacyTxType, 111}, + {common.Hash{0x02}, types.LegacyTxType, 222}, + }, }), isScheduled{nil, nil, nil}, doWait{time: txArriveTimeout / 2, step: true}, - isWaiting(map[string][]common.Hash{ - "A": {{0x02}}, + isWaiting(map[string][]announce{ + "A": { + {common.Hash{0x02}, types.LegacyTxType, 222}, + }, }), isScheduled{ - tracking: map[string][]common.Hash{ - "A": {{0x01}}, + tracking: map[string][]announce{ + "A": { + {common.Hash{0x01}, types.LegacyTxType, 111}, + }, }, fetching: map[string][]common.Hash{ "A": {{0x01}}, @@ -862,8 +877,11 @@ func TestTransactionFetcherWaitTimerResets(t *testing.T) { doWait{time: txArriveTimeout / 2, step: true}, isWaiting(nil), isScheduled{ - tracking: map[string][]common.Hash{ - "A": {{0x01}, {0x02}}, + tracking: map[string][]announce{ + "A": { + {common.Hash{0x01}, types.LegacyTxType, 111}, + {common.Hash{0x02}, types.LegacyTxType, 222}, + }, }, fetching: map[string][]common.Hash{ "A": {{0x01}}, @@ -889,17 +907,22 @@ func TestTransactionFetcherTimeoutRescheduling(t *testing.T) { }, steps: []interface{}{ // Push an initial announcement through to the scheduled stage - doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0]}}, - isWaiting(map[string][]common.Hash{ - "A": {testTxsHashes[0]}, + doTxNotify{ + peer: "A", + hashes: []common.Hash{testTxsHashes[0]}, + types: []byte{testTxs[0].Type()}, + sizes: []uint32{uint32(testTxs[0].Size())}, + }, + isWaiting(map[string][]announce{ + "A": {{testTxsHashes[0], testTxs[0].Type(), uint32(testTxs[0].Size())}}, }), isScheduled{tracking: nil, fetching: nil}, doWait{time: txArriveTimeout, step: true}, isWaiting(nil), isScheduled{ - tracking: map[string][]common.Hash{ - "A": {testTxsHashes[0]}, + tracking: map[string][]announce{ + "A": {{testTxsHashes[0], testTxs[0].Type(), uint32(testTxs[0].Size())}}, }, fetching: map[string][]common.Hash{ "A": {testTxsHashes[0]}, @@ -916,11 +939,16 @@ func TestTransactionFetcherTimeoutRescheduling(t *testing.T) { }, }, // Ensure that followup announcements don't get scheduled - doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[1]}}, + doTxNotify{ + peer: "A", + hashes: []common.Hash{testTxsHashes[1]}, + types: []byte{testTxs[1].Type()}, + sizes: []uint32{uint32(testTxs[1].Size())}, + }, doWait{time: txArriveTimeout, step: true}, isScheduled{ - tracking: map[string][]common.Hash{ - "A": {testTxsHashes[1]}, + tracking: map[string][]announce{ + "A": {{testTxsHashes[1], testTxs[1].Type(), uint32(testTxs[1].Size())}}, }, fetching: nil, dangling: map[string][]common.Hash{ @@ -931,8 +959,8 @@ func TestTransactionFetcherTimeoutRescheduling(t *testing.T) { doTxEnqueue{peer: "A", txs: []*types.Transaction{testTxs[0]}, direct: true}, isWaiting(nil), isScheduled{ - tracking: map[string][]common.Hash{ - "A": {testTxsHashes[1]}, + tracking: map[string][]announce{ + "A": {{testTxsHashes[1], testTxs[1].Type(), uint32(testTxs[1].Size())}}, }, fetching: map[string][]common.Hash{ "A": {testTxsHashes[1]}, @@ -954,16 +982,20 @@ func TestTransactionFetcherTimeoutTimerResets(t *testing.T) { ) }, steps: []interface{}{ - doTxNotify{peer: "A", hashes: []common.Hash{{0x01}}}, + doTxNotify{peer: "A", hashes: []common.Hash{{0x01}}, types: []byte{types.LegacyTxType}, sizes: []uint32{111}}, doWait{time: txArriveTimeout, step: true}, - doTxNotify{peer: "B", hashes: []common.Hash{{0x02}}}, + doTxNotify{peer: "B", hashes: []common.Hash{{0x02}}, types: []byte{types.LegacyTxType}, sizes: []uint32{222}}, doWait{time: txArriveTimeout, step: true}, isWaiting(nil), isScheduled{ - tracking: map[string][]common.Hash{ - "A": {{0x01}}, - "B": {{0x02}}, + tracking: map[string][]announce{ + "A": { + {common.Hash{0x01}, types.LegacyTxType, 111}, + }, + "B": { + {common.Hash{0x02}, types.LegacyTxType, 222}, + }, }, fetching: map[string][]common.Hash{ "A": {{0x01}}, @@ -972,8 +1004,10 @@ func TestTransactionFetcherTimeoutTimerResets(t *testing.T) { }, doWait{time: txFetchTimeout - txArriveTimeout, step: true}, isScheduled{ - tracking: map[string][]common.Hash{ - "B": {{0x02}}, + tracking: map[string][]announce{ + "B": { + {common.Hash{0x02}, types.LegacyTxType, 222}, + }, }, fetching: map[string][]common.Hash{ "B": {{0x02}}, @@ -999,9 +1033,22 @@ func TestTransactionFetcherTimeoutTimerResets(t *testing.T) { // number of them will be requested at a time. func TestTransactionFetcherRateLimiting(t *testing.T) { // Create a slew of transactions and announce them - var hashes []common.Hash + var ( + hashes []common.Hash + ts []byte + sizes []uint32 + announces []announce + ) for i := 0; i < maxTxAnnounces; i++ { - hashes = append(hashes, common.Hash{byte(i / 256), byte(i % 256)}) + hash := common.Hash{byte(i / 256), byte(i % 256)} + hashes = append(hashes, hash) + ts = append(ts, types.LegacyTxType) + sizes = append(sizes, 111) + announces = append(announces, announce{ + hash: hash, + kind: types.LegacyTxType, + size: 111, + }) } testTransactionFetcherParallel(t, txFetcherTest{ init: func() *TxFetcher { @@ -1015,15 +1062,15 @@ func TestTransactionFetcherRateLimiting(t *testing.T) { steps: []interface{}{ // Announce all the transactions, wait a bit and ensure only a small // percentage gets requested - doTxNotify{peer: "A", hashes: hashes}, + doTxNotify{peer: "A", hashes: hashes, types: ts, sizes: sizes}, doWait{time: txArriveTimeout, step: true}, isWaiting(nil), isScheduled{ - tracking: map[string][]common.Hash{ - "A": hashes, + tracking: map[string][]announce{ + "A": announces, }, fetching: map[string][]common.Hash{ - "A": hashes[1643 : 1643+maxTxRetrievals], + "A": hashes[:maxTxRetrievals], }, }, }, @@ -1065,27 +1112,27 @@ func TestTransactionFetcherBandwidthLimiting(t *testing.T) { }, doWait{time: txArriveTimeout, step: true}, isWaiting(nil), - isScheduledWithMeta{ + isScheduled{ tracking: map[string][]announce{ "A": { - {common.Hash{0x01}, typeptr(types.LegacyTxType), sizeptr(48 * 1024)}, - {common.Hash{0x02}, typeptr(types.LegacyTxType), sizeptr(48 * 1024)}, - {common.Hash{0x03}, typeptr(types.LegacyTxType), sizeptr(48 * 1024)}, - {common.Hash{0x04}, typeptr(types.LegacyTxType), sizeptr(48 * 1024)}, + {common.Hash{0x01}, types.LegacyTxType, 48 * 1024}, + {common.Hash{0x02}, types.LegacyTxType, 48 * 1024}, + {common.Hash{0x03}, types.LegacyTxType, 48 * 1024}, + {common.Hash{0x04}, types.LegacyTxType, 48 * 1024}, }, "B": { - {common.Hash{0x05}, typeptr(types.LegacyTxType), sizeptr(maxTxRetrievalSize)}, - {common.Hash{0x06}, typeptr(types.LegacyTxType), sizeptr(maxTxRetrievalSize)}, + {common.Hash{0x05}, types.LegacyTxType, maxTxRetrievalSize}, + {common.Hash{0x06}, types.LegacyTxType, maxTxRetrievalSize}, }, "C": { - {common.Hash{0x07}, typeptr(types.BlobTxType), sizeptr(params.MaxBlobGasPerBlock)}, - {common.Hash{0x08}, typeptr(types.BlobTxType), sizeptr(params.MaxBlobGasPerBlock)}, + {common.Hash{0x07}, types.BlobTxType, params.MaxBlobGasPerBlock}, + {common.Hash{0x08}, types.BlobTxType, params.MaxBlobGasPerBlock}, }, }, fetching: map[string][]common.Hash{ - "A": {{0x02}, {0x03}, {0x04}}, - "B": {{0x06}}, - "C": {{0x08}}, + "A": {{0x01}, {0x02}, {0x03}}, + "B": {{0x05}}, + "C": {{0x07}}, }, }, }, @@ -1096,13 +1143,41 @@ func TestTransactionFetcherBandwidthLimiting(t *testing.T) { // request at the same time is hard capped. func TestTransactionFetcherDoSProtection(t *testing.T) { // Create a slew of transactions and to announce them - var hashesA []common.Hash + var ( + hashesA []common.Hash + typesA []byte + sizesA []uint32 + announceA []announce + ) for i := 0; i < maxTxAnnounces+1; i++ { - hashesA = append(hashesA, common.Hash{0x01, byte(i / 256), byte(i % 256)}) + hash := common.Hash{0x01, byte(i / 256), byte(i % 256)} + hashesA = append(hashesA, hash) + typesA = append(typesA, types.LegacyTxType) + sizesA = append(sizesA, 111) + + announceA = append(announceA, announce{ + hash: hash, + kind: types.LegacyTxType, + size: 111, + }) } - var hashesB []common.Hash + var ( + hashesB []common.Hash + typesB []byte + sizesB []uint32 + announceB []announce + ) for i := 0; i < maxTxAnnounces+1; i++ { - hashesB = append(hashesB, common.Hash{0x02, byte(i / 256), byte(i % 256)}) + hash := common.Hash{0x02, byte(i / 256), byte(i % 256)} + hashesB = append(hashesB, hash) + typesB = append(typesB, types.LegacyTxType) + sizesB = append(sizesB, 111) + + announceB = append(announceB, announce{ + hash: hash, + kind: types.LegacyTxType, + size: 111, + }) } testTransactionFetcherParallel(t, txFetcherTest{ init: func() *TxFetcher { @@ -1115,45 +1190,45 @@ func TestTransactionFetcherDoSProtection(t *testing.T) { }, steps: []interface{}{ // Announce half of the transaction and wait for them to be scheduled - doTxNotify{peer: "A", hashes: hashesA[:maxTxAnnounces/2]}, - doTxNotify{peer: "B", hashes: hashesB[:maxTxAnnounces/2-1]}, + doTxNotify{peer: "A", hashes: hashesA[:maxTxAnnounces/2], types: typesA[:maxTxAnnounces/2], sizes: sizesA[:maxTxAnnounces/2]}, + doTxNotify{peer: "B", hashes: hashesB[:maxTxAnnounces/2-1], types: typesB[:maxTxAnnounces/2-1], sizes: sizesB[:maxTxAnnounces/2-1]}, doWait{time: txArriveTimeout, step: true}, // Announce the second half and keep them in the wait list - doTxNotify{peer: "A", hashes: hashesA[maxTxAnnounces/2 : maxTxAnnounces]}, - doTxNotify{peer: "B", hashes: hashesB[maxTxAnnounces/2-1 : maxTxAnnounces-1]}, + doTxNotify{peer: "A", hashes: hashesA[maxTxAnnounces/2 : maxTxAnnounces], types: typesA[maxTxAnnounces/2 : maxTxAnnounces], sizes: sizesA[maxTxAnnounces/2 : maxTxAnnounces]}, + doTxNotify{peer: "B", hashes: hashesB[maxTxAnnounces/2-1 : maxTxAnnounces-1], types: typesB[maxTxAnnounces/2-1 : maxTxAnnounces-1], sizes: sizesB[maxTxAnnounces/2-1 : maxTxAnnounces-1]}, // Ensure the hashes are split half and half - isWaiting(map[string][]common.Hash{ - "A": hashesA[maxTxAnnounces/2 : maxTxAnnounces], - "B": hashesB[maxTxAnnounces/2-1 : maxTxAnnounces-1], + isWaiting(map[string][]announce{ + "A": announceA[maxTxAnnounces/2 : maxTxAnnounces], + "B": announceB[maxTxAnnounces/2-1 : maxTxAnnounces-1], }), isScheduled{ - tracking: map[string][]common.Hash{ - "A": hashesA[:maxTxAnnounces/2], - "B": hashesB[:maxTxAnnounces/2-1], + tracking: map[string][]announce{ + "A": announceA[:maxTxAnnounces/2], + "B": announceB[:maxTxAnnounces/2-1], }, fetching: map[string][]common.Hash{ - "A": hashesA[1643 : 1643+maxTxRetrievals], - "B": append(append([]common.Hash{}, hashesB[maxTxAnnounces/2-3:maxTxAnnounces/2-1]...), hashesB[:maxTxRetrievals-2]...), + "A": hashesA[:maxTxRetrievals], + "B": hashesB[:maxTxRetrievals], }, }, // Ensure that adding even one more hash results in dropping the hash - doTxNotify{peer: "A", hashes: []common.Hash{hashesA[maxTxAnnounces]}}, - doTxNotify{peer: "B", hashes: hashesB[maxTxAnnounces-1 : maxTxAnnounces+1]}, + doTxNotify{peer: "A", hashes: []common.Hash{hashesA[maxTxAnnounces]}, types: []byte{typesA[maxTxAnnounces]}, sizes: []uint32{sizesA[maxTxAnnounces]}}, + doTxNotify{peer: "B", hashes: hashesB[maxTxAnnounces-1 : maxTxAnnounces+1], types: typesB[maxTxAnnounces-1 : maxTxAnnounces+1], sizes: sizesB[maxTxAnnounces-1 : maxTxAnnounces+1]}, - isWaiting(map[string][]common.Hash{ - "A": hashesA[maxTxAnnounces/2 : maxTxAnnounces], - "B": hashesB[maxTxAnnounces/2-1 : maxTxAnnounces], + isWaiting(map[string][]announce{ + "A": announceA[maxTxAnnounces/2 : maxTxAnnounces], + "B": announceB[maxTxAnnounces/2-1 : maxTxAnnounces], }), isScheduled{ - tracking: map[string][]common.Hash{ - "A": hashesA[:maxTxAnnounces/2], - "B": hashesB[:maxTxAnnounces/2-1], + tracking: map[string][]announce{ + "A": announceA[:maxTxAnnounces/2], + "B": announceB[:maxTxAnnounces/2-1], }, fetching: map[string][]common.Hash{ - "A": hashesA[1643 : 1643+maxTxRetrievals], - "B": append(append([]common.Hash{}, hashesB[maxTxAnnounces/2-3:maxTxAnnounces/2-1]...), hashesB[:maxTxRetrievals-2]...), + "A": hashesA[:maxTxRetrievals], + "B": hashesB[:maxTxRetrievals], }, }, }, @@ -1183,15 +1258,23 @@ func TestTransactionFetcherUnderpricedDedup(t *testing.T) { }, steps: []interface{}{ // Deliver a transaction through the fetcher, but reject as underpriced - doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0], testTxsHashes[1]}}, + doTxNotify{peer: "A", + hashes: []common.Hash{testTxsHashes[0], testTxsHashes[1]}, + types: []byte{testTxs[0].Type(), testTxs[1].Type()}, + sizes: []uint32{uint32(testTxs[0].Size()), uint32(testTxs[1].Size())}, + }, doWait{time: txArriveTimeout, step: true}, doTxEnqueue{peer: "A", txs: []*types.Transaction{testTxs[0], testTxs[1]}, direct: true}, isScheduled{nil, nil, nil}, // Try to announce the transaction again, ensure it's not scheduled back - doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0], testTxsHashes[1], testTxsHashes[2]}}, // [2] is needed to force a step in the fetcher - isWaiting(map[string][]common.Hash{ - "A": {testTxsHashes[2]}, + doTxNotify{peer: "A", + hashes: []common.Hash{testTxsHashes[0], testTxsHashes[1], testTxsHashes[2]}, + types: []byte{testTxs[0].Type(), testTxs[1].Type(), testTxs[2].Type()}, + sizes: []uint32{uint32(testTxs[0].Size()), uint32(testTxs[1].Size()), uint32(testTxs[2].Size())}, + }, // [2] is needed to force a step in the fetcher + isWaiting(map[string][]announce{ + "A": {{testTxsHashes[2], testTxs[2].Type(), uint32(testTxs[2].Size())}}, }), isScheduled{nil, nil, nil}, }, @@ -1211,21 +1294,38 @@ func TestTransactionFetcherUnderpricedDoSProtection(t *testing.T) { for i := 0; i < maxTxUnderpricedSetSize+1; i++ { txs = append(txs, types.NewTransaction(rand.Uint64(), common.Address{byte(rand.Intn(256))}, new(big.Int), 0, new(big.Int), nil)) } - hashes := make([]common.Hash, len(txs)) - for i, tx := range txs { - hashes[i] = tx.Hash() + var ( + hashes []common.Hash + ts []byte + sizes []uint32 + annos []announce + ) + for _, tx := range txs { + hashes = append(hashes, tx.Hash()) + ts = append(ts, tx.Type()) + sizes = append(sizes, uint32(tx.Size())) + annos = append(annos, announce{ + hash: tx.Hash(), + kind: tx.Type(), + size: uint32(tx.Size()), + }) } // Generate a set of steps to announce and deliver the entire set of transactions var steps []interface{} for i := 0; i < maxTxUnderpricedSetSize/maxTxRetrievals; i++ { - steps = append(steps, doTxNotify{peer: "A", hashes: hashes[i*maxTxRetrievals : (i+1)*maxTxRetrievals]}) - steps = append(steps, isWaiting(map[string][]common.Hash{ - "A": hashes[i*maxTxRetrievals : (i+1)*maxTxRetrievals], + steps = append(steps, doTxNotify{ + peer: "A", + hashes: hashes[i*maxTxRetrievals : (i+1)*maxTxRetrievals], + types: ts[i*maxTxRetrievals : (i+1)*maxTxRetrievals], + sizes: sizes[i*maxTxRetrievals : (i+1)*maxTxRetrievals], + }) + steps = append(steps, isWaiting(map[string][]announce{ + "A": annos[i*maxTxRetrievals : (i+1)*maxTxRetrievals], })) steps = append(steps, doWait{time: txArriveTimeout, step: true}) steps = append(steps, isScheduled{ - tracking: map[string][]common.Hash{ - "A": hashes[i*maxTxRetrievals : (i+1)*maxTxRetrievals], + tracking: map[string][]announce{ + "A": annos[i*maxTxRetrievals : (i+1)*maxTxRetrievals], }, fetching: map[string][]common.Hash{ "A": hashes[i*maxTxRetrievals : (i+1)*maxTxRetrievals], @@ -1253,7 +1353,12 @@ func TestTransactionFetcherUnderpricedDoSProtection(t *testing.T) { }, steps: append(steps, []interface{}{ // The preparation of the test has already been done in `steps`, add the last check - doTxNotify{peer: "A", hashes: []common.Hash{hashes[maxTxUnderpricedSetSize]}}, + doTxNotify{ + peer: "A", + hashes: []common.Hash{hashes[maxTxUnderpricedSetSize]}, + types: []byte{ts[maxTxUnderpricedSetSize]}, + sizes: []uint32{sizes[maxTxUnderpricedSetSize]}, + }, doWait{time: txArriveTimeout, step: true}, doTxEnqueue{peer: "A", txs: []*types.Transaction{txs[maxTxUnderpricedSetSize]}, direct: true}, isUnderpriced(maxTxUnderpricedSetSize), @@ -1283,18 +1388,23 @@ func TestTransactionFetcherOutOfBoundDeliveries(t *testing.T) { isScheduled{nil, nil, nil}, // Set up a few hashes into various stages - doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0]}}, + doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0]}, types: []byte{testTxs[0].Type()}, sizes: []uint32{uint32(testTxs[0].Size())}}, doWait{time: txArriveTimeout, step: true}, - doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[1]}}, + doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[1]}, types: []byte{testTxs[1].Type()}, sizes: []uint32{uint32(testTxs[1].Size())}}, doWait{time: txArriveTimeout, step: true}, - doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[2]}}, + doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[2]}, types: []byte{testTxs[2].Type()}, sizes: []uint32{uint32(testTxs[2].Size())}}, - isWaiting(map[string][]common.Hash{ - "A": {testTxsHashes[2]}, + isWaiting(map[string][]announce{ + "A": { + {testTxsHashes[2], testTxs[2].Type(), uint32(testTxs[2].Size())}, + }, }), isScheduled{ - tracking: map[string][]common.Hash{ - "A": {testTxsHashes[0], testTxsHashes[1]}, + tracking: map[string][]announce{ + "A": { + {testTxsHashes[0], testTxs[0].Type(), uint32(testTxs[0].Size())}, + {testTxsHashes[1], testTxs[1].Type(), uint32(testTxs[1].Size())}, + }, }, fetching: map[string][]common.Hash{ "A": {testTxsHashes[0]}, @@ -1330,18 +1440,23 @@ func TestTransactionFetcherDrop(t *testing.T) { }, steps: []interface{}{ // Set up a few hashes into various stages - doTxNotify{peer: "A", hashes: []common.Hash{{0x01}}}, + doTxNotify{peer: "A", hashes: []common.Hash{{0x01}}, types: []byte{types.LegacyTxType}, sizes: []uint32{111}}, doWait{time: txArriveTimeout, step: true}, - doTxNotify{peer: "A", hashes: []common.Hash{{0x02}}}, + doTxNotify{peer: "A", hashes: []common.Hash{{0x02}}, types: []byte{types.LegacyTxType}, sizes: []uint32{222}}, doWait{time: txArriveTimeout, step: true}, - doTxNotify{peer: "A", hashes: []common.Hash{{0x03}}}, + doTxNotify{peer: "A", hashes: []common.Hash{{0x03}}, types: []byte{types.LegacyTxType}, sizes: []uint32{333}}, - isWaiting(map[string][]common.Hash{ - "A": {{0x03}}, + isWaiting(map[string][]announce{ + "A": { + {common.Hash{0x03}, types.LegacyTxType, 333}, + }, }), isScheduled{ - tracking: map[string][]common.Hash{ - "A": {{0x01}, {0x02}}, + tracking: map[string][]announce{ + "A": { + {common.Hash{0x01}, types.LegacyTxType, 111}, + {common.Hash{0x02}, types.LegacyTxType, 222}, + }, }, fetching: map[string][]common.Hash{ "A": {{0x01}}, @@ -1353,12 +1468,14 @@ func TestTransactionFetcherDrop(t *testing.T) { isScheduled{nil, nil, nil}, // Push the node into a dangling (timeout) state - doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0]}}, + doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0]}, types: []byte{testTxs[0].Type()}, sizes: []uint32{uint32(testTxs[0].Size())}}, doWait{time: txArriveTimeout, step: true}, isWaiting(nil), isScheduled{ - tracking: map[string][]common.Hash{ - "A": {testTxsHashes[0]}, + tracking: map[string][]announce{ + "A": { + {testTxsHashes[0], testTxs[0].Type(), uint32(testTxs[0].Size())}, + }, }, fetching: map[string][]common.Hash{ "A": {testTxsHashes[0]}, @@ -1397,15 +1514,15 @@ func TestTransactionFetcherDropRescheduling(t *testing.T) { }, steps: []interface{}{ // Set up a few hashes into various stages - doTxNotify{peer: "A", hashes: []common.Hash{{0x01}}}, + doTxNotify{peer: "A", hashes: []common.Hash{{0x01}}, types: []byte{types.LegacyTxType}, sizes: []uint32{111}}, doWait{time: txArriveTimeout, step: true}, - doTxNotify{peer: "B", hashes: []common.Hash{{0x01}}}, + doTxNotify{peer: "B", hashes: []common.Hash{{0x01}}, types: []byte{types.LegacyTxType}, sizes: []uint32{111}}, isWaiting(nil), isScheduled{ - tracking: map[string][]common.Hash{ - "A": {{0x01}}, - "B": {{0x01}}, + tracking: map[string][]announce{ + "A": {{common.Hash{0x01}, types.LegacyTxType, 111}}, + "B": {{common.Hash{0x01}, types.LegacyTxType, 111}}, }, fetching: map[string][]common.Hash{ "A": {{0x01}}, @@ -1415,8 +1532,8 @@ func TestTransactionFetcherDropRescheduling(t *testing.T) { doDrop("A"), isWaiting(nil), isScheduled{ - tracking: map[string][]common.Hash{ - "B": {{0x01}}, + tracking: map[string][]announce{ + "B": {{common.Hash{0x01}, types.LegacyTxType, 111}}, }, fetching: map[string][]common.Hash{ "B": {{0x01}}, @@ -1443,42 +1560,57 @@ func TestInvalidAnnounceMetadata(t *testing.T) { }, steps: []interface{}{ // Initial announcement to get something into the waitlist - doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0], testTxsHashes[1]}, types: []byte{testTxs[0].Type(), testTxs[1].Type()}, sizes: []uint32{uint32(testTxs[0].Size()), uint32(testTxs[1].Size())}}, - isWaitingWithMeta(map[string][]announce{ + doTxNotify{ + peer: "A", + hashes: []common.Hash{testTxsHashes[0], testTxsHashes[1]}, + types: []byte{testTxs[0].Type(), testTxs[1].Type()}, + sizes: []uint32{uint32(testTxs[0].Size()), uint32(testTxs[1].Size())}, + }, + isWaiting(map[string][]announce{ "A": { - {testTxsHashes[0], typeptr(testTxs[0].Type()), sizeptr(uint32(testTxs[0].Size()))}, - {testTxsHashes[1], typeptr(testTxs[1].Type()), sizeptr(uint32(testTxs[1].Size()))}, + {testTxsHashes[0], testTxs[0].Type(), uint32(testTxs[0].Size())}, + {testTxsHashes[1], testTxs[1].Type(), uint32(testTxs[1].Size())}, }, }), // Announce from new peers conflicting transactions - doTxNotify{peer: "B", hashes: []common.Hash{testTxsHashes[0]}, types: []byte{testTxs[0].Type()}, sizes: []uint32{1024 + uint32(testTxs[0].Size())}}, - doTxNotify{peer: "C", hashes: []common.Hash{testTxsHashes[1]}, types: []byte{1 + testTxs[1].Type()}, sizes: []uint32{uint32(testTxs[1].Size())}}, - isWaitingWithMeta(map[string][]announce{ + doTxNotify{ + peer: "B", + hashes: []common.Hash{testTxsHashes[0]}, + types: []byte{testTxs[0].Type()}, + sizes: []uint32{1024 + uint32(testTxs[0].Size())}, + }, + doTxNotify{ + peer: "C", + hashes: []common.Hash{testTxsHashes[1]}, + types: []byte{1 + testTxs[1].Type()}, + sizes: []uint32{uint32(testTxs[1].Size())}, + }, + isWaiting(map[string][]announce{ "A": { - {testTxsHashes[0], typeptr(testTxs[0].Type()), sizeptr(uint32(testTxs[0].Size()))}, - {testTxsHashes[1], typeptr(testTxs[1].Type()), sizeptr(uint32(testTxs[1].Size()))}, + {testTxsHashes[0], testTxs[0].Type(), uint32(testTxs[0].Size())}, + {testTxsHashes[1], testTxs[1].Type(), uint32(testTxs[1].Size())}, }, "B": { - {testTxsHashes[0], typeptr(testTxs[0].Type()), sizeptr(1024 + uint32(testTxs[0].Size()))}, + {testTxsHashes[0], testTxs[0].Type(), 1024 + uint32(testTxs[0].Size())}, }, "C": { - {testTxsHashes[1], typeptr(1 + testTxs[1].Type()), sizeptr(uint32(testTxs[1].Size()))}, + {testTxsHashes[1], 1 + testTxs[1].Type(), uint32(testTxs[1].Size())}, }, }), // Schedule all the transactions for retrieval doWait{time: txArriveTimeout, step: true}, - isWaitingWithMeta(nil), - isScheduledWithMeta{ + isWaiting(nil), + isScheduled{ tracking: map[string][]announce{ "A": { - {testTxsHashes[0], typeptr(testTxs[0].Type()), sizeptr(uint32(testTxs[0].Size()))}, - {testTxsHashes[1], typeptr(testTxs[1].Type()), sizeptr(uint32(testTxs[1].Size()))}, + {testTxsHashes[0], testTxs[0].Type(), uint32(testTxs[0].Size())}, + {testTxsHashes[1], testTxs[1].Type(), uint32(testTxs[1].Size())}, }, "B": { - {testTxsHashes[0], typeptr(testTxs[0].Type()), sizeptr(1024 + uint32(testTxs[0].Size()))}, + {testTxsHashes[0], testTxs[0].Type(), 1024 + uint32(testTxs[0].Size())}, }, "C": { - {testTxsHashes[1], typeptr(1 + testTxs[1].Type()), sizeptr(uint32(testTxs[1].Size()))}, + {testTxsHashes[1], 1 + testTxs[1].Type(), uint32(testTxs[1].Size())}, }, }, fetching: map[string][]common.Hash{ @@ -1511,12 +1643,12 @@ func TestTransactionFetcherFuzzCrash01(t *testing.T) { }, steps: []interface{}{ // Get a transaction into fetching mode and make it dangling with a broadcast - doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0]}}, + doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0]}, types: []byte{testTxs[0].Type()}, sizes: []uint32{uint32(testTxs[0].Size())}}, doWait{time: txArriveTimeout, step: true}, doTxEnqueue{peer: "A", txs: []*types.Transaction{testTxs[0]}}, // Notify the dangling transaction once more and crash via a timeout - doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0]}}, + doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0]}, types: []byte{testTxs[0].Type()}, sizes: []uint32{uint32(testTxs[0].Size())}}, doWait{time: txFetchTimeout, step: true}, }, }) @@ -1539,12 +1671,12 @@ func TestTransactionFetcherFuzzCrash02(t *testing.T) { }, steps: []interface{}{ // Get a transaction into fetching mode and make it dangling with a broadcast - doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0]}}, + doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0]}, types: []byte{testTxs[0].Type()}, sizes: []uint32{uint32(testTxs[0].Size())}}, doWait{time: txArriveTimeout, step: true}, doTxEnqueue{peer: "A", txs: []*types.Transaction{testTxs[0]}}, // Notify the dangling transaction once more, re-fetch, and crash via a drop and timeout - doTxNotify{peer: "B", hashes: []common.Hash{testTxsHashes[0]}}, + doTxNotify{peer: "B", hashes: []common.Hash{testTxsHashes[0]}, types: []byte{testTxs[0].Type()}, sizes: []uint32{uint32(testTxs[0].Size())}}, doWait{time: txArriveTimeout, step: true}, doDrop("A"), doWait{time: txFetchTimeout, step: true}, @@ -1569,12 +1701,17 @@ func TestTransactionFetcherFuzzCrash03(t *testing.T) { }, steps: []interface{}{ // Get a transaction into fetching mode and make it dangling with a broadcast - doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0], testTxsHashes[1]}}, + doTxNotify{ + peer: "A", + hashes: []common.Hash{testTxsHashes[0], testTxsHashes[1]}, + types: []byte{testTxs[0].Type(), testTxs[1].Type()}, + sizes: []uint32{uint32(testTxs[0].Size()), uint32(testTxs[1].Size())}, + }, doWait{time: txFetchTimeout, step: true}, doTxEnqueue{peer: "A", txs: []*types.Transaction{testTxs[0], testTxs[1]}}, // Notify the dangling transaction once more, partially deliver, clash&crash with a timeout - doTxNotify{peer: "B", hashes: []common.Hash{testTxsHashes[0]}}, + doTxNotify{peer: "B", hashes: []common.Hash{testTxsHashes[0]}, types: []byte{testTxs[0].Type()}, sizes: []uint32{uint32(testTxs[0].Size())}}, doWait{time: txArriveTimeout, step: true}, doTxEnqueue{peer: "A", txs: []*types.Transaction{testTxs[1]}, direct: true}, @@ -1606,12 +1743,12 @@ func TestTransactionFetcherFuzzCrash04(t *testing.T) { }, steps: []interface{}{ // Get a transaction into fetching mode and make it dangling with a broadcast - doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0]}}, + doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0]}, types: []byte{testTxs[0].Type()}, sizes: []uint32{uint32(testTxs[0].Size())}}, doWait{time: txArriveTimeout, step: true}, doTxEnqueue{peer: "A", txs: []*types.Transaction{testTxs[0]}}, // Notify the dangling transaction once more, re-fetch, and crash via an in-flight disconnect - doTxNotify{peer: "B", hashes: []common.Hash{testTxsHashes[0]}}, + doTxNotify{peer: "B", hashes: []common.Hash{testTxsHashes[0]}, types: []byte{testTxs[0].Type()}, sizes: []uint32{uint32(testTxs[0].Size())}}, doWait{time: txArriveTimeout, step: true}, doFunc(func() { proceed <- struct{}{} // Allow peer A to return the failure @@ -1622,6 +1759,76 @@ func TestTransactionFetcherFuzzCrash04(t *testing.T) { }) } +// This test ensures the blob transactions will be scheduled for fetching +// once they are announced in the network. +func TestBlobTransactionAnnounce(t *testing.T) { + testTransactionFetcherParallel(t, txFetcherTest{ + init: func() *TxFetcher { + return NewTxFetcher( + func(common.Hash) bool { return false }, + nil, + func(string, []common.Hash) error { return nil }, + nil, + ) + }, + steps: []interface{}{ + // Initial announcement to get something into the waitlist + doTxNotify{peer: "A", hashes: []common.Hash{{0x01}, {0x02}}, types: []byte{types.LegacyTxType, types.LegacyTxType}, sizes: []uint32{111, 222}}, + isWaiting(map[string][]announce{ + "A": { + {common.Hash{0x01}, types.LegacyTxType, 111}, + {common.Hash{0x02}, types.LegacyTxType, 222}, + }, + }), + // Announce a blob transaction + doTxNotify{peer: "B", hashes: []common.Hash{{0x03}}, types: []byte{types.BlobTxType}, sizes: []uint32{333}}, + isWaiting(map[string][]announce{ + "A": { + {common.Hash{0x01}, types.LegacyTxType, 111}, + {common.Hash{0x02}, types.LegacyTxType, 222}, + }, + "B": { + {common.Hash{0x03}, types.BlobTxType, 333}, + }, + }), + doWait{time: 0, step: true}, // zero time, but the blob fetching should be scheduled + isWaiting(map[string][]announce{ + "A": { + {common.Hash{0x01}, types.LegacyTxType, 111}, + {common.Hash{0x02}, types.LegacyTxType, 222}, + }, + }), + isScheduled{ + tracking: map[string][]announce{ + "B": { + {common.Hash{0x03}, types.BlobTxType, 333}, + }, + }, + fetching: map[string][]common.Hash{ // Depends on deterministic test randomizer + "B": {{0x03}}, + }, + }, + doWait{time: txArriveTimeout, step: true}, // zero time, but the blob fetching should be scheduled + isWaiting(nil), + isScheduled{ + tracking: map[string][]announce{ + "A": { + {common.Hash{0x01}, types.LegacyTxType, 111}, + {common.Hash{0x02}, types.LegacyTxType, 222}, + }, + "B": { + {common.Hash{0x03}, types.BlobTxType, 333}, + }, + }, + fetching: map[string][]common.Hash{ // Depends on deterministic test randomizer + "A": {{0x01}, {0x02}}, + "B": {{0x03}}, + }, + }, + }, + }) +} + func testTransactionFetcherParallel(t *testing.T, tt txFetcherTest) { t.Parallel() testTransactionFetcher(t, tt) @@ -1652,30 +1859,6 @@ func testTransactionFetcher(t *testing.T, tt txFetcherTest) { // Crunch through all the test steps and execute them for i, step := range tt.steps { - // Auto-expand certain steps to ones with metadata - switch old := step.(type) { - case isWaiting: - new := make(isWaitingWithMeta) - for peer, hashes := range old { - for _, hash := range hashes { - new[peer] = append(new[peer], announce{hash, nil, nil}) - } - } - step = new - - case isScheduled: - new := isScheduledWithMeta{ - tracking: make(map[string][]announce), - fetching: old.fetching, - dangling: old.dangling, - } - for peer, hashes := range old.tracking { - for _, hash := range hashes { - new.tracking[peer] = append(new.tracking[peer], announce{hash, nil, nil}) - } - } - step = new - } // Process the original or expanded steps switch step := step.(type) { case doTxNotify: @@ -1710,7 +1893,7 @@ func testTransactionFetcher(t *testing.T, tt txFetcherTest) { case doFunc: step() - case isWaitingWithMeta: + case isWaiting: // We need to check that the waiting list (stage 1) internals // match with the expected set. Check the peer->hash mappings // first. @@ -1724,18 +1907,13 @@ func testTransactionFetcher(t *testing.T, tt txFetcherTest) { if meta, ok := waiting[ann.hash]; !ok { t.Errorf("step %d, peer %s: hash %x missing from waitslots", i, peer, ann.hash) } else { - if (meta == nil && (ann.kind != nil || ann.size != nil)) || - (meta != nil && (ann.kind == nil || ann.size == nil)) || - (meta != nil && (meta.kind != *ann.kind || meta.size != *ann.size)) { - t.Errorf("step %d, peer %s, hash %x: waitslot metadata mismatch: want %v, have %v/%v", i, peer, ann.hash, meta, *ann.kind, *ann.size) + if meta.kind != ann.kind || meta.size != ann.size { + t.Errorf("step %d, peer %s, hash %x: waitslot metadata mismatch: want %v, have %v/%v", i, peer, ann.hash, meta, ann.kind, ann.size) } } } for hash, meta := range waiting { - ann := announce{hash: hash} - if meta != nil { - ann.kind, ann.size = &meta.kind, &meta.size - } + ann := announce{hash: hash, kind: meta.kind, size: meta.size} if !containsAnnounce(announces, ann) { t.Errorf("step %d, peer %s: announce %v extra in waitslots", i, peer, ann) } @@ -1780,7 +1958,7 @@ func testTransactionFetcher(t *testing.T, tt txFetcherTest) { } } - case isScheduledWithMeta: + case isScheduled: // Check that all scheduled announces are accounted for and no // extra ones are present. for peer, announces := range step.tracking { @@ -1793,18 +1971,13 @@ func testTransactionFetcher(t *testing.T, tt txFetcherTest) { if meta, ok := scheduled[ann.hash]; !ok { t.Errorf("step %d, peer %s: hash %x missing from announces", i, peer, ann.hash) } else { - if (meta == nil && (ann.kind != nil || ann.size != nil)) || - (meta != nil && (ann.kind == nil || ann.size == nil)) || - (meta != nil && (meta.kind != *ann.kind || meta.size != *ann.size)) { - t.Errorf("step %d, peer %s, hash %x: announce metadata mismatch: want %v, have %v/%v", i, peer, ann.hash, meta, *ann.kind, *ann.size) + if meta.kind != ann.kind || meta.size != ann.size { + t.Errorf("step %d, peer %s, hash %x: announce metadata mismatch: want %v, have %v/%v", i, peer, ann.hash, meta, ann.kind, ann.size) } } } for hash, meta := range scheduled { - ann := announce{hash: hash} - if meta != nil { - ann.kind, ann.size = &meta.kind, &meta.size - } + ann := announce{hash: hash, kind: meta.kind, size: meta.size} if !containsAnnounce(announces, ann) { t.Errorf("step %d, peer %s: announce %x extra in announces", i, peer, hash) } @@ -1954,18 +2127,10 @@ func testTransactionFetcher(t *testing.T, tt txFetcherTest) { func containsAnnounce(slice []announce, ann announce) bool { for _, have := range slice { if have.hash == ann.hash { - if have.kind == nil || ann.kind == nil { - if have.kind != ann.kind { - return false - } - } else if *have.kind != *ann.kind { + if have.kind != ann.kind { return false } - if have.size == nil || ann.size == nil { - if have.size != ann.size { - return false - } - } else if *have.size != *ann.size { + if have.size != ann.size { return false } return true diff --git a/go.mod b/go.mod index ae6b12c0b8e6..909add8be662 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/btcsuite/btcd/btcec/v2 v2.3.4 github.com/cespare/cp v0.1.0 github.com/cloudflare/cloudflare-go v0.79.0 - github.com/cockroachdb/pebble v1.1.1 + github.com/cockroachdb/pebble v1.1.2 github.com/consensys/gnark-crypto v0.12.1 github.com/crate-crypto/go-ipa v0.0.0-20240223125850-b1e8a79f509c github.com/crate-crypto/go-kzg-4844 v1.0.0 diff --git a/go.sum b/go.sum index 4dadd40a75f0..089c05c2ecb7 100644 --- a/go.sum +++ b/go.sum @@ -122,8 +122,8 @@ github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce h1:giXvy4KSc/6g/e github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce/go.mod h1:9/y3cnZ5GKakj/H4y9r9GTjCvAFta7KLgSHPJJYc52M= github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE= github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= -github.com/cockroachdb/pebble v1.1.1 h1:XnKU22oiCLy2Xn8vp1re67cXg4SAasg/WDt1NtcRFaw= -github.com/cockroachdb/pebble v1.1.1/go.mod h1:4exszw1r40423ZsmkG/09AFEG83I0uDgfujJdbL6kYU= +github.com/cockroachdb/pebble v1.1.2 h1:CUh2IPtR4swHlEj48Rhfzw6l/d0qA31fItcIszQVIsA= +github.com/cockroachdb/pebble v1.1.2/go.mod h1:4exszw1r40423ZsmkG/09AFEG83I0uDgfujJdbL6kYU= github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30= github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo=