diff --git a/CHANGELOG.md b/CHANGELOG.md index 66b443cdf8..cbeec06c0f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,23 @@ # Changelog +## v1.6.1 +v1.6.1-alpha is a preview release, which fixes several issues of the v1.6.0-alpha, it is more reliable than v1.6.0-alpha, so mark it as beta stage. + +### FEATURE +NA + +### BUGFIX +[\#3336](https://github.com/bnb-chain/bsc/pull/3336) miner: avoid to commit a bid twice +[\#3347](https://github.com/bnb-chain/bsc/pull/3347) fix: discovery AyncFilter deadlock on shutdown +[\#3340](https://github.com/bnb-chain/bsc/pull/3340) core: rework fast node + +### IMPROVEMENT +[\#3337](https://github.com/bnb-chain/bsc/pull/3337) eth/pebble: use NoSync as write mode +[\#3332](https://github.com/bnb-chain/bsc/pull/3332) FilterMap: update bsc checkpoint file +[\#3324](https://github.com/bnb-chain/bsc/pull/3324) eth/downloader: remove InsertHeaderChain to improve sync speed +[\#3319](https://github.com/bnb-chain/bsc/pull/3319) core/rawdb: remove func AncientOffSet and ItemAmountInAncient +[\#3346](https://github.com/bnb-chain/bsc/pull/3346) cmd/geth: remove subcmd hbss2pbss and insecure-prune-all +[\#3354](https://github.com/bnb-chain/bsc/pull/3354) freezer: add debug log for out of bounds access + ## v1.6.0 v1.6.0-alpha is a preview release for upstream code sync, it catches up with [go-ethereum release [v1.16.1]](https://github.com/ethereum/go-ethereum/releases/tag/v1.16.1) and also inlcude several bug fix. diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 239ef3a1d3..021322fd90 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -146,6 +146,7 @@ var ( utils.MinerRecommitIntervalFlag, utils.MinerNewPayloadTimeoutFlag, // deprecated utils.MinerDelayLeftoverFlag, + utils.EnableBALFlag, // utils.MinerNewPayloadTimeout, utils.NATFlag, utils.NoDiscoverFlag, diff --git a/cmd/jsutils/getchainstatus.js b/cmd/jsutils/getchainstatus.js index 4ef5d68262..1eab7fddf4 100644 --- a/cmd/jsutils/getchainstatus.js +++ b/cmd/jsutils/getchainstatus.js @@ -172,6 +172,7 @@ const validatorMap = new Map([ ['0x18c44f4FBEde9826C7f257d500A65a3D5A8edebc', [ 'Nozti' , '0x95E105468b3a9E158df258ee385CA873cB566bF2', '0xa76a951b947eda0b4585730049bf08338c0e679071127f0f2f7e7dce542a655d69b24e7af4586ed20efc2764044c0b3c']], ['0xEdB69D7AE8fE7c21a33e0491e76db241C8e09a5F', ['BlkRazor' , '0x5eBAf404d466a1cc2d02684B6A3bB1D43dCB7586', '0xb23e281776590409333b1b36019390f7fadce505f55bfb98969cd3df6660bfe873b73e73d28aeef04bac40e3f4520df1']], ['0xd6Ab358AD430F65EB4Aa5a1598FF2c34489dcfdE', [ 'Saturn' , '0x54A9c15A143dBFf49539dfF4e095ec8D09464A4A', '0x835a7608cb0888fa649aa4120e81b1ab8c54c894e01b3b1d8c458563bec901ba6bb0c5f356dca5b962392872480f3b4c']], + ['0xCc767841fbB5b79B91EdF7a19EC5bd2F3D334fD8', [ 'Kraken' , '0x4279baBE4293c0826810b6C59e40F9DA9e5fd45b', '0xaa7a4c76d38b9fe7f872bbc53ac172faa56c7db2ad4b4aea3af41de2c2df7738e82827f501f206ea82ad050b4ffead8a']], // Testnet: Chapel ['0x08265dA01E1A65d62b903c7B34c08cB389bF3D99', [ 'Ararat' , '0x341e228f22D4ec16297DD05A9d6347C74c125F66', '0x96f763f030b1adcfb369c5a5df4a18e1529baffe7feaec66db3dbd1bc06810f7f6f88b7be6645418a7e2a2a3f40514c2']], ['0x7f5f2cF1aec83bF0c74DF566a41aa7ed65EA84Ea', [ 'Kita' , '0x2716756EAF7F1B4f4DbB282A80efdbf96e90A644', '0x99e3849ef31887c0f880a0feb92f356f58fbd023a82f5311fc87a5883a662e9ebbbefc90bf13aa533c2438a4113804bf']], @@ -207,6 +208,7 @@ const validatorMap = new Map([ ['0x6a5470a3B7959ab064d6815e349eD4aE2dE5210d', ['Skynet10k' , '0xDD1fD7C74BaCCA08e1b88a24199F19aB1b1b9cE4', '0x81f13afdbd6976d9784a05619405df430314e2707050b32f29ae683b9ef89d285d1a227df3e31ac147016c4c7533be70']], ['0xce6cCa0DE7b3EB3fd0CcE4bc38cceF473166e4f4', ['Infinity' , '0xc8A6Bfe0834FB99340a0Df253d79B7CaE25053b8', '0xa40f553889e9de6b4fe8005a06d7335fa061ae51ef5ba2b0c4ea477fcaa8f6de1650e318cf59824462b1831a725488da']], ['0xa7deE0bCAEb78849Ec4aD4e2f48688D2e9f2315B', ['KrakV' , '0x6563AA29C30d9f80968c2fb7DFFed092a03FBdeD', '0x848ffc9a3fac00d9fbaebcb63f2b7c0a4747d9ffecd4b484073ad03d91584cb51af29870c1c8421b757f4f6fae813288']], + ['0x32415e630B9B3489639dEE7de21274Ab64016226', ['Kraken' , '0x70Cd30d9216AF7A5654D245e9F5c649b811aB2eB', '0xa80ebd07bd9d717bd538413e8830f673e63dfad496c901de324be5d16b0496aee39352ecfb84fa58d8d8a67746f8ae6c']], ]); const builderMap = new Map([ diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 07da73acd4..3727f4fdc3 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -182,6 +182,11 @@ var ( Usage: "Chapel network: pre-configured Proof-of-Stake-Authority BSC test network", Category: flags.EthCategory, } + EnableBALFlag = &cli.BoolFlag{ + Name: "enablebal", + Usage: "Enable block access list feature, validator will generate BAL for each block", + Category: flags.EthCategory, + } // Dev mode DeveloperFlag = &cli.BoolFlag{ Name: "dev", @@ -369,8 +374,8 @@ var ( } JournalFileFlag = &cli.BoolFlag{ Name: "journalfile", - Usage: "Enable using journal file to store the TrieJournal instead of KVDB in pbss (default = false)", - Value: false, + Usage: "Enable using journal file to store the TrieJournal instead of KVDB in pbss (default = true)", + Value: true, Category: flags.StateCategory, } StateHistoryFlag = &cli.Uint64Flag{ @@ -1743,6 +1748,9 @@ func SetNodeConfig(ctx *cli.Context, cfg *node.Config) { if ctx.IsSet(DisableSnapProtocolFlag.Name) { cfg.DisableSnapProtocol = ctx.Bool(DisableSnapProtocolFlag.Name) } + if ctx.IsSet(EnableBALFlag.Name) { + cfg.EnableBAL = ctx.Bool(EnableBALFlag.Name) + } if ctx.IsSet(RangeLimitFlag.Name) { cfg.RangeLimit = ctx.Bool(RangeLimitFlag.Name) } @@ -2042,6 +2050,9 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { if ctx.IsSet(CacheNoPrefetchFlag.Name) { cfg.NoPrefetch = ctx.Bool(CacheNoPrefetchFlag.Name) } + if ctx.IsSet(EnableBALFlag.Name) { + cfg.EnableBAL = ctx.Bool(EnableBALFlag.Name) + } // Read the value from the flag no matter if it's set or not. cfg.Preimages = ctx.Bool(CachePreimagesFlag.Name) if cfg.NoPruning && !cfg.Preimages { @@ -2078,9 +2089,8 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { if ctx.IsSet(PathDBSyncFlag.Name) { cfg.PathSyncFlush = true } - if ctx.IsSet(JournalFileFlag.Name) { - cfg.JournalFileEnabled = true - } + + cfg.JournalFileEnabled = ctx.Bool(JournalFileFlag.Name) if ctx.String(GCModeFlag.Name) == "archive" { if cfg.TransactionHistory != 0 { @@ -2698,6 +2708,7 @@ func MakeChain(ctx *cli.Context, stack *node.Node, readonly bool) (*core.BlockCh options := &core.BlockChainConfig{ TrieCleanLimit: ethconfig.Defaults.TrieCleanCache, NoPrefetch: ctx.Bool(CacheNoPrefetchFlag.Name), + EnableBAL: ctx.Bool(EnableBALFlag.Name), TrieDirtyLimit: ethconfig.Defaults.TrieDirtyCache, ArchiveMode: ctx.String(GCModeFlag.Name) == "archive", TrieTimeLimit: ethconfig.Defaults.TrieTimeout, diff --git a/consensus/beacon/consensus.go b/consensus/beacon/consensus.go index 0f11aada2c..aaa1111456 100644 --- a/consensus/beacon/consensus.go +++ b/consensus/beacon/consensus.go @@ -491,6 +491,15 @@ func (beacon *Beacon) SealHash(header *types.Header) common.Hash { return beacon.ethone.SealHash(header) } +func (beacon *Beacon) SignBAL(blockAccessList *types.BlockAccessListEncode) error { + return nil +} + +// VerifyBAL verifies the BAL of the block +func (beacon *Beacon) VerifyBAL(block *types.Block, bal *types.BlockAccessListEncode) error { + return nil +} + // CalcDifficulty is the difficulty adjustment algorithm. It returns // the difficulty that a new block should have when created at time // given the parent block's time and difficulty. diff --git a/consensus/clique/clique.go b/consensus/clique/clique.go index 7e5e85c578..411ddc5fd4 100644 --- a/consensus/clique/clique.go +++ b/consensus/clique/clique.go @@ -787,3 +787,11 @@ func encodeSigHeader(w io.Writer, header *types.Header) { panic("can't encode: " + err.Error()) } } + +func (c *Clique) SignBAL(bal *types.BlockAccessListEncode) error { + return nil +} + +func (c *Clique) VerifyBAL(block *types.Block, bal *types.BlockAccessListEncode) error { + return nil +} diff --git a/consensus/consensus.go b/consensus/consensus.go index 115a28a76e..2d50a56dff 100644 --- a/consensus/consensus.go +++ b/consensus/consensus.go @@ -136,6 +136,12 @@ type Engine interface { // SealHash returns the hash of a block prior to it being sealed. SealHash(header *types.Header) common.Hash + // SignBAL signs the BAL of the block + SignBAL(blockAccessList *types.BlockAccessListEncode) error + + // VerifyBAL verifies the BAL of the block + VerifyBAL(block *types.Block, bal *types.BlockAccessListEncode) error + // CalcDifficulty is the difficulty adjustment algorithm. It returns the difficulty // that a new block should have. CalcDifficulty(chain ChainHeaderReader, time uint64, parent *types.Header) *big.Int diff --git a/consensus/ethash/ethash.go b/consensus/ethash/ethash.go index f624f73875..3027898981 100644 --- a/consensus/ethash/ethash.go +++ b/consensus/ethash/ethash.go @@ -76,3 +76,11 @@ func (ethash *Ethash) Close() error { func (ethash *Ethash) Seal(chain consensus.ChainHeaderReader, block *types.Block, results chan<- *types.Block, stop <-chan struct{}) error { panic("ethash (pow) sealing not supported any more") } + +func (ethash *Ethash) SignBAL(bal *types.BlockAccessListEncode) error { + return nil +} + +func (ethash *Ethash) VerifyBAL(block *types.Block, bal *types.BlockAccessListEncode) error { + return nil +} diff --git a/consensus/parlia/parlia.go b/consensus/parlia/parlia.go index 8379bf7738..dad3f9b50c 100644 --- a/consensus/parlia/parlia.go +++ b/consensus/parlia/parlia.go @@ -1755,6 +1755,71 @@ func (p *Parlia) Seal(chain consensus.ChainHeaderReader, block *types.Block, res return nil } +func (p *Parlia) SignBAL(blockAccessList *types.BlockAccessListEncode) error { + p.lock.RLock() + val, signFn := p.val, p.signFn + p.lock.RUnlock() + + data, err := rlp.EncodeToBytes([]interface{}{blockAccessList.Version, blockAccessList.Number, blockAccessList.Hash, blockAccessList.Accounts}) + if err != nil { + log.Error("Encode to bytes failed when sealing", "err", err) + return errors.New("encode to bytes failed") + } + + if len(data) > int(params.MaxBALSize) { + log.Error("data is too large", "dataSize", len(data), "maxSize", params.MaxBALSize) + return errors.New("data is too large") + } + + sig, err := signFn(accounts.Account{Address: val}, accounts.MimetypeParlia, data) + if err != nil { + log.Error("Sign for the block header failed when sealing", "err", err) + return errors.New("sign for the block header failed") + } + + copy(blockAccessList.SignData, sig) + return nil +} + +func (p *Parlia) VerifyBAL(block *types.Block, bal *types.BlockAccessListEncode) error { + if bal.Version != 0 { + log.Error("invalid BAL version", "version", bal.Version) + return errors.New("invalid BAL version") + } + + if len(bal.SignData) != 65 { + log.Error("invalid BAL signature", "signatureSize", len(bal.SignData)) + return errors.New("invalid BAL signature") + } + + // Recover the public key and the Ethereum address + data, err := rlp.EncodeToBytes([]interface{}{bal.Version, block.Number(), block.Hash(), bal.Accounts}) + if err != nil { + log.Error("encode to bytes failed", "err", err) + return errors.New("encode to bytes failed") + } + + if len(data) > int(params.MaxBALSize) { + log.Error("data is too large", "dataSize", len(data), "maxSize", params.MaxBALSize) + return errors.New("data is too large") + } + + pubkey, err := crypto.Ecrecover(crypto.Keccak256(data), bal.SignData) + if err != nil { + return err + } + var pubkeyAddr common.Address + copy(pubkeyAddr[:], crypto.Keccak256(pubkey[1:])[12:]) + + signer := block.Header().Coinbase + if signer != pubkeyAddr { + log.Error("BAL signer mismatch", "signer", signer, "pubkeyAddr", pubkeyAddr, "bal.Number", bal.Number, "bal.Hash", bal.Hash) + return errors.New("signer mismatch") + } + + return nil +} + func (p *Parlia) shouldWaitForCurrentBlockProcess(chain consensus.ChainHeaderReader, header *types.Header, snap *Snapshot) bool { if header.Difficulty.Cmp(diffInTurn) == 0 { return false diff --git a/consensus/parlia/parlia_test.go b/consensus/parlia/parlia_test.go index 005f90a983..b3afcd2927 100644 --- a/consensus/parlia/parlia_test.go +++ b/consensus/parlia/parlia_test.go @@ -10,6 +10,7 @@ import ( "strings" "testing" + "github.com/ethereum/go-ethereum/accounts" "github.com/ethereum/go-ethereum/common" cmath "github.com/ethereum/go-ethereum/common/math" "github.com/ethereum/go-ethereum/consensus" @@ -858,3 +859,494 @@ func (c *mockParlia) FinalizeAndAssemble(chain consensus.ChainHeaderReader, head func (c *mockParlia) CalcDifficulty(chain consensus.ChainHeaderReader, time uint64, parent *types.Header) *big.Int { return big.NewInt(1) } + +func TestSignBAL(t *testing.T) { + // Setup test environment + key, _ := crypto.GenerateKey() + addr := crypto.PubkeyToAddress(key.PublicKey) + + // Create mock signing function that succeeds + mockSignFn := func(account accounts.Account, mimeType string, data []byte) ([]byte, error) { + if account.Address != addr { + return nil, fmt.Errorf("wrong address") + } + if mimeType != accounts.MimetypeParlia { + return nil, fmt.Errorf("wrong mime type") + } + // Return a dummy 65-byte signature + sig := make([]byte, 65) + copy(sig, []byte("test_signature_data_for_testing_purposes_123456789012345678901234")) + return sig, nil + } + + // Create Parlia instance + parlia := &Parlia{ + val: addr, + signFn: mockSignFn, + } + + tests := []struct { + name string + bal *types.BlockAccessListEncode + expectedError bool + signFn SignerFn + description string + }{ + { + name: "successful signing", + bal: &types.BlockAccessListEncode{ + Version: 0, + SignData: make([]byte, 65), + Accounts: []types.AccountAccessListEncode{ + { + Address: common.HexToAddress("0x1234567890123456789012345678901234567890"), + StorageItems: []types.StorageAccessItem{ + {Key: common.HexToHash("0x01"), TxIndex: 0, Dirty: false}, + }, + }, + }, + }, + expectedError: false, + signFn: mockSignFn, + description: "Should successfully sign a valid BlockAccessListEncode", + }, + { + name: "signing function error", + bal: &types.BlockAccessListEncode{ + Version: 0, + SignData: make([]byte, 65), + Accounts: []types.AccountAccessListEncode{}, + }, + expectedError: true, + signFn: func(account accounts.Account, mimeType string, data []byte) ([]byte, error) { + return nil, fmt.Errorf("signing failed") + }, + description: "Should return error when signing function fails", + }, + { + name: "empty accounts list", + bal: &types.BlockAccessListEncode{ + Version: 0, + SignData: make([]byte, 65), + Accounts: []types.AccountAccessListEncode{}, + }, + expectedError: false, + signFn: mockSignFn, + description: "Should successfully sign even with empty accounts list", + }, + { + name: "multiple accounts", + bal: &types.BlockAccessListEncode{ + Version: 2, + SignData: make([]byte, 65), + Accounts: []types.AccountAccessListEncode{ + { + Address: common.HexToAddress("0x1111111111111111111111111111111111111111"), + StorageItems: []types.StorageAccessItem{ + {Key: common.HexToHash("0x01"), TxIndex: 0, Dirty: false}, + {Key: common.HexToHash("0x02"), TxIndex: 1, Dirty: true}, + }, + }, + { + Address: common.HexToAddress("0x2222222222222222222222222222222222222222"), + StorageItems: []types.StorageAccessItem{ + {Key: common.HexToHash("0x03"), TxIndex: 2, Dirty: false}, + }, + }, + }, + }, + expectedError: false, + signFn: mockSignFn, + description: "Should successfully sign with multiple accounts", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Set up Parlia with the test signing function + parlia.signFn = tt.signFn + + // Call SignBAL + err := parlia.SignBAL(tt.bal) + + // Check results + if tt.expectedError { + if err == nil { + t.Errorf("Expected error but got none. %s", tt.description) + } + } else { + if err != nil { + t.Errorf("Expected no error but got: %v. %s", err, tt.description) + } + // Verify signature was copied to SignData + if tt.bal != nil && len(tt.bal.SignData) != 65 { + t.Errorf("Expected SignData to be 65 bytes, got %d", len(tt.bal.SignData)) + } + // Verify signature content (for successful cases) + if tt.bal != nil && !tt.expectedError { + expectedSig := "test_signature_data_for_testing_purposes_123456789012345678901234" + if string(tt.bal.SignData[:len(expectedSig)]) != expectedSig { + t.Errorf("SignData was not properly set") + } + } + } + }) + } +} + +func TestVerifyBAL(t *testing.T) { + // Setup test environment + key, _ := crypto.GenerateKey() + addr := crypto.PubkeyToAddress(key.PublicKey) + + // Helper function to create a properly signed BAL + createSignedBAL := func(version uint32, accounts []types.AccountAccessListEncode) *types.BlockAccessListEncode { + bal := &types.BlockAccessListEncode{ + Version: version, + SignData: make([]byte, 65), + Accounts: accounts, + } + + // RLP encode the data + data, _ := rlp.EncodeToBytes([]interface{}{bal.Version, bal.Accounts}) + + // Create signature using the test key + hash := crypto.Keccak256(data) + sig, _ := crypto.Sign(hash, key) + copy(bal.SignData, sig) + + return bal + } + + // Create a Parlia instance + parlia := &Parlia{} + + tests := []struct { + name string + signer common.Address + bal *types.BlockAccessListEncode + expectedError bool + description string + }{ + { + name: "valid signature verification", + signer: addr, + bal: createSignedBAL(0, []types.AccountAccessListEncode{ + { + Address: common.HexToAddress("0x1234567890123456789012345678901234567890"), + StorageItems: []types.StorageAccessItem{ + {Key: common.HexToHash("0x01"), TxIndex: 0, Dirty: false}, + }, + }, + }), + expectedError: false, + description: "Should successfully verify a properly signed BAL", + }, + { + name: "invalid version", + signer: addr, + bal: createSignedBAL(1, []types.AccountAccessListEncode{ + { + Address: common.HexToAddress("0x1234567890123456789012345678901234567890"), + StorageItems: []types.StorageAccessItem{ + {Key: common.HexToHash("0x01"), TxIndex: 0, Dirty: false}, + }, + }, + }), + expectedError: true, + description: "Should fail when version is invalid", + }, + { + name: "invalid signature length - too short", + signer: addr, + bal: &types.BlockAccessListEncode{ + Version: 0, + SignData: make([]byte, 64), // Wrong length + Accounts: []types.AccountAccessListEncode{}, + }, + expectedError: true, + description: "Should fail when signature is too short", + }, + { + name: "invalid signature length - too long", + signer: addr, + bal: &types.BlockAccessListEncode{ + Version: 0, + SignData: make([]byte, 66), // Wrong length + Accounts: []types.AccountAccessListEncode{}, + }, + expectedError: true, + description: "Should fail when signature is too long", + }, + { + name: "empty signature", + signer: addr, + bal: &types.BlockAccessListEncode{ + Version: 0, + SignData: []byte{}, // Empty signature + Accounts: []types.AccountAccessListEncode{}, + }, + expectedError: true, + description: "Should fail with empty signature", + }, + { + name: "signer mismatch", + signer: common.HexToAddress("0x9999999999999999999999999999999999999999"), // Wrong signer + bal: createSignedBAL(0, []types.AccountAccessListEncode{ + { + Address: common.HexToAddress("0x1234567890123456789012345678901234567890"), + StorageItems: []types.StorageAccessItem{ + {Key: common.HexToHash("0x01"), TxIndex: 0, Dirty: false}, + }, + }, + }), + expectedError: true, + description: "Should fail when signer address doesn't match recovered address", + }, + { + name: "corrupted signature", + signer: addr, + bal: func() *types.BlockAccessListEncode { + bal := createSignedBAL(0, []types.AccountAccessListEncode{}) + // Corrupt the signature + bal.SignData[0] = ^bal.SignData[0] + return bal + }(), + expectedError: true, + description: "Should fail with corrupted signature", + }, + { + name: "empty accounts list", + signer: addr, + bal: createSignedBAL(0, []types.AccountAccessListEncode{}), + expectedError: false, + description: "Should successfully verify BAL with empty accounts", + }, + { + name: "multiple accounts", + signer: addr, + bal: createSignedBAL(0, []types.AccountAccessListEncode{ + { + Address: common.HexToAddress("0x1111111111111111111111111111111111111111"), + StorageItems: []types.StorageAccessItem{ + {Key: common.HexToHash("0x01"), TxIndex: 0, Dirty: false}, + {Key: common.HexToHash("0x02"), TxIndex: 1, Dirty: true}, + }, + }, + { + Address: common.HexToAddress("0x2222222222222222222222222222222222222222"), + StorageItems: []types.StorageAccessItem{ + {Key: common.HexToHash("0x03"), TxIndex: 2, Dirty: false}, + }, + }, + }), + expectedError: false, + description: "Should successfully verify BAL with multiple accounts", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := parlia.VerifyBAL(tt.signer, tt.bal) + + if tt.expectedError { + if err == nil { + t.Errorf("Expected error but got none. %s", tt.description) + } + } else { + if err != nil { + t.Errorf("Expected no error but got: %v. %s", err, tt.description) + } + } + }) + } +} + +func TestVerifyBAL_EdgeCases(t *testing.T) { + // Test with different key to ensure proper signature verification + key1, _ := crypto.GenerateKey() + key2, _ := crypto.GenerateKey() + addr1 := crypto.PubkeyToAddress(key1.PublicKey) + addr2 := crypto.PubkeyToAddress(key2.PublicKey) + + parlia := &Parlia{} + + // Create BAL signed with key1 + bal := &types.BlockAccessListEncode{ + Version: 0, + SignData: make([]byte, 65), + Accounts: []types.AccountAccessListEncode{ + { + Address: common.HexToAddress("0x1234567890123456789012345678901234567890"), + StorageItems: []types.StorageAccessItem{ + {Key: common.HexToHash("0x01"), TxIndex: 0, Dirty: false}, + }, + }, + }, + } + + // Sign with key1 + data, _ := rlp.EncodeToBytes([]interface{}{bal.Version, bal.Accounts}) + hash := crypto.Keccak256(data) + sig, _ := crypto.Sign(hash, key1) + copy(bal.SignData, sig) + + // Should succeed with addr1 + err := parlia.VerifyBAL(addr1, bal) + if err != nil { + t.Errorf("Verification with correct signer failed: %v", err) + } + + // Should fail with addr2 (different key) + err = parlia.VerifyBAL(addr2, bal) + if err == nil { + t.Error("Expected verification to fail with different signer address") + } +} + +func TestVerifyBAL_TooLargeData(t *testing.T) { + // Test with large amount of data to ensure RLP encoding works correctly + key, _ := crypto.GenerateKey() + addr := crypto.PubkeyToAddress(key.PublicKey) + parlia := &Parlia{} + + // Create BAL with many accounts + accounts := make([]types.AccountAccessListEncode, 20000) + for i := 0; i < 20000; i++ { + accounts[i] = types.AccountAccessListEncode{ + Address: common.BigToAddress(big.NewInt(int64(i))), + StorageItems: []types.StorageAccessItem{ + {Key: common.BigToHash(big.NewInt(int64(i))), TxIndex: uint32(i), Dirty: i%2 == 0}, + {Key: common.BigToHash(big.NewInt(int64(i + 1000))), TxIndex: uint32(i + 1), Dirty: i%3 == 0}, + }, + } + } + + bal := &types.BlockAccessListEncode{ + Version: 0, + SignData: make([]byte, 65), + Accounts: accounts, + } + + // Sign the large data + data, err := rlp.EncodeToBytes([]interface{}{bal.Version, bal.Accounts}) + if err != nil { + t.Fatalf("Failed to RLP encode large data: %v", err) + } + + hash := crypto.Keccak256(data) + sig, err := crypto.Sign(hash, key) + if err != nil { + t.Fatalf("Failed to sign large data: %v", err) + } + copy(bal.SignData, sig) + + // Verify the signature + err = parlia.VerifyBAL(addr, bal) + if err.Error() != "data is too large" { + t.Errorf("Failed to verify BAL with large data: %v", err) + } +} + +func TestSignBAL_VerifyBAL_Integration(t *testing.T) { + // Test complete sign-verify cycle + key, _ := crypto.GenerateKey() + addr := crypto.PubkeyToAddress(key.PublicKey) + + // Create mock signing function + mockSignFn := func(account accounts.Account, mimeType string, data []byte) ([]byte, error) { + if account.Address != addr { + return nil, fmt.Errorf("wrong address") + } + if mimeType != accounts.MimetypeParlia { + return nil, fmt.Errorf("wrong mime type") + } + // Use the actual private key to sign + hash := crypto.Keccak256(data) + return crypto.Sign(hash, key) + } + + parlia := &Parlia{ + val: addr, + signFn: mockSignFn, + } + + testCases := []struct { + name string + version uint32 + accounts []types.AccountAccessListEncode + }{ + { + name: "empty accounts", + version: 0, + accounts: []types.AccountAccessListEncode{}, + }, + { + name: "single account", + version: 0, + accounts: []types.AccountAccessListEncode{ + { + Address: common.HexToAddress("0x1234567890123456789012345678901234567890"), + StorageItems: []types.StorageAccessItem{ + {Key: common.HexToHash("0x01"), TxIndex: 0, Dirty: false}, + {Key: common.HexToHash("0x02"), TxIndex: 1, Dirty: true}, + }, + }, + }, + }, + { + name: "multiple accounts", + version: 0, + accounts: []types.AccountAccessListEncode{ + { + Address: common.HexToAddress("0x1111111111111111111111111111111111111111"), + StorageItems: []types.StorageAccessItem{ + {Key: common.HexToHash("0x01"), TxIndex: 0, Dirty: false}, + }, + }, + { + Address: common.HexToAddress("0x2222222222222222222222222222222222222222"), + StorageItems: []types.StorageAccessItem{ + {Key: common.HexToHash("0x02"), TxIndex: 1, Dirty: true}, + {Key: common.HexToHash("0x03"), TxIndex: 2, Dirty: false}, + }, + }, + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // Create BAL + bal := &types.BlockAccessListEncode{ + Version: tc.version, + SignData: make([]byte, 65), + Accounts: tc.accounts, + } + + // Sign the BAL + err := parlia.SignBAL(bal) + if err != nil { + t.Fatalf("SignBAL failed: %v", err) + } + + // Verify signature length + if len(bal.SignData) != 65 { + t.Errorf("Expected SignData to be 65 bytes, got %d", len(bal.SignData)) + } + + // Verify the BAL with correct signer + err = parlia.VerifyBAL(addr, bal) + if err != nil { + t.Errorf("VerifyBAL failed with correct signer: %v", err) + } + + // Verify should fail with wrong signer + wrongSigner := common.HexToAddress("0x9999999999999999999999999999999999999999") + err = parlia.VerifyBAL(wrongSigner, bal) + if err == nil { + t.Error("VerifyBAL should fail with wrong signer address") + } + }) + } +} diff --git a/core/blockchain.go b/core/blockchain.go index 3033927649..c8f18db6fa 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -215,6 +215,9 @@ type BlockChainConfig struct { // If the value is zero, all transactions of the entire chain will be indexed. // If the value is -1, indexing is disabled. TxLookupLimit int64 + + // EnableBAL enables the block access list feature + EnableBAL bool } // DefaultConfig returns the default config. @@ -1222,6 +1225,7 @@ func (bc *BlockChain) setHeadBeyondRoot(head uint64, time uint64, root common.Ha rawdb.DeleteReceipts(db, hash, num) rawdb.DeleteTd(db, hash, num) rawdb.DeleteBlobSidecars(db, hash, num) + rawdb.DeleteBAL(db, hash, num) } // Todo(rjl493456442) txlookup, log index, etc } @@ -1499,7 +1503,6 @@ func (bc *BlockChain) Stop() { if err := triedb.Commit(recent.Root(), true); err != nil { log.Error("Failed to commit recent state trie", "err", err) } else { - rawdb.WriteSafePointBlockNumber(bc.db, recent.NumberU64()) once.Do(func() { rawdb.WriteHeadBlockHash(bc.db, recent.Hash()) }) @@ -1510,8 +1513,6 @@ func (bc *BlockChain) Stop() { log.Info("Writing snapshot state to disk", "root", snapBase) if err := triedb.Commit(snapBase, true); err != nil { log.Error("Failed to commit recent state trie", "err", err) - } else { - rawdb.WriteSafePointBlockNumber(bc.db, bc.CurrentBlock().Number.Uint64()) } } for !bc.triegc.Empty() { @@ -1739,6 +1740,7 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [ if bc.chainConfig.IsCancun(block.Number(), block.Time()) { rawdb.WriteBlobSidecars(batch, block.Hash(), block.NumberU64(), block.Sidecars()) } + rawdb.WriteBAL(batch, block.Hash(), block.NumberU64(), block.BAL()) // Write everything belongs to the blocks into the database. So that // we can ensure all components of body is completed(body, receipts) @@ -1817,6 +1819,7 @@ func (bc *BlockChain) writeBlockWithoutState(block *types.Block, td *big.Int) (e if bc.chainConfig.IsCancun(block.Number(), block.Time()) { rawdb.WriteBlobSidecars(blockBatch, block.Hash(), block.NumberU64(), block.Sidecars()) } + rawdb.WriteBAL(blockBatch, block.Hash(), block.NumberU64(), block.BAL()) if err := blockBatch.Write(); err != nil { log.Crit("Failed to write block into disk", "err", err) } @@ -1863,6 +1866,7 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types. if bc.chainConfig.IsCancun(block.Number(), block.Time()) { rawdb.WriteBlobSidecars(blockBatch, block.Hash(), block.NumberU64(), block.Sidecars()) } + rawdb.WriteBAL(blockBatch, block.Hash(), block.NumberU64(), block.BAL()) if bc.db.HasSeparateStateStore() { rawdb.WritePreimages(bc.db.GetStateStore(), statedb.Preimages()) } else { @@ -1937,7 +1941,6 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types. } // Flush an entire trie and restart the counters bc.triedb.Commit(header.Root, true) - rawdb.WriteSafePointBlockNumber(bc.db, chosen) bc.lastWrite = chosen bc.gcproc = 0 } @@ -2419,7 +2422,7 @@ func (bc *BlockChain) processBlock(parentRoot common.Hash, block *types.Block, s defer interrupt.Store(true) // terminate the prefetch at the end needBadSharedStorage := bc.chainConfig.NeedBadSharedStorage(block.Number()) - needPrefetch := needBadSharedStorage || (!bc.cfg.NoPrefetch && len(block.Transactions()) >= prefetchTxNumber) + needPrefetch := needBadSharedStorage || (!bc.cfg.NoPrefetch && len(block.Transactions()) >= prefetchTxNumber) || block.BAL() != nil if !needPrefetch { statedb, err = state.New(parentRoot, bc.statedb) if err != nil { @@ -2457,11 +2460,17 @@ func (bc *BlockChain) processBlock(parentRoot common.Hash, block *types.Block, s storageCacheMissMeter.Mark(stats.StorageMiss) }() + interruptChan := make(chan struct{}) + defer close(interruptChan) go func(start time.Time, throwaway *state.StateDB, block *types.Block) { // Disable tracing for prefetcher executions. vmCfg := bc.cfg.VmConfig vmCfg.Tracer = nil - bc.prefetcher.Prefetch(block.Transactions(), block.Header(), block.GasLimit(), throwaway, vmCfg, &interrupt) + if block.BAL() != nil { + bc.prefetcher.PrefetchBAL(block, throwaway, interruptChan) + } else { + bc.prefetcher.Prefetch(block.Transactions(), block.Header(), block.GasLimit(), throwaway, vmCfg, &interrupt) + } blockPrefetchExecuteTimer.Update(time.Since(start)) if interrupt.Load() { diff --git a/core/blockchain_insert.go b/core/blockchain_insert.go index cf5a4977ce..1c9ba3e991 100644 --- a/core/blockchain_insert.go +++ b/core/blockchain_insert.go @@ -62,7 +62,7 @@ func (st *insertStats) report(chain []*types.Block, index int, snapDiffItems, sn context := []interface{}{ "number", end.Number(), "hash", end.Hash(), "miner", end.Coinbase(), "blocks", st.processed, "txs", txs, "blobs", blobs, "mgas", float64(st.usedGas) / 1000000, - "elapsed", common.PrettyDuration(elapsed), "mgasps", mgasps, + "elapsed", common.PrettyDuration(elapsed), "mgasps", mgasps, "BAL", end.BAL() != nil, } blockInsertMgaspsGauge.Update(int64(mgasps)) if timestamp := time.Unix(int64(end.Time()), 0); time.Since(timestamp) > time.Minute { diff --git a/core/blockchain_reader.go b/core/blockchain_reader.go index f739100af4..29f620f3d1 100644 --- a/core/blockchain_reader.go +++ b/core/blockchain_reader.go @@ -203,6 +203,11 @@ func (bc *BlockChain) GetBlock(hash common.Hash, number uint64) *types.Block { } sidecars := rawdb.ReadBlobSidecars(bc.db, hash, number) block = block.WithSidecars(sidecars) + + bal := rawdb.ReadBAL(bc.db, hash, number) + if bal != nil { + block = block.WithBAL(bal) + } // Cache the found block for next time and return bc.blockCache.Add(block.Hash(), block) return block @@ -476,6 +481,9 @@ func (bc *BlockChain) State() (*state.StateDB, error) { // StateAt returns a new mutable state based on a particular point in time. func (bc *BlockChain) StateAt(root common.Hash) (*state.StateDB, error) { stateDb, err := state.New(root, bc.statedb) + if bc.cfg.EnableBAL { + stateDb.InitBlockAccessList() + } if err != nil { return nil, err } diff --git a/core/data_availability.go b/core/data_availability.go index 6ec552d7a7..1f60efffa3 100644 --- a/core/data_availability.go +++ b/core/data_availability.go @@ -60,7 +60,7 @@ func IsDataAvailable(chain consensus.ChainHeaderReader, block *types.Block) (err // refer logic in ValidateBody if !chain.Config().IsCancun(block.Number(), block.Time()) { - if block.Sidecars() != nil { + if len(block.Sidecars()) != 0 { return errors.New("sidecars present in block body before cancun") } return nil diff --git a/core/rawdb/accessors_chain.go b/core/rawdb/accessors_chain.go index d6b30d7c2d..758275adc8 100644 --- a/core/rawdb/accessors_chain.go +++ b/core/rawdb/accessors_chain.go @@ -326,7 +326,7 @@ func ReadHeaderRange(db ethdb.Reader, number uint64, count uint64) []rlp.RawValu // read remaining from ancients, cap at 2M data, err := db.AncientRange(ChainFreezerHeaderTable, i+1-count, count, 2*1024*1024) if err != nil { - log.Error("Failed to read headers from freezer", "err", err) + log.Debug("Failed to read headers from freezer", "err", err, "start", i+1-count, "count", count, "number", number) return rlpHeaders } if uint64(len(data)) != count { @@ -858,6 +858,31 @@ func WriteAncientBlocks(db ethdb.AncientWriter, blocks []*types.Block, receipts }) } +func writeAncientBlock(op ethdb.AncientWriteOp, block *types.Block, header *types.Header, receipts rlp.RawValue, td *big.Int) error { + num := block.NumberU64() + if err := op.AppendRaw(ChainFreezerHashTable, num, block.Hash().Bytes()); err != nil { + return fmt.Errorf("can't add block %d hash: %v", num, err) + } + if err := op.Append(ChainFreezerHeaderTable, num, header); err != nil { + return fmt.Errorf("can't append block header %d: %v", num, err) + } + if err := op.Append(ChainFreezerBodiesTable, num, block.Body()); err != nil { + return fmt.Errorf("can't append block body %d: %v", num, err) + } + if err := op.Append(ChainFreezerReceiptTable, num, receipts); err != nil { + return fmt.Errorf("can't append block %d receipts: %v", num, err) + } + if err := op.Append(ChainFreezerDifficultyTable, num, td); err != nil { + return fmt.Errorf("can't append block %d total difficulty: %v", num, err) + } + if block.Sidecars() != nil { + if err := op.Append(ChainFreezerBlobSidecarTable, num, block.Sidecars()); err != nil { + return fmt.Errorf("can't append block %d blobs: %v", num, err) + } + } + return nil +} + // ReadBlobSidecarsRLP retrieves all the transaction blobs belonging to a block in RLP encoding. func ReadBlobSidecarsRLP(db ethdb.Reader, hash common.Hash, number uint64) rlp.RawValue { var data []byte @@ -908,29 +933,42 @@ func DeleteBlobSidecars(db ethdb.KeyValueWriter, hash common.Hash, number uint64 } } -func writeAncientBlock(op ethdb.AncientWriteOp, block *types.Block, header *types.Header, receipts rlp.RawValue, td *big.Int) error { - num := block.NumberU64() - if err := op.AppendRaw(ChainFreezerHashTable, num, block.Hash().Bytes()); err != nil { - return fmt.Errorf("can't add block %d hash: %v", num, err) - } - if err := op.Append(ChainFreezerHeaderTable, num, header); err != nil { - return fmt.Errorf("can't append block header %d: %v", num, err) +// ReadBALRLP retrieves all the block access list belonging to a block in RLP encoding. +func ReadBALRLP(db ethdb.Reader, hash common.Hash, number uint64) rlp.RawValue { + // BAL is only in kv DB, will not be put into ancient DB + data, _ := db.Get(blockBALKey(number, hash)) + return data +} + +// ReadBAL retrieves the block access list belonging to a block. +func ReadBAL(db ethdb.Reader, hash common.Hash, number uint64) *types.BlockAccessListEncode { + data := ReadBALRLP(db, hash, number) + if len(data) == 0 { + return nil } - if err := op.Append(ChainFreezerBodiesTable, num, block.Body()); err != nil { - return fmt.Errorf("can't append block body %d: %v", num, err) + var ret types.BlockAccessListEncode + if err := rlp.DecodeBytes(data, &ret); err != nil { + log.Error("Invalid blob array RLP", "hash", hash, "err", err) + return nil } - if err := op.Append(ChainFreezerReceiptTable, num, receipts); err != nil { - return fmt.Errorf("can't append block %d receipts: %v", num, err) + return &ret +} + +func WriteBAL(db ethdb.KeyValueWriter, hash common.Hash, number uint64, bal *types.BlockAccessListEncode) { + data, err := rlp.EncodeToBytes(bal) + if err != nil { + log.Crit("Failed to encode block BAL", "err", err) } - if err := op.Append(ChainFreezerDifficultyTable, num, td); err != nil { - return fmt.Errorf("can't append block %d total difficulty: %v", num, err) + + if err := db.Put(blockBALKey(number, hash), data); err != nil { + log.Crit("Failed to store block BAL", "err", err) } - if block.Sidecars() != nil { - if err := op.Append(ChainFreezerBlobSidecarTable, num, block.Sidecars()); err != nil { - return fmt.Errorf("can't append block %d blobs: %v", num, err) - } +} + +func DeleteBAL(db ethdb.KeyValueWriter, hash common.Hash, number uint64) { + if err := db.Delete(blockBALKey(number, hash)); err != nil { + log.Crit("Failed to delete block BAL", "err", err) } - return nil } // WriteAncientHeaderChain writes the supplied headers along with nil block @@ -975,6 +1013,7 @@ func DeleteBlock(db ethdb.KeyValueWriter, hash common.Hash, number uint64) { DeleteBody(db, hash, number) DeleteTd(db, hash, number) DeleteBlobSidecars(db, hash, number) // it is safe to delete non-exist blob + DeleteBAL(db, hash, number) } // DeleteBlockWithoutNumber removes all block data associated with a hash, except @@ -985,6 +1024,7 @@ func DeleteBlockWithoutNumber(db ethdb.KeyValueWriter, hash common.Hash, number DeleteBody(db, hash, number) DeleteTd(db, hash, number) DeleteBlobSidecars(db, hash, number) + DeleteBAL(db, hash, number) } const badBlockToKeep = 10 diff --git a/core/rawdb/accessors_chain_test.go b/core/rawdb/accessors_chain_test.go index 63c056d70c..9b7cd9ae12 100644 --- a/core/rawdb/accessors_chain_test.go +++ b/core/rawdb/accessors_chain_test.go @@ -617,8 +617,7 @@ func TestWriteAncientHeaderChain(t *testing.T) { ReceiptHash: types.EmptyReceiptsHash, }) // Write and verify the header in the database - ptd := new(big.Int) - WriteAncientHeaderChain(db, headers, ptd) + WriteAncientHeaderChain(db, headers, new(big.Int)) for _, header := range headers { if blob := ReadHeaderRLP(db, header.Hash(), header.Number.Uint64()); len(blob) == 0 { @@ -1123,3 +1122,376 @@ func TestHeadersRLPStorage(t *testing.T) { checkSequence(1, 1) // Only block 1 checkSequence(1, 2) // Genesis + block 1 } + +// Tests BAL (Block Access List) storage and retrieval operations. +func TestBALStorage(t *testing.T) { + db := NewMemoryDatabase() + + // Create test BAL data + bal := &types.BlockAccessListEncode{ + Version: 1, + SignData: make([]byte, 65), + Accounts: []types.AccountAccessListEncode{ + { + TxIndex: 0, + Address: common.HexToAddress("0x1234567890123456789012345678901234567890"), + StorageItems: []types.StorageAccessItem{ + {Key: common.HexToHash("0x01"), TxIndex: 0, Dirty: false}, + {Key: common.HexToHash("0x02"), TxIndex: 1, Dirty: true}, + }, + }, + { + TxIndex: 1, + Address: common.HexToAddress("0x2222222222222222222222222222222222222222"), + StorageItems: []types.StorageAccessItem{ + {Key: common.HexToHash("0x03"), TxIndex: 2, Dirty: false}, + }, + }, + }, + } + + // Fill SignData with test data + copy(bal.SignData, []byte("test_signature_data_for_bal_testing_12345678901234567890123456789")) + + hash := common.HexToHash("0x123456789abcdef") + number := uint64(42) + + // Test non-existent BAL retrieval + if entry := ReadBAL(db, hash, number); entry != nil { + t.Fatalf("Non-existent BAL returned: %v", entry) + } + if entry := ReadBALRLP(db, hash, number); len(entry) != 0 { + t.Fatalf("Non-existent raw BAL returned: %v", entry) + } + + // Test BAL storage and retrieval + WriteBAL(db, hash, number, bal) + if entry := ReadBAL(db, hash, number); entry == nil { + t.Fatalf("Stored BAL not found") + } else if !balEqual(entry, bal) { + t.Fatalf("Retrieved BAL mismatch: have %v, want %v", entry, bal) + } + + // Test raw BAL retrieval + if entry := ReadBALRLP(db, hash, number); len(entry) == 0 { + t.Fatalf("Stored raw BAL not found") + } + + // Test BAL deletion + DeleteBAL(db, hash, number) + if entry := ReadBAL(db, hash, number); entry != nil { + t.Fatalf("Deleted BAL still returned: %v", entry) + } + if entry := ReadBALRLP(db, hash, number); len(entry) != 0 { + t.Fatalf("Deleted raw BAL still returned: %v", entry) + } +} + +func TestBALRLPStorage(t *testing.T) { + db := NewMemoryDatabase() + + // Test different BAL configurations + testCases := []struct { + name string + bal *types.BlockAccessListEncode + hash common.Hash + number uint64 + }{ + { + name: "empty accounts", + bal: &types.BlockAccessListEncode{ + Version: 0, + SignData: make([]byte, 65), + Accounts: []types.AccountAccessListEncode{}, + }, + hash: common.HexToHash("0x1111"), + number: 1, + }, + { + name: "single account with multiple storage items", + bal: &types.BlockAccessListEncode{ + Version: 2, + SignData: make([]byte, 65), + Accounts: []types.AccountAccessListEncode{ + { + TxIndex: 0, + Address: common.HexToAddress("0xabcdefabcdefabcdefabcdefabcdefabcdefabcd"), + StorageItems: []types.StorageAccessItem{ + {Key: common.HexToHash("0x0a"), TxIndex: 0, Dirty: true}, + {Key: common.HexToHash("0x0b"), TxIndex: 1, Dirty: false}, + {Key: common.HexToHash("0x0c"), TxIndex: 2, Dirty: true}, + }, + }, + }, + }, + hash: common.HexToHash("0x2222"), + number: 2, + }, + { + name: "multiple accounts", + bal: &types.BlockAccessListEncode{ + Version: ^uint32(0), // Max uint32 value + SignData: make([]byte, 65), + Accounts: []types.AccountAccessListEncode{ + { + TxIndex: 0, + Address: common.HexToAddress("0x1111111111111111111111111111111111111111"), + StorageItems: []types.StorageAccessItem{ + {Key: common.HexToHash("0x01"), TxIndex: 0, Dirty: false}, + }, + }, + { + TxIndex: 1, + Address: common.HexToAddress("0x3333333333333333333333333333333333333333"), + StorageItems: []types.StorageAccessItem{ + {Key: common.HexToHash("0x04"), TxIndex: 3, Dirty: true}, + {Key: common.HexToHash("0x05"), TxIndex: 4, Dirty: false}, + }, + }, + { + TxIndex: 2, + Address: common.HexToAddress("0x4444444444444444444444444444444444444444"), + StorageItems: []types.StorageAccessItem{}, + }, + }, + }, + hash: common.HexToHash("0x3333"), + number: 100, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + // Fill SignData with unique test data + sigData := fmt.Sprintf("test_signature_for_%s_123456789012345678901234567890123456789012345678901234567890", tc.name) + copy(tc.bal.SignData, []byte(sigData)) + + // Store BAL + WriteBAL(db, tc.hash, tc.number, tc.bal) + + // Test RLP retrieval + rawData := ReadBALRLP(db, tc.hash, tc.number) + if len(rawData) == 0 { + t.Fatalf("Failed to store/retrieve raw BAL data") + } + + // Test structured retrieval + retrieved := ReadBAL(db, tc.hash, tc.number) + if retrieved == nil { + t.Fatalf("Failed to retrieve structured BAL") + } + + // Compare values + if !balEqual(retrieved, tc.bal) { + t.Fatalf("Retrieved BAL doesn't match stored BAL") + } + + // Test deletion + DeleteBAL(db, tc.hash, tc.number) + if ReadBAL(db, tc.hash, tc.number) != nil { + t.Fatalf("BAL not properly deleted") + } + }) + } +} + +func TestBALCorruptedData(t *testing.T) { + db := NewMemoryDatabase() + hash := common.HexToHash("0x9999") + number := uint64(123) + + // Store corrupted RLP data directly + corruptedData := []byte{0xff, 0xff, 0xff, 0xff} // Invalid RLP + if err := db.Put(blockBALKey(number, hash), corruptedData); err != nil { + t.Fatalf("Failed to store corrupted data: %v", err) + } + + // ReadBALRLP should return the corrupted data + rawData := ReadBALRLP(db, hash, number) + if !bytes.Equal(rawData, corruptedData) { + t.Fatalf("ReadBALRLP should return raw data even if corrupted") + } + + // ReadBAL should return nil for corrupted data + bal := ReadBAL(db, hash, number) + if bal != nil { + t.Fatalf("ReadBAL should return nil for corrupted data, got: %v", bal) + } +} + +func TestBALLargeData(t *testing.T) { + db := NewMemoryDatabase() + + // Create BAL with large amount of data + accounts := make([]types.AccountAccessListEncode, 1000) + for i := 0; i < 1000; i++ { + storageItems := make([]types.StorageAccessItem, 10) + for j := 0; j < 10; j++ { + storageItems[j] = types.StorageAccessItem{ + Key: common.BigToHash(big.NewInt(int64(i*10 + j))), + TxIndex: uint32(i*10 + j), + Dirty: (i+j)%2 == 0, + } + } + accounts[i] = types.AccountAccessListEncode{ + TxIndex: uint32(i), + Address: common.BigToAddress(big.NewInt(int64(i))), + StorageItems: storageItems, + } + } + + bal := &types.BlockAccessListEncode{ + Version: 12345, + SignData: make([]byte, 65), + Accounts: accounts, + } + + // Fill SignData + copy(bal.SignData, []byte("large_data_test_signature_123456789012345678901234567890123456789")) + + hash := common.HexToHash("0xaaaa") + number := uint64(999) + + // Test storage and retrieval of large data + WriteBAL(db, hash, number, bal) + + retrieved := ReadBAL(db, hash, number) + if retrieved == nil { + t.Fatalf("Failed to retrieve large BAL data") + } + + if !balEqual(retrieved, bal) { + t.Fatalf("Large BAL data integrity check failed") + } + + // Test deletion + DeleteBAL(db, hash, number) + if ReadBAL(db, hash, number) != nil { + t.Fatalf("Large BAL data not properly deleted") + } +} + +func TestBALMultipleBlocks(t *testing.T) { + db := NewMemoryDatabase() + + // Store BALs for multiple blocks + blocks := []struct { + hash common.Hash + number uint64 + bal *types.BlockAccessListEncode + }{ + { + hash: common.HexToHash("0xaaaa"), + number: 1, + bal: &types.BlockAccessListEncode{ + Version: 1, + SignData: make([]byte, 65), + Accounts: []types.AccountAccessListEncode{ + { + TxIndex: 0, + Address: common.HexToAddress("0x1111111111111111111111111111111111111111"), + StorageItems: []types.StorageAccessItem{ + {Key: common.HexToHash("0x01"), TxIndex: 0, Dirty: false}, + }, + }, + }, + }, + }, + { + hash: common.HexToHash("0xbbbb"), + number: 2, + bal: &types.BlockAccessListEncode{ + Version: 2, + SignData: make([]byte, 65), + Accounts: []types.AccountAccessListEncode{ + { + TxIndex: 0, + Address: common.HexToAddress("0x2222222222222222222222222222222222222222"), + StorageItems: []types.StorageAccessItem{ + {Key: common.HexToHash("0x02"), TxIndex: 1, Dirty: true}, + }, + }, + }, + }, + }, + { + hash: common.HexToHash("0xcccc"), + number: 3, + bal: &types.BlockAccessListEncode{ + Version: 3, + SignData: make([]byte, 65), + Accounts: []types.AccountAccessListEncode{}, + }, + }, + } + + // Store all BALs + for i, block := range blocks { + sigData := fmt.Sprintf("signature_for_block_%d_123456789012345678901234567890123456789012345678901234567890", i) + copy(block.bal.SignData, []byte(sigData)) + WriteBAL(db, block.hash, block.number, block.bal) + } + + // Verify all can be retrieved independently + for i, block := range blocks { + retrieved := ReadBAL(db, block.hash, block.number) + if retrieved == nil { + t.Fatalf("Failed to retrieve BAL for block %d", i) + } + if !balEqual(retrieved, block.bal) { + t.Fatalf("BAL mismatch for block %d", i) + } + } + + // Delete middle block + DeleteBAL(db, blocks[1].hash, blocks[1].number) + + // Verify first and third blocks still exist + if ReadBAL(db, blocks[0].hash, blocks[0].number) == nil { + t.Fatalf("Block 0 BAL was incorrectly deleted") + } + if ReadBAL(db, blocks[1].hash, blocks[1].number) != nil { + t.Fatalf("Block 1 BAL was not deleted") + } + if ReadBAL(db, blocks[2].hash, blocks[2].number) == nil { + t.Fatalf("Block 2 BAL was incorrectly deleted") + } +} + +// Helper function to compare two BlockAccessListEncode structs +func balEqual(a, b *types.BlockAccessListEncode) bool { + if a == nil && b == nil { + return true + } + if a == nil || b == nil { + return false + } + if a.Version != b.Version { + return false + } + if !bytes.Equal(a.SignData, b.SignData) { + return false + } + if len(a.Accounts) != len(b.Accounts) { + return false + } + for i, accountA := range a.Accounts { + accountB := b.Accounts[i] + if accountA.TxIndex != accountB.TxIndex { + return false + } + if accountA.Address != accountB.Address { + return false + } + if len(accountA.StorageItems) != len(accountB.StorageItems) { + return false + } + for j, storageA := range accountA.StorageItems { + storageB := accountB.StorageItems[j] + if storageA.Key != storageB.Key || storageA.TxIndex != storageB.TxIndex || storageA.Dirty != storageB.Dirty { + return false + } + } + } + return true +} diff --git a/core/rawdb/accessors_metadata.go b/core/rawdb/accessors_metadata.go index 9ab27f80af..94dcc4d05f 100644 --- a/core/rawdb/accessors_metadata.go +++ b/core/rawdb/accessors_metadata.go @@ -18,7 +18,6 @@ package rawdb import ( "encoding/json" - "math/big" "time" "github.com/ethereum/go-ethereum/common" @@ -31,7 +30,6 @@ import ( // FreezerType enumerator const ( EntireFreezerType uint64 = iota // classic ancient type - PruneFreezerType // prune ancient type ) // ReadDatabaseVersion retrieves the version number of the database. @@ -194,19 +192,3 @@ func WriteTransitionStatus(db ethdb.KeyValueWriter, data []byte) { log.Crit("Failed to store the eth2 transition status", "err", err) } } - -// ReadSafePointBlockNumber return the number of block that roothash save to disk -func ReadSafePointBlockNumber(db ethdb.KeyValueReader) uint64 { - num, _ := db.Get(LastSafePointBlockKey) - if num == nil { - return 0 - } - return new(big.Int).SetBytes(num).Uint64() -} - -// WriteSafePointBlockNumber write the number of block that roothash save to disk -func WriteSafePointBlockNumber(db ethdb.KeyValueWriter, number uint64) { - if err := db.Put(LastSafePointBlockKey, new(big.Int).SetUint64(number).Bytes()); err != nil { - log.Crit("Failed to store safe point of block number", "err", err) - } -} diff --git a/core/rawdb/accessors_state.go b/core/rawdb/accessors_state.go index bed2327573..3669efb8c0 100644 --- a/core/rawdb/accessors_state.go +++ b/core/rawdb/accessors_state.go @@ -291,28 +291,23 @@ func ReadStateHistoryList(db ethdb.AncientReaderOp, start uint64, count uint64) // history starts from one(zero for initial state). func WriteStateHistory(db ethdb.AncientWriter, id uint64, meta []byte, accountIndex []byte, storageIndex []byte, accounts []byte, storages []byte) error { _, err := db.ModifyAncients(func(op ethdb.AncientWriteOp) error { - err := op.AppendRaw(stateHistoryMeta, id-1, meta) - if err != nil { + if err := op.AppendRaw(stateHistoryMeta, id-1, meta); err != nil { log.Error("WriteStateHistory failed", "err", err) return err } - err = op.AppendRaw(stateHistoryAccountIndex, id-1, accountIndex) - if err != nil { + if err := op.AppendRaw(stateHistoryAccountIndex, id-1, accountIndex); err != nil { log.Error("WriteStateHistory failed", "err", err) return err } - err = op.AppendRaw(stateHistoryStorageIndex, id-1, storageIndex) - if err != nil { + if err := op.AppendRaw(stateHistoryStorageIndex, id-1, storageIndex); err != nil { log.Error("WriteStateHistory failed", "err", err) return err } - err = op.AppendRaw(stateHistoryAccountData, id-1, accounts) - if err != nil { + if err := op.AppendRaw(stateHistoryAccountData, id-1, accounts); err != nil { log.Error("WriteStateHistory failed", "err", err) return err } - err = op.AppendRaw(stateHistoryStorageData, id-1, storages) - if err != nil { + if err := op.AppendRaw(stateHistoryStorageData, id-1, storages); err != nil { log.Error("WriteStateHistory failed", "err", err) return err } diff --git a/core/rawdb/ancient_scheme.go b/core/rawdb/ancient_scheme.go index fd30c6e150..6620e9bd4c 100644 --- a/core/rawdb/ancient_scheme.go +++ b/core/rawdb/ancient_scheme.go @@ -45,7 +45,6 @@ const ( // chainFreezerTableConfigs configures the settings for tables in the chain freezer. // Compression is disabled for hashes as they don't compress well. -// TODO(Nathan): setting prunable properly var chainFreezerTableConfigs = map[string]freezerTableConfig{ ChainFreezerHeaderTable: {noSnappy: false, prunable: true}, ChainFreezerHashTable: {noSnappy: true, prunable: true}, @@ -54,6 +53,7 @@ var chainFreezerTableConfigs = map[string]freezerTableConfig{ ChainFreezerDifficultyTable: {noSnappy: true, prunable: true}, ChainFreezerBlobSidecarTable: {noSnappy: false, prunable: true}, } +var additionTables = []string{ChainFreezerBlobSidecarTable} // freezerTableConfig contains the settings for a freezer table. type freezerTableConfig struct { @@ -61,8 +61,6 @@ type freezerTableConfig struct { prunable bool // true for tables that can be pruned by TruncateTail } -var additionTables = []string{ChainFreezerBlobSidecarTable} - const ( // stateHistoryTableSize defines the maximum size of freezer data files. stateHistoryTableSize = 2 * 1000 * 1000 * 1000 diff --git a/core/rawdb/ancient_utils.go b/core/rawdb/ancient_utils.go index 02d50dd3e5..422c869d8d 100644 --- a/core/rawdb/ancient_utils.go +++ b/core/rawdb/ancient_utils.go @@ -24,8 +24,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/metrics" ) type tableSize struct { @@ -101,7 +99,6 @@ func inspectFreezers(db ethdb.Database) ([]freezerInfo, error) { return nil, err } - // TODO(Nathan): handle VerkleStateFreezerName file, err := os.Open(filepath.Join(datadir, MerkleStateFreezerName)) if err != nil { return nil, err @@ -143,7 +140,7 @@ func InspectFreezerTable(ancient string, freezerName string, tableName string, s ) switch freezerName { case ChainFreezerName: - path, tables = resolveChainFreezerDir(ancient), stateFreezerTableConfigs + path, tables = resolveChainFreezerDir(ancient), chainFreezerTableConfigs case MerkleStateFreezerName, VerkleStateFreezerName: if multiDatabase { @@ -169,23 +166,3 @@ func InspectFreezerTable(ancient string, freezerName string, tableName string, s table.dumpIndexStdout(start, end) return nil } - -func ResetStateFreezerTableOffset(ancient string, virtualTail uint64) error { - path, tables := filepath.Join(ancient, MerkleStateFreezerName), stateFreezerTableConfigs - - for name, config := range tables { - log.Info("Handle table", "name", name, "disableSnappy", config.noSnappy, "prunable", config.prunable) - table, err := newTable(path, name, metrics.NewInactiveMeter(), metrics.NewInactiveMeter(), metrics.NewGauge(), freezerTableSize, config, false) - if err != nil { - log.Error("New table failed", "error", err) - return err - } - // Reset the metadata of the freezer table - err = table.ResetItemsOffset(virtualTail) - if err != nil { - log.Error("Reset items offset of the table", "name", name, "error", err) - return err - } - } - return nil -} diff --git a/core/rawdb/chain_freezer.go b/core/rawdb/chain_freezer.go index 77b2c7f6d1..8fe8e5c115 100644 --- a/core/rawdb/chain_freezer.go +++ b/core/rawdb/chain_freezer.go @@ -141,7 +141,7 @@ func (f *chainFreezer) Close() error { // readHeadNumber returns the number of chain head block. 0 is returned if the // block is unknown or not available yet. -func (f *chainFreezer) readHeadNumber(db ethdb.Reader) uint64 { +func (f *chainFreezer) readHeadNumber(db ethdb.KeyValueReader) uint64 { hash := ReadHeadBlockHash(db) if hash == (common.Hash{}) { log.Error("Head block is not reachable") @@ -157,7 +157,7 @@ func (f *chainFreezer) readHeadNumber(db ethdb.Reader) uint64 { // readFinalizedNumber returns the number of finalized block. 0 is returned // if the block is unknown or not available yet. -func (f *chainFreezer) readFinalizedNumber(db ethdb.Reader) uint64 { +func (f *chainFreezer) readFinalizedNumber(db ethdb.KeyValueReader) uint64 { hash := ReadFinalizedBlockHash(db) if hash == (common.Hash{}) { return 0 @@ -172,7 +172,7 @@ func (f *chainFreezer) readFinalizedNumber(db ethdb.Reader) uint64 { // freezeThreshold returns the threshold for chain freezing. It's determined // by formula: max(finality, HEAD-params.FullImmutabilityThreshold). -func (f *chainFreezer) freezeThreshold(db ethdb.Reader) (uint64, error) { +func (f *chainFreezer) freezeThreshold(db ethdb.KeyValueReader) (uint64, error) { var ( head = f.readHeadNumber(db) final = f.readFinalizedNumber(db) @@ -237,7 +237,11 @@ func (f *chainFreezer) freeze(db ethdb.KeyValueStore, continueFreeze bool) { err error ) - // TODO(Nathan): use finalized block as the chain freeze indicator, to be activated + // BSC does not use the finalized block as the freeze indicator, for two reasons: + // 1. To retain double-signed blocks in recent days. + // 2. In pruneancient mode, frozen blocks are pruned away, which may result in too few + // blocks being retained. If the node is forcefully killed, it may fail to repair + // itself during restart. useFinalizedForFreeze := false if useFinalizedForFreeze { threshold, err = f.freezeThreshold(nfdb) @@ -538,9 +542,9 @@ func (f *chainFreezer) freezeRange(nfdb *nofreezedb, number, limit uint64) (hash if number > limit { return nil, nil } + hashes = make([]common.Hash, 0, limit-number+1) env, _ := f.freezeEnv.Load().(*ethdb.FreezerEnv) - hashes = make([]common.Hash, 0, limit-number+1) _, err = f.ModifyAncients(func(op ethdb.AncientWriteOp) error { for ; number <= limit; number++ { // Retrieve all the components of the canonical block. diff --git a/core/rawdb/database.go b/core/rawdb/database.go index 437fb7ee57..41bf1671a3 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -495,7 +495,6 @@ func Open(db ethdb.KeyValueStore, opts OpenOptions) (ethdb.Database, error) { // freezer. } } - // Freezer is consistent with the key-value database, permit combining the two if !opts.ReadOnly { frdb.wg.Add(1) @@ -639,6 +638,7 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { tds stat numHashPairings stat blobSidecars stat + bals stat hashNumPairings stat legacyTries stat stateLookups stat @@ -689,16 +689,18 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { bodies.Add(size) case bytes.HasPrefix(key, blockReceiptsPrefix) && len(key) == (len(blockReceiptsPrefix)+8+common.HashLength): receipts.Add(size) - case IsLegacyTrieNode(key, it.Value()): - legacyTries.Add(size) case bytes.HasPrefix(key, headerPrefix) && bytes.HasSuffix(key, headerTDSuffix): tds.Add(size) case bytes.HasPrefix(key, BlockBlobSidecarsPrefix): blobSidecars.Add(size) + case bytes.HasPrefix(key, BlockBALPrefix): + bals.Add(size) case bytes.HasPrefix(key, headerPrefix) && bytes.HasSuffix(key, headerHashSuffix): numHashPairings.Add(size) case bytes.HasPrefix(key, headerNumberPrefix) && len(key) == (len(headerNumberPrefix)+common.HashLength): hashNumPairings.Add(size) + case IsLegacyTrieNode(key, it.Value()): + legacyTries.Add(size) case bytes.HasPrefix(key, stateIDPrefix) && len(key) == len(stateIDPrefix)+common.HashLength: stateLookups.Add(size) case IsAccountTrieNode(key): @@ -829,6 +831,7 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { {"Key-Value store", "Receipt lists", receipts.Size(), receipts.Count()}, {"Key-Value store", "Difficulties", tds.Size(), tds.Count()}, {"Key-Value store", "BlobSidecars", blobSidecars.Size(), blobSidecars.Count()}, + {"Key-Value store", "Block access list", bals.Size(), bals.Count()}, {"Key-Value store", "Block number->hash", numHashPairings.Size(), numHashPairings.Count()}, {"Key-Value store", "Block hash->number", hashNumPairings.Size(), hashNumPairings.Count()}, {"Key-Value store", "Transaction index", txLookups.Size(), txLookups.Count()}, @@ -971,7 +974,7 @@ var knownMetadataKeys = [][]byte{ } // printChainMetadata prints out chain metadata to stderr. -func printChainMetadata(db ethdb.Reader) { +func printChainMetadata(db ethdb.KeyValueStore) { fmt.Fprintf(os.Stderr, "Chain metadata\n") for _, v := range ReadChainMetadata(db) { fmt.Fprintf(os.Stderr, " %s\n", strings.Join(v, ": ")) @@ -982,7 +985,7 @@ func printChainMetadata(db ethdb.Reader) { // ReadChainMetadata returns a set of key/value pairs that contains information // about the database chain status. This can be used for diagnostic purposes // when investigating the state of the node. -func ReadChainMetadata(db ethdb.Reader) [][]string { +func ReadChainMetadata(db ethdb.KeyValueStore) [][]string { pp := func(val *uint64) string { if val == nil { return "" diff --git a/core/rawdb/freezer.go b/core/rawdb/freezer.go index bf6cb5ec95..778d9b8cbe 100644 --- a/core/rawdb/freezer.go +++ b/core/rawdb/freezer.go @@ -416,12 +416,10 @@ func (f *Freezer) validate() error { return fmt.Errorf("freezer table %s has a differing head: %d != %d", kind, table.items.Load(), head) } if !table.config.prunable { - // TODO(Nathan): In BSC's prune feature, `table.itemHidden.Load() != 0` may return true. - // // non-prunable tables have to start at 0 - // if table.itemHidden.Load() != 0 { - // return fmt.Errorf("non-prunable freezer table '%s' has a non-zero tail: %d", kind, table.itemHidden.Load()) - // } + if table.itemHidden.Load() != 0 { + return fmt.Errorf("non-prunable freezer table '%s' has a non-zero tail: %d", kind, table.itemHidden.Load()) + } } else { // prunable tables have to have the same length if prunedTail == nil { @@ -486,12 +484,10 @@ func (f *Freezer) repair() error { return err } if !table.config.prunable { - // TODO(Nathan): In BSC's prune feature, `table.itemHidden.Load() != 0` may return true. - // // non-prunable tables have to start at 0 - // if table.itemHidden.Load() != 0 { - // panic(fmt.Sprintf("non-prunable freezer table %s has non-zero tail: %v", kind, table.itemHidden.Load())) - // } + if table.itemHidden.Load() != 0 { + panic(fmt.Sprintf("non-prunable freezer table %s has non-zero tail: %v", kind, table.itemHidden.Load())) + } } else { // prunable tables have to have the same length if err := table.truncateTail(prunedTail); err != nil { diff --git a/core/rawdb/freezer_table.go b/core/rawdb/freezer_table.go index 15a56acec0..dd29a33bd3 100644 --- a/core/rawdb/freezer_table.go +++ b/core/rawdb/freezer_table.go @@ -151,7 +151,7 @@ func newTable(path string, name string, readMeter, writeMeter *metrics.Meter, si meta *os.File ) if readonly { - // Will fail if table doesn't exist + // Will fail if table index file or meta file is not existent index, err = openFreezerFileForReadOnly(filepath.Join(path, idxName)) if err != nil { return nil, err diff --git a/core/rawdb/schema.go b/core/rawdb/schema.go index ef955dfedf..beb97f5b42 100644 --- a/core/rawdb/schema.go +++ b/core/rawdb/schema.go @@ -88,9 +88,6 @@ var ( // database. fastTxLookupLimitKey = []byte("FastTransactionLookupLimit") - //LastSafePointBlockKey tracks the block number for block state that write disk - LastSafePointBlockKey = []byte("LastSafePointBlockNumber") - // badBlockKey tracks the list of bad blocks seen by local badBlockKey = []byte("InvalidBlock") @@ -146,6 +143,8 @@ var ( BlockBlobSidecarsPrefix = []byte("blobs") + BlockBALPrefix = []byte("bal") // blockBALPrefix + blockNumber (uint64 big endian) + blockHash -> block access list + // new log index filterMapsPrefix = "fm-" filterMapsRangeKey = []byte(filterMapsPrefix + "R") @@ -216,6 +215,11 @@ func blockBlobSidecarsKey(number uint64, hash common.Hash) []byte { return append(append(BlockBlobSidecarsPrefix, encodeBlockNumber(number)...), hash.Bytes()...) } +// blockBALKey = blockBALPrefix + blockNumber (uint64 big endian) + blockHash +func blockBALKey(number uint64, hash common.Hash) []byte { + return append(append(BlockBALPrefix, encodeBlockNumber(number)...), hash.Bytes()...) +} + // txLookupKey = txLookupPrefix + hash func txLookupKey(hash common.Hash) []byte { return append(txLookupPrefix, hash.Bytes()...) diff --git a/core/state/statedb.go b/core/state/statedb.go index 09803b2fd4..419d2df074 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -138,6 +138,9 @@ type StateDB struct { accessList *accessList accessEvents *AccessEvents + // block level access list + blockAccessList *types.BlockAccessListRecord + // Transient storage transientStorage transientStorage @@ -190,6 +193,7 @@ func NewWithReader(root common.Hash, db Database, reader Reader) (*StateDB, erro preimages: make(map[common.Hash][]byte), journal: newJournal(), accessList: newAccessList(), + blockAccessList: nil, transientStorage: newTransientStorage(), } if db.TrieDB().IsVerkle() { @@ -198,6 +202,13 @@ func NewWithReader(root common.Hash, db Database, reader Reader) (*StateDB, erro return sdb, nil } +func (s *StateDB) InitBlockAccessList() { + if s.blockAccessList != nil { + log.Warn("prepareBAL blockAccessList is not nil") + } + s.blockAccessList = &types.BlockAccessListRecord{Accounts: make(map[common.Address]types.AccountAccessListRecord)} +} + func (s *StateDB) SetNeedBadSharedStorage(needBadSharedStorage bool) { s.needBadSharedStorage = needBadSharedStorage } @@ -375,6 +386,43 @@ func (s *StateDB) GetNonce(addr common.Address) uint64 { return 0 } +func (s *StateDB) PreloadAccount(addr common.Address) { + if s.Empty(addr) { + return + } + s.GetCode(addr) +} + +func (s *StateDB) PreloadStorage(addr common.Address, key common.Hash) { + if s.Empty(addr) { + return + } + s.GetState(addr, key) +} +func (s *StateDB) PreloadAccountTrie(addr common.Address) { + if s.prefetcher == nil { + return + } + + addressesToPrefetch := []common.Address{addr} + if err := s.prefetcher.prefetch(common.Hash{}, s.originalRoot, common.Address{}, addressesToPrefetch, nil, false); err != nil { + log.Error("Failed to prefetch addresses", "addresses", len(addressesToPrefetch), "err", err) + } +} + +func (s *StateDB) PreloadStorageTrie(addr common.Address, key common.Hash) { + if s.prefetcher == nil { + return + } + obj := s.getStateObject(addr) + if obj == nil { + return + } + if err := s.prefetcher.prefetch(obj.addrHash, obj.origin.Root, obj.address, nil, []common.Hash{key}, true); err != nil { + log.Error("Failed to prefetch storage slot", "addr", obj.address, "key", key, "err", err) + } +} + // GetStorageRoot retrieves the storage root from the given address or empty // if object not found. func (s *StateDB) GetStorageRoot(addr common.Address) common.Hash { @@ -424,6 +472,7 @@ func (s *StateDB) GetCodeHash(addr common.Address) common.Hash { func (s *StateDB) GetState(addr common.Address, hash common.Hash) common.Hash { stateObject := s.getStateObject(addr) if stateObject != nil { + s.blockAccessList.AddStorage(addr, hash, uint32(s.txIndex), false) return stateObject.GetState(hash) } return common.Hash{} @@ -506,6 +555,7 @@ func (s *StateDB) SetCode(addr common.Address, code []byte) (prev []byte) { } func (s *StateDB) SetState(addr common.Address, key, value common.Hash) common.Hash { + s.blockAccessList.AddStorage(addr, key, uint32(s.txIndex), true) if stateObject := s.getOrNewStateObject(addr); stateObject != nil { return stateObject.SetState(key, value) } @@ -635,6 +685,7 @@ func (s *StateDB) deleteStateObject(addr common.Address) { // getStateObject retrieves a state object given by the address, returning nil if // the object is not found or was deleted in this execution context. func (s *StateDB) getStateObject(addr common.Address) *stateObject { + s.blockAccessList.AddAccount(addr, uint32(s.txIndex)) // Prefer live objects if any is available if obj := s.stateObjects[addr]; obj != nil { return obj @@ -725,6 +776,14 @@ func (s *StateDB) CopyDoPrefetch() *StateDB { return s.copyInternal(true) } +func (s *StateDB) TransferBlockAccessList(prev *StateDB) { + if prev == nil { + return + } + s.blockAccessList = prev.blockAccessList + prev.blockAccessList = nil +} + // If doPrefetch is true, it tries to reuse the prefetcher, the copied StateDB will do active trie prefetch. // otherwise, just do inactive copy trie prefetcher. func (s *StateDB) copyInternal(doPrefetch bool) *StateDB { @@ -753,6 +812,7 @@ func (s *StateDB) copyInternal(doPrefetch bool) *StateDB { // empty lists, so we do it anyway to not blow up if we ever decide copy them // in the middle of a transaction. accessList: s.accessList.Copy(), + blockAccessList: nil, transientStorage: s.transientStorage.Copy(), journal: s.journal.copy(), } @@ -1595,3 +1655,53 @@ func (s *StateDB) IsAddressInMutations(addr common.Address) bool { _, ok := s.mutations[addr] return ok } + +func (s *StateDB) DumpAccessList(block *types.Block) { + if s.blockAccessList == nil { + return + } + accountCount := 0 + storageCount := 0 + dirtyStorageCount := 0 + for addr, account := range s.blockAccessList.Accounts { + accountCount++ + log.Debug(" DumpAccessList Address", "address", addr.Hex(), "txIndex", account.TxIndex) + for _, storageItem := range account.StorageItems { + log.Debug(" DumpAccessList Storage Item", "key", storageItem.Key.Hex(), "txIndex", storageItem.TxIndex, "dirty", storageItem.Dirty) + storageCount++ + if storageItem.Dirty { + dirtyStorageCount++ + } + } + } + log.Info("DumpAccessList", "blockNumber", block.NumberU64(), "GasUsed", block.GasUsed(), + "accountCount", accountCount, "storageCount", storageCount, "dirtyStorageCount", dirtyStorageCount) +} + +// GetEncodedBlockAccessList: convert BlockAccessListRecord to BlockAccessListEncode +func (s *StateDB) GetEncodedBlockAccessList(block *types.Block) *types.BlockAccessListEncode { + if s.blockAccessList == nil { + return nil + } + // encode block access list to rlp to propagate with the block + blockAccessList := types.BlockAccessListEncode{ + Version: 0, + Number: block.NumberU64(), + Hash: block.Hash(), + SignData: make([]byte, 65), + Accounts: make([]types.AccountAccessListEncode, 0), + } + for addr, account := range s.blockAccessList.Accounts { + accountAccessList := types.AccountAccessListEncode{ + TxIndex: account.TxIndex, + Address: addr, + StorageItems: make([]types.StorageAccessItem, 0), + } + for _, storageItem := range account.StorageItems { + accountAccessList.StorageItems = append(accountAccessList.StorageItems, storageItem) + } + blockAccessList.Accounts = append(blockAccessList.Accounts, accountAccessList) + } + + return &blockAccessList +} diff --git a/core/state_prefetcher.go b/core/state_prefetcher.go index b777b195bc..36a0a38c88 100644 --- a/core/state_prefetcher.go +++ b/core/state_prefetcher.go @@ -25,11 +25,14 @@ import ( "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" "golang.org/x/sync/errgroup" ) const prefetchMiningThread = 3 +const prefetchThreadBALSnapshot = 8 +const prefetchThreadBALTrie = 8 const checkInterval = 10 // statePrefetcher is a basic Prefetcher that executes transactions from a block @@ -129,6 +132,147 @@ func (p *statePrefetcher) Prefetch(transactions types.Transactions, header *type return } +func (p *statePrefetcher) PrefetchBALSnapshot(balPrefetch *types.BlockAccessListPrefetch, block *types.Block, txSize int, statedb *state.StateDB, interruptChan <-chan struct{}) { + accChan := make(chan struct { + txIndex uint32 + accAddr common.Address + }, prefetchThreadBALSnapshot) + + keyChan := make(chan struct { + txIndex uint32 + accAddr common.Address + key common.Hash + }, prefetchThreadBALSnapshot) + + // prefetch snapshot cache + for i := 0; i < prefetchThreadBALSnapshot; i++ { + go func() { + newStatedb := statedb.CopyDoPrefetch() + for { + select { + case accAddr := <-accChan: + log.Debug("PrefetchBALSnapshot", "txIndex", accAddr.txIndex, "accAddr", accAddr.accAddr) + newStatedb.PreloadAccount(accAddr.accAddr) + case item := <-keyChan: + log.Debug("PrefetchBALSnapshot", "txIndex", item.txIndex, "accAddr", item.accAddr, "key", item.key) + newStatedb.PreloadStorage(item.accAddr, item.key) + case <-interruptChan: + return + } + } + }() + } + for txIndex := 0; txIndex < txSize; txIndex++ { + txAccessList := balPrefetch.AccessListItems[uint32(txIndex)] + for accAddr, storageItems := range txAccessList.Accounts { + select { + case accChan <- struct { + txIndex uint32 + accAddr common.Address + }{ + txIndex: uint32(txIndex), + accAddr: accAddr, + }: + case <-interruptChan: + return + } + for _, storageItem := range storageItems { + select { + case keyChan <- struct { + txIndex uint32 + accAddr common.Address + key common.Hash + }{ + txIndex: uint32(txIndex), + accAddr: accAddr, + key: storageItem.Key, + }: + case <-interruptChan: + return + } + } + } + } + log.Debug("PrefetchBALSnapshot dispatch finished") +} + +func (p *statePrefetcher) PrefetchBALTrie(balPrefetch *types.BlockAccessListPrefetch, block *types.Block, statedb *state.StateDB, interruptChan <-chan struct{}) { + accItemsChan := make(chan struct { + txIndex uint32 + accAddr common.Address + items []types.StorageAccessItemPrefetch + }, prefetchThreadBALTrie) + + for i := 0; i < prefetchThreadBALTrie; i++ { + go func() { + newStatedb := statedb.CopyDoPrefetch() + for { + select { + case accItem := <-accItemsChan: + newStatedb.PreloadAccountTrie(accItem.accAddr) + log.Debug("PrefetchBALTrie", "txIndex", accItem.txIndex, "accAddr", accItem.accAddr) + for _, storageItem := range accItem.items { + if storageItem.Dirty { + log.Debug("PrefetchBALTrie", "txIndex", accItem.txIndex, "accAddr", accItem.accAddr, "storageItem", storageItem.Key, "dirty", storageItem.Dirty) + statedb.PreloadStorageTrie(accItem.accAddr, storageItem.Key) + } + } + case <-interruptChan: + return + } + } + }() + } + + for txIndex, txAccessList := range balPrefetch.AccessListItems { + for accAddr, storageItems := range txAccessList.Accounts { + select { + case accItemsChan <- struct { + txIndex uint32 + accAddr common.Address + items []types.StorageAccessItemPrefetch + }{ + txIndex: txIndex, + accAddr: accAddr, + items: storageItems, + }: + case <-interruptChan: + log.Warn("PrefetchBALTrie interrupted") + return + } + } + } + log.Debug("PrefetchBALTrie dispatch finished") +} + +func (p *statePrefetcher) PrefetchBAL(block *types.Block, statedb *state.StateDB, interruptChan <-chan struct{}) { + if block.BAL() == nil { + return + } + transactions := block.Transactions() + blockAccessList := block.BAL() + + // get index sorted block access list, each transaction has a list of accounts, each account has a list of storage items + // txIndex 0: + // account1: storage1_1, storage1_2, storage1_3 + // account2: storage2_1, storage2_2, storage2_3 + // txIndex 1: + // account3: storage3_1, storage3_2, storage3_3 + // ... + balPrefetch := types.BlockAccessListPrefetch{ + AccessListItems: make(map[uint32]types.TxAccessListPrefetch), + } + for _, account := range blockAccessList.Accounts { + balPrefetch.Update(&account) + } + + // prefetch snapshot cache + go p.PrefetchBALSnapshot(&balPrefetch, block, len(transactions), statedb, interruptChan) + + // prefetch MPT trie node cache + go p.PrefetchBALTrie(&balPrefetch, block, statedb, interruptChan) +} + // PrefetchMining processes the state changes according to the Ethereum rules by running // the transaction messages using the statedb, but any changes are discarded. The // only goal is to warm the state caches. Only used for mining stage. diff --git a/core/state_processor.go b/core/state_processor.go index b995b9f4e9..9d3240036d 100644 --- a/core/state_processor.go +++ b/core/state_processor.go @@ -181,7 +181,7 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg for _, receipt := range receipts { allLogs = append(allLogs, receipt.Logs...) } - + statedb.DumpAccessList(block) return &ProcessResult{ Receipts: receipts, Requests: requests, diff --git a/core/types.go b/core/types.go index 2e27fcd5d5..e27395f5c3 100644 --- a/core/types.go +++ b/core/types.go @@ -49,6 +49,8 @@ type Prefetcher interface { Prefetch(transactions types.Transactions, header *types.Header, gasLimit uint64, statedb *state.StateDB, cfg vm.Config, interrupt *atomic.Bool) // PrefetchMining used for pre-caching transaction signatures and state trie nodes. Only used for mining stage. PrefetchMining(txs TransactionsByPriceAndNonce, header *types.Header, gasLimit uint64, statedb *state.StateDB, cfg vm.Config, interruptCh <-chan struct{}, txCurr **types.Transaction) + // prefetch based on block access list + PrefetchBAL(block *types.Block, statedb *state.StateDB, interruptChan <-chan struct{}) } // Processor is an interface for processing blocks using a given initial state. diff --git a/core/types/block.go b/core/types/block.go index 62d486cf7f..4f4e47d9e1 100644 --- a/core/types/block.go +++ b/core/types/block.go @@ -235,6 +235,123 @@ type Body struct { Withdrawals []*Withdrawal `rlp:"optional"` } +// StorageAccessItem is a single storage key that is accessed in a block. +type StorageAccessItem struct { + TxIndex uint32 // index of the first transaction in the block that accessed the storage + Dirty bool // true if the storage was modified in the block, false if it was read only + Key common.Hash +} + +// AccountAccessListEncode & BlockAccessListEncode are for BAL serialization. +type AccountAccessListEncode struct { + TxIndex uint32 // index of the first transaction in the block that accessed the account + Address common.Address + StorageItems []StorageAccessItem +} + +type BlockAccessListEncode struct { + Version uint32 // Version of the access list format + Number uint64 // number of the block that the BAL is for + Hash common.Hash // hash of the block that the BAL is for + SignData []byte // sign data for BAL + Accounts []AccountAccessListEncode +} + +// TxAccessListPrefetch & BlockAccessListPrefetch are for BAL prefetch +type StorageAccessItemPrefetch struct { + Dirty bool + Key common.Hash +} + +type TxAccessListPrefetch struct { + Accounts map[common.Address][]StorageAccessItemPrefetch +} + +type BlockAccessListPrefetch struct { + AccessListItems map[uint32]TxAccessListPrefetch +} + +func (b *BlockAccessListPrefetch) Update(aclEncode *AccountAccessListEncode) { + if aclEncode == nil { + return + } + accAddr := aclEncode.Address + b.PrepareTxAccount(aclEncode.TxIndex, accAddr) + for _, storageItem := range aclEncode.StorageItems { + b.PrepareTxStorage(accAddr, storageItem) + } +} + +func (b *BlockAccessListPrefetch) PrepareTxStorage(accAddr common.Address, storageItem StorageAccessItem) { + b.PrepareTxAccount(storageItem.TxIndex, accAddr) + txAccessList := b.AccessListItems[storageItem.TxIndex] + txAccessList.Accounts[accAddr] = append(txAccessList.Accounts[accAddr], StorageAccessItemPrefetch{ + Dirty: storageItem.Dirty, + Key: storageItem.Key, + }) +} +func (b *BlockAccessListPrefetch) PrepareTxAccount(txIndex uint32, addr common.Address) { + // create the tx access list if not exists + if _, ok := b.AccessListItems[txIndex]; !ok { + b.AccessListItems[txIndex] = TxAccessListPrefetch{ + Accounts: make(map[common.Address][]StorageAccessItemPrefetch), + } + } + // create the account access list if not exists + if _, ok := b.AccessListItems[txIndex].Accounts[addr]; !ok { + b.AccessListItems[txIndex].Accounts[addr] = make([]StorageAccessItemPrefetch, 0) + } +} + +// BlockAccessListRecord & BlockAccessListRecord are used to record access list during tx execution. +type AccountAccessListRecord struct { + TxIndex uint32 // index of the first transaction in the block that accessed the account + StorageItems map[common.Hash]StorageAccessItem +} + +type BlockAccessListRecord struct { + Version uint32 // Version of the access list format + Accounts map[common.Address]AccountAccessListRecord +} + +func (b *BlockAccessListRecord) AddAccount(addr common.Address, txIndex uint32) { + if b == nil { + return + } + + if _, ok := b.Accounts[addr]; !ok { + b.Accounts[addr] = AccountAccessListRecord{ + TxIndex: txIndex, + StorageItems: make(map[common.Hash]StorageAccessItem), + } + } +} + +func (b *BlockAccessListRecord) AddStorage(addr common.Address, key common.Hash, txIndex uint32, dirty bool) { + if b == nil { + return + } + + if _, ok := b.Accounts[addr]; !ok { + b.Accounts[addr] = AccountAccessListRecord{ + TxIndex: txIndex, + StorageItems: make(map[common.Hash]StorageAccessItem), + } + } + + if _, ok := b.Accounts[addr].StorageItems[key]; !ok { + b.Accounts[addr].StorageItems[key] = StorageAccessItem{ + TxIndex: txIndex, + Dirty: dirty, + Key: key, + } + } else { + storageItem := b.Accounts[addr].StorageItems[key] + storageItem.Dirty = dirty + b.Accounts[addr].StorageItems[key] = storageItem + } +} + // Block represents an Ethereum block. // // Note the Block type tries to be 'immutable', and contains certain caches that rely @@ -274,6 +391,10 @@ type Block struct { // sidecars provides DA check sidecars BlobSidecars + + // bal provides block access list + bal *BlockAccessListEncode + balSize atomic.Uint64 } // "external" block encoding. used for eth protocol, etc. @@ -496,6 +617,19 @@ func (b *Block) Size() uint64 { return uint64(c) } +func (b *Block) BALSize() uint64 { + if b.bal == nil { + return 0 + } + if size := b.balSize.Load(); size > 0 { + return size + } + c := writeCounter(0) + rlp.Encode(&c, b.bal) + b.balSize.Store(uint64(c)) + return uint64(c) +} + func (b *Block) SetRoot(root common.Hash) { b.header.Root = root } // SanityCheck can be used to prevent that unbounded fields are @@ -508,6 +642,10 @@ func (b *Block) Sidecars() BlobSidecars { return b.sidecars } +func (b *Block) BAL() *BlockAccessListEncode { + return b.bal +} + func (b *Block) CleanSidecars() { b.sidecars = make(BlobSidecars, 0) } @@ -563,6 +701,7 @@ func (b *Block) WithSeal(header *Header) *Block { withdrawals: b.withdrawals, witness: b.witness, sidecars: b.sidecars, + bal: b.bal, } } @@ -576,6 +715,7 @@ func (b *Block) WithBody(body Body) *Block { withdrawals: slices.Clone(body.Withdrawals), witness: b.witness, sidecars: b.sidecars, + bal: b.bal, } for i := range body.Uncles { block.uncles[i] = CopyHeader(body.Uncles[i]) @@ -591,6 +731,7 @@ func (b *Block) WithWithdrawals(withdrawals []*Withdrawal) *Block { uncles: b.uncles, witness: b.witness, sidecars: b.sidecars, + bal: b.bal, } if withdrawals != nil { block.withdrawals = make([]*Withdrawal, len(withdrawals)) @@ -607,6 +748,7 @@ func (b *Block) WithSidecars(sidecars BlobSidecars) *Block { uncles: b.uncles, withdrawals: b.withdrawals, witness: b.witness, + bal: b.bal, } if sidecars != nil { block.sidecars = make(BlobSidecars, len(sidecars)) @@ -615,6 +757,23 @@ func (b *Block) WithSidecars(sidecars BlobSidecars) *Block { return block } +func (b *Block) WithBAL(bal *BlockAccessListEncode) *Block { + block := &Block{ + header: b.header, + transactions: b.transactions, + uncles: b.uncles, + withdrawals: b.withdrawals, + witness: b.witness, + sidecars: b.sidecars, + } + block.bal = bal + return block +} + +func (b *Block) UpdateBAL(bal *BlockAccessListEncode) { + b.bal = bal +} + func (b *Block) WithWitness(witness *ExecutionWitness) *Block { return &Block{ header: b.header, diff --git a/eth/backend.go b/eth/backend.go index ac1dd0fa22..2f2bed4f49 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -324,6 +324,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { options = &core.BlockChainConfig{ TrieCleanLimit: config.TrieCleanCache, NoPrefetch: config.NoPrefetch, + EnableBAL: config.EnableBAL, TrieDirtyLimit: config.TrieDirtyCache, ArchiveMode: config.NoPruning, TrieTimeLimit: config.TrieTimeout, @@ -430,6 +431,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { RequiredBlocks: config.RequiredBlocks, DirectBroadcast: config.DirectBroadcast, EnableEVNFeatures: stack.Config().EnableEVNFeatures, + EnableBAL: config.EnableBAL, EVNNodeIdsWhitelist: stack.Config().P2P.EVNNodeIdsWhitelist, ProxyedValidatorAddresses: stack.Config().P2P.ProxyedValidatorAddresses, DisablePeerTxBroadcast: config.DisablePeerTxBroadcast, diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index 968241c792..86c57071a4 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -111,6 +111,7 @@ type Config struct { NoPruning bool // Whether to disable pruning and flush everything to disk NoPrefetch bool // Whether to disable prefetching and only load state on demand + EnableBAL bool DirectBroadcast bool DisableSnapProtocol bool // Whether disable snap protocol RangeLimit bool diff --git a/eth/ethconfig/gen_config.go b/eth/ethconfig/gen_config.go index 038fec8aec..a784f2dabc 100644 --- a/eth/ethconfig/gen_config.go +++ b/eth/ethconfig/gen_config.go @@ -30,6 +30,7 @@ func (c Config) MarshalTOML() (interface{}, error) { BscDiscoveryURLs []string NoPruning bool NoPrefetch bool + EnableBAL bool DirectBroadcast bool DisableSnapProtocol bool RangeLimit bool @@ -90,6 +91,7 @@ func (c Config) MarshalTOML() (interface{}, error) { enc.BscDiscoveryURLs = c.BscDiscoveryURLs enc.NoPruning = c.NoPruning enc.NoPrefetch = c.NoPrefetch + enc.EnableBAL = c.EnableBAL enc.DirectBroadcast = c.DirectBroadcast enc.DisableSnapProtocol = c.DisableSnapProtocol enc.RangeLimit = c.RangeLimit @@ -154,6 +156,7 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { BscDiscoveryURLs []string NoPruning *bool NoPrefetch *bool + EnableBAL *bool DirectBroadcast *bool DisableSnapProtocol *bool RangeLimit *bool @@ -241,6 +244,9 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.NoPrefetch != nil { c.NoPrefetch = *dec.NoPrefetch } + if dec.EnableBAL != nil { + c.EnableBAL = *dec.EnableBAL + } if dec.DirectBroadcast != nil { c.DirectBroadcast = *dec.DirectBroadcast } diff --git a/eth/fetcher/block_fetcher.go b/eth/fetcher/block_fetcher.go index 2794726b57..ee909e9e66 100644 --- a/eth/fetcher/block_fetcher.go +++ b/eth/fetcher/block_fetcher.go @@ -889,7 +889,7 @@ func (f *BlockFetcher) importBlocks(op *blockOrHeaderInject) { hash := block.Hash() // Run the import on a new thread - log.Debug("Importing propagated block", "peer", peer, "number", block.Number(), "hash", hash) + log.Debug("Importing propagated block", "peer", peer, "number", block.Number(), "hash", hash, "balSize", block.BALSize()) go func() { // If the parent's unknown, abort insertion parent := f.getBlock(block.ParentHash()) diff --git a/eth/handler.go b/eth/handler.go index 38ba881f68..e8182068a1 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -141,6 +141,7 @@ type handlerConfig struct { PeerSet *peerSet EnableQuickBlockFetching bool EnableEVNFeatures bool + EnableBAL bool EVNNodeIdsWhitelist []enode.ID ProxyedValidatorAddresses []common.Address } @@ -150,6 +151,7 @@ type handler struct { networkID uint64 disablePeerTxBroadcast bool enableEVNFeatures bool + enableBAL bool evnNodeIdsWhitelistMap map[enode.ID]struct{} proxyedValidatorAddressMap map[common.Address]struct{} @@ -220,6 +222,7 @@ func newHandler(config *handlerConfig) (*handler, error) { requiredBlocks: config.RequiredBlocks, directBroadcast: config.DirectBroadcast, enableEVNFeatures: config.EnableEVNFeatures, + enableBAL: config.EnableBAL, evnNodeIdsWhitelistMap: make(map[enode.ID]struct{}), proxyedValidatorAddressMap: make(map[common.Address]struct{}), quitSync: make(chan struct{}), @@ -328,7 +331,7 @@ func newHandler(config *handlerConfig) (*handler, error) { if p.bscExt == nil { return nil, fmt.Errorf("peer does not support bsc protocol, peer: %v", p.ID()) } - if p.bscExt.Version() != bsc.Bsc2 { + if p.bscExt.Version() < bsc.Bsc2 { return nil, fmt.Errorf("remote peer does not support the required Bsc2 protocol version, peer: %v", p.ID()) } res, err := p.bscExt.RequestBlocksByRange(startHeight, startHash, count) @@ -340,6 +343,7 @@ func newHandler(config *handlerConfig) (*handler, error) { for i, item := range res { block := types.NewBlockWithHeader(item.Header).WithBody(types.Body{Transactions: item.Txs, Uncles: item.Uncles}) block = block.WithSidecars(item.Sidecars) + block = block.WithBAL(item.BAL) block.ReceivedAt = time.Now() block.ReceivedFrom = p.ID() if err := block.SanityCheck(); err != nil { @@ -455,12 +459,15 @@ func (h *handler) runEthPeer(peer *eth.Peer, handler eth.Handler) error { peer.Log().Error("Snapshot extension barrier failed", "err", err) return err } - bsc, err := h.peers.waitBscExtension(peer) + bscExt, err := h.peers.waitBscExtension(peer) if err != nil { peer.Log().Error("Bsc extension barrier failed", "err", err) return err } - + if bscExt != nil && bscExt.Version() == bsc.Bsc3 { + peer.CanHandleBAL.Store(true) + log.Debug("runEthPeer", "bscExt.Version", bscExt.Version(), "CanHandleBAL", peer.CanHandleBAL.Load()) + } // Execute the Ethereum handshake var ( head = h.chain.CurrentHeader() @@ -510,7 +517,7 @@ func (h *handler) runEthPeer(peer *eth.Peer, handler eth.Handler) error { } // Register the peer locally - if err := h.peers.registerPeer(peer, snap, bsc); err != nil { + if err := h.peers.registerPeer(peer, snap, bscExt); err != nil { peer.Log().Error("Ethereum peer registration failed", "err", err) return err } @@ -821,7 +828,8 @@ func (h *handler) BroadcastBlock(block *types.Block, propagate bool) { } for _, peer := range transfer { - log.Debug("broadcast block to peer", "hash", hash, "peer", peer.ID(), "EVNPeerFlag", peer.EVNPeerFlag.Load()) + log.Debug("broadcast block to peer", "hash", hash, "peer", peer.ID(), + "EVNPeerFlag", peer.EVNPeerFlag.Load(), "CanHandleBAL", peer.CanHandleBAL.Load()) peer.AsyncSendNewBlock(block, td) } @@ -834,7 +842,8 @@ func (h *handler) BroadcastBlock(block *types.Block, propagate bool) { } } for _, peer := range morePeers { - log.Debug("broadcast block to extra peer", "hash", hash, "peer", peer.ID(), "EVNPeerFlag", peer.EVNPeerFlag.Load()) + log.Debug("broadcast block to extra peer", "hash", hash, "peer", peer.ID(), + "EVNPeerFlag", peer.EVNPeerFlag.Load(), "CanHandleBAL", peer.CanHandleBAL.Load()) peer.AsyncSendNewBlock(block, td) } } @@ -845,7 +854,8 @@ func (h *handler) BroadcastBlock(block *types.Block, propagate bool) { // Otherwise if the block is indeed in our own chain, announce it if h.chain.HasBlock(hash, block.NumberU64()) { for _, peer := range peers { - log.Debug("Announced block to peer", "hash", hash, "peer", peer.ID(), "EVNPeerFlag", peer.EVNPeerFlag.Load()) + log.Debug("Announced block to peer", "hash", hash, "peer", peer.ID(), + "EVNPeerFlag", peer.EVNPeerFlag.Load(), "CanHandleBAL", peer.CanHandleBAL.Load()) peer.AsyncSendNewBlockHash(block) } log.Debug("Announced block", "hash", hash, "recipients", len(peers), "duration", common.PrettyDuration(time.Since(block.ReceivedAt))) diff --git a/eth/handler_eth.go b/eth/handler_eth.go index 4a1c2b50ea..ff8cfdf5a6 100644 --- a/eth/handler_eth.go +++ b/eth/handler_eth.go @@ -141,6 +141,9 @@ func (h *ethHandler) handleBlockBroadcast(peer *eth.Peer, packet *eth.NewBlockPa if sidecars != nil { block = block.WithSidecars(sidecars) } + if packet.Bal != nil && h.chain.Engine().VerifyBAL(block, packet.Bal) == nil { + block = block.WithBAL(packet.Bal) + } // Schedule the block for import log.Debug("handleBlockBroadcast", "peer", peer.ID(), "block", block.Number(), "hash", block.Hash()) diff --git a/eth/handler_test.go b/eth/handler_test.go index 9c91a1a9cf..03cbe7803d 100644 --- a/eth/handler_test.go +++ b/eth/handler_test.go @@ -300,7 +300,9 @@ func newTestParliaHandlerAfterCancun(t *testing.T, config *params.ChainConfig, m Alloc: types.GenesisAlloc{testAddr: {Balance: new(big.Int).SetUint64(10 * params.Ether)}}, } engine := &mockParlia{} - chain, _ := core.NewBlockChain(db, gspec, engine, nil) + cfg := core.DefaultConfig() + cfg.StateScheme = rawdb.PathScheme + chain, _ := core.NewBlockChain(db, gspec, engine, cfg) signer := types.LatestSigner(config) _, bs, _ := core.GenerateChainWithGenesis(gspec, engine, int(preCancunBlks+postCancunBlks), func(i int, gen *core.BlockGen) { diff --git a/eth/protocols/bsc/handler.go b/eth/protocols/bsc/handler.go index 37d3d29f9b..994a362e3a 100644 --- a/eth/protocols/bsc/handler.go +++ b/eth/protocols/bsc/handler.go @@ -166,15 +166,17 @@ func handleGetBlocksByRange(backend Backend, msg Decoder, peer *Peer) error { return fmt.Errorf("msg %v, cannot get start block: %v, %v", GetBlocksByRangeMsg, req.StartBlockHeight, req.StartBlockHash) } blocks = append(blocks, NewBlockData(block)) + balSize := block.BALSize() for i := uint64(1); i < req.Count; i++ { block = backend.Chain().GetBlockByHash(block.ParentHash()) if block == nil { break } + balSize += block.BALSize() blocks = append(blocks, NewBlockData(block)) } - log.Debug("reply GetBlocksByRange msg", "from", peer.id, "req", req.Count, "blocks", len(blocks)) + log.Debug("reply GetBlocksByRange msg", "from", peer.id, "req", req.Count, "blocks", len(blocks), "balSize", balSize) return p2p.Send(peer.rw, BlocksByRangeMsg, &BlocksByRangePacket{ RequestId: req.RequestId, Blocks: blocks, diff --git a/eth/protocols/bsc/protocol.go b/eth/protocols/bsc/protocol.go index 50a08599af..572c24debb 100644 --- a/eth/protocols/bsc/protocol.go +++ b/eth/protocols/bsc/protocol.go @@ -12,6 +12,7 @@ import ( const ( Bsc1 = 1 Bsc2 = 2 + Bsc3 = 3 // to BAL process ) // ProtocolName is the official short name of the `bsc` protocol used during @@ -20,11 +21,11 @@ const ProtocolName = "bsc" // ProtocolVersions are the supported versions of the `bsc` protocol (first // is primary). -var ProtocolVersions = []uint{Bsc1, Bsc2} +var ProtocolVersions = []uint{Bsc1, Bsc2, Bsc3} // protocolLengths are the number of implemented message corresponding to // different protocol versions. -var protocolLengths = map[uint]uint64{Bsc1: 2, Bsc2: 4} +var protocolLengths = map[uint]uint64{Bsc1: 2, Bsc2: 4, Bsc3: 4} // maxMessageSize is the maximum cap on the size of a protocol message. const maxMessageSize = 10 * 1024 * 1024 @@ -84,8 +85,9 @@ type BlockData struct { Header *types.Header Txs []*types.Transaction Uncles []*types.Header - Withdrawals []*types.Withdrawal `rlp:"optional"` - Sidecars types.BlobSidecars `rlp:"optional"` + Withdrawals []*types.Withdrawal `rlp:"optional"` + Sidecars types.BlobSidecars `rlp:"optional"` + BAL *types.BlockAccessListEncode `rlp:"optional"` } // NewBlockData creates a new BlockData object from a block @@ -96,6 +98,7 @@ func NewBlockData(block *types.Block) *BlockData { Uncles: block.Uncles(), Withdrawals: block.Withdrawals(), Sidecars: block.Sidecars(), + BAL: block.BAL(), } } diff --git a/eth/protocols/eth/handlers.go b/eth/protocols/eth/handlers.go index 8fb4e17d5a..81c3501cdd 100644 --- a/eth/protocols/eth/handlers.go +++ b/eth/protocols/eth/handlers.go @@ -376,6 +376,13 @@ func handleNewBlock(backend Backend, msg Decoder, peer *Peer) error { return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) } + if ann.Bal != nil { + log.Debug("handleNewBlock, BAL", "number", ann.Block.NumberU64(), "hash", ann.Block.Hash(), "peer", peer.ID(), + "version", ann.Bal.Version, "signData", len(ann.Bal.SignData), "accounts", len(ann.Bal.Accounts), "balSize", ann.Block.BALSize()) + } else { + log.Debug("handleNewBlock, no BAL", "number", ann.Block.NumberU64(), "hash", ann.Block.Hash(), "peer", peer.ID(), + "txNum", len(ann.Block.Transactions()), "balSize", ann.Block.BALSize()) + } // Now that we have our packet, perform operations using the interface methods if err := ann.sanityCheck(); err != nil { return err diff --git a/eth/protocols/eth/peer.go b/eth/protocols/eth/peer.go index 654385e410..d7ef06b265 100644 --- a/eth/protocols/eth/peer.go +++ b/eth/protocols/eth/peer.go @@ -25,6 +25,7 @@ import ( mapset "github.com/deckarep/golang-set/v2" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/rlp" @@ -311,10 +312,23 @@ func (p *Peer) AsyncSendNewBlockHash(block *types.Block) { func (p *Peer) SendNewBlock(block *types.Block, td *big.Int) error { // Mark all the block hash as known, but ensure we don't overflow our limits p.knownBlocks.Add(block.Hash()) + bal := block.BAL() + if !p.CanHandleBAL.Load() { + bal = nil + } + if bal != nil { + log.Debug("SendNewBlock", "number", block.NumberU64(), "hash", block.Hash(), "peer", p.ID(), + "balSize", block.BALSize(), "version", bal.Version, "canHandleBAL", p.CanHandleBAL.Load()) + } else { + log.Debug("SendNewBlock no BAL", "number", block.NumberU64(), "hash", block.Hash(), "peer", p.ID(), + "txNum", len(block.Transactions()), "canHandleBAL", p.CanHandleBAL.Load()) + } + return p2p.Send(p.rw, NewBlockMsg, &NewBlockPacket{ Block: block, TD: td, Sidecars: block.Sidecars(), + Bal: bal, }) } diff --git a/eth/protocols/eth/protocol.go b/eth/protocols/eth/protocol.go index 5d4566b7c2..60d8a2f6b9 100644 --- a/eth/protocols/eth/protocol.go +++ b/eth/protocols/eth/protocol.go @@ -233,7 +233,8 @@ type BlockHeadersRLPPacket struct { type NewBlockPacket struct { Block *types.Block TD *big.Int - Sidecars types.BlobSidecars `rlp:"optional"` + Sidecars types.BlobSidecars `rlp:"optional"` + Bal *types.BlockAccessListEncode `rlp:"optional"` } // sanityCheck verifies that the values are reasonable, as a DoS protection diff --git a/eth/sync_test.go b/eth/sync_test.go index cd6461c102..7bc5667437 100644 --- a/eth/sync_test.go +++ b/eth/sync_test.go @@ -138,7 +138,7 @@ func testChainSyncWithBlobs(t *testing.T, mode downloader.SyncMode, preCancunBlk // Sync up the two handlers via both `eth` and `snap` caps := []p2p.Cap{{Name: "eth", Version: ethVer}, {Name: "snap", Version: snapVer}} - emptyPipeEth, fullPipeEth := p2p.MsgPipe() + emptyPipeEth, fullPipeEth := p2p.MsgPipe(true) defer emptyPipeEth.Close() defer fullPipeEth.Close() @@ -154,7 +154,7 @@ func testChainSyncWithBlobs(t *testing.T, mode downloader.SyncMode, preCancunBlk return eth.Handle((*ethHandler)(full.handler), peer) }) - emptyPipeSnap, fullPipeSnap := p2p.MsgPipe() + emptyPipeSnap, fullPipeSnap := p2p.MsgPipe(true) defer emptyPipeSnap.Close() defer fullPipeSnap.Close() diff --git a/miner/worker.go b/miner/worker.go index ea462a96ba..8f82e10741 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -121,13 +121,13 @@ func (env *environment) copy() *environment { receipts: copyReceipts(env.receipts), committed: env.committed, } + cpy.state.TransferBlockAccessList(env.state) if env.gasPool != nil { gasPool := *env.gasPool cpy.gasPool = &gasPool } cpy.txs = make([]*types.Transaction, len(env.txs)) copy(cpy.txs, env.txs) - if env.sidecars != nil { cpy.sidecars = make(types.BlobSidecars, len(env.sidecars)) copy(cpy.sidecars, env.sidecars) @@ -664,6 +664,13 @@ func (w *worker) resultLoop() { w.recentMinedBlocks.Add(block.NumberU64(), []common.Hash{block.ParentHash()}) } + // add BAL to the block + bal := task.state.GetEncodedBlockAccessList(block) + if bal != nil && w.engine.SignBAL(bal) == nil { + block = block.WithBAL(bal) + } + task.state.DumpAccessList(block) + // Commit block and state to database. start := time.Now() status, err := w.chain.WriteBlockAndSetHead(block, receipts, logs, task.state, w.mux) @@ -680,7 +687,7 @@ func (w *worker) resultLoop() { stats.SendBlockTime.Store(time.Now().UnixMilli()) stats.StartMiningTime.Store(task.miningStartAt.UnixMilli()) log.Info("Successfully seal and write new block", "number", block.Number(), "sealhash", sealhash, "hash", hash, - "elapsed", common.PrettyDuration(time.Since(task.createdAt))) + "block size(noBal)", block.Size(), "balSize", block.BALSize(), "elapsed", common.PrettyDuration(time.Since(task.createdAt))) w.mux.Post(core.NewMinedBlockEvent{Block: block}) case <-w.exitCh: diff --git a/node/config.go b/node/config.go index bc8679756a..8e18fc7d62 100644 --- a/node/config.go +++ b/node/config.go @@ -104,6 +104,9 @@ type Config struct { // EnableQuickBlockFetching indicates whether to fetch new blocks using new messages. EnableQuickBlockFetching bool `toml:",omitempty"` + // EnableBAL enables the block access list feature + EnableBAL bool `toml:",omitempty"` + // RangeLimit enable 5000 blocks limit when handle range query RangeLimit bool `toml:",omitempty"` diff --git a/p2p/message.go b/p2p/message.go index 3ab56ee350..c61409cd8b 100644 --- a/p2p/message.go +++ b/p2p/message.go @@ -152,13 +152,21 @@ func (r *eofSignal) Read(buf []byte) (int, error) { // MsgPipe creates a message pipe. Reads on one end are matched // with writes on the other. The pipe is full-duplex, both ends // implement MsgReadWriter. -func MsgPipe() (*MsgPipeRW, *MsgPipeRW) { +func MsgPipe(args ...any) (*MsgPipeRW, *MsgPipeRW) { + noBlock := false + if len(args) > 0 { + noBlock = args[0].(bool) + } + c1, c2 := make(chan Msg), make(chan Msg) + if noBlock { + c1 = make(chan Msg, 1) + c2 = make(chan Msg, 1) + } var ( - c1, c2 = make(chan Msg), make(chan Msg) closing = make(chan struct{}) closed = new(atomic.Bool) - rw1 = &MsgPipeRW{c1, c2, closing, closed} - rw2 = &MsgPipeRW{c2, c1, closing, closed} + rw1 = &MsgPipeRW{c1, c2, closing, closed, noBlock} + rw2 = &MsgPipeRW{c2, c1, closing, closed, noBlock} ) return rw1, rw2 } @@ -173,6 +181,7 @@ type MsgPipeRW struct { r <-chan Msg closing chan struct{} closed *atomic.Bool + noBlock bool } // WriteMsg sends a message on the pipe. @@ -183,6 +192,9 @@ func (p *MsgPipeRW) WriteMsg(msg Msg) error { msg.Payload = &eofSignal{msg.Payload, msg.Size, consumed} select { case p.w <- msg: + if p.noBlock { + return nil + } if msg.Size > 0 { // wait for payload read or discard select { diff --git a/p2p/peer.go b/p2p/peer.go index e59a547fed..6d64e942e7 100644 --- a/p2p/peer.go +++ b/p2p/peer.go @@ -129,6 +129,9 @@ type Peer struct { // it indicates the peer is in the validator network, it will directly broadcast when miner/sentry broadcast mined block, // and won't broadcast any txs between EVN peers. EVNPeerFlag atomic.Bool + + // it indicates the peer can handle BAL(block access list) packet + CanHandleBAL atomic.Bool } // NewPeer returns a peer for testing purposes. diff --git a/params/network_params.go b/params/network_params.go index 804336ccb8..c9572a25e5 100644 --- a/params/network_params.go +++ b/params/network_params.go @@ -22,6 +22,9 @@ package params const ( // StableStateThreshold is the reserve number of block state save to disk before delete ancientdb StableStateThreshold uint64 = 128 + + // MaxBALSize is the maximum bytes of the rlp encoded block access list: 1MB + MaxBALSize uint64 = 1048576 ) var ( diff --git a/version/version.go b/version/version.go index e8b4e121b2..8e5e9502a5 100644 --- a/version/version.go +++ b/version/version.go @@ -17,8 +17,8 @@ package version const ( - Major = 1 // Major version component of the current release - Minor = 6 // Minor version component of the current release - Patch = 0 // Patch version component of the current release - Meta = "" // Version metadata to append to the version string + Major = 1 // Major version component of the current release + Minor = 6 // Minor version component of the current release + Patch = 100 // Patch version component of the current release + Meta = "BAL" // Version metadata to append to the version string )