From 8ab867c18cde4c592dea9b203a179934b278ea07 Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Tue, 2 Feb 2021 12:51:08 +0100 Subject: [PATCH 01/11] tests: added stData and stAccessList --- tests/state_test_util.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tests/state_test_util.go b/tests/state_test_util.go index 03dc01459c09..c36529960014 100644 --- a/tests/state_test_util.go +++ b/tests/state_test_util.go @@ -99,12 +99,22 @@ type stTransaction struct { GasPrice *big.Int `json:"gasPrice"` Nonce uint64 `json:"nonce"` To string `json:"to"` - Data []string `json:"data"` + Data stData `json:"data"` GasLimit []uint64 `json:"gasLimit"` Value []string `json:"value"` PrivateKey []byte `json:"secretKey"` } +type stData struct { + Data []string `json:"data"` + AccessList []stAccessList `json:"accessList"` +} + +type stAccessList struct { + Address string `json:"address"` + StorageKeys []string `json:"storageKeys"` +} + type stTransactionMarshaling struct { GasPrice *math.HexOrDecimal256 Nonce math.HexOrDecimal64 From 24ceafe25f8d0c0b57f676d51e68f9bb123ce0fd Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Tue, 2 Feb 2021 14:16:09 +0100 Subject: [PATCH 02/11] tests: fixed marshalling --- tests/state_test_util.go | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/tests/state_test_util.go b/tests/state_test_util.go index c36529960014..38ce7e3c84fa 100644 --- a/tests/state_test_util.go +++ b/tests/state_test_util.go @@ -106,13 +106,8 @@ type stTransaction struct { } type stData struct { - Data []string `json:"data"` - AccessList []stAccessList `json:"accessList"` -} - -type stAccessList struct { - Address string `json:"address"` - StorageKeys []string `json:"storageKeys"` + Data []string `json:"data,omitempty"` + AccessList []*types.AccessList `json:"accessList,omitempty"` } type stTransactionMarshaling struct { From b1b05a6312241a549dea5afe8e5e3fbe9172ae22 Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Tue, 2 Feb 2021 16:27:12 +0100 Subject: [PATCH 03/11] tests: fixed marshalling --- tests/state_test_util.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/state_test_util.go b/tests/state_test_util.go index 38ce7e3c84fa..44fce945732e 100644 --- a/tests/state_test_util.go +++ b/tests/state_test_util.go @@ -99,14 +99,14 @@ type stTransaction struct { GasPrice *big.Int `json:"gasPrice"` Nonce uint64 `json:"nonce"` To string `json:"to"` - Data stData `json:"data"` + Data []stData `json:"data"` GasLimit []uint64 `json:"gasLimit"` Value []string `json:"value"` PrivateKey []byte `json:"secretKey"` } type stData struct { - Data []string `json:"data,omitempty"` + Data string `json:"data,omitempty"` AccessList []*types.AccessList `json:"accessList,omitempty"` } From 9987c82892e47f964bbc3f2c74c0e9eac54aed91 Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Tue, 2 Feb 2021 16:45:06 +0100 Subject: [PATCH 04/11] tests: regenerated marshalling --- tests/gen_stenv.go | 2 ++ tests/gen_sttransaction.go | 6 ++++-- tests/gen_vmexec.go | 2 ++ tests/state_test_util.go | 16 ++++++++++------ 4 files changed, 18 insertions(+), 8 deletions(-) diff --git a/tests/gen_stenv.go b/tests/gen_stenv.go index 1d4baf2fd79c..bfecc145b46b 100644 --- a/tests/gen_stenv.go +++ b/tests/gen_stenv.go @@ -13,6 +13,7 @@ import ( var _ = (*stEnvMarshaling)(nil) +// MarshalJSON marshals as JSON. func (s stEnv) MarshalJSON() ([]byte, error) { type stEnv struct { Coinbase common.UnprefixedAddress `json:"currentCoinbase" gencodec:"required"` @@ -30,6 +31,7 @@ func (s stEnv) MarshalJSON() ([]byte, error) { return json.Marshal(&enc) } +// UnmarshalJSON unmarshals from JSON. func (s *stEnv) UnmarshalJSON(input []byte) error { type stEnv struct { Coinbase *common.UnprefixedAddress `json:"currentCoinbase" gencodec:"required"` diff --git a/tests/gen_sttransaction.go b/tests/gen_sttransaction.go index 451ffcbf43a1..c85c2684fda3 100644 --- a/tests/gen_sttransaction.go +++ b/tests/gen_sttransaction.go @@ -12,12 +12,13 @@ import ( var _ = (*stTransactionMarshaling)(nil) +// MarshalJSON marshals as JSON. func (s stTransaction) MarshalJSON() ([]byte, error) { type stTransaction struct { GasPrice *math.HexOrDecimal256 `json:"gasPrice"` Nonce math.HexOrDecimal64 `json:"nonce"` To string `json:"to"` - Data []string `json:"data"` + Data []stData `json:"data"` GasLimit []math.HexOrDecimal64 `json:"gasLimit"` Value []string `json:"value"` PrivateKey hexutil.Bytes `json:"secretKey"` @@ -38,12 +39,13 @@ func (s stTransaction) MarshalJSON() ([]byte, error) { return json.Marshal(&enc) } +// UnmarshalJSON unmarshals from JSON. func (s *stTransaction) UnmarshalJSON(input []byte) error { type stTransaction struct { GasPrice *math.HexOrDecimal256 `json:"gasPrice"` Nonce *math.HexOrDecimal64 `json:"nonce"` To *string `json:"to"` - Data []string `json:"data"` + Data []stData `json:"data"` GasLimit []math.HexOrDecimal64 `json:"gasLimit"` Value []string `json:"value"` PrivateKey *hexutil.Bytes `json:"secretKey"` diff --git a/tests/gen_vmexec.go b/tests/gen_vmexec.go index a5f01cf45695..2fe155152d0c 100644 --- a/tests/gen_vmexec.go +++ b/tests/gen_vmexec.go @@ -14,6 +14,7 @@ import ( var _ = (*vmExecMarshaling)(nil) +// MarshalJSON marshals as JSON. func (v vmExec) MarshalJSON() ([]byte, error) { type vmExec struct { Address common.UnprefixedAddress `json:"address" gencodec:"required"` @@ -37,6 +38,7 @@ func (v vmExec) MarshalJSON() ([]byte, error) { return json.Marshal(&enc) } +// UnmarshalJSON unmarshals from JSON. func (v *vmExec) UnmarshalJSON(input []byte) error { type vmExec struct { Address *common.UnprefixedAddress `json:"address" gencodec:"required"` diff --git a/tests/state_test_util.go b/tests/state_test_util.go index 44fce945732e..e1151e136de8 100644 --- a/tests/state_test_util.go +++ b/tests/state_test_util.go @@ -106,8 +106,8 @@ type stTransaction struct { } type stData struct { - Data string `json:"data,omitempty"` - AccessList []*types.AccessList `json:"accessList,omitempty"` + Data string `json:"data,omitempty"` + AccessList *types.AccessList `json:"accessList,omitempty"` } type stTransactionMarshaling struct { @@ -294,12 +294,16 @@ func (tx *stTransaction) toMessage(ps stPostState) (core.Message, error) { } value = v } - data, err := hex.DecodeString(strings.TrimPrefix(dataHex, "0x")) - if err != nil { - return nil, fmt.Errorf("invalid tx data %q", dataHex) + var data []byte + if dataHex.Data != "" { + d, err := hex.DecodeString(strings.TrimPrefix(dataHex.Data, "0x")) + if err != nil { + return nil, fmt.Errorf("invalid tx data %q", dataHex) + } + data = d } - msg := types.NewMessage(from, to, tx.Nonce, value, gasLimit, tx.GasPrice, data, nil, true) + msg := types.NewMessage(from, to, tx.Nonce, value, gasLimit, tx.GasPrice, data, dataHex.AccessList, true) return msg, nil } From 1e3cc1eedc83c86fff7f4c1a12856f89d33d90a2 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Fri, 19 Feb 2021 14:15:14 +0100 Subject: [PATCH 05/11] tests: changes in state-test format --- tests/gen_sttransaction.go | 35 +++++++++++++++++++++-------------- tests/state_test.go | 8 -------- tests/state_test_util.go | 33 ++++++++++++--------------------- 3 files changed, 33 insertions(+), 43 deletions(-) diff --git a/tests/gen_sttransaction.go b/tests/gen_sttransaction.go index c85c2684fda3..2670f4f9c8e7 100644 --- a/tests/gen_sttransaction.go +++ b/tests/gen_sttransaction.go @@ -8,6 +8,7 @@ import ( "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/common/math" + "github.com/ethereum/go-ethereum/core/types" ) var _ = (*stTransactionMarshaling)(nil) @@ -15,19 +16,21 @@ var _ = (*stTransactionMarshaling)(nil) // MarshalJSON marshals as JSON. func (s stTransaction) MarshalJSON() ([]byte, error) { type stTransaction struct { - GasPrice *math.HexOrDecimal256 `json:"gasPrice"` - Nonce math.HexOrDecimal64 `json:"nonce"` - To string `json:"to"` - Data []stData `json:"data"` - GasLimit []math.HexOrDecimal64 `json:"gasLimit"` - Value []string `json:"value"` - PrivateKey hexutil.Bytes `json:"secretKey"` + GasPrice *math.HexOrDecimal256 `json:"gasPrice"` + Nonce math.HexOrDecimal64 `json:"nonce"` + To string `json:"to"` + Data []string `json:"data"` + AccessLists []*types.AccessList `json:"accessLists,omitempty"` + GasLimit []math.HexOrDecimal64 `json:"gasLimit"` + Value []string `json:"value"` + PrivateKey hexutil.Bytes `json:"secretKey"` } var enc stTransaction enc.GasPrice = (*math.HexOrDecimal256)(s.GasPrice) enc.Nonce = math.HexOrDecimal64(s.Nonce) enc.To = s.To enc.Data = s.Data + enc.AccessLists = s.AccessLists if s.GasLimit != nil { enc.GasLimit = make([]math.HexOrDecimal64, len(s.GasLimit)) for k, v := range s.GasLimit { @@ -42,13 +45,14 @@ func (s stTransaction) MarshalJSON() ([]byte, error) { // UnmarshalJSON unmarshals from JSON. func (s *stTransaction) UnmarshalJSON(input []byte) error { type stTransaction struct { - GasPrice *math.HexOrDecimal256 `json:"gasPrice"` - Nonce *math.HexOrDecimal64 `json:"nonce"` - To *string `json:"to"` - Data []stData `json:"data"` - GasLimit []math.HexOrDecimal64 `json:"gasLimit"` - Value []string `json:"value"` - PrivateKey *hexutil.Bytes `json:"secretKey"` + GasPrice *math.HexOrDecimal256 `json:"gasPrice"` + Nonce *math.HexOrDecimal64 `json:"nonce"` + To *string `json:"to"` + Data []string `json:"data"` + AccessLists []*types.AccessList `json:"accessLists,omitempty"` + GasLimit []math.HexOrDecimal64 `json:"gasLimit"` + Value []string `json:"value"` + PrivateKey *hexutil.Bytes `json:"secretKey"` } var dec stTransaction if err := json.Unmarshal(input, &dec); err != nil { @@ -66,6 +70,9 @@ func (s *stTransaction) UnmarshalJSON(input []byte) error { if dec.Data != nil { s.Data = dec.Data } + if dec.AccessLists != nil { + s.AccessLists = dec.AccessLists + } if dec.GasLimit != nil { s.GasLimit = make([]uint64, len(dec.GasLimit)) for k, v := range dec.GasLimit { diff --git a/tests/state_test.go b/tests/state_test.go index 0f2bd3853276..b77a898c21f6 100644 --- a/tests/state_test.go +++ b/tests/state_test.go @@ -62,14 +62,6 @@ func TestState(t *testing.T) { } { st.walk(t, dir, func(t *testing.T, name string, test *StateTest) { for _, subtest := range test.Subtests() { - if subtest.Fork == "Berlin" { - // Our current berlin-tests were generated using YOLOv2 rules, hence a lot of them - // fail when berlin is defined as YOLOv3. We skip those, until they've been - // regenerated and re-imported - // TODO (@holiman) - continue - } - subtest := subtest key := fmt.Sprintf("%s/%d", subtest.Fork, subtest.Index) name := name + "/" + key diff --git a/tests/state_test_util.go b/tests/state_test_util.go index e1151e136de8..56cb3265eb1a 100644 --- a/tests/state_test_util.go +++ b/tests/state_test_util.go @@ -96,18 +96,14 @@ type stEnvMarshaling struct { //go:generate gencodec -type stTransaction -field-override stTransactionMarshaling -out gen_sttransaction.go type stTransaction struct { - GasPrice *big.Int `json:"gasPrice"` - Nonce uint64 `json:"nonce"` - To string `json:"to"` - Data []stData `json:"data"` - GasLimit []uint64 `json:"gasLimit"` - Value []string `json:"value"` - PrivateKey []byte `json:"secretKey"` -} - -type stData struct { - Data string `json:"data,omitempty"` - AccessList *types.AccessList `json:"accessList,omitempty"` + GasPrice *big.Int `json:"gasPrice"` + Nonce uint64 `json:"nonce"` + To string `json:"to"` + Data []string `json:"data"` + AccessLists []*types.AccessList `json:"accessLists,omitempty"` + GasLimit []uint64 `json:"gasLimit"` + Value []string `json:"value"` + PrivateKey []byte `json:"secretKey"` } type stTransactionMarshaling struct { @@ -294,16 +290,11 @@ func (tx *stTransaction) toMessage(ps stPostState) (core.Message, error) { } value = v } - var data []byte - if dataHex.Data != "" { - d, err := hex.DecodeString(strings.TrimPrefix(dataHex.Data, "0x")) - if err != nil { - return nil, fmt.Errorf("invalid tx data %q", dataHex) - } - data = d + data, err := hex.DecodeString(strings.TrimPrefix(dataHex, "0x")) + if err != nil { + return nil, fmt.Errorf("invalid tx data %q", dataHex) } - - msg := types.NewMessage(from, to, tx.Nonce, value, gasLimit, tx.GasPrice, data, dataHex.AccessList, true) + msg := types.NewMessage(from, to, tx.Nonce, value, gasLimit, tx.GasPrice, data, tx.AccessLists[ps.Indexes.Data], true) return msg, nil } From c2050dcc0aa38257a9bb678257991320d83fc835 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Fri, 19 Feb 2021 14:27:47 +0100 Subject: [PATCH 06/11] tests: run berlin blocktests too --- tests/block_test.go | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/tests/block_test.go b/tests/block_test.go index 84618fd0be5c..2649bae85a2f 100644 --- a/tests/block_test.go +++ b/tests/block_test.go @@ -25,7 +25,11 @@ func TestBlockchain(t *testing.T) { bt := new(testMatcher) // General state tests are 'exported' as blockchain tests, but we can run them natively. - bt.skipLoad(`^GeneralStateTests/`) + // For speedier CI-runs, the line below can be uncommented, so those are skipped. + // For now, in hardfork-times (Berlin), we run the tests both as StateTests and + // as blockchain tests, since the latter also covers things like receipt root + //bt.skipLoad(`^GeneralStateTests/`) + // Skip random failures due to selfish mining test bt.skipLoad(`.*bcForgedTest/bcForkUncle\.json`) @@ -43,15 +47,7 @@ func TestBlockchain(t *testing.T) { // test takes a lot for time and goes easily OOM because of sha3 calculation on a huge range, // using 4.6 TGas bt.skipLoad(`.*randomStatetest94.json.*`) - bt.walk(t, blockTestDir, func(t *testing.T, name string, test *BlockTest) { - if test.json.Network == "Berlin" { - // Our current berlin-tests were generated using YOLOv2 rules, hence a lot of them - // fail when berlin is defined as YOLOv3. We skip those, until they've been - // regenerated and re-imported - // TODO (@holiman) - return - } if err := bt.checkFailure(t, name+"/trie", test.Run(false)); err != nil { t.Errorf("test without snapshotter failed: %v", err) } From 67f8d32254575154e359e3d2617772d6bba55d61 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Mon, 22 Feb 2021 13:05:46 +0100 Subject: [PATCH 07/11] tests: handle legacy tests without access lists --- tests/state_test_util.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/state_test_util.go b/tests/state_test_util.go index 56cb3265eb1a..62eb9cd38279 100644 --- a/tests/state_test_util.go +++ b/tests/state_test_util.go @@ -294,7 +294,11 @@ func (tx *stTransaction) toMessage(ps stPostState) (core.Message, error) { if err != nil { return nil, fmt.Errorf("invalid tx data %q", dataHex) } - msg := types.NewMessage(from, to, tx.Nonce, value, gasLimit, tx.GasPrice, data, tx.AccessLists[ps.Indexes.Data], true) + var accessList *types.AccessList + if len(tx.AccessLists) > 0 { + accessList = tx.AccessLists[ps.Indexes.Data] + } + msg := types.NewMessage(from, to, tx.Nonce, value, gasLimit, tx.GasPrice, data, accessList, true) return msg, nil } From 3291e5edf198acab392884748f94fbbd8608d5ed Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Thu, 25 Feb 2021 11:11:08 +0100 Subject: [PATCH 08/11] changes from rebase --- tests/state_test_util.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/state_test_util.go b/tests/state_test_util.go index 62eb9cd38279..a1d4213b4ee0 100644 --- a/tests/state_test_util.go +++ b/tests/state_test_util.go @@ -294,9 +294,9 @@ func (tx *stTransaction) toMessage(ps stPostState) (core.Message, error) { if err != nil { return nil, fmt.Errorf("invalid tx data %q", dataHex) } - var accessList *types.AccessList - if len(tx.AccessLists) > 0 { - accessList = tx.AccessLists[ps.Indexes.Data] + var accessList types.AccessList + if tx.AccessLists != nil && tx.AccessLists[ps.Indexes.Data] != nil { + accessList = *tx.AccessLists[ps.Indexes.Data] } msg := types.NewMessage(from, to, tx.Nonce, value, gasLimit, tx.GasPrice, data, accessList, true) return msg, nil From ab150e1160c5822934196c4e074a5a2694602008 Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Mon, 1 Mar 2021 10:18:32 +0100 Subject: [PATCH 09/11] tests: update tests to 7.0.2 --- tests/testdata | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/testdata b/tests/testdata index 6c863f03bee8..987463367a4a 160000 --- a/tests/testdata +++ b/tests/testdata @@ -1 +1 @@ -Subproject commit 6c863f03bee8d7a66bb7a028a9f880a86a5f4975 +Subproject commit 987463367a4af0cf9245bdfaba1e51dbb1652ff2 From fd960622dbb632b57d5fe05709dcfa1d37a0f7c1 Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Mon, 1 Mar 2021 10:24:50 +0100 Subject: [PATCH 10/11] tests: update tests to v7.0.1 --- tests/testdata | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/testdata b/tests/testdata index 987463367a4a..1508126ea04c 160000 --- a/tests/testdata +++ b/tests/testdata @@ -1 +1 @@ -Subproject commit 987463367a4af0cf9245bdfaba1e51dbb1652ff2 +Subproject commit 1508126ea04cd61495b60db2f036ac823de274b1 From 70f5e99d4443fc05cbf8d6f06a70068d1a18acee Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Mon, 1 Mar 2021 10:31:40 +0100 Subject: [PATCH 11/11] tests: update tests to develop@31d6630 --- tests/testdata | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/testdata b/tests/testdata index 1508126ea04c..31d663076b66 160000 --- a/tests/testdata +++ b/tests/testdata @@ -1 +1 @@ -Subproject commit 1508126ea04cd61495b60db2f036ac823de274b1 +Subproject commit 31d663076b6678df18983d6da912d7cad4ad3416