tests: Enable invalid block blockchain tests

circle.yml

@@ -361,8 +361,11 @@ jobs:
       - run:
           name: "Execution spec tests (blockchain_tests)"
           working_directory: ~/build
+          # 4844 and 4788 temporarily turned off b/c they rely on blob features.
           command: >
-            bin/evmone-blockchaintest ~/spec-tests/fixtures/blockchain_tests
+            bin/evmone-blockchaintest
+            --gtest_filter='*:-*eip4844*:*eip4788*'
+            ~/spec-tests/fixtures/blockchain_tests
       - download_execution_spec_tests:
           release: pectra-devnet-4@v1.0.1
           fixtures_suffix: pectra-devnet-4
@@ -402,8 +405,10 @@ jobs:
       - run:
           name: "EOF pre-release execution spec tests (blockchain_tests)"
           working_directory: ~/build
+          # 4844 and 4788 temporarily turned off b/c they rely on blob features.
           command: >
             bin/evmone-blockchaintest ~/spec-tests/fixtures/blockchain_tests/osaka
+            --gtest_filter='*:-*stEIP4844*'
       - run:
           name: "EOF pre-release execution spec tests (eof_tests)"
           working_directory: ~/build
@@ -438,16 +443,18 @@ jobs:
       - run:
           name: "Blockchain tests (GeneralStateTests)"
           working_directory: ~/build
+          # 4844 and 4788 temporarily turned off b/c they rely on blob features.
           command: >
             bin/evmone-blockchaintest
-            --gtest_filter='*:-VMTests/vmPerformance.*:*.*Call50000_sha256:*.CALLBlake2f_MaxRounds'
+            --gtest_filter='*:-VMTests/vmPerformance.*:*.*Call50000_sha256:*.CALLBlake2f_MaxRounds:*eip4844*:*stEIP4844*:*eip4788*'
             ~/tests/BlockchainTests/GeneralStateTests
       - run:
           name: "Blockchain tests (ValidBlocks)"
           working_directory: ~/build
+          # 4844 and 4788 temporarily turned off b/c they rely on blob features.
           command: >
             bin/evmone-blockchaintest
-            --gtest_filter='*:-bcMultiChainTest.*:bcTotalDifficultyTest.*:bcForkStressTest.ForkStressTest:bcGasPricerTest.RPC_API_Test:bcValidBlockTest.SimpleTx3LowS'
+            --gtest_filter='*:-bcMultiChainTest.*:bcTotalDifficultyTest.*:bcForkStressTest.ForkStressTest:bcGasPricerTest.RPC_API_Test:bcValidBlockTest.SimpleTx3LowS:*bcEIP4844*'
             ~/tests/BlockchainTests/ValidBlocks
             ~/tests/LegacyTests/Cancun/BlockchainTests/ValidBlocks
       - collect_coverage_gcc

test/blockchaintest/blockchaintest.hpp

@@ -47,6 +47,7 @@ struct TestBlock
 {
     state::BlockInfo block_info;
     std::vector<state::Transaction> transactions;
+    bool valid = true;
 
     BlockHeader expected_block_header;
 };

test/blockchaintest/blockchaintest_loader.cpp

@@ -73,18 +73,6 @@ static TestBlock load_test_block(const json::json& j, evmc_revision rev)
         }
     }
 
-    if (const auto it = j.find("expectException"); it != j.end())
-    {
-        // TODO: Add support for invalid blocks.
-        throw UnsupportedTestFeature("tests with invalid blocks are not supported");
-    }
-
-    if (const auto it = j.find("transactionSequence"); it != j.end())
-    {
-        // TODO: Add support for invalid blocks.
-        throw UnsupportedTestFeature("tests with invalid transactions are not supported");
-    }
-
     if (const auto it = j.find("uncleHeaders"); it != j.end())
     {
         const auto current_block_number = tb.block_info.number;
@@ -124,7 +112,25 @@ BlockchainTest load_blockchain_test_case(const std::string& name, const json::js
     bt.rev = to_rev_schedule(j.at("network").get<std::string>());
 
     for (const auto& el : j.at("blocks"))
-        bt.test_blocks.emplace_back(load_test_block(el, bt.rev.get_revision(0)));
+    {
+        if (const auto it = el.find("expectException"); it != el.end())
+        {
+            // `rlp_decoded` holds the `FixtureBlock` element with the relevant block data for
+            // invalid blocks within a test. It should be a sibling element to `expectException`.
+
+            // TODO: Add support for invalidly rlp-encoded blocks, which do
+            // not have `rlp_decoded`.
+            if (!el.contains("rlp_decoded"))
+                throw UnsupportedTestFeature(
+                    "tests with invalidly rlp-encoded blocks are not supported");
+
+            auto test_block = load_test_block(el.at("rlp_decoded"), bt.rev.get_revision(0));
+            test_block.valid = false;
+            bt.test_blocks.emplace_back(test_block);
+        }
+        else
+            bt.test_blocks.emplace_back(load_test_block(el, bt.rev.get_revision(0)));
+    }
 
     bt.expectation.last_block_hash = from_json<hash256>(j.at("lastblockhash"));
 

test/blockchaintest/blockchaintest_runner.cpp

@@ -26,6 +26,8 @@ struct TransitionResult
     std::vector<state::Requests> requests;
     int64_t gas_used;
     state::BloomFilter bloom;
+    int64_t blob_gas_left;
+    TestState block_state;
 };
 
 namespace
@@ -34,11 +36,12 @@ TransitionResult apply_block(TestState& state, evmc::VM& vm, const state::BlockI
     const state::BlockHashes& block_hashes, const std::vector<state::Transaction>& txs,
     evmc_revision rev, std::optional<int64_t> block_reward)
 {
-    system_call(state, block, block_hashes, rev, vm);
+    TestState block_state(state);
+    system_call(block_state, block, block_hashes, rev, vm);
 
     std::vector<state::Log> txs_logs;
     int64_t block_gas_left = block.gas_limit;
-    int64_t blob_gas_left = state::BlockInfo::MAX_BLOB_GAS_PER_BLOCK;
+    int64_t blob_gas_left = 0;
 
     std::vector<RejectedTransaction> rejected_txs;
     std::vector<state::TransactionReceipt> receipts;
@@ -50,8 +53,8 @@ TransitionResult apply_block(TestState& state, evmc::VM& vm, const state::BlockI
         const auto& tx = txs[i];
 
         const auto computed_tx_hash = keccak256(rlp::encode(tx));
-        auto res =
-            transition(state, block, block_hashes, tx, rev, vm, block_gas_left, blob_gas_left);
+        auto res = test::transition(
+            block_state, block, block_hashes, tx, rev, vm, block_gas_left, blob_gas_left);
 
         if (holds_alternative<std::error_code>(res))
         {
@@ -68,7 +71,7 @@ TransitionResult apply_block(TestState& state, evmc::VM& vm, const state::BlockI
             cumulative_gas_used += receipt.gas_used;
             receipt.cumulative_gas_used = cumulative_gas_used;
             if (rev < EVMC_BYZANTIUM)
-                receipt.post_state = state::mpt_hash(state);
+                receipt.post_state = state::mpt_hash(block_state);
 
             block_gas_left -= receipt.gas_used;
             blob_gas_left -= tx.blob_gas_used();
@@ -82,11 +85,19 @@ TransitionResult apply_block(TestState& state, evmc::VM& vm, const state::BlockI
                                   state::Requests(state::Requests::Type::consolidation)} :
                               std::vector<state::Requests>{});
 
-    finalize(state, rev, block.coinbase, block_reward, block.ommers, block.withdrawals);
+    finalize(block_state, rev, block.coinbase, block_reward, block.ommers, block.withdrawals);
 
     const auto bloom = compute_bloom_filter(receipts);
+
     return {std::move(receipts), std::move(rejected_txs), std::move(requests), cumulative_gas_used,
-        bloom};
+        bloom, blob_gas_left, std::move(block_state)};
+}
+
+bool validate_block(evmc_revision, const TestBlock&, const BlockHeader&) noexcept
+{
+    // NOTE: includes only block validity unrelated to individual txs. See `apply_block`.
+
+    return true;
 }
 
 std::optional<int64_t> mining_reward(evmc_revision rev) noexcept
@@ -149,58 +160,111 @@ void run_blockchain_tests(std::span<const BlockchainTest> tests, evmc::VM& vm)
 
         TestBlockHashes block_hashes{
             {c.genesis_block_header.block_number, c.genesis_block_header.hash}};
-
-        for (const auto& test_block : c.test_blocks)
+        for (size_t i = 0; i < c.test_blocks.size(); ++i)
         {
+            const auto& test_block = c.test_blocks[i];
+            const auto& parent_header =
+                i == 0 ? c.genesis_block_header : c.test_blocks[i - 1].expected_block_header;
+
             auto bi = test_block.block_info;
 
             const auto rev = c.rev.get_revision(bi.timestamp);
 
-            const auto res = apply_block(
-                state, vm, bi, block_hashes, test_block.transactions, rev, mining_reward(rev));
-
-            block_hashes[test_block.expected_block_header.block_number] =
-                test_block.expected_block_header.hash;
-
             SCOPED_TRACE(std::string{evmc::to_string(rev)} + '/' + std::to_string(case_index) +
                          '/' + c.name + '/' + std::to_string(test_block.block_info.number));
 
-            EXPECT_EQ(state::mpt_hash(state), test_block.expected_block_header.state_root);
-
-            if (rev >= EVMC_SHANGHAI)
+            if (test_block.valid)
             {
-                EXPECT_EQ(state::mpt_hash(test_block.block_info.withdrawals),
-                    test_block.expected_block_header.withdrawal_root);
+                EXPECT_TRUE(validate_block(rev, test_block, parent_header))
+                    << "Expected block to be valid (validate_block)";
+
+                const auto res = apply_block(
+                    state, vm, bi, block_hashes, test_block.transactions, rev, mining_reward(rev));
+
+                block_hashes[test_block.expected_block_header.block_number] =
+                    test_block.expected_block_header.hash;
+                state = res.block_state;
+
+                EXPECT_TRUE(res.rejected.empty())
+                    << "Invalid transaction in block expected to be valid";
+                EXPECT_TRUE(res.blob_gas_left == 0)
+                    << "Transactions used more or less blob gas than expected in block header";
+
+                EXPECT_EQ(state::mpt_hash(state), test_block.expected_block_header.state_root);
+
+                if (rev >= EVMC_SHANGHAI)
+                {
+                    EXPECT_EQ(state::mpt_hash(test_block.block_info.withdrawals),
+                        test_block.expected_block_header.withdrawal_root);
+                }
+
+                EXPECT_EQ(state::mpt_hash(test_block.transactions),
+                    test_block.expected_block_header.transactions_root);
+                EXPECT_EQ(
+                    state::mpt_hash(res.receipts), test_block.expected_block_header.receipts_root);
+                if (rev >= EVMC_PRAGUE)
+                {
+                    EXPECT_EQ(calculate_requests_hash(res.requests),
+                        test_block.expected_block_header.requests_hash);
+                }
+                EXPECT_EQ(res.gas_used, test_block.expected_block_header.gas_used);
+                EXPECT_EQ(
+                    bytes_view{res.bloom}, bytes_view{test_block.expected_block_header.logs_bloom});
             }
-
-            EXPECT_EQ(state::mpt_hash(test_block.transactions),
-                test_block.expected_block_header.transactions_root);
-            EXPECT_EQ(
-                state::mpt_hash(res.receipts), test_block.expected_block_header.receipts_root);
-            if (rev >= EVMC_PRAGUE)
+            else
             {
-                EXPECT_EQ(calculate_requests_hash(res.requests),
-                    test_block.expected_block_header.requests_hash);
+                if (!validate_block(rev, test_block, parent_header))
+                    continue;
+
+                const auto res = apply_block(
+                    state, vm, bi, block_hashes, test_block.transactions, rev, mining_reward(rev));
+                if (!res.rejected.empty())
+                    continue;
+                if (res.blob_gas_left != 0)
+                    continue;
+
+                if (state::mpt_hash(res.block_state) != test_block.expected_block_header.state_root)
+                    continue;
+
+                if (rev >= EVMC_SHANGHAI && state::mpt_hash(test_block.block_info.withdrawals) !=
+                                                test_block.expected_block_header.withdrawal_root)
+                    continue;
+                if (state::mpt_hash(test_block.transactions) !=
+                    test_block.expected_block_header.transactions_root)
+                    continue;
+                if (state::mpt_hash(res.receipts) != test_block.expected_block_header.receipts_root)
+                    continue;
+                if (rev >= EVMC_PRAGUE && calculate_requests_hash(res.requests) !=
+                                              test_block.expected_block_header.requests_hash)
+                    continue;
+                if (res.gas_used != test_block.expected_block_header.gas_used)
+                    continue;
+                if (bytes_view{res.bloom} !=
+                    bytes_view{test_block.expected_block_header.logs_bloom})
+                    continue;
+
+                EXPECT_TRUE(false) << "Expected block to be invalid but resulted valid";
+
+                // Apply the resulting state in order to continue testing expectations, even if
+                // the test already has gone into failed state here.
+                block_hashes[test_block.expected_block_header.block_number] =
+                    test_block.expected_block_header.hash;
+                state = res.block_state;
             }
-            EXPECT_EQ(res.gas_used, test_block.expected_block_header.gas_used);
-            EXPECT_EQ(
-                bytes_view{res.bloom}, bytes_view{test_block.expected_block_header.logs_bloom});
-
-            // TODO: Add difficulty calculation verification.
         }
-
         const auto expected_post_hash =
             std::holds_alternative<TestState>(c.expectation.post_state) ?
                 state::mpt_hash(std::get<TestState>(c.expectation.post_state)) :
                 std::get<hash256>(c.expectation.post_state);
-        EXPECT_TRUE(state::mpt_hash(state) == expected_post_hash)
+        EXPECT_EQ(state::mpt_hash(state), expected_post_hash)
             << "Result state:\n"
             << print_state(state)
             << (std::holds_alternative<TestState>(c.expectation.post_state) ?
                        "\n\nExpected state:\n" +
                            print_state(std::get<TestState>(c.expectation.post_state)) :
                        "");
     }
+    // TODO: Add difficulty calculation verification.
 }
 
 }  // namespace evmone::test

test/statetest/statetest_loader.cpp

@@ -300,7 +300,8 @@ TestState from_json<TestState>(const json::json& j)
 /// Load common parts of Transaction or TestMultiTransaction.
 static void from_json_tx_common(const json::json& j, state::Transaction& o)
 {
-    o.sender = from_json<evmc::address>(j.at("sender"));
+    // `sender` is not provided for transactions in invalid blocks.
+    o.sender = load_if_exists<evmc::address>(j, "sender");
     o.nonce = from_json<uint64_t>(j.at("nonce"));
 
     if (const auto chain_id_it = j.find("chainId"); chain_id_it != j.end())