diff --git a/.github/workflows/nethermind-tests-checked.yml b/.github/workflows/nethermind-tests-checked.yml index 18830556fba5..39d31b11abba 100644 --- a/.github/workflows/nethermind-tests-checked.yml +++ b/.github/workflows/nethermind-tests-checked.yml @@ -187,6 +187,7 @@ jobs: env: TEST_CHUNK: ${{ matrix.test.chunk }} DOTNET_EnableHWIntrinsic: ${{ matrix.variant.hw-intrinsic }} + TEST_SKIP_HEAVY: 1 run: | dotnet test --project ${{ matrix.test.project }}.csproj -c release \ ${{ matrix.variant.dotnet-args }} diff --git a/.github/workflows/nethermind-tests-flat.yml b/.github/workflows/nethermind-tests-flat.yml index 85728a30d005..1b6e834a47fe 100644 --- a/.github/workflows/nethermind-tests-flat.yml +++ b/.github/workflows/nethermind-tests-flat.yml @@ -96,6 +96,7 @@ jobs: working-directory: src/Nethermind/${{ matrix.project }} env: TEST_CHUNK: ${{ matrix.chunk }} + TEST_SKIP_HEAVY: 1 run: | dotnet test --project ${{ matrix.project }}.csproj -c release diff --git a/.github/workflows/nethermind-tests.yml b/.github/workflows/nethermind-tests.yml index e01f1a83bf66..a1ee7729fed3 100644 --- a/.github/workflows/nethermind-tests.yml +++ b/.github/workflows/nethermind-tests.yml @@ -282,6 +282,7 @@ jobs: working-directory: src/Nethermind/${{ matrix.test.project }} env: TEST_CHUNK: ${{ matrix.test.chunk }} + TEST_SKIP_HEAVY: 1 run: | set +e dotnet test --project ${{ matrix.test.project }}.csproj -c release diff --git a/.gitignore b/.gitignore index 11527d8ecd5d..27aafb566091 100644 --- a/.gitignore +++ b/.gitignore @@ -454,3 +454,4 @@ keystore/ # Worktrees .worktrees/ +/.dotnet-cli diff --git a/cspell.json b/cspell.json index d33bd8201c5f..f82e47eddfee 100644 --- a/cspell.json +++ b/cspell.json @@ -6,6 +6,7 @@ "ignoreRandomStrings": true, "unknownWords": "report-common-typos", "ignorePaths": [ + ".gitignore", "**/*.json", "**/*.zst", "**/artifacts/**", diff --git a/src/Nethermind/Ethereum.Blockchain.Pyspec.Test/Amsterdam/AmsterdamTestFixture.cs b/src/Nethermind/Ethereum.Blockchain.Pyspec.Test/Amsterdam/AmsterdamTestFixture.cs index f02cf1d8901b..e95b54d24ccd 100644 --- a/src/Nethermind/Ethereum.Blockchain.Pyspec.Test/Amsterdam/AmsterdamTestFixture.cs +++ b/src/Nethermind/Ethereum.Blockchain.Pyspec.Test/Amsterdam/AmsterdamTestFixture.cs @@ -13,13 +13,17 @@ namespace Ethereum.Blockchain.Pyspec.Test.Amsterdam; /// /// Generic base for Amsterdam EIP blockchain tests. /// Wildcard is read from on . +/// In CI, only runs on Linux x64 to stay within the job timeout budget. /// [TestFixture] [Parallelizable(ParallelScope.All)] public abstract class AmsterdamBlockChainTestFixture : BlockchainTestBase { + [SetUp] + public void SkipInCiOnSlowRunners() => CiRunnerGuard.SkipIfNotLinuxX64(); + [TestCaseSource(nameof(LoadTests))] - public async Task Test(BlockchainTest test) => await RunTest(test); + public async Task Test(BlockchainTest test) => (await RunTest(test)).Pass.Should().BeTrue(); public static IEnumerable LoadTests() => new TestsSourceLoader(new LoadPyspecTestsStrategy @@ -29,6 +33,29 @@ public static IEnumerable LoadTests() => }, "fixtures/blockchain_tests/for_amsterdam", typeof(TSelf).GetCustomAttribute()!.Wildcard).LoadTests(); } +/// +/// Generic base for Amsterdam EIP engine blockchain tests. +/// Wildcard is read from on . +/// In CI, only runs on Linux x64 to stay within the job timeout budget. +/// +[TestFixture] +[Parallelizable(ParallelScope.All)] +public abstract class AmsterdamEngineBlockChainTestFixture : BlockchainTestBase +{ + [SetUp] + public void SkipInCiOnSlowRunners() => CiRunnerGuard.SkipIfNotLinuxX64(); + + [TestCaseSource(nameof(LoadTests))] + public async Task Test(BlockchainTest test) => (await RunTest(test)).Pass.Should().BeTrue(); + + public static IEnumerable LoadTests() => + new TestsSourceLoader(new LoadPyspecTestsStrategy + { + ArchiveVersion = Constants.BalArchiveVersion, + ArchiveName = Constants.BalArchiveName + }, "fixtures/blockchain_tests_engine/for_amsterdam", typeof(TSelf).GetCustomAttribute()!.Wildcard).LoadTests(); +} + /// /// Generic base for Amsterdam EIP state tests. /// Wildcard is read from on . diff --git a/src/Nethermind/Ethereum.Blockchain.Pyspec.Test/Amsterdam/Tests.cs b/src/Nethermind/Ethereum.Blockchain.Pyspec.Test/Amsterdam/Tests.cs index e05cb5312747..4d7dc1dfa41f 100644 --- a/src/Nethermind/Ethereum.Blockchain.Pyspec.Test/Amsterdam/Tests.cs +++ b/src/Nethermind/Ethereum.Blockchain.Pyspec.Test/Amsterdam/Tests.cs @@ -8,24 +8,45 @@ namespace Ethereum.Blockchain.Pyspec.Test.Amsterdam; [EipWildcard("eip7708_eth_transfer_logs")] public class Eip7708BlockChainTests : AmsterdamBlockChainTestFixture; +[EipWildcard("eip7708_eth_transfer_logs")] +public class Eip7708EngineBlockChainTests : AmsterdamEngineBlockChainTestFixture; + [EipWildcard("eip7778_block_gas_accounting_without_refunds")] public class Eip7778BlockChainTests : AmsterdamBlockChainTestFixture; +[EipWildcard("eip7778_block_gas_accounting_without_refunds")] +public class Eip7778EngineBlockChainTests : AmsterdamEngineBlockChainTestFixture; + [EipWildcard("eip7843_slotnum")] public class Eip7843BlockChainTests : AmsterdamBlockChainTestFixture; +[EipWildcard("eip7843_slotnum")] +public class Eip7843EngineBlockChainTests : AmsterdamEngineBlockChainTestFixture; + [EipWildcard("eip7928_block_level_access_lists")] public class Eip7928BlockChainTests : AmsterdamBlockChainTestFixture; +[EipWildcard("eip7928_block_level_access_lists")] +public class Eip7928EngineBlockChainTests : AmsterdamEngineBlockChainTestFixture; + [EipWildcard("eip7954_increase_max_contract_size")] public class Eip7954BlockChainTests : AmsterdamBlockChainTestFixture; +[EipWildcard("eip7954_increase_max_contract_size")] +public class Eip7954EngineBlockChainTests : AmsterdamEngineBlockChainTestFixture; + [EipWildcard("eip8024_dupn_swapn_exchange")] public class Eip8024BlockChainTests : AmsterdamBlockChainTestFixture; +[EipWildcard("eip8024_dupn_swapn_exchange")] +public class Eip8024EngineBlockChainTests : AmsterdamEngineBlockChainTestFixture; + [EipWildcard("eip8037_state_creation_gas_cost_increase")] public class Eip8037BlockChainTests : AmsterdamBlockChainTestFixture; +[EipWildcard("eip8037_state_creation_gas_cost_increase")] +public class Eip8037EngineBlockChainTests : AmsterdamEngineBlockChainTestFixture; + // State tests [EipWildcard("eip7708_eth_transfer_logs")] diff --git a/src/Nethermind/Ethereum.Blockchain.Pyspec.Test/PyspecTestFixture.cs b/src/Nethermind/Ethereum.Blockchain.Pyspec.Test/PyspecTestFixture.cs index ed80ffd0b632..fa2d26108d7b 100644 --- a/src/Nethermind/Ethereum.Blockchain.Pyspec.Test/PyspecTestFixture.cs +++ b/src/Nethermind/Ethereum.Blockchain.Pyspec.Test/PyspecTestFixture.cs @@ -1,7 +1,9 @@ // SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only +using System; using System.Collections.Generic; +using System.Runtime.InteropServices; using System.Threading.Tasks; using Ethereum.Test.Base; using FluentAssertions; @@ -25,6 +27,26 @@ public static IEnumerable LoadTests() => $"fixtures/blockchain_tests/for_{TestDirectoryHelper.GetDirectoryByConvention("BlockchainTests")}").LoadTests(); } +/// +/// Generic base for pyspec engine blockchain tests using . +/// Directory is derived by convention: strip "EngineBlockchainTests" suffix, lowercase. +/// In CI (TEST_CHUNK set), only runs on Linux x64 to stay within the job timeout budget. +/// +[TestFixture] +[Parallelizable(ParallelScope.All)] +public abstract class PyspecEngineBlockchainTestFixture : BlockchainTestBase +{ + [SetUp] + public void SkipInCiOnSlowRunners() => CiRunnerGuard.SkipIfNotLinuxX64(); + + [TestCaseSource(nameof(LoadTests))] + public async Task Test(BlockchainTest test) => (await RunTest(test)).Pass.Should().BeTrue(); + + public static IEnumerable LoadTests() => + new TestsSourceLoader(new LoadPyspecTestsStrategy(), + $"fixtures/blockchain_tests_engine/for_{TestDirectoryHelper.GetDirectoryByConvention("EngineBlockchainTests")}").LoadTests(); +} + /// /// Generic base for pyspec state tests using . /// Directory is derived by convention: strip "StateTests" suffix, lowercase. @@ -40,3 +62,23 @@ public static IEnumerable LoadTests() => new TestsSourceLoader(new LoadPyspecTestsStrategy(), $"fixtures/state_tests/for_{TestDirectoryHelper.GetDirectoryByConvention("StateTests")}").LoadTests(); } + +/// +/// Skips heavy tests in CI on runners that are too slow or running variant builds. +/// Only active when TEST_CHUNK is set (CI). Local runs always execute. +/// Set TEST_SKIP_HEAVY=1 to unconditionally skip (used by checked/no-intrinsics variants). +/// +internal static class CiRunnerGuard +{ + private static readonly bool s_isCi = !string.IsNullOrEmpty(Environment.GetEnvironmentVariable("TEST_CHUNK")); + private static readonly bool s_isLinuxX64 = OperatingSystem.IsLinux() && RuntimeInformation.ProcessArchitecture == Architecture.X64; + private static readonly bool s_skipHeavy = Environment.GetEnvironmentVariable("TEST_SKIP_HEAVY") == "1"; + + public static void SkipIfNotLinuxX64() + { + if (s_skipHeavy) + Assert.Ignore("Skipped — TEST_SKIP_HEAVY is set"); + if (s_isCi && !s_isLinuxX64) + Assert.Ignore("Skipped in CI — engine/Amsterdam tests only run on Linux x64"); + } +} diff --git a/src/Nethermind/Ethereum.Blockchain.Pyspec.Test/Tests.cs b/src/Nethermind/Ethereum.Blockchain.Pyspec.Test/Tests.cs index 07ef7b7c1924..09eff275184e 100644 --- a/src/Nethermind/Ethereum.Blockchain.Pyspec.Test/Tests.cs +++ b/src/Nethermind/Ethereum.Blockchain.Pyspec.Test/Tests.cs @@ -27,6 +27,16 @@ public class PragueBlockchainTests : PyspecBlockchainTestFixture; +// Engine blockchain tests — only forks with meaningful Engine API differences +// (e.g. blobs, execution requests, BAL). Regular BlockchainTests cover earlier forks. +// Directory derived from class name by convention (strip "EngineBlockchainTests", lowercase) + +public class CancunEngineBlockchainTests : PyspecEngineBlockchainTestFixture; + +public class PragueEngineBlockchainTests : PyspecEngineBlockchainTestFixture; + +public class OsakaEngineBlockchainTests : PyspecEngineBlockchainTestFixture; + // State tests - directory derived from class name by convention (strip "StateTests", lowercase) public class FrontierStateTests : PyspecStateTestFixture; diff --git a/src/Nethermind/Ethereum.Test.Base/BlockchainTestBase.cs b/src/Nethermind/Ethereum.Test.Base/BlockchainTestBase.cs index 61e58cb87e1e..0cf6031197f3 100644 --- a/src/Nethermind/Ethereum.Test.Base/BlockchainTestBase.cs +++ b/src/Nethermind/Ethereum.Test.Base/BlockchainTestBase.cs @@ -32,10 +32,14 @@ using Nethermind.Evm.State; using Nethermind.Init.Modules; using NUnit.Framework; -using Nethermind.Merge.Plugin.Data; -using Nethermind.Merge.Plugin; using Nethermind.JsonRpc; +using Nethermind.JsonRpc.Modules; +using Nethermind.Merge.Plugin; +using Nethermind.Merge.Plugin.Data; +using Nethermind.TxPool; +using Nethermind.Serialization.Json; using System.Reflection; +using System.Text.Json; namespace Ethereum.Test.Base; @@ -49,15 +53,9 @@ public abstract class BlockchainTestBase static BlockchainTestBase() { DifficultyCalculator = new DifficultyCalculatorWrapper(); - _logManager ??= LimboLogs.Instance; _logger = _logManager.GetClassLogger(); } - [SetUp] - public void Setup() - { - } - private class DifficultyCalculatorWrapper : IDifficultyCalculator { public IDifficultyCalculator? Wrapped { get; set; } @@ -86,11 +84,12 @@ protected async Task RunTest(BlockchainTest test, Stopwatch? bool isEngineTest = test.Blocks is null && test.EngineNewPayloads is not null; - // Post-merge pyspec blockchain_test_from_state_test fixtures expect genesis to be processed - // under the target fork rules when the fork requires it (e.g. EIP-7928 sets BlockAccessListHash). + // EIP-7928 introduces BlockAccessListHash in the block header, which must be computed + // during genesis processing. Without target fork rules at genesis, the hash field is missing + // and the genesis block header doesn't match the pyspec fixture expectation. bool genesisUsesTargetFork = test.Network.IsEip7928Enabled; - List<(ForkActivation Activation, IReleaseSpec Spec)> transitions = isEngineTest || genesisUsesTargetFork + List<(ForkActivation Activation, IReleaseSpec Spec)> transitions = genesisUsesTargetFork ? [((ForkActivation)0, test.Network)] : [((ForkActivation)0, test.GenesisSpec), ((ForkActivation)1, test.Network)]; // genesis block is always initialized with Frontier @@ -129,12 +128,16 @@ protected async Task RunTest(BlockchainTest test, Stopwatch? } IConfigProvider configProvider = new ConfigProvider(); + IBlocksConfig blocksConfig = configProvider.GetConfig(); + blocksConfig.PreWarmStateConcurrency = 0; + blocksConfig.PreWarmStateOnBlockProcessing = false; ContainerBuilder containerBuilder = new ContainerBuilder() .AddModule(new TestNethermindModule(configProvider)) .AddSingleton(specProvider) .AddSingleton(_logManager) .AddSingleton(rewardCalculator) - .AddSingleton(DifficultyCalculator); + .AddSingleton(DifficultyCalculator) + .AddSingleton(NullTxPool.Instance); if (isEngineTest) { @@ -150,7 +153,6 @@ protected async Task RunTest(BlockchainTest test, Stopwatch? IBlockValidator blockValidator = container.Resolve(); blockchainProcessor.Start(); - // Register tracer if provided for blocktest tracing if (tracer is not null) { blockchainProcessor.Tracers.Add(tracer); @@ -171,9 +173,8 @@ protected async Task RunTest(BlockchainTest test, Stopwatch? Block genesisBlock = Rlp.Decode(test.GenesisRlp.Bytes); Assert.That(genesisBlock.Header.Hash, Is.EqualTo(new Hash256(test.GenesisBlockHeader.Hash))); - ManualResetEvent genesisProcessed = new(false); - - blockTree.NewHeadBlock += (_, args) => + using ManualResetEvent genesisProcessed = new(false); + EventHandler onNewHeadBlock = (_, args) => { if (args.Block.Number == 0) { @@ -181,8 +182,7 @@ protected async Task RunTest(BlockchainTest test, Stopwatch? genesisProcessed.Set(); } }; - - blockchainProcessor.BlockRemoved += (_, args) => + EventHandler onGenesisBlockRemoved = (_, args) => { if (args.ProcessingResult != ProcessingResult.Success && args.BlockHash == genesisBlock.Header.Hash) { @@ -191,11 +191,22 @@ protected async Task RunTest(BlockchainTest test, Stopwatch? } }; - blockTree.SuggestBlock(genesisBlock); - genesisProcessed.WaitOne(_genesisProcessingTimeoutMs); - parentHeader = genesisBlock.Header; + blockTree.NewHeadBlock += onNewHeadBlock; + blockchainProcessor.BlockRemoved += onGenesisBlockRemoved; + + try + { + blockTree.SuggestBlock(genesisBlock); + Assert.That(genesisProcessed.WaitOne(_genesisProcessingTimeoutMs), Is.True, + "Timed out waiting for genesis block processing."); + parentHeader = genesisBlock.Header; + } + finally + { + blockTree.NewHeadBlock -= onNewHeadBlock; + blockchainProcessor.BlockRemoved -= onGenesisBlockRemoved; + } - // Dispose genesis block's AccountChanges genesisBlock.DisposeAccountChanges(); } @@ -206,9 +217,11 @@ protected async Task RunTest(BlockchainTest test, Stopwatch? } else if (test.EngineNewPayloads is not null) { - // engine test - IEngineRpcModule engineRpcModule = container.Resolve(); - await RunNewPayloads(test.EngineNewPayloads, engineRpcModule); + // engine test — route through JsonRpcService for realistic deserialization + IJsonRpcService rpcService = container.Resolve(); + JsonRpcUrl engineUrl = new(Uri.UriSchemeHttp, "localhost", 8551, RpcEndpoint.Http, true, ["engine"]); + JsonRpcContext rpcContext = new(RpcEndpoint.Http, url: engineUrl); + await RunNewPayloads(test.EngineNewPayloads, rpcService, rpcContext, parentHeader.Hash!); } else { @@ -311,38 +324,69 @@ private static BlockHeader SuggestBlocks(BlockchainTest test, bool failOnInvalid return parentHeader; } - private async static Task RunNewPayloads(TestEngineNewPayloadsJson[]? newPayloads, IEngineRpcModule engineRpcModule) + private static readonly Dictionary s_newPayloadParamCounts = Enumerable + .Range(1, EngineApiVersions.NewPayload.Latest) + .ToDictionary(v => v, v => (typeof(IEngineRpcModule).GetMethod($"engine_newPayloadV{v}") + ?? throw new NotSupportedException($"engine_newPayloadV{v} not found on IEngineRpcModule")).GetParameters().Length); + + private async static Task RunNewPayloads(TestEngineNewPayloadsJson[]? newPayloads, IJsonRpcService rpcService, JsonRpcContext rpcContext, Hash256 initialHeadHash) { - (ExecutionPayloadV4, string[]?, string[]?, int, int)[] payloads = [.. JsonToEthereumTest.Convert(newPayloads)]; + if (newPayloads is null || newPayloads.Length == 0) return; - // blockchain test engine - foreach ((ExecutionPayload executionPayload, string[]? blobVersionedHashes, string[]? validationError, int newPayloadVersion, int fcuVersion) in payloads) + int initialFcuVersion = int.Parse(newPayloads[0].ForkChoiceUpdatedVersion ?? EngineApiVersions.Fcu.Latest.ToString()); + AssertRpcSuccess(await SendFcu(rpcService, rpcContext, initialFcuVersion, initialHeadHash.ToString())); + + foreach (TestEngineNewPayloadsJson enginePayload in newPayloads) { - ResultWrapper res; - byte[]?[] hashes = blobVersionedHashes is null ? [] : [.. blobVersionedHashes.Select(x => Bytes.FromHexString(x))]; + int newPayloadVersion = int.Parse(enginePayload.NewPayloadVersion ?? EngineApiVersions.NewPayload.Latest.ToString()); + int fcuVersion = int.Parse(enginePayload.ForkChoiceUpdatedVersion ?? EngineApiVersions.Fcu.Latest.ToString()); + string? validationError = JsonToEthereumTest.ParseValidationError(enginePayload, newPayloadVersion); - MethodInfo newPayloadMethod = engineRpcModule.GetType().GetMethod($"engine_newPayloadV{newPayloadVersion}"); - List newPayloadParams = [executionPayload]; - if (newPayloadVersion >= 3) - { - newPayloadParams.AddRange([hashes, executionPayload.ParentBeaconBlockRoot]); - } - if (newPayloadVersion >= 4) + int paramCount = s_newPayloadParamCounts[newPayloadVersion]; + string paramsJson = "[" + string.Join(",", enginePayload.Params.Take(paramCount).Select(static p => p.GetRawText())) + "]"; + + JsonRpcResponse npResponse = await SendRpc(rpcService, rpcContext, + "engine_newPayloadV" + newPayloadVersion, paramsJson); + + // RPC-level errors (e.g. wrong payload version) are valid for negative tests + if (npResponse is JsonRpcErrorResponse errorResponse) { - newPayloadParams.Add(executionPayload.ExecutionRequests); + Assert.That(validationError, Is.Not.Null, + $"engine_newPayloadV{newPayloadVersion} RPC error: {errorResponse.Error?.Code} {errorResponse.Error?.Message}"); + continue; } - res = await (Task>)newPayloadMethod.Invoke(engineRpcModule, [.. newPayloadParams]); + PayloadStatusV1 payloadStatus = (PayloadStatusV1)((JsonRpcSuccessResponse)npResponse).Result!; + string expectedStatus = validationError is null ? PayloadStatus.Valid : PayloadStatus.Invalid; + Assert.That(payloadStatus.Status, Is.EqualTo(expectedStatus), + $"engine_newPayloadV{newPayloadVersion} returned {payloadStatus.Status}, expected {expectedStatus}. " + + $"ValidationError: {payloadStatus.ValidationError}"); - if (res.Result.ResultType == ResultType.Success) + if (payloadStatus.Status == PayloadStatus.Valid) { - ForkchoiceStateV1 fcuState = new(executionPayload.BlockHash, executionPayload.BlockHash, executionPayload.BlockHash); - MethodInfo fcuMethod = engineRpcModule.GetType().GetMethod($"engine_forkchoiceUpdatedV{fcuVersion}"); - await (Task>)fcuMethod.Invoke(engineRpcModule, [fcuState, null]); + string blockHash = enginePayload.Params[0].GetProperty("blockHash").GetString()!; + AssertRpcSuccess(await SendFcu(rpcService, rpcContext, fcuVersion, blockHash)); } } } + private static async Task SendRpc(IJsonRpcService rpcService, JsonRpcContext context, string method, string paramsJson) + { + using JsonDocument doc = JsonDocument.Parse(paramsJson); + JsonRpcRequest request = new() { JsonRpc = "2.0", Id = 1, Method = method, Params = doc.RootElement.Clone() }; + return await rpcService.SendRequestAsync(request, context); + } + + private static Task SendFcu(IJsonRpcService rpcService, JsonRpcContext context, int fcuVersion, string blockHash) => + SendRpc(rpcService, context, "engine_forkchoiceUpdatedV" + fcuVersion, + $$$"""[{"headBlockHash":"{{{blockHash}}}","safeBlockHash":"{{{blockHash}}}","finalizedBlockHash":"{{{blockHash}}}"},null]"""); + + private static void AssertRpcSuccess(JsonRpcResponse response) + { + Assert.That(response, Is.InstanceOf(), + response is JsonRpcErrorResponse err ? $"RPC error: {err.Error?.Code} {err.Error?.Message}" : "unexpected response type"); + } + private static List<(Block Block, string ExpectedException)> DecodeRlps(BlockchainTest test, bool failOnInvalidRlp) { List<(Block Block, string ExpectedException)> correctRlp = []; diff --git a/src/Nethermind/Ethereum.Test.Base/JsonToEthereumTest.cs b/src/Nethermind/Ethereum.Test.Base/JsonToEthereumTest.cs index 8e8aaabe93f2..206559b9dc7b 100644 --- a/src/Nethermind/Ethereum.Test.Base/JsonToEthereumTest.cs +++ b/src/Nethermind/Ethereum.Test.Base/JsonToEthereumTest.cs @@ -14,6 +14,7 @@ using Nethermind.Core.Specs; using Nethermind.Crypto; using Nethermind.Int256; +using Nethermind.Merge.Plugin; using Nethermind.Merge.Plugin.Data; using Nethermind.Serialization.Json; using Nethermind.Serialization.Rlp; @@ -86,46 +87,30 @@ public static BlockHeader Convert(TestBlockHeaderJson? headerJson) return header; } - public static IEnumerable<(ExecutionPayloadV4, string[]?, string[]?, int, int)> Convert(TestEngineNewPayloadsJson[]? executionPayloadsJson) + public static string? ParseValidationError(TestEngineNewPayloadsJson engineNewPayload, int newPayloadVersion) { - if (executionPayloadsJson is null) + if (engineNewPayload.ValidationError is not null) { - throw new InvalidDataException("Execution payloads JSON was null when constructing test."); + return engineNewPayload.ValidationError; } - foreach (TestEngineNewPayloadsJson engineNewPayload in executionPayloadsJson) + int validationErrorParamIndex = newPayloadVersion >= 4 ? 4 : 3; + if (engineNewPayload.Params.Length <= validationErrorParamIndex) { - TestEngineNewPayloadsJson.ParamsExecutionPayload executionPayload = engineNewPayload.Params[0].Deserialize(EthereumJsonSerializer.JsonOptions); - string[]? blobVersionedHashes = engineNewPayload.Params.Length > 1 ? engineNewPayload.Params[1].Deserialize(EthereumJsonSerializer.JsonOptions) : null; - string? parentBeaconBlockRoot = engineNewPayload.Params.Length > 2 ? engineNewPayload.Params[2].Deserialize(EthereumJsonSerializer.JsonOptions) : null; - string[]? validationError = engineNewPayload.Params.Length > 3 ? engineNewPayload.Params[3].Deserialize(EthereumJsonSerializer.JsonOptions) : null; - yield return (new ExecutionPayloadV4() - { - BaseFeePerGas = (ulong)Bytes.FromHexString(executionPayload.BaseFeePerGas).ToUnsignedBigInteger(), - BlockHash = new(executionPayload.BlockHash), - BlockNumber = (long)Bytes.FromHexString(executionPayload.BlockNumber).ToUnsignedBigInteger(), - ExtraData = Bytes.FromHexString(executionPayload.ExtraData), - FeeRecipient = new(executionPayload.FeeRecipient), - GasLimit = (long)Bytes.FromHexString(executionPayload.GasLimit).ToUnsignedBigInteger(), - GasUsed = (long)Bytes.FromHexString(executionPayload.GasUsed).ToUnsignedBigInteger(), - LogsBloom = new(Bytes.FromHexString(executionPayload.LogsBloom)), - ParentHash = new(executionPayload.ParentHash), - PrevRandao = new(executionPayload.PrevRandao), - ReceiptsRoot = new(executionPayload.ReceiptsRoot), - StateRoot = new(executionPayload.StateRoot), - Timestamp = (ulong)Bytes.FromHexString(executionPayload.Timestamp).ToUnsignedBigInteger(), - BlockAccessList = executionPayload.BlockAccessList is null ? null : Bytes.FromHexString(executionPayload.BlockAccessList), - BlobGasUsed = executionPayload.BlobGasUsed is null ? null : (ulong)Bytes.FromHexString(executionPayload.BlobGasUsed).ToUnsignedBigInteger(), - ExcessBlobGas = executionPayload.ExcessBlobGas is null ? null : (ulong)Bytes.FromHexString(executionPayload.ExcessBlobGas).ToUnsignedBigInteger(), - ParentBeaconBlockRoot = parentBeaconBlockRoot is null ? null : new(parentBeaconBlockRoot), - Withdrawals = executionPayload.Withdrawals is null ? null : [.. executionPayload.Withdrawals.Select(x => Rlp.Decode(Bytes.FromHexString(x)))], - SlotNumber = executionPayload.SlotNumber is null ? null : (ulong)Bytes.FromHexString(executionPayload.SlotNumber).ToUnsignedBigInteger(), - Transactions = [.. executionPayload.Transactions.Select(x => Bytes.FromHexString(x))], - ExecutionRequests = [] - }, blobVersionedHashes, validationError, int.Parse(engineNewPayload.NewPayloadVersion ?? "4"), int.Parse(engineNewPayload.ForkChoiceUpdatedVersion ?? "3")); + return null; } + + JsonElement validationError = engineNewPayload.Params[validationErrorParamIndex]; + return validationError.ValueKind switch + { + JsonValueKind.Null => null, + JsonValueKind.String => validationError.Deserialize(EthereumJsonSerializer.JsonOptions), + JsonValueKind.Array => string.Join("|", validationError.Deserialize(EthereumJsonSerializer.JsonOptions) ?? []), + _ => null, + }; } + public static Transaction Convert(PostStateJson postStateJson, TransactionJson transactionJson) { Transaction transaction = new() diff --git a/src/Nethermind/Ethereum.Test.Base/TestEngineNewPayloadsJson.cs b/src/Nethermind/Ethereum.Test.Base/TestEngineNewPayloadsJson.cs index f5ee28be8f10..eb06b15e353a 100644 --- a/src/Nethermind/Ethereum.Test.Base/TestEngineNewPayloadsJson.cs +++ b/src/Nethermind/Ethereum.Test.Base/TestEngineNewPayloadsJson.cs @@ -10,6 +10,7 @@ public class TestEngineNewPayloadsJson public JsonElement[] Params { get; set; } public string? NewPayloadVersion { get; set; } public string? ForkChoiceUpdatedVersion { get; set; } + public string? ValidationError { get; set; } public class ParamsExecutionPayload { @@ -29,7 +30,7 @@ public class ParamsExecutionPayload public string ExcessBlobGas { get; set; } public string BlockHash { get; set; } public string[] Transactions { get; set; } - public string[]? Withdrawals { get; set; } + public JsonElement[]? Withdrawals { get; set; } public string? BlockAccessList { get; set; } public string? SlotNumber { get; set; } } diff --git a/src/Nethermind/Ethereum.Transaction.Test/TransactionJsonTest.cs b/src/Nethermind/Ethereum.Transaction.Test/TransactionJsonTest.cs index 770611a16334..94771f0044fc 100644 --- a/src/Nethermind/Ethereum.Transaction.Test/TransactionJsonTest.cs +++ b/src/Nethermind/Ethereum.Transaction.Test/TransactionJsonTest.cs @@ -120,4 +120,5 @@ public void Invalid_pre_berlin_access_list_tx_with_empty_list_preserves_prestate "invalid AccessList tx on pre-Berlin fork should not mutate state"); result.Pass.Should().BeTrue(); } + } diff --git a/src/Nethermind/Nethermind.Blockchain.Test.Runner/BlockchainTestsBugHunter.cs b/src/Nethermind/Nethermind.Blockchain.Test.Runner/BlockchainTestsBugHunter.cs index 9ccbc224d27b..2618b1a2697c 100644 --- a/src/Nethermind/Nethermind.Blockchain.Test.Runner/BlockchainTestsBugHunter.cs +++ b/src/Nethermind/Nethermind.Blockchain.Test.Runner/BlockchainTestsBugHunter.cs @@ -28,8 +28,6 @@ public async Task> RunTestsAsync() IEnumerable tests = _testsSource.LoadTests(); foreach (BlockchainTest test in tests) { - Setup(); - Console.Write($"{test,-120} "); if (test.LoadFailure is not null) { @@ -53,7 +51,6 @@ public async Task> RunTestsAsync() Directory.CreateDirectory(directoryName); } - Setup(); await RunTest(test); } } diff --git a/src/Nethermind/Nethermind.Blockchain.Test/BlockProcessorTests.cs b/src/Nethermind/Nethermind.Blockchain.Test/BlockProcessorTests.cs index 572351d4a2ba..b5ff3db27a39 100644 --- a/src/Nethermind/Nethermind.Blockchain.Test/BlockProcessorTests.cs +++ b/src/Nethermind/Nethermind.Blockchain.Test/BlockProcessorTests.cs @@ -35,6 +35,8 @@ using System.Threading.Tasks; using Nethermind.Blockchain.Tracing; using Nethermind.Evm; +using Nethermind.Evm.Tracing; +using Nethermind.Core.BlockAccessLists; namespace Nethermind.Blockchain.Test; @@ -206,6 +208,31 @@ public void BranchProcessor_unsubscribes_from_TransactionsExecuted_after_process externalHandlerCallCount.Should().Be(1, "only the externally subscribed handler should fire, BranchProcessor should have unsubscribed"); } + [Test, MaxTime(Timeout.MaxTestTime)] + public void BlockValidationTransactionsExecutor_uses_block_gas_for_bal_validation_budget() + { + TrackingBlockAccessListWorldState stateProvider = new(TestWorldStateFactory.CreateForTest()); + stateProvider.LoadSuggestedBlockAccessList(new BlockAccessList(), 37_568); + + ITransactionProcessorAdapter transactionProcessor = Substitute.For(); + transactionProcessor.Execute(Arg.Any(), Arg.Any()).Returns(static callInfo => + { + Transaction transaction = callInfo.Arg(); + transaction.SpentGas = 63_586; + transaction.BlockGasUsed = 37_568; + return TransactionResult.Ok; + }); + + BlockProcessor.BlockValidationTransactionsExecutor txExecutor = new(transactionProcessor, stateProvider); + Block block = Build.A.Block.WithTransactions(Build.A.Transaction.SignedAndResolved().TestObject).TestObject; + BlockReceiptsTracer receiptsTracer = new(); + receiptsTracer.StartNewBlockTrace(block); + + txExecutor.ProcessTransactions(block, ProcessingOptions.NoValidation, receiptsTracer, CancellationToken.None); + + stateProvider.ValidatedGasRemaining.Should().Equal([37_568L, 0L]); + } + [Test, MaxTime(Timeout.MaxTestTime)] public void BranchProcessor_no_prewarmer_still_processes_successfully() { @@ -276,4 +303,28 @@ public Task PreWarmCaches(Block suggestedBlock, BlockHeader? parent, IReleaseSpe public CacheType ClearCaches() => default; } + + private sealed class TrackingBlockAccessListWorldState(IWorldState innerWorldState) + : WrappedWorldState(innerWorldState), IBlockAccessListBuilder + { + public bool TracingEnabled { get; set; } + public BlockAccessList GeneratedBlockAccessList { get; set; } = new(); + public List ValidatedGasRemaining { get; } = []; + + private long _gasUsed; + + public void AddAccountRead(Address address) + { + } + + public void LoadSuggestedBlockAccessList(BlockAccessList suggested, long gasUsed) => _gasUsed = gasUsed; + + public long GasUsed() + => _gasUsed; + + public void ValidateBlockAccessList(BlockHeader block, ushort index, long gasRemaining) + => ValidatedGasRemaining.Add(gasRemaining); + + public void SetBlockAccessList(Block block, IReleaseSpec spec) { } + } } diff --git a/src/Nethermind/Nethermind.Blockchain.Test/Validators/BlockValidatorTests.cs b/src/Nethermind/Nethermind.Blockchain.Test/Validators/BlockValidatorTests.cs index ac8e31e84806..f5fcafc0da49 100644 --- a/src/Nethermind/Nethermind.Blockchain.Test/Validators/BlockValidatorTests.cs +++ b/src/Nethermind/Nethermind.Blockchain.Test/Validators/BlockValidatorTests.cs @@ -261,4 +261,37 @@ public void ValidateSuggestedBlock_SuggestedBlockIsInvalid_CorrectErrorIsSet(Blo Assert.That(error, Does.StartWith(expectedError)); } + + [TestCase(30_000, true)] + [TestCase(29_999, false)] + public void ValidateSuggestedBlock_Enforces_bal_item_gas_limit_boundary(long gasLimit, bool expectedValid) + { + BlockHeader parent = Build.A.BlockHeader.TestObject; + BlockAccessList bal = Build.A.BlockAccessList.WithPrecompileChanges(parent.Hash!, timestamp: 12).TestObject; + byte[] encodedBal = Rlp.Encode(bal).Bytes; + Hash256 balHash = new(ValueKeccak.Compute(encodedBal).Bytes); + Block suggestedBlock = Build.A.Block + .WithParent(parent) + .WithGasLimit(gasLimit) + .WithBlobGasUsed(0) + .WithWithdrawals([]) + .WithBlockAccessList(bal) + .WithEncodedBlockAccessList(encodedBal) + .WithBlockAccessListHash(balHash) + .TestObject; + TxValidator txValidator = new(TestBlockchainIds.ChainId); + BlockValidator sut = new(txValidator, Always.Valid, Always.Valid, new CustomSpecProvider(((ForkActivation)0, Amsterdam.Instance)), LimboLogs.Instance); + + bool isValid = sut.ValidateSuggestedBlock(suggestedBlock, parent, out string? error); + + Assert.That(isValid, Is.EqualTo(expectedValid)); + if (expectedValid) + { + Assert.That(error, Is.Null); + } + else + { + Assert.That(error, Does.StartWith("BlockAccessListGasLimitExceeded")); + } + } } diff --git a/src/Nethermind/Nethermind.Blockchain/BlockAccessLists/IBlockAccessListStore.cs b/src/Nethermind/Nethermind.Blockchain/BlockAccessLists/IBlockAccessListStore.cs index 857f7dbaf32b..43e0edca181d 100644 --- a/src/Nethermind/Nethermind.Blockchain/BlockAccessLists/IBlockAccessListStore.cs +++ b/src/Nethermind/Nethermind.Blockchain/BlockAccessLists/IBlockAccessListStore.cs @@ -19,6 +19,11 @@ void InsertFromBlock(Block block) { Insert(block.Hash, block.BlockAccessList); } + + // Release BAL data after persistence to prevent memory accumulation in block caches. + // The data can be re-read from the store if needed. + block.GeneratedBlockAccessList = null; + block.EncodedBlockAccessList = null; } void Insert(Hash256 blockHash, byte[] bal); diff --git a/src/Nethermind/Nethermind.Consensus/EngineApiVersions.cs b/src/Nethermind/Nethermind.Consensus/EngineApiVersions.cs index 9d8285ccdec2..1d8dbe19b6f8 100644 --- a/src/Nethermind/Nethermind.Consensus/EngineApiVersions.cs +++ b/src/Nethermind/Nethermind.Consensus/EngineApiVersions.cs @@ -18,6 +18,7 @@ public static class Fcu public const int V2 = 2; // Shanghai public const int V3 = 3; // Cancun/Prague/Osaka public const int V4 = 4; // Amsterdam + public const int Latest = V4; } /// engine_newPayload method versions. @@ -28,6 +29,7 @@ public static class NewPayload public const int V3 = 3; // Cancun public const int V4 = 4; // Prague/Osaka public const int V5 = 5; // Amsterdam + public const int Latest = V5; } /// engine_getPayload method versions. diff --git a/src/Nethermind/Nethermind.Consensus/Processing/BlockProcessor.BlockValidationTransactionsExecutor.cs b/src/Nethermind/Nethermind.Consensus/Processing/BlockProcessor.BlockValidationTransactionsExecutor.cs index c74ccc0b74f5..308505da3169 100644 --- a/src/Nethermind/Nethermind.Consensus/Processing/BlockProcessor.BlockValidationTransactionsExecutor.cs +++ b/src/Nethermind/Nethermind.Consensus/Processing/BlockProcessor.BlockValidationTransactionsExecutor.cs @@ -47,7 +47,7 @@ public TxReceipt[] ProcessTransactions(Block block, ProcessingOptions processing if (gasRemaining is not null) { - gasRemaining -= currentTx.SpentGas; + gasRemaining -= currentTx.BlockGasUsed; _balBuilder.ValidateBlockAccessList(block.Header, (ushort)(i + 1), gasRemaining!.Value); } } diff --git a/src/Nethermind/Nethermind.Consensus/Processing/BlockProcessor.cs b/src/Nethermind/Nethermind.Consensus/Processing/BlockProcessor.cs index cbcf57b284ab..b3e98914c5ff 100644 --- a/src/Nethermind/Nethermind.Consensus/Processing/BlockProcessor.cs +++ b/src/Nethermind/Nethermind.Consensus/Processing/BlockProcessor.cs @@ -61,11 +61,14 @@ public partial class BlockProcessor( { if (_logger.IsTrace) _logger.Trace($"Processing block {suggestedBlock.ToString(Block.Format.Short)} ({options})"); - if (_balBuilder is not null && spec.BlockLevelAccessListsEnabled) + if (_balBuilder is not null) { - _balBuilder.TracingEnabled = true; - _balBuilder.GeneratedBlockAccessList.Clear(); - _balBuilder.LoadSuggestedBlockAccessList(suggestedBlock.BlockAccessList, suggestedBlock.GasUsed); + bool balsEnabled = spec.BlockLevelAccessListsEnabled; + _balBuilder.TracingEnabled = balsEnabled; + if (balsEnabled) + { + _balBuilder.LoadSuggestedBlockAccessList(suggestedBlock.BlockAccessList, suggestedBlock.GasUsed); + } } ApplyDaoTransition(suggestedBlock); diff --git a/src/Nethermind/Nethermind.Consensus/Validators/BlockValidator.cs b/src/Nethermind/Nethermind.Consensus/Validators/BlockValidator.cs index e186726858b3..fc2522ff6375 100644 --- a/src/Nethermind/Nethermind.Consensus/Validators/BlockValidator.cs +++ b/src/Nethermind/Nethermind.Consensus/Validators/BlockValidator.cs @@ -411,6 +411,15 @@ public virtual bool ValidateBlockLevelAccessList(Block block, IReleaseSpec spec, if (block.BlockAccessList is not null) { + int itemCount = block.BlockAccessList.ItemCount(); + if (itemCount * GasCostOf.BlockAccessListItem > block.Header.GasLimit) + { + error = BlockErrorMessages.BlockAccessListGasLimitExceeded(itemCount, block.Header.GasLimit); + if (_logger.IsWarn) _logger.Warn($"Block level access list item count {itemCount} exceeds block gas limit bound in block {block.ToString(Block.Format.FullHashAndNumber)}."); + + return false; + } + if (!ValidateBlockLevelAccessListHashMatches(block, out Hash256 blockLevelAccessListHash)) { error = BlockErrorMessages.InvalidBlockLevelAccessListHash(block.Header.BlockAccessListHash, blockLevelAccessListHash); diff --git a/src/Nethermind/Nethermind.Core.Test/Modules/FixedIpResolver.cs b/src/Nethermind/Nethermind.Core.Test/Modules/FixedIpResolver.cs index 66babdae0bb9..13f25f298043 100644 --- a/src/Nethermind/Nethermind.Core.Test/Modules/FixedIpResolver.cs +++ b/src/Nethermind/Nethermind.Core.Test/Modules/FixedIpResolver.cs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: LGPL-3.0-only using System.Net; +using System.Threading; using System.Threading.Tasks; using Nethermind.Network; using Nethermind.Network.Config; @@ -12,7 +13,7 @@ public class FixedIpResolver(INetworkConfig networkConfig) : IIPResolver { public IPAddress LocalIp => IPAddress.Parse(networkConfig.LocalIp!); public IPAddress ExternalIp => IPAddress.Parse(networkConfig.ExternalIp!); - public Task Initialize() + public Task Initialize(CancellationToken cancellationToken = default) { return Task.CompletedTask; } diff --git a/src/Nethermind/Nethermind.Core/BlockAccessLists/BlockAccessList.cs b/src/Nethermind/Nethermind.Core/BlockAccessLists/BlockAccessList.cs index 7c0e1c47dd7b..bcadac6cc236 100644 --- a/src/Nethermind/Nethermind.Core/BlockAccessLists/BlockAccessList.cs +++ b/src/Nethermind/Nethermind.Core/BlockAccessLists/BlockAccessList.cs @@ -3,7 +3,6 @@ using System; using System.Collections.Generic; -using System.Linq; using System.Text; using System.Text.Json.Serialization; using Nethermind.Int256; @@ -31,8 +30,25 @@ public BlockAccessList(SortedDictionary accountChanges) _changes = new(); } - public bool Equals(BlockAccessList? other) => - other is not null && _accountChanges.SequenceEqual(other._accountChanges); + public bool Equals(BlockAccessList? other) + { + if (other is null) + return false; + + if (_accountChanges.Count != other._accountChanges.Count) + return false; + + foreach (KeyValuePair pair in _accountChanges) + { + if (!other._accountChanges.TryGetValue(pair.Key, out AccountChanges? otherValue)) + return false; + + if (pair.Value != otherValue) + return false; + } + + return true; + } public override bool Equals(object? obj) => obj is BlockAccessList other && Equals(other); @@ -48,6 +64,14 @@ public override int GetHashCode() => public AccountChanges? GetAccountChanges(Address address) => _accountChanges.TryGetValue(address, out AccountChanges? value) ? value : null; + public int ItemCount() + { + int count = _accountChanges.Count; + foreach (AccountChanges accountChanges in _accountChanges.Values) + count += accountChanges.StorageChanges.Count + accountChanges.StorageReads.Count; + return count; + } + public void IncrementBlockAccessIndex() { _changes.Clear(); diff --git a/src/Nethermind/Nethermind.Core/Extensions/TaskExtensions.cs b/src/Nethermind/Nethermind.Core/Extensions/TaskExtensions.cs index 6963c23afde8..a21116c57635 100644 --- a/src/Nethermind/Nethermind.Core/Extensions/TaskExtensions.cs +++ b/src/Nethermind/Nethermind.Core/Extensions/TaskExtensions.cs @@ -3,12 +3,23 @@ using System; using System.Linq; +using System.Threading; using System.Threading.Tasks; namespace Nethermind.Core.Extensions; public static class TaskExtensions { + /// + /// Delay that returns immediately when cancelled instead of throwing . + /// Returns true if the delay completed normally, false if cancelled. + /// + public static async Task DelaySafe(int millisecondsDelay, CancellationToken cancellationToken) + { + try { await Task.Delay(millisecondsDelay, cancellationToken); return true; } + catch (OperationCanceledException) { return false; } + } + public static bool IsFailedButNotCanceled(this Task? task) { if (task is null || !task.IsFaulted || task.Exception is null) return false; diff --git a/src/Nethermind/Nethermind.Core/GasCostOf.cs b/src/Nethermind/Nethermind.Core/GasCostOf.cs index 862232c33eb9..cbfb47683f76 100644 --- a/src/Nethermind/Nethermind.Core/GasCostOf.cs +++ b/src/Nethermind/Nethermind.Core/GasCostOf.cs @@ -82,6 +82,7 @@ public static class GasCostOf public const long PerAuthBaseRegular = 7_500; public const long PerAuthBaseState = 23 * CostPerStateByte; public const long PerEmptyAccountState = 112 * CostPerStateByte; + public const long BlockAccessListItem = 2_000; // eip-7928 public const long TxDataNonZeroMultiplier = TxDataNonZero / TxDataZero; public const long TxDataNonZeroMultiplierEip2028 = TxDataNonZeroEip2028 / TxDataZero; diff --git a/src/Nethermind/Nethermind.Core/Messages/BlockErrorMessages.cs b/src/Nethermind/Nethermind.Core/Messages/BlockErrorMessages.cs index cb3143f2ca4a..661e88fa8663 100644 --- a/src/Nethermind/Nethermind.Core/Messages/BlockErrorMessages.cs +++ b/src/Nethermind/Nethermind.Core/Messages/BlockErrorMessages.cs @@ -165,6 +165,9 @@ public static string ExceededBlockSizeLimit(int limit) => public static string InvalidBlockLevelAccessListHash(Hash256 expected, Hash256 actual) => $"InvalidBlockLevelAccessListHash: Expected {expected}, got {actual}"; + public static string BlockAccessListGasLimitExceeded(int itemCount, long gasLimit) => + $"BlockAccessListGasLimitExceeded: BAL item count {itemCount} exceeds gas limit {gasLimit}."; + public static string ReceiptCountMismatch(int expectedCount, int actualCount) => $"ReceiptCountMismatch: Expected {expectedCount} receipts to match transaction count, but got {actualCount}."; diff --git a/src/Nethermind/Nethermind.Db.Rocks/HyperClockCacheWrapper.cs b/src/Nethermind/Nethermind.Db.Rocks/HyperClockCacheWrapper.cs index a47654fbd373..08860e6a077d 100644 --- a/src/Nethermind/Nethermind.Db.Rocks/HyperClockCacheWrapper.cs +++ b/src/Nethermind/Nethermind.Db.Rocks/HyperClockCacheWrapper.cs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: LGPL-3.0-only using System; +using System.Threading; using Microsoft.Win32.SafeHandles; using RocksDbSharp; @@ -9,16 +10,24 @@ namespace Nethermind.Db.Rocks; public class HyperClockCacheWrapper : SafeHandleZeroOrMinusOneIsInvalid { + private static readonly Lock _nativeCacheLock = new(); + public HyperClockCacheWrapper(ulong capacity = 32_000_000) : base(ownsHandle: true) { - SetHandle(RocksDbSharp.Native.Instance.rocksdb_cache_create_hyper_clock(new UIntPtr(capacity), 0)); + lock (_nativeCacheLock) + { + SetHandle(Native.Instance.rocksdb_cache_create_hyper_clock(new UIntPtr(capacity), 0)); + } } public IntPtr Handle => DangerousGetHandle(); protected override bool ReleaseHandle() { - RocksDbSharp.Native.Instance.rocksdb_cache_destroy(handle); + lock (_nativeCacheLock) + { + Native.Instance.rocksdb_cache_destroy(handle); + } return true; } diff --git a/src/Nethermind/Nethermind.Facade/Proxy/DefaultHttpClient.cs b/src/Nethermind/Nethermind.Facade/Proxy/DefaultHttpClient.cs index 29572f49e9f3..193e87c957ab 100644 --- a/src/Nethermind/Nethermind.Facade/Proxy/DefaultHttpClient.cs +++ b/src/Nethermind/Nethermind.Facade/Proxy/DefaultHttpClient.cs @@ -67,7 +67,7 @@ private async Task ExecuteAsync(Method method, string endpoint, object? pa } if (_logger.IsTrace) _logger.Trace($"HTTP {methodType} request to: {endpoint} [id: {requestId}] will be sent again in: {_retryDelayMilliseconds} ms."); - await Task.Delay(_retryDelayMilliseconds); + await Task.Delay(_retryDelayMilliseconds, cancellationToken); } } while (currentRetry <= _retries); diff --git a/src/Nethermind/Nethermind.Grpc/Clients/GrpcClient.cs b/src/Nethermind/Nethermind.Grpc/Clients/GrpcClient.cs index 2c6fc5e85105..c8866847cd96 100644 --- a/src/Nethermind/Nethermind.Grpc/Clients/GrpcClient.cs +++ b/src/Nethermind/Nethermind.Grpc/Clients/GrpcClient.cs @@ -6,6 +6,7 @@ using System.Threading; using System.Threading.Tasks; using Grpc.Core; +using Nethermind.Core.Extensions; using Nethermind.Logging; namespace Nethermind.Grpc.Clients @@ -19,6 +20,9 @@ public class GrpcClient : IGrpcClient private Channel _channel; private NethermindService.NethermindServiceClient _client; private readonly string _address; +#nullable enable + private CancellationTokenSource? _cts = new(); +#nullable restore public GrpcClient(string host, int port, int reconnectionInterval, ILogManager logManager) { @@ -63,7 +67,7 @@ private async Task TryStartAsync() _client = new NethermindService.NethermindServiceClient(_channel); while (_channel.State != ChannelState.Ready) { - await Task.Delay(_reconnectionInterval); + await Task.Delay(_reconnectionInterval, _cts?.Token ?? CancellationToken.None); } if (_logger.IsInfo) _logger.Info($"Connected gRPC client to: '{_address}'"); @@ -73,6 +77,7 @@ private async Task TryStartAsync() public Task StopAsync() { _connected = false; + CancellationTokenExtensions.CancelDisposeAndClear(ref _cts); return _channel?.ShutdownAsync() ?? Task.CompletedTask; } @@ -138,7 +143,9 @@ private async Task TryReconnectAsync() await StopAsync(); _retry++; if (_logger.IsWarn) _logger.Warn($"Retrying ({_retry}) gRPC connection to: '{_address}' in {_reconnectionInterval} ms."); - await Task.Delay(_reconnectionInterval); + // Use CancellationToken.None: _cts is already cancelled by StopAsync, and this delay + // represents backoff time before the next connection attempt. + await Task.Delay(_reconnectionInterval, CancellationToken.None); await StartAsync(); } } diff --git a/src/Nethermind/Nethermind.Init/Steps/ResolveIps.cs b/src/Nethermind/Nethermind.Init/Steps/ResolveIps.cs index 82eb7ee5ed5b..e2ebc0eab761 100644 --- a/src/Nethermind/Nethermind.Init/Steps/ResolveIps.cs +++ b/src/Nethermind/Nethermind.Init/Steps/ResolveIps.cs @@ -14,10 +14,10 @@ namespace Nethermind.Init.Steps public class ResolveIps(INetworkConfig networkConfig, IIPResolver ipResolver) : IStep { [Todo(Improve.Refactor, "Automatically scan all the references solutions?")] - public virtual async Task Execute(CancellationToken _) + public virtual async Task Execute(CancellationToken cancellationToken) { // this should be outside of Ethereum Runner I guess - await ipResolver.Initialize(); + await ipResolver.Initialize(cancellationToken); networkConfig.ExternalIp = ipResolver.ExternalIp.ToString(); networkConfig.LocalIp = ipResolver.LocalIp.ToString(); } diff --git a/src/Nethermind/Nethermind.Merge.Plugin/GC/GCKeeper.cs b/src/Nethermind/Nethermind.Merge.Plugin/GC/GCKeeper.cs index d99b02590166..ca1e0f732dd3 100644 --- a/src/Nethermind/Nethermind.Merge.Plugin/GC/GCKeeper.cs +++ b/src/Nethermind/Nethermind.Merge.Plugin/GC/GCKeeper.cs @@ -7,12 +7,13 @@ using System.Threading.Tasks; using FastEnumUtility; using Nethermind.Core; -using Nethermind.Core.Extensions; using Nethermind.Logging; namespace Nethermind.Merge.Plugin.GC; -public class GCKeeper +using Nethermind.Core.Extensions; + +public class GCKeeper : IDisposable { private static ulong _forcedGcCount = 0; private readonly Lock _lock = new(); @@ -22,6 +23,7 @@ public class GCKeeper private static readonly long _defaultSize = 512.MB; private Task _gcScheduleTask = Task.CompletedTask; private readonly Func _tryStartNoGCRegionFunc; + private CancellationTokenSource? _shutdownCts = new(); public GCKeeper(IGCStrategy gcStrategy, ILogManager logManager) { @@ -31,6 +33,11 @@ public GCKeeper(IGCStrategy gcStrategy, ILogManager logManager) _tryStartNoGCRegionFunc = TryStartNoGCRegion; } + public void Dispose() + { + CancellationTokenExtensions.CancelDisposeAndClear(ref _shutdownCts); + } + public Task TryStartNoGCRegionAsync() => Task.Run(_tryStartNoGCRegionFunc); private IDisposable TryStartNoGCRegion() @@ -165,7 +172,7 @@ private async Task ScheduleGCInternal() } else { - await Task.Delay(postBlockDelayMs); + if (!await TaskExtensions.DelaySafe(postBlockDelayMs, _shutdownCts?.Token ?? CancellationToken.None)) return; } if (GCSettings.LatencyMode != GCLatencyMode.NoGCRegion) diff --git a/src/Nethermind/Nethermind.Network.Discovery/DiscoveryApp.cs b/src/Nethermind/Nethermind.Network.Discovery/DiscoveryApp.cs index 992f86417a33..09e39b33a583 100644 --- a/src/Nethermind/Nethermind.Network.Discovery/DiscoveryApp.cs +++ b/src/Nethermind/Nethermind.Network.Discovery/DiscoveryApp.cs @@ -385,7 +385,7 @@ private async Task RunDiscoveryProcess() { byte[] randomId = new byte[64]; CancellationToken cancellationToken = _stopCts.Token; - PeriodicTimer timer = new(TimeSpan.FromMilliseconds(10)); + using PeriodicTimer timer = new(TimeSpan.FromMilliseconds(10)); long lastTickMs = Environment.TickCount64; long waitTimeTimeMs = 10; diff --git a/src/Nethermind/Nethermind.Network.Discovery/DiscoveryPersistenceManager.cs b/src/Nethermind/Nethermind.Network.Discovery/DiscoveryPersistenceManager.cs index 749cdbc63f3d..39bdb83fc4bd 100644 --- a/src/Nethermind/Nethermind.Network.Discovery/DiscoveryPersistenceManager.cs +++ b/src/Nethermind/Nethermind.Network.Discovery/DiscoveryPersistenceManager.cs @@ -106,7 +106,7 @@ public Task LoadPersistedNodes(CancellationToken cancellationToken) public async Task RunDiscoveryPersistenceCommit(CancellationToken cancellationToken) { if (_logger.IsDebug) _logger.Debug("Starting discovery persistence timer"); - PeriodicTimer timer = new PeriodicTimer(TimeSpan.FromMilliseconds(_persistenceInterval)); + using PeriodicTimer timer = new PeriodicTimer(TimeSpan.FromMilliseconds(_persistenceInterval)); while (!cancellationToken.IsCancellationRequested && await timer.WaitForNextTickAsync(cancellationToken)) { diff --git a/src/Nethermind/Nethermind.Network/IIPResolver.cs b/src/Nethermind/Nethermind.Network/IIPResolver.cs index 2388c29fa980..1d58a9863512 100644 --- a/src/Nethermind/Nethermind.Network/IIPResolver.cs +++ b/src/Nethermind/Nethermind.Network/IIPResolver.cs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: LGPL-3.0-only using System.Net; +using System.Threading; using System.Threading.Tasks; namespace Nethermind.Network @@ -10,6 +11,6 @@ public interface IIPResolver { IPAddress LocalIp { get; } IPAddress ExternalIp { get; } - Task Initialize(); + Task Initialize(CancellationToken cancellationToken = default); } } diff --git a/src/Nethermind/Nethermind.Network/IPResolver.cs b/src/Nethermind/Nethermind.Network/IPResolver.cs index 538493ef9bc6..bbc93645d0e0 100644 --- a/src/Nethermind/Nethermind.Network/IPResolver.cs +++ b/src/Nethermind/Nethermind.Network/IPResolver.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; using System.Net; +using System.Threading; using System.Threading.Tasks; using Nethermind.Core; using Nethermind.Logging; @@ -25,7 +26,7 @@ public IPResolver(INetworkConfig networkConfig, ILogManager logManager) _logManager = logManager; } - public async Task Initialize() + public async Task Initialize(CancellationToken cancellationToken = default) { try { @@ -44,7 +45,7 @@ public async Task Initialize() if (i > 0) { if (_logger.IsWarn) _logger.Warn($"External IP resolution failed (attempt {i}/{maxAttempts}). Retrying in {delaySeconds}s..."); - await Task.Delay(TimeSpan.FromSeconds(delaySeconds)); + await Task.Delay(TimeSpan.FromSeconds(delaySeconds), cancellationToken); } try diff --git a/src/Nethermind/Nethermind.Network/P2P/MessageDictionary.cs b/src/Nethermind/Nethermind.Network/P2P/MessageDictionary.cs index 78c67a1a8f3c..169a841af8e3 100644 --- a/src/Nethermind/Nethermind.Network/P2P/MessageDictionary.cs +++ b/src/Nethermind/Nethermind.Network/P2P/MessageDictionary.cs @@ -15,7 +15,7 @@ namespace Nethermind.Network.P2P; -public class MessageDictionary(Action send, TimeSpan? oldRequestThreshold = null) where T66Msg : IEth66Message +public class MessageDictionary(Action send, TimeSpan? oldRequestThreshold = null, CancellationToken cancellationToken = default) where T66Msg : IEth66Message { // The limit is largely to prevent unexpected OOM. // But the side effect is that if the peer did not respond with the message, eventually it will throw @@ -32,6 +32,7 @@ public class MessageDictionary(Action send, TimeSpan? old private readonly TimeSpan _oldRequestThreshold = oldRequestThreshold ?? DefaultOldRequestThreshold; private readonly ConcurrentDictionary> _requests = new(); + private readonly CancellationToken _cancellationToken = cancellationToken; private Task _cleanOldRequestTask = Task.CompletedTask; private int _requestCount = 0; @@ -69,25 +70,29 @@ static void ThrowTooManyOutstandingRequests() private async Task CleanOldRequests() { - while (true) + try { - await Task.Delay(_oldRequestThreshold); - - foreach (KeyValuePair> requestIdValues in _requests) + while (true) { - if (requestIdValues.Value.Elapsed > _oldRequestThreshold) + await Task.Delay(_oldRequestThreshold, _cancellationToken); + + foreach (KeyValuePair> requestIdValues in _requests) { - if (_requests.TryRemove(requestIdValues.Key, out Request request)) + if (requestIdValues.Value.Elapsed > _oldRequestThreshold) { - Interlocked.Decrement(ref _requestCount); - // Unblock waiting thread. - request.CompletionSource.TrySetException(new TimeoutException("No response received")); + if (_requests.TryRemove(requestIdValues.Key, out Request request)) + { + Interlocked.Decrement(ref _requestCount); + // Unblock waiting thread. + request.CompletionSource.TrySetException(new TimeoutException("No response received")); + } } } - } - if (Volatile.Read(ref _requestCount) == 0) break; + if (Volatile.Read(ref _requestCount) == 0) break; + } } + catch (OperationCanceledException) { } } public void Handle(long id, TData data, long size) diff --git a/src/Nethermind/Nethermind.Network/P2P/PacketSender.cs b/src/Nethermind/Nethermind.Network/P2P/PacketSender.cs index 038339a2465f..1d9073822b64 100644 --- a/src/Nethermind/Nethermind.Network/P2P/PacketSender.cs +++ b/src/Nethermind/Nethermind.Network/P2P/PacketSender.cs @@ -17,6 +17,7 @@ public class PacketSender : ChannelHandlerAdapter, IPacketSender private readonly IMessageSerializationService _messageSerializationService; private readonly ILogger _logger; private readonly TimeSpan _sendLatency; + private readonly CancellationTokenSource _cts = new(); private IChannelHandlerContext _context; private Action _delayThenWrite; private Action _observeWriteCompletion; @@ -56,7 +57,7 @@ private void SendBuffer(IByteBuffer buffer) { if (_sendLatency != TimeSpan.Zero) { - Task delayTask = Task.Delay(_sendLatency); + Task delayTask = Task.Delay(_sendLatency, _cts.Token); if (delayTask.IsCompletedSuccessfully) { StartWrite(buffer); @@ -168,4 +169,10 @@ public override void HandlerAdded(IChannelHandlerContext context) { _context = context; } + + public override void HandlerRemoved(IChannelHandlerContext context) + { + _cts.Cancel(); + _cts.Dispose(); + } } diff --git a/src/Nethermind/Nethermind.Network/PeerManager.cs b/src/Nethermind/Nethermind.Network/PeerManager.cs index 5c11d2bc15ea..2a1b3f745ae0 100644 --- a/src/Nethermind/Nethermind.Network/PeerManager.cs +++ b/src/Nethermind/Nethermind.Network/PeerManager.cs @@ -356,7 +356,7 @@ private async Task RunPeerUpdateLoop() } else { - await Task.Delay(1000); + await Task.Delay(1000, _cancellationTokenSource.Token); } } @@ -454,7 +454,7 @@ private void ResetConnectionRoundCounters() // Delay to prevent high CPU use. There is a shortcut path for newly discovered peer, so having // a lower delay probably won't do much. - await Task.Delay(TimeSpan.FromSeconds(1)); + await Task.Delay(TimeSpan.FromSeconds(1), _cancellationTokenSource.Token); return null; } diff --git a/src/Nethermind/Nethermind.Network/Rlpx/RlpxHost.cs b/src/Nethermind/Nethermind.Network/Rlpx/RlpxHost.cs index 00e151d507eb..698e783ee64d 100644 --- a/src/Nethermind/Nethermind.Network/Rlpx/RlpxHost.cs +++ b/src/Nethermind/Nethermind.Network/Rlpx/RlpxHost.cs @@ -58,6 +58,7 @@ public class RlpxHost : IRlpxHost private readonly ConcurrentDictionary _sessionActivitySubscriptions = new(); private readonly TimeSpan _shutdownQuietPeriod; private readonly TimeSpan _shutdownCloseTimeout; + private CancellationTokenSource? _shutdownCts = new(); public RlpxHost( IMessageSerializationService serializationService, @@ -333,7 +334,7 @@ private void OnChannelCloseCompleted(Task _, object? state) // The close completion is completed before actual closing or remaining packet is processed. // So usually, we do get a disconnect reason from peer, we just receive it after this. So we need to // add some delay to account for whatever is holding the network pipeline. - _ = Task.Delay(TimeSpan.FromSeconds(1)).ContinueWith( + _ = Task.Delay(TimeSpan.FromSeconds(1), _shutdownCts?.Token ?? CancellationToken.None).ContinueWith( _markDisconnectedAfterCloseDelay, state, CancellationToken.None, @@ -350,6 +351,8 @@ private void MarkDisconnectedAfterCloseDelay(Task _, object? state) public async Task Shutdown() { + CancellationTokenExtensions.CancelDisposeAndClear(ref _shutdownCts); + // Close channels first so Disconnected handlers fire while subscriptions are still active await (_bootstrapChannel?.CloseAsync().ContinueWith(t => { diff --git a/src/Nethermind/Nethermind.Network/SessionMonitor.cs b/src/Nethermind/Nethermind.Network/SessionMonitor.cs index 4483892587b9..527201f06a86 100644 --- a/src/Nethermind/Nethermind.Network/SessionMonitor.cs +++ b/src/Nethermind/Nethermind.Network/SessionMonitor.cs @@ -165,6 +165,7 @@ private void StopPingTimer() { try { + _pingTimer.Dispose(); if (_logger.IsTrace) _logger.Trace("Stopping session monitor"); CancellationTokenExtensions.CancelDisposeAndClear(ref _cancellationTokenSource); } diff --git a/src/Nethermind/Nethermind.Optimism/CL/ExecutionEngineManager.cs b/src/Nethermind/Nethermind.Optimism/CL/ExecutionEngineManager.cs index 142e8a2ff2d9..f6e2730370cc 100644 --- a/src/Nethermind/Nethermind.Optimism/CL/ExecutionEngineManager.cs +++ b/src/Nethermind/Nethermind.Optimism/CL/ExecutionEngineManager.cs @@ -57,7 +57,7 @@ public async Task Initialize() } if (_logger.IsInfo) _logger.Info($"Derived payload. Number: {payloadAttributes.Number}"); - ExecutionPayloadV3? executionPayload = await BuildBlockWithPayloadAttributes(payloadAttributes); + ExecutionPayloadV3? executionPayload = await BuildBlockWithPayloadAttributes(payloadAttributes, token); if (executionPayload is null) { return null; @@ -171,7 +171,7 @@ public async Task ProcessNewP2PExecutionPayload(ExecutionPaylo } } - private async Task BuildBlockWithPayloadAttributes(PayloadAttributesRef payloadAttributes) + private async Task BuildBlockWithPayloadAttributes(PayloadAttributesRef payloadAttributes, CancellationToken token) { var fcuResult = await l2Api.ForkChoiceUpdatedV3( _currentHead.Hash, _currentFinalizedHead.Hash, _currentSafeHead.Hash, @@ -184,14 +184,14 @@ public async Task ProcessNewP2PExecutionPayload(ExecutionPaylo } var getPayloadResult = await l2Api.GetPayloadV3(fcuResult.PayloadId!); - if (!await SendNewPayload(getPayloadResult.ExecutionPayload)) + if (!await SendNewPayload(getPayloadResult.ExecutionPayload, token)) { return null; } return getPayloadResult.ExecutionPayload; } - private async Task SendNewPayload(ExecutionPayloadV3 executionPayload) + private async Task SendNewPayload(ExecutionPayloadV3 executionPayload, CancellationToken token) { PayloadStatusV1 npResult = await l2Api.NewPayloadV3(executionPayload, executionPayload.ParentBeaconBlockRoot); @@ -199,7 +199,7 @@ private async Task SendNewPayload(ExecutionPayloadV3 executionPayload) { // retry after delay if (_logger.IsWarn) _logger.Warn($"Got Syncing after NewPayload. {executionPayload.BlockNumber}"); - await Task.Delay(100); + await Task.Delay(100, token); npResult = await l2Api.NewPayloadV3(executionPayload, executionPayload.ParentBeaconBlockRoot); } diff --git a/src/Nethermind/Nethermind.Runner/Monitoring/DataFeed.cs b/src/Nethermind/Nethermind.Runner/Monitoring/DataFeed.cs index f2dfd34b7276..131e07780369 100644 --- a/src/Nethermind/Nethermind.Runner/Monitoring/DataFeed.cs +++ b/src/Nethermind/Nethermind.Runner/Monitoring/DataFeed.cs @@ -31,6 +31,8 @@ namespace Nethermind.Runner.Monitoring; +using Nethermind.Core.Extensions; + public class DataFeed { public static long StartTime { get; set; } @@ -182,7 +184,7 @@ private async Task StartTxFlowRefresh() { while (!_lifetime.IsCancellationRequested) { - await Task.Delay(millisecondsDelay: 1000); + await TaskExtensions.DelaySafe(millisecondsDelay: 1000, _lifetime); // No subscribers, no need to prepare event data if (!HaveSubscribers) continue; @@ -212,7 +214,7 @@ private async Task SystemStatsRefresh() private async Task GetStatsTask(int delayMs) { - await Task.Delay(delayMs); + await TaskExtensions.DelaySafe(delayMs, _lifetime); Environment.ProcessCpuUsage cpuUsage = Environment.CpuUsage; long timeStamp = Stopwatch.GetTimestamp(); @@ -239,7 +241,7 @@ private async Task StartPeersRefresh() _lastTimeStamp = Stopwatch.GetTimestamp(); while (!_lifetime.IsCancellationRequested) { - await Task.Delay(millisecondsDelay: 1000); + await TaskExtensions.DelaySafe(millisecondsDelay: 1000, _lifetime); // No subscribers, no need to prepare event data if (!HaveSubscribers) continue; diff --git a/src/Nethermind/Nethermind.State.Flat/FlatDbManager.cs b/src/Nethermind/Nethermind.State.Flat/FlatDbManager.cs index 481f594e8ff2..8551b3506297 100644 --- a/src/Nethermind/Nethermind.State.Flat/FlatDbManager.cs +++ b/src/Nethermind/Nethermind.State.Flat/FlatDbManager.cs @@ -207,10 +207,15 @@ private async Task NotifyWhenSlow(string name, Func closure) _ = Task.Run(async () => { long sw = Stopwatch.GetTimestamp(); + using CancellationTokenSource cts = new(); while (true) { - Task delayTask = Task.Delay(slowTime); - if (await Task.WhenAny(jobTask, delayTask) == jobTask) break; + Task delayTask = Task.Delay(slowTime, cts.Token); + if (await Task.WhenAny(jobTask, delayTask) == jobTask) + { + await cts.CancelAsync(); + break; + } if (_logger.IsWarn) _logger.Warn($"Slow task \"{name}\". Took {Stopwatch.GetElapsedTime(sw)}"); } }); diff --git a/src/Nethermind/Nethermind.State.Flat/Persistence/RefCountingPersistenceReader.cs b/src/Nethermind/Nethermind.State.Flat/Persistence/RefCountingPersistenceReader.cs index a7e5e11d4775..0a0aab0582e9 100644 --- a/src/Nethermind/Nethermind.State.Flat/Persistence/RefCountingPersistenceReader.cs +++ b/src/Nethermind/Nethermind.State.Flat/Persistence/RefCountingPersistenceReader.cs @@ -1,8 +1,10 @@ // SPDX-FileCopyrightText: 2025 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only +using System.Threading; using System.Threading.Tasks; using Nethermind.Core; +using Nethermind.Core.Extensions; using Nethermind.Core.Crypto; using Nethermind.Core.Utils; using Nethermind.Int256; @@ -16,6 +18,7 @@ public class RefCountingPersistenceReader : RefCountingDisposable, IPersistence. private const int NoAccessors = 0; // Same as parent's constant private const int Disposing = -1; // Same as parent's constant private readonly IPersistence.IPersistenceReader _innerReader; + private CancellationTokenSource? _cts = new(); public RefCountingPersistenceReader(IPersistence.IPersistenceReader innerReader, ILogger logger) { _innerReader = innerReader; @@ -23,10 +26,10 @@ public RefCountingPersistenceReader(IPersistence.IPersistenceReader innerReader, _ = Task.Run(async () => { // Reader should be re-created every block unless something holds it for very long. - // It prevent database compaction, so this need to be closed eventually. + // It prevents database compaction, so this needs to be closed eventually. while (true) { - await Task.Delay(60_000); + if (!await Nethermind.Core.Extensions.TaskExtensions.DelaySafe(60_000, _cts?.Token ?? CancellationToken.None)) return; if (Volatile.Read(ref _leases.Value) <= NoAccessors) return; if (logger.IsWarn) logger.Warn($"Unexpected old snapshot created. Lease count {_leases.Value}. State {CurrentState}"); @@ -62,7 +65,11 @@ public IPersistence.IFlatIterator CreateStorageIterator(in ValueHash256 accountK public bool IsPreimageMode => _innerReader.IsPreimageMode; - protected override void CleanUp() => _innerReader.Dispose(); + protected override void CleanUp() + { + CancellationTokenExtensions.CancelDisposeAndClear(ref _cts); + _innerReader.Dispose(); + } public bool TryAcquire() => TryAcquireLease(); } diff --git a/src/Nethermind/Nethermind.State/ParallelWorldState.cs b/src/Nethermind/Nethermind.State/ParallelWorldState.cs index 7929c1432349..87036b7ca456 100644 --- a/src/Nethermind/Nethermind.State/ParallelWorldState.cs +++ b/src/Nethermind/Nethermind.State/ParallelWorldState.cs @@ -28,6 +28,7 @@ public class InvalidBlockLevelAccessListException(BlockHeader block, string mess public void LoadSuggestedBlockAccessList(BlockAccessList suggested, long gasUsed) { + GeneratedBlockAccessList = new(); _suggestedBlockAccessList = suggested; _gasUsed = gasUsed; } diff --git a/src/Nethermind/Nethermind.Synchronization/ParallelSync/MultiSyncModeSelector.cs b/src/Nethermind/Nethermind.Synchronization/ParallelSync/MultiSyncModeSelector.cs index 3eb5e9e616aa..499ac44ba38f 100644 --- a/src/Nethermind/Nethermind.Synchronization/ParallelSync/MultiSyncModeSelector.cs +++ b/src/Nethermind/Nethermind.Synchronization/ParallelSync/MultiSyncModeSelector.cs @@ -8,6 +8,7 @@ using System.Threading; using System.Threading.Tasks; using Nethermind.Blockchain.Synchronization; +using Nethermind.Core.Extensions; using Nethermind.Core.ServiceStopper; using Nethermind.Int256; using Nethermind.Logging; @@ -61,7 +62,7 @@ public class MultiSyncModeSelector : ISyncModeSelector private long? LastBlockThatEnabledFullSync { get; set; } private int TotalSyncLag => _syncConfig.StateMinDistanceFromHead + _syncConfig.HeaderStateDistance; - private readonly CancellationTokenSource _cancellation = new(); + private CancellationTokenSource? _cancellation = new(); public event EventHandler? Preparing; public event EventHandler? Changing; @@ -93,12 +94,12 @@ public MultiSyncModeSelector( _isSnapSyncDisabledAfterAnyStateSync = _syncProgressResolver.FindBestFullState() != 0; - _ = StartAsync(_cancellation.Token); + _ = StartAsync(_cancellation?.Token ?? CancellationToken.None); } private async Task StartAsync(CancellationToken cancellationToken) { - PeriodicTimer timer = new(TimeSpan.FromMilliseconds(_syncConfig.MultiSyncModeSelectorLoopTimerMs)); + using PeriodicTimer timer = new(TimeSpan.FromMilliseconds(_syncConfig.MultiSyncModeSelectorLoopTimerMs)); try { while (await timer.WaitForNextTickAsync(cancellationToken)) @@ -121,7 +122,7 @@ private async Task StartAsync(CancellationToken cancellationToken) public Task StopAsync() { - return _cancellation.CancelAsync(); + return _cancellation?.CancelAsync() ?? Task.CompletedTask; } string IStoppableService.Description => "sync mode selector"; @@ -697,7 +698,10 @@ private bool ShouldBeInSnapRangesPhase(Snapshot best) return (maxPeerDifficulty, number); } - public void Dispose() => _cancellation.Dispose(); + public void Dispose() + { + CancellationTokenExtensions.CancelDisposeAndClear(ref _cancellation); + } private Snapshot EnsureSnapshot(in UInt256? peerDifficulty, long peerBlock, bool inBeaconControl) { diff --git a/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs b/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs index 153c19dd12bd..b3a8c7af3c6c 100644 --- a/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs +++ b/src/Nethermind/Nethermind.Synchronization/Synchronizer.cs @@ -245,22 +245,26 @@ public async ValueTask DisposeAsync() { _syncCancellation?.Cancel(); - Task timeout = Task.Delay(FeedsTerminationTimeout); - Task completedFirst = await Task.WhenAny( - timeout, - Task.WhenAll( - fullSyncComponent.Feed.FeedTask, - fastSyncComponent.Feed.FeedTask, - stateSyncComponent.Feed.FeedTask, - snapSyncComponent.Feed.FeedTask, - fastHeaderComponent.Feed.FeedTask, - oldBodiesComponent.Feed.FeedTask, - oldReceiptsComponent.Feed.FeedTask)); + using CancellationTokenSource timeoutCts = new(); + Task timeout = Task.Delay(FeedsTerminationTimeout, timeoutCts.Token); + Task feedsTask = Task.WhenAll( + fullSyncComponent.Feed.FeedTask, + fastSyncComponent.Feed.FeedTask, + stateSyncComponent.Feed.FeedTask, + snapSyncComponent.Feed.FeedTask, + fastHeaderComponent.Feed.FeedTask, + oldBodiesComponent.Feed.FeedTask, + oldReceiptsComponent.Feed.FeedTask); + Task completedFirst = await Task.WhenAny(timeout, feedsTask); if (completedFirst == timeout) { if (_logger.IsWarn) _logger.Warn("Sync feeds dispose timeout"); } + else + { + timeoutCts.Cancel(); + } CancellationTokenExtensions.CancelDisposeAndClear(ref _syncCancellation); } diff --git a/src/Nethermind/Nethermind.Test.Runner/BlockchainTestsRunner.cs b/src/Nethermind/Nethermind.Test.Runner/BlockchainTestsRunner.cs index e384660974b5..9328a9bde8b4 100644 --- a/src/Nethermind/Nethermind.Test.Runner/BlockchainTestsRunner.cs +++ b/src/Nethermind/Nethermind.Test.Runner/BlockchainTestsRunner.cs @@ -43,8 +43,6 @@ public async Task> RunTestsAsync() if (filter is not null && test.Name is not null && !Regex.Match(test.Name, $"^({filter})").Success) continue; - Setup(); - Console.Write($"{test,-120} "); if (test.LoadFailure is not null) { diff --git a/src/Nethermind/Nethermind.Trie/Pruning/TrieStore.cs b/src/Nethermind/Nethermind.Trie/Pruning/TrieStore.cs index 5d6dc6e492ef..cd9277a5f6b4 100644 --- a/src/Nethermind/Nethermind.Trie/Pruning/TrieStore.cs +++ b/src/Nethermind/Nethermind.Trie/Pruning/TrieStore.cs @@ -583,7 +583,7 @@ private async Task TrySyncPrune() } else { - await Task.Delay(pruneDelayMs); + if (!await Nethermind.Core.Extensions.TaskExtensions.DelaySafe(pruneDelayMs, _pruningTaskCancellationTokenSource.Token)) return; } using (_pruningLock.EnterScope()) diff --git a/src/Nethermind/Nethermind.TxPool/RetryCache.cs b/src/Nethermind/Nethermind.TxPool/RetryCache.cs index 2e7863314316..fd3137812e60 100644 --- a/src/Nethermind/Nethermind.TxPool/RetryCache.cs +++ b/src/Nethermind/Nethermind.TxPool/RetryCache.cs @@ -44,7 +44,7 @@ public RetryCache(ILogManager logManager, int timeoutMs = 2500, int requestingCa _maxRetryRequests = maxRetryRequests; _mainLoopTask = Task.Run(async () => { - PeriodicTimer timer = new(TimeSpan.FromMilliseconds(_checkMs)); + using PeriodicTimer timer = new(TimeSpan.FromMilliseconds(_checkMs)); while (await timer.WaitForNextTickAsync(token)) {