Perf/simple flat

.github/workflows/nethermind-tests-flat.yml

@@ -0,0 +1,96 @@
+name: Nethermind tests (Flat DB)
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+on:
+  pull_request:
+  push:
+    branches: [master]
+  workflow_dispatch:
+
+env:
+  TEST_USE_FLAT: "1"
+  DOTNET_SYSTEM_CONSOLE_ALLOW_ANSI_COLOR_REDIRECTION: 1
+  TERM: xterm
+
+jobs:
+  tests:
+    name: Run ${{ matrix.project }}${{ matrix.chunk && format(' ({0})', matrix.chunk) || '' }}
+    runs-on: ubuntu-latest
+    continue-on-error: true
+    strategy:
+      matrix:
+        project:
+          - Ethereum.Abi.Test
+          - Ethereum.Basic.Test
+          - Ethereum.Blockchain.Block.Test
+          - Ethereum.Blockchain.Pyspec.Test
+          - Ethereum.Difficulty.Test
+          - Ethereum.HexPrefix.Test
+          - Ethereum.KeyAddress.Test
+          - Ethereum.KeyStore.Test
+          - Ethereum.Legacy.Blockchain.Block.Test
+          - Ethereum.Legacy.Transition.Test
+          - Ethereum.Legacy.VM.Test
+          - Ethereum.PoW.Test
+          - Ethereum.Rlp.Test
+          - Ethereum.Transaction.Test
+          - Ethereum.Trie.Test
+          - Nethermind.Consensus.Test
+          - Nethermind.Core.Test
+          - Nethermind.Db.Test
+          - Nethermind.Runner.Test
+        chunk: ['']
+        include:
+          - project: Ethereum.Legacy.Blockchain.Test
+            chunk: 1of4
+          - project: Ethereum.Legacy.Blockchain.Test
+            chunk: 2of4
+          - project: Ethereum.Legacy.Blockchain.Test
+            chunk: 3of4
+          - project: Ethereum.Legacy.Blockchain.Test
+            chunk: 4of4
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v6
+        with:
+          submodules: ${{ startsWith(matrix.project, 'Ethereum.') && 'recursive' || 'false' }}
+
+      - name: Set up .NET
+        uses: actions/setup-dotnet@v5
+
+      - name: ${{ matrix.project }}
+        id: test
+        working-directory: src/Nethermind/${{ matrix.project }}
+        env:
+          TEST_CHUNK: ${{ matrix.chunk }}
+        run: |
+          dotnet test --project ${{ matrix.project }}.csproj -c release
+
+      - name: Save test outcome
+        if: success() || failure()
+        run: echo "${{ steps.test.outcome == 'success' }}," >> test.outcome
+
+      - name: Upload test outcome
+        if: success() || failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ matrix.project }}${{ matrix.chunk && format('-{0}', matrix.chunk) || '' }}-flat-outcome
+          path: test.outcome
+          retention-days: 1
+
+  tests-summary:
+    name: Tests summary
+    needs: tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Download test outcomes
+        uses: actions/download-artifact@v4
+
+      - name: Ensure all tests passed
+        run: |
+          data=$(cat **/test.outcome) && data=${data%?}
+          passed=$(echo "[$data]" | jq -r 'all')
+          [[ "$passed" == "true" ]] && exit 0 || exit 1

.gitignore

@@ -445,3 +445,6 @@ FodyWeavers.xsd
 ## Nethermind
 keystore/
 /.githooks
+
+# Worktrees
+.worktrees/

Directory.Packages.props

@@ -82,6 +82,7 @@
     <PackageVersion Include="Spectre.Console" Version="0.54.0" />
     <PackageVersion Include="System.CommandLine" Version="2.0.1" />
     <PackageVersion Include="System.Configuration.ConfigurationManager" Version="10.0.1" />
+    <PackageVersion Include="System.IO.Hashing" Version="10.0.2" />
     <PackageVersion Include="System.Security.Cryptography.ProtectedData" Version="10.0.1" />
     <PackageVersion Include="TestableIO.System.IO.Abstractions.TestingHelpers" Version="22.1.0" />
     <PackageVersion Include="TestableIO.System.IO.Abstractions.Wrappers" Version="22.1.0" />

cspell.json

@@ -101,12 +101,14 @@
     "bootnodes",
     "bottlenecked",
     "browsable",
+    "bsearch",
     "btcs",
     "buildtransitive",
     "bulkset",
     "bursty",
     "buterin",
     "bylica",
+    "cacheline",
     "bytecodes",
     "callcode",
     "calldatacopy",
@@ -170,6 +172,7 @@
     "deserialised",
     "dests",
     "devirtualize",
+    "devirtualized",
     "devnet",
     "devnets",
     "devp2p",
@@ -237,7 +240,9 @@
     "extopcodes",
     "extradata",
     "extstaticcall",
+    "Exitor",
     "fastbin",
+    "Fastpath",
     "fastlz",
     "fastmod",
     "fastsync",
@@ -292,6 +297,7 @@
     "hostnames",
     "hotstuff",
     "hyperthreading",
+    "HUGEPAGE",
     "idxs",
     "iface",
     "ikvp",
@@ -345,6 +351,7 @@
     "ldarg",
     "ldfld",
     "lemire's",
+    "levelname",
     "libc",
     "libdl",
     "libp",
@@ -361,6 +368,8 @@
     "machdep",
     "machinename",
     "madv",
+    "Madvise",
+    "madvise",
     "maiboroda",
     "mainchain",
     "mallopt",
@@ -379,10 +388,12 @@
     "maxlevel",
     "maxpriorityfee",
     "mclbn",
+    "mbpk",
     "mcmc",
     "mcopy",
     "mellor",
     "memberwise",
+    "memcolumndb",
     "memin",
     "meminstart",
     "meminsz",
@@ -411,6 +422,7 @@
     "morden",
     "movbe",
     "movzx",
+    "Mpmc",
     "mres",
     "mscorlib",
     "msgrecv",
@@ -457,6 +469,7 @@
     "nodestore",
     "nodetype",
     "nofile",
+    "noninteractive",
     "nonposdao",
     "nonstring",
     "nops",
@@ -483,13 +496,15 @@
     "owlf",
     "pacaya",
     "parallelizable",
+    "patriciatree",
     "paweł",
     "pctg",
     "pearce",
     "pectra",
     "pendingtxs",
     "perfnet",
     "perfolizer",
+    "Persistences",
     "permissioned",
     "pgrep",
     "physicalcpu",
@@ -499,6 +514,7 @@
     "pkcs",
     "pmsg",
     "poacore",
+    "Populator",
     "poaps",
     "podc",
     "popcnt",
@@ -509,6 +525,7 @@
     "powm",
     "preconf",
     "preconfirmation",
+    "pregenesis",
     "predeploy",
     "prefixedf",
     "prefund",
@@ -560,6 +577,7 @@
     "resettables",
     "retesteth",
     "retf",
+    "ribbonfilter",
     "returncode",
     "returndata",
     "returndatacopy",
@@ -578,6 +596,7 @@
     "rocksdb",
     "ronin",
     "roothash",
+    "rootref",
     "rormask",
     "rpcurl",
     "runtimeconfig",
@@ -611,11 +630,13 @@
     "sload",
     "smod",
     "somelabel",
+    "Spmc",
     "spaceneth",
     "spammy",
     "sparkline",
     "spinlocks",
     "squarify",
+    "srcset",
     "ssse",
     "sstfiles",
     "sstore",
@@ -680,6 +701,7 @@
     "trienode",
     "triestore",
     "trietest",
+    "Triewarmer",
     "trietestnextprev",
     "triggerable",
     "tstore",
@@ -723,6 +745,8 @@
     "upto",
     "upvoting",
     "vbmi",
+    "verifytrie",
+    "verkle",
     "vitalik",
     "vmovups",
     "vmtrace",
@@ -734,6 +758,8 @@
     "vpor",
     "vptest",
     "vzeroupper",
+    "Wakeup",
+    "wakeup",
     "wamp",
     "warmcoinbase",
     "wblob",
@@ -744,6 +770,7 @@
     "worklet",
     "worklist",
     "worldchain",
+    "worldscope",
     "worldstate",
     "writebatch",
     "writeoptions",
@@ -758,6 +785,7 @@
     "yparity",
     "zcompressor",
     "zdecompressor",
+    "zerobytes",
     "zhizhu",
     "zstandard",
     "zstd",

src/Nethermind/Nethermind.Blockchain.Test/FullPruning/FullPruningDiskTest.cs

@@ -17,6 +17,7 @@
 using Nethermind.Core.Extensions;
 using Nethermind.Core.Test.Blockchain;
 using Nethermind.Core.Test.IO;
+using Nethermind.Core.Test.Modules;
 using Nethermind.Db;
 using Nethermind.Db.FullPruning;
 using Nethermind.Db.Rocks;
@@ -136,6 +137,12 @@ protected override async Task RunFullPruning(CancellationToken cancellationToken
         }
     }
 
+    [SetUp]
+    public void Setup()
+    {
+        if (PseudoNethermindModule.TestUseFlat) Assert.Ignore("Disabled in flat");
+    }
+
     [Test, MaxTime(Timeout.LongTestTime)]
     public async Task prune_on_disk_multiple_times()
     {

src/Nethermind/Nethermind.Consensus.Test/ShareableTxProcessingSourceTests.cs

@@ -7,6 +7,8 @@
 using Nethermind.Core;
 using Nethermind.Core.Test.Builders;
 using Nethermind.Core.Test.Modules;
+using Nethermind.Evm.State;
+using Nethermind.Evm.TransactionProcessing;
 using NUnit.Framework;
 
 namespace Nethermind.Consensus.Test;
@@ -19,8 +21,8 @@ public void OnSubsequentBuild_GiveDifferentWorldState()
         using IContainer container = new ContainerBuilder().AddModule(new TestNethermindModule()).Build();
         IShareableTxProcessorSource shareableSource = container.Resolve<IShareableTxProcessorSource>();
 
-        var scope1 = shareableSource.Build(Build.A.BlockHeader.TestObject);
-        var scope2 = shareableSource.Build(Build.A.BlockHeader.TestObject);
+        var scope1 = shareableSource.Build(IWorldState.PreGenesis);
+        var scope2 = shareableSource.Build(IWorldState.PreGenesis);
 
         scope1.WorldState.Should().NotBeSameAs(scope2.WorldState);
     }
@@ -31,9 +33,9 @@ public void OnSubsequentBuild_AfterFirstScopeDispose_GiveSameWorldState()
         using IContainer container = new ContainerBuilder().AddModule(new TestNethermindModule()).Build();
         IShareableTxProcessorSource shareableSource = container.Resolve<IShareableTxProcessorSource>();
 
-        var scope1 = shareableSource.Build(Build.A.BlockHeader.TestObject);
+        var scope1 = shareableSource.Build(IWorldState.PreGenesis);
         scope1.Dispose();
-        var scope2 = shareableSource.Build(Build.A.BlockHeader.TestObject);
+        var scope2 = shareableSource.Build(IWorldState.PreGenesis);
 
         scope1.WorldState.Should().BeSameAs(scope2.WorldState);
     }

src/Nethermind/Nethermind.Consensus/Processing/BranchProcessor.cs

@@ -70,8 +70,7 @@ public Block[] Process(BlockHeader? baseBlock, IReadOnlyList<Block> suggestedBlo
         }
         else
         {
-            BlockHeader? scopeBaseBlock = baseBlock ?? (suggestedBlock.IsGenesis ? suggestedBlock.Header : null);
-            worldStateCloser = stateProvider.BeginScope(scopeBaseBlock);
+            worldStateCloser = stateProvider.BeginScope(baseBlock);
         }
 
         CancellationTokenSource? backgroundCancellation = new();