diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index eacbb6efe8d6..328129af10d3 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -13,6 +13,7 @@ /src/Nethermind/Nethermind.Crypto @LukaszRozmej @Marchhill /src/Nethermind/Nethermind.Db.Rocks @LukaszRozmej @asdacap @damian-orzechowski /src/Nethermind/Nethermind.Era1 @ak88 @Marchhill @smartprogrammer93 +/src/Nethermind/Nethermind.EraE @svlachakis @Marchhill @smartprogrammer93 @ak88 /src/Nethermind/Nethermind.Evm.Precompiles @LukaszRozmej @benaadams @rubo @Marchhill /src/Nethermind/Nethermind.Evm @LukaszRozmej @benaadams /src/Nethermind/Nethermind.ExternalSigner.Plugin @ak88 diff --git a/src/Nethermind/Nethermind.Core/Collections/LockableConcurrentDictionary.cs b/src/Nethermind/Nethermind.Core/Collections/ConcurrentDictionaryExtensions.cs similarity index 86% rename from src/Nethermind/Nethermind.Core/Collections/LockableConcurrentDictionary.cs rename to src/Nethermind/Nethermind.Core/Collections/ConcurrentDictionaryExtensions.cs index a9111ac76e06..d19ff2058204 100644 --- a/src/Nethermind/Nethermind.Core/Collections/LockableConcurrentDictionary.cs +++ b/src/Nethermind/Nethermind.Core/Collections/ConcurrentDictionaryExtensions.cs @@ -113,6 +113,27 @@ public static void AddBy(this NonBlocking.ConcurrentDictionary (_, startValue, amount) => startValue + amount, amount ); + + /// + /// Like but + /// disposes the value created by when another thread wins the race. + /// Fast path avoids allocation when the key already exists. + /// + public static TValue GetOrAddDisposable( + this ConcurrentDictionary dictionary, TKey key, Func factory) + where TKey : notnull + where TValue : class, IDisposable + { + if (dictionary.TryGetValue(key, out TValue? existing)) + return existing; + + TValue created = factory(key); + TValue actual = dictionary.GetOrAdd(key, created); + if (!ReferenceEquals(actual, created)) + created.Dispose(); + return actual; + } + } diff --git a/src/Nethermind/Nethermind.Core/Extensions/EnumerableExtensions.cs b/src/Nethermind/Nethermind.Core/Extensions/EnumerableExtensions.cs index 0a0cf4faf049..a2f3517d4979 100644 --- a/src/Nethermind/Nethermind.Core/Extensions/EnumerableExtensions.cs +++ b/src/Nethermind/Nethermind.Core/Extensions/EnumerableExtensions.cs @@ -20,6 +20,32 @@ public static ISet AsSet(this IEnumerable enumerable) => public static ArrayPoolListRef ToPooledListRef(this IReadOnlyCollection collection) => new(collection.Count, collection); public static ArrayPoolListRef ToPooledListRef(this ReadOnlySpan span) => new(span); + public static (T Min, T Max) MinMax(this IEnumerable source) + where T : IComparable + { + T? min = default; + T? max = default; + bool hasValue = false; + foreach (T item in source) + { + if (!hasValue) + { + min = item; + max = item; + hasValue = true; + } + else + { + if (item.CompareTo(min!) < 0) min = item; + if (item.CompareTo(max!) > 0) max = item; + } + } + + return hasValue + ? (min!, max!) + : throw new InvalidOperationException("Sequence contains no elements."); + } + public static IEnumerable Shuffle(this IEnumerable enumerable, Random rng, int maxSize = 100) { using ArrayPoolList buffer = new(maxSize, enumerable); diff --git a/src/Nethermind/Nethermind.Era1/AccumulatorCalculator.cs b/src/Nethermind/Nethermind.Era1/AccumulatorCalculator.cs index 8be42473da51..142066724970 100644 --- a/src/Nethermind/Nethermind.Era1/AccumulatorCalculator.cs +++ b/src/Nethermind/Nethermind.Era1/AccumulatorCalculator.cs @@ -1,7 +1,9 @@ // SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only +using System.Buffers.Binary; using System.Runtime.InteropServices; +using System.Security.Cryptography; using Nethermind.Core.Collections; using Nethermind.Core.Crypto; using Nethermind.Int256; @@ -10,9 +12,13 @@ namespace Nethermind.Era1; // https://github.com/ethereum/portal-network-specs/blob/master/history/history-network.md#algorithms -internal class AccumulatorCalculator : IDisposable +public class AccumulatorCalculator : IDisposable { + private const int TreeDepth = 13; // log2(8192) + private const int ProofLength = 15; // 1 (HeaderRecord field) + 13 (tree) + 1 (length mixin) + private readonly ArrayPoolList> _roots = new(EraWriter.MaxEra1Size); + private readonly ArrayPoolList _totalDifficulties = new(EraWriter.MaxEra1Size); public void Add(Hash256 headerHash, UInt256 td) { @@ -20,6 +26,7 @@ public void Add(Hash256 headerHash, UInt256 td) merkleizer.Feed(headerHash.Bytes); merkleizer.Feed(td); _roots.Add(merkleizer.CalculateRoot().ToLittleEndian()); + _totalDifficulties.Add(td); } public ValueHash256 ComputeRoot() @@ -30,7 +37,60 @@ public ValueHash256 ComputeRoot() return new ValueHash256(MemoryMarshal.Cast(MemoryMarshal.CreateSpan(ref root, 1))); } - internal void Clear() => _roots.Clear(); + public ValueHash256[] GetProof(int blockIndex) + { + if (blockIndex < 0 || blockIndex >= _roots.Count) + throw new ArgumentOutOfRangeException(nameof(blockIndex), $"Block index {blockIndex} is out of range [0, {_roots.Count - 1}]."); + + int count = _roots.Count; + ValueHash256[] proof = new ValueHash256[ProofLength]; + + // Level 0: sibling of block_hash = total_difficulty as 32-byte LE SSZ chunk + proof[0] = new ValueHash256(_totalDifficulties[blockIndex].ToLittleEndian()); + + // Build the flat binary tree over MaxEra1Size HeaderRecord roots. + using ArrayPoolList flatTreeBuffer = new(2 * EraWriter.MaxEra1Size * 32, 2 * EraWriter.MaxEra1Size * 32); + Span flatTree = flatTreeBuffer.AsSpan(); + flatTree.Clear(); + for (int i = 0; i < count; i++) + { + _roots[i].Span.CopyTo(flatTree.Slice((EraWriter.MaxEra1Size + i) * 32, 32)); + } + + Span combined = stackalloc byte[64]; + for (int i = EraWriter.MaxEra1Size - 1; i >= 1; i--) + { + ReadOnlySpan left = flatTree.Slice(2 * i * 32, 32); + ReadOnlySpan right = flatTree.Slice((2 * i + 1) * 32, 32); + left.CopyTo(combined); + right.CopyTo(combined[32..]); + SHA256.TryHashData(combined, flatTree.Slice(i * 32, 32), out _); + } + + int current = EraWriter.MaxEra1Size + blockIndex; + for (int i = 0; i < TreeDepth; i++) + { + int sibling = current ^ 1; + proof[1 + i] = new ValueHash256(flatTree.Slice(sibling * 32, 32)); + current >>= 1; + } + + Span lenBytes = stackalloc byte[32]; + BinaryPrimitives.WriteUInt64LittleEndian(lenBytes, (ulong)count); + proof[14] = new ValueHash256(lenBytes); - public void Dispose() => _roots.Dispose(); + return proof; + } + + internal void Clear() + { + _roots.Clear(); + _totalDifficulties.Clear(); + } + + public void Dispose() + { + _roots.Dispose(); + _totalDifficulties.Dispose(); + } } diff --git a/src/Nethermind/Nethermind.Era1/E2StoreReader.cs b/src/Nethermind/Nethermind.Era1/E2StoreReader.cs index e4b609f552f5..bd844be12b5c 100644 --- a/src/Nethermind/Nethermind.Era1/E2StoreReader.cs +++ b/src/Nethermind/Nethermind.Era1/E2StoreReader.cs @@ -164,16 +164,18 @@ public long BlockCount } } - public ValueHash256 CalculateChecksum() + public ValueHash256 CalculateChecksum() => ComputeChecksum(_file, _fileLength); + + public static ValueHash256 ComputeChecksum(SafeFileHandle file, long fileLength) { using IncrementalHash sha = IncrementalHash.CreateHash(HashAlgorithmName.SHA256); const int bufferSize = 81920; byte[] buffer = new byte[bufferSize]; long offset = 0; - while (offset < _fileLength) + while (offset < fileLength) { - int toRead = (int)Math.Min(bufferSize, _fileLength - offset); - int read = RandomAccess.Read(_file, buffer.AsSpan(0, toRead), offset); + int toRead = (int)Math.Min(bufferSize, fileLength - offset); + int read = RandomAccess.Read(file, buffer.AsSpan(0, toRead), offset); if (read == 0) break; sha.AppendData(buffer, 0, read); offset += read; diff --git a/src/Nethermind/Nethermind.Era1/E2StoreWriter.cs b/src/Nethermind/Nethermind.Era1/E2StoreWriter.cs index 8c31388e976d..3a12fbc16628 100644 --- a/src/Nethermind/Nethermind.Era1/E2StoreWriter.cs +++ b/src/Nethermind/Nethermind.Era1/E2StoreWriter.cs @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only +using System.Buffers; using System.Diagnostics; using System.IO.Compression; using System.Runtime.InteropServices; @@ -68,4 +69,23 @@ public void Dispose() } public ValueHash256 FinalizeChecksum() => new(_checksumCalculator.GetHashAndReset()); + + /// + /// Compresses using Snappy framing and returns the result + /// in a rented buffer. The caller must return the buffer + /// via ArrayPool<byte>.Shared.Return after use. + /// + public static async Task<(byte[] Buffer, int Length)> Compress( + ReadOnlyMemory bytes, CancellationToken cancellation = default) + { + using RecyclableMemoryStream ms = RecyclableStream.GetStream(nameof(E2StoreWriter)); + using SnappyStream compressor = new(ms, CompressionMode.Compress, leaveOpen: true); + await compressor.WriteAsync(bytes, cancellation); + await compressor.FlushAsync(cancellation); + bool ok = ms.TryGetBuffer(out ArraySegment segment); + Debug.Assert(ok); + byte[] rented = ArrayPool.Shared.Rent(segment.Count); + segment.AsSpan().CopyTo(rented); + return (rented, segment.Count); + } } diff --git a/src/Nethermind/Nethermind.EraE.Test/Admin/AdminEraServiceTests.cs b/src/Nethermind/Nethermind.EraE.Test/Admin/AdminEraServiceTests.cs new file mode 100644 index 000000000000..b566b598c7d0 --- /dev/null +++ b/src/Nethermind/Nethermind.EraE.Test/Admin/AdminEraServiceTests.cs @@ -0,0 +1,93 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Config; +using Nethermind.Core; +using Nethermind.EraE.Admin; +using Nethermind.EraE.Export; +using Nethermind.EraE.Import; +using Nethermind.JsonRpc; +using Nethermind.Logging; +using NSubstitute; +using NUnit.Framework; + +namespace Nethermind.EraE.Test.Admin; + +public class AdminEraServiceTests +{ + [Test] + public void ImportHistory_WhenCalled_DelegatesToImporter() + { + IEraImporter importer = Substitute.For(); + AdminEraService sut = new( + importer, + Substitute.For(), + Substitute.For(), + LimboLogs.Instance); + + sut.ImportHistory("somewhere", 99, 999, null); + + importer.Received().Import("somewhere", 99, 999, null, Arg.Any()); + } + + [Test] + public void ImportHistory_WhenImportAlreadyRunning_ReturnsFailure() + { + IEraImporter importer = Substitute.For(); + TaskCompletionSource tcs = new(); + importer.Import("somewhere", 99, 999, null, Arg.Any()).Returns(tcs.Task); + + AdminEraService sut = new( + importer, + Substitute.For(), + Substitute.For(), + LimboLogs.Instance); + + sut.ImportHistory("somewhere", 99, 999, null); + + ResultWrapper result = sut.ImportHistory("somewhere", 99, 999, null); + Assert.That(result.Result.ResultType, Is.EqualTo(ResultType.Failure)); + + tcs.TrySetResult(); + + Assert.That(() => sut.ImportHistory("somewhere", 99, 999, null), Throws.Nothing); + } + + [Test] + public void ExportHistory_WhenCalled_DelegatesToExporter() + { + IEraExporter exporter = Substitute.For(); + AdminEraService sut = new( + Substitute.For(), + exporter, + Substitute.For(), + LimboLogs.Instance); + + sut.ExportHistory("somewhere", 99, 999); + + exporter.Received().Export("somewhere", 99, 999, Arg.Any()); + } + + [Test] + public void ExportHistory_WhenExportAlreadyRunning_ReturnsFailure() + { + IEraExporter exporter = Substitute.For(); + TaskCompletionSource tcs = new(); + exporter.Export("somewhere", 99, 999, Arg.Any()).Returns(tcs.Task); + + AdminEraService sut = new( + Substitute.For(), + exporter, + Substitute.For(), + LimboLogs.Instance); + + sut.ExportHistory("somewhere", 99, 999); + + ResultWrapper result = sut.ExportHistory("somewhere", 99, 999); + Assert.That(result.Result.ResultType, Is.EqualTo(ResultType.Failure)); + + tcs.TrySetResult(); + + Assert.That(() => sut.ExportHistory("somewhere", 99, 999), Throws.Nothing); + } +} diff --git a/src/Nethermind/Nethermind.EraE.Test/Archive/AccumulatorCalculatorTests.cs b/src/Nethermind/Nethermind.EraE.Test/Archive/AccumulatorCalculatorTests.cs new file mode 100644 index 000000000000..226f87ac3b9f --- /dev/null +++ b/src/Nethermind/Nethermind.EraE.Test/Archive/AccumulatorCalculatorTests.cs @@ -0,0 +1,108 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Core.Crypto; +using AccumulatorCalculator = Nethermind.Era1.AccumulatorCalculator; +using NUnit.Framework; + +namespace Nethermind.EraE.Test.Archive; + +public class AccumulatorCalculatorTests +{ + [Test] + public void Add_WhenCalled_DoesNotThrow() + { + using AccumulatorCalculator sut = new(); + Assert.That(() => sut.Add(Keccak.Zero, 0), Throws.Nothing); + } + + [Test] + public void ComputeRoot_WithKnownValues_ReturnsExpectedResult() + { + using AccumulatorCalculator sut = new(); + sut.Add(Keccak.Zero, 1); + sut.Add(Keccak.MaxValue, 2); + + byte[] result = sut.ComputeRoot().ToByteArray(); + + Assert.That(result, Is.EquivalentTo(new byte[] + { + 0x3E, 0xD6, 0x26, 0x52, 0xDF, 0xB7, 0xE1, 0x07, + 0x2D, 0x0F, 0x04, 0x0F, 0xEB, 0x6D, 0x00, 0x2A, + 0x9F, 0x7C, 0xE3, 0x7C, 0xF8, 0xDD, 0xB1, 0x65, + 0x49, 0xA7, 0xAC, 0x5C, 0xF8, 0xE3, 0xB7, 0x91 + })); + } + + [Test] + public void ComputeRoot_WithSameInputInTwoInstances_ReturnsSameResult() + { + using AccumulatorCalculator sut1 = new(); + using AccumulatorCalculator sut2 = new(); + + sut1.Add(Keccak.Zero, 100); + sut2.Add(Keccak.Zero, 100); + + Assert.That(sut1.ComputeRoot(), Is.EqualTo(sut2.ComputeRoot())); + } + + [Test] + public void ComputeRoot_WithDifferentInputs_ReturnsDifferentResults() + { + using AccumulatorCalculator sut1 = new(); + using AccumulatorCalculator sut2 = new(); + + sut1.Add(Keccak.Zero, 1); + sut2.Add(Keccak.MaxValue, 1); + + Assert.That(sut1.ComputeRoot(), Is.Not.EqualTo(sut2.ComputeRoot())); + } + + [TestCase(-1, TestName = "negative")] + [TestCase(1, TestName = "at_count")] + public void GetProof_WithOutOfRangeIndex_ThrowsArgumentOutOfRangeException(int index) + { + using AccumulatorCalculator sut = new(); + sut.Add(Keccak.Zero, 1); + + Assert.That(() => sut.GetProof(index), Throws.TypeOf()); + } + + [Test] + public void GetProof_WhenCalled_Returns15Elements() + { + using AccumulatorCalculator sut = new(); + sut.Add(Keccak.Zero, 42); + + Assert.That(sut.GetProof(0), Has.Length.EqualTo(15)); + } + + [TestCase(0)] + [TestCase(1)] + [TestCase(7)] + public void GetProof_WhenCalled_ProofZeroIsTotalDifficultyLE(int blockIndex) + { + using AccumulatorCalculator sut = new(); + for (int i = 0; i <= blockIndex; i++) + sut.Add(Keccak.Zero, (ulong)(i + 1)); + + byte[] expected = new byte[32]; + expected[0] = (byte)(blockIndex + 1); + Assert.That(sut.GetProof(blockIndex)[0].ToByteArray(), Is.EqualTo(expected)); + } + + [Test] + public void GetProof_WithDifferentIndices_ReturnDifferentProofs() + { + using AccumulatorCalculator sut = new(); + sut.Add(Keccak.Zero, 1); + sut.Add(Keccak.MaxValue, 2); + + ValueHash256[] proof0 = sut.GetProof(0); + ValueHash256[] proof1 = sut.GetProof(1); + + Assert.That(proof0[0], Is.Not.EqualTo(proof1[0])); + Assert.That(proof0[1], Is.Not.EqualTo(proof1[1])); + } + +} diff --git a/src/Nethermind/Nethermind.EraE.Test/Archive/EraReaderTests.cs b/src/Nethermind/Nethermind.EraE.Test/Archive/EraReaderTests.cs new file mode 100644 index 000000000000..8bdbaabf767a --- /dev/null +++ b/src/Nethermind/Nethermind.EraE.Test/Archive/EraReaderTests.cs @@ -0,0 +1,157 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using FluentAssertions; +using Nethermind.Consensus.Validators; +using Nethermind.Core; +using Nethermind.Core.Crypto; +using Nethermind.EraE.Archive; +using AccumulatorCalculator = Nethermind.Era1.AccumulatorCalculator; +using EraException = Nethermind.Era1.EraException; +using Nethermind.Specs; +using NUnit.Framework; + +namespace Nethermind.EraE.Test.Archive; + +internal class EraReaderTests +{ + [TestCase(3, 0)] + [TestCase(0, 3)] + public async Task GetBlockByNumber_ReturnsCorrectBlockNumbers(int preMergeCount, int postMergeCount) + { + int totalCount = preMergeCount + postMergeCount; + using TestEraFile file = await TestEraFile.Create(preMergeCount: preMergeCount, postMergeCount: postMergeCount); + using EraReader sut = new(file.FilePath); + + for (int i = 0; i < totalCount; i++) + { + (Block block, _) = await sut.GetBlockByNumber(i); + block.Number.Should().Be(i); + } + } + + [Test] + public async Task GetBlockByNumber_InPreMergeEpoch_TotalDifficultyIsRestored() + { + using TestEraFile file = await TestEraFile.Create(preMergeCount: 3, postMergeCount: 0); + using EraReader sut = new(file.FilePath); + + (Block block, _) = await sut.GetBlockByNumber(2); + block.TotalDifficulty.Should().Be(file.Contents[2].Block.TotalDifficulty); + } + + [Test] + public async Task ReadAccumulatorRoot_InPreMergeEpoch_Succeeds() + { + using TestEraFile file = await TestEraFile.Create(preMergeCount: 3, postMergeCount: 0); + using EraReader sut = new(file.FilePath); + + Assert.That(() => sut.ReadAccumulatorRoot(), Throws.Nothing); + } + + [Test] + public async Task VerifyContent_PreMergeEpoch_AccumulatorMatchesComputed() + { + using TestEraFile file = await TestEraFile.Create(preMergeCount: 3, postMergeCount: 0); + + using AccumulatorCalculator calculator = new(); + foreach ((Block block, _) in file.Contents) + calculator.Add(block.Hash!, block.TotalDifficulty!.Value); + + ValueHash256 expectedRoot = calculator.ComputeRoot(); + + using EraReader sut = new(file.FilePath); + ValueHash256 verifiedRoot = await sut.VerifyContent(MainnetSpecProvider.Instance, Always.Valid); + + verifiedRoot.Should().Be(expectedRoot); + } + + [Test] + public async Task GetAsyncEnumerator_InPreMergeEpoch_YieldsAllBlocks() + { + using TestEraFile file = await TestEraFile.Create(preMergeCount: 3, postMergeCount: 0); + using EraReader sut = new(file.FilePath); + + List<(Block, TxReceipt[])> result = await sut.ToListAsync(); + result.Should().HaveCount(3); + result.Select(r => r.Item1.Number).Should().BeEquivalentTo([0L, 1L, 2L]); + } + + [Test] + public async Task ReadAccumulatorRoot_InPostMergeEpoch_ThrowsEraException() + { + using TestEraFile file = await TestEraFile.Create(preMergeCount: 0, postMergeCount: 3); + using EraReader sut = new(file.FilePath); + + Assert.That(() => sut.ReadAccumulatorRoot(), Throws.TypeOf()); + } + + [Test] + public async Task VerifyContent_InPostMergeEpoch_ReturnsDefaultHash() + { + using TestEraFile file = await TestEraFile.Create(preMergeCount: 0, postMergeCount: 3); + using EraReader sut = new(file.FilePath); + + ValueHash256 result = await sut.VerifyContent(MainnetSpecProvider.Instance, Always.Valid); + result.Should().Be(default, "post-merge epochs have no accumulator"); + } + + [Test] + public async Task VerifyContent_InTransitionEpoch_AccumulatorCoversOnlyPreMergeBlocks() + { + using TestEraFile file = await TestEraFile.Create(preMergeCount: 2, postMergeCount: 2); + + using AccumulatorCalculator calculator = new(); + foreach ((Block block, _) in file.Contents.Where(c => !c.Block.Header.IsPostMerge)) + calculator.Add(block.Hash!, block.TotalDifficulty!.Value); + + ValueHash256 expectedRoot = calculator.ComputeRoot(); + + using EraReader sut = new(file.FilePath); + ValueHash256 verifiedRoot = await sut.VerifyContent(MainnetSpecProvider.Instance, Always.Valid); + + verifiedRoot.Should().Be(expectedRoot, + "transition epoch accumulator must only cover pre-merge blocks"); + } + + [Test] + public async Task GetBlockByNumber_InTransitionEpoch_FirstPostMergeBlockHasIsPostMergeTrue() + { + using TestEraFile file = await TestEraFile.Create(preMergeCount: 2, postMergeCount: 2); + using EraReader sut = new(file.FilePath); + + (Block postMergeBlock, _) = await sut.GetBlockByNumber(2); + postMergeBlock.Header.IsPostMerge.Should().BeTrue(); + } + + [Test] + public async Task GetBlockByNumber_WithBelowRangeNumber_ThrowsArgumentOutOfRangeException() + { + using TestEraFile file = await TestEraFile.Create(preMergeCount: 2, postMergeCount: 0); + using EraReader sut = new(file.FilePath); + + Assert.That(async () => await sut.GetBlockByNumber(-1), Throws.TypeOf()); + } + + [Test] + public async Task GetBlockByNumber_WithAboveRangeNumber_ThrowsArgumentOutOfRangeException() + { + using TestEraFile file = await TestEraFile.Create(preMergeCount: 2, postMergeCount: 0); + using EraReader sut = new(file.FilePath); + + Assert.That(async () => await sut.GetBlockByNumber(999), Throws.TypeOf()); + } + + [Test] + public async Task GetBlockByNumber_WithSlimEncodedReceipts_BloomIsReconstructedFromLogs() + { + using TestEraFile file = await TestEraFile.Create(preMergeCount: 1, postMergeCount: 0); + using EraReader sut = new(file.FilePath); + + (_, TxReceipt[] receipts) = await sut.GetBlockByNumber(0); + + receipts.Should().NotBeEmpty(); + receipts[0].Bloom.Should().NotBeNull("bloom must be auto-computed from logs"); + } + +} diff --git a/src/Nethermind/Nethermind.EraE.Test/Archive/EraSlimReceiptDecoderTests.cs b/src/Nethermind/Nethermind.EraE.Test/Archive/EraSlimReceiptDecoderTests.cs new file mode 100644 index 000000000000..7d3a9cfbc3c2 --- /dev/null +++ b/src/Nethermind/Nethermind.EraE.Test/Archive/EraSlimReceiptDecoderTests.cs @@ -0,0 +1,135 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using FluentAssertions; +using Nethermind.Core; +using Nethermind.Core.Crypto; +using Nethermind.Core.Test.Builders; +using Nethermind.EraE.Archive; +using Nethermind.Serialization.Rlp; +using NUnit.Framework; + +namespace Nethermind.EraE.Test.Archive; + +/// +/// Tests for covering the go-ethereum 4-field ERA receipt +/// format, which is what ethpandaops and go-ethereum-based providers emit. +/// +internal class EraSlimReceiptDecoderTests +{ + // go-ethereum 4-field format: outer_list { receipt_list { tx_type, status, gas, logs } } + // Pre-Byzantium: the "status" field is a 32-byte PostTransactionState (state root), not a status code. + // Post-Byzantium (EIP-658): the "status" field is 0x00 or 0x01 (1 byte). + + [Test] + public void Decode_GethFormat_PreByzantium_SetsPostTransactionStateNotStatusCode() + { + // Arrange: build raw go-ethereum 4-field receipt bytes with a 32-byte state root. + // Receipt structure: LIST { LIST { BYTES("") [tx_type], BYTES(<32>) [state_root], INT(0) [gas], LIST{} [logs] } } + Hash256 expectedStateRoot = TestItem.KeccakA; + + byte[] stateRootEncoded = new byte[33]; // 0xa0 + 32 bytes + stateRootEncoded[0] = 0xa0; // RLP prefix for 32-byte string + expectedStateRoot.Bytes.CopyTo(stateRootEncoded.AsSpan(1)); + + // Receipt content: tx_type(0x80) + state_root(33) + gas(0x80) + logs(0xc0) = 36 bytes + byte[] encoded = WrapAsGethReceipt([0x80, .. stateRootEncoded, 0x80, 0xc0]); + + EraSlimReceiptDecoder sut = new(); + TxReceipt[] receipts = sut.Decode(encoded.AsMemory()); + + receipts.Should().HaveCount(1); + receipts[0].PostTransactionState.Should().Be(expectedStateRoot, + "pre-Byzantium go-ethereum receipts encode the state root in the status field; " + + "the decoder must restore PostTransactionState, not StatusCode"); + receipts[0].StatusCode.Should().Be(0, + "StatusCode must not be set for pre-Byzantium receipts"); + } + + [Test] + public void Decode_GethFormat_PostByzantiumSuccess_SetsStatusCode() + { + // Receipt content: tx_type(0x80) + status(0x01) + gas(0x80) + logs(0xc0) = 4 bytes + byte[] encoded = WrapAsGethReceipt([0x80, 0x01, 0x80, 0xc0]); + + EraSlimReceiptDecoder sut = new(); + TxReceipt[] receipts = sut.Decode(encoded.AsMemory()); + + receipts.Should().HaveCount(1); + receipts[0].StatusCode.Should().Be(1); + receipts[0].PostTransactionState.Should().BeNull(); + } + + [Test] + public void Decode_GethFormat_PostByzantiumFailure_SetsStatusCode() + { + // status = 0x80 (empty bytes = 0/failure in go-ethereum encoding) + byte[] encoded = WrapAsGethReceipt([0x80, 0x80, 0x80, 0xc0]); + + EraSlimReceiptDecoder sut = new(); + TxReceipt[] receipts = sut.Decode(encoded.AsMemory()); + + receipts.Should().HaveCount(1); + receipts[0].StatusCode.Should().Be(0); + receipts[0].PostTransactionState.Should().BeNull(); + } + + [TestCase((byte)1)] + [TestCase((byte)2)] + [TestCase((byte)3)] + public void Decode_GethFormat_TypedReceipt_SetsTxType(byte txType) + { + byte[] encoded = WrapAsGethReceipt([txType, 0x01, 0x80, 0xc0]); + + EraSlimReceiptDecoder sut = new(); + TxReceipt[] receipts = sut.Decode(encoded.AsMemory()); + + receipts.Should().HaveCount(1); + receipts[0].TxType.Should().Be((TxType)txType); + receipts[0].StatusCode.Should().Be(1); + } + + [Test] + public void Decode_GethFormat_DecodesCumulativeGasUsed() + { + // gas = 100 => 0x64 + byte[] encoded = WrapAsGethReceipt([0x80, 0x01, 0x64, 0xc0]); + + EraSlimReceiptDecoder sut = new(); + TxReceipt[] receipts = sut.Decode(encoded.AsMemory()); + + receipts.Should().HaveCount(1); + receipts[0].GasUsedTotal.Should().Be(100); + } + + [Test] + public void Decode_GethFormat_InvalidStatusLength_Throws() + { + // status = 2-byte string: 0x82 0x01 0x02 + byte[] encoded = WrapAsGethReceipt([0x80, 0x82, 0x01, 0x02, 0x80, 0xc0]); + + EraSlimReceiptDecoder sut = new(); + Action act = () => sut.Decode(encoded.AsMemory()); + + act.Should().Throw(); + } + + [Test] + public void Decode_GethFormat_InvalidTxTypeLength_Throws() + { + // tx_type = 2-byte string + byte[] encoded = WrapAsGethReceipt([0x82, 0x01, 0x02, 0x01, 0x80, 0xc0]); + + EraSlimReceiptDecoder sut = new(); + Action act = () => sut.Decode(encoded.AsMemory()); + + act.Should().Throw(); + } + + // go-ethereum receipt encoding: outer_list { receipt_list { content } } + private static byte[] WrapAsGethReceipt(byte[] receiptContent) + { + byte[] receipt = [(byte)(0xc0 + receiptContent.Length), .. receiptContent]; + return [(byte)(0xc0 + receipt.Length), .. receipt]; + } +} diff --git a/src/Nethermind/Nethermind.EraE.Test/Archive/EraWriterTests.cs b/src/Nethermind/Nethermind.EraE.Test/Archive/EraWriterTests.cs new file mode 100644 index 000000000000..b40ddd048143 --- /dev/null +++ b/src/Nethermind/Nethermind.EraE.Test/Archive/EraWriterTests.cs @@ -0,0 +1,186 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using FluentAssertions; +using Nethermind.Core; +using Nethermind.Core.Specs; +using Nethermind.Core.Test.Builders; +using Nethermind.Core.Test.IO; +using Nethermind.Era1; +using Nethermind.Int256; +using NSubstitute; +using NUnit.Framework; +using E2StoreReader = Nethermind.EraE.E2Store.E2StoreReader; +using EraWriter = Nethermind.EraE.Archive.EraWriter; + +namespace Nethermind.EraE.Test.Archive; + +internal class EraWriterTests +{ + [Test] + public async Task Add_WithNonSequentialBlock_ThrowsArgumentException() + { + using EraWriter sut = CreateSut(); + + Block b0 = Build.A.Block.WithNumber(0).WithTotalDifficulty(BlockHeaderBuilder.DefaultDifficulty).TestObject; + Block b2 = Build.A.Block.WithNumber(2).WithTotalDifficulty(BlockHeaderBuilder.DefaultDifficulty).TestObject; + + await sut.Add(b0, []); + Assert.That(async () => await sut.Add(b2, []), Throws.ArgumentException); + } + + [Test] + public Task Add_WithNullBlock_ThrowsArgumentNullException() + { + using EraWriter sut = CreateSut(); + + Assert.That(async () => await sut.Add(null!, []), Throws.ArgumentNullException); + return Task.CompletedTask; + } + + [Test] + public Task Add_PreMergeBlockWithoutTotalDifficulty_ThrowsArgumentException() + { + using EraWriter sut = CreateSut(); + + Block block = Build.A.Block.WithNumber(0).WithTotalDifficulty((UInt256?)null).TestObject; + Assert.That(async () => await sut.Add(block, []), Throws.ArgumentException); + return Task.CompletedTask; + } + + [Test] + public Task Add_PreMergeBlockWithTdLowerThanDifficulty_ThrowsArgumentOutOfRangeException() + { + using EraWriter sut = CreateSut(); + + Block block = Build.A.Block.WithNumber(0) + .WithTotalDifficulty(BlockHeaderBuilder.DefaultDifficulty).TestObject; + block.Header.TotalDifficulty = 0; + + Assert.That(async () => await sut.Add(block, []), Throws.TypeOf()); + return Task.CompletedTask; + } + + [Test] + public Task Add_PostMergeBlockWithoutTotalDifficulty_Succeeds() + { + using EraWriter sut = CreateSut(); + + Block block = Build.A.Block.WithNumber(0).WithPostMergeRules().TestObject; + Assert.That(async () => await sut.Add(block, []), Throws.Nothing); + return Task.CompletedTask; + } + + [Test] + public async Task Add_AfterFinalized_ThrowsEraException() + { + using EraWriter sut = CreateSut(); + + Block block = Build.A.Block.WithNumber(0).WithTotalDifficulty(BlockHeaderBuilder.DefaultDifficulty).TestObject; + await sut.Add(block, []); + await sut.Finalize(); + + Assert.That(async () => await sut.Add(block, []), Throws.TypeOf()); + } + + [Test] + public async Task Add_WhenExceedingMaxEraSize_ThrowsArgumentException() + { + using EraWriter sut = CreateSut(); + + for (int i = 0; i < EraWriter.MaxEraSize; i++) + { + Block block = Build.A.Block.WithNumber(i).WithTotalDifficulty(BlockHeaderBuilder.DefaultDifficulty).TestObject; + await sut.Add(block, []); + } + + Block overflow = Build.A.Block.WithNumber(EraWriter.MaxEraSize).WithTotalDifficulty(BlockHeaderBuilder.DefaultDifficulty).TestObject; + Assert.That(async () => await sut.Add(overflow, []), Throws.ArgumentException); + } + + [Test] + public void Finalize_WithNoBlocksAdded_ThrowsEraException() + { + using EraWriter sut = CreateSut(); + + Assert.That(async () => await sut.Finalize(), Throws.TypeOf()); + } + + [Test] + public async Task Finalize_WhenCalledTwice_ThrowsEraException() + { + using EraWriter sut = CreateSut(); + + Block block = Build.A.Block.WithNumber(0).WithTotalDifficulty(BlockHeaderBuilder.DefaultDifficulty).TestObject; + await sut.Add(block, []); + await sut.Finalize(); + + Assert.That(async () => await sut.Finalize(), Throws.TypeOf()); + } + + [TestCase(false, TestName = "pre_merge")] + [TestCase(true, TestName = "post_merge")] + public async Task Finalize_ComponentIndexHasTotalDifficulty_MatchesMergeState(bool isPostMerge) + { + using TempPath tmpFile = TempPath.GetTempFile(); + using (EraWriter sut = new(tmpFile.Path, Substitute.For())) + { + Block block = isPostMerge + ? Build.A.Block.WithNumber(0).WithPostMergeRules().TestObject + : Build.A.Block.WithNumber(0).WithTotalDifficulty(BlockHeaderBuilder.DefaultDifficulty).TestObject; + await sut.Add(block, []); + await sut.Finalize(); + } + + using E2StoreReader reader = new(tmpFile.Path); + reader.HasTotalDifficulty.Should().Be(!isPostMerge); + } + + [TestCase(false, TestName = "pre_merge")] + [TestCase(true, TestName = "post_merge")] + public async Task Finalize_AccumulatorRootOffset_MatchesMergeState(bool isPostMerge) + { + using TempPath tmpFile = TempPath.GetTempFile(); + using (EraWriter sut = new(tmpFile.Path, Substitute.For())) + { + Block block = isPostMerge + ? Build.A.Block.WithNumber(0).WithPostMergeRules().TestObject + : Build.A.Block.WithNumber(0).WithTotalDifficulty(BlockHeaderBuilder.DefaultDifficulty).TestObject; + await sut.Add(block, []); + await sut.Finalize(); + } + + using E2StoreReader reader = new(tmpFile.Path); + if (isPostMerge) + reader.AccumulatorRootOffset.Should().Be(-1, "post-merge epoch has no AccumulatorRoot entry"); + else + reader.AccumulatorRootOffset.Should().BeGreaterThan(0); + } + + [Test] + public async Task Finalize_WithPreMergeBlock_HeaderOffsetStartsAfterVersionEntry() + { + using TempPath tmpFile = TempPath.GetTempFile(); + using (EraWriter sut = new(tmpFile.Path, Substitute.For())) + { + Block block = Build.A.Block.WithNumber(0).WithTotalDifficulty(BlockHeaderBuilder.DefaultDifficulty).TestObject; + await sut.Add(block, []); + await sut.Finalize(); + } + + using E2StoreReader reader = new(tmpFile.Path); + reader.HeaderOffset(0).Should().Be(8); + } + + [Test] + public void Dispose_WhenCalled_DisposesInnerStream() + { + MemoryStream stream = new(); + EraWriter sut = new(stream, Substitute.For()); + sut.Dispose(); + + Assert.That(() => stream.ReadByte(), Throws.TypeOf()); + } + + private static EraWriter CreateSut() => new(new MemoryStream(), Substitute.For()); +} diff --git a/src/Nethermind/Nethermind.EraE.Test/EraCliRunnerTests.cs b/src/Nethermind/Nethermind.EraE.Test/EraCliRunnerTests.cs new file mode 100644 index 000000000000..94c920cd185d --- /dev/null +++ b/src/Nethermind/Nethermind.EraE.Test/EraCliRunnerTests.cs @@ -0,0 +1,59 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.EraE.Config; +using EraException = Nethermind.Era1.EraException; +using Nethermind.EraE.Export; +using Nethermind.EraE.Import; +using Nethermind.History; +using NSubstitute; +using NUnit.Framework; + +namespace Nethermind.EraE.Test; + +public class EraCliRunnerTests +{ + [Test] + public void Run_WithImportDirectory_CallsEraImporter() + { + IEraImporter eraImporter = Substitute.For(); + IEraEConfig eraConfig = new EraEConfig + { + ImportDirectory = "import dir", + From = 99, + To = 999 + }; + EraCliRunner cliRunner = new(eraConfig, new HistoryConfig(), eraImporter, Substitute.For()); + + _ = cliRunner.Run(default); + + eraImporter.Received().Import("import dir", 99, 999, null, default); + } + + [Test] + public void Run_WithImportDirectoryAndUseAncientBarriers_ThrowsEraException() + { + IEraEConfig eraConfig = new EraEConfig { ImportDirectory = "import dir" }; + IHistoryConfig historyConfig = new HistoryConfig { Pruning = PruningModes.UseAncientBarriers }; + EraCliRunner cliRunner = new(eraConfig, historyConfig, Substitute.For(), Substitute.For()); + + Assert.That(() => cliRunner.Run(default), Throws.TypeOf()); + } + + [Test] + public void Run_WithExportDirectory_CallsEraExporter() + { + IEraExporter eraExporter = Substitute.For(); + IEraEConfig eraConfig = new EraEConfig + { + ExportDirectory = "export dir", + From = 99, + To = 999 + }; + EraCliRunner cliRunner = new(eraConfig, new HistoryConfig(), Substitute.For(), eraExporter); + + _ = cliRunner.Run(default); + + eraExporter.Received().Export("export dir", 99, 999, default); + } +} diff --git a/src/Nethermind/Nethermind.EraE.Test/EraETestModule.cs b/src/Nethermind/Nethermind.EraE.Test/EraETestModule.cs new file mode 100644 index 000000000000..1091101b89b5 --- /dev/null +++ b/src/Nethermind/Nethermind.EraE.Test/EraETestModule.cs @@ -0,0 +1,110 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Autofac; +using Nethermind.Blockchain; +using Nethermind.Blockchain.Receipts; +using Nethermind.Consensus.Validators; +using Nethermind.Core; +using Nethermind.Core.Test.Builders; +using Nethermind.Core.Test.IO; +using Nethermind.Core.Test.Modules; +using System.IO.Abstractions; +using Nethermind.EraE.Config; +using Nethermind.EraE.Export; +using Testably.Abstractions; + +namespace Nethermind.EraE.Test; + +public class EraETestModule(bool useRealValidator = false) : Module +{ + public const string TestNetwork = "abc"; + + public static ContainerBuilder BuildContainerBuilder() => + new ContainerBuilder().AddModule(new EraETestModule()); + + public static ContainerBuilder BuildContainerBuilderWithBlockTreeOfLength(int length) + { + BlockTreeBuilder blockTreeBuilder = Build.A.BlockTree(); + + return new ContainerBuilder() + .AddModule(new EraETestModule()) + .AddSingleton(blockTreeBuilder.TestObject) + .OnBuild(ctx => + { + blockTreeBuilder + .WithTransactions(ctx.Resolve()) + .OfChainLength(length); + }); + } + + // Beacon genesis: 1606824023s (1 Dec 2020). Post-merge blocks must have timestamps after this. + // The Merge was ~15 Sep 2022 (1663224162s). We use a round timestamp well past both. + private const ulong PostMergeGenesisTimestamp = 1_663_000_000; + + public static ContainerBuilder BuildContainerBuilderWithPostMergeBlockTreeOfLength(int length) + { + // Set genesis timestamp past beacon-chain genesis so SlotTime.GetSlot succeeds on all blocks. + Block genesis = Build.A.Block.Genesis.WithTimestamp(PostMergeGenesisTimestamp).WithPostMergeRules().TestObject; + BlockTreeBuilder blockTreeBuilder = Build.A.BlockTree(genesis).WithPostMergeRules(); + + return new ContainerBuilder() + .AddModule(new EraETestModule()) + .AddSingleton(PostMerge) + .AddSingleton(blockTreeBuilder.TestObject) + .OnBuild(ctx => + { + blockTreeBuilder + .WithTransactions(ctx.Resolve()) + .OfChainLength(length); + + // Post-merge blocks have TotalDifficulty=0, which never satisfies + // HeadImprovementRequirementsSatisfied (0 < MainnetTTD). Force-update head + // so the exporter can resolve blockTree.Head correctly. + Block? lastBlock = blockTreeBuilder.BlockTree.FindBlock(length - 1, BlockTreeLookupOptions.None); + if (lastBlock is not null) + blockTreeBuilder.BlockTree.UpdateMainChain(new[] { lastBlock }, true, forceUpdateHeadBlock: true); + }); + } + + // Matches PostMergeGenesisTimestamp: 1663000000s = 2022-09-12 18:26:40 UTC + public static ManualTimestamper PostMerge => + new(DateTimeOffset.FromUnixTimeSeconds((long)PostMergeGenesisTimestamp).UtcDateTime); + + public static async Task CreateExportedEraEnv(int chainLength = 512, long from = 0, long to = 0) + { + IContainer testCtx = BuildContainerBuilderWithBlockTreeOfLength(chainLength).Build(); + await testCtx.Resolve().Export(testCtx.ResolveTempDirPath(), from, to); + return testCtx; + } + + public static async Task CreateExportedPostMergeEraEnv(int chainLength = 16) + { + IContainer testCtx = BuildContainerBuilderWithPostMergeBlockTreeOfLength(chainLength).Build(); + // Start from block 1: genesis is pre-merge in all block trees. + await testCtx.Resolve().Export(testCtx.ResolveTempDirPath(), from: 1, to: 0); + return testCtx; + } + + protected override void Load(ContainerBuilder builder) + { + base.Load(builder); + builder + .AddModule(TestNethermindModule.CreateWithRealChainSpec()) + .AddModule(new EraEModule()) + .AddSingleton(new RealFileSystem()) + .AddSingleton(new EraEConfig() + { + MaxEraSize = 16, + NetworkName = TestNetwork, + }) + .AddSingleton(ManualTimestamper.PreMerge) + .AddKeyedSingleton("file", ctx => TempPath.GetTempFile()) + .AddKeyedSingleton("directory", ctx => TempPath.GetTempDirectory()); + + if (!useRealValidator) + { + builder.AddSingleton(Always.Valid); + } + } +} diff --git a/src/Nethermind/Nethermind.EraE.Test/EraFileFormatComplianceTests.cs b/src/Nethermind/Nethermind.EraE.Test/EraFileFormatComplianceTests.cs new file mode 100644 index 000000000000..fcec2b558054 --- /dev/null +++ b/src/Nethermind/Nethermind.EraE.Test/EraFileFormatComplianceTests.cs @@ -0,0 +1,188 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Buffers.Binary; +using FluentAssertions; +using NUnit.Framework; + +namespace Nethermind.EraE.Test; + +/// +/// Verifies that EraWriter produces files whose binary layout matches the EraE spec exactly: +/// TLV structure, entry type codes, section ordering, and ComponentIndex structure. +/// These tests assert spec compliance from Nethermind's perspective. +/// Cross-client interoperability testing with go-ethereum / fluffy is pending availability +/// of reference .erae test files (go-ethereum has not yet written Proof entries as of 2026-03). +/// +public class EraFileFormatComplianceTests +{ + // TLV entry header: [type: uint16 LE] [length: uint32 LE] [reserved: uint16 LE = 0x0000] + private const int EntryHeaderSize = 8; + + // Entry type codes per EraE spec (matches EntryTypes.cs) + private const ushort TypeVersion = 0x3265; + private const ushort TypeCompressedHeader = 0x03; + private const ushort TypeCompressedBody = 0x04; + private const ushort TypeCompressedSlimReceipts = 0x0a; + private const ushort TypeProof = 0x0b; + private const ushort TypeTotalDifficulty = 0x06; + private const ushort TypeAccumulatorRoot = 0x07; + private const ushort TypeComponentIndex = 0x3267; + + private const int SharedBlockCount = 3; + + private TestEraFile _preMergeFile = null!; + private TestEraFile _postMergeFile = null!; + + [OneTimeSetUp] + public async Task SetUpFiles() + { + _preMergeFile = await TestEraFile.Create(preMergeCount: SharedBlockCount, postMergeCount: 0); + _postMergeFile = await TestEraFile.Create(preMergeCount: 0, postMergeCount: SharedBlockCount); + } + + [OneTimeTearDown] + public void TearDownFiles() + { + _preMergeFile.Dispose(); + _postMergeFile.Dispose(); + } + + [Test] + public void File_PreMergeEpoch_FirstEntryIsVersion() + { + List entries = ReadAllEntries(_preMergeFile.FilePath); + + entries[0].Type.Should().Be(TypeVersion, "Version must be the first entry per spec"); + entries[0].Length.Should().Be(0, "Version entry carries no data"); + } + + [Test] + public void File_PreMergeEpoch_LastEntryIsComponentIndex() + { + List entries = ReadAllEntries(_preMergeFile.FilePath); + + entries[^1].Type.Should().Be(TypeComponentIndex, "ComponentIndex must be the last entry per spec"); + } + + [Test] + public void File_PostMergeEpoch_LastEntryIsComponentIndex() + { + List entries = ReadAllEntries(_postMergeFile.FilePath); + + entries[^1].Type.Should().Be(TypeComponentIndex); + } + + [Test] + public void File_AllEntries_HaveReservedBytesZero() + { + byte[] bytes = File.ReadAllBytes(_preMergeFile.FilePath); + + long pos = 0; + while (pos + EntryHeaderSize <= bytes.Length) + { + ushort reserved = BinaryPrimitives.ReadUInt16LittleEndian(bytes.AsSpan((int)pos + 6, 2)); + uint length = BinaryPrimitives.ReadUInt32LittleEndian(bytes.AsSpan((int)pos + 2, 4)); + + reserved.Should().Be(0, $"reserved bytes at offset {pos} must be 0x0000 per TLV spec"); + pos += EntryHeaderSize + length; + } + } + + [Test] + public void File_PreMergeEpoch_SectionOrderIsHeaderBodyReceiptsTdAccumulatorIndex() + { + List types = ReadAllEntries(_preMergeFile.FilePath).Select(e => e.Type).ToList(); + + // Section ordering: all headers before any body + types.LastIndexOf(TypeCompressedHeader).Should().BeLessThan( + types.IndexOf(TypeCompressedBody), "all headers must precede any body per spec"); + + // All bodies before any receipts + types.LastIndexOf(TypeCompressedBody).Should().BeLessThan( + types.IndexOf(TypeCompressedSlimReceipts), "all bodies must precede any receipts per spec"); + + // Receipts before TotalDifficulty + types.LastIndexOf(TypeCompressedSlimReceipts).Should().BeLessThan( + types.IndexOf(TypeTotalDifficulty), "all receipts must precede TotalDifficulty per spec"); + + // TotalDifficulty before AccumulatorRoot + types.LastIndexOf(TypeTotalDifficulty).Should().BeLessThan( + types.IndexOf(TypeAccumulatorRoot), "TotalDifficulty must precede AccumulatorRoot per spec"); + + // AccumulatorRoot before ComponentIndex + types.IndexOf(TypeAccumulatorRoot).Should().BeLessThan( + types.LastIndexOf(TypeComponentIndex), "AccumulatorRoot must precede ComponentIndex per spec"); + } + + [Test] + public void File_PostMergeEpoch_HasNoTdOrAccumulatorEntries() + { + List types = ReadAllEntries(_postMergeFile.FilePath).Select(e => e.Type).ToList(); + + types.Should().NotContain(TypeTotalDifficulty, "post-merge epochs have no TotalDifficulty entries"); + types.Should().NotContain(TypeAccumulatorRoot, "post-merge epochs have no AccumulatorRoot entry"); + } + + [Test] + public void File_PreMergeEpoch_EntryCountsMatchBlockCount() + { + List types = ReadAllEntries(_preMergeFile.FilePath).Select(e => e.Type).ToList(); + + types.Count(t => t == TypeCompressedHeader).Should().Be(SharedBlockCount); + types.Count(t => t == TypeCompressedBody).Should().Be(SharedBlockCount); + types.Count(t => t == TypeCompressedSlimReceipts).Should().Be(SharedBlockCount); + types.Count(t => t == TypeTotalDifficulty).Should().Be(SharedBlockCount); + types.Count(t => t == TypeAccumulatorRoot).Should().Be(1); + types.Count(t => t == TypeComponentIndex).Should().Be(1); + } + + [Test] + public void File_PostMergeEpoch_EntryCountsMatchBlockCount() + { + List types = ReadAllEntries(_postMergeFile.FilePath).Select(e => e.Type).ToList(); + + types.Count(t => t == TypeCompressedHeader).Should().Be(SharedBlockCount); + types.Count(t => t == TypeCompressedBody).Should().Be(SharedBlockCount); + types.Count(t => t == TypeCompressedSlimReceipts).Should().Be(SharedBlockCount); + } + + [Test] + public void File_AccumulatorRootEntry_Is32Bytes() + { + EntryRecord accEntry = ReadAllEntries(_preMergeFile.FilePath).Single(e => e.Type == TypeAccumulatorRoot); + + accEntry.Length.Should().Be(32, "AccumulatorRoot entry must be exactly 32 bytes (Bytes32)"); + } + + [Test] + public void File_TotalDifficultyEntries_AreEach32Bytes() + { + List tdEntries = ReadAllEntries(_preMergeFile.FilePath) + .Where(e => e.Type == TypeTotalDifficulty) + .ToList(); + + tdEntries.Should().AllSatisfy(e => + e.Length.Should().Be(32, "TotalDifficulty entry must be 32-byte LE uint256")); + } + + internal static List ReadAllEntries(string filePath) + { + List entries = []; + byte[] bytes = File.ReadAllBytes(filePath); + long pos = 0; + + while (pos + EntryHeaderSize <= bytes.Length) + { + ushort type = BinaryPrimitives.ReadUInt16LittleEndian(bytes.AsSpan((int)pos, 2)); + uint length = BinaryPrimitives.ReadUInt32LittleEndian(bytes.AsSpan((int)pos + 2, 4)); + entries.Add(new EntryRecord(type, length, pos)); + pos += EntryHeaderSize + length; + } + + return entries; + } + + internal sealed record EntryRecord(ushort Type, uint Length, long Offset); + +} diff --git a/src/Nethermind/Nethermind.EraE.Test/Export/EraExporterTests.cs b/src/Nethermind/Nethermind.EraE.Test/Export/EraExporterTests.cs new file mode 100644 index 000000000000..51494a124dbd --- /dev/null +++ b/src/Nethermind/Nethermind.EraE.Test/Export/EraExporterTests.cs @@ -0,0 +1,208 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Autofac; +using FluentAssertions; +using Nethermind.Blockchain; +using Nethermind.Blockchain.Receipts; +using Nethermind.Core; +using Nethermind.Core.Test.Builders; +using NSubstitute; +using NSubstitute.ReturnsExtensions; +using Nethermind.EraE.Config; +using Nethermind.EraE.E2Store; +using EraException = Nethermind.Era1.EraException; +using Nethermind.EraE.Export; +using NUnit.Framework; + +namespace Nethermind.EraE.Test.Export; + +public class EraExporterTests +{ + [TestCase(1, 0, 0, 1, 1)] + [TestCase(3, 0, 2, 1, 3)] + [TestCase(16, 0, 15, 16, 1)] + [TestCase(32, 0, 31, 16, 2)] + [TestCase(48, 8, 39, 16, 3)] + public async Task Export_WithVaryingChainLength_CreatesCorrectNumberOfEpochFiles( + int chainLength, int from, int to, int eraSize, int expectedEraFiles) + { + await using IContainer container = EraETestModule.BuildContainerBuilderWithBlockTreeOfLength(chainLength) + .AddSingleton(new EraEConfig + { + MaxEraSize = eraSize, + NetworkName = EraETestModule.TestNetwork + }) + .Build(); + + string tmpDirectory = container.ResolveTempDirPath(); + IEraExporter sut = container.Resolve(); + await sut.Export(tmpDirectory, from, to); + + string[] allFiles = System.IO.Directory.GetFiles(tmpDirectory); + string[] eraFiles = allFiles.Where(f => f.EndsWith(EraPathUtils.FileExtension)).ToArray(); + eraFiles.Length.Should().Be(expectedEraFiles); + } + + [TestCase("checksums_sha256.txt")] + [TestCase("checksums.txt")] + [TestCase("accumulators.txt")] + public async Task Export_WhenCalled_CreatesMetadataFile(string fileName) + { + await using IContainer container = EraETestModule.BuildContainerBuilderWithBlockTreeOfLength(32).Build(); + + string tmpDirectory = container.ResolveTempDirPath(); + await container.Resolve().Export(tmpDirectory, 0, 0); + + System.IO.File.Exists(System.IO.Path.Combine(tmpDirectory, fileName)) + .Should().BeTrue(); + } + + [Test] + public async Task Export_WhenCalled_ChecksumsFileHasCorrectFormat() + { + await using IContainer container = EraETestModule.BuildContainerBuilderWithBlockTreeOfLength(32).Build(); + + string tmpDirectory = container.ResolveTempDirPath(); + await container.Resolve().Export(tmpDirectory, 0, 0); + + string[] lines = await System.IO.File.ReadAllLinesAsync( + System.IO.Path.Combine(tmpDirectory, EraExporter.ChecksumsSHA256FileName)); + + foreach (string line in lines) + { + ReadOnlySpan span = line.AsSpan(); + int spaceIdx = span.IndexOf(' '); + spaceIdx.Should().BeGreaterThan(0, "each line should be ' '"); + span[(spaceIdx + 1)..].EndsWith(EraPathUtils.FileExtension.AsSpan()).Should().BeTrue(); + } + } + + [Test] + public async Task Export_WhenCalled_EraFilesHaveCorrectExtension() + { + await using IContainer container = EraETestModule.BuildContainerBuilderWithBlockTreeOfLength(16).Build(); + + string tmpDirectory = container.ResolveTempDirPath(); + await container.Resolve().Export(tmpDirectory, 0, 0); + + string[] eraFiles = System.IO.Directory.GetFiles(tmpDirectory, $"*{EraPathUtils.FileExtension}"); + eraFiles.Should().NotBeEmpty(); + eraFiles.Should().AllSatisfy(f => f.Should().EndWith(EraPathUtils.FileExtension)); + } + + [Test] + public async Task Export_WithToExceedingHeadBlock_ThrowsArgumentException() + { + await using IContainer container = EraETestModule.BuildContainerBuilderWithBlockTreeOfLength(5).Build(); + + string tmpDirectory = container.ResolveTempDirPath(); + IEraExporter sut = container.Resolve(); + + Assert.That(() => sut.Export(tmpDirectory, 0, 999), Throws.TypeOf()); + } + + [Test] + public async Task Export_WithFromExceedingTo_ThrowsArgumentException() + { + await using IContainer container = EraETestModule.BuildContainerBuilderWithBlockTreeOfLength(32).Build(); + + string tmpDirectory = container.ResolveTempDirPath(); + IEraExporter sut = container.Resolve(); + + Assert.That(() => sut.Export(tmpDirectory, 20, 10), Throws.TypeOf()); + } + + [Test] + public async Task Export_WhenReceiptsStorageReturnsNull_ThrowsEraException() + { + await using IContainer container = EraETestModule.BuildContainerBuilderWithBlockTreeOfLength(10) + .AddSingleton(Substitute.For()) + .Build(); + + container.Resolve() + .Get(Arg.Any(), Arg.Any(), Arg.Any()) + .ReturnsNull(); + + IEraExporter sut = container.Resolve(); + string tmpDirectory = container.ResolveTempDirPath(); + + Assert.That(() => sut.Export(tmpDirectory, 0, 1), Throws.TypeOf()); + } + + [Test] + public async Task Export_WithToZero_ExportsUpToHead() + { + const int chainLength = 32; + await using IContainer container = EraETestModule.BuildContainerBuilderWithBlockTreeOfLength(chainLength).Build(); + + string tmpDirectory = container.ResolveTempDirPath(); + await container.Resolve().Export(tmpDirectory, 0, to: 0); + + string[] eraFiles = System.IO.Directory.GetFiles(tmpDirectory, $"*{EraPathUtils.FileExtension}"); + eraFiles.Should().NotBeEmpty("exporting to 0 should default to head"); + } + + [Test] + public void Export_WithDestinationAsExistingFile_ThrowsArgumentException() + { + using IContainer container = EraETestModule.BuildContainerBuilderWithBlockTreeOfLength(10).Build(); + + string tmpFile = container.ResolveTempFilePath(); + System.IO.File.WriteAllText(tmpFile, "existing"); + + IEraExporter sut = container.Resolve(); + Assert.That(() => sut.Export(tmpFile, 0, 0), Throws.TypeOf()); + } + + [Test] + public async Task Export_WithPostMergeChain_ProducesEpochWithNoTdProofOrAccumulatorEntries() + { + const int chainLength = 16; + await using IContainer container = EraETestModule.BuildContainerBuilderWithPostMergeBlockTreeOfLength(chainLength).Build(); + + string tmpDirectory = container.ResolveTempDirPath(); + // Export from block 1: genesis is pre-merge in all block trees, so starting at 1 ensures a pure post-merge epoch. + await container.Resolve().Export(tmpDirectory, 1, 0); + + string[] eraFiles = Directory.GetFiles(tmpDirectory, $"*{EraPathUtils.FileExtension}"); + eraFiles.Should().HaveCount(1, "one epoch for blocks 1-15"); + + List types = EraFileFormatComplianceTests.ReadAllEntries(eraFiles[0]).Select(e => e.Type).ToList(); + types.Should().NotContain(EntryTypes.Proof, "post-merge epochs have no Proof entries"); + types.Should().NotContain(EntryTypes.TotalDifficulty, "post-merge epochs have no TotalDifficulty entries"); + types.Should().NotContain(EntryTypes.AccumulatorRoot, "post-merge epochs have no AccumulatorRoot entry"); + } + + [Test] + public void Export_WhenLastBlockBodyNotAvailable_ThrowsInvalidOperationException() + { + IBlockTree blockTree = Substitute.For(); + Block head = Build.A.Block.WithNumber(10).TestObject; + blockTree.Head.Returns(head); + blockTree.FindBlock(10, BlockTreeLookupOptions.DoNotCreateLevelIfMissing).ReturnsNull(); + + using IContainer container = EraETestModule.BuildContainerBuilderWithBlockTreeOfLength(1) + .AddSingleton(blockTree) + .Build(); + + string tmpDirectory = container.ResolveTempDirPath(); + IEraExporter sut = container.Resolve(); + + Assert.That(() => sut.Export(tmpDirectory, 0, 10), Throws.TypeOf()); + } + + [Test] + public async Task Export_WithPartialRange_CreatesOnlyEpochsInRange() + { + await using IContainer container = EraETestModule.BuildContainerBuilderWithBlockTreeOfLength(48) + .AddSingleton(new EraEConfig { MaxEraSize = 16, NetworkName = EraETestModule.TestNetwork }) + .Build(); + + string tmpDirectory = container.ResolveTempDirPath(); + await container.Resolve().Export(tmpDirectory, 16, 31); + + string[] eraFiles = System.IO.Directory.GetFiles(tmpDirectory, $"*{EraPathUtils.FileExtension}"); + eraFiles.Length.Should().Be(1, "only one epoch covers blocks 16-31"); + } +} diff --git a/src/Nethermind/Nethermind.EraE.Test/Export/EraPathUtilsTests.cs b/src/Nethermind/Nethermind.EraE.Test/Export/EraPathUtilsTests.cs new file mode 100644 index 000000000000..02f63599eec0 --- /dev/null +++ b/src/Nethermind/Nethermind.EraE.Test/Export/EraPathUtilsTests.cs @@ -0,0 +1,38 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Core.Crypto; +using Nethermind.EraE.Export; +using NUnit.Framework; + +namespace Nethermind.EraE.Test.Export; + +public class EraPathUtilsTests +{ + [TestCase("test", 0, "0x0000000000000000000000000000000000000000000000000000000000000000", "test-00000-00000000.erae")] + [TestCase("goerli", 1, "0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "goerli-00001-ffffffff.erae")] + [TestCase("mainnet", 2, "0x1122ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", "mainnet-00002-1122ffff.erae")] + public void Filename_WithValidParameters_ReturnsExpected(string network, int epoch, string hash, string expected) => Assert.That(EraPathUtils.Filename(network, epoch, new Hash256(hash)), Is.EqualTo(expected)); + + [Test] + public void Filename_WithNullNetwork_ThrowsArgumentNullException() => + Assert.That(() => EraPathUtils.Filename(null!, 0, new Hash256("0x0000000000000000000000000000000000000000000000000000000000000000")), Throws.ArgumentNullException); + + [Test] + public void Filename_WithEmptyNetwork_ThrowsArgumentException() => Assert.That(() => EraPathUtils.Filename("", 0, new Hash256("0x0000000000000000000000000000000000000000000000000000000000000000")), Throws.ArgumentException); + + [Test] + public void Filename_WithNegativeEpoch_ThrowsArgumentOutOfRangeException() => Assert.That(() => EraPathUtils.Filename("test", -1, new Hash256("0x0000000000000000000000000000000000000000000000000000000000000000")), Throws.TypeOf()); + + [Test] + public void Filename_WithNullRoot_ThrowsArgumentNullException() => + Assert.That(() => EraPathUtils.Filename("test", 0, null!), Throws.ArgumentNullException); + + [TestCase("0xaabbccdd00000000000000000000000000000000000000000000000000000000", TestName = "hash_only")] + [TestCase("0xaabbccdd00000000000000000000000000000000000000000000000000000000 test-00000-aabbccdd.erae", TestName = "hash_and_filename")] + public void ExtractHashFromChecksumEntry_ReturnsCorrectHash(string input) + { + ValueHash256 result = EraPathUtils.ExtractHashFromChecksumEntry(input); + Assert.That(result.ToByteArray()[0], Is.EqualTo(0xaa)); + } +} diff --git a/src/Nethermind/Nethermind.EraE.Test/Import/EraImporterTests.cs b/src/Nethermind/Nethermind.EraE.Test/Import/EraImporterTests.cs new file mode 100644 index 000000000000..eeb2a6a11247 --- /dev/null +++ b/src/Nethermind/Nethermind.EraE.Test/Import/EraImporterTests.cs @@ -0,0 +1,268 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Autofac; +using FluentAssertions; +using Nethermind.Blockchain; +using Nethermind.Blockchain.Receipts; +using Nethermind.Blockchain.Synchronization; +using Nethermind.Consensus.Validators; +using Nethermind.Core; +using Nethermind.Core.Test.Builders; +using EraException = Nethermind.Era1.EraException; +using EraVerificationException = Nethermind.Era1.Exceptions.EraVerificationException; +using Nethermind.EraE.Export; +using Nethermind.EraE.Import; +using NUnit.Framework; + +namespace Nethermind.EraE.Test.Import; + +public class EraImporterTests +{ + [Test] + public async Task Import_WithEmptyDirectory_ThrowsEraException() + { + using IContainer ctx = EraETestModule.BuildContainerBuilder().Build(); + + string tmpDirectory = ctx.ResolveTempDirPath(); + System.IO.Directory.CreateDirectory(tmpDirectory); + await System.IO.File.WriteAllTextAsync( + System.IO.Path.Combine(tmpDirectory, EraExporter.ChecksumsSHA256FileName), ""); + + IEraImporter sut = ctx.Resolve(); + Assert.That( + () => sut.Import(tmpDirectory, 0, 0, null), + Throws.TypeOf()); + } + + [Test] + public async Task Import_WithValidEraFiles_ImportsAllBlocksIntoTree() + { + const int chainLength = 32; + await using ImportEnvironment env = await CreateImportEnvironment(chainLength); + + await env.Sut.Import(env.ExportPath, 0, long.MaxValue, null); + + for (long i = 1; i < chainLength; i++) + { + env.TargetTree.FindBlock(i, BlockTreeLookupOptions.None).Should().NotBeNull( + $"block {i} should have been imported"); + } + } + + [Test] + public async Task Import_WithTrustedAccumulators_Succeeds() + { + await using ImportEnvironment env = await CreateImportEnvironment(); + string accumulatorPath = System.IO.Path.Combine(env.ExportPath, EraExporter.AccumulatorFileName); + + Assert.That( + () => env.Sut.Import(env.ExportPath, 0, long.MaxValue, accumulatorPath), + Throws.Nothing); + } + + [Test] + public async Task Import_WithModifiedChecksum_ThrowsEraVerificationException() + { + await using ImportEnvironment env = await CreateImportEnvironment(); + + string checksumPath = System.IO.Path.Combine(env.ExportPath, EraExporter.ChecksumsSHA256FileName); + string[] lines = await System.IO.File.ReadAllLinesAsync(checksumPath); + lines[^1] = "0x0000000000000000000000000000000000000000000000000000000000000000 " + + System.IO.Path.GetFileName(lines[^1].Split(' ')[^1]); + await System.IO.File.WriteAllLinesAsync(checksumPath, lines); + + Assert.That( + () => env.Sut.Import(env.ExportPath, 0, long.MaxValue, null), + Throws.TypeOf()); + } + + [Test] + public async Task Import_WithWrongTrustedAccumulator_ThrowsEraVerificationException() + { + await using ImportEnvironment env = await CreateImportEnvironment(); + + string fakeAccumulatorPath = System.IO.Path.Combine(env.ExportPath, "fake_accumulators.txt"); + string[] accLines = await System.IO.File.ReadAllLinesAsync( + System.IO.Path.Combine(env.ExportPath, EraExporter.AccumulatorFileName)); + string[] fakeLines = new string[accLines.Length]; + for (int i = 0; i < accLines.Length; i++) + fakeLines[i] = "0x0000000000000000000000000000000000000000000000000000000000000000 " + + accLines[i].Split(' ')[^1]; + await System.IO.File.WriteAllLinesAsync(fakeAccumulatorPath, fakeLines); + + Assert.That( + () => env.Sut.Import(env.ExportPath, 0, long.MaxValue, fakeAccumulatorPath), + Throws.TypeOf()); + } + + [Test] + public async Task Import_WithPartialRange_ImportsOnlyRequestedBlocks() + { + await using ImportEnvironment env = await CreateImportEnvironment(); + + await env.Sut.Import(env.ExportPath, 0, 15, null); + + for (long i = 1; i <= 15; i++) + env.TargetTree.FindBlock(i, BlockTreeLookupOptions.None).Should().NotBeNull($"block {i} should have been imported"); + + env.TargetTree.FindBlock(16, BlockTreeLookupOptions.None).Should().BeNull("block 16 is outside the requested range"); + } + + [Test] + public async Task Import_WhenCalledTwice_DoesNotThrowAndIsIdempotent() + { + await using ImportEnvironment env = await CreateImportEnvironment(); + + await env.Sut.Import(env.ExportPath, 0, long.MaxValue, null); + + Assert.That(() => env.Sut.Import(env.ExportPath, 0, long.MaxValue, null), Throws.Nothing, + "re-importing the same range must be idempotent"); + } + + [Test] + public async Task ExportThenImport_RoundTrip_BlocksAndReceiptsMatchOriginal() + { + const int chainLength = 32; + await using ImportEnvironment env = await CreateImportEnvironment( + chainLength, + b => b.AddSingleton(new SyncConfig { FastSync = true })); + + IReceiptStorage sourceReceipts = env.SourceCtx.Resolve(); + + await env.Sut.Import(env.ExportPath, 0, long.MaxValue, null); + + IReceiptStorage targetReceipts = env.TargetCtx.Resolve(); + IBlockTree sourceTree = env.SourceCtx.Resolve(); + + for (long i = 1; i < chainLength; i++) + { + Block? original = sourceTree.FindBlock(i, BlockTreeLookupOptions.None); + Block? imported = env.TargetTree.FindBlock(i, BlockTreeLookupOptions.None); + + imported.Should().NotBeNull($"block {i} should exist after import"); + imported!.Hash.Should().Be(original!.Hash!, $"block {i} hash must match"); + + TxReceipt[] originalReceipts = sourceReceipts.Get(original!); + bool hasReceipts = targetReceipts.HasBlock(imported.Number, imported.Hash!); + if (originalReceipts.Length > 0) + hasReceipts.Should().BeTrue($"receipts for block {i} should have been imported"); + } + } + + [Test] + public async Task ExportThenImport_PostMergeRoundTrip_BlocksAndReceiptsMatchOriginal() + { + const int chainLength = 16; + await using IContainer sourceCtx = await EraETestModule.CreateExportedPostMergeEraEnv(chainLength); + string exportPath = sourceCtx.ResolveTempDirPath(); + + IBlockTree sourceTree = sourceCtx.Resolve(); + BlockTree targetTree = Build.A.BlockTree() + .WithBlocks(sourceTree.FindBlock(0, BlockTreeLookupOptions.None)!) + .TestObject; + + await using IContainer targetCtx = EraETestModule.BuildContainerBuilder() + .AddSingleton(targetTree) + .AddSingleton(new SyncConfig { FastSync = true }) + .Build(); + + await targetCtx.Resolve().Import(exportPath, 0, long.MaxValue, null); + + IReceiptStorage sourceReceipts = sourceCtx.Resolve(); + IReceiptStorage targetReceipts = targetCtx.Resolve(); + + for (long i = 1; i < chainLength; i++) + { + Block? original = sourceTree.FindBlock(i, BlockTreeLookupOptions.None); + Block? imported = targetTree.FindBlock(i, BlockTreeLookupOptions.None); + + imported.Should().NotBeNull($"post-merge block {i} should exist after import"); + imported!.Hash.Should().Be(original!.Hash!, $"post-merge block {i} hash must match"); + + TxReceipt[] originalReceipts = sourceReceipts.Get(original!); + bool hasReceipts = targetReceipts.HasBlock(imported.Number, imported.Hash!); + if (originalReceipts.Length > 0) + hasReceipts.Should().BeTrue($"receipts for post-merge block {i} should have been imported"); + } + } + + [Test] + public async Task Import_WhenBlocksPrePopulatedWithoutTotalDifficulty_SetsCorrectTotalDifficulty() + { + // Simulate the snap sync ancient-bodies phase: block bodies exist in the tree but were + // inserted without TotalDifficulty (blockInfo.TD=0). Era import must re-insert the header + // with the correct TD — either from the era file or computed via SetTotalDifficulty. + const int chainLength = 32; + await using IContainer sourceCtx = await EraETestModule.CreateExportedEraEnv(chainLength, from: 0, to: 0); + string exportPath = sourceCtx.ResolveTempDirPath(); + + IBlockTree sourceTree = sourceCtx.Resolve(); + BlockTree targetTree = Build.A.BlockTree() + .WithBlocks(sourceTree.FindBlock(0, BlockTreeLookupOptions.None)!) + .TestObject; + + for (long i = 1; i < chainLength; i++) + { + Block block = sourceTree.FindBlock(i, BlockTreeLookupOptions.TotalDifficultyNotNeeded)!; + targetTree.Insert(block, + BlockTreeInsertBlockOptions.SaveHeader | BlockTreeInsertBlockOptions.SkipCanAcceptNewBlocks, + BlockTreeInsertHeaderOptions.TotalDifficultyNotNeeded); + } + + await using IContainer targetCtx = EraETestModule.BuildContainerBuilder() + .AddSingleton(targetTree) + .Build(); + + await targetCtx.Resolve().Import(exportPath, 0, long.MaxValue, null); + + for (long i = 1; i < chainLength; i++) + { + Block? imported = targetTree.FindBlock(i, BlockTreeLookupOptions.None); + imported.Should().NotBeNull($"block {i} should exist"); + imported!.TotalDifficulty.Should().NotBeNull($"block {i} should have TotalDifficulty after import"); + } + } + + [Test] + public async Task Import_WhenBlockFailsValidation_ThrowsEraVerificationException() + { + await using ImportEnvironment env = await CreateImportEnvironment( + configure: b => b.AddSingleton(Always.Invalid)); + + Assert.That( + () => env.Sut.Import(env.ExportPath, 0, long.MaxValue, null), + Throws.TypeOf()); + } + + private static async Task CreateImportEnvironment( + int chainLength = 32, + Action? configure = null) + { + IContainer sourceCtx = await EraETestModule.CreateExportedEraEnv(chainLength, from: 0, to: 0); + string exportPath = sourceCtx.ResolveTempDirPath(); + IBlockTree sourceTree = sourceCtx.Resolve(); + BlockTree targetTree = Build.A.BlockTree() + .WithBlocks(sourceTree.FindBlock(0, BlockTreeLookupOptions.None)!) + .TestObject; + ContainerBuilder builder = EraETestModule.BuildContainerBuilder() + .AddSingleton(targetTree); + configure?.Invoke(builder); + IContainer targetCtx = builder.Build(); + return new ImportEnvironment(sourceCtx, targetCtx, exportPath, targetTree, targetCtx.Resolve()); + } + + private sealed record ImportEnvironment( + IContainer SourceCtx, + IContainer TargetCtx, + string ExportPath, + BlockTree TargetTree, + IEraImporter Sut) : IAsyncDisposable + { + public async ValueTask DisposeAsync() + { + await TargetCtx.DisposeAsync(); + await SourceCtx.DisposeAsync(); + } + } +} diff --git a/src/Nethermind/Nethermind.EraE.Test/Nethermind.EraE.Test.csproj b/src/Nethermind/Nethermind.EraE.Test/Nethermind.EraE.Test.csproj new file mode 100644 index 000000000000..a10632ce6032 --- /dev/null +++ b/src/Nethermind/Nethermind.EraE.Test/Nethermind.EraE.Test.csproj @@ -0,0 +1,23 @@ + + + + + + enable + enable + + + + + + + + + + + + + + + + diff --git a/src/Nethermind/Nethermind.EraE.Test/Proofs/BeaconApiRootsProviderTests.cs b/src/Nethermind/Nethermind.EraE.Test/Proofs/BeaconApiRootsProviderTests.cs new file mode 100644 index 000000000000..a8e4696a7e82 --- /dev/null +++ b/src/Nethermind/Nethermind.EraE.Test/Proofs/BeaconApiRootsProviderTests.cs @@ -0,0 +1,119 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Net; +using System.Text; +using FluentAssertions; +using Nethermind.Core.Crypto; +using Nethermind.EraE.Proofs; +using NUnit.Framework; + +namespace Nethermind.EraE.Test.Proofs; + +public class BeaconApiRootsProviderTests +{ + // Production code calls FetchBlockRootAsync first (headers endpoint), then FetchStateRootAsync. + // Responses are enqueued in that order. + + private const string HeadersJson = + """{"data":{"root":"0xaabbccdd00000000000000000000000000000000000000000000000000000000"}}"""; + + private const string StateRootJson = + """{"data":{"root":"0x1122334400000000000000000000000000000000000000000000000000000000"}}"""; + + [Test] + public async Task GetBeaconRoots_WithValidSlot_ReturnsBothRoots() + { + using BeaconApiRootsProvider sut = Build( + (HttpStatusCode.OK, HeadersJson), + (HttpStatusCode.OK, StateRootJson)); + + (ValueHash256 BeaconBlockRoot, ValueHash256 StateRoot)? result = await sut.GetBeaconRoots(1); + + result.Should().NotBeNull(); + result!.Value.BeaconBlockRoot.ToByteArray()[0].Should().Be(0xaa); + result.Value.StateRoot.ToByteArray()[0].Should().Be(0x11); + } + + [Test] + public async Task GetBeaconRoots_WithSameSlotCalledTwice_MakesOnlyTwoHttpRequests() + { + // Only 2 responses enqueued — a third request would throw + SequentialHttpMessageHandler handler = new(); + handler.Enqueue(HttpStatusCode.OK, HeadersJson); + handler.Enqueue(HttpStatusCode.OK, StateRootJson); + using BeaconApiRootsProvider sut = new(new Uri("http://localhost:5052"), new HttpClient(handler)); + + await sut.GetBeaconRoots(1); + await sut.GetBeaconRoots(1); + + handler.CallCount.Should().Be(2, "cache hit on second call must skip both HTTP requests"); + } + + [Test] + public async Task GetBeaconRoots_WhenHeadersResponseMissingDataField_ReturnsNull() + { + // State root endpoint must NOT be called — handler throws on any extra request + using BeaconApiRootsProvider sut = Build((HttpStatusCode.OK, """{"other":"x"}""")); + + (ValueHash256 BeaconBlockRoot, ValueHash256 StateRoot)? result = await sut.GetBeaconRoots(1); + + result.Should().BeNull(); + } + + [Test] + public async Task GetBeaconRoots_WhenHeadersResponseIsNonSuccess_ReturnsNull() + { + using BeaconApiRootsProvider sut = Build((HttpStatusCode.NotFound, "{}")); + + (ValueHash256 BeaconBlockRoot, ValueHash256 StateRoot)? result = await sut.GetBeaconRoots(1); + + result.Should().BeNull(); + } + + [Test] + public async Task GetBeaconRoots_WhenStateRootResponseIsNonSuccess_ReturnsNull() + { + using BeaconApiRootsProvider sut = Build( + (HttpStatusCode.OK, HeadersJson), + (HttpStatusCode.NotFound, "{}")); + + (ValueHash256 BeaconBlockRoot, ValueHash256 StateRoot)? result = await sut.GetBeaconRoots(1); + + result.Should().BeNull(); + } + + private static BeaconApiRootsProvider Build(params (HttpStatusCode StatusCode, string Body)[] responses) + { + SequentialHttpMessageHandler handler = new(); + foreach ((HttpStatusCode statusCode, string body) in responses) + handler.Enqueue(statusCode, body); + return new BeaconApiRootsProvider(new Uri("http://localhost:5052"), new HttpClient(handler)); + } + + private sealed class SequentialHttpMessageHandler : HttpMessageHandler + { + private readonly Queue<(HttpStatusCode StatusCode, string Body)> _responses = new(); + + public int CallCount { get; private set; } + + public void Enqueue(HttpStatusCode statusCode, string body) => + _responses.Enqueue((statusCode, body)); + + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + CallCount++; + if (_responses.Count == 0) + throw new InvalidOperationException( + $"Unexpected HTTP request to {request.RequestUri} — no more responses queued."); + + (HttpStatusCode statusCode, string body) = _responses.Dequeue(); + HttpResponseMessage response = new(statusCode) + { + Content = new StringContent(body, Encoding.UTF8, "application/json") + }; + return Task.FromResult(response); + } + } +} diff --git a/src/Nethermind/Nethermind.EraE.Test/Proofs/BlocksRootContextTests.cs b/src/Nethermind/Nethermind.EraE.Test/Proofs/BlocksRootContextTests.cs new file mode 100644 index 000000000000..5e306602f765 --- /dev/null +++ b/src/Nethermind/Nethermind.EraE.Test/Proofs/BlocksRootContextTests.cs @@ -0,0 +1,131 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using FluentAssertions; +using Nethermind.Core; +using Nethermind.Core.Crypto; +using Nethermind.Core.Test.Builders; +using Nethermind.EraE.Proofs; +using Nethermind.Specs; +using NUnit.Framework; + +namespace Nethermind.EraE.Test.Proofs; + +public class BlocksRootContextTests +{ + [Test] + public void Constructor_WithBlockNumberBelowParis_SetsHistoricalHashesAccumulatorType() + { + using BlocksRootContext sut = new(0, specProvider: MainnetSpecProvider.Instance); + + sut.AccumulatorType.Should().Be(AccumulatorType.HistoricalHashesAccumulator); + } + + [Test] + public void Constructor_WithParisBlockNumberAndPreShanghaiTimestamp_SetsHistoricalRootsType() + { + using BlocksRootContext sut = CreateHistoricalRootsContext(); + + sut.AccumulatorType.Should().Be(AccumulatorType.HistoricalRoots); + } + + [Test] + public void Constructor_WithShanghaiBlockTimestamp_SetsHistoricalSummariesType() + { + using BlocksRootContext sut = new(MainnetSpecProvider.ParisBlockNumber + 1, MainnetSpecProvider.ShanghaiBlockTimestamp, MainnetSpecProvider.Instance); + + sut.AccumulatorType.Should().Be(AccumulatorType.HistoricalSummaries); + } + + [Test] + public void ProcessBlock_InPreMergeContext_SetsPopulated() + { + using BlocksRootContext sut = new(0, specProvider: MainnetSpecProvider.Instance); + Block block = Build.A.Block.WithNumber(0).WithTotalDifficulty(1L).TestObject; + + sut.ProcessBlock(block); + + sut.Populated.Should().BeTrue(); + } + + [Test] + public void AccumulatorRoot_WhenFinalizeContextNeverCalled_ThrowsInvalidOperationException() + { + using BlocksRootContext sut = new(0, specProvider: MainnetSpecProvider.Instance); + + sut.Invoking(c => _ = c.AccumulatorRoot).Should().Throw(); + } + + [Test] + public void AccumulatorRoot_WhenFinalizeContextCalledWithoutBlocks_ThrowsInvalidOperationException() + { + using BlocksRootContext sut = new(0, specProvider: MainnetSpecProvider.Instance); + + sut.FinalizeContext(); + + sut.Invoking(c => _ = c.AccumulatorRoot).Should().Throw(); + } + + [Test] + public void HistoricalRoot_WhenNotFinalized_ThrowsInvalidOperationException() + { + using BlocksRootContext sut = CreateHistoricalRootsContext(); + + sut.Invoking(c => _ = c.HistoricalRoot).Should().Throw(); + } + + [Test] + public void HistoricalSummary_WhenNotFinalized_ThrowsInvalidOperationException() + { + using BlocksRootContext sut = new(MainnetSpecProvider.ParisBlockNumber + 1, MainnetSpecProvider.ShanghaiBlockTimestamp, MainnetSpecProvider.Instance); + + sut.Invoking(c => _ = c.HistoricalSummary).Should().Throw(); + } + + [Test] + public void FinalizeContext_InPreMergeContext_SetsAccumulatorRoot() + { + using BlocksRootContext sut = new(0, specProvider: MainnetSpecProvider.Instance); + Block block = Build.A.Block.WithNumber(0).WithTotalDifficulty(1L).TestObject; + sut.ProcessBlock(block); + + sut.FinalizeContext(); + + sut.Invoking(c => _ = c.AccumulatorRoot).Should().NotThrow(); + } + + [Test] + public void FinalizeContext_InHistoricalRootsContext_SetsHistoricalRoot() + { + using BlocksRootContext sut = CreateHistoricalRootsContext(); + Block block = Build.A.Block.WithNumber(MainnetSpecProvider.ParisBlockNumber + 1).TestObject; + sut.ProcessBlock(block, + new ValueHash256("0xaabbccdd00000000000000000000000000000000000000000000000000000000"), + new ValueHash256("0x1122334400000000000000000000000000000000000000000000000000000000")); + + sut.FinalizeContext(); + + sut.Invoking(c => _ = c.HistoricalRoot).Should().NotThrow(); + } + + [Test] + public void FinalizeContext_InHistoricalSummariesContext_SetsHistoricalSummary() + { + using BlocksRootContext sut = new(MainnetSpecProvider.ParisBlockNumber + 1, MainnetSpecProvider.ShanghaiBlockTimestamp, MainnetSpecProvider.Instance); + Block block = Build.A.Block.WithNumber(MainnetSpecProvider.ParisBlockNumber + 1).TestObject; + sut.ProcessBlock(block, + new ValueHash256("0xaabbccdd00000000000000000000000000000000000000000000000000000000"), + new ValueHash256("0x1122334400000000000000000000000000000000000000000000000000000000")); + + sut.FinalizeContext(); + + sut.Invoking(c => _ = c.HistoricalSummary).Should().NotThrow(); + } + + private static BlocksRootContext CreateHistoricalRootsContext() + { + MainnetSpecProvider specProvider = new(); + specProvider.UpdateMergeTransitionInfo(MainnetSpecProvider.ParisBlockNumber + 1); + return new BlocksRootContext(MainnetSpecProvider.ParisBlockNumber + 1, 1_600_000_000UL, specProvider); + } +} diff --git a/src/Nethermind/Nethermind.EraE.Test/Proofs/HistoricalSummariesRpcProviderTests.cs b/src/Nethermind/Nethermind.EraE.Test/Proofs/HistoricalSummariesRpcProviderTests.cs new file mode 100644 index 000000000000..e951387d7b66 --- /dev/null +++ b/src/Nethermind/Nethermind.EraE.Test/Proofs/HistoricalSummariesRpcProviderTests.cs @@ -0,0 +1,143 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Net; +using System.Text; +using FluentAssertions; +using Nethermind.EraE.Proofs; +using NUnit.Framework; + +namespace Nethermind.EraE.Test.Proofs; + +public class HistoricalSummariesRpcProviderTests +{ + private const string TwoSummariesJson = """ + { + "data": { + "historical_summaries": [ + { + "block_summary_root": "0xaabbccdd00000000000000000000000000000000000000000000000000000000", + "state_summary_root": "0x1122334400000000000000000000000000000000000000000000000000000000" + }, + { + "block_summary_root": "0xccddee0000000000000000000000000000000000000000000000000000000000", + "state_summary_root": "0x5566770000000000000000000000000000000000000000000000000000000000" + } + ] + } + } + """; + + [Test] + public async Task GetHistoricalSummary_WithValidResponse_ReturnsExpectedSummaryAtIndex() + { + using HistoricalSummariesRpcProvider sut = Build(TwoSummariesJson); + + HistoricalSummary? result = await sut.GetHistoricalSummary(0); + + result.Should().NotBeNull(); + result!.Value.BlockSummaryRoot.ToByteArray()[0].Should().Be(0xaa); + result.Value.StateSummaryRoot.ToByteArray()[0].Should().Be(0x11); + } + + [Test] + public async Task GetHistoricalSummary_WithIndexOutOfRange_ReturnsNull() + { + using HistoricalSummariesRpcProvider sut = Build(TwoSummariesJson); + + HistoricalSummary? result = await sut.GetHistoricalSummary(999); + + result.Should().BeNull(); + } + + [Test] + public async Task GetHistoricalSummariesAsync_WithSameCallTwice_ReturnsCachedResultWithoutExtraRequest() + { + FakeHttpMessageHandler handler = new(TwoSummariesJson); + using HistoricalSummariesRpcProvider sut = new( + new Uri("http://localhost:5052"), + new HttpClient(handler)); + + await sut.GetHistoricalSummariesAsync(); + await sut.GetHistoricalSummariesAsync(); + + handler.RequestCount.Should().Be(1); + } + + [Test] + public async Task GetHistoricalSummariesAsync_WithForceRefresh_BypassesCache() + { + FakeHttpMessageHandler handler = new(TwoSummariesJson); + using HistoricalSummariesRpcProvider sut = new( + new Uri("http://localhost:5052"), + new HttpClient(handler)); + + await sut.GetHistoricalSummariesAsync(); + await sut.GetHistoricalSummariesAsync(forceRefresh: true); + + handler.RequestCount.Should().Be(2); + } + + [Test] + public async Task GetHistoricalSummariesAsync_WhenMissingDataField_ReturnsEmptyArray() + { + using HistoricalSummariesRpcProvider sut = Build("""{"other":"x"}"""); + + HistoricalSummary[] result = await sut.GetHistoricalSummariesAsync(); + + result.Should().BeEmpty(); + } + + [Test] + public async Task GetHistoricalSummariesAsync_WhenMissingHistoricalSummariesField_ReturnsEmptyArray() + { + using HistoricalSummariesRpcProvider sut = Build("""{"data":{}}"""); + + HistoricalSummary[] result = await sut.GetHistoricalSummariesAsync(); + + result.Should().BeEmpty(); + } + + [Test] + public async Task GetHistoricalSummariesAsync_WhenEntryMissingBlockSummaryRootField_SkipsEntry() + { + using HistoricalSummariesRpcProvider sut = Build(""" + { + "data": { + "historical_summaries": [ + { + "state_summary_root": "0x1122334400000000000000000000000000000000000000000000000000000000" + }, + { + "block_summary_root": "0xaabbccdd00000000000000000000000000000000000000000000000000000000", + "state_summary_root": "0x1122334400000000000000000000000000000000000000000000000000000000" + } + ] + } + } + """); + + HistoricalSummary[] result = await sut.GetHistoricalSummariesAsync(); + + result.Should().HaveCount(1); + } + + private static HistoricalSummariesRpcProvider Build(string responseBody) => + new(new Uri("http://localhost:5052"), new HttpClient(new FakeHttpMessageHandler(responseBody))); + + private sealed class FakeHttpMessageHandler(string responseBody) : HttpMessageHandler + { + public int RequestCount { get; private set; } + + protected override Task SendAsync( + HttpRequestMessage request, CancellationToken cancellationToken) + { + RequestCount++; + HttpResponseMessage response = new(HttpStatusCode.OK) + { + Content = new StringContent(responseBody, Encoding.UTF8, "application/json") + }; + return Task.FromResult(response); + } + } +} diff --git a/src/Nethermind/Nethermind.EraE.Test/Proofs/ValidatorTests.cs b/src/Nethermind/Nethermind.EraE.Test/Proofs/ValidatorTests.cs new file mode 100644 index 000000000000..052da009715e --- /dev/null +++ b/src/Nethermind/Nethermind.EraE.Test/Proofs/ValidatorTests.cs @@ -0,0 +1,69 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using FluentAssertions; +using Nethermind.Core.Crypto; +using Nethermind.Core.Specs; +using EraVerificationException = Nethermind.Era1.Exceptions.EraVerificationException; +using Nethermind.EraE.Proofs; +using NSubstitute; +using NUnit.Framework; + +namespace Nethermind.EraE.Test.Proofs; + +public class ValidatorTests +{ + [Test] + public void VerifyAccumulator_WithMatchingTrustedAccumulator_ReturnsTrue() + { + ValueHash256 root = new("0xaabbccdd00000000000000000000000000000000000000000000000000000000"); + Validator sut = BuildValidator(root); + + bool result = sut.VerifyAccumulator(0, root); + + result.Should().BeTrue(); + } + + [Test] + public void VerifyAccumulator_WithMismatchedTrustedAccumulator_ReturnsFalse() + { + ValueHash256 root = new("0xaabbccdd00000000000000000000000000000000000000000000000000000000"); + Validator sut = BuildValidator(root); + + bool result = sut.VerifyAccumulator(0, + new ValueHash256("0x1122334400000000000000000000000000000000000000000000000000000000")); + + result.Should().BeFalse(); + } + + [Test] + public void VerifyAccumulator_WhenNoTrustedAccumulatorsProvided_ReturnsTrue() + { + ISpecProvider specProvider = Substitute.For(); + specProvider.BeaconChainGenesisTimestamp.Returns((ulong?)1606824023UL); + Validator sut = new(specProvider, null, null, null); + + bool result = sut.VerifyAccumulator(0, + new ValueHash256("0xaabbccdd00000000000000000000000000000000000000000000000000000000")); + + result.Should().BeTrue(); + } + + [Test] + public void VerifyAccumulator_WhenAccumulatorNotFoundForEpoch_ThrowsEraVerificationException() + { + ValueHash256 root = new("0xaabbccdd00000000000000000000000000000000000000000000000000000000"); + Validator sut = BuildValidator(root); + + // Epoch 1 requires index 1 in the trusted set, but we only have 1 entry (index 0) + sut.Invoking(v => v.VerifyAccumulator(8192, root)) + .Should().Throw(); + } + + private static Validator BuildValidator(ValueHash256 trustedRoot) + { + ISpecProvider specProvider = Substitute.For(); + specProvider.BeaconChainGenesisTimestamp.Returns((ulong?)1606824023UL); + return new Validator(specProvider, new List { trustedRoot }, null, null); + } +} diff --git a/src/Nethermind/Nethermind.EraE.Test/ReceiptMessageDecoderSlimTests.cs b/src/Nethermind/Nethermind.EraE.Test/ReceiptMessageDecoderSlimTests.cs new file mode 100644 index 000000000000..c61bfd95c175 --- /dev/null +++ b/src/Nethermind/Nethermind.EraE.Test/ReceiptMessageDecoderSlimTests.cs @@ -0,0 +1,97 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using FluentAssertions; +using Nethermind.Core; +using Nethermind.Core.Test.Builders; +using Nethermind.Serialization.Rlp; +using NUnit.Framework; + +namespace Nethermind.EraE.Test; + +public class ReceiptMessageDecoderSlimTests +{ + private static readonly ReceiptMessageDecoder SlimDecoder = new(skipBloom: true); + private static readonly ReceiptMessageDecoder FullDecoder = new(); + + [Test] + public void Encode_WithSkipBloom_ProducesSmallerPayloadThanFull() + { + TxReceipt receipt = Build.A.Receipt + .WithTxType(TxType.EIP1559) + .WithLogs(Build.A.LogEntry.WithData([1, 2, 3]).TestObject) + .TestObject; + + byte[] slimBytes = SlimDecoder.EncodeNew(receipt, RlpBehaviors.Eip658Receipts); + byte[] fullBytes = FullDecoder.EncodeNew(receipt, RlpBehaviors.Eip658Receipts); + + slimBytes.Length.Should().BeLessThan(fullBytes.Length, + "slim receipt omits the 256-byte bloom filter"); + } + + [Test] + public void Decode_AfterSlimEncode_PreservesStatusGasAndLogs() + { + LogEntry log = Build.A.LogEntry + .WithAddress(TestItem.AddressA) + .WithData([0xDE, 0xAD, 0xBE, 0xEF]) + .TestObject; + + TxReceipt original = Build.A.Receipt + .WithTxType(TxType.EIP1559) + .WithStatusCode(1) + .WithGasUsed(21000) + .WithLogs(log) + .TestObject; + + byte[] encoded = SlimDecoder.EncodeNew(original, RlpBehaviors.Eip658Receipts); + + Rlp.ValueDecoderContext ctx = new(encoded.AsSpan()); + TxReceipt decoded = SlimDecoder.Decode(ref ctx, RlpBehaviors.Eip658Receipts)!; + + decoded.StatusCode.Should().Be(original.StatusCode); + decoded.GasUsedTotal.Should().Be(original.GasUsedTotal); + decoded.Logs!.Should().HaveCount(original.Logs!.Length); + decoded.Logs[0].Data.Should().BeEquivalentTo(original.Logs[0].Data); + } + + [Test] + public void Decode_WithSlimEncodedReceipt_BloomIsReconstructedFromLogs() + { + LogEntry log = Build.A.LogEntry + .WithAddress(TestItem.AddressA) + .WithTopics(TestItem.KeccakA) + .TestObject; + + TxReceipt original = Build.A.Receipt + .WithTxType(TxType.EIP1559) + .WithStatusCode(1) + .WithLogs(log) + .TestObject; + + byte[] encoded = SlimDecoder.EncodeNew(original, RlpBehaviors.Eip658Receipts); + Rlp.ValueDecoderContext ctx = new(encoded.AsSpan()); + TxReceipt decoded = SlimDecoder.Decode(ref ctx, RlpBehaviors.Eip658Receipts)!; + + decoded.Bloom.Should().NotBeNull(); + decoded.Bloom!.Should().Be(original.Bloom, + "bloom must be identical to what the full receipt would have"); + } + + [Test] + public void Encode_WithEmptyLogs_DecodesWithoutError() + { + TxReceipt receipt = Build.A.Receipt + .WithTxType(TxType.Legacy) + .WithLogs() + .TestObject; + + byte[] encoded = SlimDecoder.EncodeNew(receipt, RlpBehaviors.Eip658Receipts); + Rlp.ValueDecoderContext ctx = new(encoded.AsSpan()); + TxReceipt decoded = SlimDecoder.Decode(ref ctx, RlpBehaviors.Eip658Receipts)!; + + decoded.Should().NotBeNull(); + decoded.Logs.Should().NotBeNull("decoder must return an empty array, not null"); + decoded.Logs!.Should().BeEmpty(); + } +} diff --git a/src/Nethermind/Nethermind.EraE.Test/Store/EraStoreTests.cs b/src/Nethermind/Nethermind.EraE.Test/Store/EraStoreTests.cs new file mode 100644 index 000000000000..89feac6ca631 --- /dev/null +++ b/src/Nethermind/Nethermind.EraE.Test/Store/EraStoreTests.cs @@ -0,0 +1,151 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Autofac; +using FluentAssertions; +using Nethermind.Core; +using Nethermind.Core.Crypto; +using Nethermind.EraE.Config; +using EraException = Nethermind.Era1.EraException; +using Nethermind.EraE.Export; +using Nethermind.EraE.Store; +using NUnit.Framework; + +namespace Nethermind.EraE.Test.Store; + +public class EraStoreTests +{ + [TestCase(100, 16, 32)] + [TestCase(100, 16, 64)] + [TestCase(200, 50, 100)] + public async Task FirstAndLastBlock_AfterExport_MatchSpecifiedRange(int chainLength, int from, int to) + { + await using IContainer ctx = await EraETestModule.CreateExportedEraEnv(chainLength, from, to); + string tmpDirectory = ctx.ResolveTempDirPath(); + + using IEraStore eraStore = ctx.Resolve().Create(tmpDirectory, null); + + eraStore.BlockRange.Should().Be((from, to)); + } + + [Test] + public async Task FindBlockAndReceipts_WithKnownBlockNumber_ReturnsBlock() + { + await using EraStoreEnv env = await CreateDefaultEraStoreEnv(); + + (Block? block, TxReceipt[]? receipts) = await env.EraStore.FindBlockAndReceipts( + env.EraStore.BlockRange.First, ensureValidated: false); + + block.Should().NotBeNull(); + receipts.Should().NotBeNull(); + } + + [Test] + public async Task FindBlockAndReceipts_WithOutOfRangeNumber_ReturnsNull() + { + await using EraStoreEnv env = await CreateDefaultEraStoreEnv(); + + (Block? block, TxReceipt[]? receipts) = await env.EraStore.FindBlockAndReceipts(99999, ensureValidated: false); + + block.Should().BeNull(); + receipts.Should().BeNull(); + } + + [Test] + public async Task FindBlockAndReceipts_WithValidBlockNumber_ReturnsCorrectBlock() + { + await using EraStoreEnv env = await CreateDefaultEraStoreEnv(); + + long targetBlock = env.EraStore.BlockRange.First + 5; + (Block? block, _) = await env.EraStore.FindBlockAndReceipts(targetBlock, ensureValidated: false); + + block!.Number.Should().Be(targetBlock); + } + + [Test] + public async Task FindBlockAndReceipts_WithNegativeBlockNumber_ThrowsArgumentOutOfRangeException() + { + await using EraStoreEnv env = await CreateDefaultEraStoreEnv(); + + Assert.That( + async () => await env.EraStore.FindBlockAndReceipts(-1, ensureValidated: false), + Throws.TypeOf()); + } + + [Test] + public async Task NextEraStart_WhenCalled_ReturnsCorrectBoundary() + { + const int eraSize = 16; + const int chainLength = 50; + await using IContainer ctx = await EraETestModule + .BuildContainerBuilderWithBlockTreeOfLength(chainLength) + .AddSingleton(new EraEConfig { MaxEraSize = eraSize, NetworkName = EraETestModule.TestNetwork }) + .Build() + .AsTask(); + + IEraExporter exporter = ctx.Resolve(); + await exporter.Export(ctx.ResolveTempDirPath(), 0, 0); + string tmpDirectory = ctx.ResolveTempDirPath(); + + using IEraStore eraStore = ctx.Resolve().Create(tmpDirectory, null); + long nextStart = eraStore.NextEraStart(eraStore.BlockRange.First); + + nextStart.Should().Be(eraSize, "first era covers blocks 0..eraSize-1"); + } + + [Test] + public void Constructor_WithDirectoryContainingNoEraFiles_ThrowsEraException() + { + using IContainer ctx = EraETestModule.BuildContainerBuilder().Build(); + string tmpDirectory = ctx.ResolveTempDirPath(); + Directory.CreateDirectory(tmpDirectory); + + File.WriteAllText(Path.Combine(tmpDirectory, EraExporter.ChecksumsSHA256FileName), ""); + + Assert.That( + () => ctx.Resolve().Create(tmpDirectory, null), + Throws.TypeOf()); + } + + [Test] + public async Task FindBlockAndReceipts_WithValidTrustedAccumulators_Succeeds() + { + const int chainLength = 32; + await using IContainer ctx = await EraETestModule.CreateExportedEraEnv(chainLength, from: 0, to: 0); + string tmpDirectory = ctx.ResolveTempDirPath(); + + string accPath = Path.Combine(tmpDirectory, EraExporter.AccumulatorFileName); + HashSet trusted = []; + foreach (string line in await File.ReadAllLinesAsync(accPath)) + trusted.Add(EraPathUtils.ExtractHashFromChecksumEntry(line)); + + using IEraStore eraStore = ctx.Resolve().Create(tmpDirectory, trusted); + + Assert.That( + async () => await eraStore.FindBlockAndReceipts(eraStore.BlockRange.First, ensureValidated: true), + Throws.Nothing); + } + + private static async Task CreateDefaultEraStoreEnv(int chainLength = 32) + { + IContainer ctx = await EraETestModule.CreateExportedEraEnv(chainLength, from: 0, to: 0); + string tmpDirectory = ctx.ResolveTempDirPath(); + IEraStore eraStore = ctx.Resolve().Create(tmpDirectory, null); + return new EraStoreEnv(ctx, eraStore); + } +} + +file static class ContainerAsyncExtension +{ + public static Task AsTask(this IContainer container) => Task.FromResult(container); +} + +internal sealed record EraStoreEnv(IContainer Ctx, IEraStore EraStore) : IAsyncDisposable +{ + public async ValueTask DisposeAsync() + { + EraStore.Dispose(); + await Ctx.DisposeAsync(); + } +} + diff --git a/src/Nethermind/Nethermind.EraE.Test/Store/HttpRemoteEraClientIntegrationTests.cs b/src/Nethermind/Nethermind.EraE.Test/Store/HttpRemoteEraClientIntegrationTests.cs new file mode 100644 index 000000000000..a2ba597fd446 --- /dev/null +++ b/src/Nethermind/Nethermind.EraE.Test/Store/HttpRemoteEraClientIntegrationTests.cs @@ -0,0 +1,110 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Security.Cryptography; +using FluentAssertions; +using Nethermind.Core; +using Nethermind.Core.Test.IO; +using Nethermind.EraE.Store; +using NUnit.Framework; + +namespace Nethermind.EraE.Test.Store; + +/// +/// Integration tests that hit the live ethpandaops eraE server. +/// Marked [Explicit] — run manually only, never in CI. +/// They serve as a contract test: if they fail, the server format or content has changed +/// in a way that would break production import for Sepolia. +/// +[Explicit("Requires network access to data.ethpandaops.io")] +public class HttpRemoteEraClientIntegrationTests +{ + private const string SepoliaBaseUrl = "https://data.ethpandaops.io/erae/sepolia/"; + private const string ManifestFilename = "checksums_sha256.txt"; + + // Epoch 0 is the smallest file (~3.5 MB) and its filename is immutable (historical data). + private const string Epoch0Filename = "sepolia-00000-8e3e7dc9.erae"; + private const int Epoch0 = 0; + private const int MaxEraSize = 8192; + + private TempPath _downloadDir = null!; + private HttpRemoteEraClient _client = null!; + + [SetUp] + public void SetUp() + { + _downloadDir = TempPath.GetTempDirectory(); + _client = new HttpRemoteEraClient(new Uri(SepoliaBaseUrl), ManifestFilename); + } + + [TearDown] + public void TearDown() + { + _client.Dispose(); + _downloadDir.Dispose(); + } + + [Test] + public async Task FetchManifest_WithSepoliaServer_ParsesAllEntries() + { + IReadOnlyDictionary manifest = await _client.FetchManifestAsync(); + + manifest.Should().NotBeEmpty(); + manifest.Keys.Min().Should().Be(0, "epoch 0 must be the first entry"); + manifest.Keys.Should().OnlyContain(epoch => epoch >= 0); + + manifest.Should().ContainKey(Epoch0) + .WhoseValue.Filename.Should().Be(Epoch0Filename, + "epoch 0 filename is immutable — if this fails the server format has changed"); + + manifest.Values.Should().OnlyContain(entry => + entry.Sha256Hash.Length == 32, "every entry must carry a full 32-byte SHA-256 hash"); + + manifest.Values.Should().OnlyContain(entry => + entry.Filename.EndsWith(".erae"), "every entry filename must use the .erae extension"); + } + + [Test] + public async Task DownloadEpochZero_WithSepoliaServer_PassesSha256Verification() + { + IReadOnlyDictionary manifest = await _client.FetchManifestAsync(); + RemoteEraEntry epoch0Entry = manifest[Epoch0]; + + string destinationPath = Path.Join(_downloadDir.Path, epoch0Entry.Filename); + await _client.DownloadFileAsync(epoch0Entry.Filename, destinationPath); + + File.Exists(destinationPath).Should().BeTrue(); + new FileInfo(destinationPath).Length.Should().BeGreaterThan(0); + + using FileStream fs = new(destinationPath, FileMode.Open, FileAccess.Read, FileShare.Read); + byte[] actualHash = SHA256.HashData(fs); + actualHash.Should().Equal(epoch0Entry.Sha256Hash, + "SHA-256 of the downloaded file must match the server manifest — file integrity is intact"); + } + + [Test] + public async Task FindBlockAndReceipts_WithRealSepoliaEpochZero_ReturnsCorrectBlock() + { + IReadOnlyDictionary manifest = await _client.FetchManifestAsync(); + RemoteEraEntry epoch0Entry = manifest[Epoch0]; + + // Pre-download so the decorator serves from cache (avoids double download in this test) + string destinationPath = Path.Join(_downloadDir.Path, epoch0Entry.Filename); + await _client.DownloadFileAsync(epoch0Entry.Filename, destinationPath); + + using RemoteEraStoreDecorator sut = new( + localStore: null, + _client, + _downloadDir.Path, + MaxEraSize); + + // Block 1 is the first non-genesis block — epoch 0 must contain it + (Block? block, TxReceipt[]? receipts) = await sut.FindBlockAndReceipts(1, ensureValidated: false); + + block.Should().NotBeNull(); + receipts.Should().NotBeNull(); + block!.Number.Should().Be(1); + block.Hash.Should().NotBeNull("a real downloaded block must have a computed hash"); + block.ParentHash.Should().NotBeNull(); + } +} diff --git a/src/Nethermind/Nethermind.EraE.Test/Store/RemoteEraStoreDecoratorTests.cs b/src/Nethermind/Nethermind.EraE.Test/Store/RemoteEraStoreDecoratorTests.cs new file mode 100644 index 000000000000..7bf9c13907e8 --- /dev/null +++ b/src/Nethermind/Nethermind.EraE.Test/Store/RemoteEraStoreDecoratorTests.cs @@ -0,0 +1,130 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Security.Cryptography; +using Autofac; +using FluentAssertions; +using Nethermind.Core; +using Nethermind.Core.Test.Builders; +using Nethermind.Core.Test.IO; +using EraException = Nethermind.Era1.EraException; +using EraVerificationException = Nethermind.Era1.Exceptions.EraVerificationException; +using Nethermind.EraE.Export; +using Nethermind.EraE.Store; +using NSubstitute; +using NUnit.Framework; + +namespace Nethermind.EraE.Test.Store; + +public class RemoteEraStoreDecoratorTests +{ + private IRemoteEraClient _client = null!; + private TempPath _downloadDir = null!; + + [SetUp] + public void SetUp() + { + _client = Substitute.For(); + _downloadDir = TempPath.GetTempDirectory(); + } + + [TearDown] + public void TearDown() => _downloadDir.Dispose(); + + [Test] + public async Task FindBlockAndReceipts_WhenEpochMissingLocally_DownloadsAndReturnsBlock() + { + await using IContainer ctx = await EraETestModule.CreateExportedEraEnv(chainLength: 32, from: 0, to: 0); + string exportDir = ctx.ResolveTempDirPath(); + string eraFile = EraPathUtils.GetAllEraFiles(exportDir, EraETestModule.TestNetwork).First(); + string filename = Path.GetFileName(eraFile); + int epoch = ParseEpoch(filename); + byte[] sha256 = SHA256.HashData(await File.ReadAllBytesAsync(eraFile)); + + _client.FetchManifestAsync(Arg.Any()) + .Returns(new Dictionary { [epoch] = new(filename, sha256) }); + _client.DownloadFileAsync(filename, Arg.Any(), Arg.Any()) + .Returns(callInfo => CopyFile(eraFile, callInfo.ArgAt(1))); + + using RemoteEraStoreDecorator sut = new(localStore: null, _client, _downloadDir.Path, maxEraSize: 16); + + (Block? block, TxReceipt[]? receipts) = await sut.FindBlockAndReceipts(epoch * 16, ensureValidated: false); + + block.Should().NotBeNull(); + receipts.Should().NotBeNull(); + block!.Number.Should().Be(epoch * 16); + await _client.Received(1).DownloadFileAsync(filename, Arg.Any(), Arg.Any()); + } + + [Test] + public async Task FindBlockAndReceipts_WhenLocalStoreReturnsBlock_SkipsRemoteClient() + { + Block expectedBlock = Build.A.Block.WithNumber(42).TestObject; + TxReceipt[] expectedReceipts = []; + + IEraStore localStore = Substitute.For(); + localStore.FindBlockAndReceipts(42, Arg.Any(), Arg.Any()) + .Returns((expectedBlock, (TxReceipt[]?)expectedReceipts)); + + using RemoteEraStoreDecorator sut = new(localStore, _client, _downloadDir.Path, maxEraSize: 16); + + (Block? block, TxReceipt[]? _) = await sut.FindBlockAndReceipts(42, ensureValidated: false); + + block.Should().BeSameAs(expectedBlock); + await _client.DidNotReceive().FetchManifestAsync(Arg.Any()); + } + + [Test] + public async Task FindBlockAndReceipts_WhenEpochAbsentFromManifest_ThrowsEraException() + { + IEraStore localStore = Substitute.For(); + localStore.FindBlockAndReceipts(Arg.Any(), Arg.Any(), Arg.Any()) + .Returns((null, null)); + + _client.FetchManifestAsync(Arg.Any()) + .Returns(new Dictionary()); + + using RemoteEraStoreDecorator sut = new(localStore, _client, _downloadDir.Path, maxEraSize: 16); + + await sut.Invoking(s => s.FindBlockAndReceipts(5, ensureValidated: false)) + .Should().ThrowAsync() + .WithMessage("*Epoch 0*not available*"); + } + + [Test] + public async Task FindBlockAndReceipts_WhenDownloadedFileHasWrongChecksum_ThrowsAndDeletesFile() + { + string filename = "abc-00000-deadbeef.erae"; + byte[] wrongHash = new byte[32]; + + _client.FetchManifestAsync(Arg.Any()) + .Returns(new Dictionary { [0] = new(filename, wrongHash) }); + _client.DownloadFileAsync(filename, Arg.Any(), Arg.Any()) + .Returns(callInfo => + { + File.WriteAllBytes(callInfo.ArgAt(1), [1, 2, 3, 4]); + return Task.CompletedTask; + }); + + string expectedFilePath = Path.Join(_downloadDir.Path, filename); + using RemoteEraStoreDecorator sut = new(localStore: null, _client, _downloadDir.Path, maxEraSize: 16); + + await sut.Invoking(s => s.FindBlockAndReceipts(0, ensureValidated: false)) + .Should().ThrowAsync() + .WithMessage("*SHA-256*"); + + File.Exists(expectedFilePath).Should().BeFalse(); + } + + private static int ParseEpoch(string filename) + { + string[] parts = Path.GetFileNameWithoutExtension(filename).Split('-'); + return int.Parse(parts[1]); + } + + private static Task CopyFile(string source, string destination) + { + File.Copy(source, destination); + return Task.CompletedTask; + } +} diff --git a/src/Nethermind/Nethermind.EraE.Test/TestContainerExtensions.cs b/src/Nethermind/Nethermind.EraE.Test/TestContainerExtensions.cs new file mode 100644 index 000000000000..fb8ac96bfd91 --- /dev/null +++ b/src/Nethermind/Nethermind.EraE.Test/TestContainerExtensions.cs @@ -0,0 +1,16 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Autofac; +using Nethermind.Core.Test.IO; + +namespace Nethermind.EraE.Test; + +public static class TestContainerExtensions +{ + public static string ResolveTempFilePath(this IContainer container) => + container.ResolveNamed("file").Path; + + public static string ResolveTempDirPath(this IContainer container) => + container.ResolveNamed("directory").Path; +} diff --git a/src/Nethermind/Nethermind.EraE.Test/TestEraFile.cs b/src/Nethermind/Nethermind.EraE.Test/TestEraFile.cs new file mode 100644 index 000000000000..cf1e81bdae30 --- /dev/null +++ b/src/Nethermind/Nethermind.EraE.Test/TestEraFile.cs @@ -0,0 +1,66 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Blockchain.Receipts; +using Nethermind.Core; +using Nethermind.Core.Crypto; +using Nethermind.Core.Specs; +using Nethermind.Core.Test.Builders; +using Nethermind.Core.Test.IO; +using Nethermind.Int256; +using Nethermind.Serialization.Rlp; +using Nethermind.Specs; +using EraWriter = Nethermind.EraE.Archive.EraWriter; + +namespace Nethermind.EraE.Test; + +internal sealed class TestEraFile : IDisposable +{ + private readonly TempPath _tmpFile; + public string FilePath => _tmpFile.Path; + public List<(Block Block, TxReceipt[] Receipts)> Contents { get; } = []; + + private TestEraFile(TempPath tmpFile) => _tmpFile = tmpFile; + + public static async Task Create( + int preMergeCount, + int postMergeCount, + ISpecProvider? specProvider = null) + { + specProvider ??= MainnetSpecProvider.Instance; + TempPath tmpFile = TempPath.GetTempFile(); + using EraWriter writer = new(tmpFile.Path, specProvider); + TestEraFile file = new(tmpFile); + HeaderDecoder headerDecoder = new(); + + long number = 0; + UInt256 td = BlockHeaderBuilder.DefaultDifficulty; + + for (int i = 0; i < preMergeCount; i++, number++, td += BlockHeaderBuilder.DefaultDifficulty) + { + TxReceipt receipt = Build.A.Receipt.WithTxType(TxType.EIP1559).TestObject; + Block block = Build.A.Block.WithNumber(number).WithTotalDifficulty(td).TestObject; + block.Header.ReceiptsRoot = ReceiptsRootCalculator.Instance.GetReceiptsRoot( + [receipt], specProvider.GetSpec(block.Header), block.ReceiptsRoot); + block.Header.Hash = Keccak.Compute(headerDecoder.Encode(block.Header).Bytes); + file.Contents.Add((block, [receipt])); + await writer.Add(block, [receipt]); + } + + for (int i = 0; i < postMergeCount; i++, number++) + { + TxReceipt receipt = Build.A.Receipt.WithTxType(TxType.EIP1559).TestObject; + Block block = Build.A.Block.WithNumber(number).WithPostMergeRules().TestObject; + block.Header.ReceiptsRoot = ReceiptsRootCalculator.Instance.GetReceiptsRoot( + [receipt], specProvider.GetSpec(block.Header), block.ReceiptsRoot); + block.Header.Hash = Keccak.Compute(headerDecoder.Encode(block.Header).Bytes); + file.Contents.Add((block, [receipt])); + await writer.Add(block, [receipt]); + } + + await writer.Finalize(); + return file; + } + + public void Dispose() => _tmpFile.Dispose(); +} diff --git a/src/Nethermind/Nethermind.EraE/Admin/AdminEraService.cs b/src/Nethermind/Nethermind.EraE/Admin/AdminEraService.cs new file mode 100644 index 000000000000..e4aebd3d6db2 --- /dev/null +++ b/src/Nethermind/Nethermind.EraE/Admin/AdminEraService.cs @@ -0,0 +1,53 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Config; +using Nethermind.EraE.Export; +using Nethermind.EraE.Import; +using Nethermind.JsonRpc; +using Nethermind.Logging; + +namespace Nethermind.EraE.Admin; + +public sealed class AdminEraService( + IEraImporter eraImporter, + IEraExporter eraExporter, + IProcessExitSource processExit, + ILogManager logManager) : IAdminEraService +{ + private readonly ILogger _logger = logManager.GetClassLogger(); + private int _canEnterImport = 1; + private int _canEnterExport = 1; + + public ResultWrapper ExportHistory(string destination, long from, long to) + { + if (Interlocked.Exchange(ref _canEnterExport, 0) != 1) + return ResultWrapper.Fail("An export job is already running."); + + _ = EraJobRunner.RunProtected( + ct => eraExporter.Export(destination, from, to, ct), + $"EraE export was cancelled. Archives in '{destination}' may be incomplete.", + "EraE export error", + _logger, + processExit.Token, + () => Interlocked.Exchange(ref _canEnterExport, 1)); + + return ResultWrapper.Success("Started EraE export task."); + } + + public ResultWrapper ImportHistory(string source, long from, long to, string? accumulatorFile) + { + if (Interlocked.Exchange(ref _canEnterImport, 0) != 1) + return ResultWrapper.Fail("An import job is already running."); + + _ = EraJobRunner.RunProtected( + ct => eraImporter.Import(source, from, to, accumulatorFile, ct), + $"EraE import was cancelled. State from '{source}' may be incomplete.", + "EraE import error", + _logger, + processExit.Token, + () => Interlocked.Exchange(ref _canEnterImport, 1)); + + return ResultWrapper.Success("Started EraE import task."); + } +} diff --git a/src/Nethermind/Nethermind.EraE/Admin/IAdminEraService.cs b/src/Nethermind/Nethermind.EraE/Admin/IAdminEraService.cs new file mode 100644 index 000000000000..675d1b531c6a --- /dev/null +++ b/src/Nethermind/Nethermind.EraE/Admin/IAdminEraService.cs @@ -0,0 +1,12 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.JsonRpc; + +namespace Nethermind.EraE.Admin; + +public interface IAdminEraService +{ + ResultWrapper ExportHistory(string destination, long from, long to); + ResultWrapper ImportHistory(string source, long from, long to, string? accumulatorFile); +} diff --git a/src/Nethermind/Nethermind.EraE/Archive/EraReader.cs b/src/Nethermind/Nethermind.EraE/Archive/EraReader.cs new file mode 100644 index 000000000000..66536c698b70 --- /dev/null +++ b/src/Nethermind/Nethermind.EraE/Archive/EraReader.cs @@ -0,0 +1,195 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Collections.Concurrent; +using Nethermind.Consensus.Validators; +using Nethermind.Core; +using Nethermind.Core.Crypto; +using Nethermind.Core.Extensions; +using Nethermind.Core.Specs; +using Nethermind.EraE.E2Store; +using AccumulatorCalculator = Nethermind.Era1.AccumulatorCalculator; +using EraException = Nethermind.Era1.EraException; +using EraVerificationException = Nethermind.Era1.Exceptions.EraVerificationException; +using Nethermind.EraE.Proofs; +using Nethermind.Int256; +using Nethermind.Serialization.Rlp; +using Nethermind.State.Proofs; + +namespace Nethermind.EraE.Archive; + +public sealed class EraReader(E2StoreReader e2) : IAsyncEnumerable<(Block, TxReceipt[])>, IDisposable +{ + private readonly EraSlimReceiptDecoder _slimReceiptDecoder = new(); + private readonly ReceiptMessageDecoder _fullReceiptDecoder = new(); + private readonly BlockBodyDecoder _blockBodyDecoder = BlockBodyDecoder.Instance; + private readonly HeaderDecoder _headerDecoder = new(); + + public long FirstBlock => e2.First; + public long LastBlock => e2.LastBlock; + + public EraReader(string fileName) : this(new E2StoreReader(fileName)) { } + + public async IAsyncEnumerator<(Block, TxReceipt[])> GetAsyncEnumerator(CancellationToken cancellation = default) + { + for (long blockNumber = e2.First; blockNumber <= e2.LastBlock; blockNumber++) + { + (Block block, TxReceipt[] receipts) = await ReadBlockAndReceipts(blockNumber, cancellation); + yield return (block, receipts); + } + } + + public async Task<(Block, TxReceipt[])> GetBlockByNumber(long number, CancellationToken cancellation = default) + { + if (number < e2.First) + throw new ArgumentOutOfRangeException(nameof(number), $"Cannot be less than first block {e2.First}."); + if (number > e2.LastBlock) + throw new ArgumentOutOfRangeException(nameof(number), $"Cannot exceed last block {e2.LastBlock}."); + + return await ReadBlockAndReceipts(number, cancellation); + } + + public ValueHash256 ReadAccumulatorRoot() + { + long offset = e2.AccumulatorRootOffset; + if (offset < 0) + throw new EraException("This EraE file does not contain an AccumulatorRoot (post-merge epoch)."); + + e2.ReadEntryAndDecode( + offset, + static buffer => new ValueHash256(buffer.Span), + EntryTypes.AccumulatorRoot, + out ValueHash256 root); + + return root; + } + + public async Task VerifyContent( + ISpecProvider specProvider, + IBlockValidator blockValidator, + int verifyConcurrency = 0, + Validator? validator = null, + CancellationToken cancellation = default) + { + ArgumentNullException.ThrowIfNull(specProvider); + ArgumentNullException.ThrowIfNull(blockValidator); + + if (verifyConcurrency <= 0) + verifyConcurrency = Environment.ProcessorCount; + + long startBlock = e2.First; + int blockCount = (int)e2.BlockCount; + + (Hash256 Hash, UInt256 Td, bool IsPreMerge)[] blockMeta = + new (Hash256 Hash, UInt256 Td, bool IsPreMerge)[blockCount]; + + ConcurrentQueue blockNumbers = new(); + for (long n = startBlock; n <= e2.LastBlock; n++) + { + blockNumbers.Enqueue(n); + } + + Task[] workers = new Task[verifyConcurrency]; + for (int i = 0; i < verifyConcurrency; i++) + { + workers[i] = Task.Run(async () => + { + while (blockNumbers.TryDequeue(out long blockNumber)) + { + (Block block, TxReceipt[] receipts) = await ReadBlockAndReceipts(blockNumber, cancellation); + + if (!blockValidator.ValidateBodyAgainstHeader(block.Header, block.Body, out string? error)) + throw new EraVerificationException($"Mismatched block body against header: {error}. Block {blockNumber}."); + + Hash256 receiptRoot = ReceiptTrie.CalculateRoot( + specProvider.GetReceiptSpec(block.Number), receipts, _fullReceiptDecoder); + if (block.Header.ReceiptsRoot != receiptRoot) + throw new EraVerificationException($"Mismatched receipt root at block {blockNumber}."); + + // Safe: each dequeued blockNumber is unique, so each idx is written by exactly one worker. + int idx = (int)(block.Header.Number - startBlock); + blockMeta[idx] = ( + block.Header.Hash!, + block.TotalDifficulty ?? UInt256.Zero, + !block.Header.IsPostMerge); + } + }, cancellation); + } + + await Task.WhenAll(workers); + + // Accumulator verification applies to pre-merge and transition epochs only. + // Post-merge-only epochs have no AccumulatorRoot entry. + // TODO: verify HistoricalRoots / HistoricalSummaries beacon proofs for post-merge epochs. + if (!e2.HasTotalDifficulty) + return default; + + ValueHash256 storedRoot = ReadAccumulatorRoot(); + + using AccumulatorCalculator calculator = new(); + foreach ((Hash256 hash, UInt256 td, bool isPreMerge) in blockMeta) + { + if (!isPreMerge) continue; // post-merge blocks excluded from accumulator even in transition epochs + calculator.Add(hash, td); + } + + ValueHash256 computedRoot = calculator.ComputeRoot(); + if (computedRoot != storedRoot) + throw new EraVerificationException("Computed accumulator root does not match stored AccumulatorRoot."); + + // Optional chain-integrity check: verify stored root against externally trusted accumulators. + if (validator is not null && !validator.VerifyAccumulator(startBlock, storedRoot)) + throw new EraVerificationException("Stored AccumulatorRoot does not match trusted accumulator."); + + return storedRoot; + } + + public ValueHash256 CalculateChecksum() => e2.CalculateChecksum(); + + public void Dispose() => e2.Dispose(); + + private async Task<(Block, TxReceipt[])> ReadBlockAndReceipts(long blockNumber, CancellationToken cancellation) + { + long headerPos = e2.HeaderOffset(blockNumber); + (BlockHeader header, _) = await e2.ReadSnappyCompressedEntryAndDecode( + headerPos, DecodeHeader, EntryTypes.CompressedHeader, cancellation); + + // IsPostMerge is not RLP-serialized; restore it from Difficulty after decode. + // Pre-merge blocks have Difficulty > 0, post-merge blocks have Difficulty == 0 (EIP-3675). + header.IsPostMerge = header.Difficulty == 0; + + long bodyPos = e2.BodyOffset(blockNumber); + (BlockBody body, _) = await e2.ReadSnappyCompressedEntryAndDecode( + bodyPos, DecodeBody, EntryTypes.CompressedBody, cancellation); + + long receiptsPos = e2.SlimReceiptsOffset(blockNumber); + (TxReceipt[] receipts, _) = await e2.ReadSnappyCompressedEntryAndDecode( + receiptsPos, _slimReceiptDecoder.Decode, EntryTypes.CompressedSlimReceipts, cancellation); + + if (e2.HasTotalDifficulty) + { + long tdPos = e2.TotalDifficultyOffset(blockNumber); + e2.ReadEntryAndDecode( + tdPos, + static buf => new UInt256(buf.Span, isBigEndian: false), + EntryTypes.TotalDifficulty, + out UInt256 td); + header.TotalDifficulty = td; + } + + // Bloom is not stored in slim receipts; TxReceipt.Bloom auto-calculates from Logs on first access. + return (new Block(header, body), receipts); + } + + private BlockHeader DecodeHeader(Memory buffer) + { + Rlp.ValueDecoderContext ctx = new(buffer.Span); + return _headerDecoder.Decode(ref ctx)!; + } + + private BlockBody DecodeBody(Memory buffer) + { + Rlp.ValueDecoderContext ctx = new(buffer.Span); + return _blockBodyDecoder.Decode(ref ctx)!; + } +} diff --git a/src/Nethermind/Nethermind.EraE/Archive/EraSlimReceiptDecoder.cs b/src/Nethermind/Nethermind.EraE/Archive/EraSlimReceiptDecoder.cs new file mode 100644 index 000000000000..e3f56bd0439b --- /dev/null +++ b/src/Nethermind/Nethermind.EraE/Archive/EraSlimReceiptDecoder.cs @@ -0,0 +1,89 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Core; +using Nethermind.Core.Crypto; +using Nethermind.Serialization.Rlp; + +namespace Nethermind.EraE.Archive; + +internal sealed class EraSlimReceiptDecoder +{ + public TxReceipt[] Decode(Memory buffer) + { + Rlp.ValueDecoderContext ctx = new(buffer.Span); + + int outerLength = ctx.ReadSequenceLength(); + int outerEnd = ctx.Position + outerLength; + int count = ctx.PeekNumberOfItemsRemaining(outerEnd); + + TxReceipt[] receipts = new TxReceipt[count]; + for (int i = 0; i < count; i++) + { + receipts[i] = DecodeSlimReceipt(ref ctx); + } + + if (ctx.Position != outerEnd) + throw new RlpException("Receipt list was not fully consumed."); + + return receipts; + } + + private static TxReceipt DecodeSlimReceipt(ref Rlp.ValueDecoderContext ctx) + { + int sequenceLength = ctx.ReadSequenceLength(); + int receiptEnd = ctx.Position + sequenceLength; + + TxReceipt receipt = new(); + + byte[] txTypeBytes = ctx.DecodeByteArray(); + if (txTypeBytes.Length > 1) + throw new RlpException($"Invalid slim receipt tx_type length: {txTypeBytes.Length}"); + + receipt.TxType = txTypeBytes.Length == 0 + ? TxType.Legacy + : (TxType)txTypeBytes[0]; + + byte[] statusBytes = ctx.DecodeByteArray(); + switch (statusBytes.Length) + { + case 0: + receipt.StatusCode = 0; + break; + + case 1: + receipt.StatusCode = statusBytes[0]; + break; + + case Keccak.Size: + receipt.PostTransactionState = new Hash256(statusBytes); + break; + + default: + throw new RlpException( + $"Invalid slim receipt status encoding length: {statusBytes.Length}"); + } + + receipt.GasUsedTotal = ctx.DecodePositiveLong(); + + int logsLength = ctx.ReadSequenceLength(); + int logsEnd = ctx.Position + logsLength; + int logCount = ctx.PeekNumberOfItemsRemaining(logsEnd); + + LogEntry[] logs = new LogEntry[logCount]; + for (int i = 0; i < logCount; i++) + { + logs[i] = Rlp.Decode(ref ctx); + } + + if (ctx.Position != logsEnd) + throw new RlpException("Slim receipt logs list was not fully consumed."); + + receipt.Logs = logs; + + if (ctx.Position != receiptEnd) + throw new RlpException("Slim receipt sequence was not fully consumed."); + + return receipt; + } +} diff --git a/src/Nethermind/Nethermind.EraE/Archive/EraWriter.cs b/src/Nethermind/Nethermind.EraE/Archive/EraWriter.cs new file mode 100644 index 000000000000..4c0d5e906aa1 --- /dev/null +++ b/src/Nethermind/Nethermind.EraE/Archive/EraWriter.cs @@ -0,0 +1,369 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Buffers; +using System.Buffers.Binary; +using System.Diagnostics; +using System.IO.Compression; +using Microsoft.IO; +using Nethermind.Core; +using Nethermind.Core.Collections; +using Nethermind.Core.Crypto; +using Nethermind.Core.Resettables; +using Nethermind.Core.Specs; +using Nethermind.Era1; +using Nethermind.EraE.E2Store; +using Nethermind.EraE.Proofs; +using Nethermind.Int256; +using Nethermind.Serialization.Rlp; +using Snappier; +using Timestamper = Nethermind.Core.Timestamper; + +namespace Nethermind.EraE.Archive; + +/// +/// Writes an EraE archive file. +/// Each call to encodes and buffers the block's header, body, and receipts. +/// writes the buffered components in EraE section order: +/// Version | CompressedHeader* | CompressedBody* | CompressedSlimReceipts* | +/// TotalDifficulty* | AccumulatorRoot? | ComponentIndex. +/// +public sealed class EraWriter : IDisposable +{ + public const int MaxEraSize = 8192; + + private const long MillisecondsPerSecond = 1000; + private const int BeaconSlotSeconds = 12; + private const int IndexFieldSize = 8; // sizeof(long) — each ComponentIndex field is a little-endian int64 + + private readonly HeaderDecoder _headerDecoder = new(); + private readonly BlockBodyDecoder _blockBodyDecoder = new(); + private readonly E2StoreWriter _e2StoreWriter; + private readonly ISpecProvider _specProvider; + private readonly IBeaconRootsProvider? _beaconRootsProvider; + private readonly SlotTime? _slotTime; + + // Buffered per-block RLP payloads. These are written in section order during Finalize(). + private readonly ArrayPoolList _headers = new(MaxEraSize); + private readonly ArrayPoolList _bodies = new(MaxEraSize); + private readonly ArrayPoolList _receipts = new(MaxEraSize); + + // Per-block byte offsets recorded during Finalize() and written into the ComponentIndex. + // Each stores the absolute file position of the entry's TLV header. + private readonly ArrayPoolList _headerOffsets = new(MaxEraSize); + private readonly ArrayPoolList _bodyOffsets = new(MaxEraSize); + private readonly ArrayPoolList _receiptsOffsets = new(MaxEraSize); + + private readonly ArrayPoolList _totalDifficulties = new(MaxEraSize); + + private long _startNumber; + private bool _firstBlock = true; + private bool _finalized; + private int _preMergeBlockCount; + private bool _hasPostMergeBlocks; + private UInt256 _lastPreMergeTD; + private BlocksRootContext? _blocksRootContext; + + public EraWriter(string path, ISpecProvider specProvider, IBeaconRootsProvider? beaconRootsProvider = null) + : this(new E2StoreWriter(new FileStream(path, FileMode.Create)), specProvider, beaconRootsProvider) + { + } + + public EraWriter(Stream outputStream, ISpecProvider specProvider, IBeaconRootsProvider? beaconRootsProvider = null) + : this(new E2StoreWriter(outputStream), specProvider, beaconRootsProvider) + { + } + + private EraWriter(E2StoreWriter e2StoreWriter, ISpecProvider specProvider, IBeaconRootsProvider? beaconRootsProvider) + { + _e2StoreWriter = e2StoreWriter; + _specProvider = specProvider; + _beaconRootsProvider = beaconRootsProvider; + + if (beaconRootsProvider is not null && specProvider.BeaconChainGenesisTimestamp.HasValue) + { + _slotTime = new SlotTime( + specProvider.BeaconChainGenesisTimestamp.Value * MillisecondsPerSecond, + new Timestamper(), + TimeSpan.FromSeconds(BeaconSlotSeconds), + TimeSpan.Zero); + } + } + + public async Task Add(Block block, TxReceipt[] receipts, CancellationToken cancellation = default) + { + ArgumentNullException.ThrowIfNull(block); + ArgumentNullException.ThrowIfNull(receipts); + + if (_finalized) + throw new EraException("Finalize() has been called; no more blocks can be added."); + ArgumentNullException.ThrowIfNull(block.Header); + ArgumentNullException.ThrowIfNull(block.Hash); + if (_headers.Count >= MaxEraSize) + throw new ArgumentException($"Era file cannot contain more than {MaxEraSize} blocks."); + + if (_firstBlock) + { + _startNumber = block.Number; + _blocksRootContext = new BlocksRootContext(block.Number, block.Header.Timestamp, _specProvider); + _firstBlock = false; + await _e2StoreWriter.WriteEntry(EntryTypes.Version, Memory.Empty, cancellation); + } + else if (block.Number != _startNumber + _headers.Count) + { + throw new ArgumentException( + $"Blocks must be added in sequential order. Expected block {_startNumber + _headers.Count}, got {block.Number}.", + nameof(block)); + } + + IReleaseSpec spec = _specProvider.GetSpec(block.Header); + bool isPostMerge = block.Header.IsPostMerge; + + if (!isPostMerge) + { + if (block.TotalDifficulty is null) + throw new ArgumentException("Pre-merge block must have TotalDifficulty set.", nameof(block)); + if (block.TotalDifficulty < block.Difficulty) + throw new ArgumentOutOfRangeException(nameof(block.TotalDifficulty), "Cannot be less than block difficulty."); + } + + RlpBehaviors rlpBehaviors = spec.IsEip658Enabled ? RlpBehaviors.Eip658Receipts : RlpBehaviors.None; + + using (NettyRlpStream headerRlp = _headerDecoder.EncodeToNewNettyStream(block.Header, rlpBehaviors)) + { + _headers.Add(headerRlp.AsMemory().ToArray()); + } + + using (NettyRlpStream bodyRlp = _blockBodyDecoder.EncodeToNewNettyStream(block.Body, rlpBehaviors)) + { + _bodies.Add(bodyRlp.AsMemory().ToArray()); + } + + _receipts.Add(EncodeSlimReceipts(receipts, spec.IsEip658Enabled)); + + if (isPostMerge && _beaconRootsProvider is not null && _slotTime is not null) + { + long slot = (long)_slotTime.GetSlot(block.Header.Timestamp * MillisecondsPerSecond); + (ValueHash256 beaconBlockRoot, ValueHash256 stateRoot)? roots = + await _beaconRootsProvider.GetBeaconRoots(slot, cancellation); + _blocksRootContext!.ProcessBlock(block, roots?.beaconBlockRoot, roots?.stateRoot); + } + else + { + _blocksRootContext!.ProcessBlock(block); + } + + if (!isPostMerge) + { + _totalDifficulties.Add(block.TotalDifficulty!.Value); + _lastPreMergeTD = block.TotalDifficulty.Value; + _preMergeBlockCount++; + } + else + { + _totalDifficulties.Add(UInt256.Zero); + _hasPostMergeBlocks = true; + } + } + + public async Task<(ValueHash256 AccumulatorRoot, ValueHash256 Checksum)> Finalize(CancellationToken cancellation = default) + { + if (_firstBlock) + throw new EraException("No blocks have been added."); + if (_finalized) + throw new EraException("Finalize has already been called."); + + _blocksRootContext!.FinalizeContext(); + + bool isTransitionEpoch = _preMergeBlockCount > 0 && _hasPostMergeBlocks; + bool needsTd = _preMergeBlockCount > 0; + int componentCount = needsTd ? 4 : 3; + int blockCount = _headers.Count; + + if (isTransitionEpoch) + { + for (int i = _preMergeBlockCount; i < blockCount; i++) + _totalDifficulties[i] = _lastPreMergeTD; + } + + long[] tdOffsets = needsTd ? ArrayPool.Shared.Rent(blockCount) : []; + try + { + // Write sections in EraE spec order: + // Version | Header* | Body* | Receipts* | TD* | Accumulator? | ComponentIndex + + for (int i = 0; i < blockCount; i++) + { + _headerOffsets.Add(_e2StoreWriter.Position); + await WriteCompressed(EntryTypes.CompressedHeader, _headers[i], cancellation); + } + + for (int i = 0; i < blockCount; i++) + { + _bodyOffsets.Add(_e2StoreWriter.Position); + await WriteCompressed(EntryTypes.CompressedBody, _bodies[i], cancellation); + } + + for (int i = 0; i < blockCount; i++) + { + _receiptsOffsets.Add(_e2StoreWriter.Position); + await WriteCompressed(EntryTypes.CompressedSlimReceipts, _receipts[i], cancellation); + } + + if (needsTd) + { + for (int i = 0; i < blockCount; i++) + { + tdOffsets[i] = _e2StoreWriter.Position; + await _e2StoreWriter.WriteEntry( + EntryTypes.TotalDifficulty, + _totalDifficulties[i].ToLittleEndian(), + cancellation); + } + } + + ValueHash256 accumulatorRoot = default; + if (needsTd) + { + accumulatorRoot = _blocksRootContext!.AccumulatorRoot; + await _e2StoreWriter.WriteEntry(EntryTypes.AccumulatorRoot, accumulatorRoot.ToByteArray(), cancellation); + } + + // ComponentIndex + // Layout: starting_number | [header_off, body_off, receipts_off, [td_off]] * N | component_count | block_count + // Offsets are negative int64 LE, relative to start of the ComponentIndex TLV (including 8-byte header). + long componentIndexStart = _e2StoreWriter.Position; + int indexDataLength = IndexFieldSize + blockCount * componentCount * IndexFieldSize + IndexFieldSize + IndexFieldSize; + + using ArrayPoolList indexBytes = new(indexDataLength, indexDataLength); + Span span = indexBytes.AsSpan(); + + WriteInt64(span, 0, _startNumber); + + for (int i = 0; i < blockCount; i++) + { + int baseOff = IndexFieldSize + i * componentCount * IndexFieldSize; + WriteInt64(span, baseOff + IndexFieldSize * 0, _headerOffsets[i] - componentIndexStart); + WriteInt64(span, baseOff + IndexFieldSize * 1, _bodyOffsets[i] - componentIndexStart); + WriteInt64(span, baseOff + IndexFieldSize * 2, _receiptsOffsets[i] - componentIndexStart); + if (needsTd) + WriteInt64(span, baseOff + IndexFieldSize * 3, tdOffsets[i] - componentIndexStart); + } + + int tailOff = IndexFieldSize + blockCount * componentCount * IndexFieldSize; + WriteInt64(span, tailOff, componentCount); + WriteInt64(span, tailOff + IndexFieldSize, blockCount); + + await _e2StoreWriter.WriteEntry(EntryTypes.ComponentIndex, indexBytes.AsMemory(), cancellation); + await _e2StoreWriter.Flush(cancellation); + + _finalized = true; + return (accumulatorRoot, _e2StoreWriter.FinalizeChecksum()); + } + finally + { + if (needsTd) + ArrayPool.Shared.Return(tdOffsets); + } + } + + public void Dispose() + { + _blocksRootContext?.Dispose(); + _e2StoreWriter?.Dispose(); + _headers.Dispose(); + _bodies.Dispose(); + _receipts.Dispose(); + _headerOffsets.Dispose(); + _bodyOffsets.Dispose(); + _receiptsOffsets.Dispose(); + _totalDifficulties.Dispose(); + } + + /// + /// Encodes receipts in the go-ethereum slim format used by EraE files: + /// rlp([txType, postStateOrStatus, cumulativeGas, logs]) per receipt, no bloom. + /// txType is empty-bytes for legacy (type 0), single byte otherwise. + /// postStateOrStatus is the 32-byte state root pre-EIP-658, or 0x01/empty for success/failure post-EIP-658. + /// + private static byte[] EncodeSlimReceipts(TxReceipt[] receipts, bool isEip658) + { + int totalLength = 0; + foreach (TxReceipt receipt in receipts) + totalLength += Rlp.LengthOfSequence(GetReceiptContentLength(receipt, isEip658)); + + RlpStream stream = new(Rlp.LengthOfSequence(totalLength)); + stream.StartSequence(totalLength); + foreach (TxReceipt receipt in receipts) + WriteReceipt(stream, receipt, isEip658); + return stream.Data.ToArray() ?? []; + } + + private static int GetReceiptContentLength(TxReceipt receipt, bool isEip658) + { + int logsLength = 0; + if (receipt.Logs is not null) + foreach (LogEntry log in receipt.Logs) + logsLength += LogEntryDecoder.Instance.GetLength(log); + + // txType: 0x80 (empty = legacy) or single self-encoding byte (type 1/2/3) — always 1 byte + int statusLength = isEip658 ? 1 : Rlp.LengthOf(receipt.PostTransactionState); + + return 1 + statusLength + Rlp.LengthOf(receipt.GasUsedTotal) + Rlp.LengthOfSequence(logsLength); + } + + private static void WriteReceipt(RlpStream stream, TxReceipt receipt, bool isEip658) + { + int logsLength = 0; + if (receipt.Logs is not null) + { + foreach (LogEntry log in receipt.Logs) + logsLength += LogEntryDecoder.Instance.GetLength(log); + } + + int statusLength = isEip658 ? 1 : Rlp.LengthOf(receipt.PostTransactionState); + int contentLength = 1 + statusLength + Rlp.LengthOf(receipt.GasUsedTotal) + Rlp.LengthOfSequence(logsLength); + + stream.StartSequence(contentLength); + + // TxType: empty byte array for legacy, single byte for typed (EIP-2718) + if (receipt.TxType == TxType.Legacy) + stream.Encode(Array.Empty()); + else + stream.WriteByte((byte)receipt.TxType); + + // postStateOrStatus: 32-byte hash (pre-EIP-658), 0x01 (success), or empty (failure) + if (!isEip658) + stream.Encode(receipt.PostTransactionState); + else if (receipt.StatusCode == 0) + stream.Encode(Array.Empty()); + else + stream.WriteByte(receipt.StatusCode); + + stream.Encode(receipt.GasUsedTotal); + + stream.StartSequence(logsLength); + if (receipt.Logs is not null) + { + foreach (LogEntry log in receipt.Logs) + LogEntryDecoder.Instance.Encode(stream, log); + } + } + + private async Task WriteCompressed(ushort entryType, ReadOnlyMemory data, CancellationToken cancellation) + { + await using RecyclableMemoryStream ms = RecyclableStream.GetStream(nameof(EraWriter)); + await using (SnappyStream compressor = new(ms, CompressionMode.Compress, leaveOpen: true)) + { + compressor.Write(data.Span); + await compressor.FlushAsync(cancellation); + } + + bool ok = ms.TryGetBuffer(out ArraySegment segment); + Debug.Assert(ok); + await _e2StoreWriter.WriteEntry(entryType, segment.AsMemory(), cancellation); + } + + private static void WriteInt64(Span destination, int off, long value) => + BinaryPrimitives.WriteInt64LittleEndian(destination.Slice(off, IndexFieldSize), value); +} diff --git a/src/Nethermind/Nethermind.EraE/Config/EraEConfig.cs b/src/Nethermind/Nethermind.EraE/Config/EraEConfig.cs new file mode 100644 index 000000000000..876f8ff9b5ab --- /dev/null +++ b/src/Nethermind/Nethermind.EraE/Config/EraEConfig.cs @@ -0,0 +1,23 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.EraE.Archive; + +namespace Nethermind.EraE.Config; + +public class EraEConfig : IEraEConfig +{ + public string? ImportDirectory { get; set; } + public string? ExportDirectory { get; set; } + public long From { get; set; } + public long To { get; set; } + public string? TrustedAccumulatorFile { get; set; } + public int MaxEraSize { get; set; } = EraWriter.MaxEraSize; + public string? NetworkName { get; set; } + public int Concurrency { get; set; } + public long ImportBlocksBufferSize { get; set; } = 4096; + public string? BeaconNodeUrl { get; set; } + public string? RemoteBaseUrl { get; set; } + public string? RemoteDownloadDirectory { get; set; } + public string RemoteChecksumFile { get; set; } = "checksums_sha256.txt"; +} diff --git a/src/Nethermind/Nethermind.EraE/Config/IEraEConfig.cs b/src/Nethermind/Nethermind.EraE/Config/IEraEConfig.cs new file mode 100644 index 000000000000..0339e4eb20d4 --- /dev/null +++ b/src/Nethermind/Nethermind.EraE/Config/IEraEConfig.cs @@ -0,0 +1,48 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Config; + +namespace Nethermind.EraE.Config; + +public interface IEraEConfig : IConfig +{ + [ConfigItem(Description = "Directory of EraE archives to be imported.", DefaultValue = "")] + string? ImportDirectory { get; set; } + + [ConfigItem(Description = "Directory for EraE archive export.", DefaultValue = "")] + string? ExportDirectory { get; set; } + + [ConfigItem(Description = "Block number to import/export from.", DefaultValue = "0")] + long From { get; set; } + + [ConfigItem(Description = "Block number to import/export to. 0 means head.", DefaultValue = "0")] + long To { get; set; } + + [ConfigItem(Description = "Accumulator file for trusting EraE archives.", DefaultValue = "null")] + string? TrustedAccumulatorFile { get; set; } + + [ConfigItem(Description = "Max number of blocks per era file.", DefaultValue = "8192", HiddenFromDocs = true)] + int MaxEraSize { get; set; } + + [ConfigItem(Description = "Network name used for EraE directory naming. When null, inferred from chain spec.", DefaultValue = "null", HiddenFromDocs = true)] + string? NetworkName { get; set; } + + [ConfigItem(Description = "Export parallelism. 0 = ProcessorCount/4 (default, background-friendly). -1 = all cores. >0 = exact.", DefaultValue = "0", HiddenFromDocs = true)] + int Concurrency { get; set; } + + [ConfigItem(Description = "[Technical] Block buffer size during era import.", DefaultValue = "4096", HiddenFromDocs = true)] + long ImportBlocksBufferSize { get; set; } + + [ConfigItem(Description = "Beacon node URL for fetching beacon block roots and state roots during post-merge EraE export. When set, enables BeaconApiRootsProvider and HistoricalSummariesRpcProvider.", DefaultValue = "null")] + string? BeaconNodeUrl { get; set; } + + [ConfigItem(Description = "Base URL of a remote EraE archive server (e.g. https://data.ethpandaops.io/erae/{network}/). When set, missing local epoch files are downloaded on demand.", DefaultValue = "null")] + string? RemoteBaseUrl { get; set; } + + [ConfigItem(Description = "Local directory where remotely downloaded EraE files are cached. Defaults to ImportDirectory when null.", DefaultValue = "null")] + string? RemoteDownloadDirectory { get; set; } + + [ConfigItem(Description = "Filename of the checksum manifest on the remote server.", DefaultValue = "checksums_sha256.txt", HiddenFromDocs = true)] + string RemoteChecksumFile { get; set; } +} diff --git a/src/Nethermind/Nethermind.EraE/E2Store/E2StoreReader.cs b/src/Nethermind/Nethermind.EraE/E2Store/E2StoreReader.cs new file mode 100644 index 000000000000..87cd19210137 --- /dev/null +++ b/src/Nethermind/Nethermind.EraE/E2Store/E2StoreReader.cs @@ -0,0 +1,242 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Buffers.Binary; +using System.IO.Compression; +using CommunityToolkit.HighPerformance; +using Microsoft.IO; +using Microsoft.Win32.SafeHandles; +using Nethermind.Core.Collections; +using Nethermind.Core.Crypto; +using Nethermind.Core.Resettables; +using Nethermind.EraE.Exceptions; +using Snappier; +using EraException = Nethermind.Era1.EraException; +using Entry = Nethermind.Era1.Entry; +using EraWriter = Nethermind.EraE.Archive.EraWriter; + +namespace Nethermind.EraE.E2Store; + +public sealed class E2StoreReader : IDisposable +{ + private const int EntryHeaderSize = 8; + private const int IndexFieldSize = 8; // sizeof(long) — each ComponentIndex field is a little-endian int64 + private const int AccumulatorRootSize = 32; // Bytes32 hash + private const int MinComponentCount = 3; // post-merge: header, body, receipts + private const int MaxComponentCount = 5; // future: transition with both TD and proof + private const int ComponentsWithTotalDifficulty = 4; // pre-merge or transition epoch + private const int ValueSizeLimit = 1024 * 1024 * 50; + + private readonly SafeFileHandle _file; + private readonly long _fileLength; + + private long _startBlock; + private long _blockCount; + private int _componentCount; // 3 (post-merge) or 4 (pre-merge/transition) + private bool _indexLoaded; + private long _componentIndexTlvStart; // absolute position of ComponentIndex entry header + + public E2StoreReader(string filePath) + { + _file = File.OpenHandle(filePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); + _fileLength = RandomAccess.GetLength(_file); + } + + public long First + { + get + { + EnsureIndexLoaded(); + return _startBlock; + } + } + + public long LastBlock => First + _blockCount - 1; + + public long BlockCount + { + get + { + EnsureIndexLoaded(); + return _blockCount; + } + } + + public bool HasTotalDifficulty + { + get + { + EnsureIndexLoaded(); + return _componentCount >= ComponentsWithTotalDifficulty; + } + } + + public long HeaderOffset(long blockNumber) => ComponentOffset(blockNumber, 0); + + public long BodyOffset(long blockNumber) => ComponentOffset(blockNumber, 1); + + public long SlimReceiptsOffset(long blockNumber) => ComponentOffset(blockNumber, 2); + + public long TotalDifficultyOffset(long blockNumber) + { + EnsureIndexLoaded(); + if (_componentCount < ComponentsWithTotalDifficulty) + throw new EraException("This EraE file does not contain TotalDifficulty entries (post-merge epoch)."); + return ComponentOffset(blockNumber, 3); + } + + public long AccumulatorRootOffset + { + get + { + EnsureIndexLoaded(); + if (_componentCount < ComponentsWithTotalDifficulty) + return -1; + + // AccumulatorRoot is the entry immediately before ComponentIndex. + // ComponentIndex entry header is at _componentIndexTlvStart. + // Before it: AccumulatorRoot entry = [EntryHeaderSize][AccumulatorRootSize] + return _componentIndexTlvStart - EntryHeaderSize - AccumulatorRootSize; + } + } + + public long ReadEntryAndDecode(long position, Func, T> decoder, ushort expectedType, out T value) + { + Entry entry = ReadEntry(position, expectedType); + int length = (int)entry.Length; + using ArrayPoolListRef buffer = new(length, length); + RandomAccess.Read(_file, buffer.AsSpan(), position + EntryHeaderSize); + value = decoder(buffer.AsMemory()); + return (long)entry.Length + EntryHeaderSize; + } + + public async Task<(T, long)> ReadSnappyCompressedEntryAndDecode( + long position, Func, T> decoder, ushort expectedType, CancellationToken token = default) + { + Entry entry = ReadEntry(position, expectedType); + T value = await ReadEntryValueAsSnappy(position + EntryHeaderSize, entry.Length, decoder, token); + return (value, (long)entry.Length + EntryHeaderSize); + } + + private Entry ReadEntry(long position, ushort? expectedType) + { + ushort type = ReadUInt16(position); + uint length = ReadUInt32(position + 2); + ushort reserved = ReadUInt16(position + 6); + + Entry entry = new(type, length); + if (expectedType.HasValue && entry.Type != expectedType) + throw new EraException($"Expected entry type 0x{expectedType:X4} but got 0x{entry.Type:X4} at position {position}."); + if (entry.Length + (ulong)position > (ulong)_fileLength) + throw new EraFormatException($"Entry length {entry.Length} at position {position} exceeds file length {_fileLength}."); + if (entry.Length > ValueSizeLimit) + throw new EraException($"Entry size {entry.Length} exceeds limit {ValueSizeLimit}."); + if (reserved != 0) + throw new EraFormatException($"Reserved header bytes are non-zero at position {position}."); + return entry; + } + + public ValueHash256 CalculateChecksum() => + Era1.E2StoreReader.ComputeChecksum(_file, _fileLength); + + public void Dispose() => _file.Dispose(); + + private void EnsureIndexLoaded() + { + if (_indexLoaded) return; + + // ComponentIndex is the last entry in the file. + // Layout of ComponentIndex data: + // starting_number (8 bytes) + // [comp offsets] (blockCount * componentCount * 8 bytes) + // component_count (8 bytes) + // block_count (8 bytes) + + // Minimum viable file: entry header + starting_number + component_count + block_count + if (_fileLength < EntryHeaderSize + IndexFieldSize * 3) + throw new EraFormatException($"File too small ({_fileLength} bytes) to contain a valid ComponentIndex."); + + // Read block_count from the last 8 bytes of file + _blockCount = ReadInt64(_fileLength - IndexFieldSize); + if (_blockCount <= 0 || _blockCount > EraWriter.MaxEraSize) + throw new EraFormatException($"Invalid block count {_blockCount} in EraE ComponentIndex."); + + // Read component_count from the 8 bytes before block_count + _componentCount = (int)ReadInt64(_fileLength - IndexFieldSize * 2); + // 3 = post-merge (header, body, receipts) + // 4 = pre-merge or transition (+ total-difficulty), or post-merge with proof + // 5 = transition with both total-difficulty and proof (future) + if (_componentCount < MinComponentCount || _componentCount > MaxComponentCount) + throw new EraFormatException($"Invalid component count {_componentCount} in EraE ComponentIndex."); + + // Total data length = starting_number + offsets + component_count + block_count + long indexDataLength = IndexFieldSize + _blockCount * _componentCount * IndexFieldSize + IndexFieldSize + IndexFieldSize; + + // Verify entry type + long indexEntryStart = _fileLength - EntryHeaderSize - indexDataLength; + _ = ReadEntry(indexEntryStart, EntryTypes.ComponentIndex); + + _componentIndexTlvStart = indexEntryStart; + + // Read starting block number (first field of index data) + _startBlock = ReadInt64(indexEntryStart + EntryHeaderSize); + + _indexLoaded = true; + } + + private long ComponentOffset(long blockNumber, int componentIdx) + { + EnsureIndexLoaded(); + + if (blockNumber < _startBlock || blockNumber >= _startBlock + _blockCount) + throw new ArgumentOutOfRangeException(nameof(blockNumber), $"Block {blockNumber} is outside range [{_startBlock}, {_startBlock + _blockCount - 1}]."); + ArgumentOutOfRangeException.ThrowIfGreaterThanOrEqual(componentIdx, _componentCount); + + long blockIdx = blockNumber - _startBlock; + // Offset table starts at: indexEntryStart + EntryHeaderSize + IndexFieldSize (starting_number) + long offsetFieldPos = _componentIndexTlvStart + EntryHeaderSize + IndexFieldSize + + blockIdx * _componentCount * IndexFieldSize + + componentIdx * IndexFieldSize; + + long relativeOffset = ReadInt64(offsetFieldPos); + // Offset is relative to the ComponentIndex TLV start (entry header included) + return _componentIndexTlvStart + relativeOffset; + } + + private async Task ReadEntryValueAsSnappy(long offset, ulong length, Func, T> decoder, CancellationToken cancellation = default) + { + using ArrayPoolList inputBuffer = new((int)length, (int)length); + RandomAccess.Read(_file, inputBuffer.AsSpan(), offset); + Stream inputStream = inputBuffer.AsMemory().AsStream(); + + await using SnappyStream decompressor = new(inputStream, CompressionMode.Decompress, true); + await using RecyclableMemoryStream stream = RecyclableStream.GetStream(nameof(E2StoreReader)); + await decompressor.CopyToAsync(stream, cancellation); + + if (!stream.TryGetBuffer(out ArraySegment segment)) + throw new InvalidDataException("Unable to get buffer from memory stream."); + + return decoder(segment); + } + + private ushort ReadUInt16(long position) + { + Span buff = stackalloc byte[2]; + RandomAccess.Read(_file, buff, position); + return BinaryPrimitives.ReadUInt16LittleEndian(buff); + } + + private uint ReadUInt32(long position) + { + Span buff = stackalloc byte[4]; + RandomAccess.Read(_file, buff, position); + return BinaryPrimitives.ReadUInt32LittleEndian(buff); + } + + private long ReadInt64(long position) + { + Span buff = stackalloc byte[8]; + RandomAccess.Read(_file, buff, position); + return BinaryPrimitives.ReadInt64LittleEndian(buff); + } +} diff --git a/src/Nethermind/Nethermind.EraE/E2Store/EntryTypes.cs b/src/Nethermind/Nethermind.EraE/E2Store/EntryTypes.cs new file mode 100644 index 000000000000..1eefd90d1489 --- /dev/null +++ b/src/Nethermind/Nethermind.EraE/E2Store/EntryTypes.cs @@ -0,0 +1,19 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +namespace Nethermind.EraE.E2Store; + +internal static class EntryTypes +{ + // Shared with era1 + public const ushort Version = 0x3265; + public const ushort CompressedHeader = 0x03; + public const ushort CompressedBody = 0x04; + public const ushort TotalDifficulty = 0x06; + public const ushort AccumulatorRoot = 0x07; + + // EraE-specific + public const ushort CompressedSlimReceipts = 0x0a; + public const ushort Proof = 0x0b; + public const ushort ComponentIndex = 0x3267; +} diff --git a/src/Nethermind/Nethermind.EraE/EraCliRunner.cs b/src/Nethermind/Nethermind.EraE/EraCliRunner.cs new file mode 100644 index 000000000000..af70ff63fdfb --- /dev/null +++ b/src/Nethermind/Nethermind.EraE/EraCliRunner.cs @@ -0,0 +1,37 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.EraE.Config; +using Nethermind.EraE.Export; +using Nethermind.EraE.Import; +using Nethermind.History; +using EraException = Nethermind.Era1.EraException; + +namespace Nethermind.EraE; + +public sealed class EraCliRunner( + IEraEConfig eraConfig, + IHistoryConfig historyConfig, + IEraImporter eraImporter, + IEraExporter eraExporter) +{ + public async Task Run(CancellationToken token) + { + if (!string.IsNullOrEmpty(eraConfig.ImportDirectory)) + { + if (historyConfig.Pruning == PruningModes.UseAncientBarriers) + { + throw new EraException( + "EraE import is configured alongside History.Pruning=UseAncientBarriers. " + + "This would immediately prune the imported blocks. " + + "Either disable history pruning or remove the import directory."); + } + + await eraImporter.Import(eraConfig.ImportDirectory!, eraConfig.From, eraConfig.To, eraConfig.TrustedAccumulatorFile, token); + } + else if (!string.IsNullOrEmpty(eraConfig.ExportDirectory)) + { + await eraExporter.Export(eraConfig.ExportDirectory!, eraConfig.From, eraConfig.To, token); + } + } +} diff --git a/src/Nethermind/Nethermind.EraE/EraEModule.cs b/src/Nethermind/Nethermind.EraE/EraEModule.cs new file mode 100644 index 000000000000..c44a740c2c23 --- /dev/null +++ b/src/Nethermind/Nethermind.EraE/EraEModule.cs @@ -0,0 +1,79 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.IO.Abstractions; +using Autofac; +using Nethermind.Config; +using Nethermind.Consensus.Validators; +using Nethermind.Core; +using Nethermind.Core.Specs; +using Nethermind.EraE.Admin; +using Nethermind.EraE.Config; +using Nethermind.EraE.Export; +using Nethermind.EraE.Import; +using Nethermind.EraE.JsonRpc; +using Nethermind.EraE.Proofs; +using Nethermind.EraE.Store; +using Nethermind.JsonRpc.Modules; +using Nethermind.Logging; + +namespace Nethermind.EraE; + +public class EraEModule : Module +{ + protected override void Load(ContainerBuilder builder) + { + base.Load(builder); + + builder + .AddSingleton() + .AddSingleton(ctx => + { + string? url = ctx.Resolve().BeaconNodeUrl; + return string.IsNullOrWhiteSpace(url) + ? NullBeaconRootsProvider.Instance + : new BeaconApiRootsProvider(new Uri(url), ctx.Resolve(), logManager: ctx.Resolve()); + }) + .AddSingleton(ctx => + { + string? url = ctx.Resolve().BeaconNodeUrl; + return string.IsNullOrWhiteSpace(url) + ? NullHistoricalSummariesProvider.Instance + : new HistoricalSummariesRpcProvider(new Uri(url), ctx.Resolve(), logManager: ctx.Resolve()); + }) + .AddSingleton() + .AddSingleton() + .AddSingleton(ctx => + { + IEraEConfig config = ctx.Resolve(); + IRemoteEraClient? remoteClient = string.IsNullOrWhiteSpace(config.RemoteBaseUrl) + ? null + : new HttpRemoteEraClient(new Uri(config.RemoteBaseUrl), config.RemoteChecksumFile, ctx.Resolve(), logManager: ctx.Resolve()); + + return new EraStoreFactory( + ctx.Resolve(), + ctx.Resolve(), + ctx.Resolve(), + config, + ctx.Resolve(), + ctx.ResolveOptional(), + remoteClient); + }) + .AddSingleton() + .AddSingleton() + .AddSingleton(ctx => + { + ISpecProvider spec = ctx.Resolve(); + IHistoricalSummariesProvider summariesProvider = ctx.Resolve(); + IBlocksConfig blocksConfig = ctx.Resolve(); + return new Validator(spec, trustedAccumulators: null, trustedHistoricalRoots: null, summariesProvider, blocksConfig); + }) + .RegisterSingletonJsonRpcModule() + .AddDecorator((ctx, eraConfig) => + { + if (string.IsNullOrWhiteSpace(eraConfig.NetworkName)) + eraConfig.NetworkName = BlockchainIds.GetBlockchainName(ctx.Resolve().NetworkId); + return eraConfig; + }); + } +} diff --git a/src/Nethermind/Nethermind.EraE/EraJobRunner.cs b/src/Nethermind/Nethermind.EraE/EraJobRunner.cs new file mode 100644 index 000000000000..8ed36f598d78 --- /dev/null +++ b/src/Nethermind/Nethermind.EraE/EraJobRunner.cs @@ -0,0 +1,29 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Logging; + +namespace Nethermind.EraE; + +internal static class EraJobRunner +{ + internal static async Task RunProtected(Func job, string cancelledMessage, string errorMessage, ILogger logger, CancellationToken ct, Action? onFinally = null) + { + try + { + await job(ct); + } + catch (Exception e) when (e is TaskCanceledException or OperationCanceledException) + { + logger.Warn(cancelledMessage); + } + catch (Exception e) + { + logger.Error(errorMessage, e); + } + finally + { + onFinally?.Invoke(); + } + } +} diff --git a/src/Nethermind/Nethermind.EraE/Exceptions/EraFormatException.cs b/src/Nethermind/Nethermind.EraE/Exceptions/EraFormatException.cs new file mode 100644 index 000000000000..0a26ca15ebbf --- /dev/null +++ b/src/Nethermind/Nethermind.EraE/Exceptions/EraFormatException.cs @@ -0,0 +1,8 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Era1; + +namespace Nethermind.EraE.Exceptions; + +internal class EraFormatException(string message) : EraException(message); diff --git a/src/Nethermind/Nethermind.EraE/Export/EraExporter.cs b/src/Nethermind/Nethermind.EraE/Export/EraExporter.cs new file mode 100644 index 000000000000..197005627e6d --- /dev/null +++ b/src/Nethermind/Nethermind.EraE/Export/EraExporter.cs @@ -0,0 +1,332 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.IO.Abstractions; +using Nethermind.Blockchain; +using Nethermind.Blockchain.Receipts; +using Nethermind.Core; +using Nethermind.Core.Collections; +using Nethermind.Core.Crypto; +using Nethermind.Core.Extensions; +using Nethermind.Core.Specs; +using Nethermind.EraE.Archive; +using Nethermind.EraE.Config; +using EraException = Nethermind.Era1.EraException; +using Nethermind.EraE.Proofs; +using Nethermind.Logging; +using Nethermind.Serialization.Rlp; +using Nethermind.State.Proofs; + +namespace Nethermind.EraE.Export; + +public sealed class EraExporter( + IFileSystem fileSystem, + IBlockTree blockTree, + IReceiptStorage receiptStorage, + ISpecProvider specProvider, + IEraEConfig eraConfig, + ILogManager logManager, + IBeaconRootsProvider? beaconRootsProvider = null) + : IEraExporter +{ + private readonly string _networkName = string.IsNullOrWhiteSpace(eraConfig.NetworkName) + ? throw new ArgumentException("NetworkName cannot be null or whitespace.", nameof(eraConfig)) + : eraConfig.NetworkName.Trim().ToLower(); + + private readonly ILogger _logger = logManager.GetClassLogger(); + private readonly ReceiptMessageDecoder _receiptDecoder = new(); + private readonly int _eraSize = eraConfig.MaxEraSize is > 0 and <= EraWriter.MaxEraSize + ? eraConfig.MaxEraSize + : throw new ArgumentException($"MaxEraSize must be between 1 and {EraWriter.MaxEraSize} (SLOTS_PER_HISTORICAL_ROOT). Got {eraConfig.MaxEraSize}."); + + public const string AccumulatorFileName = "accumulators.txt"; + public const string ChecksumsSHA256FileName = "checksums_sha256.txt"; + public const string ChecksumsFileName = "checksums.txt"; + + private const int ProgressLogInterval = 10000; + private const int RetryDelayMs = 100; + + public Task Export(string destinationPath, long from, long to, CancellationToken cancellation = default) + { + if (fileSystem.File.Exists(destinationPath)) + throw new ArgumentException("Destination already exists as a file.", nameof(destinationPath)); + if (to == 0) to = blockTree.Head?.Number ?? 0; + if (to > (blockTree.Head?.Number ?? 0)) + throw new ArgumentException($"Cannot export beyond head block {blockTree.Head?.Number ?? 0}."); + if (from > to) + throw new ArgumentException($"Start block ({from}) must not be after end block ({to})."); + + Block? lastBlock = blockTree.FindBlock(to, BlockTreeLookupOptions.DoNotCreateLevelIfMissing); + if (lastBlock is null) + throw new InvalidOperationException( + $"Block {to} is not available. " + + "EraE export requires all block bodies to be present. " + + "Ensure the node is fully synced before exporting."); + + return DoExport(destinationPath, from, to, cancellation); + } + + private async Task DoExport(string destinationPath, long from, long to, CancellationToken cancellation) + { + int concurrency = CalculateConcurrency(eraConfig.Concurrency); + if (_logger.IsInfo) _logger.Info($"Exporting EraE blocks {from}–{to} to {destinationPath} (concurrency={concurrency})"); + + if (!fileSystem.Directory.Exists(destinationPath)) + fileSystem.Directory.CreateDirectory(destinationPath); + + ProgressLogger progress = new("EraE export", logManager); + progress.Reset(0, to - from + 1); + int totalProcessed = 0; + + long startEpoch = from / _eraSize; + long endEpoch = to / _eraSize; + long epochCount = endEpoch - startEpoch + 1; + + using ArrayPoolList epochIdxs = new((int)epochCount); + for (long i = 0; i < epochCount; i++) epochIdxs.Add(i); + + using ArrayPoolList accumulators = new((int)epochCount, (int)epochCount); + using ArrayPoolList checksums = new((int)epochCount, (int)epochCount); + using ArrayPoolList fileNames = new((int)epochCount, (int)epochCount); + + // Load existing hash manifests so TrySkipExistingEpoch can look up checksums and + // accumulator roots by filename — O(1) per epoch — instead of reading each era file. + Dictionary cachedChecksums = LoadHashFile(Path.Combine(destinationPath, ChecksumsSHA256FileName)); + Dictionary cachedAccumulators = LoadHashFile(Path.Combine(destinationPath, AccumulatorFileName)); + + await Parallel.ForEachAsync(epochIdxs, new ParallelOptions + { + MaxDegreeOfParallelism = concurrency, + CancellationToken = cancellation + }, + (epochIdx, cancel) => new ValueTask(WriteEpoch(epochIdx, cancel))); + + string accumulatorPath = Path.Combine(destinationPath, AccumulatorFileName); + fileSystem.File.Delete(accumulatorPath); + await WriteHashFile(accumulatorPath, accumulators, fileNames, cancellation); + + string checksumSha256FilePath = Path.Combine(destinationPath, ChecksumsSHA256FileName); + fileSystem.File.Delete(checksumSha256FilePath); + await WriteHashFile(checksumSha256FilePath, checksums, fileNames, cancellation); + + string checksumFilePath = Path.Combine(destinationPath, ChecksumsFileName); + fileSystem.File.Delete(checksumFilePath); + await WriteChecksumFile(checksumFilePath, checksums, cancellation); + + progress.LogProgress(); + if (_logger.IsInfo) _logger.Info($"Finished EraE export from {from} to {to}"); + + async Task WriteEpoch(long epochIdx, CancellationToken cancel) + { + int idx = (int)epochIdx; + long epoch = startEpoch + epochIdx; + // Each epoch covers [epoch * eraSize, epoch * eraSize + eraSize - 1]. + // Clamp to [from, to] to handle partial first and last epochs. + long epochBlockStart = epoch * _eraSize; + long writeFrom = Math.Max(epochBlockStart, from); + long writeTo = Math.Min(epochBlockStart + _eraSize - 1, to); + + if (TrySkipExistingEpoch(destinationPath, epoch, idx, writeFrom, writeTo, accumulators, checksums, fileNames, cachedChecksums, cachedAccumulators, ref totalProcessed)) + return; + + string placeholderPath = Path.Combine(destinationPath, EraPathUtils.Filename(_networkName, epoch, Keccak.Zero)); + + ValueHash256 accumulator; + ValueHash256 sha256; + Hash256 lastBlockHash = Keccak.Zero; + + using (EraWriter eraWriter = new(fileSystem.File.Create(placeholderPath), specProvider, beaconRootsProvider)) + { + for (long blockNumber = writeFrom; blockNumber <= writeTo; blockNumber++) + { + Block? block = blockTree.FindBlock(blockNumber, BlockTreeLookupOptions.DoNotCreateLevelIfMissing); + if (block is null) + throw new EraException($"Could not find block {blockNumber}. The node may not have finished syncing block bodies for this range."); + + // IsPostMerge is not part of the RLP encoding and defaults to false when read from + // the block store. Restore it from Difficulty (EIP-3675: post-merge Difficulty == 0). + block.Header.IsPostMerge = block.Header.Difficulty == 0; + + TxReceipt[]? receipts = receiptStorage.Get(block, true, false); + if (receipts is null || (block.Header.ReceiptsRoot != Keccak.EmptyTreeHash && receipts.Length == 0)) + throw new EraException($"Could not find receipts for block {block.ToString(Block.Format.FullHashAndNumber)}."); + + if (block.Header.ReceiptsRoot != Keccak.EmptyTreeHash) + { + Hash256 computedRoot = ReceiptTrie.CalculateRoot(specProvider.GetReceiptSpec(block.Number), receipts, _receiptDecoder); + if (computedRoot != block.Header.ReceiptsRoot) + { + throw new EraException( + $"Receipt root mismatch at block {block.ToString(Block.Format.FullHashAndNumber)}: " + + "the database contains stale or corrupt receipt data. " + + "Re-import from the original ERA source before re-exporting."); + } + } + + await eraWriter.Add(block, receipts, cancel); + lastBlockHash = block.Hash!; + + if (Interlocked.Increment(ref totalProcessed) % ProgressLogInterval == 0) + { + progress.Update(totalProcessed); + progress.LogProgress(); + } + } + + (accumulator, sha256) = await eraWriter.Finalize(cancel); + } + + // Safe concurrent indexed writes: each epoch has a unique idx slot; + // reads from these arrays happen only after Parallel.ForEachAsync completes. + accumulators[idx] = accumulator; + checksums[idx] = sha256; + // Filename uses the last block hash as the epoch identifier — same convention as go-ethereum execdb. + string finalName = EraPathUtils.Filename(_networkName, epoch, lastBlockHash); + fileNames[idx] = finalName; + + string finalPath = Path.Combine(destinationPath, finalName); + try + { + for (int attempt = 0; ; attempt++) + { + try + { + fileSystem.File.Move(placeholderPath, finalPath, true); + break; + } + catch (IOException) when (attempt < 3) + { + await Task.Delay(RetryDelayMs * (attempt + 1), cancel); + } + } + } + catch + { + fileSystem.File.Delete(placeholderPath); + throw; + } + } + } + + // Follows the same convention as FullPruningMaxDegreeOfParallelism / VisitingOptions.AdjustMaxDegreeOfParallelism: + // 0 → ProcessorCount / 4 (default: background-friendly, leaves 75% of CPU for the live node) + // -1 → ProcessorCount (aggressive: fastest export, use when the node has no other responsibilities) + // >0 → exact value + private static int CalculateConcurrency(int configured) => configured switch + { + 0 => Math.Max(Environment.ProcessorCount / 4, 1), + < 0 => Environment.ProcessorCount, + _ => configured + }; + + private bool TrySkipExistingEpoch( + string destinationPath, + long epoch, + int idx, + long writeFrom, + long writeTo, + ArrayPoolList accumulators, + ArrayPoolList checksums, + ArrayPoolList fileNames, + Dictionary cachedChecksums, + Dictionary cachedAccumulators, + ref int totalProcessed) + { + string? existingFile = null; + foreach (string f in fileSystem.Directory.EnumerateFiles(destinationPath, $"{_networkName}-{epoch:D5}-*{EraPathUtils.FileExtension}")) + { + existingFile = f; + break; + } + + if (existingFile is null) + return false; + + string fileName = Path.GetFileName(existingFile); + fileNames[idx] = fileName; + + if (cachedChecksums.TryGetValue(fileName, out ValueHash256 cachedChecksum)) + { + // Fast path: both values available from the manifest files — no era file reads needed. + checksums[idx] = cachedChecksum; + accumulators[idx] = cachedAccumulators.GetValueOrDefault(fileName); + } + else + { + // Slow path: manifest missing or stale — read the era file directly. + using EraReader reader = new(existingFile); + try + { + accumulators[idx] = reader.ReadAccumulatorRoot(); + } + catch (EraException) + { + accumulators[idx] = default; + } + checksums[idx] = reader.CalculateChecksum(); + } + + Interlocked.Add(ref totalProcessed, (int)(writeTo - writeFrom + 1)); + if (_logger.IsDebug) _logger.Debug($"Skipping already exported epoch {epoch}."); + return true; + } + + /// + /// Parses a hash manifest file (checksums or accumulators) into a filename → hash lookup. + /// Returns an empty dictionary if the file does not exist or cannot be read. + /// + private Dictionary LoadHashFile(string path) + { + Dictionary result = new(); + if (!fileSystem.File.Exists(path)) + return result; + + foreach (string line in fileSystem.File.ReadAllLines(path)) + { + // Format: "{hash} {filename}" (two spaces, same as sha256sum output) + int sep = line.IndexOf(" ", StringComparison.Ordinal); + if (sep < 0) continue; + + string hashHex = line[..sep].Trim(); + string fileName = line[(sep + 2)..].Trim(); + if (hashHex.Length != 64) continue; + result[fileName] = new ValueHash256(hashHex); + } + + return result; + } + + private async Task WriteHashFile(string path, ArrayPoolList hashes, ArrayPoolList fileNames, CancellationToken cancellation) + { + IEnumerable Lines() + { + for (int i = 0; i < hashes.Count; i++) + yield return $"{hashes[i].ToString(false)} {fileNames[i]}"; + } + await WriteLines(path, Lines(), cancellation); + } + + private async Task WriteChecksumFile(string path, ArrayPoolList hashes, CancellationToken cancellation) + { + await WriteLines(path, Lines(), cancellation); + return; + + IEnumerable Lines() + { + for (int i = 0; i < hashes.Count; i++) + yield return $"0x{hashes[i].ToString(false)}"; + } + } + + private async Task WriteLines(string path, IEnumerable lines, CancellationToken cancellation) + { + await using FileSystemStream stream = fileSystem.FileStream.New(path, FileMode.Create, FileAccess.Write, FileShare.None); + await using StreamWriter writer = new(stream); + + foreach (string line in lines) + { + cancellation.ThrowIfCancellationRequested(); + await writer.WriteLineAsync(line); + } + } +} diff --git a/src/Nethermind/Nethermind.EraE/Export/EraPathUtils.cs b/src/Nethermind/Nethermind.EraE/Export/EraPathUtils.cs new file mode 100644 index 000000000000..4618a15fb551 --- /dev/null +++ b/src/Nethermind/Nethermind.EraE/Export/EraPathUtils.cs @@ -0,0 +1,79 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.IO.Abstractions; +using Nethermind.Core.Crypto; +using Nethermind.Core.Extensions; +using EraException = Nethermind.Era1.EraException; +using Testably.Abstractions; + +namespace Nethermind.EraE.Export; + +public static class EraPathUtils +{ + private static readonly char[] _separator = ['-']; + + public const string FileExtension = ".erae"; + + public static IEnumerable GetAllEraFiles(string directoryPath, string network, IFileSystem fileSystem) + { + string[] entries = fileSystem.Directory.GetFiles(directoryPath, $"*{FileExtension}", new EnumerationOptions + { + RecurseSubdirectories = false, + MatchCasing = MatchCasing.PlatformDefault + }); + + if (entries.Length == 0) + yield break; + + foreach (string file in entries) + { + // Format: --.erae + string[] parts = Path.GetFileNameWithoutExtension(file).Split(_separator); + if (parts.Length != 3 || !network.Equals(parts[0], StringComparison.OrdinalIgnoreCase)) continue; + if (!uint.TryParse(parts[1], out _)) + throw new EraException($"Invalid erae filename: {Path.GetFileName(file)}"); + + yield return file; + } + } + + public static IEnumerable GetAllEraFiles(string directoryPath, string network) + => GetAllEraFiles(directoryPath, network, new RealFileSystem()); + + public static string Filename(string network, long epoch, Hash256 accumulatorRoot) + { + ArgumentException.ThrowIfNullOrEmpty(network); + ArgumentNullException.ThrowIfNull(accumulatorRoot); + ArgumentOutOfRangeException.ThrowIfLessThan(epoch, 0); + + return $"{network}-{epoch:D5}-{accumulatorRoot.ToString(true)[2..10]}{FileExtension}"; + } + + public static ValueHash256 ExtractHashFromChecksumEntry(string s) + { + ValueHash256 result = default; + ReadOnlySpan span = s.AsSpan(); + int idx = span.IndexOf(' '); + ReadOnlySpan token = idx == -1 ? span : span[..idx]; + Bytes.FromHexString(token, result.BytesAsSpan); + return result; + } + + public static (long Epoch, ValueHash256 Hash) ParseChecksumEntry(string line) + { + ValueHash256 hash = ExtractHashFromChecksumEntry(line); + + int nameStart = line.IndexOf(' '); + if (nameStart == -1) + throw new ArgumentException($"Invalid checksum entry (no filename): {line}"); + + string filename = line[(nameStart + 1)..].TrimStart(); + string nameWithoutExt = Path.GetFileNameWithoutExtension(filename); + string[] parts = nameWithoutExt.Split(_separator); + if (parts.Length != 3 || !long.TryParse(parts[1], out long epoch)) + throw new ArgumentException($"Invalid checksum entry filename: {filename}"); + + return (epoch, hash); + } +} diff --git a/src/Nethermind/Nethermind.EraE/Export/IEraExporter.cs b/src/Nethermind/Nethermind.EraE/Export/IEraExporter.cs new file mode 100644 index 000000000000..cde8037f483f --- /dev/null +++ b/src/Nethermind/Nethermind.EraE/Export/IEraExporter.cs @@ -0,0 +1,9 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +namespace Nethermind.EraE.Export; + +public interface IEraExporter +{ + Task Export(string destinationPath, long from, long to, CancellationToken cancellation = default); +} diff --git a/src/Nethermind/Nethermind.EraE/Import/EraImporter.cs b/src/Nethermind/Nethermind.EraE/Import/EraImporter.cs new file mode 100644 index 000000000000..25b706cff0c3 --- /dev/null +++ b/src/Nethermind/Nethermind.EraE/Import/EraImporter.cs @@ -0,0 +1,233 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Collections.Concurrent; +using System.IO.Abstractions; +using Autofac.Features.AttributeFilters; +using Nethermind.Blockchain; +using Nethermind.Blockchain.Receipts; +using Nethermind.Blockchain.Synchronization; +using Nethermind.Consensus.Validators; +using Nethermind.Core; +using Nethermind.Core.Crypto; +using Nethermind.Db; +using Nethermind.EraE.Config; +using EraImportException = Nethermind.Era1.EraImportException; +using EraVerificationException = Nethermind.Era1.Exceptions.EraVerificationException; +using Nethermind.EraE.Export; +using Nethermind.EraE.Store; +using Nethermind.Logging; + +namespace Nethermind.EraE.Import; + +public sealed class EraImporter( + IFileSystem fileSystem, + IBlockTree blockTree, + IReceiptStorage receiptStorage, + IBlockValidator blockValidator, + ILogManager logManager, + IEraEConfig eraConfig, + ISyncConfig syncConfig, + IEraStoreFactory eraStoreFactory, + [KeyFilter(DbNames.Blocks)] ITunableDb blocksDb, + [KeyFilter(DbNames.Receipts)] ITunableDb receiptsDb) + : IEraImporter +{ + private readonly ILogger _logger = logManager.GetClassLogger(); + private readonly int _maxEraSize = eraConfig.MaxEraSize; + + private const int ProgressLogInterval = 10000; + + public async Task Import(string src, long from, long to, string? accumulatorFile, CancellationToken cancellation = default) + { + if (!fileSystem.Directory.Exists(src)) + { + if (string.IsNullOrEmpty(eraConfig.RemoteBaseUrl)) + throw new ArgumentException($"Import source directory does not exist: {src}"); + fileSystem.Directory.CreateDirectory(src); + } + if (accumulatorFile != null && !fileSystem.File.Exists(accumulatorFile)) + throw new ArgumentException($"Accumulator file {accumulatorFile} does not exist."); + + ISet? trustedAccumulators = null; + if (accumulatorFile != null) + { + HashSet accumulators = []; + foreach (string line in await fileSystem.File.ReadAllLinesAsync(accumulatorFile, cancellation)) + accumulators.Add(EraPathUtils.ExtractHashFromChecksumEntry(line)); + trustedAccumulators = accumulators; + } + + using IEraStore eraStore = eraStoreFactory.Create(src, trustedAccumulators); + + (long firstBlockInStore, long lastBlockInStore) = eraStore.BlockRange; + if (to is 0 or long.MaxValue) + to = lastBlockInStore; + else if (to > lastBlockInStore) + throw new EraImportException($"Store highest block {lastBlockInStore} is lower than requested end {to}."); + + if (from == 0) + from = firstBlockInStore; + else if (from < firstBlockInStore) + throw new EraImportException($"Store first block {firstBlockInStore} is higher than requested start {from}."); + if (from > to) + throw new ArgumentException($"Start block ({from}) must not be after end block ({to})."); + + receiptsDb.Tune(ITunableDb.TuneType.HeavyWrite); + blocksDb.Tune(ITunableDb.TuneType.HeavyWrite); + + try + { + await ImportInternal(from, to, eraStore, cancellation); + } + finally + { + receiptsDb.Tune(ITunableDb.TuneType.Default); + blocksDb.Tune(ITunableDb.TuneType.Default); + } + } + + private async Task ImportInternal(long from, long to, IEraStore eraStore, CancellationToken cancellation) + { + if (_logger.IsInfo) _logger.Info($"Starting EraE import from {from} to {to}"); + + ProgressLogger progressLogger = new("EraE import", logManager); + progressLogger.Reset(0, to - from + 1); + long blocksProcessed = 0; + + using BlockTreeSuggestPacer pacer = new(blockTree, eraConfig.ImportBlocksBufferSize, eraConfig.ImportBlocksBufferSize - 1024); + long blockNumber = from; + + long suggestFromBlock = (blockTree.Head?.Number ?? 0) + 1; + if (syncConfig.FastSync && suggestFromBlock == 1) + suggestFromBlock = long.MaxValue; + + // Align to era boundary for parallel section + long nextEraStart = eraStore.NextEraStart(blockNumber); + if (nextEraStart <= to) + { + for (; blockNumber < nextEraStart; blockNumber++) + await ImportBlock(blockNumber); + } + + // Parallel historical import (blocks without state) + long partitionSize = _maxEraSize; + if (blockNumber + partitionSize < suggestFromBlock) + { + ConcurrentQueue partitionStartBlocks = new(); + for (; blockNumber + partitionSize < suggestFromBlock && blockNumber + partitionSize < to; blockNumber += partitionSize) + partitionStartBlocks.Enqueue(blockNumber); + + int concurrency = eraConfig.Concurrency == 0 ? Environment.ProcessorCount : eraConfig.Concurrency; + Task[] workers = new Task[concurrency]; + for (int i = 0; i < concurrency; i++) + { + workers[i] = Task.Run(async () => + { + while (partitionStartBlocks.TryDequeue(out long partStart)) + { + for (long j = 0; j < partitionSize; j++) + { + await ImportBlock(partStart + j); + } + } + }, cancellation); + } + + await Task.WhenAll(workers); + } + + // Sequential near-head import + for (; blockNumber <= to; blockNumber++) + await ImportBlock(blockNumber); + + progressLogger.LogProgress(); + if (_logger.IsInfo) _logger.Info($"Finished EraE import from {from} to {to}"); + + async Task ImportBlock(long number) + { + if (_logger.IsTrace) _logger.Trace($"Importing EraE block {number}"); + cancellation.ThrowIfCancellationRequested(); + + (Block? block, TxReceipt[]? receipts) = await eraStore.FindBlockAndReceipts(number, cancellation: cancellation); + + if (block is null) + throw new EraImportException($"Block {number} not found in archive."); + if (receipts is null) + throw new EraImportException($"Receipts for block {number} not found in archive."); + if (block.Number != number) + throw new EraImportException($"Unexpected block number {block.Number}, expected {number}."); + if (block.IsGenesis) + return; + if (block.IsBodyMissing) + throw new EraImportException($"Block {number} body is missing — archive may be corrupted."); + + if (suggestFromBlock <= block.Number) + { + await pacer.WaitForQueue(block.Number, cancellation); + await SuggestAndProcessBlock(block); + } + else + { + InsertBlockAndReceipts(block, receipts, to); + } + + long processed = Interlocked.Increment(ref blocksProcessed); + if (processed % ProgressLogInterval == 0) + { + progressLogger.Update(processed); + progressLogger.LogProgress(); + } + } + } + + private void InsertBlockAndReceipts(Block block, TxReceipt[] receipts, long lastBlockNumber) + { + Block? existing = blockTree.FindBlock(block.Number); + if (existing is null) + { + BlockTreeInsertHeaderOptions headerOptions = block.Header.IsPostMerge + ? BlockTreeInsertHeaderOptions.TotalDifficultyNotNeeded + : BlockTreeInsertHeaderOptions.None; + AddBlockResult result = blockTree.Insert(block, BlockTreeInsertBlockOptions.SaveHeader | BlockTreeInsertBlockOptions.SkipCanAcceptNewBlocks, headerOptions, bodiesWriteFlags: WriteFlags.DisableWAL); + if (result is not AddBlockResult.Added and not AddBlockResult.AlreadyKnown) + if (_logger.IsWarn) _logger.Warn($"Unexpected block tree insert result {result} for block {block.Number}."); + } + else if (!block.Header.IsPostMerge && existing.TotalDifficulty is null) + { + // Block body already exists (e.g. downloaded during snap sync ancient-bodies phase) but + // TotalDifficulty was not stored at that time. Re-insert so the block tree stores the + // correct TD — either from the era file (if available) or computed via SetTotalDifficulty. + AddBlockResult result = blockTree.Insert(block, BlockTreeInsertBlockOptions.SaveHeader | BlockTreeInsertBlockOptions.SkipCanAcceptNewBlocks, bodiesWriteFlags: WriteFlags.DisableWAL); + if (result is not AddBlockResult.Added and not AddBlockResult.AlreadyKnown) + if (_logger.IsWarn) _logger.Warn($"Unexpected block tree insert result {result} for block {block.Number} (TD re-insert)."); + } + if (!receiptStorage.HasBlock(block.Number, block.Hash!)) + receiptStorage.Insert(block, receipts, true, writeFlags: WriteFlags.DisableWAL, lastBlockNumber: lastBlockNumber); + } + + private async Task SuggestAndProcessBlock(Block block) + { + block.Header.TotalDifficulty = null; + + if (!blockValidator.ValidateBodyAgainstHeader(block.Header, block.Body, out string? error)) + throw new EraVerificationException($"Block validation failed: {error}"); + + AddBlockResult addResult = await blockTree.SuggestBlockAsync(block, BlockTreeSuggestOptions.ShouldProcess); + switch (addResult) + { + case AddBlockResult.AlreadyKnown: + return; + case AddBlockResult.CannotAccept: + throw new EraImportException("Block rejected from EraE archive."); + case AddBlockResult.UnknownParent: + throw new EraImportException("Unknown parent for block in EraE archive."); + case AddBlockResult.InvalidBlock: + throw new EraImportException("Invalid block in EraE archive."); + case AddBlockResult.Added: + break; + default: + throw new NotSupportedException($"Unexpected {nameof(AddBlockResult)}: {addResult}"); + } + } +} diff --git a/src/Nethermind/Nethermind.EraE/Import/IEraImporter.cs b/src/Nethermind/Nethermind.EraE/Import/IEraImporter.cs new file mode 100644 index 000000000000..bf438d67ec49 --- /dev/null +++ b/src/Nethermind/Nethermind.EraE/Import/IEraImporter.cs @@ -0,0 +1,9 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +namespace Nethermind.EraE.Import; + +public interface IEraImporter +{ + Task Import(string src, long from, long to, string? accumulatorFile = null, CancellationToken cancellation = default); +} diff --git a/src/Nethermind/Nethermind.EraE/InternalsVisibility.cs b/src/Nethermind/Nethermind.EraE/InternalsVisibility.cs new file mode 100644 index 000000000000..53ccf42eeb94 --- /dev/null +++ b/src/Nethermind/Nethermind.EraE/InternalsVisibility.cs @@ -0,0 +1,6 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("Nethermind.EraE.Test")] diff --git a/src/Nethermind/Nethermind.EraE/JsonRpc/EraAdminRpcModule.cs b/src/Nethermind/Nethermind.EraE/JsonRpc/EraAdminRpcModule.cs new file mode 100644 index 000000000000..1697ee715653 --- /dev/null +++ b/src/Nethermind/Nethermind.EraE/JsonRpc/EraAdminRpcModule.cs @@ -0,0 +1,16 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.EraE.Admin; +using Nethermind.JsonRpc; + +namespace Nethermind.EraE.JsonRpc; + +public class EraAdminRpcModule(IAdminEraService eraService) : IEraAdminRpcModule +{ + public Task> admin_exportEraHistory(string destinationPath, long from, long to) => + Task.FromResult(eraService.ExportHistory(destinationPath, from, to)); + + public Task> admin_importEraHistory(string sourcePath, long from = 0, long to = 0, string? accumulatorFile = null) => + Task.FromResult(eraService.ImportHistory(sourcePath, from, to, accumulatorFile)); +} diff --git a/src/Nethermind/Nethermind.EraE/JsonRpc/IEraAdminRpcModule.cs b/src/Nethermind/Nethermind.EraE/JsonRpc/IEraAdminRpcModule.cs new file mode 100644 index 000000000000..5cd32bd722ab --- /dev/null +++ b/src/Nethermind/Nethermind.EraE/JsonRpc/IEraAdminRpcModule.cs @@ -0,0 +1,41 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.JsonRpc; +using Nethermind.JsonRpc.Modules; + +namespace Nethermind.EraE.JsonRpc; + +[RpcModule(ModuleType.Admin)] +public interface IEraAdminRpcModule : IRpcModule +{ + [JsonRpcMethod( + Description = "Exports a range of historic blocks in erae format.", + EdgeCaseHint = "", + ExampleResponse = "\"Export task started.\"", + IsImplemented = true)] + Task> admin_exportEraHistory( + [JsonRpcParameter(Description = "Destination path to export to.", ExampleValue = "/tmp/eraeexportdir")] + string destinationPath, + [JsonRpcParameter(Description = "Start block to export from.", ExampleValue = "0")] + long from, + [JsonRpcParameter(Description = "Last block to export to. Set to 0 to export to head.", ExampleValue = "1000000")] + long to + ); + + [JsonRpcMethod( + Description = "Imports a range of historic blocks from an erae directory.", + EdgeCaseHint = "", + ExampleResponse = "\"Import task started.\"", + IsImplemented = true)] + Task> admin_importEraHistory( + [JsonRpcParameter(Description = "Source path to import from.", ExampleValue = "/tmp/eraedir")] + string sourcePath, + [JsonRpcParameter(Description = "Start block to import. Set to 0 for first available.", ExampleValue = "0")] + long from = 0, + [JsonRpcParameter(Description = "End block to import. Set to 0 for last available.", ExampleValue = "0")] + long to = 0, + [JsonRpcParameter(Description = "Accumulator file to trust. Set to null to skip accumulator verification.", ExampleValue = "null")] + string? accumulatorFile = null + ); +} diff --git a/src/Nethermind/Nethermind.EraE/Nethermind.EraE.csproj b/src/Nethermind/Nethermind.EraE/Nethermind.EraE.csproj new file mode 100644 index 000000000000..f9bb62105b5a --- /dev/null +++ b/src/Nethermind/Nethermind.EraE/Nethermind.EraE.csproj @@ -0,0 +1,26 @@ + + + + enable + enable + + + + + + + + + + + + + + + + + + + + + diff --git a/src/Nethermind/Nethermind.EraE/Proofs/BeaconApiHttpClient.cs b/src/Nethermind/Nethermind.EraE/Proofs/BeaconApiHttpClient.cs new file mode 100644 index 000000000000..cf5a8f1a3dd4 --- /dev/null +++ b/src/Nethermind/Nethermind.EraE/Proofs/BeaconApiHttpClient.cs @@ -0,0 +1,48 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Net.Http.Headers; +using System.Text.Json; +using Nethermind.Logging; + +namespace Nethermind.EraE.Proofs; + +internal sealed class BeaconApiHttpClient(HttpClient? httpClient, TimeSpan requestTimeout, ILogger logger = default) + : IDisposable +{ + private readonly HttpClient _httpClient = httpClient ?? new HttpClient(); + private readonly bool _ownsHttpClient = httpClient is null; + + public async Task GetAsync(Uri uri, CancellationToken cancellationToken) + { + using CancellationTokenSource cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + cts.CancelAfter(requestTimeout); + + using HttpRequestMessage request = new(HttpMethod.Get, uri); + request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); + + using HttpResponseMessage response = await _httpClient.SendAsync( + request, HttpCompletionOption.ResponseHeadersRead, cts.Token).ConfigureAwait(false); + + if (!response.IsSuccessStatusCode) + { + if ((int)response.StatusCode >= 500) + throw new HttpRequestException($"Beacon API returned {(int)response.StatusCode} for {uri}."); + + // 404 is expected for missed slots; other 4xx (401, 403, 429) indicate misconfiguration or quota issues. + if (response.StatusCode != System.Net.HttpStatusCode.NotFound) + logger.Warn($"Beacon API returned {(int)response.StatusCode} for {uri}. Beacon roots will not be included for this slot."); + + return null; + } + + await using Stream stream = await response.Content.ReadAsStreamAsync(cts.Token).ConfigureAwait(false); + return await JsonDocument.ParseAsync(stream, cancellationToken: cts.Token).ConfigureAwait(false); + } + + public void Dispose() + { + if (_ownsHttpClient) + _httpClient.Dispose(); + } +} diff --git a/src/Nethermind/Nethermind.EraE/Proofs/BeaconApiRetry.cs b/src/Nethermind/Nethermind.EraE/Proofs/BeaconApiRetry.cs new file mode 100644 index 000000000000..bfa6ddb3bc0d --- /dev/null +++ b/src/Nethermind/Nethermind.EraE/Proofs/BeaconApiRetry.cs @@ -0,0 +1,29 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Polly; +using Polly.Retry; + +namespace Nethermind.EraE.Proofs; + +internal sealed class BeaconApiRetry +{ + private readonly ResiliencePipeline _pipeline; + + internal BeaconApiRetry(int maxAttempts) => + _pipeline = new ResiliencePipelineBuilder() + .AddRetry(new RetryStrategyOptions + { + MaxRetryAttempts = maxAttempts - 1, + ShouldHandle = new PredicateBuilder() + .Handle() + .Handle() + .Handle(), + DelayGenerator = args => ValueTask.FromResult( + TimeSpan.FromSeconds(Math.Pow(2, args.AttemptNumber + 1))) + }) + .Build(); + + internal Task ExecuteAsync(Func> operation, CancellationToken cancellationToken) => + _pipeline.ExecuteAsync(ct => new ValueTask(operation(ct)), cancellationToken).AsTask(); +} diff --git a/src/Nethermind/Nethermind.EraE/Proofs/BeaconApiRootsProvider.cs b/src/Nethermind/Nethermind.EraE/Proofs/BeaconApiRootsProvider.cs new file mode 100644 index 000000000000..4756ae426cb3 --- /dev/null +++ b/src/Nethermind/Nethermind.EraE/Proofs/BeaconApiRootsProvider.cs @@ -0,0 +1,76 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Text.Json; +using Nethermind.Core.Crypto; +using Nethermind.Logging; + +namespace Nethermind.EraE.Proofs; + +public sealed class BeaconApiRootsProvider( + Uri baseUrl, + HttpClient? httpClient = null, + TimeSpan? requestTimeout = null, + int maxAttempts = 3, + ILogManager? logManager = null) + : IBeaconRootsProvider +{ + private readonly BeaconApiHttpClient _client = new(httpClient, requestTimeout ?? TimeSpan.FromSeconds(30), logManager?.GetClassLogger() ?? default); + private readonly BeaconApiRetry<(ValueHash256, ValueHash256)?> _beaconApiRetry = new(maxAttempts); + + public void Dispose() => _client.Dispose(); + + public Task<(ValueHash256 BeaconBlockRoot, ValueHash256 StateRoot)?> GetBeaconRoots( + long slot, CancellationToken cancellationToken = default) => + FetchWithRetryAsync(slot, cancellationToken); + + private Task<(ValueHash256, ValueHash256)?> FetchWithRetryAsync( + long slot, CancellationToken cancellationToken) => + _beaconApiRetry.ExecuteAsync(async ct => + { + ValueHash256? blockRoot = await FetchBlockRootAsync(slot, ct).ConfigureAwait(false); + if (!blockRoot.HasValue) + return null; + + ValueHash256? stateRoot = await FetchStateRootAsync(slot, ct).ConfigureAwait(false); + if (!stateRoot.HasValue) + return null; + + return (blockRoot.Value, stateRoot.Value); + }, cancellationToken); + + private async Task FetchBlockRootAsync(long slot, CancellationToken cancellationToken) + { + Uri uri = new(baseUrl, $"/eth/v1/beacon/headers/{slot}"); + using JsonDocument? document = await _client.GetAsync(uri, cancellationToken).ConfigureAwait(false); + return document is null ? null : ExtractHash(document.RootElement, ["data", "root"]); + } + + private async Task FetchStateRootAsync(long slot, CancellationToken cancellationToken) + { + Uri uri = new(baseUrl, $"/eth/v1/beacon/states/{slot}/root"); + using JsonDocument? document = await _client.GetAsync(uri, cancellationToken).ConfigureAwait(false); + return document is null ? null : ExtractHash(document.RootElement, ["data", "root"]); + } + + private static ValueHash256? ExtractHash(JsonElement root, string[] path) + { + JsonElement element = root; + foreach (string key in path) + { + if (!element.TryGetProperty(key, out JsonElement next)) + return null; + element = next; + } + + if (element.ValueKind != JsonValueKind.String) + return null; + + string? hex = element.GetString(); + if (string.IsNullOrWhiteSpace(hex)) + return null; + + try { return new ValueHash256(hex); } + catch (FormatException) { return null; } + } +} diff --git a/src/Nethermind/Nethermind.EraE/Proofs/BlockProofs.cs b/src/Nethermind/Nethermind.EraE/Proofs/BlockProofs.cs new file mode 100644 index 000000000000..2315e08f81df --- /dev/null +++ b/src/Nethermind/Nethermind.EraE/Proofs/BlockProofs.cs @@ -0,0 +1,50 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Core.Crypto; +using Nethermind.Serialization.Ssz; + +namespace Nethermind.EraE.Proofs; + +// Per the Ethereum beacon chain spec, HistoricalBatch.block_roots and .state_roots are +// fixed-length vectors of SLOTS_PER_HISTORICAL_ROOT = 8192 entries (one per slot in a period). +[SszContainer] +partial struct HistoricalBatch +{ + [SszVector(8192)] // SLOTS_PER_HISTORICAL_ROOT + public SszBytes32[] BlockRoots { get; set; } + + [SszVector(8192)] // SLOTS_PER_HISTORICAL_ROOT + public SszBytes32[] StateRoots { get; set; } + + public static HistoricalBatch From(ReadOnlySpan blockRoots, ReadOnlySpan stateRoots) + { + SszBytes32[] blockRootsData = new SszBytes32[blockRoots.Length]; + for (int i = 0; i < blockRoots.Length; i++) blockRootsData[i] = SszBytes32.From(blockRoots[i]); + SszBytes32[] stateRootsData = new SszBytes32[stateRoots.Length]; + for (int i = 0; i < stateRoots.Length; i++) stateRootsData[i] = SszBytes32.From(stateRoots[i]); + return new() { BlockRoots = blockRootsData, StateRoots = stateRootsData }; + } +} + +[SszContainer] +partial struct ValueHash256Vector +{ + [SszVector(8192)] // SLOTS_PER_HISTORICAL_ROOT + public SszBytes32[] Data { get; set; } + + public static ValueHash256Vector From(ReadOnlySpan hashesAccumulator) + { + SszBytes32[] data = new SszBytes32[hashesAccumulator.Length]; + for (int i = 0; i < hashesAccumulator.Length; i++) data[i] = SszBytes32.From(hashesAccumulator[i]); + return new() { Data = data }; + } + + public readonly ValueHash256[] Hashes() + { + ValueHash256[] result = new ValueHash256[Data.Length]; + for (int i = 0; i < Data.Length; i++) result[i] = Data[i].Hash; + return result; + } +} + diff --git a/src/Nethermind/Nethermind.EraE/Proofs/BlocksRootContext.cs b/src/Nethermind/Nethermind.EraE/Proofs/BlocksRootContext.cs new file mode 100644 index 000000000000..7d2b675d1cd5 --- /dev/null +++ b/src/Nethermind/Nethermind.EraE/Proofs/BlocksRootContext.cs @@ -0,0 +1,125 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Runtime.InteropServices; +using Nethermind.Core; +using Nethermind.Core.Collections; +using Nethermind.Core.Crypto; +using Nethermind.Core.Specs; +using AccumulatorCalculator = Nethermind.Era1.AccumulatorCalculator; +using Nethermind.Int256; + +namespace Nethermind.EraE.Proofs; + +public enum AccumulatorType +{ + HistoricalHashesAccumulator, + HistoricalRoots, + HistoricalSummaries +} + +public sealed class BlocksRootContext : IDisposable +{ + private readonly ArrayPoolList _blockRoots = new(8192); + private readonly ArrayPoolList _stateRoots = new(8192); + private readonly ArrayPoolList<(Hash256 Hash, UInt256 Td)> _blockHashes = new(8192); + + public AccumulatorType AccumulatorType { get; } + + private AccumulatorCalculator? _accumulatorCalculator; + private ValueHash256? _accumulatorRoot; + private HistoricalSummary? _historicalSummary; + private ValueHash256? _historicalRoot; + + public bool Populated { get; private set; } + public long StartingBlockNumber { get; } + public ulong? StartingBlockTimestamp { get; } + + public ValueHash256 AccumulatorRoot => + _accumulatorRoot ?? throw new InvalidOperationException("Accumulator root not set or not finalized."); + + public HistoricalSummary HistoricalSummary => + _historicalSummary ?? throw new InvalidOperationException("Historical summary not set or not finalized."); + + public ValueHash256 HistoricalRoot => + _historicalRoot ?? throw new InvalidOperationException("Historical root not set or not finalized."); + + public BlocksRootContext(long startingBlockNumber, ulong? startingBlockTimestamp = null, ISpecProvider? specProvider = null) + { + StartingBlockNumber = startingBlockNumber; + StartingBlockTimestamp = startingBlockTimestamp; + ForkActivation forkActivation = new(startingBlockNumber, startingBlockTimestamp); + AccumulatorType = GetAccumulatorType(forkActivation, specProvider); + } + + public void ProcessBlock(Block block, ValueHash256? beaconBlockRoot = null, ValueHash256? stateRoot = null) + { + switch (AccumulatorType) + { + case AccumulatorType.HistoricalHashesAccumulator: + // Only track pre-merge blocks in the accumulator. + if (!block.Header.IsPostMerge) + _blockHashes.Add((block.Header.Hash!, block.TotalDifficulty!.Value)); + break; + + case AccumulatorType.HistoricalRoots: + case AccumulatorType.HistoricalSummaries: + // Post-merge: collect beacon block roots and state roots per slot. + // Missed slots are represented by zero hashes (default ValueHash256). + _blockRoots.Add(beaconBlockRoot ?? default); + _stateRoots.Add(stateRoot ?? default); + break; + } + Populated = true; + } + + public void FinalizeContext() + { + if (!Populated) return; + + switch (AccumulatorType) + { + case AccumulatorType.HistoricalHashesAccumulator: + _accumulatorCalculator = new AccumulatorCalculator(); + foreach ((Hash256 hash, UInt256 td) in _blockHashes.AsSpan()) + _accumulatorCalculator.Add(hash, td); + _accumulatorRoot = _accumulatorCalculator.ComputeRoot(); + break; + + case AccumulatorType.HistoricalRoots: + HistoricalBatch.Merkleize( + HistoricalBatch.From(_blockRoots.AsSpan(), _stateRoots.AsSpan()), + out UInt256 historicalRoot); + _historicalRoot = UInt256ToHash(ref historicalRoot); + break; + + case AccumulatorType.HistoricalSummaries: + ValueHash256Vector.Merkleize(ValueHash256Vector.From(_blockRoots.AsSpan()), out UInt256 blockRoot); + ValueHash256Vector.Merkleize(ValueHash256Vector.From(_stateRoots.AsSpan()), out UInt256 stateRoot); + _historicalSummary = new HistoricalSummary( + UInt256ToHash(ref blockRoot), + UInt256ToHash(ref stateRoot)); + break; + } + } + + public void Dispose() + { + _accumulatorCalculator?.Dispose(); + _blockRoots.Dispose(); + _stateRoots.Dispose(); + _blockHashes.Dispose(); + } + + private static AccumulatorType GetAccumulatorType(ForkActivation forkActivation, ISpecProvider? specProvider) => + specProvider switch + { + null => AccumulatorType.HistoricalHashesAccumulator, + _ when specProvider.GetSpec(forkActivation).IsEip4895Enabled => AccumulatorType.HistoricalSummaries, + _ when specProvider.MergeBlockNumber is { BlockNumber: var merge } && forkActivation.BlockNumber >= merge => AccumulatorType.HistoricalRoots, + _ => AccumulatorType.HistoricalHashesAccumulator + }; + + private static ValueHash256 UInt256ToHash(ref UInt256 value) => + new(MemoryMarshal.Cast(MemoryMarshal.CreateSpan(ref value, 1))); +} diff --git a/src/Nethermind/Nethermind.EraE/Proofs/HistoricalSummariesRpcProvider.cs b/src/Nethermind/Nethermind.EraE/Proofs/HistoricalSummariesRpcProvider.cs new file mode 100644 index 000000000000..d895b84b26b4 --- /dev/null +++ b/src/Nethermind/Nethermind.EraE/Proofs/HistoricalSummariesRpcProvider.cs @@ -0,0 +1,99 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Text.Json; +using Nethermind.Logging; + +namespace Nethermind.EraE.Proofs; + +public sealed class HistoricalSummariesRpcProvider( + Uri baseUrl, + HttpClient? httpClient = null, + TimeSpan? requestTimeout = null, + int maxAttempts = 3, + ILogManager? logManager = null) : IHistoricalSummariesProvider +{ + private readonly BeaconApiHttpClient _beaconApiHttpClient = new(httpClient, requestTimeout ?? TimeSpan.FromSeconds(30), logManager?.GetClassLogger() ?? default); + private readonly BeaconApiRetry _beaconApiRetry = new(maxAttempts); + private readonly SemaphoreSlim _lock = new(1, 1); + private const string Endpoint = "/eth/v2/debug/beacon/states"; + + private HistoricalSummary[] _cachedSummaries = []; + + public void Dispose() + { + _beaconApiHttpClient.Dispose(); + _lock.Dispose(); + } + + public async Task GetHistoricalSummary( + int index, + bool forceRefresh = false, + CancellationToken cancellationToken = default) + { + HistoricalSummary[] summaries = await GetHistoricalSummariesAsync("head", forceRefresh, cancellationToken).ConfigureAwait(false); + return summaries.Length > index ? summaries[index] : null; + } + + public async Task GetHistoricalSummariesAsync( + string stateId = "head", + bool forceRefresh = false, + CancellationToken cancellationToken = default) + { + if (_cachedSummaries.Length > 0 && !forceRefresh) + return _cachedSummaries; + + await _lock.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + if (_cachedSummaries.Length > 0 && !forceRefresh) + return _cachedSummaries; + + _cachedSummaries = await LoadWithRetryAsync(stateId, cancellationToken).ConfigureAwait(false); + return _cachedSummaries; + } + finally + { + _lock.Release(); + } + } + + private Task LoadWithRetryAsync( + string stateId, + CancellationToken cancellationToken) => + _beaconApiRetry.ExecuteAsync( + ct => LoadHistoricalSummariesAsync(stateId, ct), + cancellationToken); + + private async Task LoadHistoricalSummariesAsync( + string stateId, + CancellationToken cancellationToken) + { + Uri fullUri = new(baseUrl, $"{Endpoint}/{stateId}"); + using JsonDocument? document = await _beaconApiHttpClient.GetAsync(fullUri, cancellationToken).ConfigureAwait(false); + return document is null ? [] : ParseHistoricalSummaries(document.RootElement); + } + + private static HistoricalSummary[] ParseHistoricalSummaries(JsonElement root) + { + if (!root.TryGetProperty("data", out JsonElement data) + || !data.TryGetProperty("historical_summaries", out JsonElement historicalSummaries) + || historicalSummaries.ValueKind != JsonValueKind.Array) + return []; + + List summaries = new(historicalSummaries.GetArrayLength()); + foreach (JsonElement summary in historicalSummaries.EnumerateArray()) + { + if (summary.TryGetProperty("block_summary_root", out JsonElement blockSummaryRootEl) + && summary.TryGetProperty("state_summary_root", out JsonElement stateSummaryRootEl)) + { + string? blockSummaryRoot = blockSummaryRootEl.GetString(); + string? stateSummaryRoot = stateSummaryRootEl.GetString(); + if (blockSummaryRoot is not null && stateSummaryRoot is not null) + summaries.Add(HistoricalSummary.From(blockSummaryRoot, stateSummaryRoot)); + } + } + + return [.. summaries]; + } +} diff --git a/src/Nethermind/Nethermind.EraE/Proofs/IBeaconRootsProvider.cs b/src/Nethermind/Nethermind.EraE/Proofs/IBeaconRootsProvider.cs new file mode 100644 index 000000000000..cdcab52847ad --- /dev/null +++ b/src/Nethermind/Nethermind.EraE/Proofs/IBeaconRootsProvider.cs @@ -0,0 +1,20 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Core.Crypto; + +namespace Nethermind.EraE.Proofs; + +/// +/// Provides beacon block roots and state roots by beacon slot. +/// Used during EraE export to generate post-merge block proofs. +/// +public interface IBeaconRootsProvider : IDisposable +{ + /// + /// Returns the beacon block root and state root for the given beacon slot, + /// or null if the data is unavailable. + /// + Task<(ValueHash256 BeaconBlockRoot, ValueHash256 StateRoot)?> GetBeaconRoots( + long slot, CancellationToken cancellationToken = default); +} diff --git a/src/Nethermind/Nethermind.EraE/Proofs/IHistoricalSummariesProvider.cs b/src/Nethermind/Nethermind.EraE/Proofs/IHistoricalSummariesProvider.cs new file mode 100644 index 000000000000..d78449b55667 --- /dev/null +++ b/src/Nethermind/Nethermind.EraE/Proofs/IHistoricalSummariesProvider.cs @@ -0,0 +1,21 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Core.Crypto; + +namespace Nethermind.EraE.Proofs; + +public readonly struct HistoricalSummary(ValueHash256 blockSummaryRoot, ValueHash256 stateSummaryRoot) +{ + public ValueHash256 BlockSummaryRoot { get; } = blockSummaryRoot; + public ValueHash256 StateSummaryRoot { get; } = stateSummaryRoot; + + public static HistoricalSummary From(string blockSummaryRootHex, string stateSummaryRootHex) => + new(new ValueHash256(blockSummaryRootHex), new ValueHash256(stateSummaryRootHex)); +} + +public interface IHistoricalSummariesProvider : IDisposable +{ + Task GetHistoricalSummary(int index, bool forceRefresh = false, CancellationToken cancellationToken = default); + Task GetHistoricalSummariesAsync(string stateId = "head", bool forceRefresh = false, CancellationToken cancellationToken = default); +} diff --git a/src/Nethermind/Nethermind.EraE/Proofs/NullBeaconRootsProvider.cs b/src/Nethermind/Nethermind.EraE/Proofs/NullBeaconRootsProvider.cs new file mode 100644 index 000000000000..efaff8eee155 --- /dev/null +++ b/src/Nethermind/Nethermind.EraE/Proofs/NullBeaconRootsProvider.cs @@ -0,0 +1,19 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Core.Crypto; + +namespace Nethermind.EraE.Proofs; + +public sealed class NullBeaconRootsProvider : IBeaconRootsProvider +{ + public static readonly NullBeaconRootsProvider Instance = new(); + + private NullBeaconRootsProvider() { } + + public Task<(ValueHash256 BeaconBlockRoot, ValueHash256 StateRoot)?> GetBeaconRoots( + long slot, CancellationToken cancellationToken = default) => + Task.FromResult<(ValueHash256, ValueHash256)?>(null); + + public void Dispose() { } +} diff --git a/src/Nethermind/Nethermind.EraE/Proofs/NullHistoricalSummariesProvider.cs b/src/Nethermind/Nethermind.EraE/Proofs/NullHistoricalSummariesProvider.cs new file mode 100644 index 000000000000..f626f258a794 --- /dev/null +++ b/src/Nethermind/Nethermind.EraE/Proofs/NullHistoricalSummariesProvider.cs @@ -0,0 +1,19 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +namespace Nethermind.EraE.Proofs; + +public sealed class NullHistoricalSummariesProvider : IHistoricalSummariesProvider +{ + public static readonly NullHistoricalSummariesProvider Instance = new(); + + private NullHistoricalSummariesProvider() { } + + public Task GetHistoricalSummary(int index, bool forceRefresh = false, CancellationToken cancellationToken = default) => + Task.FromResult(null); + + public Task GetHistoricalSummariesAsync(string stateId = "head", bool forceRefresh = false, CancellationToken cancellationToken = default) => + Task.FromResult(Array.Empty()); + + public void Dispose() { } +} diff --git a/src/Nethermind/Nethermind.EraE/Proofs/Validator.cs b/src/Nethermind/Nethermind.EraE/Proofs/Validator.cs new file mode 100644 index 000000000000..15f34d881a34 --- /dev/null +++ b/src/Nethermind/Nethermind.EraE/Proofs/Validator.cs @@ -0,0 +1,105 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Config; +using Nethermind.Core; +using Nethermind.Core.Crypto; +using Nethermind.Core.Specs; +using EraVerificationException = Nethermind.Era1.Exceptions.EraVerificationException; + +namespace Nethermind.EraE.Proofs; + +public sealed class Validator +{ + private readonly ISpecProvider _specProvider; + private readonly IHistoricalSummariesProvider? _historicalSummariesProvider; + private readonly IReadOnlyList? _trustedAccumulators; + private readonly IReadOnlyList? _trustedHistoricalRoots; + private readonly SlotTime? _slotTime; + + private const int SlotsPerHistoricalRoot = 8192; + + public Validator( + ISpecProvider specProvider, + IReadOnlyList? trustedAccumulators, + IReadOnlyList? trustedHistoricalRoots, + IHistoricalSummariesProvider? historicalSummariesProvider, + IBlocksConfig? blocksConfig = null) + { + _specProvider = specProvider; + if (specProvider.BeaconChainGenesisTimestamp.HasValue) + { + ulong secondsPerSlot = blocksConfig?.SecondsPerSlot ?? 12; + _slotTime = new SlotTime( + specProvider.BeaconChainGenesisTimestamp.Value * 1000, + new Timestamper(), + TimeSpan.FromSeconds(secondsPerSlot), + TimeSpan.Zero); + } + _trustedAccumulators = trustedAccumulators; + _trustedHistoricalRoots = trustedHistoricalRoots; + _historicalSummariesProvider = historicalSummariesProvider; + } + + public bool VerifyAccumulator(long blockNumber, ValueHash256 accumulatorRoot) + { + if (!TrustedAccumulatorsProvided()) return true; + ValueHash256? trusted = GetAccumulatorForEpoch(blockNumber / SlotsPerHistoricalRoot); + return trusted is null + ? throw new EraVerificationException("Trusted accumulator root was not provided.") + : trusted.Equals(accumulatorRoot); + } + + public async Task VerifyBlocksRootContext(BlocksRootContext context, CancellationToken cancellation = default) + { + switch (context.AccumulatorType) + { + case AccumulatorType.HistoricalHashesAccumulator: + if (!VerifyAccumulator(context.StartingBlockNumber, context.AccumulatorRoot)) + throw new EraVerificationException("Computed accumulator does not match trusted accumulator."); + break; + + case AccumulatorType.HistoricalRoots: + if (_slotTime is null) throw new EraVerificationException("Beacon chain genesis timestamp is not available for HistoricalRoots verification."); + long slot = (long)_slotTime.GetSlot(context.StartingBlockTimestamp!.Value); + ValueHash256? trustedRoot = GetHistoricalRoot(slot) ?? throw new EraVerificationException("Historical root not found."); + if (!trustedRoot.Equals(context.HistoricalRoot)) + throw new EraVerificationException("Computed historical root does not match trusted historical root."); + break; + + case AccumulatorType.HistoricalSummaries: + if (_slotTime is null) throw new EraVerificationException("Beacon chain genesis timestamp is not available for HistoricalSummaries verification."); + long summarySlot = (long)_slotTime.GetSlot(context.StartingBlockTimestamp!.Value); + HistoricalSummary? trustedSummary = await GetHistoricalSummary(summarySlot, cancellation) ?? throw new EraVerificationException("Historical summary not found."); + if (!trustedSummary.Value.BlockSummaryRoot.Equals(context.HistoricalSummary.BlockSummaryRoot)) + throw new EraVerificationException("Computed block summary root does not match trusted historical block summary root."); + if (!trustedSummary.Value.StateSummaryRoot.Equals(context.HistoricalSummary.StateSummaryRoot)) + throw new EraVerificationException("Computed state summary root does not match trusted historical state summary root."); + break; + } + } + + private bool TrustedAccumulatorsProvided() => _trustedAccumulators is { Count: > 0 }; + + private ValueHash256? GetAccumulatorForEpoch(long epochIdx) => + _trustedAccumulators is not null && _trustedAccumulators.Count > epochIdx + ? _trustedAccumulators[(int)epochIdx] + : null; + + private ValueHash256? GetHistoricalRoot(long slotNumber) + { + long idx = slotNumber / SlotsPerHistoricalRoot; + return _trustedHistoricalRoots is not null && _trustedHistoricalRoots.Count > idx + ? _trustedHistoricalRoots[(int)idx] + : null; + } + + private async Task GetHistoricalSummary(long slotNumber, CancellationToken cancellation = default) + { + long idx = slotNumber / SlotsPerHistoricalRoot; + return _historicalSummariesProvider is null + ? null + : await _historicalSummariesProvider.GetHistoricalSummary((int)idx, cancellationToken: cancellation); + } + +} diff --git a/src/Nethermind/Nethermind.EraE/Store/EraStore.cs b/src/Nethermind/Nethermind.EraE/Store/EraStore.cs new file mode 100644 index 000000000000..93af11630326 --- /dev/null +++ b/src/Nethermind/Nethermind.EraE/Store/EraStore.cs @@ -0,0 +1,223 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Collections.Concurrent; +using System.IO.Abstractions; +using Nethermind.Consensus.Validators; +using Nethermind.Core; +using Nethermind.Core.Collections; +using Nethermind.Core.Crypto; +using Nethermind.Core.Specs; +using Nethermind.EraE.Archive; +using Nethermind.EraE.E2Store; +using EraException = Nethermind.Era1.EraException; +using EraVerificationException = Nethermind.Era1.Exceptions.EraVerificationException; +using Nethermind.EraE.Export; + +namespace Nethermind.EraE.Store; + +public sealed class EraStore : IEraStore +{ + private static readonly char[] _eraSeparator = ['-']; + + private readonly ISpecProvider _specProvider; + private readonly IBlockValidator _blockValidator; + private readonly ISet? _trustedAccumulators; + private readonly Proofs.Validator? _validator; + + private readonly Dictionary _epochs; + private readonly Dictionary _checksumsByEpoch = new(); + + private readonly ConcurrentDictionary _verifiedEpochs = new(); + private readonly ConcurrentDictionary _epochLocks = new(); + + private readonly int _maxOpenFile; + private readonly ConcurrentDictionary _openedReader = new(); + + private readonly int _maxEraSize; + private readonly int _verifyConcurrency; + private volatile bool _disposed; + + private readonly Lazy<(long First, long Last)> _blockRange; + + public (long First, long Last) BlockRange => _blockRange.Value; + + private int LastEpoch { get; } + private int FirstEpoch { get; } = int.MaxValue; + + public EraStore( + ISpecProvider specProvider, + IBlockValidator blockValidator, + IFileSystem fileSystem, + string networkName, + int maxEraSize, + ISet? trustedAccumulators, + string directory, + int verifyConcurrency = 0, + Proofs.Validator? validator = null) + { + ArgumentNullException.ThrowIfNull(specProvider); + ArgumentNullException.ThrowIfNull(blockValidator); + ArgumentException.ThrowIfNullOrEmpty(networkName); + ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(maxEraSize, 0); + + _specProvider = specProvider; + _blockValidator = blockValidator; + _trustedAccumulators = trustedAccumulators; + _validator = validator; + _maxEraSize = maxEraSize; + _maxOpenFile = Environment.ProcessorCount * 2; + _verifyConcurrency = verifyConcurrency == 0 ? Environment.ProcessorCount : verifyConcurrency; + + foreach (string line in fileSystem.File.ReadAllLines(Path.Join(directory, EraExporter.ChecksumsSHA256FileName))) + { + (long checksumEpoch, ValueHash256 hash) = EraPathUtils.ParseChecksumEntry(line); + _checksumsByEpoch[checksumEpoch] = hash; + } + + bool hasEraFile = false; + _epochs = []; + + foreach (string file in EraPathUtils.GetAllEraFiles(directory, networkName, fileSystem)) + { + string[] parts = Path.GetFileNameWithoutExtension(file).Split(_eraSeparator); + if (parts.Length != 3 || !int.TryParse(parts[1], out int epoch) || epoch < 0) + throw new ArgumentException($"Malformed EraE file '{file}'.", file); + + _epochs[epoch] = file; + hasEraFile = true; + if (epoch > LastEpoch) LastEpoch = epoch; + if (epoch < FirstEpoch) FirstEpoch = epoch; + } + + if (!hasEraFile) + throw new EraException($"No relevant erae files in directory {directory}."); + + _blockRange = new Lazy<(long, long)>(() => + { + using EraRenter f = RentReader(FirstEpoch, out EraReader firstReader); + using EraRenter l = RentReader(LastEpoch, out EraReader lastReader); + return (firstReader.FirstBlock, lastReader.LastBlock); + }); + } + + private long GetEpochNumber(long blockNumber) => blockNumber / _maxEraSize; + + private EraReader GetReader(long epoch) => !_epochs.TryGetValue(epoch, out string? path) + ? throw new ArgumentOutOfRangeException(nameof(epoch), epoch, "Epoch not available.") + : new EraReader(new E2StoreReader(path)); + + private async ValueTask EnsureEpochVerified(long epoch, EraReader reader, CancellationToken cancellation) + { + if (_verifiedEpochs.TryGetValue(epoch, out bool verified) && verified) return; + + SemaphoreSlim epochLock = _epochLocks.GetOrAddDisposable(epoch, static _ => new SemaphoreSlim(1, 1)); + await epochLock.WaitAsync(cancellation).ConfigureAwait(false); + try + { + // Double-check: another worker may have verified this epoch while we were waiting + if (_verifiedEpochs.TryGetValue(epoch, out verified) && verified) return; + + // Both tasks share the same EraReader. E2StoreReader uses RandomAccess (positional I/O) + // so concurrent reads from different offsets are safe without additional locking. + Task checksumTask = Task.Run(() => + { + ValueHash256 actual = reader.CalculateChecksum(); + if (!_checksumsByEpoch.TryGetValue(epoch, out ValueHash256 expected)) + throw new EraVerificationException($"No checksum entry found for epoch {epoch}."); + if (actual != expected) + throw new EraVerificationException($"Checksum mismatch for epoch {epoch}. Got {actual}, expected {expected}."); + }, cancellation); + + Task accumulatorTask = Task.Run(async () => + { + ValueHash256 accRoot = await reader.VerifyContent(_specProvider, _blockValidator, _verifyConcurrency, _validator, cancellation).ConfigureAwait(false); + if (_trustedAccumulators != null && accRoot != default && !_trustedAccumulators.Contains(accRoot)) + throw new EraVerificationException($"AccumulatorRoot {accRoot} for epoch {epoch} is not trusted."); + }, cancellation); + + await Task.WhenAll(checksumTask, accumulatorTask).ConfigureAwait(false); + _verifiedEpochs.TryAdd(epoch, true); + } + finally + { + epochLock.Release(); + } + } + + public bool HasEpoch(long blockNumber) => _epochs.ContainsKey(GetEpochNumber(blockNumber)); + + public long NextEraStart(long blockNumber) + { + long epoch = GetEpochNumber(blockNumber); + using EraRenter _ = RentReader(epoch, out EraReader reader); + return reader.LastBlock + 1; + } + + public async Task<(Block?, TxReceipt[]?)> FindBlockAndReceipts(long number, bool ensureValidated = true, CancellationToken cancellation = default) + { + ArgumentOutOfRangeException.ThrowIfNegative(number); + + long epoch = GetEpochNumber(number); + if (!_epochs.ContainsKey(epoch)) + return (null, null); + + using EraRenter _ = RentReader(epoch, out EraReader reader); + if (ensureValidated) await EnsureEpochVerified(epoch, reader, cancellation); + (Block b, TxReceipt[] r) = await reader.GetBlockByNumber(number, cancellation); + return (b, r); + } + + private EraRenter RentReader(long epoch, out EraReader reader) + { + int shardIdx = (int)(epoch % _maxOpenFile); + if (_openedReader.TryRemove(shardIdx, out (long Epoch, EraReader Reader) opened)) + { + if (opened.Epoch == epoch) + { + reader = opened.Reader; + return new EraRenter(this, reader, epoch); + } + + if (!_openedReader.TryAdd(shardIdx, opened)) + opened.Reader.Dispose(); + } + + reader = GetReader(epoch); + return new EraRenter(this, reader, epoch); + } + + private void ReturnReader(long epoch, EraReader reader) + { + if (!_disposed) + { + int shardIdx = (int)(epoch % _maxOpenFile); + + if (_openedReader.TryAdd(shardIdx, (epoch, reader))) return; + + if (_openedReader.TryRemove(shardIdx, out (long Epoch, EraReader Reader) existing)) + existing.Reader.Dispose(); + + if (_openedReader.TryAdd(shardIdx, (epoch, reader))) return; + } + + reader.Dispose(); + } + + private readonly struct EraRenter(EraStore store, EraReader reader, long epoch) : IDisposable + { + public void Dispose() => store.ReturnReader(epoch, reader); + } + + public void Dispose() + { + if (_disposed) return; + _disposed = true; + + foreach (KeyValuePair kv in _openedReader) + kv.Value.Reader.Dispose(); + + foreach (KeyValuePair kv in _epochLocks) + kv.Value.Dispose(); + } +} diff --git a/src/Nethermind/Nethermind.EraE/Store/EraStoreFactory.cs b/src/Nethermind/Nethermind.EraE/Store/EraStoreFactory.cs new file mode 100644 index 000000000000..44bc2eb366d5 --- /dev/null +++ b/src/Nethermind/Nethermind.EraE/Store/EraStoreFactory.cs @@ -0,0 +1,60 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.IO.Abstractions; +using Nethermind.Consensus.Validators; +using Nethermind.Core.Crypto; +using Nethermind.Core.Specs; +using Nethermind.EraE.Config; +using EraException = Nethermind.Era1.EraException; +using Nethermind.EraE.Proofs; +using Nethermind.Logging; + +namespace Nethermind.EraE.Store; + +public sealed class EraStoreFactory( + ISpecProvider specProvider, + IBlockValidator blockValidator, + IFileSystem fileSystem, + IEraEConfig eraConfig, + ILogManager logManager, + Validator? validator = null, + IRemoteEraClient? remoteClient = null +) : IEraStoreFactory +{ + private readonly ILogger _logger = logManager.GetClassLogger(); + + public IEraStore Create(string src, ISet? trustedAccumulators) + { + IEraStore? localStore = null; + try + { + localStore = new EraStore( + specProvider, + blockValidator, + fileSystem, + eraConfig.NetworkName!, + eraConfig.MaxEraSize, + trustedAccumulators, + src, + eraConfig.Concurrency, + validator); + } + catch (Exception e) when (remoteClient is not null && e is EraException or FileNotFoundException or DirectoryNotFoundException) + { + if (_logger.IsDebug) _logger.Debug($"No local EraE files found in '{src}': {e.Message}. Remote will supply them on demand."); + } + + if (remoteClient is null) + { + return localStore ?? throw new EraException($"No eraE files found in '{src}' and no remote URL is configured."); + } + + string downloadDir = !string.IsNullOrWhiteSpace(eraConfig.RemoteDownloadDirectory) + ? eraConfig.RemoteDownloadDirectory + : src; + + fileSystem.Directory.CreateDirectory(downloadDir); + return new RemoteEraStoreDecorator(localStore, remoteClient, downloadDir, eraConfig.MaxEraSize); + } +} diff --git a/src/Nethermind/Nethermind.EraE/Store/HttpRemoteEraClient.cs b/src/Nethermind/Nethermind.EraE/Store/HttpRemoteEraClient.cs new file mode 100644 index 000000000000..de8ed39ff97d --- /dev/null +++ b/src/Nethermind/Nethermind.EraE/Store/HttpRemoteEraClient.cs @@ -0,0 +1,135 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using EraException = Nethermind.Era1.EraException; +using Nethermind.Logging; + +namespace Nethermind.EraE.Store; + +public sealed class HttpRemoteEraClient : IRemoteEraClient, IDisposable +{ + private readonly HttpClient _httpClient; + private readonly Uri _baseUrl; + private readonly string _manifestFilename; + private readonly bool _ownsHttpClient; + private readonly ILogger _logger; + + public HttpRemoteEraClient(Uri baseUrl, string manifestFilename, HttpClient? httpClient = null, ILogManager? logManager = null) + { + ArgumentNullException.ThrowIfNull(baseUrl); + ArgumentException.ThrowIfNullOrWhiteSpace(manifestFilename); + + _baseUrl = baseUrl.AbsoluteUri.EndsWith('/') ? baseUrl : new Uri(baseUrl.AbsoluteUri + "/"); + _manifestFilename = manifestFilename; + _ownsHttpClient = httpClient is null; + _httpClient = httpClient ?? new HttpClient(); + _logger = (logManager ?? NullLogManager.Instance).GetClassLogger(); + } + + public async Task> FetchManifestAsync(CancellationToken cancellation = default) + { + Uri manifestUri = new(_baseUrl, _manifestFilename); + + if (_logger.IsInfo) _logger.Info($"Fetching eraE manifest from {manifestUri}"); + + using HttpResponseMessage response = await _httpClient.GetAsync(manifestUri, HttpCompletionOption.ResponseHeadersRead, cancellation).ConfigureAwait(false); + + if (!response.IsSuccessStatusCode) + throw new EraException($"Failed to fetch eraE manifest from {manifestUri}: HTTP {(int)response.StatusCode} {response.ReasonPhrase}."); + + await using Stream stream = await response.Content.ReadAsStreamAsync(cancellation).ConfigureAwait(false); + using StreamReader reader = new(stream); + + Dictionary manifest = new(); + + while (await reader.ReadLineAsync(cancellation).ConfigureAwait(false) is { } line) + { + if (string.IsNullOrWhiteSpace(line)) continue; + + // Standard sha256sum format: "{hash} {filename}" (two spaces) + int separatorIdx = line.IndexOf(" ", StringComparison.Ordinal); + if (separatorIdx < 0) continue; + + string hashHex = line[..separatorIdx].Trim(); + string filename = line[(separatorIdx + 2)..].Trim(); + + if (!TryParseEpoch(filename, out int epoch)) continue; + if (!TryParseHex(hashHex, out byte[] sha256)) continue; + + manifest[epoch] = new RemoteEraEntry(filename, sha256); + } + + return manifest; + } + + public async Task DownloadFileAsync(string filename, string destinationPath, CancellationToken cancellation = default) + { + Uri fileUri = new(_baseUrl, filename); + string tmpPath = destinationPath + ".tmp"; + + string? destinationDir = Path.GetDirectoryName(destinationPath); + if (!string.IsNullOrEmpty(destinationDir)) + Directory.CreateDirectory(destinationDir); + + if (_logger.IsInfo) _logger.Info($"Downloading eraE file {filename} from {fileUri}"); + + try + { + using HttpResponseMessage response = await _httpClient.GetAsync(fileUri, HttpCompletionOption.ResponseHeadersRead, cancellation).ConfigureAwait(false); + + if (!response.IsSuccessStatusCode) + throw new EraException($"Failed to download eraE file '{filename}': HTTP {(int)response.StatusCode} {response.ReasonPhrase}."); + + long? contentLength = response.Content.Headers.ContentLength; + + await using Stream httpStream = await response.Content.ReadAsStreamAsync(cancellation).ConfigureAwait(false); + await using FileStream fileStream = new(tmpPath, FileMode.Create, FileAccess.Write, FileShare.None, bufferSize: 81920, useAsync: true); + await httpStream.CopyToAsync(fileStream, cancellation).ConfigureAwait(false); + + double mb = (contentLength ?? fileStream.Length) / 1_048_576.0; + if (_logger.IsInfo) _logger.Info($"Downloaded eraE file {filename} ({mb:F1} MB)"); + } + catch + { + if (File.Exists(tmpPath)) + File.Delete(tmpPath); + throw; + } + + File.Move(tmpPath, destinationPath, overwrite: true); + } + + private static bool TryParseEpoch(string filename, out int epoch) + { + epoch = 0; + // Expected: {network}-{epoch:05d}-{hash}.erae + ReadOnlySpan name = Path.GetFileNameWithoutExtension(filename.AsSpan()); + int first = name.IndexOf('-'); + if (first < 0) return false; + int second = name[(first + 1)..].IndexOf('-'); + if (second < 0) return false; + ReadOnlySpan epochPart = name[(first + 1)..(first + 1 + second)]; + return int.TryParse(epochPart, out epoch) && epoch >= 0; + } + + private static bool TryParseHex(string hex, out byte[] bytes) + { + bytes = []; + if (hex.Length % 2 != 0) return false; + try + { + bytes = Convert.FromHexString(hex); + return true; + } + catch (FormatException) + { + return false; + } + } + + public void Dispose() + { + if (_ownsHttpClient) + _httpClient.Dispose(); + } +} diff --git a/src/Nethermind/Nethermind.EraE/Store/IEraStore.cs b/src/Nethermind/Nethermind.EraE/Store/IEraStore.cs new file mode 100644 index 000000000000..20dfee04e9a0 --- /dev/null +++ b/src/Nethermind/Nethermind.EraE/Store/IEraStore.cs @@ -0,0 +1,17 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Core; + +namespace Nethermind.EraE.Store; + +public interface IEraStore : IDisposable +{ + Task<(Block?, TxReceipt[]?)> FindBlockAndReceipts(long number, bool ensureValidated = true, CancellationToken cancellation = default); + (long First, long Last) BlockRange { get; } + + bool HasEpoch(long blockNumber); + + /// Used for alignment when parallelizing imports so different tasks work on different files. + long NextEraStart(long blockNumber); +} diff --git a/src/Nethermind/Nethermind.EraE/Store/IEraStoreFactory.cs b/src/Nethermind/Nethermind.EraE/Store/IEraStoreFactory.cs new file mode 100644 index 000000000000..5c935c4eb6f4 --- /dev/null +++ b/src/Nethermind/Nethermind.EraE/Store/IEraStoreFactory.cs @@ -0,0 +1,11 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Core.Crypto; + +namespace Nethermind.EraE.Store; + +public interface IEraStoreFactory +{ + IEraStore Create(string src, ISet? trustedAccumulators); +} diff --git a/src/Nethermind/Nethermind.EraE/Store/IRemoteEraClient.cs b/src/Nethermind/Nethermind.EraE/Store/IRemoteEraClient.cs new file mode 100644 index 000000000000..543dc7e68cb3 --- /dev/null +++ b/src/Nethermind/Nethermind.EraE/Store/IRemoteEraClient.cs @@ -0,0 +1,19 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +namespace Nethermind.EraE.Store; + +public readonly record struct RemoteEraEntry(string Filename, byte[] Sha256Hash); + +public interface IRemoteEraClient +{ + /// Fetches and parses the remote checksum manifest into a map of epoch → entry. + Task> FetchManifestAsync(CancellationToken cancellation = default); + + /// + /// Downloads a single erae file to . + /// Uses an atomic write: streams to a .tmp file then renames on success. + /// Deletes the .tmp file on failure. + /// + Task DownloadFileAsync(string filename, string destinationPath, CancellationToken cancellation = default); +} diff --git a/src/Nethermind/Nethermind.EraE/Store/RemoteEraStoreDecorator.cs b/src/Nethermind/Nethermind.EraE/Store/RemoteEraStoreDecorator.cs new file mode 100644 index 000000000000..db317f5a041a --- /dev/null +++ b/src/Nethermind/Nethermind.EraE/Store/RemoteEraStoreDecorator.cs @@ -0,0 +1,221 @@ +// SPDX-FileCopyrightText: 2026 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Collections.Concurrent; +using System.Security.Cryptography; +using Nethermind.Core; +using Nethermind.Core.Collections; +using Nethermind.Core.Extensions; +using Nethermind.EraE.Archive; +using EraException = Nethermind.Era1.EraException; +using EraVerificationException = Nethermind.Era1.Exceptions.EraVerificationException; + +namespace Nethermind.EraE.Store; + +// Thread-safety model: +// Setup paths (BlockRange, NextEraStart) — called sequentially during initialization, +// never from an async context; sync-over-async via GetAwaiter().GetResult() is safe. +// Read paths (FindBlockAndReceipts) — called concurrently by multiple importer worker tasks; +// protected by per-epoch semaphores and a manifest lock. +public sealed class RemoteEraStoreDecorator : IEraStore +{ + private readonly IEraStore? _localStore; + private readonly IRemoteEraClient _client; + private readonly string _downloadDir; + private readonly int _maxEraSize; + // Bounded reader pool: capped at ProcessorCount*2 to stay within OS fd limits. + // Mainnet has ~1600 epochs; Linux default fd limit is 1024 — an unbounded pool would exhaust it. + private readonly int _maxOpenReaders; + private volatile bool _disposed; + + // Manifest fetched once on first remote access + private IReadOnlyDictionary? _manifest; + private readonly SemaphoreSlim _manifestLock = new(1, 1); + + // One semaphore per epoch prevents concurrent duplicate downloads. + private readonly ConcurrentDictionary _epochLocks = new(); + + // Verified epoch paths — populated after successful SHA-256 check + private readonly ConcurrentDictionary _verifiedEpochs = new(); + + // Bounded idle reader pool — readers are checked out (TryRemove) before use and returned + // (TryAdd) when done, so the eviction path can only dispose readers that are not in flight. + private readonly ConcurrentDictionary _openedReaders = new(); + + // Setup path — sequential, sync-over-async is safe (see thread-safety model above) + public (long First, long Last) BlockRange => GetBlockRangeAsync().GetAwaiter().GetResult(); + + public RemoteEraStoreDecorator( + IEraStore? localStore, + IRemoteEraClient client, + string downloadDir, + int maxEraSize) + { + ArgumentNullException.ThrowIfNull(client); + ArgumentException.ThrowIfNullOrWhiteSpace(downloadDir); + ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(maxEraSize, 0); + + _localStore = localStore; + _client = client; + _downloadDir = downloadDir; + _maxEraSize = maxEraSize; + _maxOpenReaders = Math.Max(Environment.ProcessorCount * 2, 8); + Directory.CreateDirectory(downloadDir); + } + + public async Task<(Block?, TxReceipt[]?)> FindBlockAndReceipts( + long number, bool ensureValidated = true, CancellationToken cancellation = default) + { + if (_localStore is not null) + { + (Block? b, TxReceipt[]? r) = await _localStore.FindBlockAndReceipts(number, ensureValidated, cancellation).ConfigureAwait(false); + if (b is not null) return (b, r); + } + + int epoch = (int)(number / _maxEraSize); + string localPath = await EnsureEpochAvailableAsync(epoch, cancellation).ConfigureAwait(false); + + using EraRenter renter = RentReader(epoch, localPath); + if (number > renter.Reader.LastBlock) return (null, null); + (Block block, TxReceipt[] receipts) = await renter.Reader.GetBlockByNumber(number, cancellation).ConfigureAwait(false); + return (block, receipts); + } + + public bool HasEpoch(long blockNumber) => _localStore is not null && _localStore.HasEpoch(blockNumber); + + public long NextEraStart(long blockNumber) + { + if (_localStore is not null && _localStore.HasEpoch(blockNumber)) + return _localStore.NextEraStart(blockNumber); + + int epoch = (int)(blockNumber / _maxEraSize); + // Setup path — sequential, sync-over-async is safe (see thread-safety model above) + string localPath = EnsureEpochAvailableAsync(epoch).GetAwaiter().GetResult(); + using EraReader reader = new(localPath); + return reader.LastBlock + 1; + } + + public void Dispose() + { + _disposed = true; + _localStore?.Dispose(); + _manifestLock.Dispose(); + foreach (SemaphoreSlim s in _epochLocks.Values) + s.Dispose(); + foreach (EraReader reader in _openedReaders.Values) + reader.Dispose(); + } + + private EraRenter RentReader(int epoch, string localPath) + { + // Fast path: check out an existing idle reader. + if (_openedReaders.TryRemove(epoch, out EraReader? existing)) + return new EraRenter(this, existing, epoch); + + // Evict the oldest (lowest-epoch) idle reader when the pool is at capacity. + // Import accesses epochs in ascending order, so the lowest epoch is always the + // least-recently-used and is safe to close. + if (_openedReaders.Count >= _maxOpenReaders) + { + int oldest = int.MaxValue; + foreach (int key in _openedReaders.Keys) + if (key < oldest) oldest = key; + if (_openedReaders.TryRemove(oldest, out EraReader? evicted)) + evicted.Dispose(); + } + + return new EraRenter(this, new EraReader(localPath), epoch); + } + + private void ReturnReader(int epoch, EraReader reader) + { + if (_disposed || !_openedReaders.TryAdd(epoch, reader)) + reader.Dispose(); + } + + private readonly struct EraRenter(RemoteEraStoreDecorator store, EraReader reader, int epoch) : IDisposable + { + public EraReader Reader => reader; + public void Dispose() => store.ReturnReader(epoch, reader); + } + + private async Task<(long First, long Last)> GetBlockRangeAsync(CancellationToken cancellation = default) + { + if (_localStore is not null) return _localStore.BlockRange; + + IReadOnlyDictionary manifest = await GetManifestAsync(cancellation).ConfigureAwait(false); + if (manifest.Count == 0) throw new EraException("Remote eraE manifest is empty."); + + (int minEpoch, int maxEpoch) = manifest.Keys.MinMax(); + + // First is exact: era epochs are aligned to maxEraSize boundaries. + // Last is upper-bound estimate: avoids downloading the last (potentially huge) epoch file just for validation. + // The actual last block may be slightly lower for a non-full final epoch. + // FindBlockAndReceipts returns (null, null) when number > reader.LastBlock, so importers that + // rely on this value (to=0 / auto mode) will stop naturally at the real end. + return ((long)minEpoch * _maxEraSize, (long)(maxEpoch + 1) * _maxEraSize - 1); + } + + private async Task> GetManifestAsync(CancellationToken cancellation = default) + { + if (_manifest is not null) return _manifest; + + await _manifestLock.WaitAsync(cancellation).ConfigureAwait(false); + try + { + if (_manifest is not null) return _manifest; + _manifest = await _client.FetchManifestAsync(cancellation).ConfigureAwait(false); + return _manifest; + } + finally + { + _manifestLock.Release(); + } + } + + private async Task EnsureEpochAvailableAsync(int epoch, CancellationToken cancellation = default) + { + if (_verifiedEpochs.TryGetValue(epoch, out string? cached)) + return cached; + + IReadOnlyDictionary manifest = await GetManifestAsync(cancellation).ConfigureAwait(false); + if (!manifest.TryGetValue(epoch, out RemoteEraEntry entry)) + throw new EraException($"Epoch {epoch} is not available in the remote eraE manifest."); + + string destinationPath = Path.Join(_downloadDir, entry.Filename); + + SemaphoreSlim epochLock = _epochLocks.GetOrAddDisposable(epoch, static _ => new SemaphoreSlim(1, 1)); + await epochLock.WaitAsync(cancellation).ConfigureAwait(false); + try + { + // Re-check after acquiring lock (another thread may have finished) + if (_verifiedEpochs.TryGetValue(epoch, out cached)) + return cached; + + if (!File.Exists(destinationPath)) + await _client.DownloadFileAsync(entry.Filename, destinationPath, cancellation).ConfigureAwait(false); + + VerifySha256(destinationPath, entry.Sha256Hash); + _verifiedEpochs.TryAdd(epoch, destinationPath); + return destinationPath; + } + catch (Exception) + { + if (File.Exists(destinationPath)) + File.Delete(destinationPath); + throw; + } + finally + { + epochLock.Release(); + } + } + + private static void VerifySha256(string filePath, byte[] expectedHash) + { + using FileStream fs = new(filePath, FileMode.Open, FileAccess.Read, FileShare.Read, bufferSize: 81920); + byte[] actual = SHA256.HashData(fs); + if (!actual.AsSpan().SequenceEqual(expectedHash)) + throw new EraVerificationException($"SHA-256 checksum mismatch for '{Path.GetFileName(filePath)}'."); + } +} diff --git a/src/Nethermind/Nethermind.Init/Modules/BuiltInStepsModule.cs b/src/Nethermind/Nethermind.Init/Modules/BuiltInStepsModule.cs index 46aa77b3ab18..8323cd4c125f 100644 --- a/src/Nethermind/Nethermind.Init/Modules/BuiltInStepsModule.cs +++ b/src/Nethermind/Nethermind.Init/Modules/BuiltInStepsModule.cs @@ -13,6 +13,7 @@ public class BuiltInStepsModule : Module [ typeof(ApplyMemoryHint), typeof(DatabaseMigrations), + typeof(EraEStep), typeof(EraStep), typeof(InitDatabase), typeof(InitializeBlockchain), diff --git a/src/Nethermind/Nethermind.Init/Modules/NethermindModule.cs b/src/Nethermind/Nethermind.Init/Modules/NethermindModule.cs index d52d9b27a7b8..8ae7a49acb64 100644 --- a/src/Nethermind/Nethermind.Init/Modules/NethermindModule.cs +++ b/src/Nethermind/Nethermind.Init/Modules/NethermindModule.cs @@ -17,7 +17,6 @@ using Nethermind.Crypto; using Nethermind.Db; using Nethermind.Db.LogIndex; -using Nethermind.Era1; using Nethermind.JsonRpc; using Nethermind.Logging; using Nethermind.Monitoring.Config; @@ -55,7 +54,8 @@ protected override void Load(ContainerBuilder builder) .AddModule(new PrewarmerModule(configProvider.GetConfig())) .AddModule(new BuiltInStepsModule()) .AddModule(new RpcModules(configProvider.GetConfig())) - .AddModule(new EraModule()) + .AddModule(new Era1.EraModule()) + .AddModule(new EraE.EraEModule()) .AddSource(new ConfigRegistrationSource()) .AddModule(new BlockProcessingModule(configProvider.GetConfig(), configProvider.GetConfig())) .AddModule(new BlockTreeModule(configProvider.GetConfig(), configProvider.GetConfig())) diff --git a/src/Nethermind/Nethermind.Init/Nethermind.Init.csproj b/src/Nethermind/Nethermind.Init/Nethermind.Init.csproj index 3f3637408f23..0f410fac8715 100644 --- a/src/Nethermind/Nethermind.Init/Nethermind.Init.csproj +++ b/src/Nethermind/Nethermind.Init/Nethermind.Init.csproj @@ -9,6 +9,7 @@ + diff --git a/src/Nethermind/Nethermind.Init/Steps/EraEStep.cs b/src/Nethermind/Nethermind.Init/Steps/EraEStep.cs new file mode 100644 index 000000000000..10eaef805262 --- /dev/null +++ b/src/Nethermind/Nethermind.Init/Steps/EraEStep.cs @@ -0,0 +1,18 @@ +// SPDX-FileCopyrightText: 2025 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Threading; +using System.Threading.Tasks; +using Nethermind.Api.Steps; +using Nethermind.EraE; + +namespace Nethermind.Init.Steps; + +[RunnerStepDependencies( + dependencies: [typeof(ReviewBlockTree)], + dependents: [typeof(InitializeNetwork)] +)] +public class EraEStep(EraCliRunner eraCliRunner) : IStep +{ + public async Task Execute(CancellationToken cancellationToken) => await eraCliRunner.Run(cancellationToken); +} diff --git a/src/Nethermind/Nethermind.Merkleization/Merkleizer.cs b/src/Nethermind/Nethermind.Merkleization/Merkleizer.cs index 150bc688ca16..33bc293cf539 100644 --- a/src/Nethermind/Nethermind.Merkleization/Merkleizer.cs +++ b/src/Nethermind/Nethermind.Merkleization/Merkleizer.cs @@ -10,6 +10,7 @@ using Nethermind.Core.Collections; using Nethermind.Core.Crypto; using Nethermind.Int256; +using Nethermind.Serialization.Ssz; namespace Nethermind.Merkleization; @@ -237,6 +238,27 @@ public void Feed(Bytes32 value) => public void Feed(Root value) => Feed(MemoryMarshal.Cast(value.AsSpan())[0]); + public void Feed(SszBytes32 value) => Feed(MemoryMarshal.Cast(value.Hash.BytesAsSpan)[0]); + + public void Feed(IReadOnlyList value) + { + using ArrayPoolSpan input = new(value.Count); + for (int i = 0; i < value.Count; i++) + input[i] = MemoryMarshal.Cast(value[i].Hash.BytesAsSpan)[0]; + Merkle.Merkleize(out _chunks[^1], input); + Feed(_chunks[^1]); + } + + public void Feed(IReadOnlyList value, ulong maxLength) + { + using ArrayPoolSpan subRoots = new(value.Count); + for (int i = 0; i < value.Count; i++) + subRoots[i] = MemoryMarshal.Cast(value[i].Hash.BytesAsSpan)[0]; + Merkle.Merkleize(out _chunks[^1], subRoots, maxLength); + Merkle.MixIn(ref _chunks[^1], value.Count); + Feed(_chunks[^1]); + } + public void Feed(IReadOnlyList value) { // TODO: If the above MemoryMarshal.Cast of a single Bytes32, we could use that here diff --git a/src/Nethermind/Nethermind.Runner/packages.lock.json b/src/Nethermind/Nethermind.Runner/packages.lock.json index 28adbcf316e3..aabeecec44fe 100644 --- a/src/Nethermind/Nethermind.Runner/packages.lock.json +++ b/src/Nethermind/Nethermind.Runner/packages.lock.json @@ -773,6 +773,25 @@ "Snappier": "[1.3.0, )" } }, + "nethermind.erae": { + "type": "Project", + "dependencies": { + "CommunityToolkit.HighPerformance": "[8.4.0, )", + "Nethermind.Api": "[1.37.0-unstable, )", + "Nethermind.Blockchain": "[1.37.0-unstable, )", + "Nethermind.Consensus": "[1.37.0-unstable, )", + "Nethermind.Core": "[1.37.0-unstable, )", + "Nethermind.Era1": "[1.37.0-unstable, )", + "Nethermind.History": "[1.37.0-unstable, )", + "Nethermind.JsonRpc": "[1.37.0-unstable, )", + "Nethermind.Merkleization": "[1.37.0-unstable, )", + "Nethermind.Serialization.Rlp": "[1.37.0-unstable, )", + "Nethermind.Serialization.Ssz": "[1.37.0-unstable, )", + "Nethermind.State": "[1.37.0-unstable, )", + "Polly": "[8.6.6, )", + "Snappier": "[1.3.0, )" + } + }, "nethermind.ethstats": { "type": "Project", "dependencies": { @@ -874,6 +893,7 @@ "Nethermind.Db.Rocks": "[1.37.0-unstable, )", "Nethermind.Db.Rpc": "[1.37.0-unstable, )", "Nethermind.Era1": "[1.37.0-unstable, )", + "Nethermind.EraE": "[1.37.0-unstable, )", "Nethermind.Network.Discovery": "[1.37.0-unstable, )", "Nethermind.Network.Dns": "[1.37.0-unstable, )", "Nethermind.Network.Enr": "[1.37.0-unstable, )", diff --git a/src/Nethermind/Nethermind.Serialization.Rlp/ReceiptMessageDecoder.cs b/src/Nethermind/Nethermind.Serialization.Rlp/ReceiptMessageDecoder.cs index a361d6eecd7b..bfbb46723366 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp/ReceiptMessageDecoder.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp/ReceiptMessageDecoder.cs @@ -12,11 +12,10 @@ namespace Nethermind.Serialization.Rlp [Rlp.Decoder(RlpDecoderKey.Default)] [Rlp.Decoder(RlpDecoderKey.Trie)] [method: DynamicDependency(DynamicallyAccessedMemberTypes.PublicConstructors, typeof(ReceiptMessageDecoder))] - public sealed class ReceiptMessageDecoder(bool skipStateAndStatus = false) : RlpValueDecoder + public sealed class ReceiptMessageDecoder(bool skipStateAndStatus = false, bool skipBloom = false) : RlpValueDecoder { // A 100M gas ceiling still allows roughly 266k LOG0 emissions after intrinsic gas. private static readonly RlpLimit LogsRlpLimit = RlpLimit.For(270_000, nameof(TxReceipt.Logs)); - private readonly bool _skipStateAndStatus = skipStateAndStatus; protected override TxReceipt DecodeInternal(ref Rlp.ValueDecoderContext ctx, RlpBehaviors rlpBehaviors = RlpBehaviors.None) { @@ -51,7 +50,9 @@ protected override TxReceipt DecodeInternal(ref Rlp.ValueDecoderContext ctx, Rlp txReceipt.GasUsedTotal = ctx.DecodePositiveLong(); } - txReceipt.Bloom = ctx.DecodeBloom(); + if (!skipBloom) + txReceipt.Bloom = ctx.DecodeBloom(); + // When _skipBloom is true (slim receipt), bloom is absent from the stream — nothing to skip. int lastCheck = ctx.ReadSequenceLength() + ctx.Position; @@ -94,14 +95,15 @@ static void ThrowUnexpectedReceiptField() int contentLength = 0; contentLength += Rlp.LengthOf(item.GasUsedTotal); - contentLength += Rlp.LengthOf(item.Bloom); + if (!skipBloom) + contentLength += Rlp.LengthOf(item.Bloom); int logsLength = GetLogsLength(item); contentLength += Rlp.LengthOfSequence(logsLength); bool isEip658Receipts = (rlpBehaviors & RlpBehaviors.Eip658Receipts) == RlpBehaviors.Eip658Receipts; - if (!_skipStateAndStatus) + if (!skipStateAndStatus) { contentLength += isEip658Receipts ? Rlp.LengthOf(item.StatusCode) @@ -176,7 +178,7 @@ public override void Encode(RlpStream rlpStream, TxReceipt item, RlpBehaviors rl } rlpStream.StartSequence(totalContentLength); - if (!_skipStateAndStatus) + if (!skipStateAndStatus) { if (isEip658Receipts) { @@ -189,7 +191,8 @@ public override void Encode(RlpStream rlpStream, TxReceipt item, RlpBehaviors rl } rlpStream.Encode(item.GasUsedTotal); - rlpStream.Encode(item.Bloom); + if (!skipBloom) + rlpStream.Encode(item.Bloom); rlpStream.StartSequence(logsLength); LogEntry[] logs = item.Logs; diff --git a/src/Nethermind/Nethermind.Serialization.Ssz/Crypto/Ssz.SszBytes32.cs b/src/Nethermind/Nethermind.Serialization.Ssz/Crypto/Ssz.SszBytes32.cs new file mode 100644 index 000000000000..78ecc2796885 --- /dev/null +++ b/src/Nethermind/Nethermind.Serialization.Ssz/Crypto/Ssz.SszBytes32.cs @@ -0,0 +1,37 @@ +// SPDX-FileCopyrightText: 2025 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; + +namespace Nethermind.Serialization.Ssz; + +public static partial class Ssz +{ + public static void Encode(Span span, SszBytes32 value) => + value.Hash.Bytes.CopyTo(span); + + public static void Encode(Span span, SszBytes32[] value) + { + const int typeSize = 32; + if (span.Length != value.Length * typeSize) + ThrowTargetLength(span.Length, value.Length * typeSize); + for (int i = 0; i < value.Length; i++) + Encode(span.Slice(i * typeSize, typeSize), value[i]); + } + + public static void Decode(ReadOnlySpan span, out SszBytes32 value) + { + ValidateLength(span, 32); + value = new SszBytes32(span); + } + + public static void Decode(ReadOnlySpan span, out ReadOnlySpan result) + { + const int typeSize = 32; + ValidateArrayLength(span, typeSize); + SszBytes32[] array = new SszBytes32[span.Length / typeSize]; + for (int i = 0; i < array.Length; i++) + Decode(span.Slice(i * typeSize, typeSize), out array[i]); + result = array; + } +} diff --git a/src/Nethermind/Nethermind.Serialization.Ssz/SszBytes32.cs b/src/Nethermind/Nethermind.Serialization.Ssz/SszBytes32.cs new file mode 100644 index 000000000000..36ef787c2e13 --- /dev/null +++ b/src/Nethermind/Nethermind.Serialization.Ssz/SszBytes32.cs @@ -0,0 +1,18 @@ +// SPDX-FileCopyrightText: 2025 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using Nethermind.Core.Crypto; + +namespace Nethermind.Serialization.Ssz; + +public readonly struct SszBytes32 +{ + private readonly ValueHash256 _value; + + public SszBytes32(ReadOnlySpan bytes) => _value = new ValueHash256(bytes); + + public ValueHash256 Hash => _value; + + public static SszBytes32 From(ValueHash256 hash) => new(hash.Bytes); +} diff --git a/src/Nethermind/Nethermind.Serialization.SszGenerator/SszGenerator.cs b/src/Nethermind/Nethermind.Serialization.SszGenerator/SszGenerator.cs index f66a9b5c3a88..15d1464c5813 100644 --- a/src/Nethermind/Nethermind.Serialization.SszGenerator/SszGenerator.cs +++ b/src/Nethermind/Nethermind.Serialization.SszGenerator/SszGenerator.cs @@ -64,11 +64,9 @@ private static string PartialTypeDeclaration(SszType decl) => private static string NamespaceLine(SszType decl) => string.IsNullOrEmpty(decl.Namespace) ? string.Empty : $"namespace {decl.Namespace};"; - private static bool IsClassWithAttribute(SyntaxNode syntaxNode) - { - return syntaxNode is TypeDeclarationSyntax classDeclaration && - classDeclaration.AttributeLists.Any(x => x.Attributes.Any()); - } + private static bool IsClassWithAttribute(SyntaxNode syntaxNode) => + syntaxNode is TypeDeclarationSyntax classDeclaration && + classDeclaration.AttributeLists.Any(x => x.Attributes.Any()); private static (SszType Type, List FoundTypes, Location? Location, bool IsNested)? GetClassWithAttribute(GeneratorSyntaxContext context) { @@ -504,7 +502,7 @@ private static string EncodeStatement(string target, SszProperty property, strin using System.Runtime.InteropServices; using Nethermind.Serialization.Ssz; using static Nethermind.Serialization.SszCodecHelpers; -{string.Join("\n", foundTypes.Select(x => x.Namespace).Distinct().OrderBy(x => x).Where(x => !string.IsNullOrEmpty(x)).Select(n => $"using {n};"))} +{string.Join("\n", foundTypes.Select(x => x.Namespace).Distinct().OrderBy(x => x).Where(x => !string.IsNullOrEmpty(x) && x != "Nethermind.Serialization.Ssz").Select(n => $"using {n};"))} {Whitespace} using SszLib = Nethermind.Serialization.Ssz.Ssz; {Whitespace} @@ -673,7 +671,7 @@ public static void MerkleizeProgressiveList(ReadOnlySpan<{decl.Name}> container, using System.Runtime.InteropServices; using Nethermind.Serialization.Ssz; using static Nethermind.Serialization.SszCodecHelpers; -{string.Join("\n", foundTypes.Select(x => x.Namespace).Distinct().OrderBy(x => x).Where(x => !string.IsNullOrEmpty(x)).Select(n => $"using {n};"))} +{string.Join("\n", foundTypes.Select(x => x.Namespace).Distinct().OrderBy(x => x).Where(x => !string.IsNullOrEmpty(x) && x != "Nethermind.Serialization.Ssz").Select(n => $"using {n};"))} {Whitespace} using SszLib = Nethermind.Serialization.Ssz.Ssz; {Whitespace} @@ -877,7 +875,7 @@ public static void MerkleizeProgressiveList(ReadOnlySpan<{decl.Name}> container, using System.Runtime.InteropServices; using Nethermind.Serialization.Ssz; using static Nethermind.Serialization.SszCodecHelpers; -{string.Join("\n", foundTypes.Select(x => x.Namespace).Distinct().OrderBy(x => x).Where(x => !string.IsNullOrEmpty(x)).Select(n => $"using {n};"))} +{string.Join("\n", foundTypes.Select(x => x.Namespace).Distinct().OrderBy(x => x).Where(x => !string.IsNullOrEmpty(x) && x != "Nethermind.Serialization.Ssz").Select(n => $"using {n};"))} {Whitespace} using SszLib = Nethermind.Serialization.Ssz.Ssz; {Whitespace} diff --git a/src/Nethermind/Nethermind.Serialization.SszGenerator/SszType.cs b/src/Nethermind/Nethermind.Serialization.SszGenerator/SszType.cs index 94aa480085af..b0795943f988 100644 --- a/src/Nethermind/Nethermind.Serialization.SszGenerator/SszType.cs +++ b/src/Nethermind/Nethermind.Serialization.SszGenerator/SszType.cs @@ -77,6 +77,14 @@ static SszType() Name = "BitArray", Kind = Kind.Basic, }); + + BasicTypes.Add(new SszType + { + Namespace = "Nethermind.Serialization.Ssz", + Name = "SszBytes32", + Kind = Kind.Basic, + StaticLength = 32, + }); } public static List BasicTypes { get; set; } = []; diff --git a/src/Nethermind/Nethermind.slnx b/src/Nethermind/Nethermind.slnx index 30498560cf49..1ecdce8c9942 100644 --- a/src/Nethermind/Nethermind.slnx +++ b/src/Nethermind/Nethermind.slnx @@ -49,6 +49,7 @@ + @@ -98,6 +99,7 @@ +