From b10d0ee5c2ac24ed9c3332092ca0de5c142f23a8 Mon Sep 17 00:00:00 2001 From: Stefan Bratanov Date: Thu, 3 Apr 2025 16:46:24 +0100 Subject: [PATCH 01/12] Initial cleanup --- .../DeserializableListTypeDefinition.java | 4 + .../metrics/TekuMetricCategory.java | 1 - .../services/chainstorage/StorageService.java | 15 +- .../storage/server/kvstore/DatabaseTest.java | 37 ++--- ...Archive.java => BlobSidecarsArchiver.java} | 15 +- .../teku/storage/archive/DataArchive.java | 35 ----- .../storage/archive/DataArchiveWriter.java | 27 ---- .../FileSystemBlobSidecarsArchiver.java | 131 +++++++++++++++++ .../fsarchive/BlobSidecarJsonWriter.java | 44 ------ .../archive/fsarchive/FileSystemArchive.java | 137 ------------------ .../nooparchive/DataArchiveNoopWriter.java | 28 ---- .../pegasys/teku/storage/server/Database.java | 10 +- .../server/kvstore/KvStoreDatabase.java | 27 ++-- .../storage/server/noop/NoOpDatabase.java | 34 ++--- .../server/pruner/BlobSidecarPruner.java | 20 +-- .../FileSystemBlobSidecarsArchiverTest.java} | 103 ++++++------- .../fsarchive/BlobSidecarJsonWriterTest.java | 99 ------------- .../server/pruner/BlobSidecarPrunerTest.java | 32 ++-- .../pegasys/teku/storage/store/StoreTest.java | 4 +- 19 files changed, 271 insertions(+), 532 deletions(-) rename storage/src/main/java/tech/pegasys/teku/storage/archive/{nooparchive/NoopDataArchive.java => BlobSidecarsArchiver.java} (60%) delete mode 100644 storage/src/main/java/tech/pegasys/teku/storage/archive/DataArchive.java delete mode 100644 storage/src/main/java/tech/pegasys/teku/storage/archive/DataArchiveWriter.java create mode 100644 storage/src/main/java/tech/pegasys/teku/storage/archive/filesystem/FileSystemBlobSidecarsArchiver.java delete mode 100644 storage/src/main/java/tech/pegasys/teku/storage/archive/fsarchive/BlobSidecarJsonWriter.java delete mode 100644 storage/src/main/java/tech/pegasys/teku/storage/archive/fsarchive/FileSystemArchive.java delete mode 100644 storage/src/main/java/tech/pegasys/teku/storage/archive/nooparchive/DataArchiveNoopWriter.java rename storage/src/test/java/tech/pegasys/teku/storage/archive/{fsarchive/FileSystemArchiveTest.java => filesystem/FileSystemBlobSidecarsArchiverTest.java} (59%) delete mode 100644 storage/src/test/java/tech/pegasys/teku/storage/archive/fsarchive/BlobSidecarJsonWriterTest.java diff --git a/infrastructure/json/src/main/java/tech/pegasys/teku/infrastructure/json/types/DeserializableListTypeDefinition.java b/infrastructure/json/src/main/java/tech/pegasys/teku/infrastructure/json/types/DeserializableListTypeDefinition.java index 0bcb51b361d..7d23c9c5e69 100644 --- a/infrastructure/json/src/main/java/tech/pegasys/teku/infrastructure/json/types/DeserializableListTypeDefinition.java +++ b/infrastructure/json/src/main/java/tech/pegasys/teku/infrastructure/json/types/DeserializableListTypeDefinition.java @@ -26,4 +26,8 @@ public DeserializableListTypeDefinition( final Optional maxItems) { super(itemType, Function.identity(), minItems, maxItems); } + + public DeserializableListTypeDefinition(final DeserializableTypeDefinition itemType) { + super(itemType, Function.identity(), Optional.empty(), Optional.empty()); + } } diff --git a/infrastructure/metrics/src/main/java/tech/pegasys/teku/infrastructure/metrics/TekuMetricCategory.java b/infrastructure/metrics/src/main/java/tech/pegasys/teku/infrastructure/metrics/TekuMetricCategory.java index 2a0966c0e37..5970bf7a8ed 100644 --- a/infrastructure/metrics/src/main/java/tech/pegasys/teku/infrastructure/metrics/TekuMetricCategory.java +++ b/infrastructure/metrics/src/main/java/tech/pegasys/teku/infrastructure/metrics/TekuMetricCategory.java @@ -27,7 +27,6 @@ public enum TekuMetricCategory implements MetricCategory { STORAGE("storage"), STORAGE_HOT_DB("storage_hot"), STORAGE_FINALIZED_DB("storage_finalized"), - REMOTE_VALIDATOR("remote_validator"), VALIDATOR("validator"), VALIDATOR_PERFORMANCE("validator_performance"), VALIDATOR_DUTY("validator_duty"); diff --git a/services/chainstorage/src/main/java/tech/pegasys/teku/services/chainstorage/StorageService.java b/services/chainstorage/src/main/java/tech/pegasys/teku/services/chainstorage/StorageService.java index 4f74c2c0f1d..bd569c8cee3 100644 --- a/services/chainstorage/src/main/java/tech/pegasys/teku/services/chainstorage/StorageService.java +++ b/services/chainstorage/src/main/java/tech/pegasys/teku/services/chainstorage/StorageService.java @@ -37,9 +37,8 @@ import tech.pegasys.teku.storage.api.CombinedStorageChannel; import tech.pegasys.teku.storage.api.Eth1DepositStorageChannel; import tech.pegasys.teku.storage.api.VoteUpdateChannel; -import tech.pegasys.teku.storage.archive.DataArchive; -import tech.pegasys.teku.storage.archive.fsarchive.FileSystemArchive; -import tech.pegasys.teku.storage.archive.nooparchive.NoopDataArchive; +import tech.pegasys.teku.storage.archive.BlobSidecarsArchiver; +import tech.pegasys.teku.storage.archive.filesystem.FileSystemBlobSidecarsArchiver; import tech.pegasys.teku.storage.server.BatchingVoteUpdateChannel; import tech.pegasys.teku.storage.server.ChainStorage; import tech.pegasys.teku.storage.server.CombinedStorageChannelSplitter; @@ -159,11 +158,13 @@ protected SafeFuture doStart() { pruningActiveLabelledGauge); } - final DataArchive dataArchive = + final BlobSidecarsArchiver blobSidecarsArchiver = config .getBlobsArchivePath() - .map(path -> new FileSystemArchive(Path.of(path))) - .orElse(new NoopDataArchive()); + .map( + path -> + new FileSystemBlobSidecarsArchiver(config.getSpec(), Path.of(path))) + .orElse(BlobSidecarsArchiver.NOOP); if (config.getSpec().isMilestoneSupported(SpecMilestone.DENEB)) { blobsPruner = @@ -171,7 +172,7 @@ protected SafeFuture doStart() { new BlobSidecarPruner( config.getSpec(), database, - dataArchive, + blobSidecarsArchiver, serviceConfig.getMetricsSystem(), storagePrunerAsyncRunner, serviceConfig.getTimeProvider(), diff --git a/storage/src/integration-test/java/tech/pegasys/teku/storage/server/kvstore/DatabaseTest.java b/storage/src/integration-test/java/tech/pegasys/teku/storage/server/kvstore/DatabaseTest.java index 237a17b53df..0dd8f760c60 100644 --- a/storage/src/integration-test/java/tech/pegasys/teku/storage/server/kvstore/DatabaseTest.java +++ b/storage/src/integration-test/java/tech/pegasys/teku/storage/server/kvstore/DatabaseTest.java @@ -82,7 +82,7 @@ import tech.pegasys.teku.storage.api.OnDiskStoreData; import tech.pegasys.teku.storage.api.StorageUpdate; import tech.pegasys.teku.storage.api.WeakSubjectivityUpdate; -import tech.pegasys.teku.storage.archive.fsarchive.FileSystemArchive; +import tech.pegasys.teku.storage.archive.filesystem.FileSystemBlobSidecarsArchiver; import tech.pegasys.teku.storage.client.RecentChainData; import tech.pegasys.teku.storage.server.Database; import tech.pegasys.teku.storage.server.DatabaseContext; @@ -122,7 +122,7 @@ public class DatabaseTest { private StateStorageMode storageMode; private StorageSystem storageSystem; private Database database; - private FileSystemArchive fileSystemDataArchive; + private FileSystemBlobSidecarsArchiver blobSidecarsArchiver; private RecentChainData recentChainData; private UpdatableStore store; private final List storageSystems = new ArrayList<>(); @@ -139,7 +139,7 @@ private void setupWithSpec(final Spec spec) throws IOException { this.chainProperties = new ChainProperties(spec); final Path blobsArchive = Files.createTempDirectory("blobs"); tmpDirectories.add(blobsArchive.toFile()); - this.fileSystemDataArchive = new FileSystemArchive(blobsArchive); + this.blobSidecarsArchiver = new FileSystemBlobSidecarsArchiver(spec, blobsArchive); genesisBlockAndState = chainBuilder.generateGenesis(genesisTime, true); genesisCheckpoint = getCheckpointForBlock(genesisBlockAndState.getBlock()); genesisAnchor = AnchorPoint.fromGenesisState(spec, genesisBlockAndState.getState()); @@ -300,9 +300,7 @@ public void verifyBlobsLifecycle(final DatabaseContext context) throws IOExcepti List.of(blobSidecar5_0))); // let's prune with limit to 1 - assertThat( - database.pruneOldestBlobSidecars( - UInt64.MAX_VALUE, 1, fileSystemDataArchive.getBlobSidecarWriter())) + assertThat(database.pruneOldestBlobSidecars(UInt64.MAX_VALUE, 1, blobSidecarsArchiver)) .isTrue(); assertBlobSidecarKeys( blobSidecar2_0.getSlot(), @@ -324,9 +322,7 @@ public void verifyBlobsLifecycle(final DatabaseContext context) throws IOExcepti assertThat(getSlotBlobsArchiveFile(blobSidecar2_0)).doesNotExist(); // let's prune up to slot 1 (nothing will be pruned) - assertThat( - database.pruneOldestBlobSidecars(ONE, 10, fileSystemDataArchive.getBlobSidecarWriter())) - .isFalse(); + assertThat(database.pruneOldestBlobSidecars(ONE, 10, blobSidecarsArchiver)).isFalse(); assertBlobSidecarKeys( blobSidecar2_0.getSlot(), blobSidecar5_0.getSlot(), @@ -343,9 +339,7 @@ public void verifyBlobsLifecycle(final DatabaseContext context) throws IOExcepti assertThat(database.getBlobSidecarColumnCount()).isEqualTo(4L); // let's prune all from slot 4 excluded - assertThat( - database.pruneOldestBlobSidecars( - UInt64.valueOf(3), 10, fileSystemDataArchive.getBlobSidecarWriter())) + assertThat(database.pruneOldestBlobSidecars(UInt64.valueOf(3), 10, blobSidecarsArchiver)) .isFalse(); assertBlobSidecarKeys( blobSidecar1_0.getSlot(), blobSidecar5_0.getSlot(), blobSidecarToKey(blobSidecar5_0)); @@ -359,9 +353,7 @@ public void verifyBlobsLifecycle(final DatabaseContext context) throws IOExcepti assertThat(getSlotBlobsArchiveFile(blobSidecar5_0)).doesNotExist(); // let's prune all - assertThat( - database.pruneOldestBlobSidecars( - UInt64.valueOf(5), 1, fileSystemDataArchive.getBlobSidecarWriter())) + assertThat(database.pruneOldestBlobSidecars(UInt64.valueOf(5), 1, blobSidecarsArchiver)) .isTrue(); // all empty now assertBlobSidecarKeys(ZERO, UInt64.valueOf(10)); @@ -372,8 +364,8 @@ public void verifyBlobsLifecycle(final DatabaseContext context) throws IOExcepti assertThat(getSlotBlobsArchiveFile(blobSidecar5_0)).exists(); } - private File getSlotBlobsArchiveFile(final BlobSidecar blobSidecar) { - return fileSystemDataArchive.resolve(blobSidecar.getSlotAndBlockRoot()); + private Path getSlotBlobsArchiveFile(final BlobSidecar blobSidecar) { + return blobSidecarsArchiver.resolve(blobSidecar.getSlotAndBlockRoot()); } @TestTemplate @@ -473,8 +465,7 @@ public void verifyNonCanonicalBlobsLifecycle(final DatabaseContext context) thro // Pruning with a prune limit set to 1: Only blobSidecar1 will be pruned assertThat( - database.pruneOldestNonCanonicalBlobSidecars( - UInt64.MAX_VALUE, 1, fileSystemDataArchive.getBlobSidecarWriter())) + database.pruneOldestNonCanonicalBlobSidecars(UInt64.MAX_VALUE, 1, blobSidecarsArchiver)) .isTrue(); assertNonCanonicalBlobSidecarKeys( blobSidecar2_0.getSlot(), @@ -495,9 +486,7 @@ public void verifyNonCanonicalBlobsLifecycle(final DatabaseContext context) thro assertThat(getSlotBlobsArchiveFile(blobSidecar2_0)).doesNotExist(); // Pruning up to slot 1: No blobs pruned - assertThat( - database.pruneOldestNonCanonicalBlobSidecars( - ONE, 10, fileSystemDataArchive.getBlobSidecarWriter())) + assertThat(database.pruneOldestNonCanonicalBlobSidecars(ONE, 10, blobSidecarsArchiver)) .isFalse(); assertNonCanonicalBlobSidecarKeys( blobSidecar2_0.getSlot(), @@ -520,7 +509,7 @@ public void verifyNonCanonicalBlobsLifecycle(final DatabaseContext context) thro // Prune blobs up to slot 3 assertThat( database.pruneOldestNonCanonicalBlobSidecars( - UInt64.valueOf(3), 10, fileSystemDataArchive.getBlobSidecarWriter())) + UInt64.valueOf(3), 10, blobSidecarsArchiver)) .isFalse(); assertNonCanonicalBlobSidecarKeys( blobSidecar1_0.getSlot(), blobSidecar5_0.getSlot(), blobSidecarToKey(blobSidecar5_0)); @@ -535,7 +524,7 @@ public void verifyNonCanonicalBlobsLifecycle(final DatabaseContext context) thro // Pruning all blobs assertThat( database.pruneOldestNonCanonicalBlobSidecars( - UInt64.valueOf(5), 1, fileSystemDataArchive.getBlobSidecarWriter())) + UInt64.valueOf(5), 1, blobSidecarsArchiver)) .isTrue(); // No blobs should be left assertNonCanonicalBlobSidecarKeys(ZERO, UInt64.valueOf(10)); diff --git a/storage/src/main/java/tech/pegasys/teku/storage/archive/nooparchive/NoopDataArchive.java b/storage/src/main/java/tech/pegasys/teku/storage/archive/BlobSidecarsArchiver.java similarity index 60% rename from storage/src/main/java/tech/pegasys/teku/storage/archive/nooparchive/NoopDataArchive.java rename to storage/src/main/java/tech/pegasys/teku/storage/archive/BlobSidecarsArchiver.java index 6de2de80585..82c3fac18a4 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/archive/nooparchive/NoopDataArchive.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/archive/BlobSidecarsArchiver.java @@ -11,20 +11,15 @@ * specific language governing permissions and limitations under the License. */ -package tech.pegasys.teku.storage.archive.nooparchive; +package tech.pegasys.teku.storage.archive; import java.util.List; import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar; -import tech.pegasys.teku.storage.archive.DataArchive; -import tech.pegasys.teku.storage.archive.DataArchiveWriter; +import tech.pegasys.teku.spec.datastructures.blocks.SlotAndBlockRoot; -public class NoopDataArchive implements DataArchive { +public interface BlobSidecarsArchiver { - private static final DataArchiveWriter> BLOB_SIDECAR_WRITER = - new DataArchiveNoopWriter<>(); + BlobSidecarsArchiver NOOP = (slotAndBlockRoot, blobSidecars) -> true; - @Override - public DataArchiveWriter> getBlobSidecarWriter() { - return BLOB_SIDECAR_WRITER; - } + boolean archive(SlotAndBlockRoot slotAndBlockRoot, List blobSidecars); } diff --git a/storage/src/main/java/tech/pegasys/teku/storage/archive/DataArchive.java b/storage/src/main/java/tech/pegasys/teku/storage/archive/DataArchive.java deleted file mode 100644 index 201a1fd4ef0..00000000000 --- a/storage/src/main/java/tech/pegasys/teku/storage/archive/DataArchive.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright Consensys Software Inc., 2025 - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ - -package tech.pegasys.teku.storage.archive; - -import java.io.IOException; -import java.util.List; -import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar; - -/** - * Interface for a data archive which stores prunable BlobSidecars outside the data availability - * window and could be extended later to include other data types. It is expected that the - * DataArchive is on disk or externally stored with slow write and recovery times. Initial interface - * is write only, but may be expanded to include read operations later. - */ -public interface DataArchive { - - /** - * Returns the archive writer capable of storing BlobSidecars. - * - * @return a closeable DataArchiveWriter for writing BlobSidecars - * @throws IOException throw exception if it fails to get a writer. - */ - DataArchiveWriter> getBlobSidecarWriter() throws IOException; -} diff --git a/storage/src/main/java/tech/pegasys/teku/storage/archive/DataArchiveWriter.java b/storage/src/main/java/tech/pegasys/teku/storage/archive/DataArchiveWriter.java deleted file mode 100644 index 1331365fab6..00000000000 --- a/storage/src/main/java/tech/pegasys/teku/storage/archive/DataArchiveWriter.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright Consensys Software Inc., 2025 - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ - -package tech.pegasys.teku.storage.archive; - -import java.io.Closeable; - -/** - * An interface to allow storing data that is to be pruned from the Database. If the store function - * is successful it returns true, signalling the data can be pruned. If the store function fails, - * the data was not stored and the data should not be pruned. - * - * @param the data to be stored. - */ -public interface DataArchiveWriter extends Closeable { - boolean archive(final T data); -} diff --git a/storage/src/main/java/tech/pegasys/teku/storage/archive/filesystem/FileSystemBlobSidecarsArchiver.java b/storage/src/main/java/tech/pegasys/teku/storage/archive/filesystem/FileSystemBlobSidecarsArchiver.java new file mode 100644 index 00000000000..0b679a8ae27 --- /dev/null +++ b/storage/src/main/java/tech/pegasys/teku/storage/archive/filesystem/FileSystemBlobSidecarsArchiver.java @@ -0,0 +1,131 @@ +/* + * Copyright Consensys Software Inc., 2025 + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package tech.pegasys.teku.storage.archive.filesystem; + +import static tech.pegasys.teku.infrastructure.json.types.DeserializableTypeDefinition.listOf; + +import com.google.common.annotations.VisibleForTesting; +import java.io.BufferedWriter; +import java.io.File; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; +import java.util.List; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import tech.pegasys.teku.infrastructure.json.JsonUtil; +import tech.pegasys.teku.infrastructure.json.types.SerializableTypeDefinition; +import tech.pegasys.teku.spec.Spec; +import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar; +import tech.pegasys.teku.spec.datastructures.blocks.SlotAndBlockRoot; +import tech.pegasys.teku.spec.schemas.SchemaDefinitionsDeneb; +import tech.pegasys.teku.storage.archive.BlobSidecarsArchiver; + +public class FileSystemBlobSidecarsArchiver implements BlobSidecarsArchiver { + + static final String INDEX_FILE = "index.dat"; + + private static final Logger LOG = LogManager.getLogger(); + + private final Spec spec; + private final Path baseDirectory; + private final Path indexFile; + + public FileSystemBlobSidecarsArchiver(final Spec spec, final Path baseDirectory) { + this.spec = spec; + this.baseDirectory = baseDirectory; + this.indexFile = baseDirectory.resolve(INDEX_FILE); + } + + @Override + public boolean archive( + final SlotAndBlockRoot slotAndBlockRoot, final List blobSidecars) { + if (blobSidecars == null || blobSidecars.isEmpty()) { + return true; + } + + final Path path = resolve(slotAndBlockRoot); + if (Files.exists(path)) { + LOG.error("Failed to write BlobSidecars. File exists: {}", path); + return false; + } + + try { + Files.createDirectories(path.getParent()); + } catch (final IOException __) { + LOG.error("Failed to write BlobSidecars. Could not create directories: {}", path.getParent()); + return false; + } + + try (final OutputStream output = Files.newOutputStream(path); + final BufferedWriter indexWriter = + Files.newBufferedWriter( + indexFile, + StandardCharsets.UTF_8, + StandardOpenOption.CREATE, + StandardOpenOption.APPEND)) { + writeBlobSidecars(output, slotAndBlockRoot, blobSidecars); + indexWriter.write(formatIndexFileOutput(slotAndBlockRoot)); + indexWriter.newLine(); + return true; + } catch (final IOException ex) { + LOG.error(String.format("Failed to write BlobSidecars for %s", slotAndBlockRoot), ex); + return false; + } + } + + /** + * Given a basePath, slot and block root, return where to store/find the BlobSidecar. Initial + * implementation uses blockRoot as a hex string in the directory of the first two characters. + * + * @param slotAndBlockRoot The slot and block root. + * @return a path of where to store or find the BlobSidecar + */ + @VisibleForTesting + public Path resolve(final SlotAndBlockRoot slotAndBlockRoot) { + // For blockroot 0x1a2bcd... the directory is basePath/1a/2b/1a2bcd... + // 256 * 256 directories = 65,536. + // Assume 8000 to 10000 blobs per day. With perfect hash distribution, + // all directories have one file after a week. After 1 year, expect 50 files in each directory. + String blockRootString = slotAndBlockRoot.getBlockRoot().toUnprefixedHexString(); + final String dir1 = blockRootString.substring(0, 2); + final String dir2 = blockRootString.substring(2, 4); + final String blobSidecarFilename = + dir1 + File.separator + dir2 + File.separator + blockRootString; + return baseDirectory.resolve(blobSidecarFilename); + } + + private void writeBlobSidecars( + final OutputStream out, + final SlotAndBlockRoot slotAndBlockRoot, + final List blobSidecars) + throws IOException { + final SerializableTypeDefinition> type = + listOf( + SchemaDefinitionsDeneb.required( + spec.atSlot(slotAndBlockRoot.getSlot()).getSchemaDefinitions()) + .getBlobSidecarSchema() + .getJsonTypeDefinition()); + JsonUtil.serializeToBytes(blobSidecars, type, out); + } + + private String formatIndexFileOutput(final SlotAndBlockRoot slotAndBlockRoot) { + return slotAndBlockRoot.getSlot() + + " " + + slotAndBlockRoot.getBlockRoot().toUnprefixedHexString(); + } +} diff --git a/storage/src/main/java/tech/pegasys/teku/storage/archive/fsarchive/BlobSidecarJsonWriter.java b/storage/src/main/java/tech/pegasys/teku/storage/archive/fsarchive/BlobSidecarJsonWriter.java deleted file mode 100644 index 2c85a310235..00000000000 --- a/storage/src/main/java/tech/pegasys/teku/storage/archive/fsarchive/BlobSidecarJsonWriter.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright Consensys Software Inc., 2025 - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ - -package tech.pegasys.teku.storage.archive.fsarchive; - -import static java.nio.charset.StandardCharsets.UTF_8; -import static tech.pegasys.teku.infrastructure.json.types.DeserializableTypeDefinition.listOf; - -import java.io.IOException; -import java.io.OutputStream; -import java.util.List; -import java.util.Objects; -import tech.pegasys.teku.infrastructure.json.JsonUtil; -import tech.pegasys.teku.infrastructure.json.types.SerializableTypeDefinition; -import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar; - -public class BlobSidecarJsonWriter { - - public void writeSlotBlobSidecars(final OutputStream out, final List blobSidecars) - throws IOException { - Objects.requireNonNull(out); - Objects.requireNonNull(blobSidecars); - - // Technically not possible as pruner prunes sidecars and not slots. - if (blobSidecars.isEmpty()) { - out.write("[]".getBytes(UTF_8)); - return; - } - - final SerializableTypeDefinition> type = - listOf(blobSidecars.getFirst().getSchema().getJsonTypeDefinition()); - JsonUtil.serializeToBytes(blobSidecars, type, out); - } -} diff --git a/storage/src/main/java/tech/pegasys/teku/storage/archive/fsarchive/FileSystemArchive.java b/storage/src/main/java/tech/pegasys/teku/storage/archive/fsarchive/FileSystemArchive.java deleted file mode 100644 index 0839207063b..00000000000 --- a/storage/src/main/java/tech/pegasys/teku/storage/archive/fsarchive/FileSystemArchive.java +++ /dev/null @@ -1,137 +0,0 @@ -/* - * Copyright Consensys Software Inc., 2025 - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ - -package tech.pegasys.teku.storage.archive.fsarchive; - -import java.io.BufferedWriter; -import java.io.Closeable; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.OutputStreamWriter; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.List; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar; -import tech.pegasys.teku.spec.datastructures.blocks.SlotAndBlockRoot; -import tech.pegasys.teku.storage.archive.DataArchive; -import tech.pegasys.teku.storage.archive.DataArchiveWriter; - -/** - * A file system based implementations of the DataArchive. Writes to a directory using the - * PathResolver method to decide where to write the files. - */ -public class FileSystemArchive implements DataArchive { - static final String INDEX_FILE = "index.dat"; - private static final Logger LOG = LogManager.getLogger(); - - private final Path baseDirectory; - private final BlobSidecarJsonWriter jsonWriter; - - public FileSystemArchive(final Path baseDirectory) { - this.baseDirectory = baseDirectory; - this.jsonWriter = new BlobSidecarJsonWriter(); - } - - @Override - public DataArchiveWriter> getBlobSidecarWriter() throws IOException { - - try { - final File indexFile = baseDirectory.resolve(INDEX_FILE).toFile(); - return new FileSystemBlobSidecarWriter(indexFile); - } catch (IOException e) { - LOG.warn("Unable to create BlobSidecar archive writer", e); - throw e; - } - } - - private class FileSystemBlobSidecarWriter - implements DataArchiveWriter>, Closeable { - final BufferedWriter indexWriter; - - public FileSystemBlobSidecarWriter(final File indexFile) throws IOException { - indexWriter = - new BufferedWriter( - new OutputStreamWriter( - new FileOutputStream(indexFile, true), StandardCharsets.UTF_8)); - } - - @Override - public boolean archive(final List blobSidecars) { - if (blobSidecars == null || blobSidecars.isEmpty()) { - return true; - } - - final SlotAndBlockRoot slotAndBlockRoot = blobSidecars.getFirst().getSlotAndBlockRoot(); - final File file = resolve(slotAndBlockRoot); - if (file.exists()) { - LOG.error("Failed to write BlobSidecar. File exists: {}", file.toString()); - return false; - } - - try { - Files.createDirectories(file.toPath().getParent()); - } catch (IOException e) { - LOG.error( - "Failed to write BlobSidecar. Could not make directories to: {}", - file.getParentFile().toString()); - return false; - } - - try (FileOutputStream output = new FileOutputStream(file)) { - jsonWriter.writeSlotBlobSidecars(output, blobSidecars); - indexWriter.write(formatIndexOutput(slotAndBlockRoot)); - indexWriter.newLine(); - return true; - } catch (IOException | NullPointerException e) { - LOG.error("Failed to write BlobSidecar.", e); - return false; - } - } - - private String formatIndexOutput(final SlotAndBlockRoot slotAndBlockRoot) { - return slotAndBlockRoot.getSlot() - + " " - + slotAndBlockRoot.getBlockRoot().toUnprefixedHexString(); - } - - @Override - public void close() throws IOException { - indexWriter.flush(); - indexWriter.close(); - } - } - - /** - * Given a basePath, slot and block root, return where to store/find the BlobSidecar. Initial - * implementation uses blockRoot as a hex string in the directory of the first two characters. - * - * @param slotAndBlockRoot The slot and block root. - * @return a path of where to store or find the BlobSidecar - */ - public File resolve(final SlotAndBlockRoot slotAndBlockRoot) { - // For blockroot 0x1a2bcd... the directory is basePath/1a/2b/1a2bcd... - // 256 * 256 directories = 65,536. - // Assume 8000 to 10000 blobs per day. With perfect hash distribution, - // all directories have one file after a week. After 1 year, expect 50 files in each directory. - String blockRootString = slotAndBlockRoot.getBlockRoot().toUnprefixedHexString(); - final String dir1 = blockRootString.substring(0, 2); - final String dir2 = blockRootString.substring(2, 4); - final String blobSidecarFilename = - dir1 + File.separator + dir2 + File.separator + blockRootString; - return baseDirectory.resolve(blobSidecarFilename).toFile(); - } -} diff --git a/storage/src/main/java/tech/pegasys/teku/storage/archive/nooparchive/DataArchiveNoopWriter.java b/storage/src/main/java/tech/pegasys/teku/storage/archive/nooparchive/DataArchiveNoopWriter.java deleted file mode 100644 index 00a631f5baa..00000000000 --- a/storage/src/main/java/tech/pegasys/teku/storage/archive/nooparchive/DataArchiveNoopWriter.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright Consensys Software Inc., 2025 - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ - -package tech.pegasys.teku.storage.archive.nooparchive; - -import java.io.IOException; -import tech.pegasys.teku.storage.archive.DataArchiveWriter; - -public class DataArchiveNoopWriter implements DataArchiveWriter { - - @Override - public boolean archive(final T data) { - return true; - } - - @Override - public void close() throws IOException {} -} diff --git a/storage/src/main/java/tech/pegasys/teku/storage/server/Database.java b/storage/src/main/java/tech/pegasys/teku/storage/server/Database.java index 4b1ddfd3f89..182f98ff894 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/server/Database.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/server/Database.java @@ -40,7 +40,7 @@ import tech.pegasys.teku.storage.api.UpdateResult; import tech.pegasys.teku.storage.api.WeakSubjectivityState; import tech.pegasys.teku.storage.api.WeakSubjectivityUpdate; -import tech.pegasys.teku.storage.archive.DataArchiveWriter; +import tech.pegasys.teku.storage.archive.BlobSidecarsArchiver; public interface Database extends AutoCloseable { @@ -74,16 +74,14 @@ void storeFinalizedBlocks( * * @param lastSlotToPrune inclusive, not reached if limit happens first * @param pruneLimit maximum number of slots to prune. - * @param archiveWriter write BlobSidecars to archive when pruning. + * @param blobSidecarsArchiver write BlobSidecars to archive when pruning. * @return true if number of pruned blobs reached the pruneLimit, false otherwise */ boolean pruneOldestBlobSidecars( - UInt64 lastSlotToPrune, - int pruneLimit, - final DataArchiveWriter> archiveWriter); + UInt64 lastSlotToPrune, int pruneLimit, BlobSidecarsArchiver blobSidecarsArchiver); boolean pruneOldestNonCanonicalBlobSidecars( - UInt64 lastSlotToPrune, int pruneLimit, DataArchiveWriter> archiveWriter); + UInt64 lastSlotToPrune, int pruneLimit, BlobSidecarsArchiver blobSidecarsArchiver); @MustBeClosed Stream streamBlobSidecarKeys(UInt64 startSlot, UInt64 endSlot); diff --git a/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/KvStoreDatabase.java b/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/KvStoreDatabase.java index 5e1107e433b..3bf0f208591 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/KvStoreDatabase.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/KvStoreDatabase.java @@ -71,7 +71,7 @@ import tech.pegasys.teku.storage.api.UpdateResult; import tech.pegasys.teku.storage.api.WeakSubjectivityState; import tech.pegasys.teku.storage.api.WeakSubjectivityUpdate; -import tech.pegasys.teku.storage.archive.DataArchiveWriter; +import tech.pegasys.teku.storage.archive.BlobSidecarsArchiver; import tech.pegasys.teku.storage.server.Database; import tech.pegasys.teku.storage.server.StateStorageMode; import tech.pegasys.teku.storage.server.kvstore.dataaccess.CombinedKvStoreDao; @@ -943,7 +943,7 @@ public Optional getNonCanonicalBlobSidecar(final SlotAndBlockRootAn public boolean pruneOldestBlobSidecars( final UInt64 lastSlotToPrune, final int pruneLimit, - final DataArchiveWriter> archiveWriter) { + final BlobSidecarsArchiver blobSidecarsArchiver) { final Optional earliestBlobSidecarSlot = getEarliestBlobSidecarSlot(); if (earliestBlobSidecarSlot.isPresent() && earliestBlobSidecarSlot.get().isGreaterThan(lastSlotToPrune)) { @@ -951,7 +951,7 @@ public boolean pruneOldestBlobSidecars( } try (final Stream prunableBlobKeys = streamBlobSidecarKeys(earliestBlobSidecarSlot.orElse(UInt64.ZERO), lastSlotToPrune)) { - return pruneBlobSidecars(pruneLimit, prunableBlobKeys, archiveWriter, false); + return pruneBlobSidecars(pruneLimit, prunableBlobKeys, blobSidecarsArchiver, false); } } @@ -959,7 +959,7 @@ public boolean pruneOldestBlobSidecars( public boolean pruneOldestNonCanonicalBlobSidecars( final UInt64 lastSlotToPrune, final int pruneLimit, - final DataArchiveWriter> archiveWriter) { + final BlobSidecarsArchiver blobSidecarsArchiver) { final Optional earliestBlobSidecarSlot = getEarliestBlobSidecarSlot(); if (earliestBlobSidecarSlot.isPresent() && earliestBlobSidecarSlot.get().isGreaterThan(lastSlotToPrune)) { @@ -968,14 +968,15 @@ public boolean pruneOldestNonCanonicalBlobSidecars( try (final Stream prunableNoncanonicalBlobKeys = streamNonCanonicalBlobSidecarKeys( earliestBlobSidecarSlot.orElse(UInt64.ZERO), lastSlotToPrune)) { - return pruneBlobSidecars(pruneLimit, prunableNoncanonicalBlobKeys, archiveWriter, true); + return pruneBlobSidecars( + pruneLimit, prunableNoncanonicalBlobKeys, blobSidecarsArchiver, true); } } private boolean pruneBlobSidecars( final int pruneLimit, final Stream prunableBlobKeys, - final DataArchiveWriter> archiveWriter, + final BlobSidecarsArchiver blobSidecarsArchiver, final boolean nonCanonicalBlobSidecars) { int pruned = 0; @@ -1008,11 +1009,15 @@ private boolean pruneBlobSidecars( LOG.warn("Failed to retrieve BlobSidecars for keys: {}", keys); } - // Attempt to archive the BlobSidecars. - final boolean blobSidecarArchived = archiveWriter.archive(blobSidecars); - if (!blobSidecarArchived) { - LOG.error("Failed to archive and prune BlobSidecars. Stopping pruning"); - break; + if (!keys.isEmpty()) { + // Attempt to archive the BlobSidecars. + final SlotAndBlockRoot slotAndBlockRoot = keys.getFirst().getSlotAndBlockRoot(); + final boolean blobSidecarArchived = + blobSidecarsArchiver.archive(slotAndBlockRoot, blobSidecars); + if (!blobSidecarArchived) { + LOG.error("Failed to archive and prune BlobSidecars. Stopping pruning"); + break; + } } // Remove the BlobSidecars from the database. diff --git a/storage/src/main/java/tech/pegasys/teku/storage/server/noop/NoOpDatabase.java b/storage/src/main/java/tech/pegasys/teku/storage/server/noop/NoOpDatabase.java index 185cc19e865..268eeebb543 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/server/noop/NoOpDatabase.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/server/noop/NoOpDatabase.java @@ -43,7 +43,7 @@ import tech.pegasys.teku.storage.api.UpdateResult; import tech.pegasys.teku.storage.api.WeakSubjectivityState; import tech.pegasys.teku.storage.api.WeakSubjectivityUpdate; -import tech.pegasys.teku.storage.archive.DataArchiveWriter; +import tech.pegasys.teku.storage.archive.BlobSidecarsArchiver; import tech.pegasys.teku.storage.server.Database; public class NoOpDatabase implements Database { @@ -305,6 +305,22 @@ public Optional getNonCanonicalBlobSidecar(final SlotAndBlockRootAn return Optional.empty(); } + @Override + public boolean pruneOldestBlobSidecars( + final UInt64 lastSlotToPrune, + final int pruneLimit, + final BlobSidecarsArchiver blobSidecarsArchiver) { + return false; + } + + @Override + public boolean pruneOldestNonCanonicalBlobSidecars( + final UInt64 lastSlotToPrune, + final int pruneLimit, + final BlobSidecarsArchiver blobSidecarsArchiver) { + return false; + } + @Override public Stream streamBlobSidecarKeys( final UInt64 startSlot, final UInt64 endSlot) { @@ -333,22 +349,6 @@ public Optional getEarliestBlobSidecarSlot() { return Optional.empty(); } - @Override - public boolean pruneOldestBlobSidecars( - final UInt64 lastSlotToPrune, - final int pruneLimit, - final DataArchiveWriter> archiveWriter) { - return false; - } - - @Override - public boolean pruneOldestNonCanonicalBlobSidecars( - final UInt64 lastSlotToPrune, - final int pruneLimit, - final DataArchiveWriter> archiveWriter) { - return false; - } - @Override public void close() {} } diff --git a/storage/src/main/java/tech/pegasys/teku/storage/server/pruner/BlobSidecarPruner.java b/storage/src/main/java/tech/pegasys/teku/storage/server/pruner/BlobSidecarPruner.java index fb37cb1c9e6..c35f2b8ab09 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/server/pruner/BlobSidecarPruner.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/server/pruner/BlobSidecarPruner.java @@ -13,9 +13,7 @@ package tech.pegasys.teku.storage.server.pruner; -import java.io.IOException; import java.time.Duration; -import java.util.List; import java.util.Optional; import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.atomic.AtomicLong; @@ -33,9 +31,7 @@ import tech.pegasys.teku.service.serviceutils.Service; import tech.pegasys.teku.spec.Spec; import tech.pegasys.teku.spec.config.SpecConfigDeneb; -import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar; -import tech.pegasys.teku.storage.archive.DataArchive; -import tech.pegasys.teku.storage.archive.DataArchiveWriter; +import tech.pegasys.teku.storage.archive.BlobSidecarsArchiver; import tech.pegasys.teku.storage.server.Database; import tech.pegasys.teku.storage.server.ShuttingDownException; @@ -59,12 +55,12 @@ public class BlobSidecarPruner extends Service { private final AtomicLong blobColumnSize = new AtomicLong(0); private final AtomicLong earliestBlobSidecarSlot = new AtomicLong(-1); private final boolean storeNonCanonicalBlobSidecars; - private final DataArchive dataArchive; + private final BlobSidecarsArchiver blobSidecarsArchiver; public BlobSidecarPruner( final Spec spec, final Database database, - final DataArchive dataArchive, + final BlobSidecarsArchiver blobSidecarsArchiver, final MetricsSystem metricsSystem, final AsyncRunner asyncRunner, final TimeProvider timeProvider, @@ -77,7 +73,7 @@ public BlobSidecarPruner( final boolean storeNonCanonicalBlobSidecars) { this.spec = spec; this.database = database; - this.dataArchive = dataArchive; + this.blobSidecarsArchiver = blobSidecarsArchiver; this.asyncRunner = asyncRunner; this.pruneInterval = pruneInterval; this.pruneLimit = pruneLimit; @@ -152,10 +148,10 @@ private void pruneBlobsPriorToAvailabilityWindow() { return; } LOG.debug("Pruning blobs up to slot {}, limit {}", latestPrunableSlot, pruneLimit); - try (DataArchiveWriter> archiveWriter = dataArchive.getBlobSidecarWriter()) { + try { final long blobsPruningStart = System.currentTimeMillis(); final boolean blobsPruningLimitReached = - database.pruneOldestBlobSidecars(latestPrunableSlot, pruneLimit, archiveWriter); + database.pruneOldestBlobSidecars(latestPrunableSlot, pruneLimit, blobSidecarsArchiver); logPruningResult( "Blobs pruning finished in {} ms. Limit reached: {}", blobsPruningStart, @@ -165,14 +161,12 @@ private void pruneBlobsPriorToAvailabilityWindow() { final long nonCanonicalBlobsPruningStart = System.currentTimeMillis(); final boolean nonCanonicalBlobsLimitReached = database.pruneOldestNonCanonicalBlobSidecars( - latestPrunableSlot, pruneLimit, archiveWriter); + latestPrunableSlot, pruneLimit, blobSidecarsArchiver); logPruningResult( "Non canonical Blobs pruning finished in {} ms. Limit reached: {}", nonCanonicalBlobsPruningStart, nonCanonicalBlobsLimitReached); } - } catch (IOException ex) { - LOG.error("Failed to get the BlobSidecar archive writer", ex); } catch (ShuttingDownException | RejectedExecutionException ex) { LOG.debug("Shutting down", ex); } diff --git a/storage/src/test/java/tech/pegasys/teku/storage/archive/fsarchive/FileSystemArchiveTest.java b/storage/src/test/java/tech/pegasys/teku/storage/archive/filesystem/FileSystemBlobSidecarsArchiverTest.java similarity index 59% rename from storage/src/test/java/tech/pegasys/teku/storage/archive/fsarchive/FileSystemArchiveTest.java rename to storage/src/test/java/tech/pegasys/teku/storage/archive/filesystem/FileSystemBlobSidecarsArchiverTest.java index 59f81ab9717..22a4ff46c83 100644 --- a/storage/src/test/java/tech/pegasys/teku/storage/archive/fsarchive/FileSystemArchiveTest.java +++ b/storage/src/test/java/tech/pegasys/teku/storage/archive/filesystem/FileSystemBlobSidecarsArchiverTest.java @@ -11,38 +11,40 @@ * specific language governing permissions and limitations under the License. */ -package tech.pegasys.teku.storage.archive.fsarchive; +package tech.pegasys.teku.storage.archive.filesystem; +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; -import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; -import java.util.ArrayList; import java.util.List; import java.util.stream.Stream; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; +import tech.pegasys.teku.infrastructure.json.JsonUtil; +import tech.pegasys.teku.infrastructure.json.types.DeserializableListTypeDefinition; import tech.pegasys.teku.infrastructure.unsigned.UInt64; import tech.pegasys.teku.spec.Spec; import tech.pegasys.teku.spec.TestSpecFactory; import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.Blob; import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar; import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlock; +import tech.pegasys.teku.spec.datastructures.blocks.SlotAndBlockRoot; import tech.pegasys.teku.spec.datastructures.type.SszKZGProof; import tech.pegasys.teku.spec.datastructures.util.SlotAndBlockRootAndBlobIndex; import tech.pegasys.teku.spec.logic.common.helpers.Predicates; import tech.pegasys.teku.spec.logic.versions.deneb.helpers.MiscHelpersDeneb; import tech.pegasys.teku.spec.schemas.SchemaDefinitionsDeneb; import tech.pegasys.teku.spec.util.DataStructureUtil; -import tech.pegasys.teku.storage.archive.DataArchiveWriter; -public class FileSystemArchiveTest { +public class FileSystemBlobSidecarsArchiverTest { + private static final Spec SPEC = TestSpecFactory.createMinimalDeneb(); private final Predicates predicates = new Predicates(SPEC.getGenesisSpecConfig()); private final SchemaDefinitionsDeneb schemaDefinitionsDeneb = @@ -55,12 +57,12 @@ public class FileSystemArchiveTest { private final DataStructureUtil dataStructureUtil = new DataStructureUtil(SPEC); static Path testTempDir; - static FileSystemArchive dataArchive; + static FileSystemBlobSidecarsArchiver blobSidecarsArchiver; @BeforeAll static void beforeEach() throws IOException { testTempDir = Files.createTempDirectory("blobs"); - dataArchive = new FileSystemArchive(testTempDir); + blobSidecarsArchiver = new FileSystemBlobSidecarsArchiver(SPEC, testTempDir); } @AfterEach @@ -79,75 +81,76 @@ public void tearDown() throws IOException { } } - BlobSidecar createBlobSidecar() { - final SignedBeaconBlock signedBeaconBlock = - dataStructureUtil.randomSignedBeaconBlockWithCommitments(1); - final Blob blob = dataStructureUtil.randomBlob(); - final SszKZGProof proof = dataStructureUtil.randomSszKZGProof(); - - return miscHelpersDeneb.constructBlobSidecar(signedBeaconBlock, UInt64.ZERO, blob, proof); - } - @Test void testResolve() { - SlotAndBlockRootAndBlobIndex slotAndBlockRootAndBlobIndex = + final SlotAndBlockRootAndBlobIndex slotAndBlockRootAndBlobIndex = new SlotAndBlockRootAndBlobIndex( UInt64.ONE, dataStructureUtil.randomBytes32(), UInt64.ZERO); - File file = dataArchive.resolve(slotAndBlockRootAndBlobIndex.getSlotAndBlockRoot()); + final Path path = + blobSidecarsArchiver.resolve(slotAndBlockRootAndBlobIndex.getSlotAndBlockRoot()); // Check if the file path is correct. Doesn't check the intermediate directories. - assertTrue(file.toString().startsWith(testTempDir.toString())); + assertTrue(path.toString().startsWith(testTempDir.toString())); assertTrue( - file.toString() + path.toString() .endsWith(slotAndBlockRootAndBlobIndex.getBlockRoot().toUnprefixedHexString())); } @Test - void testArchiveWithEmptyList() throws IOException { - DataArchiveWriter> blobWriter = dataArchive.getBlobSidecarWriter(); - ArrayList list = new ArrayList<>(); - assertTrue(blobWriter.archive(list)); - blobWriter.close(); + void testArchiveWithEmptyList() { + assertTrue(blobSidecarsArchiver.archive(dataStructureUtil.randomSlotAndBlockRoot(), List.of())); } @Test - void testArchiveWithNullList() throws IOException { - DataArchiveWriter> blobWriter = dataArchive.getBlobSidecarWriter(); - assertTrue(blobWriter.archive(null)); - blobWriter.close(); + void testArchiveWithNullList() { + assertTrue(blobSidecarsArchiver.archive(dataStructureUtil.randomSlotAndBlockRoot(), null)); } @Test void testWriteBlobSidecar() throws IOException { - DataArchiveWriter> blobWriter = dataArchive.getBlobSidecarWriter(); - ArrayList list = new ArrayList<>(); - BlobSidecar blobSidecar = createBlobSidecar(); - list.add(blobSidecar); - assertTrue(blobWriter.archive(list)); - blobWriter.close(); + final SlotAndBlockRoot slotAndBlockRoot = dataStructureUtil.randomSlotAndBlockRoot(); + final List blobSidecars = List.of(createBlobSidecar()); + assertTrue(blobSidecarsArchiver.archive(slotAndBlockRoot, blobSidecars)); // Check if the file was written - try (FileInputStream fis = - new FileInputStream(testTempDir.resolve(FileSystemArchive.INDEX_FILE).toFile())) { - String content = new String(fis.readAllBytes(), StandardCharsets.UTF_8); - String expected = - blobSidecar.getSlot().toString() + try (final FileInputStream indexFile = + new FileInputStream( + testTempDir.resolve(FileSystemBlobSidecarsArchiver.INDEX_FILE).toFile()); + final FileInputStream blobSidecarsFile = + new FileInputStream(blobSidecarsArchiver.resolve(slotAndBlockRoot).toFile())) { + final String indexFileContent = new String(indexFile.readAllBytes(), StandardCharsets.UTF_8); + final String expectedIndexFileContent = + slotAndBlockRoot.getSlot().toString() + " " - + blobSidecar.getSlotAndBlockRoot().getBlockRoot().toUnprefixedHexString(); - + + slotAndBlockRoot.getBlockRoot().toUnprefixedHexString(); // Windows new lines are different, so don't include new lines in the comparison. - assertTrue(content.contains(expected)); + assertTrue(indexFileContent.contains(expectedIndexFileContent)); + + final String blobSidecarsJson = + new String(blobSidecarsFile.readAllBytes(), StandardCharsets.UTF_8); + + final DeserializableListTypeDefinition blobSidecarsType = + new DeserializableListTypeDefinition<>( + schemaDefinitionsDeneb.getBlobSidecarSchema().getJsonTypeDefinition()); + + assertThat(JsonUtil.parse(blobSidecarsJson, blobSidecarsType)).isEqualTo(blobSidecars); } } @Test - void testFileAlreadyExists() throws IOException { - DataArchiveWriter> blobWriter = dataArchive.getBlobSidecarWriter(); - ArrayList list = new ArrayList<>(); - list.add(createBlobSidecar()); - assertTrue(blobWriter.archive(list)); + void testFileAlreadyExists() { + final SlotAndBlockRoot slotAndBlockRoot = dataStructureUtil.randomSlotAndBlockRoot(); + final List blobSidecars = List.of(createBlobSidecar()); + assertTrue(blobSidecarsArchiver.archive(slotAndBlockRoot, blobSidecars)); // Try to write the same file again - assertFalse(blobWriter.archive(list)); - blobWriter.close(); + assertFalse(blobSidecarsArchiver.archive(slotAndBlockRoot, blobSidecars)); + } + + private BlobSidecar createBlobSidecar() { + final SignedBeaconBlock signedBeaconBlock = + dataStructureUtil.randomSignedBeaconBlockWithCommitments(1); + final Blob blob = dataStructureUtil.randomBlob(); + final SszKZGProof proof = dataStructureUtil.randomSszKZGProof(); + return miscHelpersDeneb.constructBlobSidecar(signedBeaconBlock, UInt64.ZERO, blob, proof); } } diff --git a/storage/src/test/java/tech/pegasys/teku/storage/archive/fsarchive/BlobSidecarJsonWriterTest.java b/storage/src/test/java/tech/pegasys/teku/storage/archive/fsarchive/BlobSidecarJsonWriterTest.java deleted file mode 100644 index 974e47d87c9..00000000000 --- a/storage/src/test/java/tech/pegasys/teku/storage/archive/fsarchive/BlobSidecarJsonWriterTest.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Copyright Consensys Software Inc., 2025 - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ - -package tech.pegasys.teku.storage.archive.fsarchive; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.List; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import tech.pegasys.teku.spec.Spec; -import tech.pegasys.teku.spec.TestSpecFactory; -import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar; -import tech.pegasys.teku.spec.util.DataStructureUtil; - -public class BlobSidecarJsonWriterTest { - private static final Spec SPEC = TestSpecFactory.createMinimalDeneb(); - - BlobSidecarJsonWriter blobSidecarJsonWriter; - private DataStructureUtil dataStructureUtil; - - @BeforeEach - public void test() { - this.blobSidecarJsonWriter = new BlobSidecarJsonWriter(); - this.dataStructureUtil = new DataStructureUtil(SPEC); - } - - @Test - void testWriteSlotBlobSidecarsWithEmptyList() throws IOException { - ByteArrayOutputStream out = new ByteArrayOutputStream(); - List blobSidecars = new ArrayList<>(); - blobSidecarJsonWriter.writeSlotBlobSidecars(out, blobSidecars); - String json = out.toString(StandardCharsets.UTF_8); - assertEquals("[]", json); - } - - @Test - void testWriteSlotBlobSidecarsWithSingleElement() throws IOException { - ByteArrayOutputStream out = new ByteArrayOutputStream(); - List blobSidecars = new ArrayList<>(); - final BlobSidecar blobSidecar = - dataStructureUtil.randomBlobSidecarForBlock( - dataStructureUtil.randomSignedBeaconBlock(1), 0); - blobSidecars.add(blobSidecar); - blobSidecarJsonWriter.writeSlotBlobSidecars(out, blobSidecars); - String json = out.toString(StandardCharsets.UTF_8); - assertTrue(json.contains("index")); - assertTrue(json.contains("blob")); - assertTrue(json.contains("kzg_commitment")); - assertTrue(json.contains("kzg_proof")); - assertTrue(json.contains("signed_block_header")); - assertTrue(json.contains("parent_root")); - assertTrue(json.contains("state_root")); - assertTrue(json.contains("body_root")); - assertTrue(json.contains("signature")); - } - - @Test - void testWriteSlotBlobSidecarsNulls() { - assertThrows( - NullPointerException.class, () -> blobSidecarJsonWriter.writeSlotBlobSidecars(null, null)); - } - - @Test - void testWriteSlotBlobSidecarsNullOut() { - assertThrows( - NullPointerException.class, - () -> { - List blobSidecars = new ArrayList<>(); - blobSidecarJsonWriter.writeSlotBlobSidecars(null, blobSidecars); - }); - } - - @Test - void testWriteSlotBlobSidecarsNullList() { - assertThrows( - NullPointerException.class, - () -> { - ByteArrayOutputStream out = new ByteArrayOutputStream(); - blobSidecarJsonWriter.writeSlotBlobSidecars(out, null); - }); - } -} diff --git a/storage/src/test/java/tech/pegasys/teku/storage/server/pruner/BlobSidecarPrunerTest.java b/storage/src/test/java/tech/pegasys/teku/storage/server/pruner/BlobSidecarPrunerTest.java index 378e7306035..c6d237f2248 100644 --- a/storage/src/test/java/tech/pegasys/teku/storage/server/pruner/BlobSidecarPrunerTest.java +++ b/storage/src/test/java/tech/pegasys/teku/storage/server/pruner/BlobSidecarPrunerTest.java @@ -23,7 +23,6 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; -import java.io.IOException; import java.time.Duration; import java.util.Optional; import org.junit.jupiter.api.BeforeEach; @@ -38,8 +37,7 @@ import tech.pegasys.teku.spec.TestSpecFactory; import tech.pegasys.teku.spec.config.SpecConfig; import tech.pegasys.teku.spec.config.SpecConfigDeneb; -import tech.pegasys.teku.storage.archive.DataArchive; -import tech.pegasys.teku.storage.archive.nooparchive.NoopDataArchive; +import tech.pegasys.teku.storage.archive.BlobSidecarsArchiver; import tech.pegasys.teku.storage.server.Database; public class BlobSidecarPrunerTest { @@ -57,13 +55,13 @@ public class BlobSidecarPrunerTest { private final StubAsyncRunner asyncRunner = new StubAsyncRunner(timeProvider); private final Database database = mock(Database.class); private final StubMetricsSystem stubMetricsSystem = new StubMetricsSystem(); - private final DataArchive dataArchive = new NoopDataArchive(); + private final BlobSidecarsArchiver blobSidecarsArchiver = BlobSidecarsArchiver.NOOP; private final BlobSidecarPruner blobsPruner = new BlobSidecarPruner( spec, database, - dataArchive, + blobSidecarsArchiver, stubMetricsSystem, asyncRunner, timeProvider, @@ -118,7 +116,7 @@ void shouldNotPruneWhenLatestPrunableIncludeGenesis() { } @Test - void shouldPruneWhenLatestPrunableSlotIsGreaterThanOldestDAEpoch() throws IOException { + void shouldPruneWhenLatestPrunableSlotIsGreaterThanOldestDAEpoch() { final SpecConfig config = spec.forMilestone(SpecMilestone.DENEB).getConfig(); final SpecConfigDeneb specConfigDeneb = SpecConfigDeneb.required(config); // set current slot to MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS + 1 epoch + half epoch @@ -133,18 +131,14 @@ void shouldPruneWhenLatestPrunableSlotIsGreaterThanOldestDAEpoch() throws IOExce asyncRunner.executeDueActions(); verify(database) .pruneOldestBlobSidecars( - UInt64.valueOf((slotsPerEpoch / 2) - 1), - PRUNE_LIMIT, - dataArchive.getBlobSidecarWriter()); + UInt64.valueOf((slotsPerEpoch / 2) - 1), PRUNE_LIMIT, blobSidecarsArchiver); verify(database) .pruneOldestNonCanonicalBlobSidecars( - UInt64.valueOf((slotsPerEpoch / 2) - 1), - PRUNE_LIMIT, - dataArchive.getBlobSidecarWriter()); + UInt64.valueOf((slotsPerEpoch / 2) - 1), PRUNE_LIMIT, blobSidecarsArchiver); } @Test - void shouldUseEpochsStoreBlobs() throws IOException { + void shouldUseEpochsStoreBlobs() { final SpecConfig config = spec.forMilestone(SpecMilestone.DENEB).getConfig(); final SpecConfigDeneb specConfigDeneb = SpecConfigDeneb.required(config); final int defaultValue = specConfigDeneb.getMinEpochsForBlobSidecarsRequests(); @@ -164,7 +158,7 @@ void shouldUseEpochsStoreBlobs() throws IOException { new BlobSidecarPruner( specOverride, databaseOverride, - dataArchive, + blobSidecarsArchiver, stubMetricsSystem, asyncRunner, timeProvider, @@ -200,14 +194,10 @@ void shouldUseEpochsStoreBlobs() throws IOException { asyncRunner.executeDueActions(); verify(databaseOverride) .pruneOldestBlobSidecars( - UInt64.valueOf((slotsPerEpoch / 2) - 1), - PRUNE_LIMIT, - dataArchive.getBlobSidecarWriter()); + UInt64.valueOf((slotsPerEpoch / 2) - 1), PRUNE_LIMIT, blobSidecarsArchiver); verify(databaseOverride) .pruneOldestNonCanonicalBlobSidecars( - UInt64.valueOf((slotsPerEpoch / 2) - 1), - PRUNE_LIMIT, - dataArchive.getBlobSidecarWriter()); + UInt64.valueOf((slotsPerEpoch / 2) - 1), PRUNE_LIMIT, blobSidecarsArchiver); } @Test @@ -228,7 +218,7 @@ void shouldNotPruneWhenEpochsStoreBlobsIsMax() { new BlobSidecarPruner( specOverride, databaseOverride, - dataArchive, + blobSidecarsArchiver, stubMetricsSystem, asyncRunner, timeProvider, diff --git a/storage/src/test/java/tech/pegasys/teku/storage/store/StoreTest.java b/storage/src/test/java/tech/pegasys/teku/storage/store/StoreTest.java index 8c7aef79f74..d5153c79b59 100644 --- a/storage/src/test/java/tech/pegasys/teku/storage/store/StoreTest.java +++ b/storage/src/test/java/tech/pegasys/teku/storage/store/StoreTest.java @@ -51,7 +51,7 @@ import tech.pegasys.teku.spec.generator.ChainBuilder; import tech.pegasys.teku.storage.api.StubStorageUpdateChannel; import tech.pegasys.teku.storage.api.StubStorageUpdateChannelWithDelays; -import tech.pegasys.teku.storage.archive.nooparchive.DataArchiveNoopWriter; +import tech.pegasys.teku.storage.archive.BlobSidecarsArchiver; import tech.pegasys.teku.storage.storageSystem.InMemoryStorageSystemBuilder; import tech.pegasys.teku.storage.storageSystem.StorageSystem; import tech.pegasys.teku.storage.store.UpdatableStore.StoreTransaction; @@ -396,7 +396,7 @@ public void retrieveEarliestBlobSidecarSlot_shouldReturnUpdatedValue() { storageSystem .database() - .pruneOldestBlobSidecars(UInt64.valueOf(5), 3, new DataArchiveNoopWriter<>()); + .pruneOldestBlobSidecars(UInt64.valueOf(5), 3, BlobSidecarsArchiver.NOOP); assertThat(store.retrieveEarliestBlobSidecarSlot()) .isCompletedWithValueMatching( From 682167d2a14c9654d7c67f681786ba64bdcc6e98 Mon Sep 17 00:00:00 2001 From: Stefan Bratanov Date: Mon, 7 Apr 2025 20:02:53 +0100 Subject: [PATCH 02/12] changes --- ...eaconHandlerWithChainDataProviderTest.java | 2 + data/provider/build.gradle | 1 + .../pegasys/teku/api/ChainDataProvider.java | 4 +- .../BlobSidecarSelectorFactory.java | 10 +- .../api/AbstractChainDataProviderTest.java | 7 +- .../BlobSidecarSelectorFactoryTest.java | 5 +- .../beaconchain/BeaconChainController.java | 10 +- .../services/chainstorage/StorageService.java | 9 +- .../api/BlobSidecarsArchiveChannel.java | 41 +++++ .../teku/storage/api/StorageQueryChannel.java | 2 +- .../storage/server/kvstore/DatabaseTest.java | 8 +- .../storage/archive/BlobSidecarsArchiver.java | 28 ++- .../FileSystemBlobSidecarsArchiver.java | 166 ++++++++++++++---- .../client/CombinedChainDataClient.java | 33 ++-- .../server/kvstore/KvStoreDatabase.java | 8 +- .../FileSystemBlobSidecarsArchiverTest.java | 125 +++++++------ .../server/pruner/BlobSidecarPrunerTest.java | 2 +- 17 files changed, 338 insertions(+), 123 deletions(-) create mode 100644 storage/api/src/main/java/tech/pegasys/teku/storage/api/BlobSidecarsArchiveChannel.java diff --git a/data/beaconrestapi/src/test/java/tech/pegasys/teku/beaconrestapi/AbstractMigratedBeaconHandlerWithChainDataProviderTest.java b/data/beaconrestapi/src/test/java/tech/pegasys/teku/beaconrestapi/AbstractMigratedBeaconHandlerWithChainDataProviderTest.java index b11c340c7cc..9414f0a34fe 100644 --- a/data/beaconrestapi/src/test/java/tech/pegasys/teku/beaconrestapi/AbstractMigratedBeaconHandlerWithChainDataProviderTest.java +++ b/data/beaconrestapi/src/test/java/tech/pegasys/teku/beaconrestapi/AbstractMigratedBeaconHandlerWithChainDataProviderTest.java @@ -25,6 +25,7 @@ import tech.pegasys.teku.spec.util.DataStructureUtil; import tech.pegasys.teku.statetransition.validatorcache.ActiveValidatorCache; import tech.pegasys.teku.statetransition.validatorcache.ActiveValidatorChannel; +import tech.pegasys.teku.storage.archive.BlobSidecarsArchiver; import tech.pegasys.teku.storage.client.ChainUpdater; import tech.pegasys.teku.storage.client.CombinedChainDataClient; import tech.pegasys.teku.storage.client.RecentChainData; @@ -75,6 +76,7 @@ private void setupStorage( spec, recentChainData, combinedChainDataClient, + BlobSidecarsArchiver.NOOP, new RewardCalculator(spec, new BlockRewardCalculatorUtil(spec))); } } diff --git a/data/provider/build.gradle b/data/provider/build.gradle index a63324d0998..29199a6530d 100644 --- a/data/provider/build.gradle +++ b/data/provider/build.gradle @@ -12,6 +12,7 @@ dependencies { implementation project(':infrastructure:serviceutils') implementation project(':infrastructure:ssz') implementation project(':storage') + implementation project(':storage:api') implementation project(':beacon:sync') implementation project(':validator:api') diff --git a/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java b/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java index a1b7a1cb91c..f0ca23659cd 100644 --- a/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java +++ b/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java @@ -78,6 +78,7 @@ import tech.pegasys.teku.spec.logic.common.statetransition.epoch.status.ValidatorStatuses; import tech.pegasys.teku.spec.logic.common.statetransition.exceptions.EpochProcessingException; import tech.pegasys.teku.spec.logic.common.statetransition.exceptions.SlotProcessingException; +import tech.pegasys.teku.storage.api.BlobSidecarsArchiveChannel; import tech.pegasys.teku.storage.client.ChainDataUnavailableException; import tech.pegasys.teku.storage.client.CombinedChainDataClient; import tech.pegasys.teku.storage.client.RecentChainData; @@ -96,6 +97,7 @@ public ChainDataProvider( final Spec spec, final RecentChainData recentChainData, final CombinedChainDataClient combinedChainDataClient, + final BlobSidecarsArchiveChannel blobSidecarsArchiveChannel, final RewardCalculator rewardCalculator) { this( spec, @@ -103,7 +105,7 @@ public ChainDataProvider( combinedChainDataClient, new BlockSelectorFactory(spec, combinedChainDataClient), new StateSelectorFactory(spec, combinedChainDataClient), - new BlobSidecarSelectorFactory(spec, combinedChainDataClient), + new BlobSidecarSelectorFactory(spec, combinedChainDataClient, blobSidecarsArchiveChannel), rewardCalculator); } diff --git a/data/provider/src/main/java/tech/pegasys/teku/api/blobselector/BlobSidecarSelectorFactory.java b/data/provider/src/main/java/tech/pegasys/teku/api/blobselector/BlobSidecarSelectorFactory.java index 26ecd633d5f..6a6c8becfa7 100644 --- a/data/provider/src/main/java/tech/pegasys/teku/api/blobselector/BlobSidecarSelectorFactory.java +++ b/data/provider/src/main/java/tech/pegasys/teku/api/blobselector/BlobSidecarSelectorFactory.java @@ -28,6 +28,7 @@ import tech.pegasys.teku.spec.datastructures.blocks.SlotAndBlockRoot; import tech.pegasys.teku.spec.datastructures.blocks.blockbody.versions.deneb.BeaconBlockBodyDeneb; import tech.pegasys.teku.spec.datastructures.metadata.BlobSidecarsAndMetaData; +import tech.pegasys.teku.storage.api.BlobSidecarsArchiveChannel; import tech.pegasys.teku.storage.client.ChainHead; import tech.pegasys.teku.storage.client.CombinedChainDataClient; @@ -35,8 +36,15 @@ public class BlobSidecarSelectorFactory extends AbstractSelectorFactory indices = List.of(UInt64.ZERO, UInt64.ONE); @@ -56,7 +59,7 @@ public class BlobSidecarSelectorFactoryTest { private final List blobSidecars = data.randomBlobSidecars(3); private final BlobSidecarSelectorFactory blobSidecarSelectorFactory = - new BlobSidecarSelectorFactory(spec, client); + new BlobSidecarSelectorFactory(spec, client, blobSidecarsArchiveChannel); @Test public void headSelector_shouldGetHeadBlobSidecars() diff --git a/services/beaconchain/src/main/java/tech/pegasys/teku/services/beaconchain/BeaconChainController.java b/services/beaconchain/src/main/java/tech/pegasys/teku/services/beaconchain/BeaconChainController.java index 07d4db164bd..d6dfcd9cc40 100644 --- a/services/beaconchain/src/main/java/tech/pegasys/teku/services/beaconchain/BeaconChainController.java +++ b/services/beaconchain/src/main/java/tech/pegasys/teku/services/beaconchain/BeaconChainController.java @@ -172,6 +172,7 @@ import tech.pegasys.teku.statetransition.validation.signatures.SignatureVerificationService; import tech.pegasys.teku.statetransition.validatorcache.ActiveValidatorCache; import tech.pegasys.teku.statetransition.validatorcache.ActiveValidatorChannel; +import tech.pegasys.teku.storage.api.BlobSidecarsArchiveChannel; import tech.pegasys.teku.storage.api.ChainHeadChannel; import tech.pegasys.teku.storage.api.CombinedStorageChannel; import tech.pegasys.teku.storage.api.Eth1DepositStorageChannel; @@ -179,6 +180,7 @@ import tech.pegasys.teku.storage.api.StorageQueryChannel; import tech.pegasys.teku.storage.api.StorageUpdateChannel; import tech.pegasys.teku.storage.api.VoteUpdateChannel; +import tech.pegasys.teku.storage.archive.BlobSidecarsArchiver; import tech.pegasys.teku.storage.client.CombinedChainDataClient; import tech.pegasys.teku.storage.client.EarliestAvailableBlockSlot; import tech.pegasys.teku.storage.client.RecentChainData; @@ -283,6 +285,7 @@ public class BeaconChainController extends Service implements BeaconChainControl protected volatile KZG kzg; protected volatile BlobSidecarManager blobSidecarManager; protected volatile BlobSidecarGossipValidator blobSidecarValidator; + protected volatile BlobSidecarsArchiver blobSidecarsArchiver; protected volatile Optional terminalPowBlockMonitor = Optional.empty(); protected volatile ProposersDataManager proposersDataManager; protected volatile KeyValueStore keyValueStore; @@ -994,7 +997,12 @@ public void initValidatorApiHandler() { final ValidatorApiHandler validatorApiHandler = new ValidatorApiHandler( - new ChainDataProvider(spec, recentChainData, combinedChainDataClient, rewardCalculator), + new ChainDataProvider( + spec, + recentChainData, + combinedChainDataClient, + eventChannels.getPublisher(BlobSidecarsArchiveChannel.class, beaconAsyncRunner), + rewardCalculator), dataProvider.getNodeDataProvider(), dataProvider.getNetworkDataProvider(), combinedChainDataClient, diff --git a/services/chainstorage/src/main/java/tech/pegasys/teku/services/chainstorage/StorageService.java b/services/chainstorage/src/main/java/tech/pegasys/teku/services/chainstorage/StorageService.java index bd569c8cee3..197d991fee1 100644 --- a/services/chainstorage/src/main/java/tech/pegasys/teku/services/chainstorage/StorageService.java +++ b/services/chainstorage/src/main/java/tech/pegasys/teku/services/chainstorage/StorageService.java @@ -34,6 +34,7 @@ import tech.pegasys.teku.service.serviceutils.ServiceConfig; import tech.pegasys.teku.spec.SpecMilestone; import tech.pegasys.teku.spec.networks.Eth2Network; +import tech.pegasys.teku.storage.api.BlobSidecarsArchiveChannel; import tech.pegasys.teku.storage.api.CombinedStorageChannel; import tech.pegasys.teku.storage.api.Eth1DepositStorageChannel; import tech.pegasys.teku.storage.api.VoteUpdateChannel; @@ -163,9 +164,14 @@ protected SafeFuture doStart() { .getBlobsArchivePath() .map( path -> - new FileSystemBlobSidecarsArchiver(config.getSpec(), Path.of(path))) + new FileSystemBlobSidecarsArchiver( + config.getSpec(), Path.of(path), storagePrunerAsyncRunner)) .orElse(BlobSidecarsArchiver.NOOP); + final EventChannels eventChannels = serviceConfig.getEventChannels(); + + eventChannels.subscribe(BlobSidecarsArchiveChannel.class, blobSidecarsArchiver); + if (config.getSpec().isMilestoneSupported(SpecMilestone.DENEB)) { blobsPruner = Optional.of( @@ -184,7 +190,6 @@ protected SafeFuture doStart() { pruningActiveLabelledGauge, config.isStoreNonCanonicalBlocksEnabled())); } - final EventChannels eventChannels = serviceConfig.getEventChannels(); chainStorage = ChainStorage.create( database, diff --git a/storage/api/src/main/java/tech/pegasys/teku/storage/api/BlobSidecarsArchiveChannel.java b/storage/api/src/main/java/tech/pegasys/teku/storage/api/BlobSidecarsArchiveChannel.java new file mode 100644 index 00000000000..22330bf7f6c --- /dev/null +++ b/storage/api/src/main/java/tech/pegasys/teku/storage/api/BlobSidecarsArchiveChannel.java @@ -0,0 +1,41 @@ +/* + * Copyright Consensys Software Inc., 2025 + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package tech.pegasys.teku.storage.api; + +import java.util.List; +import java.util.Optional; +import org.apache.tuweni.bytes.Bytes32; +import tech.pegasys.teku.infrastructure.async.SafeFuture; +import tech.pegasys.teku.infrastructure.events.ChannelInterface; +import tech.pegasys.teku.infrastructure.unsigned.UInt64; +import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar; +import tech.pegasys.teku.spec.datastructures.blocks.SlotAndBlockRoot; + +public interface BlobSidecarsArchiveChannel extends ChannelInterface { + + void archive(SlotAndBlockRoot slotAndBlockRoot, List blobSidecars); + + SafeFuture>> retrieve(Bytes32 blockRoot, Optional maybeSlot); + + SafeFuture>> retrieve(UInt64 slot); + + default SafeFuture>> retrieve(final Bytes32 blockRoot) { + return retrieve(blockRoot, Optional.empty()); + } + + default SafeFuture>> retrieve( + final SlotAndBlockRoot slotAndBlockRoot) { + return retrieve(slotAndBlockRoot.getBlockRoot(), Optional.of(slotAndBlockRoot.getSlot())); + } +} diff --git a/storage/api/src/main/java/tech/pegasys/teku/storage/api/StorageQueryChannel.java b/storage/api/src/main/java/tech/pegasys/teku/storage/api/StorageQueryChannel.java index ad6bc10ed6a..a68d814dbb2 100644 --- a/storage/api/src/main/java/tech/pegasys/teku/storage/api/StorageQueryChannel.java +++ b/storage/api/src/main/java/tech/pegasys/teku/storage/api/StorageQueryChannel.java @@ -56,7 +56,7 @@ public interface StorageQueryChannel extends ChannelInterface { SafeFuture> getHotBlockAndStateByBlockRoot(Bytes32 blockRoot); SafeFuture> getHotStateAndBlockSummaryByBlockRoot( - final Bytes32 blockRoot); + Bytes32 blockRoot); /** * Returns "hot" blocks - the latest finalized block or blocks that descend from the latest diff --git a/storage/src/integration-test/java/tech/pegasys/teku/storage/server/kvstore/DatabaseTest.java b/storage/src/integration-test/java/tech/pegasys/teku/storage/server/kvstore/DatabaseTest.java index 0dd8f760c60..5fa4356c1ef 100644 --- a/storage/src/integration-test/java/tech/pegasys/teku/storage/server/kvstore/DatabaseTest.java +++ b/storage/src/integration-test/java/tech/pegasys/teku/storage/server/kvstore/DatabaseTest.java @@ -60,6 +60,7 @@ import tech.pegasys.teku.ethereum.pow.api.DepositTreeSnapshot; import tech.pegasys.teku.ethereum.pow.api.MinGenesisTimeBlockEvent; import tech.pegasys.teku.infrastructure.async.AsyncRunner; +import tech.pegasys.teku.infrastructure.async.StubAsyncRunner; import tech.pegasys.teku.infrastructure.unsigned.UInt64; import tech.pegasys.teku.spec.Spec; import tech.pegasys.teku.spec.TestSpecFactory; @@ -126,6 +127,7 @@ public class DatabaseTest { private RecentChainData recentChainData; private UpdatableStore store; private final List storageSystems = new ArrayList<>(); + private final StubAsyncRunner stubAsyncRunner = new StubAsyncRunner(); @BeforeEach public void setup() throws IOException { @@ -139,7 +141,8 @@ private void setupWithSpec(final Spec spec) throws IOException { this.chainProperties = new ChainProperties(spec); final Path blobsArchive = Files.createTempDirectory("blobs"); tmpDirectories.add(blobsArchive.toFile()); - this.blobSidecarsArchiver = new FileSystemBlobSidecarsArchiver(spec, blobsArchive); + this.blobSidecarsArchiver = + new FileSystemBlobSidecarsArchiver(spec, blobsArchive, stubAsyncRunner); genesisBlockAndState = chainBuilder.generateGenesis(genesisTime, true); genesisCheckpoint = getCheckpointForBlock(genesisBlockAndState.getBlock()); genesisAnchor = AnchorPoint.fromGenesisState(spec, genesisBlockAndState.getState()); @@ -365,7 +368,8 @@ public void verifyBlobsLifecycle(final DatabaseContext context) throws IOExcepti } private Path getSlotBlobsArchiveFile(final BlobSidecar blobSidecar) { - return blobSidecarsArchiver.resolve(blobSidecar.getSlotAndBlockRoot()); + return blobSidecarsArchiver.resolveArchivePath( + blobSidecar.getSlotAndBlockRoot().getBlockRoot()); } @TestTemplate diff --git a/storage/src/main/java/tech/pegasys/teku/storage/archive/BlobSidecarsArchiver.java b/storage/src/main/java/tech/pegasys/teku/storage/archive/BlobSidecarsArchiver.java index 82c3fac18a4..bf90653e557 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/archive/BlobSidecarsArchiver.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/archive/BlobSidecarsArchiver.java @@ -14,12 +14,34 @@ package tech.pegasys.teku.storage.archive; import java.util.List; +import java.util.Optional; +import org.apache.tuweni.bytes.Bytes32; +import tech.pegasys.teku.infrastructure.async.SafeFuture; +import tech.pegasys.teku.infrastructure.unsigned.UInt64; import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar; import tech.pegasys.teku.spec.datastructures.blocks.SlotAndBlockRoot; +import tech.pegasys.teku.storage.api.BlobSidecarsArchiveChannel; -public interface BlobSidecarsArchiver { +public interface BlobSidecarsArchiver extends BlobSidecarsArchiveChannel { - BlobSidecarsArchiver NOOP = (slotAndBlockRoot, blobSidecars) -> true; + SafeFuture>> EMPTY_FUTURE = + SafeFuture.completedFuture(Optional.empty()); - boolean archive(SlotAndBlockRoot slotAndBlockRoot, List blobSidecars); + BlobSidecarsArchiver NOOP = + new BlobSidecarsArchiver() { + @Override + public void archive( + final SlotAndBlockRoot slotAndBlockRoot, final List blobSidecars) {} + + @Override + public SafeFuture>> retrieve( + final Bytes32 blockRoot, final Optional maybeSlot) { + return EMPTY_FUTURE; + } + + @Override + public SafeFuture>> retrieve(final UInt64 slot) { + return EMPTY_FUTURE; + } + }; } diff --git a/storage/src/main/java/tech/pegasys/teku/storage/archive/filesystem/FileSystemBlobSidecarsArchiver.java b/storage/src/main/java/tech/pegasys/teku/storage/archive/filesystem/FileSystemBlobSidecarsArchiver.java index 0b679a8ae27..82609f70a53 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/archive/filesystem/FileSystemBlobSidecarsArchiver.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/archive/filesystem/FileSystemBlobSidecarsArchiver.java @@ -15,6 +15,8 @@ import static tech.pegasys.teku.infrastructure.json.types.DeserializableTypeDefinition.listOf; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonToken; import com.google.common.annotations.VisibleForTesting; import java.io.BufferedWriter; import java.io.File; @@ -25,10 +27,16 @@ import java.nio.file.Path; import java.nio.file.StandardOpenOption; import java.util.List; +import java.util.Optional; +import java.util.stream.Stream; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.tuweni.bytes.Bytes32; +import tech.pegasys.teku.infrastructure.async.AsyncRunner; +import tech.pegasys.teku.infrastructure.async.SafeFuture; import tech.pegasys.teku.infrastructure.json.JsonUtil; -import tech.pegasys.teku.infrastructure.json.types.SerializableTypeDefinition; +import tech.pegasys.teku.infrastructure.json.types.DeserializableTypeDefinition; +import tech.pegasys.teku.infrastructure.unsigned.UInt64; import tech.pegasys.teku.spec.Spec; import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar; import tech.pegasys.teku.spec.datastructures.blocks.SlotAndBlockRoot; @@ -37,71 +45,129 @@ public class FileSystemBlobSidecarsArchiver implements BlobSidecarsArchiver { - static final String INDEX_FILE = "index.dat"; + private static final String INDEX_FILE_SUFFIX = "index.dat"; + private static final long INDEX_FILE_SLOT_RANGE_SIZE = 100_000; private static final Logger LOG = LogManager.getLogger(); private final Spec spec; private final Path baseDirectory; - private final Path indexFile; + private final AsyncRunner asyncRunner; - public FileSystemBlobSidecarsArchiver(final Spec spec, final Path baseDirectory) { + public FileSystemBlobSidecarsArchiver( + final Spec spec, final Path baseDirectory, final AsyncRunner asyncRunner) { this.spec = spec; this.baseDirectory = baseDirectory; - this.indexFile = baseDirectory.resolve(INDEX_FILE); + this.asyncRunner = asyncRunner; } @Override - public boolean archive( + public void archive( final SlotAndBlockRoot slotAndBlockRoot, final List blobSidecars) { - if (blobSidecars == null || blobSidecars.isEmpty()) { - return true; - } + final Path archivePath = resolveArchivePath(slotAndBlockRoot.getBlockRoot()); - final Path path = resolve(slotAndBlockRoot); - if (Files.exists(path)) { - LOG.error("Failed to write BlobSidecars. File exists: {}", path); - return false; + if (Files.exists(archivePath)) { + LOG.error( + "Failed to archive blob sidecars for {}. File exists: {}", slotAndBlockRoot, archivePath); + return; } try { - Files.createDirectories(path.getParent()); + Files.createDirectories(archivePath.getParent()); } catch (final IOException __) { - LOG.error("Failed to write BlobSidecars. Could not create directories: {}", path.getParent()); - return false; + LOG.error( + "Failed to archive blob sidecars for {}. Could not create directories: {}", + slotAndBlockRoot, + archivePath.getParent()); + return; } - try (final OutputStream output = Files.newOutputStream(path); + final Path indexFile = resolveIndexFile(slotAndBlockRoot.getSlot()); + + try (final OutputStream output = Files.newOutputStream(archivePath); final BufferedWriter indexWriter = Files.newBufferedWriter( indexFile, StandardCharsets.UTF_8, StandardOpenOption.CREATE, StandardOpenOption.APPEND)) { - writeBlobSidecars(output, slotAndBlockRoot, blobSidecars); + if (blobSidecars.isEmpty()) { + // empty list + output.write("[]".getBytes()); + } else { + writeBlobSidecars(output, slotAndBlockRoot.getSlot(), blobSidecars); + } indexWriter.write(formatIndexFileOutput(slotAndBlockRoot)); indexWriter.newLine(); - return true; } catch (final IOException ex) { - LOG.error(String.format("Failed to write BlobSidecars for %s", slotAndBlockRoot), ex); - return false; + LOG.error(String.format("Failed to archive blob sidecars for %s", slotAndBlockRoot), ex); + } + } + + @Override + public SafeFuture>> retrieve( + final Bytes32 blockRoot, final Optional maybeSlot) { + return asyncRunner.runAsync(() -> retrieveInternal(blockRoot, maybeSlot)); + } + + private Optional> retrieveInternal( + final Bytes32 blockRoot, final Optional maybeSlot) { + try { + final Path archivePath = resolveArchivePath(blockRoot); + if (!Files.exists(archivePath)) { + return Optional.empty(); + } + final String blobSidecarsJson = Files.readString(archivePath); + final UInt64 slot = maybeSlot.orElseGet(() -> getSlot(blobSidecarsJson)); + final List blobSidecars = + JsonUtil.parse(blobSidecarsJson, getJsonTypeDefinition(slot)); + return Optional.of(blobSidecars); + } catch (IOException ex) { + LOG.error(String.format("Failed to retrieve blob sidecars for block root %s", blockRoot), ex); + return Optional.empty(); + } + } + + @Override + public SafeFuture>> retrieve(final UInt64 slot) { + return asyncRunner.runAsync(() -> retrieveInternal(slot)); + } + + private Optional> retrieveInternal(final UInt64 slot) { + final Path indexFile = resolveIndexFile(slot); + if (!Files.exists(indexFile)) { + return Optional.empty(); + } + try (final Stream lines = Files.lines(indexFile)) { + return lines + .filter(line -> line.startsWith(slot.toString())) + .findFirst() + .flatMap( + line -> { + // lines in the index file are in the format of: " " + final Bytes32 blockRoot = Bytes32.fromHexString(line.split(" ")[1]); + return retrieveInternal(blockRoot, Optional.of(slot)); + }); + } catch (IOException ex) { + LOG.error(String.format("Failed to retrieve blob sidecars for slot %s", slot), ex); + return Optional.empty(); } } /** - * Given a basePath, slot and block root, return where to store/find the BlobSidecar. Initial + * Given a basePath, block root, return where to store/find the BlobSidecar. Initial * implementation uses blockRoot as a hex string in the directory of the first two characters. * - * @param slotAndBlockRoot The slot and block root. + * @param blockRoot the block root. * @return a path of where to store or find the BlobSidecar */ @VisibleForTesting - public Path resolve(final SlotAndBlockRoot slotAndBlockRoot) { + public Path resolveArchivePath(final Bytes32 blockRoot) { // For blockroot 0x1a2bcd... the directory is basePath/1a/2b/1a2bcd... // 256 * 256 directories = 65,536. // Assume 8000 to 10000 blobs per day. With perfect hash distribution, // all directories have one file after a week. After 1 year, expect 50 files in each directory. - String blockRootString = slotAndBlockRoot.getBlockRoot().toUnprefixedHexString(); + final String blockRootString = blockRoot.toUnprefixedHexString(); final String dir1 = blockRootString.substring(0, 2); final String dir2 = blockRootString.substring(2, 4); final String blobSidecarFilename = @@ -109,18 +175,32 @@ public Path resolve(final SlotAndBlockRoot slotAndBlockRoot) { return baseDirectory.resolve(blobSidecarFilename); } + /** + * Given a basePath, slot, return where to store/find the slot -> root index file + * + * @param slot the slot. + * @return a path of where to store or find the slot -> root index file + */ + @VisibleForTesting + Path resolveIndexFile(final UInt64 slot) { + final UInt64 lowerBound = + slot.dividedBy(INDEX_FILE_SLOT_RANGE_SIZE).times(INDEX_FILE_SLOT_RANGE_SIZE); + final UInt64 upperBound = lowerBound.plus(INDEX_FILE_SLOT_RANGE_SIZE).minusMinZero(1); + final String fileName = String.format("%s-%s_%s", lowerBound, upperBound, INDEX_FILE_SUFFIX); + return baseDirectory.resolve(fileName); + } + private void writeBlobSidecars( - final OutputStream out, - final SlotAndBlockRoot slotAndBlockRoot, - final List blobSidecars) + final OutputStream out, final UInt64 slot, final List blobSidecars) throws IOException { - final SerializableTypeDefinition> type = - listOf( - SchemaDefinitionsDeneb.required( - spec.atSlot(slotAndBlockRoot.getSlot()).getSchemaDefinitions()) - .getBlobSidecarSchema() - .getJsonTypeDefinition()); - JsonUtil.serializeToBytes(blobSidecars, type, out); + JsonUtil.serializeToBytes(blobSidecars, getJsonTypeDefinition(slot), out); + } + + private DeserializableTypeDefinition> getJsonTypeDefinition(final UInt64 slot) { + return listOf( + SchemaDefinitionsDeneb.required(spec.atSlot(slot).getSchemaDefinitions()) + .getBlobSidecarSchema() + .getJsonTypeDefinition()); } private String formatIndexFileOutput(final SlotAndBlockRoot slotAndBlockRoot) { @@ -128,4 +208,20 @@ private String formatIndexFileOutput(final SlotAndBlockRoot slotAndBlockRoot) { + " " + slotAndBlockRoot.getBlockRoot().toUnprefixedHexString(); } + + /** retrieve the slot from the json content */ + private UInt64 getSlot(final String blobSidecarsJson) { + try (JsonParser parser = JsonUtil.FACTORY.createParser(blobSidecarsJson)) { + while (parser.nextToken() != null) { + if ("slot".equals(parser.currentName()) && parser.nextToken() == JsonToken.VALUE_STRING) { + return UInt64.valueOf(parser.getValueAsString()); + } + } + } catch (IOException __) { + LOG.warn("Couldn't retrieve slot from blob sidecars archive json: {}", blobSidecarsJson); + return UInt64.ZERO; + } + LOG.warn("Couldn't retrieve slot from blob sidecars archive json: {}", blobSidecarsJson); + return UInt64.ZERO; + } } diff --git a/storage/src/main/java/tech/pegasys/teku/storage/client/CombinedChainDataClient.java b/storage/src/main/java/tech/pegasys/teku/storage/client/CombinedChainDataClient.java index e4193a5cf2b..33cbbed2893 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/client/CombinedChainDataClient.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/client/CombinedChainDataClient.java @@ -177,15 +177,15 @@ public SafeFuture> getBlobSidecars( public SafeFuture> getBlobSidecars( final SlotAndBlockRoot slotAndBlockRoot, final List indices) { - final Optional> maybeBlobSidecars = - recentChainData.getBlobSidecars(slotAndBlockRoot); - if (maybeBlobSidecars.isPresent()) { - return SafeFuture.completedFuture(filterBlobSidecars(maybeBlobSidecars.get(), indices)); - } - return historicalChainData - .getBlobSidecarKeys(slotAndBlockRoot) - .thenApply(keys -> filterBlobSidecarKeys(keys, indices)) - .thenCompose(this::getBlobSidecars); + return recentChainData + .getBlobSidecars(slotAndBlockRoot) + .map(blobSidecars -> SafeFuture.completedFuture(filterBlobSidecars(blobSidecars, indices))) + .orElseGet( + () -> + historicalChainData + .getBlobSidecarKeys(slotAndBlockRoot) + .thenApply(keys -> filterBlobSidecarKeys(keys, indices)) + .thenCompose(this::getBlobSidecars)); } public SafeFuture> getAllBlobSidecars( @@ -516,13 +516,14 @@ public SafeFuture> getBlobSidecarByKey( final SlotAndBlockRootAndBlobIndex key) { final Optional> maybeBlobSidecars = recentChainData.getBlobSidecars(key.getSlotAndBlockRoot()); - if (maybeBlobSidecars.isPresent()) { - return key.getBlobIndex().isLessThan(maybeBlobSidecars.get().size()) - ? SafeFuture.completedFuture( - Optional.of(maybeBlobSidecars.get().get(key.getBlobIndex().intValue()))) - : SafeFuture.completedFuture(Optional.empty()); - } - return historicalChainData.getBlobSidecar(key); + return maybeBlobSidecars + .>>map( + blobSidecars -> + key.getBlobIndex().isLessThan(blobSidecars.size()) + ? SafeFuture.completedFuture( + Optional.of(blobSidecars.get(key.getBlobIndex().intValue()))) + : SafeFuture.completedFuture(Optional.empty())) + .orElseGet(() -> historicalChainData.getBlobSidecar(key)); } public SafeFuture> getBlobSidecarKeys( diff --git a/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/KvStoreDatabase.java b/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/KvStoreDatabase.java index 3bf0f208591..805a5f523dd 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/KvStoreDatabase.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/server/kvstore/KvStoreDatabase.java @@ -1010,14 +1010,8 @@ private boolean pruneBlobSidecars( } if (!keys.isEmpty()) { - // Attempt to archive the BlobSidecars. final SlotAndBlockRoot slotAndBlockRoot = keys.getFirst().getSlotAndBlockRoot(); - final boolean blobSidecarArchived = - blobSidecarsArchiver.archive(slotAndBlockRoot, blobSidecars); - if (!blobSidecarArchived) { - LOG.error("Failed to archive and prune BlobSidecars. Stopping pruning"); - break; - } + blobSidecarsArchiver.archive(slotAndBlockRoot, blobSidecars); } // Remove the BlobSidecars from the database. diff --git a/storage/src/test/java/tech/pegasys/teku/storage/archive/filesystem/FileSystemBlobSidecarsArchiverTest.java b/storage/src/test/java/tech/pegasys/teku/storage/archive/filesystem/FileSystemBlobSidecarsArchiverTest.java index 22a4ff46c83..bc6c3fd1688 100644 --- a/storage/src/test/java/tech/pegasys/teku/storage/archive/filesystem/FileSystemBlobSidecarsArchiverTest.java +++ b/storage/src/test/java/tech/pegasys/teku/storage/archive/filesystem/FileSystemBlobSidecarsArchiverTest.java @@ -14,21 +14,23 @@ package tech.pegasys.teku.storage.archive.filesystem; import static org.assertj.core.api.Assertions.assertThat; -import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; -import java.io.FileInputStream; import java.io.IOException; -import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.util.List; +import java.util.Optional; import java.util.stream.Stream; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; -import tech.pegasys.teku.infrastructure.json.JsonUtil; -import tech.pegasys.teku.infrastructure.json.types.DeserializableListTypeDefinition; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import tech.pegasys.teku.infrastructure.async.SafeFuture; +import tech.pegasys.teku.infrastructure.async.SafeFutureAssert; +import tech.pegasys.teku.infrastructure.async.StubAsyncRunner; import tech.pegasys.teku.infrastructure.unsigned.UInt64; import tech.pegasys.teku.spec.Spec; import tech.pegasys.teku.spec.TestSpecFactory; @@ -57,12 +59,14 @@ public class FileSystemBlobSidecarsArchiverTest { private final DataStructureUtil dataStructureUtil = new DataStructureUtil(SPEC); static Path testTempDir; + static StubAsyncRunner stubAsyncRunner; static FileSystemBlobSidecarsArchiver blobSidecarsArchiver; @BeforeAll static void beforeEach() throws IOException { testTempDir = Files.createTempDirectory("blobs"); - blobSidecarsArchiver = new FileSystemBlobSidecarsArchiver(SPEC, testTempDir); + stubAsyncRunner = new StubAsyncRunner(); + blobSidecarsArchiver = new FileSystemBlobSidecarsArchiver(SPEC, testTempDir, stubAsyncRunner); } @AfterEach @@ -82,12 +86,13 @@ public void tearDown() throws IOException { } @Test - void testResolve() { + void testResolveArchivePath() { final SlotAndBlockRootAndBlobIndex slotAndBlockRootAndBlobIndex = new SlotAndBlockRootAndBlobIndex( UInt64.ONE, dataStructureUtil.randomBytes32(), UInt64.ZERO); final Path path = - blobSidecarsArchiver.resolve(slotAndBlockRootAndBlobIndex.getSlotAndBlockRoot()); + blobSidecarsArchiver.resolveArchivePath( + slotAndBlockRootAndBlobIndex.getSlotAndBlockRoot().getBlockRoot()); // Check if the file path is correct. Doesn't check the intermediate directories. assertTrue(path.toString().startsWith(testTempDir.toString())); @@ -96,61 +101,79 @@ void testResolve() { .endsWith(slotAndBlockRootAndBlobIndex.getBlockRoot().toUnprefixedHexString())); } - @Test - void testArchiveWithEmptyList() { - assertTrue(blobSidecarsArchiver.archive(dataStructureUtil.randomSlotAndBlockRoot(), List.of())); - } - - @Test - void testArchiveWithNullList() { - assertTrue(blobSidecarsArchiver.archive(dataStructureUtil.randomSlotAndBlockRoot(), null)); + @ParameterizedTest + @MethodSource("testResolveIndexPathArguments") + void testResolveIndexPath(final UInt64 slot, final String expectedIndexFileName) { + final Path actualIndexFile = blobSidecarsArchiver.resolveIndexFile(slot); + assertThat(actualIndexFile).hasFileName(expectedIndexFileName); } @Test - void testWriteBlobSidecar() throws IOException { + void testArchiveWithEmptyList() { final SlotAndBlockRoot slotAndBlockRoot = dataStructureUtil.randomSlotAndBlockRoot(); - final List blobSidecars = List.of(createBlobSidecar()); - assertTrue(blobSidecarsArchiver.archive(slotAndBlockRoot, blobSidecars)); - - // Check if the file was written - try (final FileInputStream indexFile = - new FileInputStream( - testTempDir.resolve(FileSystemBlobSidecarsArchiver.INDEX_FILE).toFile()); - final FileInputStream blobSidecarsFile = - new FileInputStream(blobSidecarsArchiver.resolve(slotAndBlockRoot).toFile())) { - final String indexFileContent = new String(indexFile.readAllBytes(), StandardCharsets.UTF_8); - final String expectedIndexFileContent = - slotAndBlockRoot.getSlot().toString() - + " " - + slotAndBlockRoot.getBlockRoot().toUnprefixedHexString(); - // Windows new lines are different, so don't include new lines in the comparison. - assertTrue(indexFileContent.contains(expectedIndexFileContent)); - - final String blobSidecarsJson = - new String(blobSidecarsFile.readAllBytes(), StandardCharsets.UTF_8); - - final DeserializableListTypeDefinition blobSidecarsType = - new DeserializableListTypeDefinition<>( - schemaDefinitionsDeneb.getBlobSidecarSchema().getJsonTypeDefinition()); - - assertThat(JsonUtil.parse(blobSidecarsJson, blobSidecarsType)).isEqualTo(blobSidecars); - } + // archiving + blobSidecarsArchiver.archive(slotAndBlockRoot, List.of()); + + // retrieving + final SafeFuture>> blobSidecars = + blobSidecarsArchiver.retrieve(slotAndBlockRoot); + stubAsyncRunner.executeDueActions(); + // empty list + assertThat(SafeFutureAssert.safeJoin(blobSidecars)).hasValue(List.of()); } @Test - void testFileAlreadyExists() { - final SlotAndBlockRoot slotAndBlockRoot = dataStructureUtil.randomSlotAndBlockRoot(); - final List blobSidecars = List.of(createBlobSidecar()); - assertTrue(blobSidecarsArchiver.archive(slotAndBlockRoot, blobSidecars)); - // Try to write the same file again - assertFalse(blobSidecarsArchiver.archive(slotAndBlockRoot, blobSidecars)); + void testArchiveAndRetrieveBlobSidecars() { + final UInt64 slot = UInt64.valueOf(42); + final SlotAndBlockRoot slotAndBlockRoot = + new SlotAndBlockRoot(slot, dataStructureUtil.randomBytes32()); + final List blobSidecars = + List.of(createBlobSidecar(slot), createBlobSidecar(slot)); + + // archiving + blobSidecarsArchiver.archive(slotAndBlockRoot, blobSidecars); + + // test index file exists + assertThat(testTempDir.resolve("0-99999_index.dat")).exists(); + + // retrieving by root + final SafeFuture>> retrievedBlobSidecarsByRoot = + blobSidecarsArchiver.retrieve(slotAndBlockRoot.getBlockRoot()); + stubAsyncRunner.executeDueActions(); + + assertThat(SafeFutureAssert.safeJoin(retrievedBlobSidecarsByRoot)).hasValue(blobSidecars); + + // retrieving by slot (using index file) + final SafeFuture>> retrievedBlobSidecarsBySlot = + blobSidecarsArchiver.retrieve(slotAndBlockRoot.getSlot()); + stubAsyncRunner.executeDueActions(); + + assertThat(SafeFutureAssert.safeJoin(retrievedBlobSidecarsBySlot)).hasValue(blobSidecars); } - private BlobSidecar createBlobSidecar() { + private BlobSidecar createBlobSidecar(final UInt64 slot) { final SignedBeaconBlock signedBeaconBlock = - dataStructureUtil.randomSignedBeaconBlockWithCommitments(1); + dataStructureUtil.randomSignedBeaconBlockWithCommitments(slot, 1); final Blob blob = dataStructureUtil.randomBlob(); final SszKZGProof proof = dataStructureUtil.randomSszKZGProof(); return miscHelpersDeneb.constructBlobSidecar(signedBeaconBlock, UInt64.ZERO, blob, proof); } + + private static Stream testResolveIndexPathArguments() { + return Stream.of( + Arguments.of(UInt64.valueOf(0L), "0-99999_index.dat"), + Arguments.of(UInt64.valueOf(42L), "0-99999_index.dat"), + Arguments.of(UInt64.valueOf(99999L), "0-99999_index.dat"), + Arguments.of(UInt64.valueOf(100000L), "100000-199999_index.dat"), + Arguments.of(UInt64.valueOf(100001L), "100000-199999_index.dat"), + Arguments.of(UInt64.valueOf(199999L), "100000-199999_index.dat"), + Arguments.of(UInt64.valueOf(200000L), "200000-299999_index.dat"), + Arguments.of(UInt64.valueOf(999999999L), "999900000-999999999_index.dat"), + Arguments.of(UInt64.valueOf(1000000000L), "1000000000-1000099999_index.dat"), + Arguments.of(UInt64.valueOf(1L), "0-99999_index.dat"), + Arguments.of(UInt64.valueOf(99998L), "0-99999_index.dat"), + Arguments.of(UInt64.valueOf(99999L), "0-99999_index.dat"), + Arguments.of(UInt64.valueOf(100000L), "100000-199999_index.dat"), + Arguments.of(UInt64.valueOf(123456789L), "123400000-123499999_index.dat")); + } } diff --git a/storage/src/test/java/tech/pegasys/teku/storage/server/pruner/BlobSidecarPrunerTest.java b/storage/src/test/java/tech/pegasys/teku/storage/server/pruner/BlobSidecarPrunerTest.java index c6d237f2248..ed6065df9c5 100644 --- a/storage/src/test/java/tech/pegasys/teku/storage/server/pruner/BlobSidecarPrunerTest.java +++ b/storage/src/test/java/tech/pegasys/teku/storage/server/pruner/BlobSidecarPrunerTest.java @@ -55,7 +55,7 @@ public class BlobSidecarPrunerTest { private final StubAsyncRunner asyncRunner = new StubAsyncRunner(timeProvider); private final Database database = mock(Database.class); private final StubMetricsSystem stubMetricsSystem = new StubMetricsSystem(); - private final BlobSidecarsArchiver blobSidecarsArchiver = BlobSidecarsArchiver.NOOP; + private final BlobSidecarsArchiver blobSidecarsArchiver = mock(BlobSidecarsArchiver.class); private final BlobSidecarPruner blobsPruner = new BlobSidecarPruner( From ee9d6652a08dcd62c20c228e32650d02df23da3b Mon Sep 17 00:00:00 2001 From: Stefan Bratanov Date: Mon, 7 Apr 2025 22:28:08 +0100 Subject: [PATCH 03/12] changes --- .../pegasys/teku/api/ChainDataProvider.java | 4 +- .../BlobSidecarSelectorFactory.java | 10 +-- .../api/AbstractChainDataProviderTest.java | 7 +- .../BlobSidecarSelectorFactoryTest.java | 5 +- .../beaconchain/BeaconChainController.java | 10 +-- .../services/chainstorage/StorageService.java | 11 ++- .../api/BlobSidecarsArchiveChannel.java | 41 ----------- .../teku/storage/api/StorageQueryChannel.java | 6 ++ .../storage/archive/BlobSidecarsArchiver.java | 42 ++++++++--- .../FileSystemBlobSidecarsArchiver.java | 16 +---- .../client/CombinedChainDataClient.java | 70 +++++++++++++++---- .../teku/storage/server/ChainStorage.java | 32 +++++++-- .../CombinedStorageChannelSplitter.java | 11 +++ .../FileSystemBlobSidecarsArchiverTest.java | 19 ++--- .../storage/api/StubStorageQueryChannel.java | 11 +++ .../storage/storageSystem/StorageSystem.java | 4 +- 16 files changed, 168 insertions(+), 131 deletions(-) delete mode 100644 storage/api/src/main/java/tech/pegasys/teku/storage/api/BlobSidecarsArchiveChannel.java diff --git a/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java b/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java index f0ca23659cd..a1b7a1cb91c 100644 --- a/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java +++ b/data/provider/src/main/java/tech/pegasys/teku/api/ChainDataProvider.java @@ -78,7 +78,6 @@ import tech.pegasys.teku.spec.logic.common.statetransition.epoch.status.ValidatorStatuses; import tech.pegasys.teku.spec.logic.common.statetransition.exceptions.EpochProcessingException; import tech.pegasys.teku.spec.logic.common.statetransition.exceptions.SlotProcessingException; -import tech.pegasys.teku.storage.api.BlobSidecarsArchiveChannel; import tech.pegasys.teku.storage.client.ChainDataUnavailableException; import tech.pegasys.teku.storage.client.CombinedChainDataClient; import tech.pegasys.teku.storage.client.RecentChainData; @@ -97,7 +96,6 @@ public ChainDataProvider( final Spec spec, final RecentChainData recentChainData, final CombinedChainDataClient combinedChainDataClient, - final BlobSidecarsArchiveChannel blobSidecarsArchiveChannel, final RewardCalculator rewardCalculator) { this( spec, @@ -105,7 +103,7 @@ public ChainDataProvider( combinedChainDataClient, new BlockSelectorFactory(spec, combinedChainDataClient), new StateSelectorFactory(spec, combinedChainDataClient), - new BlobSidecarSelectorFactory(spec, combinedChainDataClient, blobSidecarsArchiveChannel), + new BlobSidecarSelectorFactory(spec, combinedChainDataClient), rewardCalculator); } diff --git a/data/provider/src/main/java/tech/pegasys/teku/api/blobselector/BlobSidecarSelectorFactory.java b/data/provider/src/main/java/tech/pegasys/teku/api/blobselector/BlobSidecarSelectorFactory.java index 6a6c8becfa7..26ecd633d5f 100644 --- a/data/provider/src/main/java/tech/pegasys/teku/api/blobselector/BlobSidecarSelectorFactory.java +++ b/data/provider/src/main/java/tech/pegasys/teku/api/blobselector/BlobSidecarSelectorFactory.java @@ -28,7 +28,6 @@ import tech.pegasys.teku.spec.datastructures.blocks.SlotAndBlockRoot; import tech.pegasys.teku.spec.datastructures.blocks.blockbody.versions.deneb.BeaconBlockBodyDeneb; import tech.pegasys.teku.spec.datastructures.metadata.BlobSidecarsAndMetaData; -import tech.pegasys.teku.storage.api.BlobSidecarsArchiveChannel; import tech.pegasys.teku.storage.client.ChainHead; import tech.pegasys.teku.storage.client.CombinedChainDataClient; @@ -36,15 +35,8 @@ public class BlobSidecarSelectorFactory extends AbstractSelectorFactory indices = List.of(UInt64.ZERO, UInt64.ONE); @@ -59,7 +56,7 @@ public class BlobSidecarSelectorFactoryTest { private final List blobSidecars = data.randomBlobSidecars(3); private final BlobSidecarSelectorFactory blobSidecarSelectorFactory = - new BlobSidecarSelectorFactory(spec, client, blobSidecarsArchiveChannel); + new BlobSidecarSelectorFactory(spec, client); @Test public void headSelector_shouldGetHeadBlobSidecars() diff --git a/services/beaconchain/src/main/java/tech/pegasys/teku/services/beaconchain/BeaconChainController.java b/services/beaconchain/src/main/java/tech/pegasys/teku/services/beaconchain/BeaconChainController.java index d6dfcd9cc40..07d4db164bd 100644 --- a/services/beaconchain/src/main/java/tech/pegasys/teku/services/beaconchain/BeaconChainController.java +++ b/services/beaconchain/src/main/java/tech/pegasys/teku/services/beaconchain/BeaconChainController.java @@ -172,7 +172,6 @@ import tech.pegasys.teku.statetransition.validation.signatures.SignatureVerificationService; import tech.pegasys.teku.statetransition.validatorcache.ActiveValidatorCache; import tech.pegasys.teku.statetransition.validatorcache.ActiveValidatorChannel; -import tech.pegasys.teku.storage.api.BlobSidecarsArchiveChannel; import tech.pegasys.teku.storage.api.ChainHeadChannel; import tech.pegasys.teku.storage.api.CombinedStorageChannel; import tech.pegasys.teku.storage.api.Eth1DepositStorageChannel; @@ -180,7 +179,6 @@ import tech.pegasys.teku.storage.api.StorageQueryChannel; import tech.pegasys.teku.storage.api.StorageUpdateChannel; import tech.pegasys.teku.storage.api.VoteUpdateChannel; -import tech.pegasys.teku.storage.archive.BlobSidecarsArchiver; import tech.pegasys.teku.storage.client.CombinedChainDataClient; import tech.pegasys.teku.storage.client.EarliestAvailableBlockSlot; import tech.pegasys.teku.storage.client.RecentChainData; @@ -285,7 +283,6 @@ public class BeaconChainController extends Service implements BeaconChainControl protected volatile KZG kzg; protected volatile BlobSidecarManager blobSidecarManager; protected volatile BlobSidecarGossipValidator blobSidecarValidator; - protected volatile BlobSidecarsArchiver blobSidecarsArchiver; protected volatile Optional terminalPowBlockMonitor = Optional.empty(); protected volatile ProposersDataManager proposersDataManager; protected volatile KeyValueStore keyValueStore; @@ -997,12 +994,7 @@ public void initValidatorApiHandler() { final ValidatorApiHandler validatorApiHandler = new ValidatorApiHandler( - new ChainDataProvider( - spec, - recentChainData, - combinedChainDataClient, - eventChannels.getPublisher(BlobSidecarsArchiveChannel.class, beaconAsyncRunner), - rewardCalculator), + new ChainDataProvider(spec, recentChainData, combinedChainDataClient, rewardCalculator), dataProvider.getNodeDataProvider(), dataProvider.getNetworkDataProvider(), combinedChainDataClient, diff --git a/services/chainstorage/src/main/java/tech/pegasys/teku/services/chainstorage/StorageService.java b/services/chainstorage/src/main/java/tech/pegasys/teku/services/chainstorage/StorageService.java index 197d991fee1..2f2fd36daab 100644 --- a/services/chainstorage/src/main/java/tech/pegasys/teku/services/chainstorage/StorageService.java +++ b/services/chainstorage/src/main/java/tech/pegasys/teku/services/chainstorage/StorageService.java @@ -34,7 +34,6 @@ import tech.pegasys.teku.service.serviceutils.ServiceConfig; import tech.pegasys.teku.spec.SpecMilestone; import tech.pegasys.teku.spec.networks.Eth2Network; -import tech.pegasys.teku.storage.api.BlobSidecarsArchiveChannel; import tech.pegasys.teku.storage.api.CombinedStorageChannel; import tech.pegasys.teku.storage.api.Eth1DepositStorageChannel; import tech.pegasys.teku.storage.api.VoteUpdateChannel; @@ -168,10 +167,6 @@ protected SafeFuture doStart() { config.getSpec(), Path.of(path), storagePrunerAsyncRunner)) .orElse(BlobSidecarsArchiver.NOOP); - final EventChannels eventChannels = serviceConfig.getEventChannels(); - - eventChannels.subscribe(BlobSidecarsArchiveChannel.class, blobSidecarsArchiver); - if (config.getSpec().isMilestoneSupported(SpecMilestone.DENEB)) { blobsPruner = Optional.of( @@ -195,7 +190,11 @@ protected SafeFuture doStart() { database, config.getSpec(), config.getDataStorageMode(), - config.getStateRebuildTimeoutSeconds()); + config.getStateRebuildTimeoutSeconds(), + blobSidecarsArchiver); + + final EventChannels eventChannels = serviceConfig.getEventChannels(); + final DepositStorage depositStorage = DepositStorage.create( eventChannels.getPublisher(Eth1EventsChannel.class), diff --git a/storage/api/src/main/java/tech/pegasys/teku/storage/api/BlobSidecarsArchiveChannel.java b/storage/api/src/main/java/tech/pegasys/teku/storage/api/BlobSidecarsArchiveChannel.java deleted file mode 100644 index 22330bf7f6c..00000000000 --- a/storage/api/src/main/java/tech/pegasys/teku/storage/api/BlobSidecarsArchiveChannel.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright Consensys Software Inc., 2025 - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on - * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the - * specific language governing permissions and limitations under the License. - */ - -package tech.pegasys.teku.storage.api; - -import java.util.List; -import java.util.Optional; -import org.apache.tuweni.bytes.Bytes32; -import tech.pegasys.teku.infrastructure.async.SafeFuture; -import tech.pegasys.teku.infrastructure.events.ChannelInterface; -import tech.pegasys.teku.infrastructure.unsigned.UInt64; -import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar; -import tech.pegasys.teku.spec.datastructures.blocks.SlotAndBlockRoot; - -public interface BlobSidecarsArchiveChannel extends ChannelInterface { - - void archive(SlotAndBlockRoot slotAndBlockRoot, List blobSidecars); - - SafeFuture>> retrieve(Bytes32 blockRoot, Optional maybeSlot); - - SafeFuture>> retrieve(UInt64 slot); - - default SafeFuture>> retrieve(final Bytes32 blockRoot) { - return retrieve(blockRoot, Optional.empty()); - } - - default SafeFuture>> retrieve( - final SlotAndBlockRoot slotAndBlockRoot) { - return retrieve(slotAndBlockRoot.getBlockRoot(), Optional.of(slotAndBlockRoot.getSlot())); - } -} diff --git a/storage/api/src/main/java/tech/pegasys/teku/storage/api/StorageQueryChannel.java b/storage/api/src/main/java/tech/pegasys/teku/storage/api/StorageQueryChannel.java index a68d814dbb2..5f4c1e9326f 100644 --- a/storage/api/src/main/java/tech/pegasys/teku/storage/api/StorageQueryChannel.java +++ b/storage/api/src/main/java/tech/pegasys/teku/storage/api/StorageQueryChannel.java @@ -109,4 +109,10 @@ SafeFuture> getBlobSidecarKeys( SafeFuture> getBlobSidecarKeys( SlotAndBlockRoot slotAndBlockRoot); + + // blobs archival methods + SafeFuture>> getArchivedBlobSidecars( + SlotAndBlockRoot slotAndBlockRoot); + + SafeFuture>> getArchivedBlobSidecars(UInt64 slot); } diff --git a/storage/src/main/java/tech/pegasys/teku/storage/archive/BlobSidecarsArchiver.java b/storage/src/main/java/tech/pegasys/teku/storage/archive/BlobSidecarsArchiver.java index bf90653e557..3c58eb38812 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/archive/BlobSidecarsArchiver.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/archive/BlobSidecarsArchiver.java @@ -16,16 +16,12 @@ import java.util.List; import java.util.Optional; import org.apache.tuweni.bytes.Bytes32; -import tech.pegasys.teku.infrastructure.async.SafeFuture; import tech.pegasys.teku.infrastructure.unsigned.UInt64; import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar; import tech.pegasys.teku.spec.datastructures.blocks.SlotAndBlockRoot; -import tech.pegasys.teku.storage.api.BlobSidecarsArchiveChannel; +import tech.pegasys.teku.spec.datastructures.util.SlotAndBlockRootAndBlobIndex; -public interface BlobSidecarsArchiver extends BlobSidecarsArchiveChannel { - - SafeFuture>> EMPTY_FUTURE = - SafeFuture.completedFuture(Optional.empty()); +public interface BlobSidecarsArchiver { BlobSidecarsArchiver NOOP = new BlobSidecarsArchiver() { @@ -34,14 +30,40 @@ public void archive( final SlotAndBlockRoot slotAndBlockRoot, final List blobSidecars) {} @Override - public SafeFuture>> retrieve( + public Optional> retrieve( final Bytes32 blockRoot, final Optional maybeSlot) { - return EMPTY_FUTURE; + return Optional.empty(); } @Override - public SafeFuture>> retrieve(final UInt64 slot) { - return EMPTY_FUTURE; + public Optional> retrieve(final UInt64 slot) { + return Optional.empty(); } }; + + void archive(SlotAndBlockRoot slotAndBlockRoot, List blobSidecars); + + Optional> retrieve(Bytes32 blockRoot, Optional maybeSlot); + + Optional> retrieve(UInt64 slot); + + default Optional> retrieve(final SlotAndBlockRoot slotAndBlockRoot) { + return retrieve(slotAndBlockRoot.getBlockRoot(), Optional.of(slotAndBlockRoot.getSlot())); + } + + default Optional retrieve( + final SlotAndBlockRootAndBlobIndex slotAndBlockRootAndBlobIndex) { + return retrieve( + slotAndBlockRootAndBlobIndex.getBlockRoot(), + Optional.of(slotAndBlockRootAndBlobIndex.getSlot())) + .flatMap( + blobSidecars -> + blobSidecars.stream() + .filter( + blobSidecar -> + blobSidecar + .getIndex() + .equals(slotAndBlockRootAndBlobIndex.getBlobIndex())) + .findFirst()); + } } diff --git a/storage/src/main/java/tech/pegasys/teku/storage/archive/filesystem/FileSystemBlobSidecarsArchiver.java b/storage/src/main/java/tech/pegasys/teku/storage/archive/filesystem/FileSystemBlobSidecarsArchiver.java index 82609f70a53..919375f137d 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/archive/filesystem/FileSystemBlobSidecarsArchiver.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/archive/filesystem/FileSystemBlobSidecarsArchiver.java @@ -33,7 +33,6 @@ import org.apache.logging.log4j.Logger; import org.apache.tuweni.bytes.Bytes32; import tech.pegasys.teku.infrastructure.async.AsyncRunner; -import tech.pegasys.teku.infrastructure.async.SafeFuture; import tech.pegasys.teku.infrastructure.json.JsonUtil; import tech.pegasys.teku.infrastructure.json.types.DeserializableTypeDefinition; import tech.pegasys.teku.infrastructure.unsigned.UInt64; @@ -105,12 +104,7 @@ public void archive( } @Override - public SafeFuture>> retrieve( - final Bytes32 blockRoot, final Optional maybeSlot) { - return asyncRunner.runAsync(() -> retrieveInternal(blockRoot, maybeSlot)); - } - - private Optional> retrieveInternal( + public Optional> retrieve( final Bytes32 blockRoot, final Optional maybeSlot) { try { final Path archivePath = resolveArchivePath(blockRoot); @@ -129,11 +123,7 @@ private Optional> retrieveInternal( } @Override - public SafeFuture>> retrieve(final UInt64 slot) { - return asyncRunner.runAsync(() -> retrieveInternal(slot)); - } - - private Optional> retrieveInternal(final UInt64 slot) { + public Optional> retrieve(final UInt64 slot) { final Path indexFile = resolveIndexFile(slot); if (!Files.exists(indexFile)) { return Optional.empty(); @@ -146,7 +136,7 @@ private Optional> retrieveInternal(final UInt64 slot) { line -> { // lines in the index file are in the format of: " " final Bytes32 blockRoot = Bytes32.fromHexString(line.split(" ")[1]); - return retrieveInternal(blockRoot, Optional.of(slot)); + return retrieve(blockRoot, Optional.of(slot)); }); } catch (IOException ex) { LOG.error(String.format("Failed to retrieve blob sidecars for slot %s", slot), ex); diff --git a/storage/src/main/java/tech/pegasys/teku/storage/client/CombinedChainDataClient.java b/storage/src/main/java/tech/pegasys/teku/storage/client/CombinedChainDataClient.java index 33cbbed2893..0d406fb166e 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/client/CombinedChainDataClient.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/client/CombinedChainDataClient.java @@ -172,7 +172,15 @@ public SafeFuture> getBlobSidecars( return historicalChainData .getBlobSidecarKeys(slot) .thenApply(keys -> filterBlobSidecarKeys(keys, indices)) - .thenCompose(this::getBlobSidecars); + .thenCompose(this::getBlobSidecars) + .thenCompose( + blobSidecars -> { + if (blobSidecars.isEmpty()) { + // attempt retrieving from archive + return getArchivedBlobSidecars(slot, indices); + } + return SafeFuture.completedFuture(blobSidecars); + }); } public SafeFuture> getBlobSidecars( @@ -185,7 +193,29 @@ public SafeFuture> getBlobSidecars( historicalChainData .getBlobSidecarKeys(slotAndBlockRoot) .thenApply(keys -> filterBlobSidecarKeys(keys, indices)) - .thenCompose(this::getBlobSidecars)); + .thenCompose(this::getBlobSidecars) + .thenCompose( + blobSidecars -> { + if (blobSidecars.isEmpty()) { + return getArchivedBlobSidecars(slotAndBlockRoot, indices); + } + return SafeFuture.completedFuture(blobSidecars); + })); + } + + private SafeFuture> getArchivedBlobSidecars( + final SlotAndBlockRoot slotAndBlockRoot, final List indices) { + return historicalChainData + .getArchivedBlobSidecars(slotAndBlockRoot) + .thenApply( + maybeArchivedBlobSidecars -> + maybeArchivedBlobSidecars + .map( + archivedBlobSidecars -> + archivedBlobSidecars.stream() + .filter(blobSidecar -> indices.contains(blobSidecar.getIndex())) + .toList()) + .orElse(List.of())); } public SafeFuture> getAllBlobSidecars( @@ -193,7 +223,30 @@ public SafeFuture> getAllBlobSidecars( return historicalChainData .getAllBlobSidecarKeys(slot) .thenApply(keys -> filterBlobSidecarKeys(keys, indices)) - .thenCompose(this::getAllBlobSidecars); + .thenCompose(this::getBlobSidecars) + .thenCompose( + blobSidecars -> { + if (blobSidecars.isEmpty()) { + // attempt retrieving from archive + return getArchivedBlobSidecars(slot, indices); + } + return SafeFuture.completedFuture(blobSidecars); + }); + } + + private SafeFuture> getArchivedBlobSidecars( + final UInt64 slot, final List indices) { + return historicalChainData + .getArchivedBlobSidecars(slot) + .thenApply( + maybeArchivedBlobSidecars -> + maybeArchivedBlobSidecars + .map( + archivedBlobSidecars -> + archivedBlobSidecars.stream() + .filter(blobSidecar -> indices.contains(blobSidecar.getIndex())) + .toList()) + .orElse(List.of())); } public SafeFuture> getStateAtSlotExact(final UInt64 slot) { @@ -514,9 +567,8 @@ public SafeFuture> getBlobSidecarByBlockRootAndIndex( public SafeFuture> getBlobSidecarByKey( final SlotAndBlockRootAndBlobIndex key) { - final Optional> maybeBlobSidecars = - recentChainData.getBlobSidecars(key.getSlotAndBlockRoot()); - return maybeBlobSidecars + return recentChainData + .getBlobSidecars(key.getSlotAndBlockRoot()) .>>map( blobSidecars -> key.getBlobIndex().isLessThan(blobSidecars.size()) @@ -818,12 +870,6 @@ private SafeFuture> getBlobSidecars( .thenApply(blobSidecars -> blobSidecars.stream().flatMap(Optional::stream).toList()); } - private SafeFuture> getAllBlobSidecars( - final Stream keys) { - return SafeFuture.collectAll(keys.map(this::getAllBlobSidecarByKey)) - .thenApply(blobSidecars -> blobSidecars.stream().flatMap(Optional::stream).toList()); - } - private Optional getFinalizedCheckpoint() { return Optional.ofNullable(getStore()).map(ReadOnlyStore::getFinalizedCheckpoint); } diff --git a/storage/src/main/java/tech/pegasys/teku/storage/server/ChainStorage.java b/storage/src/main/java/tech/pegasys/teku/storage/server/ChainStorage.java index 85b446b5dac..13110e23a6c 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/server/ChainStorage.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/server/ChainStorage.java @@ -48,38 +48,44 @@ import tech.pegasys.teku.storage.api.VoteUpdateChannel; import tech.pegasys.teku.storage.api.WeakSubjectivityState; import tech.pegasys.teku.storage.api.WeakSubjectivityUpdate; +import tech.pegasys.teku.storage.archive.BlobSidecarsArchiver; import tech.pegasys.teku.storage.server.state.FinalizedStateCache; public class ChainStorage implements StorageUpdateChannel, StorageQueryChannel, VoteUpdateChannel, ChainStorageFacade { private static final Logger LOG = LogManager.getLogger(); + private final Database database; private final FinalizedStateCache finalizedStateCache; - private final StateStorageMode dataStorageMode; + private final BlobSidecarsArchiver blobSidecarsArchiver; private Optional cachedStoreData = Optional.empty(); private ChainStorage( final Database database, final FinalizedStateCache finalizedStateCache, - final StateStorageMode dataStorageMode) { + final StateStorageMode dataStorageMode, + final BlobSidecarsArchiver blobSidecarsArchiver) { this.database = database; this.finalizedStateCache = finalizedStateCache; this.dataStorageMode = dataStorageMode; + this.blobSidecarsArchiver = blobSidecarsArchiver; } public static ChainStorage create( final Database database, final Spec spec, final StateStorageMode dataStorageMode, - final int stateRebuildTimeoutSeconds) { + final int stateRebuildTimeoutSeconds, + final BlobSidecarsArchiver blobSidecarsArchiver) { final int finalizedStateCacheSize = spec.getSlotsPerEpoch(SpecConfig.GENESIS_EPOCH) * 3; return new ChainStorage( database, new FinalizedStateCache( spec, database, finalizedStateCacheSize, true, stateRebuildTimeoutSeconds), - dataStorageMode); + dataStorageMode, + blobSidecarsArchiver); } private synchronized Optional getStore() { @@ -313,7 +319,12 @@ public SafeFuture> getEarliestAvailableBlobSidecarSlot() { @Override public SafeFuture> getBlobSidecar(final SlotAndBlockRootAndBlobIndex key) { - return SafeFuture.of(() -> database.getBlobSidecar(key)); + return SafeFuture.of( + () -> + database + .getBlobSidecar(key) + // retrieve from archive + .or(() -> blobSidecarsArchiver.retrieve(key))); } @Override @@ -363,4 +374,15 @@ public SafeFuture> getBlobSidecarKeys( final SlotAndBlockRoot slotAndBlockRoot) { return SafeFuture.of(() -> database.getBlobSidecarKeys(slotAndBlockRoot)); } + + @Override + public SafeFuture>> getArchivedBlobSidecars( + final SlotAndBlockRoot slotAndBlockRoot) { + return SafeFuture.of(() -> blobSidecarsArchiver.retrieve(slotAndBlockRoot)); + } + + @Override + public SafeFuture>> getArchivedBlobSidecars(final UInt64 slot) { + return SafeFuture.of(() -> blobSidecarsArchiver.retrieve(slot)); + } } diff --git a/storage/src/main/java/tech/pegasys/teku/storage/server/CombinedStorageChannelSplitter.java b/storage/src/main/java/tech/pegasys/teku/storage/server/CombinedStorageChannelSplitter.java index 163a5066a15..3639c0a57ac 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/server/CombinedStorageChannelSplitter.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/server/CombinedStorageChannelSplitter.java @@ -249,4 +249,15 @@ public SafeFuture> getBlobSidecarKeys( final SlotAndBlockRoot slotAndBlockRoot) { return asyncRunner.runAsync(() -> queryDelegate.getBlobSidecarKeys(slotAndBlockRoot)); } + + @Override + public SafeFuture>> getArchivedBlobSidecars( + final SlotAndBlockRoot slotAndBlockRoot) { + return asyncRunner.runAsync(() -> queryDelegate.getArchivedBlobSidecars(slotAndBlockRoot)); + } + + @Override + public SafeFuture>> getArchivedBlobSidecars(final UInt64 slot) { + return asyncRunner.runAsync(() -> queryDelegate.getArchivedBlobSidecars(slot)); + } } diff --git a/storage/src/test/java/tech/pegasys/teku/storage/archive/filesystem/FileSystemBlobSidecarsArchiverTest.java b/storage/src/test/java/tech/pegasys/teku/storage/archive/filesystem/FileSystemBlobSidecarsArchiverTest.java index bc6c3fd1688..31495a30a05 100644 --- a/storage/src/test/java/tech/pegasys/teku/storage/archive/filesystem/FileSystemBlobSidecarsArchiverTest.java +++ b/storage/src/test/java/tech/pegasys/teku/storage/archive/filesystem/FileSystemBlobSidecarsArchiverTest.java @@ -28,8 +28,6 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; -import tech.pegasys.teku.infrastructure.async.SafeFuture; -import tech.pegasys.teku.infrastructure.async.SafeFutureAssert; import tech.pegasys.teku.infrastructure.async.StubAsyncRunner; import tech.pegasys.teku.infrastructure.unsigned.UInt64; import tech.pegasys.teku.spec.Spec; @@ -115,11 +113,10 @@ void testArchiveWithEmptyList() { blobSidecarsArchiver.archive(slotAndBlockRoot, List.of()); // retrieving - final SafeFuture>> blobSidecars = + final Optional> blobSidecars = blobSidecarsArchiver.retrieve(slotAndBlockRoot); - stubAsyncRunner.executeDueActions(); // empty list - assertThat(SafeFutureAssert.safeJoin(blobSidecars)).hasValue(List.of()); + assertThat(blobSidecars).hasValue(List.of()); } @Test @@ -137,18 +134,16 @@ void testArchiveAndRetrieveBlobSidecars() { assertThat(testTempDir.resolve("0-99999_index.dat")).exists(); // retrieving by root - final SafeFuture>> retrievedBlobSidecarsByRoot = - blobSidecarsArchiver.retrieve(slotAndBlockRoot.getBlockRoot()); - stubAsyncRunner.executeDueActions(); + final Optional> retrievedBlobSidecarsByRoot = + blobSidecarsArchiver.retrieve(slotAndBlockRoot.getBlockRoot(), Optional.empty()); - assertThat(SafeFutureAssert.safeJoin(retrievedBlobSidecarsByRoot)).hasValue(blobSidecars); + assertThat(retrievedBlobSidecarsByRoot).hasValue(blobSidecars); // retrieving by slot (using index file) - final SafeFuture>> retrievedBlobSidecarsBySlot = + final Optional> retrievedBlobSidecarsBySlot = blobSidecarsArchiver.retrieve(slotAndBlockRoot.getSlot()); - stubAsyncRunner.executeDueActions(); - assertThat(SafeFutureAssert.safeJoin(retrievedBlobSidecarsBySlot)).hasValue(blobSidecars); + assertThat(retrievedBlobSidecarsBySlot).hasValue(blobSidecars); } private BlobSidecar createBlobSidecar(final UInt64 slot) { diff --git a/storage/src/testFixtures/java/tech/pegasys/teku/storage/api/StubStorageQueryChannel.java b/storage/src/testFixtures/java/tech/pegasys/teku/storage/api/StubStorageQueryChannel.java index de8d78bb2d9..157c9c1d56f 100644 --- a/storage/src/testFixtures/java/tech/pegasys/teku/storage/api/StubStorageQueryChannel.java +++ b/storage/src/testFixtures/java/tech/pegasys/teku/storage/api/StubStorageQueryChannel.java @@ -176,6 +176,17 @@ public SafeFuture> getBlobSidecarKeys( return SafeFuture.completedFuture(List.of()); } + @Override + public SafeFuture>> getArchivedBlobSidecars( + final SlotAndBlockRoot slotAndBlockRoot) { + return SafeFuture.completedFuture(Optional.empty()); + } + + @Override + public SafeFuture>> getArchivedBlobSidecars(final UInt64 slot) { + return SafeFuture.completedFuture(Optional.empty()); + } + @Override public SafeFuture> getBlobSidecarsBySlotAndBlockRoot( final SlotAndBlockRoot slotAndBlockRoot) { diff --git a/storage/src/testFixtures/java/tech/pegasys/teku/storage/storageSystem/StorageSystem.java b/storage/src/testFixtures/java/tech/pegasys/teku/storage/storageSystem/StorageSystem.java index a986d5cc950..a6c30d7683f 100644 --- a/storage/src/testFixtures/java/tech/pegasys/teku/storage/storageSystem/StorageSystem.java +++ b/storage/src/testFixtures/java/tech/pegasys/teku/storage/storageSystem/StorageSystem.java @@ -29,6 +29,7 @@ import tech.pegasys.teku.storage.api.FinalizedCheckpointChannel; import tech.pegasys.teku.storage.api.StubFinalizedCheckpointChannel; import tech.pegasys.teku.storage.api.TrackingChainHeadChannel; +import tech.pegasys.teku.storage.archive.BlobSidecarsArchiver; import tech.pegasys.teku.storage.client.ChainUpdater; import tech.pegasys.teku.storage.client.CombinedChainDataClient; import tech.pegasys.teku.storage.client.EarliestAvailableBlockSlot; @@ -97,7 +98,8 @@ static StorageSystem create( // Create and start storage server final ChainStorage chainStorageServer = - ChainStorage.create(database, spec, storageMode, stateRebuildTimeoutSeconds); + ChainStorage.create( + database, spec, storageMode, stateRebuildTimeoutSeconds, BlobSidecarsArchiver.NOOP); // Create recent chain data final FinalizedCheckpointChannel finalizedCheckpointChannel = From cfd4de3bdae768df18fb3d96cfc396b1573d0e9e Mon Sep 17 00:00:00 2001 From: Stefan Bratanov Date: Mon, 7 Apr 2025 22:36:13 +0100 Subject: [PATCH 04/12] fix assemble --- .../archive/filesystem/FileSystemBlobSidecarsArchiver.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/storage/src/main/java/tech/pegasys/teku/storage/archive/filesystem/FileSystemBlobSidecarsArchiver.java b/storage/src/main/java/tech/pegasys/teku/storage/archive/filesystem/FileSystemBlobSidecarsArchiver.java index 919375f137d..0449d16b8ca 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/archive/filesystem/FileSystemBlobSidecarsArchiver.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/archive/filesystem/FileSystemBlobSidecarsArchiver.java @@ -18,6 +18,8 @@ import com.fasterxml.jackson.core.JsonParser; import com.fasterxml.jackson.core.JsonToken; import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Splitter; +import com.google.common.collect.Iterables; import java.io.BufferedWriter; import java.io.File; import java.io.IOException; @@ -135,7 +137,8 @@ public Optional> retrieve(final UInt64 slot) { .flatMap( line -> { // lines in the index file are in the format of: " " - final Bytes32 blockRoot = Bytes32.fromHexString(line.split(" ")[1]); + final Bytes32 blockRoot = + Bytes32.fromHexString(Iterables.get(Splitter.on(' ').split(line), 1)); return retrieve(blockRoot, Optional.of(slot)); }); } catch (IOException ex) { From 6647b428b71af77537bd5f781b3566093de86d05 Mon Sep 17 00:00:00 2001 From: Stefan Bratanov Date: Mon, 7 Apr 2025 22:39:48 +0100 Subject: [PATCH 05/12] fix assemble --- .../pegasys/teku/services/chainstorage/StorageService.java | 3 +-- .../pegasys/teku/storage/server/kvstore/DatabaseTest.java | 5 +---- .../archive/filesystem/FileSystemBlobSidecarsArchiver.java | 6 +----- .../filesystem/FileSystemBlobSidecarsArchiverTest.java | 5 +---- 4 files changed, 4 insertions(+), 15 deletions(-) diff --git a/services/chainstorage/src/main/java/tech/pegasys/teku/services/chainstorage/StorageService.java b/services/chainstorage/src/main/java/tech/pegasys/teku/services/chainstorage/StorageService.java index 2f2fd36daab..e077e8b96ec 100644 --- a/services/chainstorage/src/main/java/tech/pegasys/teku/services/chainstorage/StorageService.java +++ b/services/chainstorage/src/main/java/tech/pegasys/teku/services/chainstorage/StorageService.java @@ -163,8 +163,7 @@ protected SafeFuture doStart() { .getBlobsArchivePath() .map( path -> - new FileSystemBlobSidecarsArchiver( - config.getSpec(), Path.of(path), storagePrunerAsyncRunner)) + new FileSystemBlobSidecarsArchiver(config.getSpec(), Path.of(path))) .orElse(BlobSidecarsArchiver.NOOP); if (config.getSpec().isMilestoneSupported(SpecMilestone.DENEB)) { diff --git a/storage/src/integration-test/java/tech/pegasys/teku/storage/server/kvstore/DatabaseTest.java b/storage/src/integration-test/java/tech/pegasys/teku/storage/server/kvstore/DatabaseTest.java index 5fa4356c1ef..fef61a7934d 100644 --- a/storage/src/integration-test/java/tech/pegasys/teku/storage/server/kvstore/DatabaseTest.java +++ b/storage/src/integration-test/java/tech/pegasys/teku/storage/server/kvstore/DatabaseTest.java @@ -60,7 +60,6 @@ import tech.pegasys.teku.ethereum.pow.api.DepositTreeSnapshot; import tech.pegasys.teku.ethereum.pow.api.MinGenesisTimeBlockEvent; import tech.pegasys.teku.infrastructure.async.AsyncRunner; -import tech.pegasys.teku.infrastructure.async.StubAsyncRunner; import tech.pegasys.teku.infrastructure.unsigned.UInt64; import tech.pegasys.teku.spec.Spec; import tech.pegasys.teku.spec.TestSpecFactory; @@ -127,7 +126,6 @@ public class DatabaseTest { private RecentChainData recentChainData; private UpdatableStore store; private final List storageSystems = new ArrayList<>(); - private final StubAsyncRunner stubAsyncRunner = new StubAsyncRunner(); @BeforeEach public void setup() throws IOException { @@ -141,8 +139,7 @@ private void setupWithSpec(final Spec spec) throws IOException { this.chainProperties = new ChainProperties(spec); final Path blobsArchive = Files.createTempDirectory("blobs"); tmpDirectories.add(blobsArchive.toFile()); - this.blobSidecarsArchiver = - new FileSystemBlobSidecarsArchiver(spec, blobsArchive, stubAsyncRunner); + this.blobSidecarsArchiver = new FileSystemBlobSidecarsArchiver(spec, blobsArchive); genesisBlockAndState = chainBuilder.generateGenesis(genesisTime, true); genesisCheckpoint = getCheckpointForBlock(genesisBlockAndState.getBlock()); genesisAnchor = AnchorPoint.fromGenesisState(spec, genesisBlockAndState.getState()); diff --git a/storage/src/main/java/tech/pegasys/teku/storage/archive/filesystem/FileSystemBlobSidecarsArchiver.java b/storage/src/main/java/tech/pegasys/teku/storage/archive/filesystem/FileSystemBlobSidecarsArchiver.java index 0449d16b8ca..2d97f5c1802 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/archive/filesystem/FileSystemBlobSidecarsArchiver.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/archive/filesystem/FileSystemBlobSidecarsArchiver.java @@ -34,7 +34,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.tuweni.bytes.Bytes32; -import tech.pegasys.teku.infrastructure.async.AsyncRunner; import tech.pegasys.teku.infrastructure.json.JsonUtil; import tech.pegasys.teku.infrastructure.json.types.DeserializableTypeDefinition; import tech.pegasys.teku.infrastructure.unsigned.UInt64; @@ -53,13 +52,10 @@ public class FileSystemBlobSidecarsArchiver implements BlobSidecarsArchiver { private final Spec spec; private final Path baseDirectory; - private final AsyncRunner asyncRunner; - public FileSystemBlobSidecarsArchiver( - final Spec spec, final Path baseDirectory, final AsyncRunner asyncRunner) { + public FileSystemBlobSidecarsArchiver(final Spec spec, final Path baseDirectory) { this.spec = spec; this.baseDirectory = baseDirectory; - this.asyncRunner = asyncRunner; } @Override diff --git a/storage/src/test/java/tech/pegasys/teku/storage/archive/filesystem/FileSystemBlobSidecarsArchiverTest.java b/storage/src/test/java/tech/pegasys/teku/storage/archive/filesystem/FileSystemBlobSidecarsArchiverTest.java index 31495a30a05..9a74920695c 100644 --- a/storage/src/test/java/tech/pegasys/teku/storage/archive/filesystem/FileSystemBlobSidecarsArchiverTest.java +++ b/storage/src/test/java/tech/pegasys/teku/storage/archive/filesystem/FileSystemBlobSidecarsArchiverTest.java @@ -28,7 +28,6 @@ import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; -import tech.pegasys.teku.infrastructure.async.StubAsyncRunner; import tech.pegasys.teku.infrastructure.unsigned.UInt64; import tech.pegasys.teku.spec.Spec; import tech.pegasys.teku.spec.TestSpecFactory; @@ -57,14 +56,12 @@ public class FileSystemBlobSidecarsArchiverTest { private final DataStructureUtil dataStructureUtil = new DataStructureUtil(SPEC); static Path testTempDir; - static StubAsyncRunner stubAsyncRunner; static FileSystemBlobSidecarsArchiver blobSidecarsArchiver; @BeforeAll static void beforeEach() throws IOException { testTempDir = Files.createTempDirectory("blobs"); - stubAsyncRunner = new StubAsyncRunner(); - blobSidecarsArchiver = new FileSystemBlobSidecarsArchiver(SPEC, testTempDir, stubAsyncRunner); + blobSidecarsArchiver = new FileSystemBlobSidecarsArchiver(SPEC, testTempDir); } @AfterEach From 0ff11e584687ee704b9285c767210b8f74bc3114 Mon Sep 17 00:00:00 2001 From: Stefan Bratanov Date: Mon, 7 Apr 2025 22:45:38 +0100 Subject: [PATCH 06/12] fix assemble --- .../archive/filesystem/FileSystemBlobSidecarsArchiver.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/src/main/java/tech/pegasys/teku/storage/archive/filesystem/FileSystemBlobSidecarsArchiver.java b/storage/src/main/java/tech/pegasys/teku/storage/archive/filesystem/FileSystemBlobSidecarsArchiver.java index 2d97f5c1802..1c88602316a 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/archive/filesystem/FileSystemBlobSidecarsArchiver.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/archive/filesystem/FileSystemBlobSidecarsArchiver.java @@ -90,7 +90,7 @@ public void archive( StandardOpenOption.APPEND)) { if (blobSidecars.isEmpty()) { // empty list - output.write("[]".getBytes()); + output.write("[]".getBytes(StandardCharsets.UTF_8)); } else { writeBlobSidecars(output, slotAndBlockRoot.getSlot(), blobSidecars); } From eea19380e687f5bc528b846afe8b6312e78f023b Mon Sep 17 00:00:00 2001 From: Stefan Bratanov Date: Mon, 7 Apr 2025 22:55:48 +0100 Subject: [PATCH 07/12] fix stuff --- .../AbstractMigratedBeaconHandlerWithChainDataProviderTest.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/data/beaconrestapi/src/test/java/tech/pegasys/teku/beaconrestapi/AbstractMigratedBeaconHandlerWithChainDataProviderTest.java b/data/beaconrestapi/src/test/java/tech/pegasys/teku/beaconrestapi/AbstractMigratedBeaconHandlerWithChainDataProviderTest.java index 9414f0a34fe..b11c340c7cc 100644 --- a/data/beaconrestapi/src/test/java/tech/pegasys/teku/beaconrestapi/AbstractMigratedBeaconHandlerWithChainDataProviderTest.java +++ b/data/beaconrestapi/src/test/java/tech/pegasys/teku/beaconrestapi/AbstractMigratedBeaconHandlerWithChainDataProviderTest.java @@ -25,7 +25,6 @@ import tech.pegasys.teku.spec.util.DataStructureUtil; import tech.pegasys.teku.statetransition.validatorcache.ActiveValidatorCache; import tech.pegasys.teku.statetransition.validatorcache.ActiveValidatorChannel; -import tech.pegasys.teku.storage.archive.BlobSidecarsArchiver; import tech.pegasys.teku.storage.client.ChainUpdater; import tech.pegasys.teku.storage.client.CombinedChainDataClient; import tech.pegasys.teku.storage.client.RecentChainData; @@ -76,7 +75,6 @@ private void setupStorage( spec, recentChainData, combinedChainDataClient, - BlobSidecarsArchiver.NOOP, new RewardCalculator(spec, new BlockRewardCalculatorUtil(spec))); } } From 313e11261ff6c8896cf52b4d325beaa29c759101 Mon Sep 17 00:00:00 2001 From: Stefan Bratanov Date: Tue, 8 Apr 2025 11:47:26 +0100 Subject: [PATCH 08/12] cleanup --- .../BlobSidecarSelectorFactory.java | 22 ++++++- .../teku/storage/api/StorageQueryChannel.java | 2 +- .../storage/archive/BlobSidecarsArchiver.java | 27 +------- .../FileSystemBlobSidecarsArchiver.java | 34 +++------- .../client/CombinedChainDataClient.java | 65 +++++-------------- .../teku/storage/server/ChainStorage.java | 7 +- .../FileSystemBlobSidecarsArchiverTest.java | 4 +- 7 files changed, 50 insertions(+), 111 deletions(-) diff --git a/data/provider/src/main/java/tech/pegasys/teku/api/blobselector/BlobSidecarSelectorFactory.java b/data/provider/src/main/java/tech/pegasys/teku/api/blobselector/BlobSidecarSelectorFactory.java index 26ecd633d5f..fd304e10063 100644 --- a/data/provider/src/main/java/tech/pegasys/teku/api/blobselector/BlobSidecarSelectorFactory.java +++ b/data/provider/src/main/java/tech/pegasys/teku/api/blobselector/BlobSidecarSelectorFactory.java @@ -178,12 +178,30 @@ private SafeFuture>> getBlobSidecarsForBlock( private SafeFuture>> getBlobSidecars( final SlotAndBlockRoot slotAndBlockRoot, final List indices) { - return client.getBlobSidecars(slotAndBlockRoot, indices).thenApply(Optional::of); + return client + .getBlobSidecars(slotAndBlockRoot, indices) + .thenCompose( + blobSidecars -> { + if (blobSidecars.isEmpty()) { + // attempt retrieving from archive + return client.getArchivedBlobSidecars(slotAndBlockRoot, indices); + } + return SafeFuture.completedFuture(Optional.of(blobSidecars)); + }); } private SafeFuture>> getBlobSidecars( final UInt64 slot, final List indices) { - return client.getBlobSidecars(slot, indices).thenApply(Optional::of); + return client + .getBlobSidecars(slot, indices) + .thenCompose( + blobSidecars -> { + if (blobSidecars.isEmpty()) { + // attempt retrieving from archive + return client.getArchivedBlobSidecars(slot, indices); + } + return SafeFuture.completedFuture(Optional.of(blobSidecars)); + }); } private Optional addMetaData( diff --git a/storage/api/src/main/java/tech/pegasys/teku/storage/api/StorageQueryChannel.java b/storage/api/src/main/java/tech/pegasys/teku/storage/api/StorageQueryChannel.java index 5f4c1e9326f..20fcba80e30 100644 --- a/storage/api/src/main/java/tech/pegasys/teku/storage/api/StorageQueryChannel.java +++ b/storage/api/src/main/java/tech/pegasys/teku/storage/api/StorageQueryChannel.java @@ -110,7 +110,7 @@ SafeFuture> getBlobSidecarKeys( SafeFuture> getBlobSidecarKeys( SlotAndBlockRoot slotAndBlockRoot); - // blobs archival methods + // Methods for retrieving archived blobs SafeFuture>> getArchivedBlobSidecars( SlotAndBlockRoot slotAndBlockRoot); diff --git a/storage/src/main/java/tech/pegasys/teku/storage/archive/BlobSidecarsArchiver.java b/storage/src/main/java/tech/pegasys/teku/storage/archive/BlobSidecarsArchiver.java index 3c58eb38812..5c8dffc509f 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/archive/BlobSidecarsArchiver.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/archive/BlobSidecarsArchiver.java @@ -15,11 +15,9 @@ import java.util.List; import java.util.Optional; -import org.apache.tuweni.bytes.Bytes32; import tech.pegasys.teku.infrastructure.unsigned.UInt64; import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar; import tech.pegasys.teku.spec.datastructures.blocks.SlotAndBlockRoot; -import tech.pegasys.teku.spec.datastructures.util.SlotAndBlockRootAndBlobIndex; public interface BlobSidecarsArchiver { @@ -30,8 +28,7 @@ public void archive( final SlotAndBlockRoot slotAndBlockRoot, final List blobSidecars) {} @Override - public Optional> retrieve( - final Bytes32 blockRoot, final Optional maybeSlot) { + public Optional> retrieve(final SlotAndBlockRoot slotAndBlockRoot) { return Optional.empty(); } @@ -43,27 +40,7 @@ public Optional> retrieve(final UInt64 slot) { void archive(SlotAndBlockRoot slotAndBlockRoot, List blobSidecars); - Optional> retrieve(Bytes32 blockRoot, Optional maybeSlot); + Optional> retrieve(SlotAndBlockRoot slotAndBlockRoot); Optional> retrieve(UInt64 slot); - - default Optional> retrieve(final SlotAndBlockRoot slotAndBlockRoot) { - return retrieve(slotAndBlockRoot.getBlockRoot(), Optional.of(slotAndBlockRoot.getSlot())); - } - - default Optional retrieve( - final SlotAndBlockRootAndBlobIndex slotAndBlockRootAndBlobIndex) { - return retrieve( - slotAndBlockRootAndBlobIndex.getBlockRoot(), - Optional.of(slotAndBlockRootAndBlobIndex.getSlot())) - .flatMap( - blobSidecars -> - blobSidecars.stream() - .filter( - blobSidecar -> - blobSidecar - .getIndex() - .equals(slotAndBlockRootAndBlobIndex.getBlobIndex())) - .findFirst()); - } } diff --git a/storage/src/main/java/tech/pegasys/teku/storage/archive/filesystem/FileSystemBlobSidecarsArchiver.java b/storage/src/main/java/tech/pegasys/teku/storage/archive/filesystem/FileSystemBlobSidecarsArchiver.java index 1c88602316a..7801d1efaff 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/archive/filesystem/FileSystemBlobSidecarsArchiver.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/archive/filesystem/FileSystemBlobSidecarsArchiver.java @@ -15,8 +15,6 @@ import static tech.pegasys.teku.infrastructure.json.types.DeserializableTypeDefinition.listOf; -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.core.JsonToken; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Splitter; import com.google.common.collect.Iterables; @@ -102,20 +100,22 @@ public void archive( } @Override - public Optional> retrieve( - final Bytes32 blockRoot, final Optional maybeSlot) { + public Optional> retrieve(final SlotAndBlockRoot slotAndBlockRoot) { try { - final Path archivePath = resolveArchivePath(blockRoot); + final Path archivePath = resolveArchivePath(slotAndBlockRoot.getBlockRoot()); if (!Files.exists(archivePath)) { return Optional.empty(); } final String blobSidecarsJson = Files.readString(archivePath); - final UInt64 slot = maybeSlot.orElseGet(() -> getSlot(blobSidecarsJson)); final List blobSidecars = - JsonUtil.parse(blobSidecarsJson, getJsonTypeDefinition(slot)); + JsonUtil.parse(blobSidecarsJson, getJsonTypeDefinition(slotAndBlockRoot.getSlot())); return Optional.of(blobSidecars); } catch (IOException ex) { - LOG.error(String.format("Failed to retrieve blob sidecars for block root %s", blockRoot), ex); + LOG.error( + String.format( + "Failed to retrieve blob sidecars for slot %s and block root %s", + slotAndBlockRoot.getSlot(), slotAndBlockRoot.getBlockRoot()), + ex); return Optional.empty(); } } @@ -135,7 +135,7 @@ public Optional> retrieve(final UInt64 slot) { // lines in the index file are in the format of: " " final Bytes32 blockRoot = Bytes32.fromHexString(Iterables.get(Splitter.on(' ').split(line), 1)); - return retrieve(blockRoot, Optional.of(slot)); + return retrieve(new SlotAndBlockRoot(slot, blockRoot)); }); } catch (IOException ex) { LOG.error(String.format("Failed to retrieve blob sidecars for slot %s", slot), ex); @@ -197,20 +197,4 @@ private String formatIndexFileOutput(final SlotAndBlockRoot slotAndBlockRoot) { + " " + slotAndBlockRoot.getBlockRoot().toUnprefixedHexString(); } - - /** retrieve the slot from the json content */ - private UInt64 getSlot(final String blobSidecarsJson) { - try (JsonParser parser = JsonUtil.FACTORY.createParser(blobSidecarsJson)) { - while (parser.nextToken() != null) { - if ("slot".equals(parser.currentName()) && parser.nextToken() == JsonToken.VALUE_STRING) { - return UInt64.valueOf(parser.getValueAsString()); - } - } - } catch (IOException __) { - LOG.warn("Couldn't retrieve slot from blob sidecars archive json: {}", blobSidecarsJson); - return UInt64.ZERO; - } - LOG.warn("Couldn't retrieve slot from blob sidecars archive json: {}", blobSidecarsJson); - return UInt64.ZERO; - } } diff --git a/storage/src/main/java/tech/pegasys/teku/storage/client/CombinedChainDataClient.java b/storage/src/main/java/tech/pegasys/teku/storage/client/CombinedChainDataClient.java index 0d406fb166e..7d558bdbd7c 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/client/CombinedChainDataClient.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/client/CombinedChainDataClient.java @@ -172,15 +172,7 @@ public SafeFuture> getBlobSidecars( return historicalChainData .getBlobSidecarKeys(slot) .thenApply(keys -> filterBlobSidecarKeys(keys, indices)) - .thenCompose(this::getBlobSidecars) - .thenCompose( - blobSidecars -> { - if (blobSidecars.isEmpty()) { - // attempt retrieving from archive - return getArchivedBlobSidecars(slot, indices); - } - return SafeFuture.completedFuture(blobSidecars); - }); + .thenCompose(this::getBlobSidecars); } public SafeFuture> getBlobSidecars( @@ -193,29 +185,7 @@ public SafeFuture> getBlobSidecars( historicalChainData .getBlobSidecarKeys(slotAndBlockRoot) .thenApply(keys -> filterBlobSidecarKeys(keys, indices)) - .thenCompose(this::getBlobSidecars) - .thenCompose( - blobSidecars -> { - if (blobSidecars.isEmpty()) { - return getArchivedBlobSidecars(slotAndBlockRoot, indices); - } - return SafeFuture.completedFuture(blobSidecars); - })); - } - - private SafeFuture> getArchivedBlobSidecars( - final SlotAndBlockRoot slotAndBlockRoot, final List indices) { - return historicalChainData - .getArchivedBlobSidecars(slotAndBlockRoot) - .thenApply( - maybeArchivedBlobSidecars -> - maybeArchivedBlobSidecars - .map( - archivedBlobSidecars -> - archivedBlobSidecars.stream() - .filter(blobSidecar -> indices.contains(blobSidecar.getIndex())) - .toList()) - .orElse(List.of())); + .thenCompose(this::getBlobSidecars)); } public SafeFuture> getAllBlobSidecars( @@ -223,30 +193,25 @@ public SafeFuture> getAllBlobSidecars( return historicalChainData .getAllBlobSidecarKeys(slot) .thenApply(keys -> filterBlobSidecarKeys(keys, indices)) - .thenCompose(this::getBlobSidecars) - .thenCompose( - blobSidecars -> { - if (blobSidecars.isEmpty()) { - // attempt retrieving from archive - return getArchivedBlobSidecars(slot, indices); - } - return SafeFuture.completedFuture(blobSidecars); - }); + .thenCompose(this::getBlobSidecars); + } + + public SafeFuture>> getArchivedBlobSidecars( + final SlotAndBlockRoot slotAndBlockRoot, final List indices) { + return historicalChainData + .getArchivedBlobSidecars(slotAndBlockRoot) + .thenApply( + maybeBlobSidecars -> + maybeBlobSidecars.map(blobSidecars -> filterBlobSidecars(blobSidecars, indices))); } - private SafeFuture> getArchivedBlobSidecars( + public SafeFuture>> getArchivedBlobSidecars( final UInt64 slot, final List indices) { return historicalChainData .getArchivedBlobSidecars(slot) .thenApply( - maybeArchivedBlobSidecars -> - maybeArchivedBlobSidecars - .map( - archivedBlobSidecars -> - archivedBlobSidecars.stream() - .filter(blobSidecar -> indices.contains(blobSidecar.getIndex())) - .toList()) - .orElse(List.of())); + maybeBlobSidecars -> + maybeBlobSidecars.map(blobSidecars -> filterBlobSidecars(blobSidecars, indices))); } public SafeFuture> getStateAtSlotExact(final UInt64 slot) { diff --git a/storage/src/main/java/tech/pegasys/teku/storage/server/ChainStorage.java b/storage/src/main/java/tech/pegasys/teku/storage/server/ChainStorage.java index 13110e23a6c..b84a9384461 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/server/ChainStorage.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/server/ChainStorage.java @@ -319,12 +319,7 @@ public SafeFuture> getEarliestAvailableBlobSidecarSlot() { @Override public SafeFuture> getBlobSidecar(final SlotAndBlockRootAndBlobIndex key) { - return SafeFuture.of( - () -> - database - .getBlobSidecar(key) - // retrieve from archive - .or(() -> blobSidecarsArchiver.retrieve(key))); + return SafeFuture.of(() -> database.getBlobSidecar(key)); } @Override diff --git a/storage/src/test/java/tech/pegasys/teku/storage/archive/filesystem/FileSystemBlobSidecarsArchiverTest.java b/storage/src/test/java/tech/pegasys/teku/storage/archive/filesystem/FileSystemBlobSidecarsArchiverTest.java index 9a74920695c..0c3b76d1f6c 100644 --- a/storage/src/test/java/tech/pegasys/teku/storage/archive/filesystem/FileSystemBlobSidecarsArchiverTest.java +++ b/storage/src/test/java/tech/pegasys/teku/storage/archive/filesystem/FileSystemBlobSidecarsArchiverTest.java @@ -130,9 +130,9 @@ void testArchiveAndRetrieveBlobSidecars() { // test index file exists assertThat(testTempDir.resolve("0-99999_index.dat")).exists(); - // retrieving by root + // retrieving by slot and block root final Optional> retrievedBlobSidecarsByRoot = - blobSidecarsArchiver.retrieve(slotAndBlockRoot.getBlockRoot(), Optional.empty()); + blobSidecarsArchiver.retrieve(slotAndBlockRoot); assertThat(retrievedBlobSidecarsByRoot).hasValue(blobSidecars); From ac7369cad9297793d78ba34ab0c465d6b9abe393 Mon Sep 17 00:00:00 2001 From: Stefan Bratanov Date: Tue, 8 Apr 2025 14:39:00 +0100 Subject: [PATCH 09/12] fix build --- .../BlobSidecarSelectorFactory.java | 14 ++++++++------ .../teku/storage/api/StorageQueryChannel.java | 7 +++---- .../FileSystemBlobSidecarsArchiver.java | 17 +++++++++++++---- .../storage/client/CombinedChainDataClient.java | 12 ++++-------- .../teku/storage/server/ChainStorage.java | 8 ++++---- .../server/CombinedStorageChannelSplitter.java | 4 ++-- .../storage/api/StubStorageQueryChannel.java | 8 ++++---- 7 files changed, 38 insertions(+), 32 deletions(-) diff --git a/data/provider/src/main/java/tech/pegasys/teku/api/blobselector/BlobSidecarSelectorFactory.java b/data/provider/src/main/java/tech/pegasys/teku/api/blobselector/BlobSidecarSelectorFactory.java index fd304e10063..59dacb11089 100644 --- a/data/provider/src/main/java/tech/pegasys/teku/api/blobselector/BlobSidecarSelectorFactory.java +++ b/data/provider/src/main/java/tech/pegasys/teku/api/blobselector/BlobSidecarSelectorFactory.java @@ -183,11 +183,12 @@ private SafeFuture>> getBlobSidecars( .thenCompose( blobSidecars -> { if (blobSidecars.isEmpty()) { - // attempt retrieving from archive + // attempt retrieving from archive (when enabled) return client.getArchivedBlobSidecars(slotAndBlockRoot, indices); } - return SafeFuture.completedFuture(Optional.of(blobSidecars)); - }); + return SafeFuture.completedFuture(blobSidecars); + }) + .thenApply(Optional::of); } private SafeFuture>> getBlobSidecars( @@ -197,11 +198,12 @@ private SafeFuture>> getBlobSidecars( .thenCompose( blobSidecars -> { if (blobSidecars.isEmpty()) { - // attempt retrieving from archive + // attempt retrieving from archive (when enabled) return client.getArchivedBlobSidecars(slot, indices); } - return SafeFuture.completedFuture(Optional.of(blobSidecars)); - }); + return SafeFuture.completedFuture(blobSidecars); + }) + .thenApply(Optional::of); } private Optional addMetaData( diff --git a/storage/api/src/main/java/tech/pegasys/teku/storage/api/StorageQueryChannel.java b/storage/api/src/main/java/tech/pegasys/teku/storage/api/StorageQueryChannel.java index 20fcba80e30..4c895e03979 100644 --- a/storage/api/src/main/java/tech/pegasys/teku/storage/api/StorageQueryChannel.java +++ b/storage/api/src/main/java/tech/pegasys/teku/storage/api/StorageQueryChannel.java @@ -110,9 +110,8 @@ SafeFuture> getBlobSidecarKeys( SafeFuture> getBlobSidecarKeys( SlotAndBlockRoot slotAndBlockRoot); - // Methods for retrieving archived blobs - SafeFuture>> getArchivedBlobSidecars( - SlotAndBlockRoot slotAndBlockRoot); + // Methods for retrieving archived blobs (when enabled) + SafeFuture> getArchivedBlobSidecars(SlotAndBlockRoot slotAndBlockRoot); - SafeFuture>> getArchivedBlobSidecars(UInt64 slot); + SafeFuture> getArchivedBlobSidecars(UInt64 slot); } diff --git a/storage/src/main/java/tech/pegasys/teku/storage/archive/filesystem/FileSystemBlobSidecarsArchiver.java b/storage/src/main/java/tech/pegasys/teku/storage/archive/filesystem/FileSystemBlobSidecarsArchiver.java index 7801d1efaff..cf43ee28080 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/archive/filesystem/FileSystemBlobSidecarsArchiver.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/archive/filesystem/FileSystemBlobSidecarsArchiver.java @@ -26,7 +26,9 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardOpenOption; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.stream.Stream; import org.apache.logging.log4j.LogManager; @@ -36,6 +38,7 @@ import tech.pegasys.teku.infrastructure.json.types.DeserializableTypeDefinition; import tech.pegasys.teku.infrastructure.unsigned.UInt64; import tech.pegasys.teku.spec.Spec; +import tech.pegasys.teku.spec.SpecMilestone; import tech.pegasys.teku.spec.datastructures.blobs.versions.deneb.BlobSidecar; import tech.pegasys.teku.spec.datastructures.blocks.SlotAndBlockRoot; import tech.pegasys.teku.spec.schemas.SchemaDefinitionsDeneb; @@ -51,6 +54,9 @@ public class FileSystemBlobSidecarsArchiver implements BlobSidecarsArchiver { private final Spec spec; private final Path baseDirectory; + private final Map>> + milestoneToTypeDefinitionCache = new HashMap<>(); + public FileSystemBlobSidecarsArchiver(final Spec spec, final Path baseDirectory) { this.spec = spec; this.baseDirectory = baseDirectory; @@ -186,10 +192,13 @@ private void writeBlobSidecars( } private DeserializableTypeDefinition> getJsonTypeDefinition(final UInt64 slot) { - return listOf( - SchemaDefinitionsDeneb.required(spec.atSlot(slot).getSchemaDefinitions()) - .getBlobSidecarSchema() - .getJsonTypeDefinition()); + return milestoneToTypeDefinitionCache.computeIfAbsent( + spec.atSlot(slot).getMilestone(), + milestone -> + listOf( + SchemaDefinitionsDeneb.required(spec.forMilestone(milestone).getSchemaDefinitions()) + .getBlobSidecarSchema() + .getJsonTypeDefinition())); } private String formatIndexFileOutput(final SlotAndBlockRoot slotAndBlockRoot) { diff --git a/storage/src/main/java/tech/pegasys/teku/storage/client/CombinedChainDataClient.java b/storage/src/main/java/tech/pegasys/teku/storage/client/CombinedChainDataClient.java index 7d558bdbd7c..78309233c5d 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/client/CombinedChainDataClient.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/client/CombinedChainDataClient.java @@ -196,22 +196,18 @@ public SafeFuture> getAllBlobSidecars( .thenCompose(this::getBlobSidecars); } - public SafeFuture>> getArchivedBlobSidecars( + public SafeFuture> getArchivedBlobSidecars( final SlotAndBlockRoot slotAndBlockRoot, final List indices) { return historicalChainData .getArchivedBlobSidecars(slotAndBlockRoot) - .thenApply( - maybeBlobSidecars -> - maybeBlobSidecars.map(blobSidecars -> filterBlobSidecars(blobSidecars, indices))); + .thenApply(blobSidecars -> filterBlobSidecars(blobSidecars, indices)); } - public SafeFuture>> getArchivedBlobSidecars( + public SafeFuture> getArchivedBlobSidecars( final UInt64 slot, final List indices) { return historicalChainData .getArchivedBlobSidecars(slot) - .thenApply( - maybeBlobSidecars -> - maybeBlobSidecars.map(blobSidecars -> filterBlobSidecars(blobSidecars, indices))); + .thenApply(blobSidecars -> filterBlobSidecars(blobSidecars, indices)); } public SafeFuture> getStateAtSlotExact(final UInt64 slot) { diff --git a/storage/src/main/java/tech/pegasys/teku/storage/server/ChainStorage.java b/storage/src/main/java/tech/pegasys/teku/storage/server/ChainStorage.java index b84a9384461..7b14357ceab 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/server/ChainStorage.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/server/ChainStorage.java @@ -371,13 +371,13 @@ public SafeFuture> getBlobSidecarKeys( } @Override - public SafeFuture>> getArchivedBlobSidecars( + public SafeFuture> getArchivedBlobSidecars( final SlotAndBlockRoot slotAndBlockRoot) { - return SafeFuture.of(() -> blobSidecarsArchiver.retrieve(slotAndBlockRoot)); + return SafeFuture.of(() -> blobSidecarsArchiver.retrieve(slotAndBlockRoot).orElse(List.of())); } @Override - public SafeFuture>> getArchivedBlobSidecars(final UInt64 slot) { - return SafeFuture.of(() -> blobSidecarsArchiver.retrieve(slot)); + public SafeFuture> getArchivedBlobSidecars(final UInt64 slot) { + return SafeFuture.of(() -> blobSidecarsArchiver.retrieve(slot).orElse(List.of())); } } diff --git a/storage/src/main/java/tech/pegasys/teku/storage/server/CombinedStorageChannelSplitter.java b/storage/src/main/java/tech/pegasys/teku/storage/server/CombinedStorageChannelSplitter.java index 3639c0a57ac..843f0b12a92 100644 --- a/storage/src/main/java/tech/pegasys/teku/storage/server/CombinedStorageChannelSplitter.java +++ b/storage/src/main/java/tech/pegasys/teku/storage/server/CombinedStorageChannelSplitter.java @@ -251,13 +251,13 @@ public SafeFuture> getBlobSidecarKeys( } @Override - public SafeFuture>> getArchivedBlobSidecars( + public SafeFuture> getArchivedBlobSidecars( final SlotAndBlockRoot slotAndBlockRoot) { return asyncRunner.runAsync(() -> queryDelegate.getArchivedBlobSidecars(slotAndBlockRoot)); } @Override - public SafeFuture>> getArchivedBlobSidecars(final UInt64 slot) { + public SafeFuture> getArchivedBlobSidecars(final UInt64 slot) { return asyncRunner.runAsync(() -> queryDelegate.getArchivedBlobSidecars(slot)); } } diff --git a/storage/src/testFixtures/java/tech/pegasys/teku/storage/api/StubStorageQueryChannel.java b/storage/src/testFixtures/java/tech/pegasys/teku/storage/api/StubStorageQueryChannel.java index 157c9c1d56f..ca24a9dbdd1 100644 --- a/storage/src/testFixtures/java/tech/pegasys/teku/storage/api/StubStorageQueryChannel.java +++ b/storage/src/testFixtures/java/tech/pegasys/teku/storage/api/StubStorageQueryChannel.java @@ -177,14 +177,14 @@ public SafeFuture> getBlobSidecarKeys( } @Override - public SafeFuture>> getArchivedBlobSidecars( + public SafeFuture> getArchivedBlobSidecars( final SlotAndBlockRoot slotAndBlockRoot) { - return SafeFuture.completedFuture(Optional.empty()); + return SafeFuture.completedFuture(List.of()); } @Override - public SafeFuture>> getArchivedBlobSidecars(final UInt64 slot) { - return SafeFuture.completedFuture(Optional.empty()); + public SafeFuture> getArchivedBlobSidecars(final UInt64 slot) { + return SafeFuture.completedFuture(List.of()); } @Override From c500d00c9b228f181ef1bce8ca6901d3af5793c8 Mon Sep 17 00:00:00 2001 From: Stefan Bratanov Date: Tue, 8 Apr 2025 15:20:00 +0100 Subject: [PATCH 10/12] fix and add tests --- .../BlobSidecarSelectorFactoryTest.java | 49 ++++++++++++++++++- 1 file changed, 47 insertions(+), 2 deletions(-) diff --git a/data/provider/src/test/java/tech/pegasys/teku/api/blobselector/BlobSidecarSelectorFactoryTest.java b/data/provider/src/test/java/tech/pegasys/teku/api/blobselector/BlobSidecarSelectorFactoryTest.java index 030618f3769..b725db051a5 100644 --- a/data/provider/src/test/java/tech/pegasys/teku/api/blobselector/BlobSidecarSelectorFactoryTest.java +++ b/data/provider/src/test/java/tech/pegasys/teku/api/blobselector/BlobSidecarSelectorFactoryTest.java @@ -119,6 +119,31 @@ public void blockRootSelector_shouldGetBlobSidecarsForFinalizedSlot() assertThat(result.get().getData()).isEqualTo(blobSidecars); } + @Test + public void blockRootSelector_shouldAttemptToGetFinalizedBlobSidecarsFromArchive() + throws ExecutionException, InterruptedException { + final UInt64 finalizedSlot = UInt64.valueOf(42); + final SignedBlockAndState blockAndState = data.randomSignedBlockAndState(100); + when(client.getChainHead()).thenReturn(Optional.of(ChainHead.create(blockAndState))); + + when(client.getFinalizedSlotByBlockRoot(block.getRoot())) + .thenReturn(SafeFuture.completedFuture(Optional.of(finalizedSlot))); + final SlotAndBlockRoot slotAndBlockRoot = new SlotAndBlockRoot(finalizedSlot, block.getRoot()); + // not available in DB + when(client.getBlobSidecars(slotAndBlockRoot, indices)) + .thenReturn(SafeFuture.completedFuture(List.of())); + // available in archive + when(client.getArchivedBlobSidecars(slotAndBlockRoot, indices)) + .thenReturn(SafeFuture.completedFuture(blobSidecars)); + + final Optional result = + blobSidecarSelectorFactory + .blockRootSelector(block.getRoot()) + .getBlobSidecars(indices) + .get(); + assertThat(result.get().getData()).isEqualTo(blobSidecars); + } + @Test public void blockRootSelector_shouldGetBlobSidecarsByRetrievingBlock() throws ExecutionException, InterruptedException { @@ -153,6 +178,23 @@ public void slotSelector_shouldGetBlobSidecarsFromFinalizedSlot() assertThat(result.get().getData()).isEqualTo(blobSidecars); } + @Test + public void slotSelector_shouldAttemptToGetFinalizedBlobSidecarsFromArchive() + throws ExecutionException, InterruptedException { + when(client.isFinalized(block.getSlot())).thenReturn(true); + // not available in DB + when(client.getBlobSidecars(block.getSlot(), indices)) + .thenReturn(SafeFuture.completedFuture(List.of())); + // available in archive + when(client.getArchivedBlobSidecars(block.getSlot(), indices)) + .thenReturn(SafeFuture.completedFuture(blobSidecars)); + when(client.isChainHeadOptimistic()).thenReturn(false); + + final Optional result = + blobSidecarSelectorFactory.slotSelector(block.getSlot()).getBlobSidecars(indices).get(); + assertThat(result.get().getData()).isEqualTo(blobSidecars); + } + @Test public void slotSelector_shouldGetBlobSidecarsByRetrievingBlockWhenSlotNotFinalized() throws ExecutionException, InterruptedException { @@ -228,6 +270,11 @@ public void shouldLookForBlobSidecarsOnlyAfterDeneb( .thenReturn(SafeFuture.completedFuture(Optional.of(block))); when(client.getBlobSidecars(any(SlotAndBlockRoot.class), anyList())) .thenReturn(SafeFuture.completedFuture(List.of())); + when(client.getArchivedBlobSidecars(any(SlotAndBlockRoot.class), anyList())) + .thenReturn(SafeFuture.completedFuture(List.of())); + when(client.getArchivedBlobSidecars(any(UInt64.class), anyList())) + .thenReturn(SafeFuture.completedFuture(List.of())); + blobSidecarSelectorFactory.slotSelector(block.getSlot()).getBlobSidecars(indices).get(); if (ctx.getSpec().isMilestoneSupported(SpecMilestone.DENEB)) { verify(client).getBlobSidecars(any(SlotAndBlockRoot.class), anyList()); @@ -273,8 +320,6 @@ public void slotSelector_whenSelectingNonFinalizedBlockMetadataReturnsFinalizedF when(client.isFinalized(block.getSlot())).thenReturn(false); when(client.getBlockAtSlotExact(block.getSlot())) .thenReturn(SafeFuture.completedFuture(Optional.of(block))); - when(client.getBlobSidecars(block.getSlot(), indices)) - .thenReturn(SafeFuture.completedFuture(blobSidecars)); when(client.isChainHeadOptimistic()).thenReturn(false); when(client.getBlobSidecars(block.getSlotAndBlockRoot(), indices)) .thenReturn(SafeFuture.completedFuture(blobSidecars)); From 4889fe06b7820fb7b56565fd1cc33f0be4c43b1c Mon Sep 17 00:00:00 2001 From: Stefan Bratanov Date: Tue, 8 Apr 2025 16:09:31 +0100 Subject: [PATCH 11/12] no need --- data/provider/build.gradle | 1 - 1 file changed, 1 deletion(-) diff --git a/data/provider/build.gradle b/data/provider/build.gradle index 29199a6530d..a63324d0998 100644 --- a/data/provider/build.gradle +++ b/data/provider/build.gradle @@ -12,7 +12,6 @@ dependencies { implementation project(':infrastructure:serviceutils') implementation project(':infrastructure:ssz') implementation project(':storage') - implementation project(':storage:api') implementation project(':beacon:sync') implementation project(':validator:api') From 0386255857ca98697df1f74a05be5205c897eb56 Mon Sep 17 00:00:00 2001 From: Stefan Bratanov Date: Wed, 9 Apr 2025 15:55:02 +0100 Subject: [PATCH 12/12] remove --- .../json/types/DeserializableListTypeDefinition.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/infrastructure/json/src/main/java/tech/pegasys/teku/infrastructure/json/types/DeserializableListTypeDefinition.java b/infrastructure/json/src/main/java/tech/pegasys/teku/infrastructure/json/types/DeserializableListTypeDefinition.java index 7d23c9c5e69..0bcb51b361d 100644 --- a/infrastructure/json/src/main/java/tech/pegasys/teku/infrastructure/json/types/DeserializableListTypeDefinition.java +++ b/infrastructure/json/src/main/java/tech/pegasys/teku/infrastructure/json/types/DeserializableListTypeDefinition.java @@ -26,8 +26,4 @@ public DeserializableListTypeDefinition( final Optional maxItems) { super(itemType, Function.identity(), minItems, maxItems); } - - public DeserializableListTypeDefinition(final DeserializableTypeDefinition itemType) { - super(itemType, Function.identity(), Optional.empty(), Optional.empty()); - } }