Skip to content

Use existing RLP methods in trie log prune batch files #10009

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
siladu merged 3 commits into from
Mar 11, 2026
Merged

Use existing RLP methods in trie log prune batch files #10009

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
0fc4993
feat: replace Java serialization with RLP in trie log prune batch files
siladu Mar 10, 2026
21fa650
copilot suggestion for unit test
siladu Mar 11, 2026
134434a
Merge branch 'main' into feat/trielog-subcommand-pr1
siladu Mar 11, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
62 changes: 5 additions & 57 deletions app/src/main/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,11 +33,7 @@
import org.hyperledger.besu.ethereum.worldstate.PathBasedExtraStorageConfiguration;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.io.PrintWriter;
import java.nio.file.Files;
import java.nio.file.Path;
Expand Down Expand Up @@ -153,26 +149,16 @@ private void saveTrieLogBatches(
final PathBasedWorldStateKeyValueStorage rootWorldStateStorage,
final List<Hash> trieLogKeys) {

try {
saveTrieLogsInFile(trieLogKeys, rootWorldStateStorage, batchFileName);
} catch (IOException e) {
LOG.error("Error saving trie logs to file: {}", e.getMessage());
throw new RuntimeException(e);
}
saveTrieLogsAsRlpInFile(trieLogKeys, rootWorldStateStorage, batchFileName);
Comment thread
siladu marked this conversation as resolved.
}

private void restoreTrieLogBatches(
final PathBasedWorldStateKeyValueStorage rootWorldStateStorage,
final long batchNumber,
final String batchFileNameBase) {

try {
LOG.info("Restoring trie logs retained from batch {}...", batchNumber);
recreateTrieLogs(rootWorldStateStorage, batchNumber, batchFileNameBase);
} catch (IOException e) {
LOG.error("Error recreating trie logs from batch {}: {}", batchNumber, e.getMessage());
throw new RuntimeException(e);
}
LOG.info("Restoring trie logs retained from batch {}...", batchNumber);
recreateTrieLogs(rootWorldStateStorage, batchNumber, batchFileNameBase);
}

private boolean deleteFiles(final String batchFileNameBase, final long numberOfBatches) {
Expand Down Expand Up @@ -257,11 +243,10 @@ private boolean validatePruneRequirements(
private void recreateTrieLogs(
final PathBasedWorldStateKeyValueStorage rootWorldStateStorage,
final long batchNumber,
final String batchFileNameBase)
throws IOException {
final String batchFileNameBase) {
// process in chunk to avoid OOM
final String batchFileName = batchFileNameBase + "-" + batchNumber;
IdentityHashMap<byte[], byte[]> trieLogsToRetain = readTrieLogsFromFile(batchFileName);
IdentityHashMap<byte[], byte[]> trieLogsToRetain = readTrieLogsAsRlpFromFile(batchFileName);
final int chunkSize = ROCKSDB_MAX_INSERTS_PER_TRANSACTION;
List<byte[]> keys = new ArrayList<>(trieLogsToRetain.keySet());

Expand Down Expand Up @@ -314,43 +299,6 @@ void validatePruneConfiguration(final DataStorageConfiguration config) {
subStorageConfiguration.getMaxLayersToLoad()));
}

private void saveTrieLogsInFile(
final List<Hash> trieLogsKeys,
final PathBasedWorldStateKeyValueStorage rootWorldStateStorage,
final String batchFileName)
throws IOException {

File file = new File(batchFileName);
if (file.exists()) {
LOG.warn("File already exists {}, skipping file creation", batchFileName);
return;
}

try (FileOutputStream fos = new FileOutputStream(file)) {
ObjectOutputStream oos = new ObjectOutputStream(fos);
oos.writeObject(getTrieLogs(trieLogsKeys, rootWorldStateStorage));
} catch (IOException e) {
LOG.error(e.getMessage());
throw new RuntimeException(e);
}
}

@SuppressWarnings("unchecked")
IdentityHashMap<byte[], byte[]> readTrieLogsFromFile(final String batchFileName) {

IdentityHashMap<byte[], byte[]> trieLogs;
try (FileInputStream fis = new FileInputStream(batchFileName);
ObjectInputStream ois = new ObjectInputStream(fis)) {

trieLogs = (IdentityHashMap<byte[], byte[]>) ois.readObject();
} catch (IOException | ClassNotFoundException e) {
LOG.error(e.getMessage());
throw new RuntimeException(e);
}

return trieLogs;
}

private void saveTrieLogsAsRlpInFile(
final List<Hash> trieLogsKeys,
final PathBasedWorldStateKeyValueStorage rootWorldStateStorage,
Expand Down
41 changes: 39 additions & 2 deletions app/src/test/java/org/hyperledger/besu/cli/subcommands/storage/TrieLogHelperTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,6 @@
import org.hyperledger.besu.ethereum.worldstate.ImmutablePathBasedExtraStorageConfiguration;
import org.hyperledger.besu.metrics.noop.NoOpMetricsSystem;

import java.io.FileNotFoundException;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
Expand Down Expand Up @@ -128,6 +127,44 @@ void mockBlockchainBase() {
when(blockchain.getBlockHeader(any(Hash.class))).thenReturn(Optional.of(blockHeader3));
}

@Test
public void pruneFailsWhenBatchFileContainsJavaSerialization(final @TempDir Path dataDir)
throws IOException {
Files.createDirectories(dataDir.resolve("database"));

DataStorageConfiguration dataStorageConfiguration =
ImmutableDataStorageConfiguration.builder()
.dataStorageFormat(BONSAI)
.pathBasedExtraStorageConfiguration(
ImmutablePathBasedExtraStorageConfiguration.builder()
.maxLayersToLoad(3L)
.limitTrieLogsEnabled(true)
.build())
.build();

mockBlockchainBase();
when(blockchain.getBlockHeader(5)).thenReturn(Optional.of(blockHeader5));
when(blockchain.getBlockHeader(4)).thenReturn(Optional.of(blockHeader4));
when(blockchain.getBlockHeader(3)).thenReturn(Optional.of(blockHeader3));

// Pre-place a Java-serialized file at the expected batch file path.
// saveTrieLogsAsRlpInFile's exists-check will skip overwriting it.
// readTrieLogsAsRlpFromFile will then fail to parse it as RLP during restore,
// confirming the code no longer silently deserializes Java-serialized objects.
Path batchFile = dataDir.resolve("database").resolve("trieLogsToRetain-1");
try (var oos =
new java.io.ObjectOutputStream(new java.io.FileOutputStream(batchFile.toFile()))) {
oos.writeObject("notrlp");
}

assertThatThrownBy(
() ->
nonValidatingTrieLogHelper.prune(
dataStorageConfiguration, inMemoryWorldState, blockchain, dataDir))
.isInstanceOf(RuntimeException.class)
.hasMessageContaining("RLP");
}

@Test
public void prune(final @TempDir Path dataDir) throws IOException {
Files.createDirectories(dataDir.resolve("database"));
Expand Down Expand Up @@ -388,7 +425,7 @@ public void exceptionWhileSavingFileStopsPruneProcess(final @TempDir Path dataDi
blockchain,
dataDir.resolve("unknownPath")))
.isInstanceOf(RuntimeException.class)
.hasCauseExactlyInstanceOf(FileNotFoundException.class);
.hasCauseInstanceOf(java.io.IOException.class);

// assert all trie logs are still in the DB
assertThat(inMemoryWorldState.getTrieLog(blockHeader1.getHash()).get())
Expand Down