feat: add --chain.pruneHistory flag (#7427)

**Motivation** - #6869 **Description** - add `MIN_EPOCHS_FOR_BLOCK_REQUESTS` config (PS we're missing a lot of the network config entries from the consensus specs) - add `--chain.pruneHistory` flag, default to false - when chain.pruneHistory is true - prune all historical blocks/states on startup and then on every subsequent finalization --------- Co-authored-by: Nico Flaig <[email protected]>
ChainSafe · Feb 7, 2025 · 90bd72f · 90bd72f
1 parent a7755ad
commit 90bd72f
Show file tree

Hide file tree

Showing 15 changed files with 375 additions and 43 deletions.
diff --git a/dashboards/lodestar_sync.json b/dashboards/lodestar_sync.json
diff --git a/docs/pages/run/beacon-management/data-retention.md b/docs/pages/run/beacon-management/data-retention.md
@@ -52,3 +52,7 @@ Configuring your node to store and prune data is key to success. On average you
 Logs can also become quite large so please check out the section on [log management](../logging-and-metrics/log-management.md) for more information.
 
 There is really only one flag that is needed to manage the data for Lodestar, [`--dataDir`](./beacon-cli#--datadir). Other than that handling log management is really the heart of the data management story. Beacon node data is what it is. Depending on the execution client that is chosen, there may be flags to help with data storage growth but that is outside the scope of this document.
+
+### Pruning history
+
+For validators seeking to store the least amount of data possible, and store no historical data, use [`--chain.pruneHistory`](./beacon-cli#--chainprunehistory). This flag will configure the beacon node to continually prune all old blocks (older than `config.MIN_EPOCHS_FOR_BLOCK_REQUESTS`) and all prior finalized states.
diff --git a/packages/beacon-node/src/chain/archiver/archiver.ts b/packages/beacon-node/src/chain/archiver/archiver.ts
@@ -7,6 +7,7 @@ import {ChainEvent} from "../emitter.js";
 import {IBeaconChain} from "../interface.js";
 import {archiveBlocks} from "./archiveBlocks.js";
 import {ArchiverOpts, StateArchiveMode, StateArchiveStrategy} from "./interface.js";
+import {pruneHistory} from "./pruneHistory.js";
 import {FrequencyStateArchiveStrategy} from "./strategies/frequencyStateArchiveStrategy.js";
 
 export const DEFAULT_STATE_ARCHIVE_MODE = StateArchiveMode.Frequency;
@@ -30,7 +31,7 @@ export class Archiver {
     private readonly chain: IBeaconChain,
     private readonly logger: Logger,
     signal: AbortSignal,
-    opts: ArchiverOpts,
+    private readonly opts: ArchiverOpts,
     private readonly metrics?: Metrics | null
   ) {
     if (opts.stateArchiveMode === StateArchiveMode.Frequency) {
@@ -98,6 +99,17 @@ export class Archiver {
         this.chain.clock.currentEpoch,
         this.archiveBlobEpochs
       );
+      if (this.opts.pruneHistory) {
+        await pruneHistory(
+          this.chain.config,
+          this.db,
+          this.logger,
+          this.metrics,
+          finalizedEpoch,
+          this.chain.clock.currentEpoch
+        );
+      }
+
       this.prevFinalized = finalized;
 
       await this.statesArchiverStrategy.onFinalizedCheckpoint(finalized, this.metrics);

diff --git a/packages/beacon-node/src/chain/archiver/interface.ts b/packages/beacon-node/src/chain/archiver/interface.ts
@@ -23,6 +23,7 @@ export interface StatesArchiverOpts {
 export type ArchiverOpts = StatesArchiverOpts & {
   disableArchiveOnCheckpoint?: boolean;
   archiveBlobEpochs?: number;
+  pruneHistory?: boolean;
 };
 
 export type ProposalStats = {

diff --git a/packages/beacon-node/src/chain/archiver/pruneHistory.ts b/packages/beacon-node/src/chain/archiver/pruneHistory.ts
@@ -0,0 +1,56 @@
+import {ChainConfig} from "@lodestar/config";
+import {computeStartSlotAtEpoch} from "@lodestar/state-transition";
+import {Epoch} from "@lodestar/types";
+import {Logger} from "@lodestar/utils";
+import {IBeaconDb} from "../../db/interface.js";
+import {Metrics} from "../../metrics/index.js";
+
+export async function pruneHistory(
+  config: ChainConfig,
+  db: IBeaconDb,
+  logger: Logger,
+  metrics: Metrics | null | undefined,
+  finalizedEpoch: Epoch,
+  currentEpoch: Epoch
+): Promise<void> {
+  const blockCutoffEpoch = Math.min(
+    // set by config, with underflow protection
+    Math.max(currentEpoch - config.MIN_EPOCHS_FOR_BLOCK_REQUESTS, 0),
+    // ensure that during (extremely lol) long periods of non-finality we don't delete unfinalized epoch data
+    finalizedEpoch
+  );
+  const blockCutoffSlot = computeStartSlotAtEpoch(blockCutoffEpoch);
+
+  logger.debug("Preparing to prune history", {
+    currentEpoch,
+    finalizedEpoch,
+    blockCutoffEpoch,
+  });
+
+  const step0 = metrics?.pruneHistory.fetchKeys.startTimer();
+  const [blocks, states] = await Promise.all([
+    db.blockArchive.keys({gte: 0, lt: blockCutoffSlot}),
+    db.stateArchive.keys({gte: 0, lt: finalizedEpoch}),
+  ]);
+  step0?.();
+
+  logger.debug("Pruning history", {
+    currentEpoch,
+    blocksToPrune: blocks.length,
+    statesToPrune: states.length,
+  });
+
+  const step1 = metrics?.pruneHistory.pruneKeys.startTimer();
+  await Promise.all([
+    // ->
+    db.blockArchive.batchDelete(blocks),
+    db.stateArchive.batchDelete(states),
+  ]);
+  step1?.();
+
+  logger.debug("Pruned history", {
+    currentEpoch,
+  });
+
+  metrics?.pruneHistory.pruneCount.inc();
+}
diff --git a/packages/beacon-node/src/chain/options.ts b/packages/beacon-node/src/chain/options.ts
@@ -106,6 +106,7 @@ export const defaultChainOptions: IChainOptions = {
   assertCorrectProgressiveBalances: false,
   archiveStateEpochFrequency: 1024,
   stateArchiveMode: DEFAULT_STATE_ARCHIVE_MODE,
+  pruneHistory: false,
   emitPayloadAttributes: false,
   // for gossip block validation, it's unlikely we see a reorg with 32 slots
   // for attestation validation, having this value ensures we don't have to regen states most of the time

diff --git a/packages/beacon-node/src/metrics/metrics/lodestar.ts b/packages/beacon-node/src/metrics/metrics/lodestar.ts
@@ -1799,5 +1799,24 @@ export function createLodestarMetrics(
         buckets: [0.0001, 0.001, 0.01, 0.1, 1],
       }),
     },
+
+    pruneHistory: {
+      pruneCount: register.gauge({
+        name: "lodestar_prune_history_prune_count_total",
+        help: "Total count of prune operations",
+      }),
+
+      fetchKeys: register.histogram({
+        name: "lodestar_prune_history_fetch_keys_time_seconds",
+        help: "Time to fetch keys in seconds",
+        buckets: [0.001, 0.01, 0.1, 1],
+      }),
+
+      pruneKeys: register.histogram({
+        name: "lodestar_prune_history_prune_keys_time_seconds",
+        help: "Time to prune keys in seconds",
+        buckets: [0.001, 0.01, 0.1, 1],
+      }),
+    },
   };
 }
diff --git a/packages/beacon-node/src/node/nodejs.ts b/packages/beacon-node/src/node/nodejs.ts
@@ -7,10 +7,11 @@ import {BeaconConfig} from "@lodestar/config";
 import type {LoggerNode} from "@lodestar/logger/node";
 import {BeaconStateAllForks} from "@lodestar/state-transition";
 import {phase0} from "@lodestar/types";
-import {sleep} from "@lodestar/utils";
+import {callFnWhenAwait, sleep} from "@lodestar/utils";
 import {ProcessShutdownCallback} from "@lodestar/validator";
 
 import {BeaconRestApiServer, getApi} from "../api/index.js";
+import {pruneHistory} from "../chain/archiver/pruneHistory.js";
 import {HistoricalStateRegen} from "../chain/historicalState/index.js";
 import {BeaconChain, IBeaconChain, initBeaconMetrics} from "../chain/index.js";
 import {IBeaconDb} from "../db/index.js";
@@ -21,6 +22,7 @@ import {MonitoringService} from "../monitoring/index.js";
 import {Network, getReqRespHandlers} from "../network/index.js";
 import {BackfillSync} from "../sync/backfill/index.js";
 import {BeaconSync, IBeaconSync} from "../sync/index.js";
+import {Clock} from "../util/clock.js";
 import {TrustedFileMode, initCKZG, loadEthereumTrustedSetup} from "../util/kzg.js";
 import {runNodeNotifier} from "./notifier.js";
 import {IBeaconNodeOptions} from "./options.js";
@@ -164,10 +166,6 @@ export class BeaconNode {
       loadEthereumTrustedSetup(TrustedFileMode.Txt, opts.chain.trustedSetup);
     }
 
-    // Prune hot db repos
-    // TODO: Should this call be awaited?
-    await db.pruneHotDb();
-
     let metrics = null;
     if (
       opts.metrics.enabled ||
@@ -187,6 +185,23 @@ export class BeaconNode {
       signal.addEventListener("abort", metrics.close, {once: true});
     }
 
+    const clock = new Clock({config, genesisTime: anchorState.genesisTime, signal});
+
+    // Prune hot db repos
+    // TODO: Should this call be awaited?
+    await db.pruneHotDb();
+
+    if (opts.chain.pruneHistory) {
+      // prune ALL stale data before starting
+      logger.info("Pruning historical data");
+      await callFnWhenAwait(
+        pruneHistory(config, db, logger, metrics, anchorState.finalizedCheckpoint.epoch, clock.currentEpoch),
+        () => logger.info("Still pruning historical data, please wait..."),
+        30_000,
+        signal
+      );
+    }
+
     const monitoring = opts.monitoring.endpoint
       ? new MonitoringService(
           "beacon",
@@ -208,6 +223,7 @@ export class BeaconNode {
 
     const chain = new BeaconChain(opts.chain, {
       config,
+      clock,
       db,
       logger: logger.child({module: LoggerModule.chain}),
       processShutdownCallback,

diff --git a/packages/cli/src/options/beaconNodeOptions/chain.ts b/packages/cli/src/options/beaconNodeOptions/chain.ts
@@ -33,6 +33,8 @@ export type ChainArgs = {
   "chain.nHistoricalStatesFileDataStore"?: boolean;
   "chain.maxBlockStates"?: number;
   "chain.maxCPStateEpochsInMemory"?: number;
+
+  "chain.pruneHistory"?: boolean;
 };
 
 export function parseArgs(args: ChainArgs): IBeaconNodeOptions["chain"] {
@@ -68,6 +70,7 @@ export function parseArgs(args: ChainArgs): IBeaconNodeOptions["chain"] {
       args["chain.nHistoricalStatesFileDataStore"] ?? defaultOptions.chain.nHistoricalStatesFileDataStore,
     maxBlockStates: args["chain.maxBlockStates"] ?? defaultOptions.chain.maxBlockStates,
     maxCPStateEpochsInMemory: args["chain.maxCPStateEpochsInMemory"] ?? defaultOptions.chain.maxCPStateEpochsInMemory,
+    pruneHistory: args["chain.pruneHistory"],
   };
 }
 
@@ -283,4 +286,11 @@ Will double processing times. Use only for debugging purposes.",
     default: defaultOptions.chain.maxCPStateEpochsInMemory,
     group: "chain",
   },
+
+  "chain.pruneHistory": {
+    description: "Prune historical blocks and state",
+    type: "boolean",
+    default: defaultOptions.chain.pruneHistory,
+    group: "chain",
+  },
 };
diff --git a/packages/config/src/chainConfig/configs/mainnet.ts b/packages/config/src/chainConfig/configs/mainnet.ts
@@ -97,6 +97,8 @@ export const chainConfig: ChainConfig = {
 
   // Networking
   // ---------------------------------------------------------------
+  // `MIN_VALIDATOR_WITHDRAWABILITY_DELAY + CHURN_LIMIT_QUOTIENT // 2` (= 33024, ~5 months)
+  MIN_EPOCHS_FOR_BLOCK_REQUESTS: 33024,
 
   // Deneb
   // `2**12` (= 4096 epochs, ~18 days)

diff --git a/packages/config/src/chainConfig/configs/minimal.ts b/packages/config/src/chainConfig/configs/minimal.ts
@@ -94,6 +94,8 @@ export const chainConfig: ChainConfig = {
 
   // Networking
   // ---------------------------------------------------------------
+  // [customized] `MIN_VALIDATOR_WITHDRAWABILITY_DELAY + CHURN_LIMIT_QUOTIENT // 2` (= 272)
+  MIN_EPOCHS_FOR_BLOCK_REQUESTS: 272,
 
   // Deneb
   // `2**12` (= 4096 epochs, ~18 days)

diff --git a/packages/config/src/chainConfig/types.ts b/packages/config/src/chainConfig/types.ts
@@ -71,6 +71,7 @@ export type ChainConfig = {
   DEPOSIT_CONTRACT_ADDRESS: Uint8Array;
 
   // Networking
+  MIN_EPOCHS_FOR_BLOCK_REQUESTS: number;
   MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS: number;
   BLOB_SIDECAR_SUBNET_COUNT: number;
   MAX_BLOBS_PER_BLOCK: number;
@@ -141,6 +142,7 @@ export const chainConfigTypes: SpecTypes<ChainConfig> = {
   DEPOSIT_CONTRACT_ADDRESS: "bytes",
 
   // Networking
+  MIN_EPOCHS_FOR_BLOCK_REQUESTS: "number",
   MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS: "number",
   BLOB_SIDECAR_SUBNET_COUNT: "number",
   MAX_BLOBS_PER_BLOCK: "number",

diff --git a/packages/utils/src/promise.ts b/packages/utils/src/promise.ts
@@ -6,25 +6,24 @@ import {ArrayToTuple, NonEmptyArray} from "./types.js";
  * While promise t is not finished, call function `fn` per `interval`
  */
 export async function callFnWhenAwait<T>(
-  p: Promise<NonNullable<T>>,
+  p: Promise<T>,
   fn: () => void,
-  interval: number
-): Promise<NonNullable<T>> {
-  let t: NonNullable<T> | undefined = undefined;
-  const logFn = async (): Promise<undefined> => {
-    while (t === undefined) {
-      await sleep(interval);
-      if (t === undefined) fn();
+  interval: number,
+  signal?: AbortSignal
+): Promise<T> {
+  let done = false;
+  const logFn = async (): Promise<void> => {
+    while (!done) {
+      await sleep(interval, signal);
+      if (!done) fn();
     }
-    return undefined;
   };
 
-  t = await Promise.race([p, logFn()]);
-  // should not happen since p doesn not resolve to undefined
-  if (t === undefined) {
-    throw new Error("Unexpected error: Timeout");
-  }
-  return t;
+  const t = await Promise.race([p, logFn()]).finally(() => {
+    done = true;
+  });
+
+  return t as T;
 }
 
 export type PromiseResult<T> = {

diff --git a/packages/utils/test/unit/promise.node.test.ts b/packages/utils/test/unit/promise.node.test.ts
@@ -11,7 +11,13 @@ describe("callFnWhenAwait util", () => {
     vi.clearAllTimers();
   });
 
-  it("should call function while awaing for promise", async () => {
+  it("should return the resolved value of promise", async () => {
+    const p = new Promise<string>((resolve) => resolve("done"));
+    const value = await callFnWhenAwait(p, vi.fn(), 1000);
+    expect(value).toBe("done");
+  });
+
+  it("should call function while awaiting for promise", async () => {
     const p = new Promise<string>((resolve) => setTimeout(() => resolve("done"), 5 * 1000));
     const stub = vi.fn();
     const result = await Promise.all([callFnWhenAwait(p, stub, 2 * 1000), vi.advanceTimersByTimeAsync(5000)]);

diff --git a/packages/validator/src/util/params.ts b/packages/validator/src/util/params.ts
@@ -135,6 +135,7 @@ function getSpecCriticalParams(localConfig: ChainConfig): Record<keyof ConfigWit
     DEPOSIT_CONTRACT_ADDRESS: true,
 
     // Networking (non-critical as those do not affect consensus)
+    MIN_EPOCHS_FOR_BLOCK_REQUESTS: false,
     MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS: false,
     BLOB_SIDECAR_SUBNET_COUNT: false,
     BLOB_SIDECAR_SUBNET_COUNT_ELECTRA: false,