feat: add hot and archive db repositories for blobSidecars

packages/beacon-node/src/api/impl/beacon/blocks/index.ts

@@ -265,9 +265,22 @@ export function getBeaconBlockApi({
       await promiseAllMaybeAsync(publishPromises);
     },
 
-    async getBlobSidecars(_blockId) {
-      // TODO DENEB: Add implementation on the DB structure change PR
-      throw Error("");
+    async getBlobSidecars(blockId) {
+      const {block, executionOptimistic} = await resolveBlockId(chain.forkChoice, db, blockId);
+      const blockRoot = config.getForkTypes(block.message.slot).BeaconBlock.hashTreeRoot(block.message);
+
+      let {blobSidecars} = (await db.blobSidecars.get(blockRoot)) ?? {};
+      if (!blobSidecars) {
+        ({blobSidecars} = (await db.blobSidecarsArchive.get(block.message.slot)) ?? {});
+      }
+
+      if (!blobSidecars) {
+        throw Error("Not found in db");
+      }
+      return {
+        executionOptimistic,
+        data: blobSidecars,
+      };
     },
   };
 }

packages/beacon-node/src/chain/archiver/archiveBlocks.ts

@@ -63,8 +63,8 @@ export async function archiveBlocks(
     });
 
     if (finalizedPostDeneb) {
-      await migrateBlobsSidecarFromHotToColdDb(config, db, finalizedCanonicalBlockRoots);
-      logger.verbose("Migrated blobsSidecar from hot DB to cold DB");
+      await migrateBlobSidecarsFromHotToColdDb(config, db, finalizedCanonicalBlockRoots);
+      logger.verbose("Migrated blobSidecars from hot DB to cold DB");
     }
   }
 
@@ -79,24 +79,24 @@ export async function archiveBlocks(
     });
 
     if (finalizedPostDeneb) {
-      await db.blobsSidecar.batchDelete(nonCanonicalBlockRoots);
+      await db.blobSidecars.batchDelete(nonCanonicalBlockRoots);
       logger.verbose("Deleted non canonical blobsSider from hot DB");
     }
   }
 
   // Delete expired blobs
   // Keep only `[max(GENESIS_EPOCH, current_epoch - MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS), current_epoch]`
   if (finalizedPostDeneb) {
-    const blobsSidecarMinEpoch = currentEpoch - MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS;
-    if (blobsSidecarMinEpoch >= config.DENEB_FORK_EPOCH) {
-      const slotsToDelete = await db.blobsSidecarArchive.keys({lt: computeStartSlotAtEpoch(blobsSidecarMinEpoch)});
+    const blobSidecarsMinEpoch = currentEpoch - MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS;
+    if (blobSidecarsMinEpoch >= config.DENEB_FORK_EPOCH) {
+      const slotsToDelete = await db.blobSidecarsArchive.keys({lt: computeStartSlotAtEpoch(blobSidecarsMinEpoch)});
       if (slotsToDelete.length > 0) {
-        await db.blobsSidecarArchive.batchDelete(slotsToDelete);
+        await db.blobSidecarsArchive.batchDelete(slotsToDelete);
         logger.verbose(
-          `blobsSidecar prune: batchDelete range ${slotsToDelete[0]}..${slotsToDelete[slotsToDelete.length - 1]}`
+          `blobSidecars prune: batchDelete range ${slotsToDelete[0]}..${slotsToDelete[slotsToDelete.length - 1]}`
         );
       } else {
-        logger.verbose(`blobsSidecar prune: no entries before epoch ${blobsSidecarMinEpoch}`);
+        logger.verbose(`blobSidecars prune: no entries before epoch ${blobSidecarsMinEpoch}`);
       }
     }
   }
@@ -163,7 +163,7 @@ async function migrateBlocksFromHotToColdDb(db: IBeaconDb, blocks: BlockRootSlot
   }
 }
 
-async function migrateBlobsSidecarFromHotToColdDb(
+async function migrateBlobSidecarsFromHotToColdDb(
   config: ChainForkConfig,
   db: IBeaconDb,
   blocks: BlockRootSlot[]
@@ -176,22 +176,22 @@ async function migrateBlobsSidecarFromHotToColdDb(
     if (canonicalBlocks.length === 0) return;
 
     // load Buffer instead of ssz deserialized to improve performance
-    const canonicalBlobsSidecarEntries: KeyValue<Slot, Uint8Array>[] = await Promise.all(
+    const canonicalBlobSidecarsEntries: KeyValue<Slot, Uint8Array>[] = await Promise.all(
       canonicalBlocks
         .filter((block) => config.getForkSeq(block.slot) >= ForkSeq.deneb)
         .map(async (block) => {
-          const bytes = await db.blobsSidecar.getBinary(block.root);
+          const bytes = await db.blobSidecars.getBinary(block.root);
           if (!bytes) {
-            throw Error(`No blobsSidecar found for slot ${block.slot} root ${toHex(block.root)}`);
+            throw Error(`No blobSidecars found for slot ${block.slot} root ${toHex(block.root)}`);
           }
           return {key: block.slot, value: bytes};
         })
     );
 
     // put to blockArchive db and delete block db
     await Promise.all([
-      db.blobsSidecarArchive.batchPutBinary(canonicalBlobsSidecarEntries),
-      db.blobsSidecar.batchDelete(canonicalBlocks.map((block) => block.root)),
+      db.blobSidecarsArchive.batchPutBinary(canonicalBlobSidecarsEntries),
+      db.blobSidecars.batchDelete(canonicalBlocks.map((block) => block.root)),
     ]);
   }
 }

packages/beacon-node/src/db/beacon.ts

@@ -15,6 +15,8 @@ import {
   SyncCommitteeRepository,
   SyncCommitteeWitnessRepository,
   BackfilledRanges,
+  BlobSidecarsRepository,
+  BlobSidecarsArchiveRepository,
   BlobsSidecarRepository,
   BlobsSidecarArchiveRepository,
   BLSToExecutionChangeRepository,
@@ -23,9 +25,14 @@ import {PreGenesisState, PreGenesisStateLastProcessedBlock} from "./single/index
 
 export class BeaconDb extends DatabaseService implements IBeaconDb {
   block: BlockRepository;
-  blobsSidecar: BlobsSidecarRepository;
   blockArchive: BlockArchiveRepository;
+
+  blobSidecars: BlobSidecarsRepository;
+  blobSidecarsArchive: BlobSidecarsArchiveRepository;
+  // TODO DENEB: cleanup post full migration
+  blobsSidecar: BlobsSidecarRepository;
   blobsSidecarArchive: BlobsSidecarArchiveRepository;
+
   stateArchive: StateArchiveRepository;
 
   voluntaryExit: VoluntaryExitRepository;
@@ -52,9 +59,14 @@ export class BeaconDb extends DatabaseService implements IBeaconDb {
 
     // Warning: If code is ever run in the constructor, must change this stub to not extend 'packages/beacon-node/test/utils/stub/beaconDb.ts' -
     this.block = new BlockRepository(this.config, this.db);
-    this.blobsSidecar = new BlobsSidecarRepository(this.config, this.db);
     this.blockArchive = new BlockArchiveRepository(this.config, this.db);
+
+    this.blobSidecars = new BlobSidecarsRepository(this.config, this.db);
+    this.blobSidecarsArchive = new BlobSidecarsArchiveRepository(this.config, this.db);
+    // TODO DENEB: cleanup post full migration
+    this.blobsSidecar = new BlobsSidecarRepository(this.config, this.db);
     this.blobsSidecarArchive = new BlobsSidecarArchiveRepository(this.config, this.db);
+
     this.stateArchive = new StateArchiveRepository(this.config, this.db);
     this.voluntaryExit = new VoluntaryExitRepository(this.config, this.db);
     this.blsToExecutionChange = new BLSToExecutionChangeRepository(this.config, this.db);

packages/beacon-node/src/db/interface.ts

@@ -14,6 +14,9 @@ import {
   SyncCommitteeRepository,
   SyncCommitteeWitnessRepository,
   BackfilledRanges,
+  BlobSidecarsRepository,
+  BlobSidecarsArchiveRepository,
+  // TODO DENEB: cleanup once fully migrated from blobsSidecar to blobSidecars
   BlobsSidecarRepository,
   BlobsSidecarArchiveRepository,
   BLSToExecutionChangeRepository,
@@ -28,10 +31,14 @@ import {PreGenesisState, PreGenesisStateLastProcessedBlock} from "./single/index
 export interface IBeaconDb {
   // unfinalized blocks
   block: BlockRepository;
-  blobsSidecar: BlobsSidecarRepository;
-
   // finalized blocks
   blockArchive: BlockArchiveRepository;
+
+  blobSidecars: BlobSidecarsRepository;
+  blobSidecarsArchive: BlobSidecarsArchiveRepository;
+
+  // TODO DENEB: cleanup following two blobs... repos once BlobsSidecar fully migrated to BlobSidecars
+  blobsSidecar: BlobsSidecarRepository;
   blobsSidecarArchive: BlobsSidecarArchiveRepository;
 
   // finalized states

packages/beacon-node/src/db/repositories/blobSidecars.ts

@@ -0,0 +1,38 @@
+import {ChainForkConfig} from "@lodestar/config";
+import {Bucket, Db, Repository} from "@lodestar/db";
+import {ssz} from "@lodestar/types";
+import {ValueOf, ContainerType} from "@chainsafe/ssz";
+
+export const blobSidecarsWrapperSsz = new ContainerType(
+  {
+    blockRoot: ssz.Root,
+    slot: ssz.Slot,
+    blobSidecars: ssz.deneb.BlobSidecars,
+  },
+  {typeName: "BlobSidecarsWrapper", jsonCase: "eth2"}
+);
+
+export type BlobSidecarsWrapper = ValueOf<typeof blobSidecarsWrapperSsz>;
+
+export const BLOB_SIDECARS_IN_WRAPPER_INDEX = 44;
+// ssz.deneb.BlobSidecars.elementType.fixedSize;
+export const BLOBSIDECAR_FIXED_SIZE = 131256;
+
+/**
+ * blobSidecarsWrapper by block root (= hash_tree_root(SignedBeaconBlockAndBlobsSidecar.beacon_block.message))
+ *
+ * Used to store unfinalized BlobsSidecar
+ */
+export class BlobSidecarsRepository extends Repository<Uint8Array, BlobSidecarsWrapper> {
+  constructor(config: ChainForkConfig, db: Db) {
+    super(config, db, Bucket.allForks_blobSidecars, blobSidecarsWrapperSsz);
+  }
+
+  /**
+   * Id is hashTreeRoot of unsigned BeaconBlock
+   */
+  getId(value: BlobSidecarsWrapper): Uint8Array {
+    const {blockRoot} = value;
+    return blockRoot;
+  }
+}

packages/beacon-node/src/db/repositories/blobSidecarsArchive.ts

@@ -0,0 +1,27 @@
+import {ChainForkConfig} from "@lodestar/config";
+import {Bucket, Db, Repository} from "@lodestar/db";
+import {Slot} from "@lodestar/types";
+import {bytesToInt} from "@lodestar/utils";
+
+import {blobSidecarsWrapperSsz, BlobSidecarsWrapper} from "./blobSidecars.js";
+
+/**
+ * blobSidecarsWrapper by slot
+ *
+ * Used to store unfinalized BlobsSidecar
+ */
+export class BlobSidecarsArchiveRepository extends Repository<Slot, BlobSidecarsWrapper> {
+  constructor(config: ChainForkConfig, db: Db) {
+    super(config, db, Bucket.allForks_blobSidecarsArchive, blobSidecarsWrapperSsz);
+  }
+
+  // Handle key as slot
+
+  getId(value: BlobSidecarsWrapper): Slot {
+    return value.slot;
+  }
+
+  decodeKey(data: Uint8Array): number {
+    return bytesToInt(super.decodeKey(data) as unknown as Uint8Array, "be");
+  }
+}

packages/beacon-node/src/db/repositories/index.ts

@@ -1,5 +1,9 @@
+export {BlobSidecarsRepository} from "./blobSidecars.js";
+export {BlobSidecarsArchiveRepository} from "./blobSidecarsArchive.js";
+// TODO DENEB: cleanup post full migration
 export {BlobsSidecarRepository} from "./blobsSidecar.js";
 export {BlobsSidecarArchiveRepository} from "./blobsSidecarArchive.js";
+
 export {BlockRepository} from "./block.js";
 export {BlockArchiveBatchPutBinaryItem, BlockArchiveRepository, BlockFilterOptions} from "./blockArchive.js";
 export {StateArchiveRepository} from "./stateArchive.js";

packages/beacon-node/test/e2e/api/impl/getBlobSidecars.test.ts

@@ -0,0 +1,31 @@
+import {expect} from "chai";
+import {config} from "@lodestar/config/default";
+import {ssz} from "@lodestar/types";
+import {GENESIS_SLOT} from "@lodestar/params";
+
+import {setupApiImplTestServer, ApiImplTestModules} from "../../../unit/api/impl/index.test.js";
+
+describe("getBlobSideCar", function () {
+  let server: ApiImplTestModules;
+
+  before(function () {
+    server = setupApiImplTestServer();
+  });
+
+  it("getBlobSideCar From BlobSidecars", async () => {
+    const block = config.getForkTypes(GENESIS_SLOT).SignedBeaconBlock.defaultValue();
+    const blobSidecars = ssz.deneb.BlobSidecars.defaultValue();
+    const wrappedBlobSidecars = {
+      blockRoot: ssz.Root.defaultValue(),
+      slot: block.message.slot,
+      blobSidecars,
+    };
+
+    server.dbStub.blockArchive.get.resolves(block);
+    server.dbStub.blobSidecars.get.resolves(wrappedBlobSidecars);
+
+    const returnedBlobSideCars = await server.blockApi.getBlobSidecars("genesis");
+
+    expect(returnedBlobSideCars.data).to.equal(blobSidecars);
+  });
+});

packages/beacon-node/test/utils/mocks/db.ts

@@ -14,6 +14,9 @@ import {
   SyncCommitteeRepository,
   SyncCommitteeWitnessRepository,
   BackfilledRanges,
+  BlobSidecarsRepository,
+  BlobSidecarsArchiveRepository,
+  // TODO DENEB: cleanup following blob repos post full migration
   BlobsSidecarRepository,
   BlobsSidecarArchiveRepository,
   BLSToExecutionChangeRepository,
@@ -30,10 +33,13 @@ export function getStubbedBeaconDb(): IBeaconDb {
   return {
     // unfinalized blocks
     block: createStubInstance(BlockRepository),
-    blobsSidecar: createStubInstance(BlobsSidecarRepository),
-
     // finalized blocks
     blockArchive: createStubInstance(BlockArchiveRepository),
+
+    blobSidecars: createStubInstance(BlobSidecarsRepository),
+    blobSidecarsArchive: createStubInstance(BlobSidecarsArchiveRepository),
+    // TODO: cleanup post full migration
+    blobsSidecar: createStubInstance(BlobsSidecarRepository),
     blobsSidecarArchive: createStubInstance(BlobsSidecarArchiveRepository),
 
     // finalized states

packages/beacon-node/test/utils/stub/beaconDb.ts

@@ -14,6 +14,8 @@ import {
   StateArchiveRepository,
   VoluntaryExitRepository,
   BLSToExecutionChangeRepository,
+  BlobSidecarsRepository,
+  BlobSidecarsArchiveRepository,
   BlobsSidecarRepository,
   BlobsSidecarArchiveRepository,
 } from "../../../src/db/repositories/index.js";
@@ -25,6 +27,9 @@ export class StubbedBeaconDb extends BeaconDb {
   block: SinonStubbedInstance<BlockRepository> & BlockRepository;
   blockArchive: SinonStubbedInstance<BlockArchiveRepository> & BlockArchiveRepository;
 
+  blobSidecars: SinonStubbedInstance<BlobSidecarsRepository> & BlobSidecarsRepository;
+  blobSidecarsArchive: SinonStubbedInstance<BlobSidecarsArchiveRepository> & BlobSidecarsArchiveRepository;
+  // TODO DENEB: cleanup following post full migration
   blobsSidecar: SinonStubbedInstance<BlobsSidecarRepository> & BlobsSidecarRepository;
   blobsSidecarArchive: SinonStubbedInstance<BlobsSidecarArchiveRepository> & BlobsSidecarArchiveRepository;
 
@@ -54,6 +59,10 @@ export class StubbedBeaconDb extends BeaconDb {
 
     this.depositDataRoot = createStubInstance(DepositDataRootRepository);
     this.eth1Data = createStubInstance(Eth1DataRepository);
+
+    this.blobSidecars = createStubInstance(BlobSidecarsRepository);
+    this.blobSidecarsArchive = createStubInstance(BlobSidecarsArchiveRepository);
+    // TODO DENEB: cleanup below post full migration
     this.blobsSidecar = createStubInstance(BlobsSidecarRepository);
     this.blobsSidecarArchive = createStubInstance(BlobsSidecarArchiveRepository);
   }

packages/db/src/schema.ts

@@ -47,8 +47,14 @@ export enum Bucket {
 
   index_stateArchiveRootIndex = 26, // State Root -> slot
 
-  allForks_blobsSidecar = 27, // DENEB BeaconBlockRoot -> BlobsSidecar
-  allForks_blobsSidecarArchive = 28, // DENEB BeaconBlockSlot -> BlobsSidecar
+  allForks_blobSidecars = 27, // DENEB BeaconBlockRoot -> BlobSidecars
+  allForks_blobSidecarsArchive = 28, // DENEB BeaconBlockSlot -> BlobSidecars
+
+  // TODO DENEB: cleanup the below buckets once BlobsSidecar fully migrated to BlobSidecars
+  // note: below buckets would not be in use till deneb hf so their number assignments
+  // can be ignored and safely deleted later on
+  allForks_blobsSidecar = 29, // DENEB BeaconBlockRoot -> BlobsSidecar
+  allForks_blobsSidecarArchive = 30, // DENEB BeaconBlockSlot -> BlobsSidecar
 
   // Lightclient server
   // altair_bestUpdatePerCommitteePeriod = 30, // DEPRECATED on v0.32.0