diff --git a/packages/beacon-node/src/api/impl/beacon/blocks/index.ts b/packages/beacon-node/src/api/impl/beacon/blocks/index.ts index 3b420dfb5d0c..0ef866962477 100644 --- a/packages/beacon-node/src/api/impl/beacon/blocks/index.ts +++ b/packages/beacon-node/src/api/impl/beacon/blocks/index.ts @@ -265,9 +265,22 @@ export function getBeaconBlockApi({ await promiseAllMaybeAsync(publishPromises); }, - async getBlobSidecars(_blockId) { - // TODO DENEB: Add implementation on the DB structure change PR - throw Error(""); + async getBlobSidecars(blockId) { + const {block, executionOptimistic} = await resolveBlockId(chain.forkChoice, db, blockId); + const blockRoot = config.getForkTypes(block.message.slot).BeaconBlock.hashTreeRoot(block.message); + + let {blobSidecars} = (await db.blobSidecars.get(blockRoot)) ?? {}; + if (!blobSidecars) { + ({blobSidecars} = (await db.blobSidecarsArchive.get(block.message.slot)) ?? {}); + } + + if (!blobSidecars) { + throw Error("Not found in db"); + } + return { + executionOptimistic, + data: blobSidecars, + }; }, }; } diff --git a/packages/beacon-node/src/chain/archiver/archiveBlocks.ts b/packages/beacon-node/src/chain/archiver/archiveBlocks.ts index 77200b59a40c..3b2b987c0235 100644 --- a/packages/beacon-node/src/chain/archiver/archiveBlocks.ts +++ b/packages/beacon-node/src/chain/archiver/archiveBlocks.ts @@ -63,8 +63,8 @@ export async function archiveBlocks( }); if (finalizedPostDeneb) { - await migrateBlobsSidecarFromHotToColdDb(config, db, finalizedCanonicalBlockRoots); - logger.verbose("Migrated blobsSidecar from hot DB to cold DB"); + await migrateBlobSidecarsFromHotToColdDb(config, db, finalizedCanonicalBlockRoots); + logger.verbose("Migrated blobSidecars from hot DB to cold DB"); } } @@ -79,7 +79,7 @@ export async function archiveBlocks( }); if (finalizedPostDeneb) { - await db.blobsSidecar.batchDelete(nonCanonicalBlockRoots); + await db.blobSidecars.batchDelete(nonCanonicalBlockRoots); logger.verbose("Deleted non canonical blobsSider from hot DB"); } } @@ -87,16 +87,16 @@ export async function archiveBlocks( // Delete expired blobs // Keep only `[max(GENESIS_EPOCH, current_epoch - MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS), current_epoch]` if (finalizedPostDeneb) { - const blobsSidecarMinEpoch = currentEpoch - MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS; - if (blobsSidecarMinEpoch >= config.DENEB_FORK_EPOCH) { - const slotsToDelete = await db.blobsSidecarArchive.keys({lt: computeStartSlotAtEpoch(blobsSidecarMinEpoch)}); + const blobSidecarsMinEpoch = currentEpoch - MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS; + if (blobSidecarsMinEpoch >= config.DENEB_FORK_EPOCH) { + const slotsToDelete = await db.blobSidecarsArchive.keys({lt: computeStartSlotAtEpoch(blobSidecarsMinEpoch)}); if (slotsToDelete.length > 0) { - await db.blobsSidecarArchive.batchDelete(slotsToDelete); + await db.blobSidecarsArchive.batchDelete(slotsToDelete); logger.verbose( - `blobsSidecar prune: batchDelete range ${slotsToDelete[0]}..${slotsToDelete[slotsToDelete.length - 1]}` + `blobSidecars prune: batchDelete range ${slotsToDelete[0]}..${slotsToDelete[slotsToDelete.length - 1]}` ); } else { - logger.verbose(`blobsSidecar prune: no entries before epoch ${blobsSidecarMinEpoch}`); + logger.verbose(`blobSidecars prune: no entries before epoch ${blobSidecarsMinEpoch}`); } } } @@ -163,7 +163,7 @@ async function migrateBlocksFromHotToColdDb(db: IBeaconDb, blocks: BlockRootSlot } } -async function migrateBlobsSidecarFromHotToColdDb( +async function migrateBlobSidecarsFromHotToColdDb( config: ChainForkConfig, db: IBeaconDb, blocks: BlockRootSlot[] @@ -176,13 +176,13 @@ async function migrateBlobsSidecarFromHotToColdDb( if (canonicalBlocks.length === 0) return; // load Buffer instead of ssz deserialized to improve performance - const canonicalBlobsSidecarEntries: KeyValue[] = await Promise.all( + const canonicalBlobSidecarsEntries: KeyValue[] = await Promise.all( canonicalBlocks .filter((block) => config.getForkSeq(block.slot) >= ForkSeq.deneb) .map(async (block) => { - const bytes = await db.blobsSidecar.getBinary(block.root); + const bytes = await db.blobSidecars.getBinary(block.root); if (!bytes) { - throw Error(`No blobsSidecar found for slot ${block.slot} root ${toHex(block.root)}`); + throw Error(`No blobSidecars found for slot ${block.slot} root ${toHex(block.root)}`); } return {key: block.slot, value: bytes}; }) @@ -190,8 +190,8 @@ async function migrateBlobsSidecarFromHotToColdDb( // put to blockArchive db and delete block db await Promise.all([ - db.blobsSidecarArchive.batchPutBinary(canonicalBlobsSidecarEntries), - db.blobsSidecar.batchDelete(canonicalBlocks.map((block) => block.root)), + db.blobSidecarsArchive.batchPutBinary(canonicalBlobSidecarsEntries), + db.blobSidecars.batchDelete(canonicalBlocks.map((block) => block.root)), ]); } } diff --git a/packages/beacon-node/src/db/beacon.ts b/packages/beacon-node/src/db/beacon.ts index ef7dc00a144f..fd7639b4c77a 100644 --- a/packages/beacon-node/src/db/beacon.ts +++ b/packages/beacon-node/src/db/beacon.ts @@ -15,6 +15,8 @@ import { SyncCommitteeRepository, SyncCommitteeWitnessRepository, BackfilledRanges, + BlobSidecarsRepository, + BlobSidecarsArchiveRepository, BlobsSidecarRepository, BlobsSidecarArchiveRepository, BLSToExecutionChangeRepository, @@ -23,9 +25,14 @@ import {PreGenesisState, PreGenesisStateLastProcessedBlock} from "./single/index export class BeaconDb extends DatabaseService implements IBeaconDb { block: BlockRepository; - blobsSidecar: BlobsSidecarRepository; blockArchive: BlockArchiveRepository; + + blobSidecars: BlobSidecarsRepository; + blobSidecarsArchive: BlobSidecarsArchiveRepository; + // TODO DENEB: cleanup post full migration + blobsSidecar: BlobsSidecarRepository; blobsSidecarArchive: BlobsSidecarArchiveRepository; + stateArchive: StateArchiveRepository; voluntaryExit: VoluntaryExitRepository; @@ -52,9 +59,14 @@ export class BeaconDb extends DatabaseService implements IBeaconDb { // Warning: If code is ever run in the constructor, must change this stub to not extend 'packages/beacon-node/test/utils/stub/beaconDb.ts' - this.block = new BlockRepository(this.config, this.db); - this.blobsSidecar = new BlobsSidecarRepository(this.config, this.db); this.blockArchive = new BlockArchiveRepository(this.config, this.db); + + this.blobSidecars = new BlobSidecarsRepository(this.config, this.db); + this.blobSidecarsArchive = new BlobSidecarsArchiveRepository(this.config, this.db); + // TODO DENEB: cleanup post full migration + this.blobsSidecar = new BlobsSidecarRepository(this.config, this.db); this.blobsSidecarArchive = new BlobsSidecarArchiveRepository(this.config, this.db); + this.stateArchive = new StateArchiveRepository(this.config, this.db); this.voluntaryExit = new VoluntaryExitRepository(this.config, this.db); this.blsToExecutionChange = new BLSToExecutionChangeRepository(this.config, this.db); diff --git a/packages/beacon-node/src/db/interface.ts b/packages/beacon-node/src/db/interface.ts index 3e0b41d6fe47..5c213052352a 100644 --- a/packages/beacon-node/src/db/interface.ts +++ b/packages/beacon-node/src/db/interface.ts @@ -14,6 +14,9 @@ import { SyncCommitteeRepository, SyncCommitteeWitnessRepository, BackfilledRanges, + BlobSidecarsRepository, + BlobSidecarsArchiveRepository, + // TODO DENEB: cleanup once fully migrated from blobsSidecar to blobSidecars BlobsSidecarRepository, BlobsSidecarArchiveRepository, BLSToExecutionChangeRepository, @@ -28,10 +31,14 @@ import {PreGenesisState, PreGenesisStateLastProcessedBlock} from "./single/index export interface IBeaconDb { // unfinalized blocks block: BlockRepository; - blobsSidecar: BlobsSidecarRepository; - // finalized blocks blockArchive: BlockArchiveRepository; + + blobSidecars: BlobSidecarsRepository; + blobSidecarsArchive: BlobSidecarsArchiveRepository; + + // TODO DENEB: cleanup following two blobs... repos once BlobsSidecar fully migrated to BlobSidecars + blobsSidecar: BlobsSidecarRepository; blobsSidecarArchive: BlobsSidecarArchiveRepository; // finalized states diff --git a/packages/beacon-node/src/db/repositories/blobSidecars.ts b/packages/beacon-node/src/db/repositories/blobSidecars.ts new file mode 100644 index 000000000000..9f90e2dc63ed --- /dev/null +++ b/packages/beacon-node/src/db/repositories/blobSidecars.ts @@ -0,0 +1,38 @@ +import {ChainForkConfig} from "@lodestar/config"; +import {Bucket, Db, Repository} from "@lodestar/db"; +import {ssz} from "@lodestar/types"; +import {ValueOf, ContainerType} from "@chainsafe/ssz"; + +export const blobSidecarsWrapperSsz = new ContainerType( + { + blockRoot: ssz.Root, + slot: ssz.Slot, + blobSidecars: ssz.deneb.BlobSidecars, + }, + {typeName: "BlobSidecarsWrapper", jsonCase: "eth2"} +); + +export type BlobSidecarsWrapper = ValueOf; + +export const BLOB_SIDECARS_IN_WRAPPER_INDEX = 44; +// ssz.deneb.BlobSidecars.elementType.fixedSize; +export const BLOBSIDECAR_FIXED_SIZE = 131256; + +/** + * blobSidecarsWrapper by block root (= hash_tree_root(SignedBeaconBlockAndBlobsSidecar.beacon_block.message)) + * + * Used to store unfinalized BlobsSidecar + */ +export class BlobSidecarsRepository extends Repository { + constructor(config: ChainForkConfig, db: Db) { + super(config, db, Bucket.allForks_blobSidecars, blobSidecarsWrapperSsz); + } + + /** + * Id is hashTreeRoot of unsigned BeaconBlock + */ + getId(value: BlobSidecarsWrapper): Uint8Array { + const {blockRoot} = value; + return blockRoot; + } +} diff --git a/packages/beacon-node/src/db/repositories/blobSidecarsArchive.ts b/packages/beacon-node/src/db/repositories/blobSidecarsArchive.ts new file mode 100644 index 000000000000..d834708cd2c6 --- /dev/null +++ b/packages/beacon-node/src/db/repositories/blobSidecarsArchive.ts @@ -0,0 +1,27 @@ +import {ChainForkConfig} from "@lodestar/config"; +import {Bucket, Db, Repository} from "@lodestar/db"; +import {Slot} from "@lodestar/types"; +import {bytesToInt} from "@lodestar/utils"; + +import {blobSidecarsWrapperSsz, BlobSidecarsWrapper} from "./blobSidecars.js"; + +/** + * blobSidecarsWrapper by slot + * + * Used to store unfinalized BlobsSidecar + */ +export class BlobSidecarsArchiveRepository extends Repository { + constructor(config: ChainForkConfig, db: Db) { + super(config, db, Bucket.allForks_blobSidecarsArchive, blobSidecarsWrapperSsz); + } + + // Handle key as slot + + getId(value: BlobSidecarsWrapper): Slot { + return value.slot; + } + + decodeKey(data: Uint8Array): number { + return bytesToInt(super.decodeKey(data) as unknown as Uint8Array, "be"); + } +} diff --git a/packages/beacon-node/src/db/repositories/index.ts b/packages/beacon-node/src/db/repositories/index.ts index 0cd99e2fadf9..00a7c96c414d 100644 --- a/packages/beacon-node/src/db/repositories/index.ts +++ b/packages/beacon-node/src/db/repositories/index.ts @@ -1,5 +1,9 @@ +export {BlobSidecarsRepository} from "./blobSidecars.js"; +export {BlobSidecarsArchiveRepository} from "./blobSidecarsArchive.js"; +// TODO DENEB: cleanup post full migration export {BlobsSidecarRepository} from "./blobsSidecar.js"; export {BlobsSidecarArchiveRepository} from "./blobsSidecarArchive.js"; + export {BlockRepository} from "./block.js"; export {BlockArchiveBatchPutBinaryItem, BlockArchiveRepository, BlockFilterOptions} from "./blockArchive.js"; export {StateArchiveRepository} from "./stateArchive.js"; diff --git a/packages/beacon-node/test/e2e/api/impl/getBlobSidecars.test.ts b/packages/beacon-node/test/e2e/api/impl/getBlobSidecars.test.ts new file mode 100644 index 000000000000..423a62a21455 --- /dev/null +++ b/packages/beacon-node/test/e2e/api/impl/getBlobSidecars.test.ts @@ -0,0 +1,31 @@ +import {expect} from "chai"; +import {config} from "@lodestar/config/default"; +import {ssz} from "@lodestar/types"; +import {GENESIS_SLOT} from "@lodestar/params"; + +import {setupApiImplTestServer, ApiImplTestModules} from "../../../unit/api/impl/index.test.js"; + +describe("getBlobSideCar", function () { + let server: ApiImplTestModules; + + before(function () { + server = setupApiImplTestServer(); + }); + + it("getBlobSideCar From BlobSidecars", async () => { + const block = config.getForkTypes(GENESIS_SLOT).SignedBeaconBlock.defaultValue(); + const blobSidecars = ssz.deneb.BlobSidecars.defaultValue(); + const wrappedBlobSidecars = { + blockRoot: ssz.Root.defaultValue(), + slot: block.message.slot, + blobSidecars, + }; + + server.dbStub.blockArchive.get.resolves(block); + server.dbStub.blobSidecars.get.resolves(wrappedBlobSidecars); + + const returnedBlobSideCars = await server.blockApi.getBlobSidecars("genesis"); + + expect(returnedBlobSideCars.data).to.equal(blobSidecars); + }); +}); diff --git a/packages/beacon-node/test/utils/mocks/db.ts b/packages/beacon-node/test/utils/mocks/db.ts index 70e60e2234c2..cdfba8873ec1 100644 --- a/packages/beacon-node/test/utils/mocks/db.ts +++ b/packages/beacon-node/test/utils/mocks/db.ts @@ -14,6 +14,9 @@ import { SyncCommitteeRepository, SyncCommitteeWitnessRepository, BackfilledRanges, + BlobSidecarsRepository, + BlobSidecarsArchiveRepository, + // TODO DENEB: cleanup following blob repos post full migration BlobsSidecarRepository, BlobsSidecarArchiveRepository, BLSToExecutionChangeRepository, @@ -30,10 +33,13 @@ export function getStubbedBeaconDb(): IBeaconDb { return { // unfinalized blocks block: createStubInstance(BlockRepository), - blobsSidecar: createStubInstance(BlobsSidecarRepository), - // finalized blocks blockArchive: createStubInstance(BlockArchiveRepository), + + blobSidecars: createStubInstance(BlobSidecarsRepository), + blobSidecarsArchive: createStubInstance(BlobSidecarsArchiveRepository), + // TODO: cleanup post full migration + blobsSidecar: createStubInstance(BlobsSidecarRepository), blobsSidecarArchive: createStubInstance(BlobsSidecarArchiveRepository), // finalized states diff --git a/packages/beacon-node/test/utils/stub/beaconDb.ts b/packages/beacon-node/test/utils/stub/beaconDb.ts index cb7e2e8cea01..1461ecbc700b 100644 --- a/packages/beacon-node/test/utils/stub/beaconDb.ts +++ b/packages/beacon-node/test/utils/stub/beaconDb.ts @@ -14,6 +14,8 @@ import { StateArchiveRepository, VoluntaryExitRepository, BLSToExecutionChangeRepository, + BlobSidecarsRepository, + BlobSidecarsArchiveRepository, BlobsSidecarRepository, BlobsSidecarArchiveRepository, } from "../../../src/db/repositories/index.js"; @@ -25,6 +27,9 @@ export class StubbedBeaconDb extends BeaconDb { block: SinonStubbedInstance & BlockRepository; blockArchive: SinonStubbedInstance & BlockArchiveRepository; + blobSidecars: SinonStubbedInstance & BlobSidecarsRepository; + blobSidecarsArchive: SinonStubbedInstance & BlobSidecarsArchiveRepository; + // TODO DENEB: cleanup following post full migration blobsSidecar: SinonStubbedInstance & BlobsSidecarRepository; blobsSidecarArchive: SinonStubbedInstance & BlobsSidecarArchiveRepository; @@ -54,6 +59,10 @@ export class StubbedBeaconDb extends BeaconDb { this.depositDataRoot = createStubInstance(DepositDataRootRepository); this.eth1Data = createStubInstance(Eth1DataRepository); + + this.blobSidecars = createStubInstance(BlobSidecarsRepository); + this.blobSidecarsArchive = createStubInstance(BlobSidecarsArchiveRepository); + // TODO DENEB: cleanup below post full migration this.blobsSidecar = createStubInstance(BlobsSidecarRepository); this.blobsSidecarArchive = createStubInstance(BlobsSidecarArchiveRepository); } diff --git a/packages/db/src/schema.ts b/packages/db/src/schema.ts index bb42cb4a4786..000cb95cbca5 100644 --- a/packages/db/src/schema.ts +++ b/packages/db/src/schema.ts @@ -47,8 +47,14 @@ export enum Bucket { index_stateArchiveRootIndex = 26, // State Root -> slot - allForks_blobsSidecar = 27, // DENEB BeaconBlockRoot -> BlobsSidecar - allForks_blobsSidecarArchive = 28, // DENEB BeaconBlockSlot -> BlobsSidecar + allForks_blobSidecars = 27, // DENEB BeaconBlockRoot -> BlobSidecars + allForks_blobSidecarsArchive = 28, // DENEB BeaconBlockSlot -> BlobSidecars + + // TODO DENEB: cleanup the below buckets once BlobsSidecar fully migrated to BlobSidecars + // note: below buckets would not be in use till deneb hf so their number assignments + // can be ignored and safely deleted later on + allForks_blobsSidecar = 29, // DENEB BeaconBlockRoot -> BlobsSidecar + allForks_blobsSidecarArchive = 30, // DENEB BeaconBlockSlot -> BlobsSidecar // Lightclient server // altair_bestUpdatePerCommitteePeriod = 30, // DEPRECATED on v0.32.0