diff --git a/packages/beacon-node/src/network/reqresp/beaconBlocksMaybeBlobsByRoot.ts b/packages/beacon-node/src/network/reqresp/beaconBlocksMaybeBlobsByRoot.ts index 5113197fc953..30791fae9540 100644 --- a/packages/beacon-node/src/network/reqresp/beaconBlocksMaybeBlobsByRoot.ts +++ b/packages/beacon-node/src/network/reqresp/beaconBlocksMaybeBlobsByRoot.ts @@ -69,7 +69,7 @@ export async function beaconBlocksMaybeBlobsByRoot( let pendingDataColumns = null; const blobIdentifiers: deneb.BlobIdentifier[] = []; - const dataColumnIdentifiers: fulu.DataColumnIdentifier[] = []; + const dataColumnsByRootIdentifiers: fulu.DataColumnsByRootIdentifier[] = []; let prevFork = null; for (const block of allBlocks) { @@ -95,8 +95,11 @@ export async function beaconBlocksMaybeBlobsByRoot( dataColumnsDataBlocks.push(block); const blobKzgCommitmentsLen = (block.data.message.body as deneb.BeaconBlockBody).blobKzgCommitments.length; const custodyColumnIndexes = blobKzgCommitmentsLen > 0 ? columns : []; - for (const columnIndex of custodyColumnIndexes) { - dataColumnIdentifiers.push({blockRoot, index: columnIndex}); + if (custodyColumnIndexes.length > 0) { + dataColumnsByRootIdentifiers.push({ + blockRoot, + columns: custodyColumnIndexes, + }); } } else { throw Error(`Invalid fork=${fork} in beaconBlocksMaybeBlobsByRoot`); @@ -139,11 +142,13 @@ export async function beaconBlocksMaybeBlobsByRoot( ...(partialDownload ? {blocks: partialDownload.blocks.length, pendingDataColumns: partialDownload.pendingDataColumns.join(" ")} : {blocks: null, pendingDataColumns: null}), - dataColumnIdentifiers: dataColumnIdentifiers.map((did) => did.index).join(" "), + dataColumnIdentifiers: dataColumnsByRootIdentifiers + .map((id) => `${id.blockRoot}: ${id.columns.join(" ")}`) + .join(" "), peerClient, }); - if (dataColumnIdentifiers.length > 0) { - allDataColumnsSidecars = await network.sendDataColumnSidecarsByRoot(peerId, dataColumnIdentifiers); + if (dataColumnsByRootIdentifiers.length > 0) { + allDataColumnsSidecars = await network.sendDataColumnSidecarsByRoot(peerId, dataColumnsByRootIdentifiers); } else { if (partialDownload !== null) { return partialDownload; @@ -513,7 +518,6 @@ export async function unavailableBeaconBlobsByRootPostFulu( const {dataColumnsCache, resolveAvailability} = cachedData as CachedDataColumns; // resolve missing blobs - const dataColumnIdentifiers: fulu.DataColumnIdentifier[] = []; const slot = block.message.slot; const blockRoot = config.getForkTypes(slot).BeaconBlock.hashTreeRoot(block.message); @@ -577,13 +581,9 @@ export async function unavailableBeaconBlobsByRootPostFulu( return unavailableBlockInput; } - for (const columnIndex of columns) { - dataColumnIdentifiers.push({blockRoot, index: columnIndex}); - } - let allDataColumnSidecars: fulu.DataColumnSidecar[]; - if (dataColumnIdentifiers.length > 0) { - allDataColumnSidecars = await network.sendDataColumnSidecarsByRoot(peerId, dataColumnIdentifiers); + if (columns.length > 0) { + allDataColumnSidecars = await network.sendDataColumnSidecarsByRoot(peerId, [{blockRoot, columns}]); } else { allDataColumnSidecars = []; } diff --git a/packages/beacon-node/src/network/reqresp/handlers/dataColumnSidecarsByRoot.ts b/packages/beacon-node/src/network/reqresp/handlers/dataColumnSidecarsByRoot.ts index 5ca3554c838d..8d3796bb66cd 100644 --- a/packages/beacon-node/src/network/reqresp/handlers/dataColumnSidecarsByRoot.ts +++ b/packages/beacon-node/src/network/reqresp/handlers/dataColumnSidecarsByRoot.ts @@ -26,18 +26,8 @@ export async function* onDataColumnSidecarsByRoot( chain.config.FULU_FORK_EPOCH ); - // In sidecars by root request, it can be expected that sidecar requests will be come - // clustured by blockroots, and this helps us save db lookups once we load sidecars - // for a root - let lastFetchedSideCars: { - blockRoot: RootHex; - bytes: Uint8Array; - dataColumnsIndex: Uint8Array; - columnsSize: number; - } | null = null; - - for (const dataColumnIdentifier of requestBody) { - const {blockRoot, index} = dataColumnIdentifier; + for (const dataColumnsByRootIdentifier of requestBody) { + const {blockRoot, columns} = dataColumnsByRootIdentifier; const blockRootHex = toHex(blockRoot); const block = chain.forkChoice.getBlockHex(blockRootHex); @@ -50,61 +40,57 @@ export async function* onDataColumnSidecarsByRoot( continue; } - // Check if we need to load sidecars for a new block root - if (lastFetchedSideCars === null || lastFetchedSideCars.blockRoot !== blockRootHex) { - const dataColumnSidecarsBytesWrapped = await db.dataColumnSidecars.getBinary(fromHex(block.blockRoot)); - if (!dataColumnSidecarsBytesWrapped) { - // Handle the same to onBeaconBlocksByRange - throw new ResponseError(RespStatus.SERVER_ERROR, `No item for root ${block.blockRoot} slot ${block.slot}`); - } - - const retrivedColumnsLen = ssz.Uint8.deserialize( - dataColumnSidecarsBytesWrapped.slice(NUM_COLUMNS_IN_WRAPPER_INDEX, COLUMN_SIZE_IN_WRAPPER_INDEX) - ); - const retrievedColumnsSizeBytes = dataColumnSidecarsBytesWrapped.slice( - COLUMN_SIZE_IN_WRAPPER_INDEX, - CUSTODY_COLUMNS_IN_IN_WRAPPER_INDEX - ); - const columnsSize = ssz.UintNum64.deserialize(retrievedColumnsSizeBytes); - const dataColumnSidecarsBytes = dataColumnSidecarsBytesWrapped.slice( - DATA_COLUMN_SIDECARS_IN_WRAPPER_INDEX + 4 * retrivedColumnsLen - ); + const dataColumnSidecarsBytesWrapped = await db.dataColumnSidecars.getBinary(fromHex(block.blockRoot)); + if (!dataColumnSidecarsBytesWrapped) { + // Handle the same to onBeaconBlocksByRange + throw new ResponseError(RespStatus.SERVER_ERROR, `No item for root ${block.blockRoot} slot ${block.slot}`); + } - const dataColumnsIndex = dataColumnSidecarsBytesWrapped.slice( - CUSTODY_COLUMNS_IN_IN_WRAPPER_INDEX, - CUSTODY_COLUMNS_IN_IN_WRAPPER_INDEX + NUMBER_OF_COLUMNS - ); + const retrivedColumnsLen = ssz.Uint8.deserialize( + dataColumnSidecarsBytesWrapped.slice(NUM_COLUMNS_IN_WRAPPER_INDEX, COLUMN_SIZE_IN_WRAPPER_INDEX) + ); + const retrievedColumnsSizeBytes = dataColumnSidecarsBytesWrapped.slice( + COLUMN_SIZE_IN_WRAPPER_INDEX, + CUSTODY_COLUMNS_IN_IN_WRAPPER_INDEX + ); + const columnsSize = ssz.UintNum64.deserialize(retrievedColumnsSizeBytes); + const dataColumnSidecarsBytes = dataColumnSidecarsBytesWrapped.slice( + DATA_COLUMN_SIDECARS_IN_WRAPPER_INDEX + 4 * retrivedColumnsLen + ); - // const storedColumns = Array.from({length: NUMBER_OF_COLUMNS}, (_v, i) => i).filter( - // (i) => dataColumnsIndex[i] > 0 - // ); - // const columnsLen = dataColumnSidecarsBytes.length / columnsSize; - // console.log( - // `onDataColumnSidecarsByRoot: slot=${block.slot} columnsSize=${columnsSize} storedColumnsLen=${columnsLen} retrivedColumnsLen=${retrivedColumnsLen} dataColumnSidecarsBytesWrapped=${dataColumnSidecarsBytesWrapped.length} storedColumns=${storedColumns.join(" ")}` - // ); + const dataColumnsIndex = dataColumnSidecarsBytesWrapped.slice( + CUSTODY_COLUMNS_IN_IN_WRAPPER_INDEX, + CUSTODY_COLUMNS_IN_IN_WRAPPER_INDEX + NUMBER_OF_COLUMNS + ); - lastFetchedSideCars = {blockRoot: blockRootHex, bytes: dataColumnSidecarsBytes, columnsSize, dataColumnsIndex}; - } + // const storedColumns = Array.from({length: NUMBER_OF_COLUMNS}, (_v, i) => i).filter( + // (i) => dataColumnsIndex[i] > 0 + // ); + // const columnsLen = dataColumnSidecarsBytes.length / columnsSize; + // console.log( + // `onDataColumnSidecarsByRoot: slot=${block.slot} columnsSize=${columnsSize} storedColumnsLen=${columnsLen} retrivedColumnsLen=${retrivedColumnsLen} dataColumnSidecarsBytesWrapped=${dataColumnSidecarsBytesWrapped.length} storedColumns=${storedColumns.join(" ")}` + // ); - const dataIndex = (lastFetchedSideCars.dataColumnsIndex[index] ?? 0) - 1; - if (dataIndex < 0) { - throw new ResponseError(RespStatus.SERVER_ERROR, `dataColumnSidecar index=${index} not custodied`); - } - const {columnsSize} = lastFetchedSideCars; + for (const index of columns) { + const dataIndex = (dataColumnsIndex[index] ?? 0) - 1; + if (dataIndex < 0) { + throw new ResponseError(RespStatus.SERVER_ERROR, `dataColumnSidecar index=${index} not custodied`); + } - const dataColumnSidecarBytes = lastFetchedSideCars.bytes.slice( - dataIndex * columnsSize, - (dataIndex + 1) * columnsSize - ); - if (dataColumnSidecarBytes.length !== columnsSize) { - throw Error( - `Inconsistent state, dataColumnSidecar blockRoot=${blockRootHex} index=${index} dataColumnSidecarBytes=${dataColumnSidecarBytes.length} expected=${columnsSize}` + const dataColumnSidecarBytes = dataColumnSidecarsBytes.slice( + dataIndex * columnsSize, + (dataIndex + 1) * columnsSize ); - } + if (dataColumnSidecarBytes.length !== columnsSize) { + throw Error( + `Inconsistent state, dataColumnSidecar blockRoot=${blockRootHex} index=${index} dataColumnSidecarBytes=${dataColumnSidecarBytes.length} expected=${columnsSize}` + ); + } - yield { - data: dataColumnSidecarBytes, - fork: chain.config.getForkName(block.slot), - }; + yield { + data: dataColumnSidecarBytes, + fork: chain.config.getForkName(block.slot), + }; + } } } diff --git a/packages/types/src/fulu/sszTypes.ts b/packages/types/src/fulu/sszTypes.ts index b8605c605764..d22da6e26c8f 100644 --- a/packages/types/src/fulu/sszTypes.ts +++ b/packages/types/src/fulu/sszTypes.ts @@ -5,6 +5,7 @@ import { FIELD_ELEMENTS_PER_EXT_BLOB, KZG_COMMITMENTS_INCLUSION_PROOF_DEPTH, MAX_BLOB_COMMITMENTS_PER_BLOCK, + MAX_REQUEST_BLOCKS_DENEB, MAX_REQUEST_DATA_COLUMN_SIDECARS, NUMBER_OF_COLUMNS, } from "@lodestar/params"; @@ -64,17 +65,17 @@ export const MatrixEntry = new ContainerType( // ReqResp types // ============= -export const DataColumnIdentifier = new ContainerType( +export const DataColumnsByRootIdentifier = new ContainerType( { blockRoot: Root, - index: ColumnIndex, + columns: new ListBasicType(ColumnIndex, NUMBER_OF_COLUMNS), }, - {typeName: "DataColumnIdentifier", jsonCase: "eth2"} + {typeName: "DataColumnsByRootIdentifier", jsonCase: "eth2"} ); export const DataColumnSidecarsByRootRequest = new ListCompositeType( - DataColumnIdentifier, - MAX_REQUEST_DATA_COLUMN_SIDECARS + DataColumnsByRootIdentifier, + MAX_REQUEST_BLOCKS_DENEB ); export const DataColumnSidecarsByRangeRequest = new ContainerType( diff --git a/packages/types/src/fulu/types.ts b/packages/types/src/fulu/types.ts index 1589652cc026..21acae97fd51 100644 --- a/packages/types/src/fulu/types.ts +++ b/packages/types/src/fulu/types.ts @@ -14,7 +14,7 @@ export type DataColumnSidecar = ValueOf; export type DataColumnSidecars = ValueOf; export type MatrixEntry = ValueOf; -export type DataColumnIdentifier = ValueOf; +export type DataColumnsByRootIdentifier = ValueOf; export type DataColumnSidecarsByRootRequest = ValueOf; export type DataColumnSidecarsByRangeRequest = ValueOf;