feat: update dataColumnSidecarsByRoot to new spec

packages/beacon-node/src/network/reqresp/beaconBlocksMaybeBlobsByRoot.ts

@@ -69,7 +69,7 @@ export async function beaconBlocksMaybeBlobsByRoot(
   let pendingDataColumns = null;
 
   const blobIdentifiers: deneb.BlobIdentifier[] = [];
-  const dataColumnIdentifiers: fulu.DataColumnIdentifier[] = [];
+  const dataColumnsByRootIdentifiers: fulu.DataColumnsByRootIdentifier[] = [];
 
   let prevFork = null;
   for (const block of allBlocks) {
@@ -95,8 +95,11 @@ export async function beaconBlocksMaybeBlobsByRoot(
       dataColumnsDataBlocks.push(block);
       const blobKzgCommitmentsLen = (block.data.message.body as deneb.BeaconBlockBody).blobKzgCommitments.length;
       const custodyColumnIndexes = blobKzgCommitmentsLen > 0 ? columns : [];
-      for (const columnIndex of custodyColumnIndexes) {
-        dataColumnIdentifiers.push({blockRoot, index: columnIndex});
+      if (custodyColumnIndexes.length > 0) {
+        dataColumnsByRootIdentifiers.push({
+          blockRoot,
+          columns: custodyColumnIndexes,
+        });
       }
     } else {
       throw Error(`Invalid fork=${fork} in beaconBlocksMaybeBlobsByRoot`);
@@ -139,11 +142,13 @@ export async function beaconBlocksMaybeBlobsByRoot(
       ...(partialDownload
         ? {blocks: partialDownload.blocks.length, pendingDataColumns: partialDownload.pendingDataColumns.join(" ")}
         : {blocks: null, pendingDataColumns: null}),
-      dataColumnIdentifiers: dataColumnIdentifiers.map((did) => did.index).join(" "),
+      dataColumnIdentifiers: dataColumnsByRootIdentifiers
+        .map((id) => `${id.blockRoot}: ${id.columns.join(" ")}`)
+        .join(" "),
       peerClient,
     });
-    if (dataColumnIdentifiers.length > 0) {
-      allDataColumnsSidecars = await network.sendDataColumnSidecarsByRoot(peerId, dataColumnIdentifiers);
+    if (dataColumnsByRootIdentifiers.length > 0) {
+      allDataColumnsSidecars = await network.sendDataColumnSidecarsByRoot(peerId, dataColumnsByRootIdentifiers);
     } else {
       if (partialDownload !== null) {
         return partialDownload;
@@ -513,7 +518,6 @@ export async function unavailableBeaconBlobsByRootPostFulu(
   const {dataColumnsCache, resolveAvailability} = cachedData as CachedDataColumns;
 
   // resolve missing blobs
-  const dataColumnIdentifiers: fulu.DataColumnIdentifier[] = [];
   const slot = block.message.slot;
   const blockRoot = config.getForkTypes(slot).BeaconBlock.hashTreeRoot(block.message);
 
@@ -577,13 +581,9 @@ export async function unavailableBeaconBlobsByRootPostFulu(
       return unavailableBlockInput;
     }
 
-    for (const columnIndex of columns) {
-      dataColumnIdentifiers.push({blockRoot, index: columnIndex});
-    }
-
     let allDataColumnSidecars: fulu.DataColumnSidecar[];
-    if (dataColumnIdentifiers.length > 0) {
-      allDataColumnSidecars = await network.sendDataColumnSidecarsByRoot(peerId, dataColumnIdentifiers);
+    if (columns.length > 0) {
+      allDataColumnSidecars = await network.sendDataColumnSidecarsByRoot(peerId, [{blockRoot, columns}]);
     } else {
       allDataColumnSidecars = [];
     }

packages/beacon-node/src/network/reqresp/handlers/dataColumnSidecarsByRoot.ts

@@ -26,18 +26,8 @@ export async function* onDataColumnSidecarsByRoot(
     chain.config.FULU_FORK_EPOCH
   );
 
-  // In sidecars by root request, it can be expected that sidecar requests will be come
-  // clustured by blockroots, and this helps us save db lookups once we load sidecars
-  // for a root
-  let lastFetchedSideCars: {
-    blockRoot: RootHex;
-    bytes: Uint8Array;
-    dataColumnsIndex: Uint8Array;
-    columnsSize: number;
-  } | null = null;
-
-  for (const dataColumnIdentifier of requestBody) {
-    const {blockRoot, index} = dataColumnIdentifier;
+  for (const dataColumnsByRootIdentifier of requestBody) {
+    const {blockRoot, columns} = dataColumnsByRootIdentifier;
     const blockRootHex = toHex(blockRoot);
     const block = chain.forkChoice.getBlockHex(blockRootHex);
 
@@ -50,61 +40,57 @@ export async function* onDataColumnSidecarsByRoot(
       continue;
     }
 
-    // Check if we need to load sidecars for a new block root
-    if (lastFetchedSideCars === null || lastFetchedSideCars.blockRoot !== blockRootHex) {
-      const dataColumnSidecarsBytesWrapped = await db.dataColumnSidecars.getBinary(fromHex(block.blockRoot));
-      if (!dataColumnSidecarsBytesWrapped) {
-        // Handle the same to onBeaconBlocksByRange
-        throw new ResponseError(RespStatus.SERVER_ERROR, `No item for root ${block.blockRoot} slot ${block.slot}`);
-      }
-
-      const retrivedColumnsLen = ssz.Uint8.deserialize(
-        dataColumnSidecarsBytesWrapped.slice(NUM_COLUMNS_IN_WRAPPER_INDEX, COLUMN_SIZE_IN_WRAPPER_INDEX)
-      );
-      const retrievedColumnsSizeBytes = dataColumnSidecarsBytesWrapped.slice(
-        COLUMN_SIZE_IN_WRAPPER_INDEX,
-        CUSTODY_COLUMNS_IN_IN_WRAPPER_INDEX
-      );
-      const columnsSize = ssz.UintNum64.deserialize(retrievedColumnsSizeBytes);
-      const dataColumnSidecarsBytes = dataColumnSidecarsBytesWrapped.slice(
-        DATA_COLUMN_SIDECARS_IN_WRAPPER_INDEX + 4 * retrivedColumnsLen
-      );
+    const dataColumnSidecarsBytesWrapped = await db.dataColumnSidecars.getBinary(fromHex(block.blockRoot));
+    if (!dataColumnSidecarsBytesWrapped) {
+      // Handle the same to onBeaconBlocksByRange
+      throw new ResponseError(RespStatus.SERVER_ERROR, `No item for root ${block.blockRoot} slot ${block.slot}`);
+    }
 
-      const dataColumnsIndex = dataColumnSidecarsBytesWrapped.slice(
-        CUSTODY_COLUMNS_IN_IN_WRAPPER_INDEX,
-        CUSTODY_COLUMNS_IN_IN_WRAPPER_INDEX + NUMBER_OF_COLUMNS
-      );
+    const retrivedColumnsLen = ssz.Uint8.deserialize(
+      dataColumnSidecarsBytesWrapped.slice(NUM_COLUMNS_IN_WRAPPER_INDEX, COLUMN_SIZE_IN_WRAPPER_INDEX)
+    );
+    const retrievedColumnsSizeBytes = dataColumnSidecarsBytesWrapped.slice(
+      COLUMN_SIZE_IN_WRAPPER_INDEX,
+      CUSTODY_COLUMNS_IN_IN_WRAPPER_INDEX
+    );
+    const columnsSize = ssz.UintNum64.deserialize(retrievedColumnsSizeBytes);
+    const dataColumnSidecarsBytes = dataColumnSidecarsBytesWrapped.slice(
+      DATA_COLUMN_SIDECARS_IN_WRAPPER_INDEX + 4 * retrivedColumnsLen
+    );
 
-      // const storedColumns = Array.from({length: NUMBER_OF_COLUMNS}, (_v, i) => i).filter(
-      //   (i) => dataColumnsIndex[i] > 0
-      // );
-      // const columnsLen = dataColumnSidecarsBytes.length / columnsSize;
-      // console.log(
-      //   `onDataColumnSidecarsByRoot: slot=${block.slot} columnsSize=${columnsSize} storedColumnsLen=${columnsLen} retrivedColumnsLen=${retrivedColumnsLen} dataColumnSidecarsBytesWrapped=${dataColumnSidecarsBytesWrapped.length} storedColumns=${storedColumns.join(" ")}`
-      // );
+    const dataColumnsIndex = dataColumnSidecarsBytesWrapped.slice(
+      CUSTODY_COLUMNS_IN_IN_WRAPPER_INDEX,
+      CUSTODY_COLUMNS_IN_IN_WRAPPER_INDEX + NUMBER_OF_COLUMNS
+    );
 
-      lastFetchedSideCars = {blockRoot: blockRootHex, bytes: dataColumnSidecarsBytes, columnsSize, dataColumnsIndex};
-    }
+    // const storedColumns = Array.from({length: NUMBER_OF_COLUMNS}, (_v, i) => i).filter(
+    //   (i) => dataColumnsIndex[i] > 0
+    // );
+    // const columnsLen = dataColumnSidecarsBytes.length / columnsSize;
+    // console.log(
+    //   `onDataColumnSidecarsByRoot: slot=${block.slot} columnsSize=${columnsSize} storedColumnsLen=${columnsLen} retrivedColumnsLen=${retrivedColumnsLen} dataColumnSidecarsBytesWrapped=${dataColumnSidecarsBytesWrapped.length} storedColumns=${storedColumns.join(" ")}`
+    // );
 
-    const dataIndex = (lastFetchedSideCars.dataColumnsIndex[index] ?? 0) - 1;
-    if (dataIndex < 0) {
-      throw new ResponseError(RespStatus.SERVER_ERROR, `dataColumnSidecar index=${index} not custodied`);
-    }
-    const {columnsSize} = lastFetchedSideCars;
+    for (const index of columns) {
+      const dataIndex = (dataColumnsIndex[index] ?? 0) - 1;
+      if (dataIndex < 0) {
+        throw new ResponseError(RespStatus.SERVER_ERROR, `dataColumnSidecar index=${index} not custodied`);
+      }
 
-    const dataColumnSidecarBytes = lastFetchedSideCars.bytes.slice(
-      dataIndex * columnsSize,
-      (dataIndex + 1) * columnsSize
-    );
-    if (dataColumnSidecarBytes.length !== columnsSize) {
-      throw Error(
-        `Inconsistent state, dataColumnSidecar blockRoot=${blockRootHex} index=${index} dataColumnSidecarBytes=${dataColumnSidecarBytes.length} expected=${columnsSize}`
+      const dataColumnSidecarBytes = dataColumnSidecarsBytes.slice(
+        dataIndex * columnsSize,
+        (dataIndex + 1) * columnsSize
       );
-    }
+      if (dataColumnSidecarBytes.length !== columnsSize) {
+        throw Error(
+          `Inconsistent state, dataColumnSidecar blockRoot=${blockRootHex} index=${index} dataColumnSidecarBytes=${dataColumnSidecarBytes.length} expected=${columnsSize}`
+        );
+      }
 
-    yield {
-      data: dataColumnSidecarBytes,
-      fork: chain.config.getForkName(block.slot),
-    };
+      yield {
+        data: dataColumnSidecarBytes,
+        fork: chain.config.getForkName(block.slot),
+      };
+    }
   }
 }

packages/types/src/fulu/sszTypes.ts

@@ -5,6 +5,7 @@ import {
   FIELD_ELEMENTS_PER_EXT_BLOB,
   KZG_COMMITMENTS_INCLUSION_PROOF_DEPTH,
   MAX_BLOB_COMMITMENTS_PER_BLOCK,
+  MAX_REQUEST_BLOCKS_DENEB,
   MAX_REQUEST_DATA_COLUMN_SIDECARS,
   NUMBER_OF_COLUMNS,
 } from "@lodestar/params";
@@ -64,17 +65,17 @@ export const MatrixEntry = new ContainerType(
 // ReqResp types
 // =============
 
-export const DataColumnIdentifier = new ContainerType(
+export const DataColumnsByRootIdentifier = new ContainerType(
   {
     blockRoot: Root,
-    index: ColumnIndex,
+    columns: new ListBasicType(ColumnIndex, NUMBER_OF_COLUMNS),
   },
-  {typeName: "DataColumnIdentifier", jsonCase: "eth2"}
+  {typeName: "DataColumnsByRootIdentifier", jsonCase: "eth2"}
 );
 
 export const DataColumnSidecarsByRootRequest = new ListCompositeType(
-  DataColumnIdentifier,
-  MAX_REQUEST_DATA_COLUMN_SIDECARS
+  DataColumnsByRootIdentifier,
+  MAX_REQUEST_BLOCKS_DENEB
 );
 
 export const DataColumnSidecarsByRangeRequest = new ContainerType(

packages/types/src/fulu/types.ts

@@ -14,7 +14,7 @@ export type DataColumnSidecar = ValueOf<typeof ssz.DataColumnSidecar>;
 export type DataColumnSidecars = ValueOf<typeof ssz.DataColumnSidecars>;
 export type MatrixEntry = ValueOf<typeof ssz.MatrixEntry>;
 
-export type DataColumnIdentifier = ValueOf<typeof ssz.DataColumnIdentifier>;
+export type DataColumnsByRootIdentifier = ValueOf<typeof ssz.DataColumnsByRootIdentifier>;
 export type DataColumnSidecarsByRootRequest = ValueOf<typeof ssz.DataColumnSidecarsByRootRequest>;
 export type DataColumnSidecarsByRangeRequest = ValueOf<typeof ssz.DataColumnSidecarsByRangeRequest>;