fix: deserialize DataColumnSidecar without cloning Uint8Array

packages/beacon-node/src/chain/errors/dataColumnSidecarError.ts

@@ -36,6 +36,7 @@ export enum DataColumnSidecarErrorCode {
   PROPOSAL_SIGNATURE_INVALID = "DATA_COLUMN_SIDECAR_ERROR_PROPOSAL_SIGNATURE_INVALID",
   INCLUSION_PROOF_INVALID = "DATA_COLUMN_SIDECAR_ERROR_INCLUSION_PROOF_INVALID",
   INCORRECT_PROPOSER = "DATA_COLUMN_SIDECAR_ERROR_INCORRECT_PROPOSER",
+  SSZ_DESERIALIZATION_FAILED = "DATA_COLUMN_SIDECAR_ERROR_SSZ_DESERIALIZATION_FAILED",
 }
 
 export type DataColumnSidecarErrorType =
@@ -97,7 +98,8 @@ export type DataColumnSidecarErrorType =
       actual: number;
     }
   | {code: DataColumnSidecarErrorCode.INVALID_KZG_PROOF_BATCH; slot: number; reason: string}
-  | {code: DataColumnSidecarErrorCode.INCORRECT_PROPOSER; actualProposerIndex: number; expectedProposerIndex: number};
+  | {code: DataColumnSidecarErrorCode.INCORRECT_PROPOSER; actualProposerIndex: number; expectedProposerIndex: number}
+  | {code: DataColumnSidecarErrorCode.SSZ_DESERIALIZATION_FAILED; length: number};
 
 export class DataColumnSidecarGossipError extends GossipActionError<DataColumnSidecarErrorType> {}
 export class DataColumnSidecarValidationError extends LodestarError<DataColumnSidecarErrorType> {}

packages/beacon-node/src/network/processor/gossipHandlers.ts

@@ -67,6 +67,7 @@ import {validateLightClientOptimisticUpdate} from "../../chain/validation/lightC
 import {OpSource} from "../../chain/validatorMonitor.js";
 import {Metrics} from "../../metrics/index.js";
 import {kzgCommitmentToVersionedHash} from "../../util/blobs.js";
+import {deserializeDataColumnSidecarUnsafe} from "../../util/sszBytes.js";
 import {INetworkCore} from "../core/index.js";
 import {NetworkEventBus} from "../events.js";
 import {
@@ -547,7 +548,16 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
       seenTimestampSec,
     }: GossipHandlerParamGeneric<GossipType.data_column_sidecar>) => {
       const {serializedData} = gossipData;
-      const dataColumnSidecar = sszDeserialize(topic, serializedData);
+      // as we have cloned the ssz bytes getting thru worker boundary, it's safe to use unsafe deserialization here to avoid extra copy
+      const dataColumnSidecar = deserializeDataColumnSidecarUnsafe(serializedData);
+      if (dataColumnSidecar === null) {
+        // should not happen, if yes could be our bug in deserializeDataColumnSidecarUnsafe() so track this for investigation
+        throw new GossipActionError(GossipAction.IGNORE, {
+          code: DataColumnSidecarErrorCode.SSZ_DESERIALIZATION_FAILED,
+          length: serializedData.length,
+        });
+      }
+
       const dataColumnSlot = dataColumnSidecar.signedBlockHeader.message.slot;
       const index = dataColumnSidecar.index;
 

packages/beacon-node/src/network/reqresp/utils/collect.ts

@@ -1,6 +1,7 @@
 import {Type} from "@chainsafe/ssz";
 import {RequestError, RequestErrorCode, ResponseIncoming} from "@lodestar/reqresp";
-import {WithBytes} from "@lodestar/types";
+import {WithBytes, ssz} from "@lodestar/types";
+import {deserializeDataColumnSidecarUnsafe} from "../../../util/sszBytes.js";
 import {ResponseTypeGetter} from "../types.js";
 
 /**
@@ -77,6 +78,15 @@ export async function collectMaxResponseTypedWithBytes<T>(
 /** Light wrapper on type to wrap deserialize errors */
 export function sszDeserializeResponse<T>(type: Type<T>, bytes: Uint8Array): T {
   try {
+    if (type === ssz.fulu.DataColumnSidecar) {
+      // as we have cloned the ssz bytes getting thru worker boundary, it's safe to use unsafe deserialization here to avoid extra copy
+      const dataColumnSidecar = deserializeDataColumnSidecarUnsafe(bytes);
+      if (dataColumnSidecar === null) {
+        throw new Error("Failed to deserialize DataColumnSidecar");
+      }
+      return dataColumnSidecar as unknown as T;
+    }
+
     return type.deserialize(bytes);
   } catch (e) {
     throw new RequestError({code: RequestErrorCode.INVALID_RESPONSE_SSZ, errorMessage: (e as Error).message});

packages/beacon-node/src/util/sszBytes.ts

@@ -6,10 +6,12 @@ import {
   ForkName,
   ForkPostDeneb,
   ForkSeq,
+  KZG_COMMITMENTS_INCLUSION_PROOF_DEPTH,
   MAX_COMMITTEES_PER_SLOT,
   isForkPostElectra,
 } from "@lodestar/params";
-import {BLSSignature, CommitteeIndex, RootHex, Slot, ValidatorIndex, ssz} from "@lodestar/types";
+import {BLSSignature, CommitteeIndex, RootHex, Slot, ValidatorIndex, phase0, ssz} from "@lodestar/types";
+import {DataColumnSidecar} from "@lodestar/types/fulu";
 
 export type BlockRootHex = RootHex;
 // pre-electra, AttestationData is used to cache attestations
@@ -417,6 +419,150 @@ export function getSlotFromDataColumnSidecarSerialized(data: Uint8Array): Slot |
   return getSlotFromOffset(data, SLOT_BYTES_POSITION_IN_SIGNED_DATA_COLUMN_SIDECAR);
 }
 
+/**
+ * Deserialize DataColumnSidecar using the backed array itself instead of copying (which the ssz lib does)
+ * This method is unsafe if the input data is shared and modified later
+ */
+export function deserializeDataColumnSidecarUnsafe(data: Uint8Array): DataColumnSidecar | null {
+  let offset = 0;
+  const index = getIndexFromOffset(data, 0);
+  if (index === null) return null;
+  // index field is 8 bytes
+  offset += ssz.ColumnIndex.fixedSize;
+  const columnStartOffset = getUint32(data, offset);
+  // column field is not fixed size
+  offset += VARIABLE_FIELD_OFFSET;
+  const kzgCommitmentsStartOffset = getUint32(data, offset);
+  // kzgCommitments field is not fixed size
+  offset += VARIABLE_FIELD_OFFSET;
+  const kzgProofsStartOffset = getUint32(data, offset);
+  // kzgProofs field is not fixed size
+  offset += VARIABLE_FIELD_OFFSET;
+  const signedBlockHeader = deserializeSignedBlockHeaderUnsafe(
+    Uint8Array.prototype.subarray.call(data, offset, offset + SIGNED_BLOCK_HEADER_SIZE)
+  );
+  if (signedBlockHeader === null) return null;
+  // signedBlockHeader field is fixed size 208 bytes
+  offset += SIGNED_BLOCK_HEADER_SIZE;
+  const inclusionProofSize = ssz.fulu.KzgCommitmentsInclusionProof.fixedSize;
+  // this should not happen, just want to make the compiler happy
+  if (inclusionProofSize === null) return null;
+  const kzgCommitmentsInclusionProofData = Uint8Array.prototype.subarray.call(
+    data,
+    offset,
+    offset + inclusionProofSize
+  );
+  const kzgCommitmentsInclusionProof = new Array<Uint8Array>(KZG_COMMITMENTS_INCLUSION_PROOF_DEPTH);
+  for (let i = 0; i < KZG_COMMITMENTS_INCLUSION_PROOF_DEPTH; i++) {
+    kzgCommitmentsInclusionProof[i] = Uint8Array.prototype.subarray.call(
+      kzgCommitmentsInclusionProofData,
+      i * BYTES_PER_FIELD_ELEMENT,
+      (i + 1) * BYTES_PER_FIELD_ELEMENT
+    );
+  }
+
+  // deserialize for dynamic fields
+  const columnData = Uint8Array.prototype.subarray.call(data, columnStartOffset, kzgCommitmentsStartOffset);
+  if (columnData.length % ssz.fulu.Cell.fixedSize !== 0) return null;
+  const numCells = Math.floor(columnData.length / ssz.fulu.Cell.fixedSize);
+  const column = new Array<Uint8Array>(numCells);
+  for (let i = 0; i < numCells; i++) {
+    column[i] = Uint8Array.prototype.subarray.call(
+      columnData,
+      i * ssz.fulu.Cell.fixedSize,
+      (i + 1) * ssz.fulu.Cell.fixedSize
+    );
+  }
+
+  const kzgCommitmentsData = Uint8Array.prototype.subarray.call(data, kzgCommitmentsStartOffset, kzgProofsStartOffset);
+  if (kzgCommitmentsData.length % ssz.deneb.KZGCommitment.fixedSize !== 0) return null;
+  const numKzgCommitments = Math.floor(kzgCommitmentsData.length / ssz.deneb.KZGCommitment.fixedSize);
+  const kzgCommitments = new Array<Uint8Array>(numKzgCommitments);
+  for (let i = 0; i < numKzgCommitments; i++) {
+    kzgCommitments[i] = Uint8Array.prototype.subarray.call(
+      kzgCommitmentsData,
+      i * ssz.deneb.KZGCommitment.fixedSize,
+      (i + 1) * ssz.deneb.KZGCommitment.fixedSize
+    );
+  }
+
+  const kzgProofsData = Uint8Array.prototype.subarray.call(
+    data,
+    kzgProofsStartOffset,
+    // this is the last dynamic field
+    data.length
+  );
+  if (kzgProofsData.length % ssz.deneb.KZGProof.fixedSize !== 0) return null;
+  const numKzgProofs = Math.floor(kzgProofsData.length / ssz.deneb.KZGProof.fixedSize);
+  const kzgProofs = new Array<Uint8Array>(numKzgProofs);
+  for (let i = 0; i < numKzgProofs; i++) {
+    kzgProofs[i] = Uint8Array.prototype.subarray.call(
+      kzgProofsData,
+      i * ssz.deneb.KZGProof.fixedSize,
+      (i + 1) * ssz.deneb.KZGProof.fixedSize
+    );
+  }
+
+  return {
+    index,
+    column,
+    kzgCommitments,
+    kzgProofs,
+    signedBlockHeader,
+    kzgCommitmentsInclusionProof,
+  };
+}
+
+/** SignedBeaconBlockHeader is 208 bytes fixed size
+ *    message: BeaconBlockHeader - 112 bytes
+ *      slot: Slot - 8 bytes
+ *      proposer_index: ValidatorIndex - 8 bytes
+ *      parent_root: Root - 32 bytes
+ *      state_root: Root - 32 bytes
+ *      body_root: Root - 32 bytes
+ *    signature: BLSSignature - 96 bytes
+ */
+const SIGNED_BLOCK_HEADER_SIZE = 208;
+
+/**
+ * Deserialize SignedBeaconBlockHeader using the backed array itself instead of copying (which the ssz lib does)
+ * This method is unsafe if the input data is shared and modified later
+ */
+export function deserializeSignedBlockHeaderUnsafe(data: Uint8Array): phase0.SignedBeaconBlockHeader | null {
+  if (data.length !== SIGNED_BLOCK_HEADER_SIZE) return null;
+
+  let offset = 0;
+  const slot = getSlotFromOffset(data, offset);
+  if (slot === null) return null;
+  // slot is 8 bytes
+  offset += ssz.Slot.fixedSize;
+  const proposerIndex = getIndexFromOffset(data, offset);
+  if (proposerIndex === null) return null;
+  // proposerIndex is 8 bytes
+  offset += 8;
+  const parentRoot = Uint8Array.prototype.subarray.call(data, offset, offset + ssz.Root.fixedSize);
+  // parentRoot is 32 bytes
+  offset += ssz.Root.fixedSize;
+  const stateRoot = Uint8Array.prototype.subarray.call(data, offset, offset + ssz.Root.fixedSize);
+  // stateRoot is 32 bytes
+  offset += ssz.Root.fixedSize;
+  const bodyRoot = Uint8Array.prototype.subarray.call(data, offset, offset + ssz.Root.fixedSize);
+  // bodyRoot is 32 bytes
+  offset += ssz.Root.fixedSize;
+  // signature is 96 bytes
+  const signature = Uint8Array.prototype.subarray.call(data, offset, offset + ssz.BLSSignature.fixedSize);
+  return {
+    message: {
+      slot,
+      proposerIndex,
+      parentRoot,
+      stateRoot,
+      bodyRoot,
+    },
+    signature,
+  };
+}
+
 /**
  * BeaconState of all forks (up until Electra, check with new forks)
  * class BeaconState(Container):
@@ -453,7 +599,7 @@ export function getSlotFromBeaconStateSerialized(data: Uint8Array): Slot | null
  * If the high bytes are not zero, return null
  */
 function getSlotFromOffset(data: Uint8Array, offset: number): Slot | null {
-  return checkSlotHighBytes(data, offset) ? getSlotFromOffsetTrusted(data, offset) : null;
+  return checkSlotHighBytes(data, offset) ? getUint32(data, offset) : null;
 }
 
 /**
@@ -466,7 +612,7 @@ function getIndexFromOffset(data: Uint8Array, offset: number): (ValidatorIndex |
 /**
  * Read only the first 4 bytes of Slot, max value is 4,294,967,295 will be reached 1634 years after genesis
  */
-function getSlotFromOffsetTrusted(data: Uint8Array, offset: number): Slot {
+function getUint32(data: Uint8Array, offset: number): Slot {
   return (data[offset] | (data[offset + 1] << 8) | (data[offset + 2] << 16) | (data[offset + 3] << 24)) >>> 0;
 }
 

packages/beacon-node/test/unit/util/sszBytes.test.ts

@@ -20,6 +20,8 @@ import {
 import {fromHex, toHex, toRootHex} from "@lodestar/utils";
 import {kzg} from "../../../src/util/kzg.js";
 import {
+  deserializeDataColumnSidecarUnsafe,
+  deserializeSignedBlockHeaderUnsafe,
   getAggregationBitsFromAttestationSerialized,
   getAttDataFromAttestationSerialized,
   getAttDataFromSignedAggregateAndProofElectra,
@@ -406,6 +408,64 @@ describe("BeaconState ssz serialized picking", () => {
   });
 });
 
+describe("SignedBeaconBlockHeader deserialization", () => {
+  const slot = 1_000_000;
+  const header = ssz.phase0.SignedBeaconBlockHeader.defaultValue();
+  header.message.slot = slot;
+  const bytes = ssz.phase0.SignedBeaconBlockHeader.serialize(header);
+
+  it("should deserialize SignedBeaconBlockHeader correctly", () => {
+    const deserialized = ssz.phase0.SignedBeaconBlockHeader.deserialize(bytes);
+    expect(deserialized.message.slot).toBe(slot);
+    const deserialized2 = deserializeSignedBlockHeaderUnsafe(bytes);
+    if (deserialized2 === null) {
+      throw new Error("cannot deserialize SignedBeaconBlockHeader");
+    }
+    expect(deserialized2.message.slot).toBe(slot);
+    expect(ssz.phase0.SignedBeaconBlockHeader.equals(deserialized, deserialized2)).toBe(true);
+  });
+
+  it("return null if invalid data", () => {
+    const invalidDataSizes = [0, 100, 200, 300];
+    for (const size of invalidDataSizes) {
+      expect(deserializeSignedBlockHeaderUnsafe(Buffer.alloc(size))).toBeNull();
+    }
+  });
+});
+
+describe("DataColumnSidecar deserialization", () => {
+  const slot = 1_000_000;
+  const sidecar = ssz.fulu.DataColumnSidecar.defaultValue();
+  sidecar.signedBlockHeader.message.slot = slot;
+  for (const numBlobs of [0, 1, 21, 42]) {
+    for (let i = 0; i < numBlobs; i++) {
+      sidecar.kzgCommitments.push(Buffer.alloc(48, i));
+      sidecar.kzgProofs.push(Buffer.alloc(48, i));
+    }
+    sidecar.kzgCommitmentsInclusionProof[0][0] = 100;
+    const bytes = ssz.fulu.DataColumnSidecar.serialize(sidecar);
+
+    it(`should deserialize DataColumnSidecar correctly blobs=${numBlobs}`, () => {
+      const deserialized = ssz.fulu.DataColumnSidecar.deserialize(bytes);
+      expect(deserialized.signedBlockHeader.message.slot).toBe(slot);
+      const deserialized2 = deserializeDataColumnSidecarUnsafe(bytes);
+      if (deserialized2 === null) {
+        throw new Error("cannot deserialize DataColumnSidecar");
+      }
+      expect(deserialized2.signedBlockHeader.message.slot).toBe(slot);
+      expect(deserialized2.kzgCommitmentsInclusionProof[0][0]).toBe(100);
+      expect(ssz.fulu.DataColumnSidecar.equals(deserialized, deserialized2)).toBe(true);
+    });
+  }
+
+  it("return null if invalid data", () => {
+    const invalidDataSizes = [0, 100, 200, 300];
+    for (const size of invalidDataSizes) {
+      expect(deserializeDataColumnSidecarUnsafe(Buffer.alloc(size))).toBeNull();
+    }
+  });
+});
+
 function phase0SingleAttestationFromValues(
   slot: Slot,
   blockRoot: RootHex,