feat: glamsterdam-devnet-7

.wordlist.txt

@@ -169,6 +169,7 @@ flamegraphs
 floodsub
 fsSL
 getNetworkIdentity
+gloas
 gnosis
 gpg
 heapdump

package.json

@@ -44,6 +44,7 @@
     "@chainsafe/benchmark": "^2.0.2",
     "@chainsafe/biomejs-config": "^1.0.0",
     "@chainsafe/caxa": "3.0.6",
+    "@chainsafe/ssz": "^1.6.2",
     "@lerna-lite/cli": "^4.9.4",
     "@lerna-lite/exec": "^4.9.4",
     "@lerna-lite/publish": "^4.9.4",

packages/api/package.json

@@ -74,8 +74,8 @@
     "check-readme": "pnpm exec ts-node ../../scripts/check_readme.ts"
   },
   "dependencies": {
-    "@chainsafe/persistent-merkle-tree": "^1.2.5",
-    "@chainsafe/ssz": "^1.4.0",
+    "@chainsafe/persistent-merkle-tree": "^1.3.0",
+    "@chainsafe/ssz": "^1.6.2",
     "@lodestar/config": "workspace:^",
     "@lodestar/params": "workspace:^",
     "@lodestar/types": "workspace:^",

packages/api/src/beacon/routes/beacon/block.ts

@@ -17,6 +17,7 @@ import {
   SignedBlockContents,
   Slot,
   deneb,
+  fulu,
   gloas,
   ssz,
   sszTypesFor,
@@ -190,8 +191,13 @@ export type Endpoints = {
    */
   publishExecutionPayloadEnvelope: Endpoint<
     "POST",
-    {signedExecutionPayloadEnvelope: gloas.SignedExecutionPayloadEnvelope},
-    {body: unknown; headers: {[MetaHeader.Version]: string}},
+    {
+      signedExecutionPayloadEnvelope: gloas.SignedExecutionPayloadEnvelope;
+      blobs?: deneb.Blobs;
+      kzgProofs?: fulu.KZGProofs;
+      broadcastValidation?: BroadcastValidation;
+    },
+    {body: unknown; headers: {[MetaHeader.Version]: string}; query: {broadcast_validation?: string}},
     EmptyResponseData,
     EmptyMeta
   >;
@@ -453,39 +459,98 @@ export function getDefinitions(config: ChainForkConfig): RouteDefinitions<Endpoi
       url: "/eth/v1/beacon/execution_payload_envelopes",
       method: "POST",
       req: {
-        writeReqJson: ({signedExecutionPayloadEnvelope}) => {
+        writeReqJson: ({signedExecutionPayloadEnvelope, blobs, kzgProofs, broadcastValidation}) => {
           const fork = config.getForkName(signedExecutionPayloadEnvelope.message.payload.slotNumber);
+          const types = getPostGloasForkTypes(fork);
+          if ((blobs === undefined) !== (kzgProofs === undefined)) {
+            throw Error("blobs and kzgProofs must both be supplied or both omitted");
+          }
+          const hasBlobs = blobs !== undefined && kzgProofs !== undefined;
+          const body = hasBlobs
+            ? types.SignedExecutionPayloadEnvelopeContents.toJson({
+                signedExecutionPayloadEnvelope,
+                kzgProofs,
+                blobs,
+              })
+            : types.SignedExecutionPayloadEnvelope.toJson(signedExecutionPayloadEnvelope);
           return {
-            body: getPostGloasForkTypes(fork).SignedExecutionPayloadEnvelope.toJson(signedExecutionPayloadEnvelope),
+            body,
             headers: {
               [MetaHeader.Version]: fork,
             },
+            query: {broadcast_validation: broadcastValidation},
           };
         },
-        parseReqJson: ({body, headers}) => {
+        parseReqJson: ({body, headers, query}) => {
           const fork = toForkName(fromHeaders(headers, MetaHeader.Version));
+          const types = getPostGloasForkTypes(fork);
+          // Discriminate by the wrapper's top-level key. SignedExecutionPayloadEnvelopeContents
+          // serializes to {signed_execution_payload_envelope, kzg_proofs, blobs}; the bare envelope
+          // serializes to {message, signature}.
+          const isContents = body !== null && typeof body === "object" && "signed_execution_payload_envelope" in body;
+          if (isContents) {
+            const contents = types.SignedExecutionPayloadEnvelopeContents.fromJson(body);
+            return {
+              signedExecutionPayloadEnvelope: contents.signedExecutionPayloadEnvelope,
+              blobs: contents.blobs,
+              kzgProofs: contents.kzgProofs,
+              broadcastValidation: query.broadcast_validation as BroadcastValidation,
+            };
+          }
           return {
-            signedExecutionPayloadEnvelope: getPostGloasForkTypes(fork).SignedExecutionPayloadEnvelope.fromJson(body),
+            signedExecutionPayloadEnvelope: types.SignedExecutionPayloadEnvelope.fromJson(body),
+            broadcastValidation: query.broadcast_validation as BroadcastValidation,
           };
         },
-        writeReqSsz: ({signedExecutionPayloadEnvelope}) => {
+        writeReqSsz: ({signedExecutionPayloadEnvelope, blobs, kzgProofs, broadcastValidation}) => {
           const fork = config.getForkName(signedExecutionPayloadEnvelope.message.payload.slotNumber);
+          const types = getPostGloasForkTypes(fork);
+          if ((blobs === undefined) !== (kzgProofs === undefined)) {
+            throw Error("blobs and kzgProofs must both be supplied or both omitted");
+          }
+          const hasBlobs = blobs !== undefined && kzgProofs !== undefined;
+          const body = hasBlobs
+            ? types.SignedExecutionPayloadEnvelopeContents.serialize({
+                signedExecutionPayloadEnvelope,
+                kzgProofs,
+                blobs,
+              })
+            : types.SignedExecutionPayloadEnvelope.serialize(signedExecutionPayloadEnvelope);
           return {
-            body: getPostGloasForkTypes(fork).SignedExecutionPayloadEnvelope.serialize(signedExecutionPayloadEnvelope),
+            body,
             headers: {
               [MetaHeader.Version]: fork,
             },
+            query: {broadcast_validation: broadcastValidation},
           };
         },
-        parseReqSsz: ({body, headers}) => {
+        parseReqSsz: ({body, headers, query}) => {
           const fork = toForkName(fromHeaders(headers, MetaHeader.Version));
+          const types = getPostGloasForkTypes(fork);
+          // SSZ has no in-band Contents-vs-envelope discriminator; spec uses a single
+          // `application/octet-stream` Content-Type for both shapes. Discriminate by the
+          // first SSZ offset: SignedExecutionPayloadEnvelopeContents has 3 variable fields
+          // so its first offset is 12; SignedExecutionPayloadEnvelope's variable `message`
+          // sits after a 96-byte signature plus the 4-byte offset header, so its first
+          // offset is 100.
+          const firstOffset = body.length >= 4 ? body[0] | (body[1] << 8) | (body[2] << 16) | (body[3] << 24) : 0;
+          if (firstOffset === 12) {
+            const contents = types.SignedExecutionPayloadEnvelopeContents.deserialize(body);
+            return {
+              signedExecutionPayloadEnvelope: contents.signedExecutionPayloadEnvelope,
+              blobs: contents.blobs,
+              kzgProofs: contents.kzgProofs,
+              broadcastValidation: query.broadcast_validation as BroadcastValidation,
+            };
+          }
           return {
-            signedExecutionPayloadEnvelope:
-              getPostGloasForkTypes(fork).SignedExecutionPayloadEnvelope.deserialize(body),
+            signedExecutionPayloadEnvelope: types.SignedExecutionPayloadEnvelope.deserialize(body),
+            broadcastValidation: query.broadcast_validation as BroadcastValidation,
           };
         },
         schema: {
           body: Schema.Object,
+          query: {broadcast_validation: Schema.String},
           headers: {[MetaHeader.Version]: Schema.String},
         },
       },

packages/api/test/unit/beacon/publishExecutionPayloadEnvelope.test.ts

@@ -0,0 +1,123 @@
+import {describe, expect, it} from "vitest";
+import {createChainForkConfig, defaultChainConfig} from "@lodestar/config";
+import {ssz} from "@lodestar/types";
+import {BroadcastValidation, getDefinitions} from "../../../src/beacon/routes/beacon/block.js";
+import {WireFormat} from "../../../src/utils/wireFormat.js";
+
+function lowercaseKeys<T extends Record<string, string>>(headers: T): T {
+  return Object.fromEntries(Object.entries(headers).map(([k, v]) => [k.toLowerCase(), v])) as T;
+}
+
+describe("publishExecutionPayloadEnvelope route", () => {
+  const config = createChainForkConfig({...defaultChainConfig, GLOAS_FORK_EPOCH: 0});
+  const definitions = getDefinitions(config);
+  const route = definitions.publishExecutionPayloadEnvelope;
+
+  const signedExecutionPayloadEnvelope = ssz.gloas.SignedExecutionPayloadEnvelope.defaultValue();
+
+  describe("JSON wire format", () => {
+    it("round-trips a bare SignedExecutionPayloadEnvelope", () => {
+      const written = route.req.writeReqJson({signedExecutionPayloadEnvelope});
+      expect(written.body).toHaveProperty("message");
+      expect(written.body).toHaveProperty("signature");
+      expect(written.body).not.toHaveProperty("signed_execution_payload_envelope");
+
+      const parsed = route.req.parseReqJson({
+        body: written.body,
+        headers: lowercaseKeys(written.headers),
+        query: written.query ?? {},
+      });
+      expect(parsed.signedExecutionPayloadEnvelope).toEqual(signedExecutionPayloadEnvelope);
+      expect(parsed.blobs).toBeUndefined();
+      expect(parsed.kzgProofs).toBeUndefined();
+    });
+
+    it("round-trips a SignedExecutionPayloadEnvelopeContents wrapper when blobs are supplied", () => {
+      const blobs = [ssz.deneb.Blob.defaultValue()];
+      const kzgProofs = Array.from({length: 128}, () => ssz.deneb.KZGProof.defaultValue());
+
+      const written = route.req.writeReqJson({
+        signedExecutionPayloadEnvelope,
+        blobs,
+        kzgProofs,
+        broadcastValidation: BroadcastValidation.consensus,
+      });
+      expect(written.body).toHaveProperty("signed_execution_payload_envelope");
+      expect(written.body).toHaveProperty("kzg_proofs");
+      expect(written.body).toHaveProperty("blobs");
+      expect(written.query?.broadcast_validation).toBe(BroadcastValidation.consensus);
+
+      const parsed = route.req.parseReqJson({
+        body: written.body,
+        headers: lowercaseKeys(written.headers),
+        query: written.query ?? {},
+      });
+      expect(parsed.signedExecutionPayloadEnvelope).toEqual(signedExecutionPayloadEnvelope);
+      expect(parsed.blobs).toEqual(blobs);
+      expect(parsed.kzgProofs).toEqual(kzgProofs);
+      expect(parsed.broadcastValidation).toBe(BroadcastValidation.consensus);
+    });
+  });
+
+  describe("SSZ wire format", () => {
+    it("round-trips a bare SignedExecutionPayloadEnvelope", () => {
+      const written = route.req.writeReqSsz({signedExecutionPayloadEnvelope});
+      const parsed = route.req.parseReqSsz({
+        body: written.body,
+        headers: lowercaseKeys(written.headers),
+        query: written.query ?? {},
+      });
+      expect(parsed.signedExecutionPayloadEnvelope).toEqual(signedExecutionPayloadEnvelope);
+      expect(parsed.blobs).toBeUndefined();
+      expect(parsed.kzgProofs).toBeUndefined();
+    });
+
+    it("round-trips a SignedExecutionPayloadEnvelopeContents wrapper", () => {
+      const blobs = [ssz.deneb.Blob.defaultValue()];
+      const kzgProofs = Array.from({length: 128}, () => ssz.deneb.KZGProof.defaultValue());
+
+      const written = route.req.writeReqSsz({signedExecutionPayloadEnvelope, blobs, kzgProofs});
+      const parsed = route.req.parseReqSsz({
+        body: written.body,
+        headers: lowercaseKeys(written.headers),
+        query: written.query ?? {},
+      });
+      expect(parsed.signedExecutionPayloadEnvelope).toEqual(signedExecutionPayloadEnvelope);
+      expect(parsed.blobs).toEqual(blobs);
+      expect(parsed.kzgProofs).toEqual(kzgProofs);
+    });
+  });
+
+  it("uses SSZ as the default request wire format", () => {
+    expect(route.init?.requestWireFormat).toBe(WireFormat.ssz);
+  });
+
+  describe("partial blobs/kzgProofs rejection", () => {
+    const blobs = [ssz.deneb.Blob.defaultValue()];
+    const kzgProofs = Array.from({length: 128}, () => ssz.deneb.KZGProof.defaultValue());
+
+    it("writeReqJson throws when only blobs are supplied", () => {
+      expect(() => route.req.writeReqJson({signedExecutionPayloadEnvelope, blobs})).toThrow(
+        /blobs and kzgProofs must both be supplied/
+      );
+    });
+
+    it("writeReqJson throws when only kzgProofs are supplied", () => {
+      expect(() => route.req.writeReqJson({signedExecutionPayloadEnvelope, kzgProofs})).toThrow(
+        /blobs and kzgProofs must both be supplied/
+      );
+    });
+
+    it("writeReqSsz throws when only blobs are supplied", () => {
+      expect(() => route.req.writeReqSsz({signedExecutionPayloadEnvelope, blobs})).toThrow(
+        /blobs and kzgProofs must both be supplied/
+      );
+    });
+
+    it("writeReqSsz throws when only kzgProofs are supplied", () => {
+      expect(() => route.req.writeReqSsz({signedExecutionPayloadEnvelope, kzgProofs})).toThrow(
+        /blobs and kzgProofs must both be supplied/
+      );
+    });
+  });
+});

packages/beacon-node/package.json

@@ -114,10 +114,10 @@
     "@chainsafe/enr": "^6.0.1",
     "@chainsafe/libp2p-noise": "^17.0.0",
     "@chainsafe/libp2p-quic": "^2.0.1",
-    "@chainsafe/persistent-merkle-tree": "^1.2.5",
+    "@chainsafe/persistent-merkle-tree": "^1.3.0",
     "@chainsafe/prometheus-gc-stats": "^1.0.0",
     "@chainsafe/snappy-wasm": "^0.5.0",
-    "@chainsafe/ssz": "^1.4.0",
+    "@chainsafe/ssz": "^1.6.2",
     "@chainsafe/threads": "^1.11.3",
     "@crate-crypto/node-eth-kzg": "0.9.1",
     "@fastify/bearer-auth": "^10.0.1",

packages/beacon-node/src/api/impl/beacon/blocks/index.ts

@@ -48,6 +48,7 @@ import {
   ProduceFullFulu,
   ProduceFullGloas,
 } from "../../../../chain/produceBlock/index.js";
+import {validateCellsAndKzgCommitments} from "../../../../chain/produceBlock/validateBlobsAndKzgCommitments.js";
 import {validateGossipBlock} from "../../../../chain/validation/block.js";
 import {validateApiExecutionPayloadBid} from "../../../../chain/validation/executionPayloadBid.js";
 import {validateApiExecutionPayloadEnvelope} from "../../../../chain/validation/executionPayloadEnvelope.js";
@@ -651,7 +652,9 @@ export function getBeaconBlockApi({
     publishBlockV2,
     publishBlindedBlockV2,
 
-    async publishExecutionPayloadEnvelope({signedExecutionPayloadEnvelope}) {
+    async publishExecutionPayloadEnvelope({signedExecutionPayloadEnvelope, blobs, kzgProofs, broadcastValidation}) {
+      // TODO GLOAS: honor broadcastValidation (gossip|consensus|consensus_and_equivocation) per beacon-APIs PR #580
+      void broadcastValidation;
       const seenTimestampSec = Date.now() / 1000;
       const envelope = signedExecutionPayloadEnvelope.message;
       const slot = envelope.payload.slotNumber;
@@ -685,11 +688,52 @@ export function getBeaconBlockApi({
 
       await validateApiExecutionPayloadEnvelope(chain, signedExecutionPayloadEnvelope);
 
+      const payloadInput = chain.seenPayloadEnvelopeInputCache.get(blockRootHex);
+      if (!payloadInput) {
+        // The block is awaited above (queuing if the envelope arrived first), and both the API and
+        // gossip import paths seed the PayloadEnvelopeInput before importing the block, so the input
+        // should exist here.
+        throw new ApiError(404, `PayloadEnvelopeInput not found for block root ${blockRootHex}`);
+      }
+
+      if ((blobs === undefined) !== (kzgProofs === undefined)) {
+        throw new ApiError(400, "blobs and kzgProofs must both be supplied or both omitted");
+      }
+      const hasSuppliedBlobs = blobs !== undefined && kzgProofs !== undefined;
       const isSelfBuild = envelope.builderIndex === BUILDER_INDEX_SELF_BUILD;
       let dataColumnSidecars: gloas.DataColumnSidecar[] = [];
 
-      if (isSelfBuild) {
-        // For self-builds, construct and publish data column sidecars from cached block production data
+      if (hasSuppliedBlobs) {
+        const expectedBlobs = payloadInput.getBlobKzgCommitments().length;
+        if (blobs.length !== expectedBlobs) {
+          throw new ApiError(400, `Expected ${expectedBlobs} blobs to match bid kzg_commitments, got ${blobs.length}`);
+        }
+        if (kzgProofs.length !== blobs.length * NUMBER_OF_COLUMNS) {
+          throw new ApiError(
+            400,
+            `Expected ${blobs.length * NUMBER_OF_COLUMNS} kzg_proofs for ${blobs.length} blobs, got ${kzgProofs.length}`
+          );
+        }
+        if (blobs.length > 0) {
+          const timer = metrics?.peerDas.dataColumnSidecarComputationTime.startTimer();
+          const cells = blobs.map((blob) => kzg.computeCells(blob));
+          try {
+            await validateCellsAndKzgCommitments(payloadInput.getBlobKzgCommitments(), kzgProofs, cells);
+          } catch (e) {
+            throw new ApiError(
+              400,
+              `Invalid supplied blobs/kzg_proofs against bid kzg_commitments: ${(e as Error).message}`
+            );
+          }
+          const cellsAndProofs = cells.map((rowCells, rowIndex) => ({
+            cells: rowCells,
+            proofs: kzgProofs.slice(rowIndex * NUMBER_OF_COLUMNS, (rowIndex + 1) * NUMBER_OF_COLUMNS),
+          }));
+          dataColumnSidecars = getGloasDataColumnSidecars(slot, envelope.beaconBlockRoot, cellsAndProofs);
+          timer?.();
+        }
+      } else if (isSelfBuild) {
+        // Stateful self-build path: reconstruct data column sidecars from cached cells + proofs.
         const cachedResult = chain.blockProductionCache.get(blockRootHex) as ProduceFullGloas | undefined;
         if (cachedResult === undefined) {
           throw new ApiError(404, `No cached block production result found for block root ${blockRootHex}`);
@@ -714,8 +758,6 @@ export function getBeaconBlockApi({
           dataColumnSidecars = getGloasDataColumnSidecars(slot, envelope.beaconBlockRoot, cellsAndProofs);
           timer?.();
         }
-      } else {
-        // TODO GLOAS: will this api be used by builders or only for self-building?
       }
 
       // If called near a slot boundary (e.g. late in slot N-1), hold briefly so gossip aligns with slot N.
@@ -724,14 +766,6 @@ export function getBeaconBlockApi({
         await sleep(msToBlockSlot);
       }
 
-      const payloadInput = chain.seenPayloadEnvelopeInputCache.get(blockRootHex);
-      if (!payloadInput) {
-        // The block is awaited above (queuing if the envelope arrived first), and both the API and
-        // gossip import paths seed the PayloadEnvelopeInput before importing the block, so the input
-        // should exist here.
-        throw new ApiError(404, `PayloadEnvelopeInput not found for block root ${blockRootHex}`);
-      }
-
       payloadInput.addPayloadEnvelope({
         envelope: signedExecutionPayloadEnvelope,
         source: PayloadEnvelopeInputSource.api,