feat: builder selection, broadcast validation, stateless flow for gloas block production

.wordlist.txt

@@ -45,6 +45,7 @@ Flamegraphs
 GPG
 Geth
 Github
+Gloas
 Goerli
 Golang
 Gossipsub

packages/api/src/beacon/routes/beacon/block.ts

@@ -18,6 +18,7 @@ import {
   Slot,
   deneb,
   gloas,
+  isSignedExecutionPayloadEnvelopeContents,
   ssz,
   sszTypesFor,
 } from "@lodestar/types";
@@ -32,6 +33,7 @@ import {
   ExecutionOptimisticFinalizedAndVersionMeta,
   MetaHeader,
 } from "../../../utils/metadata.js";
+import {toBoolean} from "../../../utils/serdes.js";
 import {WireFormat} from "../../../utils/wireFormat.js";
 
 export const BlockHeaderResponseType = new ContainerType({
@@ -187,11 +189,22 @@ export type Endpoints = {
    * Instructs the beacon node to broadcast a signed execution payload envelope to the network,
    * to be gossiped for payload validation. A success response (20x) indicates that the envelope
    * passed gossip validation and was successfully broadcast onto the network.
+   *
+   * The request body is either a `SignedExecutionPayloadEnvelopeContents` (envelope with blobs
+   * and KZG proofs, stateless flow) or a `SignedExecutionPayloadEnvelope` (stateful flow,
+   * the beacon node attaches blobs and KZG proofs from its block production cache).
    */
   publishExecutionPayloadEnvelope: Endpoint<
     "POST",
-    {signedExecutionPayloadEnvelope: gloas.SignedExecutionPayloadEnvelope},
-    {body: unknown; headers: {[MetaHeader.Version]: string}},
+    {
+      signedEnvelope: gloas.SignedExecutionPayloadEnvelopeContents | gloas.SignedExecutionPayloadEnvelope;
+      broadcastValidation?: BroadcastValidation;
+    },
+    {
+      body: unknown;
+      headers: {[MetaHeader.Version]: string; [MetaHeader.BlobDataIncluded]: string};
+      query: {broadcast_validation?: string};
+    },
     EmptyResponseData,
     EmptyMeta
   >;
@@ -453,40 +466,64 @@ export function getDefinitions(config: ChainForkConfig): RouteDefinitions<Endpoi
       url: "/eth/v1/beacon/execution_payload_envelopes",
       method: "POST",
       req: {
-        writeReqJson: ({signedExecutionPayloadEnvelope}) => {
-          const fork = config.getForkName(signedExecutionPayloadEnvelope.message.payload.slotNumber);
+        writeReqJson: ({signedEnvelope, broadcastValidation}) => {
+          const blobDataIncluded = isSignedExecutionPayloadEnvelopeContents(signedEnvelope);
+          const fork = config.getForkName(
+            (blobDataIncluded ? signedEnvelope.signedExecutionPayloadEnvelope : signedEnvelope).message.payload
+              .slotNumber
+          );
           return {
-            body: getPostGloasForkTypes(fork).SignedExecutionPayloadEnvelope.toJson(signedExecutionPayloadEnvelope),
+            body: blobDataIncluded
+              ? getPostGloasForkTypes(fork).SignedExecutionPayloadEnvelopeContents.toJson(signedEnvelope)
+              : getPostGloasForkTypes(fork).SignedExecutionPayloadEnvelope.toJson(signedEnvelope),
             headers: {
               [MetaHeader.Version]: fork,
+              [MetaHeader.BlobDataIncluded]: blobDataIncluded.toString(),
             },
+            query: {broadcast_validation: broadcastValidation},
           };
         },
-        parseReqJson: ({body, headers}) => {
+        parseReqJson: ({body, headers, query}) => {
           const fork = toForkName(fromHeaders(headers, MetaHeader.Version));
+          const blobDataIncluded = toBoolean(fromHeaders(headers, MetaHeader.BlobDataIncluded));
           return {
-            signedExecutionPayloadEnvelope: getPostGloasForkTypes(fork).SignedExecutionPayloadEnvelope.fromJson(body),
+            signedEnvelope: blobDataIncluded
+              ? getPostGloasForkTypes(fork).SignedExecutionPayloadEnvelopeContents.fromJson(body)
+              : getPostGloasForkTypes(fork).SignedExecutionPayloadEnvelope.fromJson(body),
+            broadcastValidation: query.broadcast_validation as BroadcastValidation,
           };
         },
-        writeReqSsz: ({signedExecutionPayloadEnvelope}) => {
-          const fork = config.getForkName(signedExecutionPayloadEnvelope.message.payload.slotNumber);
+        writeReqSsz: ({signedEnvelope, broadcastValidation}) => {
+          const blobDataIncluded = isSignedExecutionPayloadEnvelopeContents(signedEnvelope);
+          const fork = config.getForkName(
+            (blobDataIncluded ? signedEnvelope.signedExecutionPayloadEnvelope : signedEnvelope).message.payload
+              .slotNumber
+          );
           return {
-            body: getPostGloasForkTypes(fork).SignedExecutionPayloadEnvelope.serialize(signedExecutionPayloadEnvelope),
+            body: blobDataIncluded
+              ? getPostGloasForkTypes(fork).SignedExecutionPayloadEnvelopeContents.serialize(signedEnvelope)
+              : getPostGloasForkTypes(fork).SignedExecutionPayloadEnvelope.serialize(signedEnvelope),
             headers: {
               [MetaHeader.Version]: fork,
+              [MetaHeader.BlobDataIncluded]: blobDataIncluded.toString(),
             },
+            query: {broadcast_validation: broadcastValidation},
           };
         },
-        parseReqSsz: ({body, headers}) => {
+        parseReqSsz: ({body, headers, query}) => {
           const fork = toForkName(fromHeaders(headers, MetaHeader.Version));
+          const blobDataIncluded = toBoolean(fromHeaders(headers, MetaHeader.BlobDataIncluded));
           return {
-            signedExecutionPayloadEnvelope:
-              getPostGloasForkTypes(fork).SignedExecutionPayloadEnvelope.deserialize(body),
+            signedEnvelope: blobDataIncluded
+              ? getPostGloasForkTypes(fork).SignedExecutionPayloadEnvelopeContents.deserialize(body)
+              : getPostGloasForkTypes(fork).SignedExecutionPayloadEnvelope.deserialize(body),
+            broadcastValidation: query.broadcast_validation as BroadcastValidation,
           };
         },
         schema: {
           body: Schema.Object,
-          headers: {[MetaHeader.Version]: Schema.String},
+          query: {broadcast_validation: Schema.String},
+          headers: {[MetaHeader.Version]: Schema.String, [MetaHeader.BlobDataIncluded]: Schema.String},
         },
       },
       resp: EmptyResponseCodec,

packages/api/src/beacon/routes/validator.ts

@@ -38,6 +38,7 @@ import {
   EmptyResponseCodec,
   EmptyResponseData,
   JsonOnlyReq,
+  WithMeta,
   WithVersion,
 } from "../../utils/codecs.js";
 import {getPostBellatrixForkTypes, getPostGloasForkTypes, toForkName} from "../../utils/fork.js";
@@ -98,6 +99,8 @@ export const ProduceBlockV4MetaType = new ContainerType(
     ...VersionType.fields,
     /** Consensus rewards paid to the proposer for this block, in Wei */
     consensusBlockValue: ssz.UintBn64,
+    /** Specifies whether the response contains full block contents or only the beacon block */
+    executionPayloadIncluded: ssz.Boolean,
   },
   {jsonCase: "eth2"}
 );
@@ -421,6 +424,12 @@ export type Endpoints = {
    * Post-Gloas, proposers submit execution payload bids rather than full execution payloads,
    * so there is no longer a concept of blinded or unblinded blocks. Builders release the payload later.
    * This endpoint is specific to the post-Gloas forks and is not backwards compatible with previous forks.
+   *
+   * When self-building and `includePayload` is true, the response contains the full `BlockContents`
+   * (block, execution payload envelope, KZG proofs and blobs) which enables stateless envelope
+   * publishing via any beacon node. When `includePayload` is false, only the `BeaconBlock` is
+   * returned and the beacon node caches the envelope and blobs internally.
+   * When committing to a builder bid, only the `BeaconBlock` is returned in either case.
    */
   produceBlockV4: Endpoint<
     "GET",
@@ -433,6 +442,8 @@ export type Endpoints = {
       graffiti?: string;
       skipRandaoVerification?: boolean;
       builderBoostFactor?: UintBn64;
+      /** Include execution payload envelope and blobs in the response when self-building */
+      includePayload: boolean;
     } & Omit<ExtraProduceBlockOpts, "blindedLocal">,
     {
       params: {slot: number};
@@ -444,16 +455,18 @@ export type Endpoints = {
         builder_selection?: string;
         builder_boost_factor?: string;
         strict_fee_recipient_check?: boolean;
+        include_payload: boolean;
       };
     },
-    BeaconBlock<ForkPostGloas>,
+    BeaconBlock<ForkPostGloas> | BlockContents<ForkPostGloas>,
     ProduceBlockV4Meta
   >;
 
   /**
    * Get execution payload envelope.
-   * Retrieves execution payload envelope for a given slot and beacon block root.
-   * The envelope contains the full execution payload along with associated metadata.
+   * Retrieves the cached execution payload envelope for a given slot and beacon block root,
+   * to be signed and published via `publishExecutionPayloadEnvelope`.
+   * Used in the stateful (`includePayload=false`) local build flow.
    */
   getExecutionPayloadEnvelope: Endpoint<
     "GET",
@@ -906,6 +919,7 @@ export function getDefinitions(config: ChainForkConfig): RouteDefinitions<Endpoi
           builderSelection,
           builderBoostFactor,
           strictFeeRecipientCheck,
+          includePayload,
         }) => ({
           params: {slot},
           query: {
@@ -916,6 +930,7 @@ export function getDefinitions(config: ChainForkConfig): RouteDefinitions<Endpoi
             builder_selection: builderSelection,
             builder_boost_factor: builderBoostFactor?.toString(),
             strict_fee_recipient_check: strictFeeRecipientCheck,
+            include_payload: includePayload,
           },
         }),
         parseReq: ({params, query}) => ({
@@ -927,6 +942,7 @@ export function getDefinitions(config: ChainForkConfig): RouteDefinitions<Endpoi
           builderSelection: query.builder_selection as BuilderSelection,
           builderBoostFactor: parseBuilderBoostFactor(query.builder_boost_factor),
           strictFeeRecipientCheck: query.strict_fee_recipient_check,
+          includePayload: query.include_payload,
         }),
         schema: {
           params: {slot: Schema.UintRequired},
@@ -938,21 +954,31 @@ export function getDefinitions(config: ChainForkConfig): RouteDefinitions<Endpoi
             builder_selection: Schema.String,
             builder_boost_factor: Schema.String,
             strict_fee_recipient_check: Schema.Boolean,
+            include_payload: Schema.BooleanRequired,
           },
         },
       },
       resp: {
-        data: WithVersion((fork) => getPostGloasForkTypes(fork).BeaconBlock),
+        data: WithMeta(
+          ({version, executionPayloadIncluded}) =>
+            (executionPayloadIncluded
+              ? getPostGloasForkTypes(version).BlockContents
+              : getPostGloasForkTypes(version).BeaconBlock) as Type<
+              BeaconBlock<ForkPostGloas> | BlockContents<ForkPostGloas>
+            >
+        ),
         meta: {
           toJson: (meta) => ProduceBlockV4MetaType.toJson(meta),
           fromJson: (val) => ProduceBlockV4MetaType.fromJson(val),
           toHeadersObject: (meta) => ({
             [MetaHeader.Version]: meta.version,
             [MetaHeader.ConsensusBlockValue]: meta.consensusBlockValue.toString(),
+            [MetaHeader.ExecutionPayloadIncluded]: meta.executionPayloadIncluded.toString(),
           }),
           fromHeaders: (headers) => ({
             version: toForkName(headers.getRequired(MetaHeader.Version)),
             consensusBlockValue: BigInt(headers.getRequired(MetaHeader.ConsensusBlockValue)),
+            executionPayloadIncluded: toBoolean(headers.getRequired(MetaHeader.ExecutionPayloadIncluded)),
           }),
         },
       },

packages/api/src/utils/metadata.ts

@@ -74,8 +74,10 @@ export type ExecutionOptimisticAndDependentRootMeta = ValueOf<typeof ExecutionOp
 
 export enum MetaHeader {
   Version = "Eth-Consensus-Version",
+  BlobDataIncluded = "Eth-Blob-Data-Included",
   ConsensusBlockValue = "Eth-Consensus-Block-Value",
   ExecutionPayloadBlinded = "Eth-Execution-Payload-Blinded",
+  ExecutionPayloadIncluded = "Eth-Execution-Payload-Included",
   ExecutionPayloadValue = "Eth-Execution-Payload-Value",
 
   /* Lodestar-specific (non-standardized) headers */

packages/api/src/utils/schema.ts

@@ -34,6 +34,7 @@ export enum Schema {
   ObjectArray,
   AnyArray,
   Boolean,
+  BooleanRequired,
 }
 
 /**
@@ -71,6 +72,7 @@ function getJsonSchemaItem(schema: Schema): JsonSchema {
       return {type: "array"};
 
     case Schema.Boolean:
+    case Schema.BooleanRequired:
       return {type: "boolean"};
   }
 }
@@ -81,6 +83,7 @@ function isRequired(schema: Schema): boolean {
     case Schema.StringRequired:
     case Schema.UintOrStringRequired:
     case Schema.StringArrayRequired:
+    case Schema.BooleanRequired:
       return true;
 
     default:

packages/api/test/unit/beacon/testData/beacon.ts

@@ -80,7 +80,10 @@ export const testData: GenericServerTestCases<Endpoints> = {
     res: undefined,
   },
   publishExecutionPayloadEnvelope: {
-    args: {signedExecutionPayloadEnvelope: ssz.gloas.SignedExecutionPayloadEnvelope.defaultValue()},
+    args: {
+      signedEnvelope: ssz.gloas.SignedExecutionPayloadEnvelopeContents.defaultValue(),
+      broadcastValidation: BroadcastValidation.gossip,
+    },
     res: undefined,
   },
   publishExecutionPayloadBid: {

packages/api/test/unit/beacon/testData/validator.ts

@@ -89,12 +89,14 @@ export const testData: GenericServerTestCases<Endpoints> = {
       feeRecipient,
       builderSelection: BuilderSelection.ExecutionAlways,
       strictFeeRecipientCheck: true,
+      includePayload: true,
     },
     res: {
-      data: ssz.gloas.BeaconBlock.defaultValue(),
+      data: ssz.gloas.BlockContents.defaultValue(),
       meta: {
         version: ForkName.gloas,
         consensusBlockValue: ssz.Wei.defaultValue(),
+        executionPayloadIncluded: true,
       },
     },
   },

packages/api/test/utils/checkAgainstSpec.ts

@@ -113,6 +113,11 @@ export function runTestCheckAgainstSpec<Es extends Record<string, Endpoint>>(
           stringifyProperties(reqJson.params ?? {});
           stringifyProperties(reqJson.query ?? {});
 
+          // Parse headers back to typed values as the spec defines boolean headers as `{schema: type: boolean}`
+          if (reqJson.headers) {
+            reqJson.headers = parseHeaders(reqJson.headers as Record<string, string>) as typeof reqJson.headers;
+          }
+
           const ignoredProperties = ignoredProperty?.request;
           if (ignoredProperties) {
             // Remove ignored properties from schema validation
@@ -218,13 +223,18 @@ function prettyAjvErrors(errors: ErrorObject[] | null | undefined): string {
   return errors.map((e) => `${e.instancePath ?? "."} - ${e.message}`).join("\n");
 }
 
-type StringifiedProperty = string | StringifiedProperty[];
+type StringifiedProperty = string | boolean | StringifiedProperty[];
 
 function stringifyProperty(value: unknown): StringifiedProperty {
   if (typeof value === "number") {
     return value.toString(10);
   }
 
+  // The spec defines boolean query params as `{schema: type: boolean}`, keep them as booleans
+  if (typeof value === "boolean") {
+    return value;
+  }
+
   if (Array.isArray(value)) {
     return value.map(stringifyProperty);
   }