chore: follow up on #9390 on EIP-7688 changes

packages/beacon-node/src/network/reqresp/protocols.ts

@@ -1,6 +1,6 @@
 import {BeaconConfig} from "@lodestar/config";
-import {ForkName} from "@lodestar/params";
-import {ContextBytesFactory, ContextBytesType, Encoding} from "@lodestar/reqresp";
+import {ForkName, MAX_DATA_COLUMN_SIDECAR_SIZE, isForkPostGloas} from "@lodestar/params";
+import {ContextBytesFactory, ContextBytesType, Encoding, TypeSizes} from "@lodestar/reqresp";
 import {rateLimitQuotas} from "./rateLimit.js";
 import {ProtocolNoHandler, ReqRespMethod, Version, requestSszTypeByMethod, responseSszTypeByMethod} from "./types.js";
 
@@ -143,15 +143,43 @@ type ProtocolSummary = {
 };
 
 function toProtocol(protocol: ProtocolSummary) {
-  return (fork: ForkName, config: BeaconConfig): ProtocolNoHandler => ({
-    method: protocol.method,
-    version: protocol.version,
-    encoding: Encoding.SSZ_SNAPPY,
-    contextBytes: toContextBytes(protocol.contextBytesType, config),
-    inboundRateLimits: rateLimitQuotas(fork, config)[protocol.method],
-    requestSizes: requestSszTypeByMethod(fork, config)[protocol.method],
-    responseSizes: (fork) => responseSszTypeByMethod[protocol.method](fork, protocol.version),
-  });
+  return (fork: ForkName, config: BeaconConfig): ProtocolNoHandler => {
+    const requestType = requestSszTypeByMethod(fork, config)[protocol.method];
+    return {
+      method: protocol.method,
+      version: protocol.version,
+      encoding: Encoding.SSZ_SNAPPY,
+      contextBytes: toContextBytes(protocol.contextBytesType, config),
+      inboundRateLimits: rateLimitQuotas(fork, config)[protocol.method],
+      requestSizes: requestType === null ? null : clampTypeSizes(requestType, protocol.method, fork, config),
+      responseSizes: (fork) =>
+        clampTypeSizes(responseSszTypeByMethod[protocol.method](fork, protocol.version), protocol.method, fork, config),
+    };
+  };
+}
+
+/**
+ * Bound the sizes accepted from the ssz-snappy length-prefix. Gloas progressive containers have broad
+ * theoretical SSZ max sizes so the preset p2p bounds must be used instead.
+ *
+ * The length-prefix must be within the size bounds derived from the payload SSZ type or `MAX_PAYLOAD_SIZE`,
+ * whichever is smaller, see
+ * https://github.com/ethereum/consensus-specs/blob/v1.7.0-alpha.12/specs/phase0/p2p-interface.md#encoding-strategies.
+ * Type-specific SSZ bounds supersede the bounds derived from the SSZ type, see
+ * https://github.com/ethereum/consensus-specs/blob/v1.7.0-alpha.12/specs/gloas/p2p-interface.md#type-specific-ssz-bounds.
+ */
+function clampTypeSizes(type: TypeSizes, method: ReqRespMethod, fork: ForkName, config: BeaconConfig): TypeSizes {
+  let typeSpecificBound = config.MAX_PAYLOAD_SIZE;
+  if (isForkPostGloas(fork)) {
+    switch (method) {
+      case ReqRespMethod.DataColumnSidecarsByRange:
+      case ReqRespMethod.DataColumnSidecarsByRoot:
+        typeSpecificBound = MAX_DATA_COLUMN_SIDECAR_SIZE;
+        break;
+    }
+  }
+
+  return {minSize: type.minSize, maxSize: Math.min(type.maxSize, typeSpecificBound)};
 }
 
 function toContextBytes(type: ContextBytesType, config: BeaconConfig): ContextBytesFactory {

packages/beacon-node/test/spec/presets/fast_confirmation.test.ts

@@ -724,10 +724,8 @@ specTestIterator(
     skippedRunners: [],
     skippedTestSuites: [
       ...(defaultSkipOpts.skippedTestSuites ?? []),
-      // TODO-GLOAS: lodestar's fast-confirmation rule is block-root based and does not model the
-      // ePBS payload_status dimension required by specs/gloas/fast-confirmation.md (PTC payload
-      // presence/timeliness, get_node_for_root with PAYLOAD_STATUS_PENDING). Head/justified/
-      // finalized/proposer-head all match; only getConfirmedRoot diverges.
+      // TODO-GLOAS: The fast-confirmation runner does not process the execution_payload steps or
+      // execution_payload_envelope_* files required by these vectors.
       /^gloas\/fast_confirmation\/.*/,
     ],
   }

packages/state-transition/src/cache/stateCache.ts

@@ -1,5 +1,6 @@
 import {BeaconConfig} from "@lodestar/config";
 import {loadState} from "../util/loadState/loadState.js";
+import {isViewDUNodesPopulated} from "../util/ssz.js";
 import {EpochCache, EpochCacheImmutableData, EpochCacheOpts} from "./epochCache.js";
 import {RewardCache, createEmptyRewardCache} from "./rewardCache.js";
 import {
@@ -254,9 +255,9 @@ export function isCachedBeaconState<T extends BeaconStateAllForks>(
 // This cache is populated during epoch transition, and should be preserved for performance.
 // If the cache is missing too often, means that our clone strategy is not working well.
 export function isStateValidatorsNodesPopulated(state: CachedBeaconStateAllForks): boolean {
-  return (state.validators as unknown as {nodesPopulated?: boolean}).nodesPopulated === true;
+  return isViewDUNodesPopulated(state.validators);
 }
 
 export function isStateBalancesNodesPopulated(state: CachedBeaconStateAllForks): boolean {
-  return (state.balances as unknown as {nodesPopulated?: boolean}).nodesPopulated === true;
+  return isViewDUNodesPopulated(state.balances);
 }

packages/state-transition/src/lightClient/spec/utils.ts

@@ -191,11 +191,11 @@ export function nextSyncCommitteeGindexAtFork(fork: ForkName): number {
   return NEXT_SYNC_COMMITTEE_GINDEX;
 }
 
-export function getGindexDepth(gindex: number): number {
+function getGindexDepth(gindex: number): number {
   return Math.floor(Math.log2(gindex));
 }
 
-export function getGindexIndex(gindex: number): number {
+function getGindexIndex(gindex: number): number {
   return gindex - 2 ** getGindexDepth(gindex);
 }
 

packages/state-transition/src/metrics.ts

@@ -3,6 +3,7 @@ import {ProposerRewardType} from "./block/types.js";
 import {EpochTransitionStep} from "./epoch/index.js";
 import {StateCloneSource, StateHashTreeRootSource} from "./stateTransition.js";
 import {CachedBeaconStateAllForks} from "./types.js";
+import {isViewDUNodesPopulated} from "./util/ssz.js";
 
 export type BeaconStateTransitionMetrics = ReturnType<typeof getMetrics>;
 
@@ -159,9 +160,9 @@ export function onPostStateMetrics(postState: CachedBeaconStateAllForks, metrics
 // This cache is populated during epoch transition, and should be preserved for performance.
 // If the cache is missing too often, means that our clone strategy is not working well.
 function isValidatorsNodesPopulated(state: CachedBeaconStateAllForks): boolean {
-  return (state.validators as unknown as {nodesPopulated?: boolean}).nodesPopulated === true;
+  return isViewDUNodesPopulated(state.validators);
 }
 
 function isBalancesNodesPopulated(state: CachedBeaconStateAllForks): boolean {
-  return (state.balances as unknown as {nodesPopulated?: boolean}).nodesPopulated === true;
+  return isViewDUNodesPopulated(state.balances);
 }

packages/state-transition/src/util/loadState/loadState.ts

@@ -25,10 +25,29 @@ export function loadState(
 ): MigrateStateOutput {
   // casting only to make typescript happy
   const stateType = getStateTypeFromBytes(config, stateBytes) as typeof ssz.capella.BeaconState;
+  const fork = getForkFromStateBytes(config, stateBytes);
+  const seedFork = config.getForkSeq(seedState.slot);
+
   const dataView = new DataView(stateBytes.buffer, stateBytes.byteOffset, stateBytes.byteLength);
   const fieldRanges = stateType.getFieldRanges(dataView, 0, stateBytes.length);
   const allFields = Object.keys(stateType.fields);
   const validatorsFieldIndex = allFields.indexOf("validators");
+  const validatorsRange = fieldRanges[validatorsFieldIndex];
+  const newValidatorsBytes = stateBytes.subarray(validatorsRange.start, validatorsRange.end);
+
+  // EIP-7688 replaces List with ProgressiveList for validators and inactivityScores at gloas,
+  // changing the merkle tree shape. Seed nodes cannot be reused when one state is pre-gloas
+  // and the other post-gloas.
+  const crossesGloasFork =
+    (fork >= ForkSeq.gloas && seedFork < ForkSeq.gloas) || (fork < ForkSeq.gloas && seedFork >= ForkSeq.gloas);
+  if (crossesGloasFork) {
+    const migratedState = stateType.deserializeToViewDU(stateBytes) as BeaconStateAllForks;
+    // modified validators must still be reported so that the pubkey cache is refreshed for
+    // any index that differs from the seed state, which may not be an ancestor of this state
+    const modifiedValidators = findModifiedAndAppendedValidators(seedState, newValidatorsBytes, seedValidatorsBytes);
+    return {state: migratedState, modifiedValidators};
+  }
+
   // start with default view has the same performance to start with seed state
   // and it is not fork dependent
   const migratedState = deserializeContainerIgnoreFields(
@@ -39,19 +58,10 @@ export function loadState(
   ) as BeaconStateAllForks;
 
   // validators are rarely changed
-  const validatorsRange = fieldRanges[validatorsFieldIndex];
-  const modifiedValidators = loadValidators(
-    migratedState,
-    seedState,
-    stateBytes.subarray(validatorsRange.start, validatorsRange.end),
-    seedValidatorsBytes
-  );
+  const modifiedValidators = loadValidators(migratedState, seedState, newValidatorsBytes, seedValidatorsBytes);
 
   // inactivityScores are rarely changed
   // this saves ~500ms of hashTreeRoot() time of state
-  const fork = getForkFromStateBytes(config, stateBytes);
-  const seedFork = config.getForkSeq(seedState.slot);
-
   if (fork >= ForkSeq.altair && seedFork >= ForkSeq.altair) {
     const inactivityScoresIndex = allFields.indexOf("inactivityScores");
     const inactivityScoresRange = fieldRanges[inactivityScoresIndex];
@@ -141,7 +151,9 @@ function loadInactivityScores(
     }
   } else {
     if (newValidator - 1 < 0) {
-      migratedState.inactivityScores = ssz.altair.InactivityScores.defaultViewDU();
+      // use the state's own field type, the list shape differs between altair (List) and gloas (ProgressiveList)
+      const inactivityScoresType = (migratedState.type as typeof ssz.altair.BeaconState).fields.inactivityScores;
+      migratedState.inactivityScores = inactivityScoresType.defaultViewDU();
     } else {
       migratedState.inactivityScores = migratedState.inactivityScores.sliceTo(newValidator - 1);
     }
@@ -177,6 +189,33 @@ function loadInactivityScores(
  * @param migratedState state to be migrated, the validators are loaded to this state
  * @returns modified validator indices
  */
+/**
+ * Find indices of validators whose serialized bytes differ from the seed state, plus indices
+ * appended past the seed state's validator count. Unlike loadValidators() this only diffs
+ * bytes and does not share the seed state's tree.
+ */
+function findModifiedAndAppendedValidators(
+  seedState: BeaconStateAllForks,
+  newValidatorsBytes: Uint8Array,
+  seedStateValidatorsBytes?: Uint8Array
+): number[] {
+  const seedValidatorCount = seedState.validators.length;
+  const newValidatorCount = Math.floor(newValidatorsBytes.length / VALIDATOR_BYTES_SIZE);
+  const minValidatorCount = Math.min(seedValidatorCount, newValidatorCount);
+  const seedValidatorsBytes = seedStateValidatorsBytes ?? seedState.validators.serialize();
+  const modifiedValidators: number[] = [];
+  findModifiedValidators(
+    seedValidatorsBytes.subarray(0, minValidatorCount * VALIDATOR_BYTES_SIZE),
+    newValidatorsBytes.subarray(0, minValidatorCount * VALIDATOR_BYTES_SIZE),
+    modifiedValidators
+  );
+
+  for (let validatorIndex = seedValidatorCount; validatorIndex < newValidatorCount; validatorIndex++) {
+    modifiedValidators.push(validatorIndex);
+  }
+  return modifiedValidators;
+}
+
 function loadValidators(
   migratedState: BeaconStateAllForks,
   seedState: BeaconStateAllForks,

packages/state-transition/src/util/ssz.ts

@@ -1,7 +1,7 @@
 import {BranchNode, LeafNode, Node, zeroNode} from "@chainsafe/persistent-merkle-tree";
 import {progressiveSubtreeFillToContents} from "@chainsafe/ssz";
 
-// TODO: move these utils to @chainsafe/ssz (progressive.ts, next to progressiveSubtreeFillToContents)
+// TODO: move these utils to @chainsafe/ssz, see https://github.com/ChainSafe/ssz/issues/542
 
 /** Root node (chunks + length mix-in) of a zero-filled ProgressiveListBasicType of `length` items */
 export function zeroProgressiveListBasicRootNode(itemsPerChunk: number, length: number): Node {
@@ -15,6 +15,14 @@ export function zeroProgressiveListBasicRootNode(itemsPerChunk: number, length:
   return new BranchNode(zeroProgressiveNode(numSubtrees), LeafNode.fromUint32(length));
 }
 
+/**
+ * Check if an array-type ViewDU (ListBasic, ListComposite or their progressive equivalents) has its
+ * internal nodes cache populated. The flag is a private attribute maintained by all of these classes.
+ */
+export function isViewDUNodesPopulated(view: unknown): boolean {
+  return (view as {nodesPopulated?: boolean}).nodesPopulated === true;
+}
+
 /**
  * Root node of a progressive list from its chunk/element nodes + length mix-in.
  * `nodes` are packed 32-byte chunk leaves for basic lists, or element root nodes for composite lists.

packages/state-transition/test/unit/util/loadState.test.ts

@@ -3,7 +3,7 @@ import {createChainForkConfig} from "@lodestar/config";
 import {mainnetChainConfig} from "@lodestar/config/networks";
 import {ForkName, SLOTS_PER_EPOCH} from "@lodestar/params";
 import {ssz} from "@lodestar/types";
-import {BeaconStateAltair} from "../../../src/types.js";
+import {BeaconStateAllForks, BeaconStateAltair} from "../../../src/types.js";
 import {loadState, loadStateAndValidators} from "../../../src/util/loadState/loadState.js";
 
 describe("loadStateAndValidators", () => {
@@ -106,3 +106,60 @@ describe("loadState does not poison seed state's cache", () => {
     expect(postState.hashTreeRoot()).toEqual(originalRoot);
   });
 });
+
+describe("loadState across the gloas fork boundary", () => {
+  // EIP-7688 replaces List with ProgressiveList for validators and inactivityScores at gloas,
+  // changing the merkle tree shape. loadState() must not reuse the seed state's list nodes when
+  // the seed state is on the other side of the gloas fork, else hashTreeRoot() is silently wrong.
+  const numValidator = 10;
+  const gloasForkEpoch = 10;
+  const config = createChainForkConfig({
+    ALTAIR_FORK_EPOCH: 0,
+    BELLATRIX_FORK_EPOCH: 0,
+    CAPELLA_FORK_EPOCH: 0,
+    DENEB_FORK_EPOCH: 0,
+    ELECTRA_FORK_EPOCH: 0,
+    FULU_FORK_EPOCH: 0,
+    GLOAS_FORK_EPOCH: gloasForkEpoch,
+  });
+  const preGloasSlot = (gloasForkEpoch - 1) * SLOTS_PER_EPOCH;
+  const postGloasSlot = gloasForkEpoch * SLOTS_PER_EPOCH;
+
+  function buildState(slot: number, validatorCount: number): BeaconStateAllForks {
+    const state = config.getForkTypes(slot).BeaconState.defaultViewDU() as BeaconStateAltair;
+    state.slot = slot;
+    for (let i = 0; i < validatorCount; i++) {
+      const validator = ssz.phase0.Validator.defaultViewDU();
+      validator.pubkey = new Uint8Array(48).fill(i);
+      state.validators.push(validator);
+      state.balances.push(32 * 1e9);
+      state.inactivityScores.push(i);
+    }
+    state.commit();
+    return state;
+  }
+
+  it("loads a gloas state from a fulu seed state", () => {
+    const seedState = buildState(preGloasSlot, numValidator);
+    const targetState = buildState(postGloasSlot, numValidator + 2);
+    // simulate a diverged branch where an overlapping index holds a different validator
+    const validator = targetState.validators.get(1);
+    validator.pubkey = new Uint8Array(48).fill(0xaa);
+    targetState.validators.set(1, validator);
+    targetState.commit();
+
+    const {state: loadedState, modifiedValidators} = loadState(config, seedState, targetState.serialize());
+    expect(loadedState.hashTreeRoot()).toEqual(targetState.hashTreeRoot());
+    // modified and appended validators must be reported for the pubkey cache
+    expect(modifiedValidators).toEqual([1, numValidator, numValidator + 1]);
+  });
+
+  it("loads a fulu state from a gloas seed state", () => {
+    const seedState = buildState(postGloasSlot, numValidator);
+    const targetState = buildState(preGloasSlot, numValidator);
+
+    const {state: loadedState, modifiedValidators} = loadState(config, seedState, targetState.serialize());
+    expect(loadedState.hashTreeRoot()).toEqual(targetState.hashTreeRoot());
+    expect(modifiedValidators).toEqual([]);
+  });
+});