(WIP)perf(state-transition): use native PTC sampling

packages/beacon-node/test/spec/utils/specTestIterator.ts

@@ -108,12 +108,6 @@ export const defaultSkipOpts: SkipOpts = {
     // Enable this after https://github.com/ChainSafe/lodestar/issues/9666 is resolved
     // The case name embeds the generation seed, so it changes whenever comptests are regenerated.
     /fork_choice_compliance\/block_tree_test\/pyspec_tests\/block_tree_test_17_381675768_1$/,
-    // TODO GLOAS: gloas/heze take ~23-24s on the mainnet preset (~7.5x pre-gloas) because every
-    // post-gloas slot writes into the SLOTS_PER_HISTORICAL_ROOT-wide executionPayloadAvailability
-    // bitvector, and this suite steps 8192 slots. That is 76-81% of the 30s sanity/slots timeout,
-    // so skip rather than raise the timeout and hide the regression.
-    // Enable this after https://github.com/ChainSafe/lodestar/issues/9771 is resolved
-    /^(gloas|heze)\/sanity\/slots\/pyspec_tests\/historical_accumulator$/,
   ],
   // TODO GLOAS: Investigate why networking tests are failing since alpha.5
   skippedRunners: ["networking"],

packages/state-transition/src/epoch/processPtcWindow.ts

@@ -1,4 +1,4 @@
-import {MIN_SEED_LOOKAHEAD} from "@lodestar/params";
+import {MIN_SEED_LOOKAHEAD, SLOTS_PER_EPOCH} from "@lodestar/params";
 import {ssz} from "@lodestar/types";
 import {CachedBeaconStateGloas, EpochTransitionCache} from "../types.js";
 import {computeEpochShuffling} from "../util/epochShuffling.js";
@@ -22,17 +22,22 @@ export function processPtcWindow(state: CachedBeaconStateGloas, cache: EpochTran
     state,
     nextEpoch,
     nextEpochShuffling.committees,
-    state.epochCtx.effectiveBalanceIncrements
+    state.epochCtx.effectiveBalanceIncrements,
+    nextEpochShuffling.shuffling
   );
 
   // Stash for finalProcessEpoch to shift into epoch cache
   cache.nextEpochPayloadTimelinessCommittees = newNextPayloadTimelinessCommittees;
 
-  // Write shifted window to state: current(N) + next(N+1) + newlyComputed(N+2)
-  // From the perspective of upcoming epoch N+1, this is previous + current + next
-  state.ptcWindow = ssz.gloas.PtcWindow.toViewDU([
-    ...state.epochCtx.payloadTimelinessCommittees,
-    ...state.epochCtx.nextPayloadTimelinessCommittees,
-    ...newNextPayloadTimelinessCommittees,
-  ]);
+  const ptcWindow = state.ptcWindow;
+  const retainedLength = ptcWindow.length - SLOTS_PER_EPOCH;
+  for (let i = 0; i < retainedLength; i++) {
+    ptcWindow.set(i, ptcWindow.getReadonly(i + SLOTS_PER_EPOCH));
+  }
+  for (let i = 0; i < SLOTS_PER_EPOCH; i++) {
+    ptcWindow.set(
+      retainedLength + i,
+      ssz.gloas.PayloadTimelinessCommittee.toViewDU(newNextPayloadTimelinessCommittees[i])
+    );
+  }
 }

packages/state-transition/src/slot/index.ts

@@ -1,3 +1,4 @@
+import {HashComputationGroup} from "@chainsafe/persistent-merkle-tree";
 import {ForkSeq, SLOTS_PER_HISTORICAL_ROOT} from "@lodestar/params";
 import {byteArrayEquals} from "@lodestar/utils";
 import {ZERO_HASH} from "../constants/index.js";
@@ -12,13 +13,14 @@ export {upgradeStateToFulu} from "./upgradeStateToFulu.js";
 export {upgradeStateToGloas} from "./upgradeStateToGloas.js";
 export {upgradeStateToHeze} from "./upgradeStateToHeze.js";
 
+const slotHcGroup = new HashComputationGroup();
+
 /**
  * Dial state to next slot. Common for all forks
  */
 export function processSlot(fork: ForkSeq, state: CachedBeaconStateAllForks): void {
   // Cache state root
-  // Note: .hashTreeRoot() automatically commits() pending changes
-  const previousStateRoot = state.hashTreeRoot();
+  const previousStateRoot = fork >= ForkSeq.gloas ? state.batchHashTreeRoot(slotHcGroup) : state.hashTreeRoot();
   state.stateRoots.set(state.slot % SLOTS_PER_HISTORICAL_ROOT, previousStateRoot);
 
   // Cache latest block header state root

packages/state-transition/src/util/gloas.ts

@@ -220,7 +220,8 @@ export function initializePtcWindow(state: CachedBeaconStateFulu): Uint32Array[]
         state,
         epoch,
         shuffling.committees,
-        state.epochCtx.effectiveBalanceIncrements
+        state.epochCtx.effectiveBalanceIncrements,
+        shuffling.shuffling
       )
     );
   }

packages/state-transition/src/util/seed.ts

@@ -1,4 +1,5 @@
 import {digest} from "@chainsafe/as-sha256";
+import lodestarZ from "@chainsafe/lodestar-z";
 import {
   computeProposerIndex as nativeComputeProposerIndex,
   computeSyncCommitteeIndices as nativeComputeSyncCommitteeIndices,
@@ -275,25 +276,46 @@ export function computePayloadTimelinessCommitteesForEpoch(
   state: BeaconStateAllForks,
   epoch: number,
   committees: Uint32Array[][],
-  effectiveBalanceIncrements: EffectiveBalanceIncrements
+  effectiveBalanceIncrements: EffectiveBalanceIncrements,
+  shuffling?: Uint32Array
 ): Uint32Array[] {
   const epochSeed = getSeed(state, epoch, DOMAIN_PTC_ATTESTER);
   const startSlot = epoch * SLOTS_PER_EPOCH;
-  const result: Uint32Array[] = new Array(SLOTS_PER_EPOCH);
-
-  // Pre-allocate slot seed buffer once, reuse across all slots
-  const slotSeedInput = new Uint8Array(epochSeed.length + 8);
-  slotSeedInput.set(epochSeed, 0);
-  const slotSeedView = new DataView(slotSeedInput.buffer, slotSeedInput.byteOffset, slotSeedInput.byteLength);
-
+  const slotOffsets = new Uint32Array(SLOTS_PER_EPOCH + 1);
+  let shufflingLength = 0;
   for (let i = 0; i < SLOTS_PER_EPOCH; i++) {
-    const slot = startSlot + i;
-    // Write slot as little-endian uint64 (fits in uint32 range)
-    slotSeedView.setUint32(epochSeed.length, slot, true);
-    slotSeedView.setUint32(epochSeed.length + 4, 0, true);
-    const slotSeed = digest(slotSeedInput);
+    slotOffsets[i] = shufflingLength;
+    for (const committee of committees[i]) {
+      shufflingLength += committee.length;
+    }
+  }
+  slotOffsets[SLOTS_PER_EPOCH] = shufflingLength;
+
+  const flatShuffling = shuffling ?? new Uint32Array(shufflingLength);
+  if (shuffling === undefined) {
+    let offset = 0;
+    for (const slotCommittees of committees) {
+      for (const committee of slotCommittees) {
+        flatShuffling.set(committee, offset);
+        offset += committee.length;
+      }
+    }
+  }
 
-    result[i] = computePayloadTimelinessCommitteeForSlot(slotSeed, committees[i], effectiveBalanceIncrements);
+  const flatResult = lodestarZ.shuffle.computePtcIndicesForEpoch(
+    epochSeed,
+    startSlot,
+    SLOTS_PER_EPOCH,
+    flatShuffling,
+    slotOffsets,
+    effectiveBalanceIncrements,
+    PTC_SIZE,
+    MAX_EFFECTIVE_BALANCE_ELECTRA,
+    EFFECTIVE_BALANCE_INCREMENT
+  );
+  const result: Uint32Array[] = new Array(SLOTS_PER_EPOCH);
+  for (let i = 0; i < SLOTS_PER_EPOCH; i++) {
+    result[i] = flatResult.slice(i * PTC_SIZE, (i + 1) * PTC_SIZE);
   }
   return result;
 }

packages/state-transition/test/unit/util/seed.test.ts

@@ -1,14 +1,19 @@
 import crypto from "node:crypto";
 import {describe, expect, it} from "vitest";
+import {digest} from "@chainsafe/as-sha256";
 import {toHexString} from "@chainsafe/ssz";
-import {ForkSeq, GENESIS_EPOCH, GENESIS_SLOT, SLOTS_PER_EPOCH} from "@lodestar/params";
+import {DOMAIN_PTC_ATTESTER, ForkSeq, GENESIS_EPOCH, GENESIS_SLOT, SLOTS_PER_EPOCH} from "@lodestar/params";
 import {bytesToInt} from "@lodestar/utils";
 import {generateState} from "../../../src/testUtils/state.js";
 import {
+  computeEpochShuffling,
+  computePayloadTimelinessCommitteeForSlot,
   computePayloadTimelinessCommitteeIndices,
+  computePayloadTimelinessCommitteesForEpoch,
   computeProposerIndex,
   getNextSyncCommitteeIndices,
   getRandaoMix,
+  getSeed,
   naiveComputePayloadTimelinessCommitteeIndices,
   naiveComputeProposerIndex,
   naiveGetNextSyncCommitteeIndices,
@@ -94,6 +99,42 @@ describe("computePayloadTimelinessCommitteeIndices", () => {
     const result = computePayloadTimelinessCommitteeIndices(effectiveBalanceIncrements, indices, seed);
     expect(result).toEqual(new Uint32Array(expected));
   });
+
+  it("should compute an epoch with the same per-slot results", () => {
+    const epochValidatorCount = 16_384;
+    const epochIndices = Uint32Array.from({length: epochValidatorCount}, (_, i) => i);
+    const epochEffectiveBalanceIncrements = new Uint16Array(epochValidatorCount).fill(32);
+    const state = generateState();
+    const epoch = 0;
+    const shuffling = computeEpochShuffling(state, epochIndices, epoch);
+    const epochSeed = getSeed(state, epoch, DOMAIN_PTC_ATTESTER);
+    const slotSeedInput = new Uint8Array(epochSeed.length + 8);
+    slotSeedInput.set(epochSeed);
+    const slotSeedView = new DataView(slotSeedInput.buffer);
+    const expected = new Array<Uint32Array>(SLOTS_PER_EPOCH);
+    for (let i = 0; i < SLOTS_PER_EPOCH; i++) {
+      slotSeedView.setUint32(epochSeed.length, i, true);
+      slotSeedView.setUint32(epochSeed.length + 4, 0, true);
+      expected[i] = computePayloadTimelinessCommitteeForSlot(
+        digest(slotSeedInput),
+        shuffling.committees[i],
+        epochEffectiveBalanceIncrements
+      );
+    }
+
+    expect(
+      computePayloadTimelinessCommitteesForEpoch(
+        state,
+        epoch,
+        shuffling.committees,
+        epochEffectiveBalanceIncrements,
+        shuffling.shuffling
+      )
+    ).toEqual(expected);
+    expect(
+      computePayloadTimelinessCommitteesForEpoch(state, epoch, shuffling.committees, epochEffectiveBalanceIncrements)
+    ).toEqual(expected);
+  });
 });
 
 describe("number from 2 bytes bytesToInt", () => {

pnpm-workspace.yaml

@@ -14,7 +14,7 @@ minimumReleaseAgeExclude:
 nodeLinker: isolated
 
 catalog:
-  "@chainsafe/lodestar-z": "^1.0.0"
+  "@chainsafe/lodestar-z": "github:ChainSafe/lodestar-z#848d561f45eaa3539e81c82de11ac52c3e27b218"
   "@vitest/browser": "^4.0.7"
   "@vitest/browser-playwright": "^4.0.7"
   "@vitest/coverage-v8": "^4.0.7"