diff --git a/packages/beacon-node/test/spec/utils/specTestIterator.ts b/packages/beacon-node/test/spec/utils/specTestIterator.ts index 8d727807d98a..d05aeb17461a 100644 --- a/packages/beacon-node/test/spec/utils/specTestIterator.ts +++ b/packages/beacon-node/test/spec/utils/specTestIterator.ts @@ -108,12 +108,6 @@ export const defaultSkipOpts: SkipOpts = { // Enable this after https://github.com/ChainSafe/lodestar/issues/9666 is resolved // The case name embeds the generation seed, so it changes whenever comptests are regenerated. /fork_choice_compliance\/block_tree_test\/pyspec_tests\/block_tree_test_17_381675768_1$/, - // TODO GLOAS: gloas/heze take ~23-24s on the mainnet preset (~7.5x pre-gloas) because every - // post-gloas slot writes into the SLOTS_PER_HISTORICAL_ROOT-wide executionPayloadAvailability - // bitvector, and this suite steps 8192 slots. That is 76-81% of the 30s sanity/slots timeout, - // so skip rather than raise the timeout and hide the regression. - // Enable this after https://github.com/ChainSafe/lodestar/issues/9771 is resolved - /^(gloas|heze)\/sanity\/slots\/pyspec_tests\/historical_accumulator$/, ], // TODO GLOAS: Investigate why networking tests are failing since alpha.5 skippedRunners: ["networking"], diff --git a/packages/state-transition/src/epoch/processPtcWindow.ts b/packages/state-transition/src/epoch/processPtcWindow.ts index bdb629b00978..519e4dcb1a07 100644 --- a/packages/state-transition/src/epoch/processPtcWindow.ts +++ b/packages/state-transition/src/epoch/processPtcWindow.ts @@ -1,4 +1,4 @@ -import {MIN_SEED_LOOKAHEAD} from "@lodestar/params"; +import {MIN_SEED_LOOKAHEAD, SLOTS_PER_EPOCH} from "@lodestar/params"; import {ssz} from "@lodestar/types"; import {CachedBeaconStateGloas, EpochTransitionCache} from "../types.js"; import {computeEpochShuffling} from "../util/epochShuffling.js"; @@ -22,17 +22,22 @@ export function processPtcWindow(state: CachedBeaconStateGloas, cache: EpochTran state, nextEpoch, nextEpochShuffling.committees, - state.epochCtx.effectiveBalanceIncrements + state.epochCtx.effectiveBalanceIncrements, + nextEpochShuffling.shuffling ); // Stash for finalProcessEpoch to shift into epoch cache cache.nextEpochPayloadTimelinessCommittees = newNextPayloadTimelinessCommittees; - // Write shifted window to state: current(N) + next(N+1) + newlyComputed(N+2) - // From the perspective of upcoming epoch N+1, this is previous + current + next - state.ptcWindow = ssz.gloas.PtcWindow.toViewDU([ - ...state.epochCtx.payloadTimelinessCommittees, - ...state.epochCtx.nextPayloadTimelinessCommittees, - ...newNextPayloadTimelinessCommittees, - ]); + const ptcWindow = state.ptcWindow; + const retainedLength = ptcWindow.length - SLOTS_PER_EPOCH; + for (let i = 0; i < retainedLength; i++) { + ptcWindow.set(i, ptcWindow.getReadonly(i + SLOTS_PER_EPOCH)); + } + for (let i = 0; i < SLOTS_PER_EPOCH; i++) { + ptcWindow.set( + retainedLength + i, + ssz.gloas.PayloadTimelinessCommittee.toViewDU(newNextPayloadTimelinessCommittees[i]) + ); + } } diff --git a/packages/state-transition/src/slot/index.ts b/packages/state-transition/src/slot/index.ts index 4022590901d3..d6001c79a35b 100644 --- a/packages/state-transition/src/slot/index.ts +++ b/packages/state-transition/src/slot/index.ts @@ -1,3 +1,4 @@ +import {HashComputationGroup} from "@chainsafe/persistent-merkle-tree"; import {ForkSeq, SLOTS_PER_HISTORICAL_ROOT} from "@lodestar/params"; import {byteArrayEquals} from "@lodestar/utils"; import {ZERO_HASH} from "../constants/index.js"; @@ -12,13 +13,14 @@ export {upgradeStateToFulu} from "./upgradeStateToFulu.js"; export {upgradeStateToGloas} from "./upgradeStateToGloas.js"; export {upgradeStateToHeze} from "./upgradeStateToHeze.js"; +const slotHcGroup = new HashComputationGroup(); + /** * Dial state to next slot. Common for all forks */ export function processSlot(fork: ForkSeq, state: CachedBeaconStateAllForks): void { // Cache state root - // Note: .hashTreeRoot() automatically commits() pending changes - const previousStateRoot = state.hashTreeRoot(); + const previousStateRoot = fork >= ForkSeq.gloas ? state.batchHashTreeRoot(slotHcGroup) : state.hashTreeRoot(); state.stateRoots.set(state.slot % SLOTS_PER_HISTORICAL_ROOT, previousStateRoot); // Cache latest block header state root diff --git a/packages/state-transition/src/util/gloas.ts b/packages/state-transition/src/util/gloas.ts index f7b33ab64ae7..b7b16146e9d2 100644 --- a/packages/state-transition/src/util/gloas.ts +++ b/packages/state-transition/src/util/gloas.ts @@ -220,7 +220,8 @@ export function initializePtcWindow(state: CachedBeaconStateFulu): Uint32Array[] state, epoch, shuffling.committees, - state.epochCtx.effectiveBalanceIncrements + state.epochCtx.effectiveBalanceIncrements, + shuffling.shuffling ) ); } diff --git a/packages/state-transition/src/util/seed.ts b/packages/state-transition/src/util/seed.ts index be01506f6cd4..31340ee21888 100644 --- a/packages/state-transition/src/util/seed.ts +++ b/packages/state-transition/src/util/seed.ts @@ -1,4 +1,5 @@ import {digest} from "@chainsafe/as-sha256"; +import lodestarZ from "@chainsafe/lodestar-z"; import { computeProposerIndex as nativeComputeProposerIndex, computeSyncCommitteeIndices as nativeComputeSyncCommitteeIndices, @@ -275,25 +276,46 @@ export function computePayloadTimelinessCommitteesForEpoch( state: BeaconStateAllForks, epoch: number, committees: Uint32Array[][], - effectiveBalanceIncrements: EffectiveBalanceIncrements + effectiveBalanceIncrements: EffectiveBalanceIncrements, + shuffling?: Uint32Array ): Uint32Array[] { const epochSeed = getSeed(state, epoch, DOMAIN_PTC_ATTESTER); const startSlot = epoch * SLOTS_PER_EPOCH; - const result: Uint32Array[] = new Array(SLOTS_PER_EPOCH); - - // Pre-allocate slot seed buffer once, reuse across all slots - const slotSeedInput = new Uint8Array(epochSeed.length + 8); - slotSeedInput.set(epochSeed, 0); - const slotSeedView = new DataView(slotSeedInput.buffer, slotSeedInput.byteOffset, slotSeedInput.byteLength); - + const slotOffsets = new Uint32Array(SLOTS_PER_EPOCH + 1); + let shufflingLength = 0; for (let i = 0; i < SLOTS_PER_EPOCH; i++) { - const slot = startSlot + i; - // Write slot as little-endian uint64 (fits in uint32 range) - slotSeedView.setUint32(epochSeed.length, slot, true); - slotSeedView.setUint32(epochSeed.length + 4, 0, true); - const slotSeed = digest(slotSeedInput); + slotOffsets[i] = shufflingLength; + for (const committee of committees[i]) { + shufflingLength += committee.length; + } + } + slotOffsets[SLOTS_PER_EPOCH] = shufflingLength; + + const flatShuffling = shuffling ?? new Uint32Array(shufflingLength); + if (shuffling === undefined) { + let offset = 0; + for (const slotCommittees of committees) { + for (const committee of slotCommittees) { + flatShuffling.set(committee, offset); + offset += committee.length; + } + } + } - result[i] = computePayloadTimelinessCommitteeForSlot(slotSeed, committees[i], effectiveBalanceIncrements); + const flatResult = lodestarZ.shuffle.computePtcIndicesForEpoch( + epochSeed, + startSlot, + SLOTS_PER_EPOCH, + flatShuffling, + slotOffsets, + effectiveBalanceIncrements, + PTC_SIZE, + MAX_EFFECTIVE_BALANCE_ELECTRA, + EFFECTIVE_BALANCE_INCREMENT + ); + const result: Uint32Array[] = new Array(SLOTS_PER_EPOCH); + for (let i = 0; i < SLOTS_PER_EPOCH; i++) { + result[i] = flatResult.slice(i * PTC_SIZE, (i + 1) * PTC_SIZE); } return result; } diff --git a/packages/state-transition/test/unit/util/seed.test.ts b/packages/state-transition/test/unit/util/seed.test.ts index c208e4c73f0e..b466d88fd8ff 100644 --- a/packages/state-transition/test/unit/util/seed.test.ts +++ b/packages/state-transition/test/unit/util/seed.test.ts @@ -1,14 +1,19 @@ import crypto from "node:crypto"; import {describe, expect, it} from "vitest"; +import {digest} from "@chainsafe/as-sha256"; import {toHexString} from "@chainsafe/ssz"; -import {ForkSeq, GENESIS_EPOCH, GENESIS_SLOT, SLOTS_PER_EPOCH} from "@lodestar/params"; +import {DOMAIN_PTC_ATTESTER, ForkSeq, GENESIS_EPOCH, GENESIS_SLOT, SLOTS_PER_EPOCH} from "@lodestar/params"; import {bytesToInt} from "@lodestar/utils"; import {generateState} from "../../../src/testUtils/state.js"; import { + computeEpochShuffling, + computePayloadTimelinessCommitteeForSlot, computePayloadTimelinessCommitteeIndices, + computePayloadTimelinessCommitteesForEpoch, computeProposerIndex, getNextSyncCommitteeIndices, getRandaoMix, + getSeed, naiveComputePayloadTimelinessCommitteeIndices, naiveComputeProposerIndex, naiveGetNextSyncCommitteeIndices, @@ -94,6 +99,42 @@ describe("computePayloadTimelinessCommitteeIndices", () => { const result = computePayloadTimelinessCommitteeIndices(effectiveBalanceIncrements, indices, seed); expect(result).toEqual(new Uint32Array(expected)); }); + + it("should compute an epoch with the same per-slot results", () => { + const epochValidatorCount = 16_384; + const epochIndices = Uint32Array.from({length: epochValidatorCount}, (_, i) => i); + const epochEffectiveBalanceIncrements = new Uint16Array(epochValidatorCount).fill(32); + const state = generateState(); + const epoch = 0; + const shuffling = computeEpochShuffling(state, epochIndices, epoch); + const epochSeed = getSeed(state, epoch, DOMAIN_PTC_ATTESTER); + const slotSeedInput = new Uint8Array(epochSeed.length + 8); + slotSeedInput.set(epochSeed); + const slotSeedView = new DataView(slotSeedInput.buffer); + const expected = new Array(SLOTS_PER_EPOCH); + for (let i = 0; i < SLOTS_PER_EPOCH; i++) { + slotSeedView.setUint32(epochSeed.length, i, true); + slotSeedView.setUint32(epochSeed.length + 4, 0, true); + expected[i] = computePayloadTimelinessCommitteeForSlot( + digest(slotSeedInput), + shuffling.committees[i], + epochEffectiveBalanceIncrements + ); + } + + expect( + computePayloadTimelinessCommitteesForEpoch( + state, + epoch, + shuffling.committees, + epochEffectiveBalanceIncrements, + shuffling.shuffling + ) + ).toEqual(expected); + expect( + computePayloadTimelinessCommitteesForEpoch(state, epoch, shuffling.committees, epochEffectiveBalanceIncrements) + ).toEqual(expected); + }); }); describe("number from 2 bytes bytesToInt", () => { diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5e6fd90965bc..faa6ddd2f871 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -7,8 +7,8 @@ settings: catalogs: default: '@chainsafe/lodestar-z': - specifier: ^1.0.0 - version: 1.0.0 + specifier: github:ChainSafe/lodestar-z#848d561f45eaa3539e81c82de11ac52c3e27b218 + version: 0.1.2 '@vitest/browser': specifier: ^4.0.7 version: 4.0.7 @@ -43,7 +43,7 @@ importers: version: link:packages/cli '@chainsafe/lodestar-z': specifier: 'catalog:' - version: 1.0.0 + version: https://codeload.github.com/ChainSafe/lodestar-z/tar.gz/848d561f45eaa3539e81c82de11ac52c3e27b218 devDependencies: '@actions/core': specifier: ^1.11.1 @@ -204,7 +204,7 @@ importers: version: 2.1.3 '@chainsafe/lodestar-z': specifier: 'catalog:' - version: 1.0.0 + version: https://codeload.github.com/ChainSafe/lodestar-z/tar.gz/848d561f45eaa3539e81c82de11ac52c3e27b218 '@chainsafe/persistent-merkle-tree': specifier: ^1.3.1 version: 1.3.1 @@ -391,7 +391,7 @@ importers: dependencies: '@chainsafe/lodestar-z': specifier: 'catalog:' - version: 1.0.0 + version: https://codeload.github.com/ChainSafe/lodestar-z/tar.gz/848d561f45eaa3539e81c82de11ac52c3e27b218 '@chainsafe/ssz': specifier: ^1.6.3 version: 1.6.3 @@ -437,7 +437,7 @@ importers: version: 6.0.2 '@chainsafe/lodestar-z': specifier: 'catalog:' - version: 1.0.0 + version: https://codeload.github.com/ChainSafe/lodestar-z/tar.gz/848d561f45eaa3539e81c82de11ac52c3e27b218 '@chainsafe/persistent-merkle-tree': specifier: ^1.3.1 version: 1.3.1 @@ -610,7 +610,7 @@ importers: dependencies: '@chainsafe/lodestar-z': specifier: 'catalog:' - version: 1.0.0 + version: https://codeload.github.com/ChainSafe/lodestar-z/tar.gz/848d561f45eaa3539e81c82de11ac52c3e27b218 '@lodestar/config': specifier: workspace:^ version: link:../config @@ -771,7 +771,7 @@ importers: version: 1.2.5 '@chainsafe/lodestar-z': specifier: 'catalog:' - version: 1.0.0 + version: https://codeload.github.com/ChainSafe/lodestar-z/tar.gz/848d561f45eaa3539e81c82de11ac52c3e27b218 '@chainsafe/persistent-merkle-tree': specifier: ^1.3.1 version: 1.3.1 @@ -808,7 +808,7 @@ importers: version: 3.1.0 '@chainsafe/lodestar-z': specifier: 'catalog:' - version: 1.0.0 + version: https://codeload.github.com/ChainSafe/lodestar-z/tar.gz/848d561f45eaa3539e81c82de11ac52c3e27b218 '@lodestar/api': specifier: workspace:^ version: link:../api @@ -879,7 +879,7 @@ importers: dependencies: '@chainsafe/lodestar-z': specifier: 'catalog:' - version: 1.0.0 + version: https://codeload.github.com/ChainSafe/lodestar-z/tar.gz/848d561f45eaa3539e81c82de11ac52c3e27b218 '@chainsafe/ssz': specifier: ^1.6.3 version: 1.6.3 @@ -1231,42 +1231,9 @@ packages: resolution: {integrity: sha512-y+diflUZN1hEnGhPnTPk3DUjtPigiGNf6bfXPuRSDwoB2obat0BHW7J2ZncayYTE0TAlel1sr66lLaLEO4VAdQ==} engines: {node: '>= 22'} - '@chainsafe/lodestar-z-aarch64-apple-darwin@1.0.0': - resolution: {integrity: sha512-JSnHWAWbzyOIuLb/YXRgk9ok8tlQllglDVXjSrdEYDEJTG6Z4ui7W4II0vxCLBEMA6I3OP2aGRswhcrb97wlpA==} - cpu: [arm64] - os: [darwin] - - '@chainsafe/lodestar-z-aarch64-unknown-linux-gnu@1.0.0': - resolution: {integrity: sha512-8P1bd7XExwDbCg6AvzYqW80zRbQ1CNitX5WV4WXPoam+6Mokn2sHjk2PK3X/pw+akeDL6cxAKOLW1ncb5H0+sg==} - cpu: [arm64] - os: [linux] - libc: [glibc] - - '@chainsafe/lodestar-z-aarch64-unknown-linux-musl@1.0.0': - resolution: {integrity: sha512-AvK4dZdRlNwH5d7TvAt71ogat2ea+It5Hd9l4ayCAHi4opOdn9XFd8zJCbdLpQbyUV7IYQMdZ6tIg5d4zWeArg==} - cpu: [arm64] - os: [linux] - libc: [musl] - - '@chainsafe/lodestar-z-x86_64-apple-darwin@1.0.0': - resolution: {integrity: sha512-NL2iU2aKXO2saxco7VU86Xa5zQ7aJ4ViCOOUtZ4m83kYP0L8fkgGF6SCxOoZnz8sO6pa3FzbE5tsooiCOko1kg==} - cpu: [x64] - os: [darwin] - - '@chainsafe/lodestar-z-x86_64-unknown-linux-gnu@1.0.0': - resolution: {integrity: sha512-2ZfLGzOAXmQ2TKCarX3SOW+MEkHmFdGIpyeFfITVLuZgUX9FAuU4oRmiXMSvXpb/YEmHSXojPFXKj8A9QMT5Fg==} - cpu: [x64] - os: [linux] - libc: [glibc] - - '@chainsafe/lodestar-z-x86_64-unknown-linux-musl@1.0.0': - resolution: {integrity: sha512-7dMTqdthJvk9CiI78SDpnIqfyi2pen5H/3WYcficAJ7vElIkUsHFDbEJr9iYopudZbzp25tkhlO8r/Ryn37mnw==} - cpu: [x64] - os: [linux] - libc: [musl] - - '@chainsafe/lodestar-z@1.0.0': - resolution: {integrity: sha512-eMh8raOCZN4/8K5O9AT/4Lr+/3yR5Dwk8NCxarQVwfchLCzWt2/Ae/un1klglEtFS8pl+GN+B9jO8zjKZw6ofg==} + '@chainsafe/lodestar-z@https://codeload.github.com/ChainSafe/lodestar-z/tar.gz/848d561f45eaa3539e81c82de11ac52c3e27b218': + resolution: {tarball: https://codeload.github.com/ChainSafe/lodestar-z/tar.gz/848d561f45eaa3539e81c82de11ac52c3e27b218} + version: 0.1.2 '@chainsafe/netmask@2.0.0': resolution: {integrity: sha512-I3Z+6SWUoaljh3TBzCnCxjlUyN8tA+NAk5L6m9IxvCf1BENQTePzPMis97CoN/iMW1St3WN+AWCCRp+TTBRiDg==} @@ -6881,34 +6848,9 @@ snapshots: '@chainsafe/libp2p-quic-linux-x64-musl': 2.1.3 '@chainsafe/libp2p-quic-win32-x64-msvc': 2.1.3 - '@chainsafe/lodestar-z-aarch64-apple-darwin@1.0.0': - optional: true - - '@chainsafe/lodestar-z-aarch64-unknown-linux-gnu@1.0.0': - optional: true - - '@chainsafe/lodestar-z-aarch64-unknown-linux-musl@1.0.0': - optional: true - - '@chainsafe/lodestar-z-x86_64-apple-darwin@1.0.0': - optional: true - - '@chainsafe/lodestar-z-x86_64-unknown-linux-gnu@1.0.0': - optional: true - - '@chainsafe/lodestar-z-x86_64-unknown-linux-musl@1.0.0': - optional: true - - '@chainsafe/lodestar-z@1.0.0': + '@chainsafe/lodestar-z@https://codeload.github.com/ChainSafe/lodestar-z/tar.gz/848d561f45eaa3539e81c82de11ac52c3e27b218': dependencies: '@chainsafe/zapi': 4.0.0 - optionalDependencies: - '@chainsafe/lodestar-z-aarch64-apple-darwin': 1.0.0 - '@chainsafe/lodestar-z-aarch64-unknown-linux-gnu': 1.0.0 - '@chainsafe/lodestar-z-aarch64-unknown-linux-musl': 1.0.0 - '@chainsafe/lodestar-z-x86_64-apple-darwin': 1.0.0 - '@chainsafe/lodestar-z-x86_64-unknown-linux-gnu': 1.0.0 - '@chainsafe/lodestar-z-x86_64-unknown-linux-musl': 1.0.0 '@chainsafe/netmask@2.0.0': dependencies: diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 43a740f45981..c75831538246 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -14,7 +14,7 @@ minimumReleaseAgeExclude: nodeLinker: isolated catalog: - "@chainsafe/lodestar-z": "^1.0.0" + "@chainsafe/lodestar-z": "github:ChainSafe/lodestar-z#848d561f45eaa3539e81c82de11ac52c3e27b218" "@vitest/browser": "^4.0.7" "@vitest/browser-playwright": "^4.0.7" "@vitest/coverage-v8": "^4.0.7"