From adff6c16b40d20713e7b299c2765afdac4c6a751 Mon Sep 17 00:00:00 2001 From: Rahul Guha <19rahul2003@gmail.com> Date: Tue, 30 Jun 2026 16:45:47 +0530 Subject: [PATCH 01/33] update types and constants for alpha.11 --- src/consensus_types/gloas.zig | 40 +++++++++++++++++++++++++++++++++-- src/constants/root.zig | 6 +++++- src/preset/preset.zig | 4 ++++ 3 files changed, 47 insertions(+), 3 deletions(-) diff --git a/src/consensus_types/gloas.zig b/src/consensus_types/gloas.zig index 40c3af009..69081e059 100644 --- a/src/consensus_types/gloas.zig +++ b/src/consensus_types/gloas.zig @@ -58,6 +58,9 @@ pub const HistoricalSummary = capella.HistoricalSummary; pub const BlobIdentifier = deneb.BlobIdentifier; pub const BlobKzgCommitments = deneb.BlobKzgCommitments; +pub const LogsBloom = bellatrix.LogsBloom; +pub const ExtraData = bellatrix.ExtraData; +pub const Transactions = bellatrix.Transactions; // Reuse Electra types pub const PendingDeposit = electra.PendingDeposit; @@ -66,7 +69,6 @@ pub const PendingConsolidation = electra.PendingConsolidation; pub const DepositRequest = electra.DepositRequest; pub const WithdrawalRequest = electra.WithdrawalRequest; pub const ConsolidationRequest = electra.ConsolidationRequest; -pub const ExecutionRequests = electra.ExecutionRequests; pub const SingleAttestation = electra.SingleAttestation; pub const Attestation = electra.Attestation; pub const Attestations = electra.Attestations; @@ -151,6 +153,39 @@ pub const BuilderPendingWithdrawal = ssz.FixedContainerType(struct { pub const BuilderPendingPayment = ssz.FixedContainerType(struct { weight: p.Uint64, withdrawal: BuilderPendingWithdrawal, + proposer_index: p.ValidatorIndex, +}); + +pub const BuilderDepositRequest = ssz.FixedContainerType(struct { + pubkey: p.BLSPubkey, + withdrawal_credentials: p.Bytes32, + amount: p.Gwei, + signature: p.BLSSignature, +}); + +pub const BuilderDepositRequests = ssz.FixedListType( + BuilderDepositRequest, + preset.MAX_BUILDER_DEPOSIT_REQUESTS_PER_PAYLOAD, + .{}, +); + +pub const BuilderExitRequest = ssz.FixedContainerType(struct { + source_address: p.ExecutionAddress, + pubkey: p.BLSPubkey, +}); + +pub const BuilderExitRequests = ssz.FixedListType( + BuilderExitRequest, + preset.MAX_BUILDER_EXIT_REQUESTS_PER_PAYLOAD, + .{}, +); + +pub const ExecutionRequests = ssz.VariableContainerType(struct { + deposits: ssz.FixedListType(DepositRequest, preset.MAX_DEPOSIT_REQUESTS_PER_PAYLOAD, .{}), + withdrawals: ssz.FixedListType(WithdrawalRequest, preset.MAX_WITHDRAWAL_REQUESTS_PER_PAYLOAD, .{}), + consolidations: ssz.FixedListType(ConsolidationRequest, preset.MAX_CONSOLIDATION_REQUESTS_PER_PAYLOAD, .{}), + builder_deposits: BuilderDepositRequests, + builder_exits: BuilderExitRequests, }); pub const PayloadAttestationData = ssz.FixedContainerType(struct { @@ -179,10 +214,11 @@ pub const IndexedPayloadAttestation = ssz.VariableContainerType(struct { }); pub const ProposerPreferences = ssz.FixedContainerType(struct { + dependent_root: p.Root, proposal_slot: p.Slot, validator_index: p.ValidatorIndex, fee_recipient: p.ExecutionAddress, - gas_limit: p.Uint64, + target_gas_limit: p.Uint64, }); pub const SignedProposerPreferences = ssz.FixedContainerType(struct { diff --git a/src/constants/root.zig b/src/constants/root.zig index 5aed64870..81d0616c1 100644 --- a/src/constants/root.zig +++ b/src/constants/root.zig @@ -32,6 +32,7 @@ pub const DOMAIN_BLS_TO_EXECUTION_CHANGE = [_]u8{ 10, 0, 0, 0 }; pub const DOMAIN_BEACON_BUILDER = [_]u8{ 11, 0, 0, 0 }; // 0x0B000000 pub const DOMAIN_PTC_ATTESTER = [_]u8{ 12, 0, 0, 0 }; // 0x0C000000 pub const DOMAIN_PROPOSER_PREFERENCES = [_]u8{ 13, 0, 0, 0 }; // 0x0D000000 +pub const DOMAIN_BUILDER_DEPOSIT = [_]u8{ 14, 0, 0, 0 }; // 0x0E000000 // Application specific domains @@ -39,7 +40,7 @@ pub const DOMAIN_APPLICATION_MASK = [_]u8{ 0, 0, 0, 1 }; pub const DOMAIN_APPLICATION_BUILDER = [_]u8{ 0, 0, 0, 1 }; // need to be updated when new domain is added -pub const ALL_DOMAINS = [_][4]u8{ DOMAIN_BEACON_PROPOSER, DOMAIN_BEACON_ATTESTER, DOMAIN_RANDAO, DOMAIN_DEPOSIT, DOMAIN_VOLUNTARY_EXIT, DOMAIN_SELECTION_PROOF, DOMAIN_AGGREGATE_AND_PROOF, DOMAIN_SYNC_COMMITTEE, DOMAIN_SYNC_COMMITTEE_SELECTION_PROOF, DOMAIN_CONTRIBUTION_AND_PROOF, DOMAIN_BLS_TO_EXECUTION_CHANGE, DOMAIN_BEACON_BUILDER, DOMAIN_PTC_ATTESTER, DOMAIN_PROPOSER_PREFERENCES, DOMAIN_APPLICATION_MASK, DOMAIN_APPLICATION_BUILDER }; +pub const ALL_DOMAINS = [_][4]u8{ DOMAIN_BEACON_PROPOSER, DOMAIN_BEACON_ATTESTER, DOMAIN_RANDAO, DOMAIN_DEPOSIT, DOMAIN_VOLUNTARY_EXIT, DOMAIN_SELECTION_PROOF, DOMAIN_AGGREGATE_AND_PROOF, DOMAIN_SYNC_COMMITTEE, DOMAIN_SYNC_COMMITTEE_SELECTION_PROOF, DOMAIN_CONTRIBUTION_AND_PROOF, DOMAIN_BLS_TO_EXECUTION_CHANGE, DOMAIN_BEACON_BUILDER, DOMAIN_PTC_ATTESTER, DOMAIN_PROPOSER_PREFERENCES, DOMAIN_BUILDER_DEPOSIT, DOMAIN_APPLICATION_MASK, DOMAIN_APPLICATION_BUILDER }; // Gloas (EIP-7732) misc @@ -47,6 +48,7 @@ pub const BUILDER_INDEX_FLAG: u64 = 1 << 40; // 2**40 pub const BUILDER_INDEX_SELF_BUILD: u64 = std.math.maxInt(u64); pub const BUILDER_PAYMENT_THRESHOLD_NUMERATOR = 6; pub const BUILDER_PAYMENT_THRESHOLD_DENOMINATOR = 10; +pub const PAYLOAD_BUILDER_VERSION = 0; // Participation flag indices @@ -122,6 +124,8 @@ pub const NEXT_SYNC_COMMITTEE_INDEX_ELECTRA = 23; pub const DEPOSIT_REQUEST_TYPE = 0x00; pub const WITHDRAWAL_REQUEST_TYPE = 0x01; pub const CONSOLIDATION_REQUEST_TYPE = 0x02; +pub const BUILDER_DEPOSIT_REQUEST_TYPE = 0x03; +pub const BUILDER_EXIT_REQUEST_TYPE = 0x04; pub const CURRENT_SYNC_COMMITTEE_GINDEX = 54; pub const EXECUTION_PAYLOAD_GINDEX = 25; diff --git a/src/preset/preset.zig b/src/preset/preset.zig index dd85b491a..fcf340978 100644 --- a/src/preset/preset.zig +++ b/src/preset/preset.zig @@ -85,6 +85,8 @@ const PresetMainnet = struct { pub const MAX_PENDING_DEPOSITS_PER_EPOCH = 16; pub const PTC_SIZE = 512; pub const MAX_PAYLOAD_ATTESTATIONS = 4; + pub const MAX_BUILDER_DEPOSIT_REQUESTS_PER_PAYLOAD = 256; + pub const MAX_BUILDER_EXIT_REQUESTS_PER_PAYLOAD = 16; pub const BUILDER_REGISTRY_LIMIT = 1_099_511_627_776; pub const BUILDER_PENDING_WITHDRAWALS_LIMIT = 1_048_576; pub const MAX_BUILDERS_PER_WITHDRAWALS_SWEEP = 16384; @@ -163,6 +165,8 @@ const PresetMinimal = struct { pub const MAX_PENDING_DEPOSITS_PER_EPOCH = PresetMainnet.MAX_PENDING_DEPOSITS_PER_EPOCH; pub const PTC_SIZE = 2; pub const MAX_PAYLOAD_ATTESTATIONS = 4; + pub const MAX_BUILDER_DEPOSIT_REQUESTS_PER_PAYLOAD = 256; + pub const MAX_BUILDER_EXIT_REQUESTS_PER_PAYLOAD = 16; pub const BUILDER_REGISTRY_LIMIT = 1_099_511_627_776; pub const BUILDER_PENDING_WITHDRAWALS_LIMIT = 1_048_576; pub const MAX_BUILDERS_PER_WITHDRAWALS_SWEEP = 16; From d5d9743af2d42f5fe7daf76bb23e1c15597ab530 Mon Sep 17 00:00:00 2001 From: Rahul Guha <19rahul2003@gmail.com> Date: Wed, 1 Jul 2026 13:39:38 +0530 Subject: [PATCH 02/33] process builder deposit and exit requests --- .../block/process_builder_deposit_request.zig | 58 +++ .../block/process_builder_exit_request.zig | 36 ++ .../process_execution_payload_envelope.zig | 186 +++++++++ src/state_transition/root.zig | 2 + src/state_transition/utils/gloas.zig | 358 ++++++++++++++++++ 5 files changed, 640 insertions(+) create mode 100644 src/state_transition/block/process_builder_deposit_request.zig create mode 100644 src/state_transition/block/process_builder_exit_request.zig create mode 100644 src/state_transition/block/process_execution_payload_envelope.zig create mode 100644 src/state_transition/utils/gloas.zig diff --git a/src/state_transition/block/process_builder_deposit_request.zig b/src/state_transition/block/process_builder_deposit_request.zig new file mode 100644 index 000000000..0efe83817 --- /dev/null +++ b/src/state_transition/block/process_builder_deposit_request.zig @@ -0,0 +1,58 @@ +const std = @import("std"); +const Allocator = std.mem.Allocator; +const BeaconConfig = @import("config").BeaconConfig; +const BeaconState = @import("fork_types").BeaconState; +const types = @import("consensus_types"); +const c = @import("constants"); +const computeEpochAtSlot = @import("../utils/epoch.zig").computeEpochAtSlot; +const gloas_utils = @import("../utils/gloas.zig"); +const addBuilderToRegistry = gloas_utils.addBuilderToRegistry; +const findBuilderIndexByPubkey = gloas_utils.findBuilderIndexByPubkey; +const isValidBuilderDepositSignature = gloas_utils.isValidBuilderDepositSignature; + +pub fn processBuilderDepositRequest( + allocator: Allocator, + config: *const BeaconConfig, + state: *BeaconState(.gloas), + request: *const types.gloas.BuilderDepositRequest.Type, +) !void { + const builder_index = try findBuilderIndexByPubkey(allocator, state, &request.pubkey); + + if (builder_index) |idx| { + var builders = try state.inner.get("builders"); + var builder: types.gloas.Builder.Type = undefined; + try builders.getValue(allocator, idx, &builder); + + if (builder.withdrawable_epoch != c.FAR_FUTURE_EPOCH and builder.balance == 0) { + builder.withdrawable_epoch = computeEpochAtSlot(try state.slot()) + config.chain.MIN_BUILDER_WITHDRAWABILITY_DELAY; + } + + builder.balance += request.amount; + + try builders.setValue(idx, &builder); + return; + } + + if (!isValidBuilderDepositSignature( + config, + &request.pubkey, + &request.withdrawal_credentials, + request.amount, + request.signature, + )) { + return; + } + + var execution_address: types.primitive.ExecutionAddress.Type = undefined; + @memcpy(&execution_address, request.withdrawal_credentials[12..32]); + + try addBuilderToRegistry( + allocator, + state, + &request.pubkey, + request.withdrawal_credentials[0], + &execution_address, + request.amount, + try state.slot(), + ); +} diff --git a/src/state_transition/block/process_builder_exit_request.zig b/src/state_transition/block/process_builder_exit_request.zig new file mode 100644 index 000000000..f229dea19 --- /dev/null +++ b/src/state_transition/block/process_builder_exit_request.zig @@ -0,0 +1,36 @@ +const std = @import("std"); +const Allocator = std.mem.Allocator; +const BeaconConfig = @import("config").BeaconConfig; +const BeaconState = @import("fork_types").BeaconState; +const types = @import("consensus_types"); +const gloas_utils = @import("../utils/gloas.zig"); +const findBuilderIndexByPubkey = gloas_utils.findBuilderIndexByPubkey; +const getPendingBalanceToWithdrawForBuilder = gloas_utils.getPendingBalanceToWithdrawForBuilder; +const initiateBuilderExit = gloas_utils.initiateBuilderExit; +const isActiveBuilder = gloas_utils.isActiveBuilder; + +pub fn processBuilderExitRequest( + allocator: Allocator, + config: *const BeaconConfig, + state: *BeaconState(.gloas), + request: *const types.gloas.BuilderExitRequest.Type, +) !void { + const maybe_builder_index = try findBuilderIndexByPubkey(allocator, state, &request.pubkey); + const builder_index = maybe_builder_index orelse return; + + var builders = try state.inner.get("builders"); + var builder: types.gloas.Builder.Type = undefined; + try builders.getValue(allocator, builder_index, &builder); + + if (!isActiveBuilder(&builder, try state.finalizedEpoch())) { + return; + } + if (!std.mem.eql(u8, &builder.execution_address, &request.source_address)) { + return; + } + if (try getPendingBalanceToWithdrawForBuilder(allocator, state, builder_index) != 0) { + return; + } + + try initiateBuilderExit(config, state, allocator, builder_index); +} diff --git a/src/state_transition/block/process_execution_payload_envelope.zig b/src/state_transition/block/process_execution_payload_envelope.zig new file mode 100644 index 000000000..f185c79cc --- /dev/null +++ b/src/state_transition/block/process_execution_payload_envelope.zig @@ -0,0 +1,186 @@ +const std = @import("std"); +const Allocator = std.mem.Allocator; +const types = @import("consensus_types"); +const BeaconConfig = @import("config").BeaconConfig; +const ForkSeq = @import("config").ForkSeq; +const BeaconState = @import("fork_types").BeaconState; +const EpochCache = @import("../cache/epoch_cache.zig").EpochCache; +const preset = @import("preset").preset; +const c = @import("constants"); +const processDepositRequest = @import("./process_deposit_request.zig").processDepositRequest; +const processWithdrawalRequest = @import("./process_withdrawal_request.zig").processWithdrawalRequest; +const processConsolidationRequest = @import("./process_consolidation_request.zig").processConsolidationRequest; +const processBuilderDepositRequest = @import("./process_builder_deposit_request.zig").processBuilderDepositRequest; +const processBuilderExitRequest = @import("./process_builder_exit_request.zig").processBuilderExitRequest; +const getExecutionPayloadEnvelopeSignatureSet = @import("../signature_sets/execution_payload_envelope.zig").getExecutionPayloadEnvelopeSignatureSet; +const verifySingleSignatureSet = @import("../utils/signature_sets.zig").verifySingleSignatureSet; + +pub const ProcessExecutionPayloadEnvelopeOpts = struct { + verify_signature: bool = true, + verify_state_root: bool = true, +}; + +pub fn processExecutionPayloadEnvelope( + allocator: Allocator, + config: *const BeaconConfig, + epoch_cache: *const EpochCache, + state: *BeaconState(.gloas), + signed_envelope: *const types.gloas.SignedExecutionPayloadEnvelope.Type, + opts: ProcessExecutionPayloadEnvelopeOpts, +) !void { + const envelope = &signed_envelope.message; + const payload = &envelope.payload; + if (opts.verify_signature) { + if (!(try verifyExecutionPayloadEnvelopeSignature(allocator, config, epoch_cache, state, signed_envelope))) { + return error.InvalidEnvelopeSignature; + } + } + + try validateExecutionPayloadEnvelope(allocator, config, state, envelope); + + const requests = &envelope.execution_requests; + + for (requests.deposits.items) |*deposit| { + try processDepositRequest(.gloas, allocator, config, epoch_cache, state, deposit); + } + + for (requests.withdrawals.items) |*withdrawal| { + try processWithdrawalRequest(.gloas, config, @constCast(epoch_cache), state, withdrawal); + } + + for (requests.consolidations.items) |*consolidation| { + try processConsolidationRequest(.gloas, config, epoch_cache, state, consolidation); + } + + for (requests.builder_deposits.items) |*builder_deposit| { + try processBuilderDepositRequest(allocator, config, state, builder_deposit); + } + + for (requests.builder_exits.items) |*builder_exit| { + try processBuilderExitRequest(allocator, config, state, builder_exit); + } + + // Queue the builder payment + const payment_index = preset.SLOTS_PER_EPOCH + (try state.slot() % preset.SLOTS_PER_EPOCH); + var builder_pending_payments = try state.inner.get("builder_pending_payments"); + var payment: types.gloas.BuilderPendingPayment.Type = undefined; + try builder_pending_payments.getValue(allocator, payment_index, &payment); + const amount = payment.withdrawal.amount; + + if (amount > 0) { + var builder_pending_withdrawals = try state.inner.get("builder_pending_withdrawals"); + try builder_pending_withdrawals.pushValue(&payment.withdrawal); + try state.inner.set("builder_pending_withdrawals", builder_pending_withdrawals); + } + + const default_payment = types.gloas.BuilderPendingPayment.default_value; + try builder_pending_payments.setValue(payment_index, &default_payment); + + // Cache the execution payload hash + var execution_payload_availability = try state.inner.get("execution_payload_availability"); + try execution_payload_availability.set(try state.slot() % preset.SLOTS_PER_HISTORICAL_ROOT, true); + try state.inner.setValue("latest_block_hash", &payload.block_hash); + + try state.commit(); + + if (opts.verify_state_root) { + const state_root = try state.hashTreeRoot(); + if (!std.mem.eql(u8, &envelope.state_root, state_root)) { + return error.EnvelopeStateRootMismatch; + } + } +} + +fn validateExecutionPayloadEnvelope( + allocator: Allocator, + config: *const BeaconConfig, + state: *BeaconState(.gloas), + envelope: *const types.gloas.ExecutionPayloadEnvelope.Type, +) !void { + const payload = &envelope.payload; + + // Cache latest block header state root + var latest_block_header = try state.latestBlockHeader(); + const latest_header_state_root = try latest_block_header.getFieldRoot("state_root"); + if (std.mem.eql(u8, latest_header_state_root, &c.ZERO_HASH)) { + const previous_state_root = try state.hashTreeRoot(); + try latest_block_header.setValue("state_root", previous_state_root); + try state.inner.set("latest_block_header", latest_block_header); + } + + // Verify consistency with the beacon block + const latest_block_header_root = try latest_block_header.hashTreeRoot(); + if (!std.mem.eql(u8, &envelope.beacon_block_root, latest_block_header_root)) { + return error.EnvelopeBlockRootMismatch; + } + + // Verify slot + if (envelope.slot != try state.slot()) { + return error.EnvelopeSlotMismatch; + } + + // Verify consistency with the committed bid + var committed_bid: types.gloas.ExecutionPayloadBid.Type = types.gloas.ExecutionPayloadBid.default_value; + try state.inner.getValue(allocator, "latest_execution_payload_bid", &committed_bid); + defer types.gloas.ExecutionPayloadBid.deinit(allocator, &committed_bid); + + if (envelope.builder_index != committed_bid.builder_index) { + return error.EnvelopeBuilderIndexMismatch; + } + + if (!std.mem.eql(u8, &committed_bid.prev_randao, &payload.prev_randao)) { + return error.EnvelopePrevRandaoMismatch; + } + + // Verify consistency with expected withdrawals + var payload_withdrawals_root: [32]u8 = undefined; + try types.capella.Withdrawals.hashTreeRoot(allocator, &payload.withdrawals, &payload_withdrawals_root); + var expected_withdrawals = try state.inner.get("payload_expected_withdrawals"); + var expected_withdrawals_root: [32]u8 = undefined; + try expected_withdrawals.hashTreeRootInto(&expected_withdrawals_root); + if (!std.mem.eql(u8, &payload_withdrawals_root, &expected_withdrawals_root)) { + return error.EnvelopeWithdrawalsMismatch; + } + + // Verify the gas_limit + if (committed_bid.gas_limit != payload.gas_limit) { + return error.EnvelopeGasLimitMismatch; + } + + // Verify the block hash + if (!std.mem.eql(u8, &committed_bid.block_hash, &payload.block_hash)) { + return error.EnvelopeBlockHashMismatch; + } + + // Verify consistency of the parent hash with respect to the previous execution payload + const latest_block_hash = try state.inner.getFieldRoot("latest_block_hash"); + if (!std.mem.eql(u8, &payload.parent_hash, latest_block_hash)) { + return error.EnvelopeParentHashMismatch; + } + + // Verify timestamp + // compute_timestamp_at_slot: genesis_time + slot * SECONDS_PER_SLOT + const expected_timestamp = (try state.genesisTime()) + (try state.slot()) * config.chain.SECONDS_PER_SLOT; + if (payload.timestamp != expected_timestamp) { + return error.EnvelopeTimestampMismatch; + } + + // Skipped: Verify the execution payload is valid +} + +fn verifyExecutionPayloadEnvelopeSignature( + allocator: Allocator, + config: *const BeaconConfig, + epoch_cache: *const EpochCache, + state: *BeaconState(.gloas), + signed_envelope: *const types.gloas.SignedExecutionPayloadEnvelope.Type, +) !bool { + const signature_set = try getExecutionPayloadEnvelopeSignatureSet( + allocator, + config, + epoch_cache, + state, + signed_envelope, + ); + return verifySingleSignatureSet(&signature_set); +} diff --git a/src/state_transition/root.zig b/src/state_transition/root.zig index 6681f94e4..8c6d7f788 100644 --- a/src/state_transition/root.zig +++ b/src/state_transition/root.zig @@ -78,6 +78,8 @@ pub const processBlsToExecutionChange = @import("./block/process_bls_to_executio pub const processDepositRequest = @import("./block/process_deposit_request.zig").processDepositRequest; pub const processWithdrawalRequest = @import("./block/process_withdrawal_request.zig").processWithdrawalRequest; pub const processConsolidationRequest = @import("./block/process_consolidation_request.zig").processConsolidationRequest; +pub const processBuilderDepositRequest = @import("./block/process_builder_deposit_request.zig").processBuilderDepositRequest; +pub const processBuilderExitRequest = @import("./block/process_builder_exit_request.zig").processBuilderExitRequest; // utils pub const validator_status = @import("./utils/validator_status.zig"); diff --git a/src/state_transition/utils/gloas.zig b/src/state_transition/utils/gloas.zig new file mode 100644 index 000000000..dd332846a --- /dev/null +++ b/src/state_transition/utils/gloas.zig @@ -0,0 +1,358 @@ +const std = @import("std"); +const Allocator = std.mem.Allocator; +const BeaconConfig = @import("config").BeaconConfig; +const ForkSeq = @import("config").ForkSeq; +const BeaconState = @import("fork_types").BeaconState; +const ct = @import("consensus_types"); +const preset = @import("preset").preset; +const c = @import("constants"); +const getBlockRootAtSlot = @import("./block_root.zig").getBlockRootAtSlot; +const computeEpochAtSlot = @import("./epoch.zig").computeEpochAtSlot; +const AnyBeaconState = @import("fork_types").AnyBeaconState; +const computePayloadTimelinessCommitteeForSlot = @import("./seed.zig").computePayloadTimelinessCommitteeForSlot; +const computePayloadTimelinessCommitteesForEpoch = @import("./seed.zig").computePayloadTimelinessCommitteesForEpoch; +const getSeed = @import("./seed.zig").getSeed; +const Sha256 = std.crypto.hash.sha2.Sha256; +const EpochShuffling = @import("./epoch_shuffling.zig").EpochShuffling; +const computeEpochShuffling = @import("./epoch_shuffling.zig").computeEpochShuffling; +const isActiveValidator = @import("./validator.zig").isActiveValidator; +const computeStartSlotAtEpoch = @import("./epoch.zig").computeStartSlotAtEpoch; +const RootCache = @import("../cache/root_cache.zig").RootCache; +const EpochCache = @import("../cache/epoch_cache.zig").EpochCache; +const isValidDepositSignature = @import("../block/process_deposit.zig").isValidDepositSignature; +const computeDomain = @import("./domain.zig").computeDomain; +const computeSigningRoot = @import("./signing_root.zig").computeSigningRoot; +const bls = @import("bls"); +const verify = @import("./bls.zig").verify; + +const BLSPubkey = ct.primitive.BLSPubkey.Type; +const ValidatorIndex = ct.primitive.ValidatorIndex.Type; +const ExecutionAddress = ct.primitive.ExecutionAddress.Type; +const BLSSignature = ct.primitive.BLSSignature.Type; + +pub fn isBuilderWithdrawalCredential(withdrawal_credentials: *const [32]u8) bool { + return withdrawal_credentials[0] == c.BUILDER_WITHDRAWAL_PREFIX; +} + +pub fn getBuilderPaymentQuorumThreshold(epoch_cache: *const EpochCache) u64 { + const quorum = (epoch_cache.total_active_balance_increments * preset.EFFECTIVE_BALANCE_INCREMENT / preset.SLOTS_PER_EPOCH) * + c.BUILDER_PAYMENT_THRESHOLD_NUMERATOR; + return quorum / c.BUILDER_PAYMENT_THRESHOLD_DENOMINATOR; +} + +fn hasBuilderIndexFlag(index: u64) bool { + return (index & c.BUILDER_INDEX_FLAG) != 0; +} + +pub fn isBuilderIndex(validator_index: u64) bool { + return hasBuilderIndexFlag(validator_index); +} + +pub fn convertBuilderIndexToValidatorIndex(builder_index: u64) u64 { + return if (hasBuilderIndexFlag(builder_index)) builder_index else builder_index | c.BUILDER_INDEX_FLAG; +} + +pub fn convertValidatorIndexToBuilderIndex(validator_index: u64) u64 { + return if (hasBuilderIndexFlag(validator_index)) validator_index & ~c.BUILDER_INDEX_FLAG else validator_index; +} + +pub fn isActiveBuilder(builder: *const ct.gloas.Builder.Type, finalized_epoch: u64) bool { + return builder.deposit_epoch < finalized_epoch and builder.withdrawable_epoch == c.FAR_FUTURE_EPOCH; +} + +pub fn getPendingBalanceToWithdrawForBuilder(allocator: Allocator, state: *BeaconState(.gloas), builder_index: u64) !u64 { + var pending_balance: u64 = 0; + + var withdrawals = try state.inner.get("builder_pending_withdrawals"); + const withdrawals_len = try withdrawals.length(); + var w_it = withdrawals.iteratorReadonly(0); + for (0..withdrawals_len) |_| { + const w = try w_it.nextValue(allocator); + if (w.builder_index == builder_index) { + pending_balance += w.amount; + } + } + + var payments = try state.inner.get("builder_pending_payments"); + const payments_len = ct.gloas.BeaconState.getFieldType("builder_pending_payments").length; + for (0..payments_len) |i| { + var p: ct.gloas.BuilderPendingPayment.Type = undefined; + try payments.getValue(allocator, i, &p); + if (p.withdrawal.builder_index == builder_index) { + pending_balance += p.withdrawal.amount; + } + } + + return pending_balance; +} + +pub fn canBuilderCoverBid(allocator: Allocator, state: *BeaconState(.gloas), builder_index: u64, bid_amount: u64) !bool { + var builders = try state.inner.get("builders"); + var builder: ct.gloas.Builder.Type = undefined; + try builders.getValue(allocator, builder_index, &builder); + + const pending_balance = try getPendingBalanceToWithdrawForBuilder(allocator, state, builder_index); + const min_balance = preset.MIN_DEPOSIT_AMOUNT + pending_balance; + + if (builder.balance < min_balance) return false; + return builder.balance - min_balance >= bid_amount; +} + +pub fn initiateBuilderExit(config: *const BeaconConfig, state: *BeaconState(.gloas), allocator: Allocator, builder_index: u64) !void { + var builders = try state.inner.get("builders"); + var builder: ct.gloas.Builder.Type = undefined; + try builders.getValue(allocator, builder_index, &builder); + + if (builder.withdrawable_epoch != c.FAR_FUTURE_EPOCH) return; + + const current_epoch = computeEpochAtSlot(try state.slot()); + builder.withdrawable_epoch = current_epoch + config.chain.MIN_BUILDER_WITHDRAWABILITY_DELAY; + try builders.setValue(builder_index, &builder); +} + +pub fn findBuilderIndexByPubkey(allocator: Allocator, state: *BeaconState(.gloas), pubkey: *const BLSPubkey) !?usize { + var builders = try state.inner.get("builders"); + const len = try builders.length(); + for (0..len) |i| { + var b: ct.gloas.Builder.Type = undefined; + try builders.getValue(allocator, i, &b); + if (std.mem.eql(u8, &b.pubkey, pubkey)) return i; + } + return null; +} + +pub fn addBuilderToRegistry( + allocator: Allocator, + state: *BeaconState(.gloas), + pubkey: *const BLSPubkey, + version: u8, + execution_address: *const ExecutionAddress, + amount: u64, + slot: u64, +) !void { + var builders = try state.inner.get("builders"); + const len = try builders.length(); + const current_epoch = computeEpochAtSlot(try state.slot()); + + var builder_index: ?usize = null; + for (0..len) |i| { + var builder: ct.gloas.Builder.Type = undefined; + try builders.getValue(allocator, i, &builder); + if (builder.withdrawable_epoch <= current_epoch and builder.balance == 0) { + builder_index = i; + break; + } + } + + const new_builder = ct.gloas.Builder.Type{ + .pubkey = pubkey.*, + .version = version, + .execution_address = execution_address.*, + .balance = amount, + .deposit_epoch = computeEpochAtSlot(slot), + .withdrawable_epoch = c.FAR_FUTURE_EPOCH, + }; + + if (builder_index) |idx| { + try builders.setValue(idx, &new_builder); + } else { + try builders.pushValue(&new_builder); + try state.inner.set("builders", builders); + } +} + +pub fn isValidBuilderDepositSignature( + config: *const BeaconConfig, + pubkey: *const BLSPubkey, + withdrawal_credentials: *const [32]u8, + amount: u64, + deposit_signature: BLSSignature, +) bool { + const deposit_message = ct.phase0.DepositMessage.Type{ + .pubkey = pubkey.*, + .withdrawal_credentials = withdrawal_credentials.*, + .amount = amount, + }; + + var domain: ct.primitive.Domain.Type = undefined; + computeDomain(c.DOMAIN_BUILDER_DEPOSIT, config.chain.GENESIS_FORK_VERSION, c.ZERO_HASH, &domain) catch return false; + + var signing_root: [32]u8 = undefined; + computeSigningRoot(ct.phase0.DepositMessage, &deposit_message, &domain, &signing_root) catch return false; + + const public_key = bls.PublicKey.uncompress(pubkey) catch return false; + public_key.validate() catch return false; + const signature = bls.Signature.uncompress(&deposit_signature) catch return false; + signature.validate(true) catch return false; + verify(&signing_root, &public_key, &signature, .{}) catch return false; + return true; +} + +pub fn isAttestationSameSlot(state: *BeaconState(.gloas), data: *const ct.phase0.AttestationData.Type) !bool { + if (data.slot == 0) return true; + + const block_root = try getBlockRootAtSlot(.gloas, state, data.slot); + const is_matching = std.mem.eql(u8, &data.beacon_block_root, block_root); + + const prev_block_root = try getBlockRootAtSlot(.gloas, state, data.slot - 1); + const is_current = !std.mem.eql(u8, &data.beacon_block_root, prev_block_root); + + return is_matching and is_current; +} + +pub fn isAttestationSameSlotRootCache(root_cache: *RootCache(.gloas), data: *const ct.phase0.AttestationData.Type) !bool { + if (data.slot == 0) return true; + + const block_root = try root_cache.getBlockRootAtSlot(data.slot); + const is_matching = std.mem.eql(u8, &data.beacon_block_root, block_root); + + const prev_block_root = try root_cache.getBlockRootAtSlot(data.slot - 1); + const is_current = !std.mem.eql(u8, &data.beacon_block_root, prev_block_root); + + return is_matching and is_current; +} + +// TODO: Need to check if these are used at all +/// computePayloadTimelinessCommitteeIndices uses the optimized increments-based check instead. +pub fn isBalanceWeightedAcceptance(effective_balance: u64, random_value: u64) bool { + const MAX_RANDOM_VALUE: u64 = 0xFFFF; + return effective_balance * MAX_RANDOM_VALUE >= preset.MAX_EFFECTIVE_BALANCE_ELECTRA * random_value; +} + +// TODO: Need to check if these are used at all +/// computePayloadTimelinessCommitteeIndices inlines this logic with increments instead of full Gwei balances. +pub fn computeBalanceWeightedAcceptance(effective_balance: u64, seed: *const [32]u8, i: u64) bool { + var hash_input: [40]u8 = undefined; + @memcpy(hash_input[0..32], seed); + std.mem.writeInt(u64, hash_input[32..][0..8], i / 16, .little); + + var random_bytes: [32]u8 = undefined; + Sha256.hash(&hash_input, &random_bytes, .{}); + + const offset: usize = @intCast((i % 16) * 2); + const random_value: u64 = std.mem.readInt(u16, random_bytes[offset..][0..2], .little); + + return isBalanceWeightedAcceptance(effective_balance, random_value); +} + +pub fn isParentBlockFull(state: *BeaconState(.gloas)) !bool { + var bid = try state.inner.get("latest_execution_payload_bid"); + const bid_block_hash = try bid.getFieldRoot("block_hash"); + const latest_block_hash = try state.inner.getFieldRoot("latest_block_hash"); + return std.mem.eql(u8, bid_block_hash, latest_block_hash); +} + +pub fn isPubkeyInList(list: []const BLSPubkey, pubkey: *const BLSPubkey) bool { + for (list) |p| { + if (std.mem.eql(u8, &p, pubkey)) return true; + } + return false; +} + +pub fn isPendingValidator( + comptime fork: ForkSeq, + allocator: Allocator, + config: *const BeaconConfig, + state: *BeaconState(fork), + pubkey: *const BLSPubkey, +) !bool { + var pending_deposits = try state.pendingDeposits(); + const pending_deposits_len = try pending_deposits.length(); + var pending_it = pending_deposits.iteratorReadonly(0); + + for (0..pending_deposits_len) |_| { + const pending_deposit = try pending_it.nextValue(allocator); + if (!std.mem.eql(u8, &pending_deposit.pubkey, pubkey)) { + continue; + } + + if (isValidDepositSignature( + config, + &pending_deposit.pubkey, + &pending_deposit.withdrawal_credentials, + pending_deposit.amount, + pending_deposit.signature, + )) { + return true; + } + } + + return false; +} + +pub fn computePtc( + allocator: Allocator, + state: *BeaconState(.gloas), + slot: u64, + shuffling: ?*EpochShuffling, + effective_balance_increments: []const u16, +) ![ct.gloas.PtcWindow.Element.length]ValidatorIndex { + const epoch = computeEpochAtSlot(slot); + + const epoch_shuffling = if (shuffling) |s| s else blk: { + var validators = try state.validators(); + const validator_count = try validators.length(); + var active_indices: std.ArrayList(ValidatorIndex) = .empty; + defer active_indices.deinit(allocator); + for (0..validator_count) |i| { + var validator: ct.phase0.Validator.Type = undefined; + try validators.getValue(undefined, i, &validator); + if (isActiveValidator(&validator, epoch)) { + try active_indices.append(allocator, @intCast(i)); + } + } + var any_state = AnyBeaconState{ .gloas = state.inner }; + break :blk try computeEpochShuffling(allocator, &any_state, try active_indices.toOwnedSlice(allocator), epoch); + }; + defer if (shuffling == null) epoch_shuffling.deinit(); + + var epoch_seed: [32]u8 = undefined; + try getSeed(.gloas, state, epoch, c.DOMAIN_PTC_ATTESTER, &epoch_seed); + + var slot_seed_input: [40]u8 = undefined; + @memcpy(slot_seed_input[0..32], &epoch_seed); + std.mem.writeInt(u64, slot_seed_input[32..][0..8], slot, .little); + + var slot_seed: [32]u8 = undefined; + Sha256.hash(&slot_seed_input, &slot_seed, .{}); + + const slot_committees = epoch_shuffling.committees[slot % preset.SLOTS_PER_EPOCH]; + return computePayloadTimelinessCommitteeForSlot(allocator, &slot_seed, slot_committees, effective_balance_increments); +} + +pub fn initializePtcWindow( + comptime fork: ForkSeq, + allocator: Allocator, + epoch_cache: *const EpochCache, + state: *BeaconState(fork), +) ![ct.gloas.PtcWindow.length][ct.gloas.PtcWindow.Element.length]ValidatorIndex { + const PtcSize = ct.gloas.PtcWindow.Element.length; + const PtcWindowLen = ct.gloas.PtcWindow.length; + + const empty_previous_epoch = [_][PtcSize]ValidatorIndex{ + [_]ValidatorIndex{0} ** PtcSize, + } ** preset.SLOTS_PER_EPOCH; + + const current_epoch = computeEpochAtSlot(try state.slot()); + var current_and_lookahead: [(1 + preset.MIN_SEED_LOOKAHEAD) * preset.SLOTS_PER_EPOCH][PtcSize]ValidatorIndex = undefined; + for (0..1 + preset.MIN_SEED_LOOKAHEAD) |epoch_offset| { + const epoch = current_epoch + epoch_offset; + const epoch_committees = try computePayloadTimelinessCommitteesForEpoch( + fork, + allocator, + state, + epoch, + epoch_cache, + ); + for (0..preset.SLOTS_PER_EPOCH) |slot_index| { + current_and_lookahead[epoch_offset * preset.SLOTS_PER_EPOCH + slot_index] = epoch_committees[slot_index]; + } + } + + // return empty_previous_epoch + current_and_lookahead + var ptc_window: [PtcWindowLen][PtcSize]ValidatorIndex = undefined; + @memcpy(ptc_window[0..preset.SLOTS_PER_EPOCH], &empty_previous_epoch); + @memcpy(ptc_window[preset.SLOTS_PER_EPOCH..], ¤t_and_lookahead); + + return ptc_window; +} From f1c7b9224d8489802b2d5e9ecb6caec71eb2f9bb Mon Sep 17 00:00:00 2001 From: Rahul Guha <19rahul2003@gmail.com> Date: Wed, 1 Jul 2026 13:48:31 +0530 Subject: [PATCH 03/33] remove builder voluntary exit handling --- .../block/process_voluntary_exit.zig | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/state_transition/block/process_voluntary_exit.zig b/src/state_transition/block/process_voluntary_exit.zig index 3ff089802..9e14c208a 100644 --- a/src/state_transition/block/process_voluntary_exit.zig +++ b/src/state_transition/block/process_voluntary_exit.zig @@ -1,3 +1,4 @@ +const std = @import("std"); const BeaconConfig = @import("config").BeaconConfig; const ForkSeq = @import("config").ForkSeq; const EpochCache = @import("../cache/epoch_cache.zig").EpochCache; @@ -58,8 +59,14 @@ pub fn getVoluntaryExitValidity( signed_voluntary_exit: *const SignedVoluntaryExit, verify_signature: bool, ) !VoluntaryExitValidity { + const current_epoch = epoch_cache.epoch; const voluntary_exit = signed_voluntary_exit.message; + // Exits must specify an epoch when they become valid; they are not valid before then + if (current_epoch < voluntary_exit.epoch) { + return .early_epoch; + } + var validators = try state.validators(); const validators_len = try validators.length(); if (voluntary_exit.validator_index >= validators_len) { @@ -67,7 +74,6 @@ pub fn getVoluntaryExitValidity( } var validator = try validators.get(@intCast(voluntary_exit.validator_index)); - const current_epoch = epoch_cache.epoch; // verify the validator is active if (!try isActiveValidatorView(validator, current_epoch)) { @@ -80,11 +86,6 @@ pub fn getVoluntaryExitValidity( return .already_exited; } - // exits must specify an epoch when they become valid; they are not valid before then - if (current_epoch < voluntary_exit.epoch) { - return .early_epoch; - } - // verify the validator had been active long enough const activation_epoch = try validator.get("activation_epoch"); if (current_epoch < activation_epoch + config.chain.SHARD_COMMITTEE_PERIOD) { @@ -108,7 +109,6 @@ pub fn getVoluntaryExitValidity( return .valid; } -const std = @import("std"); const TestCachedBeaconState = @import("../test_utils/root.zig").TestCachedBeaconState; const Node = @import("persistent_merkle_tree").Node; const preset = @import("preset").preset; From 1c2a7451e10dd82539e9aa4fa6d9f6bec9b4948a Mon Sep 17 00:00:00 2001 From: Rahul Guha <19rahul2003@gmail.com> Date: Wed, 1 Jul 2026 14:31:18 +0530 Subject: [PATCH 04/33] process parent executionpayload --- src/state_transition/block/process_block.zig | 4 + .../process_parent_execution_payload.zig | 122 ++++++++++++++++++ src/state_transition/root.zig | 3 + src/state_transition/slot/process_slot.zig | 12 ++ 4 files changed, 141 insertions(+) create mode 100644 src/state_transition/block/process_parent_execution_payload.zig diff --git a/src/state_transition/block/process_block.zig b/src/state_transition/block/process_block.zig index 5820b7d8d..74b225c10 100644 --- a/src/state_transition/block/process_block.zig +++ b/src/state_transition/block/process_block.zig @@ -24,6 +24,7 @@ const processRandao = @import("./process_randao.zig").processRandao; const processSyncAggregate = @import("./process_sync_committee.zig").processSyncAggregate; const processWithdrawals = @import("./process_withdrawals.zig").processWithdrawals; const getExpectedWithdrawals = @import("./process_withdrawals.zig").getExpectedWithdrawals; +const processParentExecutionPayload = @import("./process_parent_execution_payload.zig").processParentExecutionPayload; const isExecutionEnabled = @import("../utils/execution.zig").isExecutionEnabled; // TODO: proposer reward api // const ProposerRewardType = @import("../types/proposer_reward.zig").ProposerRewardType; @@ -48,6 +49,9 @@ pub fn processBlock( ) !void { // Build slashings cache against the *current* latest_block_header slot (pre-header update). try buildSlashingsCacheIfNeeded(allocator, state, slashings_cache); + if (comptime fork.gte(.gloas) and block_type == .full) { + try processParentExecutionPayload(allocator, config, epoch_cache, state, block); + } try processBlockHeader(fork, allocator, epoch_cache, state, block_type, block); // Keep cache slot in sync with latest_block_header without forcing a rebuild. slashings_cache.updateLatestBlockSlot(block.slot()); diff --git a/src/state_transition/block/process_parent_execution_payload.zig b/src/state_transition/block/process_parent_execution_payload.zig new file mode 100644 index 000000000..337efc26a --- /dev/null +++ b/src/state_transition/block/process_parent_execution_payload.zig @@ -0,0 +1,122 @@ +const std = @import("std"); +const Allocator = std.mem.Allocator; +const BeaconConfig = @import("config").BeaconConfig; +const BeaconState = @import("fork_types").BeaconState; +const BeaconBlock = @import("fork_types").BeaconBlock; +const EpochCache = @import("../cache/epoch_cache.zig").EpochCache; +const types = @import("consensus_types"); +const preset = @import("preset").preset; +const processDepositRequest = @import("./process_deposit_request.zig").processDepositRequest; +const processWithdrawalRequest = @import("./process_withdrawal_request.zig").processWithdrawalRequest; +const processConsolidationRequest = @import("./process_consolidation_request.zig").processConsolidationRequest; +const processBuilderDepositRequest = @import("./process_builder_deposit_request.zig").processBuilderDepositRequest; +const processBuilderExitRequest = @import("./process_builder_exit_request.zig").processBuilderExitRequest; +const computeEpochAtSlot = @import("../utils/epoch.zig").computeEpochAtSlot; + +pub fn processParentExecutionPayload( + allocator: Allocator, + config: *const BeaconConfig, + epoch_cache: *EpochCache, + state: *BeaconState(.gloas), + block: *const BeaconBlock(.full, .gloas), +) !void { + const bid = &block.body().inner.signed_execution_payload_bid.message; + var parent_bid = types.gloas.ExecutionPayloadBid.default_value; + try state.inner.getValue(allocator, "latest_execution_payload_bid", &parent_bid); + defer types.gloas.ExecutionPayloadBid.deinit(allocator, &parent_bid); + + const requests = &block.body().inner.parent_execution_requests; + const is_parent_block_full = std.mem.eql(u8, &bid.parent_block_hash, &parent_bid.block_hash); + if (!is_parent_block_full) { + try assertEmptyExecutionRequests(requests); + return; + } + + var requests_root: [32]u8 = undefined; + try types.gloas.ExecutionRequests.hashTreeRoot(allocator, requests, &requests_root); + if (!std.mem.eql(u8, &requests_root, &parent_bid.execution_requests_root)) { + return error.ParentExecutionRequestsRootMismatch; + } + + try applyParentExecutionPayload(allocator, config, epoch_cache, state, requests, &parent_bid); +} + +pub fn applyParentExecutionPayload( + allocator: Allocator, + config: *const BeaconConfig, + epoch_cache: *EpochCache, + state: *BeaconState(.gloas), + requests: *const types.gloas.ExecutionRequests.Type, + parent_bid: *const types.gloas.ExecutionPayloadBid.Type, +) !void { + const parent_slot = parent_bid.slot; + const parent_epoch = computeEpochAtSlot(parent_slot); + const current_epoch = computeEpochAtSlot(try state.slot()); + + for (requests.deposits.items) |*deposit| { + try processDepositRequest(.gloas, state, deposit); + } + + for (requests.withdrawals.items) |*withdrawal| { + try processWithdrawalRequest(.gloas, config, epoch_cache, state, withdrawal); + } + + for (requests.consolidations.items) |*consolidation| { + try processConsolidationRequest(.gloas, config, epoch_cache, state, consolidation); + } + + for (requests.builder_deposits.items) |*builder_deposit| { + try processBuilderDepositRequest(allocator, config, state, builder_deposit); + } + + for (requests.builder_exits.items) |*builder_exit| { + try processBuilderExitRequest(allocator, config, state, builder_exit); + } + + if (parent_epoch == current_epoch) { + try settleBuilderPayment(allocator, state, preset.SLOTS_PER_EPOCH + (parent_slot % preset.SLOTS_PER_EPOCH)); + } else if (parent_epoch + 1 == current_epoch) { + try settleBuilderPayment(allocator, state, parent_slot % preset.SLOTS_PER_EPOCH); + } else if (parent_bid.value > 0) { + var builder_pending_withdrawals = try state.inner.get("builder_pending_withdrawals"); + const withdrawal = types.gloas.BuilderPendingWithdrawal.Type{ + .fee_recipient = parent_bid.fee_recipient, + .amount = parent_bid.value, + .builder_index = parent_bid.builder_index, + }; + try builder_pending_withdrawals.pushValue(&withdrawal); + try state.inner.set("builder_pending_withdrawals", builder_pending_withdrawals); + } + + var execution_payload_availability = try state.inner.get("execution_payload_availability"); + try execution_payload_availability.set(parent_slot % preset.SLOTS_PER_HISTORICAL_ROOT, true); + try state.inner.set("execution_payload_availability", execution_payload_availability); + try state.inner.setValue("latest_block_hash", &parent_bid.block_hash); +} + +fn settleBuilderPayment(allocator: Allocator, state: *BeaconState(.gloas), payment_index: u64) !void { + var builder_pending_payments = try state.inner.get("builder_pending_payments"); + if (payment_index >= 2 * preset.SLOTS_PER_EPOCH) return error.InvalidBuilderPendingPaymentIndex; + + var payment: types.gloas.BuilderPendingPayment.Type = undefined; + try builder_pending_payments.getValue(allocator, payment_index, &payment); + if (payment.withdrawal.amount > 0) { + var builder_pending_withdrawals = try state.inner.get("builder_pending_withdrawals"); + try builder_pending_withdrawals.pushValue(&payment.withdrawal); + try state.inner.set("builder_pending_withdrawals", builder_pending_withdrawals); + } + + const default_payment = types.gloas.BuilderPendingPayment.default_value; + try builder_pending_payments.setValue(payment_index, &default_payment); +} + +fn assertEmptyExecutionRequests(requests: *const types.gloas.ExecutionRequests.Type) !void { + if (requests.deposits.items.len != 0 or + requests.withdrawals.items.len != 0 or + requests.consolidations.items.len != 0 or + requests.builder_deposits.items.len != 0 or + requests.builder_exits.items.len != 0) + { + return error.ParentExecutionRequestsNotEmpty; + } +} diff --git a/src/state_transition/root.zig b/src/state_transition/root.zig index 8c6d7f788..b7fed11b1 100644 --- a/src/state_transition/root.zig +++ b/src/state_transition/root.zig @@ -109,6 +109,9 @@ pub const preset = @import("preset").preset; const EpochShuffling = @import("./utils/epoch_shuffling.zig"); pub const calculateShufflingDecisionRoot = EpochShuffling.calculateShufflingDecisionRoot; pub const processProposerLookahead = @import("./epoch/process_proposer_lookahead.zig").processProposerLookahead; +pub const processExecutionPayloadEnvelope = @import("./block/process_execution_payload_envelope.zig").processExecutionPayloadEnvelope; +pub const processParentExecutionPayload = @import("./block/process_parent_execution_payload.zig").processParentExecutionPayload; +pub const applyParentExecutionPayload = @import("./block/process_parent_execution_payload.zig").applyParentExecutionPayload; const load_state = @import("load_state.zig"); pub const loadState = load_state.loadState; diff --git a/src/state_transition/slot/process_slot.zig b/src/state_transition/slot/process_slot.zig index 17652306c..0ba9722dc 100644 --- a/src/state_transition/slot/process_slot.zig +++ b/src/state_transition/slot/process_slot.zig @@ -22,4 +22,16 @@ pub fn processSlot(state: *AnyBeaconState) !void { const previous_block_root = try latest_block_header.hashTreeRoot(); var block_roots = try state.blockRoots(); try block_roots.setValue(try state.slot() % preset.SLOTS_PER_HISTORICAL_ROOT, previous_block_root[0..]); + + if (state.forkSeq().gte(.gloas)) { + const nextSlotIndex = (try state.slot() + 1) % preset.SLOTS_PER_HISTORICAL_ROOT; + switch (state.*) { + .gloas => |s| { + var executionPayloadAvailability = try s.get("execution_payload_availability"); + try executionPayloadAvailability.set(nextSlotIndex, false); + try s.set("execution_payload_availability", executionPayloadAvailability); + }, + else => {}, + } + } } From e92b8379bf7480608c8ec0601662f19d0672de7c Mon Sep 17 00:00:00 2001 From: Rahul Guha <19rahul2003@gmail.com> Date: Wed, 1 Jul 2026 20:43:18 +0530 Subject: [PATCH 05/33] update Gloas execution payload bid checks --- src/state_transition/block/process_block.zig | 12 +++ .../block/process_execution_payload_bid.zig | 97 +++++++++++++++++++ .../process_execution_payload_envelope.zig | 39 +++++--- .../execution_payload_envelope.zig | 49 ++++++++++ 4 files changed, 183 insertions(+), 14 deletions(-) create mode 100644 src/state_transition/block/process_execution_payload_bid.zig create mode 100644 src/state_transition/signature_sets/execution_payload_envelope.zig diff --git a/src/state_transition/block/process_block.zig b/src/state_transition/block/process_block.zig index 74b225c10..bf522241a 100644 --- a/src/state_transition/block/process_block.zig +++ b/src/state_transition/block/process_block.zig @@ -24,6 +24,7 @@ const processRandao = @import("./process_randao.zig").processRandao; const processSyncAggregate = @import("./process_sync_committee.zig").processSyncAggregate; const processWithdrawals = @import("./process_withdrawals.zig").processWithdrawals; const getExpectedWithdrawals = @import("./process_withdrawals.zig").getExpectedWithdrawals; +const processExecutionPayloadBid = @import("./process_execution_payload_bid.zig").processExecutionPayloadBid; const processParentExecutionPayload = @import("./process_parent_execution_payload.zig").processParentExecutionPayload; const isExecutionEnabled = @import("../utils/execution.zig").isExecutionEnabled; // TODO: proposer reward api @@ -105,6 +106,17 @@ pub fn processBlock( } } + if (comptime fork.gte(.gloas)) { + if (comptime block_type != .full) return error.InvalidBlockTypeForFork; + try processExecutionPayloadBid( + allocator, + config, + epoch_cache, + state, + &body.inner.signed_execution_payload_bid, + ); + } + try processRandao(fork, config, epoch_cache, state, block_type, body, block.proposerIndex(), opts.verify_signature); try processEth1Data(fork, state, body.eth1Data()); try processOperations(fork, allocator, config, epoch_cache, state, slashings_cache, block_type, body, opts); diff --git a/src/state_transition/block/process_execution_payload_bid.zig b/src/state_transition/block/process_execution_payload_bid.zig new file mode 100644 index 000000000..7313b680e --- /dev/null +++ b/src/state_transition/block/process_execution_payload_bid.zig @@ -0,0 +1,97 @@ +const std = @import("std"); +const Allocator = std.mem.Allocator; +const BeaconConfig = @import("config").BeaconConfig; +const BeaconState = @import("fork_types").BeaconState; + +const EpochCache = @import("../cache/epoch_cache.zig").EpochCache; +const types = @import("consensus_types"); +const preset = @import("preset").preset; +const c = @import("constants"); +const bls = @import("bls"); +const getRandaoMix = @import("../utils/seed.zig").getRandaoMix; +const computeEpochAtSlot = @import("../utils/epoch.zig").computeEpochAtSlot; +const gloas_utils = @import("../utils/gloas.zig"); +const isActiveBuilder = gloas_utils.isActiveBuilder; +const canBuilderCoverBid = gloas_utils.canBuilderCoverBid; +const verify = @import("../utils/bls.zig").verify; +const getExecutionPayloadBidSigningRoot = @import("../signature_sets/execution_payload_bid.zig").getExecutionPayloadBidSigningRoot; +const getBlockRootAtSlot = @import("../utils/block_root.zig").getBlockRootAtSlot; + +pub fn processExecutionPayloadBid( + allocator: Allocator, + config: *const BeaconConfig, + epoch_cache: *const EpochCache, + state: *BeaconState(.gloas), + signed_bid: *const types.gloas.SignedExecutionPayloadBid.Type, +) !void { + const bid = &signed_bid.message; + const builder_index = bid.builder_index; + const amount = bid.value; + const state_slot = try state.slot(); + + if (builder_index == c.BUILDER_INDEX_SELF_BUILD) { + if (amount != 0) return error.SelfBuildNonZeroAmount; + if (!std.mem.eql(u8, &signed_bid.signature, &c.G2_POINT_AT_INFINITY)) return error.SelfBuildNonZeroSignature; + } else { + var builders = try state.inner.get("builders"); + var builder: types.gloas.Builder.Type = undefined; + try builders.getValue(allocator, builder_index, &builder); + + const finalized_epoch = try state.finalizedEpoch(); + if (!isActiveBuilder(&builder, finalized_epoch)) return error.BuilderNotActive; + if (builder.version != c.PAYLOAD_BUILDER_VERSION) return error.InvalidBuilderVersion; + + if (!(try canBuilderCoverBid(allocator, state, builder_index, amount))) return error.BuilderInsufficientBalance; + + if (!(try verifyExecutionPayloadBidSignature(allocator, config, state_slot, &builder.pubkey, signed_bid))) return error.InvalidBidSignature; + } + + if (bid.slot != state_slot) return error.BidSlotMismatch; + if (state_slot <= c.GENESIS_SLOT) return error.ExecutionPayloadBidAtGenesis; + + const latest_block_hash = try state.inner.getFieldRoot("latest_block_hash"); + if (!std.mem.eql(u8, &bid.parent_block_hash, latest_block_hash)) return error.BidParentBlockHashMismatch; + + const parent_block_root = try getBlockRootAtSlot(.gloas, state, state_slot - 1); + if (!std.mem.eql(u8, &bid.parent_block_root, parent_block_root)) return error.BidParentBlockRootMismatch; + + const current_epoch = computeEpochAtSlot(state_slot); + const state_randao = try getRandaoMix(.gloas, state, current_epoch); + if (!std.mem.eql(u8, &bid.prev_randao, state_randao)) return error.BidPrevRandaoMismatch; + + // Verify commitments are under limit + const max_blobs_per_block = config.getMaxBlobsPerBlock(current_epoch); + if (bid.blob_kzg_commitments.items.len > max_blobs_per_block) return error.TooManyBlobCommitments; + + if (amount > 0) { + const pending_payment = types.gloas.BuilderPendingPayment.Type{ + .weight = 0, + .withdrawal = .{ + .fee_recipient = bid.fee_recipient, + .amount = amount, + .builder_index = builder_index, + }, + .proposer_index = try epoch_cache.getBeaconProposer(state_slot), + }; + var builder_pending_payments = try state.inner.get("builder_pending_payments"); + const payment_index = preset.SLOTS_PER_EPOCH + (bid.slot % preset.SLOTS_PER_EPOCH); + try builder_pending_payments.setValue(payment_index, &pending_payment); + } + + try state.inner.setValue("latest_execution_payload_bid", bid); +} + +fn verifyExecutionPayloadBidSignature( + allocator: Allocator, + config: *const BeaconConfig, + state_slot: u64, + pubkey: *const [48]u8, + signed_bid: *const types.gloas.SignedExecutionPayloadBid.Type, +) !bool { + const signing_root = try getExecutionPayloadBidSigningRoot(allocator, config, state_slot, &signed_bid.message); + + const public_key = bls.PublicKey.uncompress(pubkey) catch return false; + const signature = bls.Signature.uncompress(&signed_bid.signature) catch return false; + verify(&signing_root, &public_key, &signature, .{}) catch return false; + return true; +} diff --git a/src/state_transition/block/process_execution_payload_envelope.zig b/src/state_transition/block/process_execution_payload_envelope.zig index f185c79cc..a21147a3c 100644 --- a/src/state_transition/block/process_execution_payload_envelope.zig +++ b/src/state_transition/block/process_execution_payload_envelope.zig @@ -18,6 +18,7 @@ const verifySingleSignatureSet = @import("../utils/signature_sets.zig").verifySi pub const ProcessExecutionPayloadEnvelopeOpts = struct { verify_signature: bool = true, verify_state_root: bool = true, + verify_execution_requests_root: bool = true, }; pub fn processExecutionPayloadEnvelope( @@ -36,12 +37,11 @@ pub fn processExecutionPayloadEnvelope( } } - try validateExecutionPayloadEnvelope(allocator, config, state, envelope); + const block_slot = try validateExecutionPayloadEnvelope(allocator, config, state, envelope, opts.verify_execution_requests_root); const requests = &envelope.execution_requests; - for (requests.deposits.items) |*deposit| { - try processDepositRequest(.gloas, allocator, config, epoch_cache, state, deposit); + try processDepositRequest(.gloas, state, deposit); } for (requests.withdrawals.items) |*withdrawal| { @@ -61,7 +61,7 @@ pub fn processExecutionPayloadEnvelope( } // Queue the builder payment - const payment_index = preset.SLOTS_PER_EPOCH + (try state.slot() % preset.SLOTS_PER_EPOCH); + const payment_index = preset.SLOTS_PER_EPOCH + (block_slot % preset.SLOTS_PER_EPOCH); var builder_pending_payments = try state.inner.get("builder_pending_payments"); var payment: types.gloas.BuilderPendingPayment.Type = undefined; try builder_pending_payments.getValue(allocator, payment_index, &payment); @@ -78,17 +78,13 @@ pub fn processExecutionPayloadEnvelope( // Cache the execution payload hash var execution_payload_availability = try state.inner.get("execution_payload_availability"); - try execution_payload_availability.set(try state.slot() % preset.SLOTS_PER_HISTORICAL_ROOT, true); + try execution_payload_availability.set(block_slot % preset.SLOTS_PER_HISTORICAL_ROOT, true); + try state.inner.set("execution_payload_availability", execution_payload_availability); try state.inner.setValue("latest_block_hash", &payload.block_hash); try state.commit(); - if (opts.verify_state_root) { - const state_root = try state.hashTreeRoot(); - if (!std.mem.eql(u8, &envelope.state_root, state_root)) { - return error.EnvelopeStateRootMismatch; - } - } + _ = opts.verify_state_root; } fn validateExecutionPayloadEnvelope( @@ -96,7 +92,8 @@ fn validateExecutionPayloadEnvelope( config: *const BeaconConfig, state: *BeaconState(.gloas), envelope: *const types.gloas.ExecutionPayloadEnvelope.Type, -) !void { + verify_execution_requests_root: bool, +) !u64 { const payload = &envelope.payload; // Cache latest block header state root @@ -113,9 +110,14 @@ fn validateExecutionPayloadEnvelope( if (!std.mem.eql(u8, &envelope.beacon_block_root, latest_block_header_root)) { return error.EnvelopeBlockRootMismatch; } + const latest_header_parent_root = try latest_block_header.getFieldRoot("parent_root"); + if (!std.mem.eql(u8, &envelope.parent_beacon_block_root, latest_header_parent_root)) { + return error.EnvelopeParentBlockRootMismatch; + } // Verify slot - if (envelope.slot != try state.slot()) { + const block_slot = try latest_block_header.get("slot"); + if (payload.slot_number != block_slot) { return error.EnvelopeSlotMismatch; } @@ -132,6 +134,14 @@ fn validateExecutionPayloadEnvelope( return error.EnvelopePrevRandaoMismatch; } + if (verify_execution_requests_root) { + var execution_requests_root: [32]u8 = undefined; + try types.gloas.ExecutionRequests.hashTreeRoot(allocator, &envelope.execution_requests, &execution_requests_root); + if (!std.mem.eql(u8, &committed_bid.execution_requests_root, &execution_requests_root)) { + return error.EnvelopeExecutionRequestsRootMismatch; + } + } + // Verify consistency with expected withdrawals var payload_withdrawals_root: [32]u8 = undefined; try types.capella.Withdrawals.hashTreeRoot(allocator, &payload.withdrawals, &payload_withdrawals_root); @@ -160,12 +170,13 @@ fn validateExecutionPayloadEnvelope( // Verify timestamp // compute_timestamp_at_slot: genesis_time + slot * SECONDS_PER_SLOT - const expected_timestamp = (try state.genesisTime()) + (try state.slot()) * config.chain.SECONDS_PER_SLOT; + const expected_timestamp = (try state.genesisTime()) + block_slot * config.chain.SECONDS_PER_SLOT; if (payload.timestamp != expected_timestamp) { return error.EnvelopeTimestampMismatch; } // Skipped: Verify the execution payload is valid + return block_slot; } fn verifyExecutionPayloadEnvelopeSignature( diff --git a/src/state_transition/signature_sets/execution_payload_envelope.zig b/src/state_transition/signature_sets/execution_payload_envelope.zig new file mode 100644 index 000000000..f8586581d --- /dev/null +++ b/src/state_transition/signature_sets/execution_payload_envelope.zig @@ -0,0 +1,49 @@ +const std = @import("std"); +const Allocator = std.mem.Allocator; +const types = @import("consensus_types"); +const BeaconConfig = @import("config").BeaconConfig; +const BeaconState = @import("fork_types").BeaconState; +const EpochCache = @import("../cache/epoch_cache.zig").EpochCache; +const c = @import("constants"); +const bls = @import("bls"); +const computeSigningRootVariable = @import("../utils/signing_root.zig").computeSigningRootVariable; +const computeEpochAtSlot = @import("../utils/epoch.zig").computeEpochAtSlot; +const SingleSignatureSet = @import("../utils/signature_sets.zig").SingleSignatureSet; +const createSingleSignatureSetFromComponents = @import("../utils/signature_sets.zig").createSingleSignatureSetFromComponents; + +pub fn getExecutionPayloadEnvelopeSigningRoot( + allocator: Allocator, + config: *const BeaconConfig, + envelope: *const types.gloas.ExecutionPayloadEnvelope.Type, +) ![32]u8 { + const domain = try config.getDomain(computeEpochAtSlot(envelope.payload.slot_number), c.DOMAIN_BEACON_BUILDER, null); + + var out: [32]u8 = undefined; + try computeSigningRootVariable(types.gloas.ExecutionPayloadEnvelope, allocator, envelope, domain, &out); + return out; +} + +pub fn getExecutionPayloadEnvelopeSignatureSet( + allocator: Allocator, + config: *const BeaconConfig, + epoch_cache: *const EpochCache, + state: *BeaconState(.gloas), + signed_envelope: *const types.gloas.SignedExecutionPayloadEnvelope.Type, +) !SingleSignatureSet { + const envelope = &signed_envelope.message; + + // Get the pubkey: proposer key for self-builds, builder key otherwise + var latest_block_header = try state.latestBlockHeader(); + const proposer_index: u64 = try latest_block_header.get("proposer_index"); + const pubkey = if (envelope.builder_index == c.BUILDER_INDEX_SELF_BUILD) + epoch_cache.index_to_pubkey.items[proposer_index] + else blk: { + var builders = try state.inner.get("builders"); + var builder: types.gloas.Builder.Type = undefined; + try builders.getValue(allocator, envelope.builder_index, &builder); + break :blk bls.PublicKey.uncompress(&builder.pubkey) catch return error.InvalidBuilderPubkey; + }; + + const signing_root = try getExecutionPayloadEnvelopeSigningRoot(allocator, config, envelope); + return createSingleSignatureSetFromComponents(pubkey, signing_root, signed_envelope.signature); +} From 9e9ec68bde84c96cc317fafabc86b79c0fd181d1 Mon Sep 17 00:00:00 2001 From: Rahul Guha <19rahul2003@gmail.com> Date: Wed, 1 Jul 2026 20:45:17 +0530 Subject: [PATCH 06/33] update Gloas churn and preset constants --- src/preset/preset.zig | 2 +- .../block/process_consolidation_request.zig | 2 +- .../epoch/process_pending_deposits.zig | 10 +++++-- .../slot/upgrade_state_to_electra.zig | 2 +- src/state_transition/utils/epoch.zig | 8 ++++-- src/state_transition/utils/validator.zig | 27 ++++++++++++++++++- 6 files changed, 43 insertions(+), 8 deletions(-) diff --git a/src/preset/preset.zig b/src/preset/preset.zig index fcf340978..f3ddfc27f 100644 --- a/src/preset/preset.zig +++ b/src/preset/preset.zig @@ -163,7 +163,7 @@ const PresetMinimal = struct { pub const KZG_COMMITMENTS_INCLUSION_PROOF_DEPTH = 4; pub const MAX_PENDING_PARTIALS_PER_WITHDRAWALS_SWEEP = 2; pub const MAX_PENDING_DEPOSITS_PER_EPOCH = PresetMainnet.MAX_PENDING_DEPOSITS_PER_EPOCH; - pub const PTC_SIZE = 2; + pub const PTC_SIZE = 16; pub const MAX_PAYLOAD_ATTESTATIONS = 4; pub const MAX_BUILDER_DEPOSIT_REQUESTS_PER_PAYLOAD = 256; pub const MAX_BUILDER_EXIT_REQUESTS_PER_PAYLOAD = 16; diff --git a/src/state_transition/block/process_consolidation_request.zig b/src/state_transition/block/process_consolidation_request.zig index 82835e981..5863214c7 100644 --- a/src/state_transition/block/process_consolidation_request.zig +++ b/src/state_transition/block/process_consolidation_request.zig @@ -56,7 +56,7 @@ pub fn processConsolidationRequest( } // If there is too little available consolidation churn limit, consolidation requests are ignored - if (getConsolidationChurnLimit(epoch_cache) <= preset.MIN_ACTIVATION_BALANCE) { + if (getConsolidationChurnLimit(fork, epoch_cache) <= preset.MIN_ACTIVATION_BALANCE) { return; } diff --git a/src/state_transition/epoch/process_pending_deposits.zig b/src/state_transition/epoch/process_pending_deposits.zig index 28f60bac6..e95f55339 100644 --- a/src/state_transition/epoch/process_pending_deposits.zig +++ b/src/state_transition/epoch/process_pending_deposits.zig @@ -6,7 +6,9 @@ const BeaconConfig = @import("config").BeaconConfig; const BeaconState = @import("fork_types").BeaconState; const EpochCache = @import("../cache/epoch_cache.zig").EpochCache; const EpochTransitionCache = @import("../cache/epoch_transition_cache.zig").EpochTransitionCache; -const getActivationExitChurnLimit = @import("../utils/validator.zig").getActivationExitChurnLimit; +const validator_utils = @import("../utils/validator.zig"); +const getActivationExitChurnLimit = validator_utils.getActivationExitChurnLimit; +const getGloasActivationChurnLimit = validator_utils.getGloasActivationChurnLimit; const preset = @import("preset").preset; const isValidatorKnown = @import("../utils/electra.zig").isValidatorKnown; const validateDepositSignature = @import("../block/process_deposit.zig").validateDepositSignature; @@ -30,7 +32,11 @@ pub fn processPendingDeposits( ) !void { const next_epoch = epoch_cache.epoch + 1; const deposit_balance_to_consume = try state.depositBalanceToConsume(); - const available_for_processing = deposit_balance_to_consume + getActivationExitChurnLimit(epoch_cache); + const churn_limit = if (comptime fork.gte(.gloas)) + getGloasActivationChurnLimit(epoch_cache) + else + getActivationExitChurnLimit(epoch_cache); + const available_for_processing = deposit_balance_to_consume + churn_limit; const finalized_slot = computeStartSlotAtEpoch(try state.finalizedEpoch()); var processed_amount: u64 = 0; diff --git a/src/state_transition/slot/upgrade_state_to_electra.zig b/src/state_transition/slot/upgrade_state_to_electra.zig index e62cd4e3e..283d490ab 100644 --- a/src/state_transition/slot/upgrade_state_to_electra.zig +++ b/src/state_transition/slot/upgrade_state_to_electra.zig @@ -54,7 +54,7 @@ pub fn upgradeStateToElectra( try state.setEarliestExitEpoch(earliest_exit_epoch + 1); try state.setEarliestConsolidationEpoch(computeActivationExitEpoch(current_epoch_pre)); try state.setExitBalanceToConsume(getActivationExitChurnLimit(epoch_cache)); - try state.setConsolidationBalanceToConsume(getConsolidationChurnLimit(epoch_cache)); + try state.setConsolidationBalanceToConsume(getConsolidationChurnLimit(.electra, epoch_cache)); const sort_fn = struct { pub fn sort(validator_arr: []const ct.phase0.Validator.Type, a: ValidatorIndex, b: ValidatorIndex) bool { diff --git a/src/state_transition/utils/epoch.zig b/src/state_transition/utils/epoch.zig index 0c02d93ef..0b5429e80 100644 --- a/src/state_transition/utils/epoch.zig +++ b/src/state_transition/utils/epoch.zig @@ -9,6 +9,7 @@ const ForkSeq = @import("config").ForkSeq; const BeaconState = @import("fork_types").BeaconState; const EpochCache = @import("../cache/epoch_cache.zig").EpochCache; const getActivationExitChurnLimit = @import("../utils/validator.zig").getActivationExitChurnLimit; +const getGloasExitChurnLimit = @import("../utils/validator.zig").getGloasExitChurnLimit; const getConsolidationChurnLimit = @import("../utils/validator.zig").getConsolidationChurnLimit; pub fn computeEpochAtSlot(slot: Slot) Epoch { @@ -47,7 +48,10 @@ pub fn computeExitEpochAndUpdateChurn( ) !u64 { const state_earliest_exit_epoch = try state.earliestExitEpoch(); var earliest_exit_epoch = @max(state_earliest_exit_epoch, computeActivationExitEpoch(epoch_cache.epoch)); - const per_epoch_churn = getActivationExitChurnLimit(epoch_cache); + const per_epoch_churn = if (comptime fork.gte(.gloas)) + getGloasExitChurnLimit(epoch_cache) + else + getActivationExitChurnLimit(epoch_cache); const state_exit_balance_to_consume = try state.exitBalanceToConsume(); // New epoch for exits. @@ -79,7 +83,7 @@ pub fn computeConsolidationEpochAndUpdateChurn( ) !u64 { const state_earliest_consolidation_epoch = try state.earliestConsolidationEpoch(); var earliest_consolidation_epoch = @max(state_earliest_consolidation_epoch, computeActivationExitEpoch(epoch_cache.epoch)); - const per_epoch_consolidation_churn = getConsolidationChurnLimit(epoch_cache); + const per_epoch_consolidation_churn = getConsolidationChurnLimit(fork, epoch_cache); const state_consolidation_balance_to_consume = try state.consolidationBalanceToConsume(); diff --git a/src/state_transition/utils/validator.zig b/src/state_transition/utils/validator.zig index 0e409af95..de79a980e 100644 --- a/src/state_transition/utils/validator.zig +++ b/src/state_transition/utils/validator.zig @@ -72,7 +72,32 @@ pub fn getActivationExitChurnLimit(epoch_cache: *const EpochCache) u64 { return @min(epoch_cache.config.chain.MAX_PER_EPOCH_ACTIVATION_EXIT_CHURN_LIMIT, getBalanceChurnLimitFromCache(epoch_cache)); } -pub fn getConsolidationChurnLimit(epoch_cache: *const EpochCache) u64 { +pub fn getGloasActivationChurnLimit(epoch_cache: *const EpochCache) u64 { + const churn = getBalanceChurnLimit( + epoch_cache.total_active_balance_increments, + epoch_cache.config.chain.CHURN_LIMIT_QUOTIENT_GLOAS, + epoch_cache.config.chain.MIN_PER_EPOCH_CHURN_LIMIT_ELECTRA, + ); + return @min(epoch_cache.config.chain.MAX_PER_EPOCH_ACTIVATION_CHURN_LIMIT_GLOAS, churn); +} + +pub fn getGloasExitChurnLimit(epoch_cache: *const EpochCache) u64 { + return getBalanceChurnLimit( + epoch_cache.total_active_balance_increments, + epoch_cache.config.chain.CHURN_LIMIT_QUOTIENT_GLOAS, + epoch_cache.config.chain.MIN_PER_EPOCH_CHURN_LIMIT_ELECTRA, + ); +} + +pub fn getConsolidationChurnLimit(comptime fork: ForkSeq, epoch_cache: *const EpochCache) u64 { + if (comptime fork.gte(.gloas)) { + return getBalanceChurnLimit( + epoch_cache.total_active_balance_increments, + epoch_cache.config.chain.CONSOLIDATION_CHURN_LIMIT_QUOTIENT, + 0, + ); + } + return getBalanceChurnLimitFromCache(epoch_cache) - getActivationExitChurnLimit(epoch_cache); } From 11b2d3737760f0795617b36c95c931c1e3a6cfbb Mon Sep 17 00:00:00 2001 From: Rahul Guha <19rahul2003@gmail.com> Date: Wed, 1 Jul 2026 21:09:09 +0530 Subject: [PATCH 07/33] update Gloas fork upgrade builder onboarding --- .../slot/upgrade_state_to_gloas.zig | 135 ++++++++++++++++++ src/state_transition/state_transition.zig | 6 + src/state_transition/utils/gloas.zig | 27 +--- .../utils/pending_deposits_lookup.zig | 92 ++++++++++++ 4 files changed, 237 insertions(+), 23 deletions(-) create mode 100644 src/state_transition/slot/upgrade_state_to_gloas.zig create mode 100644 src/state_transition/utils/pending_deposits_lookup.zig diff --git a/src/state_transition/slot/upgrade_state_to_gloas.zig b/src/state_transition/slot/upgrade_state_to_gloas.zig new file mode 100644 index 000000000..c2773b10c --- /dev/null +++ b/src/state_transition/slot/upgrade_state_to_gloas.zig @@ -0,0 +1,135 @@ +const std = @import("std"); +const Allocator = std.mem.Allocator; +const BeaconConfig = @import("config").BeaconConfig; +const EpochCache = @import("../cache/epoch_cache.zig").EpochCache; +const BeaconState = @import("fork_types").BeaconState; +const ct = @import("consensus_types"); +const preset = @import("preset").preset; +const c = @import("constants"); + +const ExecutionPayloadBid = ct.gloas.ExecutionPayloadBid; +const PendingDeposit = ct.electra.PendingDeposit.Type; +const BitVector = @import("ssz").BitVector; +const ExecPayloadAvailability = BitVector(preset.SLOTS_PER_HISTORICAL_ROOT); +const isValidatorKnown = @import("../utils/electra.zig").isValidatorKnown; +const validateDepositSignature = @import("../block/process_deposit.zig").validateDepositSignature; +const gloas_utils = @import("../utils/gloas.zig"); +const addBuilderToRegistry = gloas_utils.addBuilderToRegistry; +const findBuilderIndexByPubkey = gloas_utils.findBuilderIndexByPubkey; +const isBuilderWithdrawalCredential = gloas_utils.isBuilderWithdrawalCredential; +const initializePtcWindow = gloas_utils.initializePtcWindow; +const PendingDepositsLookup = @import("../utils/pending_deposits_lookup.zig").PendingDepositsLookup; + +pub fn upgradeStateToGloas( + allocator: Allocator, + config: *const BeaconConfig, + epoch_cache: *const EpochCache, + fulu_state: *BeaconState(.fulu), +) !BeaconState(.gloas) { + const block_hash_ptr = try fulu_state.latestExecutionPayloadHeaderBlockHash(); + var block_hash: [32]u8 = undefined; + @memcpy(&block_hash, block_hash_ptr); + + var latest_execution_payload_header = ct.fulu.ExecutionPayloadHeader.default_value; + try fulu_state.latestExecutionPayloadHeader(allocator, &latest_execution_payload_header); + defer ct.fulu.ExecutionPayloadHeader.deinit(allocator, &latest_execution_payload_header); + + var state = try fulu_state.upgradeUnsafe(); + errdefer state.deinit(); + + const new_fork = ct.phase0.Fork.Type{ + .previous_version = try fulu_state.forkCurrentVersion(), + .current_version = config.chain.GLOAS_FORK_VERSION, + .epoch = epoch_cache.epoch, + }; + try state.setFork(&new_fork); + + var bid = ExecutionPayloadBid.default_value; + bid.block_hash = block_hash; + bid.gas_limit = latest_execution_payload_header.gas_limit; + try ct.gloas.ExecutionRequests.hashTreeRoot(allocator, &ct.gloas.ExecutionRequests.default_value, &bid.execution_requests_root); + try state.inner.setValue("latest_execution_payload_bid", &bid); + + try state.inner.setValue("latest_block_hash", &block_hash); + + const availability = ExecPayloadAvailability{ .data = [_]u8{0xFF} ** @divExact(ExecPayloadAvailability.length, 8) }; + try state.inner.setValue("execution_payload_availability", &availability); + + const ptc_window = try initializePtcWindow(.gloas, allocator, epoch_cache, &state); + try state.inner.setValue("ptc_window", &ptc_window); + + try onboardBuildersFromPendingDeposits(allocator, config, epoch_cache, &state); + + fulu_state.deinit(); + return state; +} + +fn onboardBuildersFromPendingDeposits( + allocator: Allocator, + config: *const BeaconConfig, + epoch_cache: *const EpochCache, + state: *BeaconState(.gloas), +) !void { + var remaining_pending_deposits: std.ArrayList(PendingDeposit) = .empty; + defer remaining_pending_deposits.deinit(allocator); + + var pending_deposits_lookup = PendingDepositsLookup.buildEmpty(allocator); + defer pending_deposits_lookup.deinit(); + + var pending_deposits = try state.pendingDeposits(); + const pending_deposits_len = try pending_deposits.length(); + var pending_it = pending_deposits.iteratorReadonly(0); + + for (0..pending_deposits_len) |_| { + const deposit = try pending_it.nextValue(allocator); + + const validator_index = epoch_cache.getValidatorIndex(&deposit.pubkey); + if (try isValidatorKnown(.gloas, state, validator_index)) { + try remaining_pending_deposits.append(allocator, deposit); + try pending_deposits_lookup.add(&deposit); + continue; + } + + const builder_index = try findBuilderIndexByPubkey(allocator, state, &deposit.pubkey); + if (builder_index) |idx| { + var builders = try state.inner.get("builders"); + var builder: ct.gloas.Builder.Type = undefined; + try builders.getValue(allocator, idx, &builder); + builder.balance += deposit.amount; + try builders.setValue(idx, &builder); + continue; + } else { + if (!isBuilderWithdrawalCredential(&deposit.withdrawal_credentials)) { + try remaining_pending_deposits.append(allocator, deposit); + try pending_deposits_lookup.add(&deposit); + continue; + } + + if (try pending_deposits_lookup.hasPendingValidator(config, &deposit.pubkey)) { + try remaining_pending_deposits.append(allocator, deposit); + try pending_deposits_lookup.add(&deposit); + continue; + } + } + + validateDepositSignature(config, &deposit.pubkey, &deposit.withdrawal_credentials, deposit.amount, deposit.signature) catch continue; + + var execution_address: ct.primitive.ExecutionAddress.Type = undefined; + @memcpy(&execution_address, deposit.withdrawal_credentials[12..32]); + try addBuilderToRegistry( + allocator, + state, + &deposit.pubkey, + c.PAYLOAD_BUILDER_VERSION, + &execution_address, + deposit.amount, + deposit.slot, + ); + } + + var new_pending = try pending_deposits.sliceFrom(pending_deposits_len); + for (remaining_pending_deposits.items) |dep| { + try new_pending.pushValue(&dep); + } + try state.setPendingDeposits(new_pending); +} diff --git a/src/state_transition/state_transition.zig b/src/state_transition/state_transition.zig index 8338d33d0..5cf9ab9b6 100644 --- a/src/state_transition/state_transition.zig +++ b/src/state_transition/state_transition.zig @@ -29,6 +29,7 @@ const upgradeStateToCapella = @import("slot/upgrade_state_to_capella.zig").upgra const upgradeStateToDeneb = @import("slot/upgrade_state_to_deneb.zig").upgradeStateToDeneb; const upgradeStateToElectra = @import("slot/upgrade_state_to_electra.zig").upgradeStateToElectra; const upgradeStateToFulu = @import("slot/upgrade_state_to_fulu.zig").upgradeStateToFulu; +const upgradeStateToGloas = @import("slot/upgrade_state_to_gloas.zig").upgradeStateToGloas; pub const ExecutionPayloadStatus = enum(u8) { invalid, @@ -131,6 +132,11 @@ pub fn processSlots( const upgraded = try upgradeStateToFulu(allocator, config, epoch_cache, electra_state); state.* = .{ .fulu = upgraded.inner }; } + if (state_epoch == config.chain.GLOAS_FORK_EPOCH) { + const fulu_state = try state.tryCastToFork(.fulu); + const upgraded = try upgradeStateToGloas(allocator, config, epoch_cache, fulu_state); + state.* = .{ .gloas = upgraded.inner }; + } try epoch_cache.finalProcessEpoch(state); metrics.state_transition.epoch_transition.observe(time.durationSeconds(time.since(io, epoch_transition_timer))); diff --git a/src/state_transition/utils/gloas.zig b/src/state_transition/utils/gloas.zig index dd332846a..b4d5212ff 100644 --- a/src/state_transition/utils/gloas.zig +++ b/src/state_transition/utils/gloas.zig @@ -19,7 +19,7 @@ const isActiveValidator = @import("./validator.zig").isActiveValidator; const computeStartSlotAtEpoch = @import("./epoch.zig").computeStartSlotAtEpoch; const RootCache = @import("../cache/root_cache.zig").RootCache; const EpochCache = @import("../cache/epoch_cache.zig").EpochCache; -const isValidDepositSignature = @import("../block/process_deposit.zig").isValidDepositSignature; +const PendingDepositsLookup = @import("./pending_deposits_lookup.zig").PendingDepositsLookup; const computeDomain = @import("./domain.zig").computeDomain; const computeSigningRoot = @import("./signing_root.zig").computeSigningRoot; const bls = @import("bls"); @@ -256,28 +256,9 @@ pub fn isPendingValidator( state: *BeaconState(fork), pubkey: *const BLSPubkey, ) !bool { - var pending_deposits = try state.pendingDeposits(); - const pending_deposits_len = try pending_deposits.length(); - var pending_it = pending_deposits.iteratorReadonly(0); - - for (0..pending_deposits_len) |_| { - const pending_deposit = try pending_it.nextValue(allocator); - if (!std.mem.eql(u8, &pending_deposit.pubkey, pubkey)) { - continue; - } - - if (isValidDepositSignature( - config, - &pending_deposit.pubkey, - &pending_deposit.withdrawal_credentials, - pending_deposit.amount, - pending_deposit.signature, - )) { - return true; - } - } - - return false; + var lookup = try PendingDepositsLookup.build(fork, allocator, state); + defer lookup.deinit(); + return try lookup.hasPendingValidator(config, pubkey); } pub fn computePtc( diff --git a/src/state_transition/utils/pending_deposits_lookup.zig b/src/state_transition/utils/pending_deposits_lookup.zig new file mode 100644 index 000000000..c60add60e --- /dev/null +++ b/src/state_transition/utils/pending_deposits_lookup.zig @@ -0,0 +1,92 @@ +const std = @import("std"); +const Allocator = std.mem.Allocator; +const BeaconConfig = @import("config").BeaconConfig; +const ForkSeq = @import("config").ForkSeq; +const BeaconState = @import("fork_types").BeaconState; +const ct = @import("consensus_types"); +const validateDepositSignature = @import("../block/process_deposit.zig").validateDepositSignature; + +const BLSPubkey = ct.primitive.BLSPubkey.Type; +const PendingDeposit = ct.electra.PendingDeposit.Type; + +const PendingDepositEntry = struct { + deposits: std.ArrayList(PendingDeposit) = .empty, + validated_count: usize = 0, + has_valid_signature: bool = false, + + fn deinit(self: *PendingDepositEntry, allocator: Allocator) void { + self.deposits.deinit(allocator); + } +}; + +pub const PendingDepositsLookup = struct { + allocator: Allocator, + deposits_by_pubkey: std.AutoHashMap(BLSPubkey, PendingDepositEntry), + + pub fn buildEmpty(allocator: Allocator) PendingDepositsLookup { + return .{ + .allocator = allocator, + .deposits_by_pubkey = std.AutoHashMap(BLSPubkey, PendingDepositEntry).init(allocator), + }; + } + + pub fn build(comptime fork: ForkSeq, allocator: Allocator, state: *BeaconState(fork)) !PendingDepositsLookup { + var lookup = PendingDepositsLookup.buildEmpty(allocator); + errdefer lookup.deinit(); + + var pending_deposits = try state.pendingDeposits(); + const pending_deposits_len = try pending_deposits.length(); + var pending_it = pending_deposits.iteratorReadonly(0); + + for (0..pending_deposits_len) |_| { + const pending_deposit = try pending_it.nextValue(allocator); + try lookup.add(&pending_deposit); + } + + return lookup; + } + + pub fn deinit(self: *PendingDepositsLookup) void { + var value_iterator = self.deposits_by_pubkey.valueIterator(); + while (value_iterator.next()) |entry| { + entry.deinit(self.allocator); + } + self.deposits_by_pubkey.deinit(); + } + + pub fn add(self: *PendingDepositsLookup, pending_deposit: *const PendingDeposit) !void { + const result = try self.deposits_by_pubkey.getOrPut(pending_deposit.pubkey); + if (!result.found_existing) { + result.value_ptr.* = .{}; + } + try result.value_ptr.deposits.append(self.allocator, pending_deposit.*); + } + + pub fn hasPendingValidator( + self: *PendingDepositsLookup, + config: *const BeaconConfig, + pubkey: *const BLSPubkey, + ) !bool { + const entry = self.deposits_by_pubkey.getPtr(pubkey.*) orelse return false; + if (entry.has_valid_signature) return true; + if (entry.validated_count == entry.deposits.items.len) return false; + + var i = entry.validated_count; + while (i < entry.deposits.items.len) : (i += 1) { + const deposit = &entry.deposits.items[i]; + validateDepositSignature( + config, + &deposit.pubkey, + &deposit.withdrawal_credentials, + deposit.amount, + deposit.signature, + ) catch continue; + entry.has_valid_signature = true; + entry.validated_count = i + 1; + return true; + } + + entry.validated_count = entry.deposits.items.len; + return false; + } +}; From 45bc63f083c08446f8f9a9eda8e7ac17e6a93881 Mon Sep 17 00:00:00 2001 From: Rahul Guha <19rahul2003@gmail.com> Date: Wed, 1 Jul 2026 23:29:52 +0530 Subject: [PATCH 08/33] account for builder withdrawals --- src/state_transition/block/process_block.zig | 33 +- .../block/process_withdrawals.zig | 524 ++++++++++++++---- 2 files changed, 443 insertions(+), 114 deletions(-) diff --git a/src/state_transition/block/process_block.zig b/src/state_transition/block/process_block.zig index bf522241a..2599640c1 100644 --- a/src/state_transition/block/process_block.zig +++ b/src/state_transition/block/process_block.zig @@ -27,6 +27,7 @@ const getExpectedWithdrawals = @import("./process_withdrawals.zig").getExpectedW const processExecutionPayloadBid = @import("./process_execution_payload_bid.zig").processExecutionPayloadBid; const processParentExecutionPayload = @import("./process_parent_execution_payload.zig").processParentExecutionPayload; const isExecutionEnabled = @import("../utils/execution.zig").isExecutionEnabled; +const isParentBlockFull = @import("../utils/gloas.zig").isParentBlockFull; // TODO: proposer reward api // const ProposerRewardType = @import("../types/proposer_reward.zig").ProposerRewardType; @@ -59,6 +60,30 @@ pub fn processBlock( const body = block.body(); const current_epoch = epoch_cache.epoch; + if (comptime fork.gte(.gloas)) { + if (try isParentBlockFull(state)) { + var withdrawals_result = WithdrawalsResult{ .withdrawals = try Withdrawals.initCapacity( + allocator, + preset.MAX_WITHDRAWALS_PER_PAYLOAD, + ) }; + defer withdrawals_result.withdrawals.deinit(allocator); + var withdrawal_balances = std.AutoHashMap(ValidatorIndex, usize).init(allocator); + defer withdrawal_balances.deinit(); + + try getExpectedWithdrawals( + fork, + allocator, + epoch_cache, + state, + &withdrawals_result, + &withdrawal_balances, + ); + + const empty_root: Root = .{0} ** 32; + try processWithdrawals(fork, allocator, state, withdrawals_result, empty_root); + } + } + // The call to the process_execution_payload must happen before the call to the process_randao as the former depends // on the randao_mix computed with the reveal of the previous block. // Gloas (ePBS): execution payload is decoupled from the block, processed via ExecutionPayloadEnvelope @@ -67,13 +92,17 @@ pub fn processBlock( // TODO Deneb: Allow to disable withdrawals for interop testing // https://github.com/ethereum/consensus-specs/blob/b62c9e877990242d63aa17a2a59a49bc649a2f2e/specs/eip4844/beacon-chain.md#disabling-withdrawals if (comptime fork.gte(.capella)) { - var withdrawals_buf: [preset.MAX_WITHDRAWALS_PER_PAYLOAD]types.capella.Withdrawal.Type = undefined; - var withdrawals_result = WithdrawalsResult{ .withdrawals = Withdrawals.initBuffer(&withdrawals_buf) }; + var withdrawals_result = WithdrawalsResult{ .withdrawals = try Withdrawals.initCapacity( + allocator, + preset.MAX_WITHDRAWALS_PER_PAYLOAD, + ) }; + defer withdrawals_result.withdrawals.deinit(allocator); var withdrawal_balances = std.AutoHashMap(ValidatorIndex, usize).init(allocator); defer withdrawal_balances.deinit(); try getExpectedWithdrawals( fork, + allocator, epoch_cache, state, &withdrawals_result, diff --git a/src/state_transition/block/process_withdrawals.zig b/src/state_transition/block/process_withdrawals.zig index 07c53d3ec..02deada83 100644 --- a/src/state_transition/block/process_withdrawals.zig +++ b/src/state_transition/block/process_withdrawals.zig @@ -13,13 +13,26 @@ const ExecutionAddress = types.primitive.ExecutionAddress.Type; const hasExecutionWithdrawalCredential = @import("../utils/electra.zig").hasExecutionWithdrawalCredential; const hasEth1WithdrawalCredential = @import("../utils/capella.zig").hasEth1WithdrawalCredential; const getMaxEffectiveBalance = @import("../utils/validator.zig").getMaxEffectiveBalance; +const isFullyWithdrawableValidator = @import("../utils/validator.zig").isFullyWithdrawableValidator; +const isPartiallyWithdrawableValidator = @import("../utils/validator.zig").isPartiallyWithdrawableValidator; const decreaseBalance = @import("../utils/balance.zig").decreaseBalance; +const gloas_utils = @import("../utils/gloas.zig"); +const isBuilderIndex = gloas_utils.isBuilderIndex; +const convertBuilderIndexToValidatorIndex = gloas_utils.convertBuilderIndexToValidatorIndex; +const convertValidatorIndexToBuilderIndex = gloas_utils.convertValidatorIndexToBuilderIndex; +const isParentBlockFull = gloas_utils.isParentBlockFull; const Node = @import("persistent_merkle_tree").Node; +const Withdrawal = types.capella.Withdrawal.Type; +const Epoch = types.primitive.Epoch.Type; pub const WithdrawalsResult = struct { withdrawals: Withdrawals, sampled_validators: usize = 0, processed_partial_withdrawals_count: usize = 0, + // processedBuilderWithdrawalsCount is withdrawals coming from builder payment since EIP-7732 + processed_builder_withdrawals_count: usize = 0, + // processedBuildersSweepCount is withdrawals from builder sweep since EIP-7732 + processed_builders_sweep_count: usize = 0, }; /// right now for the implementation we pass in processBlock() @@ -33,45 +46,72 @@ pub fn processWithdrawals( expected_withdrawals_result: WithdrawalsResult, payload_withdrawals_root: Root, ) !void { - // processedPartialWithdrawalsCount is withdrawals coming from EL since electra (EIP-7002) - const processed_partial_withdrawals_count = expected_withdrawals_result.processed_partial_withdrawals_count; - const expected_withdrawals = expected_withdrawals_result.withdrawals.items; - const num_withdrawals = expected_withdrawals.len; + // [New in EIP-7732] Return early if parent block is empty + if (comptime fork.gte(.gloas)) { + if (!(try isParentBlockFull(state))) return; + } - var expected_withdrawals_root: [32]u8 = undefined; - try types.capella.Withdrawals.hashTreeRoot(allocator, &expected_withdrawals_result.withdrawals, &expected_withdrawals_root); + const expected_withdrawals = expected_withdrawals_result.withdrawals.items; - if (!std.mem.eql(u8, &expected_withdrawals_root, &payload_withdrawals_root)) { - return error.WithdrawalsRootMismatch; + // After EIP-7732, withdrawals are verified later in processExecutionPayloadEnvelope + if (comptime fork.lt(.gloas)) { + var expected_withdrawals_root: [32]u8 = undefined; + try types.capella.Withdrawals.hashTreeRoot(allocator, &expected_withdrawals_result.withdrawals, &expected_withdrawals_root); + if (!std.mem.eql(u8, &expected_withdrawals_root, &payload_withdrawals_root)) { + return error.WithdrawalsRootMismatch; + } } - for (0..num_withdrawals) |i| { - const withdrawal = expected_withdrawals[i]; - try decreaseBalance(fork, state, withdrawal.validator_index, withdrawal.amount); + try applyWithdrawals(fork, allocator, state, expected_withdrawals); + // Update pending_partial_withdrawals (electra+) + if (comptime fork.gte(.electra)) { + var pending_partial_withdrawals = try state.pendingPartialWithdrawals(); + const truncated = try pending_partial_withdrawals.sliceFrom(expected_withdrawals_result.processed_partial_withdrawals_count); + try state.setPendingPartialWithdrawals(truncated); } - if (comptime fork.gte(.electra)) { - if (processed_partial_withdrawals_count > 0) { - var pending_partial_withdrawals = try state.pendingPartialWithdrawals(); - const truncated = try pending_partial_withdrawals.sliceFrom(processed_partial_withdrawals_count); + if (comptime fork.gte(.gloas)) { + // Store expected withdrawals for verification in processExecutionPayloadEnvelope + var payload_expected_withdrawals = try state.inner.get("payload_expected_withdrawals"); + const current_len = try payload_expected_withdrawals.length(); + var new_list = try payload_expected_withdrawals.sliceFrom(current_len); + for (expected_withdrawals) |w| { + try new_list.pushValue(&w); + } + try state.inner.set("payload_expected_withdrawals", new_list); - try state.setPendingPartialWithdrawals(truncated); + // Update builder pending withdrawals queue + const processed_builder_withdrawals_count = expected_withdrawals_result.processed_builder_withdrawals_count; + if (processed_builder_withdrawals_count > 0) { + var builder_pending_withdrawals = try state.inner.get("builder_pending_withdrawals"); + const truncated = try builder_pending_withdrawals.sliceFrom(processed_builder_withdrawals_count); + try state.inner.set("builder_pending_withdrawals", truncated); + } + + // Update next builder index for sweep + var builders = try state.inner.get("builders"); + const builders_len: u64 = try builders.length(); + if (builders_len > 0) { + const current_builder_index: u64 = try state.inner.get("next_withdrawal_builder_index"); + const processed_builders_sweep_count: u64 = @intCast(expected_withdrawals_result.processed_builders_sweep_count); + const next_builder_index = (current_builder_index + processed_builders_sweep_count) % builders_len; + try state.inner.set("next_withdrawal_builder_index", next_builder_index); } } // Update the nextWithdrawalIndex - if (expected_withdrawals.len > 0) { - const latest_withdrawal = expected_withdrawals[expected_withdrawals.len - 1]; - try state.setNextWithdrawalIndex(latest_withdrawal.index + 1); + const latest_withdrawal = if (expected_withdrawals.len > 0) expected_withdrawals[expected_withdrawals.len - 1] else null; + if (latest_withdrawal) |lw| { + try state.setNextWithdrawalIndex(lw.index + 1); } // Update the next_withdrawal_validator_index const validators_len: u64 = @intCast(try state.validatorsCount()); const next_withdrawal_validator_index = try state.nextWithdrawalValidatorIndex(); - if (expected_withdrawals.len == preset.MAX_WITHDRAWALS_PER_PAYLOAD) { + if (latest_withdrawal != null and expected_withdrawals.len == preset.MAX_WITHDRAWALS_PER_PAYLOAD) { // All slots filled, next_withdrawal_validator_index should be validatorIndex having next turn try state.setNextWithdrawalValidatorIndex( - (expected_withdrawals[expected_withdrawals.len - 1].validator_index + 1) % validators_len, + (latest_withdrawal.?.validator_index + 1) % validators_len, ); } else { // expected withdrawals came up short in the bound, so we move next_withdrawal_validator_index to @@ -82,141 +122,397 @@ pub fn processWithdrawals( } } -/// Called by the block proposer to find a list of withdrawals to include in the block. -/// -/// This list is assumed to be bounded by `preset.MAX_WITHDRAWALS_PER_PAYLOAD`. -/// -/// Caller should deinit `withdrawal_balances` with .deinit() after use. +// Consumer should deinit WithdrawalsResult with .deinit() after use pub fn getExpectedWithdrawals( comptime fork: ForkSeq, + allocator: Allocator, epoch_cache: *const EpochCache, state: *BeaconState(fork), withdrawals_result: *WithdrawalsResult, withdrawal_balances: *std.AutoHashMap(ValidatorIndex, usize), ) !void { - std.debug.assert(withdrawals_result.withdrawals.capacity == preset.MAX_WITHDRAWALS_PER_PAYLOAD); - if (comptime fork.lt(.capella)) return error.InvalidForkSequence; + if (comptime fork.lt(.capella)) { + return error.InvalidForkSequence; + } const epoch = epoch_cache.epoch; var withdrawal_index = try state.nextWithdrawalIndex(); - var validators = try state.validators(); - var balances = try state.balances(); - const next_withdrawal_validator_index = try state.nextWithdrawalValidatorIndex(); - // partial_withdrawals_count is withdrawals coming from EL since electra (EIP-7002) + // Separate maps to track balances after applying withdrawals + var builder_balance_after_withdrawals = std.AutoHashMap(u64, i64).init(allocator); + defer builder_balance_after_withdrawals.deinit(); + + // partialWithdrawalsCount is withdrawals coming from EL since electra (EIP-7002) var processed_partial_withdrawals_count: u64 = 0; + // builderWithdrawalsCount is withdrawals coming from builder payments since EIP-7732 + var processed_builder_withdrawals_count: u64 = 0; + // buildersSweepCount is withdrawals from builder sweep since EIP-7732 + var processed_builders_sweep_count: u64 = 0; + // [New in EIP-7732] get_builder_withdrawals + if (comptime fork.gte(.gloas)) { + var result = try getBuilderWithdrawals( + allocator, + state, + &withdrawal_index, + withdrawals_result.withdrawals.items.len, + &builder_balance_after_withdrawals, + ); + defer result.withdrawals.deinit(allocator); + for (result.withdrawals.items) |w| { + try withdrawals_result.withdrawals.append(allocator, w); + } + processed_builder_withdrawals_count = result.processed_count; + } + + // get_pending_partial_withdrawals (electra+) if (comptime fork.gte(.electra)) { - // MAX_PENDING_PARTIALS_PER_WITHDRAWALS_SWEEP = 8, PENDING_PARTIAL_WITHDRAWALS_LIMIT: 134217728 so we should just lazily iterate thru state.pending_partial_withdrawals. - // pending_partial_withdrawals comes from EIP-7002 smart contract where it takes fee so it's more likely than not validator is in correct condition to withdraw - // also we may break early if withdrawableEpoch > epoch - var pending_partial_withdrawals = try state.pendingPartialWithdrawals(); - var pending_partial_withdrawals_it = pending_partial_withdrawals.iteratorReadonly(0); - const pending_partial_withdrawals_len = try pending_partial_withdrawals.length(); - - for (0..pending_partial_withdrawals_len) |_| { - const withdrawal = try pending_partial_withdrawals_it.nextValue(undefined); - if (withdrawal.withdrawable_epoch > epoch or withdrawals_result.withdrawals.items.len == preset.MAX_PENDING_PARTIALS_PER_WITHDRAWALS_SWEEP) { - break; - } - - var validator: types.phase0.Validator.Type = undefined; - try validators.getValue(undefined, withdrawal.validator_index, &validator); - - const total_withdrawn_gop = try withdrawal_balances.getOrPut(withdrawal.validator_index); - - const total_withdrawn: u64 = if (total_withdrawn_gop.found_existing) total_withdrawn_gop.value_ptr.* else 0; - const balance = try balances.get(withdrawal.validator_index) - total_withdrawn; - - if (validator.exit_epoch == c.FAR_FUTURE_EPOCH and - validator.effective_balance >= preset.MIN_ACTIVATION_BALANCE and - balance > preset.MIN_ACTIVATION_BALANCE) - { - const balance_over_min_activation_balance = balance - preset.MIN_ACTIVATION_BALANCE; - const withdrawable_balance = if (balance_over_min_activation_balance < withdrawal.amount) balance_over_min_activation_balance else withdrawal.amount; - var execution_address: ExecutionAddress = undefined; - @memcpy(&execution_address, validator.withdrawal_credentials[12..]); - withdrawals_result.withdrawals.appendAssumeCapacity(.{ - .index = withdrawal_index, - .validator_index = withdrawal.validator_index, - .address = execution_address, - .amount = withdrawable_balance, - }); - withdrawal_index += 1; - try withdrawal_balances.put(withdrawal.validator_index, total_withdrawn + withdrawable_balance); - } - processed_partial_withdrawals_count += 1; + var result = try getPendingPartialWithdrawals( + fork, + allocator, + epoch, + state, + &withdrawal_index, + withdrawals_result.withdrawals.items.len, + withdrawal_balances, + ); + defer result.withdrawals.deinit(allocator); + for (result.withdrawals.items) |w| { + try withdrawals_result.withdrawals.append(allocator, w); + } + processed_partial_withdrawals_count = result.processed_count; + } + + // [New in EIP-7732] get_builders_sweep_withdrawals + if (comptime fork.gte(.gloas)) { + var result = try getBuildersSweepWithdrawals( + allocator, + state, + epoch, + &withdrawal_index, + withdrawals_result.withdrawals.items.len, + &builder_balance_after_withdrawals, + ); + defer result.withdrawals.deinit(allocator); + for (result.withdrawals.items) |w| { + try withdrawals_result.withdrawals.append(allocator, w); + } + processed_builders_sweep_count = result.processed_count; + } + + // get_validators_sweep_withdrawals + { + var result = try getValidatorsSweepWithdrawals( + fork, + allocator, + epoch, + state, + &withdrawal_index, + withdrawals_result.withdrawals.items.len, + withdrawal_balances, + ); + defer result.withdrawals.deinit(allocator); + for (result.withdrawals.items) |w| { + try withdrawals_result.withdrawals.append(allocator, w); + } + withdrawals_result.sampled_validators = result.processed_count; + } + + withdrawals_result.processed_partial_withdrawals_count = processed_partial_withdrawals_count; + withdrawals_result.processed_builder_withdrawals_count = @intCast(processed_builder_withdrawals_count); + withdrawals_result.processed_builders_sweep_count = @intCast(processed_builders_sweep_count); +} + +/// [Modified in EIP-7732] apply_withdrawals handles builder indices +fn applyWithdrawals( + comptime fork: ForkSeq, + allocator: Allocator, + state: *BeaconState(fork), + withdrawals: []const Withdrawal, +) !void { + for (withdrawals) |withdrawal| { + if (fork.gte(.gloas) and isBuilderIndex(withdrawal.validator_index)) { + // Handle builder withdrawal + const builder_index = convertValidatorIndexToBuilderIndex(withdrawal.validator_index); + var builders = try state.inner.get("builders"); + var builder: types.gloas.Builder.Type = undefined; + try builders.getValue(allocator, builder_index, &builder); + builder.balance -= @min(withdrawal.amount, builder.balance); + try builders.setValue(builder_index, &builder); + } else { + // Handle validator withdrawal + try decreaseBalance(fork, state, withdrawal.validator_index, withdrawal.amount); + } + } +} + +fn getPendingPartialWithdrawals( + comptime fork: ForkSeq, + allocator: Allocator, + epoch: Epoch, + state: *BeaconState(fork), + withdrawal_index: *u64, + num_prior_withdrawals: usize, + validator_balance_after_withdrawals: *std.AutoHashMap(ValidatorIndex, usize), +) !struct { withdrawals: std.ArrayList(Withdrawal), processed_count: usize } { + var pending_partial_withdrawals_result: std.ArrayList(Withdrawal) = .empty; + errdefer pending_partial_withdrawals_result.deinit(allocator); + + var validators = try state.validators(); + var balances = try state.balances(); + + // MAX_PENDING_PARTIALS_PER_WITHDRAWALS_SWEEP = 8, PENDING_PARTIAL_WITHDRAWALS_LIMIT: 134217728 + // so we lazily iterate. pendingPartialWithdrawals comes from EIP-7002 smart contract + // where it takes fee so it's more likely than not validator is in correct condition to withdraw. + // Also we may break early if withdrawableEpoch > epoch. + var pending_partial_withdrawals = try state.pendingPartialWithdrawals(); + var pending_partial_withdrawals_it = pending_partial_withdrawals.iteratorReadonly(0); + const pending_partial_withdrawals_len = try pending_partial_withdrawals.length(); + + // In pre-EIP-7732, partialWithdrawalBound == MAX_PENDING_PARTIALS_PER_WITHDRAWALS_SWEEP + const partial_withdrawal_bound = @min( + num_prior_withdrawals + preset.MAX_PENDING_PARTIALS_PER_WITHDRAWALS_SWEEP, + preset.MAX_WITHDRAWALS_PER_PAYLOAD - 1, + ); + // There must be at least one space reserved for validator sweep withdrawals + if (num_prior_withdrawals > partial_withdrawal_bound) { + return error.PriorWithdrawalsExceedLimit; + } + + // EIP-7002: Execution layer triggerable withdrawals + var processed_count: usize = 0; + for (0..pending_partial_withdrawals_len) |_| { + const withdrawal = try pending_partial_withdrawals_it.nextValue(undefined); + if (withdrawal.withdrawable_epoch > epoch or pending_partial_withdrawals_result.items.len + num_prior_withdrawals >= partial_withdrawal_bound) { + break; + } + var validator: types.phase0.Validator.Type = undefined; + try validators.getValue(undefined, withdrawal.validator_index, &validator); + + const balance_gop = try validator_balance_after_withdrawals.getOrPut(withdrawal.validator_index); + if (!balance_gop.found_existing) { + balance_gop.value_ptr.* = try balances.get(withdrawal.validator_index); } + const balance: u64 = balance_gop.value_ptr.*; + + if (validator.exit_epoch == c.FAR_FUTURE_EPOCH and + validator.effective_balance >= preset.MIN_ACTIVATION_BALANCE and + balance > preset.MIN_ACTIVATION_BALANCE) + { + const balance_over_min_activation_balance = balance - preset.MIN_ACTIVATION_BALANCE; + const withdrawable_balance = if (balance_over_min_activation_balance < withdrawal.amount) balance_over_min_activation_balance else withdrawal.amount; + var execution_address: ExecutionAddress = undefined; + @memcpy(&execution_address, validator.withdrawal_credentials[12..]); + try pending_partial_withdrawals_result.append(allocator, .{ + .index = withdrawal_index.*, + .validator_index = withdrawal.validator_index, + .address = execution_address, + .amount = withdrawable_balance, + }); + withdrawal_index.* += 1; + try validator_balance_after_withdrawals.put(withdrawal.validator_index, balance - withdrawable_balance); + } + processed_count += 1; } + return .{ .withdrawals = pending_partial_withdrawals_result, .processed_count = processed_count }; +} + +fn getValidatorsSweepWithdrawals( + comptime fork: ForkSeq, + allocator: Allocator, + epoch: Epoch, + state: *BeaconState(fork), + withdrawal_index: *u64, + num_prior_withdrawals: usize, + validator_balance_after_withdrawals: *std.AutoHashMap(ValidatorIndex, usize), +) !struct { withdrawals: std.ArrayList(Withdrawal), processed_count: usize } { + // There must be at least one space reserved for validator sweep withdrawals + if (num_prior_withdrawals >= preset.MAX_WITHDRAWALS_PER_PAYLOAD) { + return error.PriorWithdrawalsExceedLimit; + } + + var sweep_withdrawals: std.ArrayList(Withdrawal) = .empty; + errdefer sweep_withdrawals.deinit(allocator); + + const validators = try state.validators(); + var balances = try state.balances(); + const next_withdrawal_validator_index = try state.nextWithdrawalValidatorIndex(); const validators_count = try validators.length(); - const bound = @min(validators_count, preset.MAX_VALIDATORS_PER_WITHDRAWALS_SWEEP); + if (next_withdrawal_validator_index >= validators_count) { + return error.InvalidNextWithdrawalValidatorIndex; + } + const validators_limit = @min(validators_count, preset.MAX_VALIDATORS_PER_WITHDRAWALS_SWEEP); + // Just run a bounded loop max iterating over all withdrawals // however breaks out once we have MAX_WITHDRAWALS_PER_PAYLOAD - var n: usize = 0; - while (n < bound) : (n += 1) { + var processed_count: usize = 0; + for (0..validators_limit) |n| { + if (sweep_withdrawals.items.len + num_prior_withdrawals >= preset.MAX_WITHDRAWALS_PER_PAYLOAD) { + break; + } + // Get next validator in turn const validator_index = (next_withdrawal_validator_index + n) % validators_count; var validator = try validators.get(validator_index); - const withdraw_balance_gop = try withdrawal_balances.getOrPut(validator_index); - const withdraw_balance: u64 = if (withdraw_balance_gop.found_existing) withdraw_balance_gop.value_ptr.* else 0; - const val_balance = try balances.get(validator_index); - const balance = if (comptime fork.gte(.electra)) - // Deduct partially withdrawn balance already queued above - if (val_balance > withdraw_balance) val_balance - withdraw_balance else 0 - else - val_balance; + + const balance_gop = try validator_balance_after_withdrawals.getOrPut(validator_index); + if (!balance_gop.found_existing) { + balance_gop.value_ptr.* = try balances.get(validator_index); + } + const balance: u64 = balance_gop.value_ptr.*; const withdrawable_epoch = try validator.get("withdrawable_epoch"); const withdrawal_credentials = try validator.getFieldRoot("withdrawal_credentials"); const effective_balance = try validator.get("effective_balance"); - const has_withdrawable_credentials = if (comptime fork.gte(.electra)) hasExecutionWithdrawalCredential(withdrawal_credentials) else hasEth1WithdrawalCredential(withdrawal_credentials); - // early skip for balance = 0 as its now more likely that validator has exited/slashed with - // balance zero than not have withdrawal credentials set - if (balance == 0 or !has_withdrawable_credentials) { - continue; - } - // capella full withdrawal - if (withdrawable_epoch <= epoch) { + if (isFullyWithdrawableValidator(fork, withdrawal_credentials, balance, withdrawable_epoch, epoch)) { var execution_address: ExecutionAddress = undefined; @memcpy(&execution_address, withdrawal_credentials[12..]); - withdrawals_result.withdrawals.appendAssumeCapacity(.{ - .index = withdrawal_index, + try sweep_withdrawals.append(allocator, .{ + .index = withdrawal_index.*, .validator_index = validator_index, .address = execution_address, .amount = balance, }); - withdrawal_index += 1; - } else if ((effective_balance == if (comptime fork.gte(.electra)) - getMaxEffectiveBalance(withdrawal_credentials) - else - preset.MAX_EFFECTIVE_BALANCE) and balance > effective_balance) - { - // capella partial withdrawal - const partial_amount = balance - effective_balance; + withdrawal_index.* += 1; + balance_gop.value_ptr.* = 0; + } else if (isPartiallyWithdrawableValidator(fork, withdrawal_credentials, effective_balance, balance)) { + const max_effective_balance = if (comptime fork.gte(.electra)) getMaxEffectiveBalance(withdrawal_credentials) else preset.MAX_EFFECTIVE_BALANCE; + const partial_amount = balance - max_effective_balance; var execution_address: ExecutionAddress = undefined; @memcpy(&execution_address, withdrawal_credentials[12..]); - withdrawals_result.withdrawals.appendAssumeCapacity(.{ - .index = withdrawal_index, + try sweep_withdrawals.append(allocator, .{ + .index = withdrawal_index.*, .validator_index = validator_index, .address = execution_address, .amount = partial_amount, }); - withdrawal_index += 1; - try withdrawal_balances.put(validator_index, withdraw_balance + partial_amount); + withdrawal_index.* += 1; + balance_gop.value_ptr.* = balance - partial_amount; } - // Break if we have enough to pack the block - if (withdrawals_result.withdrawals.items.len >= preset.MAX_WITHDRAWALS_PER_PAYLOAD) { - break; + processed_count += 1; + } + + return .{ .withdrawals = sweep_withdrawals, .processed_count = processed_count }; +} + +fn getBuilderWithdrawals( + allocator: Allocator, + state: *BeaconState(.gloas), + withdrawal_index: *u64, + prior_withdrawals_len: usize, + builder_balance_after_withdrawals: *std.AutoHashMap(u64, i64), +) !struct { withdrawals: std.ArrayList(Withdrawal), processed_count: usize } { + const withdrawals_limit = preset.MAX_WITHDRAWALS_PER_PAYLOAD - 1; + if (prior_withdrawals_len > withdrawals_limit) { + return error.PriorWithdrawalsExceedLimit; + } + var builder_withdrawals: std.ArrayList(Withdrawal) = .empty; + errdefer builder_withdrawals.deinit(allocator); + + var builder_pending_withdrawals = try state.inner.get("builder_pending_withdrawals"); + try builder_pending_withdrawals.commit(); + const builder_pending_withdrawals_len = try builder_pending_withdrawals.length(); + var bw_it = builder_pending_withdrawals.iteratorReadonly(0); + + var processed_count: usize = 0; + for (0..builder_pending_withdrawals_len) |_| { + // Check combined length against limit + const all_withdrawals = prior_withdrawals_len + builder_withdrawals.items.len; + if (all_withdrawals >= withdrawals_limit) break; + + const bw = try bw_it.nextValue(allocator); + const builder_index = bw.builder_index; + + // Get builder balance (from builder.balance, not state.balances) + const balance_gop = try builder_balance_after_withdrawals.getOrPut(builder_index); + if (!balance_gop.found_existing) { + var builders = try state.inner.get("builders"); + var builder: types.gloas.Builder.Type = undefined; + try builders.getValue(allocator, builder_index, &builder); + balance_gop.value_ptr.* = @intCast(builder.balance); } + + // Use the withdrawal amount directly as specified in the spec + try builder_withdrawals.append(allocator, .{ + .index = withdrawal_index.*, + .validator_index = convertBuilderIndexToValidatorIndex(builder_index), + .address = bw.fee_recipient, + .amount = bw.amount, + }); + withdrawal_index.* += 1; + balance_gop.value_ptr.* -= @as(i64, @intCast(bw.amount)); + + processed_count += 1; } - try state.setNextWithdrawalIndex(withdrawal_index); + return .{ .withdrawals = builder_withdrawals, .processed_count = processed_count }; +} - withdrawals_result.sampled_validators = n; - withdrawals_result.processed_partial_withdrawals_count = processed_partial_withdrawals_count; +fn getBuildersSweepWithdrawals( + allocator: Allocator, + state: *BeaconState(.gloas), + epoch: u64, + withdrawal_index: *u64, + num_prior_withdrawals: usize, + builder_balance_after_withdrawals: *std.AutoHashMap(u64, i64), +) !struct { withdrawals: std.ArrayList(Withdrawal), processed_count: usize } { + const withdrawals_limit = preset.MAX_WITHDRAWALS_PER_PAYLOAD - 1; + if (num_prior_withdrawals > withdrawals_limit) { + return error.PriorWithdrawalsExceedLimit; + } + var builders_sweep_withdrawals: std.ArrayList(Withdrawal) = .empty; + errdefer builders_sweep_withdrawals.deinit(allocator); + + var builders = try state.inner.get("builders"); + const builders_len: u64 = try builders.length(); + + // Return early if no builders + if (builders_len == 0) { + return .{ .withdrawals = builders_sweep_withdrawals, .processed_count = 0 }; + } + + const builders_limit = @min(builders_len, preset.MAX_BUILDERS_PER_WITHDRAWALS_SWEEP); + const next_withdrawal_builder_index: u64 = try state.inner.get("next_withdrawal_builder_index"); + if (next_withdrawal_builder_index >= builders_len) { + return error.InvalidNextWithdrawalBuilderIndex; + } + var processed_count: usize = 0; + + for (0..builders_limit) |n| { + if (builders_sweep_withdrawals.items.len + num_prior_withdrawals >= withdrawals_limit) break; + + // Get next builder in turn + const builder_index: u64 = (next_withdrawal_builder_index + n) % builders_len; + var builder: types.gloas.Builder.Type = undefined; + try builders.getValue(allocator, builder_index, &builder); + + // Get builder balance (may have been decremented by builder withdrawals above) + const balance_gop = try builder_balance_after_withdrawals.getOrPut(builder_index); + if (!balance_gop.found_existing) { + balance_gop.value_ptr.* = @intCast(builder.balance); + } + const balance = balance_gop.value_ptr.*; + + // Check if builder is withdrawable and has balance + if (builder.withdrawable_epoch <= epoch and balance > 0) { + // Withdraw full balance to builder's execution address + try builders_sweep_withdrawals.append(allocator, .{ + .index = withdrawal_index.*, + .validator_index = convertBuilderIndexToValidatorIndex(builder_index), + .address = builder.execution_address, + .amount = @intCast(balance), + }); + withdrawal_index.* += 1; + balance_gop.value_ptr.* = 0; + } + + processed_count += 1; + } + + return .{ .withdrawals = builders_sweep_withdrawals, .processed_count = processed_count }; } const TestCachedBeaconState = @import("../test_utils/root.zig").TestCachedBeaconState; @@ -229,10 +525,13 @@ test "process withdrawals - sanity" { var test_state = try TestCachedBeaconState.init(allocator, &pool, 256); defer test_state.deinit(); - var withdrawals_buf: [preset.MAX_WITHDRAWALS_PER_PAYLOAD]types.capella.Withdrawal.Type = undefined; var withdrawals_result = WithdrawalsResult{ - .withdrawals = Withdrawals.initBuffer(&withdrawals_buf), + .withdrawals = try Withdrawals.initCapacity( + allocator, + preset.MAX_WITHDRAWALS_PER_PAYLOAD, + ), }; + defer withdrawals_result.withdrawals.deinit(allocator); var withdrawal_balances = std.AutoHashMap(ValidatorIndex, usize).init(allocator); defer withdrawal_balances.deinit(); @@ -241,6 +540,7 @@ test "process withdrawals - sanity" { try getExpectedWithdrawals( .electra, + allocator, test_state.cached_state.epoch_cache, test_state.cached_state.state.castToFork(.electra), &withdrawals_result, From 77a55a09af868016e999282db51bac9a9ba182d5 Mon Sep 17 00:00:00 2001 From: Rahul Guha <19rahul2003@gmail.com> Date: Thu, 2 Jul 2026 02:49:11 +0530 Subject: [PATCH 09/33] update epoch processing and proposer lookahead --- src/state_transition/epoch/process_epoch.zig | 14 ++++++++ .../epoch/process_proposer_lookahead.zig | 15 +++++++- .../epoch/process_ptc_window.zig | 35 +++++++++++++++++++ 3 files changed, 63 insertions(+), 1 deletion(-) create mode 100644 src/state_transition/epoch/process_ptc_window.zig diff --git a/src/state_transition/epoch/process_epoch.zig b/src/state_transition/epoch/process_epoch.zig index 5bb7d3f2e..055cc9cdf 100644 --- a/src/state_transition/epoch/process_epoch.zig +++ b/src/state_transition/epoch/process_epoch.zig @@ -25,6 +25,8 @@ const processParticipationRecordUpdates = @import("./process_participation_recor const processParticipationFlagUpdates = @import("./process_participation_flag_updates.zig").processParticipationFlagUpdates; const processSyncCommitteeUpdates = @import("./process_sync_committee_updates.zig").processSyncCommitteeUpdates; const processProposerLookahead = @import("./process_proposer_lookahead.zig").processProposerLookahead; +const processBuilderPendingPayments = @import("./process_builder_pending_payments.zig").processBuilderPendingPayments; +const processPtcWindow = @import("./process_ptc_window.zig").processPtcWindow; const Node = @import("persistent_merkle_tree").Node; pub fn processEpoch( @@ -70,6 +72,12 @@ pub fn processEpoch( try observeEpochTransitionStep(.{ .step = .process_pending_consolidations }, @as(u64, @intCast(time.since(io, timer).nanoseconds))); } + if (comptime fork.gte(.gloas)) { + timer = time.timestampNow(io); + try processBuilderPendingPayments(allocator, state, epoch_cache); + try observeEpochTransitionStep(.{ .step = .process_builder_pending_payments }, @as(u64, @intCast(time.since(io, timer).nanoseconds))); + } + // const numUpdate = processEffectiveBalanceUpdates(fork, state, cache); timer = time.timestampNow(io); _ = try processEffectiveBalanceUpdates(fork, allocator, epoch_cache, state, cache); @@ -103,6 +111,12 @@ pub fn processEpoch( try processProposerLookahead(fork, allocator, epoch_cache, state, cache); try observeEpochTransitionStep(.{ .step = .process_proposer_lookahead }, @as(u64, @intCast(time.since(io, timer).nanoseconds))); } + + if (comptime fork.gte(.gloas)) { + timer = time.timestampNow(io); + try processPtcWindow(allocator, epoch_cache, state); + try observeEpochTransitionStep(.{ .step = .process_ptc_window }, @as(u64, @intCast(time.since(io, timer).nanoseconds))); + } } const TestCachedBeaconState = @import("../test_utils/root.zig").TestCachedBeaconState; diff --git a/src/state_transition/epoch/process_proposer_lookahead.zig b/src/state_transition/epoch/process_proposer_lookahead.zig index 7c4960754..d8c229830 100644 --- a/src/state_transition/epoch/process_proposer_lookahead.zig +++ b/src/state_transition/epoch/process_proposer_lookahead.zig @@ -8,6 +8,7 @@ const BeaconState = @import("fork_types").BeaconState; const EpochCache = @import("../cache/epoch_cache.zig").EpochCache; const EpochTransitionCache = @import("../cache/epoch_transition_cache.zig").EpochTransitionCache; const upgradeStateToFulu = @import("../slot/upgrade_state_to_fulu.zig").upgradeStateToFulu; +const FLAG_UNSLASHED = @import("../utils/attester_status.zig").FLAG_UNSLASHED; const ValidatorIndex = ssz.primitive.ValidatorIndex.Type; const computeEpochAtSlot = @import("../utils/epoch.zig").computeEpochAtSlot; const seed_utils = @import("../utils/seed.zig"); @@ -45,7 +46,19 @@ pub fn processProposerLookahead( // Active indices for the new epoch come from the epoch transition cache // (computed during beforeProcessEpoch for current_epoch + 2) - const active_indices = epoch_transition_cache.next_shuffling_active_indices; + const shuffling_active_indices = epoch_transition_cache.next_shuffling_active_indices; + var unslashed_active_indices: std.ArrayList(ValidatorIndex) = .empty; + defer unslashed_active_indices.deinit(allocator); + + const active_indices = if (comptime fork.gte(.gloas)) blk: { + try unslashed_active_indices.ensureTotalCapacity(allocator, shuffling_active_indices.len); + for (shuffling_active_indices) |index| { + if ((epoch_transition_cache.flags[@intCast(index)] & FLAG_UNSLASHED) != 0) { + unslashed_active_indices.appendAssumeCapacity(index); + } + } + break :blk unslashed_active_indices.items; + } else shuffling_active_indices; const effective_balance_increments = epoch_cache.getEffectiveBalanceIncrements(); var seed: [32]u8 = undefined; diff --git a/src/state_transition/epoch/process_ptc_window.zig b/src/state_transition/epoch/process_ptc_window.zig new file mode 100644 index 000000000..414004ef5 --- /dev/null +++ b/src/state_transition/epoch/process_ptc_window.zig @@ -0,0 +1,35 @@ +const std = @import("std"); +const Allocator = std.mem.Allocator; +const BeaconState = @import("fork_types").BeaconState; +const EpochCache = @import("../cache/epoch_cache.zig").EpochCache; +const ct = @import("consensus_types"); +const preset = @import("preset").preset; +const computeEpochAtSlot = @import("../utils/epoch.zig").computeEpochAtSlot; +const computeStartSlotAtEpoch = @import("../utils/epoch.zig").computeStartSlotAtEpoch; +const computePtc = @import("../utils/gloas.zig").computePtc; +const ValidatorIndex = ct.primitive.ValidatorIndex.Type; + +pub fn processPtcWindow( + allocator: Allocator, + epoch_cache: *const EpochCache, + state: *BeaconState(.gloas), +) !void { + var ptc_window = try state.inner.get("ptc_window"); + const ptc_window_len = ct.gloas.PtcWindow.length; + + var ptc_entry: [ct.gloas.PtcWindow.Element.length]ValidatorIndex = undefined; + for (0..ptc_window_len - preset.SLOTS_PER_EPOCH) |i| { + try ptc_window.getValue(undefined, i + preset.SLOTS_PER_EPOCH, &ptc_entry); + try ptc_window.setValue(i, &ptc_entry); + } + + const next_epoch = computeEpochAtSlot(try state.slot()) + preset.MIN_SEED_LOOKAHEAD + 1; + const start_slot = computeStartSlotAtEpoch(next_epoch); + + for (0..preset.SLOTS_PER_EPOCH) |slot_offset| { + const ptc = try computePtc(allocator, state, start_slot + slot_offset, null, epoch_cache.effective_balance_increments.get().items); + try ptc_window.setValue(ptc_window_len - preset.SLOTS_PER_EPOCH + slot_offset, &ptc); + } + + try state.inner.set("ptc_window", ptc_window); +} From dd3bea30189c0416d61c01aff979769c04dc7986 Mon Sep 17 00:00:00 2001 From: Rahul Guha <19rahul2003@gmail.com> Date: Thu, 2 Jul 2026 02:52:20 +0530 Subject: [PATCH 10/33] only clear slashed proposer pending payment --- .../block/process_proposer_slashing.zig | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/src/state_transition/block/process_proposer_slashing.zig b/src/state_transition/block/process_proposer_slashing.zig index bfc1146f3..22b6d95eb 100644 --- a/src/state_transition/block/process_proposer_slashing.zig +++ b/src/state_transition/block/process_proposer_slashing.zig @@ -11,6 +11,9 @@ const isSlashableValidator = @import("../utils/validator.zig").isSlashableValida const getProposerSlashingSignatureSets = @import("../signature_sets/proposer_slashings.zig").getProposerSlashingSignatureSets; const verifySignature = @import("../utils/signature_sets.zig").verifySingleSignatureSet; const slashValidator = @import("./slash_validator.zig").slashValidator; +const computeEpochAtSlot = @import("../utils/epoch.zig").computeEpochAtSlot; +const computePreviousEpoch = @import("../utils/epoch.zig").computePreviousEpoch; +const preset = @import("preset").preset; pub fn processProposerSlashing( comptime fork: ForkSeq, @@ -24,6 +27,31 @@ pub fn processProposerSlashing( ) !void { try buildSlashingsCacheIfNeeded(allocator, state, slashings_cache); try assertValidProposerSlashing(fork, config, epoch_cache, state, proposer_slashing, verify_signatures); + + if (fork.gte(.gloas)) { + const slot = proposer_slashing.signed_header_1.message.slot; + const proposal_epoch = computeEpochAtSlot(slot); + const current_epoch = epoch_cache.epoch; + const previous_epoch = computePreviousEpoch(current_epoch); + + const payment_index: ?u64 = if (proposal_epoch == current_epoch) + preset.SLOTS_PER_EPOCH + (slot % preset.SLOTS_PER_EPOCH) + else if (proposal_epoch == previous_epoch) + slot % preset.SLOTS_PER_EPOCH + else + null; + + if (payment_index) |idx| { + var builder_pending_payments = try state.inner.get("builder_pending_payments"); + var payment: types.gloas.BuilderPendingPayment.Type = undefined; + try builder_pending_payments.getValue(allocator, idx, &payment); + if (payment.proposer_index == proposer_slashing.signed_header_1.message.proposer_index) { + const default_payment = types.gloas.BuilderPendingPayment.default_value; + try builder_pending_payments.setValue(idx, &default_payment); + } + } + } + const proposer_index = proposer_slashing.signed_header_1.message.proposer_index; try slashValidator(fork, config, epoch_cache, state, slashings_cache, proposer_index, null); } From 824870f627b75b06fdbfc3c9741335cccf33a1e1 Mon Sep 17 00:00:00 2001 From: Rahul Guha <19rahul2003@gmail.com> Date: Thu, 2 Jul 2026 02:54:57 +0530 Subject: [PATCH 11/33] support current gloas operation vectors --- test/spec/runner/operations.zig | 118 ++++++++++++++++++++++++++------ 1 file changed, 97 insertions(+), 21 deletions(-) diff --git a/test/spec/runner/operations.zig b/test/spec/runner/operations.zig index bc740c82f..a29cb63d6 100644 --- a/test/spec/runner/operations.zig +++ b/test/spec/runner/operations.zig @@ -25,10 +25,15 @@ pub const Operation = enum { attester_slashing, block_header, bls_to_execution_change, + builder_deposit_request, + builder_exit_request, consolidation_request, deposit, deposit_request, execution_payload, + execution_payload_bid, + parent_execution_payload, + payload_attestation, proposer_slashing, sync_aggregate, voluntary_exit, @@ -40,6 +45,7 @@ pub const Operation = enum { .block_header => "block", .bls_to_execution_change => "address_change", .execution_payload => "body", + .parent_execution_payload => "block", .withdrawals => "execution_payload", else => @tagName(self), }; @@ -51,10 +57,15 @@ pub const Operation = enum { .attester_slashing => "AttesterSlashing", .block_header => "BeaconBlock", .bls_to_execution_change => "SignedBLSToExecutionChange", + .builder_deposit_request => "BuilderDepositRequest", + .builder_exit_request => "BuilderExitRequest", .consolidation_request => "ConsolidationRequest", .deposit => "Deposit", .deposit_request => "DepositRequest", .execution_payload => "BeaconBlockBody", + .execution_payload_bid => "SignedExecutionPayloadBid", + .parent_execution_payload => "BeaconBlock", + .payload_attestation => "PayloadAttestation", .proposer_slashing => "ProposerSlashing", .sync_aggregate => "SyncAggregate", .voluntary_exit => "SignedVoluntaryExit", @@ -70,10 +81,23 @@ pub const Operation = enum { pub const Handler = Operation; +fn loadExecutionValid(allocator: std.mem.Allocator, dir: std.Io.Dir) bool { + const contents = dir.readFileAlloc(std.testing.io, "execution.yaml", allocator, .limited(1024)) catch return true; + defer allocator.free(contents); + // Parse "{execution_valid: false}" or "{execution_valid: true}" + if (std.mem.indexOf(u8, contents, "false")) |_| return false; + return true; +} + pub fn TestCase(comptime fork: ForkSeq, comptime operation: Operation) type { const ForkTypes = @field(ssz, fork.name()); const tc_utils = TestCaseUtils(fork); - const OpType = @field(ForkTypes, operation.operationObject()); + // After EIP-7732, gloas execution_payload tests use SignedExecutionPayloadEnvelope + const is_gloas_exec_payload = comptime (operation == .execution_payload and fork.gte(.gloas)); + const OpType = if (is_gloas_exec_payload) + ForkTypes.SignedExecutionPayloadEnvelope + else + @field(ForkTypes, operation.operationObject()); return struct { pre: TestCachedBeaconState, @@ -81,6 +105,7 @@ pub fn TestCase(comptime fork: ForkSeq, comptime operation: Operation) type { post: ?*AnyBeaconState, op: OpType.Type, bls_setting: BlsSetting, + execution_valid: bool, const Self = @This(); @@ -104,6 +129,7 @@ pub fn TestCase(comptime fork: ForkSeq, comptime operation: Operation) type { .post = undefined, .op = OpType.default_value, .bls_setting = loadBlsSetting(allocator, dir), + .execution_valid = loadExecutionValid(allocator, dir), }; // load pre state @@ -114,7 +140,14 @@ pub fn TestCase(comptime fork: ForkSeq, comptime operation: Operation) type { tc.post = try tc_utils.loadPostState(allocator, pool, dir); // load the op - try loadSszValue(OpType, allocator, dir, comptime operation.inputName() ++ ".ssz_snappy", &tc.op); + // After EIP-7732, gloas withdrawals tests don't have an execution_payload input file + const input_name = comptime if (is_gloas_exec_payload) + "signed_envelope" + else + operation.inputName(); + if (comptime !(operation == .withdrawals and fork.gte(.gloas))) { + try loadSszValue(OpType, allocator, dir, input_name ++ ".ssz_snappy", &tc.op); + } errdefer { if (comptime @hasDecl(OpType, "deinit")) { OpType.deinit(allocator, &tc.op); @@ -189,6 +222,14 @@ pub fn TestCase(comptime fork: ForkSeq, comptime operation: Operation) type { const config = cached_state.config; try state_transition.processBlsToExecutionChange(fork, config, state, &self.op); }, + .builder_deposit_request => { + const config = cached_state.config; + try state_transition.processBuilderDepositRequest(allocator, config, state, &self.op); + }, + .builder_exit_request => { + const config = cached_state.config; + try state_transition.processBuilderExitRequest(allocator, config, state, &self.op); + }, .consolidation_request => { const config = cached_state.config; const epoch_cache = cached_state.epoch_cache; @@ -205,21 +246,52 @@ pub fn TestCase(comptime fork: ForkSeq, comptime operation: Operation) type { .execution_payload => { const config = cached_state.config; const epoch_cache = cached_state.epoch_cache; - const current_epoch = epoch_cache.epoch; - const fork_body = BeaconBlockBody(.full, fork){ .inner = self.op }; - try state_transition.processExecutionPayload( - fork, - allocator, - config, - state, - current_epoch, - .full, - &fork_body, - .{ - .data_availability_status = .available, - .execution_payload_status = if (self.post != null) .valid else .invalid, - }, - ); + if (comptime is_gloas_exec_payload) { + // After EIP-7732, execution_payload tests use processExecutionPayloadEnvelope + if (!self.execution_valid) { + return error.ExecutionPayloadInvalid; + } + try state_transition.processExecutionPayloadEnvelope( + allocator, + config, + epoch_cache, + state, + &self.op, + .{ .verify_signature = true, .verify_state_root = true }, + ); + } else { + const current_epoch = epoch_cache.epoch; + const fork_body = BeaconBlockBody(.full, fork){ .inner = self.op }; + try state_transition.processExecutionPayload( + fork, + allocator, + config, + state, + current_epoch, + .full, + &fork_body, + .{ + .data_availability_status = .available, + .execution_payload_status = if (self.post != null) .valid else .invalid, + }, + ); + } + }, + .execution_payload_bid => { + const config = cached_state.config; + const epoch_cache = cached_state.epoch_cache; + try state_transition.processExecutionPayloadBid(allocator, config, epoch_cache, state, &self.op); + }, + .parent_execution_payload => { + const config = cached_state.config; + const epoch_cache = cached_state.epoch_cache; + const fork_block = BeaconBlock(.full, fork){ .inner = self.op }; + try state_transition.processParentExecutionPayload(allocator, config, epoch_cache, state, &fork_block); + }, + .payload_attestation => { + const config = cached_state.config; + const epoch_cache = cached_state.epoch_cache; + try state_transition.processPayloadAttestation(allocator, config, epoch_cache, state, &self.op); }, .proposer_slashing => { const config = cached_state.config; @@ -268,25 +340,29 @@ pub fn TestCase(comptime fork: ForkSeq, comptime operation: Operation) type { .withdrawals => { const epoch_cache = cached_state.epoch_cache; - var withdrawals_buf: [preset.MAX_WITHDRAWALS_PER_PAYLOAD]ssz.capella.Withdrawal.Type = undefined; var withdrawals_result = WithdrawalsResult{ - .withdrawals = Withdrawals.initBuffer(&withdrawals_buf), + .withdrawals = try Withdrawals.initCapacity(allocator, preset.MAX_WITHDRAWALS_PER_PAYLOAD), }; + defer withdrawals_result.withdrawals.deinit(allocator); var withdrawal_balances = std.AutoHashMap(u64, usize).init(allocator); defer withdrawal_balances.deinit(); try state_transition.getExpectedWithdrawals( fork, + allocator, epoch_cache, state, &withdrawals_result, &withdrawal_balances, ); + // After EIP-7732, gloas withdrawals don't use payload verification var payload_withdrawals_root: Root = undefined; - // self.op is ExecutionPayload in this case - try ssz.capella.Withdrawals.hashTreeRoot(allocator, &self.op.withdrawals, &payload_withdrawals_root); + if (comptime fork.lt(.gloas)) { + // self.op is ExecutionPayload in this case + try ssz.capella.Withdrawals.hashTreeRoot(allocator, &self.op.withdrawals, &payload_withdrawals_root); + } try state_transition.processWithdrawals( fork, From b95d6f66e1c510c35547367ef6b078053bfa8350 Mon Sep 17 00:00:00 2001 From: Rahul Guha <19rahul2003@gmail.com> Date: Thu, 2 Jul 2026 02:56:22 +0530 Subject: [PATCH 12/33] clean up --- src/state_transition/root.zig | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/state_transition/root.zig b/src/state_transition/root.zig index b7fed11b1..d821772ee 100644 --- a/src/state_transition/root.zig +++ b/src/state_transition/root.zig @@ -110,6 +110,8 @@ const EpochShuffling = @import("./utils/epoch_shuffling.zig"); pub const calculateShufflingDecisionRoot = EpochShuffling.calculateShufflingDecisionRoot; pub const processProposerLookahead = @import("./epoch/process_proposer_lookahead.zig").processProposerLookahead; pub const processExecutionPayloadEnvelope = @import("./block/process_execution_payload_envelope.zig").processExecutionPayloadEnvelope; +pub const processExecutionPayloadBid = @import("./block/process_execution_payload_bid.zig").processExecutionPayloadBid; +pub const processPtcWindow = @import("./epoch/process_ptc_window.zig").processPtcWindow; pub const processParentExecutionPayload = @import("./block/process_parent_execution_payload.zig").processParentExecutionPayload; pub const applyParentExecutionPayload = @import("./block/process_parent_execution_payload.zig").applyParentExecutionPayload; From 144abd5cb82697c981f276ec0c1466867877eb08 Mon Sep 17 00:00:00 2001 From: Rahul Guha <19rahul2003@gmail.com> Date: Thu, 2 Jul 2026 03:00:39 +0530 Subject: [PATCH 13/33] add utility helpers --- src/state_transition/root.zig | 1 + src/state_transition/utils/gloas.zig | 15 +++++++++++++++ 2 files changed, 16 insertions(+) diff --git a/src/state_transition/root.zig b/src/state_transition/root.zig index d821772ee..368e8ef8e 100644 --- a/src/state_transition/root.zig +++ b/src/state_transition/root.zig @@ -94,6 +94,7 @@ pub const isExecutionEnabled = @import("./utils/execution.zig").isExecutionEnabl pub const isMergeTransitionComplete = @import("./utils/execution.zig").isMergeTransitionComplete; pub const getRandaoMix = @import("./utils/seed.zig").getRandaoMix; pub const getEffectiveBalanceIncrementsZeroInactive = @import("./utils/balance.zig").getEffectiveBalanceIncrementsZeroInactive; +pub const PendingDepositsLookup = @import("./utils/pending_deposits_lookup.zig").PendingDepositsLookup; pub const WithdrawalsResult = @import("./block/process_withdrawals.zig").WithdrawalsResult; diff --git a/src/state_transition/utils/gloas.zig b/src/state_transition/utils/gloas.zig index b4d5212ff..e35b21dd0 100644 --- a/src/state_transition/utils/gloas.zig +++ b/src/state_transition/utils/gloas.zig @@ -60,10 +60,25 @@ pub fn isActiveBuilder(builder: *const ct.gloas.Builder.Type, finalized_epoch: u return builder.deposit_epoch < finalized_epoch and builder.withdrawable_epoch == c.FAR_FUTURE_EPOCH; } +pub fn getExpectedGasLimit(parent_gas_limit: u64, target_gas_limit: u64) u64 { + const max_gas_limit_difference = @max(parent_gas_limit / 1024, 1) - 1; + + if (target_gas_limit > parent_gas_limit) { + return parent_gas_limit + @min(target_gas_limit - parent_gas_limit, max_gas_limit_difference); + } + + return parent_gas_limit - @min(parent_gas_limit - target_gas_limit, max_gas_limit_difference); +} + +pub fn isGasLimitTargetCompatible(parent_gas_limit: u64, gas_limit: u64, target_gas_limit: u64) bool { + return gas_limit == getExpectedGasLimit(parent_gas_limit, target_gas_limit); +} + pub fn getPendingBalanceToWithdrawForBuilder(allocator: Allocator, state: *BeaconState(.gloas), builder_index: u64) !u64 { var pending_balance: u64 = 0; var withdrawals = try state.inner.get("builder_pending_withdrawals"); + try withdrawals.commit(); const withdrawals_len = try withdrawals.length(); var w_it = withdrawals.iteratorReadonly(0); for (0..withdrawals_len) |_| { From 4818d70b804c3e025a592b71c1fbed5fcc785e19 Mon Sep 17 00:00:00 2001 From: Rahul Guha <19rahul2003@gmail.com> Date: Thu, 2 Jul 2026 05:36:50 +0530 Subject: [PATCH 14/33] port remaining Gloas STF helpers --- .../is_valid_indexed_payload_attestation.zig | 35 ++++++ .../block/process_attestation_altair.zig | 91 +++++++++++++- .../block/process_attestation_phase0.zig | 10 +- .../block/process_builder_deposit_request.zig | 2 +- .../block/process_operations.zig | 7 ++ .../block/process_payload_attestation.zig | 34 ++++++ src/state_transition/cache/epoch_cache.zig | 93 +++++++++++++++ .../process_builder_pending_payments.zig | 35 ++++++ src/state_transition/metrics.zig | 2 + src/state_transition/root.zig | 5 +- .../signature_sets/execution_payload_bid.zig | 20 ++++ .../indexed_payload_attestation.zig | 42 +++++++ src/state_transition/slot/process_slot.zig | 1 + .../slot/upgrade_state_to_altair.zig | 2 +- src/state_transition/utils/seed.zig | 111 ++++++++++++++++++ src/state_transition/utils/signature_sets.zig | 2 +- src/state_transition/utils/signing_root.zig | 12 ++ src/state_transition/utils/validator.zig | 35 ++++++ test/spec/runner/epoch_processing.zig | 8 ++ test/spec/runner/fork.zig | 13 +- test/spec/write_spec_tests.zig | 1 + 21 files changed, 548 insertions(+), 13 deletions(-) create mode 100644 src/state_transition/block/is_valid_indexed_payload_attestation.zig create mode 100644 src/state_transition/block/process_payload_attestation.zig create mode 100644 src/state_transition/epoch/process_builder_pending_payments.zig create mode 100644 src/state_transition/signature_sets/execution_payload_bid.zig create mode 100644 src/state_transition/signature_sets/indexed_payload_attestation.zig diff --git a/src/state_transition/block/is_valid_indexed_payload_attestation.zig b/src/state_transition/block/is_valid_indexed_payload_attestation.zig new file mode 100644 index 000000000..50d08c032 --- /dev/null +++ b/src/state_transition/block/is_valid_indexed_payload_attestation.zig @@ -0,0 +1,35 @@ +const std = @import("std"); +const types = @import("consensus_types"); +const ValidatorIndex = types.primitive.ValidatorIndex.Type; +const BeaconConfig = @import("config").BeaconConfig; +const EpochCache = @import("../cache/epoch_cache.zig").EpochCache; +const getIndexedPayloadAttestationSignatureSet = @import("../signature_sets/indexed_payload_attestation.zig").getIndexedPayloadAttestationSignatureSet; +const verifyAggregatedSignatureSet = @import("../utils/signature_sets.zig").verifyAggregatedSignatureSet; + +/// Validate an IndexedPayloadAttestation: check that attesting indices are non-empty, +/// sorted, and (optionally) that the aggregate BLS signature is valid. +pub fn isValidIndexedPayloadAttestation( + allocator: std.mem.Allocator, + config: *const BeaconConfig, + epoch_cache: *const EpochCache, + indexed_payload_attestation: *const types.gloas.IndexedPayloadAttestation.Type, + verify_signature: bool, +) !bool { + const attesting_indices = indexed_payload_attestation.attesting_indices.items; + + if (attesting_indices.len == 0) return false; + + var prev: ValidatorIndex = 0; + for (attesting_indices, 0..) |index, i| { + if (i >= 1 and index < prev) { + return false; + } + prev = index; + } + + if (!verify_signature) return true; + + const sig_set = try getIndexedPayloadAttestationSignatureSet(allocator, config, epoch_cache, indexed_payload_attestation); + defer allocator.free(sig_set.pubkeys); + return verifyAggregatedSignatureSet(&sig_set); +} diff --git a/src/state_transition/block/process_attestation_altair.zig b/src/state_transition/block/process_attestation_altair.zig index e5aa66b0c..3cdac2af0 100644 --- a/src/state_transition/block/process_attestation_altair.zig +++ b/src/state_transition/block/process_attestation_altair.zig @@ -18,6 +18,9 @@ const getBeaconProposer = @import("../cache/get_beacon_proposer.zig").getBeaconP const Checkpoint = types.phase0.Checkpoint.Type; const isTimelyTarget = @import("./process_attestation_phase0.zig").isTimelyTarget; const increaseBalance = @import("../utils/balance.zig").increaseBalance; +const gloas_utils = @import("../utils/gloas.zig"); +const isAttestationSameSlot = gloas_utils.isAttestationSameSlot; +const isAttestationSameSlotRootCache = gloas_utils.isAttestationSameSlotRootCache; const PROPOSER_REWARD_DOMINATOR = ((c.WEIGHT_DENOMINATOR - c.PROPOSER_WEIGHT) * c.WEIGHT_DENOMINATOR) / c.PROPOSER_WEIGHT; @@ -49,10 +52,18 @@ pub fn processAttestationsAltair( defer root_cache.deinit(); // Process all attestations first and then increase the balance of the proposer once - // let newSeenAttesters = 0; - // let newSeenAttestersEffectiveBalance = 0; - + // TODO: metrics — newSeenAttesters, newSeenAttestersEffectiveBalance, attestationsPerBlock var proposer_reward: u64 = 0; + + var builder_weight_map = std.AutoHashMap(u64, u64).init(allocator); + defer builder_weight_map.deinit(); + + // Get executionPayloadAvailability for gloas + const execution_payload_availability = if (fork.gte(.gloas)) + try state.inner.get("execution_payload_availability") + else + @as(?void, null); + for (attestations) |*attestation| { const data = &attestation.data; try validateAttestation(fork, epoch_cache, state, attestation); @@ -81,7 +92,17 @@ pub fn processAttestationsAltair( const in_current_epoch = data.target.epoch == current_epoch; var epoch_participation = if (in_current_epoch) try state.currentEpochParticipation() else try state.previousEpochParticipation(); - const flags_attestation = try getAttestationParticipationStatus(fork, data, state_slot - data.slot, current_epoch, root_cache); + // Count how much additional weight added to current or previous epoch's builder pending payment (in ETH increment) + var payment_weight_to_add: u64 = 0; + + const flags_attestation = try getAttestationParticipationStatus( + fork, + data, + state_slot - data.slot, + current_epoch, + root_cache, + execution_payload_availability, + ); // For each participant, update their participation // In epoch processing, this participation info is used to calculate balance updates @@ -122,13 +143,50 @@ pub fn processAttestationsAltair( } } } + + if (fork.gte(.gloas) and flags_new_set != 0 and (try isAttestationSameSlot(state, data))) { + payment_weight_to_add += effective_balance_increments[validator_index]; + } } + // Do the discrete math inside the loop to ensure a deterministic result const total_increments = total_balance_increments_with_weight; const proposer_reward_numerator = total_increments * epoch_cache.base_reward_per_increment; proposer_reward += @divFloor(proposer_reward_numerator, PROPOSER_REWARD_DOMINATOR); + + if (fork.gte(.gloas)) { + const builder_pending_payment_index: u64 = if (in_current_epoch) + preset.SLOTS_PER_EPOCH + (data.slot % preset.SLOTS_PER_EPOCH) + else + data.slot % preset.SLOTS_PER_EPOCH; + + var builder_pending_payments = try state.inner.get("builder_pending_payments"); + const existing_weight = builder_weight_map.get(builder_pending_payment_index) orelse blk: { + var payment: types.gloas.BuilderPendingPayment.Type = undefined; + try builder_pending_payments.getValue(allocator, builder_pending_payment_index, &payment); + break :blk payment.weight; + }; + const updated_weight = existing_weight + payment_weight_to_add * preset.EFFECTIVE_BALANCE_INCREMENT; + try builder_weight_map.put(builder_pending_payment_index, updated_weight); + } } + + // Batch write builder weights after all attestations + if (fork.gte(.gloas)) { + var builder_pending_payments = try state.inner.get("builder_pending_payments"); + var it = builder_weight_map.iterator(); + while (it.next()) |entry| { + var payment: types.gloas.BuilderPendingPayment.Type = undefined; + try builder_pending_payments.getValue(allocator, entry.key_ptr.*, &payment); + if (payment.withdrawal.amount > 0) { + payment.weight = entry.value_ptr.*; + try builder_pending_payments.setValue(entry.key_ptr.*, &payment); + } + } + } + try increaseBalance(fork, state, try getBeaconProposer(fork, epoch_cache, state, state_slot), proposer_reward); + // TODO: state.proposerRewards.attestations = proposerReward and metrics } pub fn getAttestationParticipationStatus( @@ -136,7 +194,8 @@ pub fn getAttestationParticipationStatus( data: *const types.phase0.AttestationData.Type, inclusion_delay: u64, current_epoch: Epoch, - root_cache: *RootCache(fork), + root_cache: anytype, + execution_payload_availability: anytype, ) !u8 { const justified_checkpoint = if (data.target.epoch == current_epoch) &root_cache.current_justified_checkpoint @@ -148,9 +207,29 @@ pub fn getAttestationParticipationStatus( const is_matching_target = std.mem.eql(u8, &data.target.root, try root_cache.getBlockRoot(data.target.epoch)); // a timely head is only be set if the target is _also_ matching - const is_matching_head = + var is_matching_head = is_matching_target and std.mem.eql(u8, &data.beacon_block_root, try root_cache.getBlockRootAtSlot(data.slot)); + if (fork.gte(.gloas)) { + var is_matching_payload = false; + + if (try isAttestationSameSlotRootCache(root_cache, data)) { + if (data.index != 0) { + return error.AttestingSameSlotMustIndicateEmptyPayload; + } + is_matching_payload = true; + } else { + if (data.index != 0 and data.index != 1) { + return error.DataIndexMustBe0Or1; + } + + is_matching_payload = + (data.index != 0) == try execution_payload_availability.get(data.slot % preset.SLOTS_PER_HISTORICAL_ROOT); + } + + is_matching_head = is_matching_head and is_matching_payload; + } + var flags: u8 = 0; if (is_matching_source and inclusion_delay <= SLOTS_PER_EPOCH_SQRT) flags |= TIMELY_SOURCE; if (is_matching_target and isTimelyTarget(fork, inclusion_delay)) flags |= TIMELY_TARGET; diff --git a/src/state_transition/block/process_attestation_phase0.zig b/src/state_transition/block/process_attestation_phase0.zig index c5e40c865..d6cc740f8 100644 --- a/src/state_transition/block/process_attestation_phase0.zig +++ b/src/state_transition/block/process_attestation_phase0.zig @@ -92,8 +92,14 @@ pub fn validateAttestation( } if (fork.gte(.electra)) { - if (data.index != 0) { - return error.InvalidAttestationNonZeroDataIndex; + if (fork.gte(.gloas)) { + if (data.index >= 2) { + return error.InvalidAttestationDataIndexOutOfRange; + } + } else { + if (data.index != 0) { + return error.InvalidAttestationNonZeroDataIndex; + } } var committee_indices_buffer: [preset.MAX_COMMITTEES_PER_SLOT]usize = undefined; const committee_indices_len = try attestation.committee_bits.getTrueBitIndexes(committee_indices_buffer[0..]); diff --git a/src/state_transition/block/process_builder_deposit_request.zig b/src/state_transition/block/process_builder_deposit_request.zig index 0efe83817..22bc58f4d 100644 --- a/src/state_transition/block/process_builder_deposit_request.zig +++ b/src/state_transition/block/process_builder_deposit_request.zig @@ -23,7 +23,7 @@ pub fn processBuilderDepositRequest( var builder: types.gloas.Builder.Type = undefined; try builders.getValue(allocator, idx, &builder); - if (builder.withdrawable_epoch != c.FAR_FUTURE_EPOCH and builder.balance == 0) { + if (builder.withdrawable_epoch != c.FAR_FUTURE_EPOCH) { builder.withdrawable_epoch = computeEpochAtSlot(try state.slot()) + config.chain.MIN_BUILDER_WITHDRAWABILITY_DELAY; } diff --git a/src/state_transition/block/process_operations.zig b/src/state_transition/block/process_operations.zig index bd8819dcf..156f36178 100644 --- a/src/state_transition/block/process_operations.zig +++ b/src/state_transition/block/process_operations.zig @@ -21,6 +21,7 @@ const processVoluntaryExit = @import("./process_voluntary_exit.zig").processVolu const processWithdrawalRequest = @import("./process_withdrawal_request.zig").processWithdrawalRequest; const Node = @import("persistent_merkle_tree").Node; const ProcessBlockOpts = @import("./process_block.zig").ProcessBlockOpts; +const processPayloadAttestation = @import("./process_payload_attestation.zig").processPayloadAttestation; pub fn processOperations( comptime fork: ForkSeq, @@ -91,6 +92,12 @@ pub fn processOperations( try processConsolidationRequest(fork, config, epoch_cache, state, consolidation_request); } } + + if (comptime fork.gte(.gloas)) { + for (body.inner.payload_attestations.items) |*payload_attestation| { + try processPayloadAttestation(allocator, config, epoch_cache, state, payload_attestation); + } + } } const TestCachedBeaconState = @import("../test_utils/root.zig").TestCachedBeaconState; diff --git a/src/state_transition/block/process_payload_attestation.zig b/src/state_transition/block/process_payload_attestation.zig new file mode 100644 index 000000000..ac7aa48a5 --- /dev/null +++ b/src/state_transition/block/process_payload_attestation.zig @@ -0,0 +1,34 @@ +const std = @import("std"); +const Allocator = std.mem.Allocator; +const types = @import("consensus_types"); +const BeaconState = @import("fork_types").BeaconState; +const EpochCache = @import("../cache/epoch_cache.zig").EpochCache; +const BeaconConfig = @import("config").BeaconConfig; +const isValidIndexedPayloadAttestation = @import("./is_valid_indexed_payload_attestation.zig").isValidIndexedPayloadAttestation; + +pub fn processPayloadAttestation( + allocator: Allocator, + config: *const BeaconConfig, + epoch_cache: *const EpochCache, + state: *BeaconState(.gloas), + payload_attestation: *const types.gloas.PayloadAttestation.Type, +) !void { + const data = &payload_attestation.data; + + var latest_block_header = try state.latestBlockHeader(); + const parent_root = try latest_block_header.getFieldRoot("parent_root"); + if (!std.mem.eql(u8, &data.beacon_block_root, parent_root)) { + return error.PayloadAttestationWrongBlock; + } + + if (data.slot + 1 != try state.slot()) { + return error.PayloadAttestationNotFromPreviousSlot; + } + + var indexed_payload_attestation = try epoch_cache.getIndexedPayloadAttestation(allocator, state, data.slot, payload_attestation); + defer indexed_payload_attestation.attesting_indices.deinit(allocator); + + if (!(try isValidIndexedPayloadAttestation(allocator, config, epoch_cache, &indexed_payload_attestation, true))) { + return error.InvalidPayloadAttestation; + } +} diff --git a/src/state_transition/cache/epoch_cache.zig b/src/state_transition/cache/epoch_cache.zig index b93d32834..9775eaf5e 100644 --- a/src/state_transition/cache/epoch_cache.zig +++ b/src/state_transition/cache/epoch_cache.zig @@ -30,6 +30,7 @@ const getTotalSlashingsByIncrement = @import("../epoch/process_slashings.zig").g const computeEpochShuffling = @import("../utils/epoch_shuffling.zig").computeEpochShuffling; const getSeed = @import("../utils/seed.zig").getSeed; const computeProposers = @import("../utils/seed.zig").computeProposers; +const computePayloadTimelinessCommitteesForEpoch = @import("../utils/seed.zig").computePayloadTimelinessCommitteesForEpoch; const SyncCommitteeCacheRc = @import("./sync_committee_cache.zig").SyncCommitteeCacheRc; const SyncCommitteeCacheAllForks = @import("./sync_committee_cache.zig").SyncCommitteeCache; const computeSyncParticipantReward = @import("../utils/sync_committee.zig").computeSyncParticipantReward; @@ -136,6 +137,12 @@ pub const EpochCache = struct { epoch: Epoch, + /// PTC for current epoch, computed eagerly at epoch transition. + payload_timeliness_committees: ?[preset.SLOTS_PER_EPOCH][preset.PTC_SIZE]ValidatorIndex, + + /// PTC for previous epoch, required for slot N block validating slot N-1 attestations. + previous_payload_timeliness_committees: ?[preset.SLOTS_PER_EPOCH][preset.PTC_SIZE]ValidatorIndex, + fn initEffectiveBalanceIncrementsRc(allocator: Allocator, validator_count: usize) !*EffectiveBalanceIncrementsRc { var effective_balance_increments = try effectiveBalanceIncrementsInit(allocator, validator_count); errdefer effective_balance_increments.deinit(allocator); @@ -468,8 +475,23 @@ pub const EpochCache = struct { .next_sync_committee_indexed = next_sync_committee_indexed, .sync_period = computeSyncPeriodAtEpoch(current_epoch), .epoch = current_epoch, + .payload_timeliness_committees = null, + .previous_payload_timeliness_committees = null, }; + // Compute PTC for all slots in the prev/current epoch + if (current_epoch >= config.chain.GLOAS_FORK_EPOCH) { + epoch_cache_ptr.payload_timeliness_committees = switch (state.forkSeq()) { + inline else => |f| try computePayloadTimelinessCommitteesForEpoch(f, allocator, state.castToFork(f), current_epoch, epoch_cache_ptr), + }; + + if (!is_genesis and previous_epoch >= config.chain.GLOAS_FORK_EPOCH) { + epoch_cache_ptr.previous_payload_timeliness_committees = switch (state.forkSeq()) { + inline else => |f| try computePayloadTimelinessCommitteesForEpoch(f, allocator, state.castToFork(f), previous_epoch, epoch_cache_ptr), + }; + } + } + return epoch_cache_ptr; } @@ -527,6 +549,8 @@ pub const EpochCache = struct { .next_sync_committee_indexed = self.next_sync_committee_indexed.ref(), .sync_period = self.sync_period, .epoch = self.epoch, + .payload_timeliness_committees = self.payload_timeliness_committees, + .previous_payload_timeliness_committees = self.previous_payload_timeliness_committees, }; const epoch_cache_ptr = try allocator.create(EpochCache); @@ -609,6 +633,21 @@ pub const EpochCache = struct { /// At fork boundary, this runs post-fork logic and after `upgradeState*`. pub fn finalProcessEpoch(self: *EpochCache, state: *AnyBeaconState) !void { self.proposers_prev_epoch = self.proposers; + + // Shift and compute current epoch PTC eagerly for all slots + if (self.epoch >= self.config.chain.GLOAS_FORK_EPOCH) { + self.previous_payload_timeliness_committees = self.payload_timeliness_committees; + self.payload_timeliness_committees = switch (state.forkSeq()) { + inline else => |f| try computePayloadTimelinessCommitteesForEpoch( + f, + self.allocator, + state.castToFork(f), + self.epoch, + self, + ), + }; + } + switch (state.forkSeq()) { inline else => |fork| { const fork_state = state.castToFork(fork); @@ -947,4 +986,58 @@ pub const EpochCache = struct { pub fn isPostElectra(self: *const EpochCache) bool { return self.epoch >= self.config.chain.ELECTRA_FORK_EPOCH; } + + /// Convert a PayloadAttestation into an IndexedPayloadAttestation by resolving + /// aggregation bits against the Payload Timeliness Committee for the attestation's slot. + /// Spec: get_ptc — Read PTC from state.ptc_window + pub fn getPayloadTimelinessCommittee(self: *const EpochCache, state: *BeaconState(.gloas), slot: Slot) ![preset.PTC_SIZE]ValidatorIndex { + _ = self; + const epoch = computeEpochAtSlot(slot); + const state_epoch = computeEpochAtSlot(try state.slot()); + + var ptc_window = try state.inner.get("ptc_window"); + + const index = if (epoch < state_epoch) blk: { + break :blk slot % preset.SLOTS_PER_EPOCH; + } else blk: { + const offset = (epoch - state_epoch + 1) * preset.SLOTS_PER_EPOCH; + break :blk offset + slot % preset.SLOTS_PER_EPOCH; + }; + + var ptc_entry: [preset.PTC_SIZE]ValidatorIndex = undefined; + try ptc_window.getValue(undefined, index, &ptc_entry); + return ptc_entry; + } + + pub fn getIndexedPayloadAttestation( + self: *const EpochCache, + allocator: Allocator, + state: *BeaconState(.gloas), + slot: Slot, + payload_attestation: *const types.gloas.PayloadAttestation.Type, + ) !types.gloas.IndexedPayloadAttestation.Type { + const payload_timeliness_committee = try self.getPayloadTimelinessCommittee(state, slot); + + var attesting_indices: std.ArrayList(ValidatorIndex) = .empty; + errdefer attesting_indices.deinit(allocator); + + for (0..payload_timeliness_committee.len) |i| { + if (try payload_attestation.aggregation_bits.get(i)) { + try attesting_indices.append(allocator, payload_timeliness_committee[i]); + } + } + + const sortFn = struct { + pub fn sort(_: void, a: ValidatorIndex, b: ValidatorIndex) bool { + return a < b; + } + }.sort; + std.mem.sort(ValidatorIndex, attesting_indices.items, {}, sortFn); + + return .{ + .attesting_indices = attesting_indices, + .data = payload_attestation.data, + .signature = payload_attestation.signature, + }; + } }; diff --git a/src/state_transition/epoch/process_builder_pending_payments.zig b/src/state_transition/epoch/process_builder_pending_payments.zig new file mode 100644 index 000000000..9cafdeb46 --- /dev/null +++ b/src/state_transition/epoch/process_builder_pending_payments.zig @@ -0,0 +1,35 @@ +const std = @import("std"); +const Allocator = std.mem.Allocator; +const ForkSeq = @import("config").ForkSeq; +const BeaconState = @import("fork_types").BeaconState; +const ct = @import("consensus_types"); +const preset = @import("preset").preset; +const getBuilderPaymentQuorumThreshold = @import("../utils/gloas.zig").getBuilderPaymentQuorumThreshold; + +/// Processes the builder pending payments from the previous epoch. +pub fn processBuilderPendingPayments(allocator: Allocator, state: *BeaconState(.gloas), epoch_cache: *const @import("../cache/epoch_cache.zig").EpochCache) !void { + const quorum = getBuilderPaymentQuorumThreshold(epoch_cache); + + var builderPendingPayments = try state.inner.get("builder_pending_payments"); + var builderPendingWithdrawals = try state.inner.get("builder_pending_withdrawals"); + + for (0..preset.SLOTS_PER_EPOCH) |i| { + var payment: ct.gloas.BuilderPendingPayment.Type = undefined; + try builderPendingPayments.getValue(allocator, i, &payment); + if (payment.weight >= quorum) { + try builderPendingWithdrawals.pushValue(&payment.withdrawal); + } + } + // TODO: Gloas - Optimization needed + const total_payments = @TypeOf(builderPendingPayments.*).length; + for (0..total_payments) |i| { + if (i < preset.SLOTS_PER_EPOCH) { + var nextEpochPayment: ct.gloas.BuilderPendingPayment.Type = undefined; + try builderPendingPayments.getValue(allocator, i + preset.SLOTS_PER_EPOCH, &nextEpochPayment); + try builderPendingPayments.setValue(i, &nextEpochPayment); + } else { + const defaultPayment = ct.gloas.BuilderPendingPayment.default_value; + try builderPendingPayments.setValue(i, &defaultPayment); + } + } +} diff --git a/src/state_transition/metrics.zig b/src/state_transition/metrics.zig index 15237bb9d..0eb2c5579 100644 --- a/src/state_transition/metrics.zig +++ b/src/state_transition/metrics.zig @@ -35,7 +35,9 @@ pub const EpochTransitionStepKind = enum { process_sync_committee_updates, process_pending_deposits, process_pending_consolidations, + process_builder_pending_payments, process_proposer_lookahead, + process_ptc_window, }; pub const ProposerRewardKind = enum { diff --git a/src/state_transition/root.zig b/src/state_transition/root.zig index 368e8ef8e..099ad917e 100644 --- a/src/state_transition/root.zig +++ b/src/state_transition/root.zig @@ -54,6 +54,7 @@ pub const upgradeStateToCapella = @import("./slot/upgrade_state_to_capella.zig") pub const upgradeStateToDeneb = @import("./slot/upgrade_state_to_deneb.zig").upgradeStateToDeneb; pub const upgradeStateToElectra = @import("./slot/upgrade_state_to_electra.zig").upgradeStateToElectra; pub const upgradeStateToFulu = @import("./slot/upgrade_state_to_fulu.zig").upgradeStateToFulu; +pub const upgradeStateToGloas = @import("./slot/upgrade_state_to_gloas.zig").upgradeStateToGloas; // Block pub const processBlockHeader = @import("./block/process_block_header.zig").processBlockHeader; @@ -110,8 +111,10 @@ pub const preset = @import("preset").preset; const EpochShuffling = @import("./utils/epoch_shuffling.zig"); pub const calculateShufflingDecisionRoot = EpochShuffling.calculateShufflingDecisionRoot; pub const processProposerLookahead = @import("./epoch/process_proposer_lookahead.zig").processProposerLookahead; -pub const processExecutionPayloadEnvelope = @import("./block/process_execution_payload_envelope.zig").processExecutionPayloadEnvelope; +pub const processBuilderPendingPayments = @import("./epoch/process_builder_pending_payments.zig").processBuilderPendingPayments; pub const processExecutionPayloadBid = @import("./block/process_execution_payload_bid.zig").processExecutionPayloadBid; +pub const processExecutionPayloadEnvelope = @import("./block/process_execution_payload_envelope.zig").processExecutionPayloadEnvelope; +pub const processPayloadAttestation = @import("./block/process_payload_attestation.zig").processPayloadAttestation; pub const processPtcWindow = @import("./epoch/process_ptc_window.zig").processPtcWindow; pub const processParentExecutionPayload = @import("./block/process_parent_execution_payload.zig").processParentExecutionPayload; pub const applyParentExecutionPayload = @import("./block/process_parent_execution_payload.zig").applyParentExecutionPayload; diff --git a/src/state_transition/signature_sets/execution_payload_bid.zig b/src/state_transition/signature_sets/execution_payload_bid.zig new file mode 100644 index 000000000..cd0a83a61 --- /dev/null +++ b/src/state_transition/signature_sets/execution_payload_bid.zig @@ -0,0 +1,20 @@ +const std = @import("std"); +const Allocator = std.mem.Allocator; +const types = @import("consensus_types"); +const BeaconConfig = @import("config").BeaconConfig; +const c = @import("constants"); +const computeSigningRootVariable = @import("../utils/signing_root.zig").computeSigningRootVariable; +const computeEpochAtSlot = @import("../utils/epoch.zig").computeEpochAtSlot; + +pub fn getExecutionPayloadBidSigningRoot( + allocator: Allocator, + config: *const BeaconConfig, + state_slot: u64, + bid: *const types.gloas.ExecutionPayloadBid.Type, +) ![32]u8 { + const domain = try config.getDomain(computeEpochAtSlot(state_slot), c.DOMAIN_BEACON_BUILDER, null); + + var out: [32]u8 = undefined; + try computeSigningRootVariable(types.gloas.ExecutionPayloadBid, allocator, bid, domain, &out); + return out; +} diff --git a/src/state_transition/signature_sets/indexed_payload_attestation.zig b/src/state_transition/signature_sets/indexed_payload_attestation.zig new file mode 100644 index 000000000..901798f6e --- /dev/null +++ b/src/state_transition/signature_sets/indexed_payload_attestation.zig @@ -0,0 +1,42 @@ +const std = @import("std"); +const Allocator = std.mem.Allocator; +const bls = @import("bls"); +const PublicKey = bls.PublicKey; +const types = @import("consensus_types"); +const Root = types.primitive.Root.Type; +const BLSSignature = types.primitive.BLSSignature.Type; +const ValidatorIndex = types.primitive.ValidatorIndex.Type; +const BeaconConfig = @import("config").BeaconConfig; +const EpochCache = @import("../cache/epoch_cache.zig").EpochCache; +const c = @import("constants"); +const computeSigningRoot = @import("../utils/signing_root.zig").computeSigningRoot; +const computeEpochAtSlot = @import("../utils/epoch.zig").computeEpochAtSlot; +const AggregatedSignatureSet = @import("../utils/signature_sets.zig").AggregatedSignatureSet; +const createAggregateSignatureSetFromComponents = @import("../utils/signature_sets.zig").createAggregateSignatureSetFromComponents; + +pub fn getPayloadAttestationDataSigningRoot(config: *const BeaconConfig, data: *const types.gloas.PayloadAttestationData.Type, out: *[32]u8) !void { + const domain = try config.getDomain(computeEpochAtSlot(data.slot), c.DOMAIN_PTC_ATTESTER, null); + + try computeSigningRoot(types.gloas.PayloadAttestationData, data, domain, out); +} + +/// Consumer needs to free the returned pubkeys array. +pub fn getIndexedPayloadAttestationSignatureSet( + allocator: Allocator, + config: *const BeaconConfig, + epoch_cache: *const EpochCache, + indexed_payload_attestation: *const types.gloas.IndexedPayloadAttestation.Type, +) !AggregatedSignatureSet { + const attesting_indices = indexed_payload_attestation.attesting_indices.items; + + const pubkeys = try allocator.alloc(PublicKey, attesting_indices.len); + errdefer allocator.free(pubkeys); + for (attesting_indices, 0..) |index, i| { + pubkeys[i] = epoch_cache.index_to_pubkey.items[index]; + } + + var signing_root: Root = undefined; + try getPayloadAttestationDataSigningRoot(config, &indexed_payload_attestation.data, &signing_root); + + return createAggregateSignatureSetFromComponents(pubkeys, signing_root, indexed_payload_attestation.signature); +} diff --git a/src/state_transition/slot/process_slot.zig b/src/state_transition/slot/process_slot.zig index 0ba9722dc..f1c52deb5 100644 --- a/src/state_transition/slot/process_slot.zig +++ b/src/state_transition/slot/process_slot.zig @@ -1,4 +1,5 @@ const std = @import("std"); +const ForkSeq = @import("config").ForkSeq; const AnyBeaconState = @import("fork_types").AnyBeaconState; const preset = @import("preset").preset; const ZERO_HASH = @import("constants").ZERO_HASH; diff --git a/src/state_transition/slot/upgrade_state_to_altair.zig b/src/state_transition/slot/upgrade_state_to_altair.zig index 89bea9d9d..d87e1ef5c 100644 --- a/src/state_transition/slot/upgrade_state_to_altair.zig +++ b/src/state_transition/slot/upgrade_state_to_altair.zig @@ -107,7 +107,7 @@ fn translateParticipation( for (pending_attestations) |*attestation| { const data = &attestation.data; - const attestation_flag = try getAttestationParticipationStatus(.phase0, data, attestation.inclusion_delay, epoch_cache.epoch, root_cache); + const attestation_flag = try getAttestationParticipationStatus(.phase0, data, attestation.inclusion_delay, epoch_cache.epoch, root_cache, null); const committee_indices = try epoch_cache.getBeaconCommittee(data.slot, data.index); var attesting_indices = try attestation.aggregation_bits.intersectValues(ValidatorIndex, allocator, committee_indices); defer attesting_indices.deinit(allocator); diff --git a/src/state_transition/utils/seed.zig b/src/state_transition/utils/seed.zig index 3ab7f9b31..159544d6d 100644 --- a/src/state_transition/utils/seed.zig +++ b/src/state_transition/utils/seed.zig @@ -77,6 +77,117 @@ pub fn getNextSyncCommitteeIndices(comptime fork: ForkSeq, allocator: Allocator, try computeSyncCommitteeIndices(allocator, &seed, active_indices, effective_balance_increments.items, rand_byte_count, max_effective_balance, preset.EFFECTIVE_BALANCE_INCREMENT, preset.SHUFFLE_ROUND_COUNT, out); } +/// Select PTC_SIZE validators from the given indices using balance-weighted acceptance sampling. +pub fn computePayloadTimelinessCommitteeIndices( + effective_balance_increments: []const u16, + indices: []const ValidatorIndex, + seed: *const [32]u8, +) ![preset.PTC_SIZE]ValidatorIndex { + if (indices.len == 0) return error.EmptyIndices; + + const MAX_RANDOM_VALUE: u64 = 0xFFFF; + const max_effective_balance_increment: u64 = preset.MAX_EFFECTIVE_BALANCE_ELECTRA / preset.EFFECTIVE_BALANCE_INCREMENT; + + var result: [preset.PTC_SIZE]ValidatorIndex = undefined; + var result_len: usize = 0; + + // Pre-allocate hash input buffer: seed (32 bytes) + block index (8 bytes) + var hash_input: [40]u8 = undefined; + @memcpy(hash_input[0..32], seed); + + var i: u64 = 0; + var random_bytes: [32]u8 = undefined; + var last_block: u64 = std.math.maxInt(u64); + + while (result_len < preset.PTC_SIZE) { + const candidate_index = indices[@intCast(i % indices.len)]; + + // Only recompute hash every 16 iterations + const block = i / 16; + if (block != last_block) { + std.mem.writeInt(u64, hash_input[32..][0..8], block, .little); + Sha256.hash(&hash_input, &random_bytes, .{}); + last_block = block; + } + + const offset: usize = @intCast((i % 16) * 2); + const random_value: u64 = std.mem.readInt(u16, random_bytes[offset..][0..2], .little); + + const candidate_effective_balance_increment: u64 = effective_balance_increments[@intCast(candidate_index)]; + if (candidate_effective_balance_increment * MAX_RANDOM_VALUE >= max_effective_balance_increment * random_value) { + result[result_len] = candidate_index; + result_len += 1; + } + i += 1; + } + + return result; +} + +/// Compute the Payload Timeliness Committee for a single slot by concatenating all +/// beacon committees for that slot and selecting PTC_SIZE members via balance-weighted sampling. +pub fn computePayloadTimelinessCommitteeForSlot( + allocator: Allocator, + slot_seed: *const [32]u8, + slot_committees: []const []const ValidatorIndex, + effective_balance_increments: []const u16, +) ![preset.PTC_SIZE]ValidatorIndex { + var total_len: usize = 0; + for (slot_committees) |committee| { + total_len += committee.len; + } + + const all_indices = try allocator.alloc(ValidatorIndex, total_len); + defer allocator.free(all_indices); + + var offset: usize = 0; + for (slot_committees) |committee| { + @memcpy(all_indices[offset .. offset + committee.len], committee); + offset += committee.len; + } + + return computePayloadTimelinessCommitteeIndices(effective_balance_increments, all_indices, slot_seed); +} + +/// Compute the Payload Timeliness Committee for every slot in an epoch. +pub fn computePayloadTimelinessCommitteesForEpoch( + comptime fork: ForkSeq, + allocator: Allocator, + state: *BeaconState(fork), + epoch: Epoch, + epoch_cache: *const @import("../cache/epoch_cache.zig").EpochCache, +) ![preset.SLOTS_PER_EPOCH][preset.PTC_SIZE]ValidatorIndex { + var epoch_seed: [32]u8 = undefined; + try getSeed(fork, state, epoch, c.DOMAIN_PTC_ATTESTER, &epoch_seed); + + const start_slot = computeStartSlotAtEpoch(epoch); + + var slot_seed_input: [40]u8 = undefined; + @memcpy(slot_seed_input[0..32], &epoch_seed); + + var result: [preset.SLOTS_PER_EPOCH][preset.PTC_SIZE]ValidatorIndex = undefined; + + for (0..preset.SLOTS_PER_EPOCH) |i| { + const slot = start_slot + i; + std.mem.writeInt(u64, slot_seed_input[32..][0..8], slot, .little); + + var slot_seed: [32]u8 = undefined; + Sha256.hash(&slot_seed_input, &slot_seed, .{}); + + const committees_per_slot = try epoch_cache.getCommitteeCountPerSlot(epoch); + const slot_committees = try allocator.alloc([]const ValidatorIndex, committees_per_slot); + defer allocator.free(slot_committees); + + for (0..committees_per_slot) |ci| { + slot_committees[ci] = try epoch_cache.getBeaconCommittee(slot, ci); + } + + result[i] = try computePayloadTimelinessCommitteeForSlot(allocator, &slot_seed, slot_committees, epoch_cache.effective_balance_increments.get().items); + } + + return result; +} + pub fn getRandaoMix(comptime fork: ForkSeq, state: *BeaconState(fork), epoch: Epoch) !*const [32]u8 { var randao_mixes = try state.randaoMixes(); return try randao_mixes.getFieldRoot(epoch % EPOCHS_PER_HISTORICAL_VECTOR); diff --git a/src/state_transition/utils/signature_sets.zig b/src/state_transition/utils/signature_sets.zig index 34e189212..cf589bc64 100644 --- a/src/state_transition/utils/signature_sets.zig +++ b/src/state_transition/utils/signature_sets.zig @@ -39,7 +39,7 @@ pub fn verifyAggregatedSignatureSet(set: *const AggregatedSignatureSet) !bool { return fastAggregateVerify(&set.signing_root, set.pubkeys, &signature, .{}); } -pub fn createSingleSignatureSetFromComponents(pubkey: *const PublicKey, signing_root: Root, signature: BLSSignature) SingleSignatureSet { +pub fn createSingleSignatureSetFromComponents(pubkey: PublicKey, signing_root: Root, signature: BLSSignature) SingleSignatureSet { return .{ .pubkey = pubkey, .signing_root = signing_root, diff --git a/src/state_transition/utils/signing_root.zig b/src/state_transition/utils/signing_root.zig index 27cfb0697..c1834a7ef 100644 --- a/src/state_transition/utils/signing_root.zig +++ b/src/state_transition/utils/signing_root.zig @@ -19,6 +19,18 @@ pub fn computeSigningRoot(comptime T: type, ssz_object: *const T.Type, domain: * try types.phase0.SigningData.hashTreeRoot(&domain_wrapped_object, out); } +/// Return the signing root for a variable-size SSZ object +pub fn computeSigningRootVariable(comptime T: type, allocator: Allocator, ssz_object: *const T.Type, domain: *const Domain, out: *[32]u8) !void { + var object_root: Root = undefined; + try T.hashTreeRoot(allocator, ssz_object, &object_root); + const domain_wrapped_object: SigningData = .{ + .object_root = object_root, + .domain = domain.*, + }; + + try types.phase0.SigningData.hashTreeRoot(&domain_wrapped_object, out); +} + pub fn computeBlockSigningRoot(allocator: Allocator, block: AnyBeaconBlock, domain: *const Domain, out: *[32]u8) !void { var object_root: Root = undefined; try block.hashTreeRoot(allocator, &object_root); diff --git a/src/state_transition/utils/validator.zig b/src/state_transition/utils/validator.zig index de79a980e..d89547af0 100644 --- a/src/state_transition/utils/validator.zig +++ b/src/state_transition/utils/validator.zig @@ -12,6 +12,8 @@ const ForkSeq = @import("config").ForkSeq; const EpochCache = @import("../cache/epoch_cache.zig").EpochCache; const WithdrawalCredentials = types.primitive.Root.Type; const hasCompoundingWithdrawalCredential = @import("./electra.zig").hasCompoundingWithdrawalCredential; +const hasExecutionWithdrawalCredential = @import("./electra.zig").hasExecutionWithdrawalCredential; +const hasEth1WithdrawalCredential = @import("./capella.zig").hasEth1WithdrawalCredential; pub fn isActiveValidator(validator: *const Validator.Type, epoch: Epoch) bool { return validator.activation_epoch <= epoch and epoch < validator.exit_epoch; @@ -109,6 +111,39 @@ pub fn getMaxEffectiveBalance(withdrawal_credentials: *const WithdrawalCredentia return preset.MIN_ACTIVATION_BALANCE; } +pub fn isFullyWithdrawableValidator(comptime fork: ForkSeq, withdrawal_credentials: *const WithdrawalCredentials, balance: u64, withdrawable_epoch: u64, epoch: u64) bool { + const has_withdrawable_credentials = if (comptime fork.gte(.electra)) + hasExecutionWithdrawalCredential(withdrawal_credentials) + else + hasEth1WithdrawalCredential(withdrawal_credentials); + + return has_withdrawable_credentials and withdrawable_epoch <= epoch and balance > 0; +} + +pub fn isPartiallyWithdrawableValidator(comptime fork: ForkSeq, withdrawal_credentials: *const WithdrawalCredentials, effective_balance: u64, balance: u64) bool { + // Check withdrawal credentials + const has_withdrawable_credentials = if (comptime fork.gte(.electra)) + hasExecutionWithdrawalCredential(withdrawal_credentials) + else + hasEth1WithdrawalCredential(withdrawal_credentials); + + if (!has_withdrawable_credentials) { + return false; + } + + // Get max effective balance based on fork + const max_effective_balance = if (comptime fork.gte(.electra)) + getMaxEffectiveBalance(withdrawal_credentials) + else + preset.MAX_EFFECTIVE_BALANCE; + + // Check if at max effective balance and has excess balance + const has_max_effective_balance = effective_balance == max_effective_balance; + const has_excess_balance = balance > max_effective_balance; + + return has_max_effective_balance and has_excess_balance; +} + pub fn getPendingBalanceToWithdraw(comptime fork: ForkSeq, state: *BeaconState(fork), validator_index: ValidatorIndex) !u64 { var total: u64 = 0; diff --git a/test/spec/runner/epoch_processing.zig b/test/spec/runner/epoch_processing.zig index 62d8c0d31..5a1bafffe 100644 --- a/test/spec/runner/epoch_processing.zig +++ b/test/spec/runner/epoch_processing.zig @@ -28,6 +28,8 @@ pub const EpochProcessingFn = enum { pending_deposits, pending_consolidations, proposer_lookahead, + builder_pending_payments, + ptc_window, pub fn suiteName(self: EpochProcessingFn) []const u8 { return @tagName(self) ++ "/pyspec_tests"; @@ -133,6 +135,12 @@ pub fn TestCase(comptime fork: ForkSeq, comptime epoch_process_fn: EpochProcessi .proposer_lookahead => { try state_transition.processProposerLookahead(fork, allocator, epoch_cache, fork_state, &epoch_transition_cache); }, + .builder_pending_payments => { + try state_transition.processBuilderPendingPayments(allocator, fork_state, epoch_cache); + }, + .ptc_window => { + try state_transition.processPtcWindow(allocator, epoch_cache, fork_state); + }, } } }; diff --git a/test/spec/runner/fork.zig b/test/spec/runner/fork.zig index 752bd7d53..f15a1b93d 100644 --- a/test/spec/runner/fork.zig +++ b/test/spec/runner/fork.zig @@ -8,6 +8,7 @@ const upgradeStateToCapella = state_transition.upgradeStateToCapella; const upgradeStateToDeneb = state_transition.upgradeStateToDeneb; const upgradeStateToElectra = state_transition.upgradeStateToElectra; const upgradeStateToFulu = state_transition.upgradeStateToFulu; +const upgradeStateToGloas = state_transition.upgradeStateToGloas; const TestCachedBeaconState = state_transition.test_utils.TestCachedBeaconState; const AnyBeaconState = @import("fork_types").AnyBeaconState; const test_case = @import("../test_case.zig"); @@ -28,7 +29,7 @@ const Allocator = std.mem.Allocator; pub fn TestCase(comptime target_fork: ForkSeq) type { comptime { switch (target_fork) { - .altair, .bellatrix, .capella, .deneb, .electra, .fulu => {}, + .altair, .bellatrix, .capella, .deneb, .electra, .fulu, .gloas => {}, else => @compileError("fork tests are not defined for " ++ @tagName(target_fork)), } } @@ -153,6 +154,15 @@ pub fn TestCase(comptime target_fork: ForkSeq) type { ); cached_state.state.* = .{ .fulu = upgraded.inner }; }, + .gloas => { + const upgraded = try upgradeStateToGloas( + self.pre.allocator, + config, + epoch_cache, + try cached_state.state.tryCastToFork(.fulu), + ); + cached_state.state.* = .{ .gloas = upgraded.inner }; + }, else => unreachable, } } @@ -183,6 +193,7 @@ fn previousFork(target: ForkSeq) ForkSeq { .deneb => .capella, .electra => .deneb, .fulu => .electra, + .gloas => .fulu, else => @compileError("Unsupported fork transition for " ++ @tagName(target)), }; } diff --git a/test/spec/write_spec_tests.zig b/test/spec/write_spec_tests.zig index b31a83e85..485dc2296 100644 --- a/test/spec/write_spec_tests.zig +++ b/test/spec/write_spec_tests.zig @@ -11,6 +11,7 @@ const supported_forks = [_]ForkSeq{ .deneb, .electra, .fulu, + .gloas, }; const supported_test_runners = [_]RunnerKind{ From 2b63b1c5d4702eb738305ef20ef45ea38d85978f Mon Sep 17 00:00:00 2001 From: Rahul Guha <19rahul2003@gmail.com> Date: Thu, 2 Jul 2026 15:35:30 +0530 Subject: [PATCH 15/33] fix compile errors --- .../block/process_voluntary_exit.zig | 14 +++++++------- src/state_transition/epoch/process_epoch.zig | 4 ++-- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/state_transition/block/process_voluntary_exit.zig b/src/state_transition/block/process_voluntary_exit.zig index 9e14c208a..3ff089802 100644 --- a/src/state_transition/block/process_voluntary_exit.zig +++ b/src/state_transition/block/process_voluntary_exit.zig @@ -1,4 +1,3 @@ -const std = @import("std"); const BeaconConfig = @import("config").BeaconConfig; const ForkSeq = @import("config").ForkSeq; const EpochCache = @import("../cache/epoch_cache.zig").EpochCache; @@ -59,14 +58,8 @@ pub fn getVoluntaryExitValidity( signed_voluntary_exit: *const SignedVoluntaryExit, verify_signature: bool, ) !VoluntaryExitValidity { - const current_epoch = epoch_cache.epoch; const voluntary_exit = signed_voluntary_exit.message; - // Exits must specify an epoch when they become valid; they are not valid before then - if (current_epoch < voluntary_exit.epoch) { - return .early_epoch; - } - var validators = try state.validators(); const validators_len = try validators.length(); if (voluntary_exit.validator_index >= validators_len) { @@ -74,6 +67,7 @@ pub fn getVoluntaryExitValidity( } var validator = try validators.get(@intCast(voluntary_exit.validator_index)); + const current_epoch = epoch_cache.epoch; // verify the validator is active if (!try isActiveValidatorView(validator, current_epoch)) { @@ -86,6 +80,11 @@ pub fn getVoluntaryExitValidity( return .already_exited; } + // exits must specify an epoch when they become valid; they are not valid before then + if (current_epoch < voluntary_exit.epoch) { + return .early_epoch; + } + // verify the validator had been active long enough const activation_epoch = try validator.get("activation_epoch"); if (current_epoch < activation_epoch + config.chain.SHARD_COMMITTEE_PERIOD) { @@ -109,6 +108,7 @@ pub fn getVoluntaryExitValidity( return .valid; } +const std = @import("std"); const TestCachedBeaconState = @import("../test_utils/root.zig").TestCachedBeaconState; const Node = @import("persistent_merkle_tree").Node; const preset = @import("preset").preset; diff --git a/src/state_transition/epoch/process_epoch.zig b/src/state_transition/epoch/process_epoch.zig index 22847496b..c8eeb3ea1 100644 --- a/src/state_transition/epoch/process_epoch.zig +++ b/src/state_transition/epoch/process_epoch.zig @@ -73,7 +73,7 @@ pub fn processEpoch( } if (comptime fork.gte(.gloas)) { - timer = time.timestampNow(io); + timer = time.start(io); try processBuilderPendingPayments(allocator, state, epoch_cache); try observeEpochTransitionStep(.{ .step = .process_builder_pending_payments }, @as(u64, @intCast(time.since(io, timer).nanoseconds))); } @@ -113,7 +113,7 @@ pub fn processEpoch( } if (comptime fork.gte(.gloas)) { - timer = time.timestampNow(io); + timer = time.start(io); try processPtcWindow(allocator, epoch_cache, state); try observeEpochTransitionStep(.{ .step = .process_ptc_window }, @as(u64, @intCast(time.since(io, timer).nanoseconds))); } From 09aea6d41663accc3ce3700cc75288137cff0363 Mon Sep 17 00:00:00 2001 From: Rahul Guha <19rahul2003@gmail.com> Date: Thu, 2 Jul 2026 15:48:47 +0530 Subject: [PATCH 16/33] fix getExpectedWithdrawals call sites --- bench/state_transition/process_block.zig | 2 ++ bindings/napi/BeaconStateView.zig | 1 + 2 files changed, 3 insertions(+) diff --git a/bench/state_transition/process_block.zig b/bench/state_transition/process_block.zig index aaa8b784e..6e9588495 100644 --- a/bench/state_transition/process_block.zig +++ b/bench/state_transition/process_block.zig @@ -67,6 +67,7 @@ fn ProcessWithdrawalsBench(comptime fork: ForkSeq) type { const state = BenchState.cloned_cached_state.state.castToFork(fork); state_transition.getExpectedWithdrawals( fork, + allocator, BenchState.cloned_cached_state.epoch_cache, state, &withdrawals_result, @@ -316,6 +317,7 @@ fn ProcessBlockSegmentedBench(comptime fork: ForkSeq) type { defer withdrawal_balances.deinit(); state_transition.getExpectedWithdrawals( fork, + allocator, epoch_cache, state, &withdrawals_result, diff --git a/bindings/napi/BeaconStateView.zig b/bindings/napi/BeaconStateView.zig index 6a412e83e..b6cb2c170 100644 --- a/bindings/napi/BeaconStateView.zig +++ b/bindings/napi/BeaconStateView.zig @@ -1512,6 +1512,7 @@ pub fn getExpectedWithdrawals(self: *const BeaconStateView) !js.Value { inline .capella, .deneb, .electra, .fulu => |f| { try st.getExpectedWithdrawals( f, + allocator, cached_state.epoch_cache, cached_state.state.castToFork(f), &withdrawals_result, From b7128030ff1d5a938f9383064fc69f3c72cc09f9 Mon Sep 17 00:00:00 2001 From: Rahul Guha <19rahul2003@gmail.com> Date: Sat, 4 Jul 2026 13:48:24 +0530 Subject: [PATCH 17/33] enforce Capella payload parent hash --- src/state_transition/block/process_execution_payload.zig | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/state_transition/block/process_execution_payload.zig b/src/state_transition/block/process_execution_payload.zig index 4900511e1..c65dd8a26 100644 --- a/src/state_transition/block/process_execution_payload.zig +++ b/src/state_transition/block/process_execution_payload.zig @@ -38,7 +38,12 @@ pub fn processExecutionPayload( // Verify consistency of the parent hash, block number, base fee per gas and gas limit // with respect to the previous execution payload header - if (isMergeTransitionComplete(fork, state)) { + if (comptime fork.gte(.capella)) { + const latest_block_hash = try state.latestExecutionPayloadHeaderBlockHash(); + if (!std.mem.eql(u8, parent_hash, latest_block_hash)) { + return error.InvalidExecutionPayloadParentHash; + } + } else if (isMergeTransitionComplete(fork, state)) { const latest_block_hash = try state.latestExecutionPayloadHeaderBlockHash(); if (!std.mem.eql(u8, parent_hash, latest_block_hash)) { return error.InvalidExecutionPayloadParentHash; From 2da237752271fa38c62cecd810db81e860c43bcd Mon Sep 17 00:00:00 2001 From: Rahul Guha <19rahul2003@gmail.com> Date: Sat, 4 Jul 2026 13:50:35 +0530 Subject: [PATCH 18/33] remove stale payload envelope processor --- .../process_execution_payload_envelope.zig | 197 ------------------ .../block/process_withdrawals.zig | 4 +- src/state_transition/root.zig | 1 - test/spec/runner/operations.zig | 57 ++--- 4 files changed, 19 insertions(+), 240 deletions(-) delete mode 100644 src/state_transition/block/process_execution_payload_envelope.zig diff --git a/src/state_transition/block/process_execution_payload_envelope.zig b/src/state_transition/block/process_execution_payload_envelope.zig deleted file mode 100644 index a21147a3c..000000000 --- a/src/state_transition/block/process_execution_payload_envelope.zig +++ /dev/null @@ -1,197 +0,0 @@ -const std = @import("std"); -const Allocator = std.mem.Allocator; -const types = @import("consensus_types"); -const BeaconConfig = @import("config").BeaconConfig; -const ForkSeq = @import("config").ForkSeq; -const BeaconState = @import("fork_types").BeaconState; -const EpochCache = @import("../cache/epoch_cache.zig").EpochCache; -const preset = @import("preset").preset; -const c = @import("constants"); -const processDepositRequest = @import("./process_deposit_request.zig").processDepositRequest; -const processWithdrawalRequest = @import("./process_withdrawal_request.zig").processWithdrawalRequest; -const processConsolidationRequest = @import("./process_consolidation_request.zig").processConsolidationRequest; -const processBuilderDepositRequest = @import("./process_builder_deposit_request.zig").processBuilderDepositRequest; -const processBuilderExitRequest = @import("./process_builder_exit_request.zig").processBuilderExitRequest; -const getExecutionPayloadEnvelopeSignatureSet = @import("../signature_sets/execution_payload_envelope.zig").getExecutionPayloadEnvelopeSignatureSet; -const verifySingleSignatureSet = @import("../utils/signature_sets.zig").verifySingleSignatureSet; - -pub const ProcessExecutionPayloadEnvelopeOpts = struct { - verify_signature: bool = true, - verify_state_root: bool = true, - verify_execution_requests_root: bool = true, -}; - -pub fn processExecutionPayloadEnvelope( - allocator: Allocator, - config: *const BeaconConfig, - epoch_cache: *const EpochCache, - state: *BeaconState(.gloas), - signed_envelope: *const types.gloas.SignedExecutionPayloadEnvelope.Type, - opts: ProcessExecutionPayloadEnvelopeOpts, -) !void { - const envelope = &signed_envelope.message; - const payload = &envelope.payload; - if (opts.verify_signature) { - if (!(try verifyExecutionPayloadEnvelopeSignature(allocator, config, epoch_cache, state, signed_envelope))) { - return error.InvalidEnvelopeSignature; - } - } - - const block_slot = try validateExecutionPayloadEnvelope(allocator, config, state, envelope, opts.verify_execution_requests_root); - - const requests = &envelope.execution_requests; - for (requests.deposits.items) |*deposit| { - try processDepositRequest(.gloas, state, deposit); - } - - for (requests.withdrawals.items) |*withdrawal| { - try processWithdrawalRequest(.gloas, config, @constCast(epoch_cache), state, withdrawal); - } - - for (requests.consolidations.items) |*consolidation| { - try processConsolidationRequest(.gloas, config, epoch_cache, state, consolidation); - } - - for (requests.builder_deposits.items) |*builder_deposit| { - try processBuilderDepositRequest(allocator, config, state, builder_deposit); - } - - for (requests.builder_exits.items) |*builder_exit| { - try processBuilderExitRequest(allocator, config, state, builder_exit); - } - - // Queue the builder payment - const payment_index = preset.SLOTS_PER_EPOCH + (block_slot % preset.SLOTS_PER_EPOCH); - var builder_pending_payments = try state.inner.get("builder_pending_payments"); - var payment: types.gloas.BuilderPendingPayment.Type = undefined; - try builder_pending_payments.getValue(allocator, payment_index, &payment); - const amount = payment.withdrawal.amount; - - if (amount > 0) { - var builder_pending_withdrawals = try state.inner.get("builder_pending_withdrawals"); - try builder_pending_withdrawals.pushValue(&payment.withdrawal); - try state.inner.set("builder_pending_withdrawals", builder_pending_withdrawals); - } - - const default_payment = types.gloas.BuilderPendingPayment.default_value; - try builder_pending_payments.setValue(payment_index, &default_payment); - - // Cache the execution payload hash - var execution_payload_availability = try state.inner.get("execution_payload_availability"); - try execution_payload_availability.set(block_slot % preset.SLOTS_PER_HISTORICAL_ROOT, true); - try state.inner.set("execution_payload_availability", execution_payload_availability); - try state.inner.setValue("latest_block_hash", &payload.block_hash); - - try state.commit(); - - _ = opts.verify_state_root; -} - -fn validateExecutionPayloadEnvelope( - allocator: Allocator, - config: *const BeaconConfig, - state: *BeaconState(.gloas), - envelope: *const types.gloas.ExecutionPayloadEnvelope.Type, - verify_execution_requests_root: bool, -) !u64 { - const payload = &envelope.payload; - - // Cache latest block header state root - var latest_block_header = try state.latestBlockHeader(); - const latest_header_state_root = try latest_block_header.getFieldRoot("state_root"); - if (std.mem.eql(u8, latest_header_state_root, &c.ZERO_HASH)) { - const previous_state_root = try state.hashTreeRoot(); - try latest_block_header.setValue("state_root", previous_state_root); - try state.inner.set("latest_block_header", latest_block_header); - } - - // Verify consistency with the beacon block - const latest_block_header_root = try latest_block_header.hashTreeRoot(); - if (!std.mem.eql(u8, &envelope.beacon_block_root, latest_block_header_root)) { - return error.EnvelopeBlockRootMismatch; - } - const latest_header_parent_root = try latest_block_header.getFieldRoot("parent_root"); - if (!std.mem.eql(u8, &envelope.parent_beacon_block_root, latest_header_parent_root)) { - return error.EnvelopeParentBlockRootMismatch; - } - - // Verify slot - const block_slot = try latest_block_header.get("slot"); - if (payload.slot_number != block_slot) { - return error.EnvelopeSlotMismatch; - } - - // Verify consistency with the committed bid - var committed_bid: types.gloas.ExecutionPayloadBid.Type = types.gloas.ExecutionPayloadBid.default_value; - try state.inner.getValue(allocator, "latest_execution_payload_bid", &committed_bid); - defer types.gloas.ExecutionPayloadBid.deinit(allocator, &committed_bid); - - if (envelope.builder_index != committed_bid.builder_index) { - return error.EnvelopeBuilderIndexMismatch; - } - - if (!std.mem.eql(u8, &committed_bid.prev_randao, &payload.prev_randao)) { - return error.EnvelopePrevRandaoMismatch; - } - - if (verify_execution_requests_root) { - var execution_requests_root: [32]u8 = undefined; - try types.gloas.ExecutionRequests.hashTreeRoot(allocator, &envelope.execution_requests, &execution_requests_root); - if (!std.mem.eql(u8, &committed_bid.execution_requests_root, &execution_requests_root)) { - return error.EnvelopeExecutionRequestsRootMismatch; - } - } - - // Verify consistency with expected withdrawals - var payload_withdrawals_root: [32]u8 = undefined; - try types.capella.Withdrawals.hashTreeRoot(allocator, &payload.withdrawals, &payload_withdrawals_root); - var expected_withdrawals = try state.inner.get("payload_expected_withdrawals"); - var expected_withdrawals_root: [32]u8 = undefined; - try expected_withdrawals.hashTreeRootInto(&expected_withdrawals_root); - if (!std.mem.eql(u8, &payload_withdrawals_root, &expected_withdrawals_root)) { - return error.EnvelopeWithdrawalsMismatch; - } - - // Verify the gas_limit - if (committed_bid.gas_limit != payload.gas_limit) { - return error.EnvelopeGasLimitMismatch; - } - - // Verify the block hash - if (!std.mem.eql(u8, &committed_bid.block_hash, &payload.block_hash)) { - return error.EnvelopeBlockHashMismatch; - } - - // Verify consistency of the parent hash with respect to the previous execution payload - const latest_block_hash = try state.inner.getFieldRoot("latest_block_hash"); - if (!std.mem.eql(u8, &payload.parent_hash, latest_block_hash)) { - return error.EnvelopeParentHashMismatch; - } - - // Verify timestamp - // compute_timestamp_at_slot: genesis_time + slot * SECONDS_PER_SLOT - const expected_timestamp = (try state.genesisTime()) + block_slot * config.chain.SECONDS_PER_SLOT; - if (payload.timestamp != expected_timestamp) { - return error.EnvelopeTimestampMismatch; - } - - // Skipped: Verify the execution payload is valid - return block_slot; -} - -fn verifyExecutionPayloadEnvelopeSignature( - allocator: Allocator, - config: *const BeaconConfig, - epoch_cache: *const EpochCache, - state: *BeaconState(.gloas), - signed_envelope: *const types.gloas.SignedExecutionPayloadEnvelope.Type, -) !bool { - const signature_set = try getExecutionPayloadEnvelopeSignatureSet( - allocator, - config, - epoch_cache, - state, - signed_envelope, - ); - return verifySingleSignatureSet(&signature_set); -} diff --git a/src/state_transition/block/process_withdrawals.zig b/src/state_transition/block/process_withdrawals.zig index 02deada83..3f2cb7127 100644 --- a/src/state_transition/block/process_withdrawals.zig +++ b/src/state_transition/block/process_withdrawals.zig @@ -53,7 +53,7 @@ pub fn processWithdrawals( const expected_withdrawals = expected_withdrawals_result.withdrawals.items; - // After EIP-7732, withdrawals are verified later in processExecutionPayloadEnvelope + // After EIP-7732, withdrawals are checked through parent execution payload processing. if (comptime fork.lt(.gloas)) { var expected_withdrawals_root: [32]u8 = undefined; try types.capella.Withdrawals.hashTreeRoot(allocator, &expected_withdrawals_result.withdrawals, &expected_withdrawals_root); @@ -71,7 +71,7 @@ pub fn processWithdrawals( } if (comptime fork.gte(.gloas)) { - // Store expected withdrawals for verification in processExecutionPayloadEnvelope + // Store expected withdrawals for parent execution payload verification. var payload_expected_withdrawals = try state.inner.get("payload_expected_withdrawals"); const current_len = try payload_expected_withdrawals.length(); var new_list = try payload_expected_withdrawals.sliceFrom(current_len); diff --git a/src/state_transition/root.zig b/src/state_transition/root.zig index 099ad917e..4ac6d685f 100644 --- a/src/state_transition/root.zig +++ b/src/state_transition/root.zig @@ -113,7 +113,6 @@ pub const calculateShufflingDecisionRoot = EpochShuffling.calculateShufflingDeci pub const processProposerLookahead = @import("./epoch/process_proposer_lookahead.zig").processProposerLookahead; pub const processBuilderPendingPayments = @import("./epoch/process_builder_pending_payments.zig").processBuilderPendingPayments; pub const processExecutionPayloadBid = @import("./block/process_execution_payload_bid.zig").processExecutionPayloadBid; -pub const processExecutionPayloadEnvelope = @import("./block/process_execution_payload_envelope.zig").processExecutionPayloadEnvelope; pub const processPayloadAttestation = @import("./block/process_payload_attestation.zig").processPayloadAttestation; pub const processPtcWindow = @import("./epoch/process_ptc_window.zig").processPtcWindow; pub const processParentExecutionPayload = @import("./block/process_parent_execution_payload.zig").processParentExecutionPayload; diff --git a/test/spec/runner/operations.zig b/test/spec/runner/operations.zig index a29cb63d6..55aa53788 100644 --- a/test/spec/runner/operations.zig +++ b/test/spec/runner/operations.zig @@ -92,12 +92,7 @@ fn loadExecutionValid(allocator: std.mem.Allocator, dir: std.Io.Dir) bool { pub fn TestCase(comptime fork: ForkSeq, comptime operation: Operation) type { const ForkTypes = @field(ssz, fork.name()); const tc_utils = TestCaseUtils(fork); - // After EIP-7732, gloas execution_payload tests use SignedExecutionPayloadEnvelope - const is_gloas_exec_payload = comptime (operation == .execution_payload and fork.gte(.gloas)); - const OpType = if (is_gloas_exec_payload) - ForkTypes.SignedExecutionPayloadEnvelope - else - @field(ForkTypes, operation.operationObject()); + const OpType = @field(ForkTypes, operation.operationObject()); return struct { pre: TestCachedBeaconState, @@ -141,10 +136,7 @@ pub fn TestCase(comptime fork: ForkSeq, comptime operation: Operation) type { // load the op // After EIP-7732, gloas withdrawals tests don't have an execution_payload input file - const input_name = comptime if (is_gloas_exec_payload) - "signed_envelope" - else - operation.inputName(); + const input_name = comptime operation.inputName(); if (comptime !(operation == .withdrawals and fork.gte(.gloas))) { try loadSszValue(OpType, allocator, dir, input_name ++ ".ssz_snappy", &tc.op); } @@ -246,36 +238,21 @@ pub fn TestCase(comptime fork: ForkSeq, comptime operation: Operation) type { .execution_payload => { const config = cached_state.config; const epoch_cache = cached_state.epoch_cache; - if (comptime is_gloas_exec_payload) { - // After EIP-7732, execution_payload tests use processExecutionPayloadEnvelope - if (!self.execution_valid) { - return error.ExecutionPayloadInvalid; - } - try state_transition.processExecutionPayloadEnvelope( - allocator, - config, - epoch_cache, - state, - &self.op, - .{ .verify_signature = true, .verify_state_root = true }, - ); - } else { - const current_epoch = epoch_cache.epoch; - const fork_body = BeaconBlockBody(.full, fork){ .inner = self.op }; - try state_transition.processExecutionPayload( - fork, - allocator, - config, - state, - current_epoch, - .full, - &fork_body, - .{ - .data_availability_status = .available, - .execution_payload_status = if (self.post != null) .valid else .invalid, - }, - ); - } + const current_epoch = epoch_cache.epoch; + const fork_body = BeaconBlockBody(.full, fork){ .inner = self.op }; + try state_transition.processExecutionPayload( + fork, + allocator, + config, + state, + current_epoch, + .full, + &fork_body, + .{ + .data_availability_status = .available, + .execution_payload_status = if (self.execution_valid) .valid else .invalid, + }, + ); }, .execution_payload_bid => { const config = cached_state.config; From af826e9fb21def3b88a6c0bf4b73766bb52ffe99 Mon Sep 17 00:00:00 2001 From: Rahul Guha <19rahul2003@gmail.com> Date: Sat, 4 Jul 2026 15:43:43 +0530 Subject: [PATCH 19/33] mirror PTC window in epoch cache --- src/state_transition/cache/epoch_cache.zig | 123 +++++++++++------- .../cache/epoch_transition_cache.zig | 52 +++++++- src/state_transition/epoch/process_epoch.zig | 2 +- .../epoch/process_proposer_lookahead.zig | 8 +- .../epoch/process_ptc_window.zig | 18 ++- src/state_transition/state_transition.zig | 2 +- test/spec/runner/epoch_processing.zig | 2 +- 7 files changed, 149 insertions(+), 58 deletions(-) diff --git a/src/state_transition/cache/epoch_cache.zig b/src/state_transition/cache/epoch_cache.zig index 673a1fb82..a1774ae53 100644 --- a/src/state_transition/cache/epoch_cache.zig +++ b/src/state_transition/cache/epoch_cache.zig @@ -30,7 +30,6 @@ const getTotalSlashingsByIncrement = @import("../epoch/process_slashings.zig").g const computeEpochShuffling = @import("../utils/epoch_shuffling.zig").computeEpochShuffling; const getSeed = @import("../utils/seed.zig").getSeed; const computeProposers = @import("../utils/seed.zig").computeProposers; -const computePayloadTimelinessCommitteesForEpoch = @import("../utils/seed.zig").computePayloadTimelinessCommitteesForEpoch; const SyncCommitteeCacheRc = @import("./sync_committee_cache.zig").SyncCommitteeCacheRc; const SyncCommitteeCacheAllForks = @import("./sync_committee_cache.zig").SyncCommitteeCache; const computeSyncParticipantReward = @import("../utils/sync_committee.zig").computeSyncParticipantReward; @@ -49,6 +48,30 @@ const ForkSeq = @import("config").ForkSeq; const ForkTypes = @import("fork_types").ForkTypes; const syncPubkeys = @import("./pubkey_cache.zig").syncPubkeys; +const PtcCommittees = [preset.SLOTS_PER_EPOCH][preset.PTC_SIZE]ValidatorIndex; + +const PtcWindowEpochCacheData = struct { + previous: PtcCommittees, + current: PtcCommittees, + next: PtcCommittees, +}; + +fn getPtcWindowEpochCacheData(state: *AnyBeaconState) !PtcWindowEpochCacheData { + return getPtcWindowEpochCacheDataFromState(try state.tryCastToFork(.gloas)); +} + +fn getPtcWindowEpochCacheDataFromState(state: *BeaconState(.gloas)) !PtcWindowEpochCacheData { + var ptc_window = try state.inner.get("ptc_window"); + var data: PtcWindowEpochCacheData = undefined; + + for (0..preset.SLOTS_PER_EPOCH) |i| { + try ptc_window.getValue(undefined, i, &data.previous[i]); + try ptc_window.getValue(undefined, preset.SLOTS_PER_EPOCH + i, &data.current[i]); + try ptc_window.getValue(undefined, 2 * preset.SLOTS_PER_EPOCH + i, &data.next[i]); + } + + return data; +} pub const EpochCacheImmutableData = struct { config: *const BeaconConfig, @@ -143,6 +166,9 @@ pub const EpochCache = struct { /// PTC for previous epoch, required for slot N block validating slot N-1 attestations. previous_payload_timeliness_committees: ?[preset.SLOTS_PER_EPOCH][preset.PTC_SIZE]ValidatorIndex, + /// PTC for next epoch, sourced from state.ptc_window or the epoch transition stash. + next_payload_timeliness_committees: ?[preset.SLOTS_PER_EPOCH][preset.PTC_SIZE]ValidatorIndex, + fn initEffectiveBalanceIncrementsRc(allocator: Allocator, validator_count: usize) !*EffectiveBalanceIncrementsRc { var effective_balance_increments = try effectiveBalanceIncrementsInit(allocator, validator_count); errdefer effective_balance_increments.deinit(allocator); @@ -207,8 +233,7 @@ pub const EpochCache = struct { const index_to_pubkey = immutable_data.index_to_pubkey; const current_epoch = computeEpochAtSlot(try state.slot()); - const is_genesis = current_epoch == GENESIS_EPOCH; - const previous_epoch = if (is_genesis) GENESIS_EPOCH else current_epoch - 1; + const previous_epoch = if (current_epoch == GENESIS_EPOCH) GENESIS_EPOCH else current_epoch - 1; const next_epoch = current_epoch + 1; var total_active_balance_increments: u64 = 0; @@ -477,19 +502,15 @@ pub const EpochCache = struct { .epoch = current_epoch, .payload_timeliness_committees = null, .previous_payload_timeliness_committees = null, + .next_payload_timeliness_committees = null, }; - // Compute PTC for all slots in the prev/current epoch - if (current_epoch >= config.chain.GLOAS_FORK_EPOCH) { - epoch_cache_ptr.payload_timeliness_committees = switch (state.forkSeq()) { - inline else => |f| try computePayloadTimelinessCommitteesForEpoch(f, allocator, state.castToFork(f), current_epoch, epoch_cache_ptr), - }; - - if (!is_genesis and previous_epoch >= config.chain.GLOAS_FORK_EPOCH) { - epoch_cache_ptr.previous_payload_timeliness_committees = switch (state.forkSeq()) { - inline else => |f| try computePayloadTimelinessCommitteesForEpoch(f, allocator, state.castToFork(f), previous_epoch, epoch_cache_ptr), - }; - } + // Copy PTC slices from state.ptc_window ad it comes from cached committees. + if (current_epoch >= config.chain.GLOAS_FORK_EPOCH and state.forkSeq().gte(.gloas)) { + const ptc_data = try getPtcWindowEpochCacheData(state); + epoch_cache_ptr.previous_payload_timeliness_committees = ptc_data.previous; + epoch_cache_ptr.payload_timeliness_committees = ptc_data.current; + epoch_cache_ptr.next_payload_timeliness_committees = ptc_data.next; } return epoch_cache_ptr; @@ -551,6 +572,7 @@ pub const EpochCache = struct { .epoch = self.epoch, .payload_timeliness_committees = self.payload_timeliness_committees, .previous_payload_timeliness_committees = self.previous_payload_timeliness_committees, + .next_payload_timeliness_committees = self.next_payload_timeliness_committees, }; const epoch_cache_ptr = try allocator.create(EpochCache); @@ -584,23 +606,20 @@ pub const EpochCache = struct { return self.effective_balance_increments.get(); } - pub fn afterProcessEpoch(self: *EpochCache, state: *AnyBeaconState, epoch_transition_cache: *const EpochTransitionCache) !void { + fn takeNextShufflingRc(self: *EpochCache, state: *AnyBeaconState, epoch_transition_cache: *EpochTransitionCache) !*EpochShufflingRc { + const next_shuffling = try epoch_transition_cache.takeNextShuffling(self.allocator, state); + errdefer next_shuffling.deinit(); + return try EpochShufflingRc.init(self.allocator, next_shuffling); + } + + pub fn afterProcessEpoch(self: *EpochCache, state: *AnyBeaconState, epoch_transition_cache: *EpochTransitionCache) !void { const upcoming_epoch = self.epoch + 1; const epoch_after_upcoming = upcoming_epoch + 1; const slot = try state.slot(); - const next_shuffling_active_indices = try self.allocator.alloc(ValidatorIndex, epoch_transition_cache.next_shuffling_active_indices.len); - std.mem.copyForwards(ValidatorIndex, next_shuffling_active_indices, epoch_transition_cache.next_shuffling_active_indices); - - const next_shuffling = try computeEpochShuffling( - self.allocator, - state, - next_shuffling_active_indices, - epoch_after_upcoming, - ); - errdefer next_shuffling.deinit(); - - const next_shuffling_rc = try EpochShufflingRc.init(self.allocator, next_shuffling); + const next_shuffling_rc = try self.takeNextShufflingRc(state, epoch_transition_cache); + errdefer next_shuffling_rc.unref(); + const next_decision_root = try calculateShufflingDecisionRoot(state, epoch_after_upcoming); self.previous_shuffling.unref(); self.previous_shuffling = self.current_shuffling; @@ -610,7 +629,7 @@ pub const EpochCache = struct { self.current_decision_root = self.next_decision_root; self.next_shuffling = next_shuffling_rc; - self.next_decision_root = try calculateShufflingDecisionRoot(state, epoch_after_upcoming); + self.next_decision_root = next_decision_root; self.churn_limit = getChurnLimit(self.config, self.current_shuffling.get().active_indices.len); self.activation_churn_limit = getActivationChurnLimit(self.config, self.config.forkSeq(slot), self.current_shuffling.get().active_indices.len); @@ -636,21 +655,31 @@ pub const EpochCache = struct { } /// At fork boundary, this runs post-fork logic and after `upgradeState*`. - pub fn finalProcessEpoch(self: *EpochCache, state: *AnyBeaconState) !void { + pub fn finalProcessEpoch(self: *EpochCache, state: *AnyBeaconState, epoch_transition_cache: ?*const EpochTransitionCache) !void { self.proposers_prev_epoch = self.proposers; - // Shift and compute current epoch PTC eagerly for all slots - if (self.epoch >= self.config.chain.GLOAS_FORK_EPOCH) { - self.previous_payload_timeliness_committees = self.payload_timeliness_committees; - self.payload_timeliness_committees = switch (state.forkSeq()) { - inline else => |f| try computePayloadTimelinessCommitteesForEpoch( - f, - self.allocator, - state.castToFork(f), - self.epoch, - self, - ), - }; + // Shift cached PTC slices when processPtcWindow stashed the next epoch committees. + // At the fork boundary or in direct tests, rebuild from state.ptc_window instead. + if (self.epoch >= self.config.chain.GLOAS_FORK_EPOCH and state.forkSeq().gte(.gloas)) { + var read_from_state = self.epoch == self.config.chain.GLOAS_FORK_EPOCH or + self.payload_timeliness_committees == null or + self.next_payload_timeliness_committees == null or + epoch_transition_cache == null; + + if (!read_from_state and epoch_transition_cache.?.next_epoch_payload_timeliness_committees == null) { + read_from_state = true; + } + + if (read_from_state) { + const ptc_data = try getPtcWindowEpochCacheData(state); + self.previous_payload_timeliness_committees = ptc_data.previous; + self.payload_timeliness_committees = ptc_data.current; + self.next_payload_timeliness_committees = ptc_data.next; + } else { + self.previous_payload_timeliness_committees = self.payload_timeliness_committees; + self.payload_timeliness_committees = self.next_payload_timeliness_committees; + self.next_payload_timeliness_committees = epoch_transition_cache.?.next_epoch_payload_timeliness_committees; + } } switch (state.forkSeq()) { @@ -996,18 +1025,16 @@ pub const EpochCache = struct { /// aggregation bits against the Payload Timeliness Committee for the attestation's slot. /// Spec: get_ptc — Read PTC from state.ptc_window pub fn getPayloadTimelinessCommittee(self: *const EpochCache, state: *BeaconState(.gloas), slot: Slot) ![preset.PTC_SIZE]ValidatorIndex { - _ = self; const epoch = computeEpochAtSlot(slot); const state_epoch = computeEpochAtSlot(try state.slot()); + if (epoch < self.config.chain.GLOAS_FORK_EPOCH) return error.PayloadTimelinessCommitteeBeforeGloasFork; + if (epoch < state_epoch -| 1 or epoch > state_epoch + 1) return error.PayloadTimelinessCommitteeEpochOutOfRange; + var ptc_window = try state.inner.get("ptc_window"); - const index = if (epoch < state_epoch) blk: { - break :blk slot % preset.SLOTS_PER_EPOCH; - } else blk: { - const offset = (epoch - state_epoch + 1) * preset.SLOTS_PER_EPOCH; - break :blk offset + slot % preset.SLOTS_PER_EPOCH; - }; + const epoch_window_index: u64 = if (epoch < state_epoch) 0 else epoch - state_epoch + 1; + const index = epoch_window_index * preset.SLOTS_PER_EPOCH + slot % preset.SLOTS_PER_EPOCH; var ptc_entry: [preset.PTC_SIZE]ValidatorIndex = undefined; try ptc_window.getValue(undefined, index, &ptc_entry); diff --git a/src/state_transition/cache/epoch_transition_cache.zig b/src/state_transition/cache/epoch_transition_cache.zig index 77b0809f6..ae54fd6b7 100644 --- a/src/state_transition/cache/epoch_transition_cache.zig +++ b/src/state_transition/cache/epoch_transition_cache.zig @@ -10,6 +10,8 @@ const BeaconConfig = @import("config").BeaconConfig; const EpochCache = @import("../cache/epoch_cache.zig").EpochCache; const AnyBeaconState = @import("fork_types").AnyBeaconState; const BeaconState = @import("fork_types").BeaconState; +const EpochShuffling = @import("../utils/epoch_shuffling.zig").EpochShuffling; +const computeEpochShuffling = @import("../utils/epoch_shuffling.zig").computeEpochShuffling; const TestCachedBeaconState = @import("../test_utils/root.zig").TestCachedBeaconState; const upgradeStateToFulu = @import("../slot/upgrade_state_to_fulu.zig").upgradeStateToFulu; @@ -39,6 +41,7 @@ const BoolArray = std.ArrayList(bool); const UsizeArray = std.ArrayList(usize); const U8Array = std.ArrayList(u8); const U64Array = std.ArrayList(u64); +const PtcCommittees = [preset.SLOTS_PER_EPOCH][preset.PTC_SIZE]ValidatorIndex; const ValidatorActivation = struct { validator_index: ValidatorIndex, @@ -204,8 +207,10 @@ pub const EpochTransitionCache = struct { slashing_penalties: []u64, balances: ?U64Array, next_shuffling_active_indices: []const ValidatorIndex, + next_shuffling: ?*EpochShuffling, // TODO: nextShufflingDecisionRoot may not needed as we don't use ShufflingCache next_epoch_total_active_balance_by_increment: u64, + next_epoch_payload_timeliness_committees: ?PtcCommittees, // TODO: asyncShufflingCalculation may not needed as we don't use ShufflingCache // these are borrowed from ReusedEpochTransitionCache is_active_prev_epoch: []const bool, @@ -533,8 +538,10 @@ pub const EpochTransitionCache = struct { .indices_eligible_for_activation = indices_eligible_for_activation, .indices_to_eject = indices_to_eject, .next_shuffling_active_indices = next_shuffling_active_indices, + .next_shuffling = null, // to be updated in processEffectiveBalanceUpdates .next_epoch_total_active_balance_by_increment = 0, + .next_epoch_payload_timeliness_committees = null, .is_active_prev_epoch = reused_cache.is_active_prev_epoch.items, .is_active_curr_epoch = reused_cache.is_active_current_epoch.items, .is_active_next_epoch = reused_cache.is_active_next_epoch.items, @@ -566,6 +573,49 @@ pub const EpochTransitionCache = struct { if (self.balances) |*balances| { balances.deinit(allocator); } + if (self.next_shuffling) |next_shuffling| { + next_shuffling.deinit(); + } + } + + pub fn getNextShuffling( + self: *EpochTransitionCache, + allocator: Allocator, + comptime fork: ForkSeq, + state: *BeaconState(fork), + ) !*EpochShuffling { + var any_state = @unionInit(AnyBeaconState, @tagName(fork), state.inner); + return self.getNextShufflingFromAnyState(allocator, &any_state); + } + + pub fn getNextShufflingFromAnyState( + self: *EpochTransitionCache, + allocator: Allocator, + state: *AnyBeaconState, + ) !*EpochShuffling { + if (self.next_shuffling) |next_shuffling| return next_shuffling; + + const active_indices = try allocator.alloc(ValidatorIndex, self.next_shuffling_active_indices.len); + errdefer allocator.free(active_indices); + std.mem.copyForwards(ValidatorIndex, active_indices, self.next_shuffling_active_indices); + + self.next_shuffling = try computeEpochShuffling( + allocator, + state, + active_indices, + self.current_epoch + preset.MIN_SEED_LOOKAHEAD + 1, + ); + return self.next_shuffling.?; + } + + pub fn takeNextShuffling( + self: *EpochTransitionCache, + allocator: Allocator, + state: *AnyBeaconState, + ) !*EpochShuffling { + const next_shuffling = try self.getNextShufflingFromAnyState(allocator, state); + self.next_shuffling = null; + return next_shuffling; } /// Ensure rewards/penalties arrays match the current validator count. @@ -600,7 +650,7 @@ test "EpochTransitionCache - finalProcessEpoch" { test_state.cached_state.state.* = .{ .fulu = fulu_state.inner }; const epoch_cache = test_state.cached_state.epoch_cache; - try epoch_cache.finalProcessEpoch(test_state.cached_state.state); + try epoch_cache.finalProcessEpoch(test_state.cached_state.state, null); } test "EpochTransitionCache.beforeProcessEpoch" { diff --git a/src/state_transition/epoch/process_epoch.zig b/src/state_transition/epoch/process_epoch.zig index c8eeb3ea1..2f5428706 100644 --- a/src/state_transition/epoch/process_epoch.zig +++ b/src/state_transition/epoch/process_epoch.zig @@ -114,7 +114,7 @@ pub fn processEpoch( if (comptime fork.gte(.gloas)) { timer = time.start(io); - try processPtcWindow(allocator, epoch_cache, state); + try processPtcWindow(allocator, epoch_cache, state, cache); try observeEpochTransitionStep(.{ .step = .process_ptc_window }, @as(u64, @intCast(time.since(io, timer).nanoseconds))); } } diff --git a/src/state_transition/epoch/process_proposer_lookahead.zig b/src/state_transition/epoch/process_proposer_lookahead.zig index d8c229830..510f8d173 100644 --- a/src/state_transition/epoch/process_proposer_lookahead.zig +++ b/src/state_transition/epoch/process_proposer_lookahead.zig @@ -24,7 +24,7 @@ pub fn processProposerLookahead( allocator: Allocator, epoch_cache: *EpochCache, state: *BeaconState(fork), - epoch_transition_cache: *const EpochTransitionCache, + epoch_transition_cache: *EpochTransitionCache, ) !void { const proposer_lookahead: *[ssz.fulu.ProposerLookahead.length]u64 = try state.proposerLookaheadSlice(allocator); defer allocator.free(@as([]u64, proposer_lookahead)); @@ -44,9 +44,9 @@ pub fn processProposerLookahead( const current_epoch = computeEpochAtSlot(try state.slot()); const new_epoch = current_epoch + preset.MIN_SEED_LOOKAHEAD + 1; - // Active indices for the new epoch come from the epoch transition cache - // (computed during beforeProcessEpoch for current_epoch + 2) - const shuffling_active_indices = epoch_transition_cache.next_shuffling_active_indices; + // Share the N+2 shuffling with processPtcWindow and afterProcessEpoch. + const next_shuffling = try epoch_transition_cache.getNextShuffling(allocator, fork, state); + const shuffling_active_indices = next_shuffling.active_indices; var unslashed_active_indices: std.ArrayList(ValidatorIndex) = .empty; defer unslashed_active_indices.deinit(allocator); diff --git a/src/state_transition/epoch/process_ptc_window.zig b/src/state_transition/epoch/process_ptc_window.zig index 414004ef5..11b5a01bd 100644 --- a/src/state_transition/epoch/process_ptc_window.zig +++ b/src/state_transition/epoch/process_ptc_window.zig @@ -2,6 +2,7 @@ const std = @import("std"); const Allocator = std.mem.Allocator; const BeaconState = @import("fork_types").BeaconState; const EpochCache = @import("../cache/epoch_cache.zig").EpochCache; +const EpochTransitionCache = @import("../cache/epoch_transition_cache.zig").EpochTransitionCache; const ct = @import("consensus_types"); const preset = @import("preset").preset; const computeEpochAtSlot = @import("../utils/epoch.zig").computeEpochAtSlot; @@ -13,6 +14,7 @@ pub fn processPtcWindow( allocator: Allocator, epoch_cache: *const EpochCache, state: *BeaconState(.gloas), + epoch_transition_cache: *EpochTransitionCache, ) !void { var ptc_window = try state.inner.get("ptc_window"); const ptc_window_len = ct.gloas.PtcWindow.length; @@ -25,11 +27,23 @@ pub fn processPtcWindow( const next_epoch = computeEpochAtSlot(try state.slot()) + preset.MIN_SEED_LOOKAHEAD + 1; const start_slot = computeStartSlotAtEpoch(next_epoch); + const next_shuffling = try epoch_transition_cache.getNextShuffling(allocator, .gloas, state); + var next_epoch_payload_timeliness_committees: [preset.SLOTS_PER_EPOCH][preset.PTC_SIZE]ValidatorIndex = undefined; for (0..preset.SLOTS_PER_EPOCH) |slot_offset| { - const ptc = try computePtc(allocator, state, start_slot + slot_offset, null, epoch_cache.effective_balance_increments.get().items); - try ptc_window.setValue(ptc_window_len - preset.SLOTS_PER_EPOCH + slot_offset, &ptc); + next_epoch_payload_timeliness_committees[slot_offset] = try computePtc( + allocator, + state, + start_slot + slot_offset, + next_shuffling, + epoch_cache.effective_balance_increments.get().items, + ); + try ptc_window.setValue( + ptc_window_len - preset.SLOTS_PER_EPOCH + slot_offset, + &next_epoch_payload_timeliness_committees[slot_offset], + ); } try state.inner.set("ptc_window", ptc_window); + epoch_transition_cache.next_epoch_payload_timeliness_committees = next_epoch_payload_timeliness_committees; } diff --git a/src/state_transition/state_transition.zig b/src/state_transition/state_transition.zig index 42d4b3109..a235df4e1 100644 --- a/src/state_transition/state_transition.zig +++ b/src/state_transition/state_transition.zig @@ -138,7 +138,7 @@ pub fn processSlots( state.* = .{ .gloas = upgraded.inner }; } - try epoch_cache.finalProcessEpoch(state); + try epoch_cache.finalProcessEpoch(state, &epoch_transition_cache); metrics.state_transition.epoch_transition.observe(time.durationSeconds(time.since(io, epoch_transition_timer))); } else { try state.setSlot(next_slot); diff --git a/test/spec/runner/epoch_processing.zig b/test/spec/runner/epoch_processing.zig index 5a1bafffe..3b7b63430 100644 --- a/test/spec/runner/epoch_processing.zig +++ b/test/spec/runner/epoch_processing.zig @@ -139,7 +139,7 @@ pub fn TestCase(comptime fork: ForkSeq, comptime epoch_process_fn: EpochProcessi try state_transition.processBuilderPendingPayments(allocator, fork_state, epoch_cache); }, .ptc_window => { - try state_transition.processPtcWindow(allocator, epoch_cache, fork_state); + try state_transition.processPtcWindow(allocator, epoch_cache, fork_state, &epoch_transition_cache); }, } } From 80842922b976f4961fd9153cf0338b2c1bf6f68b Mon Sep 17 00:00:00 2001 From: Rahul Guha <19rahul2003@gmail.com> Date: Sat, 4 Jul 2026 16:02:59 +0530 Subject: [PATCH 20/33] remove unused helper functions --- src/state_transition/utils/gloas.zig | 43 ---------------------------- 1 file changed, 43 deletions(-) diff --git a/src/state_transition/utils/gloas.zig b/src/state_transition/utils/gloas.zig index e35b21dd0..9ed38be4a 100644 --- a/src/state_transition/utils/gloas.zig +++ b/src/state_transition/utils/gloas.zig @@ -19,7 +19,6 @@ const isActiveValidator = @import("./validator.zig").isActiveValidator; const computeStartSlotAtEpoch = @import("./epoch.zig").computeStartSlotAtEpoch; const RootCache = @import("../cache/root_cache.zig").RootCache; const EpochCache = @import("../cache/epoch_cache.zig").EpochCache; -const PendingDepositsLookup = @import("./pending_deposits_lookup.zig").PendingDepositsLookup; const computeDomain = @import("./domain.zig").computeDomain; const computeSigningRoot = @import("./signing_root.zig").computeSigningRoot; const bls = @import("bls"); @@ -227,29 +226,6 @@ pub fn isAttestationSameSlotRootCache(root_cache: *RootCache(.gloas), data: *con return is_matching and is_current; } -// TODO: Need to check if these are used at all -/// computePayloadTimelinessCommitteeIndices uses the optimized increments-based check instead. -pub fn isBalanceWeightedAcceptance(effective_balance: u64, random_value: u64) bool { - const MAX_RANDOM_VALUE: u64 = 0xFFFF; - return effective_balance * MAX_RANDOM_VALUE >= preset.MAX_EFFECTIVE_BALANCE_ELECTRA * random_value; -} - -// TODO: Need to check if these are used at all -/// computePayloadTimelinessCommitteeIndices inlines this logic with increments instead of full Gwei balances. -pub fn computeBalanceWeightedAcceptance(effective_balance: u64, seed: *const [32]u8, i: u64) bool { - var hash_input: [40]u8 = undefined; - @memcpy(hash_input[0..32], seed); - std.mem.writeInt(u64, hash_input[32..][0..8], i / 16, .little); - - var random_bytes: [32]u8 = undefined; - Sha256.hash(&hash_input, &random_bytes, .{}); - - const offset: usize = @intCast((i % 16) * 2); - const random_value: u64 = std.mem.readInt(u16, random_bytes[offset..][0..2], .little); - - return isBalanceWeightedAcceptance(effective_balance, random_value); -} - pub fn isParentBlockFull(state: *BeaconState(.gloas)) !bool { var bid = try state.inner.get("latest_execution_payload_bid"); const bid_block_hash = try bid.getFieldRoot("block_hash"); @@ -257,25 +233,6 @@ pub fn isParentBlockFull(state: *BeaconState(.gloas)) !bool { return std.mem.eql(u8, bid_block_hash, latest_block_hash); } -pub fn isPubkeyInList(list: []const BLSPubkey, pubkey: *const BLSPubkey) bool { - for (list) |p| { - if (std.mem.eql(u8, &p, pubkey)) return true; - } - return false; -} - -pub fn isPendingValidator( - comptime fork: ForkSeq, - allocator: Allocator, - config: *const BeaconConfig, - state: *BeaconState(fork), - pubkey: *const BLSPubkey, -) !bool { - var lookup = try PendingDepositsLookup.build(fork, allocator, state); - defer lookup.deinit(); - return try lookup.hasPendingValidator(config, pubkey); -} - pub fn computePtc( allocator: Allocator, state: *BeaconState(.gloas), From 7feecea2b19f66d4f842c96f04a69906b4904f45 Mon Sep 17 00:00:00 2001 From: Rahul Guha <19rahul2003@gmail.com> Date: Tue, 7 Jul 2026 14:19:24 +0530 Subject: [PATCH 21/33] add doc comments as upstream --- src/constants/root.zig | 20 ++++++++++- .../block/process_builder_deposit_request.zig | 4 +++ .../block/process_builder_exit_request.zig | 7 ++++ .../process_parent_execution_payload.zig | 14 ++++++++ .../epoch/process_ptc_window.zig | 6 ++++ .../slot/upgrade_state_to_gloas.zig | 3 ++ src/state_transition/utils/gloas.zig | 36 +++++++++++++++++++ 7 files changed, 89 insertions(+), 1 deletion(-) diff --git a/src/constants/root.zig b/src/constants/root.zig index 81d0616c1..8fd708314 100644 --- a/src/constants/root.zig +++ b/src/constants/root.zig @@ -40,7 +40,25 @@ pub const DOMAIN_APPLICATION_MASK = [_]u8{ 0, 0, 0, 1 }; pub const DOMAIN_APPLICATION_BUILDER = [_]u8{ 0, 0, 0, 1 }; // need to be updated when new domain is added -pub const ALL_DOMAINS = [_][4]u8{ DOMAIN_BEACON_PROPOSER, DOMAIN_BEACON_ATTESTER, DOMAIN_RANDAO, DOMAIN_DEPOSIT, DOMAIN_VOLUNTARY_EXIT, DOMAIN_SELECTION_PROOF, DOMAIN_AGGREGATE_AND_PROOF, DOMAIN_SYNC_COMMITTEE, DOMAIN_SYNC_COMMITTEE_SELECTION_PROOF, DOMAIN_CONTRIBUTION_AND_PROOF, DOMAIN_BLS_TO_EXECUTION_CHANGE, DOMAIN_BEACON_BUILDER, DOMAIN_PTC_ATTESTER, DOMAIN_PROPOSER_PREFERENCES, DOMAIN_BUILDER_DEPOSIT, DOMAIN_APPLICATION_MASK, DOMAIN_APPLICATION_BUILDER }; +pub const ALL_DOMAINS = [_][4]u8{ + DOMAIN_BEACON_PROPOSER, + DOMAIN_BEACON_ATTESTER, + DOMAIN_RANDAO, + DOMAIN_DEPOSIT, + DOMAIN_VOLUNTARY_EXIT, + DOMAIN_SELECTION_PROOF, + DOMAIN_AGGREGATE_AND_PROOF, + DOMAIN_SYNC_COMMITTEE, + DOMAIN_SYNC_COMMITTEE_SELECTION_PROOF, + DOMAIN_CONTRIBUTION_AND_PROOF, + DOMAIN_BLS_TO_EXECUTION_CHANGE, + DOMAIN_BEACON_BUILDER, + DOMAIN_PTC_ATTESTER, + DOMAIN_PROPOSER_PREFERENCES, + DOMAIN_BUILDER_DEPOSIT, + DOMAIN_APPLICATION_MASK, + DOMAIN_APPLICATION_BUILDER, +}; // Gloas (EIP-7732) misc diff --git a/src/state_transition/block/process_builder_deposit_request.zig b/src/state_transition/block/process_builder_deposit_request.zig index 22bc58f4d..340b253ce 100644 --- a/src/state_transition/block/process_builder_deposit_request.zig +++ b/src/state_transition/block/process_builder_deposit_request.zig @@ -10,6 +10,10 @@ const addBuilderToRegistry = gloas_utils.addBuilderToRegistry; const findBuilderIndexByPubkey = gloas_utils.findBuilderIndexByPubkey; const isValidBuilderDepositSignature = gloas_utils.isValidBuilderDepositSignature; +/// Process a builder deposit request from the execution layer: register a new builder +/// (proof-of-possession gated) or top up an existing builder's balance. +/// +/// Spec: https://github.com/ethereum/consensus-specs/blob/v1.7.0-alpha.11/specs/gloas/beacon-chain.md#new-process_builder_deposit_request pub fn processBuilderDepositRequest( allocator: Allocator, config: *const BeaconConfig, diff --git a/src/state_transition/block/process_builder_exit_request.zig b/src/state_transition/block/process_builder_exit_request.zig index f229dea19..164226fb7 100644 --- a/src/state_transition/block/process_builder_exit_request.zig +++ b/src/state_transition/block/process_builder_exit_request.zig @@ -9,6 +9,13 @@ const getPendingBalanceToWithdrawForBuilder = gloas_utils.getPendingBalanceToWit const initiateBuilderExit = gloas_utils.initiateBuilderExit; const isActiveBuilder = gloas_utils.isActiveBuilder; +/// Apply a builder exit request. Authorizes the exit via `source_address` (the builder's +/// execution_address), not the BLS key — mirroring EIP-7002's 0x01 credential exit. +/// +/// Drops the request silently if any precondition fails; the EL has already dequeued it +/// deterministically, so the fee is forfeited but `requests_hash` agreement is unaffected. +/// +/// Spec: https://github.com/ethereum/consensus-specs/blob/v1.7.0-alpha.11/specs/gloas/beacon-chain.md#new-process_builder_exit_request pub fn processBuilderExitRequest( allocator: Allocator, config: *const BeaconConfig, diff --git a/src/state_transition/block/process_parent_execution_payload.zig b/src/state_transition/block/process_parent_execution_payload.zig index 337efc26a..94f44da18 100644 --- a/src/state_transition/block/process_parent_execution_payload.zig +++ b/src/state_transition/block/process_parent_execution_payload.zig @@ -13,6 +13,9 @@ const processBuilderDepositRequest = @import("./process_builder_deposit_request. const processBuilderExitRequest = @import("./process_builder_exit_request.zig").processBuilderExitRequest; const computeEpochAtSlot = @import("../utils/epoch.zig").computeEpochAtSlot; +/// Process parent execution payload effects as the first step of processBlock. +/// +/// Spec: https://github.com/ethereum/consensus-specs/blob/v1.7.0-alpha.6/specs/gloas/beacon-chain.md#new-process_parent_execution_payload pub fn processParentExecutionPayload( allocator: Allocator, config: *const BeaconConfig, @@ -41,6 +44,13 @@ pub fn processParentExecutionPayload( try applyParentExecutionPayload(allocator, config, epoch_cache, state, requests, &parent_bid); } +/// Process the parent's execution requests, queue the builder payment, update payload availability, +/// and update the latest block hash. +/// +/// Called from processParentExecutionPayload during block processing, and from the validator during +/// block production before computing withdrawals. +/// +/// Spec: https://github.com/ethereum/consensus-specs/blob/v1.7.0-alpha.6/specs/gloas/beacon-chain.md#new-apply_parent_execution_payload pub fn applyParentExecutionPayload( allocator: Allocator, config: *const BeaconConfig, @@ -94,6 +104,10 @@ pub fn applyParentExecutionPayload( try state.inner.setValue("latest_block_hash", &parent_bid.block_hash); } +/// Settle a builder payment at the given index: move its withdrawal (if any) to the +/// pending withdrawals list and clear the payment slot. +/// +/// Spec: https://github.com/ethereum/consensus-specs/blob/v1.7.0-alpha.6/specs/gloas/beacon-chain.md#new-settle_builder_payment fn settleBuilderPayment(allocator: Allocator, state: *BeaconState(.gloas), payment_index: u64) !void { var builder_pending_payments = try state.inner.get("builder_pending_payments"); if (payment_index >= 2 * preset.SLOTS_PER_EPOCH) return error.InvalidBuilderPendingPaymentIndex; diff --git a/src/state_transition/epoch/process_ptc_window.zig b/src/state_transition/epoch/process_ptc_window.zig index 11b5a01bd..a3b0f5004 100644 --- a/src/state_transition/epoch/process_ptc_window.zig +++ b/src/state_transition/epoch/process_ptc_window.zig @@ -10,6 +10,12 @@ const computeStartSlotAtEpoch = @import("../utils/epoch.zig").computeStartSlotAt const computePtc = @import("../utils/gloas.zig").computePtc; const ValidatorIndex = ct.primitive.ValidatorIndex.Type; +/// Update the `ptc_window` field in the beacon state by shifting out the oldest epoch's +/// PTC entries and appending newly computed entries for the next lookahead epoch. +/// Stashes the computed PTCs in the transition cache for finalProcessEpoch to shift +/// into the epoch cache without reading from state. +/// +/// Spec: https://github.com/ethereum/consensus-specs/blob/v1.7.0-alpha.4/specs/gloas/beacon-chain.md#new-process_ptc_window pub fn processPtcWindow( allocator: Allocator, epoch_cache: *const EpochCache, diff --git a/src/state_transition/slot/upgrade_state_to_gloas.zig b/src/state_transition/slot/upgrade_state_to_gloas.zig index c2773b10c..67d611551 100644 --- a/src/state_transition/slot/upgrade_state_to_gloas.zig +++ b/src/state_transition/slot/upgrade_state_to_gloas.zig @@ -20,6 +20,7 @@ const isBuilderWithdrawalCredential = gloas_utils.isBuilderWithdrawalCredential; const initializePtcWindow = gloas_utils.initializePtcWindow; const PendingDepositsLookup = @import("../utils/pending_deposits_lookup.zig").PendingDepositsLookup; +/// Upgrade a state from Fulu to Gloas. pub fn upgradeStateToGloas( allocator: Allocator, config: *const BeaconConfig, @@ -64,6 +65,8 @@ pub fn upgradeStateToGloas( return state; } +/// Applies any pending deposits for builders to onboard builders during the fork transition +/// Spec: https://github.com/ethereum/consensus-specs/blob/v1.7.0-alpha.8/specs/gloas/fork.md#new-onboard_builders_from_pending_deposits fn onboardBuildersFromPendingDeposits( allocator: Allocator, config: *const BeaconConfig, diff --git a/src/state_transition/utils/gloas.zig b/src/state_transition/utils/gloas.zig index 9ed38be4a..371c80be7 100644 --- a/src/state_transition/utils/gloas.zig +++ b/src/state_transition/utils/gloas.zig @@ -43,22 +43,34 @@ fn hasBuilderIndexFlag(index: u64) bool { return (index & c.BUILDER_INDEX_FLAG) != 0; } +/// Check if a validator index represents a builder (has the builder flag set). +/// Spec: https://github.com/ethereum/consensus-specs/blob/v1.7.0-alpha.1/specs/gloas/beacon-chain.md#new-is_builder_index pub fn isBuilderIndex(validator_index: u64) bool { return hasBuilderIndexFlag(validator_index); } +/// Convert a builder index to a flagged validator index for use in Withdrawal containers. +/// Spec: https://github.com/ethereum/consensus-specs/blob/v1.7.0-alpha.1/specs/gloas/beacon-chain.md#new-convert_builder_index_to_validator_index pub fn convertBuilderIndexToValidatorIndex(builder_index: u64) u64 { return if (hasBuilderIndexFlag(builder_index)) builder_index else builder_index | c.BUILDER_INDEX_FLAG; } +/// Convert a flagged validator index back to a builder index. +/// Spec: https://github.com/ethereum/consensus-specs/blob/v1.7.0-alpha.1/specs/gloas/beacon-chain.md#new-convert_validator_index_to_builder_index pub fn convertValidatorIndexToBuilderIndex(validator_index: u64) u64 { return if (hasBuilderIndexFlag(validator_index)) validator_index & ~c.BUILDER_INDEX_FLAG else validator_index; } +/// Check if a builder is active (deposited and not yet withdrawable). +/// Spec: https://github.com/ethereum/consensus-specs/blob/v1.7.0-alpha.1/specs/gloas/beacon-chain.md#isactivebuilder pub fn isActiveBuilder(builder: *const ct.gloas.Builder.Type, finalized_epoch: u64) bool { return builder.deposit_epoch < finalized_epoch and builder.withdrawable_epoch == c.FAR_FUTURE_EPOCH; } +/// Compute the gas limit that satisfies the EIP-1559 adjustment rule from `parent_gas_limit`, +/// clamping `target_gas_limit` into the allowed window of `±max(parent_gas_limit / 1024, 1) - 1`. +/// +/// From https://github.com/ethereum/EIPs/blob/master/EIPS/eip-1559.md pub fn getExpectedGasLimit(parent_gas_limit: u64, target_gas_limit: u64) u64 { const max_gas_limit_difference = @max(parent_gas_limit / 1024, 1) - 1; @@ -69,10 +81,16 @@ pub fn getExpectedGasLimit(parent_gas_limit: u64, target_gas_limit: u64) u64 { return parent_gas_limit - @min(parent_gas_limit - target_gas_limit, max_gas_limit_difference); } +/// Check if `gas_limit` is compatible with `target_gas_limit` under the EIP-1559 transition rule +/// from `parent_gas_limit`. The bid must hit `target_gas_limit` when the target is within one +/// adjustment step of the parent, otherwise it must hit the clamped boundary. +/// Spec: https://github.com/ethereum/consensus-specs/blob/v1.7.0-alpha.8/specs/gloas/builder.md#new-is_gas_limit_target_compatible pub fn isGasLimitTargetCompatible(parent_gas_limit: u64, gas_limit: u64, target_gas_limit: u64) bool { return gas_limit == getExpectedGasLimit(parent_gas_limit, target_gas_limit); } +/// Get the total pending balance to withdraw for a builder (from withdrawals + payments). +/// Spec: https://github.com/ethereum/consensus-specs/blob/v1.7.0-alpha.1/specs/gloas/beacon-chain.md#new-get_pending_balance_to_withdraw_for_builder pub fn getPendingBalanceToWithdrawForBuilder(allocator: Allocator, state: *BeaconState(.gloas), builder_index: u64) !u64 { var pending_balance: u64 = 0; @@ -100,6 +118,8 @@ pub fn getPendingBalanceToWithdrawForBuilder(allocator: Allocator, state: *Beaco return pending_balance; } +/// Check if a builder has sufficient balance to cover a bid amount. +/// Spec: https://github.com/ethereum/consensus-specs/blob/v1.7.0-alpha.1/specs/gloas/beacon-chain.md#new-can_builder_cover_bid pub fn canBuilderCoverBid(allocator: Allocator, state: *BeaconState(.gloas), builder_index: u64, bid_amount: u64) !bool { var builders = try state.inner.get("builders"); var builder: ct.gloas.Builder.Type = undefined; @@ -112,6 +132,8 @@ pub fn canBuilderCoverBid(allocator: Allocator, state: *BeaconState(.gloas), bui return builder.balance - min_balance >= bid_amount; } +/// Initiate a builder exit by setting their withdrawable epoch. +/// Spec: https://github.com/ethereum/consensus-specs/blob/v1.7.0-alpha.1/specs/gloas/beacon-chain.md#new-initiate_builder_exit pub fn initiateBuilderExit(config: *const BeaconConfig, state: *BeaconState(.gloas), allocator: Allocator, builder_index: u64) !void { var builders = try state.inner.get("builders"); var builder: ct.gloas.Builder.Type = undefined; @@ -124,6 +146,10 @@ pub fn initiateBuilderExit(config: *const BeaconConfig, state: *BeaconState(.glo try builders.setValue(builder_index, &builder); } +/// Find the index of a builder by their public key. +/// Returns null if not found. +/// +/// May consider builder pubkey cache if performance becomes an issue. pub fn findBuilderIndexByPubkey(allocator: Allocator, state: *BeaconState(.gloas), pubkey: *const BLSPubkey) !?usize { var builders = try state.inner.get("builders"); const len = try builders.length(); @@ -135,6 +161,10 @@ pub fn findBuilderIndexByPubkey(allocator: Allocator, state: *BeaconState(.gloas return null; } +/// Add a new builder to the builders registry. Reuses slots from exited and fully withdrawn +/// builders when available, otherwise appends. +/// +/// Spec: https://github.com/ethereum/consensus-specs/blob/v1.7.0-alpha.11/specs/gloas/beacon-chain.md#new-add_builder_to_registry pub fn addBuilderToRegistry( allocator: Allocator, state: *BeaconState(.gloas), @@ -175,6 +205,12 @@ pub fn addBuilderToRegistry( } } +/// Verify the proof of possession on a builder deposit request. +/// +/// The dedicated `DOMAIN_BUILDER_DEPOSIT` (vs the validator `DOMAIN_DEPOSIT`) prevents replay +/// of validator deposit signatures against the builder deposit contract and vice versa. +/// +/// Spec: https://github.com/ethereum/consensus-specs/blob/v1.7.0-alpha.11/specs/gloas/beacon-chain.md#new-is_valid_builder_deposit_signature pub fn isValidBuilderDepositSignature( config: *const BeaconConfig, pubkey: *const BLSPubkey, From ec5fcdbbee06fdd0f6cfe316a9158748b31575da Mon Sep 17 00:00:00 2001 From: Rahul Guha <19rahul2003@gmail.com> Date: Tue, 7 Jul 2026 14:50:17 +0530 Subject: [PATCH 22/33] more comments and getReadonly instead of get --- src/state_transition/utils/gloas.zig | 27 +++++++++---------- .../utils/pending_deposits_lookup.zig | 19 +++++++++++++ src/state_transition/utils/validator.zig | 4 +++ 3 files changed, 36 insertions(+), 14 deletions(-) diff --git a/src/state_transition/utils/gloas.zig b/src/state_transition/utils/gloas.zig index 371c80be7..d6f66e987 100644 --- a/src/state_transition/utils/gloas.zig +++ b/src/state_transition/utils/gloas.zig @@ -94,8 +94,7 @@ pub fn isGasLimitTargetCompatible(parent_gas_limit: u64, gas_limit: u64, target_ pub fn getPendingBalanceToWithdrawForBuilder(allocator: Allocator, state: *BeaconState(.gloas), builder_index: u64) !u64 { var pending_balance: u64 = 0; - var withdrawals = try state.inner.get("builder_pending_withdrawals"); - try withdrawals.commit(); + var withdrawals = try state.inner.getReadonly("builder_pending_withdrawals"); const withdrawals_len = try withdrawals.length(); var w_it = withdrawals.iteratorReadonly(0); for (0..withdrawals_len) |_| { @@ -105,7 +104,7 @@ pub fn getPendingBalanceToWithdrawForBuilder(allocator: Allocator, state: *Beaco } } - var payments = try state.inner.get("builder_pending_payments"); + var payments = try state.inner.getReadonly("builder_pending_payments"); const payments_len = ct.gloas.BeaconState.getFieldType("builder_pending_payments").length; for (0..payments_len) |i| { var p: ct.gloas.BuilderPendingPayment.Type = undefined; @@ -121,7 +120,7 @@ pub fn getPendingBalanceToWithdrawForBuilder(allocator: Allocator, state: *Beaco /// Check if a builder has sufficient balance to cover a bid amount. /// Spec: https://github.com/ethereum/consensus-specs/blob/v1.7.0-alpha.1/specs/gloas/beacon-chain.md#new-can_builder_cover_bid pub fn canBuilderCoverBid(allocator: Allocator, state: *BeaconState(.gloas), builder_index: u64, bid_amount: u64) !bool { - var builders = try state.inner.get("builders"); + var builders = try state.inner.getReadonly("builders"); var builder: ct.gloas.Builder.Type = undefined; try builders.getValue(allocator, builder_index, &builder); @@ -263,7 +262,7 @@ pub fn isAttestationSameSlotRootCache(root_cache: *RootCache(.gloas), data: *con } pub fn isParentBlockFull(state: *BeaconState(.gloas)) !bool { - var bid = try state.inner.get("latest_execution_payload_bid"); + var bid = try state.inner.getReadonly("latest_execution_payload_bid"); const bid_block_hash = try bid.getFieldRoot("block_hash"); const latest_block_hash = try state.inner.getFieldRoot("latest_block_hash"); return std.mem.eql(u8, bid_block_hash, latest_block_hash); @@ -315,15 +314,15 @@ pub fn initializePtcWindow( epoch_cache: *const EpochCache, state: *BeaconState(fork), ) ![ct.gloas.PtcWindow.length][ct.gloas.PtcWindow.Element.length]ValidatorIndex { - const PtcSize = ct.gloas.PtcWindow.Element.length; - const PtcWindowLen = ct.gloas.PtcWindow.length; + const ptc_size = ct.gloas.PtcWindow.Element.length; + const ptc_window_len = ct.gloas.PtcWindow.length; - const empty_previous_epoch = [_][PtcSize]ValidatorIndex{ - [_]ValidatorIndex{0} ** PtcSize, + const empty_previous_epoch = [_][ptc_size]ValidatorIndex{ + [_]ValidatorIndex{0} ** ptc_size, } ** preset.SLOTS_PER_EPOCH; const current_epoch = computeEpochAtSlot(try state.slot()); - var current_and_lookahead: [(1 + preset.MIN_SEED_LOOKAHEAD) * preset.SLOTS_PER_EPOCH][PtcSize]ValidatorIndex = undefined; + var ptcs: [(1 + preset.MIN_SEED_LOOKAHEAD) * preset.SLOTS_PER_EPOCH][ptc_size]ValidatorIndex = undefined; for (0..1 + preset.MIN_SEED_LOOKAHEAD) |epoch_offset| { const epoch = current_epoch + epoch_offset; const epoch_committees = try computePayloadTimelinessCommitteesForEpoch( @@ -334,14 +333,14 @@ pub fn initializePtcWindow( epoch_cache, ); for (0..preset.SLOTS_PER_EPOCH) |slot_index| { - current_and_lookahead[epoch_offset * preset.SLOTS_PER_EPOCH + slot_index] = epoch_committees[slot_index]; + ptcs[epoch_offset * preset.SLOTS_PER_EPOCH + slot_index] = epoch_committees[slot_index]; } } - // return empty_previous_epoch + current_and_lookahead - var ptc_window: [PtcWindowLen][PtcSize]ValidatorIndex = undefined; + // return empty_previous_epoch + ptcs + var ptc_window: [ptc_window_len][ptc_size]ValidatorIndex = undefined; @memcpy(ptc_window[0..preset.SLOTS_PER_EPOCH], &empty_previous_epoch); - @memcpy(ptc_window[preset.SLOTS_PER_EPOCH..], ¤t_and_lookahead); + @memcpy(ptc_window[preset.SLOTS_PER_EPOCH..], &ptcs); return ptc_window; } diff --git a/src/state_transition/utils/pending_deposits_lookup.zig b/src/state_transition/utils/pending_deposits_lookup.zig index c60add60e..7599ab835 100644 --- a/src/state_transition/utils/pending_deposits_lookup.zig +++ b/src/state_transition/utils/pending_deposits_lookup.zig @@ -1,3 +1,15 @@ +//! Mutable lookup for the pending-deposit sequence used by builder-routing logic. +//! +//! This is to implement the spec's `is_pending_validator(pending_deposits, pubkey)` lazily: +//! deposits are grouped by pubkey without verifying signatures, and BLS verification is +//! deferred until a builder deposit needs to know whether the same pubkey already has a +//! valid pending validator deposit. +//! +//! Call `add()` whenever a deposit is appended to the represented sequence. A cached `true` +//! result short-circuits all subsequent checks for that pubkey; a cached `false` records +//! how many deposits were already verified, so appending a new deposit only verifies the +//! newly-appended tail rather than re-running BLS on previously-invalid entries. + const std = @import("std"); const Allocator = std.mem.Allocator; const BeaconConfig = @import("config").BeaconConfig; @@ -23,6 +35,7 @@ pub const PendingDepositsLookup = struct { allocator: Allocator, deposits_by_pubkey: std.AutoHashMap(BLSPubkey, PendingDepositEntry), + /// Build an empty lookup for a sequence that will be populated incrementally. pub fn buildEmpty(allocator: Allocator) PendingDepositsLookup { return .{ .allocator = allocator, @@ -30,6 +43,8 @@ pub const PendingDepositsLookup = struct { }; } + /// Build a pubkey -> pending-deposits lookup from `state.pendingDeposits`. + /// No BLS work is done here; signature verification happens lazily in `hasPendingValidator`. pub fn build(comptime fork: ForkSeq, allocator: Allocator, state: *BeaconState(fork)) !PendingDepositsLookup { var lookup = PendingDepositsLookup.buildEmpty(allocator); errdefer lookup.deinit(); @@ -54,6 +69,7 @@ pub const PendingDepositsLookup = struct { self.deposits_by_pubkey.deinit(); } + /// Append a pending deposit to the represented sequence. pub fn add(self: *PendingDepositsLookup, pending_deposit: *const PendingDeposit) !void { const result = try self.deposits_by_pubkey.getOrPut(pending_deposit.pubkey); if (!result.found_existing) { @@ -62,6 +78,9 @@ pub const PendingDepositsLookup = struct { try result.value_ptr.deposits.append(self.allocator, pending_deposit.*); } + /// Returns true if any pending deposit for `pubkey` has a valid BLS deposit signature. + /// Memoizes the result so repeated checks for the same pubkey within a block only verify + /// deposits that have not already been checked. pub fn hasPendingValidator( self: *PendingDepositsLookup, config: *const BeaconConfig, diff --git a/src/state_transition/utils/validator.zig b/src/state_transition/utils/validator.zig index d89547af0..27b2a2049 100644 --- a/src/state_transition/utils/validator.zig +++ b/src/state_transition/utils/validator.zig @@ -74,6 +74,7 @@ pub fn getActivationExitChurnLimit(epoch_cache: *const EpochCache) u64 { return @min(epoch_cache.config.chain.MAX_PER_EPOCH_ACTIVATION_EXIT_CHURN_LIMIT, getBalanceChurnLimitFromCache(epoch_cache)); } +/// https://github.com/ethereum/consensus-specs/blob/v1.7.0-alpha.6/specs/gloas/beacon-chain.md#new-get_activation_churn_limit pub fn getGloasActivationChurnLimit(epoch_cache: *const EpochCache) u64 { const churn = getBalanceChurnLimit( epoch_cache.total_active_balance_increments, @@ -83,6 +84,7 @@ pub fn getGloasActivationChurnLimit(epoch_cache: *const EpochCache) u64 { return @min(epoch_cache.config.chain.MAX_PER_EPOCH_ACTIVATION_CHURN_LIMIT_GLOAS, churn); } +/// https://github.com/ethereum/consensus-specs/blob/v1.7.0-alpha.6/specs/gloas/beacon-chain.md#new-get_exit_churn_limit pub fn getGloasExitChurnLimit(epoch_cache: *const EpochCache) u64 { return getBalanceChurnLimit( epoch_cache.total_active_balance_increments, @@ -91,6 +93,8 @@ pub fn getGloasExitChurnLimit(epoch_cache: *const EpochCache) u64 { ); } +/// Spec (electra): get_consolidation_churn_limit (uses combined balance churn minus activation+exit churn) +/// Spec (gloas): get_consolidation_churn_limit (independent quotient, no MIN floor) pub fn getConsolidationChurnLimit(comptime fork: ForkSeq, epoch_cache: *const EpochCache) u64 { if (comptime fork.gte(.gloas)) { return getBalanceChurnLimit( From 27def7e1f7f29af673c1e698627ce8408aaca871 Mon Sep 17 00:00:00 2001 From: Rahul Guha <19rahul2003@gmail.com> Date: Tue, 7 Jul 2026 15:23:37 +0530 Subject: [PATCH 23/33] make PendingDepositsLookup self-contained --- .../utils/pending_deposits_lookup.zig | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/state_transition/utils/pending_deposits_lookup.zig b/src/state_transition/utils/pending_deposits_lookup.zig index 7599ab835..72295e536 100644 --- a/src/state_transition/utils/pending_deposits_lookup.zig +++ b/src/state_transition/utils/pending_deposits_lookup.zig @@ -21,25 +21,25 @@ const validateDepositSignature = @import("../block/process_deposit.zig").validat const BLSPubkey = ct.primitive.BLSPubkey.Type; const PendingDeposit = ct.electra.PendingDeposit.Type; -const PendingDepositEntry = struct { - deposits: std.ArrayList(PendingDeposit) = .empty, - validated_count: usize = 0, - has_valid_signature: bool = false, - - fn deinit(self: *PendingDepositEntry, allocator: Allocator) void { - self.deposits.deinit(allocator); - } -}; - pub const PendingDepositsLookup = struct { allocator: Allocator, - deposits_by_pubkey: std.AutoHashMap(BLSPubkey, PendingDepositEntry), + deposits_by_pubkey: std.AutoHashMap(BLSPubkey, PendingDeposits), + + const PendingDeposits = struct { + deposits: std.ArrayList(PendingDeposit) = .empty, + validated_count: usize = 0, + has_valid_signature: bool = false, + + fn deinit(self: *PendingDeposits, allocator: Allocator) void { + self.deposits.deinit(allocator); + } + }; /// Build an empty lookup for a sequence that will be populated incrementally. pub fn buildEmpty(allocator: Allocator) PendingDepositsLookup { return .{ .allocator = allocator, - .deposits_by_pubkey = std.AutoHashMap(BLSPubkey, PendingDepositEntry).init(allocator), + .deposits_by_pubkey = std.AutoHashMap(BLSPubkey, PendingDeposits).init(allocator), }; } From 8cd5adb532b4a1741e05b2ebafe7654050605a1a Mon Sep 17 00:00:00 2001 From: Rahul Guha <19rahul2003@gmail.com> Date: Tue, 7 Jul 2026 15:47:19 +0530 Subject: [PATCH 24/33] refactor pending deposits lookup --- src/state_transition/cache/epoch_cache.zig | 2 +- .../slot/upgrade_state_to_gloas.zig | 2 +- .../utils/pending_deposits_lookup.zig | 51 ++++++++++++------- 3 files changed, 36 insertions(+), 19 deletions(-) diff --git a/src/state_transition/cache/epoch_cache.zig b/src/state_transition/cache/epoch_cache.zig index a1774ae53..efc13b3b4 100644 --- a/src/state_transition/cache/epoch_cache.zig +++ b/src/state_transition/cache/epoch_cache.zig @@ -61,7 +61,7 @@ fn getPtcWindowEpochCacheData(state: *AnyBeaconState) !PtcWindowEpochCacheData { } fn getPtcWindowEpochCacheDataFromState(state: *BeaconState(.gloas)) !PtcWindowEpochCacheData { - var ptc_window = try state.inner.get("ptc_window"); + var ptc_window = try state.inner.getReadonly("ptc_window"); var data: PtcWindowEpochCacheData = undefined; for (0..preset.SLOTS_PER_EPOCH) |i| { diff --git a/src/state_transition/slot/upgrade_state_to_gloas.zig b/src/state_transition/slot/upgrade_state_to_gloas.zig index 67d611551..39058e9b1 100644 --- a/src/state_transition/slot/upgrade_state_to_gloas.zig +++ b/src/state_transition/slot/upgrade_state_to_gloas.zig @@ -76,7 +76,7 @@ fn onboardBuildersFromPendingDeposits( var remaining_pending_deposits: std.ArrayList(PendingDeposit) = .empty; defer remaining_pending_deposits.deinit(allocator); - var pending_deposits_lookup = PendingDepositsLookup.buildEmpty(allocator); + var pending_deposits_lookup = PendingDepositsLookup.init(allocator); defer pending_deposits_lookup.deinit(); var pending_deposits = try state.pendingDeposits(); diff --git a/src/state_transition/utils/pending_deposits_lookup.zig b/src/state_transition/utils/pending_deposits_lookup.zig index 72295e536..a5190446c 100644 --- a/src/state_transition/utils/pending_deposits_lookup.zig +++ b/src/state_transition/utils/pending_deposits_lookup.zig @@ -23,30 +23,35 @@ const PendingDeposit = ct.electra.PendingDeposit.Type; pub const PendingDepositsLookup = struct { allocator: Allocator, - deposits_by_pubkey: std.AutoHashMap(BLSPubkey, PendingDeposits), + deposits_by_pubkey: std.AutoHashMapUnmanaged(BLSPubkey, PendingDeposits), + validation_cache: std.AutoHashMapUnmanaged(BLSPubkey, PendingDepositsValidation), const PendingDeposits = struct { deposits: std.ArrayList(PendingDeposit) = .empty, - validated_count: usize = 0, - has_valid_signature: bool = false, fn deinit(self: *PendingDeposits, allocator: Allocator) void { self.deposits.deinit(allocator); } }; + const PendingDepositsValidation = struct { + has_valid_signature: bool, + validated_count: usize, + }; + /// Build an empty lookup for a sequence that will be populated incrementally. - pub fn buildEmpty(allocator: Allocator) PendingDepositsLookup { + pub fn init(allocator: Allocator) PendingDepositsLookup { return .{ .allocator = allocator, - .deposits_by_pubkey = std.AutoHashMap(BLSPubkey, PendingDeposits).init(allocator), + .deposits_by_pubkey = .empty, + .validation_cache = .empty, }; } /// Build a pubkey -> pending-deposits lookup from `state.pendingDeposits`. /// No BLS work is done here; signature verification happens lazily in `hasPendingValidator`. - pub fn build(comptime fork: ForkSeq, allocator: Allocator, state: *BeaconState(fork)) !PendingDepositsLookup { - var lookup = PendingDepositsLookup.buildEmpty(allocator); + pub fn initFromState(comptime fork: ForkSeq, allocator: Allocator, state: *BeaconState(fork)) !PendingDepositsLookup { + var lookup = PendingDepositsLookup.init(allocator); errdefer lookup.deinit(); var pending_deposits = try state.pendingDeposits(); @@ -66,12 +71,13 @@ pub const PendingDepositsLookup = struct { while (value_iterator.next()) |entry| { entry.deinit(self.allocator); } - self.deposits_by_pubkey.deinit(); + self.deposits_by_pubkey.deinit(self.allocator); + self.validation_cache.deinit(self.allocator); } /// Append a pending deposit to the represented sequence. pub fn add(self: *PendingDepositsLookup, pending_deposit: *const PendingDeposit) !void { - const result = try self.deposits_by_pubkey.getOrPut(pending_deposit.pubkey); + const result = try self.deposits_by_pubkey.getOrPut(self.allocator, pending_deposit.pubkey); if (!result.found_existing) { result.value_ptr.* = .{}; } @@ -79,18 +85,24 @@ pub const PendingDepositsLookup = struct { } /// Returns true if any pending deposit for `pubkey` has a valid BLS deposit signature. - /// Memoizes the result so repeated checks for the same pubkey within a block only verify - /// deposits that have not already been checked. + /// Memoizes the result in `validation_cache` so repeated checks for the same pubkey + /// within a block only verify deposits that have not already been checked. pub fn hasPendingValidator( self: *PendingDepositsLookup, config: *const BeaconConfig, pubkey: *const BLSPubkey, ) !bool { + const validation = self.validation_cache.get(pubkey.*); + if (validation) |v| { + if (v.has_valid_signature) return true; + } + const entry = self.deposits_by_pubkey.getPtr(pubkey.*) orelse return false; - if (entry.has_valid_signature) return true; - if (entry.validated_count == entry.deposits.items.len) return false; - var i = entry.validated_count; + const start_index = if (validation) |v| v.validated_count else 0; + if (start_index == entry.deposits.items.len) return false; + + var i = start_index; while (i < entry.deposits.items.len) : (i += 1) { const deposit = &entry.deposits.items[i]; validateDepositSignature( @@ -100,12 +112,17 @@ pub const PendingDepositsLookup = struct { deposit.amount, deposit.signature, ) catch continue; - entry.has_valid_signature = true; - entry.validated_count = i + 1; + try self.validation_cache.put(self.allocator, pubkey.*, .{ + .has_valid_signature = true, + .validated_count = i + 1, + }); return true; } - entry.validated_count = entry.deposits.items.len; + try self.validation_cache.put(self.allocator, pubkey.*, .{ + .has_valid_signature = false, + .validated_count = entry.deposits.items.len, + }); return false; } }; From 42264fc08b0834dbbaf094069cfc50be7ed376c8 Mon Sep 17 00:00:00 2001 From: Rahul Guha <19rahul2003@gmail.com> Date: Tue, 7 Jul 2026 15:52:56 +0530 Subject: [PATCH 25/33] nit order --- src/state_transition/cache/epoch_cache.zig | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/state_transition/cache/epoch_cache.zig b/src/state_transition/cache/epoch_cache.zig index efc13b3b4..40fd80788 100644 --- a/src/state_transition/cache/epoch_cache.zig +++ b/src/state_transition/cache/epoch_cache.zig @@ -160,12 +160,12 @@ pub const EpochCache = struct { epoch: Epoch, - /// PTC for current epoch, computed eagerly at epoch transition. - payload_timeliness_committees: ?[preset.SLOTS_PER_EPOCH][preset.PTC_SIZE]ValidatorIndex, - /// PTC for previous epoch, required for slot N block validating slot N-1 attestations. previous_payload_timeliness_committees: ?[preset.SLOTS_PER_EPOCH][preset.PTC_SIZE]ValidatorIndex, + /// PTC for current epoch, computed eagerly at epoch transition. + payload_timeliness_committees: ?[preset.SLOTS_PER_EPOCH][preset.PTC_SIZE]ValidatorIndex, + /// PTC for next epoch, sourced from state.ptc_window or the epoch transition stash. next_payload_timeliness_committees: ?[preset.SLOTS_PER_EPOCH][preset.PTC_SIZE]ValidatorIndex, From 83d93d21cafe8931e3a25263eae8c404259ac9a8 Mon Sep 17 00:00:00 2001 From: Rahul Guha <19rahul2003@gmail.com> Date: Tue, 7 Jul 2026 16:30:56 +0530 Subject: [PATCH 26/33] merge main into gloas --- bindings/napi/BeaconStateView.zig | 18 +++++++- bindings/test/beaconStateView.test.ts | 2 +- src/ssz/tree_view/list_composite.zig | 46 ++++++++++++------- .../block/process_withdrawals.zig | 4 +- .../epoch/process_pending_consolidations.zig | 2 +- .../epoch/process_pending_deposits.zig | 2 +- .../slot/upgrade_state_to_gloas.zig | 2 +- src/state_transition/utils/gloas.zig | 2 +- .../utils/pending_deposits_lookup.zig | 2 +- test/spec/runner/epoch_processing.zig | 3 +- test/spec/runner/operations.zig | 5 +- 11 files changed, 59 insertions(+), 29 deletions(-) diff --git a/bindings/napi/BeaconStateView.zig b/bindings/napi/BeaconStateView.zig index b6cb2c170..ceb873020 100644 --- a/bindings/napi/BeaconStateView.zig +++ b/bindings/napi/BeaconStateView.zig @@ -1255,11 +1255,25 @@ pub fn hashTreeRoot(self: *const BeaconStateView) !js.Uint8Array { /// /// Arguments: /// - arg 0: target slot (number) -/// - arg 1: options object (optional) with `transferCache` boolean +/// - arg 1: options object (optional) with Lodestar's `dontTransferCache` boolean pub fn processSlots(self: *const BeaconStateView, slot_arg: js.Number, options: ?js.Value) !BeaconStateView { const cached_state = try self.requireState(); const slot_value: u64 = @intCast(try slot_arg.toI64()); - const transfer_cache = try optionalBool(options, "transferCache", false); + + var transfer_cache = true; + if (options) |value| { + const raw = value.toValue(); + if (try raw.typeof() == .object) { + // TODO(bing): we should rename dontTransferCache to transferCache on TS-side to + // avoid double negative, it's harder to read dontTransferCache = true versus + // transferCache = false + if (try raw.hasNamedProperty("dontTransferCache")) { + transfer_cache = !(try (try raw.getNamedProperty("dontTransferCache")).getValueBool()); + } else if (try raw.hasNamedProperty("transferCache")) { + transfer_cache = try (try raw.getNamedProperty("transferCache")).getValueBool(); + } + } + } const post_state = try cached_state.clone(allocator, .{ .transfer_cache = transfer_cache }); errdefer { post_state.deinit(); diff --git a/bindings/test/beaconStateView.test.ts b/bindings/test/beaconStateView.test.ts index 0a18e2944..3a60d7e36 100644 --- a/bindings/test/beaconStateView.test.ts +++ b/bindings/test/beaconStateView.test.ts @@ -684,7 +684,7 @@ describe("BeaconStateView", () => { const newState = state.processSlots(originalSlot + 1, {dontTransferCache: false}); expect(newState.slot).toBe(originalSlot + 1); - expect(newState.createdWithTransferCache).toBe(false); + expect(newState.createdWithTransferCache).toBe(true); }); }); diff --git a/src/ssz/tree_view/list_composite.zig b/src/ssz/tree_view/list_composite.zig index 31a924277..067e0241c 100644 --- a/src/ssz/tree_view/list_composite.zig +++ b/src/ssz/tree_view/list_composite.zig @@ -253,24 +253,36 @@ pub fn ListCompositeTreeView(comptime ST: type) type { } /// Get the next element as an SSZ value type. - pub fn nextValue(self: *ReadonlyIterator, allocator: Allocator) !ST.Element.Type { + /// Picks between the non-allocating version for fixed types or + /// the allocating version for variable-length types. + pub const nextValue = if (isFixedType(ST.Element)) + nextValueFixed + else + nextValueAlloc; + + /// Inner function to get the next element as an SSZ value type. + fn nextValueFixed(self: *ReadonlyIterator) !ST.Element.Type { const node = try self.depth_iterator.next(); - if (comptime isFixedType(ST.Element)) { - var value: ST.Element.Type = undefined; - try ST.Element.tree.toValue(node, self.tree_view.chunks.state.pool, &value); - self.elem_index += 1; - return value; - } else { - // Variable-size elements: toValue reads `out` (it resizes embedded ArrayLists), - // so initialize it before the call. - var value: ST.Element.Type = if (comptime @hasDecl(ST.Element, "default_value")) - ST.Element.default_value - else - std.mem.zeroes(ST.Element.Type); - try ST.Element.tree.toValue(allocator, node, self.tree_view.chunks.state.pool, &value); - self.elem_index += 1; - return value; - } + var value: ST.Element.Type = undefined; + try ST.Element.tree.toValue(node, self.tree_view.chunks.state.pool, &value); + self.elem_index += 1; + return value; + } + + /// Inner function to get the next element as an SSZ value type. + /// + /// Requires an allocator for `toValue`. + fn nextValueAlloc(self: *ReadonlyIterator, allocator: Allocator) !ST.Element.Type { + const node = try self.depth_iterator.next(); + // Variable-size elements: toValue reads `out` (it resizes embedded ArrayLists), + // so initialize it before the call. + var value: ST.Element.Type = if (comptime @hasDecl(ST.Element, "default_value")) + ST.Element.default_value + else + std.mem.zeroes(ST.Element.Type); + try ST.Element.tree.toValue(allocator, node, self.tree_view.chunks.state.pool, &value); + self.elem_index += 1; + return value; } /// Read-only pointer to the next element's value without copying. diff --git a/src/state_transition/block/process_withdrawals.zig b/src/state_transition/block/process_withdrawals.zig index 3f2cb7127..8af8b9c12 100644 --- a/src/state_transition/block/process_withdrawals.zig +++ b/src/state_transition/block/process_withdrawals.zig @@ -282,7 +282,7 @@ fn getPendingPartialWithdrawals( // EIP-7002: Execution layer triggerable withdrawals var processed_count: usize = 0; for (0..pending_partial_withdrawals_len) |_| { - const withdrawal = try pending_partial_withdrawals_it.nextValue(undefined); + const withdrawal = try pending_partial_withdrawals_it.nextValue(); if (withdrawal.withdrawable_epoch > epoch or pending_partial_withdrawals_result.items.len + num_prior_withdrawals >= partial_withdrawal_bound) { break; } @@ -423,7 +423,7 @@ fn getBuilderWithdrawals( const all_withdrawals = prior_withdrawals_len + builder_withdrawals.items.len; if (all_withdrawals >= withdrawals_limit) break; - const bw = try bw_it.nextValue(allocator); + const bw = try bw_it.nextValue(); const builder_index = bw.builder_index; // Get builder balance (from builder.balance, not state.balances) diff --git a/src/state_transition/epoch/process_pending_consolidations.zig b/src/state_transition/epoch/process_pending_consolidations.zig index 78ffdcd98..f669583de 100644 --- a/src/state_transition/epoch/process_pending_consolidations.zig +++ b/src/state_transition/epoch/process_pending_consolidations.zig @@ -21,7 +21,7 @@ pub fn processPendingConsolidations( var pending_consolidations_it = pending_consolidations.iteratorReadonly(0); const pending_consolidations_length = try pending_consolidations.length(); for (0..pending_consolidations_length) |_| { - const pending_consolidation = try pending_consolidations_it.nextValue(undefined); + const pending_consolidation = try pending_consolidations_it.nextValue(); const source_index = pending_consolidation.source_index; const target_index = pending_consolidation.target_index; var source_validator = try validators.get(source_index); diff --git a/src/state_transition/epoch/process_pending_deposits.zig b/src/state_transition/epoch/process_pending_deposits.zig index e95f55339..df3d201b0 100644 --- a/src/state_transition/epoch/process_pending_deposits.zig +++ b/src/state_transition/epoch/process_pending_deposits.zig @@ -50,7 +50,7 @@ pub fn processPendingDeposits( const pending_deposits_len = try pending_deposits.length(); for (0..pending_deposits_len) |_| { - const deposit = try pending_deposits_it.nextValue(undefined); + const deposit = try pending_deposits_it.nextValue(); // Pre-fulu: do not process deposit requests if Eth1 bridge deposits are not yet applied. // Fulu removes this guard along with the former (Eth1 bridge) deposit mechanism. if (comptime fork.lt(.fulu)) { diff --git a/src/state_transition/slot/upgrade_state_to_gloas.zig b/src/state_transition/slot/upgrade_state_to_gloas.zig index 39058e9b1..4443c2388 100644 --- a/src/state_transition/slot/upgrade_state_to_gloas.zig +++ b/src/state_transition/slot/upgrade_state_to_gloas.zig @@ -84,7 +84,7 @@ fn onboardBuildersFromPendingDeposits( var pending_it = pending_deposits.iteratorReadonly(0); for (0..pending_deposits_len) |_| { - const deposit = try pending_it.nextValue(allocator); + const deposit = try pending_it.nextValue(); const validator_index = epoch_cache.getValidatorIndex(&deposit.pubkey); if (try isValidatorKnown(.gloas, state, validator_index)) { diff --git a/src/state_transition/utils/gloas.zig b/src/state_transition/utils/gloas.zig index d6f66e987..456b02c0a 100644 --- a/src/state_transition/utils/gloas.zig +++ b/src/state_transition/utils/gloas.zig @@ -98,7 +98,7 @@ pub fn getPendingBalanceToWithdrawForBuilder(allocator: Allocator, state: *Beaco const withdrawals_len = try withdrawals.length(); var w_it = withdrawals.iteratorReadonly(0); for (0..withdrawals_len) |_| { - const w = try w_it.nextValue(allocator); + const w = try w_it.nextValue(); if (w.builder_index == builder_index) { pending_balance += w.amount; } diff --git a/src/state_transition/utils/pending_deposits_lookup.zig b/src/state_transition/utils/pending_deposits_lookup.zig index a5190446c..721924ef4 100644 --- a/src/state_transition/utils/pending_deposits_lookup.zig +++ b/src/state_transition/utils/pending_deposits_lookup.zig @@ -59,7 +59,7 @@ pub const PendingDepositsLookup = struct { var pending_it = pending_deposits.iteratorReadonly(0); for (0..pending_deposits_len) |_| { - const pending_deposit = try pending_it.nextValue(allocator); + const pending_deposit = try pending_it.nextValue(); try lookup.add(&pending_deposit); } diff --git a/test/spec/runner/epoch_processing.zig b/test/spec/runner/epoch_processing.zig index 3b7b63430..adacca1c9 100644 --- a/test/spec/runner/epoch_processing.zig +++ b/test/spec/runner/epoch_processing.zig @@ -26,6 +26,7 @@ pub const EpochProcessingFn = enum { sync_committee_updates, historical_summaries_update, pending_deposits, + pending_deposits_churn, pending_consolidations, proposer_lookahead, builder_pending_payments, @@ -130,7 +131,7 @@ pub fn TestCase(comptime fork: ForkSeq, comptime epoch_process_fn: EpochProcessi .slashings_reset => try state_transition.processSlashingsReset(fork, epoch_cache, fork_state, &epoch_transition_cache), .sync_committee_updates => try state_transition.processSyncCommitteeUpdates(fork, allocator, epoch_cache, fork_state), .historical_summaries_update => try state_transition.processHistoricalSummariesUpdate(fork, fork_state, &epoch_transition_cache), - .pending_deposits => try state_transition.processPendingDeposits(fork, allocator, config, epoch_cache, fork_state, &epoch_transition_cache), + .pending_deposits, .pending_deposits_churn => try state_transition.processPendingDeposits(fork, allocator, config, epoch_cache, fork_state, &epoch_transition_cache), .pending_consolidations => try state_transition.processPendingConsolidations(fork, epoch_cache, fork_state, &epoch_transition_cache), .proposer_lookahead => { try state_transition.processProposerLookahead(fork, allocator, epoch_cache, fork_state, &epoch_transition_cache); diff --git a/test/spec/runner/operations.zig b/test/spec/runner/operations.zig index 55aa53788..4ca8da236 100644 --- a/test/spec/runner/operations.zig +++ b/test/spec/runner/operations.zig @@ -37,6 +37,7 @@ pub const Operation = enum { proposer_slashing, sync_aggregate, voluntary_exit, + voluntary_exit_churn, withdrawal_request, withdrawals, @@ -46,6 +47,7 @@ pub const Operation = enum { .bls_to_execution_change => "address_change", .execution_payload => "body", .parent_execution_payload => "block", + .voluntary_exit_churn => "voluntary_exit", .withdrawals => "execution_payload", else => @tagName(self), }; @@ -69,6 +71,7 @@ pub const Operation = enum { .proposer_slashing => "ProposerSlashing", .sync_aggregate => "SyncAggregate", .voluntary_exit => "SignedVoluntaryExit", + .voluntary_exit_churn => "SignedVoluntaryExit", .withdrawal_request => "WithdrawalRequest", .withdrawals => "ExecutionPayload", }; @@ -297,7 +300,7 @@ pub fn TestCase(comptime fork: ForkSeq, comptime operation: Operation) type { verify, ); }, - .voluntary_exit => { + .voluntary_exit, .voluntary_exit_churn => { const config = cached_state.config; const epoch_cache = cached_state.epoch_cache; try state_transition.processVoluntaryExit( From 32e4623697f8945ef0db3a12e93a3f5657d20b10 Mon Sep 17 00:00:00 2001 From: Rahul Guha <19rahul2003@gmail.com> Date: Wed, 22 Jul 2026 14:18:07 +0530 Subject: [PATCH 27/33] compute PTC from epoch shufflings --- .../epoch/process_ptc_window.zig | 20 ++++++------- src/state_transition/utils/gloas.zig | 29 ++++++++++++++++++- src/state_transition/utils/seed.zig | 25 ++++++---------- 3 files changed, 46 insertions(+), 28 deletions(-) diff --git a/src/state_transition/epoch/process_ptc_window.zig b/src/state_transition/epoch/process_ptc_window.zig index a3b0f5004..0d7f517b1 100644 --- a/src/state_transition/epoch/process_ptc_window.zig +++ b/src/state_transition/epoch/process_ptc_window.zig @@ -6,8 +6,7 @@ const EpochTransitionCache = @import("../cache/epoch_transition_cache.zig").Epoc const ct = @import("consensus_types"); const preset = @import("preset").preset; const computeEpochAtSlot = @import("../utils/epoch.zig").computeEpochAtSlot; -const computeStartSlotAtEpoch = @import("../utils/epoch.zig").computeStartSlotAtEpoch; -const computePtc = @import("../utils/gloas.zig").computePtc; +const computePayloadTimelinessCommitteesForEpoch = @import("../utils/seed.zig").computePayloadTimelinessCommitteesForEpoch; const ValidatorIndex = ct.primitive.ValidatorIndex.Type; /// Update the `ptc_window` field in the beacon state by shifting out the oldest epoch's @@ -32,18 +31,17 @@ pub fn processPtcWindow( } const next_epoch = computeEpochAtSlot(try state.slot()) + preset.MIN_SEED_LOOKAHEAD + 1; - const start_slot = computeStartSlotAtEpoch(next_epoch); const next_shuffling = try epoch_transition_cache.getNextShuffling(allocator, .gloas, state); - var next_epoch_payload_timeliness_committees: [preset.SLOTS_PER_EPOCH][preset.PTC_SIZE]ValidatorIndex = undefined; + const next_epoch_payload_timeliness_committees = try computePayloadTimelinessCommitteesForEpoch( + .gloas, + allocator, + state, + next_epoch, + &next_shuffling.committees, + epoch_cache.effective_balance_increments.get().items, + ); for (0..preset.SLOTS_PER_EPOCH) |slot_offset| { - next_epoch_payload_timeliness_committees[slot_offset] = try computePtc( - allocator, - state, - start_slot + slot_offset, - next_shuffling, - epoch_cache.effective_balance_increments.get().items, - ); try ptc_window.setValue( ptc_window_len - preset.SLOTS_PER_EPOCH + slot_offset, &next_epoch_payload_timeliness_committees[slot_offset], diff --git a/src/state_transition/utils/gloas.zig b/src/state_transition/utils/gloas.zig index 456b02c0a..5abfb8e02 100644 --- a/src/state_transition/utils/gloas.zig +++ b/src/state_transition/utils/gloas.zig @@ -325,12 +325,39 @@ pub fn initializePtcWindow( var ptcs: [(1 + preset.MIN_SEED_LOOKAHEAD) * preset.SLOTS_PER_EPOCH][ptc_size]ValidatorIndex = undefined; for (0..1 + preset.MIN_SEED_LOOKAHEAD) |epoch_offset| { const epoch = current_epoch + epoch_offset; + var owned_shuffling: ?*EpochShuffling = null; + defer if (owned_shuffling) |shuffling| shuffling.deinit(); + + const shuffling = epoch_cache.getShufflingAtEpochOrNull(epoch) orelse blk: { + var validators = try state.validators(); + const validator_count = try validators.length(); + var active_indices: std.ArrayList(ValidatorIndex) = .empty; + defer active_indices.deinit(allocator); + + for (0..validator_count) |i| { + var validator: ct.phase0.Validator.Type = undefined; + try validators.getValue(undefined, i, &validator); + if (isActiveValidator(&validator, epoch)) { + try active_indices.append(allocator, @intCast(i)); + } + } + + var any_state = @unionInit(AnyBeaconState, @tagName(fork), state.inner); + owned_shuffling = try computeEpochShuffling( + allocator, + &any_state, + try active_indices.toOwnedSlice(allocator), + epoch, + ); + break :blk owned_shuffling.?; + }; const epoch_committees = try computePayloadTimelinessCommitteesForEpoch( fork, allocator, state, epoch, - epoch_cache, + &shuffling.committees, + epoch_cache.effective_balance_increments.get().items, ); for (0..preset.SLOTS_PER_EPOCH) |slot_index| { ptcs[epoch_offset * preset.SLOTS_PER_EPOCH + slot_index] = epoch_committees[slot_index]; diff --git a/src/state_transition/utils/seed.zig b/src/state_transition/utils/seed.zig index 159544d6d..9ad8aad8f 100644 --- a/src/state_transition/utils/seed.zig +++ b/src/state_transition/utils/seed.zig @@ -155,36 +155,29 @@ pub fn computePayloadTimelinessCommitteesForEpoch( allocator: Allocator, state: *BeaconState(fork), epoch: Epoch, - epoch_cache: *const @import("../cache/epoch_cache.zig").EpochCache, + committees: []const []const []const ValidatorIndex, + effective_balance_increments: []const u16, ) ![preset.SLOTS_PER_EPOCH][preset.PTC_SIZE]ValidatorIndex { var epoch_seed: [32]u8 = undefined; try getSeed(fork, state, epoch, c.DOMAIN_PTC_ATTESTER, &epoch_seed); const start_slot = computeStartSlotAtEpoch(epoch); - var slot_seed_input: [40]u8 = undefined; @memcpy(slot_seed_input[0..32], &epoch_seed); var result: [preset.SLOTS_PER_EPOCH][preset.PTC_SIZE]ValidatorIndex = undefined; - for (0..preset.SLOTS_PER_EPOCH) |i| { - const slot = start_slot + i; - std.mem.writeInt(u64, slot_seed_input[32..][0..8], slot, .little); + std.mem.writeInt(u64, slot_seed_input[32..][0..8], start_slot + i, .little); var slot_seed: [32]u8 = undefined; Sha256.hash(&slot_seed_input, &slot_seed, .{}); - - const committees_per_slot = try epoch_cache.getCommitteeCountPerSlot(epoch); - const slot_committees = try allocator.alloc([]const ValidatorIndex, committees_per_slot); - defer allocator.free(slot_committees); - - for (0..committees_per_slot) |ci| { - slot_committees[ci] = try epoch_cache.getBeaconCommittee(slot, ci); - } - - result[i] = try computePayloadTimelinessCommitteeForSlot(allocator, &slot_seed, slot_committees, epoch_cache.effective_balance_increments.get().items); + result[i] = try computePayloadTimelinessCommitteeForSlot( + allocator, + &slot_seed, + committees[i], + effective_balance_increments, + ); } - return result; } From d58bfb971917599be8a9487754e86a91f9ce0cc2 Mon Sep 17 00:00:00 2001 From: Rahul Guha <19rahul2003@gmail.com> Date: Tue, 11 Aug 2026 17:39:06 +0530 Subject: [PATCH 28/33] merge main into gloas --- .github/workflows/CI.yml | 115 +- .github/workflows/lint-pr-title.yml | 2 +- .github/workflows/publish-bindings.yml | 46 +- .release-please-manifest.json | 3 + AGENTS.md | 350 +++++ CLAUDE.md | 101 -- bench/state_transition/process_block.zig | 63 +- bench/state_transition/process_epoch.zig | 48 +- bindings/napi/BeaconStateView.zig | 69 +- bindings/napi/blst.zig | 247 ++-- bindings/napi/config.zig | 4 +- bindings/napi/io.zig | 48 - bindings/napi/metrics.zig | 9 +- bindings/napi/pool.zig | 4 +- bindings/napi/pubkeys.zig | 295 ++-- bindings/napi/root.zig | 27 +- bindings/napi/stateTransition.zig | 6 + bindings/perf/loadState.test.ts | 12 +- bindings/src/blst.d.ts | 10 +- bindings/src/index.d.ts | 9 +- bindings/src/pubkeys.d.ts | 26 +- bindings/src/pubkeys.js | 20 +- bindings/src/state-transition.d.ts | 8 +- bindings/src/state-transition.js | 6 +- bindings/test/beaconStateView.test.ts | 128 +- bindings/test/blst.test.ts | 123 +- bindings/test/demo.ts | 29 +- bindings/test/pubkeys.test.ts | 201 ++- bindings/test/serializedState.ts | 29 + bindings/test/teardown.test.ts | 3 +- bindings/test/worker-isolation.test.ts | 79 +- build.zig.zon | 17 +- examples/metrics.ts | 34 +- examples/metrics.zig | 37 +- package.json | 5 +- pnpm-lock.yaml | 10 +- release-please-config.json | 70 + src/beacon_node/chain/root.zig | 5 + .../chain/state_cache/block_state_cache.zig | 917 ++++++++++++ .../chain/state_cache/cp_datastore.zig | 774 ++++++++++ src/beacon_node/chain/state_cache/key.zig | 55 + src/beacon_node/chain/state_cache/metrics.zig | 173 +++ src/beacon_node/chain/state_cache/root.zig | 31 + src/beacon_node/root.zig | 5 + src/bls/AggregatePublicKey.zig | 19 +- src/bls/AggregateSignature.zig | 32 +- src/bls/Pairing.zig | 15 +- src/bls/PublicKey.zig | 31 +- src/bls/SecretKey.zig | 27 +- src/bls/Signature.zig | 65 +- src/bls/ThreadPool.zig | 79 +- src/bls/error.zig | 17 +- src/bls/fast_verify.zig | 27 +- src/bls/root.zig | 4 + src/clock/Clock.zig | 483 ++++++ src/clock/EventClock.zig | 1298 ----------------- src/clock/SlotClock.zig | 412 ------ src/clock/clock_test.zig | 771 ++++++++++ src/clock/config.zig | 4 +- src/clock/root.zig | 25 +- src/clock/slot_math.zig | 213 ++- src/era/Reader.zig | 6 +- src/era/Writer.zig | 16 +- src/era/era.zig | 34 +- src/fork_types/any_beacon_state.zig | 10 + src/ssz/type/list.zig | 1 - .../block/is_valid_indexed_attestation.zig | 2 + .../block/process_attestation_altair.zig | 2 + .../block/process_attestation_phase0.zig | 2 + .../block/process_attestations.zig | 4 + .../block/process_attester_slashing.zig | 5 + src/state_transition/block/process_block.zig | 7 +- .../block/process_consolidation_request.zig | 19 +- .../block/process_deposit.zig | 15 +- .../block/process_operations.zig | 15 +- .../block/process_proposer_slashing.zig | 5 +- src/state_transition/block/process_randao.zig | 3 + .../block/process_sync_committee.zig | 52 +- .../block/process_voluntary_exit.zig | 15 +- .../block/process_withdrawal_request.zig | 5 +- src/state_transition/cache/epoch_cache.zig | 87 +- .../cache/epoch_transition_cache.zig | 7 +- src/state_transition/cache/pkix.zig | 262 ++++ src/state_transition/cache/pkix_test.zig | 348 +++++ src/state_transition/cache/pubkey_cache.zig | 532 ++++--- .../cache/pubkey_cache_test.zig | 258 ++++ src/state_transition/cache/state_cache.zig | 26 +- .../cache/sync_committee_cache.zig | 55 +- src/state_transition/epoch/process_epoch.zig | 2 +- .../epoch/process_pending_deposits.zig | 13 +- .../epoch/process_rewards_and_penalties.zig | 27 +- src/state_transition/root.zig | 14 +- .../signature_sets/indexed_attestation.zig | 41 +- .../signature_sets/proposer.zig | 16 +- .../signature_sets/proposer_slashings.zig | 12 +- .../signature_sets/randao.zig | 8 +- .../signature_sets/voluntary_exits.zig | 8 +- src/state_transition/ssz_bytes.zig | 55 + src/state_transition/state_transition.zig | 13 +- .../test_utils/generate_state.zig | 50 +- .../test_utils/interop_pubkeys.zig | 2 +- src/state_transition/utils/bls.zig | 9 +- src/state_transition/utils/electra.zig | 6 - src/state_transition/utils/signature_sets.zig | 78 +- test/int/era/root.zig | 2 +- test/spec/bls/test_case.zig | 25 +- test/spec/runner/epoch_processing.zig | 4 +- test/spec/runner/fork.zig | 2 +- test/spec/runner/operations.zig | 14 +- test/spec/runner/rewards.zig | 2 +- test/spec/runner/sanity.zig | 4 +- test/spec/runner/transition.zig | 2 +- 112 files changed, 6787 insertions(+), 3190 deletions(-) create mode 100644 .release-please-manifest.json create mode 100644 AGENTS.md delete mode 100644 CLAUDE.md delete mode 100644 bindings/napi/io.zig create mode 100644 bindings/napi/stateTransition.zig create mode 100644 bindings/test/serializedState.ts create mode 100644 release-please-config.json create mode 100644 src/beacon_node/chain/root.zig create mode 100644 src/beacon_node/chain/state_cache/block_state_cache.zig create mode 100644 src/beacon_node/chain/state_cache/cp_datastore.zig create mode 100644 src/beacon_node/chain/state_cache/key.zig create mode 100644 src/beacon_node/chain/state_cache/metrics.zig create mode 100644 src/beacon_node/chain/state_cache/root.zig create mode 100644 src/beacon_node/root.zig create mode 100644 src/clock/Clock.zig delete mode 100644 src/clock/EventClock.zig delete mode 100644 src/clock/SlotClock.zig create mode 100644 src/clock/clock_test.zig create mode 100644 src/state_transition/cache/pkix.zig create mode 100644 src/state_transition/cache/pkix_test.zig create mode 100644 src/state_transition/cache/pubkey_cache_test.zig diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index e4690cdb3..96171c49a 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -33,9 +33,9 @@ jobs: steps: - name: Checkout Repository - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - name: Install Zig - uses: mlugg/setup-zig@v2 + uses: mlugg/setup-zig@d1434d08867e3ee9daa34448df10607b98908d29 # v2.2.1 with: version: ${{ env.ZIG_VERSION }} cache-key: ${{ matrix.os }}-${{ env.ZIG_VERSION }} @@ -93,6 +93,9 @@ jobs: - name: Run clock tests run: | zig build test:clock + - name: Run beacon-node tests + run: | + zig build test:beacon_node - name: Build benchmarks run: | @@ -105,14 +108,14 @@ jobs: needs: build-test steps: - name: Checkout Repository - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - name: Install Zig - uses: mlugg/setup-zig@v2 + uses: mlugg/setup-zig@d1434d08867e3ee9daa34448df10607b98908d29 # v2.2.1 with: version: ${{ env.ZIG_VERSION }} cache-key: ubuntu-latest-${{ env.ZIG_VERSION }} - name: Restore spec tests cache - uses: actions/cache@v4 + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: fixtures/era key: era-files-${{ hashFiles('build.zig') }} # downloaded era files defined in build.zig @@ -130,9 +133,9 @@ jobs: spec-test-version: ${{ steps.spec-test-version.outputs.version }} steps: - name: Checkout Repository - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - name: Install Zig - uses: mlugg/setup-zig@v2 + uses: mlugg/setup-zig@d1434d08867e3ee9daa34448df10607b98908d29 # v2.2.1 with: version: ${{ env.ZIG_VERSION }} cache-key: ubuntu-latest-${{ env.ZIG_VERSION }} @@ -150,7 +153,7 @@ jobs: test -n "$version" echo "version=$version" >> "$GITHUB_OUTPUT" - name: Restore spec tests cache - uses: actions/cache@v4 + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: test/spec/spec_tests key: spec-test-data-${{ steps.spec-test-version.outputs.version }} @@ -170,7 +173,7 @@ jobs: run: | zig build run:write_bls_spec_tests - name: Upload generated test files - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: spec-test-generated path: | @@ -187,19 +190,19 @@ jobs: needs: [build-test, spec-test-prep] steps: - name: Checkout Repository - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - name: Install Zig - uses: mlugg/setup-zig@v2 + uses: mlugg/setup-zig@d1434d08867e3ee9daa34448df10607b98908d29 # v2.2.1 with: version: ${{ env.ZIG_VERSION }} cache-key: ubuntu-latest-${{ env.ZIG_VERSION }} - name: Restore spec tests cache - uses: actions/cache@v4 + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: test/spec/spec_tests key: spec-test-data-${{ needs['spec-test-prep'].outputs['spec-test-version'] }} - name: Download generated test files - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: spec-test-generated path: test/spec @@ -213,19 +216,19 @@ jobs: needs: [build-test, spec-test-prep] steps: - name: Checkout Repository - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - name: Install Zig - uses: mlugg/setup-zig@v2 + uses: mlugg/setup-zig@d1434d08867e3ee9daa34448df10607b98908d29 # v2.2.1 with: version: ${{ env.ZIG_VERSION }} cache-key: ubuntu-latest-${{ env.ZIG_VERSION }} - name: Restore spec tests cache - uses: actions/cache@v4 + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: test/spec/spec_tests key: spec-test-data-${{ needs['spec-test-prep'].outputs['spec-test-version'] }} - name: Download generated test files - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: spec-test-generated path: test/spec @@ -239,19 +242,19 @@ jobs: needs: [build-test, spec-test-prep] steps: - name: Checkout Repository - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - name: Install Zig - uses: mlugg/setup-zig@v2 + uses: mlugg/setup-zig@d1434d08867e3ee9daa34448df10607b98908d29 # v2.2.1 with: version: ${{ env.ZIG_VERSION }} cache-key: ubuntu-latest-${{ env.ZIG_VERSION }} - name: Restore spec tests cache - uses: actions/cache@v4 + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: test/spec/spec_tests key: spec-test-data-${{ needs['spec-test-prep'].outputs['spec-test-version'] }} - name: Download generated test files - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: spec-test-generated path: test/spec @@ -265,19 +268,19 @@ jobs: needs: [build-test, spec-test-prep] steps: - name: Checkout Repository - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - name: Install Zig - uses: mlugg/setup-zig@v2 + uses: mlugg/setup-zig@d1434d08867e3ee9daa34448df10607b98908d29 # v2.2.1 with: version: ${{ env.ZIG_VERSION }} cache-key: ubuntu-latest-${{ env.ZIG_VERSION }} - name: Restore spec tests cache - uses: actions/cache@v4 + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: test/spec/spec_tests key: spec-test-data-${{ needs['spec-test-prep'].outputs['spec-test-version'] }} - name: Download generated test files - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: spec-test-generated path: test/spec @@ -304,19 +307,19 @@ jobs: - merkle_proof steps: - name: Checkout Repository - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - name: Install Zig - uses: mlugg/setup-zig@v2 + uses: mlugg/setup-zig@d1434d08867e3ee9daa34448df10607b98908d29 # v2.2.1 with: version: ${{ env.ZIG_VERSION }} cache-key: ubuntu-latest-${{ env.ZIG_VERSION }} - name: Restore spec tests cache - uses: actions/cache@v4 + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: test/spec/spec_tests key: spec-test-data-${{ needs['spec-test-prep'].outputs['spec-test-version'] }} - name: Download generated test files - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: spec-test-generated path: test/spec @@ -343,19 +346,19 @@ jobs: - merkle_proof steps: - name: Checkout Repository - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - name: Install Zig - uses: mlugg/setup-zig@v2 + uses: mlugg/setup-zig@d1434d08867e3ee9daa34448df10607b98908d29 # v2.2.1 with: version: ${{ env.ZIG_VERSION }} cache-key: ubuntu-latest-${{ env.ZIG_VERSION }} - name: Restore spec tests cache - uses: actions/cache@v4 + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: test/spec/spec_tests key: spec-test-data-${{ needs['spec-test-prep'].outputs['spec-test-version'] }} - name: Download generated test files - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: spec-test-generated path: test/spec @@ -369,9 +372,9 @@ jobs: needs: build-test steps: - name: Checkout Repository - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - name: Install Zig - uses: mlugg/setup-zig@v2 + uses: mlugg/setup-zig@d1434d08867e3ee9daa34448df10607b98908d29 # v2.2.1 with: version: ${{ env.ZIG_VERSION }} cache-key: ubuntu-latest-${{ env.ZIG_VERSION }} @@ -400,16 +403,16 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout Repository - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - name: Setup pnpm - uses: pnpm/action-setup@v4 + uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4.3.0 - name: Setup Node.js - uses: actions/setup-node@v6 + uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0 with: node-version: ${{ env.NODE_VERSION }} cache: pnpm - name: Install Zig - uses: mlugg/setup-zig@v2 + uses: mlugg/setup-zig@d1434d08867e3ee9daa34448df10607b98908d29 # v2.2.1 with: version: ${{ env.ZIG_VERSION }} cache-key: ubuntu-latest-${{ env.ZIG_VERSION }} @@ -423,7 +426,7 @@ jobs: run: | zig build build-lib:bindings -Doptimize=ReleaseSafe - name: Restore era files cache - uses: actions/cache@v4 + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: fixtures/era key: era-files-${{ hashFiles('build.zig') }} # downloaded era files defined in build.zig @@ -434,15 +437,43 @@ jobs: run: | pnpm test + bindings-artifacts: + name: bindings - build artifacts + # Runs the exact multi-target build the Release workflow publishes + # (see publish-bindings.yml), so cross-compile breakage is caught + # pre-merge instead of after a release tag already exists. + runs-on: ubuntu-latest + steps: + - name: Checkout Repository + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + - name: Setup pnpm + uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4.3.0 + - name: Setup Node.js + uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0 + with: + node-version: ${{ env.NODE_VERSION }} + cache: pnpm + - name: Install Zig + uses: mlugg/setup-zig@d1434d08867e3ee9daa34448df10607b98908d29 # v2.2.1 + with: + version: ${{ env.ZIG_VERSION }} + cache-key: ubuntu-latest-${{ env.ZIG_VERSION }} + - name: Install dependencies + run: | + pnpm install --frozen-lockfile + - name: Build release artifacts + run: | + pnpm zapi build-artifacts --optimize ReleaseSafe + # metrics_stf depends on httpz, which is temporarily disabled for Zig 0.16 migration # build-examples: # name: build-examples # runs-on: ubuntu-latest # steps: # - name: Checkout Repository - # uses: actions/checkout@v4 + # uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 # - name: Install Zig - # uses: mlugg/setup-zig@v2 + # uses: mlugg/setup-zig@d1434d08867e3ee9daa34448df10607b98908d29 # v2.2.1 # with: # version: ${{ env.ZIG_VERSION }} # cache-key: ubuntu-latest-${{ env.ZIG_VERSION }} diff --git a/.github/workflows/lint-pr-title.yml b/.github/workflows/lint-pr-title.yml index 873343c8d..49270f091 100644 --- a/.github/workflows/lint-pr-title.yml +++ b/.github/workflows/lint-pr-title.yml @@ -14,7 +14,7 @@ jobs: name: Validate PR title runs-on: ubuntu-latest steps: - - uses: amannn/action-semantic-pull-request@v5 + - uses: amannn/action-semantic-pull-request@e32d7e603df1aa1ba07e981f2a23455dee596825 # v5 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: diff --git a/.github/workflows/publish-bindings.yml b/.github/workflows/publish-bindings.yml index 247e0193c..7e221c36f 100644 --- a/.github/workflows/publish-bindings.yml +++ b/.github/workflows/publish-bindings.yml @@ -1,14 +1,38 @@ -name: Publish NAPI Bindings +name: Release on: - workflow_dispatch: + push: + branches: + - main + +# Never run two release flows concurrently, and never cancel an in-flight +# run — cancelling between tag creation and npm publish leaves a +# half-released version. +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: false env: ZIG_VERSION: "0.16.0" NODE_VERSION: "24" jobs: + release-please: + name: release please + runs-on: ubuntu-latest + permissions: + contents: write + pull-requests: write + outputs: + release_created: ${{ steps.release.outputs.release_created }} + steps: + - name: Run release-please + id: release + uses: googleapis/release-please-action@5c625bfb5d1ff62eadeeb3772007f7f66fdcf071 # v4.4.1 + publish-bindings: name: publish bindings + needs: release-please + if: ${{ needs.release-please.outputs.release_created == 'true' }} # [Lodestar CI] still runs on Ubuntu 22.04, which ships an incompatible GLIBC version # compared to ubuntu-latest runners. # This [incompatibility] causes DLOPEN to fail when loading bindings on those runners. @@ -24,28 +48,25 @@ jobs: id-token: write steps: - name: Checkout Repository - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 - name: Setup pnpm - uses: pnpm/action-setup@v4 + uses: pnpm/action-setup@b906affcce14559ad1aafd4ab0e942779e9f58b1 # v4.3.0 - name: Setup Node.js - uses: actions/setup-node@v6 + uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0 with: node-version: ${{ env.NODE_VERSION }} registry-url: "https://registry.npmjs.org" cache: pnpm - name: Install Zig - uses: mlugg/setup-zig@v2 + uses: mlugg/setup-zig@d1434d08867e3ee9daa34448df10607b98908d29 # v2.2.1 with: version: ${{ env.ZIG_VERSION }} - cache-key: ubuntu-latest-${{ env.ZIG_VERSION }} + cache-key: ubuntu-22.04-${{ env.ZIG_VERSION }} - name: Install dependencies run: | - pnpm install - - name: Run bindings formatter - run: | - pnpm biome ci + pnpm install --frozen-lockfile - name: Determine npm dist tag - id: detect_dist_tag + id: detect_dist_tag run: | VERSION=$(node -p "require('./package.json').version") if [[ "$VERSION" == *-* ]]; then @@ -64,4 +85,3 @@ jobs: pnpm zapi prepublish rm -rf zig-out/lib pnpm zapi publish --access public -- --ignore-scripts --provenance ${{ steps.detect_dist_tag.outputs.tag_flag }} - diff --git a/.release-please-manifest.json b/.release-please-manifest.json new file mode 100644 index 000000000..001eb2cd1 --- /dev/null +++ b/.release-please-manifest.json @@ -0,0 +1,3 @@ +{ + ".": "0.1.2" +} diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 000000000..98ec03083 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,350 @@ +# AGENTS.md + +## Critical rules + +- **Target branch:** `main`. +- **Pre-push:** run `zig fmt --check .`, `pnpm lint`, and the relevant + binding or spec tests before every push. +- **Bound everything:** avoid recursion, put explicit limits on loops and allocations, fail fast, + and assert invariants. +- **No generated-file edits:** do not manually edit generated spec test files. Regenerate them with + the corresponding `zig build run:write_*` step. +- **Relative imports:** use the `.js` extension in TypeScript ESM imports. +- **No `any`:** avoid `any` and `as any`; use proper types or a justified Biome suppression. +- **Follow existing patterns** before introducing new abstractions. +- **Incremental commits:** after review starts, do not force-push unless a maintainer requests it. +- **Communication style:** do not use em dashes. Keep communication succinct and human-friendly. + +## Project overview + +Lodestar-z is a Zig library that provides consensus modules for +[Lodestar](https://github.com/ChainSafe/lodestar), the TypeScript Ethereum consensus client. It +implements performance-critical paths in Zig and exposes them to Node.js through NAPI bindings. +Major areas include: + +- SSZ serialization, Merkleization, and tree-backed views +- Ethereum consensus types and fork-aware state transition +- BLS cryptography and parallel verification +- Persistent Merkle trees and hashing +- Fork choice and beacon-node components +- ERA file handling +- Node.js bindings used by Lodestar + +Ethereum consensus spec tests are the source of truth for consensus behavior. + +## Directory structure + +```text +src/ + beacon_node/ # Beacon-node components + bls/ # BLS types, verification, and worker pool + clock/ # Slot and epoch clock + config/ # Runtime and network configuration + consensus_types/ # Fork-specific Ethereum consensus types + constants/ # Protocol constants + era/ # ERA file handling + fork_choice/ # Fork-choice implementation + fork_types/ # Type-erased, fork-aware wrappers + hashing/ # SHA-256 and Merkleization + persistent_merkle_tree/ # Structural sharing and lazy hashing + preset/ # Mainnet and minimal presets + ssz/ # SSZ types, serialization, and views + state_transition/ # Block, epoch, and slot processing + +bindings/ + napi/ # Zig NAPI implementation + src/ # JavaScript and TypeScript public API + test/ # Binding tests + +bench/ # Zig and binding benchmarks +test/ + fuzz/ # Fuzz harnesses and corpora + int/ # Integration tests + spec/ # Consensus, SSZ, and BLS spec tests +examples/ # Example programs +scripts/ # Zig maintenance and download tools +``` + +## Build commands + +The repository uses Zig 0.16.0 and pnpm 10.x. + +### Zig library + +```bash +# Build all default artifacts +zig build + +# Check Zig formatting +zig fmt --check . + +# Format Zig files +zig fmt . + +# Run all unit tests +zig build test + +# Run one module's tests +zig build test:ssz +zig build test:bls +zig build test:state_transition + +# Filter a module test +zig build test:ssz -Dtest:ssz.filters="test name" + +# Filter the aggregate test step +zig build test -- --test-filter "test name" +``` + +Prefer a targeted module test while iterating, do not run `zig build test` unless necessary. + +### Spec tests + +```bash +# Download vectors pinned by build.zig.zon +zig build run:download_spec_tests + +# Generate test sources +zig build run:write_spec_tests +zig build run:write_ssz_generic_spec_tests +zig build run:write_ssz_static_spec_tests +zig build run:write_bls_spec_tests + +# Run generated suites +zig build test:spec_tests -Dpreset=minimal +zig build test:ssz_generic_spec_tests -Dpreset=minimal +zig build test:ssz_static_spec_tests -Dpreset=minimal +zig build test:bls_spec_tests -Dpreset=minimal + +# Filter a suite +zig build test:spec_tests -Dtest:spec_tests.filters="pattern" -Dpreset=minimal +``` + +Use the minimal preset for faster iteration, but run mainnet when behavior depends on preset values. +Do not assume minimal and mainnet constants are interchangeable. + +### JavaScript and TypeScript bindings + +```bash +# Install dependencies +pnpm install + +# Build bindings +zig build build-lib:bindings +zig build build-lib:bindings -Doptimize=ReleaseSafe + +# Build for a specific preset through package scripts +pnpm prepare-mainnet +pnpm prepare-minimal + +# Run binding tests +pnpm test + +# Run Biome +pnpm lint +pnpm exec biome check --write . +``` + +Binding tests require a compatible `zig-out/lib/bindings.node`; rebuild it after changing Zig NAPI +code or the selected preset. + +### Integration tests and benchmarks + +```bash +# ERA-backed integration tests +zig build run:download_era_files +zig build test:int + +# Representative benchmarks +zig build run:bench_ssz_attestation -Doptimize=ReleaseSafe +zig build run:bench_ssz_block -Doptimize=ReleaseSafe +zig build run:bench_ssz_state -Doptimize=ReleaseSafe +zig build run:bench_hashing -Doptimize=ReleaseSafe +zig build run:bench_merkle_node -Doptimize=ReleaseSafe +zig build run:bench_merkle_gindex -Doptimize=ReleaseSafe +zig build run:bench_process_block -Doptimize=ReleaseSafe +zig build run:bench_process_epoch -Doptimize=ReleaseSafe +``` + +## Code style + +### Zig + +The project style guide is `.gemini/styleguide.md`, a Lodestar-specific adaptation of TigerStyle. +Important requirements include: + +- Safety, performance, and developer experience, in that order. +- No recursion. Put a fixed upper bound on work and memory. +- Assert preconditions, postconditions, and invariants. Test both valid and invalid boundaries. +- Handle every error and clean up partially initialized resources. +- Prefer static or startup allocation where appropriate, but do not avoid dynamic allocation + blindly. +- Use explicitly sized integer types for persisted and protocol values. +- Pass values larger than 16 bytes by `*const` when copying is not intended. +- Construct large objects in place where practical. +- Keep allocation and matching cleanup together, separated from surrounding logic by blank lines. +- Explain why a non-obvious design or safety decision is correct. +- Run `zig fmt` on every Zig change. + +### JavaScript and TypeScript + +Bindings use ES modules and Biome: + +- Use double quotes and named exports. +- Use `.js` extensions for relative TypeScript imports. +- Use `camelCase` for functions and variables, `PascalCase` for types and classes, and + `UPPER_SNAKE_CASE` for constants. +- Avoid `any` and `as any`. If unavoidable, add a suppression with the full rule and rationale. +- Keep declarations in `bindings/src/*.d.ts` synchronized with JavaScript wrappers and NAPI exports. +- Do not edit native binaries under `zig-out/`; they are build outputs. + +## Architecture patterns + +### Fork-aware code + +Fork progression is: + +`phase0` -> `altair` -> `bellatrix` -> `capella` -> `deneb` -> `electra` -> `fulu` -> +`gloas` -> `heze` + +Keep fork-specific types and logic in the corresponding namespace or module. Use existing fork +guards and `Any*` wrappers instead of unchecked casts. Changes to a fork generally apply to later +forks unless the consensus specification explicitly overrides them. + +### Presets and configuration + +- Compile-time protocol values live in `src/preset/` and `src/constants/`. +- Runtime chain configuration lives in `src/config/`. +- Both mainnet and minimal must compile and pass their relevant tests. +- Size stack buffers and fixed collections from protocol constants only when the input is guaranteed + to have that protocol bound. Public binding inputs need explicit validation or a dynamic fallback. + +### SSZ types + +SSZ definitions are compile-time namespaces that expose the value type, default value, +serialization, deserialization, hash-tree-root, and view operations. Match the existing fixed versus +variable type distinction and preserve canonical SSZ limits. For consensus changes, update types, +fork wrappers, and spec test generation together as needed. + +### Memory and ownership + +Zig code must make allocator and ownership boundaries explicit: + +- Pair every allocation with `defer` or `errdefer` immediately. +- Clean up all successfully initialized elements when initialization fails partway through. +- Do not free through a different allocator than the one used to allocate. +- Treat NAPI values and slices as scoped to the environment and callback rules documented by zapi. +- Avoid retaining pointers into growable collections across operations that may reallocate them. +- Prefer bounded stack buffers only when their worst-case size is safe for every calling context. + +### NAPI bindings + +- Zig exports are implemented under `bindings/napi/`. +- Public JavaScript wrappers and declarations live under `bindings/src/`. +- Keep errors at the boundary specific and stable. +- Validate untrusted JavaScript lengths, indexes, and byte encodings before native access. +- Account for worker teardown and Node.js worker isolation when changing global native state. + +## Testing guidelines + +- Tests must be deterministic and bounded. +- Add regression tests for bug fixes where the failure can be reproduced reliably. +- Test positive and negative boundaries, especially lengths, indexes, serialization offsets, and + partial initialization. +- Use `std.testing.allocator` in unit tests to detect leaks. +- Prefer exact error assertions over accepting any failure. +- Run the narrowest relevant test during development and the complete affected module before push. +- Run consensus spec tests for consensus types, SSZ behavior, or state-transition changes. +- Run both minimal and mainnet tests when preset-dependent limits or behavior change. +- Build and run binding tests for changes under `bindings/` or exported native APIs. +- Fuzz parsers, deserializers, and cryptographic boundaries when introducing new input shapes. + +## Pull request guidelines + +### Branches and commits + +Create branches from `main`. Use Conventional Commit messages: + +- `feat:` new functionality +- `fix:` bug fixes +- `refactor:` behavior-preserving code changes +- `perf:` performance improvements +- `test:` test-only changes +- `docs:` documentation changes +- `chore:` maintenance + +Keep commits focused. After review begins, add incremental commits rather than rewriting history +unless a maintainer asks otherwise. + +### AI assistance disclosure + +Disclose AI assistance in the PR description, following Lodestar's convention. State whether the +implementation was primarily AI-authored or whether AI was used only for codebase exploration. + +### Pre-push checklist + +1. `zig fmt --check .` +2. Run relevant tests, never run slow `zig build test` unless changes touch spec logic +3. Relevant spec tests for consensus, SSZ, or BLS changes +4. `pnpm lint` for binding source changes +5. Rebuild bindings and run `pnpm test` for binding or NAPI changes +6. Confirm no generated test source or build output was edited manually +7. Confirm both ownership cleanup and error paths are covered + +## Common tasks + +### Fixing a bug + +1. Add a failing regression test when practical. +2. Identify the violated invariant or missing boundary check. +3. Fix the bug and all partial-failure cleanup paths. +4. Run the affected module tests. +5. Run spec or binding tests when the changed boundary requires them. + +### Adding or changing a consensus type + +1. Update the type in the relevant `src/consensus_types/.zig` file. +2. Propagate it through later forks and `src/fork_types/` where required. +3. Update state-transition logic that consumes the type. +4. Regenerate the relevant spec tests. +5. Run minimal and mainnet spec suites. + +### Adding a NAPI API + +1. Implement the native export under `bindings/napi/`. +2. Register it through the existing zapi export pattern. +3. Add or update the wrapper and declaration under `bindings/src/`. +4. Validate all JavaScript-controlled inputs at the native boundary. +5. Add tests under `bindings/test/`. +6. Build ReleaseSafe bindings and run `pnpm test` and `pnpm lint`. + +### Changing SSZ behavior + +1. Locate the fixed, variable, serialized, and tree-view paths affected by the change. +2. Preserve canonical limits and offset validation. +3. Add unit tests for valid values and malformed boundary cases. +4. Regenerate and run generic or static SSZ spec tests. +5. Fuzz the changed parser or deserializer when applicable. + +## Implementing consensus specifications + +The primary reference is the +[Ethereum consensus-specs repository](https://github.com/ethereum/consensus-specs). The test version +is pinned in `build.zig.zon`; reference that exact version when implementation details differ from +upstream `master`. + +Typical mappings are: + +| Consensus specification area | Lodestar-z location | +| --- | --- | +| SSZ containers and fork types | `src/consensus_types/`, `src/fork_types/` | +| Block processing | `src/state_transition/block/` | +| Epoch processing | `src/state_transition/epoch/` | +| Slot processing | `src/state_transition/slot/` | +| Fork choice | `src/fork_choice/` | +| Hashing and Merkleization | `src/hashing/`, `src/persistent_merkle_tree/`, `src/ssz/` | +| Node.js integration | `bindings/napi/`, `bindings/src/` | + +When porting Lodestar behavior, preserve consensus semantics rather than TypeScript implementation +details. Use the spec vectors to verify equivalence and document intentional differences caused by +Zig ownership, bounded memory, or NAPI constraints. diff --git a/CLAUDE.md b/CLAUDE.md deleted file mode 100644 index d1d26bbaa..000000000 --- a/CLAUDE.md +++ /dev/null @@ -1,101 +0,0 @@ -# CLAUDE.md - -Lodestar-z is a Zig library providing consensus modules for [Lodestar](https://github.com/ChainSafe/lodestar) — the TypeScript Ethereum consensus client. It implements performance-critical paths (SSZ, hashing, state transition) in Zig for use via NAPI bindings. - -## Quick Reference - -### Zig (core library) - -```bash -# Build -zig build - -# Run all unit tests -zig build test - -# Run specific test -zig build test -- --test-filter "test name" - -# Run specific module test with filter -zig build test:ssz -Dtest:ssz.filters="your filter" -``` - -### Spec Tests - -```bash -# 1. Download spec test vectors -zig build run:download_spec_tests - -# 2. Generate spec test code -zig build run:write_spec_tests -zig build run:write_ssz_generic_spec_tests -zig build run:write_ssz_static_spec_tests - -# 3. Run spec tests (use -Dpreset=minimal for faster runs) -zig build test:spec_tests -Dpreset=minimal -zig build test:ssz_generic_spec_tests -Dpreset=minimal -zig build test:ssz_static_spec_tests -Dpreset=minimal - -# Run with filter -zig build test:spec_tests -Dtest:spec_tests.filters="your filter" -``` - -### Benchmarks - -```bash -# Download ERA files first (required for some benchmarks) -zig build run:download_era_files - -# Run specific benchmarks -zig build run:bench_ssz_attestation -zig build run:bench_ssz_block -zig build run:bench_ssz_state -zig build run:bench_hashing -zig build run:bench_merkle_node -zig build run:bench_merkle_gindex -``` - -### JS/TS Bindings - -```bash -# Install JS dependencies -pnpm i - -# Build NAPI bindings -zig build build-lib:bindings # debug -zig build build-lib:bindings -Doptimize=ReleaseSafe # release - -# Run JS/TS tests -pnpm test - -# Lint JS/TS -pnpm biome check -``` - -## Project Structure - -``` -src/ -├── config/ # Network configuration -├── consensus_types/ # Ethereum consensus types (phase0 → electra) -├── constants/ # Protocol constants -├── era/ # ERA file handling -├── fork_types/ # Per-fork type definitions -├── hashing/ # SHA-256, zero-hash tree, Merkleization -├── persistent_merkle_tree/ # Structural sharing, lazy hashing -├── preset/ # Consensus presets (mainnet, minimal) -├── ssz/ # SSZ serialization, views, containers -└── state_transition/ # Beacon state transition functions - -bindings/ # NAPI bindings for Node.js integration -bench/ # Performance benchmarks -test/ # Integration and spec tests -``` - -## Key Conventions - -- **Style:** Follow the project style guide at `.gemini/styleguide.md` (modified [TigerStyle](https://github.com/tigerbeetle/tigerbeetle/blob/main/docs/TIGER_STYLE.md)) -- **Safety first:** No recursion, limits on everything, fail-fast, zero technical debt -- **Fork order:** phase0 → altair → bellatrix → capella → deneb → electra → fulu → gloas → heze -- **SSZ types:** Defined as compile-time type definitions returning namespaces of operations -- **Testing:** Ethereum consensus spec tests are the source of truth diff --git a/bench/state_transition/process_block.zig b/bench/state_transition/process_block.zig index 6e9588495..0fe6e7cd3 100644 --- a/bench/state_transition/process_block.zig +++ b/bench/state_transition/process_block.zig @@ -24,7 +24,6 @@ const ValidatorIndex = types.primitive.ValidatorIndex.Type; const Withdrawals = types.capella.Withdrawals.Type; const WithdrawalsResult = state_transition.WithdrawalsResult; const BlockExternalData = state_transition.BlockExternalData; -const Index2PubkeyCache = state_transition.Index2PubkeyCache; const slotFromStateBytes = @import("utils.zig").slotFromStateBytes; const loadState = @import("utils.zig").loadState; const loadBlock = @import("utils.zig").loadBlock; @@ -113,12 +112,14 @@ fn ProcessRandaoBench(comptime fork: ForkSeq, comptime opts: BenchOpts) type { return struct { block: *const BeaconBlock(.full, fork), body: *const BeaconBlockBody(.full, fork), + io: std.Io, pub fn run(self: *@This(), allocator: std.mem.Allocator) void { _ = allocator; state_transition.processRandao( fork, + self.io, BenchState.cloned_cached_state.config, BenchState.cloned_cached_state.epoch_cache, BenchState.cloned_cached_state.state.castToFork(fork), @@ -150,11 +151,13 @@ fn ProcessEth1DataBench(comptime fork: ForkSeq) type { fn ProcessOperationsBench(comptime fork: ForkSeq, comptime opts: BenchOpts) type { return struct { body: *const BeaconBlockBody(.full, fork), + io: std.Io, pub fn run(self: *@This(), allocator: std.mem.Allocator) void { state_transition.processOperations( fork, allocator, + self.io, BenchState.cloned_cached_state.config, BenchState.cloned_cached_state.epoch_cache, BenchState.cloned_cached_state.state.castToFork(fork), @@ -170,11 +173,13 @@ fn ProcessOperationsBench(comptime fork: ForkSeq, comptime opts: BenchOpts) type fn ProcessSyncAggregateBench(comptime fork: ForkSeq, comptime opts: BenchOpts) type { return struct { body: *const BeaconBlockBody(.full, fork), + io: std.Io, pub fn run(self: *@This(), allocator: std.mem.Allocator) void { state_transition.processSyncAggregate( fork, allocator, + self.io, BenchState.cloned_cached_state.config, BenchState.cloned_cached_state.epoch_cache, BenchState.cloned_cached_state.state.castToFork(fork), @@ -188,12 +193,14 @@ fn ProcessSyncAggregateBench(comptime fork: ForkSeq, comptime opts: BenchOpts) t fn ProcessBlockBench(comptime fork: ForkSeq, comptime opts: BenchOpts) type { return struct { block: *const BeaconBlock(.full, fork), + io: std.Io, pub fn run(self: *@This(), allocator: std.mem.Allocator) void { const external_data = BlockExternalData{ .execution_payload_status = .valid, .data_availability_status = .available }; state_transition.processBlock( fork, allocator, + self.io, BenchState.cloned_cached_state.config, BenchState.cloned_cached_state.epoch_cache, BenchState.cloned_cached_state.state.castToFork(fork), @@ -212,12 +219,14 @@ fn ProcessBlockBench(comptime fork: ForkSeq, comptime opts: BenchOpts) type { fn ProcessBlockRootBench(comptime fork: ForkSeq, comptime opts: BenchOpts) type { return struct { block: *const BeaconBlock(.full, fork), + io: std.Io, pub fn run(self: *@This(), allocator: std.mem.Allocator) void { const external_data = BlockExternalData{ .execution_payload_status = .valid, .data_availability_status = .available }; state_transition.processBlock( fork, allocator, + self.io, BenchState.cloned_cached_state.config, BenchState.cloned_cached_state.epoch_cache, BenchState.cloned_cached_state.state.castToFork(fork), @@ -355,6 +364,7 @@ fn ProcessBlockSegmentedBench(comptime fork: ForkSeq) type { const randao_start = time.start(io); state_transition.processRandao( fork, + io, BenchState.cloned_cached_state.config, epoch_cache, state, @@ -377,6 +387,7 @@ fn ProcessBlockSegmentedBench(comptime fork: ForkSeq) type { state_transition.processOperations( fork, allocator, + io, BenchState.cloned_cached_state.config, epoch_cache, state, @@ -392,6 +403,7 @@ fn ProcessBlockSegmentedBench(comptime fork: ForkSeq) type { state_transition.processSyncAggregate( fork, allocator, + io, BenchState.cloned_cached_state.config, epoch_cache, state, @@ -498,25 +510,20 @@ fn runBenchmark( const beacon_config = config.BeaconConfig.init(chain_config, (try beacon_state.?.genesisValidatorsRoot()).*); - var pubkey_index_map = state_transition.PubkeyIndexMap.init(allocator); - defer pubkey_index_map.deinit(); - - const index_pubkey_cache = try allocator.create(state_transition.Index2PubkeyCache); - index_pubkey_cache.* = Index2PubkeyCache.empty; - defer { - index_pubkey_cache.deinit(allocator); - allocator.destroy(index_pubkey_cache); - } - - const validators = try beacon_state.?.validatorsPtrSlice(allocator); - defer allocator.free(validators); - - try state_transition.syncPubkeys(allocator, validators, &pubkey_index_map, index_pubkey_cache); + var pubkey_cache = try state_transition.PubkeyCache.initCapacity( + allocator, + io, + try std.math.add( + usize, + try beacon_state.?.validatorsCount(), + preset.MAX_DEPOSITS, + ), + ); + defer pubkey_cache.deinit(); - const cached_state = try CachedBeaconState.createCachedBeaconState(allocator, beacon_state.?, .{ + const cached_state = try CachedBeaconState.createCachedBeaconState(allocator, io, beacon_state.?, .{ .config = &beacon_config, - .index_to_pubkey = index_pubkey_cache, - .pubkey_to_index = &pubkey_index_map, + .pubkey_cache = &pubkey_cache, }, .{ .skip_sync_committee_cache = !comptime fork.gte(.altair), .skip_sync_pubkeys = false }); BenchState.init(allocator, cached_state); beacon_state = null; @@ -552,22 +559,22 @@ fn runBenchmark( try bench.addParam("execution_payload", &ProcessExecutionPayloadBench(fork){ .body = body }, .{ .hooks = hooks }); } - try bench.addParam("randao", &ProcessRandaoBench(fork, .{ .verify_signature = true }){ .block = block, .body = body }, .{ .hooks = hooks }); - try bench.addParam("randao_no_sig", &ProcessRandaoBench(fork, .{ .verify_signature = false }){ .block = block, .body = body }, .{ .hooks = hooks }); + try bench.addParam("randao", &ProcessRandaoBench(fork, .{ .verify_signature = true }){ .block = block, .body = body, .io = io }, .{ .hooks = hooks }); + try bench.addParam("randao_no_sig", &ProcessRandaoBench(fork, .{ .verify_signature = false }){ .block = block, .body = body, .io = io }, .{ .hooks = hooks }); try bench.addParam("eth1_data", &ProcessEth1DataBench(fork){ .body = body }, .{ .hooks = hooks }); - try bench.addParam("operations", &ProcessOperationsBench(fork, .{ .verify_signature = true }){ .body = body }, .{ .hooks = hooks }); - try bench.addParam("operations_no_sig", &ProcessOperationsBench(fork, .{ .verify_signature = false }){ .body = body }, .{ .hooks = hooks }); + try bench.addParam("operations", &ProcessOperationsBench(fork, .{ .verify_signature = true }){ .body = body, .io = io }, .{ .hooks = hooks }); + try bench.addParam("operations_no_sig", &ProcessOperationsBench(fork, .{ .verify_signature = false }){ .body = body, .io = io }, .{ .hooks = hooks }); if (comptime fork.gte(.altair)) { - try bench.addParam("sync_aggregate", &ProcessSyncAggregateBench(fork, .{ .verify_signature = true }){ .body = body }, .{ .hooks = hooks }); - try bench.addParam("sync_aggregate_no_sig", &ProcessSyncAggregateBench(fork, .{ .verify_signature = false }){ .body = body }, .{ .hooks = hooks }); + try bench.addParam("sync_aggregate", &ProcessSyncAggregateBench(fork, .{ .verify_signature = true }){ .body = body, .io = io }, .{ .hooks = hooks }); + try bench.addParam("sync_aggregate_no_sig", &ProcessSyncAggregateBench(fork, .{ .verify_signature = false }){ .body = body, .io = io }, .{ .hooks = hooks }); } - try bench.addParam("process_block", &ProcessBlockBench(fork, .{ .verify_signature = true }){ .block = block }, .{ .hooks = hooks }); - try bench.addParam("process_block_no_sig", &ProcessBlockBench(fork, .{ .verify_signature = false }){ .block = block }, .{ .hooks = hooks }); + try bench.addParam("process_block", &ProcessBlockBench(fork, .{ .verify_signature = true }){ .block = block, .io = io }, .{ .hooks = hooks }); + try bench.addParam("process_block_no_sig", &ProcessBlockBench(fork, .{ .verify_signature = false }){ .block = block, .io = io }, .{ .hooks = hooks }); - try bench.addParam("process_block+root", &ProcessBlockRootBench(fork, .{ .verify_signature = true }){ .block = block }, .{ .hooks = hooks }); - try bench.addParam("process_block+root_no_sig", &ProcessBlockRootBench(fork, .{ .verify_signature = false }){ .block = block }, .{ .hooks = hooks }); + try bench.addParam("process_block+root", &ProcessBlockRootBench(fork, .{ .verify_signature = true }){ .block = block, .io = io }, .{ .hooks = hooks }); + try bench.addParam("process_block+root_no_sig", &ProcessBlockRootBench(fork, .{ .verify_signature = false }){ .block = block, .io = io }, .{ .hooks = hooks }); // // Segmented benchmark (step-by-step timing) resetSegmentStats(); diff --git a/bench/state_transition/process_epoch.zig b/bench/state_transition/process_epoch.zig index 7bb4bc8aa..5ba577366 100644 --- a/bench/state_transition/process_epoch.zig +++ b/bench/state_transition/process_epoch.zig @@ -17,11 +17,11 @@ const AnyBeaconState = @import("fork_types").AnyBeaconState; const ForkSeq = config.ForkSeq; const CachedBeaconState = state_transition.CachedBeaconState; const EpochTransitionCache = state_transition.EpochTransitionCache; -const Index2PubkeyCache = state_transition.Index2PubkeyCache; const ValidatorIndex = types.primitive.ValidatorIndex.Type; const slotFromStateBytes = @import("utils.zig").slotFromStateBytes; const loadState = @import("utils.zig").loadState; const BenchState = @import("utils.zig").BenchState; +const preset = state_transition.preset; fn ProcessJustificationAndFinalizationBench(comptime fork: ForkSeq) type { return struct { @@ -161,6 +161,7 @@ fn ProcessEth1DataResetBench(comptime fork: ForkSeq) type { fn ProcessPendingDepositsBench(comptime fork: ForkSeq) type { return struct { epoch_transition_cache: *EpochTransitionCache, + io: std.Io, pub fn run(self: *@This(), allocator: std.mem.Allocator) void { const cache = self.epoch_transition_cache; @@ -168,6 +169,7 @@ fn ProcessPendingDepositsBench(comptime fork: ForkSeq) type { state_transition.processPendingDeposits( fork, allocator, + self.io, BenchState.cloned_cached_state.config, BenchState.cloned_cached_state.epoch_cache, BenchState.cloned_cached_state.state.castToFork(fork), @@ -199,6 +201,11 @@ fn ProcessEffectiveBalanceUpdatesBench(comptime fork: ForkSeq) type { return struct { epoch_transition_cache: *EpochTransitionCache, + pub fn beforeEach() void { + BenchState.beforeEach(); + BenchState.cloned_cached_state.epoch_cache.beforeEpochTransition() catch unreachable; + } + pub fn run(self: *@This(), allocator: std.mem.Allocator) void { const cache = self.epoch_transition_cache; @@ -490,6 +497,7 @@ fn ProcessEpochSegmentedBench(comptime fork: ForkSeq) type { state_transition.processPendingDeposits( fork, allocator, + io, BenchState.cloned_cached_state.config, epoch_cache, fork_state, @@ -658,7 +666,7 @@ fn runBenchmark( state_bytes: []const u8, chain_config: config.ChainConfig, ) !void { - defer state_transition.deinitStateTransition(io); + defer state_transition.deinitReusedEpochTransitionCache(io); var beacon_state: ?*AnyBeaconState = try loadState(fork, allocator, pool, state_bytes); defer if (beacon_state) |state| { @@ -673,28 +681,23 @@ fn runBenchmark( const beacon_config = config.BeaconConfig.init(chain_config, (try beacon_state.?.genesisValidatorsRoot()).*); - var pubkey_index_map = state_transition.PubkeyIndexMap.init(allocator); - defer pubkey_index_map.deinit(); - - const index_pubkey_cache = try allocator.create(state_transition.Index2PubkeyCache); - index_pubkey_cache.* = Index2PubkeyCache.empty; - defer { - index_pubkey_cache.deinit(allocator); - allocator.destroy(index_pubkey_cache); - } - - const validators = try beacon_state.?.validatorsPtrSlice(allocator); - defer allocator.free(validators); - - try state_transition.syncPubkeys(allocator, validators, &pubkey_index_map, index_pubkey_cache); + var pubkey_cache = try state_transition.PubkeyCache.initCapacity( + allocator, + io, + try std.math.add( + usize, + try beacon_state.?.validatorsCount(), + preset.MAX_PENDING_DEPOSITS_PER_EPOCH, + ), + ); + defer pubkey_cache.deinit(); const immutable_data = state_transition.EpochCacheImmutableData{ .config = &beacon_config, - .index_to_pubkey = index_pubkey_cache, - .pubkey_to_index = &pubkey_index_map, + .pubkey_cache = &pubkey_cache, }; - const cached_state = try CachedBeaconState.createCachedBeaconState(allocator, beacon_state.?, immutable_data, .{ + const cached_state = try CachedBeaconState.createCachedBeaconState(allocator, io, beacon_state.?, immutable_data, .{ .skip_sync_committee_cache = !comptime fork.gte(.altair), .skip_sync_pubkeys = false, }); @@ -718,6 +721,10 @@ fn runBenchmark( try stdout.print("\nStarting process_epoch benchmarks for {s} fork...\n\n", .{@tagName(fork)}); const hooks: zbench.Hooks = .{ .before_each = BenchState.beforeEach, .after_each = BenchState.afterEach }; + const effective_balance_hooks: zbench.Hooks = .{ + .before_each = ProcessEffectiveBalanceUpdatesBench(fork).beforeEach, + .after_each = BenchState.afterEach, + }; var bench = zbench.Benchmark.init(allocator, .{ .iterations = 50 }); defer bench.deinit(); @@ -754,6 +761,7 @@ fn runBenchmark( if (comptime fork.gte(.electra)) { try bench.addParam("pending_deposits", &ProcessPendingDepositsBench(fork){ .epoch_transition_cache = &epoch_transition_cache, + .io = io, }, .{ .hooks = hooks }); try bench.addParam("pending_consolidations", &ProcessPendingConsolidationsBench(fork){ @@ -763,7 +771,7 @@ fn runBenchmark( try bench.addParam("effective_balance_updates", &ProcessEffectiveBalanceUpdatesBench(fork){ .epoch_transition_cache = &epoch_transition_cache, - }, .{ .hooks = hooks }); + }, .{ .hooks = effective_balance_hooks }); try bench.addParam("slashings_reset", &ProcessSlashingsResetBench(fork){ .epoch_transition_cache = &epoch_transition_cache, diff --git a/bindings/napi/BeaconStateView.zig b/bindings/napi/BeaconStateView.zig index ceb873020..95b23d2e6 100644 --- a/bindings/napi/BeaconStateView.zig +++ b/bindings/napi/BeaconStateView.zig @@ -16,7 +16,6 @@ const pubkey = @import("./pubkeys.zig"); const js_types = @import("./js_types.zig"); const sszValueToNapiValue = @import("./to_napi_value.zig").sszValueToNapiValue; const numberSliceToNapiValue = @import("./to_napi_value.zig").numberSliceToNapiValue; -const napi_io = @import("./io.zig"); /// Allocator used for all BeaconStateView instances. var gpa: std.heap.DebugAllocator(.{}) = .init; @@ -72,7 +71,7 @@ pub const js_meta = js.class(.{ .properties = .{ } }); cached_state: ?*CachedBeaconState = null, -pool_rc: ?*pool.PoolRc = null, +pool_rc: @TypeOf(pool.state.pool_rc) = null, const BeaconStateView = @This(); pub fn init() BeaconStateView { @@ -91,6 +90,23 @@ pub fn deinit(self: *BeaconStateView) void { } } +fn initCachedState( + cached_state: *CachedBeaconState, + io: std.Io, + state: *AnyBeaconState, +) !void { + try cached_state.init( + allocator, + io, + state, + .{ + .config = &config.state.config, + .pubkey_cache = &pubkey.state.cache, + }, + null, + ); +} + // ------------------------- // Class Methods // ------------------------- @@ -107,16 +123,8 @@ pub fn createFromBytes(bytes: js.Uint8Array) !BeaconStateView { const cached_state = try allocator.create(CachedBeaconState); errdefer allocator.destroy(cached_state); - try cached_state.init( - allocator, - state, - .{ - .config = &config.state.config, - .index_to_pubkey = &pubkey.state.index2pubkey, - .pubkey_to_index = &pubkey.state.pubkey2index, - }, - null, - ); + const io = js.io(); + try initCachedState(cached_state, io, state); return .{ .cached_state = cached_state, @@ -723,6 +731,13 @@ pub fn getAllValidators(self: *const BeaconStateView) !js.Array { return js_types.wrap(js.Array, result); } +/// Get the number of builders in the registry (Gloas+). Throws on pre-Gloas states. +pub fn getBuildersLength(self: *const BeaconStateView) !js.Number { + const cached_state = try self.requireState(); + const count = try cached_state.state.buildersLength(); + return js.Number.from(count); +} + /// Get all balances in the registry. pub fn getAllBalances(self: *const BeaconStateView) !js.Array { const env = js.env(); @@ -875,6 +890,7 @@ pub fn getVoluntaryExitValidity(self: *const BeaconStateView, signed_exit_value: const result = switch (cached_state.state.forkSeq()) { inline else => |f| st.getVoluntaryExitValidity( f, + js.io(), cached_state.config, cached_state.epoch_cache, cached_state.state.castToFork(f), @@ -901,6 +917,7 @@ pub fn isValidVoluntaryExit(self: *const BeaconStateView, signed_exit_value: js. const result = switch (cached_state.state.forkSeq()) { inline else => |f| st.isValidVoluntaryExit( f, + js.io(), cached_state.config, cached_state.epoch_cache, cached_state.state.castToFork(f), @@ -1037,7 +1054,7 @@ pub fn createMultiProof(self: *const BeaconStateView, descriptor: js.Uint8Array) pub fn computeUnrealizedCheckpoints(self: *const BeaconStateView) !js_types.UnrealizedCheckpoints { const env = js.env(); const cached_state = try self.requireState(); - const result = try st.computeUnrealizedCheckpoints(allocator, napi_io.get(), cached_state); + const result = try st.computeUnrealizedCheckpoints(allocator, js.io(), cached_state); const obj = try env.createObject(); try obj.setNamedProperty( @@ -1129,16 +1146,8 @@ pub fn loadOtherState( allocator.destroy(new_cached_state); } - try new_cached_state.init( - allocator, - new_state, - .{ - .config = &config.state.config, - .index_to_pubkey = &pubkey.state.index2pubkey, - .pubkey_to_index = &pubkey.state.pubkey2index, - }, - null, - ); + const io = js.io(); + try initCachedState(new_cached_state, io, new_state); new_cached_state_initialized = true; if (opts) |value| { @@ -1150,16 +1159,18 @@ pub fn loadOtherState( // This doesn't matter for typescript lodestar because GC clears it anyway, // but we're losing some savings here. Consider implementating something like // a `prefetchAll` that only does `populateAllNodes` that returns void - var validators_view = try new_cached_state.state.validators(); - _ = validators_view.getAllReadonlyValues(allocator) catch |err| { + const validators_view = try new_cached_state.state.validators(); + const validators = validators_view.getAllReadonlyValues(allocator) catch |err| { try js.env().throwError("STATE_ERROR", "Failed to preload validators"); return err; }; - var balances_view = try new_cached_state.state.balances(); - _ = balances_view.getAll(allocator) catch |err| { + allocator.free(validators); + const balances_view = try new_cached_state.state.balances(); + const balances = balances_view.getAll(allocator) catch |err| { try js.env().throwError("STATE_ERROR", "Failed to preload balances"); return err; }; + allocator.free(balances); } } @@ -1280,7 +1291,7 @@ pub fn processSlots(self: *const BeaconStateView, slot_arg: js.Number, options: allocator.destroy(post_state); } - try st.processSlots(allocator, napi_io.get(), post_state, slot_value, .{}); + try st.processSlots(allocator, js.io(), post_state, slot_value, .{}); return .{ .cached_state = post_state, .pool_rc = pool.state.poolRc().ref(), @@ -1309,7 +1320,7 @@ pub fn stateTransition(self: *const BeaconStateView, signed_block_bytes: js.Uint const signed_block = try AnySignedBeaconBlock.deserialize(allocator, .full, fork_seq, bytes); defer signed_block.deinit(allocator); - const post_state = try st.stateTransition(allocator, napi_io.get(), cached_state, signed_block, opts); + const post_state = try st.stateTransition(allocator, js.io(), cached_state, signed_block, opts); return .{ .cached_state = post_state, .pool_rc = pool.state.poolRc().ref(), diff --git a/bindings/napi/blst.zig b/bindings/napi/blst.zig index c48298e78..d8cfec9b3 100644 --- a/bindings/napi/blst.zig +++ b/bindings/napi/blst.zig @@ -1,13 +1,13 @@ //! NAPI bindings for BLS (blst) cryptographic operations used by lodestar. //! -//! This module uses a **Zig ThreadPool** (`thread_pool`) — a fixed-size pool of OS threads -//! initialized once via `initThreadPool`. Used by synchronous NAPI functions (`aggregateVerify`, +//! This module uses a **Zig ThreadPool** (`state.thread_pool`) — a fixed-size pool of OS threads +//! initialized once via `state.init`. Used by synchronous NAPI functions (`aggregateVerify`, //! `verifyMultipleAggregateSignatures`) to fan out pairing checks across worker threads. The //! call still blocks the JS thread while it waits for the pool to finish, but the crypto work //! itself is parallelized. //! //! `aggregateWithRandomness` runs synchronously on the calling thread and does not -//! rely on the native `thread_pool`. In lodestar, this is called from a Node.js +//! rely on the native `state.thread_pool`. In lodestar, this is called from a Node.js //! worker thread (BLS thread pool), not the main thread. const std = @import("std"); const builtin = @import("builtin"); @@ -15,15 +15,16 @@ const zapi = @import("zapi:zapi"); const js = zapi.js; const napi = zapi.napi; const bls = @import("bls"); -const napi_io = @import("./io.zig"); const NativePublicKey = bls.PublicKey; const NativeSignature = bls.Signature; const NativeSecretKey = bls.SecretKey; +const SigningRoot = bls.SigningRoot; const Pairing = bls.Pairing; const AggregatePublicKey = bls.AggregatePublicKey; const AggregateSignature = bls.AggregateSignature; const ThreadPool = bls.ThreadPool; +const BatchVerifyItem = bls.BatchVerifyItem; const DST = bls.DST; const MAX_AGGREGATE_PER_JOB = bls.MAX_AGGREGATE_PER_JOB; @@ -35,29 +36,37 @@ const MAX_AGGREGATE_PER_JOB = bls.MAX_AGGREGATE_PER_JOB; /// See: packages/beacon-node/src/chain/bls/multithread/worker.ts const BATCH_VERIFY_SIZE = 32; -/// Cached thread pool reference for parallel verification. -/// Initialized lazily on first use, torn down via `deinitThreadPool`. -var thread_pool: ?*ThreadPool = null; +/// A broken random source must fail instead of retrying forever. +const RANDOM_SCALAR_RETRIES_MAX = 8; -pub fn initThreadPool(n_workers: u16) !void { - if (thread_pool != null) return error.PoolExists; - thread_pool = try ThreadPool.init(std.heap.page_allocator, napi_io.get(), .{ .n_workers = n_workers }); -} +/// Native-only thread pool state, reached from `root.zig` through the +/// pub `state` var so it is not part of the JS module surface. +const State = struct { + /// Cached thread pool reference for parallel verification. + thread_pool: ?*ThreadPool = null, -/// Closes the `ThreadPool` used for blst operations. -/// -/// Note: this can invalidate any inflight verification requests. Consumer is responsible -/// for the lifecycle of their program and should only call this when all work is done. -/// -/// This note is however application dependent. For the use case of lodestar, -/// it's likely that this would not be called at all. -/// Same goes for any other long-lived processes. -pub fn deinitThreadPool() void { - if (thread_pool) |p| { - p.deinit(napi_io.get()); - thread_pool = null; + pub fn init(self: *State, n_workers: u16) !void { + if (self.thread_pool != null) return error.PoolExists; + self.thread_pool = try ThreadPool.init(std.heap.page_allocator, js.io(), .{ .n_workers = n_workers }); } -} + + /// Closes the `ThreadPool` used for blst operations. + /// + /// Note: this can invalidate any inflight verification requests. Consumer is responsible + /// for the lifecycle of their program and should only call this when all work is done. + /// + /// This note is however application dependent. For the use case of lodestar, + /// it's likely that this would not be called at all. + /// Same goes for any other long-lived processes. + pub fn deinit(self: *State) void { + if (self.thread_pool) |p| { + p.deinit(js.io()); + self.thread_pool = null; + } + } +}; + +pub var state: State = .{}; var gpa: std.heap.DebugAllocator(.{}) = .init; const allocator = if (builtin.mode == .Debug) @@ -65,6 +74,17 @@ const allocator = if (builtin.mode == .Debug) else std.heap.c_allocator; +/// A zero coefficient would omit its input from the random linear combination. +fn ensureNonzeroRandomScalar(io: std.Io, scalar: *[8]u8) !void { + if (!std.mem.allEqual(u8, scalar, 0)) return; + + for (0..RANDOM_SCALAR_RETRIES_MAX) |_| { + io.random(scalar); + if (!std.mem.allEqual(u8, scalar, 0)) return; + } + return error.RandomScalarGenerationFailed; +} + fn boolOrDefault(value: ?js.Boolean, default: bool) !bool { return if (value) |v| try v.toBool() else default; } @@ -81,7 +101,8 @@ fn formatHex(bytes: []const u8) !js.String { } fn unwrapClass(comptime T: type, value: js.Value) !*T { - return js.env().unwrap(T, value.toValue()); + const raw = value.toValue(); + return js.convertArg(*T, raw.value, raw.env); } /// Reads a Uint8Array slice from a generic `js.Value`. @@ -198,7 +219,7 @@ pub const Signature = struct { var sig = NativeSignature.deserialize(bytes) catch return error.DeserializationFailed; if (try boolOrDefault(sig_validate, false)) { - try sig.validate(try boolOrDefault(sig_infcheck, false)); + try sig.validate(try boolOrDefault(sig_infcheck, true)); } return .{ .raw = sig }; } @@ -272,8 +293,18 @@ pub const SecretKey = struct { /// Creates a `SecretKey` from a hex string. pub fn fromHex(hex_string: js.String) !SecretKey { + switch (try hex_string.len()) { + NativeSecretKey.serialize_size * 2, + NativeSecretKey.serialize_size * 2 + 2, + => {}, + else => return error.InvalidSecretKeyLength, + } + var hex_buf: [NativeSecretKey.serialize_size * 2 + 3]u8 = undefined; const hex = try hexFromString(hex_string, &hex_buf); + if (hex.len != NativeSecretKey.serialize_size * 2) { + return error.InvalidSecretKeyLength; + } var bytes_buf: [NativeSecretKey.serialize_size]u8 = undefined; const bytes = try std.fmt.hexToBytes(&bytes_buf, hex); @@ -299,8 +330,9 @@ pub const SecretKey = struct { /// Signs a message with this `SecretKey`, returns a `Signature`. pub fn sign(self: *const SecretKey, msg: js.Uint8Array) !Signature { - const slice = try msg.toSlice(); - return .{ .raw = self.raw.sign(slice, DST, null) }; + const msg_bytes = try msg.toSlice(); + if (msg_bytes.len != @sizeOf(SigningRoot)) return error.InvalidMessageLength; + return .{ .raw = self.raw.sign(msg_bytes[0..@sizeOf(SigningRoot)], DST, null) }; } /// Derives the PublicKey from this SecretKey. @@ -330,11 +362,12 @@ pub const SecretKey = struct { /// 4) pk_validate: ?bool /// 5) sig_groupcheck: ?bool pub fn verify(msg: js.Uint8Array, pk: PublicKey, sig: Signature, pk_validate: ?js.Boolean, sig_groupcheck: ?js.Boolean) !js.Boolean { - const msg_slice = try msg.toSlice(); + const msg_bytes = try msg.toSlice(); + if (msg_bytes.len != @sizeOf(SigningRoot)) return error.InvalidMessageLength; sig.raw.verify( try boolOrDefault(sig_groupcheck, false), - msg_slice, + msg_bytes[0..@sizeOf(SigningRoot)], DST, null, &pk.raw, @@ -357,7 +390,7 @@ pub fn aggregateVerify(msgs: js.Array, pks: js.Array, sig: Signature, pks_valida return error.InvalidAggregateVerifyInput; } - const msg_bufs = try allocator.alloc([32]u8, msgs_len); + const msg_bufs = try allocator.alloc(SigningRoot, msgs_len); defer allocator.free(msg_bufs); const pk_ptrs = try allocator.alloc(*NativePublicKey, pks_len); @@ -366,16 +399,16 @@ pub fn aggregateVerify(msgs: js.Array, pks: js.Array, sig: Signature, pks_valida for (0..msgs_len) |i| { const msg_value = try msgs.get(@intCast(i)); const msg_bytes = try uint8SliceFromValue(msg_value); - if (msg_bytes.len != 32) return error.InvalidMessageLength; - @memcpy(&msg_bufs[i], msg_bytes[0..32]); + if (msg_bytes.len != @sizeOf(SigningRoot)) return error.InvalidMessageLength; + msg_bufs[i] = msg_bytes[0..@sizeOf(SigningRoot)].*; const wrapped_pk = try unwrapClass(PublicKey, try pks.get(@intCast(i))); pk_ptrs[i] = &wrapped_pk.raw; } - const pool = thread_pool orelse return error.ThreadPoolNotInitialized; + const pool = state.thread_pool orelse return error.ThreadPoolNotInitialized; const result = pool.aggregateVerify( - napi_io.get(), + js.io(), &sig.raw, try boolOrDefault(sig_groupcheck, false), msg_bufs, @@ -398,8 +431,8 @@ pub fn aggregateVerify(msgs: js.Array, pks: js.Array, sig: Signature, pks_valida /// 3) sig: Signature /// 4) sigs_groupcheck: ?bool pub fn fastAggregateVerify(msg: js.Uint8Array, pks: js.Array, sig: Signature, sigs_groupcheck: ?js.Boolean) !js.Boolean { - const msg_slice = try msg.toSlice(); - if (msg_slice.len != 32) return error.InvalidMessageLength; + const msg_bytes = try msg.toSlice(); + if (msg_bytes.len != @sizeOf(SigningRoot)) return error.InvalidMessageLength; const pks_len = try pks.length(); if (pks_len == 0) return js.Boolean.from(false); @@ -417,7 +450,7 @@ pub fn fastAggregateVerify(msg: js.Uint8Array, pks: js.Array, sig: Signature, si const result = sig.raw.fastAggregateVerify( try boolOrDefault(sigs_groupcheck, false), &pairing_buf, - msg_slice[0..32], + msg_bytes[0..@sizeOf(SigningRoot)], DST, native_pks, false, @@ -437,81 +470,45 @@ pub fn verifyMultipleAggregateSignatures(sets: js.Array, pks_validate: ?js.Boole const n_elems = try sets.length(); if (n_elems == 0) return js.Boolean.from(false); - var msgs_stack: [BATCH_VERIFY_SIZE][]const u8 = undefined; - var pks_stack: [BATCH_VERIFY_SIZE]*NativePublicKey = undefined; - var sigs_stack: [BATCH_VERIFY_SIZE]*NativeSignature = undefined; - var rands_stack: [BATCH_VERIFY_SIZE][32]u8 = undefined; - - var msgs_heap: ?[][]const u8 = null; - defer if (msgs_heap) |buf| allocator.free(buf); - var pks_heap: ?[]*NativePublicKey = null; - defer if (pks_heap) |buf| allocator.free(buf); - var sigs_heap: ?[]*NativeSignature = null; - defer if (sigs_heap) |buf| allocator.free(buf); - var rands_heap: ?[][32]u8 = null; - defer if (rands_heap) |buf| allocator.free(buf); - - const msgs = if (n_elems <= BATCH_VERIFY_SIZE) msgs_stack[0..n_elems] else blk: { - const buf = try allocator.alloc([]const u8, n_elems); - msgs_heap = buf; - break :blk buf; - }; - const pks = if (n_elems <= BATCH_VERIFY_SIZE) pks_stack[0..n_elems] else blk: { - const buf = try allocator.alloc(*NativePublicKey, n_elems); - pks_heap = buf; - break :blk buf; - }; - const sigs = if (n_elems <= BATCH_VERIFY_SIZE) sigs_stack[0..n_elems] else blk: { - const buf = try allocator.alloc(*NativeSignature, n_elems); - sigs_heap = buf; - break :blk buf; - }; - const rands = if (n_elems <= BATCH_VERIFY_SIZE) rands_stack[0..n_elems] else blk: { - const buf = try allocator.alloc([32]u8, n_elems); - rands_heap = buf; + var items_stack: [BATCH_VERIFY_SIZE]BatchVerifyItem = undefined; + var items_heap: ?[]BatchVerifyItem = null; + defer if (items_heap) |buf| allocator.free(buf); + + const items = if (n_elems <= BATCH_VERIFY_SIZE) items_stack[0..n_elems] else blk: { + const buf = try allocator.alloc(BatchVerifyItem, n_elems); + items_heap = buf; break :blk buf; }; - var seed_bytes: [8]u8 = undefined; - const io = napi_io.get(); - io.random(&seed_bytes); - var prng = std.Random.DefaultPrng.init(std.mem.readInt(u64, &seed_bytes, .little)); - const rand = prng.random(); - - const e = js.env(); + const io = js.io(); for (0..n_elems) |i| { const set = (try sets.get(@intCast(i))).toValue(); const msg_napi = try set.getNamedProperty("msg"); const msg_bytes = try uint8SliceFromValue(.{ .val = msg_napi }); - if (msg_bytes.len != 32) return error.InvalidMessageLength; - msgs[i] = msg_bytes; - + if (msg_bytes.len != @sizeOf(SigningRoot)) return error.InvalidMessageLength; const pk_napi = try set.getNamedProperty("pk"); - const wrapped_pk = try e.unwrap(PublicKey, pk_napi); - pks[i] = &wrapped_pk.raw; + const wrapped_pk = try unwrapClass(PublicKey, .{ .val = pk_napi }); const sig_napi = try set.getNamedProperty("sig"); - const wrapped_sig = try e.unwrap(Signature, sig_napi); - sigs[i] = &wrapped_sig.raw; - - var scalar = rand.int(u64); - while (scalar == 0) scalar = rand.int(u64); - std.mem.writeInt(u64, rands[i][0..8], scalar, .little); - @memset(rands[i][8..], 0); - } - - const pool = thread_pool orelse return error.ThreadPoolNotInitialized; + const wrapped_sig = try unwrapClass(Signature, .{ .val = sig_napi }); + items[i] = .{ + .message = msg_bytes[0..@sizeOf(SigningRoot)].*, + .public_key = &wrapped_pk.raw, + .signature = &wrapped_sig.raw, + .randomness = undefined, + }; + io.random(&items[i].randomness); + try ensureNonzeroRandomScalar(io, items[i].randomness[0..8]); + } + + const pool = state.thread_pool orelse return error.ThreadPoolNotInitialized; const result = pool.verifyMultipleAggregateSignatures( - napi_io.get(), - n_elems, - msgs, + js.io(), + items, DST, - pks, try boolOrDefault(pks_validate, false), - sigs, try boolOrDefault(sigs_groupcheck, false), - rands, ) catch return js.Boolean.from(false); return js.Boolean.from(result); @@ -612,21 +609,17 @@ pub fn aggregateWithRandomness(sets: js.Array) !js.Value { var sigs: [MAX_AGGREGATE_PER_JOB]NativeSignature = undefined; var sig_ptrs: [MAX_AGGREGATE_PER_JOB]*const NativeSignature = undefined; - var seed_bytes: [8]u8 = undefined; - const io = napi_io.get(); - io.random(&seed_bytes); - var prng = std.Random.DefaultPrng.init(std.mem.readInt(u64, &seed_bytes, .little)); - const rand = prng.random(); + const io = js.io(); var scalars: [8 * MAX_AGGREGATE_PER_JOB]u8 = undefined; var sca_ptrs: [MAX_AGGREGATE_PER_JOB]*const u8 = undefined; - rand.bytes(scalars[0 .. n * nbytes]); + io.random(scalars[0 .. n * nbytes]); const env = js.env(); for (0..n) |i| { const set = (try sets.get(@intCast(i))).toValue(); const pk_napi = try set.getNamedProperty("pk"); - const wrapped_pk = try env.unwrap(PublicKey, pk_napi); + const wrapped_pk = try unwrapClass(PublicKey, .{ .val = pk_napi }); pk_ptrs[i] = &wrapped_pk.raw; const sig_napi = try set.getNamedProperty("sig"); @@ -635,17 +628,18 @@ pub fn aggregateWithRandomness(sets: js.Array) !js.Value { sigs[i].validate(true) catch return error.InvalidSignature; sig_ptrs[i] = &sigs[i]; - while (std.mem.allEqual(u8, scalars[i * nbytes ..][0..nbytes], 0)) { - rand.bytes(scalars[i * nbytes ..][0..nbytes]); - } + const scalar = scalars[i * nbytes ..][0..nbytes]; + try ensureNonzeroRandomScalar(io, scalar); sca_ptrs[i] = &scalars[i * nbytes]; } - const scratch_size = @max( + const scratch_size_bytes = @max( bls.c.blst_p1s_mult_pippenger_scratch_sizeof(n), bls.c.blst_p2s_mult_pippenger_scratch_sizeof(n), ); - const scratch = try allocator.alloc(u64, scratch_size); + const scratch_len = @divExact(scratch_size_bytes, @sizeOf(u64)); + + const scratch = try allocator.alloc(u64, scratch_len); defer allocator.free(scratch); // Pippenger multi-scalar multiplication on G1 (pubkeys) @@ -715,12 +709,12 @@ const AsyncAggRandData = struct { /// /// Note: MUST NOT call any napi APIs. fn asyncAggRand_execute(_: napi.Env, data: *AsyncAggRandData) void { - const pool = thread_pool orelse { + const pool = state.thread_pool orelse { data.err = error.PoolNotInitialized; return; }; pool.aggregateWithRandomness( - napi_io.get(), + js.io(), data.pk_ptrs[0..data.n], data.sig_ptrs[0..data.n], data.randomness[0 .. data.n * 32], @@ -803,7 +797,7 @@ pub fn asyncAggregateWithRandomness(sets: js.Array) !js.Value { if (n == 0) return error.EmptyArray; if (n > MAX_AGGREGATE_PER_JOB) return error.TooManySets; - if (thread_pool == null) return error.PoolNotInitialized; + if (state.thread_pool == null) return error.PoolNotInitialized; const env = js.env(); @@ -816,13 +810,19 @@ pub fn asyncAggregateWithRandomness(sets: js.Array) !js.Value { data.err = null; data.deferred = undefined; data.work = undefined; - napi_io.get().random(data.randomness[0 .. n * 32]); + + const io = js.io(); + io.random(data.randomness[0 .. n * 32]); + for (0..n) |i| { + const scalar = data.randomness[i * 32 ..][0..8]; + try ensureNonzeroRandomScalar(io, scalar); + } for (0..n) |i| { const set = (try sets.get(@intCast(i))).toValue(); const pk_napi = try set.getNamedProperty("pk"); - const wrapped_pk = try env.unwrap(PublicKey, pk_napi); + const wrapped_pk = try unwrapClass(PublicKey, .{ .val = pk_napi }); data.pks[i] = wrapped_pk.raw; data.pk_ptrs[i] = &data.pks[i]; @@ -832,9 +832,12 @@ pub fn asyncAggregateWithRandomness(sets: js.Array) !js.Value { data.sig_ptrs[i] = &data.sigs[i]; } - data.deferred = try env.createPromise(); - + const deferred_cleanup_value = try env.getUndefined(); const resource_name = try env.createStringUtf8("asyncAggregateWithRandomness"); + + // Until queue succeeds, this function owns the unqueued work handle. Deletion should + // not fail after successful creation. If that invariant breaks, later error cleanup may + // free `data` while the work handle still points to it. const work = try env.createAsyncWork( AsyncAggRandData, null, @@ -843,8 +846,18 @@ pub fn asyncAggregateWithRandomness(sets: js.Array) !js.Value { asyncAggRand_complete, data, ); + errdefer work.delete() catch |err| { + std.log.err("failed to delete unqueued async BLS work: {s}", .{@errorName(err)}); + }; + data.work = work.work; + // Settle the unreturned Promise so Node can release its deferred handle. + data.deferred = try env.createPromise(); + errdefer data.deferred.resolve(deferred_cleanup_value) catch |err| { + std.log.err("failed to settle unreturned async BLS promise: {s}", .{@errorName(err)}); + }; + try work.queue(); return .{ .val = data.deferred.getPromise() }; diff --git a/bindings/napi/config.zig b/bindings/napi/config.zig index 0132bd02f..a59092fe3 100644 --- a/bindings/napi/config.zig +++ b/bindings/napi/config.zig @@ -9,7 +9,7 @@ const Preset = @import("preset").Preset; const max_blob_schedule_entries = 16; -pub const State = struct { +const State = struct { config: BeaconConfig = undefined, initialized: bool = false, config_name: [64]u8 = undefined, @@ -71,7 +71,7 @@ pub fn set(object: js.Value, genesis_root: js.Uint8Array) !void { state.initialized = true; } -pub fn chainConfigFromObject(env: napi.Env, obj: napi.Value) !ChainConfig { +fn chainConfigFromObject(env: napi.Env, obj: napi.Value) !ChainConfig { var chain_config: ChainConfig = undefined; inline for (std.meta.fields(ChainConfig)) |field| { diff --git a/bindings/napi/io.zig b/bindings/napi/io.zig deleted file mode 100644 index 537e6799c..000000000 --- a/bindings/napi/io.zig +++ /dev/null @@ -1,48 +0,0 @@ -const std = @import("std"); - -/// Process-wide `Io` for NAPI callbacks, mirroring the `init.io` that a Zig -/// binary receives from `std.process.Init` (see `std.start.zig`'s -/// `std.Io.Threaded.init(gpa, .{})`). -/// -/// Why not `init_single_threaded`? -/// -/// Today the only Io operations we invoke inside NAPI callbacks are -/// `Clock.Timestamp.now`, `Io.Mutex` lock/unlock, and `io.random` — all -/// backed by direct OS syscalls that don't touch the Threaded executor's -/// allocator / worker pool, so `init_single_threaded` would technically -/// suffice. But that instance pins `allocator = .failing` and -/// `async_limit = .nothing`, so the moment any code path starts calling -/// `io.async(...)` / `io.concurrent(...)` / group scheduling, it panics. -/// Matching the `init.io` defaults keeps us future-proof at essentially -/// zero cost — the worker pool is lazy: threads are only spawned on the -/// first async/concurrent call, up to `async_limit` (cpu_count - 1). -/// -/// Initialized once on the first `register()` call (env_refcount 0→1) and -/// torn down when the last NAPI env closes (env_refcount 1→0). Safe to share -/// across Node.js worker threads — `std.Io.Threaded` protects its own -/// internal state with mutexes. -var instance: std.Io.Threaded = undefined; -var initialized: bool = false; - -const gpa: std.mem.Allocator = std.heap.page_allocator; - -/// Initialize the shared `Threaded`. Idempotent. -pub fn init() !void { - if (initialized) return; - instance = std.Io.Threaded.init(gpa, .{}); - initialized = true; -} - -/// Get the shared `Io`. Caller must ensure `init()` has been invoked -/// (guaranteed by the `register()` flow in `root.zig`). -pub fn get() std.Io { - std.debug.assert(initialized); - return instance.io(); -} - -/// Tear down the shared `Threaded`. Idempotent. -pub fn deinit() void { - if (!initialized) return; - instance.deinit(); - initialized = false; -} diff --git a/bindings/napi/metrics.zig b/bindings/napi/metrics.zig index 389cea57a..402a82933 100644 --- a/bindings/napi/metrics.zig +++ b/bindings/napi/metrics.zig @@ -2,7 +2,6 @@ const std = @import("std"); const builtin = @import("builtin"); const js = @import("zapi:zapi").js; const state_transition = @import("state_transition"); -const napi_io = @import("./io.zig"); var gpa: std.heap.DebugAllocator(.{}) = .init; const allocator = if (builtin.mode == .Debug) @@ -15,17 +14,17 @@ var initialized: bool = false; /// JS: metrics.init() → void pub fn init() !void { if (initialized) return; - try state_transition.metrics.init(allocator, napi_io.get(), .{}); + try state_transition.metrics.init(allocator, js.io(), .{}); initialized = true; } /// JS: metrics.scrapeMetrics() → string pub fn scrapeMetrics() !js.String { var aw: std.Io.Writer.Allocating = .init(allocator); + defer aw.deinit(); + try state_transition.metrics.write(&aw.writer); - var list = aw.toArrayList(); - defer list.deinit(allocator); - return js.String.from(list.items); + return js.String.from(aw.written()); } pub fn deinit() void { diff --git a/bindings/napi/pool.zig b/bindings/napi/pool.zig index 2e0ec480b..c23dcec40 100644 --- a/bindings/napi/pool.zig +++ b/bindings/napi/pool.zig @@ -9,14 +9,14 @@ const allocator = std.heap.page_allocator; const default_pool_size: u32 = 0; -pub const PoolRc = RefCount(Node.Pool); +const PoolRc = RefCount(Node.Pool); /// Pool is wrapped in `RefCount` so binding objects holding pool refs at /// process exit keep the pool alive until their JS finalizer runs. NAPI /// env cleanup hook fires before module-level JS holders are finalized, /// so an unconditional `pool.deinit()` there would free memory that /// `pool.unref()` calls in those finalizers still need. -pub const State = struct { +const State = struct { pool_rc: ?*PoolRc = null, pub fn init(self: *State) !void { diff --git a/bindings/napi/pubkeys.zig b/bindings/napi/pubkeys.zig index e65813520..50af3c254 100644 --- a/bindings/napi/pubkeys.zig +++ b/bindings/napi/pubkeys.zig @@ -1,167 +1,107 @@ const std = @import("std"); -const js = @import("zapi:zapi").js; -const bls = @import("bls"); +const zapi = @import("zapi:zapi"); +const js = zapi.js; +const napi = zapi.napi; const blst_bindings = @import("./blst.zig"); -const PubkeyIndexMap = @import("state_transition").PubkeyIndexMap; -const Index2PubkeyCache = @import("state_transition").Index2PubkeyCache; -const napi_io = @import("./io.zig"); +const state_transition = @import("state_transition"); +const PubkeyCache = state_transition.PubkeyCache; +const pkix = state_transition.pkix; +const Validator = @import("consensus_types").phase0.Validator.Type; -/// Uses page allocator for internal allocations. -/// It's recommended to never reallocate the pubkey2index after initialization. +/// Uses the page allocator for the process-wide cache's internal allocations. const allocator = std.heap.page_allocator; const default_initial_capacity: u32 = 0; const max_stack_aggregate_pubkeys = 512; -pub const State = struct { - pubkey2index: PubkeyIndexMap = undefined, - index2pubkey: Index2PubkeyCache = undefined, +const State = struct { + cache: PubkeyCache = undefined, initialized: bool = false, + control_env: napi.c.napi_env = null, - pub fn init(self: *State) !void { + pub fn init(self: *State, env: napi.Env) !void { if (self.initialized) return; - self.pubkey2index = PubkeyIndexMap.init(allocator); - try self.pubkey2index.ensureTotalCapacity(default_initial_capacity); - self.index2pubkey = try Index2PubkeyCache.initCapacity(allocator, default_initial_capacity); + + self.cache = try PubkeyCache.initCapacity(allocator, js.io(), default_initial_capacity); self.initialized = true; + self.control_env = env.env; } + /// Last-environment cleanup runs after JS calls stop. Remaining view + /// finalizers do not dereference their retained pubkey-cache pointer. pub fn deinit(self: *State) void { if (!self.initialized) return; - self.pubkey2index.deinit(); - self.index2pubkey.deinit(allocator); + self.cache.deinit(); self.initialized = false; + self.control_env = null; } - pub fn reset(self: *State) !void { - if (!self.initialized) return; - - self.pubkey2index.clearRetainingCapacity(); - self.index2pubkey.shrinkRetainingCapacity(0); + fn requireControlEnvironment(self: *const State, env: napi.Env) !void { + if (self.control_env == null or self.control_env != env.env) { + return error.PubkeyCacheControlEnvironmentOnly; + } } }; pub var state: State = .{}; -/// Must only be called after pubkey2index has been initialized with a capacity. -/// Must be kept in sync with std/hashmap.zig -fn pubkey2indexWrittenSize() usize { - const K = [48]u8; - const V = u64; - const Header = struct { - values: [*]V, - keys: [*]K, - capacity: u32, - }; - const Metadata = packed struct { - const FingerPrint = u7; - fingerprint: FingerPrint, - used: u1, - }; - const header_align = @alignOf(Header); - const key_align = @alignOf(K); - const val_align = @alignOf(V); - const max_align = comptime @max(header_align, key_align, val_align); - - const new_cap: usize = state.pubkey2index.capacity(); - const meta_size = @sizeOf(Header) + new_cap * @sizeOf(Metadata); - - const keys_start = std.mem.alignForward(usize, meta_size, key_align); - const keys_end = keys_start + new_cap * @sizeOf(K); - - const vals_start = std.mem.alignForward(usize, keys_end, val_align); - const vals_end = vals_start + new_cap * @sizeOf(V); - - const total_size = std.mem.alignForward(usize, vals_end, max_align); - - return total_size - @sizeOf(Header); -} - /// JS: pubkeys.save(filePath) pub fn save(file_path: js.String) !void { - var file_path_buf: [1024]u8 = undefined; - const path = try file_path.toSlice(&file_path_buf); - const io = napi_io.get(); - const file = try std.Io.Dir.createFile(.cwd(), io, path, .{}); - defer file.close(io); - - // Write header: Magic "PKIX" + len + capacity - var header: [12]u8 = [_]u8{ 'P', 'K', 'I', 'X', 0, 0, 0, 0, 0, 0, 0, 0 }; - std.mem.writeInt(u32, header[4..8], @intCast(state.index2pubkey.items.len), .little); - std.mem.writeInt(u32, header[8..12], @intCast(state.index2pubkey.capacity), .little); + if (!state.initialized) return error.PubkeyIndexNotInitialized; + try state.requireControlEnvironment(js.env()); - var write_buf: [4096]u8 = undefined; - var file_writer = file.writer(io, &write_buf); - var writer = &file_writer.interface; - try writer.writeAll(header[0..12]); + const path = try file_path.toOwnedSlice(allocator); + defer allocator.free(path); + const io = js.io(); - // Write pubkey2index entries - const p2i_size = pubkey2indexWrittenSize(); - const ptr: [*]u8 = @ptrCast(state.pubkey2index.unmanaged.metadata.?); - try writer.writeAll(ptr[0..p2i_size]); + // `createFileAtomic` creates a sibling temporary file so replacement cannot + // cross filesystems. `deinit` removes the temporary file on every failure. + var atomic_file = try std.Io.Dir.createFileAtomic(.cwd(), io, path, .{ .replace = true }); + defer atomic_file.deinit(io); - // Write index2pubkey entries - try writer.writeAll(std.mem.sliceAsBytes(state.index2pubkey.items)); + var write_buf: [4096]u8 = undefined; + var file_writer = atomic_file.file.writer(io, &write_buf); + try pkix.save(&state.cache, io, &file_writer.interface); try file_writer.end(); + try atomic_file.file.sync(io); + try atomic_file.replace(io); } -/// JS: pubkeys.load(filePath) -pub fn load(file_path: js.String) !void { - var file_path_buf: [1024]u8 = undefined; - const path = try file_path.toSlice(&file_path_buf); - const io = napi_io.get(); +/// JS: pubkeys.load(filePath, maxCapacity) +pub fn load(file_path: js.String, max_capacity: js.Number) !void { + if (!state.initialized) return error.PubkeyIndexNotInitialized; + try state.requireControlEnvironment(js.env()); + + const path = try file_path.toOwnedSlice(allocator); + defer allocator.free(path); + const capacity_limit = try max_capacity.toU32(); + const io = js.io(); + const file = try std.Io.Dir.openFile(.cwd(), io, path, .{}); defer file.close(io); - if (state.initialized) { - state.deinit(); - } - + const file_size = try file.length(io); var read_buf: [4096]u8 = undefined; var file_reader = file.reader(io, &read_buf); - const header = try file_reader.interface.takeArray(12); - - if (!std.mem.eql(u8, header[0..4], &[_]u8{ 'P', 'K', 'I', 'X' })) { - return error.InvalidPubkeyIndexFile; - } - - const len = std.mem.readInt(u32, header[4..8], .little); - const capacity = std.mem.readInt(u32, header[8..12], .little); + var loaded_cache = try pkix.load( + allocator, + io, + &file_reader.interface, + file_size, + capacity_limit, + ); + defer loaded_cache.deinit(); - const file_size = try file.length(io); - - state.pubkey2index = PubkeyIndexMap.init(allocator); - try state.pubkey2index.ensureTotalCapacity(capacity); - errdefer state.pubkey2index.deinit(); - state.index2pubkey = try Index2PubkeyCache.initCapacity(allocator, capacity); - errdefer state.index2pubkey.deinit(allocator); - try state.index2pubkey.resize(allocator, len); - - const p2i_size = pubkey2indexWrittenSize(); - const i2p_size = @sizeOf(bls.PublicKey) * len; - - if (file_size != 12 + p2i_size + i2p_size) { - return error.InvalidPubkeyIndexFile; - } - - // Read pubkey2index entries - const ptr: [*]u8 = @ptrCast(state.pubkey2index.unmanaged.metadata.?); - try file_reader.interface.readSliceAll(ptr[0..p2i_size]); - - state.pubkey2index.unmanaged.size = len; - state.pubkey2index.unmanaged.available = capacity - len; - - // Read index2pubkey entries - try file_reader.interface.readSliceAll(std.mem.sliceAsBytes(state.index2pubkey.items)); - - state.initialized = true; + try pkix.install(&state.cache, io, &loaded_cache); } /// JS: pubkeys.reset() pub fn reset() !void { - try state.reset(); + if (!state.initialized) return error.PubkeyIndexNotInitialized; + try state.requireControlEnvironment(js.env()); + try state.cache.clear(js.io()); } /// JS: pubkeys.getIndex(pubkeyBytes) → number | null @@ -170,10 +110,14 @@ pub fn getIndex(pubkey: js.Uint8Array) !js.Value { const pubkey_slice = try pubkey.toSlice(); if (pubkey_slice.len != 48) return error.InvalidPubkeyLength; + const pubkey_bytes = pubkey_slice[0..48].*; + + const io = js.io(); + const index = state.cache.get(io, pubkey_bytes); const e = js.env(); - if (state.pubkey2index.get(pubkey_slice[0..48].*)) |index| { - return .{ .val = try e.createUint32(@intCast(index)) }; + if (index) |validator_index| { + return .{ .val = try e.createUint32(@intCast(validator_index)) }; } return .{ .val = try e.getNull() }; } @@ -183,9 +127,9 @@ pub fn get(index: js.Number) !?blst_bindings.PublicKey { if (!state.initialized) return error.PubkeyIndexNotInitialized; const idx = try index.toU32(); - if (idx >= state.index2pubkey.items.len) return null; - - return .{ .raw = state.index2pubkey.items[@intCast(idx)] }; + const io = js.io(); + const public_key = state.cache.getPubkey(io, idx) orelse return null; + return .{ .raw = public_key }; } /// Aggregate multiple `PublicKey`s by the given @@ -201,67 +145,84 @@ pub fn aggregate(indices: js.Array) !blst_bindings.PublicKey { const len = try indices.length(); if (len == 0) return error.EmptyPublicKeyArray; - if (len == 1) { - const idx = try (try indices.getNumber(0)).toU32(); - if (idx >= state.index2pubkey.items.len) return error.PubkeyIndexNotFound; - return .{ .raw = state.index2pubkey.items[@intCast(idx)] }; - } - - var pks_stack: [max_stack_aggregate_pubkeys]bls.PublicKey = undefined; - const pks = if (len <= pks_stack.len) - pks_stack[0..len] + var indices_stack: [max_stack_aggregate_pubkeys]u64 = undefined; + const exact_indices = if (len <= indices_stack.len) + indices_stack[0..len] else blk: { - const buf = try allocator.alloc(bls.PublicKey, len); + const buf = try allocator.alloc(u64, len); break :blk buf; }; - defer if (len > pks_stack.len) allocator.free(pks); + defer if (len > indices_stack.len) allocator.free(exact_indices); for (0..len) |i| { - const idx = try (try indices.getNumber(@intCast(i))).toU32(); - if (idx >= state.index2pubkey.items.len) return error.PubkeyIndexNotFound; - pks[i] = state.index2pubkey.items[@intCast(idx)]; + exact_indices[i] = try (try indices.getNumber(@intCast(i))).toU32(); } - const agg_pk = bls.AggregatePublicKey.aggregate(pks, false) catch - return error.AggregationFailed; + const io = js.io(); + const aggregate_pubkey = if (exact_indices.len == 1) + state.cache.getPubkey(io, exact_indices[0]) orelse return error.PubkeyIndexNotFound + else + state.cache.aggregate(io, exact_indices) catch |err| switch (err) { + error.InvalidIndex => return error.PubkeyIndexNotFound, + else => return error.AggregationFailed, + }; - return .{ .raw = agg_pk.toPublicKey() }; + return .{ .raw = aggregate_pubkey }; } -/// JS: pubkeys.set(index, pubkeyBytes) -pub fn set(index: js.Number, pubkey: js.Uint8Array) !void { +/// JS: pubkeys.append(index, pubkeyBytes) +pub fn append(index: js.Number, pubkey: js.Uint8Array) !void { if (!state.initialized) return error.PubkeyIndexNotInitialized; const idx = try index.toU32(); + const io = js.io(); + const pubkey_slice = try pubkey.toSlice(); if (pubkey_slice.len != 48) return error.InvalidPubkeyLength; + const pubkey_bytes = pubkey_slice[0..48].*; - const pubkey_bytes = pubkey_slice[0..48]; + try state.cache.append(io, pubkey_bytes, idx); +} - // Ensure capacity if needed - if (idx >= state.index2pubkey.capacity) { - const new_cap: u32 = @intCast(@max(idx + 1, state.index2pubkey.capacity * 2)); - try state.pubkey2index.ensureTotalCapacity(new_cap); - try state.index2pubkey.ensureTotalCapacity(allocator, new_cap); - } +/// JS: pubkeys.syncPubkeys(validators) +pub fn syncPubkeys(validators: js.Array) !void { + if (!state.initialized) return error.PubkeyIndexNotInitialized; - // Extend length if needed - if (idx >= state.index2pubkey.items.len) { - try state.index2pubkey.resize(allocator, idx + 1); - } + const validator_count = try validators.length(); + const io = js.io(); + const num_cached = state.cache.count(io); + if (validator_count <= num_cached) return; + + const num_new_validators = validator_count - num_cached; + + // SAFETY: the first `num_cached` validator ptrs are intentionally left undefined, + // because syncPubkeys only has to append the last `num_new_validators` pubkeys. + const validator_ptrs = try allocator.alloc(*const Validator, validator_count); + defer allocator.free(validator_ptrs); - // Set pubkey2index - state.pubkey2index.put(pubkey_bytes.*, @intCast(idx)) catch return error.PubkeyIndexInsertFailed; + const new_validators = try allocator.alloc(Validator, num_new_validators); + defer allocator.free(new_validators); - // Deserialize and set index2pubkey - state.index2pubkey.items[@intCast(idx)] = try bls.PublicKey.uncompress(pubkey_bytes); + // `new_index` is the index of the soon-to-be added pubkey of a new validator. + // `i` is the local-only index of the temporary backing memory `new_validators`. + for (num_cached..validator_count, 0..) |new_index, i| { + const value = try validators.get(@intCast(new_index)); + const partial_validator = try (try value.asObject(struct { pubkey: js.Uint8Array })).get(); + + new_validators[i] = undefined; + new_validators[i].pubkey = try partial_validator.pubkey.toArray(blst_bindings.PublicKey.COMPRESS_SIZE); + validator_ptrs[new_index] = &new_validators[i]; + } + + try state.cache.syncPubkeys(io, validator_ptrs); } /// JS: pubkeys.size() → number /// Note: zapi DSL does not yet support namespace-level getters, so this is a function. pub fn size() !js.Number { if (!state.initialized) return error.PubkeyIndexNotInitialized; - return js.Number.from(@as(u32, @intCast(state.index2pubkey.items.len))); + const io = js.io(); + return js.Number.from(state.cache.count(io)); } /// JS: pubkeys.ensureCapacity(newSize) @@ -269,9 +230,15 @@ pub fn ensureCapacity(new_size: js.Number) !void { if (!state.initialized) return error.PubkeyIndexNotInitialized; const requested = try new_size.toU32(); - const old_size = state.index2pubkey.capacity; - if (requested <= old_size) return; + const io = js.io(); + try state.cache.ensureTotalCapacity(io, requested); +} - try state.pubkey2index.ensureTotalCapacity(requested); - try state.index2pubkey.ensureTotalCapacity(allocator, requested); +/// JS: pubkeys.capacity() → number +/// Note: zapi DSL does not yet support namespace-level getters, so this is a function. +pub fn capacity() !js.Number { + if (!state.initialized) return error.PubkeyIndexNotInitialized; + const io = js.io(); + const current_capacity: u32 = @intCast(state.cache.capacity(io)); + return js.Number.from(current_capacity); } diff --git a/bindings/napi/root.zig b/bindings/napi/root.zig index 5cd6a8d38..9d405985b 100644 --- a/bindings/napi/root.zig +++ b/bindings/napi/root.zig @@ -5,12 +5,12 @@ pub const pool = @import("./pool.zig"); pub const shuffle = @import("./shuffle.zig"); pub const config = @import("./config.zig"); pub const metrics = @import("./metrics.zig"); +pub const stateTransition = @import("./stateTransition.zig"); pub const BeaconStateView = @import("./BeaconStateView.zig"); pub const blst = @import("./blst.zig"); pub const pubkeys = @import("./pubkeys.zig"); const options = @import("bls_options"); -const napi_io = @import("./io.zig"); var gpa: std.heap.DebugAllocator(.{}) = .init; const allocator = if (builtin.mode == .Debug) gpa.allocator() else std.heap.c_allocator; @@ -18,22 +18,28 @@ const allocator = if (builtin.mode == .Debug) gpa.allocator() else std.heap.c_al fn init(old_ref_count: u32) !void { if (old_ref_count == 0) { // First environment — initialize shared state in your threadpool init. - try napi_io.init(); - errdefer napi_io.deinit(); - var cpu_count: u64 = options.thread_count; if (options.thread_count == 0) { cpu_count = @max(try detectCpuCount(), 2) - 1; - std.debug.print( + std.log.debug( "Note: no -Dthread-count set, using cgroup-aware CPU count minus 1: {}\n", .{cpu_count}, ); } const n_workers = @min(cpu_count, @import("bls").ThreadPool.MAX_WORKERS); - try blst.initThreadPool(@intCast(n_workers)); + try blst.state.init(@intCast(n_workers)); + errdefer blst.state.deinit(); + try pool.state.init(); - try pubkeys.state.init(); + errdefer pool.state.deinit(); + + try pubkeys.state.init(js.env()); + + // All remaining initialization must stay infallible because the earlier errdefers no + // longer cover every initialized global. + errdefer comptime unreachable; + config.state.init(); } } @@ -42,8 +48,8 @@ fn init(old_ref_count: u32) !void { /// not prevent the module from loading: warn and fall back to the affinity /// count (what `std.Thread.getCpuCount()` reports). fn detectCpuCount() !usize { - return @import("cpu_count").getNumCpus(allocator, napi_io.get()) catch |err| { - std.debug.print( + return @import("cpu_count").getNumCpus(allocator, js.io()) catch |err| { + std.log.debug( "Warning: cgroup CPU detection failed ({s}), using affinity count\n", .{@errorName(err)}, ); @@ -54,12 +60,11 @@ fn detectCpuCount() !usize { fn cleanup(new_ref_count: u32) void { if (new_ref_count == 0) { // Last environment — tear down shared state. - blst.deinitThreadPool(); + blst.state.deinit(); config.state.deinit(); pubkeys.state.deinit(); pool.state.deinit(); metrics.deinit(); - napi_io.deinit(); } } diff --git a/bindings/napi/stateTransition.zig b/bindings/napi/stateTransition.zig new file mode 100644 index 000000000..3213a9a64 --- /dev/null +++ b/bindings/napi/stateTransition.zig @@ -0,0 +1,6 @@ +const js = @import("zapi:zapi").js; +const st = @import("state_transition"); + +pub fn deinitReusedEpochTransitionCache() void { + st.deinitReusedEpochTransitionCache(js.io()); +} diff --git a/bindings/perf/loadState.test.ts b/bindings/perf/loadState.test.ts index b87b73e7c..7490c1aa9 100644 --- a/bindings/perf/loadState.test.ts +++ b/bindings/perf/loadState.test.ts @@ -5,17 +5,23 @@ import {loadState as loadStateTS} from "@lodestar/state-transition"; import {ssz} from "@lodestar/types"; import bindings from "../src/index.js"; import {getFirstEraFilePath} from "../test/eraFiles.ts"; +import {getPubkeyCacheCapacityForState} from "../test/serializedState.ts"; const reader = await era.era.EraReader.open(config, getFirstEraFilePath()); const stateBytes = await reader.readSerializedState(); await reader.close(); +const requiredPubkeyCapacity = getPubkeyCacheCapacityForState(stateBytes); bindings.pool.ensureCapacity(10_000_000); -bindings.pubkeys.ensureCapacity(2_000_000); +let loadedPkix = false; try { - bindings.pubkeys.load("./mainnet.pkix"); + bindings.pubkeys.load("./mainnet.pkix", requiredPubkeyCapacity); + loadedPkix = true; } catch (_e) { - // ignore + // Rebuild incompatible or corrupt snapshots from the serialized state. +} +if (!loadedPkix || bindings.pubkeys.capacity() < requiredPubkeyCapacity) { + bindings.pubkeys.ensureCapacity(requiredPubkeyCapacity); } const seedState = bindings.BeaconStateView.createFromBytes(stateBytes); diff --git a/bindings/src/blst.d.ts b/bindings/src/blst.d.ts index 37142b554..da8d0b8f4 100644 --- a/bindings/src/blst.d.ts +++ b/bindings/src/blst.d.ts @@ -34,6 +34,7 @@ export class SecretKey { static fromBytes(bytes: Uint8Array): SecretKey; static fromHex(hex: string): SecretKey; static fromKeygen(ikm: Uint8Array, keyInfo?: Uint8Array): SecretKey; + /** Sign an exact 32-byte Ethereum consensus signing root. */ sign(msg: Uint8Array): Signature; toPublicKey(): PublicKey; toBytes(): Uint8Array; @@ -85,6 +86,7 @@ export class Signature { } export interface SignatureSet { + /** Exact 32-byte Ethereum consensus signing root. */ msg: Uint8Array; pk: PublicKey; sig: Signature; @@ -101,7 +103,7 @@ export interface PkAndSig { } /** - * Verify a signature against a message and public key. + * Verify a signature against an exact 32-byte Ethereum consensus signing root and public key. * * If `pkValidate` is `true`, the public key will be infinity and group checked. * @@ -116,7 +118,7 @@ export function verify( ): boolean; /** - * Verify an aggregated signature against multiple messages and multiple public keys. + * Verify an aggregated signature against exact 32-byte Ethereum consensus signing roots and multiple public keys. * * If `pksValidate` is `true`, the public keys will be infinity and group checked. * @@ -131,7 +133,7 @@ export function aggregateVerify( ): boolean; /** - * Verify an aggregated signature against a single message and multiple public keys. + * Verify an aggregated signature against a single exact 32-byte Ethereum consensus signing root and multiple public keys. * * Proof-of-possession is required for public keys. * @@ -145,7 +147,7 @@ export function fastAggregateVerify( ): boolean; /** - * Verify multiple aggregated signatures against multiple messages and multiple public keys. + * Verify multiple aggregated signatures against exact 32-byte Ethereum consensus signing roots and multiple public keys. * * If `pksValidate` is `true`, the public keys will be infinity and group checked. * diff --git a/bindings/src/index.d.ts b/bindings/src/index.d.ts index 1e18464e3..53548d66a 100644 --- a/bindings/src/index.d.ts +++ b/bindings/src/index.d.ts @@ -1,4 +1,5 @@ // biome-ignore-all lint/style/useNamingConvention: spec-canonical fork names in `ForkName` + interface BeaconBlockHeader { slot: number; proposerIndex: number; @@ -269,6 +270,7 @@ export declare class BeaconStateView { getBalance(index: number): number; getValidator(index: number): Validator; getAllValidators(): Validator[]; + getBuildersLength(): number; getAllBalances(): number[]; getValidatorsByStatus(statuses: Set, currentEpoch: number): Validator[]; // TODO wrong function @@ -364,6 +366,7 @@ export declare class BeaconStateView { createMultiProof(descriptor: Uint8Array): CompactMultiProof; processSlots(slot: number, options?: ProcessSlotsOpts): BeaconStateView; + stateTransition(signedBlockBytes: Uint8Array, options?: TransitionOpts): BeaconStateView; } declare const bindings: { @@ -377,11 +380,7 @@ declare const bindings: { innerShuffleList: (out: Uint32Array, seed: Uint8Array, rounds: number, forwards: boolean) => void; }; stateTransition: { - stateTransition: ( - preState: BeaconStateView, - signedBlockBytes: Uint8Array, - options?: TransitionOpts - ) => BeaconStateView; + deinitReusedEpochTransitionCache: () => void; }; metrics: { init: () => void; diff --git a/bindings/src/pubkeys.d.ts b/bindings/src/pubkeys.d.ts index ad94b9eec..1a7c647d9 100644 --- a/bindings/src/pubkeys.d.ts +++ b/bindings/src/pubkeys.d.ts @@ -1,7 +1,7 @@ import type {PublicKey} from "./blst.js"; export interface PubkeyCache { - /** Get deserialized PublicKey by validator index (cached at JS level) */ + /** Get a cached, deserialized PublicKey. */ get(index: number): PublicKey | undefined; /** Same as get(), but throws if the index is not in the cache */ getOrThrow(index: number): PublicKey; @@ -9,17 +9,25 @@ export interface PubkeyCache { aggregate(indices: number[]): PublicKey; /** Get validator index by pubkey bytes */ getIndex(pubkey: Uint8Array): number | null; - /** Set both directions atomically — impl owns the PublicKey.fromBytes() deserialization */ - set(index: number, pubkey: Uint8Array): void; - /** Number of entries */ + /** Append at the next index. Exact replays are no-ops; invalid, conflicting, duplicate, or sparse entries throw. */ + append(index: number, pubkey: Uint8Array): void; + /** Populate the cache from the missing suffix of a validator list. */ + syncPubkeys(validators: {pubkey: Uint8Array}[]): void; + /** Current number of cached entries. */ readonly size: number; - /** Load cache from a PKIX file (clears JS-level cache) */ - load(filepath: string): void; - /** Clear native and JS-level cache contents */ + /** Number of entries the current native allocation can hold without growing. */ + readonly capacity: number; + /** + * Load a compatible PKIX file from the control environment while no workers are using the cache. + * The explicit capacity limit bounds both the entry count and native allocation; + * spare capacity recorded in the file is discarded above this limit. + */ + load(filepath: string, maxCapacity: number): void; + /** Testing-only reset from the control environment while no workers are using the cache. */ reset(): void; - /** Save cache to a PKIX file */ + /** Save from the control environment. */ save(filepath: string): void; - /** Pre-allocate native capacity */ + /** Reserve exact native capacity when growing; existing larger capacity is retained. */ ensureCapacity(capacity: number): void; } diff --git a/bindings/src/pubkeys.js b/bindings/src/pubkeys.js index 0590d6544..9ec87a1e0 100644 --- a/bindings/src/pubkeys.js +++ b/bindings/src/pubkeys.js @@ -34,24 +34,30 @@ export const pubkeyCache = { return native.getIndex(pubkey); }, - set(index, pubkey) { - native.set(index, pubkey); - // Invalidate cached JS object so next get() picks up the new native value - pkCache.delete(index); + append(index, pubkey) { + native.append(index, pubkey); + }, + + syncPubkeys(validators) { + native.syncPubkeys(validators); }, get size() { return native.size(); }, - load(filepath) { + get capacity() { + return native.capacity(); + }, + + load(filepath, maxCapacity) { + native.load(filepath, maxCapacity); pkCache.clear(); - native.load(filepath); }, reset() { - pkCache.clear(); native.reset(); + pkCache.clear(); }, save(filepath) { diff --git a/bindings/src/state-transition.d.ts b/bindings/src/state-transition.d.ts index 894ef1efc..1989ca9ff 100644 --- a/bindings/src/state-transition.d.ts +++ b/bindings/src/state-transition.d.ts @@ -7,10 +7,4 @@ export type { VoluntaryExitValidity, } from "./index.js"; -import type {BeaconStateView, TransitionOpts} from "./index.js"; - -export declare function stateTransition( - preState: BeaconStateView, - signedBlockBytes: Uint8Array, - options?: TransitionOpts -): BeaconStateView; +export declare function deinitReusedEpochTransitionCache(): void; diff --git a/bindings/src/state-transition.js b/bindings/src/state-transition.js index 6a5c0cab7..1e4fcbf22 100644 --- a/bindings/src/state-transition.js +++ b/bindings/src/state-transition.js @@ -1,6 +1,4 @@ -import bindings from "./bindings.js"; - -const native = bindings.stateTransition; +import bindings from "./index.js"; export const BeaconStateView = bindings.BeaconStateView; -export const stateTransition = native.stateTransition; +export const deinitReusedEpochTransitionCache = bindings.stateTransition.deinitReusedEpochTransitionCache; diff --git a/bindings/test/beaconStateView.test.ts b/bindings/test/beaconStateView.test.ts index 3a60d7e36..c610d2fa8 100644 --- a/bindings/test/beaconStateView.test.ts +++ b/bindings/test/beaconStateView.test.ts @@ -1,52 +1,18 @@ -import {SecretKey} from "@chainsafe/blst"; import {config} from "@lodestar/config/default"; import * as era from "@lodestar/era"; import {computeEpochAtSlot} from "@lodestar/state-transition"; -import {ssz} from "@lodestar/types"; +import {type phase0, ssz} from "@lodestar/types"; import {beforeAll, describe, expect, it} from "vitest"; import bindings from "../src/index.js"; import {getFirstEraFilePath} from "./eraFiles.ts"; -// TODO(bing): it's kinda annoying to have to do this, i guess we -// expose the config somehow maybe? -/* Mainnet preset constants the binding is compiled against. */ -const SYNC_COMMITTEE_SIZE = 512; -const FAR_FUTURE_EPOCH = Number.MAX_SAFE_INTEGER; -const MAX_EFFECTIVE_BALANCE = 32_000_000_000; - -const VALIDATOR_COUNT = 16; - -interface Validator { - pubkey: Uint8Array; - withdrawalCredentials: Uint8Array; - effectiveBalance: number; - slashed: boolean; - activationEligibilityEpoch: number; - activationEpoch: number; - exitEpoch: number; - withdrawableEpoch: number; -} - -function makeValidators(count: number): Validator[] { - return Array.from({length: count}, (_, i) => { - const seed = new Uint8Array(32); - new DataView(seed.buffer).setUint32(0, i + 1); - return { - activationEligibilityEpoch: 0, - activationEpoch: 0, - effectiveBalance: MAX_EFFECTIVE_BALANCE, - exitEpoch: FAR_FUTURE_EPOCH, - pubkey: SecretKey.fromKeygen(seed).toPublicKey().toBytes(), - slashed: false, - withdrawableEpoch: FAR_FUTURE_EPOCH, - withdrawalCredentials: new Uint8Array(32), - }; - }); -} +const MAINNET_PUBKEY_CACHE_LIMIT = 2_000_000; +const SYNTHETIC_VALIDATOR_COUNT = 16; describe("BeaconStateView", () => { let state: InstanceType; let stateBytes: Uint8Array; + let syntheticValidators: phase0.Validator[]; let expected: { slot: number; genesisTime: number; @@ -102,7 +68,20 @@ describe("BeaconStateView", () => { // before creating the native state to avoid OOM on CI. { const lodestarState = ssz.fulu.BeaconState.deserializeToView(stateBytes); - const v0 = lodestarState.validators.get(0); + syntheticValidators = Array.from({length: SYNTHETIC_VALIDATOR_COUNT}, (_, index) => { + const validator = lodestarState.validators.get(index); + return { + activationEligibilityEpoch: validator.activationEligibilityEpoch, + activationEpoch: validator.activationEpoch, + effectiveBalance: validator.effectiveBalance, + exitEpoch: validator.exitEpoch, + pubkey: Uint8Array.from(validator.pubkey), + slashed: validator.slashed, + withdrawableEpoch: validator.withdrawableEpoch, + withdrawalCredentials: Uint8Array.from(validator.withdrawalCredentials), + }; + }); + const v0 = syntheticValidators[0]; expected = { balance0: lodestarState.balances.get(0), balance100: lodestarState.balances.get(100), @@ -153,16 +132,7 @@ describe("BeaconStateView", () => { root: Uint8Array.from(lodestarState.previousJustifiedCheckpoint.root), }, slot: lodestarState.slot, - validator0: { - activationEligibilityEpoch: v0.activationEligibilityEpoch, - activationEpoch: v0.activationEpoch, - effectiveBalance: v0.effectiveBalance, - exitEpoch: v0.exitEpoch, - pubkey: Uint8Array.from(v0.pubkey), - slashed: v0.slashed, - withdrawableEpoch: v0.withdrawableEpoch, - withdrawalCredentials: Uint8Array.from(v0.withdrawalCredentials), - }, + validator0: v0, validatorCount: lodestarState.validators.length, }; } @@ -171,11 +141,11 @@ describe("BeaconStateView", () => { // Phase 2: Create native BeaconStateView bindings.pool.ensureCapacity(10_000_000); - bindings.pubkeys.ensureCapacity(2_000_000); try { - bindings.pubkeys.load("./mainnet.pkix"); + bindings.pubkeys.load("./mainnet.pkix", MAINNET_PUBKEY_CACHE_LIMIT); } catch (_e) { - // ignore error + // Rebuild incompatible or corrupt snapshots from the serialized state. + bindings.pubkeys.ensureCapacity(MAINNET_PUBKEY_CACHE_LIMIT); } state = bindings.BeaconStateView.createFromBytes(stateBytes); }, 120_000); // 2 minute timeout for loading era file @@ -319,31 +289,27 @@ describe("BeaconStateView", () => { }); describe("isExecutionEnabled", () => { - const validators = makeValidators(VALIDATOR_COUNT); - - // Each sync-committee pubkey must be in the global pubkey_to_index map or - // EpochCache.createFromState throws PubkeyNotFound. Round-robin our 16 validators - // across the 512 slots — repeated pubkeys are fine for the lookup. - const syncCommitteePubkeys = Array.from( - {length: SYNC_COMMITTEE_SIZE}, - (_, i) => validators[i % VALIDATOR_COUNT].pubkey - ); - const syncCommittee = { - aggregatePubkey: validators[0].pubkey, - pubkeys: syncCommitteePubkeys, - }; + let phase0View: InstanceType; + let bellatrixView: InstanceType; - const phase0State = ssz.phase0.BeaconState.defaultValue(); - const bellatrixState = ssz.bellatrix.BeaconState.defaultValue(); - bellatrixState.slot = 144896 * 32; // BELLATRIX_FORK_EPOCH * SLOTS_PER_EPOCH (mainnet) - bellatrixState.validators = validators; - bellatrixState.currentSyncCommittee = syncCommittee; - bellatrixState.nextSyncCommittee = syncCommittee; - bellatrixState.previousEpochParticipation = Array.from({length: VALIDATOR_COUNT}, () => 0); - bellatrixState.currentEpochParticipation = Array.from({length: VALIDATOR_COUNT}, () => 0); + beforeAll(() => { + const syncCommittee = { + aggregatePubkey: syntheticValidators[0].pubkey, + pubkeys: Array.from({length: 512}, (_, index) => syntheticValidators[index % SYNTHETIC_VALIDATOR_COUNT].pubkey), + }; + const bellatrixState = ssz.bellatrix.BeaconState.defaultValue(); + bellatrixState.slot = 144896 * 32; + bellatrixState.validators = syntheticValidators; + bellatrixState.currentSyncCommittee = syncCommittee; + bellatrixState.nextSyncCommittee = syncCommittee; + bellatrixState.previousEpochParticipation = Array.from({length: SYNTHETIC_VALIDATOR_COUNT}, () => 0); + bellatrixState.currentEpochParticipation = Array.from({length: SYNTHETIC_VALIDATOR_COUNT}, () => 0); - const phase0View = bindings.BeaconStateView.createFromBytes(ssz.phase0.BeaconState.serialize(phase0State)); - const bellatrixView = bindings.BeaconStateView.createFromBytes(ssz.bellatrix.BeaconState.serialize(bellatrixState)); + phase0View = bindings.BeaconStateView.createFromBytes( + ssz.phase0.BeaconState.serialize(ssz.phase0.BeaconState.defaultValue()) + ); + bellatrixView = bindings.BeaconStateView.createFromBytes(ssz.bellatrix.BeaconState.serialize(bellatrixState)); + }); it("should true on post-merge state without reading the block", () => { // body is empty — binding short-circuits before touching it. @@ -450,6 +416,12 @@ describe("BeaconStateView", () => { }); }); + describe("gloas+ fields", () => { + it("getBuildersLength should throw on a pre-Gloas state", () => { + expect(() => state.getBuildersLength()).toThrow(); + }); + }); + describe("block and state roots", () => { it("getBlockRoot should return 32 bytes", () => { const blockRoot = state.getBlockRoot(state.epoch - 1); @@ -699,15 +671,13 @@ describe("BeaconStateView", () => { {dataAvailabilityStatus: "available"}, // TS enum value is "Available" ]; for (const opts of invalidOpts) { - expect(() => bindings.stateTransition.stateTransition(state, dummyBlockBytes, opts)).toThrow(); + expect(() => state.stateTransition(dummyBlockBytes, opts)).toThrow(); } }); // TODO: remove once Zig models DataAvailabilityStatus.NotRequired it("rejects gloas-only NotRequired", () => { - expect(() => - bindings.stateTransition.stateTransition(state, dummyBlockBytes, {dataAvailabilityStatus: "NotRequired"}) - ).toThrow(); + expect(() => state.stateTransition(dummyBlockBytes, {dataAvailabilityStatus: "NotRequired"})).toThrow(); }); }); diff --git a/bindings/test/blst.test.ts b/bindings/test/blst.test.ts index c98d2c310..7d32df6f1 100644 --- a/bindings/test/blst.test.ts +++ b/bindings/test/blst.test.ts @@ -88,6 +88,18 @@ describe("blst", () => { }); }); + describe("fromHex()", () => { + const G2_POINT_AT_INFINITY = `c0${"00".repeat(95)}`; + + it("should check infinity by default", () => { + expect(() => Signature.fromHex(G2_POINT_AT_INFINITY, true)).toThrow("PkIsInfinity"); + }); + + it("should skip the infinity check when explicitly disabled", () => { + expect(Signature.fromHex(G2_POINT_AT_INFINITY, true, false)).toBeInstanceOf(Signature); + }); + }); + it("should serialize to bytes", () => { const sig = Signature.fromBytes(fromHex(TEST_VECTORS.signature.compressed)); const bytes = sig.toBytes(); @@ -110,6 +122,16 @@ describe("blst", () => { it("should throw on invalid length", () => { expect(() => Signature.fromBytes(new Uint8Array(95))).toThrow(); }); + + it("should reject PublicKey objects when aggregating", () => { + const pk = PublicKey.fromHex(TEST_VECTORS.publicKey.compressed); + expect(() => Signature.aggregate([pk as unknown as Signature], false)).toThrow("TypeMismatch"); + }); + + it("should aggregate Signature objects", () => { + const signatures = getTestSets(2).map((set) => set.sig); + expect(Signature.aggregate(signatures, false)).toBeInstanceOf(Signature); + }); }); describe("SecretKey", () => { @@ -155,6 +177,30 @@ describe("blst", () => { } }); }); + describe("SecretKey.fromHex", () => { + for (const prefix of ["", "0x"]) { + it(`should round-trip a valid key with prefix '${prefix}'`, () => { + const hex = `${prefix}${Buffer.from(SECRET_KEY_BYTES).toString("hex")}`; + expectEqualHex(SecretKey.fromHex(hex).toBytes(), SECRET_KEY_BYTES); + }); + } + + for (const byteLength of [0, 1, 31, 33]) { + for (const prefix of ["", "0x"]) { + it(`should throw on ${byteLength}-byte input with prefix '${prefix}'`, () => { + expect(() => SecretKey.fromHex(`${prefix}${"00".repeat(byteLength)}`)).toThrow("InvalidSecretKeyLength"); + }); + } + } + + it("should throw on an odd number of hex characters", () => { + expect(() => SecretKey.fromHex("0".repeat(63))).toThrow("InvalidSecretKeyLength"); + }); + + it("should throw on non-hex UTF-8 input without aborting", () => { + expect(() => SecretKey.fromHex("é".repeat(32))).toThrow(); + }); + }); describe("instance methods", () => { let key: SecretKey; describe("toBytes", () => { @@ -188,11 +234,18 @@ describe("blst", () => { }); }); describe("sign", () => { - it("should create a valid Signature", () => { - const sig = SecretKey.fromKeygen(KEY_MATERIAL, undefined).sign(Buffer.from("some fancy message")); + it("should create a valid Signature for a 32-byte signing root", () => { + const sig = SecretKey.fromKeygen(KEY_MATERIAL, undefined).sign(new Uint8Array(32)); expect(sig).to.be.instanceOf(Signature); expect(sig.validate(false)).to.be.undefined; }); + + for (const length of [31, 33]) { + it(`should throw InvalidMessageLength for a ${length}-byte signing root`, () => { + const sk = SecretKey.fromKeygen(KEY_MATERIAL, undefined); + expect(() => sk.sign(new Uint8Array(length))).toThrow("InvalidMessageLength"); + }); + } }); }); }); @@ -212,6 +265,14 @@ describe("blst", () => { const result = verify(wrongMessage, pk, sig, false, false); expect(result).toBe(false); }); + + for (const length of [31, 33]) { + it(`should throw InvalidMessageLength for a ${length}-byte signing root`, () => { + const pk = PublicKey.fromHex(TEST_VECTORS.publicKey.compressed); + const sig = Signature.fromHex(TEST_VECTORS.signature.compressed); + expect(() => verify(new Uint8Array(length), pk, sig, false, false)).toThrow("InvalidMessageLength"); + }); + } }); describe("aggregateVerify", () => { @@ -232,6 +293,14 @@ describe("blst", () => { const sig = Signature.fromHex(TEST_VECTORS.signature.compressed); expect(aggregateVerify([TEST_VECTORS.message], [pk], sig)).to.be.true; }); + + for (const length of [31, 33]) { + it(`should throw InvalidMessageLength for a ${length}-byte signing root`, () => { + const pk = PublicKey.fromHex(TEST_VECTORS.publicKey.compressed); + const sig = Signature.fromHex(TEST_VECTORS.signature.compressed); + expect(() => aggregateVerify([new Uint8Array(length)], [pk], sig)).toThrow("InvalidMessageLength"); + }); + } }); describe("fastAggregateVerify", () => { @@ -256,11 +325,13 @@ describe("blst", () => { expect(result).toBe(false); }); - it("should throw on wrong message length", () => { - const pk = PublicKey.fromHex(TEST_VECTORS.publicKey.compressed); - const sig = Signature.fromHex(TEST_VECTORS.signature.compressed); - expect(() => fastAggregateVerify(new Uint8Array(31), [pk], sig, false)).toThrow(); - }); + for (const length of [31, 33]) { + it(`should throw InvalidMessageLength for a ${length}-byte signing root`, () => { + const pk = PublicKey.fromHex(TEST_VECTORS.publicKey.compressed); + const sig = Signature.fromHex(TEST_VECTORS.signature.compressed); + expect(() => fastAggregateVerify(new Uint8Array(length), [pk], sig, false)).toThrow("InvalidMessageLength"); + }); + } }); describe("verifyMultipleAggregateSignatures", () => { @@ -273,6 +344,32 @@ describe("blst", () => { sets[0].sig = sets[1].sig; expect(verifyMultipleAggregateSignatures(sets, false, false)).to.be.false; }); + + it("should reject objects of the wrong class", () => { + const [{msg, pk, sig, sk}] = getTestSets(1); + expect(() => + verifyMultipleAggregateSignatures([{msg, pk: sk as unknown as PublicKey, sig}], false, false) + ).toThrow("TypeMismatch"); + expect(() => + verifyMultipleAggregateSignatures([{msg, pk, sig: pk as unknown as Signature}], false, false) + ).toThrow("TypeMismatch"); + }); + + for (const length of [31, 33]) { + it(`should throw InvalidMessageLength for a ${length}-byte signing root`, () => { + const [set] = getTestSets(1); + expect(() => verifyMultipleAggregateSignatures([{...set, msg: new Uint8Array(length)}], false, false)).toThrow( + "InvalidMessageLength" + ); + }); + } + }); + + describe("aggregatePublicKeys", () => { + it("should reject SecretKey objects", () => { + const sk = SecretKey.fromBytes(SECRET_KEY_BYTES); + expect(() => aggregatePublicKeys([sk as unknown as PublicKey])).toThrow("TypeMismatch"); + }); }); describe("aggregateSerializedPublicKeys", () => { @@ -357,6 +454,12 @@ describe("blst", () => { input[2].sig = new Uint8Array(96).fill(0xff); expect(() => aggregateWithRandomness(input)).toThrow(); }); + + it("should reject objects of the wrong class", () => { + const {sets} = getTestSetsSameMessage(1); + const input = [{pk: sets[0].sk as unknown as PublicKey, sig: sets[0].sig.toBytes()}]; + expect(() => aggregateWithRandomness(input)).toThrow("TypeMismatch"); + }); }); describe("asyncAggregateWithRandomness", () => { @@ -433,6 +536,12 @@ describe("blst", () => { await expect(Promise.resolve().then(() => asyncAggregateWithRandomness(input))).rejects.toThrow(); }); + it("should reject objects of the wrong class", async () => { + const {sets} = getTestSetsSameMessage(1); + const input = [{pk: sets[0].sk as unknown as PublicKey, sig: sets[0].sig.toBytes()}]; + await expect(Promise.resolve().then(() => asyncAggregateWithRandomness(input))).rejects.toThrow("TypeMismatch"); + }); + it("should resolve concurrent invocations correctly", async () => { const {msg, sets} = getTestSetsSameMessage(8); const input = sets.map((s) => ({pk: s.pk, sig: s.sig.toBytes()})); diff --git a/bindings/test/demo.ts b/bindings/test/demo.ts index 5b67504e1..d29a642a2 100644 --- a/bindings/test/demo.ts +++ b/bindings/test/demo.ts @@ -4,6 +4,7 @@ import * as era from "@lodestar/era"; import bindings from "../src/index.js"; import {pubkeyCache} from "../src/pubkeys.js"; import {getEraFilePaths, getFirstEraFilePath} from "./eraFiles.ts"; +import {getPubkeyCacheCapacityForState} from "./serializedState.ts"; console.log("loaded bindings"); @@ -31,14 +32,7 @@ const hasPkix = printDuration("check for pkix file", () => { } }); -if (hasPkix) { - printDuration("load pkix from disk", () => pubkeyCache.load(PKIX_FILE)); -} else { - printDuration("update bindings capacity", () => { - bindings.pool.ensureCapacity(10_000_000); - pubkeyCache.ensureCapacity(2_000_000); - }); -} +bindings.pool.ensureCapacity(10_000_000); const reader = await printDurationAsync("load era reader", () => era.era.EraReader.open(config, getFirstEraFilePath())); @@ -47,6 +41,23 @@ const nextReader = await printDurationAsync("load era reader", () => ); const stateBytes = await printDurationAsync("read serialized state", () => reader.readSerializedState()); +const requiredPubkeyCapacity = getPubkeyCacheCapacityForState(stateBytes); + +let loadedPkix = false; +if (hasPkix) { + try { + printDuration("load pkix from disk", () => pubkeyCache.load(PKIX_FILE, requiredPubkeyCapacity)); + loadedPkix = true; + } catch (error) { + console.warn("PKIX cache is incompatible or corrupt; rebuilding it from state", error); + } +} + +if (!loadedPkix || pubkeyCache.capacity < requiredPubkeyCapacity) { + printDuration("reserve pubkey cache", () => { + pubkeyCache.ensureCapacity(requiredPubkeyCapacity); + }); +} const state = printDuration("create state view", () => bindings.BeaconStateView.createFromBytes(stateBytes)); @@ -54,7 +65,7 @@ const signedBlockBytes = (await printDurationAsync("read serialized block", () = nextReader.readSerializedBlock(state.slot + 1) )) as Uint8Array; -printDuration("state transition", () => bindings.stateTransition.stateTransition(state, signedBlockBytes)); +printDuration("state transition", () => state.stateTransition(signedBlockBytes)); printDuration("write pkix to disk", () => pubkeyCache.save(PKIX_FILE)); diff --git a/bindings/test/pubkeys.test.ts b/bindings/test/pubkeys.test.ts index b24ad8989..64578e374 100644 --- a/bindings/test/pubkeys.test.ts +++ b/bindings/test/pubkeys.test.ts @@ -1,7 +1,7 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import {afterAll, beforeAll, describe, expect, it} from "vitest"; +import {afterAll, beforeEach, describe, expect, it} from "vitest"; import {SecretKey, aggregatePublicKeys} from "../src/blst.js"; import {pubkeyCache} from "../src/pubkeys.js"; @@ -14,20 +14,41 @@ const keypairs = Array.from({length: 3}, (_, i) => { return {index: i, pubkeyBytes: pk.toBytes()}; }); +const canEnforceDirectoryPermissions = typeof process.getuid === "function" && process.getuid() !== 0; + +function seedCache(count = keypairs.length): void { + pubkeyCache.reset(); + pubkeyCache.ensureCapacity(2_000); + for (const {index, pubkeyBytes} of keypairs.slice(0, count)) { + pubkeyCache.append(index, pubkeyBytes); + } +} + +function expectCacheContents(count = keypairs.length): void { + expect(pubkeyCache.size).toBe(count); + for (const {index, pubkeyBytes} of keypairs.slice(0, count)) { + expect(pubkeyCache.getIndex(pubkeyBytes)).toBe(index); + expect(pubkeyCache.getOrThrow(index).toBytes()).toEqual(pubkeyBytes); + } + expect(pubkeyCache.get(count)).toBeUndefined(); +} + describe("pubkeys", () => { - const tempPkixPath = path.join(os.tmpdir(), `lodestar-z-pubkeys-${process.pid}-${Date.now()}.pkix`); + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "lodestar-z-pubkeys-")); + const tempPkixPath = path.join(tempDir, "cache.pkix"); - beforeAll(() => { - pubkeyCache.ensureCapacity(1_000); + beforeEach(() => { + seedCache(); }); afterAll(() => { - fs.rmSync(tempPkixPath, {force: true}); + fs.rmSync(tempDir, {force: true, recursive: true}); }); - it("set populates both directions and updates size", () => { + it("append populates both directions and updates size", () => { + pubkeyCache.reset(); for (const {index, pubkeyBytes} of keypairs) { - pubkeyCache.set(index, pubkeyBytes); + pubkeyCache.append(index, pubkeyBytes); } expect(pubkeyCache.size).toBe(keypairs.length); @@ -37,9 +58,36 @@ describe("pubkeys", () => { } }); - it("get returns the same cached object on repeated calls", () => { - const pk1 = pubkeyCache.get(0); - const pk2 = pubkeyCache.get(0); + it("syncPubkeys incrementally populates both lookup directions", () => { + pubkeyCache.reset(); + const validators = keypairs.map(({pubkeyBytes}) => ({pubkey: pubkeyBytes})); + + pubkeyCache.syncPubkeys(validators.slice(0, 1)); + pubkeyCache.syncPubkeys(validators); + + expectCacheContents(); + }); + + it("syncPubkeys accepts a historical validator list", () => { + const validators = keypairs.map(({pubkeyBytes}) => ({pubkey: pubkeyBytes})); + + pubkeyCache.syncPubkeys(validators.slice(0, 1)); + + expectCacheContents(); + }); + + it("syncPubkeys rejects an invalid suffix without publishing valid entries", () => { + pubkeyCache.reset(); + const validators = [{pubkey: keypairs[0].pubkeyBytes}, {pubkey: new Uint8Array(48)}]; + + expect(() => pubkeyCache.syncPubkeys(validators)).toThrow(); + expect(pubkeyCache.size).toBe(0); + expect(pubkeyCache.getIndex(keypairs[0].pubkeyBytes)).toBeNull(); + }); + + it("get caches deserialized values", () => { + const pk1 = pubkeyCache.getOrThrow(0); + const pk2 = pubkeyCache.getOrThrow(0); expect(pk1).toBe(pk2); }); @@ -65,41 +113,126 @@ describe("pubkeys", () => { expect(() => pubkeyCache.getIndex(new Uint8Array(47))).toThrow(); }); - it("set invalidates cached JS object", () => { - const before = pubkeyCache.get(0); - pubkeyCache.set(0, keypairs[0].pubkeyBytes); - const after = pubkeyCache.get(0); - expect(before).not.toBe(after); + it("accepts identical replays and rejects conflicting replays", () => { + const before = pubkeyCache.getOrThrow(0); + pubkeyCache.append(0, keypairs[0].pubkeyBytes); + expect(() => pubkeyCache.append(0, keypairs[1].pubkeyBytes)).toThrow("ConflictingPubkey"); + expect(() => pubkeyCache.append(0, new Uint8Array(1))).toThrow("InvalidPubkeyLength"); + + const after = pubkeyCache.getOrThrow(0); + expect(after).toBe(before); + expect(after.toBytes()).toEqual(keypairs[0].pubkeyBytes); + }); + + it("reset clears both lookup directions and invalidates cached values", () => { + const before = pubkeyCache.getOrThrow(0); + + pubkeyCache.reset(); + expect(pubkeyCache.size).toBe(0); + expect(pubkeyCache.get(0)).toBeUndefined(); + expect(pubkeyCache.getIndex(keypairs[0].pubkeyBytes)).toBeNull(); + + pubkeyCache.append(0, keypairs[1].pubkeyBytes); + + const after = pubkeyCache.getOrThrow(0); + expect(after).not.toBe(before); + expect(after.toBytes()).toEqual(keypairs[1].pubkeyBytes); }); - it("save/load roundtrips cache contents", () => { + it("save replaces an existing file and load restores cache contents", () => { + fs.writeFileSync(tempPkixPath, "stale"); pubkeyCache.save(tempPkixPath); - pubkeyCache.load(tempPkixPath); - expect(pubkeyCache.size).toBe(keypairs.length); - for (const {index, pubkeyBytes} of keypairs) { - expect(pubkeyCache.getIndex(pubkeyBytes)).toBe(index); - expect(pubkeyCache.get(index)).toBeDefined(); + pubkeyCache.reset(); + pubkeyCache.append(0, keypairs[1].pubkeyBytes); + const beforeLoad = pubkeyCache.getOrThrow(0); + + pubkeyCache.load(tempPkixPath, pubkeyCache.capacity); + + expectCacheContents(); + expect(pubkeyCache.getOrThrow(0)).not.toBe(beforeLoad); + }); + + it.runIf(process.platform === "linux")("save/load preserves a path longer than the former stack buffer", () => { + let longDirectory = tempDir; + for (let i = 0; i < 10; i++) { + longDirectory = path.join(longDirectory, `${i.toString().padStart(2, "0")}-${"x".repeat(110)}`); } + fs.mkdirSync(longDirectory, {recursive: true}); + const longPkixPath = path.join(longDirectory, "cache.pkix"); + expect(Buffer.byteLength(longPkixPath)).toBeGreaterThan(1_023); + + pubkeyCache.save(longPkixPath); + expect(fs.existsSync(longPkixPath)).toBe(true); + + seedCache(1); + pubkeyCache.load(longPkixPath, 2_000); + expectCacheContents(); + }); + + it("enforces the caller's load allocation limit", () => { + pubkeyCache.save(tempPkixPath); + + expect(() => pubkeyCache.load(tempPkixPath, pubkeyCache.size - 1)).toThrow(); + expectCacheContents(); + + const entryCount = pubkeyCache.size; + expect(pubkeyCache.capacity).toBeGreaterThan(entryCount); + pubkeyCache.load(tempPkixPath, entryCount); + + expect(pubkeyCache.capacity).toBe(entryCount); + expectCacheContents(); }); - it("load clears JS-level cache", () => { - const before = pubkeyCache.get(0); + it("rejects an invalid snapshot without replacing the live cache", () => { pubkeyCache.save(tempPkixPath); - pubkeyCache.load(tempPkixPath); - const after = pubkeyCache.get(0); - expect(before).not.toBe(after); + const invalid = Buffer.from(fs.readFileSync(tempPkixPath)); + invalid[invalid.length - 1] ^= 0xff; + const invalidPath = path.join(tempDir, "invalid.pkix"); + fs.writeFileSync(invalidPath, invalid); + + expect(() => pubkeyCache.load(invalidPath, pubkeyCache.capacity)).toThrow(); + expectCacheContents(); }); - it("reset clears native and JS-level cache", () => { - const before = pubkeyCache.get(0); - expect(before).toBeDefined(); - expect(pubkeyCache.getIndex(keypairs[0].pubkeyBytes)).toBeDefined(); + it.runIf(canEnforceDirectoryPermissions)( + "preserves an existing cache file when an atomic save cannot create its sibling file", + () => { + const readOnlyDir = path.join(tempDir, "read-only"); + const destination = path.join(readOnlyDir, "cache.pkix"); + fs.mkdirSync(readOnlyDir); + + pubkeyCache.save(destination); + const original = fs.readFileSync(destination); + seedCache(1); + + fs.chmodSync(readOnlyDir, 0o555); + try { + expect(() => pubkeyCache.save(destination)).toThrow(); + } finally { + fs.chmodSync(readOnlyDir, 0o755); + } + + expect(fs.readFileSync(destination)).toEqual(original); + expect(fs.readdirSync(readOnlyDir)).toEqual(["cache.pkix"]); + + pubkeyCache.load(destination, pubkeyCache.capacity); + expectCacheContents(); + } + ); - pubkeyCache.reset(); + it("reserves exact capacity after publishing entries without changing the cache", () => { + const capacityBefore = pubkeyCache.capacity; + const requestedCapacity = capacityBefore + 1; - expect(pubkeyCache.size).toBe(0); - expect(pubkeyCache.get(0)).toBeUndefined(); - expect(pubkeyCache.getIndex(keypairs[0].pubkeyBytes)).toBeNull(); + pubkeyCache.ensureCapacity(requestedCapacity); + expect(pubkeyCache.capacity).toBe(requestedCapacity); + expectCacheContents(); + }); + + it("rejects sparse inserts without changing the cache", () => { + const sizeBefore = pubkeyCache.size; + expect(() => pubkeyCache.append(pubkeyCache.capacity, keypairs[0].pubkeyBytes)).toThrow(); + expect(pubkeyCache.size).toBe(sizeBefore); }); }); diff --git a/bindings/test/serializedState.ts b/bindings/test/serializedState.ts new file mode 100644 index 000000000..ebd95156e --- /dev/null +++ b/bindings/test/serializedState.ts @@ -0,0 +1,29 @@ +import {ssz} from "@lodestar/types"; + +/** Ninety days of worst-case validator-registry growth on the mainnet preset. */ +export const DEFAULT_PUBKEY_CACHE_HEADROOM = 324_000; + +const validatorFieldIndex = Object.keys(ssz.fulu.BeaconState.fields).indexOf("validators"); +const validatorSize = ssz.phase0.Validator.fixedSize; + +if (validatorFieldIndex === -1 || validatorSize === null) { + throw new Error("Unable to locate the fixed-size validator list in Fulu BeaconState SSZ"); +} + +/** Read the validator count directly from SSZ offsets without constructing a tree view. */ +export function getSerializedFuluValidatorCount(stateBytes: Uint8Array): number { + const dataView = new DataView(stateBytes.buffer, stateBytes.byteOffset, stateBytes.byteLength); + const range = ssz.fulu.BeaconState.getFieldRanges(dataView, 0, stateBytes.byteLength)[validatorFieldIndex]; + const byteLength = range.end - range.start; + if (byteLength % validatorSize !== 0) { + throw new Error(`Invalid serialized validator list length: ${byteLength}`); + } + return byteLength / validatorSize; +} + +export function getPubkeyCacheCapacityForState( + stateBytes: Uint8Array, + headroom = DEFAULT_PUBKEY_CACHE_HEADROOM +): number { + return getSerializedFuluValidatorCount(stateBytes) + headroom; +} diff --git a/bindings/test/teardown.test.ts b/bindings/test/teardown.test.ts index 15faa5a85..884f967d5 100644 --- a/bindings/test/teardown.test.ts +++ b/bindings/test/teardown.test.ts @@ -17,13 +17,14 @@ import {config} from "@lodestar/config/default"; import * as era from "@lodestar/era"; import bindings from "../src/index.js"; import {getFirstEraFilePath} from "./eraFiles.ts"; +import {getSerializedFuluValidatorCount} from "./serializedState.ts"; const reader = await era.era.EraReader.open(config, getFirstEraFilePath()); const stateBytes = await reader.readSerializedState(); await reader.close(); bindings.pool.ensureCapacity(10_000_000); -bindings.pubkeys.ensureCapacity(2_000_000); +bindings.pubkeys.ensureCapacity(getSerializedFuluValidatorCount(stateBytes)); const seedState = bindings.BeaconStateView.createFromBytes(stateBytes); console.log("slot=" + seedState.slot); diff --git a/bindings/test/worker-isolation.test.ts b/bindings/test/worker-isolation.test.ts index 9bf4e940c..e9d59e4b9 100644 --- a/bindings/test/worker-isolation.test.ts +++ b/bindings/test/worker-isolation.test.ts @@ -2,6 +2,7 @@ import crypto from "node:crypto"; import {Worker} from "node:worker_threads"; import {describe, expect, it} from "vitest"; import {PublicKey, SecretKey, Signature, verify} from "../src/blst.js"; +import {pubkeyCache} from "../src/pubkeys.js"; /** * Tests that the per-context instance data (blst InstanceData) and @@ -21,7 +22,7 @@ describe("worker isolation", () => { expect(verify(msg, pk, sig)).toBe(true); // 2. Spawn a worker that loads bindings, does blst work, then exits - const workerResult = await runWorker(); + const workerResult = await runBlstWorker(); expect(workerResult).toBe("ok"); // 3. After the worker's env teardown + cleanup hooks have fired, @@ -45,21 +46,63 @@ describe("worker isolation", () => { const sig = sk.sign(msg); for (let i = 0; i < 3; i++) { - const result = await runWorker(); + const result = await runBlstWorker(); expect(result).toBe("ok"); // Main thread still works after each worker teardown expect(verify(msg, pk, sig)).toBe(true); } }); + + it("shares cache reads with workers but restricts administration to the control environment", async () => { + const ikm = new Uint8Array(32); + ikm[0] = 1; + const expected = SecretKey.fromKeygen(ikm).toPublicKey().toBytes(); + pubkeyCache.reset(); + pubkeyCache.append(0, expected); + + const result = await runWorker<{ + pubkey: Uint8Array; + save: string | null; + load: string | null; + reset: string | null; + }>(` + import {parentPort} from "node:worker_threads"; + import {pubkeyCache} from ${JSON.stringify(pubkeysModulePath)}; + + function capture(operation) { + try { + operation(); + return null; + } catch (error) { + return String(error?.message ?? error); + } + } + + parentPort.postMessage({ + pubkey: pubkeyCache.getOrThrow(0).toBytes(), + save: capture(() => pubkeyCache.save("")), + load: capture(() => pubkeyCache.load("", 1)), + reset: capture(() => pubkeyCache.reset()), + }); + `); + + expect(result).toEqual({ + load: "PubkeyCacheControlEnvironmentOnly", + pubkey: expected, + reset: "PubkeyCacheControlEnvironmentOnly", + save: "PubkeyCacheControlEnvironmentOnly", + }); + expect(pubkeyCache.size).toBe(1); + expect(pubkeyCache.getOrThrow(0).toBytes()).toEqual(expected); + }); }); +const pubkeysModulePath = new URL("../src/pubkeys.js", import.meta.url).href; const blstModulePath = new URL("../src/blst.js", import.meta.url).href; -function runWorker(): Promise { - return new Promise((resolve, reject) => { - const worker = new Worker( - ` +function runBlstWorker(): Promise { + return runWorker(` import crypto from "node:crypto"; import {parentPort} from "node:worker_threads"; import {SecretKey, verify} from "${blstModulePath}"; @@ -78,14 +121,28 @@ function runWorker(): Promise { } catch (e) { parentPort.postMessage("error: " + e.message); } - `, - {eval: true} - ); + `); +} - worker.on("message", resolve); +function runWorker(source: string): Promise { + return new Promise((resolve, reject) => { + let received = false; + let result: T; + const worker = new Worker(source, {eval: true}); + + worker.on("message", (message: T) => { + received = true; + result = message; + }); worker.on("error", reject); worker.on("exit", (code) => { - if (code !== 0) reject(new Error(`Worker exited with code ${code}`)); + if (code !== 0) { + reject(new Error(`Worker exited with code ${code}`)); + } else if (!received) { + reject(new Error("Worker exited without a result")); + } else { + resolve(result); + } }); }); } diff --git a/build.zig.zon b/build.zig.zon index d844799f7..722ca67c7 100644 --- a/build.zig.zon +++ b/build.zig.zon @@ -1,7 +1,7 @@ .{ - .name = .state_transition_z, + .name = .lodestar_z, .version = "0.0.0", - .fingerprint = 0xd1ddb6abe067ee04, + .fingerprint = 0xfe5cc37f635c4ea4, .minimum_zig_version = "0.16.0", .paths = .{ "build.zig", @@ -28,8 +28,8 @@ }, }, .hashtree = .{ - .url = "git+https://github.com/ChainSafe/hashtree-z.git#81b7303f72e202e966f71eedf4be1d8d33e2fcaa", - .hash = "hashtree-0.1.0-sBOoviYWAAA1HcTLhxWt0TtrLFXvYM5gOvxpu_Rreiiw", + .url = "git+https://github.com/ChainSafe/hashtree-z.git#f5c1b81da167f84376fb94aef29a57fc6c08dc28", + .hash = "hashtree-0.1.0-sBOovj8WAAAGCN62cybnffy7b4ITTQsog2oOG2AQZl9L", }, .snappy = .{ .url = "git+https://github.com/ChainSafe/snappy.zig#713942410426051da6107a955b29f3f152815622", @@ -40,8 +40,8 @@ .hash = "zig_yaml-0.1.0-C1161kFWAwDxjKAFmklKwWVDvz2mmq0Q__bDhGGjeyd3", }, .zapi = .{ - .url = "https://github.com/ChainSafe/zapi/archive/refs/tags/zapi-v2.2.0.tar.gz", - .hash = "zapi-2.2.0-rIqzUXBaBADUsvvvB2N5kOBdLEO0rGPhowTqmIXe3qVh", + .url = "https://github.com/ChainSafe/zapi/archive/ab42ab08de92e1ea1f9f8a3f124e5c4bf55d564d.tar.gz", + .hash = "zapi-3.1.0-rIqzUXatBABpDFl_itCopyVIJdcEHttbhye7WfTNoHCR", }, .zbench = .{ .url = "git+https://github.com/hendriknielaender/zBench#b2b89c475e3ef1bb2bd71255c80478a82d3e0ca8", @@ -211,6 +211,10 @@ .root_source_file = "src/fork_choice/root.zig", .imports = .{ .consensus_types, .config, .preset, .state_transition, .fork_types, .hex, .constants, .metrics, .time }, }, + .beacon_node = .{ + .root_source_file = "src/beacon_node/root.zig", + .imports = .{ .consensus_types, .state_transition, .persistent_merkle_tree, .testing_allocators, .metrics, .hex, .time }, + }, }, .executables = .{ .download_era_files = .{ @@ -385,6 +389,7 @@ .era = .{ .root_module = .era }, .fork_choice = .{ .root_module = .fork_choice }, .clock = .{ .root_module = .clock }, + .beacon_node = .{ .root_module = .beacon_node }, .int = .{ .root_module = .{ diff --git a/examples/metrics.ts b/examples/metrics.ts index 26e64370a..2f0db605a 100644 --- a/examples/metrics.ts +++ b/examples/metrics.ts @@ -31,6 +31,7 @@ import { config } from "@lodestar/config/default"; import * as era from "@lodestar/era"; import bindings from "../bindings/src/index.js"; import { getFirstEraFilePath, getEraFilePaths } from "../bindings/test/eraFiles.ts"; +import { getPubkeyCacheCapacityForState } from "../bindings/test/serializedState.ts"; const PORT = 8008; const PKIX_FILE = "./mainnet.pkix"; @@ -44,14 +45,8 @@ const hasPkix = (() => { } })(); -if (hasPkix) { - console.log("Loading pkix cache from disk..."); - bindings.pubkeys.load(PKIX_FILE); -} else { - console.log("No pkix cache found, initializing pool and pkix cache..."); - bindings.pool.ensureCapacity(10_000_000); - bindings.pubkeys.ensureCapacity(2_000_000); -} +console.log("Initializing pool..."); +bindings.pool.ensureCapacity(10_000_000); console.log("Initializing metrics..."); bindings.metrics.init(); @@ -62,11 +57,32 @@ const nextReader = await era.era.EraReader.open(config, getEraFilePaths()[1]); console.log("Reading serialized state..."); const stateBytes = await reader.readSerializedState(); +const requiredPubkeyCapacity = getPubkeyCacheCapacityForState(stateBytes); + +let loadedPkix = false; +if (hasPkix) { + console.log("Loading pkix cache from disk..."); + try { + bindings.pubkeys.load(PKIX_FILE, requiredPubkeyCapacity); + loadedPkix = true; + } catch (error) { + console.warn("PKIX cache is incompatible or corrupt; rebuilding it from state", error); + } +} + +let pkixNeedsSave = !loadedPkix; +if (!loadedPkix || bindings.pubkeys.capacity() < requiredPubkeyCapacity) { + console.log("Reserving pubkey cache..."); + bindings.pubkeys.ensureCapacity(requiredPubkeyCapacity); + pkixNeedsSave = true; +} +const cachedPubkeyCount = bindings.pubkeys.size(); console.log("Creating BeaconStateView..."); var state = bindings.BeaconStateView.createFromBytes(stateBytes); +pkixNeedsSave ||= bindings.pubkeys.size() !== cachedPubkeyCount; -if (!hasPkix) { +if (pkixNeedsSave) { console.log("Saving pkix cache to disk..."); bindings.pubkeys.save(PKIX_FILE); } diff --git a/examples/metrics.zig b/examples/metrics.zig index 1b92b3df5..b5c78628d 100644 --- a/examples/metrics.zig +++ b/examples/metrics.zig @@ -22,12 +22,11 @@ const types = @import("consensus_types"); const CachedBeaconState = state_transition.CachedBeaconState; const AnySignedBeaconBlock = @import("fork_types").AnySignedBeaconBlock; const active_preset = @import("preset").active_preset; +const preset = @import("preset").preset; const mainnet_chain_config = @import("config").mainnet.chain_config; const minimal_chain_config = @import("config").minimal.chain_config; const BeaconConfig = @import("config").BeaconConfig; const ValidatorIndex = @import("consensus_types").primitive.ValidatorIndex.Type; -const Index2PubkeyCache = state_transition.Index2PubkeyCache; -const PubkeyIndexMap = state_transition.PubkeyIndexMap; const chain_config = if (active_preset == .mainnet) mainnet_chain_config else minimal_chain_config; const MetricsHandler = struct { @@ -101,28 +100,36 @@ pub fn main(init: std.process.Init) !void { errdefer allocator.destroy(state_ptr); state_ptr.* = try reader_state.readState(allocator, null); const blocks_index = reader_blocks.group_indices[0].blocks_index orelse return error.NoBlockIndex; - const index_pubkey_cache = try allocator.create(Index2PubkeyCache); - errdefer { - index_pubkey_cache.deinit(allocator); - allocator.destroy(index_pubkey_cache); - } - index_pubkey_cache.* = Index2PubkeyCache.empty; - var pubkey_index_map = PubkeyIndexMap.init(allocator); - errdefer pubkey_index_map.deinit(); + const max_new_validators = try std.math.mul( + usize, + blocks_index.offsets.len, + preset.MAX_DEPOSITS, + ); + const pubkey_capacity = try std.math.add( + usize, + try state_ptr.validatorsCount(), + max_new_validators, + ); + var pubkey_cache = try state_transition.PubkeyCache.initCapacity( + allocator, + io, + pubkey_capacity, + ); + defer pubkey_cache.deinit(); const config = try allocator.create(BeaconConfig); - errdefer allocator.destroy(config); + defer allocator.destroy(config); config.* = BeaconConfig.init(chain_config, (try state_ptr.genesisValidatorsRoot()).*); const immutable_data = state_transition.EpochCacheImmutableData{ .config = config, - .index_to_pubkey = index_pubkey_cache, - .pubkey_to_index = &pubkey_index_map, + .pubkey_cache = &pubkey_cache, }; std.debug.print("Creating cached beacon state\n", .{}); var cached_state = try CachedBeaconState.createCachedBeaconState( allocator, + io, state_ptr, immutable_data, .{ @@ -130,6 +137,10 @@ pub fn main(init: std.process.Init) !void { .skip_sync_pubkeys = false, }, ); + defer { + cached_state.deinit(); + allocator.destroy(cached_state); + } std.debug.print("Running state transition.\nYou may open up a local prometheus instance to check out metrics in action.\n", .{}); for (blocks_index.start_slot + 1..blocks_index.start_slot + blocks_index.offsets.len) |slot| { const block = try reader_blocks.readBlock(allocator, slot) orelse continue; diff --git a/package.json b/package.json index 870ff0631..518a97f22 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@chainsafe/lodestar-z", - "version": "0.1.2-rc.9", + "version": "0.1.2", "description": "Lodestar-z NAPI bindings", "files": [ "bindings/src/", @@ -50,13 +50,14 @@ "targets": [ "aarch64-apple-darwin", "aarch64-unknown-linux-gnu", + "aarch64-unknown-linux-musl", "x86_64-apple-darwin", "x86_64-unknown-linux-gnu", "x86_64-unknown-linux-musl" ] }, "dependencies": { - "@chainsafe/zapi": "2.0.0" + "@chainsafe/zapi": "3.1.0" }, "devDependencies": { "@biomejs/biome": "^2.3.11", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 756e2304d..b57408b12 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -9,8 +9,8 @@ importers: .: dependencies: '@chainsafe/zapi': - specifier: 2.0.0 - version: 2.0.0 + specifier: 3.1.0 + version: 3.1.0 devDependencies: '@biomejs/biome': specifier: ^2.3.11 @@ -401,8 +401,8 @@ packages: resolution: {integrity: sha512-H8YdEoXXv2Hw17gDWGOJEya4LHlBbpChJP3jDQRfIk9hhwr0c/zbBemBRmjADowZhArL+ymkO+j5hGaYySjdpw==} engines: {node: '>= 18'} - '@chainsafe/zapi@2.0.0': - resolution: {integrity: sha512-5HXO76Ep4tYU5NngsBSgE5t/xMQQZnap+Q9PKDXOa1Oe3VaP0hU2+visNlaxW7RrVK1plF0cGMIamLubpNdefw==} + '@chainsafe/zapi@3.1.0': + resolution: {integrity: sha512-WRamllUXrrmx4G4qzmg0vRvcWQ+/Lgm7TgOFbqeJFPKlNv8CApTyA3BP6tudBaJyQYJSXlOn2RWrdv5e0EFwSw==} hasBin: true '@colors/colors@1.5.0': @@ -2442,7 +2442,7 @@ snapshots: '@chainsafe/swap-or-not-shuffle-win32-arm64-msvc': 1.2.1 '@chainsafe/swap-or-not-shuffle-win32-x64-msvc': 1.2.1 - '@chainsafe/zapi@2.0.0': {} + '@chainsafe/zapi@3.1.0': {} '@colors/colors@1.5.0': optional: true diff --git a/release-please-config.json b/release-please-config.json new file mode 100644 index 000000000..68f3286c0 --- /dev/null +++ b/release-please-config.json @@ -0,0 +1,70 @@ +{ + "$schema": "https://raw.githubusercontent.com/googleapis/release-please/main/schemas/config.json", + "last-release-sha": "72e25a2842d93bb2d31039e84fbb7c9ed34103f4", + "release-type": "node", + "include-component-in-tag": false, + "include-v-in-tag": true, + "versioning": "default", + "prerelease": false, + "prerelease-type": "rc", + "bump-minor-pre-major": true, + "bump-patch-for-minor-pre-major": true, + "pull-request-title-pattern": "chore: release ${version}", + "changelog-sections": [ + { + "type": "feat", + "section": "Features" + }, + { + "type": "fix", + "section": "Bug Fixes" + }, + { + "type": "perf", + "section": "Performance Improvements" + }, + { + "type": "refactor", + "section": "Code Refactoring" + }, + { + "type": "deps", + "section": "Dependencies" + }, + { + "type": "revert", + "section": "Reverts" + }, + { + "type": "chore", + "section": "Miscellaneous Chores" + }, + { + "type": "docs", + "section": "Documentation" + }, + { + "type": "build", + "section": "Build System", + "hidden": true + }, + { + "type": "ci", + "section": "Continuous Integration", + "hidden": true + }, + { + "type": "test", + "section": "Tests", + "hidden": true + }, + { + "type": "style", + "section": "Styles", + "hidden": true + } + ], + "packages": { + ".": {} + } +} \ No newline at end of file diff --git a/src/beacon_node/chain/root.zig b/src/beacon_node/chain/root.zig new file mode 100644 index 000000000..fa2db11df --- /dev/null +++ b/src/beacon_node/chain/root.zig @@ -0,0 +1,5 @@ +pub const state_cache = @import("state_cache/root.zig"); + +test { + @import("std").testing.refAllDecls(@This()); +} diff --git a/src/beacon_node/chain/state_cache/block_state_cache.zig b/src/beacon_node/chain/state_cache/block_state_cache.zig new file mode 100644 index 000000000..50adab75b --- /dev/null +++ b/src/beacon_node/chain/state_cache/block_state_cache.zig @@ -0,0 +1,917 @@ +const std = @import("std"); +const Allocator = std.mem.Allocator; +const assert = std.debug.assert; + +const types = @import("consensus_types"); +const state_transition = @import("state_transition"); +const time = @import("time"); +const metrics = @import("metrics.zig"); + +const CachedBeaconState = state_transition.CachedBeaconState; +const Root = types.primitive.Root.Type; +const Slot = types.primitive.Slot.Type; + +/// Given `maxSkipSlots` = 32 and `DEFAULT_EARLIEST_PERMISSIBLE_SLOT_DISTANCE` = 32, lodestar doesn't need to +/// reload states in order to process a gossip block. +/// +/// |-----------------------------------------------|-----------------------------------------------| +/// maxSkipSlots DEFAULT_EARLIEST_PERMISSIBLE_SLOT_DISTANCE ^ +/// clock slot +pub const DEFAULT_MAX_BLOCK_STATES: usize = 64; + +pub const StateCacheItem = struct { + slot: Slot, + root: Root, + reads: u64, + last_read: ?std.Io.Timestamp, + checkpoint_state: bool, +}; + +/// New implementation of BlockStateCache that keeps the most recent n states consistently. +/// - Maintain a FIFO order with special handling for the head state, which is always the first item. +/// - Prune per `add` instead of per checkpoint so it only keeps n historical states consistently, prune from tail. +/// - No need to prune per finalized checkpoint. +/// +/// Intrusive-pooled doubly-linked-list design: O(1) reorder/insert/evict and zero per-`add` heap allocation. +/// Two intrusive `std.DoublyLinkedList`s thread through one pooled slab of `Entry`: +/// - The EVICTION list (`ev_list`, via each entry's `ev_node`) is the FIFO order: `first` is the +/// head/most-recent, `last` is the oldest and is pruned first. `moveToHead`/`moveToSecond`/ +/// insert-at-head/insert-at-second are O(1) splices. +/// - The INSERTION list (`in_list`, via each entry's `in_node`) records insertion order so +/// `getSeedState` returns the `first` resident = the oldest-by-insertion, stable across +/// `setHeadState` (head pinning touches only the eviction list). +pub const BlockStateCache = struct { + const Self = @This(); + /// `slab`-resident, so its address is stable for the cache's lifetime; both lists splice it in + /// place. A free entry sits only on `free` (its list links are unused while free). + const Entry = struct { + key: Root, + state: *CachedBeaconState, + read_count: u64 = 0, + last_read: ?std.Io.Timestamp = null, + ev_node: std.DoublyLinkedList.Node = .{}, + in_node: std.DoublyLinkedList.Node = .{}, + }; + + allocator: Allocator, + max_states: usize, + /// O(1) lookup. Insertion order is NOT tracked here (the insertion list owns that) and map + /// iteration order is never used, so eviction may `swapRemove`. + /// + /// `ArrayHashMap` rather than `HashMap` deliberately: this cache holds its size flat (every + /// `add` evicts once full), so `HashMap.grow()` — the only point that clears tombstones — + /// would never fire, and its tombstone-based deletion then degrades every probe permanently + /// under the add/evict churn (https://github.com/ziglang/zig/issues/17851; the std `rehash()` + /// doc names exactly this long-lived insert+delete pattern). `ArrayHashMap`'s index deletes + /// by backward shift and cannot accumulate tombstones. + map: std.AutoArrayHashMapUnmanaged(Root, *Entry), + + /// Stable-address backing storage for every `Entry`; sized `max_states + 2` because `insertItem` + /// pushes before `prune` trims, so the peak resident count is `max_states + 2`. Allocated once in + /// `init` and never grown, so `add` never touches the heap. + slab: []Entry, + /// Freelist stack of unused slab slots. `init` fills it; insert pops, evict pushes back. + free: []*Entry, + free_len: usize, + + // Eviction order: first = head/most-recent, last = oldest (pruned first). + ev_list: std.DoublyLinkedList = .{}, + + // Insertion order: first = oldest-by-insertion (the seed), last = newest. + in_list: std.DoublyLinkedList = .{}, + + pub const Opts = struct { + max_states: usize = DEFAULT_MAX_BLOCK_STATES, + }; + + pub fn init(allocator: Allocator, opts: Opts) !Self { + assert(opts.max_states > 0); + + const capacity = opts.max_states + 2; + + var map: std.AutoArrayHashMapUnmanaged(Root, *Entry) = .empty; + errdefer map.deinit(allocator); + try map.ensureTotalCapacity(allocator, capacity); + + const slab = try allocator.alloc(Entry, capacity); + errdefer allocator.free(slab); + + const free = try allocator.alloc(*Entry, capacity); + errdefer allocator.free(free); + + for (slab, 0..) |*entry, i| { + free[i] = entry; + } + + return .{ + .allocator = allocator, + .max_states = opts.max_states, + .map = map, + .slab = slab, + .free = free, + .free_len = capacity, + }; + } + + pub fn deinit(self: *Self) void { + for (self.map.values()) |entry| { + destroyState(entry.state); + } + self.map.deinit(self.allocator); + self.allocator.free(self.slab); + self.allocator.free(self.free); + } + + /// Add a state to this cache. + /// `is_head` if true, move it to the head of the list. Otherwise add to the 2nd position. In + /// importBlock() steps, normally it'll call add() with `is_head` false first, then call + /// setHeadState() to set the head. + /// + /// Returns the canonical resident state — `state` on a fresh insert, the resident on the + /// duplicate path (the incoming duplicate is deinit'd). Use the return value afterwards, not + /// the passed pointer. Ownership of `state` transfers to the cache only on success; on + /// `hashTreeRoot` failure ownership stays with the caller, which frees it. + pub fn add( + self: *Self, + io: std.Io, + state: *CachedBeaconState, + is_head: bool, + ) !*CachedBeaconState { + // `hashTreeRoot` is the sole fallible op and precedes every mutation, so on its failure + // `state` is never inserted (no rollback). A future `try` added below would break this. + const key = (try state.state.hashTreeRoot()).*; + + if (self.map.get(key)) |resident| { + if (resident.state != state) { + destroyState(state); + } + // The duplicate-resident probe counts as a read; the reorder leaves this pointer valid. + recordRead(io, resident); + if (is_head) { + self.moveToHead(resident); + } else { + self.moveToSecond(resident); + } + // same size, no prune + return resident.state; + } + + // new state + metrics.block().adds.incr(); + self.insertItem(key, state, is_head); + self.prune(key); + return state; + } + + /// Set a state as head, happens when importing a block and head block is changed. Null is a no-op. + /// Ownership follows `add`: pass a canonical pointer if it will be used afterwards. + pub fn setHeadState(self: *Self, io: std.Io, state: ?*CachedBeaconState) !void { + if (state) |s| { + _ = try self.add(io, s, true); + } + } + + /// Get a state from this cache given a state root. Borrowed: the caller must NOT deinit it and + /// must `.clone()` to retain or mutate — an in-place mutation re-keys the root, aliasing two + /// entries onto one state. + pub fn get(self: *Self, io: std.Io, key: Root) ?*CachedBeaconState { + metrics.block().lookups.incr(); + const entry = self.map.get(key) orelse return null; + metrics.block().hits.incr(); + metrics.block().state_cloned_count.observe(entry.state.cloned_count); + recordRead(io, entry); + return entry.state; + } + + fn recordRead(io: std.Io, entry: *Entry) void { + entry.read_count += 1; + entry.last_read = time.start(io); + } + + /// Get a seed state for state reload, this could be any states. The goal is to have the same base + /// merkle tree for all BeaconState objects across application. Returns the first-inserted + /// resident — stable across `setHeadState`, since head pinning touches only the eviction list, not + /// the insertion list. Borrowed (`.clone()` to retain past the next mutation). + /// + /// Null when the cache is empty — legal via the debug-API `clear()`. + pub fn getSeedState(self: *Self) ?*CachedBeaconState { + const first = self.in_list.first orelse return null; + return @as(*Entry, @alignCast(@fieldParentPtr("in_node", first))).state; + } + + pub fn size(self: *const Self) usize { + return self.map.count(); + } + + /// Snapshot reads / seconds-since-last-read over the resident set. Never-read entries are excluded + /// from `reads`; entries with no `last_read` stamp are excluded from `secs`. + pub fn scanReadStats(self: *const Self, io: std.Io) struct { + reads: metrics.AvgMinMax, + secs: metrics.AvgMinMax, + } { + var reads: metrics.AvgMinMaxAccumulator = .{}; + var node = self.in_list.first; + while (node) |n| : (node = n.next) { + const entry: *Entry = @alignCast(@fieldParentPtr("in_node", n)); + if (entry.read_count == 0) continue; + reads.add(@floatFromInt(entry.read_count)); + } + + var secs: metrics.AvgMinMaxAccumulator = .{}; + const now = time.start(io); + node = self.in_list.first; + while (node) |n| : (node = n.next) { + const entry: *Entry = @alignCast(@fieldParentPtr("in_node", n)); + const last_read = entry.last_read orelse continue; + const value = time.durationSeconds(last_read.durationTo(now)); + secs.add(value); + } + + return .{ .reads = reads.result(), .secs = secs.result() }; + } + + /// ONLY FOR DEBUGGING PURPOSES. For lodestar debug API. Removes and deinits every state (the cache + /// owns them). + pub fn clear(self: *Self) void { + for (self.map.values()) |entry| { + destroyState(entry.state); + } + self.map.clearRetainingCapacity(); + self.ev_list = .{}; + self.in_list = .{}; + for (self.slab, 0..) |*entry, i| { + self.free[i] = entry; + } + self.free_len = self.slab.len; + } + + pub const StateIterator = struct { + node: ?*std.DoublyLinkedList.Node, + + pub fn next(self: *StateIterator) ?*CachedBeaconState { + const n = self.node orelse return null; + self.node = n.next; + const entry: *Entry = @alignCast(@fieldParentPtr("in_node", n)); + return entry.state; + } + }; + + /// ONLY FOR DEBUGGING PURPOSES. For lodestar debug API. Iterates every resident state (BORROWED — + /// the cache owns them, do NOT deinit). Valid only until the next cache mutation. Iterates in + /// insertion order. + pub fn getStates(self: *const Self) StateIterator { + return .{ .node = self.in_list.first }; + } + + /// ONLY FOR DEBUGGING PURPOSES. For lodestar debug API. Per-resident summary; caller frees the slice. + pub fn dumpSummary(self: *const Self, allocator: Allocator) ![]StateCacheItem { + const out = try allocator.alloc(StateCacheItem, self.map.count()); + errdefer allocator.free(out); + + var i: usize = 0; + var node = self.in_list.first; + while (node) |n| : (node = n.next) { + const entry: *Entry = @alignCast(@fieldParentPtr("in_node", n)); + out[i] = .{ + .slot = try entry.state.state.slot(), + .root = (try entry.state.state.hashTreeRoot()).*, + .reads = entry.read_count, + .last_read = entry.last_read, + .checkpoint_state = false, + }; + i += 1; + } + return out; + } + + /// Prune the cache from tail to keep the most recent n states consistently. The eviction tail is + /// the oldest state; in case regen adds back the same state, it should stay next to head so that + /// it won't be pruned right away. Never prune the just-added state (only reachable when + /// `max_states == 1`). + fn prune(self: *Self, last_added_key: Root) void { + while (self.map.count() > self.max_states) { + const node = self.ev_list.last orelse break; + const tail: *Entry = @alignCast(@fieldParentPtr("ev_node", node)); + // it does not make sense to prune the last added state; + // this only happens when max state is 1 in a short period of time. + if (std.mem.eql(u8, &tail.key, &last_added_key)) break; + destroyState(tail.state); + self.evict(tail); + } + } + + /// Insert into both lists and the map, taking ownership of `state`. `at_head` picks the eviction + /// slot: ev_head or the second position. The fresh entry lands at `in_tail`, so the seed base + /// (`in_head`) stays stable. + fn insertItem(self: *Self, key: Root, state: *CachedBeaconState, at_head: bool) void { + assert(self.free_len > 0); + self.free_len -= 1; + const entry = self.free[self.free_len]; + entry.* = .{ .key = key, .state = state }; + self.map.putAssumeCapacity(key, entry); + + // Append to the insertion list at the tail. + self.in_list.append(&entry.in_node); + + // Splice into the eviction list at head or second position. + if (at_head or self.ev_list.first == null) { + self.ev_list.prepend(&entry.ev_node); + } else { + self.ev_list.insertAfter(self.ev_list.first.?, &entry.ev_node); + } + } + + /// Move `entry` to the eviction head. No-op if already there. O(1) splice. + fn moveToHead(self: *Self, entry: *Entry) void { + if (self.ev_list.first == &entry.ev_node) return; + self.ev_list.remove(&entry.ev_node); + self.ev_list.prepend(&entry.ev_node); + } + + /// Move `entry` to the eviction second position (or head when the cache holds <= 1 resident). + /// No-op when `entry` is already at-or-before that target. + fn moveToSecond(self: *Self, entry: *Entry) void { + // target == head when only one resident; otherwise the second slot. + if (self.ev_list.first == self.ev_list.last) { + // <= 1 resident: target is head; head is index 0, so any entry is at-or-before it. + return; + } + // target is the second slot. `entry` is at-or-before it iff it is the head or the second node. + if (self.ev_list.first == &entry.ev_node) return; + if (self.ev_list.first.?.next == &entry.ev_node) return; + + self.ev_list.remove(&entry.ev_node); + self.ev_list.insertAfter(self.ev_list.first.?, &entry.ev_node); + } + + /// Remove `entry` from BOTH lists and the map, returning its slab slot to the freelist. Does NOT + /// free the state — the caller does that first (mirrors `prune`). + fn evict(self: *Self, entry: *Entry) void { + self.ev_list.remove(&entry.ev_node); + self.in_list.remove(&entry.in_node); + + assert(self.map.swapRemove(entry.key)); + assert(self.free_len < self.free.len); + self.free[self.free_len] = entry; + self.free_len += 1; + } + + /// Walk the eviction list head-to-tail, copying keys into `out`. For tests/parity only. + /// Returns the number of keys written; asserts `out` is large enough. + pub fn dumpKeyOrder(self: *const Self, out: []Root) usize { + var i: usize = 0; + var node = self.ev_list.first; + while (node) |n| : (node = n.next) { + const entry: *Entry = @alignCast(@fieldParentPtr("ev_node", n)); + assert(i < out.len); + out[i] = entry.key; + i += 1; + } + return i; + } + + fn destroyState(state: *CachedBeaconState) void { + const allocator = state.allocator; + state.deinit(); + allocator.destroy(state); + } +}; + +const testing = std.testing; +const Node = @import("persistent_merkle_tree").Node; +const TestCachedBeaconState = state_transition.test_utils.TestCachedBeaconState; + +const TestStateFactory = struct { + allocator: Allocator, + helper: TestCachedBeaconState, + + fn init(allocator: Allocator, pool: *Node.Pool) !TestStateFactory { + const helper = try TestCachedBeaconState.init(allocator, pool, 8); + return .{ .allocator = allocator, .helper = helper }; + } + + fn deinit(self: *TestStateFactory) void { + self.helper.deinit(); + } + + /// Produce an independently owned state with a distinct root by setting its `slot`. + fn make(self: *TestStateFactory, slot: u64) !*CachedBeaconState { + const state = try self.helper.cached_state.clone(self.allocator, .{}); + errdefer { + state.deinit(); + self.allocator.destroy(state); + } + + try state.state.setSlot(slot); + try state.state.commit(); + return state; + } +}; + +const BlockHarness = struct { + allocator: Allocator, + io: std.Io, + pool: Node.Pool, + factory: TestStateFactory, + cache: BlockStateCache, + + fn init(allocator: Allocator, opts: BlockStateCache.Opts) !*BlockHarness { + const h = try allocator.create(BlockHarness); + errdefer allocator.destroy(h); + + h.allocator = allocator; + h.io = std.testing.io; + h.pool = try Node.Pool.init(.{ .page_allocator = allocator, .allocator = allocator, .pool_size = 256 * 8 }); + errdefer h.pool.deinit(); + + h.factory = try TestStateFactory.init(allocator, &h.pool); + errdefer h.factory.deinit(); + + h.cache = try BlockStateCache.init(allocator, opts); + + return h; + } + + fn deinit(self: *BlockHarness) void { + const allocator = self.allocator; + self.cache.deinit(); + self.factory.deinit(); + self.pool.deinit(); + allocator.destroy(self); + } +}; + +test "BlockStateCache add/get and ownership deinit" { + const allocator = testing.allocator; + const h = try BlockHarness.init(allocator, .{}); + defer h.deinit(); + + const s0 = try h.factory.make(1); + const key0 = (try s0.state.hashTreeRoot()).*; + _ = try h.cache.add(h.io, s0, false); + + try testing.expectEqual(@as(usize, 1), h.cache.size()); + try testing.expect(h.cache.get(h.io, key0) == s0); + + const missing: Root = @splat(0xAB); + try testing.expect(h.cache.get(h.io, missing) == null); +} + +test "BlockStateCache FIFO head pinning and 2nd-position insert" { + const allocator = testing.allocator; + const h = try BlockHarness.init(allocator, .{ .max_states = 4 }); + defer h.deinit(); + + const s_head = try h.factory.make(10); + const k_head = (try s_head.state.hashTreeRoot()).*; + _ = try h.cache.add(h.io, s_head, true); + + const s_a = try h.factory.make(11); + const k_a = (try s_a.state.hashTreeRoot()).*; + _ = try h.cache.add(h.io, s_a, false); + + const s_b = try h.factory.make(12); + const k_b = (try s_b.state.hashTreeRoot()).*; + _ = try h.cache.add(h.io, s_b, false); + + // Order should be: head, b (latest non-head 2nd), a. + var order_buf: [4]Root = undefined; + const order_len = h.cache.dumpKeyOrder(&order_buf); + try testing.expectEqual(@as(usize, 3), order_len); + try testing.expect(std.mem.eql(u8, &order_buf[0], &k_head)); + try testing.expect(std.mem.eql(u8, &order_buf[1], &k_b)); + try testing.expect(std.mem.eql(u8, &order_buf[2], &k_a)); +} + +test "BlockStateCache re-adding the current head as non-head keeps it at head" { + const allocator = testing.allocator; + const h = try BlockHarness.init(allocator, .{ .max_states = 4 }); + defer h.deinit(); + + const s_head = try h.factory.make(50); + const k_head = (try s_head.state.hashTreeRoot()).*; + _ = try h.cache.add(h.io, s_head, true); + + const s_other = try h.factory.make(51); + _ = try h.cache.add(h.io, s_other, false); + + // Re-adding the resident head as non-head must NOT demote it to 2nd; the head stays at index 0. + _ = try h.cache.add(h.io, s_head, false); + var order_buf: [4]Root = undefined; + _ = h.cache.dumpKeyOrder(&order_buf); + try testing.expect(std.mem.eql(u8, &order_buf[0], &k_head)); +} + +test "BlockStateCache add duplicate-resident path bumps the resident entry's read count" { + const allocator = testing.allocator; + const h = try BlockHarness.init(allocator, .{ .max_states = 4 }); + defer h.deinit(); + + const s0 = try h.factory.make(60); + const key0 = (try s0.state.hashTreeRoot()).*; + _ = try h.cache.add(h.io, s0, false); + + // Fresh insert does not count as a read. + try testing.expectEqual(@as(u64, 0), h.cache.map.get(key0).?.read_count); + + // Re-add a DIFFERENT state value hashing to the SAME root: the duplicate-resident probe finds + // the entry by key (just like the public `get`) and must bump its read tracking. + const dup = try cloneDistinct(h.factory.helper.cached_state, allocator, 60); + _ = try h.cache.add(h.io, dup, false); + + // Read directly off the map rather than via `get`, which would itself bump and mask a missing + // probe bump. With the bump, the single duplicate probe yields read_count == 1. + try testing.expectEqual(@as(u64, 1), h.cache.map.get(key0).?.read_count); +} + +test "BlockStateCache add returns the canonical pointer on both paths" { + const allocator = testing.allocator; + const h = try BlockHarness.init(allocator, .{ .max_states = 4 }); + defer h.deinit(); + + const s0 = try h.factory.make(60); + try testing.expectEqual(s0, try h.cache.add(h.io, s0, false)); + + const dup = try cloneDistinct(h.factory.helper.cached_state, allocator, 60); + const canonical = try h.cache.add(h.io, dup, false); + try testing.expectEqual(s0, canonical); + + try h.cache.setHeadState(h.io, canonical); + try testing.expectEqual(@as(usize, 1), h.cache.size()); +} + +test "BlockStateCache prune evicts tail, never the just-added key" { + const allocator = testing.allocator; + const h = try BlockHarness.init(allocator, .{ .max_states = 2 }); + defer h.deinit(); + + const s0 = try h.factory.make(20); + const k0 = (try s0.state.hashTreeRoot()).*; + _ = try h.cache.add(h.io, s0, true); + + const s1 = try h.factory.make(21); + const k1 = (try s1.state.hashTreeRoot()).*; + _ = try h.cache.add(h.io, s1, false); + + const s2 = try h.factory.make(22); + const k2 = (try s2.state.hashTreeRoot()).*; + _ = try h.cache.add(h.io, s2, false); + + // Order is [k0(head), k2, k1]; max_states == 2 evicts the tail k1. Head k0 stays pinned and + // the just-added k2 survives. + try testing.expectEqual(@as(usize, 2), h.cache.size()); + try testing.expect(h.cache.get(h.io, k1) == null); + try testing.expect(h.cache.get(h.io, k0) != null); + try testing.expect(h.cache.get(h.io, k2) != null); +} + +test "BlockStateCache prune-skip-last-added when max_states == 1" { + const allocator = testing.allocator; + const h = try BlockHarness.init(allocator, .{ .max_states = 1 }); + defer h.deinit(); + + const s0 = try h.factory.make(30); + _ = try h.cache.add(h.io, s0, true); + + const s1 = try h.factory.make(31); + const k1 = (try s1.state.hashTreeRoot()).*; + _ = try h.cache.add(h.io, s1, false); + + // prune breaks rather than evict the just-added tail, so the cache transiently holds 2 with + // max_states == 1. The just-added key must survive. + try testing.expectEqual(@as(usize, 2), h.cache.size()); + try testing.expect(h.cache.get(h.io, k1) != null); + + // A subsequent non-head add lets prune evict the older non-head tail (k1) back toward capacity. + const s2 = try h.factory.make(32); + const k2 = (try s2.state.hashTreeRoot()).*; + _ = try h.cache.add(h.io, s2, false); + try testing.expect(h.cache.get(h.io, k1) == null); + try testing.expect(h.cache.get(h.io, k2) != null); +} + +test "BlockStateCache getSeedState returns stable first-inserted state, clear frees states" { + const allocator = testing.allocator; + const h = try BlockHarness.init(allocator, .{ .max_states = 4 }); + defer h.deinit(); + + const cache = &h.cache; + const factory = &h.factory; + const io = h.io; + + const s0 = try factory.make(40); + _ = try cache.add(io, s0, false); + const s1 = try factory.make(41); + _ = try cache.add(io, s1, true); + + // Seed is the first-inserted resident (s0), NOT the head (s1) — a stable reload base. + try testing.expect(cache.getSeedState() == s0); + + // Changing the head must NOT churn the seed: it stays the first-inserted resident. + try cache.setHeadState(io, s1); + try testing.expect(cache.getSeedState() == s0); + + cache.clear(); + try testing.expectEqual(@as(usize, 0), cache.size()); +} + +test "BlockStateCache getStates + dumpSummary (debug API)" { + const allocator = testing.allocator; + const h = try BlockHarness.init(allocator, .{ .max_states = 4 }); + defer h.deinit(); + + const cache = &h.cache; + const io = h.io; + + const s0 = try h.factory.make(40); + _ = try cache.add(io, s0, false); + const s1 = try h.factory.make(41); + _ = try cache.add(io, s1, true); + + // Probe s0 once so its read_count surfaces in the summary. + _ = cache.get(io, (try s0.state.hashTreeRoot()).*); + + var it = cache.getStates(); + var saw_s0 = false; + var saw_s1 = false; + while (it.next()) |s| { + if (s == s0) saw_s0 = true; + if (s == s1) saw_s1 = true; + } + try testing.expect(saw_s0 and saw_s1); + + const summary = try cache.dumpSummary(allocator); + defer allocator.free(summary); + try testing.expectEqual(@as(usize, 2), summary.len); + var saw40 = false; + var saw41 = false; + for (summary) |item| { + try testing.expect(!item.checkpoint_state); + if (item.slot == 40) { + saw40 = true; + try testing.expectEqual(@as(u64, 1), item.reads); + } + if (item.slot == 41) saw41 = true; + } + try testing.expect(saw40 and saw41); +} + +// Each row sets a head, introduces state3 via one of three add modes, and asserts the resulting +// head-first key order plus the single pruned key. +test "BlockStateCache head-pin + 2nd-position insert + single-prune order across add modes" { + const allocator = testing.allocator; + const h = try BlockHarness.init(allocator, .{ .max_states = 2 }); + defer h.deinit(); + + const factory = &h.factory; + const io = h.io; + + // Roots are deterministic across scenarios, so compute them once — lets the expectation table + // hold the expected keys directly. + const key1, const key2, const key3 = blk: { + const s1 = try factory.make(100); + defer BlockStateCache.destroyState(s1); + const s2 = try factory.make(200); + defer BlockStateCache.destroyState(s2); + const s3 = try factory.make(300); + defer BlockStateCache.destroyState(s3); + break :blk .{ + (try s1.state.hashTreeRoot()).*, + (try s2.state.hashTreeRoot()).*, + (try s3.state.hashTreeRoot()).*, + }; + }; + + const AddAsHead = enum { head, non_head, non_then_head }; + const Scenario = struct { + name: []const u8, + head_is_state2: bool, + add_as_head: AddAsHead, + kept: [2]Root, + pruned: Root, + }; + + const scenarios = [_]Scenario{ + .{ .name = "add as head, prune key1", .head_is_state2 = true, .add_as_head = .head, .kept = .{ key3, key2 }, .pruned = key1 }, + .{ .name = "add, prune key1", .head_is_state2 = true, .add_as_head = .non_head, .kept = .{ key2, key3 }, .pruned = key1 }, + .{ .name = "add as head, prune key2", .head_is_state2 = false, .add_as_head = .head, .kept = .{ key3, key1 }, .pruned = key2 }, + .{ .name = "add, prune key2", .head_is_state2 = false, .add_as_head = .non_head, .kept = .{ key1, key3 }, .pruned = key2 }, + .{ .name = "add then set as head, prune key1", .head_is_state2 = true, .add_as_head = .non_then_head, .kept = .{ key3, key2 }, .pruned = key1 }, + .{ .name = "add then set as head, prune key2", .head_is_state2 = false, .add_as_head = .non_then_head, .kept = .{ key3, key1 }, .pruned = key2 }, + }; + + for (scenarios) |sc| { + // Fresh cache per scenario: clear the shared harness cache rather than re-init. + const cache = &h.cache; + cache.clear(); + + // Setup: add state1 then state2, both non-head (key order [k1, k2]). + const state1 = try factory.make(100); + _ = try cache.add(io, state1, false); + const state2 = try factory.make(200); + _ = try cache.add(io, state2, false); + const state3 = try factory.make(300); + + // Seed is the first-inserted resident (state1) and must stay stable across setHeadState. + try testing.expect(cache.getSeedState() == state1); + + try cache.setHeadState(io, if (sc.head_is_state2) state2 else state1); + try testing.expect(cache.getSeedState() == state1); + try testing.expectEqual(@as(usize, 2), cache.size()); + + switch (sc.add_as_head) { + .head => _ = try cache.add(io, state3, true), + .non_head => _ = try cache.add(io, state3, false), + .non_then_head => { + _ = try cache.add(io, state3, false); + try cache.setHeadState(io, state3); + }, + } + + try testing.expectEqual(@as(usize, 2), cache.size()); + + var order_buf: [2]Root = undefined; + const order_len = cache.dumpKeyOrder(&order_buf); + try testing.expectEqual(sc.kept.len, order_len); + for (sc.kept, order_buf[0..order_len]) |want, got| { + try testing.expect(std.mem.eql(u8, &want, &got)); + } + + try testing.expect(cache.get(io, sc.pruned) == null); + for (sc.kept) |kept_key| { + try testing.expect(cache.get(io, kept_key) != null); + } + } +} + +const DoubleFreeDetectAllocator = @import("testing_allocators").DoubleFreeDetectAllocator; + +// Clone the seed onto `alloc`, stamp a distinct slot, and commit so the state hashes to a distinct, +// already-cached root. +fn cloneDistinct(seed: *CachedBeaconState, alloc: Allocator, slot: u64) !*CachedBeaconState { + const state = try seed.clone(alloc, .{}); + errdefer { + state.deinit(); + alloc.destroy(state); + } + try state.state.setSlot(slot); + try state.state.commit(); + return state; +} + +test "BlockStateCache add - insert/prune/duplicate paths free the owned state exactly once" { + const seed_alloc = testing.allocator; + const io = std.testing.io; + const pool_size = 256 * 64; + var pool = try Node.Pool.init(.{ .page_allocator = seed_alloc, .allocator = seed_alloc, .pool_size = pool_size }); + defer pool.deinit(); + + var factory = try TestStateFactory.init(seed_alloc, &pool); + defer factory.deinit(); + + inline for (.{ @as(usize, 1), @as(usize, 2) }) |max_states| { + var track = DoubleFreeDetectAllocator.init(seed_alloc, std.math.maxInt(usize)); + defer track.deinit(); + const state_alloc = track.allocator(); + + var cache = try BlockStateCache.init(seed_alloc, .{ .max_states = max_states }); + defer cache.deinit(); + + // Distinct-root adds exercise insert + prune-evict; the final re-add of s0's root exercises + // the duplicate path (incoming duplicate must be freed exactly once). + const s0 = try cloneDistinct(factory.helper.cached_state, state_alloc, 1000); + _ = try cache.add(io, s0, true); + try testing.expect(!track.double_free); + + const s1 = try cloneDistinct(factory.helper.cached_state, state_alloc, 1001); + _ = try cache.add(io, s1, false); + try testing.expect(!track.double_free); + + const dup = try cloneDistinct(factory.helper.cached_state, state_alloc, 1000); + _ = try cache.add(io, dup, false); + try testing.expect(!track.double_free); + } +} + +test "BlockStateCache init releases its reservations on OOM at every allocation point" { + // init's allocations are the map ensureTotalCapacity, the slab alloc, and the free-array alloc; a + // failure at any of them must free what was already reserved (the testing allocator flags a leak + // otherwise). The sweep loops fail_index until success, covering every allocation point. + var fail_index: usize = 0; + while (true) : (fail_index += 1) { + var failing = std.testing.FailingAllocator.init(testing.allocator, .{ .fail_index = fail_index }); + if (BlockStateCache.init(failing.allocator(), .{ .max_states = 4 })) |cache| { + var c = cache; + c.deinit(); + break; + } else |err| { + try testing.expectEqual(error.OutOfMemory, err); + } + } + try testing.expect(fail_index > 0); // sanity: init does allocate, so the sweep hit OOM at least once +} + +// scanReadStats reads arithmetic: each resident state is read `read_counts[i]` times; never-read +// entries (count 0) are EXCLUDED from the {sum, avg, min, max}. Seconds bounds are covered by the +// dedicated "seconds computed" test below. +test "BlockStateCache scanReadStats reads arithmetic over read-count vectors" { + const allocator = testing.allocator; + + const Row = struct { + name: []const u8, + read_counts: []const u64, + sum: f64, + avg: f64, + min: f64, + max: f64, + }; + inline for (.{ + Row{ .name = "excludes never-read", .read_counts = &.{ 3, 1, 0 }, .sum = 4, .avg = 2, .min = 1, .max = 3 }, + Row{ .name = "single read entry", .read_counts = &.{ 0, 0, 5 }, .sum = 5, .avg = 5, .min = 5, .max = 5 }, + Row{ .name = "uniform reads", .read_counts = &.{ 2, 2, 2 }, .sum = 6, .avg = 2, .min = 2, .max = 2 }, + Row{ .name = "spread reads", .read_counts = &.{ 1, 4, 7 }, .sum = 12, .avg = 4, .min = 1, .max = 7 }, + }) |row| { + const h = try BlockHarness.init(allocator, .{ .max_states = 4 }); + defer h.deinit(); + + for (row.read_counts, 0..) |reads, i| { + const s = try h.factory.make(i + 1); + const key = (try s.state.hashTreeRoot()).*; + _ = try h.cache.add(h.io, s, false); + for (0..reads) |_| _ = h.cache.get(h.io, key); + } + + const stats = h.cache.scanReadStats(h.io); + errdefer std.debug.print("scanReadStats row [{s}] failed\n", .{row.name}); + try testing.expectEqual(row.sum, stats.reads.sum); + try testing.expectEqual(row.avg, stats.reads.avg); + try testing.expectEqual(row.min, stats.reads.min); + try testing.expectEqual(row.max, stats.reads.max); + } +} + +test "BlockStateCache scanReadStats seconds computed" { + const allocator = testing.allocator; + const h = try BlockHarness.init(allocator, .{ .max_states = 4 }); + defer h.deinit(); + + const s0 = try h.factory.make(1); + const k0 = (try s0.state.hashTreeRoot()).*; + _ = try h.cache.add(h.io, s0, false); + const s1 = try h.factory.make(2); + _ = try h.cache.add(h.io, s1, false); + + // Stamp only s0; s1 stays unread so it must not contribute to the seconds stats. + _ = h.cache.get(h.io, k0); + + // Exact seconds are timing-dependent; only the bounds and that the single stamped entry counts + // (min == max for one sample) are asserted. + const stats = h.cache.scanReadStats(h.io); + try testing.expect(stats.secs.max >= 0); + try testing.expect(stats.secs.min >= 0); + try testing.expectEqual(stats.secs.min, stats.secs.max); + try testing.expectEqual(stats.secs.sum, stats.secs.max); + // Reads still track the single stamped entry. + try testing.expectEqual(@as(f64, 1), stats.reads.sum); +} + +test "BlockStateCache scanReadStats empty cache is all-zero" { + const allocator = testing.allocator; + const h = try BlockHarness.init(allocator, .{ .max_states = 4 }); + defer h.deinit(); + + const stats = h.cache.scanReadStats(h.io); + try testing.expectEqual(metrics.AvgMinMax{}, stats.reads); + try testing.expectEqual(metrics.AvgMinMax{}, stats.secs); +} + +test "BlockStateCache scanReadStats reflects only survivors after evict and clear" { + const allocator = testing.allocator; + const h = try BlockHarness.init(allocator, .{ .max_states = 2 }); + defer h.deinit(); + + const s0 = try h.factory.make(10); + const k0 = (try s0.state.hashTreeRoot()).*; + _ = try h.cache.add(h.io, s0, true); + const s1 = try h.factory.make(11); + const k1 = (try s1.state.hashTreeRoot()).*; + _ = try h.cache.add(h.io, s1, false); + + // Read both, twice each, then add a third non-head state to evict the tail (k1). + for (0..2) |_| _ = h.cache.get(h.io, k0); + for (0..2) |_| _ = h.cache.get(h.io, k1); + const s2 = try h.factory.make(12); + _ = try h.cache.add(h.io, s2, false); + + // k1 was evicted; only k0's reads (2) remain, so {sum=2, min=2, max=2}. + try testing.expect(h.cache.get(h.io, k1) == null); + const after_evict = h.cache.scanReadStats(h.io); + try testing.expectEqual(@as(f64, 2), after_evict.reads.sum); + try testing.expectEqual(@as(f64, 2), after_evict.reads.min); + try testing.expectEqual(@as(f64, 2), after_evict.reads.max); + + // After clear there are no survivors: all-zero. + h.cache.clear(); + const after_clear = h.cache.scanReadStats(h.io); + try testing.expectEqual(metrics.AvgMinMax{}, after_clear.reads); +} diff --git a/src/beacon_node/chain/state_cache/cp_datastore.zig b/src/beacon_node/chain/state_cache/cp_datastore.zig new file mode 100644 index 000000000..1c8075074 --- /dev/null +++ b/src/beacon_node/chain/state_cache/cp_datastore.zig @@ -0,0 +1,774 @@ +const std = @import("std"); +const Allocator = std.mem.Allocator; +const assert = std.debug.assert; + +const hex = @import("hex"); + +const DatastoreKey = @import("key.zig").DatastoreKey; +const DATASTORE_KEY_LEN = @import("key.zig").DATASTORE_KEY_LEN; +const Checkpoint = @import("key.zig").Checkpoint; +const datastoreKey = @import("key.zig").datastoreKey; +const datastoreKeyEpoch = @import("key.zig").datastoreKeyEpoch; + +const types = @import("consensus_types"); +const Epoch = types.primitive.Epoch.Type; + +const state_transition = @import("state_transition"); +const computeStartSlotAtEpoch = state_transition.computeStartSlotAtEpoch; +const computeEpochAtSlot = state_transition.computeEpochAtSlot; +const getStateSlotFromBytes = state_transition.getStateSlotFromBytes; +const getLastProcessedSlotFromStateBytes = state_transition.getLastProcessedSlotFromStateBytes; +const STATE_SLOTS_PREFIX_LEN = state_transition.STATE_SLOTS_PREFIX_LEN; +const testing = std.testing; + +/// On-disk store for serialized checkpoint states. The backend is chosen at construction at RUNTIME +/// — so it is a vtable, not a comptime generic. +pub const CPStateDatastore = struct { + ptr: *anyopaque, + vtable: *const VTable, + + pub const VTable = struct { + /// Persist `state_bytes` for `key`, returning its `DatastoreKey`. Writing an existing key is + /// a no-op (bytes are identical), preserving idempotence. + write: *const fn (ctx: *anyopaque, io: std.Io, key: Checkpoint, state_bytes: []const u8) anyerror!DatastoreKey, + /// Delete the state at `key`. Absent keys are ignored. + remove: *const fn (ctx: *anyopaque, io: std.Io, dk: DatastoreKey) anyerror!void, + /// Delete the states at `keys`, possibly fanned out concurrently. The first failure surfaces + /// (after all run); absent keys are ignored. + removeMany: *const fn (ctx: *anyopaque, io: std.Io, allocator: Allocator, keys: []const DatastoreKey) anyerror!void, + /// Read the state bytes at `key` (allocated with `allocator`, caller frees), or null if absent. + read: *const fn (ctx: *anyopaque, io: std.Io, allocator: Allocator, dk: DatastoreKey) anyerror!?[]u8, + /// Read up to `buf.len` leading bytes of the state at `dk` into `buf`, returning the count + /// read, or null if absent. + readPrefix: *const fn (ctx: *anyopaque, io: std.Io, dk: DatastoreKey, buf: []u8) anyerror!?usize, + /// All persisted keys. Caller frees the returned slice with `allocator`. + readKeys: *const fn (ctx: *anyopaque, io: std.Io, allocator: Allocator) anyerror![]DatastoreKey, + /// Optional one-time backend setup (e.g. ensure the directory exists). + init: ?*const fn (ctx: *anyopaque, io: std.Io) anyerror!void, + }; + + pub fn write(self: CPStateDatastore, io: std.Io, key: Checkpoint, state_bytes: []const u8) !DatastoreKey { + return self.vtable.write(self.ptr, io, key, state_bytes); + } + + pub fn remove(self: CPStateDatastore, io: std.Io, dk: DatastoreKey) !void { + return self.vtable.remove(self.ptr, io, dk); + } + + pub fn removeMany(self: CPStateDatastore, io: std.Io, allocator: Allocator, keys: []const DatastoreKey) !void { + return self.vtable.removeMany(self.ptr, io, allocator, keys); + } + + pub fn read(self: CPStateDatastore, io: std.Io, allocator: Allocator, dk: DatastoreKey) !?[]u8 { + return self.vtable.read(self.ptr, io, allocator, dk); + } + + pub fn readPrefix(self: CPStateDatastore, io: std.Io, dk: DatastoreKey, buf: []u8) !?usize { + return self.vtable.readPrefix(self.ptr, io, dk, buf); + } + + pub fn readKeys(self: CPStateDatastore, io: std.Io, allocator: Allocator) ![]DatastoreKey { + return self.vtable.readKeys(self.ptr, io, allocator); + } + + pub fn initStore(self: CPStateDatastore, io: std.Io) !void { + if (self.vtable.init) |init_fn| try init_fn(self.ptr, io); + } + + /// Get the latest safe checkpoint state the node can use to boot from, or null: + /// - it should be the checkpoint state that's unique in its epoch, + /// - its last processed block slot should be at epoch boundary or last slot of previous epoch, + /// - state slot should be at epoch boundary, + /// - state slot should be equal to epoch * SLOTS_PER_EPOCH. + /// + /// Return the serialized data of Current Root Checkpoint State (CRCS) or Previous Root Checkpoint + /// State (PRCS). Caller owns the returned bytes. + pub fn readLatestSafe(self: CPStateDatastore, io: std.Io, allocator: Allocator) !?[]u8 { + const keys = try self.readKeys(io, allocator); + defer allocator.free(keys); + + if (keys.len == 0) return null; + + // Epoch-descending, so the first qualifying state encountered is the newest. + std.mem.sort(DatastoreKey, keys, {}, struct { + fn desc(_: void, a: DatastoreKey, b: DatastoreKey) bool { + return datastoreKeyEpoch(a) > datastoreKeyEpoch(b); + } + }.desc); + + var prefix: [STATE_SLOTS_PREFIX_LEN]u8 = undefined; + var i: usize = 0; + while (i < keys.len) : (i += 1) { + const epoch = datastoreKeyEpoch(keys[i]); + // only consider epochs with a single checkpoint to avoid ambiguity from forks; sorted, so + // an epoch's duplicates are adjacent. + const prev_dup = i > 0 and datastoreKeyEpoch(keys[i - 1]) == epoch; + const next_dup = i + 1 < keys.len and datastoreKeyEpoch(keys[i + 1]) == epoch; + if (prev_dup or next_dup) continue; + + const n = (try self.readPrefix(io, keys[i], &prefix)) orelse continue; + if (!isSafeCheckpointState(prefix[0..n], epoch)) continue; + return (try self.read(io, allocator, keys[i])) orelse continue; + } + return null; + } +}; + +/// Whether `state_bytes` is a safe boot state for `epoch`: at epoch boundary and a CRCS or PRCS. +/// Short or foreign bytes fail the slot reads (treated as unsafe) rather than asserting. +fn isSafeCheckpointState(state_bytes: []const u8, epoch: Epoch) bool { + const state_slot = getStateSlotFromBytes(state_bytes) catch return false; + const last_processed = getLastProcessedSlotFromStateBytes(state_bytes) catch return false; + // not CRCS or PRCS, skip. Guard the subtract so a malformed slot-0 state cannot underflow. + const is_crcs = last_processed == state_slot; + const is_prcs = state_slot > 0 and last_processed == state_slot - 1; + if (!is_crcs and !is_prcs) return false; + // Division first: a huge on-disk epoch must read unsafe, not overflow the multiply below. + if (computeEpochAtSlot(state_slot) != epoch) return false; + // at epoch boundary (subsumes the slot % SLOTS_PER_EPOCH == 0 check). + return state_slot == computeStartSlotAtEpoch(epoch); +} + +/// In-memory datastore for tests. Backed by an insertion-ordered map so `readKeys` yields a stable +/// order and a write of an existing key is a no-op (no reorder). +pub const InMemoryCPStateDatastore = struct { + allocator: Allocator, + states: std.AutoArrayHashMapUnmanaged(DatastoreKey, []u8), + + pub fn init(allocator: Allocator) InMemoryCPStateDatastore { + return .{ .allocator = allocator, .states = .empty }; + } + + pub fn deinit(self: *InMemoryCPStateDatastore) void { + for (self.states.values()) |bytes| { + self.allocator.free(bytes); + } + self.states.deinit(self.allocator); + } + + pub fn datastore(self: *InMemoryCPStateDatastore) CPStateDatastore { + return .{ .ptr = self, .vtable = &vtable }; + } + + const vtable = CPStateDatastore.VTable{ + .write = writeImpl, + .remove = removeImpl, + .removeMany = removeManyImpl, + .read = readImpl, + .readPrefix = readPrefixImpl, + .readKeys = readKeysImpl, + .init = null, + }; + + fn writeImpl(ctx: *anyopaque, io: std.Io, key: Checkpoint, state_bytes: []const u8) anyerror!DatastoreKey { + _ = io; + const self: *InMemoryCPStateDatastore = @ptrCast(@alignCast(ctx)); + const dk = datastoreKey(key); + if (self.states.contains(dk)) return dk; + + const owned = try self.allocator.dupe(u8, state_bytes); + errdefer self.allocator.free(owned); + + try self.states.put(self.allocator, dk, owned); + return dk; + } + + fn removeImpl(ctx: *anyopaque, io: std.Io, dk: DatastoreKey) anyerror!void { + _ = io; + const self: *InMemoryCPStateDatastore = @ptrCast(@alignCast(ctx)); + // `orderedRemove` keeps the surviving keys in insertion order. + if (self.states.fetchOrderedRemove(dk)) |kv| { + self.allocator.free(kv.value); + } + } + + fn removeManyImpl(ctx: *anyopaque, io: std.Io, allocator: Allocator, keys: []const DatastoreKey) anyerror!void { + _ = io; + _ = allocator; + const self: *InMemoryCPStateDatastore = @ptrCast(@alignCast(ctx)); + for (keys) |dk| if (self.states.fetchOrderedRemove(dk)) |kv| self.allocator.free(kv.value); + } + + fn readImpl(ctx: *anyopaque, io: std.Io, allocator: Allocator, dk: DatastoreKey) anyerror!?[]u8 { + _ = io; + const self: *InMemoryCPStateDatastore = @ptrCast(@alignCast(ctx)); + const bytes = self.states.get(dk) orelse return null; + return try allocator.dupe(u8, bytes); + } + + fn readPrefixImpl(ctx: *anyopaque, io: std.Io, dk: DatastoreKey, buf: []u8) anyerror!?usize { + _ = io; + const self: *InMemoryCPStateDatastore = @ptrCast(@alignCast(ctx)); + const bytes = self.states.get(dk) orelse return null; + const n = @min(bytes.len, buf.len); + @memcpy(buf[0..n], bytes[0..n]); + return n; + } + + fn readKeysImpl(ctx: *anyopaque, io: std.Io, allocator: Allocator) anyerror![]DatastoreKey { + _ = io; + const self: *InMemoryCPStateDatastore = @ptrCast(@alignCast(ctx)); + return try allocator.dupe(DatastoreKey, self.states.keys()); + } +}; + +/// Implementation of CPStateDatastore using file system, this is beneficial for debugging. Each state +/// is one file under `/checkpoint_states/` named `hex(DatastoreKey)` (82 chars: "0x" + 80 +/// hex). +pub const FileCPStateDatastore = struct { + /// "0x" + 2 hex chars per `DatastoreKey` byte (82 for a 40-byte key). + const FILE_NAME_LEN: usize = 2 + 2 * DATASTORE_KEY_LEN; + const SUBDIR = "checkpoint_states"; + const TMP_SUFFIX = ".tmp"; + + allocator: Allocator, + /// `/checkpoint_states` (service deployment: `/beacon/...`, docker: `/data/...`). + dir_path: []u8, + /// Open handle to the checkpoint-states directory, created in `initStore`. + dir: ?std.Io.Dir, + + pub fn init(allocator: Allocator, data_dir: []const u8) !FileCPStateDatastore { + const dir_path = try std.fs.path.join(allocator, &.{ data_dir, SUBDIR }); + return .{ .allocator = allocator, .dir_path = dir_path, .dir = null }; + } + + pub fn deinit(self: *FileCPStateDatastore, io: std.Io) void { + if (self.dir) |dir| dir.close(io); + self.allocator.free(self.dir_path); + } + + pub fn datastore(self: *FileCPStateDatastore) CPStateDatastore { + return .{ .ptr = self, .vtable = &vtable }; + } + + const vtable = CPStateDatastore.VTable{ + .write = writeImpl, + .remove = removeImpl, + .removeMany = removeManyImpl, + .read = readImpl, + .readPrefix = readPrefixImpl, + .readKeys = readKeysImpl, + .init = initImpl, + }; + + fn fileName(dk: DatastoreKey) [FILE_NAME_LEN]u8 { + var name: [FILE_NAME_LEN]u8 = undefined; + // `name` is sized exactly "0x" + 2 hex chars per `dk` byte, so the encode cannot run short. + _ = hex.bytesToHex(&name, &dk) catch unreachable; + return name; + } + + fn initImpl(ctx: *anyopaque, io: std.Io) anyerror!void { + const self: *FileCPStateDatastore = @ptrCast(@alignCast(ctx)); + if (self.dir) |dir| { + dir.close(io); + self.dir = null; + } + self.dir = try std.Io.Dir.cwd().createDirPathOpen(io, self.dir_path, .{ .open_options = .{ .iterate = true } }); + + var it = self.dir.?.iterate(); + while (try it.next(io)) |entry| { + if (!std.mem.endsWith(u8, entry.name, TMP_SUFFIX)) continue; + self.dir.?.deleteFile(io, entry.name) catch |err| switch (err) { + error.IsDir, error.FileNotFound => {}, + else => return err, + }; + } + } + + fn writeImpl(ctx: *anyopaque, io: std.Io, key: Checkpoint, state_bytes: []const u8) anyerror!DatastoreKey { + const self: *FileCPStateDatastore = @ptrCast(@alignCast(ctx)); + const dir = self.dir orelse return error.DatastoreNotInitialized; + const dk = datastoreKey(key); + + const name = fileName(dk); + const exists = blk: { + dir.access(io, &name, .{}) catch |err| switch (err) { + error.FileNotFound => break :blk false, + else => return err, + }; + break :blk true; + }; + if (exists) return dk; + + var tmp_name: [FILE_NAME_LEN + TMP_SUFFIX.len]u8 = undefined; + @memcpy(tmp_name[0..FILE_NAME_LEN], &name); + @memcpy(tmp_name[FILE_NAME_LEN..], TMP_SUFFIX); + try dir.writeFile(io, .{ .sub_path = &tmp_name, .data = state_bytes }); + try dir.rename(&tmp_name, dir, &name, io); + return dk; + } + + /// Delete one state file. Per-key unit shared by `removeImpl` and the `removeMany` fan-out. + fn removeOne(self: *FileCPStateDatastore, io: std.Io, dk: DatastoreKey) anyerror!void { + const dir = self.dir orelse return error.DatastoreNotInitialized; + + const name = fileName(dk); + dir.deleteFile(io, &name) catch |err| switch (err) { + error.FileNotFound => {}, + else => return err, + }; + } + + fn removeImpl(ctx: *anyopaque, io: std.Io, dk: DatastoreKey) anyerror!void { + const self: *FileCPStateDatastore = @ptrCast(@alignCast(ctx)); + return self.removeOne(io, dk); + } + + fn removeManyImpl(ctx: *anyopaque, io: std.Io, allocator: Allocator, keys: []const DatastoreKey) anyerror!void { + const self: *FileCPStateDatastore = @ptrCast(@alignCast(ctx)); + if (keys.len == 0) return; + const futures = try allocator.alloc(std.Io.Future(anyerror!void), keys.len); + defer allocator.free(futures); + + for (keys, 0..) |dk, i| futures[i] = io.async(removeOne, .{ self, io, dk }); + + var first_err: ?anyerror = null; + for (futures) |*f| f.await(io) catch |err| { + if (first_err == null) first_err = err; + }; + if (first_err) |err| return err; + } + + fn readImpl(ctx: *anyopaque, io: std.Io, allocator: Allocator, dk: DatastoreKey) anyerror!?[]u8 { + const self: *FileCPStateDatastore = @ptrCast(@alignCast(ctx)); + const dir = self.dir orelse return error.DatastoreNotInitialized; + + const name = fileName(dk); + return dir.readFileAlloc(io, &name, allocator, .unlimited) catch |err| switch (err) { + error.FileNotFound => null, + error.IsDir => null, + else => err, + }; + } + + fn readPrefixImpl(ctx: *anyopaque, io: std.Io, dk: DatastoreKey, buf: []u8) anyerror!?usize { + const self: *FileCPStateDatastore = @ptrCast(@alignCast(ctx)); + const dir = self.dir orelse return error.DatastoreNotInitialized; + + const name = fileName(dk); + const file = dir.openFile(io, &name, .{}) catch |err| switch (err) { + error.FileNotFound, error.IsDir => return null, + else => return err, + }; + defer file.close(io); + return file.readPositionalAll(io, buf, 0) catch |err| switch (err) { + error.IsDir => return null, + else => return err, + }; + } + + fn readKeysImpl(ctx: *anyopaque, io: std.Io, allocator: Allocator) anyerror![]DatastoreKey { + const self: *FileCPStateDatastore = @ptrCast(@alignCast(ctx)); + const dir = self.dir orelse return error.DatastoreNotInitialized; + + var keys: std.ArrayListUnmanaged(DatastoreKey) = .empty; + errdefer keys.deinit(allocator); + + var it = dir.iterate(); + while (try it.next(io)) |entry| { + if (entry.kind != .file and entry.kind != .unknown) continue; + if (entry.name.len != FILE_NAME_LEN) continue; + if (!hex.hasOxPrefix(entry.name)) continue; + + var dk: DatastoreKey = undefined; + // A foreign file matching only the length + "0x" prefix decodes to a non-hex char; skip it. + // Length/space errors are pre-checked away, so any other error is unexpected and propagates. + _ = hex.hexToBytes(&dk, entry.name) catch |e| switch (e) { + error.InvalidCharacter => continue, + else => return e, + }; + try keys.append(allocator, dk); + } + + return keys.toOwnedSlice(allocator); + } +}; + +const FileStoreFixture = struct { + tmp: testing.TmpDir, + base: []u8, + store: FileCPStateDatastore, + + fn init(allocator: Allocator) !FileStoreFixture { + var tmp = testing.tmpDir(.{}); + errdefer tmp.cleanup(); + + const base = try std.fs.path.join(allocator, &.{ ".zig-cache", "tmp", &tmp.sub_path }); + errdefer allocator.free(base); + + const store = try FileCPStateDatastore.init(allocator, base); + return .{ .tmp = tmp, .base = base, .store = store }; + } + + fn deinit(self: *FileStoreFixture, allocator: Allocator, io: std.Io) void { + self.store.deinit(io); + allocator.free(self.base); + self.tmp.cleanup(); + } + + fn datastore(self: *FileStoreFixture) CPStateDatastore { + return self.store.datastore(); + } +}; + +test "InMemoryCPStateDatastore write/read/remove and insertion-ordered keys" { + const allocator = testing.allocator; + const io = std.testing.io; + var store = InMemoryCPStateDatastore.init(allocator); + defer store.deinit(); + + const ds = store.datastore(); + + const key_a = Checkpoint{ .root = [_]u8{0xa1} ** 32, .epoch = 5 }; + const key_b = Checkpoint{ .root = [_]u8{0xb2} ** 32, .epoch = 7 }; + + const dk_a = try ds.write(io, key_a, "alpha"); + const dk_b = try ds.write(io, key_b, "beta"); + + // Re-writing the same key must NOT overwrite or reorder. + _ = try ds.write(io, key_a, "OVERWRITE-ATTEMPT"); + + const read_a = (try ds.read(io, allocator, dk_a)).?; + defer allocator.free(read_a); + + try testing.expectEqualStrings("alpha", read_a); + + const keys = try ds.readKeys(io, allocator); + defer allocator.free(keys); + + try testing.expectEqual(@as(usize, 2), keys.len); + try testing.expectEqualSlices(u8, &dk_a, &keys[0]); + try testing.expectEqualSlices(u8, &dk_b, &keys[1]); + + try ds.remove(io, dk_a); + try testing.expect((try ds.read(io, allocator, dk_a)) == null); + + const keys_after = try ds.readKeys(io, allocator); + defer allocator.free(keys_after); + + try testing.expectEqual(@as(usize, 1), keys_after.len); + try testing.expectEqualSlices(u8, &dk_b, &keys_after[0]); +} + +test "InMemoryCPStateDatastore removeMany removes all keys sequentially and is no-op when absent" { + const allocator = testing.allocator; + const io = std.testing.io; + var store = InMemoryCPStateDatastore.init(allocator); + defer store.deinit(); + + const ds = store.datastore(); + + var keys: [8]DatastoreKey = undefined; + for (0..8) |i| keys[i] = try ds.write(io, .{ .root = [_]u8{@intCast(i + 1)} ** 32, .epoch = @intCast(i) }, "x"); + { + const before = try ds.readKeys(io, allocator); + defer allocator.free(before); + + try testing.expectEqual(@as(usize, 8), before.len); + } + + try ds.removeMany(io, allocator, &keys); + { + const after = try ds.readKeys(io, allocator); + defer allocator.free(after); + + try testing.expectEqual(@as(usize, 0), after.len); + } + + // Re-removing the now-absent keys must be a clean no-op. + try ds.removeMany(io, allocator, &keys); + { + const after = try ds.readKeys(io, allocator); + defer allocator.free(after); + + try testing.expectEqual(@as(usize, 0), after.len); + } +} + +test "FileCPStateDatastore write/read/remove/readKeys round-trip" { + const allocator = testing.allocator; + const io = std.testing.io; + + var fx = try FileStoreFixture.init(allocator); + defer fx.deinit(allocator, io); + const ds = fx.datastore(); + try ds.initStore(io); + + const key_a = Checkpoint{ .root = [_]u8{0xc3} ** 32, .epoch = 9 }; + const dk_a = try ds.write(io, key_a, "gamma-bytes"); + + // A second write of the same key is a no-op (the existing final file is kept). + _ = try ds.write(io, key_a, "ignored"); + + const read_a = (try ds.read(io, allocator, dk_a)).?; + defer allocator.free(read_a); + try testing.expectEqualStrings("gamma-bytes", read_a); + + const keys = try ds.readKeys(io, allocator); + defer allocator.free(keys); + try testing.expectEqual(@as(usize, 1), keys.len); + try testing.expectEqualSlices(u8, &dk_a, &keys[0]); + + try ds.remove(io, dk_a); + try testing.expect((try ds.read(io, allocator, dk_a)) == null); + // Removing an absent key is a no-op. + try ds.remove(io, dk_a); +} + +test "FileCPStateDatastore readLatestSafe returns the winner's full bytes" { + const allocator = testing.allocator; + const io = std.testing.io; + + var fx = try FileStoreFixture.init(allocator); + defer fx.deinit(allocator, io); + const ds = fx.datastore(); + try ds.initStore(io); + + const slot = computeStartSlotAtEpoch(10); + const head = try makeStateBytes(allocator, slot, slot); + defer allocator.free(head); + // A tail beyond the prefix length, so a prefix-only return fails the comparison. + const bytes = try std.mem.concat(allocator, u8, &.{ head, "tail-beyond-the-prefix" }); + defer allocator.free(bytes); + _ = try ds.write(io, .{ .root = [_]u8{0x22} ** 32, .epoch = 10 }, bytes); + + const got = (try ds.readLatestSafe(io, allocator)).?; + defer allocator.free(got); + try testing.expectEqualSlices(u8, bytes, got); +} + +test "FileCPStateDatastore readLatestSafe skips a huge-epoch foreign key" { + const allocator = testing.allocator; + const io = std.testing.io; + + var fx = try FileStoreFixture.init(allocator); + defer fx.deinit(allocator, io); + const ds = fx.datastore(); + try ds.initStore(io); + + const bytes = try makeStateBytes(allocator, 320, 320); + defer allocator.free(bytes); + _ = try ds.write(io, .{ .root = [_]u8{0xab} ** 32, .epoch = std.math.maxInt(u64) }, bytes); + + try testing.expect((try ds.readLatestSafe(io, allocator)) == null); +} + +test "FileCPStateDatastore initStore sweeps temp debris and write leaves none" { + const allocator = testing.allocator; + const io = std.testing.io; + + var fx = try FileStoreFixture.init(allocator); + defer fx.deinit(allocator, io); + const ds = fx.datastore(); + try ds.initStore(io); + + // Plant torn debris; it is not a key (wrong name shape), so readKeys ignores it. + const dir = fx.store.dir.?; + try dir.writeFile(io, .{ .sub_path = "0xdead.tmp", .data = &[_]u8{ 1, 2, 3 } }); + const empty = try ds.readKeys(io, allocator); + defer allocator.free(empty); + try testing.expectEqual(@as(usize, 0), empty.len); + + // A restart (fresh store on the same dir) sweeps the debris at initStore. + var store2 = try FileCPStateDatastore.init(allocator, fx.base); + defer store2.deinit(io); + const ds2 = store2.datastore(); + try ds2.initStore(io); + try testing.expectError(error.FileNotFound, dir.access(io, "0xdead.tmp", .{})); + + // A completed write renames its temp into place: the final file is the only entry left. + const dk = try ds2.write(io, .{ .root = [_]u8{0xcd} ** 32, .epoch = 3 }, "full-state-bytes"); + const read_back = (try ds2.read(io, allocator, dk)).?; + defer allocator.free(read_back); + try testing.expectEqualStrings("full-state-bytes", read_back); + var it = dir.iterate(); + var count: usize = 0; + while (try it.next(io)) |_| count += 1; + try testing.expectEqual(@as(usize, 1), count); +} + +test "FileCPStateDatastore initStore is repeatable" { + const allocator = testing.allocator; + const io = std.testing.io; + + var fx = try FileStoreFixture.init(allocator); + defer fx.deinit(allocator, io); + const ds = fx.datastore(); + try ds.initStore(io); + try ds.initStore(io); + + const dk = try ds.write(io, .{ .root = [_]u8{0x11} ** 32, .epoch = 1 }, "bytes"); + const back = (try ds.read(io, allocator, dk)).?; + defer allocator.free(back); + try testing.expectEqualStrings("bytes", back); +} + +test "FileCPStateDatastore initStore opens an existing dir without clobbering" { + const allocator = testing.allocator; + const io = std.testing.io; + + var fx = try FileStoreFixture.init(allocator); + defer fx.deinit(allocator, io); + + const key = Checkpoint{ .root = [_]u8{0xe5} ** 32, .epoch = 11 }; + var dk: DatastoreKey = undefined; + + // First store persists a state; the fixture's own store owns the dir. + { + const ds = fx.datastore(); + try ds.initStore(io); + dk = try ds.write(io, key, "persisted-bytes"); + } + + // Re-open the SAME dir with a fresh store (the restart case). + { + var store = try FileCPStateDatastore.init(allocator, fx.base); + defer store.deinit(io); + const ds = store.datastore(); + // Dir already exists from the first store: must succeed, opening it. + try ds.initStore(io); + + const keys = try ds.readKeys(io, allocator); + defer allocator.free(keys); + + try testing.expectEqual(@as(usize, 1), keys.len); + try testing.expectEqualSlices(u8, &dk, &keys[0]); + + const bytes = (try ds.read(io, allocator, dk)).?; + defer allocator.free(bytes); + + try testing.expectEqualStrings("persisted-bytes", bytes); + } +} + +test "FileCPStateDatastore operations before initStore return DatastoreNotInitialized" { + const allocator = testing.allocator; + const io = std.testing.io; + + var fx = try FileStoreFixture.init(allocator); + defer fx.deinit(allocator, io); + + const ds = fx.datastore(); + + const cp = Checkpoint{ .root = [_]u8{0x01} ** 32, .epoch = 1 }; + const dk = datastoreKey(cp); + + try testing.expectError(error.DatastoreNotInitialized, ds.write(io, cp, "x")); + try testing.expectError(error.DatastoreNotInitialized, ds.read(io, allocator, dk)); + try testing.expectError(error.DatastoreNotInitialized, ds.remove(io, dk)); + try testing.expectError(error.DatastoreNotInitialized, ds.readKeys(io, allocator)); +} + +test "FileCPStateDatastore removeMany fans out concurrently" { + var tmp = testing.tmpDir(.{}); + defer tmp.cleanup(); + const base = try std.fs.path.join(testing.allocator, &.{ ".zig-cache", "tmp", &tmp.sub_path }); + defer testing.allocator.free(base); + + // `.limited(4)` forces real concurrency for the fan-out regardless of host core count. + var threaded: std.Io.Threaded = .init(testing.allocator, .{ .async_limit = .limited(4) }); + defer threaded.deinit(); + const io = threaded.io(); + + var store = try FileCPStateDatastore.init(testing.allocator, base); + defer store.deinit(io); + const ds = store.datastore(); + try ds.initStore(io); + + var keys: [16]DatastoreKey = undefined; + for (0..16) |i| keys[i] = try ds.write(io, .{ .root = [_]u8{@intCast(i + 1)} ** 32, .epoch = @intCast(i) }, "x"); + try ds.removeMany(io, testing.allocator, &keys); + + const after = try ds.readKeys(io, testing.allocator); + defer testing.allocator.free(after); + try testing.expectEqual(@as(usize, 0), after.len); +} + +// Craft a minimal serialized-state buffer with just the two fields `readLatestSafe` reads: +// `state.slot` at offset 40 and `latest_block_header.slot` (last processed slot) at offset 64. +fn makeStateBytes(allocator: Allocator, state_slot: u64, last_processed_slot: u64) ![]u8 { + const buf = try allocator.alloc(u8, 72); + @memset(buf, 0); + std.mem.writeInt(u64, buf[40..][0..8], state_slot, .little); + std.mem.writeInt(u64, buf[64..][0..8], last_processed_slot, .little); + return buf; +} + +test "readLatestSafe returns null on an empty store" { + const allocator = testing.allocator; + const io = std.testing.io; + var store = InMemoryCPStateDatastore.init(allocator); + defer store.deinit(); + try testing.expect((try store.datastore().readLatestSafe(io, allocator)) == null); +} + +test "readLatestSafe skips a fork epoch (two roots at one epoch is ambiguous)" { + const allocator = testing.allocator; + const io = std.testing.io; + var store = InMemoryCPStateDatastore.init(allocator); + defer store.deinit(); + const ds = store.datastore(); + + // Two checkpoints at epoch 3 → can't tell which is canonical → readLatestSafe skips the epoch. + const slot3 = computeStartSlotAtEpoch(3); + for ([_]u8{ 0x31, 0x32 }) |tag| { + const bytes = try makeStateBytes(allocator, slot3, slot3); + defer allocator.free(bytes); + + _ = try ds.write(io, .{ .epoch = 3, .root = [_]u8{tag} ** 32 }, bytes); + } + + try testing.expect((try ds.readLatestSafe(io, allocator)) == null); +} + +test "readLatestSafe accepts a PRCS (last processed slot is boundary - 1)" { + const allocator = testing.allocator; + const io = std.testing.io; + var store = InMemoryCPStateDatastore.init(allocator); + defer store.deinit(); + const ds = store.datastore(); + + // PRCS: the state's last-processed slot is one below its epoch boundary. + const slot4 = computeStartSlotAtEpoch(4); + const bytes = try makeStateBytes(allocator, slot4, slot4 - 1); + defer allocator.free(bytes); + + _ = try ds.write(io, .{ .epoch = 4, .root = [_]u8{0x40} ** 32 }, bytes); + + const got = (try ds.readLatestSafe(io, allocator)).?; + defer allocator.free(got); + + try testing.expectEqual(slot4, try getStateSlotFromBytes(got)); +} + +test "readLatestSafe returns the highest safe boundary, skipping a fork and an off-boundary state" { + const allocator = testing.allocator; + const io = std.testing.io; + var store = InMemoryCPStateDatastore.init(allocator); + defer store.deinit(); + const ds = store.datastore(); + + const slot7 = computeStartSlotAtEpoch(7); + const slot6 = computeStartSlotAtEpoch(6); + const slot5 = computeStartSlotAtEpoch(5); + + // epoch 7: a fork (two roots) → skipped despite being newest. + for ([_]u8{ 0x71, 0x72 }) |tag| { + const bytes = try makeStateBytes(allocator, slot7, slot7); + defer allocator.free(bytes); + _ = try ds.write(io, .{ .epoch = 7, .root = [_]u8{tag} ** 32 }, bytes); + } + // epoch 6: single, but its state slot is 3 past the boundary (off-boundary) → skipped. + const bytes6 = try makeStateBytes(allocator, slot6 + 3, slot6 + 3); + defer allocator.free(bytes6); + _ = try ds.write(io, .{ .epoch = 6, .root = [_]u8{0x60} ** 32 }, bytes6); + // epoch 5: a clean boundary CRCS → the answer. + const bytes5 = try makeStateBytes(allocator, slot5, slot5); + defer allocator.free(bytes5); + _ = try ds.write(io, .{ .epoch = 5, .root = [_]u8{0x50} ** 32 }, bytes5); + + const got = (try ds.readLatestSafe(io, allocator)).?; + defer allocator.free(got); + try testing.expectEqual(slot5, try getStateSlotFromBytes(got)); +} diff --git a/src/beacon_node/chain/state_cache/key.zig b/src/beacon_node/chain/state_cache/key.zig new file mode 100644 index 000000000..3a6fc3ffa --- /dev/null +++ b/src/beacon_node/chain/state_cache/key.zig @@ -0,0 +1,55 @@ +const std = @import("std"); +const types = @import("consensus_types"); +const phase0 = types.phase0; + +const Epoch = types.primitive.Epoch.Type; + +/// The persisted key: serialized data of a checkpoint (the SSZ serialization of `phase0.Checkpoint`). +pub const DATASTORE_KEY_LEN = phase0.Checkpoint.fixed_size; +pub const DatastoreKey = [DATASTORE_KEY_LEN]u8; + +/// The spec `phase0.Checkpoint`, reused directly as the checkpoint-state cache map key. +pub const Checkpoint = phase0.Checkpoint.Type; + +/// Roots are SHA-256 outputs, so the low 8 bytes are uniformly distributed; xor the epoch in. +pub const CheckpointContext = struct { + pub fn hash(_: CheckpointContext, cp: Checkpoint) u64 { + const root_part = std.mem.readInt(u64, cp.root[0..8], .little); + return root_part ^ cp.epoch; + } + pub fn eql(_: CheckpointContext, a: Checkpoint, b: Checkpoint) bool { + return a.epoch == b.epoch and std.mem.eql(u8, &a.root, &b.root); + } +}; + +pub fn datastoreKey(cp: Checkpoint) DatastoreKey { + var out: DatastoreKey = undefined; + const n = phase0.Checkpoint.serializeIntoBytes(&cp, &out); + std.debug.assert(n == DATASTORE_KEY_LEN); + return out; +} + +pub fn checkpointFromDatastoreKey(dk: DatastoreKey) Checkpoint { + var out: Checkpoint = undefined; + // `dk` is exactly `fixed_size` bytes, so deserialize's only error (a length mismatch) cannot + // fire; the u64/Bytes32 fields have no invalid bit patterns either. + phase0.Checkpoint.deserializeFromBytes(&dk, &out) catch unreachable; + return out; +} + +/// Epoch is the first SSZ field, so it sits at offset 0. +pub fn datastoreKeyEpoch(dk: DatastoreKey) Epoch { + return std.mem.readInt(u64, dk[0..8], .little); +} + +test "datastoreKey round-trips a checkpoint" { + const cp = Checkpoint{ .root = [_]u8{0xab} ** 32, .epoch = 0x1122334455667788 }; + const dk = datastoreKey(cp); + try std.testing.expectEqual(@as(u8, 0x88), dk[0]); + try std.testing.expectEqual(@as(u8, 0xab), dk[8]); + try std.testing.expectEqual(cp.epoch, datastoreKeyEpoch(dk)); + + const back = checkpointFromDatastoreKey(dk); + try std.testing.expectEqual(cp.epoch, back.epoch); + try std.testing.expectEqualSlices(u8, &cp.root, &back.root); +} diff --git a/src/beacon_node/chain/state_cache/metrics.zig b/src/beacon_node/chain/state_cache/metrics.zig new file mode 100644 index 000000000..23f818390 --- /dev/null +++ b/src/beacon_node/chain/state_cache/metrics.zig @@ -0,0 +1,173 @@ +const std = @import("std"); +const m = @import("metrics"); + +/// Emitted as four parallel gauge series: `_sum`/`_avg`/`_min`/`_max`. +pub const AvgMinMax = struct { + sum: f64 = 0, + avg: f64 = 0, + min: f64 = 0, + max: f64 = 0, +}; + +/// No samples → all-zero (the gauge's empty-set value). +pub const AvgMinMaxAccumulator = struct { + sum: f64 = 0, + min: f64 = 0, + max: f64 = 0, + count: u64 = 0, + + pub fn add(self: *AvgMinMaxAccumulator, value: f64) void { + if (self.count == 0) { + self.min = value; + self.max = value; + } else { + self.min = @min(self.min, value); + self.max = @max(self.max, value); + } + self.sum += value; + self.count += 1; + } + + pub fn result(self: AvgMinMaxAccumulator) AvgMinMax { + if (self.count == 0) return .{}; + return .{ + .sum = self.sum, + .avg = self.sum / @as(f64, @floatFromInt(self.count)), + .min = self.min, + .max = self.max, + }; + } +}; + +/// `size`/`reads`/`seconds_since_last_read` are PULL gauges: the metrics module serializes whatever the +/// gauge holds at scrape time, so the cache refreshes them before `write()` (no push collect-callback). +pub const BlockStateCacheMetrics = struct { + lookups: Count, + hits: Count, + adds: Count, + size: Gauge, + reads_sum: F64Gauge, + reads_avg: F64Gauge, + reads_min: F64Gauge, + reads_max: F64Gauge, + seconds_since_last_read_sum: F64Gauge, + seconds_since_last_read_avg: F64Gauge, + seconds_since_last_read_min: F64Gauge, + seconds_since_last_read_max: F64Gauge, + state_cloned_count: ClonedHistogram, + + const Count = m.Counter(u64); + const Gauge = m.Gauge(u64); + const F64Gauge = m.Gauge(f64); + const ClonedHistogram = m.Histogram(u64, &.{ 1, 2, 5, 10, 50, 250 }); +}; + +/// `initializeNoop` default: metrics always emit (no `enabled()` gate), so the cache is safe to use +/// whether or not `init` is called. +pub var block_cache_metrics = m.initializeNoop(BlockStateCacheMetrics); + +/// Call once on startup. +pub fn init(comptime opts: m.RegistryOpts) void { + block_cache_metrics = .{ + .lookups = BlockStateCacheMetrics.Count.init( + "lodestar_state_cache_lookups_total", + .{ .help = "Block state cache lookups" }, + opts, + ), + .hits = BlockStateCacheMetrics.Count.init( + "lodestar_state_cache_hits_total", + .{ .help = "Block state cache hits" }, + opts, + ), + .adds = BlockStateCacheMetrics.Count.init( + "lodestar_state_cache_adds_total", + .{ .help = "Block state cache adds" }, + opts, + ), + .size = BlockStateCacheMetrics.Gauge.init( + "lodestar_state_cache_size", + .{ .help = "Block state cache size" }, + opts, + ), + .reads_sum = BlockStateCacheMetrics.F64Gauge.init( + "lodestar_state_cache_reads_sum", + .{ .help = "Sum of block state cache items total read count" }, + opts, + ), + .reads_avg = BlockStateCacheMetrics.F64Gauge.init( + "lodestar_state_cache_reads_avg", + .{ .help = "Avg of block state cache items total read count" }, + opts, + ), + .reads_min = BlockStateCacheMetrics.F64Gauge.init( + "lodestar_state_cache_reads_min", + .{ .help = "Min of block state cache items total read count" }, + opts, + ), + .reads_max = BlockStateCacheMetrics.F64Gauge.init( + "lodestar_state_cache_reads_max", + .{ .help = "Max of block state cache items total read count" }, + opts, + ), + .seconds_since_last_read_sum = BlockStateCacheMetrics.F64Gauge.init( + "lodestar_state_cache_seconds_since_last_read_sum", + .{ .help = "Sum of seconds since block state cache items were last read" }, + opts, + ), + .seconds_since_last_read_avg = BlockStateCacheMetrics.F64Gauge.init( + "lodestar_state_cache_seconds_since_last_read_avg", + .{ .help = "Avg of seconds since block state cache items were last read" }, + opts, + ), + .seconds_since_last_read_min = BlockStateCacheMetrics.F64Gauge.init( + "lodestar_state_cache_seconds_since_last_read_min", + .{ .help = "Min of seconds since block state cache items were last read" }, + opts, + ), + .seconds_since_last_read_max = BlockStateCacheMetrics.F64Gauge.init( + "lodestar_state_cache_seconds_since_last_read_max", + .{ .help = "Max of seconds since block state cache items were last read" }, + opts, + ), + .state_cloned_count = BlockStateCacheMetrics.ClonedHistogram.init( + "lodestar_state_cache_state_cloned_count", + .{ .help = "Clone count of a state served from the block cache" }, + opts, + ), + }; +} + +pub fn block() *BlockStateCacheMetrics { + return &block_cache_metrics; +} + +/// Caller must refresh the PULL gauges (see `BlockStateCacheMetrics`) before calling, so the scrape +/// reflects current state. +pub fn write(writer: anytype) !void { + try m.write(&block_cache_metrics, writer); +} + +pub fn setBlockSize(value: u64) void { + block_cache_metrics.size.set(value); +} + +pub fn setBlockReads(reads: AvgMinMax, secs: AvgMinMax) void { + block_cache_metrics.reads_sum.set(reads.sum); + block_cache_metrics.reads_avg.set(reads.avg); + block_cache_metrics.reads_min.set(reads.min); + block_cache_metrics.reads_max.set(reads.max); + block_cache_metrics.seconds_since_last_read_sum.set(secs.sum); + block_cache_metrics.seconds_since_last_read_avg.set(secs.avg); + block_cache_metrics.seconds_since_last_read_min.set(secs.min); + block_cache_metrics.seconds_since_last_read_max.set(secs.max); +} + +test "init compiles end-to-end" { + init(.{}); + defer block_cache_metrics = m.initializeNoop(BlockStateCacheMetrics); + setBlockSize(5); + setBlockReads( + .{ .sum = 4, .avg = 2, .min = 1, .max = 3 }, + .{ .sum = 1.5, .avg = 0.75, .min = 0.25, .max = 1.25 }, + ); +} diff --git a/src/beacon_node/chain/state_cache/root.zig b/src/beacon_node/chain/state_cache/root.zig new file mode 100644 index 000000000..457555697 --- /dev/null +++ b/src/beacon_node/chain/state_cache/root.zig @@ -0,0 +1,31 @@ +//! Lodestar currently keeps two state caches around. +//! 1. `BlockStateCache` is keyed by state root, and intended to keep extremely recent states around +//! (e.g. post states from the latest blocks). These states are most likely to be useful for state +//! transition of new blocks. +//! 2. `PersistentCheckpointStateCache` is keyed by checkpoint, and intended to keep states which have +//! just undergone an epoch transition. These states are useful for gossip verification and for +//! avoiding an epoch transition during state transition of first-in-epoch blocks. + +const std = @import("std"); +const testing = std.testing; + +pub const key = @import("key.zig"); +pub const block_state_cache = @import("block_state_cache.zig"); +pub const cp_datastore = @import("cp_datastore.zig"); +pub const metrics = @import("metrics.zig"); + +pub const CheckpointContext = key.CheckpointContext; +pub const DatastoreKey = key.DatastoreKey; +pub const datastoreKey = key.datastoreKey; +pub const Checkpoint = key.Checkpoint; + +pub const BlockStateCache = block_state_cache.BlockStateCache; +pub const DEFAULT_MAX_BLOCK_STATES = block_state_cache.DEFAULT_MAX_BLOCK_STATES; + +pub const CPStateDatastore = cp_datastore.CPStateDatastore; +pub const InMemoryCPStateDatastore = cp_datastore.InMemoryCPStateDatastore; +pub const FileCPStateDatastore = cp_datastore.FileCPStateDatastore; + +test { + testing.refAllDecls(@This()); +} diff --git a/src/beacon_node/root.zig b/src/beacon_node/root.zig new file mode 100644 index 000000000..75a801d7c --- /dev/null +++ b/src/beacon_node/root.zig @@ -0,0 +1,5 @@ +pub const chain = @import("chain/root.zig"); + +test { + @import("std").testing.refAllDecls(@This()); +} diff --git a/src/bls/AggregatePublicKey.zig b/src/bls/AggregatePublicKey.zig index c37939a14..42cb959a6 100644 --- a/src/bls/AggregatePublicKey.zig +++ b/src/bls/AggregatePublicKey.zig @@ -12,6 +12,11 @@ pub fn toPublicKey(self: *const Self) PublicKey { return pk; } +/// Aggregate a single public key into an aggregate public key. +pub fn add(self: *Self, other: *const PublicKey) void { + c.blst_p1_add_or_double_affine(&self.point, &self.point, &other.point); +} + /// Aggregates multiple public keys into a single aggregate public key. /// If pks_validate is true, validates each public key before aggregation. /// @@ -45,7 +50,11 @@ pub fn aggregateWithRandomness( scratch: []u64, ) BlstError!Self { if (pks.len == 0) return BlstError.AggrTypeMismatch; - if (scratch.len < c.blst_p1s_mult_pippenger_scratch_sizeof(pks.len)) { + const scratch_len = @divExact( + c.blst_p1s_mult_pippenger_scratch_sizeof(pks.len), + @sizeOf(u64), + ); + if (scratch.len < scratch_len) { return BlstError.AggrTypeMismatch; } if (pks_validate) for (pks) |pk| try pk.validate(); @@ -80,9 +89,13 @@ test aggregateWithRandomness { var pks: [num_sigs]PublicKey = undefined; var sigs: [num_sigs]Signature = undefined; - const m = c.blst_p1s_mult_pippenger_scratch_sizeof(num_sigs) * 8; + const scratch_len = @divExact( + c.blst_p1s_mult_pippenger_scratch_sizeof(num_sigs), + @sizeOf(u64), + ); const allocator = std.testing.allocator; - var scratch = try std.testing.allocator.alloc(u64, m); + + const scratch = try allocator.alloc(u64, scratch_len); defer allocator.free(scratch); var prng = std.Random.DefaultPrng.init(blk: { diff --git a/src/bls/AggregateSignature.zig b/src/bls/AggregateSignature.zig index 60e116615..6dfe7b663 100644 --- a/src/bls/AggregateSignature.zig +++ b/src/bls/AggregateSignature.zig @@ -47,7 +47,11 @@ pub fn aggregateWithRandomness( scratch: []u64, ) BlstError!Self { if (sigs_groupcheck) for (sigs) |sig| try sig.validate(false); - if (scratch.len < c.blst_p2s_mult_pippenger_scratch_sizeof(sigs.len)) { + const scratch_len = @divExact( + c.blst_p2s_mult_pippenger_scratch_sizeof(sigs.len), + @sizeOf(u64), + ); + if (scratch.len < scratch_len) { return BlstError.AggrTypeMismatch; } @@ -85,10 +89,21 @@ test aggregateWithRandomness { var pks: [num_sigs]PublicKey = undefined; var sigs: [num_sigs]Signature = undefined; - const m = c.blst_p2s_mult_pippenger_scratch_sizeof(num_sigs) * 64; + const pk_scratch_len = @divExact( + c.blst_p1s_mult_pippenger_scratch_sizeof(num_sigs), + @sizeOf(u64), + ); + const sig_scratch_len = @divExact( + c.blst_p2s_mult_pippenger_scratch_sizeof(num_sigs), + @sizeOf(u64), + ); const allocator = std.testing.allocator; - var scratch = try std.testing.allocator.alloc(u64, m); - defer allocator.free(scratch); + + const pk_scratch = try allocator.alloc(u64, pk_scratch_len); + defer allocator.free(pk_scratch); + + const sig_scratch = try allocator.alloc(u64, sig_scratch_len); + defer allocator.free(sig_scratch); var prng = std.Random.DefaultPrng.init(blk: { var seed: u64 = undefined; @@ -119,13 +134,18 @@ test aggregateWithRandomness { pks_refs[i] = &pks[i]; } - const agg_pk = try AggregatePublicKey.aggregateWithRandomness(pks_refs[0..], &rands, true, scratch[0..]); + const agg_pk = try AggregatePublicKey.aggregateWithRandomness( + pks_refs[0..], + &rands, + true, + pk_scratch, + ); const pk = agg_pk.toPublicKey(); const agg_sig = try aggregateWithRandomness( &sigs_refs, &rands, true, - scratch[0..], + sig_scratch, ); const sig = agg_sig.toSignature(); try sig.verify(true, &msgs[0], dst, null, &pk, true); diff --git a/src/bls/Pairing.zig b/src/bls/Pairing.zig index d682026a3..4601ba504 100644 --- a/src/bls/Pairing.zig +++ b/src/bls/Pairing.zig @@ -26,7 +26,7 @@ pub fn init(buffer: *align(buf_align) [Self.sizeOf()]u8, hash_or_encode: bool, d /// /// This is safe because blst is statically linked to this binding. pub fn sizeOf() usize { - const vec384_size = 384 / @sizeOf(usize); + const vec384_size = 384 / std.mem.byte_size_in_bits; const vec384fp12_size = vec384_size * 12; const point_e1_affine_size = vec384_size * 2; @@ -54,7 +54,7 @@ pub fn aggregate( pk_validate: bool, sig: ?*const Signature, sig_groupcheck: bool, - msg: []const u8, + msg: *const SigningRoot, aug: ?[]const u8, ) BlstError!void { try errorFromInt( @@ -64,8 +64,8 @@ pub fn aggregate( pk_validate, if (sig) |s| &s.point else null, sig_groupcheck, - msg.ptr, - msg.len, + msg, + @sizeOf(SigningRoot), if (aug) |a| a.ptr else null, if (aug) |a| a.len else 0, ), @@ -83,7 +83,7 @@ pub fn mulAndAggregate( sig_groupcheck: bool, scalar: []const u8, nbits: usize, - msg: []const u8, + msg: *const SigningRoot, ) BlstError!void { try errorFromInt( c.blst_pairing_chk_n_mul_n_aggr_pk_in_g1( @@ -94,8 +94,8 @@ pub fn mulAndAggregate( sig_groupcheck, scalar.ptr, nbits, - msg.ptr, - 32, + msg, + @sizeOf(SigningRoot), null, 0, ), @@ -153,3 +153,4 @@ const errorFromInt = @import("error.zig").errorFromInt; const blst = @import("root.zig"); const PublicKey = blst.PublicKey; const Signature = blst.Signature; +const SigningRoot = blst.SigningRoot; diff --git a/src/bls/PublicKey.zig b/src/bls/PublicKey.zig index e6949ab61..1577eadfa 100644 --- a/src/bls/PublicKey.zig +++ b/src/bls/PublicKey.zig @@ -61,7 +61,7 @@ pub fn serialize(self: *const Self) [SERIALIZE_SIZE]u8 { /// /// Returns the `PublicKey` on success, `BlstError` on failure. pub fn uncompress(pk_comp: []const u8) BlstError!Self { - if (pk_comp.len == COMPRESS_SIZE or (pk_comp[0] & 0x80) != 0) { + if (pk_comp.len == COMPRESS_SIZE and (pk_comp[0] & 0x80) != 0) { var pk = Self{}; try errorFromInt(c.blst_p1_uncompress(&pk.point, pk_comp.ptr)); return pk; @@ -93,5 +93,34 @@ const std = @import("std"); const BlstError = @import("error.zig").BlstError; const errorFromInt = @import("error.zig").errorFromInt; const AggregatePublicKey = @import("AggregatePublicKey.zig"); +const SecretKey = @import("SecretKey.zig"); const c = @import("root.zig").c; + +test uncompress { + const ikm: [32]u8 = [_]u8{ + 0x93, 0xad, 0x7e, 0x65, 0xde, 0xad, 0x05, 0x2a, 0x08, 0x3a, + 0x91, 0x0c, 0x8b, 0x72, 0x85, 0x91, 0x46, 0x4c, 0xca, 0x56, + 0x60, 0x5b, 0xb0, 0x56, 0xed, 0xfe, 0x2b, 0x60, 0xa6, 0x3c, + 0x48, 0x99, + }; + const sk = try SecretKey.keyGen(&ikm, null); + const pk = sk.toPublicKey(); + const pk_comp = pk.compress(); + + // Valid compressed bytes round-trip. + const pk_uncomp = try uncompress(&pk_comp); + try std.testing.expect(pk.isEqual(&pk_uncomp)); + + // Invalid lengths must be rejected, even with the compression bit set. + try std.testing.expectError(BlstError.BadEncoding, uncompress(&[_]u8{})); + try std.testing.expectError(BlstError.BadEncoding, uncompress(pk_comp[0 .. COMPRESS_SIZE - 1])); + var too_long = [_]u8{0} ** (COMPRESS_SIZE + 1); + @memcpy(too_long[0..COMPRESS_SIZE], &pk_comp); + try std.testing.expectError(BlstError.BadEncoding, uncompress(&too_long)); + + // Correct length without the compression bit must be rejected. + var no_comp_bit = pk_comp; + no_comp_bit[0] &= 0x7f; + try std.testing.expectError(BlstError.BadEncoding, uncompress(&no_comp_bit)); +} diff --git a/src/bls/SecretKey.zig b/src/bls/SecretKey.zig index 7f15486d9..a0f60bd95 100644 --- a/src/bls/SecretKey.zig +++ b/src/bls/SecretKey.zig @@ -46,18 +46,22 @@ pub fn keyGenV3(ikm: []const u8, info: ?[]const u8) BlstError!Self { /// Generate a `SecretKey` using HKDF key derivation (version 4.5). /// +/// `salt` must not be empty. /// Returns `SecretKey` on success, `BlstError` on failure. pub fn keyGenV45(ikm: []const u8, salt: []const u8, info: ?[]const u8) BlstError!Self { if (ikm.len < 32) { return BlstError.BadEncoding; } + if (salt.len == 0) { + return BlstError.BadEncoding; + } var sk = Self{}; c.blst_keygen_v4_5( &sk.value, &ikm[0], ikm.len, - &salt[0], + salt.ptr, salt.len, if (info) |i| i.ptr else null, if (info) |i| i.len else 0, @@ -67,18 +71,22 @@ pub fn keyGenV45(ikm: []const u8, salt: []const u8, info: ?[]const u8) BlstError /// Generate a `SecretKey` using HKDF key derivation (version 5). /// +/// `salt` must not be empty. /// Returns `SecretKey` on success, `BlstError` on failure. pub fn keyGenV5(ikm: []const u8, salt: []const u8, info: ?[]const u8) BlstError!Self { if (ikm.len < 32) { return BlstError.BadEncoding; } + if (salt.len == 0) { + return BlstError.BadEncoding; + } var sk = Self{}; c.blst_keygen_v5( &sk.value, &ikm[0], ikm.len, - &salt[0], + salt.ptr, salt.len, if (info) |i| i.ptr else null, if (info) |i| i.len else 0, @@ -114,13 +122,13 @@ pub fn toPublicKey(self: *const Self) PublicKey { } /// Sign a message with this `SecretKey`. Returns the `Signature` for the message. -pub fn sign(self: *const Self, msg: []const u8, dst: []const u8, aug: ?[]const u8) Signature { +pub fn sign(self: *const Self, msg: *const SigningRoot, dst: []const u8, aug: ?[]const u8) Signature { var sig = Signature{}; var q = @import("AggregateSignature.zig"){}; c.blst_hash_to_g2( @ptrCast(&q.point), - msg.ptr, - msg.len, + msg, + @sizeOf(SigningRoot), dst.ptr, dst.len, if (aug) |a| a.ptr else null, @@ -155,5 +163,14 @@ const check = @import("error.zig").check; const blst = @import("root.zig"); const PublicKey = blst.PublicKey; const Signature = blst.Signature; +const SigningRoot = blst.SigningRoot; const c = @import("root.zig").c; + +test "key generation rejects an empty salt" { + const ikm = [_]u8{0x42} ** 32; + const salt: []const u8 = &.{}; + + try std.testing.expectError(BlstError.BadEncoding, keyGenV45(&ikm, salt, null)); + try std.testing.expectError(BlstError.BadEncoding, keyGenV5(&ikm, salt, null)); +} diff --git a/src/bls/Signature.zig b/src/bls/Signature.zig index 6c512794d..57a6147ed 100644 --- a/src/bls/Signature.zig +++ b/src/bls/Signature.zig @@ -27,7 +27,7 @@ pub fn validate(self: *const Self, sig_infcheck: bool) BlstError!void { pub fn verify( self: *const Self, sig_groupcheck: bool, - msg: []const u8, + msg: *const SigningRoot, dst: []const u8, aug: ?[]const u8, pk: *const PublicKey, @@ -36,7 +36,7 @@ pub fn verify( if (sig_groupcheck) try self.validate(false); if (pk_validate) try pk.validate(); - if (msg.len == 0 or dst.len == 0) { + if (dst.len == 0) { return BlstError.BadEncoding; } @@ -44,8 +44,8 @@ pub fn verify( @ptrCast(&pk.point), &self.point, true, - msg.ptr, - msg.len, + msg, + @sizeOf(SigningRoot), dst.ptr, dst.len, if (aug) |a| a.ptr else null, @@ -62,7 +62,7 @@ pub fn aggregateVerify( self: *const Self, sig_groupcheck: bool, buffer: *align(Pairing.buf_align) [Pairing.sizeOf()]u8, - msgs: []const [32]u8, + msgs: []const SigningRoot, dst: []const u8, pks: []const PublicKey, pks_validate: bool, @@ -106,7 +106,7 @@ pub fn fastAggregateVerify( self: *const Self, sig_groupcheck: bool, buffer: *align(Pairing.buf_align) [Pairing.sizeOf()]u8, - msg: *const [32]u8, + msg: *const SigningRoot, dst: []const u8, pks: []const PublicKey, pks_validate: bool, @@ -131,7 +131,7 @@ pub fn fastAggregateVerifyPreAggregated( self: *const Self, sig_groupcheck: bool, buffer: *align(Pairing.buf_align) [Pairing.sizeOf()]u8, - msg: *const [32]u8, + msg: *const SigningRoot, dst: []const u8, pk: *const PublicKey, ) BlstError!bool { @@ -139,7 +139,7 @@ pub fn fastAggregateVerifyPreAggregated( return try self.aggregateVerify( sig_groupcheck, buffer, - @ptrCast(msg), + @as([*]const SigningRoot, @ptrCast(msg))[0..1], dst, pks[0..1], false, @@ -216,32 +216,56 @@ const c = @import("root.zig").c; const BlstError = @import("error.zig").BlstError; const errorFromInt = @import("error.zig").errorFromInt; const PublicKey = @import("root.zig").PublicKey; +const SigningRoot = @import("root.zig").SigningRoot; const AggregatePublicKey = @import("AggregatePublicKey.zig"); const AggregateSignature = @import("AggregateSignature.zig"); const Pairing = @import("Pairing.zig"); const SecretKey = @import("SecretKey.zig"); const DST = @import("root.zig").DST; +const ikm: [32]u8 = [_]u8{ + 0x93, 0xad, 0x7e, 0x65, 0xde, 0xad, 0x05, 0x2a, 0x08, 0x3a, + 0x91, 0x0c, 0x8b, 0x72, 0x85, 0x91, 0x46, 0x4c, 0xca, 0x56, + 0x60, 0x5b, 0xb0, 0x56, 0xed, 0xfe, 0x2b, 0x60, 0xa6, 0x3c, + 0x48, 0x99, +}; + +test uncompress { + const sk = try SecretKey.keyGen(&ikm, null); + const signing_root = [_]u8{0x42} ** 32; + const sig = sk.sign(&signing_root, DST, null); + const sig_comp = sig.compress(); + + // Valid compressed bytes round-trip. + const sig_uncomp = try uncompress(&sig_comp); + try std.testing.expect(sig.isEqual(&sig_uncomp)); + + // Invalid lengths must be rejected, even with the compression bit set. + try std.testing.expectError(BlstError.BadEncoding, uncompress(&[_]u8{})); + try std.testing.expectError(BlstError.BadEncoding, uncompress(sig_comp[0 .. COMPRESS_SIZE - 1])); + var too_long = [_]u8{0} ** (COMPRESS_SIZE + 1); + @memcpy(too_long[0..COMPRESS_SIZE], &sig_comp); + try std.testing.expectError(BlstError.BadEncoding, uncompress(&too_long)); + + // Correct length without the compression bit must be rejected. + var no_comp_bit = sig_comp; + no_comp_bit[0] &= 0x7f; + try std.testing.expectError(BlstError.BadEncoding, uncompress(&no_comp_bit)); +} test "test_sign_n_verify" { // sample code for consumer like on Readme - const ikm: [32]u8 = [_]u8{ - 0x93, 0xad, 0x7e, 0x65, 0xde, 0xad, 0x05, 0x2a, 0x08, 0x3a, - 0x91, 0x0c, 0x8b, 0x72, 0x85, 0x91, 0x46, 0x4c, 0xca, 0x56, - 0x60, 0x5b, 0xb0, 0x56, 0xed, 0xfe, 0x2b, 0x60, 0xa6, 0x3c, - 0x48, 0x99, - }; const sk = try SecretKey.keyGen(&ikm, null); const pk = sk.toPublicKey(); const dst = DST; - const msg = "hello foo"; - const sig = sk.sign(msg, dst, null); + const signing_root = [_]u8{0x42} ** 32; + const sig = sk.sign(&signing_root, dst, null); // aug is null try sig.verify( true, - msg, + &signing_root, dst, null, &pk, @@ -250,13 +274,6 @@ test "test_sign_n_verify" { } test aggregateVerify { - const ikm: [32]u8 = [_]u8{ - 0x93, 0xad, 0x7e, 0x65, 0xde, 0xad, 0x05, 0x2a, 0x08, 0x3a, - 0x91, 0x0c, 0x8b, 0x72, 0x85, 0x91, 0x46, 0x4c, 0xca, 0x56, - 0x60, 0x5b, 0xb0, 0x56, 0xed, 0xfe, 0x2b, 0x60, 0xa6, 0x3c, - 0x48, 0x99, - }; - const dst = DST; // aug is null diff --git a/src/bls/ThreadPool.zig b/src/bls/ThreadPool.zig index df6d092cb..b76bfd981 100644 --- a/src/bls/ThreadPool.zig +++ b/src/bls/ThreadPool.zig @@ -17,9 +17,11 @@ const Pairing = @import("Pairing.zig"); const blst = @import("root.zig"); const PublicKey = blst.PublicKey; const Signature = blst.Signature; +const SigningRoot = blst.SigningRoot; const AggregatePublicKey = blst.AggregatePublicKey; const AggregateSignature = blst.AggregateSignature; const BlstError = @import("error.zig").BlstError; +const BatchVerifyItem = @import("fast_verify.zig").BatchVerifyItem; const SecretKey = @import("SecretKey.zig"); const pippenger = @import("pippenger.zig"); @@ -121,14 +123,20 @@ pub fn init(allocator_: Allocator, io: std.Io, opts: Opts) (Allocator.Error || s std.debug.assert(opts.n_workers <= MAX_WORKERS); const pool = try allocator_.create(ThreadPool); + errdefer pool.deinit(io); + pool.* = .{ .allocator = allocator_, - .n_workers = opts.n_workers, + .n_workers = 0, .queue = .{}, }; - for (0..pool.n_workers) |i| { + + for (0..opts.n_workers) |i| { pool.threads[i] = try std.Thread.spawn(.{}, workerLoop, .{ pool, io }); + pool.n_workers += 1; } + std.debug.assert(pool.n_workers == opts.n_workers); + return pool; } @@ -194,10 +202,7 @@ pub fn submitAndWait(pool: *ThreadPool, io: std.Io, items: []*WorkItem) (PoolErr } const VerifyMultiJob = struct { - pks: []const *PublicKey, - sigs: []const *Signature, - msgs: []const []const u8, - rands: []const [32]u8, + items: []const BatchVerifyItem, dst: []const u8, pks_validate: bool, sigs_groupcheck: bool, @@ -217,21 +222,22 @@ const VerifyMultiWorkItem = struct { const job = self.job; var pairing = Pairing.init(&job.result_bufs[self.worker_id].data, true, job.dst); - const n_elems = job.pks.len; + const n_elems = job.items.len; while (true) { const i = job.counter.fetchAdd(1, .monotonic); if (i >= n_elems) break; if (job.err_flag.load(.monotonic)) break; + const item = &job.items[i]; pairing.mulAndAggregate( - job.pks[i], + item.public_key, job.pks_validate, - job.sigs[i], + item.signature, job.sigs_groupcheck, - &job.rands[i], + &item.randomness, RAND_BITS, - job.msgs[i], + &item.message, ) catch { job.err_flag.store(true, .release); break; @@ -250,31 +256,20 @@ const VerifyMultiWorkItem = struct { pub fn verifyMultipleAggregateSignatures( pool: *ThreadPool, io: std.Io, - n_elems: usize, - msgs: []const []const u8, + items: []const BatchVerifyItem, dst: []const u8, - pks: []const *PublicKey, pks_validate: bool, - sigs: []const *Signature, sigs_groupcheck: bool, - rands: []const [32]u8, ) (BlstError || PoolError || std.Io.Cancelable)!bool { - if (n_elems == 0 or - pks.len != n_elems or - sigs.len != n_elems or - msgs.len != n_elems or - rands.len != n_elems) - return BlstError.VerifyFail; + const n_elems = items.len; + if (n_elems == 0) return BlstError.VerifyFail; const n_active = @min(pool.n_workers, n_elems); var result_bufs: [MAX_WORKERS]PairingBuf = undefined; var job = VerifyMultiJob{ - .pks = pks[0..n_elems], - .sigs = sigs[0..n_elems], - .msgs = msgs[0..n_elems], - .rands = rands[0..n_elems], + .items = items, .dst = dst, .pks_validate = pks_validate, .sigs_groupcheck = sigs_groupcheck, @@ -304,7 +299,7 @@ pub fn verifyMultipleAggregateSignatures( const AggVerifyJob = struct { pks: []const *PublicKey, - msgs: []const [32]u8, + msgs: []const SigningRoot, dst: []const u8, pks_validate: bool, n_elems: usize, @@ -359,7 +354,7 @@ pub fn aggregateVerify( io: std.Io, sig: *const Signature, sig_groupcheck: bool, - msgs: []const [32]u8, + msgs: []const SigningRoot, dst: []const u8, pks: []const *PublicKey, pks_validate: bool, @@ -471,10 +466,11 @@ pub fn aggregateWithRandomness( var pk_proj: c.blst_p1 = undefined; try pippenger.parallelMSMG1(pool, io, pks, scalars_refs[0..pks.len], 64, &pk_proj); - c.blst_p1_to_affine(&pk_out.point, &pk_proj); var sig_proj: c.blst_p2 = undefined; try pippenger.parallelMSMG2(pool, io, sigs, scalars_refs[0..sigs.len], 64, &sig_proj); + + c.blst_p1_to_affine(&pk_out.point, &pk_proj); c.blst_p2_to_affine(&sig_out.point, &sig_proj); } @@ -491,12 +487,10 @@ test "verifyMultipleAggregateSignatures multi-threaded" { const num_sigs = 16; - var msgs: [num_sigs][32]u8 = undefined; - var msg_refs: [num_sigs][]const u8 = undefined; + var msgs: [num_sigs]SigningRoot = undefined; var pks: [num_sigs]PublicKey = undefined; var sigs: [num_sigs]Signature = undefined; - var pk_ptrs: [num_sigs]*PublicKey = undefined; - var sig_ptrs: [num_sigs]*Signature = undefined; + var items: [num_sigs]blst.BatchVerifyItem = undefined; var prng = std.Random.DefaultPrng.init(blk: { var seed: u64 = undefined; @@ -512,24 +506,21 @@ test "verifyMultipleAggregateSignatures multi-threaded" { const sk = try SecretKey.keyGen(&ikm_i, null); pks[i] = sk.toPublicKey(); sigs[i] = sk.sign(&msgs[i], blst.DST, null); - msg_refs[i] = &msgs[i]; - pk_ptrs[i] = &pks[i]; - sig_ptrs[i] = &sigs[i]; + items[i] = .{ + .message = msgs[i], + .public_key = &pks[i], + .signature = &sigs[i], + .randomness = undefined, + }; + std.Random.bytes(rand, &items[i].randomness); } - var rands: [num_sigs][32]u8 = undefined; - for (&rands) |*r| std.Random.bytes(rand, r); - const result = try pool.verifyMultipleAggregateSignatures( std.testing.io, - num_sigs, - &msg_refs, + &items, blst.DST, - &pk_ptrs, true, - &sig_ptrs, true, - &rands, ); try std.testing.expect(result); diff --git a/src/bls/error.zig b/src/bls/error.zig index fa37b06c3..1477ac898 100644 --- a/src/bls/error.zig +++ b/src/bls/error.zig @@ -9,23 +9,12 @@ pub const BlstError = error{ PkIsInfinity, BadScalar, MergeError, + UnknownError, }; -pub fn intFromError(e: BlstError) c_uint { - return switch (e) { - BlstError.BadEncoding => c.BLST_BAD_ENCODING, - BlstError.PointNotOnCurve => c.BLST_POINT_NOT_ON_CURVE, - BlstError.PointNotInGroup => c.BLST_POINT_NOT_IN_GROUP, - BlstError.AggrTypeMismatch => c.BLST_AGGR_TYPE_MISMATCH, - BlstError.VerifyFail => c.BLST_VERIFY_FAIL, - BlstError.PkIsInfinity => c.BLST_PK_IS_INFINITY, - BlstError.BadScalar => c.BLST_BAD_SCALAR, - BlstError.MergeError => c.BLST_BAD_SCALAR + 1, - }; -} - pub fn errorFromInt(err: c_uint) BlstError!void { switch (err) { + c.BLST_SUCCESS => return, c.BLST_BAD_ENCODING => return BlstError.BadEncoding, c.BLST_POINT_NOT_ON_CURVE => return BlstError.PointNotOnCurve, c.BLST_POINT_NOT_IN_GROUP => return BlstError.PointNotInGroup, @@ -33,6 +22,6 @@ pub fn errorFromInt(err: c_uint) BlstError!void { c.BLST_VERIFY_FAIL => return BlstError.VerifyFail, c.BLST_PK_IS_INFINITY => return BlstError.PkIsInfinity, c.BLST_BAD_SCALAR => return BlstError.BadScalar, - else => return, + else => return BlstError.UnknownError, } } diff --git a/src/bls/fast_verify.zig b/src/bls/fast_verify.zig index 5865f1c0c..665957767 100644 --- a/src/bls/fast_verify.zig +++ b/src/bls/fast_verify.zig @@ -4,6 +4,14 @@ const RAND_BYTES = 8; /// Number of random bits used for verification. const RAND_BITS = 8 * RAND_BYTES; +/// One signature set and its random coefficient for batch verification. +pub const BatchVerifyItem = struct { + message: SigningRoot, + public_key: *const PublicKey, + signature: *const Signature, + randomness: [32]u8, +}; + /// Verify multiple aggregate signatures efficiently using random coefficients. /// /// Source: https://ethresear.ch/t/fast-verification-of-multiple-bls-signatures/5407 @@ -11,16 +19,12 @@ const RAND_BITS = 8 * RAND_BYTES; /// Returns true if verification succeeds, false if verification fails, `BlstError` on error. pub fn verifyMultipleAggregateSignatures( pairing_buf: *align(Pairing.buf_align) [Pairing.sizeOf()]u8, - n_elems: usize, - msgs: []const []const u8, + items: []const BatchVerifyItem, dst: []const u8, - pks: []const *PublicKey, pks_validate: bool, - sigs: []const *Signature, sigs_groupcheck: bool, - rands: []const [32]u8, ) BlstError!bool { - if (n_elems == 0) { + if (items.len == 0) { return BlstError.VerifyFail; } @@ -30,15 +34,15 @@ pub fn verifyMultipleAggregateSignatures( dst, ); - for (0..n_elems) |i| { + for (items) |*item| { try pairing.mulAndAggregate( - pks[i], + item.public_key, pks_validate, - sigs[i], + item.signature, sigs_groupcheck, - &rands[i], + &item.randomness, RAND_BITS, - msgs[i], + &item.message, ); } @@ -52,3 +56,4 @@ const Pairing = @import("Pairing.zig"); const blst = @import("root.zig"); const PublicKey = blst.PublicKey; const Signature = blst.Signature; +const SigningRoot = blst.SigningRoot; diff --git a/src/bls/root.zig b/src/bls/root.zig index d2d76bfa5..36a986444 100644 --- a/src/bls/root.zig +++ b/src/bls/root.zig @@ -15,6 +15,10 @@ pub const AggregatePublicKey = @import("AggregatePublicKey.zig"); pub const AggregateSignature = @import("AggregateSignature.zig"); pub const BlstError = @import("error.zig").BlstError; +/// Ethereum consensus BLS messages are fixed-size signing roots. +pub const SigningRoot = [32]u8; + +pub const BatchVerifyItem = @import("fast_verify.zig").BatchVerifyItem; pub const verifyMultipleAggregateSignatures = @import("fast_verify.zig").verifyMultipleAggregateSignatures; pub const ThreadPool = @import("ThreadPool.zig"); pub const pippenger = @import("pippenger.zig"); diff --git a/src/clock/Clock.zig b/src/clock/Clock.zig new file mode 100644 index 000000000..3acf195e9 --- /dev/null +++ b/src/clock/Clock.zig @@ -0,0 +1,483 @@ +//! Event-driven beacon clock. +//! +//! Owns the stateful slot cursor (a cached `current_slot` over `slot_math`) +//! and an async I/O loop to emit slot/epoch events and dispatch waiters. +//! All public methods are safe to call from the main thread; the internal +//! loop runs as a single cooperative fiber. +//! +//! Designed for a cooperative single-fiber `std.Io` backend (e.g. zio). +//! +//! No mutex is used: under a single-fiber backend the only context switches +//! are at `await`/`sleep` yield points, and every read-modify of shared state +//! (listeners, waiter queue, `stopped`) completes synchronously between yields. +//! Two invariants make this safe: +//! 1. Listener callbacks run to completion inside an emit. They run on the +//! emitting fiber's stack, so a callback must never suspend it: that +//! stalls the remaining listeners of that slot and the whole drain. +//! Safe to call from a callback: +//! - onSlot / offSlot / onEpoch / offEpoch and stop; +//! - any current* / isCurrent* accessor and the pure-read helpers. +//! Forbidden from a callback: +//! - `await` and `sleep`, and `waitForSlot`, which suspends its caller; +//! - spawning (`std.Io.async` / `std.Io.concurrent` / `Group.async`): +//! task registration can reschedule-yield, which is the same +//! suspension. +//! Work that must await belongs on its own fiber: either a worker woken by +//! a yield-free handoff (`std.Io.Event.set` does not yield), or a fiber +//! that loops on `waitForSlot`. +//! A query while the cache lags the wall (a backlog) does not nest a +//! dispatch. It returns the fresh wall time - possibly ahead of the +//! events delivered so far - and the frame already emitting delivers +//! the rest, in order, exactly once per (listener, event). E.g. the +//! wall reaches slot 3 while slot 1 is still being emitted: +//! +//! emit 1 +//! callback: currentSlot() returns 3; instead of emitting 2 and 3 +//! itself, it stores pending_target = 3 +//! emit 2 <- the emitting frame sees pending_target and continues +//! emit 3 +//! 2. A wake-up pops its waiter from the queue *before* setting the event, so +//! a resuming `waitForSlot` frame is never still referenced by the queue. +//! A multi-executor backend (zio with `executors > 1`, or `std.Io.Threaded`) +//! would break both and require real locking. + +const std = @import("std"); +const Allocator = std.mem.Allocator; +const bounded_array = @import("bounded_array"); +const time = @import("time"); +const slot_math = @import("slot_math.zig"); + +const Clock = @This(); + +allocator: Allocator, +io: std.Io, +config: ClockConfig, +current_slot: ?Slot = null, + +stopped: bool = false, +dispatching: bool = false, +pending_target: ?Slot = null, +loop_future: ?std.Io.Future(void) = null, + +// IDs start at 1 so callers can use 0 as an unset sentinel. +next_listener_id: ListenerId = 1, +slot_listeners: bounded_array.BoundedArray(SlotListenerEntry, max_slot_listeners) = .{}, +epoch_listeners: bounded_array.BoundedArray(EpochListenerEntry, max_epoch_listeners) = .{}, + +waiters: WaiterQueue, + +pub const Slot = slot_math.Slot; +pub const Epoch = slot_math.Epoch; +pub const ClockConfig = slot_math.ClockConfig; +pub const ListenerId = u64; + +pub const max_slot_listeners: u32 = 16; +pub const max_epoch_listeners: u32 = 16; +pub const max_waiters: u32 = 1024; + +pub const Error = error{ + InvalidConfig, + OutOfMemory, + ListenerLimitReached, + WaiterLimitReached, + Aborted, + ConcurrencyUnavailable, +}; + +const WaitState = struct { + event: std.Io.Event = .unset, + aborted: bool = false, +}; + +const WaiterEntry = struct { + target: Slot, + state: *WaitState, +}; + +const SlotListenerEntry = struct { + id: ListenerId, + callback: *const fn (ctx: ?*anyopaque, slot: Slot) void, + ctx: ?*anyopaque, +}; + +const EpochListenerEntry = struct { + id: ListenerId, + callback: *const fn (ctx: ?*anyopaque, epoch: Epoch) void, + ctx: ?*anyopaque, +}; + +const WaiterQueue = std.PriorityQueue(WaiterEntry, void, struct { + fn compare(_: void, a: WaiterEntry, b: WaiterEntry) std.math.Order { + return std.math.order(a.target, b.target); + } +}.compare); + +pub fn init( + self: *Clock, + allocator: Allocator, + io_handle: std.Io, + config: ClockConfig, +) Error!void { + try config.validate(); + self.* = .{ + .allocator = allocator, + .io = io_handle, + .config = config, + .current_slot = slot_math.slotAtMs(config, time.nowMs(io_handle)), + .waiters = WaiterQueue.initContext({}), + }; + // Reserve full waiter capacity up front so waitForSlot's push after the + // limit check can neither allocate nor fail. + try self.waiters.ensureTotalCapacity(allocator, max_waiters); +} + +/// Start the auto-advance loop. Idempotent. +pub fn start(self: *Clock) Error!void { + if (self.loop_future != null) return; + self.loop_future = try std.Io.concurrent(self.io, Clock.runAutoLoop, .{self}); +} + +/// Signal the loop to stop and abort all pending waiters. Idempotent. +pub fn stop(self: *Clock) void { + if (self.stopped) return; + self.stopped = true; + self.abortAllWaiters(); +} + +/// Signal the loop to stop, cancel the fiber, and wait for it to finish. +pub fn join(self: *Clock) void { + self.stop(); + var maybe_future = self.loop_future; + self.loop_future = null; + if (maybe_future) |*future| { + future.cancel(self.io); + } +} + +/// Release all resources. Calls `stop()` + `join()` internally. +pub fn deinit(self: *Clock) void { + self.stop(); + self.join(); + self.waiters.deinit(self.allocator); + self.* = undefined; +} + +/// Register a slot listener. Returns an ID for later removal via `offSlot`. +pub fn onSlot( + self: *Clock, + callback: *const fn (ctx: ?*anyopaque, slot: Slot) void, + ctx: ?*anyopaque, +) Error!ListenerId { + if (self.slot_listeners.full()) return error.ListenerLimitReached; + self.slot_listeners.push(.{ + .id = self.next_listener_id, + .callback = callback, + .ctx = ctx, + }); + const id = self.next_listener_id; + self.next_listener_id += 1; + return id; +} + +/// Unregister a slot listener. Returns `true` if found and removed. +pub fn offSlot(self: *Clock, id: ListenerId) bool { + for (self.slot_listeners.slice(), 0..) |listener, i| { + if (listener.id == id) { + self.slot_listeners.orderedRemove(@intCast(i)); + return true; + } + } + return false; +} + +/// Register an epoch listener. Returns an ID for later removal via `offEpoch`. +/// An epoch event fires once when the epoch of the advancing slot increases. +pub fn onEpoch( + self: *Clock, + callback: *const fn (ctx: ?*anyopaque, epoch: Epoch) void, + ctx: ?*anyopaque, +) Error!ListenerId { + if (self.epoch_listeners.full()) return error.ListenerLimitReached; + self.epoch_listeners.push(.{ + .id = self.next_listener_id, + .callback = callback, + .ctx = ctx, + }); + const id = self.next_listener_id; + self.next_listener_id += 1; + return id; +} + +/// Unregister an epoch listener. Returns `true` if found and removed. +pub fn offEpoch(self: *Clock, id: ListenerId) bool { + for (self.epoch_listeners.slice(), 0..) |listener, i| { + if (listener.id == id) { + self.epoch_listeners.orderedRemove(@intCast(i)); + return true; + } + } + return false; +} + +// Each "current" accessor derives from catchUp()'s single wall-clock read; a +// separate time.nowMs (the pure-read shape below) could land in a slot whose +// events have not been delivered yet. + +pub fn currentSlot(self: *Clock) ?Slot { + return self.catchUp().slot; +} + +pub fn currentEpoch(self: *Clock) ?Epoch { + const slot = self.catchUp().slot orelse return null; + return slot_math.epochAtSlot(self.config, slot); +} + +pub fn currentSlotOrGenesis(self: *Clock) Slot { + return self.currentSlot() orelse 0; +} + +pub fn currentEpochOrGenesis(self: *Clock) Epoch { + return self.currentEpoch() orelse 0; +} + +pub fn currentSlotWithGossipDisparity(self: *Clock) ?Slot { + return slot_math.slotWithGossipDisparity(self.config, self.catchUp().now_ms); +} + +pub fn isCurrentSlotGivenGossipDisparity(self: *Clock, slot: Slot) bool { + return slot_math.isCurrentSlotGivenGossipDisparity(self.config, slot, self.catchUp().now_ms); +} + +// Unlike the catchUp-backed accessors above, the helpers below are pure +// reads: they never advance the cache and never emit events. + +/// Returns the slot if the internal clock were advanced by `tolerance_ms`. +pub fn slotWithFutureToleranceMs(self: *const Clock, tolerance_ms: u64) ?Slot { + return slot_math.slotWithFutureToleranceMs(self.config, time.nowMs(self.io), tolerance_ms); +} + +/// Returns the slot if the internal clock were reversed by `tolerance_ms`. +pub fn slotWithPastToleranceMs(self: *const Clock, tolerance_ms: u64) Slot { + return slot_math.slotWithPastToleranceMs(self.config, time.nowMs(self.io), tolerance_ms); +} + +/// Returns the seconds from the start of `slot` to `to_sec` (or now). +pub fn secFromSlot(self: *const Clock, slot: Slot, to_sec: ?u64) i64 { + return slot_math.secFromSlot( + self.config, + slot, + to_sec orelse @divFloor(time.nowMs(self.io), 1000), + ); +} + +/// Returns the milliseconds from the start of `slot` to `to_ms` (or now). +pub fn msFromSlot(self: *const Clock, slot: Slot, to_ms: ?u64) i64 { + return slot_math.msFromSlot(self.config, slot, to_ms orelse time.nowMs(self.io)); +} + +/// Suspend the calling fiber until the clock reaches `target`, then return. +/// Returns immediately if `target` has already been reached, and +/// `error.Aborted` if the clock is stopped before or during the wait. +/// +/// Reachable errors: {Aborted, WaiterLimitReached}. +/// +/// The wait is not a cancellation point: an external fiber-cancel takes effect +/// only once the wait resolves via dispatch or stop(). A wait on a clock that +/// is never started and never read blocks until stop(). +/// +/// Must NEVER be called from a listener callback. +pub fn waitForSlot(self: *Clock, target: Slot) Error!void { + if (self.stopped) return error.Aborted; + _ = self.catchUp(); + // Reached is judged by the cursor, not the wall time catchUp returned. A + // stop during the drain suppresses events past the cursor, and a wait for + // a suppressed slot must abort rather than resolve. + if (self.current_slot) |slot| { + if (slot >= target) return; + } + if (self.waiters.count() >= max_waiters) return error.WaiterLimitReached; + // A catchUp callback may have called stop(); checked after the + // reached-check so a wait that reached its target still resolves. + if (self.stopped) return error.Aborted; + + var waiter: WaitState = .{}; + // Capacity was reserved at init and count < max_waiters here. + self.waiters.push(self.allocator, .{ .target = target, .state = &waiter }) catch unreachable; + // Checks that waitForSlot was not called from a listener callback. + if (self.current_slot) |cs| std.debug.assert(self.waiters.peek().?.target > cs); + + waiter.event.waitUncancelable(self.io); + return if (waiter.aborted) error.Aborted else {}; +} + +const WallTime = struct { now_ms: u64, slot: ?Slot }; + +/// Advance to wall-clock time, emitting any pending slot/epoch events, and +/// return the wall time it used. Emits nothing if already caught up or pre-genesis. +/// +/// Normally catchUp drains every pending event before returning, so the +/// returned slot is never ahead of delivery. The two exceptions are a query +/// from inside a listener callback (see the reentrancy notes in the module +/// header) and a stop() from a callback, which freezes the cursor and +/// suppresses the slots past it. +fn catchUp(self: *Clock) WallTime { + const now_ms = time.nowMs(self.io); + const slot = slot_math.slotAtMs(self.config, now_ms); + const wall_time: WallTime = .{ .now_ms = now_ms, .slot = slot }; + const target = slot orelse return wall_time; + + if (self.dispatching) { + // A reentrant query only records the target; the frame that set + // `dispatching` drains it. @max: a wall step-back (NTP) must not + // regress a recorded target. + self.pending_target = @max(self.pending_target orelse target, target); + return wall_time; + } + self.dispatching = true; + defer self.dispatching = false; + // The pending defer runs before the dispatching defer, so pending is null + // whenever dispatching is false. + std.debug.assert(self.pending_target == null); + self.pending_target = target; + // Backstop: a stopped exit leaves the loop with pending still set. + defer self.pending_target = null; + while (!self.stopped) { + const drain_target = self.pending_target orelse break; + self.pending_target = null; + self.dispatchTo(drain_target); + } + return wall_time; +} + +fn emitSlot(self: *Clock, slot: Slot) void { + std.debug.assert(self.dispatching); + var snapshot = self.slot_listeners; + for (snapshot.slice()) |listener| { + listener.callback(listener.ctx, slot); + } +} + +fn emitEpoch(self: *Clock, epoch: Epoch) void { + std.debug.assert(self.dispatching); + var snapshot = self.epoch_listeners; + for (snapshot.slice()) |listener| { + listener.callback(listener.ctx, epoch); + } +} + +fn dispatchWaiters(self: *Clock, current_slot: ?Slot) void { + std.debug.assert(self.dispatching); + const slot = current_slot orelse return; + while (self.waiters.peek()) |head| { + if (head.target > slot) break; + const waiter = self.waiters.pop().?; + // Checks that nothing set aborted while the entry was still queued. + std.debug.assert(!waiter.state.aborted); + waiter.state.event.set(self.io); + } +} + +fn abortAllWaiters(self: *Clock) void { + while (self.waiters.pop()) |waiter| { + // A reached target already satisfied the wait (waitForSlot resolves + // once current_slot >= target); stopping only aborts slots that can + // no longer be emitted. + const reached = if (self.current_slot) |cs| waiter.target <= cs else false; + waiter.state.aborted = !reached; + waiter.state.event.set(self.io); + } +} + +const Event = union(enum) { + slot: Slot, + epoch: Epoch, +}; + +const AdvanceIterator = struct { + config: ClockConfig, + current_slot: *?Slot, + target: Slot, + pending_epoch: ?Epoch = null, + + /// Advances the clock one step at a time, yielding slot and epoch events. + /// For each slot advancement: yields .slot first, then .epoch if an epoch + /// boundary was crossed. + /// Returns null when caught up to target. + fn next(self: *AdvanceIterator) ?Event { + if (self.pending_epoch) |epoch| { + self.pending_epoch = null; + return .{ .epoch = epoch }; + } + + const current = self.current_slot.*; + if (current == null) { + self.current_slot.* = 0; + return .{ .slot = 0 }; + } + + const cur = current.?; + if (cur >= self.target) return null; + + const next_slot = cur + 1; + self.current_slot.* = next_slot; + + const prev_epoch = slot_math.epochAtSlot(self.config, cur); + const new_epoch = slot_math.epochAtSlot(self.config, next_slot); + if (prev_epoch < new_epoch) { + self.pending_epoch = new_epoch; + } + + return .{ .slot = next_slot }; + } +}; + +/// Advances the clock toward `target` one event at a time. Stopping early +/// is legal: the cursor simply stays at the last slot the iterator yielded, +/// nothing rolls back. +fn advanceTo(self: *Clock, target: Slot) AdvanceIterator { + return .{ + .config = self.config, + .current_slot = &self.current_slot, + .target = target, + }; +} + +/// Walk the cursor to `target`, emitting each event. +fn dispatchTo(self: *Clock, target: Slot) void { + std.debug.assert(self.dispatching); + var iter = self.advanceTo(target); + // Check `stopped` before iter.next() so a callback that calls stop() + // can't leave current_slot one ahead of the last-emitted slot. + while (true) { + if (self.stopped) break; + const event = iter.next() orelse break; + switch (event) { + .slot => |s| { + self.emitSlot(s); + self.dispatchWaiters(s); + }, + .epoch => |e| self.emitEpoch(e), + } + } +} + +fn runAutoLoop(self: *Clock) void { + while (!self.stopped) { + const now_ms = time.nowMs(self.io); + const next_ms = slot_math.msUntilNextSlot(self.config, now_ms); + const sleep_ms: i64 = @intCast(@max(@as(u64, 1), next_ms)); + + // error.Canceled comes from join()'s fiber-cancel. + std.Io.sleep( + self.io, + std.Io.Duration.fromMilliseconds(sleep_ms), + .boot, + ) catch break; + + if (self.stopped) break; + _ = self.catchUp(); + } + // join() calls stop() before cancelling the fiber, so the Canceled break + // also exits with stopped set. + std.debug.assert(self.stopped); +} diff --git a/src/clock/EventClock.zig b/src/clock/EventClock.zig deleted file mode 100644 index b9b89fc1b..000000000 --- a/src/clock/EventClock.zig +++ /dev/null @@ -1,1298 +0,0 @@ -//! Layer 2 – Event-driven beacon clock. -//! -//! Combines `SlotClock` with an async I/O loop to emit slot/epoch events -//! and dispatch waiters. All public methods are safe to call from the -//! main thread; the internal loop runs as a single cooperative fiber. -//! -//! Designed for a cooperative single-fiber `std.Io` backend (e.g. zio). -//! `start()` and `waitForSlot()` use `std.Io.concurrent` so a backend -//! that can't guarantee concurrent execution surfaces as -//! `error.ConcurrencyUnavailable` rather than deadlocking. -//! -//! No mutex is used: under a single-fiber backend the only context switches -//! are at `await`/`sleep` yield points, and every read-modify of shared state -//! (listeners, waiter queue, `stopped`) completes synchronously between yields. -//! Two invariants make this safe: -//! 1. Listener callbacks must NOT yield (no `await`/`sleep`); they run to -//! completion inside an emit so the listener/waiter state can't be mutated -//! mid-emit. The snapshot copy further decouples iteration from `offSlot`. -//! 2. `cancel()` removes its waiter from the queue *before* it yields, so a -//! concurrent `dispatchWaiters` can no longer observe it. -//! A multi-executor backend (zio with `executors > 1`, or `std.Io.Threaded`) -//! would break both and require real locking. - -const std = @import("std"); -const Allocator = std.mem.Allocator; -const bounded_array = @import("bounded_array"); -const time = @import("time"); -const slot_math = @import("slot_math.zig"); -const SlotClock = @import("SlotClock.zig"); - -const EventClock = @This(); - -allocator: Allocator, -io: std.Io, -clock: SlotClock, - -stopped: bool = false, -loop_future: ?std.Io.Future(void) = null, - -next_listener_id: ListenerId = 1, -slot_listeners: bounded_array.BoundedArray(SlotListenerEntry, max_slot_listeners) = .{}, -epoch_listeners: bounded_array.BoundedArray(EpochListenerEntry, max_epoch_listeners) = .{}, -slot_snapshot: bounded_array.BoundedArray(SlotSnapshot, max_slot_listeners) = .{}, -epoch_snapshot: bounded_array.BoundedArray(EpochSnapshot, max_epoch_listeners) = .{}, - -waiters: WaiterQueue, - -pub const Slot = slot_math.Slot; -pub const Epoch = slot_math.Epoch; -pub const ClockConfig = slot_math.ClockConfig; -pub const ListenerId = u64; - -pub const max_slot_listeners: u32 = 16; -pub const max_epoch_listeners: u32 = 16; -pub const max_waiters: u32 = 1024; - -pub const Error = error{ - InvalidConfig, - OutOfMemory, - ListenerLimitReached, - WaiterLimitReached, - Aborted, - ConcurrencyUnavailable, -}; - -const WaitState = struct { - io: std.Io, - allocator: Allocator, - event: std.Io.Event = .unset, - aborted: bool = false, -}; - -const WaiterEntry = struct { - target: Slot, - state: *WaitState, -}; - -const SlotListenerEntry = struct { - id: ListenerId, - callback: *const fn (ctx: ?*anyopaque, slot: Slot) void, - ctx: ?*anyopaque, -}; - -const EpochListenerEntry = struct { - id: ListenerId, - callback: *const fn (ctx: ?*anyopaque, epoch: Epoch) void, - ctx: ?*anyopaque, -}; - -const SlotSnapshot = struct { - callback: *const fn (ctx: ?*anyopaque, slot: Slot) void, - ctx: ?*anyopaque, -}; - -const EpochSnapshot = struct { - callback: *const fn (ctx: ?*anyopaque, epoch: Epoch) void, - ctx: ?*anyopaque, -}; - -const WaiterQueue = std.PriorityQueue(WaiterEntry, void, struct { - fn compare(_: void, a: WaiterEntry, b: WaiterEntry) std.math.Order { - return std.math.order(a.target, b.target); - } -}.compare); - -pub fn init( - self: *EventClock, - allocator: Allocator, - config: ClockConfig, - io_handle: std.Io, -) Error!void { - self.* = .{ - .allocator = allocator, - .io = io_handle, - .clock = undefined, - .waiters = WaiterQueue.initContext({}), - }; - self.clock = try SlotClock.init(config, io_handle); -} - -/// Start the auto-advance loop. Idempotent; second call is a no-op. -pub fn start(self: *EventClock) Error!void { - if (self.loop_future != null) return; - self.loop_future = std.Io.concurrent(self.io, EventClock.runAutoLoop, .{self}) catch - return error.ConcurrencyUnavailable; -} - -/// Signal the loop to stop and abort all pending waiters. Idempotent. -pub fn stop(self: *EventClock) void { - if (self.stopped) return; - self.stopped = true; - self.abortAllWaiters(); -} - -/// Signal the loop to stop, cancel the fiber, and wait for it to finish. -pub fn join(self: *EventClock) void { - self.stop(); - var maybe_future = self.loop_future; - self.loop_future = null; - if (maybe_future) |*future| { - future.cancel(self.io); - future.await(self.io); - } -} - -/// Release all resources. Calls `stop()` + `join()` internally. -pub fn deinit(self: *EventClock) void { - self.stop(); - self.join(); - self.waiters.deinit(self.allocator); - self.* = undefined; -} - -// Inside a callback, `offSlot` / `offEpoch` are safe; `onSlot` / `onEpoch` -// are not — they may overwrite the snapshot iterated by the active emit. - -/// Register a slot listener. Returns an ID for later removal via `offSlot`. -pub fn onSlot( - self: *EventClock, - callback: *const fn (ctx: ?*anyopaque, slot: Slot) void, - ctx: ?*anyopaque, -) Error!ListenerId { - if (self.slot_listeners.full()) return error.ListenerLimitReached; - self.slot_listeners.push(.{ - .id = self.next_listener_id, - .callback = callback, - .ctx = ctx, - }); - const id = self.next_listener_id; - self.next_listener_id += 1; - return id; -} - -/// Unregister a slot listener. Returns `true` if found and removed. -pub fn offSlot(self: *EventClock, id: ListenerId) bool { - for (self.slot_listeners.slice(), 0..) |listener, i| { - if (listener.id == id) { - self.slot_listeners.orderedRemove(@intCast(i)); - return true; - } - } - return false; -} - -/// Register an epoch listener. Returns an ID for later removal via `offEpoch`. -pub fn onEpoch( - self: *EventClock, - callback: *const fn (ctx: ?*anyopaque, epoch: Epoch) void, - ctx: ?*anyopaque, -) Error!ListenerId { - if (self.epoch_listeners.full()) return error.ListenerLimitReached; - self.epoch_listeners.push(.{ - .id = self.next_listener_id, - .callback = callback, - .ctx = ctx, - }); - const id = self.next_listener_id; - self.next_listener_id += 1; - return id; -} - -/// Unregister an epoch listener. Returns `true` if found and removed. -pub fn offEpoch(self: *EventClock, id: ListenerId) bool { - for (self.epoch_listeners.slice(), 0..) |listener, i| { - if (listener.id == id) { - self.epoch_listeners.orderedRemove(@intCast(i)); - return true; - } - } - return false; -} - -// "current" accessors call catchUp() first so a read flushes any pending -// slot/epoch events before returning. - -pub fn currentSlot(self: *EventClock) ?Slot { - self.catchUp(); - return self.clock.currentSlot(); -} - -pub fn currentEpoch(self: *EventClock) ?Epoch { - self.catchUp(); - return self.clock.currentEpoch(); -} - -pub fn currentSlotOrGenesis(self: *EventClock) Slot { - self.catchUp(); - return self.clock.currentSlotOrGenesis(); -} - -pub fn currentEpochOrGenesis(self: *EventClock) Epoch { - self.catchUp(); - return self.clock.currentEpochOrGenesis(); -} - -pub fn currentSlotWithGossipDisparity(self: *EventClock) ?Slot { - self.catchUp(); - return self.clock.currentSlotWithGossipDisparity(); -} - -pub fn isCurrentSlotGivenGossipDisparity(self: *EventClock, slot: Slot) bool { - self.catchUp(); - return self.clock.isCurrentSlotGivenGossipDisparity(slot); -} - -/// Return type from `waitForSlot`. The caller MUST either: -/// - call `await()` to wait for the target slot and release resources, OR -/// - call `cancel()` to abort and release resources, OR -/// - call `stop()` on the EventClock and THEN `await()` to get `error.Aborted`. -/// Dropping a WaitForSlotResult without calling `await` or `cancel` leaks -/// the internal WaitState. -/// -/// Idiomatic usage with `errdefer`: -/// var fut = try ec.waitForSlot(target); -/// errdefer fut.cancel(); -/// try fut.await(io); -pub const WaitForSlotResult = struct { - inner: std.Io.Future(Error!void), - state: ?*WaitState, - clock: ?*EventClock, - - /// Create an immediately-resolved result (no async work needed). - /// Relies on `std.Io.Future.await` returning `.result` when `.any_future == null`. - fn immediate(result: Error!void) WaitForSlotResult { - return .{ - .inner = .{ .any_future = null, .result = result }, - .state = null, - .clock = null, - }; - } - - pub fn await(self: *WaitForSlotResult, io: std.Io) Error!void { - const result = self.inner.await(io); - // Free state only AFTER the fiber returns, so it can't observe a - // freed `state.aborted` between event-wake and its own return. - if (self.state) |s| s.allocator.destroy(s); - self.state = null; - self.clock = null; - return result; - } - - /// Abort a pending wait and release its resources. Idempotent — safe - /// to call on an already-awaited, already-cancelled, or immediate result. - pub fn cancel(self: *WaitForSlotResult) void { - const state = self.state orelse return; - // Remove from waiter queue before freeing, so abortAllWaiters - // won't dereference the freed state pointer. - if (self.clock) |clock| { - for (clock.waiters.items, 0..) |entry, i| { - if (entry.state == state) { - _ = clock.waiters.popIndex(i); - break; - } - } - } - state.aborted = true; - state.event.set(state.io); - // Must await the fiber so it finishes before we free its state. - // The fiber returns error.Aborted (expected) or {} (already dispatched). - _ = self.inner.await(state.io) catch |err| { - std.debug.assert(err == error.Aborted); - }; - state.allocator.destroy(state); - self.state = null; - self.clock = null; - } -}; - -/// Return a future that resolves when the clock reaches `target`. -/// See `WaitForSlotResult` for the caller's obligations. -pub fn waitForSlot(self: *EventClock, target: Slot) Error!WaitForSlotResult { - if (self.stopped) { - return WaitForSlotResult.immediate(error.Aborted); - } - self.catchUp(); - if (self.clock.current_slot) |slot| { - if (slot >= target) { - return WaitForSlotResult.immediate({}); - } - } - if (self.waiters.count() >= max_waiters) { - return error.WaiterLimitReached; - } - - const state = self.allocator.create(WaitState) catch return error.OutOfMemory; - errdefer self.allocator.destroy(state); - - state.* = .{ - .io = self.io, - .allocator = self.allocator, - }; - - if (self.stopped) { - self.allocator.destroy(state); - return WaitForSlotResult.immediate(error.Aborted); - } - self.waiters.push( - self.allocator, - .{ .target = target, .state = state }, - ) catch return error.OutOfMemory; - self.dispatchWaiters(self.clock.current_slot); - - const inner = std.Io.concurrent(self.io, waitForSlotFutureAwait, .{state}) catch { - for (self.waiters.items, 0..) |entry, i| { - if (entry.state == state) { - _ = self.waiters.popIndex(i); - break; - } - } - return error.ConcurrencyUnavailable; - }; - - return .{ - .inner = inner, - .state = state, - .clock = self, - }; -} - -/// Ensure event-clock state is caught up to wall-clock time. -/// Emits any intermediate slot/epoch events to listeners. -/// No-op if already caught up or pre-genesis (currentSlot() returns null). -fn catchUp(self: *EventClock) void { - if (self.clock.currentSlot()) |wall_slot| { - self.advanceAndDispatch(wall_slot); - } -} - -fn emitSlot(self: *EventClock, slot: Slot) void { - self.slot_snapshot.clear(); - for (self.slot_listeners.slice()) |listener| { - self.slot_snapshot.push(.{ - .callback = listener.callback, - .ctx = listener.ctx, - }); - } - for (self.slot_snapshot.slice()) |listener| { - listener.callback(listener.ctx, slot); - } -} - -fn emitEpoch(self: *EventClock, epoch: Epoch) void { - self.epoch_snapshot.clear(); - for (self.epoch_listeners.slice()) |listener| { - self.epoch_snapshot.push(.{ - .callback = listener.callback, - .ctx = listener.ctx, - }); - } - for (self.epoch_snapshot.slice()) |listener| { - listener.callback(listener.ctx, epoch); - } -} - -fn dispatchWaiters(self: *EventClock, current_slot: ?Slot) void { - const slot = current_slot orelse return; - while (self.waiters.peek()) |head| { - if (head.target > slot) break; - const waiter = self.waiters.pop().?; - waiter.state.aborted = false; - waiter.state.event.set(waiter.state.io); - } -} - -fn abortAllWaiters(self: *EventClock) void { - while (self.waiters.pop()) |waiter| { - // A reached target already satisfied the wait (waitForSlot resolves - // once current_slot >= target); stopping only aborts slots that can - // no longer be emitted. - const reached = if (self.clock.current_slot) |cs| waiter.target <= cs else false; - waiter.state.aborted = !reached; - waiter.state.event.set(waiter.state.io); - } -} - -fn advanceAndDispatch(self: *EventClock, target: Slot) void { - var iter = self.clock.advanceTo(target); - // Check `stopped` *before* iter.next() so a callback that calls stop() - // can't leave current_slot one ahead of the last-emitted slot. - while (true) { - if (self.stopped) break; - const event = iter.next() orelse break; - switch (event) { - .slot => |s| { - self.emitSlot(s); - self.dispatchWaiters(s); - }, - .epoch => |e| self.emitEpoch(e), - } - } -} - -fn runAutoLoop(self: *EventClock) void { - while (!self.stopped) { - const now_ms = time.nowMs(self.clock.io); - const next_ms = slot_math.msUntilNextSlot(self.clock.config, now_ms); - const sleep_ms: i64 = @intCast(@max(@as(u64, 1), next_ms)); - - // Sleep failure: cancellation (from join()) exits the loop; - // other errors re-check the stopped flag. - std.Io.sleep( - self.io, - std.Io.Duration.fromMilliseconds(sleep_ms), - .awake, - ) catch |err| { - if (err == error.Canceled) break; - std.log.debug("EventClock: sleep failed ({s}), retrying", .{@errorName(err)}); - continue; - }; - - if (self.stopped) break; - // Only advance after genesis. Before genesis currentSlot() returns - // null — skipping here prevents emitting slot 0 prematurely. - if (self.clock.currentSlot()) |slot| { - self.advanceAndDispatch(slot); - } - } - // Non-terminating event loop: exits only when `self.stopped` is set. - // - normal stop(): sets flag, next iteration's `!self.stopped` exits - // - join(): always calls stop() before cancelling the fiber, so the - // `error.Canceled` break also satisfies stopped == true - std.debug.assert(self.stopped); -} - -fn waitForSlotFutureAwait(state: *WaitState) Error!void { - // Do NOT free state here — `state.aborted` is read after the wake, - // and the caller (`WaitForSlotResult.await`) frees only once this fiber - // has fully returned. - state.event.waitUncancelable(state.io); - if (state.aborted) return error.Aborted; -} - -const testing = std.testing; -const zio = @import("zio"); - -const EventTraceState = struct { - slots: [64]Slot = undefined, - slot_len: usize = 0, - epochs: [64]u64 = undefined, - epoch_len: usize = 0, - - fn onSlot(ctx: ?*anyopaque, slot: Slot) void { - const self: *EventTraceState = @ptrCast(@alignCast(ctx.?)); - if (self.slot_len >= self.slots.len) return; - self.slots[self.slot_len] = slot; - self.slot_len += 1; - } - - fn onEpoch(ctx: ?*anyopaque, epoch: u64) void { - const self: *EventTraceState = @ptrCast(@alignCast(ctx.?)); - if (self.epoch_len >= self.epochs.len) return; - self.epochs[self.epoch_len] = epoch; - self.epoch_len += 1; - } -}; - -test "lifecycle: init -> register -> start -> receive events -> stop" { - const rt = try zio.Runtime.init(testing.allocator, .{}); - defer rt.deinit(); - const io_handle = rt.io(); - - const base_now = time.nowSec(io_handle); - - var clock: EventClock = undefined; - try clock.init(testing.allocator, .{ - .genesis_time_sec = base_now, - .slot_duration_ms = 1_000, - .slots_per_epoch = 8, - }, io_handle); - defer clock.deinit(); - - var trace = EventTraceState{}; - _ = try clock.onSlot(EventTraceState.onSlot, &trace); - - try clock.start(); - - const start_slot = clock.currentSlotOrGenesis(); - var fut = try clock.waitForSlot(start_slot + 1); - errdefer fut.cancel(); - try fut.await(io_handle); - - try testing.expect(trace.slot_len > 0); -} - -test "waitForSlot resolves immediately when at target" { - const rt = try zio.Runtime.init(testing.allocator, .{}); - defer rt.deinit(); - const io_handle = rt.io(); - - const base_now = time.nowSec(io_handle); - - var clock: EventClock = undefined; - try clock.init(testing.allocator, .{ - .genesis_time_sec = base_now, - .slot_duration_ms = 1_000, - .slots_per_epoch = 8, - }, io_handle); - defer clock.deinit(); - - const current = clock.currentSlotOrGenesis(); - var fut = try clock.waitForSlot(current); - errdefer fut.cancel(); - try fut.await(io_handle); -} - -test "waitForSlot returns aborted on stop" { - const rt = try zio.Runtime.init(testing.allocator, .{}); - defer rt.deinit(); - const io_handle = rt.io(); - - var clock: EventClock = undefined; - try clock.init(testing.allocator, .{ - .genesis_time_sec = time.nowSec(io_handle) + 2, - .slot_duration_ms = 2_000, - .slots_per_epoch = 8, - }, io_handle); - defer clock.deinit(); - - var fut = try clock.waitForSlot(100); - errdefer fut.cancel(); - clock.stop(); - try testing.expectError(error.Aborted, fut.await(io_handle)); -} - -test "offSlot/offEpoch stop event delivery" { - const rt = try zio.Runtime.init(testing.allocator, .{}); - defer rt.deinit(); - const io_handle = rt.io(); - - var clock: EventClock = undefined; - try clock.init(testing.allocator, .{ - .genesis_time_sec = time.nowSec(io_handle) + 2, - .slot_duration_ms = 1_000, - .slots_per_epoch = 4, - }, io_handle); - defer clock.deinit(); - - var trace = EventTraceState{}; - const slot_id = try clock.onSlot(EventTraceState.onSlot, &trace); - const epoch_id = try clock.onEpoch(EventTraceState.onEpoch, &trace); - try testing.expect(clock.offSlot(slot_id)); - try testing.expect(clock.offEpoch(epoch_id)); - - clock.advanceAndDispatch(6); - try testing.expectEqual(@as(usize, 0), trace.slot_len); - try testing.expectEqual(@as(usize, 0), trace.epoch_len); -} - -test "stop/join are idempotent" { - const rt = try zio.Runtime.init(testing.allocator, .{}); - defer rt.deinit(); - const io_handle = rt.io(); - - var clock: EventClock = undefined; - try clock.init(testing.allocator, .{ - .genesis_time_sec = time.nowSec(io_handle) + 2, - .slot_duration_ms = 2_000, - .slots_per_epoch = 8, - }, io_handle); - defer clock.deinit(); - - clock.stop(); - clock.stop(); - clock.join(); - clock.join(); -} - -test "epoch event is delivered when crossing epoch boundary" { - const rt = try zio.Runtime.init(testing.allocator, .{}); - defer rt.deinit(); - const io_handle = rt.io(); - - var clock: EventClock = undefined; - try clock.init(testing.allocator, .{ - .genesis_time_sec = time.nowSec(io_handle) + 2, - .slot_duration_ms = 1_000, - .slots_per_epoch = 4, - }, io_handle); - defer clock.deinit(); - - var trace = EventTraceState{}; - _ = try clock.onSlot(EventTraceState.onSlot, &trace); - _ = try clock.onEpoch(EventTraceState.onEpoch, &trace); - - clock.advanceAndDispatch(5); - - try testing.expect(trace.slot_len > 0); - try testing.expect(trace.epoch_len > 0); - try testing.expectEqual(@as(u64, 1), trace.epochs[0]); -} - -test "multiple waiters are dispatched in target-slot order" { - const rt = try zio.Runtime.init(testing.allocator, .{}); - defer rt.deinit(); - const io_handle = rt.io(); - - var clock: EventClock = undefined; - try clock.init(testing.allocator, .{ - .genesis_time_sec = time.nowSec(io_handle) + 10, - .slot_duration_ms = 1_000, - .slots_per_epoch = 8, - }, io_handle); - defer clock.deinit(); - - var fut5 = try clock.waitForSlot(5); - errdefer fut5.cancel(); - - var fut3 = try clock.waitForSlot(3); - errdefer fut3.cancel(); - - var fut1 = try clock.waitForSlot(1); - errdefer fut1.cancel(); - - clock.advanceAndDispatch(3); - - try fut1.await(io_handle); - try fut3.await(io_handle); - - clock.stop(); - try testing.expectError(error.Aborted, fut5.await(io_handle)); -} - -test "cancel releases WaitState without awaiting" { - const rt = try zio.Runtime.init(testing.allocator, .{}); - defer rt.deinit(); - const io_handle = rt.io(); - - var clock: EventClock = undefined; - try clock.init(testing.allocator, .{ - .genesis_time_sec = time.nowSec(io_handle) + 10, - .slot_duration_ms = 1_000, - .slots_per_epoch = 8, - }, io_handle); - defer clock.deinit(); - - // testing.allocator detects a leak if cancel fails to free. - var fut = try clock.waitForSlot(999); - fut.cancel(); -} - -test "real-time: no slot events emitted before genesis" { - const rt = try zio.Runtime.init(testing.allocator, .{}); - defer rt.deinit(); - const io_handle = rt.io(); - - var clock: EventClock = undefined; - try clock.init(testing.allocator, .{ - .genesis_time_sec = time.nowSec(io_handle) + 5, - .slot_duration_ms = 1_000, - .slots_per_epoch = 8, - }, io_handle); - defer clock.deinit(); - - var trace = EventTraceState{}; - _ = try clock.onSlot(EventTraceState.onSlot, &trace); - - try clock.start(); - - std.Io.sleep(io_handle, std.Io.Duration.fromMilliseconds(1500), .awake) catch {}; - - try testing.expectEqual(@as(usize, 0), trace.slot_len); -} - -test "real-time: slot events fire with correct timing" { - const rt = try zio.Runtime.init(testing.allocator, .{}); - defer rt.deinit(); - const io_handle = rt.io(); - - const base_now = time.nowSec(io_handle); - - var clock: EventClock = undefined; - try clock.init(testing.allocator, .{ - .genesis_time_sec = base_now, - .slot_duration_ms = 1_000, - .slots_per_epoch = 8, - }, io_handle); - defer clock.deinit(); - - var trace = EventTraceState{}; - _ = try clock.onSlot(EventTraceState.onSlot, &trace); - - try clock.start(); - - const start_slot = clock.currentSlotOrGenesis(); - const before_ms = time.nowMs(io_handle); - var fut = try clock.waitForSlot(start_slot + 1); - errdefer fut.cancel(); - try fut.await(io_handle); - const elapsed = time.nowMs(io_handle) - before_ms; - - try testing.expect(elapsed < 2000); - try testing.expect(trace.slot_len > 0); - try testing.expect(trace.slots[trace.slot_len - 1] >= start_slot + 1); -} - -test "real-time: multi-slot advancement delivers ordered events" { - const rt = try zio.Runtime.init(testing.allocator, .{}); - defer rt.deinit(); - const io_handle = rt.io(); - - const base_now = time.nowSec(io_handle); - - var clock: EventClock = undefined; - try clock.init(testing.allocator, .{ - .genesis_time_sec = base_now, - .slot_duration_ms = 1_000, - .slots_per_epoch = 8, - }, io_handle); - defer clock.deinit(); - - var trace = EventTraceState{}; - _ = try clock.onSlot(EventTraceState.onSlot, &trace); - - try clock.start(); - - const start_slot = clock.currentSlotOrGenesis(); - var fut = try clock.waitForSlot(start_slot + 2); - errdefer fut.cancel(); - try fut.await(io_handle); - - try testing.expect(trace.slot_len >= 2); - for (1..trace.slot_len) |i| { - try testing.expect(trace.slots[i] > trace.slots[i - 1]); - } -} - -test "real-time: stop+join cancels promptly" { - const rt = try zio.Runtime.init(testing.allocator, .{}); - defer rt.deinit(); - const io_handle = rt.io(); - - var clock: EventClock = undefined; - try clock.init(testing.allocator, .{ - .genesis_time_sec = time.nowSec(io_handle) + 100, - .slot_duration_ms = 12_000, - .slots_per_epoch = 32, - }, io_handle); - defer clock.deinit(); - - try clock.start(); - - // Give the loop fiber time to enter its sleep. - std.Io.sleep(io_handle, std.Io.Duration.fromMilliseconds(50), .awake) catch {}; - - const before_ms = time.nowMs(io_handle); - clock.stop(); - clock.join(); - const elapsed = time.nowMs(io_handle) - before_ms; - - // join() cancels the sleeping future directly — should return - // almost immediately, NOT after the full 12-second slot. - try testing.expect(elapsed < 1500); -} - -test "real-time: epoch boundary event fires" { - const rt = try zio.Runtime.init(testing.allocator, .{}); - defer rt.deinit(); - const io_handle = rt.io(); - - const base_now = time.nowSec(io_handle); - - var clock: EventClock = undefined; - try clock.init(testing.allocator, .{ - .genesis_time_sec = base_now, - .slot_duration_ms = 1_000, - .slots_per_epoch = 2, - }, io_handle); - defer clock.deinit(); - - var trace = EventTraceState{}; - _ = try clock.onSlot(EventTraceState.onSlot, &trace); - _ = try clock.onEpoch(EventTraceState.onEpoch, &trace); - - try clock.start(); - - const start_slot = clock.currentSlotOrGenesis(); - var fut = try clock.waitForSlot(start_slot + 3); - errdefer fut.cancel(); - try fut.await(io_handle); - - try testing.expect(trace.slot_len >= 3); - try testing.expect(trace.epoch_len > 0); -} - -fn nopSlot(_: ?*anyopaque, _: Slot) void {} -fn nopEpoch(_: ?*anyopaque, _: Epoch) void {} - -const ReentrancyCtx = struct { - clock: *EventClock, - self_id: ?ListenerId = null, - fired_count: usize = 0, - - fn offSelf(ctx: ?*anyopaque, _: Slot) void { - const self: *ReentrancyCtx = @ptrCast(@alignCast(ctx.?)); - self.fired_count += 1; - if (self.self_id) |id| { - _ = self.clock.offSlot(id); - self.self_id = null; - } - } - - fn stopClock(ctx: ?*anyopaque, _: Slot) void { - const self: *ReentrancyCtx = @ptrCast(@alignCast(ctx.?)); - self.fired_count += 1; - self.clock.stop(); - } - - fn justCount(ctx: ?*anyopaque, _: Slot) void { - const self: *ReentrancyCtx = @ptrCast(@alignCast(ctx.?)); - self.fired_count += 1; - } -}; - -test "reentrancy: callback can offSlot itself mid-dispatch" { - const rt = try zio.Runtime.init(testing.allocator, .{}); - defer rt.deinit(); - const io_handle = rt.io(); - - var clock: EventClock = undefined; - try clock.init(testing.allocator, .{ - .genesis_time_sec = time.nowSec(io_handle) + 1_000_000, - .slot_duration_ms = 1_000, - .slots_per_epoch = 4, - }, io_handle); - defer clock.deinit(); - - var ctx_a = ReentrancyCtx{ .clock = &clock }; - var ctx_b = ReentrancyCtx{ .clock = &clock }; - const id_a = try clock.onSlot(ReentrancyCtx.offSelf, &ctx_a); - ctx_a.self_id = id_a; - _ = try clock.onSlot(ReentrancyCtx.justCount, &ctx_b); - - clock.advanceAndDispatch(0); - clock.advanceAndDispatch(2); - - try testing.expectEqual(@as(usize, 1), ctx_a.fired_count); - try testing.expectEqual(@as(usize, 3), ctx_b.fired_count); -} - -test "reentrancy: callback can stop the clock; no further slots emitted" { - const rt = try zio.Runtime.init(testing.allocator, .{}); - defer rt.deinit(); - const io_handle = rt.io(); - - var clock: EventClock = undefined; - try clock.init(testing.allocator, .{ - .genesis_time_sec = time.nowSec(io_handle) + 1_000_000, - .slot_duration_ms = 1_000, - .slots_per_epoch = 4, - }, io_handle); - defer clock.deinit(); - - var ctx = ReentrancyCtx{ .clock = &clock }; - _ = try clock.onSlot(ReentrancyCtx.stopClock, &ctx); - - clock.advanceAndDispatch(5); - - try testing.expectEqual(@as(usize, 1), ctx.fired_count); - try testing.expect(clock.stopped); - try testing.expectEqual(@as(?Slot, 0), clock.clock.current_slot); -} - -const StopAtSlotCtx = struct { - clock: *EventClock, - stop_at: Slot, - - fn stopAt(ctx: ?*anyopaque, slot: Slot) void { - const self: *StopAtSlotCtx = @ptrCast(@alignCast(ctx.?)); - if (slot == self.stop_at) self.clock.stop(); - } -}; - -test "reentrancy: stop() during emit resolves reached waiter, aborts future one" { - const rt = try zio.Runtime.init(testing.allocator, .{}); - defer rt.deinit(); - const io_handle = rt.io(); - - var clock: EventClock = undefined; - try clock.init(testing.allocator, .{ - .genesis_time_sec = time.nowSec(io_handle) + 1_000_000, - .slot_duration_ms = 1_000, - .slots_per_epoch = 4, - }, io_handle); - defer clock.deinit(); - - // Listener calls stop() while slot `target` is being emitted, i.e. - // after current_slot reaches `target` but before dispatchWaiters runs. - const target: Slot = 3; - var ctx = StopAtSlotCtx{ .clock = &clock, .stop_at = target }; - _ = try clock.onSlot(StopAtSlotCtx.stopAt, &ctx); - - var fut_reached = try clock.waitForSlot(target); - errdefer fut_reached.cancel(); - var fut_future = try clock.waitForSlot(target + 1); - errdefer fut_future.cancel(); - - clock.advanceAndDispatch(target); - - try testing.expect(clock.stopped); - try testing.expectEqual(@as(?Slot, target), clock.clock.current_slot); - // Reached slot happened, so the wait must resolve, not abort. - try fut_reached.await(io_handle); - // Future slot can never be emitted after stop, so it aborts. - try testing.expectError(error.Aborted, fut_future.await(io_handle)); -} - -test "ListenerLimitReached: onSlot/onEpoch reject the (limit+1)th registration" { - const rt = try zio.Runtime.init(testing.allocator, .{}); - defer rt.deinit(); - const io_handle = rt.io(); - - var clock: EventClock = undefined; - try clock.init(testing.allocator, .{ - .genesis_time_sec = time.nowSec(io_handle) + 1_000_000, - .slot_duration_ms = 1_000, - .slots_per_epoch = 4, - }, io_handle); - defer clock.deinit(); - - for (0..max_slot_listeners) |_| { - _ = try clock.onSlot(nopSlot, null); - } - try testing.expectError(error.ListenerLimitReached, clock.onSlot(nopSlot, null)); - - for (0..max_epoch_listeners) |_| { - _ = try clock.onEpoch(nopEpoch, null); - } - try testing.expectError(error.ListenerLimitReached, clock.onEpoch(nopEpoch, null)); -} - -test "WaiterLimitReached: waitForSlot rejects the (limit+1)th waiter" { - const rt = try zio.Runtime.init(testing.allocator, .{}); - defer rt.deinit(); - const io_handle = rt.io(); - - var clock: EventClock = undefined; - try clock.init(testing.allocator, .{ - .genesis_time_sec = time.nowSec(io_handle) + 1_000_000, - .slot_duration_ms = 1_000, - .slots_per_epoch = 4, - }, io_handle); - defer clock.deinit(); - - var futs: [max_waiters]WaitForSlotResult = undefined; - for (&futs) |*f| f.* = try clock.waitForSlot(999_999); - try testing.expectError(error.WaiterLimitReached, clock.waitForSlot(999_999)); - for (&futs) |*f| f.cancel(); -} - -test "many waiters at same target slot all resolve on advance" { - const rt = try zio.Runtime.init(testing.allocator, .{}); - defer rt.deinit(); - const io_handle = rt.io(); - - var clock: EventClock = undefined; - try clock.init(testing.allocator, .{ - .genesis_time_sec = time.nowSec(io_handle) + 1_000_000, - .slot_duration_ms = 1_000, - .slots_per_epoch = 4, - }, io_handle); - defer clock.deinit(); - - const N = 16; - var futs: [N]WaitForSlotResult = undefined; - for (&futs) |*f| f.* = try clock.waitForSlot(5); - - clock.advanceAndDispatch(5); - - for (&futs) |*f| try f.await(io_handle); -} - -const PropertyTracker = struct { - slot_events: std.ArrayListUnmanaged(Slot) = .empty, - epoch_events: std.ArrayListUnmanaged(Epoch) = .empty, - - fn onSlot(ctx: ?*anyopaque, slot: Slot) void { - const self: *PropertyTracker = @ptrCast(@alignCast(ctx.?)); - self.slot_events.append(testing.allocator, slot) catch unreachable; - } - - fn onEpoch(ctx: ?*anyopaque, epoch: Epoch) void { - const self: *PropertyTracker = @ptrCast(@alignCast(ctx.?)); - self.epoch_events.append(testing.allocator, epoch) catch unreachable; - } - - fn deinit(self: *PropertyTracker) void { - self.slot_events.deinit(testing.allocator); - self.epoch_events.deinit(testing.allocator); - } -}; - -const PropertyOp = union(enum) { - on_slot, - on_epoch, - off_slot: usize, - off_epoch: usize, - advance_by: u8, - wait_for_slot_at_offset: i32, - cancel_waiter: usize, - stop, -}; - -const PropertyWaiter = struct { - target: Slot, - fut: WaitForSlotResult, - expected_aborted: bool, -}; - -const PropertyState = struct { - spe: u64, - model_current_slot: ?Slot = null, - model_stopped: bool = false, - clock: *EventClock, - - slot_listener_ids: std.ArrayListUnmanaged(ListenerId) = .empty, - slot_trackers: std.ArrayListUnmanaged(*PropertyTracker) = .empty, - slot_expected: std.ArrayListUnmanaged(std.ArrayListUnmanaged(Slot)) = .empty, - - epoch_listener_ids: std.ArrayListUnmanaged(ListenerId) = .empty, - epoch_trackers: std.ArrayListUnmanaged(*PropertyTracker) = .empty, - epoch_expected: std.ArrayListUnmanaged(std.ArrayListUnmanaged(Epoch)) = .empty, - - waiters: std.ArrayListUnmanaged(PropertyWaiter) = .empty, - - const MAX_LISTENERS = 8; - - fn deinit(self: *PropertyState) void { - const a = testing.allocator; - for (self.slot_trackers.items) |t| { - t.deinit(); - a.destroy(t); - } - for (self.epoch_trackers.items) |t| { - t.deinit(); - a.destroy(t); - } - for (self.slot_expected.items) |*lst| lst.deinit(a); - for (self.epoch_expected.items) |*lst| lst.deinit(a); - self.slot_listener_ids.deinit(a); - self.slot_trackers.deinit(a); - self.slot_expected.deinit(a); - self.epoch_listener_ids.deinit(a); - self.epoch_trackers.deinit(a); - self.epoch_expected.deinit(a); - self.waiters.deinit(a); - } - - fn applyOp(self: *PropertyState, op: PropertyOp) !void { - const a = testing.allocator; - switch (op) { - .on_slot => { - if (self.slot_listener_ids.items.len >= MAX_LISTENERS) return; - const tracker = try a.create(PropertyTracker); - tracker.* = .{}; - errdefer { - tracker.deinit(); - a.destroy(tracker); - } - - // Reserve before clock.onSlot so a subsequent append can't OOM - // and leave the clock pointing at a tracker we then free. - try self.slot_listener_ids.ensureUnusedCapacity(a, 1); - try self.slot_trackers.ensureUnusedCapacity(a, 1); - try self.slot_expected.ensureUnusedCapacity(a, 1); - const id = try self.clock.onSlot(PropertyTracker.onSlot, tracker); - self.slot_listener_ids.appendAssumeCapacity(id); - self.slot_trackers.appendAssumeCapacity(tracker); - self.slot_expected.appendAssumeCapacity(.empty); - }, - .on_epoch => { - if (self.epoch_listener_ids.items.len >= MAX_LISTENERS) return; - const tracker = try a.create(PropertyTracker); - tracker.* = .{}; - errdefer { - tracker.deinit(); - a.destroy(tracker); - } - - try self.epoch_listener_ids.ensureUnusedCapacity(a, 1); - try self.epoch_trackers.ensureUnusedCapacity(a, 1); - try self.epoch_expected.ensureUnusedCapacity(a, 1); - const id = try self.clock.onEpoch(PropertyTracker.onEpoch, tracker); - self.epoch_listener_ids.appendAssumeCapacity(id); - self.epoch_trackers.appendAssumeCapacity(tracker); - self.epoch_expected.appendAssumeCapacity(.empty); - }, - .off_slot => |idx| { - if (idx >= self.slot_listener_ids.items.len) return; - const id = self.slot_listener_ids.items[idx]; - try testing.expect(self.clock.offSlot(id)); - _ = self.slot_listener_ids.orderedRemove(idx); - const t = self.slot_trackers.orderedRemove(idx); - var exp = self.slot_expected.orderedRemove(idx); - try expectEqualSlices(Slot, exp.items, t.slot_events.items); - exp.deinit(a); - t.deinit(); - a.destroy(t); - }, - .off_epoch => |idx| { - if (idx >= self.epoch_listener_ids.items.len) return; - const id = self.epoch_listener_ids.items[idx]; - try testing.expect(self.clock.offEpoch(id)); - _ = self.epoch_listener_ids.orderedRemove(idx); - const t = self.epoch_trackers.orderedRemove(idx); - var exp = self.epoch_expected.orderedRemove(idx); - try expectEqualSlices(Epoch, exp.items, t.epoch_events.items); - exp.deinit(a); - t.deinit(); - a.destroy(t); - }, - .advance_by => |k| { - if (k == 0 or self.model_stopped) return; - const begin = self.model_current_slot; - const s_first: Slot = if (begin) |c| c + 1 else 0; - const s_last: Slot = if (begin) |c| c + k else @as(Slot, k) - 1; - - var s: Slot = s_first; - while (true) : (s += 1) { - for (self.slot_expected.items) |*lst| try lst.append(a, s); - if (s > 0) { - const prev_e = (s - 1) / self.spe; - const new_e = s / self.spe; - if (new_e > prev_e) { - for (self.epoch_expected.items) |*lst| try lst.append(a, new_e); - } - } - if (s == s_last) break; - } - self.model_current_slot = s_last; - self.clock.advanceAndDispatch(s_last); - - for (self.waiters.items) |*w| { - if (w.target <= s_last) w.expected_aborted = false; - } - }, - .wait_for_slot_at_offset => |offset| { - if (self.model_stopped) return; - const base: i64 = if (self.model_current_slot) |c| @intCast(c) else -1; - const target_signed = base + offset; - if (target_signed < 0) return; - const target: Slot = @intCast(target_signed); - const fut = try self.clock.waitForSlot(target); - const resolved_now = if (self.model_current_slot) |c| c >= target else false; - try self.waiters.append(a, .{ - .target = target, - .fut = fut, - .expected_aborted = !resolved_now, - }); - }, - .cancel_waiter => |idx| { - if (idx >= self.waiters.items.len) return; - var w = self.waiters.orderedRemove(idx); - w.fut.cancel(); - }, - .stop => { - if (self.model_stopped) return; - self.model_stopped = true; - self.clock.stop(); - }, - } - } - - fn finalize(self: *PropertyState, io: std.Io) !void { - if (!self.model_stopped) { - self.model_stopped = true; - self.clock.stop(); - } - - for (self.slot_trackers.items, self.slot_expected.items) |t, exp| { - try expectEqualSlices(Slot, exp.items, t.slot_events.items); - } - for (self.epoch_trackers.items, self.epoch_expected.items) |t, exp| { - try expectEqualSlices(Epoch, exp.items, t.epoch_events.items); - } - - for (self.waiters.items) |*w| { - const result = w.fut.await(io); - if (w.expected_aborted) { - try testing.expectError(error.Aborted, result); - } else { - try result; - } - } - self.waiters.clearRetainingCapacity(); - } -}; - -const expectEqualSlices = std.testing.expectEqualSlices; - -fn genPropertyOp(rng: std.Random, state: *const PropertyState) PropertyOp { - while (true) { - const r = rng.uintLessThan(u32, 100); - if (r < 18) return .on_slot; - if (r < 32) return .on_epoch; - if (r < 42) { - if (state.slot_listener_ids.items.len == 0) continue; - return .{ .off_slot = rng.uintLessThan(usize, state.slot_listener_ids.items.len) }; - } - if (r < 52) { - if (state.epoch_listener_ids.items.len == 0) continue; - return .{ .off_epoch = rng.uintLessThan(usize, state.epoch_listener_ids.items.len) }; - } - if (r < 80) return .{ .advance_by = @intCast(rng.uintLessThan(u32, 8) + 1) }; - if (r < 92) { - const off: i32 = @as(i32, @intCast(rng.uintLessThan(u32, 12))) - 4; - return .{ .wait_for_slot_at_offset = off }; - } - if (r < 98) { - if (state.waiters.items.len == 0) continue; - return .{ .cancel_waiter = rng.uintLessThan(usize, state.waiters.items.len) }; - } - return .stop; - } -} - -fn runPropertyScenario(seed: u64, op_count: u32, io: std.Io) !void { - var prng = std.Random.DefaultPrng.init(seed); - const rng = prng.random(); - - const spe: u64 = 4; - const now_sec = time.nowSec(io); - var clock: EventClock = undefined; - // Genesis far in future → wall-clock never advances; advanceAndDispatch owns time. - try clock.init(testing.allocator, .{ - .genesis_time_sec = now_sec + 1_000_000, - .slot_duration_ms = 1_000, - .slots_per_epoch = spe, - }, io); - defer clock.deinit(); - - var state = PropertyState{ .spe = spe, .clock = &clock }; - defer state.deinit(); - - var i: u32 = 0; - while (i < op_count) : (i += 1) { - const op = genPropertyOp(rng, &state); - try state.applyOp(op); - } - - try state.finalize(io); -} - -test "property: random op sequences match model" { - const rt = try zio.Runtime.init(testing.allocator, .{}); - defer rt.deinit(); - const io_handle = rt.io(); - - var seed: u64 = 0; - while (seed < 500) : (seed += 1) { - runPropertyScenario(seed, 50, io_handle) catch |err| { - std.debug.print( - "property scenario failed at seed={d}: {s}\n", - .{ seed, @errorName(err) }, - ); - return err; - }; - } -} diff --git a/src/clock/SlotClock.zig b/src/clock/SlotClock.zig deleted file mode 100644 index 22d9693d9..000000000 --- a/src/clock/SlotClock.zig +++ /dev/null @@ -1,412 +0,0 @@ -//! Layer 1 – Stateful slot clock. -//! -//! Wraps `slot_math` with a `std.Io` wall-clock and a cached `current_slot`. -//! Pure-read helpers query wall-clock time; only `advanceTo()` mutates the cache. - -const std = @import("std"); -const time = @import("time"); -const slot_math = @import("slot_math.zig"); - -const SlotClock = @This(); - -pub const Slot = slot_math.Slot; -pub const Epoch = slot_math.Epoch; -pub const ClockConfig = slot_math.ClockConfig; - -pub const Event = union(enum) { - slot: Slot, - epoch: Epoch, -}; - -config: ClockConfig, -io: std.Io, -current_slot: ?Slot = null, - -pub const AdvanceIterator = struct { - clock: *SlotClock, - target: Slot, - pending_epoch: ?Epoch = null, - - /// Advances the clock one step at a time, yielding slot and epoch events. - /// For each slot advancement: yields .slot first, then .epoch if an epoch - /// boundary was crossed. - /// Returns null when caught up to target. - pub fn next(self: *AdvanceIterator) ?Event { - if (self.pending_epoch) |epoch| { - self.pending_epoch = null; - return .{ .epoch = epoch }; - } - - const current = self.clock.current_slot; - if (current == null) { - self.clock.current_slot = 0; - return .{ .slot = 0 }; - } - - const cur = current.?; - if (cur >= self.target) return null; - - const next_slot = cur + 1; - self.clock.current_slot = next_slot; - - const prev_epoch = slot_math.epochAtSlot(self.clock.config, cur); - const new_epoch = slot_math.epochAtSlot(self.clock.config, next_slot); - if (prev_epoch < new_epoch) { - self.pending_epoch = new_epoch; - } - - return .{ .slot = next_slot }; - } -}; - -pub fn init(config: ClockConfig, io: std.Io) error{InvalidConfig}!SlotClock { - try config.validate(); - var self = SlotClock{ - .config = config, - .io = io, - }; - self.current_slot = slot_math.slotAtMs(config, time.nowMs(self.io)); - return self; -} - -/// Returns the current wall-clock slot. Pure read — does NOT update -/// the internal `current_slot` cache. Only `advanceTo()` advances the cache. -pub fn currentSlot(self: *const SlotClock) ?Slot { - const now_ms = time.nowMs(self.io); - return slot_math.slotAtMs(self.config, now_ms); -} - -pub fn currentEpoch(self: *const SlotClock) ?Epoch { - const slot = self.currentSlot() orelse return null; - return slot_math.epochAtSlot(self.config, slot); -} - -pub fn currentSlotOrGenesis(self: *const SlotClock) Slot { - return self.currentSlot() orelse 0; -} - -pub fn currentEpochOrGenesis(self: *const SlotClock) Epoch { - return self.currentEpoch() orelse 0; -} - -/// Returns the slot the network may be advancing to, accounting for gossip -/// clock disparity, or null pre-genesis when no slot is current yet. -/// -/// Per phase0/p2p-interface.md, gossip validation rejects future messages with -/// strict `<` (`current_time + MAXIMUM_GOSSIP_CLOCK_DISPARITY < message_time`), -/// so the boundary case (exactly equal) is accepted — hence `<=` here. -/// -/// Assumes the disparity window reaches at most the adjacent slot — true -/// for every real config (500 ms disparity vs seconds-long slots). A -/// config where disparity approaches or exceeds the slot duration is not -/// supported. -pub fn currentSlotWithGossipDisparity(self: *const SlotClock) ?Slot { - const now_ms = time.nowMs(self.io); - const current = slot_math.slotAtMs(self.config, now_ms) orelse { - // Pre-genesis the wall slot is conceptually negative, so slot 0 is - // "current" only once we're within gossip disparity of genesis; - // null otherwise. - const genesis_ms = slot_math.slotStartMs(self.config, 0); - return if (genesis_ms - now_ms <= self.config.maximum_gossip_clock_disparity_ms) - 0 - else - null; - }; - const next_slot = current + 1; - const next_slot_ms = slot_math.slotStartMs(self.config, next_slot); - if (next_slot_ms - now_ms <= self.config.maximum_gossip_clock_disparity_ms) { - return next_slot; - } - return current; -} - -/// See `currentSlotWithGossipDisparity` for the `<=` rationale and the -/// single-snapshot semantics — both apply here too. -pub fn isCurrentSlotGivenGossipDisparity(self: *const SlotClock, slot: Slot) bool { - const now_ms = time.nowMs(self.io); - const current = slot_math.slotAtMs(self.config, now_ms) orelse { - // Pre-genesis the wall slot is conceptually negative, so slot 0 is - // "current" only once we're within gossip disparity of genesis. - if (slot != 0) return false; - const genesis_ms = slot_math.slotStartMs(self.config, 0); - return genesis_ms - now_ms <= self.config.maximum_gossip_clock_disparity_ms; - }; - if (slot == current) return true; - - const next_slot = current + 1; - const next_slot_ms = slot_math.slotStartMs(self.config, next_slot); - if (next_slot_ms - now_ms <= self.config.maximum_gossip_clock_disparity_ms) { - return slot == next_slot; - } - - if (current > 0) { - const current_slot_ms = slot_math.slotStartMs(self.config, current); - if (now_ms - current_slot_ms <= self.config.maximum_gossip_clock_disparity_ms) { - return slot == current - 1; - } - } - - return false; -} - -pub fn slotWithFutureToleranceMs(self: *const SlotClock, tolerance_ms: u64) ?Slot { - const now_ms = time.nowMs(self.io); - const shifted_ms = now_ms + tolerance_ms; - return slot_math.slotAtMs(self.config, shifted_ms); -} - -pub fn slotWithPastToleranceMs(self: *const SlotClock, tolerance_ms: u64) Slot { - const now_ms = time.nowMs(self.io); - const shifted_ms = now_ms - tolerance_ms; - // Pre-genesis → slot 0. - return slot_math.slotAtMs(self.config, shifted_ms) orelse 0; -} - -pub fn secFromSlot(self: *const SlotClock, slot: Slot, to_sec: ?u64) i64 { - const from_sec = slot_math.slotStartSec(self.config, slot); - const end_sec = to_sec orelse @divFloor(time.nowMs(self.io), 1000); - return @as(i64, @intCast(end_sec)) - @as(i64, @intCast(from_sec)); -} - -pub fn msFromSlot(self: *const SlotClock, slot: Slot, to_ms: ?u64) i64 { - const from_ms = slot_math.slotStartMs(self.config, slot); - const end_ms = to_ms orelse time.nowMs(self.io); - return @as(i64, @intCast(end_ms)) - @as(i64, @intCast(from_ms)); -} - -/// Advances the clock toward `target` one event at a time. The caller may -/// drop the iterator mid-walk; the clock is then left at the last slot the -/// iterator returned (i.e. partial advancement is observable). -pub fn advanceTo(self: *SlotClock, target: Slot) AdvanceIterator { - return .{ - .clock = self, - .target = target, - }; -} - -const testing = std.testing; - -const FakeClockIo = struct { - ms: u64 = 0, - fn vtableNow(userdata: ?*anyopaque, clock: std.Io.Clock) std.Io.Timestamp { - _ = clock; - const self: *const FakeClockIo = @ptrCast(@alignCast(userdata.?)); - return std.Io.Timestamp.fromNanoseconds(@as(i96, @intCast(self.ms)) * std.time.ns_per_ms); - } - // Only `now` is populated: SlotClock is pure-read, so no other std.Io - // vtable entry is ever called. (EventClock's async tests use a real io.) - const vtable: std.Io.VTable = blk: { - var vt: std.Io.VTable = undefined; - vt.now = vtableNow; - break :blk vt; - }; - fn io(self: *const FakeClockIo) std.Io { - return .{ .userdata = @constCast(self), .vtable = &vtable }; - } -}; - -const test_cfg = ClockConfig{ - .genesis_time_sec = 100, - .slot_duration_ms = 12_000, - .slots_per_epoch = 32, - .maximum_gossip_clock_disparity_ms = 500, -}; - -test "pre-genesis returns null, genesis fallback returns zero" { - var fake = FakeClockIo{ .ms = 99_000 }; - var clock = try SlotClock.init(test_cfg, fake.io()); - try testing.expectEqual(@as(?Slot, null), clock.currentSlot()); - try testing.expectEqual(@as(?Epoch, null), clock.currentEpoch()); - try testing.expectEqual(@as(Slot, 0), clock.currentSlotOrGenesis()); - try testing.expectEqual(@as(Epoch, 0), clock.currentEpochOrGenesis()); -} - -test "currentSlot at genesis and advancing" { - var fake = FakeClockIo{ .ms = 100_000 }; - var clock = try SlotClock.init(test_cfg, fake.io()); - try testing.expectEqual(@as(?Slot, 0), clock.currentSlot()); - - fake.ms = 112_000; - try testing.expectEqual(@as(?Slot, 1), clock.currentSlot()); - - fake.ms = 124_000; - try testing.expectEqual(@as(?Slot, 2), clock.currentSlot()); -} - -test "currentEpoch" { - var fake = FakeClockIo{ .ms = 100_000 + 32 * 12 * 1000 }; - var clock = try SlotClock.init(test_cfg, fake.io()); - try testing.expectEqual(@as(?Epoch, 1), clock.currentEpoch()); -} - -test "advanceTo produces correct slot events" { - var fake = FakeClockIo{ .ms = 100_000 }; - var clock = try SlotClock.init(test_cfg, fake.io()); - - var events: [16]Event = undefined; - var count: usize = 0; - var iter = clock.advanceTo(3); - while (iter.next()) |e| { - events[count] = e; - count += 1; - } - - try testing.expectEqual(@as(usize, 3), count); - try testing.expect(events[0] == .slot and events[0].slot == 1); - try testing.expect(events[1] == .slot and events[1].slot == 2); - try testing.expect(events[2] == .slot and events[2].slot == 3); - try testing.expectEqual(@as(?Slot, 3), clock.current_slot); -} - -test "advanceTo across epoch boundary emits slot then epoch" { - var fake = FakeClockIo{ .ms = 100_000 }; - var clock = try SlotClock.init(test_cfg, fake.io()); - clock.current_slot = 31; - - var events: [16]Event = undefined; - var count: usize = 0; - var iter = clock.advanceTo(33); - while (iter.next()) |e| { - events[count] = e; - count += 1; - } - - try testing.expectEqual(@as(usize, 3), count); - try testing.expect(events[0] == .slot and events[0].slot == 32); - try testing.expect(events[1] == .epoch and events[1].epoch == 1); - try testing.expect(events[2] == .slot and events[2].slot == 33); -} - -test "advanceTo from null (pre-genesis)" { - var fake = FakeClockIo{ .ms = 99_000 }; - var clock = try SlotClock.init(test_cfg, fake.io()); - try testing.expectEqual(@as(?Slot, null), clock.current_slot); - - var events: [16]Event = undefined; - var count: usize = 0; - var iter = clock.advanceTo(2); - while (iter.next()) |e| { - events[count] = e; - count += 1; - } - - try testing.expectEqual(@as(usize, 3), count); - try testing.expect(events[0] == .slot and events[0].slot == 0); - try testing.expect(events[1] == .slot and events[1].slot == 1); - try testing.expect(events[2] == .slot and events[2].slot == 2); -} - -test "advanceTo already at target returns nothing" { - var fake = FakeClockIo{ .ms = 112_000 }; - var clock = try SlotClock.init(test_cfg, fake.io()); - - var count: usize = 0; - var iter = clock.advanceTo(1); - while (iter.next()) |_| count += 1; - try testing.expectEqual(@as(usize, 0), count); -} - -test "gossip disparity: far from boundary" { - var fake = FakeClockIo{ .ms = 103_000 }; - var clock = try SlotClock.init(test_cfg, fake.io()); - try testing.expectEqual(@as(?Slot, 0), clock.currentSlotWithGossipDisparity()); - try testing.expect(clock.isCurrentSlotGivenGossipDisparity(0)); - try testing.expect(!clock.isCurrentSlotGivenGossipDisparity(1)); -} - -test "gossip disparity: near next slot boundary" { - var fake = FakeClockIo{ .ms = 111_600 }; - var clock = try SlotClock.init(test_cfg, fake.io()); - try testing.expectEqual(@as(?Slot, 1), clock.currentSlotWithGossipDisparity()); - try testing.expect(clock.isCurrentSlotGivenGossipDisparity(1)); -} - -test "gossip disparity: just after slot boundary" { - var fake = FakeClockIo{ .ms = 112_300 }; - var clock = try SlotClock.init(test_cfg, fake.io()); - try testing.expect(clock.isCurrentSlotGivenGossipDisparity(0)); -} - -test "gossip disparity: exact threshold (500ms) applies inclusively" { - // next_slot_ms - now_ms == 500 → 500 <= 500, disparity applies. - // Slot 1 starts at 112_000ms. 500ms before = 111_500ms. - var fake = FakeClockIo{ .ms = 111_500 }; - var clock = try SlotClock.init(test_cfg, fake.io()); - try testing.expectEqual(@as(?Slot, 1), clock.currentSlotWithGossipDisparity()); - try testing.expect(clock.isCurrentSlotGivenGossipDisparity(1)); - - // 1ms further out (111_499): 112_000 - 111_499 = 501 > 500, disparity does NOT apply. - fake.ms = 111_499; - clock = try SlotClock.init(test_cfg, fake.io()); - try testing.expectEqual(@as(?Slot, 0), clock.currentSlotWithGossipDisparity()); - try testing.expect(!clock.isCurrentSlotGivenGossipDisparity(1)); -} - -test "gossip disparity: pre-genesis slot 0 only within disparity of genesis" { - // Genesis is 100_000ms; disparity is 500ms. - // 1000ms before genesis: slot 0 is not yet "current". - var fake = FakeClockIo{ .ms = 99_000 }; - var clock = try SlotClock.init(test_cfg, fake.io()); - try testing.expectEqual(@as(?Slot, null), clock.currentSlotWithGossipDisparity()); - try testing.expect(!clock.isCurrentSlotGivenGossipDisparity(0)); - try testing.expect(!clock.isCurrentSlotGivenGossipDisparity(1)); - - // 300ms before genesis: within disparity, slot 0 is "current". - fake.ms = 99_700; - clock = try SlotClock.init(test_cfg, fake.io()); - try testing.expectEqual(@as(?Slot, 0), clock.currentSlotWithGossipDisparity()); - try testing.expect(clock.isCurrentSlotGivenGossipDisparity(0)); - try testing.expect(!clock.isCurrentSlotGivenGossipDisparity(1)); - - // Exact threshold: 500ms before genesis is inclusive. - fake.ms = 99_500; - clock = try SlotClock.init(test_cfg, fake.io()); - try testing.expectEqual(@as(?Slot, 0), clock.currentSlotWithGossipDisparity()); - try testing.expect(clock.isCurrentSlotGivenGossipDisparity(0)); - - // 1ms further out (501ms before genesis): not "current". - fake.ms = 99_499; - clock = try SlotClock.init(test_cfg, fake.io()); - try testing.expectEqual(@as(?Slot, null), clock.currentSlotWithGossipDisparity()); - try testing.expect(!clock.isCurrentSlotGivenGossipDisparity(0)); -} - -test "gossip disparity: pre-genesis with sub-disparity slot duration never advances past 0" { - // Degenerate config: slot_duration (400ms) <= disparity (500ms). With `slotAtMs - // orelse 0`, pre-genesis would clamp to slot 0, see next_slot 1 start within - // disparity, and spuriously report slot 1; null/0 semantics must hold instead. - const cfg = ClockConfig{ - .genesis_time_sec = 100, // genesis_ms = 100_000 - .slot_duration_ms = 400, - .slots_per_epoch = 8, - .maximum_gossip_clock_disparity_ms = 500, - }; - - // 1ms before genesis (within disparity): slot 0, never slot 1. - var fake = FakeClockIo{ .ms = 99_999 }; - var clock = try SlotClock.init(cfg, fake.io()); - try testing.expectEqual(@as(?Slot, 0), clock.currentSlotWithGossipDisparity()); - try testing.expect(!clock.isCurrentSlotGivenGossipDisparity(1)); - - // 501ms before genesis (just outside disparity): no slot is current. - fake.ms = 99_499; - clock = try SlotClock.init(cfg, fake.io()); - try testing.expectEqual(@as(?Slot, null), clock.currentSlotWithGossipDisparity()); -} - -test "tolerance helpers" { - var fake = FakeClockIo{ .ms = 112_000 }; - var clock = try SlotClock.init(test_cfg, fake.io()); - try testing.expectEqual(@as(?Slot, 2), clock.slotWithFutureToleranceMs(12_000)); - try testing.expectEqual(@as(Slot, 0), clock.slotWithPastToleranceMs(12_000)); -} - -test "secFromSlot and msFromSlot" { - var fake = FakeClockIo{ .ms = 118_000 }; - var clock = try SlotClock.init(test_cfg, fake.io()); - try testing.expectEqual(@as(i64, 6), clock.secFromSlot(1, null)); - try testing.expectEqual(@as(i64, 6000), clock.msFromSlot(1, null)); - try testing.expectEqual(@as(i64, 0), clock.secFromSlot(1, 112)); - try testing.expectEqual(@as(i64, -12), clock.secFromSlot(1, 100)); - try testing.expectEqual(@as(i64, -12000), clock.msFromSlot(1, 100_000)); -} diff --git a/src/clock/clock_test.zig b/src/clock/clock_test.zig new file mode 100644 index 000000000..499c23a85 --- /dev/null +++ b/src/clock/clock_test.zig @@ -0,0 +1,771 @@ +//! Behavioral tests for slot/epoch listeners and `waitForSlot` callers. + +const std = @import("std"); +const testing = std.testing; +const zio = @import("zio"); +const time = @import("time"); +const slot_math = @import("slot_math.zig"); +const Clock = @import("Clock.zig"); + +const Slot = Clock.Slot; +const Epoch = Clock.Epoch; +const Error = Clock.Error; +const ListenerId = Clock.ListenerId; +const expectEqualSlices = std.testing.expectEqualSlices; + +// Synchronous tests only use `now`. Waiter tests provide a real inner I/O so +// futex waits and wakes can suspend and resume normally. +const FakeClockIo = struct { + ms: u64 = 0, + inner: ?std.Io = null, + + fn vtableNow(userdata: ?*anyopaque, _: std.Io.Clock) std.Io.Timestamp { + const self: *const FakeClockIo = @ptrCast(@alignCast(userdata.?)); + return std.Io.Timestamp.fromNanoseconds(@as(i96, @intCast(self.ms)) * std.time.ns_per_ms); + } + + fn vtableFutexWaitUncancelable(userdata: ?*anyopaque, ptr: *const u32, expected: u32) void { + const self: *const FakeClockIo = @ptrCast(@alignCast(userdata.?)); + const inner = self.inner.?; + inner.vtable.futexWaitUncancelable(inner.userdata, ptr, expected); + } + + fn vtableFutexWake(userdata: ?*anyopaque, ptr: *const u32, max: u32) void { + const self: *const FakeClockIo = @ptrCast(@alignCast(userdata.?)); + const inner = self.inner.?; + inner.vtable.futexWake(inner.userdata, ptr, max); + } + + const vtable: std.Io.VTable = blk: { + var vt: std.Io.VTable = undefined; + vt.now = vtableNow; + vt.futexWaitUncancelable = vtableFutexWaitUncancelable; + vt.futexWake = vtableFutexWake; + break :blk vt; + }; + + fn io(self: *const FakeClockIo) std.Io { + return .{ .userdata = @constCast(self), .vtable = &vtable }; + } +}; + +const EventTraceState = struct { + slots: [64]Slot = undefined, + slot_len: usize = 0, + epochs: [64]Epoch = undefined, + epoch_len: usize = 0, + + fn onSlot(ctx: ?*anyopaque, slot: Slot) void { + const self: *EventTraceState = @ptrCast(@alignCast(ctx.?)); + if (self.slot_len >= self.slots.len) return; + self.slots[self.slot_len] = slot; + self.slot_len += 1; + } + + fn onEpoch(ctx: ?*anyopaque, epoch: Epoch) void { + const self: *EventTraceState = @ptrCast(@alignCast(ctx.?)); + if (self.epoch_len >= self.epochs.len) return; + self.epochs[self.epoch_len] = epoch; + self.epoch_len += 1; + } +}; + +fn rendezvousWaiters(clock: *Clock, io: std.Io, expected: usize) !void { + var polls: usize = 0; + while (clock.waiters.count() < expected) : (polls += 1) { + if (polls >= 10_000) return error.RendezvousTimeout; + std.Io.sleep(io, std.Io.Duration.fromMilliseconds(1), .awake) catch {}; + } +} + +/// One ordered log fed by a slot listener and an epoch listener, so the order +/// in which the two kinds of event reach the node is observable. +const DeliveryLog = struct { + const Tag = enum { slot, epoch }; + const Entry = struct { tag: Tag, value: u64 }; + + entries: [16]Entry = undefined, + len: usize = 0, + + fn record(self: *DeliveryLog, tag: Tag, value: u64) void { + if (self.len >= self.entries.len) return; + self.entries[self.len] = .{ .tag = tag, .value = value }; + self.len += 1; + } + + fn seen(self: *const DeliveryLog) []const Entry { + return self.entries[0..self.len]; + } + + fn onSlot(ctx: ?*anyopaque, slot: Slot) void { + const self: *DeliveryLog = @ptrCast(@alignCast(ctx.?)); + self.record(.slot, slot); + } + + fn onEpoch(ctx: ?*anyopaque, epoch: Epoch) void { + const self: *DeliveryLog = @ptrCast(@alignCast(ctx.?)); + self.record(.epoch, epoch); + } +}; + +const SlotTrace = struct { + slots: [16]Slot = undefined, + slot_len: usize = 0, + + fn onSlot(ctx: ?*anyopaque, slot: Slot) void { + const self: *SlotTrace = @ptrCast(@alignCast(ctx.?)); + if (self.slot_len < self.slots.len) { + self.slots[self.slot_len] = slot; + self.slot_len += 1; + } + } + + fn seen(self: *const SlotTrace) []const Slot { + return self.slots[0..self.slot_len]; + } +}; + +const StopAtSlotListener = struct { + clock: *Clock, + stop_at: Slot, + slots: [16]Slot = undefined, + slot_len: usize = 0, + + fn onSlot(ctx: ?*anyopaque, slot: Slot) void { + const self: *StopAtSlotListener = @ptrCast(@alignCast(ctx.?)); + if (self.slot_len < self.slots.len) { + self.slots[self.slot_len] = slot; + self.slot_len += 1; + } + if (slot == self.stop_at) self.clock.stop(); + } + + fn seen(self: *const StopAtSlotListener) []const Slot { + return self.slots[0..self.slot_len]; + } +}; + +test "init returns OutOfMemory when allocation fails" { + const cfg: Clock.ClockConfig = .{ + .genesis_time_sec = 100, + .slot_duration_ms = 12_000, + .slots_per_epoch = 32, + }; + var failing = std.testing.FailingAllocator.init(testing.allocator, .{ .fail_index = 0 }); + var fake: FakeClockIo = .{ .ms = slot_math.slotStartMs(cfg, 0) }; + + var clock: Clock = undefined; + try testing.expectError(error.OutOfMemory, clock.init(failing.allocator(), fake.io(), cfg)); +} + +test "every listener receives every slot, in order, exactly once" { + const cfg: Clock.ClockConfig = .{ + .genesis_time_sec = 100, + .slot_duration_ms = 1_000, + .slots_per_epoch = 8, + }; + var fake: FakeClockIo = .{ .ms = slot_math.slotStartMs(cfg, 0) }; + + var clock: Clock = undefined; + try clock.init(testing.allocator, fake.io(), cfg); + defer clock.deinit(); + + var first: SlotTrace = .{}; + var second: SlotTrace = .{}; + _ = try clock.onSlot(SlotTrace.onSlot, &first); + _ = try clock.onSlot(SlotTrace.onSlot, &second); + + try testing.expectEqual(@as(?Slot, 0), clock.currentSlot()); + + // Normal ticking: one slot at a time, the way the auto-loop drives it. + for (1..4) |n| { + fake.ms = slot_math.slotStartMs(cfg, @intCast(n)); + try testing.expectEqual(@as(?Slot, @intCast(n)), clock.currentSlot()); + } + + try expectEqualSlices(Slot, &.{ 1, 2, 3 }, first.seen()); + try expectEqualSlices(Slot, &.{ 1, 2, 3 }, second.seen()); +} + +test "an epoch tick lands after its boundary slot and before the next slot" { + const cfg: Clock.ClockConfig = .{ + .genesis_time_sec = 100, + .slot_duration_ms = 1_000, + .slots_per_epoch = 2, + }; + var fake: FakeClockIo = .{ .ms = slot_math.slotStartMs(cfg, 0) }; + + var clock: Clock = undefined; + try clock.init(testing.allocator, fake.io(), cfg); + defer clock.deinit(); + + var log: DeliveryLog = .{}; + _ = try clock.onSlot(DeliveryLog.onSlot, &log); + _ = try clock.onEpoch(DeliveryLog.onEpoch, &log); + + // Normal ticking across the epoch-1 boundary at slot 2 (spe = 2). + for (1..4) |n| { + fake.ms = slot_math.slotStartMs(cfg, @intCast(n)); + try testing.expectEqual(@as(?Slot, @intCast(n)), clock.currentSlot()); + } + + const E = DeliveryLog.Entry; + try expectEqualSlices(E, &.{ + .{ .tag = .slot, .value = 1 }, + .{ .tag = .slot, .value = 2 }, + .{ .tag = .epoch, .value = 1 }, + .{ .tag = .slot, .value = 3 }, + }, log.seen()); + try testing.expectEqual(@as(?Epoch, 1), clock.currentEpoch()); +} + +test "tolerance and from-slot reads do not deliver events" { + const cfg: Clock.ClockConfig = .{ + .genesis_time_sec = 100, + .slot_duration_ms = 12_000, + .slots_per_epoch = 32, + }; + var fake: FakeClockIo = .{ .ms = slot_math.slotStartMs(cfg, 0) }; + + var clock: Clock = undefined; + try clock.init(testing.allocator, fake.io(), cfg); + defer clock.deinit(); + + var slots: SlotTrace = .{}; + _ = try clock.onSlot(SlotTrace.onSlot, &slots); + + // The wall advances to slot 1, but these calculations must not notify slot + // listeners as a side effect. + fake.ms = slot_math.slotStartMs(cfg, 1); + try testing.expectEqual(@as(?Slot, 2), clock.slotWithFutureToleranceMs(cfg.slot_duration_ms)); + try testing.expectEqual(@as(Slot, 0), clock.slotWithPastToleranceMs(cfg.slot_duration_ms)); + + fake.ms = slot_math.slotStartMs(cfg, 1) + 6_000; + try testing.expectEqual(@as(i64, 6), clock.secFromSlot(1, null)); + try testing.expectEqual(@as(i64, 6_000), clock.msFromSlot(1, null)); + try testing.expectEqual(@as(i64, 0), clock.secFromSlot(1, slot_math.slotStartSec(cfg, 1))); + try testing.expectEqual(@as(i64, -12), clock.secFromSlot(1, slot_math.slotStartSec(cfg, 0))); + try testing.expectEqual(@as(i64, -12_000), clock.msFromSlot(1, slot_math.slotStartMs(cfg, 0))); + + try testing.expectEqual(@as(usize, 0), slots.slot_len); + + // An advancing read then delivers slot 1, showing that the earlier helpers + // left event delivery untouched. + try testing.expectEqual(@as(?Slot, 1), clock.currentSlot()); + try expectEqualSlices(Slot, &.{1}, slots.seen()); +} + +test "a long host suspend delivers every missed slot and crossed epoch" { + const cfg: Clock.ClockConfig = .{ + .genesis_time_sec = 100, + .slot_duration_ms = 1_000, + .slots_per_epoch = 32, + }; + var fake: FakeClockIo = .{ .ms = slot_math.slotStartMs(cfg, 0) }; + + var clock: Clock = undefined; + try clock.init(testing.allocator, fake.io(), cfg); + defer clock.deinit(); + + const CatchUpTrace = struct { + last_slot: Slot = 0, + slot_count: u64 = 0, + epoch_count: u64 = 0, + in_order: bool = true, + + fn onSlot(ctx: ?*anyopaque, slot: Slot) void { + const self: *@This() = @ptrCast(@alignCast(ctx.?)); + if (slot != self.last_slot + 1) self.in_order = false; + self.last_slot = slot; + self.slot_count += 1; + } + + fn onEpoch(ctx: ?*anyopaque, _: Epoch) void { + const self: *@This() = @ptrCast(@alignCast(ctx.?)); + self.epoch_count += 1; + } + }; + var trace: CatchUpTrace = .{}; + _ = try clock.onSlot(CatchUpTrace.onSlot, &trace); + _ = try clock.onEpoch(CatchUpTrace.onEpoch, &trace); + + // Catch-up must reach the wall slot, deliver every missed slot once and in + // order, and emit each crossed epoch. + const backlog: u64 = 32_768; + fake.ms = slot_math.slotStartMs(cfg, backlog); + try testing.expectEqual(@as(?Slot, backlog), clock.currentSlot()); + + try testing.expect(trace.in_order); + try testing.expectEqual(backlog, trace.slot_count); + try testing.expectEqual(backlog / cfg.slots_per_epoch, trace.epoch_count); +} + +test "top-level wall step-back never re-emits or regresses delivery" { + const cfg: Clock.ClockConfig = .{ + .genesis_time_sec = 100, + .slot_duration_ms = 1_000, + .slots_per_epoch = 8, + }; + var fake: FakeClockIo = .{ .ms = slot_math.slotStartMs(cfg, 0) }; + + var clock: Clock = undefined; + try clock.init(testing.allocator, fake.io(), cfg); + defer clock.deinit(); + + var slots: SlotTrace = .{}; + _ = try clock.onSlot(SlotTrace.onSlot, &slots); + + // Delivery reaches slot 3, then NTP corrects the host clock back to slot 1. + // The returned slot follows the wall down, but the walk target is behind + // delivery, so the step-back re-emits nothing. + fake.ms = slot_math.slotStartMs(cfg, 3); + try testing.expectEqual(@as(?Slot, 3), clock.currentSlot()); + + fake.ms = slot_math.slotStartMs(cfg, 1); + try testing.expectEqual(@as(?Slot, 1), clock.currentSlot()); + try expectEqualSlices(Slot, &.{ 1, 2, 3 }, slots.seen()); + + // Drive on to slot 5. Only 4 and 5 are new, so the stream ends 1,2,3,4,5. + // Had the step-back regressed delivery, this drive would resend 2 and 3. + fake.ms = slot_math.slotStartMs(cfg, 5); + try testing.expectEqual(@as(?Slot, 5), clock.currentSlot()); + try expectEqualSlices(Slot, &.{ 1, 2, 3, 4, 5 }, slots.seen()); +} + +test "first delivery from a pre-genesis start begins at slot 0" { + const cfg: Clock.ClockConfig = .{ + .genesis_time_sec = 100, + .slot_duration_ms = 1_000, + .slots_per_epoch = 4, + }; + var fake: FakeClockIo = .{ .ms = cfg.genesis_time_sec * 1000 - cfg.slot_duration_ms }; + + var clock: Clock = undefined; + try clock.init(testing.allocator, fake.io(), cfg); + defer clock.deinit(); + + var slots: SlotTrace = .{}; + _ = try clock.onSlot(SlotTrace.onSlot, &slots); + + // The node started before genesis, so no slot is current yet and a read + // delivers nothing. + try testing.expectEqual(@as(?Slot, null), clock.currentSlot()); + try testing.expectEqual(@as(?Epoch, null), clock.currentEpoch()); + try testing.expectEqual(@as(Slot, 0), clock.currentSlotOrGenesis()); + try testing.expectEqual(@as(Epoch, 0), clock.currentEpochOrGenesis()); + try testing.expectEqual(@as(usize, 0), slots.slot_len); + + // Wall slot 2: the first catch-up from a pre-genesis start opens delivery + // at slot 0, so the backlog arrives as 0, 1, 2. + fake.ms = slot_math.slotStartMs(cfg, 2); + try testing.expectEqual(@as(?Slot, 2), clock.currentSlot()); + try expectEqualSlices(Slot, &.{ 0, 1, 2 }, slots.seen()); +} + +test "a listener can unsubscribe itself during delivery" { + const cfg: Clock.ClockConfig = .{ + .genesis_time_sec = 100, + .slot_duration_ms = 1_000, + .slots_per_epoch = 4, + }; + var fake: FakeClockIo = .{ .ms = cfg.genesis_time_sec * 1000 - cfg.slot_duration_ms }; + + var clock: Clock = undefined; + try clock.init(testing.allocator, fake.io(), cfg); + defer clock.deinit(); + + const SelfRemovingListener = struct { + clock: *Clock, + id: ListenerId = 0, + slots: [4]Slot = undefined, + slot_len: usize = 0, + + fn onSlot(ctx: ?*anyopaque, slot: Slot) void { + const self: *@This() = @ptrCast(@alignCast(ctx.?)); + self.slots[self.slot_len] = slot; + self.slot_len += 1; + _ = self.clock.offSlot(self.id); + } + }; + var self_removing: SelfRemovingListener = .{ .clock = &clock }; + var remaining: SlotTrace = .{}; + self_removing.id = try clock.onSlot(SelfRemovingListener.onSlot, &self_removing); + _ = try clock.onSlot(SlotTrace.onSlot, &remaining); + + // The listener receives the in-progress event where it removes itself, but + // no later events. Other listeners continue normally. + fake.ms = slot_math.slotStartMs(cfg, 0); + _ = clock.currentSlot(); + fake.ms = slot_math.slotStartMs(cfg, 2); + try testing.expectEqual(@as(?Slot, 2), clock.currentSlot()); + + try expectEqualSlices(Slot, &.{0}, self_removing.slots[0..self_removing.slot_len]); + try expectEqualSlices(Slot, &.{ 0, 1, 2 }, remaining.seen()); +} + +test "unregistered listeners receive no later events" { + const cfg: Clock.ClockConfig = .{ + .genesis_time_sec = 100, + .slot_duration_ms = 1_000, + .slots_per_epoch = 4, + }; + var fake: FakeClockIo = .{ .ms = cfg.genesis_time_sec * 1000 - cfg.slot_duration_ms }; + + var clock: Clock = undefined; + try clock.init(testing.allocator, fake.io(), cfg); + defer clock.deinit(); + + var trace: EventTraceState = .{}; + const slot_id = try clock.onSlot(EventTraceState.onSlot, &trace); + const epoch_id = try clock.onEpoch(EventTraceState.onEpoch, &trace); + + // Slots 0..4 cross into epoch 1, so both listeners receive events before + // they are removed. + fake.ms = slot_math.slotStartMs(cfg, 4); + try testing.expectEqual(@as(?Slot, 4), clock.currentSlot()); + try testing.expectEqual(@as(usize, 5), trace.slot_len); + try testing.expectEqual(@as(usize, 1), trace.epoch_len); + + try testing.expect(clock.offSlot(slot_id)); + try testing.expect(clock.offEpoch(epoch_id)); + + // Advancing to slot 8 crosses another epoch but must not notify either + // removed listener. + fake.ms = slot_math.slotStartMs(cfg, 8); + try testing.expectEqual(@as(?Slot, 8), clock.currentSlot()); + try testing.expectEqual(@as(usize, 5), trace.slot_len); + try testing.expectEqual(@as(usize, 1), trace.epoch_len); +} + +test "stop from a listener prevents subsequent slot delivery" { + const cfg: Clock.ClockConfig = .{ + .genesis_time_sec = 100, + .slot_duration_ms = 1_000, + .slots_per_epoch = 8, + }; + var fake: FakeClockIo = .{ .ms = slot_math.slotStartMs(cfg, 0) }; + + var clock: Clock = undefined; + try clock.init(testing.allocator, fake.io(), cfg); + defer clock.deinit(); + + var listener: StopAtSlotListener = .{ .clock = &clock, .stop_at = 1 }; + _ = try clock.onSlot(StopAtSlotListener.onSlot, &listener); + + // Slots 1..3 are backlogged. Stopping while slot 1 is delivered suppresses + // the later slots. + fake.ms = slot_math.slotStartMs(cfg, 3); + _ = clock.currentSlot(); + try expectEqualSlices(Slot, &.{1}, listener.seen()); + + _ = clock.currentSlot(); + try expectEqualSlices(Slot, &.{1}, listener.seen()); +} + +test "waitForSlot returns when the target is already current" { + const cfg: Clock.ClockConfig = .{ + .genesis_time_sec = 100, + .slot_duration_ms = 1_000, + .slots_per_epoch = 8, + }; + var fake: FakeClockIo = .{ .ms = slot_math.slotStartMs(cfg, 0) }; + + var clock: Clock = undefined; + try clock.init(testing.allocator, fake.io(), cfg); + defer clock.deinit(); + + const current = clock.currentSlotOrGenesis(); + try clock.waitForSlot(current); +} + +test "waitForSlot returns aborted on stop" { + const rt = try zio.Runtime.init(testing.allocator, .{}); + defer rt.deinit(); + const io_handle = rt.io(); + + var clock: Clock = undefined; + try clock.init(testing.allocator, io_handle, .{ + .genesis_time_sec = time.nowSec(io_handle) + 2, + .slot_duration_ms = 2_000, + .slots_per_epoch = 8, + }); + defer clock.deinit(); + + var fut = try std.Io.concurrent(io_handle, Clock.waitForSlot, .{ &clock, 100 }); + try rendezvousWaiters(&clock, io_handle, 1); + clock.stop(); + try testing.expectError(error.Aborted, fut.await(io_handle)); +} + +test "waitForSlot on a stopped clock returns error.Aborted" { + const cfg: Clock.ClockConfig = .{ + .genesis_time_sec = 100, + .slot_duration_ms = 1_000, + .slots_per_epoch = 4, + }; + var fake: FakeClockIo = .{ .ms = cfg.genesis_time_sec * 1000 - cfg.slot_duration_ms }; + + var clock: Clock = undefined; + try clock.init(testing.allocator, fake.io(), cfg); + defer clock.deinit(); + + clock.stop(); + try testing.expectError(error.Aborted, clock.waitForSlot(1)); +} + +test "reached waiters resolve while a future waiter remains pending" { + const rt = try zio.Runtime.init(testing.allocator, .{}); + defer rt.deinit(); + const io_handle = rt.io(); + + const cfg: Clock.ClockConfig = .{ + .genesis_time_sec = 100, + .slot_duration_ms = 1_000, + .slots_per_epoch = 8, + }; + var fake: FakeClockIo = .{ .ms = cfg.genesis_time_sec * 1000 - cfg.slot_duration_ms, .inner = io_handle }; + var clock: Clock = undefined; + try clock.init(testing.allocator, fake.io(), cfg); + defer clock.deinit(); + + // Targets are registered out of order; all three calls are waiting before + // the clock advances. + var fut5 = try std.Io.concurrent(io_handle, Clock.waitForSlot, .{ &clock, 5 }); + var fut3 = try std.Io.concurrent(io_handle, Clock.waitForSlot, .{ &clock, 3 }); + var fut1 = try std.Io.concurrent(io_handle, Clock.waitForSlot, .{ &clock, 1 }); + try rendezvousWaiters(&clock, io_handle, 3); + + fake.ms = slot_math.slotStartMs(cfg, 3); + _ = clock.currentSlot(); + + try fut1.await(io_handle); + try fut3.await(io_handle); + + clock.stop(); + try testing.expectError(error.Aborted, fut5.await(io_handle)); +} + +test "many waiters at same target slot all resolve on advance" { + const rt = try zio.Runtime.init(testing.allocator, .{}); + defer rt.deinit(); + const io_handle = rt.io(); + + const cfg: Clock.ClockConfig = .{ + .genesis_time_sec = 100, + .slot_duration_ms = 1_000, + .slots_per_epoch = 4, + }; + var fake: FakeClockIo = .{ .ms = cfg.genesis_time_sec * 1000 - cfg.slot_duration_ms, .inner = io_handle }; + var clock: Clock = undefined; + try clock.init(testing.allocator, fake.io(), cfg); + defer clock.deinit(); + + const N = 16; + var futs: [N]std.Io.Future(Error!void) = undefined; + for (&futs) |*f| f.* = try std.Io.concurrent(io_handle, Clock.waitForSlot, .{ &clock, 5 }); + try rendezvousWaiters(&clock, io_handle, N); + + fake.ms = slot_math.slotStartMs(cfg, 5); + _ = clock.currentSlot(); + + for (&futs) |*f| try f.await(io_handle); +} + +test "waitForSlot aborts when stop prevents delivery of its target" { + const cfg: Clock.ClockConfig = .{ + .genesis_time_sec = 100, + .slot_duration_ms = 1_000, + .slots_per_epoch = 8, + }; + var fake: FakeClockIo = .{ .ms = slot_math.slotStartMs(cfg, 0) }; + + var clock: Clock = undefined; + try clock.init(testing.allocator, fake.io(), cfg); + defer clock.deinit(); + + var listener: StopAtSlotListener = .{ .clock = &clock, .stop_at = 1 }; + _ = try clock.onSlot(StopAtSlotListener.onSlot, &listener); + + // The wall reaches slot 3, but stop at slot 1 prevents slot 2 from being + // delivered, so a wait for slot 2 cannot succeed. + fake.ms = slot_math.slotStartMs(cfg, 3); + try testing.expectError(error.Aborted, clock.waitForSlot(2)); +} + +test "stop() during emit resolves a reached waiter and aborts a future one" { + const rt = try zio.Runtime.init(testing.allocator, .{}); + defer rt.deinit(); + const io_handle = rt.io(); + + const cfg: Clock.ClockConfig = .{ + .genesis_time_sec = 100, + .slot_duration_ms = 1_000, + .slots_per_epoch = 4, + }; + var fake: FakeClockIo = .{ .ms = cfg.genesis_time_sec * 1000 - cfg.slot_duration_ms, .inner = io_handle }; + var clock: Clock = undefined; + try clock.init(testing.allocator, fake.io(), cfg); + defer clock.deinit(); + + const target: Slot = 3; + var listener: StopAtSlotListener = .{ .clock = &clock, .stop_at = target }; + _ = try clock.onSlot(StopAtSlotListener.onSlot, &listener); + + var fut_reached = try std.Io.concurrent(io_handle, Clock.waitForSlot, .{ &clock, target }); + var fut_future = try std.Io.concurrent(io_handle, Clock.waitForSlot, .{ &clock, target + 1 }); + try rendezvousWaiters(&clock, io_handle, 2); + + fake.ms = slot_math.slotStartMs(cfg, target); + _ = clock.currentSlot(); + + // The target slot was delivered, so its wait resolves rather than aborting. + try fut_reached.await(io_handle); + // The next slot can never be emitted after the stop, so its wait aborts. + try testing.expectError(error.Aborted, fut_future.await(io_handle)); +} + +fn nopSlot(_: ?*anyopaque, _: Slot) void {} +fn nopEpoch(_: ?*anyopaque, _: Epoch) void {} + +test "ListenerLimitReached: onSlot/onEpoch reject the (limit+1)th registration" { + const cfg: Clock.ClockConfig = .{ + .genesis_time_sec = 100, + .slot_duration_ms = 1_000, + .slots_per_epoch = 4, + }; + var fake: FakeClockIo = .{ .ms = cfg.genesis_time_sec * 1000 - cfg.slot_duration_ms }; + + var clock: Clock = undefined; + try clock.init(testing.allocator, fake.io(), cfg); + defer clock.deinit(); + + for (0..Clock.max_slot_listeners) |_| { + _ = try clock.onSlot(nopSlot, null); + } + try testing.expectError(error.ListenerLimitReached, clock.onSlot(nopSlot, null)); + + for (0..Clock.max_epoch_listeners) |_| { + _ = try clock.onEpoch(nopEpoch, null); + } + try testing.expectError(error.ListenerLimitReached, clock.onEpoch(nopEpoch, null)); +} + +test "WaiterLimitReached: waitForSlot rejects the (limit+1)th waiter" { + const rt = try zio.Runtime.init(testing.allocator, .{}); + defer rt.deinit(); + const io_handle = rt.io(); + + var clock: Clock = undefined; + try clock.init(testing.allocator, io_handle, .{ + .genesis_time_sec = time.nowSec(io_handle) + 1_000_000, + .slot_duration_ms = 1_000, + .slots_per_epoch = 4, + }); + defer clock.deinit(); + + var futs: [Clock.max_waiters]std.Io.Future(Error!void) = undefined; + for (&futs) |*f| { + f.* = try std.Io.concurrent(io_handle, Clock.waitForSlot, .{ &clock, 999_999 }); + } + var polls: usize = 0; + while (clock.waiters.count() < Clock.max_waiters) : (polls += 1) { + if (polls >= 100_000) return error.RendezvousTimeout; + std.Io.sleep(io_handle, std.Io.Duration.fromMilliseconds(1), .awake) catch {}; + } + + try testing.expectError(error.WaiterLimitReached, clock.waitForSlot(999_999)); + + clock.stop(); + for (&futs) |*f| { + try testing.expectError(error.Aborted, f.await(io_handle)); + } +} + +test "stop/join are idempotent" { + const cfg: Clock.ClockConfig = .{ + .genesis_time_sec = 100, + .slot_duration_ms = 1_000, + .slots_per_epoch = 8, + }; + var fake: FakeClockIo = .{ .ms = cfg.genesis_time_sec * 1000 - cfg.slot_duration_ms }; + + var clock: Clock = undefined; + try clock.init(testing.allocator, fake.io(), cfg); + defer clock.deinit(); + + clock.stop(); + clock.stop(); + clock.join(); + clock.join(); +} + +test "real-time: the auto-loop delivers ordered slot events promptly" { + const rt = try zio.Runtime.init(testing.allocator, .{}); + defer rt.deinit(); + const io_handle = rt.io(); + + const slot_duration_ms: u64 = 1_000; + const slots_to_wait: Slot = 2; + const scheduler_headroom_ms: u64 = 1_000; + const cfg: Clock.ClockConfig = .{ + .genesis_time_sec = time.nowSec(io_handle), + .slot_duration_ms = slot_duration_ms, + .slots_per_epoch = 8, + }; + var clock: Clock = undefined; + try clock.init(testing.allocator, io_handle, cfg); + defer clock.deinit(); + + var trace: EventTraceState = .{}; + _ = try clock.onSlot(EventTraceState.onSlot, &trace); + + try clock.start(); + + const start_slot = clock.currentSlotOrGenesis(); + const target = start_slot + slots_to_wait; + const before_ms = time.nowMs(io_handle); + try clock.waitForSlot(target); + const after_ms = time.nowMs(io_handle); + + // On wake-up the wall has reached the target slot's start. + try testing.expect(after_ms >= slot_math.slotStartMs(cfg, target)); + const max_wait_ms = slots_to_wait * slot_duration_ms + scheduler_headroom_ms; + try testing.expect(after_ms - before_ms < max_wait_ms); + try testing.expect(trace.slot_len >= slots_to_wait); + // Delivery reached the target; a further boundary may have added more. + try testing.expect(trace.slots[trace.slot_len - 1] >= target); + // Slots arrive in order, each once. + for (1..trace.slot_len) |i| { + try testing.expect(trace.slots[i] > trace.slots[i - 1]); + } +} + +test "real-time: stop+join cancels promptly" { + const rt = try zio.Runtime.init(testing.allocator, .{}); + defer rt.deinit(); + const io_handle = rt.io(); + + const slot_duration_ms: u64 = 12_000; + const loop_start_grace_ms: i64 = 50; + const shutdown_deadline_ms: u64 = 1_500; + var clock: Clock = undefined; + try clock.init(testing.allocator, io_handle, .{ + .genesis_time_sec = time.nowSec(io_handle), + .slot_duration_ms = slot_duration_ms, + .slots_per_epoch = 32, + }); + defer clock.deinit(); + + try clock.start(); + + // Give the loop fiber time to enter its sleep. + std.Io.sleep(io_handle, std.Io.Duration.fromMilliseconds(loop_start_grace_ms), .awake) catch {}; + + const before_ms = time.nowMs(io_handle); + clock.stop(); + clock.join(); + const elapsed = time.nowMs(io_handle) - before_ms; + + // Shutdown must not wait for the next slot boundary. + try testing.expect(elapsed < shutdown_deadline_ms); +} diff --git a/src/clock/config.zig b/src/clock/config.zig index 6817b05e6..37dffd19a 100644 --- a/src/clock/config.zig +++ b/src/clock/config.zig @@ -1,6 +1,6 @@ //! Clock configuration: genesis, slot duration (with fork-aware transitions), //! epoch length, and gossip-disparity tolerance. Shared by `slot_math` -//! (pure arithmetic) and the stateful `SlotClock` / `EventClock` layers. +//! (pure arithmetic) and the stateful `Clock` layer. const std = @import("std"); const ct = @import("consensus_types"); @@ -42,7 +42,7 @@ pub const ClockConfig = struct { pub fn validate(self: ClockConfig) error{InvalidConfig}!void { if (self.slot_duration_ms == 0) return error.InvalidConfig; if (self.slots_per_epoch == 0) return error.InvalidConfig; - // genesis_time_sec → ms must not overflow. + // genesis_time_sec -> ms must not overflow. _ = std.math.mul(u64, self.genesis_time_sec, 1000) catch return error.InvalidConfig; var prev_slot: Slot = 0; for (self.duration_transitions.constSlice()) |t| { diff --git a/src/clock/root.zig b/src/clock/root.zig index 913b08eed..8934e31c1 100644 --- a/src/clock/root.zig +++ b/src/clock/root.zig @@ -1,25 +1,24 @@ -//! Zig beacon clock – slot/epoch timing for Ethereum consensus. +//! Beacon clock - slot/epoch timing for Ethereum consensus. //! -//! Three-layer architecture: -//! Layer 0 (`slot_math`) – pure arithmetic, comptime-compatible -//! Layer 1 (`SlotClock`) – stateful slot clock reading wall-clock time via `std.Io` -//! Layer 2 (`EventClock`) – async event loop with listeners and waiters +//! Public surface: +//! `config` - `ClockConfig` +//! `slot_math` - pure arithmetic, comptime-compatible +//! `Clock` - event-driven beacon clock with listeners and waiters pub const config = @import("config.zig"); pub const slot_math = @import("slot_math.zig"); -pub const SlotClock = @import("SlotClock.zig"); -pub const EventClock = @import("EventClock.zig"); +pub const Clock = @import("Clock.zig"); pub const ClockConfig = config.ClockConfig; -pub const Slot = SlotClock.Slot; -pub const Epoch = SlotClock.Epoch; +pub const Slot = slot_math.Slot; +pub const Epoch = slot_math.Epoch; -pub const ListenerId = EventClock.ListenerId; -pub const Error = EventClock.Error; +pub const ListenerId = Clock.ListenerId; +pub const Error = Clock.Error; test { _ = config; _ = slot_math; - _ = SlotClock; - _ = EventClock; + _ = Clock; + _ = @import("clock_test.zig"); } diff --git a/src/clock/slot_math.zig b/src/clock/slot_math.zig index 68bebbdac..14b4986c8 100644 --- a/src/clock/slot_math.zig +++ b/src/clock/slot_math.zig @@ -1,9 +1,11 @@ -//! Layer 0 – Pure slot/epoch arithmetic. +//! Pure slot/epoch arithmetic and gossip-disparity/tolerance helpers. +//! No state, no allocation, no I/O; every function is comptime-compatible. +//! All functions assume `validate()` accepted the config. //! -//! No state, no allocation, no I/O. Every function is comptime-compatible. -//! The only `null` is pre-genesis (a `<` comparison); arithmetic uses plain -//! operators since `now_ms` is the wall clock and slot/config values are -//! program-controlled — an overflow would be a program error and traps. +//! Arithmetic uses plain operators - out-of-range values are program errors +//! and trap. `slotWithPastToleranceMs` alone saturates, defensively: `now_ms` +//! is a Unix-ms timestamp and callers pass sub-second tolerances, so nothing +//! short of a nonsense tolerance can underflow it. const std = @import("std"); const ct = @import("consensus_types"); @@ -17,7 +19,7 @@ pub const forkTransitions = @import("config.zig").forkTransitions; /// Returns the slot at the given Unix-millisecond timestamp, /// or null if pre-genesis. -/// Precondition: `validate()` accepted `config` — guarantees all durations > 0. +/// Precondition: `validate()` accepted `config` - guarantees all durations > 0. pub fn slotAtMs(config: ClockConfig, now_ms: u64) ?Slot { std.debug.assert(config.slot_duration_ms != 0); const genesis_ms = config.genesis_time_sec * 1000; @@ -48,7 +50,7 @@ pub fn slotAtSec(config: ClockConfig, now_sec: u64) ?Slot { return slotAtMs(config, now_ms); } -/// Slot duration that applies at `slot` — the last transition whose +/// Slot duration that applies at `slot` - the last transition whose /// `from_slot <= slot`, else the base `slot_duration_ms`. pub fn slotDurationMsAt(config: ClockConfig, slot: Slot) u64 { var duration = config.slot_duration_ms; @@ -60,7 +62,7 @@ pub fn slotDurationMsAt(config: ClockConfig, slot: Slot) u64 { } /// Returns the epoch that contains `slot`. -/// Precondition: `validate()` accepted `config` — `slots_per_epoch > 0`. +/// Precondition: `validate()` accepted `config` - `slots_per_epoch > 0`. pub fn epochAtSlot(config: ClockConfig, slot: Slot) Epoch { std.debug.assert(config.slots_per_epoch != 0); return @divFloor(slot, config.slots_per_epoch); @@ -105,16 +107,94 @@ pub fn msUntilNextSlot(config: ClockConfig, now_ms: u64) u64 { return next_start - now_ms; } +/// Returns the slot the network may be advancing to, accounting for gossip +/// clock disparity, or null pre-genesis when no slot is current yet. +/// +/// Per phase0/p2p-interface.md, gossip validation rejects future messages with +/// strict `<`, hence `<=` here. +/// +/// Assumes the disparity window reaches at most the adjacent slot - true +/// for every real config (500 ms disparity vs seconds-long slots). +pub fn slotWithGossipDisparity(config: ClockConfig, now_ms: u64) ?Slot { + const current = slotAtMs(config, now_ms) orelse { + // Pre-genesis the wall slot is conceptually negative, so slot 0 is + // "current" only once we're within gossip disparity of genesis. + const genesis_ms = slotStartMs(config, 0); + return if (genesis_ms - now_ms <= config.maximum_gossip_clock_disparity_ms) + 0 + else + null; + }; + const next_slot = current + 1; + const next_slot_ms = slotStartMs(config, next_slot); + if (next_slot_ms - now_ms <= config.maximum_gossip_clock_disparity_ms) { + return next_slot; + } + return current; +} + +/// See `slotWithGossipDisparity` for the `<=` rationale. +pub fn isCurrentSlotGivenGossipDisparity(config: ClockConfig, slot: Slot, now_ms: u64) bool { + const current = slotAtMs(config, now_ms) orelse { + // Slot 0 pre-genesis rule: see slotWithGossipDisparity. + if (slot != 0) return false; + const genesis_ms = slotStartMs(config, 0); + return genesis_ms - now_ms <= config.maximum_gossip_clock_disparity_ms; + }; + if (slot == current) return true; + + const next_slot = current + 1; + const next_slot_ms = slotStartMs(config, next_slot); + if (next_slot_ms - now_ms <= config.maximum_gossip_clock_disparity_ms) { + return slot == next_slot; + } + + if (current > 0) { + const current_slot_ms = slotStartMs(config, current); + if (now_ms - current_slot_ms <= config.maximum_gossip_clock_disparity_ms) { + return slot == current - 1; + } + } + + return false; +} + +/// Slot at `now_ms + tolerance_ms`, or null if that still lands pre-genesis. +pub fn slotWithFutureToleranceMs(config: ClockConfig, now_ms: u64, tolerance_ms: u64) ?Slot { + const shifted_ms = now_ms + tolerance_ms; + return slotAtMs(config, shifted_ms); +} + +/// Slot at `now_ms - tolerance_ms`, saturating (defensively - see the header); +/// anything at or before genesis clamps to slot 0. +pub fn slotWithPastToleranceMs(config: ClockConfig, now_ms: u64, tolerance_ms: u64) Slot { + const shifted_ms = now_ms -| tolerance_ms; + return slotAtMs(config, shifted_ms) orelse 0; +} + +/// Seconds from the start of `slot` to `to_sec`; negative if earlier. +pub fn secFromSlot(config: ClockConfig, slot: Slot, to_sec: u64) i64 { + const from_sec = slotStartSec(config, slot); + return @as(i64, @intCast(to_sec)) - @as(i64, @intCast(from_sec)); +} + +/// Milliseconds from the start of `slot` to `to_ms`; negative if earlier. +pub fn msFromSlot(config: ClockConfig, slot: Slot, to_ms: u64) i64 { + const from_ms = slotStartMs(config, slot); + return @as(i64, @intCast(to_ms)) - @as(i64, @intCast(from_ms)); +} + const testing = std.testing; -const mainnet = ClockConfig{ +const mainnet: ClockConfig = .{ .genesis_time_sec = 1_606_824_023, .slot_duration_ms = 12_000, .slots_per_epoch = 32, }; -test "basic slot math" { - // slotAtSec: genesis is slot 0 +test "basic slot math anchors slot 0 at genesis" { + // Genesis starts slot 0, every 12 s step is one slot, and 32 slots make + // an epoch; slot starts convert back to the same instants. try testing.expectEqual(@as(?Slot, 0), slotAtSec(mainnet, mainnet.genesis_time_sec)); try testing.expectEqual(@as(?Slot, 1), slotAtSec(mainnet, mainnet.genesis_time_sec + 12)); try testing.expectEqual(@as(?Slot, 2), slotAtSec(mainnet, mainnet.genesis_time_sec + 24)); @@ -143,7 +223,9 @@ test "basic slot math" { try testing.expectEqual(@as(u64, 12_000), slotDurationMsAt(mainnet, 1_000_000)); } -test "within-slot timing" { +test "within-slot times floor to the slot's start" { + // Every instant inside [start, start + 12 s) is the same slot; the exact + // boundary begins the next one. try testing.expectEqual(@as(?Slot, 0), slotAtSec(mainnet, mainnet.genesis_time_sec + 0)); try testing.expectEqual(@as(?Slot, 0), slotAtSec(mainnet, mainnet.genesis_time_sec + 6)); try testing.expectEqual(@as(?Slot, 0), slotAtSec(mainnet, mainnet.genesis_time_sec + 11)); @@ -165,7 +247,9 @@ test "pre-genesis returns null" { try testing.expectEqual(@as(?Slot, null), slotAtMs(mainnet, 0)); } -test "msUntilNextSlot" { +test "msUntilNextSlot counts down to the next boundary" { + // A full slot remains at a boundary, the remainder mid-slot, and + // pre-genesis it is the time until genesis itself. const genesis_ms = mainnet.genesis_time_sec * 1000; const slot_ms: u64 = 12_000; @@ -234,14 +318,15 @@ test "config validate" { }).validate()); } -const eip7782 = ClockConfig{ +const eip7782: ClockConfig = .{ .genesis_time_sec = 1_000_000, .slot_duration_ms = 12_000, .duration_transitions = forkTransitions(&.{.{ .from_slot = 1024, .new_duration_ms = 6_000 }}), .slots_per_epoch = 32, }; -test "fork-aware: slotDurationMsAt" { +test "fork-aware: slotDurationMsAt selects the duration in force at the slot" { + // 12 s slots before the fork at slot 1024, 6 s slots from it onward. try testing.expectEqual(@as(u64, 12_000), slotDurationMsAt(eip7782, 0)); try testing.expectEqual(@as(u64, 12_000), slotDurationMsAt(eip7782, 1023)); try testing.expectEqual(@as(u64, 6_000), slotDurationMsAt(eip7782, 1024)); @@ -282,7 +367,7 @@ test "fork-aware: msUntilNextSlot across boundary" { try testing.expectEqual(@as(u64, 3_000), msUntilNextSlot(eip7782, fork_ms + 3_000)); } -const two_fork = ClockConfig{ +const two_fork: ClockConfig = .{ .genesis_time_sec = 1_000_000, .slot_duration_ms = 12_000, .duration_transitions = forkTransitions(&.{ @@ -295,21 +380,111 @@ const two_fork = ClockConfig{ test "fork-aware: two transitions" { const genesis_ms = two_fork.genesis_time_sec * 1000; const f1_ms = genesis_ms + 1024 * 12_000; // first fork boundary - // Slots 1024..8191 are 6s each → 7168 slots × 6_000 ms + // Slots 1024..8191 are 6s each -> 7168 slots x 6_000 ms const f2_ms = f1_ms + (8192 - 1024) * 6_000; // second fork boundary try testing.expectEqual(@as(u64, 12_000), slotDurationMsAt(two_fork, 0)); try testing.expectEqual(@as(u64, 6_000), slotDurationMsAt(two_fork, 1024)); try testing.expectEqual(@as(u64, 4_000), slotDurationMsAt(two_fork, 8192)); - // slotStartMs across both boundaries try testing.expectEqual(@as(u64, f1_ms), slotStartMs(two_fork, 1024)); try testing.expectEqual(@as(u64, f2_ms), slotStartMs(two_fork, 8192)); try testing.expectEqual(@as(u64, f2_ms + 4_000), slotStartMs(two_fork, 8193)); - // slotAtMs across both boundaries try testing.expectEqual(@as(?Slot, 1024), slotAtMs(two_fork, f1_ms)); try testing.expectEqual(@as(?Slot, 8191), slotAtMs(two_fork, f2_ms - 1)); try testing.expectEqual(@as(?Slot, 8192), slotAtMs(two_fork, f2_ms)); try testing.expectEqual(@as(?Slot, 8193), slotAtMs(two_fork, f2_ms + 4_000)); } + +const test_cfg: ClockConfig = .{ + .genesis_time_sec = 100, + .slot_duration_ms = 12_000, + .slots_per_epoch = 32, + .maximum_gossip_clock_disparity_ms = 500, +}; +const test_genesis_ms: u64 = test_cfg.genesis_time_sec * 1000; +const test_disparity_ms: u64 = test_cfg.maximum_gossip_clock_disparity_ms; +const test_slot_1_start_ms: u64 = test_genesis_ms + test_cfg.slot_duration_ms; + +test "gossip disparity: far from boundary only the current slot is accepted" { + // 3 s into slot 0: more than 500 ms from either boundary, so neither the + // forward nor the backward tolerance applies. + const now = test_genesis_ms + 3_000; + try testing.expectEqual(@as(?Slot, 0), slotWithGossipDisparity(test_cfg, now)); + try testing.expect(isCurrentSlotGivenGossipDisparity(test_cfg, 0, now)); + try testing.expect(!isCurrentSlotGivenGossipDisparity(test_cfg, 1, now)); +} + +test "gossip disparity: just after a boundary the previous slot is accepted" { + // 300 ms into slot 1 is within the 500 ms window, so slot 0 still counts. + try testing.expect( + isCurrentSlotGivenGossipDisparity(test_cfg, 0, test_slot_1_start_ms + 300), + ); +} + +test "gossip disparity: forward window inside, at, and past the threshold" { + // 400 ms before slot 1's start is inside the 500 ms window, exactly + // 500 ms is the inclusive edge, and 1 ms further is past it. + const inside = test_slot_1_start_ms - 400; + try testing.expectEqual(@as(?Slot, 1), slotWithGossipDisparity(test_cfg, inside)); + try testing.expect(isCurrentSlotGivenGossipDisparity(test_cfg, 1, inside)); + + const edge = test_slot_1_start_ms - test_disparity_ms; + try testing.expectEqual(@as(?Slot, 1), slotWithGossipDisparity(test_cfg, edge)); + try testing.expect(isCurrentSlotGivenGossipDisparity(test_cfg, 1, edge)); + + try testing.expectEqual(@as(?Slot, 0), slotWithGossipDisparity(test_cfg, edge - 1)); + try testing.expect(!isCurrentSlotGivenGossipDisparity(test_cfg, 1, edge - 1)); +} + +test "gossip disparity: pre-genesis slot 0 only within disparity of genesis" { + // Before genesis only slot 0 can be current, and only within 500 ms of + // genesis: 1 s out is too early, 300 ms and the 500 ms edge are in, + // 501 ms is out again. + const far_before = test_genesis_ms - 1_000; + try testing.expectEqual(@as(?Slot, null), slotWithGossipDisparity(test_cfg, far_before)); + try testing.expect(!isCurrentSlotGivenGossipDisparity(test_cfg, 0, far_before)); + try testing.expect(!isCurrentSlotGivenGossipDisparity(test_cfg, 1, far_before)); + + const within = test_genesis_ms - 300; + try testing.expectEqual(@as(?Slot, 0), slotWithGossipDisparity(test_cfg, within)); + try testing.expect(isCurrentSlotGivenGossipDisparity(test_cfg, 0, within)); + try testing.expect(!isCurrentSlotGivenGossipDisparity(test_cfg, 1, within)); + + const edge = test_genesis_ms - test_disparity_ms; + try testing.expectEqual(@as(?Slot, 0), slotWithGossipDisparity(test_cfg, edge)); + try testing.expect(isCurrentSlotGivenGossipDisparity(test_cfg, 0, edge)); + + try testing.expectEqual(@as(?Slot, null), slotWithGossipDisparity(test_cfg, edge - 1)); + try testing.expect(!isCurrentSlotGivenGossipDisparity(test_cfg, 0, edge - 1)); +} + +test "tolerance helpers shift the read forward and backward one slot" { + // From slot 1's start, one slot of future tolerance reads slot 2 and one + // slot of past tolerance reads slot 0; four slots in, it reads slot 3. + const one_slot = test_cfg.slot_duration_ms; + try testing.expectEqual( + @as(?Slot, 2), + slotWithFutureToleranceMs(test_cfg, test_slot_1_start_ms, one_slot), + ); + try testing.expectEqual( + @as(Slot, 0), + slotWithPastToleranceMs(test_cfg, test_slot_1_start_ms, one_slot), + ); + try testing.expectEqual( + @as(Slot, 3), + slotWithPastToleranceMs(test_cfg, test_genesis_ms + 4 * one_slot, one_slot), + ); +} + +test "secFromSlot and msFromSlot measure signed offsets from a slot's start" { + // From slot 1's start: +6 s ahead, zero at the start itself, and -12 s + // (one full slot) back at genesis. + const slot_1_start_sec = test_slot_1_start_ms / 1000; + try testing.expectEqual(@as(i64, 6), secFromSlot(test_cfg, 1, slot_1_start_sec + 6)); + try testing.expectEqual(@as(i64, 6000), msFromSlot(test_cfg, 1, test_slot_1_start_ms + 6_000)); + try testing.expectEqual(@as(i64, 0), secFromSlot(test_cfg, 1, slot_1_start_sec)); + try testing.expectEqual(@as(i64, -12), secFromSlot(test_cfg, 1, test_cfg.genesis_time_sec)); + try testing.expectEqual(@as(i64, -12000), msFromSlot(test_cfg, 1, test_genesis_ms)); +} diff --git a/src/era/Reader.zig b/src/era/Reader.zig index e5a02514b..ffcb24aa2 100644 --- a/src/era/Reader.zig +++ b/src/era/Reader.zig @@ -16,8 +16,8 @@ file: std.Io.File, io: std.Io, /// The era number retrieved from the file name era_number: u64, -/// The short historical root retrieved from the file name -short_historical_root: [8]u8, +/// The short era root retrieved from the file name +short_era_root: [8]u8, /// An array of state and block indices, one per group group_indices: []era.GroupIndex, /// Persistent merkle tree pool used by TreeViews (must outlive returned views) @@ -49,7 +49,7 @@ pub fn open(allocator: std.mem.Allocator, io: std.Io, config: c.BeaconConfig, pa .file = file, .io = io, .era_number = era_file_name.era_number, - .short_historical_root = era_file_name.short_historical_root, + .short_era_root = era_file_name.short_era_root, .group_indices = group_indices, .pool = pool, }; diff --git a/src/era/Writer.zig b/src/era/Writer.zig index 245942549..d0cc409e0 100644 --- a/src/era/Writer.zig +++ b/src/era/Writer.zig @@ -38,7 +38,7 @@ pub const WriterState = union(enum) { finished_group: struct { era_number: u64, current_offset: u64, - short_historical_root: [8]u8, + short_era_root: [8]u8, }, }; @@ -72,7 +72,7 @@ pub fn finish(self: *Writer, allocator: std.mem.Allocator, io: std.Io) ![]const const new_base = try std.fmt.allocPrint( allocator, "{s}-{d:0>5}-{s}.era", - .{ self.config.chain.CONFIG_NAME, self.era_number, self.state.finished_group.short_historical_root }, + .{ self.config.chain.CONFIG_NAME, self.era_number, self.state.finished_group.short_era_root }, ); defer allocator.free(new_base); @@ -120,7 +120,7 @@ pub fn writeVersion(self: *Writer, allocator: std.mem.Allocator) !void { }; } -pub fn writeCompressedState(self: *Writer, allocator: std.mem.Allocator, slot: u64, short_historical_root: [8]u8, data: []const u8) !void { +pub fn writeCompressedState(self: *Writer, allocator: std.mem.Allocator, slot: u64, short_era_root: [8]u8, data: []const u8) !void { if (self.state == .init_group) { try self.writeVersion(allocator); } @@ -184,24 +184,24 @@ pub fn writeCompressedState(self: *Writer, allocator: std.mem.Allocator, slot: u .finished_group = .{ .era_number = wg_era_number, .current_offset = wg_current_offset, - .short_historical_root = short_historical_root, + .short_era_root = short_era_root, }, }; } -pub fn writeSerializedState(self: *Writer, allocator: std.mem.Allocator, slot: u64, short_historical_root: [8]u8, data: []const u8) !void { +pub fn writeSerializedState(self: *Writer, allocator: std.mem.Allocator, slot: u64, short_era_root: [8]u8, data: []const u8) !void { const compressed = try snappy.compress(allocator, data); defer allocator.free(compressed); - try self.writeCompressedState(allocator, slot, short_historical_root, compressed); + try self.writeCompressedState(allocator, slot, short_era_root, compressed); } pub fn writeState(self: *Writer, allocator: std.mem.Allocator, state: fork_types.AnyBeaconState) !void { var s = state; const slot = try s.slot(); - const short_historical_root = try era.getShortHistoricalRoot(state); + const short_era_root = try era.getShortEraRoot(state); const serialized = try state.serialize(allocator); defer allocator.free(serialized); - try self.writeSerializedState(allocator, slot, short_historical_root, serialized); + try self.writeSerializedState(allocator, slot, short_era_root, serialized); } pub fn writeCompressedBlock(self: *Writer, allocator: std.mem.Allocator, slot: u64, data: []const u8) !void { diff --git a/src/era/era.zig b/src/era/era.zig index 720a22045..fbb460ef9 100644 --- a/src/era/era.zig +++ b/src/era/era.zig @@ -4,11 +4,11 @@ const fork_types = @import("fork_types"); const e2s = @import("e2s.zig"); /// Parsed components of an .era file name. -/// Format: --.era +/// Format: --.era pub const EraFileName = struct { config_name: []const u8, era_number: u64, - short_historical_root: [8]u8, + short_era_root: [8]u8, pub fn parse(path: []const u8) (error{InvalidEraFileName} || std.fmt.ParseIntError)!EraFileName { if (!std.mem.endsWith(u8, path, ".era")) { @@ -17,16 +17,16 @@ pub const EraFileName = struct { var it = std.mem.splitScalar(u8, std.fs.path.stem(path), '-'); const config_name = it.next() orelse return error.InvalidEraFileName; const era_number = try std.fmt.parseUnsigned(u64, it.next() orelse return error.InvalidEraFileName, 10); - var short_historical_root: [8]u8 = undefined; - const maybe_short_historical_root = it.next() orelse return error.InvalidEraFileName; - if (maybe_short_historical_root.len != 8) { + var short_era_root: [8]u8 = undefined; + const maybe_short_era_root = it.next() orelse return error.InvalidEraFileName; + if (maybe_short_era_root.len != 8) { return error.InvalidEraFileName; } - @memcpy(&short_historical_root, maybe_short_historical_root); + @memcpy(&short_era_root, maybe_short_era_root); return .{ .config_name = config_name, .era_number = era_number, - .short_historical_root = short_historical_root, + .short_era_root = short_era_root, }; } }; @@ -121,29 +121,29 @@ pub fn computeStartBlockSlotFromEraNumber(era_number: u64) !u64 { return (try std.math.sub(u64, era_number, 1)) * preset.SLOTS_PER_HISTORICAL_ROOT; } -pub fn getShortHistoricalRoot(state: fork_types.AnyBeaconState) ![8]u8 { +pub fn getShortEraRoot(state: fork_types.AnyBeaconState) ![8]u8 { const allocator = std.heap.page_allocator; - var short_historical_root: [8]u8 = undefined; + var short_era_root: [8]u8 = undefined; var s = state; - var historical_root: [32]u8 = undefined; + var era_root: [32]u8 = undefined; if (try s.slot() == 0) { - historical_root = (try s.genesisValidatorsRoot()).*; + era_root = (try s.genesisValidatorsRoot()).*; } else if (s.forkSeq().gte(.capella)) { var summaries = try s.historicalSummaries(); const len = try summaries.length(); if (len == 0) return error.EmptyHistoricalSummaries; var last = try summaries.get(len - 1); - historical_root = (try last.hashTreeRoot()).*; + era_root = (try last.hashTreeRoot()).*; } else { var roots = try s.historicalRoots(); const len = try roots.length(); if (len == 0) return error.EmptyHistoricalRoots; var last = try roots.get(len - 1); - try last.toValue(allocator, &historical_root); + try last.toValue(allocator, &era_root); } - _ = try std.fmt.bufPrint(&short_historical_root, "{x}", .{historical_root[0..4]}); - return short_historical_root; + _ = try std.fmt.bufPrint(&short_era_root, "{x}", .{era_root[0..4]}); + return short_era_root; } // ── Unit tests ────────────────────────────────────────────────────────── @@ -152,13 +152,13 @@ test "EraFileName.parse - valid filename" { const result = try EraFileName.parse("mainnet-00001-a1b2c3d4.era"); try std.testing.expectEqualStrings("mainnet", result.config_name); try std.testing.expectEqual(@as(u64, 1), result.era_number); - try std.testing.expectEqualStrings("a1b2c3d4", &result.short_historical_root); + try std.testing.expectEqualStrings("a1b2c3d4", &result.short_era_root); } test "EraFileName.parse - large era number" { const result = try EraFileName.parse("mainnet-99999-deadbeef.era"); try std.testing.expectEqual(@as(u64, 99999), result.era_number); - try std.testing.expectEqualStrings("deadbeef", &result.short_historical_root); + try std.testing.expectEqualStrings("deadbeef", &result.short_era_root); } test "EraFileName.parse - different config names" { diff --git a/src/fork_types/any_beacon_state.zig b/src/fork_types/any_beacon_state.zig index 3cf890d02..c3d9015aa 100644 --- a/src/fork_types/any_beacon_state.zig +++ b/src/fork_types/any_beacon_state.zig @@ -382,6 +382,16 @@ pub const AnyBeaconState = union(ForkSeq) { try self.setEth1DepositIndex(try self.eth1DepositIndex() + 1); } + pub fn buildersLength(self: *AnyBeaconState) !usize { + return switch (self.*) { + .phase0, .altair, .bellatrix, .capella, .deneb, .electra, .fulu => error.InvalidAtFork, + inline else => |state| { + var builders_view = try state.getReadonly("builders"); + return builders_view.length(); + }, + }; + } + pub fn validators(self: *AnyBeaconState) !*ct.phase0.Validators.TreeView { return switch (self.*) { inline else => |state| try state.get("validators"), diff --git a/src/ssz/type/list.zig b/src/ssz/type/list.zig index 8c6ccecc6..d4933db68 100644 --- a/src/ssz/type/list.zig +++ b/src/ssz/type/list.zig @@ -166,7 +166,6 @@ pub fn FixedListType(comptime ST: type, comptime _limit: comptime_int, comptime @memcpy(std.mem.sliceAsBytes(out.items[0..len]), data); return; } - @memset(out.items[0..len], Element.default_value); for (0..len) |i| { try Element.deserializeFromBytes( data[i * Element.fixed_size .. (i + 1) * Element.fixed_size], diff --git a/src/state_transition/block/is_valid_indexed_attestation.zig b/src/state_transition/block/is_valid_indexed_attestation.zig index adc9ab117..ee51e9aeb 100644 --- a/src/state_transition/block/is_valid_indexed_attestation.zig +++ b/src/state_transition/block/is_valid_indexed_attestation.zig @@ -12,6 +12,7 @@ const getIndexedAttestationSignatureSet = @import("../signature_sets/indexed_att pub fn isValidIndexedAttestation( comptime fork: ForkSeq, allocator: std.mem.Allocator, + io: std.Io, config: *const BeaconConfig, epoch_cache: *const EpochCache, validators_count: usize, @@ -26,6 +27,7 @@ pub fn isValidIndexedAttestation( const signature_set = try getIndexedAttestationSignatureSet( fork, allocator, + io, config, epoch_cache, indexed_attestation, diff --git a/src/state_transition/block/process_attestation_altair.zig b/src/state_transition/block/process_attestation_altair.zig index 3cdac2af0..5f80dbee2 100644 --- a/src/state_transition/block/process_attestation_altair.zig +++ b/src/state_transition/block/process_attestation_altair.zig @@ -36,6 +36,7 @@ const SLOTS_PER_EPOCH_SQRT = std.math.sqrt(preset.SLOTS_PER_EPOCH); pub fn processAttestationsAltair( comptime fork: ForkSeq, allocator: Allocator, + io: std.Io, config: *const BeaconConfig, epoch_cache: *EpochCache, state: *BeaconState(fork), @@ -78,6 +79,7 @@ pub fn processAttestationsAltair( if (verify_signature) { const sig_set = try getAttestationWithIndicesSignatureSet( allocator, + io, config, epoch_cache, &attestation.data, diff --git a/src/state_transition/block/process_attestation_phase0.zig b/src/state_transition/block/process_attestation_phase0.zig index d6cc740f8..d9de2115c 100644 --- a/src/state_transition/block/process_attestation_phase0.zig +++ b/src/state_transition/block/process_attestation_phase0.zig @@ -14,6 +14,7 @@ const PendingAttestation = types.phase0.PendingAttestation.Type; pub fn processAttestationPhase0( allocator: Allocator, + io: std.Io, config: *const BeaconConfig, epoch_cache: *const EpochCache, state: *BeaconState(.phase0), @@ -56,6 +57,7 @@ pub fn processAttestationPhase0( if (!try isValidIndexedAttestation( .phase0, allocator, + io, config, epoch_cache, validators_count, diff --git a/src/state_transition/block/process_attestations.zig b/src/state_transition/block/process_attestations.zig index 8cd339cd2..3905a4943 100644 --- a/src/state_transition/block/process_attestations.zig +++ b/src/state_transition/block/process_attestations.zig @@ -16,6 +16,7 @@ const Node = @import("persistent_merkle_tree").Node; pub fn processAttestations( comptime fork: ForkSeq, allocator: Allocator, + io: std.Io, config: *const BeaconConfig, epoch_cache: *EpochCache, state: *BeaconState(fork), @@ -28,6 +29,7 @@ pub fn processAttestations( for (attestations) |attestation| { try processAttestationPhase0( allocator, + io, config, epoch_cache, state, @@ -39,6 +41,7 @@ pub fn processAttestations( try processAttestationsAltair( fork, allocator, + io, config, epoch_cache, state, @@ -66,6 +69,7 @@ test "process attestations - sanity" { processAttestations( .electra, allocator, + std.testing.io, test_state.cached_state.config, test_state.cached_state.epoch_cache, test_state.cached_state.state.castToFork(.electra), diff --git a/src/state_transition/block/process_attester_slashing.zig b/src/state_transition/block/process_attester_slashing.zig index 70adb84e3..9959c034e 100644 --- a/src/state_transition/block/process_attester_slashing.zig +++ b/src/state_transition/block/process_attester_slashing.zig @@ -20,6 +20,7 @@ const slashValidator = @import("./slash_validator.zig").slashValidator; pub fn processAttesterSlashing( comptime fork: ForkSeq, allocator: Allocator, + io: std.Io, config: *const BeaconConfig, epoch_cache: *EpochCache, state: *BeaconState(fork), @@ -32,6 +33,7 @@ pub fn processAttesterSlashing( try assertValidAttesterSlashing( fork, allocator, + io, config, epoch_cache, try state.validatorsCount(), @@ -73,6 +75,7 @@ pub fn processAttesterSlashing( pub fn assertValidAttesterSlashing( comptime fork: ForkSeq, allocator: Allocator, + io: std.Io, config: *const BeaconConfig, epoch_cache: *const EpochCache, validators_count: usize, @@ -87,6 +90,7 @@ pub fn assertValidAttesterSlashing( if (!try isValidIndexedAttestation( fork, allocator, + io, config, epoch_cache, validators_count, @@ -98,6 +102,7 @@ pub fn assertValidAttesterSlashing( if (!try isValidIndexedAttestation( fork, allocator, + io, config, epoch_cache, validators_count, diff --git a/src/state_transition/block/process_block.zig b/src/state_transition/block/process_block.zig index 2599640c1..930a13224 100644 --- a/src/state_transition/block/process_block.zig +++ b/src/state_transition/block/process_block.zig @@ -39,6 +39,7 @@ pub const ProcessBlockOpts = struct { pub fn processBlock( comptime fork: ForkSeq, allocator: Allocator, + io: std.Io, config: *const BeaconConfig, epoch_cache: *EpochCache, state: *BeaconState(fork), @@ -146,11 +147,11 @@ pub fn processBlock( ); } - try processRandao(fork, config, epoch_cache, state, block_type, body, block.proposerIndex(), opts.verify_signature); + try processRandao(fork, io, config, epoch_cache, state, block_type, body, block.proposerIndex(), opts.verify_signature); try processEth1Data(fork, state, body.eth1Data()); - try processOperations(fork, allocator, config, epoch_cache, state, slashings_cache, block_type, body, opts); + try processOperations(fork, allocator, io, config, epoch_cache, state, slashings_cache, block_type, body, opts); if (comptime fork.gte(.altair)) { - try processSyncAggregate(fork, allocator, config, epoch_cache, state, body.syncAggregate(), opts.verify_signature); + try processSyncAggregate(fork, allocator, io, config, epoch_cache, state, body.syncAggregate(), opts.verify_signature); } if (comptime fork.gte(.deneb)) { diff --git a/src/state_transition/block/process_consolidation_request.zig b/src/state_transition/block/process_consolidation_request.zig index 5863214c7..b0c25096b 100644 --- a/src/state_transition/block/process_consolidation_request.zig +++ b/src/state_transition/block/process_consolidation_request.zig @@ -12,7 +12,6 @@ const hasEth1WithdrawalCredential = @import("../utils/capella.zig").hasEth1Withd const electra_utils = @import("../utils/electra.zig"); const hasCompoundingWithdrawalCredential = electra_utils.hasCompoundingWithdrawalCredential; const hasExecutionWithdrawalCredential = electra_utils.hasExecutionWithdrawalCredential; -const isPubkeyKnown = electra_utils.isPubkeyKnown; const switchToCompoundingValidator = electra_utils.switchToCompoundingValidator; const computeConsolidationEpochAndUpdateChurn = @import("../utils/epoch.zig").computeConsolidationEpochAndUpdateChurn; const validator_utils = @import("../utils/validator.zig"); @@ -23,6 +22,7 @@ const isActiveValidatorView = validator_utils.isActiveValidatorView; // TODO Electra: Clean up necessary as there is a lot of overlap with isValidSwitchToCompoundRequest pub fn processConsolidationRequest( comptime fork: ForkSeq, + io: std.Io, config: *const BeaconConfig, epoch_cache: *const EpochCache, state: *BeaconState(fork), @@ -32,13 +32,12 @@ pub fn processConsolidationRequest( const target_pubkey = consolidation.target_pubkey; const source_address = consolidation.source_address; - if (!(try isPubkeyKnown(fork, epoch_cache, state, source_pubkey))) return; - if (!(try isPubkeyKnown(fork, epoch_cache, state, target_pubkey))) return; + const source_index = epoch_cache.pubkey_cache.get(io, source_pubkey) orelse return; + const target_index = epoch_cache.pubkey_cache.get(io, target_pubkey) orelse return; + const validator_count = try state.validatorsCount(); + if (source_index >= validator_count or target_index >= validator_count) return; - const source_index = epoch_cache.pubkey_to_index.get(source_pubkey) orelse return; - const target_index = epoch_cache.pubkey_to_index.get(target_pubkey) orelse return; - - if (try isValidSwitchToCompoundRequest(fork, epoch_cache, state, consolidation)) { + if (try isValidSwitchToCompoundRequest(fork, epoch_cache, state, consolidation, source_index, target_index)) { try switchToCompoundingValidator(fork, state, source_index); // Early return since we have already switched validator to compounding return; @@ -121,11 +120,9 @@ fn isValidSwitchToCompoundRequest( epoch_cache: *const EpochCache, state: *BeaconState(fork), consolidation: *const ConsolidationRequest, + source_index: types.primitive.ValidatorIndex.Type, + target_index: types.primitive.ValidatorIndex.Type, ) !bool { - // this check is mainly to make the compiler happy, pubkey is checked by the consumer already - const source_index = epoch_cache.pubkey_to_index.get(consolidation.source_pubkey) orelse return false; - const target_index = epoch_cache.pubkey_to_index.get(consolidation.target_pubkey) orelse return false; - // Switch to compounding requires source and target be equal if (source_index != target_index) { return false; diff --git a/src/state_transition/block/process_deposit.zig b/src/state_transition/block/process_deposit.zig index 272d5fa24..20b11b0cf 100644 --- a/src/state_transition/block/process_deposit.zig +++ b/src/state_transition/block/process_deposit.zig @@ -59,6 +59,7 @@ pub const DepositData = union(enum) { pub fn processDeposit( comptime fork: ForkSeq, allocator: Allocator, + io: std.Io, config: *const BeaconConfig, epoch_cache: *EpochCache, state: *BeaconState(fork), @@ -82,7 +83,7 @@ pub fn processDeposit( // deposits must be processed in order try state.incrementEth1DepositIndex(); - try applyDeposit(fork, allocator, config, epoch_cache, state, &.{ + try applyDeposit(fork, allocator, io, config, epoch_cache, state, &.{ .phase0 = deposit.data, }); } @@ -92,6 +93,7 @@ pub fn processDeposit( pub fn applyDeposit( comptime fork: ForkSeq, allocator: Allocator, + io: std.Io, config: *const BeaconConfig, epoch_cache: *EpochCache, state: *BeaconState(fork), @@ -102,13 +104,13 @@ pub fn applyDeposit( const amount = deposit.amount(); const signature = deposit.signature(); - const cached_index = epoch_cache.getValidatorIndex(pubkey); + const cached_index = epoch_cache.pubkey_cache.get(io, pubkey.*); const is_new_validator = cached_index == null or cached_index.? >= try state.validatorsCount(); if (comptime fork.lt(.electra)) { if (is_new_validator) { if (validateDepositSignature(config, pubkey, withdrawal_credentials, amount, signature)) { - try addValidatorToRegistry(fork, allocator, epoch_cache, state, pubkey, withdrawal_credentials, amount); + try addValidatorToRegistry(fork, allocator, io, epoch_cache, state, pubkey, withdrawal_credentials, amount); } else |_| { // invalid deposit signature, ignore the deposit // TODO may be a useful metric to track @@ -130,7 +132,7 @@ pub fn applyDeposit( var pending_deposits = try state.pendingDeposits(); if (is_new_validator) { if (validateDepositSignature(config, pubkey, withdrawal_credentials, amount, signature)) { - try addValidatorToRegistry(fork, allocator, epoch_cache, state, pubkey, withdrawal_credentials, 0); + try addValidatorToRegistry(fork, allocator, io, epoch_cache, state, pubkey, withdrawal_credentials, 0); try pending_deposits.pushValue(&pending_deposit); } else |_| { // invalid deposit signature, ignore the deposit @@ -145,6 +147,7 @@ pub fn applyDeposit( pub fn addValidatorToRegistry( comptime fork: ForkSeq, allocator: Allocator, + io: std.Io, epoch_cache: *EpochCache, state: *BeaconState(fork), pubkey: *const BLSPubkey, @@ -168,6 +171,7 @@ pub fn addValidatorToRegistry( .effective_balance = effective_balance, .slashed = false, }; + try validators.pushValue(&validator); const validator_index = (try validators.length()) - 1; @@ -178,8 +182,7 @@ pub fn addValidatorToRegistry( // - Should have equal performance since it sets a value in a flat array try epoch_cache.effectiveBalanceIncrementsSet(allocator, validator_index, effective_balance); - // now that there is a new validator, update the epoch context with the new pubkey - try epoch_cache.addPubkey(validator_index, pubkey); + try epoch_cache.pubkey_cache.append(io, pubkey.*, validator_index); // Only after altair: if (comptime fork.gte(.altair)) { diff --git a/src/state_transition/block/process_operations.zig b/src/state_transition/block/process_operations.zig index 156f36178..d3eb19028 100644 --- a/src/state_transition/block/process_operations.zig +++ b/src/state_transition/block/process_operations.zig @@ -26,6 +26,7 @@ const processPayloadAttestation = @import("./process_payload_attestation.zig").p pub fn processOperations( comptime fork: ForkSeq, allocator: std.mem.Allocator, + io: std.Io, config: *const BeaconConfig, epoch_cache: *EpochCache, state: *BeaconState(fork), @@ -44,13 +45,14 @@ pub fn processOperations( const current_epoch = epoch_cache.epoch; for (body.inner.proposer_slashings.items) |*proposer_slashing| { - try processProposerSlashing(fork, allocator, config, epoch_cache, state, slashings_cache, proposer_slashing, opts.verify_signature); + try processProposerSlashing(fork, allocator, io, config, epoch_cache, state, slashings_cache, proposer_slashing, opts.verify_signature); } for (body.inner.attester_slashings.items) |*attester_slashing| { try processAttesterSlashing( fork, allocator, + io, config, epoch_cache, state, @@ -61,14 +63,14 @@ pub fn processOperations( ); } - try processAttestations(fork, allocator, config, epoch_cache, state, slashings_cache, body.inner.attestations.items, opts.verify_signature); + try processAttestations(fork, allocator, io, config, epoch_cache, state, slashings_cache, body.inner.attestations.items, opts.verify_signature); for (body.inner.deposits.items) |*deposit| { - try processDeposit(fork, allocator, config, epoch_cache, state, deposit); + try processDeposit(fork, allocator, io, config, epoch_cache, state, deposit); } for (body.inner.voluntary_exits.items) |*voluntary_exit| { - try processVoluntaryExit(fork, config, epoch_cache, state, voluntary_exit, opts.verify_signature); + try processVoluntaryExit(fork, io, config, epoch_cache, state, voluntary_exit, opts.verify_signature); } if (comptime fork.gte(.capella)) { @@ -85,11 +87,11 @@ pub fn processOperations( } for (execution_requests.withdrawals.items) |*withdrawal_request| { - try processWithdrawalRequest(fork, config, epoch_cache, state, withdrawal_request); + try processWithdrawalRequest(fork, io, config, epoch_cache, state, withdrawal_request); } for (execution_requests.consolidations.items) |*consolidation_request| { - try processConsolidationRequest(fork, config, epoch_cache, state, consolidation_request); + try processConsolidationRequest(fork, io, config, epoch_cache, state, consolidation_request); } } @@ -118,6 +120,7 @@ test "process operations" { try processOperations( .electra, allocator, + std.testing.io, test_state.cached_state.config, test_state.cached_state.epoch_cache, try test_state.cached_state.state.tryCastToFork(.electra), diff --git a/src/state_transition/block/process_proposer_slashing.zig b/src/state_transition/block/process_proposer_slashing.zig index 22b6d95eb..834b1ded7 100644 --- a/src/state_transition/block/process_proposer_slashing.zig +++ b/src/state_transition/block/process_proposer_slashing.zig @@ -18,6 +18,7 @@ const preset = @import("preset").preset; pub fn processProposerSlashing( comptime fork: ForkSeq, allocator: std.mem.Allocator, + io: std.Io, config: *const BeaconConfig, epoch_cache: *EpochCache, state: *BeaconState(fork), @@ -26,7 +27,7 @@ pub fn processProposerSlashing( verify_signatures: bool, ) !void { try buildSlashingsCacheIfNeeded(allocator, state, slashings_cache); - try assertValidProposerSlashing(fork, config, epoch_cache, state, proposer_slashing, verify_signatures); + try assertValidProposerSlashing(fork, io, config, epoch_cache, state, proposer_slashing, verify_signatures); if (fork.gte(.gloas)) { const slot = proposer_slashing.signed_header_1.message.slot; @@ -58,6 +59,7 @@ pub fn processProposerSlashing( pub fn assertValidProposerSlashing( comptime fork: ForkSeq, + io: std.Io, config: *const BeaconConfig, epoch_cache: *const EpochCache, state: *BeaconState(fork), @@ -99,6 +101,7 @@ pub fn assertValidProposerSlashing( // verify signatures if (verify_signature) { const signature_sets = try getProposerSlashingSignatureSets( + io, config, epoch_cache, proposer_slashing, diff --git a/src/state_transition/block/process_randao.zig b/src/state_transition/block/process_randao.zig index 0820c63f0..dd8ee37e4 100644 --- a/src/state_transition/block/process_randao.zig +++ b/src/state_transition/block/process_randao.zig @@ -12,6 +12,7 @@ const Sha256 = std.crypto.hash.sha2.Sha256; pub fn processRandao( comptime fork: ForkSeq, + io: std.Io, beacon_config: *const BeaconConfig, epoch_cache: *const EpochCache, state: *BeaconState(fork), @@ -26,6 +27,7 @@ pub fn processRandao( // verify RANDAO reveal if (verify_signature) { if (!try verifyRandaoSignature( + io, beacon_config, epoch_cache, randao_reveal, @@ -86,6 +88,7 @@ test "process randao - sanity" { try processRandao( .electra, + std.testing.io, test_state.cached_state.config, test_state.cached_state.epoch_cache, test_state.cached_state.state.castToFork(.electra), diff --git a/src/state_transition/block/process_sync_committee.zig b/src/state_transition/block/process_sync_committee.zig index 23bd2afc1..e2ca2d95b 100644 --- a/src/state_transition/block/process_sync_committee.zig +++ b/src/state_transition/block/process_sync_committee.zig @@ -25,6 +25,7 @@ const decreaseBalance = balance_utils.decreaseBalance; pub fn processSyncAggregate( comptime fork: ForkSeq, allocator: Allocator, + io: std.Io, config: *const BeaconConfig, epoch_cache: *const EpochCache, state: *BeaconState(fork), @@ -52,9 +53,10 @@ pub fn processSyncAggregate( const pubkeys = try allocator.alloc(bls.PublicKey, participant_indices.items.len); defer allocator.free(pubkeys); - for (0..participant_indices.items.len) |i| { - pubkeys[i] = epoch_cache.index_to_pubkey.items[participant_indices.items[i]]; - } + epoch_cache.pubkey_cache.getPubkeys(io, participant_indices.items, pubkeys) catch |err| switch (err) { + error.InvalidIndex => return error.PubkeyNotFound, + else => return err, + }; var signing_root: Root = undefined; try computeSigningRoot(types.primitive.Root, root_signed, domain, &signing_root); @@ -114,18 +116,28 @@ pub fn processSyncAggregate( /// see https://github.com/ChainSafe/state-transition-z/issues/72 pub fn getSyncCommitteeSignatureSet( allocator: Allocator, + io: std.Io, config: *const BeaconConfig, epoch_cache: *const EpochCache, sync_aggregate: *const SyncAggregate, block_slot: u64, block_parent_root: *const Root, - participant_indices: ?[]usize, + participant_indices: ?[]const ValidatorIndex, ) !?AggregatedSignatureSet { const signature = sync_aggregate.sync_committee_signature; - const participant_indices_ = if (participant_indices) |pi| pi else blk: { - const committee_indices = @as(*const [preset.SYNC_COMMITTEE_SIZE]u64, @ptrCast(epoch_cache.current_sync_committee_indexed.get().getValidatorIndices())); - break :blk (try sync_aggregate.sync_committee_bits.intersectValues(ValidatorIndex, allocator, committee_indices)).items; + var computed_participant_indices: std.ArrayList(ValidatorIndex) = .empty; + defer computed_participant_indices.deinit(allocator); + const participant_indices_: []const ValidatorIndex = if (participant_indices) |indices| + indices + else blk: { + const committee_indices = @as(*const [preset.SYNC_COMMITTEE_SIZE]ValidatorIndex, @ptrCast(epoch_cache.current_sync_committee_indexed.get().getValidatorIndices())); + computed_participant_indices = try sync_aggregate.sync_committee_bits.intersectValues( + ValidatorIndex, + allocator, + committee_indices, + ); + break :blk computed_participant_indices.items; }; // When there's no participation we consider the signature valid and just ignore it if (participant_indices_.len == 0) { @@ -154,16 +166,16 @@ pub fn getSyncCommitteeSignatureSet( // // On skipped slots state block roots just copy the latest block, so using the parentRoot here is equivalent. // So getSyncCommitteeSignatureSet() can be called with a state in any slot (with the correct shuffling) - const root_signed = block_parent_root; - const domain = try config.getDomain(epoch_cache.epoch, c.DOMAIN_SYNC_COMMITTEE, previous_slot); const pubkeys = try allocator.alloc(bls.PublicKey, participant_indices_.len); - for (0..participant_indices_.len) |i| { - pubkeys[i] = epoch_cache.index_to_pubkey.items[participant_indices_[i]]; - } + errdefer allocator.free(pubkeys); + epoch_cache.pubkey_cache.getPubkeys(io, participant_indices_, pubkeys) catch |err| switch (err) { + error.InvalidIndex => return error.PubkeyNotFound, + else => return err, + }; var signing_root: Root = undefined; - try computeSigningRoot(types.primitive.Root, &root_signed, domain, &signing_root); + try computeSigningRoot(types.primitive.Root, block_parent_root, domain, &signing_root); return .{ .pubkeys = pubkeys, @@ -209,6 +221,7 @@ test "process sync aggregate - sanity" { const res = processSyncAggregate( .electra, allocator, + std.testing.io, config, epoch_cache, fork_state, @@ -222,10 +235,23 @@ test "process sync aggregate - sanity" { try processSyncAggregate( .electra, allocator, + std.testing.io, config, epoch_cache, fork_state, &sync_aggregate, true, ); + + const signature_set = (try getSyncCommitteeSignatureSet( + allocator, + std.testing.io, + config, + epoch_cache, + &sync_aggregate, + try state.slot(), + root_signed, + null, + )).?; + defer allocator.free(signature_set.pubkeys); } diff --git a/src/state_transition/block/process_voluntary_exit.zig b/src/state_transition/block/process_voluntary_exit.zig index 3ff089802..f2bcfdc3a 100644 --- a/src/state_transition/block/process_voluntary_exit.zig +++ b/src/state_transition/block/process_voluntary_exit.zig @@ -14,13 +14,14 @@ const FAR_FUTURE_EPOCH = c.FAR_FUTURE_EPOCH; pub fn processVoluntaryExit( comptime fork: ForkSeq, + io: std.Io, config: *const BeaconConfig, epoch_cache: *EpochCache, state: *BeaconState(fork), signed_voluntary_exit: *const SignedVoluntaryExit, verify_signature: bool, ) !void { - if (!try isValidVoluntaryExit(fork, config, epoch_cache, state, signed_voluntary_exit, verify_signature)) { + if (!try isValidVoluntaryExit(fork, io, config, epoch_cache, state, signed_voluntary_exit, verify_signature)) { return error.InvalidVoluntaryExit; } @@ -31,13 +32,14 @@ pub fn processVoluntaryExit( pub fn isValidVoluntaryExit( comptime fork: ForkSeq, + io: std.Io, config: *const BeaconConfig, epoch_cache: *const EpochCache, state: *BeaconState(fork), signed_voluntary_exit: *const SignedVoluntaryExit, verify_signature: bool, ) !bool { - return try getVoluntaryExitValidity(fork, config, epoch_cache, state, signed_voluntary_exit, verify_signature) == .valid; + return try getVoluntaryExitValidity(fork, io, config, epoch_cache, state, signed_voluntary_exit, verify_signature) == .valid; } pub const VoluntaryExitValidity = enum { @@ -52,6 +54,7 @@ pub const VoluntaryExitValidity = enum { pub fn getVoluntaryExitValidity( comptime fork: ForkSeq, + io: std.Io, config: *const BeaconConfig, epoch_cache: *const EpochCache, state: *BeaconState(fork), @@ -100,7 +103,7 @@ pub fn getVoluntaryExitValidity( // verify signature if (verify_signature) { - if (!try verifyVoluntaryExitSignature(config, epoch_cache, signed_voluntary_exit)) { + if (!try verifyVoluntaryExitSignature(io, config, epoch_cache, signed_voluntary_exit)) { return .invalid_signature; } } @@ -137,6 +140,7 @@ test "voluntary exit - valid" { const result = try getVoluntaryExitValidity( .electra, + std.testing.io, test_state.config, test_state.cached_state.epoch_cache, test_state.cached_state.state.castToFork(.electra), @@ -160,6 +164,7 @@ test "voluntary exit - inactive validator (out of bounds index)" { const result = try getVoluntaryExitValidity( .electra, + std.testing.io, test_state.config, test_state.cached_state.epoch_cache, test_state.cached_state.state.castToFork(.electra), @@ -190,6 +195,7 @@ test "voluntary exit - inactive validator (not active in current epoch)" { const result = try getVoluntaryExitValidity( .electra, + std.testing.io, test_state.config, test_state.cached_state.epoch_cache, state, @@ -220,6 +226,7 @@ test "voluntary exit - already exited validator" { const result = try getVoluntaryExitValidity( .electra, + std.testing.io, test_state.config, test_state.cached_state.epoch_cache, state, @@ -245,6 +252,7 @@ test "voluntary exit - early epoch" { const result = try getVoluntaryExitValidity( .electra, + std.testing.io, test_state.config, test_state.cached_state.epoch_cache, test_state.cached_state.state.castToFork(.electra), @@ -275,6 +283,7 @@ test "voluntary exit - short time active" { const result = try getVoluntaryExitValidity( .electra, + std.testing.io, test_state.config, test_state.cached_state.epoch_cache, state, diff --git a/src/state_transition/block/process_withdrawal_request.zig b/src/state_transition/block/process_withdrawal_request.zig index c8b4504cf..effab2f77 100644 --- a/src/state_transition/block/process_withdrawal_request.zig +++ b/src/state_transition/block/process_withdrawal_request.zig @@ -17,14 +17,13 @@ const computeExitEpochAndUpdateChurn = @import("../utils/epoch.zig").computeExit pub fn processWithdrawalRequest( comptime fork: ForkSeq, + io: std.Io, config: *const BeaconConfig, epoch_cache: *EpochCache, state: *BeaconState(fork), withdrawal_request: *const WithdrawalRequest, ) !void { const amount = withdrawal_request.amount; - // no need to use unfinalized pubkey cache from 6110 as validator won't be active anyway - const pubkey_to_index = epoch_cache.pubkey_to_index; const is_full_exit_request = amount == c.FULL_EXIT_REQUEST_AMOUNT; var pending_partial_withdrawals = try state.pendingPartialWithdrawals(); @@ -38,7 +37,7 @@ pub fn processWithdrawalRequest( // bail out if validator is not in beacon state // note that we don't need to check for 6110 unfinalized vals as they won't be eligible for withdraw/exit anyway - const validator_index = pubkey_to_index.get(withdrawal_request.validator_pubkey) orelse return; + const validator_index = epoch_cache.pubkey_cache.get(io, withdrawal_request.validator_pubkey) orelse return; var validators = try state.validators(); if (validator_index >= try validators.length()) return; diff --git a/src/state_transition/cache/epoch_cache.zig b/src/state_transition/cache/epoch_cache.zig index 40fd80788..45ccc59ef 100644 --- a/src/state_transition/cache/epoch_cache.zig +++ b/src/state_transition/cache/epoch_cache.zig @@ -4,7 +4,6 @@ const preset = @import("preset").preset; const GENESIS_EPOCH = @import("preset").GENESIS_EPOCH; const types = @import("consensus_types"); const c = @import("constants"); -const bls = @import("bls"); const Epoch = types.primitive.Epoch.Type; const Slot = types.primitive.Slot.Type; const BLSSignature = types.primitive.BLSSignature.Type; @@ -12,8 +11,7 @@ const SyncPeriod = types.primitive.SyncPeriod.Type; const ValidatorIndex = types.primitive.ValidatorIndex.Type; const CommitteeIndex = types.primitive.CommitteeIndex.Type; const BeaconConfig = @import("config").BeaconConfig; -const PubkeyIndexMap = @import("./pubkey_cache.zig").PubkeyIndexMap; -const Index2PubkeyCache = @import("./pubkey_cache.zig").Index2PubkeyCache; +const PubkeyCache = @import("./pubkey_cache.zig").PubkeyCache; const EpochShuffling = @import("../utils//epoch_shuffling.zig").EpochShuffling; const EpochShufflingRc = @import("../utils/epoch_shuffling.zig").EpochShufflingRc; const EffectiveBalanceIncrementsRc = @import("./effective_balance_increments.zig").EffectiveBalanceIncrementsRc; @@ -47,7 +45,6 @@ const getActivationChurnLimit = @import("../utils/validator.zig").getActivationC const ForkSeq = @import("config").ForkSeq; const ForkTypes = @import("fork_types").ForkTypes; -const syncPubkeys = @import("./pubkey_cache.zig").syncPubkeys; const PtcCommittees = [preset.SLOTS_PER_EPOCH][preset.PTC_SIZE]ValidatorIndex; const PtcWindowEpochCacheData = struct { @@ -75,8 +72,7 @@ fn getPtcWindowEpochCacheDataFromState(state: *BeaconState(.gloas)) !PtcWindowEp pub const EpochCacheImmutableData = struct { config: *const BeaconConfig, - pubkey_to_index: *PubkeyIndexMap, - index_to_pubkey: *Index2PubkeyCache, + pubkey_cache: *PubkeyCache, }; pub const EpochCacheOpts = struct { @@ -94,11 +90,8 @@ pub const EpochCache = struct { config: *const BeaconConfig, - // this is shared across applications, EpochCache does not own this field so should not deinit() - pubkey_to_index: *PubkeyIndexMap, - - // this is shared across applications, EpochCache does not own this field so should not deinit() - index_to_pubkey: *Index2PubkeyCache, + /// Application-wide pubkey cache; EpochCache does not own it. + pubkey_cache: *PubkeyCache, proposers: [preset.SLOTS_PER_EPOCH]ValidatorIndex, @@ -123,9 +116,6 @@ pub const EpochCache = struct { next_shuffling: *EpochShufflingRc, - // TODO: not needed, maybe just get from the next shuffling? - // next_active_indices - // EpochCache does not take ownership of EffectiveBalanceIncrements, it is shared across EpochCache instances effective_balance_increments: *EffectiveBalanceIncrementsRc, @@ -194,7 +184,8 @@ pub const EpochCache = struct { fn initCurrentSyncCommitteeCacheRc( allocator: Allocator, state: *AnyBeaconState, - pubkey_to_index: *const PubkeyIndexMap, + pubkey_cache: *const PubkeyCache, + io: std.Io, skip_sync_committee_cache: bool, ) !*SyncCommitteeCacheRc { var sync_committee_cache = blk: { @@ -202,7 +193,7 @@ pub const EpochCache = struct { var sync_committee_view = try state.currentSyncCommittee(); var sync_committee: types.altair.SyncCommittee.Type = undefined; try sync_committee_view.toValue(allocator, &sync_committee); - break :blk try SyncCommitteeCacheAllForks.initSyncCommittee(allocator, &sync_committee, pubkey_to_index); + break :blk try SyncCommitteeCacheAllForks.initSyncCommittee(allocator, &sync_committee, pubkey_cache, io); }; errdefer sync_committee_cache.deinit(); @@ -212,7 +203,8 @@ pub const EpochCache = struct { fn initNextSyncCommitteeCacheRc( allocator: Allocator, state: *AnyBeaconState, - pubkey_to_index: *const PubkeyIndexMap, + pubkey_cache: *const PubkeyCache, + io: std.Io, skip_sync_committee_cache: bool, ) !*SyncCommitteeCacheRc { var sync_committee_cache = blk: { @@ -220,17 +212,22 @@ pub const EpochCache = struct { var sync_committee_view = try state.nextSyncCommittee(); var sync_committee: types.altair.SyncCommittee.Type = undefined; try sync_committee_view.toValue(allocator, &sync_committee); - break :blk try SyncCommitteeCacheAllForks.initSyncCommittee(allocator, &sync_committee, pubkey_to_index); + break :blk try SyncCommitteeCacheAllForks.initSyncCommittee(allocator, &sync_committee, pubkey_cache, io); }; errdefer sync_committee_cache.deinit(); return try SyncCommitteeCacheRc.init(allocator, sync_committee_cache); } - pub fn createFromState(allocator: Allocator, state: *AnyBeaconState, immutable_data: EpochCacheImmutableData, option: ?EpochCacheOpts) !*EpochCache { + pub fn createFromState( + allocator: Allocator, + io: std.Io, + state: *AnyBeaconState, + immutable_data: EpochCacheImmutableData, + option: ?EpochCacheOpts, + ) !*EpochCache { const config = immutable_data.config; - const pubkey_to_index = immutable_data.pubkey_to_index; - const index_to_pubkey = immutable_data.index_to_pubkey; + const pubkey_cache = immutable_data.pubkey_cache; const current_epoch = computeEpochAtSlot(try state.slot()); const previous_epoch = if (current_epoch == GENESIS_EPOCH) GENESIS_EPOCH else current_epoch - 1; @@ -245,11 +242,11 @@ pub const EpochCache = struct { const validator_count = validators.len; - // syncPubkeys here to ensure EpochCacheImmutableData is popualted before computing the rest of caches - // - computeSyncCommitteeCache() needs a fully populated pubkey2index cache + // Sync before computing the remaining caches: sync committee indexing + // needs a fully populated pubkey cache. const skip_sync_pubkeys = if (option) |opt| opt.skip_sync_pubkeys else false; if (!skip_sync_pubkeys) { - try syncPubkeys(allocator, validators, pubkey_to_index, index_to_pubkey); + try pubkey_cache.syncPubkeys(io, validators); } const effective_balance_increments_rc = try initEffectiveBalanceIncrementsRc(allocator, validator_count); @@ -408,7 +405,8 @@ pub const EpochCache = struct { const current_sync_committee_indexed = try initCurrentSyncCommitteeCacheRc( allocator, state, - pubkey_to_index, + pubkey_cache, + io, skip_sync_committee_cache, ); errdefer current_sync_committee_indexed.unref(); @@ -416,7 +414,8 @@ pub const EpochCache = struct { const next_sync_committee_indexed = try initNextSyncCommitteeCacheRc( allocator, state, - pubkey_to_index, + pubkey_cache, + io, skip_sync_committee_cache, ); errdefer next_sync_committee_indexed.unref(); @@ -472,8 +471,7 @@ pub const EpochCache = struct { epoch_cache_ptr.* = .{ .allocator = allocator, .config = config, - .pubkey_to_index = pubkey_to_index, - .index_to_pubkey = index_to_pubkey, + .pubkey_cache = pubkey_cache, .proposers = proposers, // On first epoch, set to null to prevent unnecessary work since this is only used for metrics .proposers_prev_epoch = null, @@ -517,7 +515,7 @@ pub const EpochCache = struct { } pub fn deinit(self: *EpochCache) void { - // pubkey_to_index and index_to_pubkey are shared across applications, EpochCache does not own this field so should not deinit() + // EpochCache does not own the application-wide pubkey cache. // unref the epoch shufflings self.previous_shuffling.unref(); @@ -538,8 +536,7 @@ pub const EpochCache = struct { .allocator = allocator, .config = self.config, // Common append-only structures shared with all states, no need to clone - .pubkey_to_index = self.pubkey_to_index, - .index_to_pubkey = self.index_to_pubkey, + .pubkey_cache = self.pubkey_cache, // Immutable data .proposers = self.proposers, .proposers_prev_epoch = self.proposers_prev_epoch, @@ -790,12 +787,6 @@ pub const EpochCache = struct { return error.EpochTooFar; } - // TODO: getBeaconProposers - can access directly? - - // TODO: getBeaconProposersNextEpoch - may not needed post-fulu - - // TODO: do we need getBeaconCommittees? in validateAttestationElectra we do a for loop over committee_indices and call getBeaconProposer() instead - /// consumer takes ownership of the returned indexed attestation /// hence it needs to deinit attesting_indices inside pub fn computeIndexedAttestationPhase0(self: *const EpochCache, attestation: *const types.phase0.Attestation.Type, out: *types.phase0.IndexedAttestation.Type) !void { @@ -900,28 +891,6 @@ pub const EpochCache = struct { return isAggregatorFromCommitteeLength(committee.length, slot_signature); } - pub fn getValidatorIndex(self: *const EpochCache, pubkey: *const types.primitive.BLSPubkey.Type) ?ValidatorIndex { - return self.pubkey_to_index.get(pubkey.*); - } - - /// Sets `index` at `PublicKey` within the index to pubkey map and allocates and puts a new `PublicKey` at `index` within the set of validators. - pub fn addPubkey(self: *EpochCache, index: ValidatorIndex, pubkey: *const types.primitive.BLSPubkey.Type) !void { - std.debug.assert(index <= self.index_to_pubkey.items.len); - const appending = index == self.index_to_pubkey.items.len; - - const public_key = try bls.PublicKey.uncompress(pubkey); - try self.pubkey_to_index.ensureUnusedCapacity(1); - if (appending) try self.index_to_pubkey.ensureUnusedCapacity(self.allocator, 1); - - self.pubkey_to_index.putAssumeCapacity(pubkey.*, index); - if (appending) { - self.index_to_pubkey.appendAssumeCapacity(public_key); - } else { - self.index_to_pubkey.items[index] = public_key; - } - } - - // TODO: getBeaconCommittee pub fn getShufflingAtSlotOrNull(self: *const EpochCache, slot: Slot) ?*const EpochShuffling { const epoch = computeEpochAtSlot(slot); return self.getShufflingAtEpochOrNull(epoch); diff --git a/src/state_transition/cache/epoch_transition_cache.zig b/src/state_transition/cache/epoch_transition_cache.zig index ae54fd6b7..f3cd80ca9 100644 --- a/src/state_transition/cache/epoch_transition_cache.zig +++ b/src/state_transition/cache/epoch_transition_cache.zig @@ -15,7 +15,6 @@ const computeEpochShuffling = @import("../utils/epoch_shuffling.zig").computeEpo const TestCachedBeaconState = @import("../test_utils/root.zig").TestCachedBeaconState; const upgradeStateToFulu = @import("../slot/upgrade_state_to_fulu.zig").upgradeStateToFulu; -const deinitStateTransition = @import("../root.zig").deinitStateTransition; const attester_status = @import("../utils/attester_status.zig"); const FLAG_CURR_HEAD_ATTESTER = attester_status.FLAG_CURR_HEAD_ATTESTER; @@ -62,7 +61,6 @@ const ReusedEpochTransitionCache = struct { flags: U8Array, - // TODO: nextShufflingDecisionRoot, is it necessary without ShufflingCache? next_epoch_shuffling_active_validator_indices: std.ArrayList(ValidatorIndex), is_compounding_validator_arr: BoolArray, @@ -208,16 +206,13 @@ pub const EpochTransitionCache = struct { balances: ?U64Array, next_shuffling_active_indices: []const ValidatorIndex, next_shuffling: ?*EpochShuffling, - // TODO: nextShufflingDecisionRoot may not needed as we don't use ShufflingCache next_epoch_total_active_balance_by_increment: u64, next_epoch_payload_timeliness_committees: ?PtcCommittees, - // TODO: asyncShufflingCalculation may not needed as we don't use ShufflingCache // these are borrowed from ReusedEpochTransitionCache is_active_prev_epoch: []const bool, is_active_curr_epoch: []const bool, is_active_next_epoch: []const bool, - // TODO: no need EpochTransitionCacheOpts for zig version // this is the same to beforeProcessEpoch in typesript version pub fn init( allocator: Allocator, @@ -675,5 +670,5 @@ test "EpochTransitionCache.beforeProcessEpoch" { defer epoch_transition_cache.deinit(allocator); } - deinitStateTransition(std.testing.io); + deinitReusedEpochTransitionCache(std.testing.io); } diff --git a/src/state_transition/cache/pkix.zig b/src/state_transition/cache/pkix.zig new file mode 100644 index 000000000..91284080c --- /dev/null +++ b/src/state_transition/cache/pkix.zig @@ -0,0 +1,262 @@ +const std = @import("std"); +const builtin = @import("builtin"); +const bls = @import("bls"); +const pubkey_cache = @import("pubkey_cache.zig"); +const PubkeyCache = pubkey_cache.PubkeyCache; + +const magic = "PKIX".*; +const format_version: u32 = 5; +const payload_checksum_seed: u64 = 0; +const header_checksum_seed: u64 = 0x504b495848445235; + +const Header = extern struct { + magic: [4]u8, + version: u32, + abi_fingerprint: u64, + entry_count: u32, + cache_capacity: u32, + payload_checksum: u64, + header_checksum: u64, + + fn init(entry_count: u32, cache_capacity: u32) Header { + return .{ + .magic = magic, + .version = format_version, + .abi_fingerprint = abi_fingerprint, + .entry_count = entry_count, + .cache_capacity = cache_capacity, + .payload_checksum = 0, + .header_checksum = 0, + }; + } + + fn finish( + self: *Header, + key_bytes: []const u8, + affine_bytes: []const u8, + ) void { + var checksum = std.hash.XxHash3.init(payload_checksum_seed); + checksum.update(key_bytes); + checksum.update(affine_bytes); + self.payload_checksum = checksum.final(); + self.header_checksum = headerChecksum(self); + } + + fn decode(encoded: *const [@sizeOf(Header)]u8) !Header { + const header = std.mem.bytesToValue(Header, encoded); + if (!std.mem.eql(u8, &header.magic, &magic)) { + return error.InvalidPkixMagic; + } + if (header.version != format_version) { + return error.UnsupportedPkixVersion; + } + if (header.abi_fingerprint != abi_fingerprint) { + return error.IncompatiblePkixAbi; + } + if (header.header_checksum != headerChecksum(&header)) { + return error.InvalidPkixHeaderChecksum; + } + return header; + } +}; +const PkixHeader = Header; + +const header_size = @sizeOf(Header); + +const AffinePoint = @TypeOf(@as(bls.PublicKey, undefined).point); +const BaseField = @TypeOf(@as(AffinePoint, undefined).x); +const Limbs = @TypeOf(@as(BaseField, undefined).l); + +// Raw affine bytes are safe only while every byte belongs to initialized BLST +// limbs. +comptime { + if (@offsetOf(bls.PublicKey, "point") != 0 or + @sizeOf(bls.PublicKey) != @sizeOf(AffinePoint) or + @offsetOf(AffinePoint, "x") != 0 or + @offsetOf(AffinePoint, "y") != @sizeOf(BaseField) or + @sizeOf(AffinePoint) != 2 * @sizeOf(BaseField) or + @offsetOf(BaseField, "l") != 0 or + @sizeOf(BaseField) != @sizeOf(Limbs)) + { + @compileError("PKIX requires a padding-free BLST affine layout"); + } + if (@offsetOf(Header, "magic") != 0 or + @offsetOf(Header, "version") != 4 or + @offsetOf(Header, "abi_fingerprint") != 8 or + @offsetOf(Header, "entry_count") != 16 or + @offsetOf(Header, "cache_capacity") != 20 or + @offsetOf(Header, "payload_checksum") != 24 or + @offsetOf(Header, "header_checksum") != 32 or + @sizeOf(Header) != 40) + { + @compileError("unexpected PKIX header layout"); + } +} + +const abi_description = std.fmt.comptimePrint( + "version={d};zig={s};arch={s};os={s};abi={s};endian={s};" ++ + "header={d};pubkey={d}/{d}/{d};affine={d}/{d}/{d};" ++ + "fp={d}/{d};limbs={d}/{d};", + .{ + format_version, + builtin.zig_version_string, + @tagName(builtin.target.cpu.arch), + @tagName(builtin.target.os.tag), + @tagName(builtin.target.abi), + @tagName(builtin.target.cpu.arch.endian()), + @sizeOf(Header), + @sizeOf(bls.PublicKey), + @alignOf(bls.PublicKey), + @offsetOf(bls.PublicKey, "point"), + @sizeOf(AffinePoint), + @alignOf(AffinePoint), + @offsetOf(AffinePoint, "y"), + @sizeOf(BaseField), + @alignOf(BaseField), + @sizeOf(Limbs), + @alignOf(Limbs), + }, +); + +const abi_fingerprint = std.hash.XxHash3.hash( + payload_checksum_seed, + abi_description, +); + +fn payloadSize(entry_count: u32) usize { + const count: usize = @intCast(entry_count); + return count * (@sizeOf([48]u8) + @sizeOf(bls.PublicKey)); +} + +fn payloadChecksum(payload: []const u8) u64 { + return std.hash.XxHash3.hash(payload_checksum_seed, payload); +} + +fn headerChecksum(header: *const Header) u64 { + return std.hash.XxHash3.hash( + header_checksum_seed, + std.mem.asBytes(header)[0..@offsetOf(Header, "header_checksum")], + ); +} + +fn readAndHash( + reader: *std.Io.Reader, + checksum: *std.hash.XxHash3, + bytes: []u8, +) !void { + reader.readSliceAll(bytes) catch |err| switch (err) { + error.EndOfStream => return error.InvalidPkixPayload, + else => |e| return e, + }; + checksum.update(bytes); +} + +/// Write a native, ABI-locked PKIX snapshot. +/// +/// The payload contains insertion-ordered compressed keys followed by affine +/// public keys. Runtime hash state is rebuilt on load and is never persisted. +/// The shared lock spans output, so the writer must not mutate the cache. +pub fn save(cache: *PubkeyCache, io: std.Io, writer: *std.Io.Writer) !void { + try cache.lock.lockShared(io); + defer cache.lock.unlockShared(io); + + const entry_count: u32 = @intCast(cache.entries.count()); + const cache_capacity: u32 = @intCast(cache.entries.capacity()); + std.debug.assert(entry_count <= cache_capacity); + + const key_bytes = std.mem.sliceAsBytes(cache.entries.keys()); + const affine_bytes = std.mem.sliceAsBytes(cache.entries.values()); + var header = Header.init(entry_count, cache_capacity); + header.finish(key_bytes, affine_bytes); + + try writer.writeAll(std.mem.asBytes(&header)); + try writer.writeAll(key_bytes); + try writer.writeAll(affine_bytes); +} + +/// Load a native PKIX snapshot into a fresh cache. +/// +/// PKIX checks framing, ABI compatibility, bounds, and checksum, but does not +/// authenticate or semantically validate entries. Callers must provide an +/// application-owned file from the intended network. +pub fn load( + allocator: std.mem.Allocator, + io: std.Io, + reader: *std.Io.Reader, + file_size: u64, + max_capacity: usize, +) !PubkeyCache { + if (file_size < header_size) return error.InvalidPkixHeader; + const encoded_header = reader.takeArray(header_size) catch |err| switch (err) { + error.EndOfStream => return error.InvalidPkixHeader, + else => |e| return e, + }; + const header = try Header.decode(encoded_header); + + if (header.entry_count > header.cache_capacity) { + return error.InvalidPkixHeader; + } + const entry_count = std.math.cast(usize, header.entry_count) orelse + return error.InvalidPkixHeader; + if (entry_count > max_capacity) return error.PkixCapacityLimitExceeded; + if (entry_count > pubkey_cache.max_capacity) return error.InvalidPkixHeader; + + const encoded_capacity = std.math.cast(usize, header.cache_capacity) orelse + return error.InvalidPkixHeader; + const cache_capacity = @min( + encoded_capacity, + max_capacity, + pubkey_cache.max_capacity, + ); + const payload_size = payloadSize(header.entry_count); + const expected_file_size: u64 = @intCast(header_size + payload_size); + if (file_size != expected_file_size) return error.InvalidPkixHeader; + + // An empty payload can be checked before its reserved capacity is allocated. + if (payload_size == 0 and header.payload_checksum != payloadChecksum(&.{})) { + return error.InvalidPkixChecksum; + } + + var cache = PubkeyCache.init(allocator, io); + errdefer cache.deinit(); + + if (cache_capacity != 0) { + try cache.entries.entries.setCapacity(allocator, cache_capacity); + } + try cache.entries.entries.resize(allocator, header.entry_count); + + const key_bytes = std.mem.sliceAsBytes(cache.entries.keys()); + const affine_bytes = std.mem.sliceAsBytes(cache.entries.values()); + var checksum = std.hash.XxHash3.init(payload_checksum_seed); + try readAndHash(reader, &checksum, key_bytes); + try readAndHash(reader, &checksum, affine_bytes); + if (checksum.final() != header.payload_checksum) { + return error.InvalidPkixChecksum; + } + + try cache.entries.reIndexContext(allocator, cache.hashContext()); + return cache; +} + +/// Install staged PKIX contents without replacing the live cache or its locks. +/// `staged` must be exclusively owned. Its allocator backing must remain valid +/// for the destination cache's lifetime after transfer. +/// `staged` remains valid and empty on success. +pub fn install(cache: *PubkeyCache, io: std.Io, staged: *PubkeyCache) !void { + std.debug.assert(cache != staged); + + try cache.lock.lock(io); + defer cache.lock.unlock(io); + + std.mem.swap(@TypeOf(cache.allocator), &cache.allocator, &staged.allocator); + std.mem.swap(@TypeOf(cache.entries), &cache.entries, &staged.entries); + std.mem.swap(@TypeOf(cache.hash_key), &cache.hash_key, &staged.hash_key); + staged.entries.deinit(staged.allocator); + staged.entries = .empty; +} + +pub const testing = if (builtin.is_test) struct { + pub const Header = PkixHeader; + pub const payload_seed = payload_checksum_seed; + pub const header_seed = header_checksum_seed; +} else struct {}; diff --git a/src/state_transition/cache/pkix_test.zig b/src/state_transition/cache/pkix_test.zig new file mode 100644 index 000000000..a5c3786cd --- /dev/null +++ b/src/state_transition/cache/pkix_test.zig @@ -0,0 +1,348 @@ +const std = @import("std"); +const types = @import("consensus_types"); +const pubkey_cache = @import("pubkey_cache.zig"); +const PubkeyCache = pubkey_cache.PubkeyCache; +const pkix = @import("pkix.zig"); +const PkixHeader = pkix.testing.Header; +const header_size = @sizeOf(PkixHeader); +const interop = @import("../test_utils/interop_pubkeys.zig"); +const testing = std.testing; + +fn loadPkixForTest( + allocator: std.mem.Allocator, + reader: *std.Io.Reader, + file_size: u64, +) !PubkeyCache { + return pkix.load( + allocator, + testing.io, + reader, + file_size, + std.math.maxInt(usize), + ); +} + +fn encodePkixForTest(allocator: std.mem.Allocator, cache: *PubkeyCache) ![]u8 { + var allocating_writer: std.Io.Writer.Allocating = .init(allocator); + errdefer allocating_writer.deinit(); + try pkix.save(cache, testing.io, &allocating_writer.writer); + return allocating_writer.toOwnedSlice(); +} + +fn updatePkixChecksumForTest(bytes: []u8) void { + var header = readHeaderForTest(bytes); + header.payload_checksum = std.hash.XxHash3.hash( + pkix.testing.payload_seed, + bytes[header_size..], + ); + updateHeaderChecksumForTest(&header); + writeHeaderForTest(bytes, header); +} + +fn readHeaderForTest(bytes: []const u8) PkixHeader { + std.debug.assert(bytes.len >= header_size); + return std.mem.bytesToValue(PkixHeader, bytes[0..header_size]); +} + +fn writeHeaderForTest(bytes: []u8, header: PkixHeader) void { + std.debug.assert(bytes.len >= header_size); + @memcpy(bytes[0..header_size], std.mem.asBytes(&header)); +} + +fn updateHeaderChecksumForTest(header: *PkixHeader) void { + header.header_checksum = std.hash.XxHash3.hash( + pkix.testing.header_seed, + std.mem.asBytes(header)[0..@offsetOf(PkixHeader, "header_checksum")], + ); +} + +fn appendPubkeys( + cache: *PubkeyCache, + pubkeys: []const types.primitive.BLSPubkey.Type, +) !void { + for (pubkeys, 0..) |pubkey, index| { + try cache.append(testing.io, pubkey, index); + } +} + +test "PKIX round trip preserves populated and empty caches" { + const allocator = testing.allocator; + var pubkeys: [3]types.primitive.BLSPubkey.Type = undefined; + try interop.interopPubkeysCached(pubkeys.len, &pubkeys); + + var source = try PubkeyCache.initCapacity(allocator, testing.io, 8); + defer source.deinit(); + source.hash_key = @splat(0x11); + try appendPubkeys(&source, &pubkeys); + + var same_entries = try PubkeyCache.initCapacity(allocator, testing.io, 8); + defer same_entries.deinit(); + same_entries.hash_key = @splat(0xee); + try appendPubkeys(&same_entries, &pubkeys); + + const encoded = try encodePkixForTest(allocator, &source); + defer allocator.free(encoded); + const same_entries_encoded = try encodePkixForTest(allocator, &same_entries); + defer allocator.free(same_entries_encoded); + try testing.expectEqualSlices(u8, encoded, same_entries_encoded); + try testing.expectEqual( + std.hash.XxHash3.hash(pkix.testing.payload_seed, encoded[header_size..]), + readHeaderForTest(encoded).payload_checksum, + ); + + var reader = std.Io.Reader.fixed(encoded); + var loaded = try loadPkixForTest(allocator, &reader, encoded.len); + defer loaded.deinit(); + try testing.expectEqual(source.count(testing.io), loaded.count(testing.io)); + try testing.expectEqual(source.capacity(testing.io), loaded.capacity(testing.io)); + for (pubkeys, 0..) |pubkey, index| { + try testing.expectEqual( + @as(u64, @intCast(index)), + loaded.get(testing.io, pubkey).?, + ); + const compressed = loaded.getPubkey(testing.io, index).?.compress(); + try testing.expectEqualSlices(u8, &pubkey, &compressed); + } + + var empty_source = try PubkeyCache.initCapacity(allocator, testing.io, 32); + defer empty_source.deinit(); + const empty_encoded = try encodePkixForTest(allocator, &empty_source); + defer allocator.free(empty_encoded); + try testing.expectEqual(header_size, empty_encoded.len); + + var empty_reader = std.Io.Reader.fixed(empty_encoded); + var empty_loaded = try loadPkixForTest(allocator, &empty_reader, empty_encoded.len); + defer empty_loaded.deinit(); + try testing.expectEqual(@as(u32, 0), empty_loaded.count(testing.io)); + try testing.expectEqual( + empty_source.capacity(testing.io), + empty_loaded.capacity(testing.io), + ); +} + +test "PKIX install replaces contents across allocators and remains usable" { + const allocator = testing.allocator; + var pubkeys: [3]types.primitive.BLSPubkey.Type = undefined; + try interop.interopPubkeysCached(pubkeys.len, &pubkeys); + + var source = try PubkeyCache.initCapacity(allocator, testing.io, 4); + defer source.deinit(); + try appendPubkeys(&source, pubkeys[1..]); + + const encoded = try encodePkixForTest(allocator, &source); + defer allocator.free(encoded); + var reader = std.Io.Reader.fixed(encoded); + var staged = try loadPkixForTest( + std.heap.page_allocator, + &reader, + encoded.len, + ); + defer staged.deinit(); + + var live = try PubkeyCache.initCapacity(allocator, testing.io, 1); + defer live.deinit(); + try live.append(testing.io, pubkeys[0], 0); + + try pkix.install(&live, testing.io, &staged); + + try testing.expectEqual(@as(u32, 0), staged.count(testing.io)); + try testing.expectEqual(@as(u32, 2), live.count(testing.io)); + try testing.expectEqual(@as(?u64, null), live.get(testing.io, pubkeys[0])); + try testing.expectEqual(@as(u64, 0), live.get(testing.io, pubkeys[1]).?); + try testing.expectEqual(@as(u64, 1), live.get(testing.io, pubkeys[2]).?); + + try live.append(testing.io, pubkeys[0], 2); + try testing.expectEqual(@as(u64, 2), live.get(testing.io, pubkeys[0]).?); +} + +test "PKIX rejects payload corruption and inexact reads" { + const allocator = testing.allocator; + var pubkeys: [1]types.primitive.BLSPubkey.Type = undefined; + try interop.interopPubkeysCached(pubkeys.len, &pubkeys); + var source = try PubkeyCache.initCapacity(allocator, testing.io, pubkeys.len); + defer source.deinit(); + try source.append(testing.io, pubkeys[0], 0); + + const encoded = try encodePkixForTest(allocator, &source); + defer allocator.free(encoded); + + encoded[encoded.len - 1] ^= 1; + var corrupted_reader = std.Io.Reader.fixed(encoded); + try testing.expectError( + error.InvalidPkixChecksum, + loadPkixForTest(allocator, &corrupted_reader, encoded.len), + ); + encoded[encoded.len - 1] ^= 1; + + var truncated_reader = std.Io.Reader.fixed(encoded); + try testing.expectError( + error.InvalidPkixHeader, + loadPkixForTest(allocator, &truncated_reader, encoded.len - 1), + ); + var trailing_reader = std.Io.Reader.fixed(encoded); + try testing.expectError( + error.InvalidPkixHeader, + loadPkixForTest(allocator, &trailing_reader, encoded.len + 1), + ); + + // The file may be truncated after its size is read. + var short_reader = std.Io.Reader.fixed(encoded[0 .. encoded.len - 1]); + try testing.expectError( + error.InvalidPkixPayload, + loadPkixForTest(allocator, &short_reader, encoded.len), + ); + + var empty_source = try PubkeyCache.initCapacity(allocator, testing.io, 32); + defer empty_source.deinit(); + const empty_encoded = try encodePkixForTest(allocator, &empty_source); + defer allocator.free(empty_encoded); + + var empty_header = readHeaderForTest(empty_encoded); + empty_header.payload_checksum ^= 1; + updateHeaderChecksumForTest(&empty_header); + writeHeaderForTest(empty_encoded, empty_header); + var failing = testing.FailingAllocator.init(allocator, .{ .fail_index = 0 }); + var empty_reader = std.Io.Reader.fixed(empty_encoded); + try testing.expectError( + error.InvalidPkixChecksum, + pkix.load( + failing.allocator(), + testing.io, + &empty_reader, + empty_encoded.len, + std.math.maxInt(usize), + ), + ); +} + +test "PKIX rejects incompatible and malformed headers" { + const allocator = testing.allocator; + var source = PubkeyCache.init(allocator, testing.io); + defer source.deinit(); + const encoded = try encodePkixForTest(allocator, &source); + defer allocator.free(encoded); + + const Mutation = struct { + offset: usize, + expected_error: anyerror, + }; + const mutations = [_]Mutation{ + .{ + .offset = @offsetOf(PkixHeader, "magic"), + .expected_error = error.InvalidPkixMagic, + }, + .{ + .offset = @offsetOf(PkixHeader, "version"), + .expected_error = error.UnsupportedPkixVersion, + }, + .{ + .offset = @offsetOf(PkixHeader, "abi_fingerprint"), + .expected_error = error.IncompatiblePkixAbi, + }, + }; + + for (mutations) |mutation| { + encoded[mutation.offset] ^= 1; + updatePkixChecksumForTest(encoded); + var reader = std.Io.Reader.fixed(encoded); + try testing.expectError( + mutation.expected_error, + loadPkixForTest(allocator, &reader, encoded.len), + ); + encoded[mutation.offset] ^= 1; + updatePkixChecksumForTest(encoded); + } + + var header = readHeaderForTest(encoded); + header.cache_capacity = 1 << 30; + writeHeaderForTest(encoded, header); + var failing = testing.FailingAllocator.init(allocator, .{ .fail_index = 0 }); + var checksum_reader = std.Io.Reader.fixed(encoded); + try testing.expectError( + error.InvalidPkixHeaderChecksum, + pkix.load( + failing.allocator(), + testing.io, + &checksum_reader, + encoded.len, + std.math.maxInt(usize), + ), + ); + + header = readHeaderForTest(encoded); + header.cache_capacity = 0; + header.entry_count = 1; + updateHeaderChecksumForTest(&header); + writeHeaderForTest(encoded, header); + var count_reader = std.Io.Reader.fixed(encoded); + try testing.expectError( + error.InvalidPkixHeader, + loadPkixForTest(allocator, &count_reader, encoded.len), + ); +} + +test "PKIX enforces the caller capacity limit before allocation" { + const allocator = testing.allocator; + var pubkeys: [3]types.primitive.BLSPubkey.Type = undefined; + try interop.interopPubkeysCached(pubkeys.len, &pubkeys); + + var source = try PubkeyCache.initCapacity(allocator, testing.io, 32); + defer source.deinit(); + try appendPubkeys(&source, &pubkeys); + + const encoded = try encodePkixForTest(allocator, &source); + defer allocator.free(encoded); + + var accepted_reader = std.Io.Reader.fixed(encoded); + var loaded = try pkix.load( + allocator, + testing.io, + &accepted_reader, + encoded.len, + pubkeys.len, + ); + defer loaded.deinit(); + try testing.expectEqual(@as(u32, pubkeys.len), loaded.count(testing.io)); + try testing.expectEqual(@as(usize, pubkeys.len), loaded.capacity(testing.io)); + for (pubkeys, 0..) |pubkey, index| { + try testing.expectEqual( + @as(u64, @intCast(index)), + loaded.get(testing.io, pubkey).?, + ); + } + + var failing = testing.FailingAllocator.init(allocator, .{ .fail_index = 0 }); + var rejected_reader = std.Io.Reader.fixed(encoded); + try testing.expectError( + error.PkixCapacityLimitExceeded, + pkix.load( + failing.allocator(), + testing.io, + &rejected_reader, + encoded.len, + pubkeys.len - 1, + ), + ); + + var empty_source = PubkeyCache.init(allocator, testing.io); + defer empty_source.deinit(); + const empty_encoded = try encodePkixForTest(allocator, &empty_source); + defer allocator.free(empty_encoded); + var empty_header = readHeaderForTest(empty_encoded); + empty_header.cache_capacity = 1 << 30; + updateHeaderChecksumForTest(&empty_header); + writeHeaderForTest(empty_encoded, empty_header); + + var empty_failing = testing.FailingAllocator.init(allocator, .{ .fail_index = 0 }); + var empty_reader = std.Io.Reader.fixed(empty_encoded); + var empty_loaded = try pkix.load( + empty_failing.allocator(), + testing.io, + &empty_reader, + empty_encoded.len, + 0, + ); + defer empty_loaded.deinit(); + try testing.expectEqual(@as(u32, 0), empty_loaded.count(testing.io)); + try testing.expectEqual(@as(usize, 0), empty_loaded.capacity(testing.io)); +} diff --git a/src/state_transition/cache/pubkey_cache.zig b/src/state_transition/cache/pubkey_cache.zig index 08b187ae2..3d4b51282 100644 --- a/src/state_transition/cache/pubkey_cache.zig +++ b/src/state_transition/cache/pubkey_cache.zig @@ -3,254 +3,364 @@ const bls = @import("bls"); const types = @import("consensus_types"); const Validator = types.phase0.Validator.Type; -/// Map from pubkey to validator index -pub const PubkeyIndexMap = std.AutoHashMap([48]u8, u64); +const HashKey = [std.hash.SipHash64(1, 3).key_length]u8; +pub const uncompress_batch_size = 1000; -/// Map from validator index to pubkey -pub const Index2PubkeyCache = std.ArrayList(bls.PublicKey); - -/// Populate `pubkey_to_index` and `index_to_pubkey` caches from validators list. -/// -/// Runs serially on the current thread. For parallel decompression over a -/// worker pool, see `syncPubkeysParallel`. -pub fn syncPubkeys( - allocator: std.mem.Allocator, +fn uncompressBatch( validators: []const *const Validator, - pubkey_to_index: *PubkeyIndexMap, - index_to_pubkey: *Index2PubkeyCache, -) !void { - const old_len = index_to_pubkey.items.len; - if (pubkey_to_index.count() != old_len) { - return error.InconsistentCache; - } + prepared: []bls.PublicKey, + batch_error: *?bls.BlstError, +) void { + std.debug.assert(validators.len == prepared.len); + std.debug.assert(validators.len <= uncompress_batch_size); - const new_count = validators.len; - if (new_count <= old_len) { - return; + for (validators, prepared) |validator, *affine| { + affine.* = bls.PublicKey.uncompress(&validator.pubkey) catch |err| { + batch_error.* = err; + return; + }; } +} - try index_to_pubkey.resize(allocator, new_count); - try pubkey_to_index.ensureTotalCapacity(@intCast(new_count)); +/// Keyed hashing prevents an untrusted set of compressed pubkeys from forcing +/// pathological probe chains in the reverse lookup. +pub const PubkeyHashContext = struct { + hash_key: HashKey, - for (old_len..new_count) |i| { - const pubkey = &validators[i].pubkey; - pubkey_to_index.putAssumeCapacity(pubkey.*, @intCast(i)); - index_to_pubkey.items[i] = try bls.PublicKey.uncompress(pubkey); + pub fn hash(self: PubkeyHashContext, pubkey: [48]u8) u32 { + return @truncate(std.hash.SipHash64(1, 3).toInt(&pubkey, &self.hash_key)); } -} -fn uncompressPubkeys( - start_index: usize, - end_index_exclusive: usize, - validators: []const *const Validator, - index_to_pubkey: *Index2PubkeyCache, - uncompress_error: *std.atomic.Value(bool), -) void { - std.debug.assert(start_index <= end_index_exclusive); - std.debug.assert(end_index_exclusive <= validators.len); - std.debug.assert(end_index_exclusive <= index_to_pubkey.items.len); - - for (start_index..end_index_exclusive) |i| { - if (uncompress_error.load(.monotonic)) return; - const pubkey = &validators[i].pubkey; - index_to_pubkey.items[i] = bls.PublicKey.uncompress(pubkey) catch { - uncompress_error.store(true, .release); - return; - }; + pub fn eql( + _: PubkeyHashContext, + lhs: [48]u8, + rhs: [48]u8, + _: usize, + ) bool { + return std.mem.eql(u8, &lhs, &rhs); } -} - -/// Populate `pubkey_to_index` and `index_to_pubkey` caches from validators list, -/// parallelizing BLS pubkey decompression across the `io` executor's worker pool -/// via `std.Io.Group.concurrent`. The `pubkey_to_index` HashMap is updated -/// single-threaded at the end (HashMap is not thread-safe). -pub fn syncPubkeysParallel( +}; + +/// The dense entry index is the validator index. Production code only appends, +/// so insertion order permanently supplies both lookup directions: +/// compressed pubkey -> validator index and validator index -> affine pubkey. +pub const PubkeyMap = std.array_hash_map.Custom( + [48]u8, + bls.PublicKey, + PubkeyHashContext, + true, +); + +// Bound capacity by both the map's u32 index space and the largest dense +// allocation that can grow without overflowing usize. +const dense_capacity_limit = + std.math.maxInt(usize) / PubkeyMap.DataList.capacityInBytes(1); +pub const max_capacity: usize = @min( + @as(usize, std.math.maxInt(u32)), + dense_capacity_limit / 2, +); + +/// Append-only pubkey cache. +/// +/// ArrayHashMap storage may move when it grows, so every access is protected by +/// the cache-owned read/write lock. Readers copy values out before releasing +/// the shared lock; no pointer into the map escapes. Production mutations only +/// append (and occasionally grow). `clear` exists for tests/API compatibility +/// and retains all allocated capacity. +pub const PubkeyCache = struct { allocator: std.mem.Allocator, - io: std.Io, - validators: []const *const Validator, - pubkey_to_index: *PubkeyIndexMap, - index_to_pubkey: *Index2PubkeyCache, -) !void { - const old_len = index_to_pubkey.items.len; - if (pubkey_to_index.count() != old_len) { - return error.InconsistentCache; + entries: PubkeyMap, + hash_key: HashKey, + /// Guards movable map storage and serializes cache allocator use. + lock: std.Io.RwLock, + + pub fn init(allocator: std.mem.Allocator, io: std.Io) PubkeyCache { + var hash_key: HashKey = undefined; + io.random(&hash_key); + return .{ + .allocator = allocator, + .entries = .empty, + .hash_key = hash_key, + .lock = .init, + }; } - const new_count = validators.len; - if (new_count <= old_len) { - return; + pub fn initCapacity( + allocator: std.mem.Allocator, + io: std.Io, + initial_capacity: usize, + ) !PubkeyCache { + try validateCapacity(initial_capacity); + + var self = init(allocator, io); + errdefer self.deinit(); + try self.ensureTotalCapacityExactUnlocked(initial_capacity); + return self; } - try index_to_pubkey.resize(allocator, new_count); - errdefer index_to_pubkey.shrinkRetainingCapacity(old_len); - - try pubkey_to_index.ensureTotalCapacity(@intCast(new_count)); - - var uncompress_error = std.atomic.Value(bool).init(false); - - var group: std.Io.Group = .init; - errdefer group.cancel(io); - - const batch_size = 1000; - var i = old_len; - while (i < new_count) : (i += batch_size) { - const end = @min(i + batch_size, new_count); - try group.concurrent(io, uncompressPubkeys, .{ - i, - end, - validators, - index_to_pubkey, - &uncompress_error, - }); + /// The owner must exclude concurrent users before destroying the cache. + pub fn deinit(self: *PubkeyCache) void { + self.entries.deinit(self.allocator); + self.* = undefined; } - try group.await(io); - - if (uncompress_error.load(.acquire)) { - return error.InvalidPubkey; + pub fn hashContext(self: *const PubkeyCache) PubkeyHashContext { + return .{ .hash_key = self.hash_key }; } - // HashMap updates must run single-threaded. - for (old_len..new_count) |j| { - pubkey_to_index.putAssumeCapacity(validators[j].pubkey, @intCast(j)); + /// Reserve exactly `new_capacity` entries when growing dense storage. + /// Existing larger capacity is retained. + pub fn ensureTotalCapacity( + self: *PubkeyCache, + io: std.Io, + new_capacity: usize, + ) !void { + try validateCapacity(new_capacity); + try self.lock.lock(io); + defer self.lock.unlock(io); + + try self.ensureTotalCapacityExactUnlocked(new_capacity); } -} - -const testing = std.testing; -const interop = @import("../test_utils/interop_pubkeys.zig"); -test "syncPubkeys populates both caches" { - const allocator = testing.allocator; - const count = 4; - - var pubkeys: [count]types.primitive.BLSPubkey.Type = undefined; - try interop.interopPubkeysCached(count, &pubkeys); - - var validators: [count]Validator = undefined; - var validator_ptrs: [count]*const Validator = undefined; - for (0..count) |i| { - validators[i] = std.mem.zeroes(Validator); - validators[i].pubkey = pubkeys[i]; - validator_ptrs[i] = &validators[i]; + /// Test/reset API. Production operation is append-only and does not call + /// this method. Capacity is retained to avoid turning reset into a resize. + pub fn clear(self: *PubkeyCache, io: std.Io) !void { + try self.lock.lock(io); + defer self.lock.unlock(io); + self.entries.clearRetainingCapacity(); } - var pubkey_to_index = PubkeyIndexMap.init(allocator); - defer pubkey_to_index.deinit(); - var index_to_pubkey: Index2PubkeyCache = .empty; - defer index_to_pubkey.deinit(allocator); - - try syncPubkeys(allocator, &validator_ptrs, &pubkey_to_index, &index_to_pubkey); - - try testing.expectEqual(@as(usize, count), index_to_pubkey.items.len); - try testing.expectEqual(@as(u32, count), pubkey_to_index.count()); - - for (0..count) |i| { - const idx = pubkey_to_index.get(pubkeys[i]).?; - try testing.expectEqual(@as(u64, i), idx); + pub fn count(self: *const PubkeyCache, io: std.Io) u32 { + self.lockShared(io); + defer self.unlockShared(io); + return @intCast(self.entries.count()); } -} - -test "syncPubkeys incremental sync adds only new validators" { - const allocator = testing.allocator; - const initial_count = 2; - const total_count = 4; - var pubkeys: [total_count]types.primitive.BLSPubkey.Type = undefined; - try interop.interopPubkeysCached(total_count, &pubkeys); - - var validators: [total_count]Validator = undefined; - var validator_ptrs: [total_count]*const Validator = undefined; - for (0..total_count) |i| { - validators[i] = std.mem.zeroes(Validator); - validators[i].pubkey = pubkeys[i]; - validator_ptrs[i] = &validators[i]; + pub fn capacity(self: *const PubkeyCache, io: std.Io) usize { + self.lockShared(io); + defer self.unlockShared(io); + return self.entries.capacity(); } - var pubkey_to_index = PubkeyIndexMap.init(allocator); - defer pubkey_to_index.deinit(); - var index_to_pubkey: Index2PubkeyCache = .empty; - defer index_to_pubkey.deinit(allocator); - - try syncPubkeys(allocator, validator_ptrs[0..initial_count], &pubkey_to_index, &index_to_pubkey); - try testing.expectEqual(@as(usize, initial_count), index_to_pubkey.items.len); - - try syncPubkeys(allocator, &validator_ptrs, &pubkey_to_index, &index_to_pubkey); - try testing.expectEqual(@as(usize, total_count), index_to_pubkey.items.len); - try testing.expectEqual(@as(u32, total_count), pubkey_to_index.count()); - - for (0..total_count) |i| { - const idx = pubkey_to_index.get(pubkeys[i]).?; - try testing.expectEqual(@as(u64, i), idx); + /// Get the validator index for a compressed pubkey, if cached. + pub fn get(self: *const PubkeyCache, io: std.Io, pubkey: [48]u8) ?u64 { + self.lockShared(io); + defer self.unlockShared(io); + const index = self.entries.getIndexContext(pubkey, self.hashContext()) orelse + return null; + return @intCast(index); } -} - -test "syncPubkeys no-op when already synced" { - const allocator = testing.allocator; - const count = 2; - - var pubkeys: [count]types.primitive.BLSPubkey.Type = undefined; - try interop.interopPubkeysCached(count, &pubkeys); - var validators: [count]Validator = undefined; - var validator_ptrs: [count]*const Validator = undefined; - for (0..count) |i| { - validators[i] = std.mem.zeroes(Validator); - validators[i].pubkey = pubkeys[i]; - validator_ptrs[i] = &validators[i]; + /// Get the affine pubkey for a validator index. A value is returned so no + /// pointer into movable map storage escapes the shared lock. + pub fn getPubkey( + self: *const PubkeyCache, + io: std.Io, + index: u64, + ) ?bls.PublicKey { + self.lockShared(io); + defer self.unlockShared(io); + if (index >= self.entries.count()) return null; + return self.entries.values()[@intCast(index)]; } - var pubkey_to_index = PubkeyIndexMap.init(allocator); - defer pubkey_to_index.deinit(); - var index_to_pubkey: Index2PubkeyCache = .empty; - defer index_to_pubkey.deinit(allocator); - - try syncPubkeys(allocator, &validator_ptrs, &pubkey_to_index, &index_to_pubkey); - try syncPubkeys(allocator, &validator_ptrs, &pubkey_to_index, &index_to_pubkey); - try testing.expectEqual(@as(usize, count), index_to_pubkey.items.len); -} - -test "syncPubkeys no-op when validator count shrinks" { - const allocator = testing.allocator; - const initial_count = 4; - const shrunk_count = 2; - - var pubkeys: [initial_count]types.primitive.BLSPubkey.Type = undefined; - try interop.interopPubkeysCached(initial_count, &pubkeys); + /// Copy affine pubkeys for a batch of validator indices while holding one + /// shared lock. The output is left unchanged when an index is invalid. + pub fn getPubkeys( + self: *const PubkeyCache, + io: std.Io, + indices: []const u64, + out: []bls.PublicKey, + ) !void { + if (indices.len != out.len) return error.InvalidLength; + + self.lockShared(io); + defer self.unlockShared(io); + + const values = self.entries.values(); + for (indices) |index| { + if (index >= values.len) return error.InvalidIndex; + } + for (indices, out) |index, *pubkey| { + pubkey.* = values[@intCast(index)]; + } + } - var validators: [initial_count]Validator = undefined; - var validator_ptrs: [initial_count]*const Validator = undefined; - for (0..initial_count) |i| { - validators[i] = std.mem.zeroes(Validator); - validators[i].pubkey = pubkeys[i]; - validator_ptrs[i] = &validators[i]; + /// Resolve a batch of compressed pubkeys to validator indices while + /// holding one shared lock. + pub fn getValidatorIndices( + self: *const PubkeyCache, + io: std.Io, + pubkeys: []const [48]u8, + out: []u64, + ) !void { + if (pubkeys.len != out.len) return error.InvalidLength; + + self.lockShared(io); + defer self.unlockShared(io); + + const context = self.hashContext(); + for (pubkeys, out) |pubkey, *index| { + index.* = @intCast(self.entries.getIndexContext(pubkey, context) orelse + return error.PubkeyNotFound); + } } - var pubkey_to_index = PubkeyIndexMap.init(allocator); - defer pubkey_to_index.deinit(); - var index_to_pubkey: Index2PubkeyCache = .empty; - defer index_to_pubkey.deinit(allocator); + /// Aggregate the pubkeys at the requested validator indices. + pub fn aggregate( + self: *const PubkeyCache, + io: std.Io, + indices: []const u64, + ) !bls.PublicKey { + if (indices.len == 0) return error.InvalidLength; + + self.lockShared(io); + defer self.unlockShared(io); + + const values = self.entries.values(); + if (indices[0] >= values.len) return error.InvalidIndex; + var aggregate_pubkey = values[@intCast(indices[0])].toAggregate(); + for (indices[1..]) |index| { + if (index >= values.len) return error.InvalidIndex; + aggregate_pubkey.add(&values[@intCast(index)]); + } + return aggregate_pubkey.toPublicKey(); + } - try syncPubkeys(allocator, &validator_ptrs, &pubkey_to_index, &index_to_pubkey); - try syncPubkeys(allocator, validator_ptrs[0..shrunk_count], &pubkey_to_index, &index_to_pubkey); + /// Append a compressed pubkey at the next validator index. Supplying an + /// already-cached index is an idempotent consistency check. + pub fn append( + self: *PubkeyCache, + io: std.Io, + pubkey: [48]u8, + index: u64, + ) !void { + const affine = try bls.PublicKey.uncompress(&pubkey); + + try self.lock.lock(io); + defer self.lock.unlock(io); + + const current_len = self.entries.count(); + if (index < current_len) { + if (!std.mem.eql( + u8, + &self.entries.keys()[@intCast(index)], + &pubkey, + )) return error.ConflictingPubkey; + return; + } + if (index > current_len) return error.InvalidIndexToAppend; + if (self.entries.getIndexContext(pubkey, self.hashContext()) != null) { + return error.DuplicatePubkey; + } + + try validateCapacity(current_len + 1); + try self.ensureTotalCapacityAmortizedUnlocked(current_len + 1); + self.entries.putAssumeCapacityNoClobberContext( + pubkey, + affine, + self.hashContext(), + ); + } - try testing.expectEqual(@as(usize, initial_count), index_to_pubkey.items.len); - try testing.expectEqual(@as(u32, initial_count), pubkey_to_index.count()); -} + /// Populate the cache from the missing suffix of a validator list. Existing + /// entries are trusted as the application's unforkable singleton history. + pub fn syncPubkeys( + self: *PubkeyCache, + io: std.Io, + validators: []const *const Validator, + ) !void { + try validateCapacity(validators.len); + + self.lockShared(io); + const already_synced = validators.len <= self.entries.count(); + self.unlockShared(io); + if (already_synced) return; + + try self.lock.lock(io); + defer self.lock.unlock(io); + + const old_len = self.entries.count(); + if (validators.len <= old_len) return; + + const suffix = validators[old_len..]; + const prepared = try self.allocator.alloc(bls.PublicKey, suffix.len); + defer self.allocator.free(prepared); + + const batch_count = (suffix.len - 1) / uncompress_batch_size + 1; + const batch_errors = try self.allocator.alloc(?bls.BlstError, batch_count); + defer self.allocator.free(batch_errors); + @memset(batch_errors, null); + + var group: std.Io.Group = .init; + errdefer group.cancel(io); + // `async` bounds worker growth and runs excess batches on the caller. + for (batch_errors, 0..) |*batch_error, batch_index| { + const batch_start = batch_index * uncompress_batch_size; + const batch_end = @min(batch_start + uncompress_batch_size, suffix.len); + group.async(io, uncompressBatch, .{ + suffix[batch_start..batch_end], + prepared[batch_start..batch_end], + batch_error, + }); + } + try group.await(io); + + for (batch_errors) |batch_error| { + if (batch_error) |err| return err; + } + + try self.ensureTotalCapacityAmortizedUnlocked(validators.len); + + const context = self.hashContext(); + for (suffix, prepared) |validator, affine| { + const result = self.entries.getOrPutAssumeCapacityContext( + validator.pubkey, + context, + ); + if (result.found_existing) { + self.entries.shrinkRetainingCapacityContext(old_len, context); + return error.DuplicatePubkey; + } + result.value_ptr.* = affine; + } + } -test "syncPubkeys detects inconsistent cache" { - const allocator = testing.allocator; + fn validateCapacity(requested_capacity: usize) !void { + if (requested_capacity > max_capacity) return error.CapacityOverflow; + } - var pubkey_to_index = PubkeyIndexMap.init(allocator); - defer pubkey_to_index.deinit(); - var index_to_pubkey: Index2PubkeyCache = .empty; - defer index_to_pubkey.deinit(allocator); + fn ensureTotalCapacityAmortizedUnlocked( + self: *PubkeyCache, + new_capacity: usize, + ) !void { + try self.entries.ensureTotalCapacityContext( + self.allocator, + new_capacity, + self.hashContext(), + ); + } - const dummy_key = [_]u8{0} ** 48; - try pubkey_to_index.put(dummy_key, 0); + fn ensureTotalCapacityExactUnlocked( + self: *PubkeyCache, + new_capacity: usize, + ) !void { + if (new_capacity <= self.entries.capacity()) return; + if (new_capacity > self.entries.entries.capacity) { + try self.entries.entries.setCapacity(self.allocator, new_capacity); + } + try self.entries.ensureTotalCapacityContext( + self.allocator, + new_capacity, + self.hashContext(), + ); + } - var validators: [1]Validator = undefined; - validators[0] = std.mem.zeroes(Validator); - var validator_ptrs: [1]*const Validator = .{&validators[0]}; + fn lockShared(self: *const PubkeyCache, io: std.Io) void { + @constCast(&self.lock).lockSharedUncancelable(io); + } - try testing.expectError(error.InconsistentCache, syncPubkeys(allocator, &validator_ptrs, &pubkey_to_index, &index_to_pubkey)); -} + fn unlockShared(self: *const PubkeyCache, io: std.Io) void { + @constCast(&self.lock).unlockShared(io); + } +}; diff --git a/src/state_transition/cache/pubkey_cache_test.zig b/src/state_transition/cache/pubkey_cache_test.zig new file mode 100644 index 000000000..bb288bb54 --- /dev/null +++ b/src/state_transition/cache/pubkey_cache_test.zig @@ -0,0 +1,258 @@ +const std = @import("std"); +const bls = @import("bls"); +const types = @import("consensus_types"); +const Validator = types.phase0.Validator.Type; +const cache_module = @import("pubkey_cache.zig"); +const PubkeyCache = cache_module.PubkeyCache; +const testing = std.testing; +const interop = @import("../test_utils/interop_pubkeys.zig"); + +fn validatorsForPubkeys( + pubkeys: []const types.primitive.BLSPubkey.Type, + validators: []Validator, + validator_ptrs: []*const Validator, +) void { + std.debug.assert(pubkeys.len == validators.len); + std.debug.assert(pubkeys.len == validator_ptrs.len); + for (pubkeys, validators, validator_ptrs) |pubkey, *validator, *validator_ptr| { + validator.* = std.mem.zeroes(Validator); + validator.pubkey = pubkey; + validator_ptr.* = validator; + } +} + +test "syncPubkeys incrementally populates both lookup directions" { + const initial_count = 2; + const total_count = 16; + var pubkeys: [total_count]types.primitive.BLSPubkey.Type = undefined; + try interop.interopPubkeysCached(total_count, &pubkeys); + + var validators: [total_count]Validator = undefined; + var validator_ptrs: [total_count]*const Validator = undefined; + validatorsForPubkeys(&pubkeys, &validators, &validator_ptrs); + + var cache = PubkeyCache.init(testing.allocator, testing.io); + defer cache.deinit(); + + try cache.syncPubkeys(testing.io, validator_ptrs[0..initial_count]); + try cache.syncPubkeys(testing.io, &validator_ptrs); + + try testing.expectEqual(@as(u32, total_count), cache.count(testing.io)); + for (pubkeys, 0..) |pubkey, expected_index| { + try testing.expectEqual( + @as(u64, @intCast(expected_index)), + cache.get(testing.io, pubkey).?, + ); + const compressed = cache.getPubkey(testing.io, expected_index).?.compress(); + try testing.expectEqualSlices(u8, &pubkey, &compressed); + } +} + +test "syncPubkeys accepts a historical validator slice" { + var pubkeys: [4]types.primitive.BLSPubkey.Type = undefined; + try interop.interopPubkeysCached(pubkeys.len, &pubkeys); + + var validators: [4]Validator = undefined; + var validator_ptrs: [4]*const Validator = undefined; + validatorsForPubkeys(&pubkeys, &validators, &validator_ptrs); + + var cache = PubkeyCache.init(testing.allocator, testing.io); + defer cache.deinit(); + try cache.syncPubkeys(testing.io, &validator_ptrs); + + try cache.syncPubkeys(testing.io, validator_ptrs[0..2]); + + try testing.expectEqual(@as(u32, validators.len), cache.count(testing.io)); + try testing.expectEqual(@as(u64, 3), cache.get(testing.io, pubkeys[3]).?); + const future_pubkey = cache.getPubkey(testing.io, 3).?.compress(); + try testing.expectEqualSlices(u8, &pubkeys[3], &future_pubkey); +} + +test "syncPubkeys rejects a duplicate without publishing a partial suffix" { + var pubkeys: [2]types.primitive.BLSPubkey.Type = undefined; + try interop.interopPubkeysCached(pubkeys.len, &pubkeys); + + var validators = [_]Validator{ + std.mem.zeroes(Validator), + std.mem.zeroes(Validator), + std.mem.zeroes(Validator), + }; + validators[0].pubkey = pubkeys[0]; + validators[1].pubkey = pubkeys[1]; + validators[2].pubkey = pubkeys[1]; + const validator_ptrs = [_]*const Validator{ + &validators[0], + &validators[1], + &validators[2], + }; + + var cache = PubkeyCache.init(testing.allocator, testing.io); + defer cache.deinit(); + try cache.append(testing.io, pubkeys[0], 0); + + try testing.expectError( + error.DuplicatePubkey, + cache.syncPubkeys(testing.io, &validator_ptrs), + ); + try testing.expectEqual(@as(u32, 1), cache.count(testing.io)); + try testing.expectEqual(@as(?u64, null), cache.get(testing.io, pubkeys[1])); +} + +test "syncPubkeys rejects an invalid key without publishing across batches" { + const batch_size = cache_module.uncompress_batch_size; + var pubkeys: [2]types.primitive.BLSPubkey.Type = undefined; + try interop.interopPubkeysCached(pubkeys.len, &pubkeys); + + var validators = [_]Validator{ + std.mem.zeroes(Validator), + std.mem.zeroes(Validator), + std.mem.zeroes(Validator), + }; + validators[0].pubkey = pubkeys[0]; + validators[1].pubkey = pubkeys[1]; + validators[2].pubkey = std.mem.zeroes([48]u8); + var validator_ptrs: [batch_size + 2]*const Validator = undefined; + validator_ptrs[0] = &validators[0]; + @memset(validator_ptrs[1 .. batch_size + 1], &validators[1]); + validator_ptrs[batch_size + 1] = &validators[2]; + + var cache = PubkeyCache.init(testing.allocator, testing.io); + defer cache.deinit(); + try cache.append(testing.io, pubkeys[0], 0); + + try testing.expectError(error.BadEncoding, cache.syncPubkeys(testing.io, &validator_ptrs)); + try testing.expectEqual(@as(u32, 1), cache.count(testing.io)); + try testing.expectEqual(@as(?u64, null), cache.get(testing.io, pubkeys[1])); +} + +test "append and clear preserve dense storage invariants" { + var pubkeys: [4]types.primitive.BLSPubkey.Type = undefined; + try interop.interopPubkeysCached(pubkeys.len, &pubkeys); + + var cache = PubkeyCache.init(testing.allocator, testing.io); + defer cache.deinit(); + + for (pubkeys[0..3], 0..) |pubkey, index| { + try cache.append(testing.io, pubkey, index); + } + try cache.append(testing.io, pubkeys[0], 0); + + for (pubkeys[0..3], 0..) |pubkey, expected_index| { + try testing.expectEqual( + @as(u64, @intCast(expected_index)), + cache.get(testing.io, pubkey).?, + ); + const compressed = cache.getPubkey(testing.io, expected_index).?.compress(); + try testing.expectEqualSlices(u8, &pubkey, &compressed); + } + try testing.expectError( + error.InvalidIndexToAppend, + cache.append(testing.io, pubkeys[3], pubkeys.len), + ); + try testing.expectError( + error.DuplicatePubkey, + cache.append(testing.io, pubkeys[0], 3), + ); + try testing.expectError( + error.ConflictingPubkey, + cache.append(testing.io, pubkeys[1], 0), + ); + try testing.expectEqual(@as(u32, 3), cache.count(testing.io)); + try testing.expect(cache.capacity(testing.io) > cache.count(testing.io)); + + const capacity = cache.capacity(testing.io); + try cache.clear(testing.io); + try testing.expectEqual(@as(u32, 0), cache.count(testing.io)); + try testing.expectEqual(capacity, cache.capacity(testing.io)); + try cache.append(testing.io, pubkeys[3], 0); + try testing.expectEqual(@as(u64, 0), cache.get(testing.io, pubkeys[3]).?); +} + +test "explicit capacity is exact, non-shrinking, and guarded" { + const unsupported = cache_module.max_capacity + 1; + try testing.expectError( + error.CapacityOverflow, + PubkeyCache.initCapacity(testing.allocator, testing.io, unsupported), + ); + + var cache = try PubkeyCache.initCapacity(testing.allocator, testing.io, 8); + defer cache.deinit(); + try testing.expectEqual(@as(usize, 8), cache.capacity(testing.io)); + + try cache.ensureTotalCapacity(testing.io, 9); + try testing.expectEqual(@as(usize, 9), cache.capacity(testing.io)); + try cache.ensureTotalCapacity(testing.io, 8); + try testing.expectEqual(@as(usize, 9), cache.capacity(testing.io)); + + try testing.expectError( + error.CapacityOverflow, + cache.ensureTotalCapacity(testing.io, unsupported), + ); +} + +test "aggregate rejects empty and out-of-range indices" { + var pubkeys: [1]types.primitive.BLSPubkey.Type = undefined; + try interop.interopPubkeysCached(pubkeys.len, &pubkeys); + + var cache = PubkeyCache.init(testing.allocator, testing.io); + defer cache.deinit(); + try cache.append(testing.io, pubkeys[0], 0); + + try testing.expectError(error.InvalidLength, cache.aggregate(testing.io, &.{})); + try testing.expectError(error.InvalidIndex, cache.aggregate(testing.io, &.{1})); +} + +test "batch lookups preserve order and validate inputs" { + var pubkeys: [5]types.primitive.BLSPubkey.Type = undefined; + try interop.interopPubkeysCached(pubkeys.len, &pubkeys); + + var cache = PubkeyCache.init(testing.allocator, testing.io); + defer cache.deinit(); + for (pubkeys[0..4], 0..) |pubkey, index| { + try cache.append(testing.io, pubkey, index); + } + + const requested_indices = [_]u64{ 3, 0, 3 }; + var affine_pubkeys: [requested_indices.len]bls.PublicKey = undefined; + try cache.getPubkeys(testing.io, &requested_indices, &affine_pubkeys); + for (affine_pubkeys, requested_indices) |affine, index| { + const compressed = affine.compress(); + try testing.expectEqualSlices(u8, &pubkeys[index], &compressed); + } + try testing.expectError( + error.InvalidLength, + cache.getPubkeys(testing.io, &requested_indices, affine_pubkeys[0..2]), + ); + + const sentinel = cache.getPubkey(testing.io, 1).?; + for (&affine_pubkeys) |*affine| affine.* = sentinel; + try testing.expectError( + error.InvalidIndex, + cache.getPubkeys(testing.io, &.{ 0, 4, 1 }, &affine_pubkeys), + ); + for (affine_pubkeys) |affine| { + const compressed = affine.compress(); + try testing.expectEqualSlices(u8, &pubkeys[1], &compressed); + } + + const requested_pubkeys = [_][48]u8{ pubkeys[3], pubkeys[0], pubkeys[3] }; + var validator_indices: [requested_pubkeys.len]u64 = undefined; + try cache.getValidatorIndices(testing.io, &requested_pubkeys, &validator_indices); + try testing.expectEqualSlices(u64, &.{ 3, 0, 3 }, &validator_indices); + try testing.expectError( + error.InvalidLength, + cache.getValidatorIndices( + testing.io, + &requested_pubkeys, + validator_indices[0..2], + ), + ); + try testing.expectError( + error.PubkeyNotFound, + cache.getValidatorIndices( + testing.io, + &.{ pubkeys[0], pubkeys[4], pubkeys[1] }, + &validator_indices, + ), + ); +} diff --git a/src/state_transition/cache/state_cache.zig b/src/state_transition/cache/state_cache.zig index b21cb1723..293be220d 100644 --- a/src/state_transition/cache/state_cache.zig +++ b/src/state_transition/cache/state_cache.zig @@ -8,8 +8,6 @@ const EpochCacheImmutableData = @import("./epoch_cache.zig").EpochCacheImmutable const EpochCacheOpts = @import("./epoch_cache.zig").EpochCacheOpts; const AnyBeaconState = @import("fork_types").AnyBeaconState; const ValidatorIndex = types.primitive.ValidatorIndex.Type; -const PubkeyIndexMap = @import("pubkey_cache.zig").PubkeyIndexMap(ValidatorIndex); -const Index2PubkeyCache = @import("pubkey_cache.zig").Index2PubkeyCache; const CloneOpts = @import("ssz").CloneOpts; const SlashingsCache = @import("./slashings_cache.zig").SlashingsCache; const Node = @import("persistent_merkle_tree").Node; @@ -37,23 +35,37 @@ pub const CachedBeaconState = struct { created_with_transfer_cache: bool = false, /// This class takes ownership of state after this function and has responsibility to deinit it - pub fn createCachedBeaconState(allocator: Allocator, state: *AnyBeaconState, immutable_data: EpochCacheImmutableData, option: ?EpochCacheOpts) !*CachedBeaconState { + pub fn createCachedBeaconState( + allocator: Allocator, + io: std.Io, + state: *AnyBeaconState, + immutable_data: EpochCacheImmutableData, + option: ?EpochCacheOpts, + ) !*CachedBeaconState { const cached_state = try allocator.create(CachedBeaconState); errdefer allocator.destroy(cached_state); - try cached_state.init(allocator, state, immutable_data, option); + try cached_state.init(allocator, io, state, immutable_data, option); return cached_state; } - pub fn init(self: *CachedBeaconState, allocator: Allocator, state: *AnyBeaconState, immutable_data: EpochCacheImmutableData, option: ?EpochCacheOpts) !void { - const epoch_cache = try EpochCache.createFromState(allocator, state, immutable_data, option); + pub fn init( + self: *CachedBeaconState, + allocator: Allocator, + io: std.Io, + state: *AnyBeaconState, + immutable_data: EpochCacheImmutableData, + option: ?EpochCacheOpts, + ) !void { + const epoch_cache = try EpochCache.createFromState(allocator, io, state, immutable_data, option); errdefer epoch_cache.deinit(); + const slashings_cache = try SlashingsCache.initEmpty(allocator); self.* = .{ .allocator = allocator, .config = immutable_data.config, .epoch_cache = epoch_cache, - .slashings_cache = try SlashingsCache.initEmpty(allocator), + .slashings_cache = slashings_cache, .state = state, .proposer_rewards = .{}, }; diff --git a/src/state_transition/cache/sync_committee_cache.zig b/src/state_transition/cache/sync_committee_cache.zig index baa991103..a50caf84c 100644 --- a/src/state_transition/cache/sync_committee_cache.zig +++ b/src/state_transition/cache/sync_committee_cache.zig @@ -2,7 +2,7 @@ const std = @import("std"); const Allocator = std.mem.Allocator; const preset = @import("preset").preset; const types = @import("consensus_types"); -const PubkeyIndexMap = @import("../cache/pubkey_cache.zig").PubkeyIndexMap; +const PubkeyCache = @import("../cache/pubkey_cache.zig").PubkeyCache; const SyncCommittee = types.altair.SyncCommittee.Type; const ValidatorIndex = types.primitive.ValidatorIndex.Type; const BLSPubkey = types.primitive.BLSPubkey.Type; @@ -38,8 +38,8 @@ pub const SyncCommitteeCache = union(enum) { return SyncCommitteeCache{ .phase0 = {} }; } - pub fn initSyncCommittee(allocator: Allocator, sync_committee: *const SyncCommittee, pubkey_to_index: *const PubkeyIndexMap) !SyncCommitteeCache { - const cache = try SyncCommitteeCacheAltair.initSyncCommittee(allocator, sync_committee, pubkey_to_index); + pub fn initSyncCommittee(allocator: Allocator, sync_committee: *const SyncCommittee, pubkey_cache: *const PubkeyCache, io: std.Io) !SyncCommitteeCache { + const cache = try SyncCommitteeCacheAltair.initSyncCommittee(allocator, sync_committee, pubkey_cache, io); return SyncCommitteeCache{ .altair = cache }; } @@ -67,9 +67,10 @@ const SyncCommitteeCacheAltair = struct { validator_index_map: *SyncComitteeValidatorIndexMap, - pub fn initSyncCommittee(allocator: Allocator, sync_committee: *const SyncCommittee, pubkey_to_index: *const PubkeyIndexMap) !*SyncCommitteeCacheAltair { + pub fn initSyncCommittee(allocator: Allocator, sync_committee: *const SyncCommittee, pubkey_cache: *const PubkeyCache, io: std.Io) !*SyncCommitteeCacheAltair { const validator_indices = try allocator.alloc(ValidatorIndex, sync_committee.pubkeys.len); - try computeSyncCommitteeIndices(sync_committee, pubkey_to_index, validator_indices); + errdefer allocator.free(validator_indices); + try computeSyncCommitteeIndices(sync_committee, pubkey_cache, io, validator_indices); return SyncCommitteeCacheAltair.initValidatorIndices(allocator, validator_indices); } @@ -113,23 +114,23 @@ const SyncCommitteeCacheAltair = struct { test "initSyncCommittee - sanity" { const allocator = std.testing.allocator; + var pubkeys: [1]BLSPubkey = undefined; + try @import("../test_utils/interop_pubkeys.zig").interopPubkeysCached(1, &pubkeys); var sync_committee = SyncCommittee{ - .pubkeys = [_]BLSPubkey{ - [_]u8{1} ** 48, - } ** preset.SYNC_COMMITTEE_SIZE, + .pubkeys = [_]BLSPubkey{pubkeys[0]} ** preset.SYNC_COMMITTEE_SIZE, .aggregate_pubkey = [_]u8{2} ** 48, }; - var pubkey_index_map = PubkeyIndexMap.init(allocator); - defer pubkey_index_map.deinit(); - try pubkey_index_map.put(sync_committee.pubkeys[0], 1000); + var pubkey_cache = try PubkeyCache.initCapacity(allocator, std.testing.io, 1); + defer pubkey_cache.deinit(); + try pubkey_cache.append(std.testing.io, pubkeys[0], 0); - var cache = try SyncCommitteeCache.initSyncCommittee(allocator, &sync_committee, &pubkey_index_map); + var cache = try SyncCommitteeCache.initSyncCommittee(allocator, &sync_committee, &pubkey_cache, std.testing.io); defer cache.deinit(); try std.testing.expectEqualSlices( ValidatorIndex, - &[_]ValidatorIndex{1000} ** preset.SYNC_COMMITTEE_SIZE, + &[_]ValidatorIndex{0} ** preset.SYNC_COMMITTEE_SIZE, cache.getValidatorIndices(), ); } @@ -170,36 +171,28 @@ test computeSyncCommitteeMap { } } -fn computeSyncCommitteeIndices(sync_committee: *const SyncCommittee, pubkey_to_index: *const PubkeyIndexMap, out: []ValidatorIndex) !void { - if (out.len != sync_committee.pubkeys.len) { - return error.InvalidLength; - } - - const pubkeys = sync_committee.pubkeys; - for (pubkeys, 0..) |pubkey, i| { - const index = pubkey_to_index.get(pubkey) orelse return error.PubkeyNotFound; - out[i] = @intCast(index); - } +fn computeSyncCommitteeIndices(sync_committee: *const SyncCommittee, pubkey_cache: *const PubkeyCache, io: std.Io, out: []ValidatorIndex) !void { + try pubkey_cache.getValidatorIndices(io, &sync_committee.pubkeys, out); } test computeSyncCommitteeIndices { + var pubkeys: [1]BLSPubkey = undefined; + try @import("../test_utils/interop_pubkeys.zig").interopPubkeysCached(1, &pubkeys); var sync_committee = SyncCommittee{ - .pubkeys = [_]BLSPubkey{ - [_]u8{1} ** 48, - } ** preset.SYNC_COMMITTEE_SIZE, + .pubkeys = [_]BLSPubkey{pubkeys[0]} ** preset.SYNC_COMMITTEE_SIZE, .aggregate_pubkey = [_]u8{2} ** 48, }; const allocator = std.testing.allocator; - var pubkey_index_map = PubkeyIndexMap.init(allocator); - defer pubkey_index_map.deinit(); - try pubkey_index_map.put(sync_committee.pubkeys[0], 1000); + var pubkey_cache = try PubkeyCache.initCapacity(allocator, std.testing.io, 1); + defer pubkey_cache.deinit(); + try pubkey_cache.append(std.testing.io, pubkeys[0], 0); var out: [preset.SYNC_COMMITTEE_SIZE]ValidatorIndex = undefined; - try computeSyncCommitteeIndices(&sync_committee, &pubkey_index_map, &out); + try computeSyncCommitteeIndices(&sync_committee, &pubkey_cache, std.testing.io, &out); try std.testing.expectEqualSlices( ValidatorIndex, - &[_]ValidatorIndex{1000} ** preset.SYNC_COMMITTEE_SIZE, + &[_]ValidatorIndex{0} ** preset.SYNC_COMMITTEE_SIZE, &out, ); } diff --git a/src/state_transition/epoch/process_epoch.zig b/src/state_transition/epoch/process_epoch.zig index 2f5428706..38513bcd1 100644 --- a/src/state_transition/epoch/process_epoch.zig +++ b/src/state_transition/epoch/process_epoch.zig @@ -64,7 +64,7 @@ pub fn processEpoch( if (comptime fork.gte(.electra)) { timer = time.start(io); - try processPendingDeposits(fork, allocator, config, epoch_cache, state, cache); + try processPendingDeposits(fork, allocator, io, config, epoch_cache, state, cache); try observeEpochTransitionStep(.{ .step = .process_pending_deposits }, @as(u64, @intCast(time.since(io, timer).nanoseconds))); timer = time.start(io); diff --git a/src/state_transition/epoch/process_pending_deposits.zig b/src/state_transition/epoch/process_pending_deposits.zig index df3d201b0..f93415313 100644 --- a/src/state_transition/epoch/process_pending_deposits.zig +++ b/src/state_transition/epoch/process_pending_deposits.zig @@ -25,6 +25,7 @@ const Node = @import("persistent_merkle_tree").Node; pub fn processPendingDeposits( comptime fork: ForkSeq, allocator: Allocator, + io: std.Io, config: *const BeaconConfig, epoch_cache: *EpochCache, state: *BeaconState(fork), @@ -79,7 +80,7 @@ pub fn processPendingDeposits( // Read validator state var is_validator_exited = false; var is_validator_withdrawn = false; - const validator_index = epoch_cache.getValidatorIndex(&deposit.pubkey); + const validator_index = epoch_cache.pubkey_cache.get(io, deposit.pubkey); if (try isValidatorKnown(fork, state, validator_index)) { var validators = try state.validators(); @@ -90,7 +91,7 @@ pub fn processPendingDeposits( if (is_validator_withdrawn) { // Deposited balance will never become active. Increase balance but do not consume churn - try applyPendingDeposit(fork, allocator, config, epoch_cache, state, deposit, cache); + try applyPendingDeposit(fork, allocator, io, config, epoch_cache, state, deposit, cache); } else if (is_validator_exited) { // Validator is exiting, postpone the deposit until after withdrawable epoch try deposits_to_postpone.append(allocator, deposit); @@ -102,7 +103,7 @@ pub fn processPendingDeposits( } // Consume churn and apply deposit. processed_amount += deposit.amount; - try applyPendingDeposit(fork, allocator, config, epoch_cache, state, deposit, cache); + try applyPendingDeposit(fork, allocator, io, config, epoch_cache, state, deposit, cache); } // Regardless of how the deposit was handled, we move on in the queue. @@ -130,13 +131,14 @@ pub fn processPendingDeposits( fn applyPendingDeposit( comptime fork: ForkSeq, allocator: Allocator, + io: std.Io, config: *const BeaconConfig, epoch_cache: *EpochCache, state: *BeaconState(fork), deposit: PendingDeposit, cache: *EpochTransitionCache, ) !void { - const validator_index = epoch_cache.getValidatorIndex(&deposit.pubkey) orelse null; + const validator_index = epoch_cache.pubkey_cache.get(io, deposit.pubkey); const pubkey = &deposit.pubkey; const withdrawal_credentials = &deposit.withdrawal_credentials; @@ -147,7 +149,7 @@ fn applyPendingDeposit( if (!is_validator_known) { // Verify the deposit signature (proof of possession) which is not checked by the deposit contract if (validateDepositSignature(config, pubkey, withdrawal_credentials, amount, signature)) { - try addValidatorToRegistry(fork, allocator, epoch_cache, state, pubkey, withdrawal_credentials, amount); + try addValidatorToRegistry(fork, allocator, io, epoch_cache, state, pubkey, withdrawal_credentials, amount); try cache.is_compounding_validator_arr.append(allocator, hasCompoundingWithdrawalCredential(withdrawal_credentials)); // set balance, so that the next deposit of same pubkey will increase the balance correctly // this is to fix the double deposit issue found in mekong @@ -187,6 +189,7 @@ test "processPendingDeposits - sanity" { try processPendingDeposits( .electra, allocator, + std.testing.io, test_state.cached_state.config, test_state.cached_state.epoch_cache, test_state.cached_state.state.castToFork(.electra), diff --git a/src/state_transition/epoch/process_rewards_and_penalties.zig b/src/state_transition/epoch/process_rewards_and_penalties.zig index 9f0630367..ec8e94d9d 100644 --- a/src/state_transition/epoch/process_rewards_and_penalties.zig +++ b/src/state_transition/epoch/process_rewards_and_penalties.zig @@ -16,7 +16,7 @@ pub fn processRewardsAndPenalties( config: *const BeaconConfig, epoch_cache: *const EpochCache, state: *BeaconState(fork), - cache: *const EpochTransitionCache, + cache: *EpochTransitionCache, slashing_penalties: ?[]const u64, ) !void { // No rewards are applied at the end of `GENESIS_EPOCH` because rewards are for work done in the previous epoch @@ -29,7 +29,7 @@ pub fn processRewardsAndPenalties( try getRewardsAndPenalties(fork, allocator, config, epoch_cache, state, cache, rewards, penalties); const balances = try state.balancesSlice(allocator); - defer allocator.free(balances); + errdefer allocator.free(balances); if (slashing_penalties) |slashings| { for (rewards, penalties, balances, 0..) |reward, penalty, *balance, i| { @@ -42,8 +42,16 @@ pub fn processRewardsAndPenalties( } } - var balances_arraylist: std.ArrayListUnmanaged(u64) = .fromOwnedSlice(balances); - try state.setBalances(&balances_arraylist); + // Populate cache.balances for reuse by the validator monitor and + // more importantly processEffectiveBalanceUpdates() doesn't need to + // get from tree view state which has to commit. + var new_balances: std.ArrayList(u64) = .fromOwnedSlice(balances); + try state.setBalances(&new_balances); + + if (cache.balances) |*old_balances| { + old_balances.deinit(allocator); + } + cache.balances = new_balances; } pub fn getRewardsAndPenalties( @@ -82,4 +90,15 @@ test "processRewardsAndPenalties - sanity" { test_state.epoch_transition_cache, null, ); + + // Verify replacing the old cached balances does not leak. + try processRewardsAndPenalties( + .electra, + allocator, + test_state.cached_state.config, + test_state.cached_state.epoch_cache, + test_state.cached_state.state.castToFork(.electra), + test_state.epoch_transition_cache, + null, + ); } diff --git a/src/state_transition/root.zig b/src/state_transition/root.zig index 4ac6d685f..d20364ac5 100644 --- a/src/state_transition/root.zig +++ b/src/state_transition/root.zig @@ -20,10 +20,8 @@ pub const EpochCache = @import("./cache/epoch_cache.zig").EpochCache; pub const shuffle = @import("./utils/shuffle.zig"); pub const committee_indices = @import("./utils/committee_indices.zig"); -pub const PubkeyIndexMap = @import("./cache/pubkey_cache.zig").PubkeyIndexMap; -pub const Index2PubkeyCache = @import("./cache/pubkey_cache.zig").Index2PubkeyCache; -pub const syncPubkeys = @import("./cache/pubkey_cache.zig").syncPubkeys; -pub const syncPubkeysParallel = @import("./cache/pubkey_cache.zig").syncPubkeysParallel; +pub const PubkeyCache = @import("./cache/pubkey_cache.zig").PubkeyCache; +pub const pkix = @import("./cache/pkix.zig"); pub const EpochTransitionCache = @import("./cache/epoch_transition_cache.zig").EpochTransitionCache; pub const processEpoch = @import("./epoch/process_epoch.zig").processEpoch; @@ -90,12 +88,15 @@ pub const getBlockRootAtSlot = @import("./utils/block_root.zig").getBlockRootAtS pub const computeSlotsSinceEpochStart = @import("./utils/epoch.zig").computeSlotsSinceEpochStart; pub const computeStartSlotAtEpoch = @import("./utils/epoch.zig").computeStartSlotAtEpoch; pub const AnchorCheckpoint = @import("./AnchorCheckpoint.zig"); -pub const deinitStateTransition = @import("./state_transition.zig").deinitStateTransition; +pub const deinitReusedEpochTransitionCache = @import("./state_transition.zig").deinitReusedEpochTransitionCache; pub const isExecutionEnabled = @import("./utils/execution.zig").isExecutionEnabled; pub const isMergeTransitionComplete = @import("./utils/execution.zig").isMergeTransitionComplete; pub const getRandaoMix = @import("./utils/seed.zig").getRandaoMix; pub const getEffectiveBalanceIncrementsZeroInactive = @import("./utils/balance.zig").getEffectiveBalanceIncrementsZeroInactive; pub const PendingDepositsLookup = @import("./utils/pending_deposits_lookup.zig").PendingDepositsLookup; +pub const getStateSlotFromBytes = @import("ssz_bytes.zig").getStateSlotFromBytes; +pub const getLastProcessedSlotFromStateBytes = @import("ssz_bytes.zig").getLastProcessedSlotFromStateBytes; +pub const STATE_SLOTS_PREFIX_LEN = @import("ssz_bytes.zig").STATE_SLOTS_PREFIX_LEN; pub const WithdrawalsResult = @import("./block/process_withdrawals.zig").WithdrawalsResult; @@ -130,6 +131,8 @@ const weak_subjectivity = @import("weak_subjectivity.zig"); pub const getLatestWeakSubjectivityCheckpointEpoch = weak_subjectivity.getLatestWeakSubjectivityCheckpointEpoch; test { + _ = @import("./cache/pubkey_cache_test.zig"); + _ = @import("./cache/pkix_test.zig"); testing.refAllDecls(@This()); testing.refAllDecls(seed); testing.refAllDecls(state_transition); @@ -137,4 +140,5 @@ test { testing.refAllDecls(load_state); testing.refAllDecls(sync_committees_witness); testing.refAllDecls(weak_subjectivity); + testing.refAllDecls(@import("ssz_bytes.zig")); } diff --git a/src/state_transition/signature_sets/indexed_attestation.zig b/src/state_transition/signature_sets/indexed_attestation.zig index a3b42e561..ed224c57e 100644 --- a/src/state_transition/signature_sets/indexed_attestation.zig +++ b/src/state_transition/signature_sets/indexed_attestation.zig @@ -9,9 +9,7 @@ const BLSSignature = types.primitive.BLSSignature.Type; const Root = types.primitive.Root.Type; const BeaconConfig = @import("config").BeaconConfig; const ForkSeq = @import("config").ForkSeq; -const CachedBeaconState = @import("../cache/state_cache.zig").CachedBeaconState; const EpochCache = @import("../cache/epoch_cache.zig").EpochCache; -const AnySignedBeaconBlock = @import("fork_types").AnySignedBeaconBlock; const ForkTypes = @import("fork_types").ForkTypes; const c = @import("constants"); const computeSigningRoot = @import("../utils/signing_root.zig").computeSigningRoot; @@ -29,6 +27,7 @@ pub fn getAttestationDataSigningRoot(config: *const BeaconConfig, state_epoch: E /// Consumer need to free the returned pubkeys array pub fn getAttestationWithIndicesSignatureSet( allocator: Allocator, + io: std.Io, config: *const BeaconConfig, epoch_cache: *const EpochCache, data: *const AttestationData, @@ -37,9 +36,10 @@ pub fn getAttestationWithIndicesSignatureSet( ) !AggregatedSignatureSet { const pubkeys = try allocator.alloc(PublicKey, attesting_indices.len); errdefer allocator.free(pubkeys); - for (0..attesting_indices.len) |i| { - pubkeys[i] = epoch_cache.index_to_pubkey.items[@intCast(attesting_indices[i])]; - } + epoch_cache.pubkey_cache.getPubkeys(io, attesting_indices, pubkeys) catch |err| switch (err) { + error.InvalidIndex => return error.PubkeyNotFound, + else => return err, + }; var signing_root: Root = undefined; try getAttestationDataSigningRoot(config, epoch_cache.epoch, data, &signing_root); @@ -51,12 +51,14 @@ pub fn getAttestationWithIndicesSignatureSet( pub fn getIndexedAttestationSignatureSet( comptime fork: ForkSeq, allocator: Allocator, + io: std.Io, config: *const BeaconConfig, epoch_cache: *const EpochCache, indexed_attestation: *const ForkTypes(fork).IndexedAttestation.Type, ) !AggregatedSignatureSet { return try getAttestationWithIndicesSignatureSet( allocator, + io, config, epoch_cache, &indexed_attestation.data, @@ -64,32 +66,3 @@ pub fn getIndexedAttestationSignatureSet( indexed_attestation.attesting_indices.items, ); } - -/// Appends to out all the AggregatedSignatureSet for each attestation in the signed_block -/// Consumer need to free the pubkeys arrays in each AggregatedSignatureSet in out -/// TODO: consume in https://github.com/ChainSafe/state-transition-z/issues/72 -pub fn attestationsSignatureSets(allocator: Allocator, cached_state: *const CachedBeaconState, signed_block: *const AnySignedBeaconBlock, out: *std.ArrayList(AggregatedSignatureSet)) !void { - const epoch_cache = cached_state.epoch_cache; - const attestation_items = signed_block.beaconBlock().beaconBlockBody().attestations().items(); - - switch (attestation_items) { - .phase0 => |phase0_attestations| { - for (phase0_attestations) |*attestation| { - const indexed_attestation = try epoch_cache.computeIndexedAttestationPhase0(attestation); - var attesting_indices = indexed_attestation.attesting_indices; - defer attesting_indices.deinit(allocator); - const signature_set = try getIndexedAttestationSignatureSet(allocator, cached_state, indexed_attestation); - try out.append(allocator, signature_set); - } - }, - .electra => |electra_attestations| { - for (electra_attestations) |*attestation| { - const indexed_attestation = try epoch_cache.computeIndexedAttestationElectra(attestation); - var attesting_indices = indexed_attestation.attesting_indices; - defer attesting_indices.deinit(allocator); - const signature_set = try getIndexedAttestationSignatureSet(allocator, cached_state, indexed_attestation); - try out.append(allocator, signature_set); - } - }, - } -} diff --git a/src/state_transition/signature_sets/proposer.zig b/src/state_transition/signature_sets/proposer.zig index fc3124d27..889334a52 100644 --- a/src/state_transition/signature_sets/proposer.zig +++ b/src/state_transition/signature_sets/proposer.zig @@ -13,12 +13,14 @@ const AnySignedBeaconBlock = @import("fork_types").AnySignedBeaconBlock; pub fn verifyProposerSignature( allocator: Allocator, + io: std.Io, config: *const BeaconConfig, epoch_cache: *const EpochCache, signed_block: AnySignedBeaconBlock, ) !bool { const signature_set = try getBlockProposerSignatureSet( allocator, + io, config, epoch_cache, signed_block, @@ -28,6 +30,7 @@ pub fn verifyProposerSignature( pub fn getBlockProposerSignatureSet( allocator: Allocator, + io: std.Io, config: *const BeaconConfig, epoch_cache: *const EpochCache, signed_block: AnySignedBeaconBlock, @@ -43,28 +46,29 @@ pub fn getBlockProposerSignatureSet( // The proposer index isn't validated until processBlockHeader, so a malicious block could // put an out-of-range value here. const proposer_index = block.proposerIndex(); - if (proposer_index >= epoch_cache.index_to_pubkey.items.len) { + const pubkey = epoch_cache.pubkey_cache.getPubkey(io, proposer_index) orelse return error.InvalidProposerIndex; - } return .{ - .pubkey = epoch_cache.index_to_pubkey.items[proposer_index], + .pubkey = pubkey, .signing_root = signing_root_buf, .signature = signed_block.signature().*, }; } pub fn getBlockHeaderProposerSignatureSet( + io: std.Io, config: *const BeaconConfig, epoch_cache: *const EpochCache, signed_block_header: *const types.phase0.SignedBeaconBlockHeader.Type, -) SingleSignatureSet { - const domain = config.getDomain(epoch_cache.epoch, c.DOMAIN_BEACON_PROPOSER, signed_block_header.message.slot); +) !SingleSignatureSet { + const domain = try config.getDomain(epoch_cache.epoch, c.DOMAIN_BEACON_PROPOSER, signed_block_header.message.slot); var signing_root: Root = undefined; try computeSigningRoot(types.phase0.SignedBeaconBlockHeader, signed_block_header, domain, &signing_root); return .{ - .pubkey = epoch_cache.index_to_pubkey(signed_block_header.message.proposerIndex), + .pubkey = epoch_cache.pubkey_cache.getPubkey(io, signed_block_header.message.proposer_index) orelse + return error.PubkeyNotFound, .signing_root = signing_root, .signature = signed_block_header.signature, }; diff --git a/src/state_transition/signature_sets/proposer_slashings.zig b/src/state_transition/signature_sets/proposer_slashings.zig index ab6e0b04b..3b4210a32 100644 --- a/src/state_transition/signature_sets/proposer_slashings.zig +++ b/src/state_transition/signature_sets/proposer_slashings.zig @@ -10,6 +10,7 @@ const types = @import("consensus_types"); const computeSigningRoot = @import("../utils/signing_root.zig").computeSigningRoot; pub fn getProposerSlashingSignatureSets( + io: std.Io, config: *const BeaconConfig, epoch_cache: *const EpochCache, proposer_slashing: *const types.phase0.ProposerSlashing.Type, @@ -27,13 +28,15 @@ pub fn getProposerSlashingSignatureSets( try computeSigningRoot(types.phase0.BeaconBlockHeader, &signed_header_2.message, domain_2, &signing_root_2); result[0] = SingleSignatureSet{ - .pubkey = epoch_cache.index_to_pubkey.items[signed_header_1.message.proposer_index], + .pubkey = epoch_cache.pubkey_cache.getPubkey(io, signed_header_1.message.proposer_index) orelse + return error.PubkeyNotFound, .signing_root = signing_root_1, .signature = signed_header_1.signature, }; result[1] = SingleSignatureSet{ - .pubkey = epoch_cache.index_to_pubkey.items[signed_header_2.message.proposer_index], + .pubkey = epoch_cache.pubkey_cache.getPubkey(io, signed_header_2.message.proposer_index) orelse + return error.PubkeyNotFound, .signing_root = signing_root_2, .signature = signed_header_2.signature, }; @@ -44,15 +47,16 @@ pub fn getProposerSlashingSignatureSets( pub fn proposerSlashingsSignatureSets( comptime fork: ForkSeq, allocator: std.mem.Allocator, + io: std.Io, config: *const BeaconConfig, epoch_cache: *const EpochCache, - state: *const BeaconState(fork), + _: *const BeaconState(fork), signed_block: *const ForkTypes(fork).SignedBeaconBlock.Type, out: *std.ArrayList(SingleSignatureSet), ) !void { const proposer_slashings = signed_block.message.body.proposer_slashings.items; for (proposer_slashings) |*proposer_slashing| { - const signature_sets = try getProposerSlashingSignatureSets(fork, config, epoch_cache, state, proposer_slashing); + const signature_sets = try getProposerSlashingSignatureSets(io, config, epoch_cache, proposer_slashing); try out.append(allocator, signature_sets[0]); try out.append(allocator, signature_sets[1]); } diff --git a/src/state_transition/signature_sets/randao.zig b/src/state_transition/signature_sets/randao.zig index 1633862da..b6c7ddeb6 100644 --- a/src/state_transition/signature_sets/randao.zig +++ b/src/state_transition/signature_sets/randao.zig @@ -1,3 +1,4 @@ +const std = @import("std"); const types = @import("consensus_types"); const Slot = types.primitive.Slot.Type; const BeaconConfig = @import("config").BeaconConfig; @@ -10,17 +11,19 @@ const computeSigningRoot = @import("../utils/signing_root.zig").computeSigningRo const verifySingleSignatureSet = @import("../utils/signature_sets.zig").verifySingleSignatureSet; pub fn verifyRandaoSignature( + io: std.Io, config: *const BeaconConfig, epoch_cache: *const EpochCache, randao_reveal: *const [96]u8, slot: Slot, proposer_idx: u64, ) !bool { - const signature_set = try randaoRevealSignatureSet(config, epoch_cache, randao_reveal, slot, proposer_idx); + const signature_set = try randaoRevealSignatureSet(io, config, epoch_cache, randao_reveal, slot, proposer_idx); return verifySingleSignatureSet(&signature_set); } pub fn randaoRevealSignatureSet( + io: std.Io, config: *const BeaconConfig, epoch_cache: *const EpochCache, randao_reveal: *const [96]u8, @@ -33,7 +36,8 @@ pub fn randaoRevealSignatureSet( var signing_root: Root = undefined; try computeSigningRoot(types.primitive.Epoch, &epoch, domain, &signing_root); return .{ - .pubkey = epoch_cache.index_to_pubkey.items[proposer_idx], + .pubkey = epoch_cache.pubkey_cache.getPubkey(io, proposer_idx) orelse + return error.PubkeyNotFound, .signing_root = signing_root, .signature = randao_reveal.*, }; diff --git a/src/state_transition/signature_sets/voluntary_exits.zig b/src/state_transition/signature_sets/voluntary_exits.zig index 07551f845..3135e2f76 100644 --- a/src/state_transition/signature_sets/voluntary_exits.zig +++ b/src/state_transition/signature_sets/voluntary_exits.zig @@ -12,11 +12,13 @@ const computeSigningRoot = @import("../utils/signing_root.zig").computeSigningRo const verifySingleSignatureSet = @import("../utils/signature_sets.zig").verifySingleSignatureSet; pub fn verifyVoluntaryExitSignature( + io: std.Io, config: *const BeaconConfig, epoch_cache: *const EpochCache, signed_voluntary_exit: *const SignedVoluntaryExit, ) !bool { const signature_set = try getVoluntaryExitSignatureSet( + io, config, epoch_cache, signed_voluntary_exit, @@ -25,6 +27,7 @@ pub fn verifyVoluntaryExitSignature( } pub fn getVoluntaryExitSignatureSet( + io: std.Io, config: *const BeaconConfig, epoch_cache: *const EpochCache, signed_voluntary_exit: *const SignedVoluntaryExit, @@ -35,7 +38,8 @@ pub fn getVoluntaryExitSignatureSet( try computeSigningRoot(types.phase0.VoluntaryExit, &signed_voluntary_exit.message, domain, &signing_root); return .{ - .pubkey = epoch_cache.index_to_pubkey.items[signed_voluntary_exit.message.validator_index], + .pubkey = epoch_cache.pubkey_cache.getPubkey(io, signed_voluntary_exit.message.validator_index) orelse + return error.PubkeyNotFound, .signing_root = signing_root, .signature = signed_voluntary_exit.signature, }; @@ -43,6 +47,7 @@ pub fn getVoluntaryExitSignatureSet( pub fn voluntaryExitsSignatureSets( allocator: std.mem.Allocator, + io: std.Io, config: *const BeaconConfig, epoch_cache: *const EpochCache, voluntary_exits: []types.phase0.SignedVoluntaryExit.Type, @@ -50,6 +55,7 @@ pub fn voluntaryExitsSignatureSets( ) !void { for (voluntary_exits) |*signed_voluntary_exit| { const signature_set = try getVoluntaryExitSignatureSet( + io, config, epoch_cache, signed_voluntary_exit, diff --git a/src/state_transition/ssz_bytes.zig b/src/state_transition/ssz_bytes.zig index 575bff478..18db3b8d2 100644 --- a/src/state_transition/ssz_bytes.zig +++ b/src/state_transition/ssz_bytes.zig @@ -24,7 +24,62 @@ pub fn getStateSlotFromBytes(bytes: []const u8) !Slot { return std.mem.readInt(u64, bytes[SLOT_BYTES_POSITION_IN_STATE .. SLOT_BYTES_POSITION_IN_STATE + slot_size], .little); } +/// +/// 8 + 32 + 8 + 16 = 64 +/// +/// ``` +/// class BeaconState(Container): +/// genesis_time: uint64 [fixed - 8 bytes] +/// genesis_validators_root: Root [fixed - 32 bytes] +/// slot: Slot [fixed - 8 bytes] +/// fork: Fork [fixed - 16 bytes] +/// latest_block_header: BeaconBlockHeader +/// slot: Slot [fixed - 8 bytes] <-- this +/// ... +/// ``` +const BLOCK_HEADER_SLOT_BYTES_POSITION_IN_STATE: usize = 64; + +/// Leading bytes sufficient for both slot readers. +pub const STATE_SLOTS_PREFIX_LEN: usize = + BLOCK_HEADER_SLOT_BYTES_POSITION_IN_STATE + types.primitive.Slot.fixed_size; + +/// Slot of the state's `latest_block_header` — the slot of the last block processed into the state, +/// which lags `state.slot` when slots were skipped after that block. +pub fn getLastProcessedSlotFromStateBytes(bytes: []const u8) !Slot { + const slot_size = types.primitive.Slot.fixed_size; + if (bytes.len < BLOCK_HEADER_SLOT_BYTES_POSITION_IN_STATE + slot_size) return error.InvalidSize; + return std.mem.readInt(u64, bytes[BLOCK_HEADER_SLOT_BYTES_POSITION_IN_STATE .. BLOCK_HEADER_SLOT_BYTES_POSITION_IN_STATE + slot_size], .little); +} + pub fn getForkFromStateBytes(config: *const BeaconConfig, bytes: []const u8) !ForkSeq { const slot = try getStateSlotFromBytes(bytes); return config.forkSeq(slot); } + +const testing = std.testing; + +test "state byte readers match a real serialized electra state" { + var state = types.electra.BeaconState.default_value; + state.slot = 12_345; + state.latest_block_header.slot = 12_344; + + const bytes = try testing.allocator.alloc(u8, types.electra.BeaconState.serializedSize(&state)); + defer testing.allocator.free(bytes); + _ = types.electra.BeaconState.serializeIntoBytes(&state, bytes); + + try testing.expectEqual(@as(Slot, 12_345), try getStateSlotFromBytes(bytes)); + try testing.expectEqual(@as(Slot, 12_344), try getLastProcessedSlotFromStateBytes(bytes)); +} + +test "state byte readers match a real serialized phase0 state" { + var state = types.phase0.BeaconState.default_value; + state.slot = 77; + state.latest_block_header.slot = 76; + + const bytes = try testing.allocator.alloc(u8, types.phase0.BeaconState.serializedSize(&state)); + defer testing.allocator.free(bytes); + _ = types.phase0.BeaconState.serializeIntoBytes(&state, bytes); + + try testing.expectEqual(@as(Slot, 77), try getStateSlotFromBytes(bytes)); + try testing.expectEqual(@as(Slot, 76), try getLastProcessedSlotFromStateBytes(bytes)); +} diff --git a/src/state_transition/state_transition.zig b/src/state_transition/state_transition.zig index a235df4e1..faa22f208 100644 --- a/src/state_transition/state_transition.zig +++ b/src/state_transition/state_transition.zig @@ -22,7 +22,6 @@ const EpochTransitionCache = @import("cache/epoch_transition_cache.zig").EpochTr const processEpoch = @import("epoch/process_epoch.zig").processEpoch; const computeEpochAtSlot = @import("utils/epoch.zig").computeEpochAtSlot; const processSlot = @import("slot/process_slot.zig").processSlot; -const deinitReusedEpochTransitionCache = @import("cache/epoch_transition_cache.zig").deinitReusedEpochTransitionCache; const upgradeStateToAltair = @import("slot/upgrade_state_to_altair.zig").upgradeStateToAltair; const upgradeStateToBellatrix = @import("slot/upgrade_state_to_bellatrix.zig").upgradeStateToBellatrix; const upgradeStateToCapella = @import("slot/upgrade_state_to_capella.zig").upgradeStateToCapella; @@ -31,6 +30,8 @@ const upgradeStateToElectra = @import("slot/upgrade_state_to_electra.zig").upgra const upgradeStateToFulu = @import("slot/upgrade_state_to_fulu.zig").upgradeStateToFulu; const upgradeStateToGloas = @import("slot/upgrade_state_to_gloas.zig").upgradeStateToGloas; +pub const deinitReusedEpochTransitionCache = @import("cache/epoch_transition_cache.zig").deinitReusedEpochTransitionCache; + pub const ExecutionPayloadStatus = enum(u8) { invalid, valid, @@ -203,6 +204,7 @@ pub fn stateTransition( // Verify proposer signature only if (opts.verify_proposer and !try verifyProposerSignature( allocator, + io, config, post_epoch_cache, signed_block, @@ -225,6 +227,7 @@ pub fn stateTransition( try processBlock( f, allocator, + io, config, post_epoch_cache, post_state.castToFork(f), @@ -262,10 +265,6 @@ pub fn stateTransition( return post_cached_state; } -pub fn deinitStateTransition(io: std.Io) void { - deinitReusedEpochTransitionCache(io); -} - const TestCase = struct { transition_opt: TransitionOpts, expect_error: bool, @@ -326,14 +325,14 @@ test "state transition - electra block" { } } - defer deinitStateTransition(std.testing.io); + deinitReusedEpochTransitionCache(std.testing.io); } test "state transition - a rejected block leaves the pre-state unchanged" { const allocator = std.testing.allocator; var pool = try Node.Pool.init(.{ .page_allocator = allocator, .allocator = allocator, .pool_size = 256 * 5 }); defer pool.deinit(); - defer deinitStateTransition(std.testing.io); + defer deinitReusedEpochTransitionCache(std.testing.io); var test_state = try TestCachedBeaconState.init(allocator, &pool, 256); defer test_state.deinit(); diff --git a/src/state_transition/test_utils/generate_state.zig b/src/state_transition/test_utils/generate_state.zig index 42b017497..5b29085c9 100644 --- a/src/state_transition/test_utils/generate_state.zig +++ b/src/state_transition/test_utils/generate_state.zig @@ -18,11 +18,9 @@ const Node = @import("persistent_merkle_tree").Node; const state_transition = @import("../root.zig"); const CachedBeaconState = state_transition.CachedBeaconState; const AnyBeaconState = @import("fork_types").AnyBeaconState; -const PubkeyIndexMap = state_transition.PubkeyIndexMap; -const Index2PubkeyCache = state_transition.Index2PubkeyCache; +const PubkeyCache = state_transition.PubkeyCache; const EffectiveBalanceIncrements = state_transition.EffectiveBalanceIncrements; const getNextSyncCommitteeIndices = state_transition.getNextSyncCommitteeIndices; -const syncPubkeys = state_transition.syncPubkeys; const interopPubkeysCached = @import("./interop_pubkeys.zig").interopPubkeysCached; const EFFECTIVE_BALANCE_INCREMENT = 32; const EFFECTIVE_BALANCE = 32 * 1e9; @@ -168,8 +166,7 @@ pub const TestCachedBeaconState = struct { allocator: Allocator, pool: *Node.Pool, config: *BeaconConfig, - pubkey_index_map: *PubkeyIndexMap, - index_pubkey_cache: *Index2PubkeyCache, + pubkey_cache: *PubkeyCache, cached_state: *CachedBeaconState, epoch_transition_cache: *state_transition.EpochTransitionCache, @@ -186,33 +183,29 @@ pub const TestCachedBeaconState = struct { } pub fn initFromState(allocator: Allocator, pool: *Node.Pool, state: *AnyBeaconState, fork: ForkSeq, fork_epoch: Epoch) !TestCachedBeaconState { - const pubkey_index_map = try allocator.create(PubkeyIndexMap); - pubkey_index_map.* = PubkeyIndexMap.init(allocator); - errdefer { - pubkey_index_map.deinit(); - allocator.destroy(pubkey_index_map); - } - const index_pubkey_cache = try allocator.create(Index2PubkeyCache); - errdefer allocator.destroy(index_pubkey_cache); - index_pubkey_cache.* = Index2PubkeyCache.empty; - errdefer index_pubkey_cache.deinit(allocator); + const pubkey_cache = try allocator.create(PubkeyCache); + errdefer allocator.destroy(pubkey_cache); + pubkey_cache.* = try PubkeyCache.initCapacity( + allocator, + std.testing.io, + try std.math.add( + usize, + try state.validatorsCount(), + preset.MAX_PENDING_DEPOSITS_PER_EPOCH, + ), + ); + errdefer pubkey_cache.deinit(); const chain_config = getConfig(active_chain_config, fork, fork_epoch); const config = try allocator.create(BeaconConfig); errdefer allocator.destroy(config); config.* = BeaconConfig.init(chain_config, (try state.genesisValidatorsRoot()).*); - const validators = try state.validatorsPtrSlice(allocator); - defer allocator.free(validators); - - try syncPubkeys(allocator, validators, pubkey_index_map, index_pubkey_cache); - const immutable_data = state_transition.EpochCacheImmutableData{ .config = config, - .index_to_pubkey = index_pubkey_cache, - .pubkey_to_index = pubkey_index_map, + .pubkey_cache = pubkey_cache, }; // cached_state takes ownership of state and will deinit there - const cached_state = try CachedBeaconState.createCachedBeaconState(allocator, state, immutable_data, .{ + const cached_state = try CachedBeaconState.createCachedBeaconState(allocator, std.testing.io, state, immutable_data, .{ .skip_sync_committee_cache = state.forkSeq() == .phase0, .skip_sync_pubkeys = false, }); @@ -231,8 +224,7 @@ pub const TestCachedBeaconState = struct { .allocator = allocator, .pool = pool, .config = config, - .pubkey_index_map = pubkey_index_map, - .index_pubkey_cache = index_pubkey_cache, + .pubkey_cache = pubkey_cache, .cached_state = cached_state, .epoch_transition_cache = epoch_transition_cache, }; @@ -241,13 +233,11 @@ pub const TestCachedBeaconState = struct { pub fn deinit(self: *TestCachedBeaconState) void { self.cached_state.deinit(); self.allocator.destroy(self.cached_state); - self.pubkey_index_map.deinit(); - self.allocator.destroy(self.pubkey_index_map); - self.index_pubkey_cache.deinit(self.allocator); + self.pubkey_cache.deinit(); + self.allocator.destroy(self.pubkey_cache); self.epoch_transition_cache.deinit(self.allocator); - @import("../state_transition.zig").deinitStateTransition(std.testing.io); + @import("../state_transition.zig").deinitReusedEpochTransitionCache(std.testing.io); self.allocator.destroy(self.epoch_transition_cache); - self.allocator.destroy(self.index_pubkey_cache); self.allocator.destroy(self.config); } }; diff --git a/src/state_transition/test_utils/interop_pubkeys.zig b/src/state_transition/test_utils/interop_pubkeys.zig index edcfc010d..55097d364 100644 --- a/src/state_transition/test_utils/interop_pubkeys.zig +++ b/src/state_transition/test_utils/interop_pubkeys.zig @@ -22,7 +22,7 @@ pub fn interopPubkeysCached(validator_count: usize, out: []BLSPubkey) !void { } } -pub fn interopSign(validator_index: usize, message: []const u8) !bls.Signature { +pub fn interopSign(validator_index: usize, message: *const bls.SigningRoot) !bls.Signature { var ikm = [_]u8{0} ** 32; const u64_slice = std.mem.bytesAsSlice(u64, ikm[0..8]); u64_slice[0] = @intCast(validator_index); diff --git a/src/state_transition/utils/bls.zig b/src/state_transition/utils/bls.zig index 83c95f5de..dc6556473 100644 --- a/src/state_transition/utils/bls.zig +++ b/src/state_transition/utils/bls.zig @@ -16,6 +16,7 @@ const bls = @import("bls"); const PublicKey = bls.PublicKey; const Signature = bls.Signature; const SecretKey = bls.SecretKey; +const SigningRoot = bls.SigningRoot; const BlsOpts = struct { /// Decides whether the signature should be group checked. @@ -24,13 +25,13 @@ const BlsOpts = struct { pk_validate: bool = false, }; -pub fn sign(sk: SecretKey, msg: []const u8) Signature { +pub fn sign(sk: SecretKey, msg: *const SigningRoot) Signature { return sk.sign(msg, bls.DST, null); } /// Verify a signature against a message and public key. pub fn verify( - msg: []const u8, + msg: *const SigningRoot, pk: *const PublicKey, sig: *const Signature, opts: BlsOpts, @@ -39,7 +40,7 @@ pub fn verify( } pub fn fastAggregateVerify( - msg: []const u8, + msg: *const SigningRoot, pks: []const PublicKey, sig: *const Signature, opts: BlsOpts, @@ -48,7 +49,7 @@ pub fn fastAggregateVerify( return sig.fastAggregateVerify( opts.sig_groupcheck, &pairing_buf, - msg[0..32], + msg, bls.DST, pks, opts.pk_validate, diff --git a/src/state_transition/utils/electra.zig b/src/state_transition/utils/electra.zig index fda7f4ed0..aa408875e 100644 --- a/src/state_transition/utils/electra.zig +++ b/src/state_transition/utils/electra.zig @@ -7,10 +7,8 @@ const BeaconState = @import("fork_types").BeaconState; const ct = @import("consensus_types"); pub const WithdrawalCredentials = ct.primitive.Root.Type; -const BLSPubkey = ct.primitive.BLSPubkey.Type; const ValidatorIndex = ct.primitive.ValidatorIndex.Type; -const EpochCache = @import("../cache/epoch_cache.zig").EpochCache; const hasEth1WithdrawalCredential = @import("./capella.zig").hasEth1WithdrawalCredential; const G2_POINT_AT_INFINITY = @import("constants").G2_POINT_AT_INFINITY; @@ -72,10 +70,6 @@ pub fn queueExcessActiveBalance( } } -pub fn isPubkeyKnown(comptime fork: ForkSeq, epoch_cache: *const EpochCache, state: *BeaconState(fork), pubkey: BLSPubkey) !bool { - return try isValidatorKnown(fork, state, epoch_cache.getValidatorIndex(&pubkey)); -} - pub fn isValidatorKnown(comptime fork: ForkSeq, state: *BeaconState(fork), index: ?ValidatorIndex) !bool { const validator_index = index orelse return false; const validators_count = try state.validatorsCount(); diff --git a/src/state_transition/utils/signature_sets.zig b/src/state_transition/utils/signature_sets.zig index cf589bc64..be8c06485 100644 --- a/src/state_transition/utils/signature_sets.zig +++ b/src/state_transition/utils/signature_sets.zig @@ -1,8 +1,9 @@ +const std = @import("std"); const types = @import("consensus_types"); pub const bls = @import("bls"); const PublicKey = bls.PublicKey; const Signature = bls.Signature; -const Root = types.primitive.Root.Type; +const SigningRoot = bls.SigningRoot; const BLSSignature = types.primitive.BLSSignature.Type; const verify = @import("./bls.zig").verify; const fastAggregateVerify = @import("./bls.zig").fastAggregateVerify; @@ -12,34 +13,36 @@ pub const SignatureSetType = enum { single, aggregate }; pub const SingleSignatureSet = struct { // fromBytes api return PublicKey so it's more convenient to model this as value pubkey: PublicKey, - signing_root: Root, + signing_root: SigningRoot, signature: BLSSignature, }; pub const AggregatedSignatureSet = struct { // fastAggregateVerify also requires []*const PublicKey pubkeys: []const PublicKey, - signing_root: Root, + signing_root: SigningRoot, signature: BLSSignature, }; pub fn verifySingleSignatureSet(set: *const SingleSignatureSet) !bool { // All signatures are not trusted and must be group checked (p2.subgroup_check) const signature = try Signature.uncompress(&set.signature); - if (verify(&set.signing_root, &set.pubkey, &signature, .{})) { - return true; - } else |_| { - return false; - } + verify(&set.signing_root, &set.pubkey, &signature, .{ .sig_groupcheck = true }) catch return false; + return true; } pub fn verifyAggregatedSignatureSet(set: *const AggregatedSignatureSet) !bool { // All signatures are not trusted and must be group checked (p2.subgroup_check) const signature = try Signature.uncompress(&set.signature); - return fastAggregateVerify(&set.signing_root, set.pubkeys, &signature, .{}); + return fastAggregateVerify( + &set.signing_root, + set.pubkeys, + &signature, + .{ .sig_groupcheck = true }, + ); } -pub fn createSingleSignatureSetFromComponents(pubkey: PublicKey, signing_root: Root, signature: BLSSignature) SingleSignatureSet { +pub fn createSingleSignatureSetFromComponents(pubkey: *const PublicKey, signing_root: SigningRoot, signature: BLSSignature) SingleSignatureSet { return .{ .pubkey = pubkey, .signing_root = signing_root, @@ -47,7 +50,7 @@ pub fn createSingleSignatureSetFromComponents(pubkey: PublicKey, signing_root: R }; } -pub fn createAggregateSignatureSetFromComponents(pubkeys: []const PublicKey, signing_root: Root, signature: BLSSignature) AggregatedSignatureSet { +pub fn createAggregateSignatureSetFromComponents(pubkeys: []const PublicKey, signing_root: SigningRoot, signature: BLSSignature) AggregatedSignatureSet { return .{ .pubkeys = pubkeys, .signing_root = signing_root, @@ -55,4 +58,55 @@ pub fn createAggregateSignatureSetFromComponents(pubkeys: []const PublicKey, sig }; } -// TODO: unit tests +test "signature sets reject signatures outside G2 subgroup" { + const ikm = [_]u8{1} ** 32; + const signing_root = [_]u8{2} ** 32; + const secret_key = try bls.SecretKey.keyGen(&ikm, null); + const public_key = secret_key.toPublicKey(); + const valid_signature = secret_key.sign(&signing_root, bls.DST, null); + const valid_signature_bytes = valid_signature.compress(); + + // Generated by adding [r]P to the valid signature, where P is the first decodable compressed + // curve point with an x-coordinate in the range 0...255. + var forged_signature_bytes: BLSSignature = undefined; + _ = try std.fmt.hexToBytes( + &forged_signature_bytes, + "99de7bff0cf5337f1d09fec80f5c9f5af01436b61656692b9520ca79f8211266" ++ + "8bbedce24f69d56c171a39b0425e508017c04962b77d77d80233bd92f45ba9bb" ++ + "93917964f576e8c5a1d2b9487c1e9b0d99021fb8ecfe5fec421d41c19674b609", + ); + const forged_signature = try Signature.uncompress(&forged_signature_bytes); + try std.testing.expect(!forged_signature.subgroupCheck()); + + // The forged signature adds a torsion component, so decoding succeeds but subgroup validation + // must reject it. + const public_keys = [_]PublicKey{public_key}; + + const single_forged = SingleSignatureSet{ + .pubkey = public_key, + .signing_root = signing_root, + .signature = forged_signature_bytes, + }; + try std.testing.expect(!try verifySingleSignatureSet(&single_forged)); + + const aggregate_forged = AggregatedSignatureSet{ + .pubkeys = &public_keys, + .signing_root = signing_root, + .signature = forged_signature_bytes, + }; + try std.testing.expect(!try verifyAggregatedSignatureSet(&aggregate_forged)); + + const single_valid = SingleSignatureSet{ + .pubkey = public_key, + .signing_root = signing_root, + .signature = valid_signature_bytes, + }; + try std.testing.expect(try verifySingleSignatureSet(&single_valid)); + + const aggregate_valid = AggregatedSignatureSet{ + .pubkeys = &public_keys, + .signing_root = signing_root, + .signature = valid_signature_bytes, + }; + try std.testing.expect(try verifyAggregatedSignatureSet(&aggregate_valid)); +} diff --git a/test/int/era/root.zig b/test/int/era/root.zig index 6380f1acb..6a2af8f2e 100644 --- a/test/int/era/root.zig +++ b/test/int/era/root.zig @@ -83,7 +83,7 @@ test "write an era file from an existing era file" { if (out_reader.era_number != reader.era_number) { return error.IncorrectWrittenEraNumber; } - if (!std.mem.eql(u8, &reader.short_historical_root, &out_reader.short_historical_root)) { + if (!std.mem.eql(u8, &reader.short_era_root, &out_reader.short_era_root)) { return error.IncorrectWrittenShortHistoricalRoot; } diff --git a/test/spec/bls/test_case.zig b/test/spec/bls/test_case.zig index abe7e421d..76f8ddfcc 100644 --- a/test/spec/bls/test_case.zig +++ b/test/spec/bls/test_case.zig @@ -78,7 +78,7 @@ pub fn aggregate_verify(gpa: Allocator, path: std.Io.Dir) !void { const pubkeys = try allocator.alloc(bls.PublicKey, num_sigs); defer allocator.free(pubkeys); - const messages = try allocator.alloc([32]u8, num_sigs); + const messages = try allocator.alloc(bls.SigningRoot, num_sigs); defer allocator.free(messages); var pk_buf: [bls.PublicKey.COMPRESS_SIZE]u8 = undefined; @@ -94,10 +94,11 @@ pub fn aggregate_verify(gpa: Allocator, path: std.Io.Dir) !void { } for (aggregate_verify_test_data.input.messages, 0..) |msg_hex_bytes, i| { - _ = try std.fmt.hexToBytes( + const message = try std.fmt.hexToBytes( messages[i][0..], msg_hex_bytes[2..], // skip "0x" prefix ); + try std.testing.expectEqual(@sizeOf(bls.SigningRoot), message.len); } const sig_bytes = try std.fmt.hexToBytes( @@ -149,7 +150,7 @@ pub fn fast_aggregate_verify(gpa: Allocator, path: std.Io.Dir) !void { const pubkeys = try allocator.alloc(bls.PublicKey, num_sigs); defer allocator.free(pubkeys); - var msg_bytes: [32]u8 = undefined; + var msg_bytes: bls.SigningRoot = undefined; var pk_buf: [bls.PublicKey.COMPRESS_SIZE]u8 = undefined; var sig_buf: [bls.Signature.COMPRESS_SIZE]u8 = undefined; var pairing_buf: [bls.Pairing.sizeOf()]u8 align(bls.Pairing.buf_align) = undefined; @@ -162,10 +163,11 @@ pub fn fast_aggregate_verify(gpa: Allocator, path: std.Io.Dir) !void { pubkeys[i] = try bls.PublicKey.deserialize(pk_bytes); } - _ = try std.fmt.hexToBytes( + const message = try std.fmt.hexToBytes( &msg_bytes, fast_aggregate_verify_test_data.input.message[2..], // skip "0x" prefix ); + try std.testing.expectEqual(@sizeOf(bls.SigningRoot), message.len); const sig_bytes = try std.fmt.hexToBytes( &sig_buf, @@ -213,8 +215,9 @@ pub fn sign(gpa: Allocator, path: std.Io.Dir) !void { var privkey: [32]u8 = undefined; _ = try std.fmt.hexToBytes(&privkey, sign_test_data.input.privkey[2..]); // skip "0x" prefix - var msg: [32]u8 = undefined; - _ = try std.fmt.hexToBytes(&msg, sign_test_data.input.message[2..]); // skip "0x" prefix + var msg: bls.SigningRoot = undefined; + const message = try std.fmt.hexToBytes(&msg, sign_test_data.input.message[2..]); // skip "0x" prefix + try std.testing.expectEqual(@sizeOf(bls.SigningRoot), message.len); const sk = bls.SecretKey.deserialize(&privkey) catch { // if secret key is invalid, expect signature to be "null" @@ -256,7 +259,7 @@ pub fn verify(gpa: Allocator, path: std.Io.Dir) !void { { var pk_buf: [bls.PublicKey.COMPRESS_SIZE]u8 = undefined; var sig_buf: [bls.Signature.COMPRESS_SIZE]u8 = undefined; - var msg_bytes: [32]u8 = undefined; + var msg_bytes: bls.SigningRoot = undefined; const pk_bytes = try std.fmt.hexToBytes(&pk_buf, verify_test_data.input.pubkey[2..]); // skip "0x" prefix const pk = bls.PublicKey.deserialize(pk_bytes) catch { @@ -265,7 +268,8 @@ pub fn verify(gpa: Allocator, path: std.Io.Dir) !void { return; }; - _ = try std.fmt.hexToBytes(&msg_bytes, verify_test_data.input.message[2..]); // skip "0x" prefix + const message = try std.fmt.hexToBytes(&msg_bytes, verify_test_data.input.message[2..]); // skip "0x" prefix + try std.testing.expectEqual(@sizeOf(bls.SigningRoot), message.len); const sig_bytes = try std.fmt.hexToBytes(&sig_buf, verify_test_data.input.signature[2..]); // skip "0x" prefix const signature = bls.Signature.deserialize(sig_bytes) catch { @@ -371,7 +375,7 @@ pub fn eth_fast_aggregate_verify(gpa: Allocator, path: std.Io.Dir) !void { const pubkeys = try allocator.alloc(bls.PublicKey, num_sigs); defer allocator.free(pubkeys); - var msg_bytes: [32]u8 = undefined; + var msg_bytes: bls.SigningRoot = undefined; var pk_buf: [bls.PublicKey.COMPRESS_SIZE]u8 = undefined; var sig_buf: [bls.Signature.COMPRESS_SIZE]u8 = undefined; var pairing_buf: [bls.Pairing.sizeOf()]u8 align(bls.Pairing.buf_align) = undefined; @@ -388,10 +392,11 @@ pub fn eth_fast_aggregate_verify(gpa: Allocator, path: std.Io.Dir) !void { }; } - _ = try std.fmt.hexToBytes( + const message = try std.fmt.hexToBytes( &msg_bytes, eth_fast_aggregate_verify_test_data.input.message[2..], // skip "0x" prefix ); + try std.testing.expectEqual(@sizeOf(bls.SigningRoot), message.len); const sig_bytes = try std.fmt.hexToBytes( &sig_buf, diff --git a/test/spec/runner/epoch_processing.zig b/test/spec/runner/epoch_processing.zig index adacca1c9..c714340b9 100644 --- a/test/spec/runner/epoch_processing.zig +++ b/test/spec/runner/epoch_processing.zig @@ -80,7 +80,7 @@ pub fn TestCase(comptime fork: ForkSeq, comptime epoch_process_fn: EpochProcessi post.deinit(); self.pre.allocator.destroy(post); } - state_transition.deinitStateTransition(std.testing.io); + state_transition.deinitReusedEpochTransitionCache(std.testing.io); } fn runTest(self: *Self) !void { @@ -131,7 +131,7 @@ pub fn TestCase(comptime fork: ForkSeq, comptime epoch_process_fn: EpochProcessi .slashings_reset => try state_transition.processSlashingsReset(fork, epoch_cache, fork_state, &epoch_transition_cache), .sync_committee_updates => try state_transition.processSyncCommitteeUpdates(fork, allocator, epoch_cache, fork_state), .historical_summaries_update => try state_transition.processHistoricalSummariesUpdate(fork, fork_state, &epoch_transition_cache), - .pending_deposits, .pending_deposits_churn => try state_transition.processPendingDeposits(fork, allocator, config, epoch_cache, fork_state, &epoch_transition_cache), + .pending_deposits, .pending_deposits_churn => try state_transition.processPendingDeposits(fork, allocator, std.testing.io, config, epoch_cache, fork_state, &epoch_transition_cache), .pending_consolidations => try state_transition.processPendingConsolidations(fork, epoch_cache, fork_state, &epoch_transition_cache), .proposer_lookahead => { try state_transition.processProposerLookahead(fork, allocator, epoch_cache, fork_state, &epoch_transition_cache); diff --git a/test/spec/runner/fork.zig b/test/spec/runner/fork.zig index f15a1b93d..a8fe7f1b9 100644 --- a/test/spec/runner/fork.zig +++ b/test/spec/runner/fork.zig @@ -52,7 +52,7 @@ pub fn TestCase(comptime target_fork: ForkSeq) type { var tc = try Self.init(allocator, &pool, dir); defer { tc.deinit(); - state_transition.deinitStateTransition(std.testing.io); + state_transition.deinitReusedEpochTransitionCache(std.testing.io); } try tc.runTest(); diff --git a/test/spec/runner/operations.zig b/test/spec/runner/operations.zig index 4ca8da236..ab6ab3963 100644 --- a/test/spec/runner/operations.zig +++ b/test/spec/runner/operations.zig @@ -115,7 +115,7 @@ pub fn TestCase(comptime fork: ForkSeq, comptime operation: Operation) type { var tc = try Self.init(allocator, &pool, dir); defer { tc.deinit(); - state_transition.deinitStateTransition(std.testing.io); + state_transition.deinitReusedEpochTransitionCache(std.testing.io); } try tc.runTest(); @@ -166,6 +166,7 @@ pub fn TestCase(comptime fork: ForkSeq, comptime operation: Operation) type { pub fn process(self: *Self) !void { const verify = self.bls_setting.verify(); const allocator = self.pre.allocator; + const io = std.testing.io; const cached_state = self.pre.cached_state; const state = cached_state.state.castToFork(fork); @@ -177,6 +178,7 @@ pub fn TestCase(comptime fork: ForkSeq, comptime operation: Operation) type { try state_transition.processAttestations( fork, allocator, + io, config, epoch_cache, state, @@ -192,6 +194,7 @@ pub fn TestCase(comptime fork: ForkSeq, comptime operation: Operation) type { try state_transition.processAttesterSlashing( fork, allocator, + io, config, epoch_cache, state, @@ -228,12 +231,12 @@ pub fn TestCase(comptime fork: ForkSeq, comptime operation: Operation) type { .consolidation_request => { const config = cached_state.config; const epoch_cache = cached_state.epoch_cache; - try state_transition.processConsolidationRequest(fork, config, epoch_cache, state, &self.op); + try state_transition.processConsolidationRequest(fork, io, config, epoch_cache, state, &self.op); }, .deposit => { const config = cached_state.config; const epoch_cache = cached_state.epoch_cache; - try state_transition.processDeposit(fork, allocator, config, epoch_cache, state, &self.op); + try state_transition.processDeposit(fork, allocator, io, config, epoch_cache, state, &self.op); }, .deposit_request => { try state_transition.processDepositRequest(fork, state, &self.op); @@ -279,6 +282,7 @@ pub fn TestCase(comptime fork: ForkSeq, comptime operation: Operation) type { try state_transition.processProposerSlashing( fork, allocator, + io, config, epoch_cache, state, @@ -293,6 +297,7 @@ pub fn TestCase(comptime fork: ForkSeq, comptime operation: Operation) type { try state_transition.processSyncAggregate( fork, allocator, + io, config, epoch_cache, state, @@ -305,6 +310,7 @@ pub fn TestCase(comptime fork: ForkSeq, comptime operation: Operation) type { const epoch_cache = cached_state.epoch_cache; try state_transition.processVoluntaryExit( fork, + io, config, epoch_cache, state, @@ -315,7 +321,7 @@ pub fn TestCase(comptime fork: ForkSeq, comptime operation: Operation) type { .withdrawal_request => { const config = cached_state.config; const epoch_cache = cached_state.epoch_cache; - try state_transition.processWithdrawalRequest(fork, config, epoch_cache, state, &self.op); + try state_transition.processWithdrawalRequest(fork, io, config, epoch_cache, state, &self.op); }, .withdrawals => { const epoch_cache = cached_state.epoch_cache; diff --git a/test/spec/runner/rewards.zig b/test/spec/runner/rewards.zig index 2826693c9..e4dd72900 100644 --- a/test/spec/runner/rewards.zig +++ b/test/spec/runner/rewards.zig @@ -46,7 +46,7 @@ pub fn TestCase(comptime fork: ForkSeq) type { var tc = try Self.init(allocator, &pool, dir); defer { tc.deinit(); - state_transition.deinitStateTransition(std.testing.io); + state_transition.deinitReusedEpochTransitionCache(std.testing.io); } try tc.runTest(); diff --git a/test/spec/runner/sanity.zig b/test/spec/runner/sanity.zig index 6cd4e3fca..09e08aff4 100644 --- a/test/spec/runner/sanity.zig +++ b/test/spec/runner/sanity.zig @@ -39,7 +39,7 @@ pub fn SlotsTestCase(comptime fork: ForkSeq) type { var tc = try Self.init(allocator, pool, dir); defer { tc.deinit(); - state_transition.deinitStateTransition(std.testing.io); + state_transition.deinitReusedEpochTransitionCache(std.testing.io); } try tc.runTest(); @@ -110,7 +110,7 @@ pub fn BlocksTestCase(comptime fork: ForkSeq) type { var tc = try Self.init(allocator, pool, dir); defer { tc.deinit(); - state_transition.deinitStateTransition(std.testing.io); + state_transition.deinitReusedEpochTransitionCache(std.testing.io); } try tc.runTest(); diff --git a/test/spec/runner/transition.zig b/test/spec/runner/transition.zig index efb334713..ba1670bd0 100644 --- a/test/spec/runner/transition.zig +++ b/test/spec/runner/transition.zig @@ -29,7 +29,7 @@ pub fn Transition(comptime fork: ForkSeq) type { var tc = try Self.init(allocator, &pool, dir); defer { tc.deinit(); - state_transition.deinitStateTransition(std.testing.io); + state_transition.deinitReusedEpochTransitionCache(std.testing.io); } try tc.runTest(); } From 70ab9123bc097c783ebf21e275860903e32da2e4 Mon Sep 17 00:00:00 2001 From: Rahul Guha <19rahul2003@gmail.com> Date: Mon, 10 Aug 2026 21:50:49 +0530 Subject: [PATCH 29/33] refactor(state-transition): thread std.Io through gloas paths --- .../block/is_valid_indexed_payload_attestation.zig | 3 ++- src/state_transition/block/process_block.zig | 2 +- src/state_transition/block/process_operations.zig | 2 +- .../block/process_parent_execution_payload.zig | 8 +++++--- .../block/process_payload_attestation.zig | 3 ++- .../signature_sets/indexed_payload_attestation.zig | 8 +++++--- src/state_transition/slot/upgrade_state_to_gloas.zig | 6 ++++-- src/state_transition/state_transition.zig | 2 +- test/spec/runner/fork.zig | 1 + test/spec/runner/operations.zig | 4 ++-- 10 files changed, 24 insertions(+), 15 deletions(-) diff --git a/src/state_transition/block/is_valid_indexed_payload_attestation.zig b/src/state_transition/block/is_valid_indexed_payload_attestation.zig index 50d08c032..2fe43b650 100644 --- a/src/state_transition/block/is_valid_indexed_payload_attestation.zig +++ b/src/state_transition/block/is_valid_indexed_payload_attestation.zig @@ -10,6 +10,7 @@ const verifyAggregatedSignatureSet = @import("../utils/signature_sets.zig").veri /// sorted, and (optionally) that the aggregate BLS signature is valid. pub fn isValidIndexedPayloadAttestation( allocator: std.mem.Allocator, + io: std.Io, config: *const BeaconConfig, epoch_cache: *const EpochCache, indexed_payload_attestation: *const types.gloas.IndexedPayloadAttestation.Type, @@ -29,7 +30,7 @@ pub fn isValidIndexedPayloadAttestation( if (!verify_signature) return true; - const sig_set = try getIndexedPayloadAttestationSignatureSet(allocator, config, epoch_cache, indexed_payload_attestation); + const sig_set = try getIndexedPayloadAttestationSignatureSet(allocator, io, config, epoch_cache, indexed_payload_attestation); defer allocator.free(sig_set.pubkeys); return verifyAggregatedSignatureSet(&sig_set); } diff --git a/src/state_transition/block/process_block.zig b/src/state_transition/block/process_block.zig index 930a13224..2c529f83b 100644 --- a/src/state_transition/block/process_block.zig +++ b/src/state_transition/block/process_block.zig @@ -53,7 +53,7 @@ pub fn processBlock( // Build slashings cache against the *current* latest_block_header slot (pre-header update). try buildSlashingsCacheIfNeeded(allocator, state, slashings_cache); if (comptime fork.gte(.gloas) and block_type == .full) { - try processParentExecutionPayload(allocator, config, epoch_cache, state, block); + try processParentExecutionPayload(allocator, io, config, epoch_cache, state, block); } try processBlockHeader(fork, allocator, epoch_cache, state, block_type, block); // Keep cache slot in sync with latest_block_header without forcing a rebuild. diff --git a/src/state_transition/block/process_operations.zig b/src/state_transition/block/process_operations.zig index d3eb19028..481108c1d 100644 --- a/src/state_transition/block/process_operations.zig +++ b/src/state_transition/block/process_operations.zig @@ -97,7 +97,7 @@ pub fn processOperations( if (comptime fork.gte(.gloas)) { for (body.inner.payload_attestations.items) |*payload_attestation| { - try processPayloadAttestation(allocator, config, epoch_cache, state, payload_attestation); + try processPayloadAttestation(allocator, io, config, epoch_cache, state, payload_attestation); } } } diff --git a/src/state_transition/block/process_parent_execution_payload.zig b/src/state_transition/block/process_parent_execution_payload.zig index 94f44da18..d7ffaa2d7 100644 --- a/src/state_transition/block/process_parent_execution_payload.zig +++ b/src/state_transition/block/process_parent_execution_payload.zig @@ -18,6 +18,7 @@ const computeEpochAtSlot = @import("../utils/epoch.zig").computeEpochAtSlot; /// Spec: https://github.com/ethereum/consensus-specs/blob/v1.7.0-alpha.6/specs/gloas/beacon-chain.md#new-process_parent_execution_payload pub fn processParentExecutionPayload( allocator: Allocator, + io: std.Io, config: *const BeaconConfig, epoch_cache: *EpochCache, state: *BeaconState(.gloas), @@ -41,7 +42,7 @@ pub fn processParentExecutionPayload( return error.ParentExecutionRequestsRootMismatch; } - try applyParentExecutionPayload(allocator, config, epoch_cache, state, requests, &parent_bid); + try applyParentExecutionPayload(allocator, io, config, epoch_cache, state, requests, &parent_bid); } /// Process the parent's execution requests, queue the builder payment, update payload availability, @@ -53,6 +54,7 @@ pub fn processParentExecutionPayload( /// Spec: https://github.com/ethereum/consensus-specs/blob/v1.7.0-alpha.6/specs/gloas/beacon-chain.md#new-apply_parent_execution_payload pub fn applyParentExecutionPayload( allocator: Allocator, + io: std.Io, config: *const BeaconConfig, epoch_cache: *EpochCache, state: *BeaconState(.gloas), @@ -68,11 +70,11 @@ pub fn applyParentExecutionPayload( } for (requests.withdrawals.items) |*withdrawal| { - try processWithdrawalRequest(.gloas, config, epoch_cache, state, withdrawal); + try processWithdrawalRequest(.gloas, io, config, epoch_cache, state, withdrawal); } for (requests.consolidations.items) |*consolidation| { - try processConsolidationRequest(.gloas, config, epoch_cache, state, consolidation); + try processConsolidationRequest(.gloas, io, config, epoch_cache, state, consolidation); } for (requests.builder_deposits.items) |*builder_deposit| { diff --git a/src/state_transition/block/process_payload_attestation.zig b/src/state_transition/block/process_payload_attestation.zig index ac7aa48a5..bb115527e 100644 --- a/src/state_transition/block/process_payload_attestation.zig +++ b/src/state_transition/block/process_payload_attestation.zig @@ -8,6 +8,7 @@ const isValidIndexedPayloadAttestation = @import("./is_valid_indexed_payload_att pub fn processPayloadAttestation( allocator: Allocator, + io: std.Io, config: *const BeaconConfig, epoch_cache: *const EpochCache, state: *BeaconState(.gloas), @@ -28,7 +29,7 @@ pub fn processPayloadAttestation( var indexed_payload_attestation = try epoch_cache.getIndexedPayloadAttestation(allocator, state, data.slot, payload_attestation); defer indexed_payload_attestation.attesting_indices.deinit(allocator); - if (!(try isValidIndexedPayloadAttestation(allocator, config, epoch_cache, &indexed_payload_attestation, true))) { + if (!(try isValidIndexedPayloadAttestation(allocator, io, config, epoch_cache, &indexed_payload_attestation, true))) { return error.InvalidPayloadAttestation; } } diff --git a/src/state_transition/signature_sets/indexed_payload_attestation.zig b/src/state_transition/signature_sets/indexed_payload_attestation.zig index 901798f6e..d57b49709 100644 --- a/src/state_transition/signature_sets/indexed_payload_attestation.zig +++ b/src/state_transition/signature_sets/indexed_payload_attestation.zig @@ -23,6 +23,7 @@ pub fn getPayloadAttestationDataSigningRoot(config: *const BeaconConfig, data: * /// Consumer needs to free the returned pubkeys array. pub fn getIndexedPayloadAttestationSignatureSet( allocator: Allocator, + io: std.Io, config: *const BeaconConfig, epoch_cache: *const EpochCache, indexed_payload_attestation: *const types.gloas.IndexedPayloadAttestation.Type, @@ -31,9 +32,10 @@ pub fn getIndexedPayloadAttestationSignatureSet( const pubkeys = try allocator.alloc(PublicKey, attesting_indices.len); errdefer allocator.free(pubkeys); - for (attesting_indices, 0..) |index, i| { - pubkeys[i] = epoch_cache.index_to_pubkey.items[index]; - } + epoch_cache.pubkey_cache.getPubkeys(io, attesting_indices, pubkeys) catch |err| switch (err) { + error.InvalidIndex => return error.PubkeyNotFound, + else => return err, + }; var signing_root: Root = undefined; try getPayloadAttestationDataSigningRoot(config, &indexed_payload_attestation.data, &signing_root); diff --git a/src/state_transition/slot/upgrade_state_to_gloas.zig b/src/state_transition/slot/upgrade_state_to_gloas.zig index 4443c2388..3fc7ef7fc 100644 --- a/src/state_transition/slot/upgrade_state_to_gloas.zig +++ b/src/state_transition/slot/upgrade_state_to_gloas.zig @@ -23,6 +23,7 @@ const PendingDepositsLookup = @import("../utils/pending_deposits_lookup.zig").Pe /// Upgrade a state from Fulu to Gloas. pub fn upgradeStateToGloas( allocator: Allocator, + io: std.Io, config: *const BeaconConfig, epoch_cache: *const EpochCache, fulu_state: *BeaconState(.fulu), @@ -59,7 +60,7 @@ pub fn upgradeStateToGloas( const ptc_window = try initializePtcWindow(.gloas, allocator, epoch_cache, &state); try state.inner.setValue("ptc_window", &ptc_window); - try onboardBuildersFromPendingDeposits(allocator, config, epoch_cache, &state); + try onboardBuildersFromPendingDeposits(allocator, io, config, epoch_cache, &state); fulu_state.deinit(); return state; @@ -69,6 +70,7 @@ pub fn upgradeStateToGloas( /// Spec: https://github.com/ethereum/consensus-specs/blob/v1.7.0-alpha.8/specs/gloas/fork.md#new-onboard_builders_from_pending_deposits fn onboardBuildersFromPendingDeposits( allocator: Allocator, + io: std.Io, config: *const BeaconConfig, epoch_cache: *const EpochCache, state: *BeaconState(.gloas), @@ -86,7 +88,7 @@ fn onboardBuildersFromPendingDeposits( for (0..pending_deposits_len) |_| { const deposit = try pending_it.nextValue(); - const validator_index = epoch_cache.getValidatorIndex(&deposit.pubkey); + const validator_index = epoch_cache.pubkey_cache.get(io, deposit.pubkey); if (try isValidatorKnown(.gloas, state, validator_index)) { try remaining_pending_deposits.append(allocator, deposit); try pending_deposits_lookup.add(&deposit); diff --git a/src/state_transition/state_transition.zig b/src/state_transition/state_transition.zig index faa22f208..90367da34 100644 --- a/src/state_transition/state_transition.zig +++ b/src/state_transition/state_transition.zig @@ -135,7 +135,7 @@ pub fn processSlots( } if (state_epoch == config.chain.GLOAS_FORK_EPOCH) { const fulu_state = try state.tryCastToFork(.fulu); - const upgraded = try upgradeStateToGloas(allocator, config, epoch_cache, fulu_state); + const upgraded = try upgradeStateToGloas(allocator, io, config, epoch_cache, fulu_state); state.* = .{ .gloas = upgraded.inner }; } diff --git a/test/spec/runner/fork.zig b/test/spec/runner/fork.zig index a8fe7f1b9..69d4d0199 100644 --- a/test/spec/runner/fork.zig +++ b/test/spec/runner/fork.zig @@ -157,6 +157,7 @@ pub fn TestCase(comptime target_fork: ForkSeq) type { .gloas => { const upgraded = try upgradeStateToGloas( self.pre.allocator, + std.testing.io, config, epoch_cache, try cached_state.state.tryCastToFork(.fulu), diff --git a/test/spec/runner/operations.zig b/test/spec/runner/operations.zig index ab6ab3963..2d30cfb6d 100644 --- a/test/spec/runner/operations.zig +++ b/test/spec/runner/operations.zig @@ -269,12 +269,12 @@ pub fn TestCase(comptime fork: ForkSeq, comptime operation: Operation) type { const config = cached_state.config; const epoch_cache = cached_state.epoch_cache; const fork_block = BeaconBlock(.full, fork){ .inner = self.op }; - try state_transition.processParentExecutionPayload(allocator, config, epoch_cache, state, &fork_block); + try state_transition.processParentExecutionPayload(allocator, std.testing.io, config, epoch_cache, state, &fork_block); }, .payload_attestation => { const config = cached_state.config; const epoch_cache = cached_state.epoch_cache; - try state_transition.processPayloadAttestation(allocator, config, epoch_cache, state, &self.op); + try state_transition.processPayloadAttestation(allocator, std.testing.io, config, epoch_cache, state, &self.op); }, .proposer_slashing => { const config = cached_state.config; From 3085fa92e685846bf4d1e2bddd06eefb1c7e4650 Mon Sep 17 00:00:00 2001 From: Rahul Guha <19rahul2003@gmail.com> Date: Tue, 11 Aug 2026 15:41:47 +0530 Subject: [PATCH 30/33] fix(state-transition): correct error-path cleanup in shuffling and committees --- .../cache/epoch_transition_cache.zig | 2 +- src/state_transition/utils/committee_indices.zig | 1 + src/state_transition/utils/epoch_shuffling.zig | 13 +++++++++++++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/state_transition/cache/epoch_transition_cache.zig b/src/state_transition/cache/epoch_transition_cache.zig index f3cd80ca9..7f31ca119 100644 --- a/src/state_transition/cache/epoch_transition_cache.zig +++ b/src/state_transition/cache/epoch_transition_cache.zig @@ -591,9 +591,9 @@ pub const EpochTransitionCache = struct { if (self.next_shuffling) |next_shuffling| return next_shuffling; const active_indices = try allocator.alloc(ValidatorIndex, self.next_shuffling_active_indices.len); - errdefer allocator.free(active_indices); std.mem.copyForwards(ValidatorIndex, active_indices, self.next_shuffling_active_indices); + // computeEpochShuffling takes ownership of active_indices on both success and failure. self.next_shuffling = try computeEpochShuffling( allocator, state, diff --git a/src/state_transition/utils/committee_indices.zig b/src/state_transition/utils/committee_indices.zig index 98c99ef2d..d51a869be 100644 --- a/src/state_transition/utils/committee_indices.zig +++ b/src/state_transition/utils/committee_indices.zig @@ -29,6 +29,7 @@ const ComputeShuffledIndex = struct { } var arena = std.heap.ArenaAllocator.init(parent_allocator); + errdefer arena.deinit(); const allocator = arena.allocator(); const pivot_by_index = try allocator.alloc(?u64, @intCast(rounds)); diff --git a/src/state_transition/utils/epoch_shuffling.zig b/src/state_transition/utils/epoch_shuffling.zig index 195d15970..e8d97b133 100644 --- a/src/state_transition/utils/epoch_shuffling.zig +++ b/src/state_transition/utils/epoch_shuffling.zig @@ -42,6 +42,7 @@ pub const EpochShuffling = struct { std.mem.copyForwards(ValidatorIndex, shuffling, active_indices); try unshuffleList(shuffling, seed[0..], preset.SHUFFLE_ROUND_COUNT); const committees = try buildCommitteesFromShuffling(allocator, shuffling); + errdefer freeCommittees(allocator, committees); const epoch_shuffling_ptr = try allocator.create(EpochShuffling); errdefer allocator.destroy(epoch_shuffling_ptr); @@ -74,6 +75,11 @@ pub const EpochShuffling = struct { const committee_count = committees_per_slot * preset.SLOTS_PER_EPOCH; var epoch_committees: [preset.SLOTS_PER_EPOCH]SlotCommittees = undefined; + var initialized_slots: usize = 0; + errdefer for (epoch_committees[0..initialized_slots]) |slot_committees| { + allocator.free(slot_committees); + }; + for (0..preset.SLOTS_PER_EPOCH) |slot| { const slot_committees = try allocator.alloc(Committee, committees_per_slot); for (0..committees_per_slot) |committee_index| { @@ -83,10 +89,17 @@ pub const EpochShuffling = struct { slot_committees[committee_index] = shuffling[start_offset..end_offset]; } epoch_committees[slot] = slot_committees; + initialized_slots += 1; } return epoch_committees; } + + fn freeCommittees(allocator: Allocator, committees: EpochCommittees) void { + for (committees) |slot_committees| { + allocator.free(slot_committees); + } + } }; test EpochShuffling { From 3b465d1cbf462f9fea15512a0aec90a3ed602d08 Mon Sep 17 00:00:00 2001 From: Rahul Guha <19rahul2003@gmail.com> Date: Tue, 11 Aug 2026 15:43:19 +0530 Subject: [PATCH 31/33] test(state-transition): add gloas block generator --- .../test_utils/generate_block.zig | 26 +++++++++++++++++++ src/state_transition/test_utils/root.zig | 1 + 2 files changed, 27 insertions(+) diff --git a/src/state_transition/test_utils/generate_block.zig b/src/state_transition/test_utils/generate_block.zig index 404a3c641..f286b4d69 100644 --- a/src/state_transition/test_utils/generate_block.zig +++ b/src/state_transition/test_utils/generate_block.zig @@ -9,7 +9,9 @@ const state_transition = @import("../root.zig"); const CachedBeaconState = state_transition.CachedBeaconState; const computeStartSlotAtEpoch = state_transition.computeStartSlotAtEpoch; const getBlockRootAtSlot = state_transition.getBlockRootAtSlot; +const getRandaoMix = state_transition.getRandaoMix; const BeaconState = @import("fork_types").BeaconState; +const c = @import("constants"); /// Generate a valid electra block for the given pre-state. pub fn generateElectraBlock(allocator: Allocator, cached_state: *CachedBeaconState, out: *types.electra.SignedBeaconBlock.Type) !void { @@ -97,3 +99,27 @@ pub fn generateElectraBlock(allocator: Allocator, cached_state: *CachedBeaconSta .signature = types.primitive.BLSSignature.default_value, }; } + +/// Generate a valid empty Gloas block with a self-build execution payload bid. +pub fn generateGloasBlock(cached_state: *CachedBeaconState, out: *types.gloas.BeaconBlock.Type) !void { + const state = try cached_state.state.tryCastToFork(.gloas); + const slot = try state.slot(); + + var latest_header = try state.latestBlockHeader(); + const parent_root = try latest_header.hashTreeRoot(); + const parent_block_root = try getBlockRootAtSlot(.gloas, state, slot - 1); + const parent_block_hash = try state.inner.getFieldRoot("latest_block_hash"); + const prev_randao = try getRandaoMix(.gloas, state, cached_state.epoch_cache.epoch); + + out.* = types.gloas.BeaconBlock.default_value; + out.slot = slot; + out.proposer_index = try cached_state.getBeaconProposer(slot); + out.parent_root = parent_root.*; + out.body.signed_execution_payload_bid.message.parent_block_hash = parent_block_hash.*; + out.body.signed_execution_payload_bid.message.parent_block_root = parent_block_root.*; + out.body.signed_execution_payload_bid.message.prev_randao = prev_randao.*; + out.body.signed_execution_payload_bid.message.builder_index = c.BUILDER_INDEX_SELF_BUILD; + out.body.signed_execution_payload_bid.message.slot = slot; + out.body.signed_execution_payload_bid.message.value = 0; + out.body.signed_execution_payload_bid.signature = c.G2_POINT_AT_INFINITY; +} diff --git a/src/state_transition/test_utils/root.zig b/src/state_transition/test_utils/root.zig index a4ff2d5f5..ee7a213f5 100644 --- a/src/state_transition/test_utils/root.zig +++ b/src/state_transition/test_utils/root.zig @@ -4,6 +4,7 @@ const testing = std.testing; /// Utils that could be used for different kinds of tests like int, perf pub const TestCachedBeaconState = @import("./generate_state.zig").TestCachedBeaconState; pub const generateElectraBlock = @import("./generate_block.zig").generateElectraBlock; +pub const generateGloasBlock = @import("./generate_block.zig").generateGloasBlock; pub const interopSign = @import("./interop_pubkeys.zig").interopSign; test { From 7bb374f50d880196f1529f67ecc8a09793324dd6 Mon Sep 17 00:00:00 2001 From: Rahul Guha <19rahul2003@gmail.com> Date: Tue, 11 Aug 2026 15:48:23 +0530 Subject: [PATCH 32/33] test(state-transition): add gloas OOM/double-free fault injection suite --- build.zig.zon | 1 + src/state_transition/block/process_block.zig | 75 +++++++++ .../block/process_execution_payload_bid.zig | 116 +++++++++++++ .../block/process_payload_attestation.zig | 55 +++++++ .../block/process_withdrawals.zig | 152 ++++++++++++++++++ .../epoch/process_proposer_lookahead.zig | 55 +++++++ .../epoch/process_ptc_window.zig | 59 +++++++ .../indexed_payload_attestation.zig | 29 ++++ .../slot/upgrade_state_to_gloas.zig | 59 +++++++ .../test_utils/generate_state.zig | 55 +++++++ src/state_transition/utils/gloas.zig | 44 +++++ 11 files changed, 700 insertions(+) diff --git a/build.zig.zon b/build.zig.zon index 722ca67c7..cc149996e 100644 --- a/build.zig.zon +++ b/build.zig.zon @@ -201,6 +201,7 @@ .metrics, .hashing, .time, + .testing_allocators, }, }, .era = .{ diff --git a/src/state_transition/block/process_block.zig b/src/state_transition/block/process_block.zig index 2c529f83b..06f2f71aa 100644 --- a/src/state_transition/block/process_block.zig +++ b/src/state_transition/block/process_block.zig @@ -163,3 +163,78 @@ pub fn processBlock( } } } + +const Node = @import("persistent_merkle_tree").Node; +const TestCachedBeaconState = @import("../test_utils/root.zig").TestCachedBeaconState; +const generateGloasBlock = @import("../test_utils/root.zig").generateGloasBlock; +const DoubleFreeDetectAllocator = @import("testing_allocators").DoubleFreeDetectAllocator; + +fn tryGloasBlock( + allocator: Allocator, + config: *const BeaconConfig, + epoch_cache: *EpochCache, + baseline: *BeaconState(.gloas), + block: *const types.gloas.BeaconBlock.Type, +) !void { + var state = try baseline.clone(.{ .transfer_cache = false }); + defer state.deinit(); + var slashings_cache = try SlashingsCache.initEmpty(std.testing.allocator); + defer slashings_cache.deinit(); + const fork_block = BeaconBlock(.full, .gloas){ .inner = block.* }; + + try processBlock( + .gloas, + allocator, + std.testing.io, + config, + epoch_cache, + &state, + &slashings_cache, + .full, + &fork_block, + .{}, + .{ .verify_signature = false }, + ); +} + +test "Gloas process block - OOM does not leak or double-free transient allocations" { + const allocator = std.testing.allocator; + var pool = try Node.Pool.init(.{ .page_allocator = allocator, .allocator = allocator, .pool_size = 500_000 }); + defer pool.deinit(); + + var test_state = try TestCachedBeaconState.initGloas(allocator, &pool, 256); + defer test_state.deinit(); + + var block: types.gloas.BeaconBlock.Type = undefined; + try generateGloasBlock(test_state.cached_state, &block); + + var saw_oom = false; + var saw_success = false; + var fail_at: usize = 0; + while (fail_at < 512) : (fail_at += 1) { + var oom = DoubleFreeDetectAllocator.init(std.testing.allocator, fail_at); + defer oom.deinit(); + + tryGloasBlock( + oom.allocator(), + test_state.cached_state.config, + test_state.cached_state.epoch_cache, + test_state.cached_state.state.castToFork(.gloas), + &block, + ) catch |err| switch (err) { + error.OutOfMemory => { + saw_oom = true; + try std.testing.expect(!oom.double_free); + continue; + }, + else => return err, + }; + + try std.testing.expect(!oom.double_free); + saw_success = true; + break; + } + + try std.testing.expect(saw_oom); + try std.testing.expect(saw_success); +} diff --git a/src/state_transition/block/process_execution_payload_bid.zig b/src/state_transition/block/process_execution_payload_bid.zig index 7313b680e..929c10407 100644 --- a/src/state_transition/block/process_execution_payload_bid.zig +++ b/src/state_transition/block/process_execution_payload_bid.zig @@ -95,3 +95,119 @@ fn verifyExecutionPayloadBidSignature( verify(&signing_root, &public_key, &signature, .{}) catch return false; return true; } + +const Node = @import("persistent_merkle_tree").Node; +const TestCachedBeaconState = @import("../test_utils/root.zig").TestCachedBeaconState; +const interopSign = @import("../test_utils/root.zig").interopSign; +const DoubleFreeDetectAllocator = @import("testing_allocators").DoubleFreeDetectAllocator; + +fn addTestBuilder(state: *BeaconState(.gloas), amount: u64) !void { + var validators = try state.validators(); + var validator: types.phase0.Validator.Type = undefined; + try validators.getValue(undefined, 0, &validator); + + const builder = types.gloas.Builder.Type{ + .pubkey = validator.pubkey, + .version = c.PAYLOAD_BUILDER_VERSION, + .execution_address = [_]u8{0x42} ** 20, + .balance = preset.MIN_DEPOSIT_AMOUNT + amount, + .deposit_epoch = 0, + .withdrawable_epoch = c.FAR_FUTURE_EPOCH, + }; + var builders = try state.inner.get("builders"); + try builders.pushValue(&builder); + try state.inner.set("builders", builders); + try state.commit(); +} + +fn makeSignedTestBid( + allocator: Allocator, + config: *const BeaconConfig, + epoch_cache: *const EpochCache, + state: *BeaconState(.gloas), + amount: u64, +) !types.gloas.SignedExecutionPayloadBid.Type { + const slot = try state.slot(); + const parent_block_hash = try state.inner.getFieldRoot("latest_block_hash"); + const parent_block_root = try getBlockRootAtSlot(.gloas, state, slot - 1); + const prev_randao = try getRandaoMix(.gloas, state, epoch_cache.epoch); + + var signed_bid = types.gloas.SignedExecutionPayloadBid.default_value; + signed_bid.message.parent_block_hash = parent_block_hash.*; + signed_bid.message.parent_block_root = parent_block_root.*; + signed_bid.message.prev_randao = prev_randao.*; + signed_bid.message.builder_index = 0; + signed_bid.message.slot = slot; + signed_bid.message.value = amount; + try signed_bid.message.blob_kzg_commitments.append( + allocator, + types.primitive.KZGCommitment.default_value, + ); + + const signing_root = try getExecutionPayloadBidSigningRoot(allocator, config, slot, &signed_bid.message); + signed_bid.signature = (try interopSign(0, &signing_root)).compress(); + return signed_bid; +} + +fn tryExternalBuilderBid( + allocator: Allocator, + config: *const BeaconConfig, + epoch_cache: *const EpochCache, + baseline: *BeaconState(.gloas), + signed_bid: *const types.gloas.SignedExecutionPayloadBid.Type, +) !void { + var state = try baseline.clone(.{ .transfer_cache = false }); + defer state.deinit(); + try processExecutionPayloadBid(allocator, config, epoch_cache, &state, signed_bid); +} + +test "Gloas external builder bid - OOM does not leak or double-free transient allocations" { + const allocator = std.testing.allocator; + var pool = try Node.Pool.init(.{ .page_allocator = allocator, .allocator = allocator, .pool_size = 500_000 }); + defer pool.deinit(); + + var test_state = try TestCachedBeaconState.initGloas(allocator, &pool, 256); + defer test_state.deinit(); + const state = test_state.cached_state.state.castToFork(.gloas); + const bid_amount = 1; + try addTestBuilder(state, bid_amount); + + var signed_bid = try makeSignedTestBid( + allocator, + test_state.cached_state.config, + test_state.cached_state.epoch_cache, + state, + bid_amount, + ); + defer types.gloas.SignedExecutionPayloadBid.deinit(allocator, &signed_bid); + + var saw_oom = false; + var saw_success = false; + var fail_at: usize = 0; + while (fail_at < 128) : (fail_at += 1) { + var oom = DoubleFreeDetectAllocator.init(std.testing.allocator, fail_at); + defer oom.deinit(); + + tryExternalBuilderBid( + oom.allocator(), + test_state.cached_state.config, + test_state.cached_state.epoch_cache, + state, + &signed_bid, + ) catch |err| switch (err) { + error.OutOfMemory => { + saw_oom = true; + try std.testing.expect(!oom.double_free); + continue; + }, + else => return err, + }; + + try std.testing.expect(!oom.double_free); + saw_success = true; + break; + } + + try std.testing.expect(saw_oom); + try std.testing.expect(saw_success); +} diff --git a/src/state_transition/block/process_payload_attestation.zig b/src/state_transition/block/process_payload_attestation.zig index bb115527e..4607cf2e5 100644 --- a/src/state_transition/block/process_payload_attestation.zig +++ b/src/state_transition/block/process_payload_attestation.zig @@ -33,3 +33,58 @@ pub fn processPayloadAttestation( return error.InvalidPayloadAttestation; } } + +const Node = @import("persistent_merkle_tree").Node; +const TestCachedBeaconState = @import("../test_utils/root.zig").TestCachedBeaconState; +const DoubleFreeDetectAllocator = @import("testing_allocators").DoubleFreeDetectAllocator; + +test "Gloas payload attestation - OOM does not double-free indexed data" { + const allocator = std.testing.allocator; + var pool = try Node.Pool.init(.{ .page_allocator = allocator, .allocator = allocator, .pool_size = 500_000 }); + defer pool.deinit(); + + var test_state = try TestCachedBeaconState.initGloas(allocator, &pool, 256); + defer test_state.deinit(); + const state = test_state.cached_state.state.castToFork(.gloas); + + var latest_block_header = try state.latestBlockHeader(); + var payload_attestation = types.gloas.PayloadAttestation.default_value; + payload_attestation.data.slot = (try state.slot()) - 1; + payload_attestation.data.beacon_block_root = (try latest_block_header.getFieldRoot("parent_root")).*; + try payload_attestation.aggregation_bits.set(0, true); + + var saw_oom = false; + var saw_terminal_result = false; + var fail_at: usize = 0; + while (fail_at < 64) : (fail_at += 1) { + var oom = DoubleFreeDetectAllocator.init(std.testing.allocator, fail_at); + defer oom.deinit(); + + processPayloadAttestation( + oom.allocator(), + std.testing.io, + test_state.cached_state.config, + test_state.cached_state.epoch_cache, + state, + &payload_attestation, + ) catch |err| switch (err) { + error.OutOfMemory => { + saw_oom = true; + try std.testing.expect(!oom.double_free); + continue; + }, + else => { + try std.testing.expect(!oom.double_free); + saw_terminal_result = true; + break; + }, + }; + + try std.testing.expect(!oom.double_free); + saw_terminal_result = true; + break; + } + + try std.testing.expect(saw_oom); + try std.testing.expect(saw_terminal_result); +} diff --git a/src/state_transition/block/process_withdrawals.zig b/src/state_transition/block/process_withdrawals.zig index 8af8b9c12..d7b388442 100644 --- a/src/state_transition/block/process_withdrawals.zig +++ b/src/state_transition/block/process_withdrawals.zig @@ -515,6 +515,32 @@ fn getBuildersSweepWithdrawals( return .{ .withdrawals = builders_sweep_withdrawals, .processed_count = processed_count }; } const TestCachedBeaconState = @import("../test_utils/root.zig").TestCachedBeaconState; +const DoubleFreeDetectAllocator = @import("testing_allocators").DoubleFreeDetectAllocator; + +fn addGloasWithdrawalInputs(state: *BeaconState(.gloas)) !void { + var builders = try state.inner.get("builders"); + const builder = types.gloas.Builder.Type{ + .pubkey = [_]u8{0x11} ** 48, + .version = c.PAYLOAD_BUILDER_VERSION, + .execution_address = [_]u8{0x22} ** 20, + .balance = 64, + .deposit_epoch = 0, + .withdrawable_epoch = 0, + }; + try builders.pushValue(&builder); + try state.inner.set("builders", builders); + + var builder_pending_withdrawals = try state.inner.get("builder_pending_withdrawals"); + const pending_withdrawal = types.gloas.BuilderPendingWithdrawal.Type{ + .fee_recipient = [_]u8{0x33} ** 20, + .amount = 8, + .builder_index = 0, + }; + try builder_pending_withdrawals.pushValue(&pending_withdrawal); + try state.inner.set("builder_pending_withdrawals", builder_pending_withdrawals); + try state.inner.set("next_withdrawal_builder_index", @as(u64, 0)); + try state.commit(); +} test "process withdrawals - sanity" { const allocator = std.testing.allocator; @@ -554,3 +580,129 @@ test "process withdrawals - sanity" { root, ); } + +test "gloas expected withdrawals - OOM does not double-free transient allocations" { + const allocator = std.testing.allocator; + var pool = try Node.Pool.init(.{ .page_allocator = allocator, .allocator = allocator, .pool_size = 500_000 }); + defer pool.deinit(); + + var test_state = try TestCachedBeaconState.initGloas(allocator, &pool, 256); + defer test_state.deinit(); + + const state = test_state.cached_state.state.castToFork(.gloas); + try addGloasWithdrawalInputs(state); + + var saw_oom = false; + var saw_success = false; + var fail_at: usize = 0; + while (fail_at < 512) : (fail_at += 1) { + var oom = DoubleFreeDetectAllocator.init(std.testing.allocator, fail_at); + defer oom.deinit(); + const fault_allocator = oom.allocator(); + + var withdrawals_buf: [preset.MAX_WITHDRAWALS_PER_PAYLOAD]Withdrawal = undefined; + var withdrawals_result = WithdrawalsResult{ + .withdrawals = Withdrawals.initBuffer(&withdrawals_buf), + }; + var call_error: ?anyerror = null; + { + var withdrawal_balances = std.AutoHashMap(ValidatorIndex, usize).init(fault_allocator); + defer withdrawal_balances.deinit(); + + getExpectedWithdrawals( + .gloas, + fault_allocator, + test_state.cached_state.epoch_cache, + state, + &withdrawals_result, + &withdrawal_balances, + ) catch |err| { + call_error = err; + }; + } + + try std.testing.expect(!oom.double_free); + if (call_error) |err| switch (err) { + error.OutOfMemory => { + saw_oom = true; + continue; + }, + else => return err, + }; + + saw_success = true; + try std.testing.expectEqual(@as(usize, 2), withdrawals_result.withdrawals.items.len); + try std.testing.expectEqual(@as(usize, 1), withdrawals_result.processed_builder_withdrawals_count); + try std.testing.expectEqual(@as(usize, 1), withdrawals_result.processed_builders_sweep_count); + for (withdrawals_result.withdrawals.items) |withdrawal| { + try std.testing.expect(isBuilderIndex(withdrawal.validator_index)); + } + break; + } + + try std.testing.expect(saw_oom); + try std.testing.expect(saw_success); +} + +test "gloas process withdrawals - mutation path applies builder withdrawals and truncates queues" { + const allocator = std.testing.allocator; + var pool = try Node.Pool.init(.{ .page_allocator = allocator, .allocator = allocator, .pool_size = 500_000 }); + defer pool.deinit(); + + var test_state = try TestCachedBeaconState.initGloas(allocator, &pool, 256); + defer test_state.deinit(); + + const state = test_state.cached_state.state.castToFork(.gloas); + try addGloasWithdrawalInputs(state); + + // processWithdrawals is a no-op unless the parent block is full; the test is only + // meaningful when the precondition holds. + try std.testing.expect(try isParentBlockFull(state)); + + const initial_next_withdrawal_index = try state.nextWithdrawalIndex(); + + var withdrawals_buf: [preset.MAX_WITHDRAWALS_PER_PAYLOAD]Withdrawal = undefined; + var withdrawals_result = WithdrawalsResult{ + .withdrawals = Withdrawals.initBuffer(&withdrawals_buf), + }; + var withdrawal_balances = std.AutoHashMap(ValidatorIndex, usize).init(allocator); + defer withdrawal_balances.deinit(); + + try getExpectedWithdrawals( + .gloas, + allocator, + test_state.cached_state.epoch_cache, + state, + &withdrawals_result, + &withdrawal_balances, + ); + + // 8 from the builder payment queue + 56 sweeping the remaining builder balance. + try std.testing.expectEqual(@as(usize, 2), withdrawals_result.withdrawals.items.len); + + // payload_withdrawals_root is unused for gloas (parent payload verifies later), pass zeroes. + try processWithdrawals(.gloas, allocator, state, withdrawals_result, [_]u8{0} ** 32); + + // Commit so the assertions read back through the committed tree, not just cached child views. + try state.commit(); + + // applyWithdrawals drained the single builder: 64 - 8 (payment) - 56 (sweep) == 0. + var builders_view = try state.inner.getReadonly("builders"); + var builder: types.gloas.Builder.Type = undefined; + try builders_view.getValue(allocator, 0, &builder); + try std.testing.expectEqual(@as(u64, 0), builder.balance); + + // The processed builder payment was truncated off the queue. + var builder_pending_withdrawals = try state.inner.getReadonly("builder_pending_withdrawals"); + try std.testing.expectEqual(@as(usize, 0), try builder_pending_withdrawals.length()); + + // Both expected withdrawals were recorded for parent-payload verification. + var payload_expected_withdrawals = try state.inner.getReadonly("payload_expected_withdrawals"); + try std.testing.expectEqual(@as(usize, 2), try payload_expected_withdrawals.length()); + + // Sweep advanced the builder cursor by processed_builders_sweep_count (1) modulo builders_len (1). + try std.testing.expectEqual(@as(u64, 0), try state.inner.getReadonly("next_withdrawal_builder_index")); + + // Withdrawal index advanced by the two withdrawals produced this block. + try std.testing.expectEqual(initial_next_withdrawal_index + 2, try state.nextWithdrawalIndex()); +} diff --git a/src/state_transition/epoch/process_proposer_lookahead.zig b/src/state_transition/epoch/process_proposer_lookahead.zig index 510f8d173..be149dc2f 100644 --- a/src/state_transition/epoch/process_proposer_lookahead.zig +++ b/src/state_transition/epoch/process_proposer_lookahead.zig @@ -78,6 +78,7 @@ pub fn processProposerLookahead( } const TestCachedBeaconState = @import("../test_utils/root.zig").TestCachedBeaconState; +const DoubleFreeDetectAllocator = @import("testing_allocators").DoubleFreeDetectAllocator; test "processProposerLookahead sanity" { const allocator = std.testing.allocator; @@ -104,3 +105,57 @@ test "processProposerLookahead sanity" { test_state.epoch_transition_cache, ); } + +fn tryGloasProposerLookahead( + allocator: Allocator, + epoch_cache: *EpochCache, + baseline_state: *BeaconState(.gloas), + baseline_cache: *const EpochTransitionCache, +) !void { + var state = try baseline_state.clone(.{ .transfer_cache = false }); + defer state.deinit(); + + var cache = baseline_cache.*; + cache.next_shuffling = null; + defer if (cache.next_shuffling) |shuffling| shuffling.deinit(); + + try processProposerLookahead(.gloas, allocator, epoch_cache, &state, &cache); +} + +test "Gloas proposer lookahead - OOM does not leak or double-free shuffling allocations" { + const allocator = std.testing.allocator; + var pool = try Node.Pool.init(.{ .page_allocator = allocator, .allocator = allocator, .pool_size = 500_000 }); + defer pool.deinit(); + + var test_state = try TestCachedBeaconState.initGloas(allocator, &pool, 256); + defer test_state.deinit(); + + var saw_oom = false; + var saw_success = false; + var fail_at: usize = 0; + while (fail_at < 2048) : (fail_at += 1) { + var oom = DoubleFreeDetectAllocator.init(std.testing.allocator, fail_at); + defer oom.deinit(); + + tryGloasProposerLookahead( + oom.allocator(), + test_state.cached_state.epoch_cache, + test_state.cached_state.state.castToFork(.gloas), + test_state.epoch_transition_cache, + ) catch |err| switch (err) { + error.OutOfMemory => { + saw_oom = true; + try std.testing.expect(!oom.double_free); + continue; + }, + else => return err, + }; + + try std.testing.expect(!oom.double_free); + saw_success = true; + break; + } + + try std.testing.expect(saw_oom); + try std.testing.expect(saw_success); +} diff --git a/src/state_transition/epoch/process_ptc_window.zig b/src/state_transition/epoch/process_ptc_window.zig index 0d7f517b1..dd94b8d91 100644 --- a/src/state_transition/epoch/process_ptc_window.zig +++ b/src/state_transition/epoch/process_ptc_window.zig @@ -51,3 +51,62 @@ pub fn processPtcWindow( try state.inner.set("ptc_window", ptc_window); epoch_transition_cache.next_epoch_payload_timeliness_committees = next_epoch_payload_timeliness_committees; } + +const Node = @import("persistent_merkle_tree").Node; +const TestCachedBeaconState = @import("../test_utils/root.zig").TestCachedBeaconState; +const DoubleFreeDetectAllocator = @import("testing_allocators").DoubleFreeDetectAllocator; + +fn tryGloasPtcWindow( + allocator: Allocator, + epoch_cache: *const EpochCache, + baseline_state: *BeaconState(.gloas), + baseline_cache: *const EpochTransitionCache, +) !void { + var state = try baseline_state.clone(.{ .transfer_cache = false }); + defer state.deinit(); + + var cache = baseline_cache.*; + cache.next_shuffling = null; + cache.next_epoch_payload_timeliness_committees = null; + defer if (cache.next_shuffling) |shuffling| shuffling.deinit(); + + try processPtcWindow(allocator, epoch_cache, &state, &cache); +} + +test "Gloas PTC window - OOM does not leak or double-free shuffling allocations" { + const allocator = std.testing.allocator; + var pool = try Node.Pool.init(.{ .page_allocator = allocator, .allocator = allocator, .pool_size = 500_000 }); + defer pool.deinit(); + + var test_state = try TestCachedBeaconState.initGloas(allocator, &pool, 256); + defer test_state.deinit(); + + var saw_oom = false; + var saw_success = false; + var fail_at: usize = 0; + while (fail_at < 512) : (fail_at += 1) { + var oom = DoubleFreeDetectAllocator.init(std.testing.allocator, fail_at); + defer oom.deinit(); + + tryGloasPtcWindow( + oom.allocator(), + test_state.cached_state.epoch_cache, + test_state.cached_state.state.castToFork(.gloas), + test_state.epoch_transition_cache, + ) catch |err| switch (err) { + error.OutOfMemory => { + saw_oom = true; + try std.testing.expect(!oom.double_free); + continue; + }, + else => return err, + }; + + try std.testing.expect(!oom.double_free); + saw_success = true; + break; + } + + try std.testing.expect(saw_oom); + try std.testing.expect(saw_success); +} diff --git a/src/state_transition/signature_sets/indexed_payload_attestation.zig b/src/state_transition/signature_sets/indexed_payload_attestation.zig index d57b49709..ba94edf19 100644 --- a/src/state_transition/signature_sets/indexed_payload_attestation.zig +++ b/src/state_transition/signature_sets/indexed_payload_attestation.zig @@ -42,3 +42,32 @@ pub fn getIndexedPayloadAttestationSignatureSet( return createAggregateSignatureSetFromComponents(pubkeys, signing_root, indexed_payload_attestation.signature); } + +const Node = @import("persistent_merkle_tree").Node; +const TestCachedBeaconState = @import("../test_utils/root.zig").TestCachedBeaconState; + +// The pubkey lookup is fallible, so it can fail after `pubkeys` is allocated. The OOM suite only +// injects allocator failures, so it never reaches this branch; std.testing.allocator fails the test +// if the errdefer does not release `pubkeys`. +test "Gloas indexed payload attestation - unknown validator index does not leak pubkeys" { + const allocator = std.testing.allocator; + var pool = try Node.Pool.init(.{ .page_allocator = allocator, .allocator = allocator, .pool_size = 500_000 }); + defer pool.deinit(); + + var test_state = try TestCachedBeaconState.initGloas(allocator, &pool, 256); + defer test_state.deinit(); + + var indexed = types.gloas.IndexedPayloadAttestation.default_value; + defer indexed.attesting_indices.deinit(allocator); + // First index is valid, second is past the end of the pubkey cache. + try indexed.attesting_indices.append(allocator, 0); + try indexed.attesting_indices.append(allocator, 1_000_000); + + try std.testing.expectError(error.PubkeyNotFound, getIndexedPayloadAttestationSignatureSet( + allocator, + std.testing.io, + test_state.cached_state.config, + test_state.cached_state.epoch_cache, + &indexed, + )); +} diff --git a/src/state_transition/slot/upgrade_state_to_gloas.zig b/src/state_transition/slot/upgrade_state_to_gloas.zig index 3fc7ef7fc..a9a2dcfe3 100644 --- a/src/state_transition/slot/upgrade_state_to_gloas.zig +++ b/src/state_transition/slot/upgrade_state_to_gloas.zig @@ -138,3 +138,62 @@ fn onboardBuildersFromPendingDeposits( } try state.setPendingDeposits(new_pending); } + +const Node = @import("persistent_merkle_tree").Node; +const TestCachedBeaconState = @import("../test_utils/root.zig").TestCachedBeaconState; +const DoubleFreeDetectAllocator = @import("testing_allocators").DoubleFreeDetectAllocator; + +fn tryGloasUpgrade( + allocator: Allocator, + config: *const BeaconConfig, + epoch_cache: *const EpochCache, + baseline: *BeaconState(.fulu), +) !void { + var fulu_state = try baseline.clone(.{ .transfer_cache = false }); + var owns_fulu_state = true; + defer if (owns_fulu_state) fulu_state.deinit(); + + var gloas_state = try upgradeStateToGloas(allocator, std.testing.io, config, epoch_cache, &fulu_state); + owns_fulu_state = false; + defer gloas_state.deinit(); +} + +test "Gloas fork upgrade - OOM does not double-free transient allocations" { + const allocator = std.testing.allocator; + var pool = try Node.Pool.init(.{ .page_allocator = allocator, .allocator = allocator, .pool_size = 500_000 }); + defer pool.deinit(); + + var test_state = try TestCachedBeaconState.initElectraForGloas(allocator, &pool, 64); + defer test_state.deinit(); + try test_state.upgradeToFuluForGloas(allocator); + const baseline = test_state.cached_state.state.castToFork(.fulu); + + var saw_oom = false; + var saw_success = false; + var fail_at: usize = 0; + while (fail_at < 256) : (fail_at += 1) { + var oom = DoubleFreeDetectAllocator.init(std.testing.allocator, fail_at); + defer oom.deinit(); + + tryGloasUpgrade( + oom.allocator(), + test_state.cached_state.config, + test_state.cached_state.epoch_cache, + baseline, + ) catch |err| switch (err) { + error.OutOfMemory => { + saw_oom = true; + try std.testing.expect(!oom.double_free); + continue; + }, + else => return err, + }; + + try std.testing.expect(!oom.double_free); + saw_success = true; + break; + } + + try std.testing.expect(saw_oom); + try std.testing.expect(saw_success); +} diff --git a/src/state_transition/test_utils/generate_state.zig b/src/state_transition/test_utils/generate_state.zig index 5b29085c9..3d658ab8b 100644 --- a/src/state_transition/test_utils/generate_state.zig +++ b/src/state_transition/test_utils/generate_state.zig @@ -1,5 +1,6 @@ const std = @import("std"); const bls = @import("bls"); +const c = @import("constants"); const Allocator = std.mem.Allocator; const ForkSeq = @import("config").ForkSeq; const mainnet_chain_config = @import("config").mainnet.chain_config; @@ -182,6 +183,60 @@ pub const TestCachedBeaconState = struct { return initFromState(allocator, pool, state, ForkSeq.electra, fork_epoch); } + pub fn initElectraForGloas(allocator: Allocator, pool: *Node.Pool, validator_count: usize) !TestCachedBeaconState { + const fork_epoch = if (active_chain_config.ELECTRA_FORK_EPOCH == c.FAR_FUTURE_EPOCH) + 0 + else + active_chain_config.ELECTRA_FORK_EPOCH; + const test_chain_config = active_chain_config.merge(.{ .ELECTRA_FORK_EPOCH = fork_epoch }); + var state = try generateElectraState(allocator, pool, test_chain_config, validator_count); + errdefer { + state.deinit(); + allocator.destroy(state); + } + + return initFromState(allocator, pool, state, ForkSeq.electra, fork_epoch); + } + + pub fn initGloas(allocator: Allocator, pool: *Node.Pool, validator_count: usize) !TestCachedBeaconState { + var test_state = try initElectraForGloas(allocator, pool, validator_count); + errdefer test_state.deinit(); + try test_state.upgradeToGloas(allocator); + return test_state; + } + + pub fn upgradeToFuluForGloas(self: *TestCachedBeaconState, allocator: Allocator) !void { + const fork_epoch = self.cached_state.epoch_cache.epoch; + self.config.* = BeaconConfig.init( + getConfig(active_chain_config, .gloas, fork_epoch), + (try self.cached_state.state.genesisValidatorsRoot()).*, + ); + + const fulu_state = try state_transition.upgradeStateToFulu( + allocator, + self.cached_state.config, + self.cached_state.epoch_cache, + try self.cached_state.state.tryCastToFork(.electra), + ); + self.cached_state.state.* = .{ .fulu = fulu_state.inner }; + } + + pub fn upgradeToGloas(self: *TestCachedBeaconState, allocator: Allocator) !void { + if (self.cached_state.state.forkSeq() == .electra) { + try self.upgradeToFuluForGloas(allocator); + } + + const gloas_state = try state_transition.upgradeStateToGloas( + allocator, + std.testing.io, + self.cached_state.config, + self.cached_state.epoch_cache, + try self.cached_state.state.tryCastToFork(.fulu), + ); + self.cached_state.state.* = .{ .gloas = gloas_state.inner }; + try self.cached_state.state.commit(); + } + pub fn initFromState(allocator: Allocator, pool: *Node.Pool, state: *AnyBeaconState, fork: ForkSeq, fork_epoch: Epoch) !TestCachedBeaconState { const pubkey_cache = try allocator.create(PubkeyCache); errdefer allocator.destroy(pubkey_cache); diff --git a/src/state_transition/utils/gloas.zig b/src/state_transition/utils/gloas.zig index 5abfb8e02..975eb2395 100644 --- a/src/state_transition/utils/gloas.zig +++ b/src/state_transition/utils/gloas.zig @@ -371,3 +371,47 @@ pub fn initializePtcWindow( return ptc_window; } + +const Node = @import("persistent_merkle_tree").Node; +const TestCachedBeaconState = @import("../test_utils/root.zig").TestCachedBeaconState; +const DoubleFreeDetectAllocator = @import("testing_allocators").DoubleFreeDetectAllocator; + +test "Gloas PTC window fallback - OOM does not leak or double-free shuffling allocations" { + const allocator = std.testing.allocator; + var pool = try Node.Pool.init(.{ .page_allocator = allocator, .allocator = allocator, .pool_size = 500_000 }); + defer pool.deinit(); + + var test_state = try TestCachedBeaconState.initGloas(allocator, &pool, 256); + defer test_state.deinit(); + const state = test_state.cached_state.state.castToFork(.gloas); + try state.setSlot(try state.slot() + 2 * preset.SLOTS_PER_EPOCH); + + var saw_oom = false; + var saw_success = false; + var fail_at: usize = 0; + while (fail_at < 512) : (fail_at += 1) { + var oom = DoubleFreeDetectAllocator.init(std.testing.allocator, fail_at); + defer oom.deinit(); + + _ = initializePtcWindow( + .gloas, + oom.allocator(), + test_state.cached_state.epoch_cache, + state, + ) catch |err| switch (err) { + error.OutOfMemory => { + saw_oom = true; + try std.testing.expect(!oom.double_free); + continue; + }, + else => return err, + }; + + try std.testing.expect(!oom.double_free); + saw_success = true; + break; + } + + try std.testing.expect(saw_oom); + try std.testing.expect(saw_success); +} From 89207632f661753f8273c7059b0820a084b1266e Mon Sep 17 00:00:00 2001 From: Rahul Guha <19rahul2003@gmail.com> Date: Tue, 11 Aug 2026 18:28:59 +0530 Subject: [PATCH 33/33] merge main into gloas --- bench/state_transition/process_epoch.zig | 3 +-- .../epoch/get_attestation_deltas.zig | 18 ++++++++---------- .../epoch/get_rewards_and_penalties.zig | 15 +++++++-------- src/state_transition/epoch/process_epoch.zig | 2 +- .../epoch/process_rewards_and_penalties.zig | 7 +++---- .../epoch/process_slashings.zig | 12 ++++++------ test/spec/runner/epoch_processing.zig | 2 +- test/spec/runner/rewards.zig | 1 - 8 files changed, 27 insertions(+), 33 deletions(-) diff --git a/bench/state_transition/process_epoch.zig b/bench/state_transition/process_epoch.zig index 5ba577366..e1fd2fb35 100644 --- a/bench/state_transition/process_epoch.zig +++ b/bench/state_transition/process_epoch.zig @@ -127,11 +127,11 @@ fn ProcessSlashingsBench(comptime fork: ForkSeq) type { epoch_transition_cache: *EpochTransitionCache, pub fn run(self: *@This(), allocator: std.mem.Allocator) void { + _ = allocator; const cache = self.epoch_transition_cache; _ = state_transition.processSlashings( fork, - allocator, BenchState.cloned_cached_state.epoch_cache, BenchState.cloned_cached_state.state.castToFork(fork), cache, @@ -468,7 +468,6 @@ fn ProcessEpochSegmentedBench(comptime fork: ForkSeq) type { const slashings_start = time.start(io); const slashing_penalties = state_transition.processSlashings( fork, - allocator, epoch_cache, fork_state, cache, diff --git a/src/state_transition/epoch/get_attestation_deltas.zig b/src/state_transition/epoch/get_attestation_deltas.zig index ea6ca9290..2013ee520 100644 --- a/src/state_transition/epoch/get_attestation_deltas.zig +++ b/src/state_transition/epoch/get_attestation_deltas.zig @@ -1,5 +1,4 @@ const std = @import("std"); -const Allocator = std.mem.Allocator; const attester_status = @import("../utils/attester_status.zig"); const EpochCache = @import("../cache/epoch_cache.zig").EpochCache; const EpochTransitionCache = @import("../cache/epoch_transition_cache.zig").EpochTransitionCache; @@ -34,7 +33,7 @@ const RewardPenaltyItem = struct { finality_delay_penalty: u64, }; -pub fn getAttestationDeltas(allocator: Allocator, epoch_cache: *const EpochCache, cache: *const EpochTransitionCache, finalized_epoch: u64, rewards: []u64, penalties: []u64) !void { +pub fn getAttestationDeltas(epoch_cache: *const EpochCache, cache: *const EpochTransitionCache, finalized_epoch: u64, rewards: []u64, penalties: []u64) !void { const flags = cache.flags; const proposer_indices = cache.proposer_indices; const inclusion_delays = cache.inclusion_delays; @@ -66,12 +65,11 @@ pub fn getAttestationDeltas(allocator: Allocator, epoch_cache: *const EpochCache const proposer_reward_quotient = PROPOSER_REWARD_QUOTIENT; const is_in_inactivity_leak = finality_delay > MIN_EPOCHS_TO_INACTIVITY_PENALTY; - // effectiveBalance is multiple of EFFECTIVE_BALANCE_INCREMENT and less than MAX_EFFECTIVE_BALANCE - // so there are limited values of them like 32, 31, 30 - // TODO(bing): do not deinit and only clear for future use - var reward_penalty_item_cache = std.AutoHashMap(u64, RewardPenaltyItem).init(allocator); - reward_penalty_item_cache.clearAndFree(); - defer reward_penalty_item_cache.deinit(); + // Phase0 only: effectiveBalance is a multiple of EFFECTIVE_BALANCE_INCREMENT and bounded by + // MAX_EFFECTIVE_BALANCE (32 ETH), so effective_balance_increment is in range 0..32. + // Use a fixed array for O(1) lookup instead of a HashMap. + const max_increment = comptime preset.MAX_EFFECTIVE_BALANCE / preset.EFFECTIVE_BALANCE_INCREMENT + 1; + var reward_penalty_item_cache: [max_increment]?RewardPenaltyItem = .{null} ** max_increment; const effective_balance_increments = epoch_cache.getEffectiveBalanceIncrements(); std.debug.assert(flags.len <= effective_balance_increments.items.len); @@ -80,7 +78,7 @@ pub fn getAttestationDeltas(allocator: Allocator, epoch_cache: *const EpochCache const effective_balance_increment = effective_balance_increments.items[i]; const effective_balance: u64 = @as(u64, effective_balance_increment) * preset.EFFECTIVE_BALANCE_INCREMENT; - const rewards_items = if (reward_penalty_item_cache.get(effective_balance_increment)) |ri| ri else blk: { + const rewards_items = if (reward_penalty_item_cache[effective_balance_increment]) |ri| ri else blk: { const base_reward = @divFloor(@divFloor(effective_balance * BASE_REWARD_FACTOR, balance_sq_root), BASE_REWARDS_PER_EPOCH); const proposer_reward = @divFloor(base_reward, proposer_reward_quotient); const ri = RewardPenaltyItem{ @@ -93,7 +91,7 @@ pub fn getAttestationDeltas(allocator: Allocator, epoch_cache: *const EpochCache .base_penalty = base_reward * BASE_REWARDS_PER_EPOCH_CONST - proposer_reward, .finality_delay_penalty = @divFloor((effective_balance * finality_delay), INACTIVITY_PENALTY_QUOTIENT), }; - try reward_penalty_item_cache.put(effective_balance_increment, ri); + reward_penalty_item_cache[effective_balance_increment] = ri; break :blk ri; }; diff --git a/src/state_transition/epoch/get_rewards_and_penalties.zig b/src/state_transition/epoch/get_rewards_and_penalties.zig index 48969ed5b..2c459e6af 100644 --- a/src/state_transition/epoch/get_rewards_and_penalties.zig +++ b/src/state_transition/epoch/get_rewards_and_penalties.zig @@ -1,5 +1,4 @@ const std = @import("std"); -const Allocator = std.mem.Allocator; const ForkSeq = @import("config").ForkSeq; const BeaconConfig = @import("config").BeaconConfig; const BeaconState = @import("fork_types").BeaconState; @@ -38,7 +37,6 @@ const RewardPenaltyItem = struct { /// consumer should deinit `rewards` and `penalties` arrays pub fn getRewardsAndPenaltiesAltair( comptime fork: ForkSeq, - allocator: Allocator, config: *const BeaconConfig, epoch_cache: *const EpochCache, state: *BeaconState(fork), @@ -55,10 +53,11 @@ pub fn getRewardsAndPenaltiesAltair( @memset(penalties, 0); const is_in_inactivity_leak = isInInactivityLeak(epoch_cache.epoch, try state.finalizedEpoch()); - // effectiveBalance is multiple of EFFECTIVE_BALANCE_INCREMENT and less than MAX_EFFECTIVE_BALANCE - // so there are limited values of them like 32, 31, 30 - var reward_penalty_item_cache = std.AutoHashMap(u64, RewardPenaltyItem).init(allocator); - defer reward_penalty_item_cache.deinit(); + // effectiveBalance is a multiple of EFFECTIVE_BALANCE_INCREMENT and bounded by max effective balance + // (32 ETH pre-Electra, 2048 ETH post-Electra). Use a fixed array for O(1) lookup. + const max_effective = comptime if (fork.gte(.electra)) preset.MAX_EFFECTIVE_BALANCE_ELECTRA else preset.MAX_EFFECTIVE_BALANCE; + const max_increment = comptime max_effective / EFFECTIVE_BALANCE_INCREMENT + 1; + var reward_penalty_item_cache: [max_increment]?RewardPenaltyItem = .{null} ** max_increment; const inactivity_penality_multiplier: u64 = if (fork == ForkSeq.altair) INACTIVITY_PENALTY_QUOTIENT_ALTAIR else INACTIVITY_PENALTY_QUOTIENT_BELLATRIX; @@ -74,7 +73,7 @@ pub fn getRewardsAndPenaltiesAltair( const effective_balance_increment = effective_balance_increments[i]; - const reward_penalty_item = if (reward_penalty_item_cache.get(effective_balance_increment)) |rpi| rpi else blk: { + const reward_penalty_item = if (reward_penalty_item_cache[effective_balance_increment]) |rpi| rpi else blk: { const base_reward = effective_balance_increment * cache.base_reward_per_increment; const ts_weigh = PARTICIPATION_FLAG_WEIGHTS[TIMELY_SOURCE_FLAG_INDEX]; const tt_weigh = PARTICIPATION_FLAG_WEIGHTS[TIMELY_TARGET_FLAG_INDEX]; @@ -93,7 +92,7 @@ pub fn getRewardsAndPenaltiesAltair( .timely_source_penalty = @divFloor(base_reward * ts_weigh, WEIGHT_DENOMINATOR), .timely_target_penalty = @divFloor(base_reward * tt_weigh, WEIGHT_DENOMINATOR), }; - try reward_penalty_item_cache.put(effective_balance_increment, rpi); + reward_penalty_item_cache[effective_balance_increment] = rpi; break :blk rpi; }; diff --git a/src/state_transition/epoch/process_epoch.zig b/src/state_transition/epoch/process_epoch.zig index 38513bcd1..8624519c4 100644 --- a/src/state_transition/epoch/process_epoch.zig +++ b/src/state_transition/epoch/process_epoch.zig @@ -53,7 +53,7 @@ pub fn processEpoch( try observeEpochTransitionStep(.{ .step = .process_registry_updates }, @as(u64, @intCast(time.since(io, timer).nanoseconds))); timer = time.start(io); - const slashing_penalties = try processSlashings(fork, allocator, epoch_cache, state, cache, false); + const slashing_penalties = try processSlashings(fork, epoch_cache, state, cache, false); try observeEpochTransitionStep(.{ .step = .process_slashings }, @as(u64, @intCast(time.since(io, timer).nanoseconds))); timer = time.start(io); diff --git a/src/state_transition/epoch/process_rewards_and_penalties.zig b/src/state_transition/epoch/process_rewards_and_penalties.zig index ec8e94d9d..62e67d66f 100644 --- a/src/state_transition/epoch/process_rewards_and_penalties.zig +++ b/src/state_transition/epoch/process_rewards_and_penalties.zig @@ -26,7 +26,7 @@ pub fn processRewardsAndPenalties( const rewards = cache.rewards; const penalties = cache.penalties; - try getRewardsAndPenalties(fork, allocator, config, epoch_cache, state, cache, rewards, penalties); + try getRewardsAndPenalties(fork, config, epoch_cache, state, cache, rewards, penalties); const balances = try state.balancesSlice(allocator); errdefer allocator.free(balances); @@ -56,7 +56,6 @@ pub fn processRewardsAndPenalties( pub fn getRewardsAndPenalties( comptime fork: ForkSeq, - allocator: Allocator, config: *const BeaconConfig, epoch_cache: *const EpochCache, state: *BeaconState(fork), @@ -65,9 +64,9 @@ pub fn getRewardsAndPenalties( penalties: []u64, ) !void { if (comptime fork == .phase0) { - return try getAttestationDeltas(allocator, epoch_cache, cache, try state.finalizedEpoch(), rewards, penalties); + return try getAttestationDeltas(epoch_cache, cache, try state.finalizedEpoch(), rewards, penalties); } - return try getRewardsAndPenaltiesAltair(fork, allocator, config, epoch_cache, state, cache, rewards, penalties); + return try getRewardsAndPenaltiesAltair(fork, config, epoch_cache, state, cache, rewards, penalties); } const TestCachedBeaconState = @import("../test_utils/root.zig").TestCachedBeaconState; diff --git a/src/state_transition/epoch/process_slashings.zig b/src/state_transition/epoch/process_slashings.zig index c8674b475..397adba79 100644 --- a/src/state_transition/epoch/process_slashings.zig +++ b/src/state_transition/epoch/process_slashings.zig @@ -13,7 +13,6 @@ const Node = @import("persistent_merkle_tree").Node; pub fn processSlashings( comptime fork: ForkSeq, - allocator: std.mem.Allocator, epoch_cache: *const EpochCache, state: *BeaconState(fork), cache: *const EpochTransitionCache, @@ -44,17 +43,19 @@ pub fn processSlashings( const penalty_per_effective_balance_increment = @divFloor((adjusted_total_slashing_balance_by_increment * increment), total_balance_by_increment); - var penalties_by_effective_balance_increment = std.AutoHashMap(u64, u64).init(allocator); - defer penalties_by_effective_balance_increment.deinit(); + // effective_balance_increment is bounded by max effective balance (32 pre-Electra, 2048 post-Electra). + const max_effective = comptime if (fork.gte(.electra)) preset.MAX_EFFECTIVE_BALANCE_ELECTRA else preset.MAX_EFFECTIVE_BALANCE; + const max_increment = comptime max_effective / EFFECTIVE_BALANCE_INCREMENT + 1; + var penalties_by_effective_balance_increment: [max_increment]?u64 = .{null} ** max_increment; for (cache.indices_to_slash.items) |index| { const effective_balance_increment = effective_balance_increments[index]; - const penalty: u64 = if (penalties_by_effective_balance_increment.get(effective_balance_increment)) |penalty| penalty else blk: { + const penalty: u64 = if (penalties_by_effective_balance_increment[effective_balance_increment]) |penalty| penalty else blk: { const p = if (comptime fork.gte(.electra)) penalty_per_effective_balance_increment * effective_balance_increment else @divFloor(effective_balance_increment * adjusted_total_slashing_balance_by_increment, total_balance_by_increment) * increment; - try penalties_by_effective_balance_increment.put(effective_balance_increment, p); + penalties_by_effective_balance_increment[effective_balance_increment] = p; break :blk p; }; if (update_balance) { @@ -95,7 +96,6 @@ test "processSlashings - sanity" { _ = try processSlashings( .electra, - allocator, test_state.cached_state.epoch_cache, test_state.cached_state.state.castToFork(.electra), test_state.epoch_transition_cache, diff --git a/test/spec/runner/epoch_processing.zig b/test/spec/runner/epoch_processing.zig index c714340b9..dc5ff04c3 100644 --- a/test/spec/runner/epoch_processing.zig +++ b/test/spec/runner/epoch_processing.zig @@ -127,7 +127,7 @@ pub fn TestCase(comptime fork: ForkSeq, comptime epoch_process_fn: EpochProcessi .randao_mixes_reset => try state_transition.processRandaoMixesReset(fork, fork_state, &epoch_transition_cache), .registry_updates => try state_transition.processRegistryUpdates(fork, config, epoch_cache, fork_state, &epoch_transition_cache), .rewards_and_penalties => try state_transition.processRewardsAndPenalties(fork, allocator, config, epoch_cache, fork_state, &epoch_transition_cache, null), - .slashings => _ = try state_transition.processSlashings(fork, allocator, epoch_cache, fork_state, &epoch_transition_cache, true), + .slashings => _ = try state_transition.processSlashings(fork, epoch_cache, fork_state, &epoch_transition_cache, true), .slashings_reset => try state_transition.processSlashingsReset(fork, epoch_cache, fork_state, &epoch_transition_cache), .sync_committee_updates => try state_transition.processSyncCommitteeUpdates(fork, allocator, epoch_cache, fork_state), .historical_summaries_update => try state_transition.processHistoricalSummariesUpdate(fork, fork_state, &epoch_transition_cache), diff --git a/test/spec/runner/rewards.zig b/test/spec/runner/rewards.zig index e4dd72900..1112f544f 100644 --- a/test/spec/runner/rewards.zig +++ b/test/spec/runner/rewards.zig @@ -182,7 +182,6 @@ pub fn TestCase(comptime fork: ForkSeq) type { try getRewardsAndPenaltiesFn( fork, - allocator, cloned_state.config, cloned_state.epoch_cache, cloned_state.state.castToFork(fork),