From d87cb9722560f626efbafc139de7693258a5b90e Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Fri, 10 Jul 2026 14:59:35 +0100 Subject: [PATCH 1/3] feat(bindings): configurable pubkey cache growth step --- bindings/napi/pubkeys.zig | 33 ++++++++++++++++++++++++--------- bindings/src/pubkeys.d.ts | 9 +++++++-- bindings/src/pubkeys.js | 8 ++++++-- bindings/test/pubkeys.test.ts | 15 +++++++++++++++ 4 files changed, 52 insertions(+), 13 deletions(-) diff --git a/bindings/napi/pubkeys.zig b/bindings/napi/pubkeys.zig index e65813520..0d1c1b9da 100644 --- a/bindings/napi/pubkeys.zig +++ b/bindings/napi/pubkeys.zig @@ -16,6 +16,8 @@ const max_stack_aggregate_pubkeys = 512; pub const State = struct { pubkey2index: PubkeyIndexMap = undefined, index2pubkey: Index2PubkeyCache = undefined, + /// Capacity added when a set() outgrows the current capacity, doubles when 0 + growth_step: u32 = 0, initialized: bool = false, pub fn init(self: *State) !void { @@ -128,14 +130,14 @@ pub fn load(file_path: js.String) !void { } const len = std.mem.readInt(u32, header[4..8], .little); - const capacity = std.mem.readInt(u32, header[8..12], .little); + const saved_capacity = std.mem.readInt(u32, header[8..12], .little); const file_size = try file.length(io); state.pubkey2index = PubkeyIndexMap.init(allocator); - try state.pubkey2index.ensureTotalCapacity(capacity); + try state.pubkey2index.ensureTotalCapacity(saved_capacity); errdefer state.pubkey2index.deinit(); - state.index2pubkey = try Index2PubkeyCache.initCapacity(allocator, capacity); + state.index2pubkey = try Index2PubkeyCache.initCapacity(allocator, saved_capacity); errdefer state.index2pubkey.deinit(allocator); try state.index2pubkey.resize(allocator, len); @@ -151,7 +153,7 @@ pub fn load(file_path: js.String) !void { try file_reader.interface.readSliceAll(ptr[0..p2i_size]); state.pubkey2index.unmanaged.size = len; - state.pubkey2index.unmanaged.available = capacity - len; + state.pubkey2index.unmanaged.available = saved_capacity - len; // Read index2pubkey entries try file_reader.interface.readSliceAll(std.mem.sliceAsBytes(state.index2pubkey.items)); @@ -240,9 +242,11 @@ pub fn set(index: js.Number, pubkey: js.Uint8Array) !void { // Ensure capacity if needed if (idx >= state.index2pubkey.capacity) { - const new_cap: u32 = @intCast(@max(idx + 1, state.index2pubkey.capacity * 2)); + const current = state.index2pubkey.capacity; + const grown = if (state.growth_step > 0) current + state.growth_step else current * 2; + const new_cap: u32 = @intCast(@max(idx + 1, grown)); try state.pubkey2index.ensureTotalCapacity(new_cap); - try state.index2pubkey.ensureTotalCapacity(allocator, new_cap); + try state.index2pubkey.ensureTotalCapacityPrecise(allocator, new_cap); } // Extend length if needed @@ -264,14 +268,25 @@ pub fn size() !js.Number { return js.Number.from(@as(u32, @intCast(state.index2pubkey.items.len))); } -/// JS: pubkeys.ensureCapacity(newSize) -pub fn ensureCapacity(new_size: js.Number) !void { +/// JS: pubkeys.ensureCapacity(newSize, growthStep?) +pub fn ensureCapacity(new_size: js.Number, growth_step: ?js.Number) !void { if (!state.initialized) return error.PubkeyIndexNotInitialized; + if (growth_step) |step| { + state.growth_step = try step.toU32(); + } + const requested = try new_size.toU32(); const old_size = state.index2pubkey.capacity; if (requested <= old_size) return; try state.pubkey2index.ensureTotalCapacity(requested); - try state.index2pubkey.ensureTotalCapacity(allocator, requested); + try state.index2pubkey.ensureTotalCapacityPrecise(allocator, requested); +} + +/// JS: pubkeys.capacity() → number +/// Note: zapi DSL does not yet support namespace-level getters, so this is a function. +pub fn capacity() !js.Number { + if (!state.initialized) return error.PubkeyIndexNotInitialized; + return js.Number.from(@as(u32, @intCast(state.index2pubkey.capacity))); } diff --git a/bindings/src/pubkeys.d.ts b/bindings/src/pubkeys.d.ts index ad94b9eec..908777d34 100644 --- a/bindings/src/pubkeys.d.ts +++ b/bindings/src/pubkeys.d.ts @@ -13,14 +13,19 @@ export interface PubkeyCache { set(index: number, pubkey: Uint8Array): void; /** Number of entries */ readonly size: number; + /** Allocated native capacity */ + readonly capacity: number; /** Load cache from a PKIX file (clears JS-level cache) */ load(filepath: string): void; /** Clear native and JS-level cache contents */ reset(): void; /** Save cache to a PKIX file */ save(filepath: string): void; - /** Pre-allocate native capacity */ - ensureCapacity(capacity: number): void; + /** + * Pre-allocate native capacity. `growthStep` sets how much extra capacity is reserved + * when a set() outgrows the current capacity, capacity doubles if never configured. + */ + ensureCapacity(capacity: number, growthStep?: number): void; } export declare const pubkeyCache: PubkeyCache; diff --git a/bindings/src/pubkeys.js b/bindings/src/pubkeys.js index 0590d6544..73afe9f9c 100644 --- a/bindings/src/pubkeys.js +++ b/bindings/src/pubkeys.js @@ -44,6 +44,10 @@ export const pubkeyCache = { return native.size(); }, + get capacity() { + return native.capacity(); + }, + load(filepath) { pkCache.clear(); native.load(filepath); @@ -58,7 +62,7 @@ export const pubkeyCache = { native.save(filepath); }, - ensureCapacity(capacity) { - native.ensureCapacity(capacity); + ensureCapacity(capacity, growthStep) { + native.ensureCapacity(capacity, growthStep); }, }; diff --git a/bindings/test/pubkeys.test.ts b/bindings/test/pubkeys.test.ts index b24ad8989..e68582cf0 100644 --- a/bindings/test/pubkeys.test.ts +++ b/bindings/test/pubkeys.test.ts @@ -102,4 +102,19 @@ describe("pubkeys", () => { expect(pubkeyCache.get(0)).toBeUndefined(); expect(pubkeyCache.getIndex(keypairs[0].pubkeyBytes)).toBeNull(); }); + + it("exposes native capacity", () => { + expect(pubkeyCache.capacity).toBe(1_000); + }); + + it("doubles capacity on growth when no growth step is configured", () => { + pubkeyCache.set(1_000, keypairs[0].pubkeyBytes); + expect(pubkeyCache.capacity).toBe(2_000); + }); + + it("grows capacity by the configured growth step", () => { + pubkeyCache.ensureCapacity(pubkeyCache.capacity, 8); + pubkeyCache.set(2_000, keypairs[1].pubkeyBytes); + expect(pubkeyCache.capacity).toBe(2_008); + }); }); From 6b317120b044f72391103b68c504c226705f5588 Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Fri, 10 Jul 2026 15:11:35 +0100 Subject: [PATCH 2/3] refactor(bindings): derive pubkey cache growth step from preset --- bindings/napi/pubkeys.zig | 20 +++++++++----------- bindings/src/pubkeys.d.ts | 6 +++--- bindings/src/pubkeys.js | 4 ++-- bindings/test/pubkeys.test.ts | 19 ++++++++++--------- 4 files changed, 24 insertions(+), 25 deletions(-) diff --git a/bindings/napi/pubkeys.zig b/bindings/napi/pubkeys.zig index 0d1c1b9da..f83fc5fce 100644 --- a/bindings/napi/pubkeys.zig +++ b/bindings/napi/pubkeys.zig @@ -5,6 +5,7 @@ const blst_bindings = @import("./blst.zig"); const PubkeyIndexMap = @import("state_transition").PubkeyIndexMap; const Index2PubkeyCache = @import("state_transition").Index2PubkeyCache; const napi_io = @import("./io.zig"); +const preset = @import("preset").preset; /// Uses page allocator for internal allocations. /// It's recommended to never reallocate the pubkey2index after initialization. @@ -13,11 +14,14 @@ const allocator = std.heap.page_allocator; const default_initial_capacity: u32 = 0; const max_stack_aggregate_pubkeys = 512; +/// Capacity added when a set() outgrows the current capacity. Covers ~3 months of +/// worst-case validator registry growth (MAX_PENDING_DEPOSITS_PER_EPOCH new validators +/// per epoch at 12s slots), so growth stays proportionate at any network scale. +const growth_step: u32 = preset.MAX_PENDING_DEPOSITS_PER_EPOCH * ((90 * 24 * 60 * 60) / (12 * preset.SLOTS_PER_EPOCH)); + pub const State = struct { pubkey2index: PubkeyIndexMap = undefined, index2pubkey: Index2PubkeyCache = undefined, - /// Capacity added when a set() outgrows the current capacity, doubles when 0 - growth_step: u32 = 0, initialized: bool = false, pub fn init(self: *State) !void { @@ -242,9 +246,7 @@ pub fn set(index: js.Number, pubkey: js.Uint8Array) !void { // Ensure capacity if needed if (idx >= state.index2pubkey.capacity) { - const current = state.index2pubkey.capacity; - const grown = if (state.growth_step > 0) current + state.growth_step else current * 2; - const new_cap: u32 = @intCast(@max(idx + 1, grown)); + const new_cap: u32 = @intCast(@max(idx + 1, state.index2pubkey.capacity + growth_step)); try state.pubkey2index.ensureTotalCapacity(new_cap); try state.index2pubkey.ensureTotalCapacityPrecise(allocator, new_cap); } @@ -268,14 +270,10 @@ pub fn size() !js.Number { return js.Number.from(@as(u32, @intCast(state.index2pubkey.items.len))); } -/// JS: pubkeys.ensureCapacity(newSize, growthStep?) -pub fn ensureCapacity(new_size: js.Number, growth_step: ?js.Number) !void { +/// JS: pubkeys.ensureCapacity(newSize) +pub fn ensureCapacity(new_size: js.Number) !void { if (!state.initialized) return error.PubkeyIndexNotInitialized; - if (growth_step) |step| { - state.growth_step = try step.toU32(); - } - const requested = try new_size.toU32(); const old_size = state.index2pubkey.capacity; if (requested <= old_size) return; diff --git a/bindings/src/pubkeys.d.ts b/bindings/src/pubkeys.d.ts index 908777d34..5a278203a 100644 --- a/bindings/src/pubkeys.d.ts +++ b/bindings/src/pubkeys.d.ts @@ -22,10 +22,10 @@ export interface PubkeyCache { /** Save cache to a PKIX file */ save(filepath: string): void; /** - * Pre-allocate native capacity. `growthStep` sets how much extra capacity is reserved - * when a set() outgrows the current capacity, capacity doubles if never configured. + * Pre-allocate native capacity. When a set() outgrows the current capacity, it grows + * by a fixed step covering ~3 months of worst-case validator registry growth. */ - ensureCapacity(capacity: number, growthStep?: number): void; + ensureCapacity(capacity: number): void; } export declare const pubkeyCache: PubkeyCache; diff --git a/bindings/src/pubkeys.js b/bindings/src/pubkeys.js index 73afe9f9c..3aa6b37ac 100644 --- a/bindings/src/pubkeys.js +++ b/bindings/src/pubkeys.js @@ -62,7 +62,7 @@ export const pubkeyCache = { native.save(filepath); }, - ensureCapacity(capacity, growthStep) { - native.ensureCapacity(capacity, growthStep); + ensureCapacity(capacity) { + native.ensureCapacity(capacity); }, }; diff --git a/bindings/test/pubkeys.test.ts b/bindings/test/pubkeys.test.ts index e68582cf0..81e3e8d2b 100644 --- a/bindings/test/pubkeys.test.ts +++ b/bindings/test/pubkeys.test.ts @@ -107,14 +107,15 @@ describe("pubkeys", () => { expect(pubkeyCache.capacity).toBe(1_000); }); - it("doubles capacity on growth when no growth step is configured", () => { - pubkeyCache.set(1_000, keypairs[0].pubkeyBytes); - expect(pubkeyCache.capacity).toBe(2_000); - }); - - it("grows capacity by the configured growth step", () => { - pubkeyCache.ensureCapacity(pubkeyCache.capacity, 8); - pubkeyCache.set(2_000, keypairs[1].pubkeyBytes); - expect(pubkeyCache.capacity).toBe(2_008); + it("grows capacity by a fixed step instead of doubling", () => { + const cap0 = pubkeyCache.capacity; + pubkeyCache.set(cap0, keypairs[0].pubkeyBytes); + const cap1 = pubkeyCache.capacity; + const step = cap1 - cap0; + expect(step).toBeGreaterThan(0); + + pubkeyCache.set(cap1, keypairs[1].pubkeyBytes); + const cap2 = pubkeyCache.capacity; + expect(cap2 - cap1).toBe(step); }); }); From e397aa47779edf91486e2175600eeda419e7563c Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Fri, 10 Jul 2026 16:09:13 +0100 Subject: [PATCH 3/3] fix(bindings): keep growth curve slack on pubkey cache reservation --- bindings/napi/pubkeys.zig | 5 ++++- bindings/test/pubkeys.test.ts | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/bindings/napi/pubkeys.zig b/bindings/napi/pubkeys.zig index f83fc5fce..d3a3ee7c2 100644 --- a/bindings/napi/pubkeys.zig +++ b/bindings/napi/pubkeys.zig @@ -279,7 +279,10 @@ pub fn ensureCapacity(new_size: js.Number) !void { if (requested <= old_size) return; try state.pubkey2index.ensureTotalCapacity(requested); - try state.index2pubkey.ensureTotalCapacityPrecise(allocator, requested); + // Not precise on purpose, the growth curve overshoot leaves slack for states with + // slightly more validators than reserved, which the zig-side syncPubkeys cannot + // grow safely (it does not own the backing allocator) + try state.index2pubkey.ensureTotalCapacity(allocator, requested); } /// JS: pubkeys.capacity() → number diff --git a/bindings/test/pubkeys.test.ts b/bindings/test/pubkeys.test.ts index 81e3e8d2b..140c52289 100644 --- a/bindings/test/pubkeys.test.ts +++ b/bindings/test/pubkeys.test.ts @@ -104,7 +104,7 @@ describe("pubkeys", () => { }); it("exposes native capacity", () => { - expect(pubkeyCache.capacity).toBe(1_000); + expect(pubkeyCache.capacity).toBeGreaterThanOrEqual(1_000); }); it("grows capacity by a fixed step instead of doubling", () => {