feat: add compute shuffled index

index.d.ts

@@ -1,9 +1,18 @@
 /* auto-generated by NAPI-RS */
 /* eslint-disable */
+export declare class ComputeShuffledIndex {
+  constructor(seed: Uint8Array, indexCount: number, rounds: number)
+  get(index: number): number
+}
+
 export declare function asyncShuffleList(activeIndices: Uint32Array, seed: Uint8Array, rounds: number): Promise<Uint32Array>
 
 export declare function asyncUnshuffleList(activeIndices: Uint32Array, seed: Uint8Array, rounds: number): Promise<Uint32Array>
 
+export declare function computeProposerIndexElectra(seed: Uint8Array, activeIndices: Uint32Array, effectiveBalanceIncrements: Uint16Array, maxEffectiveBalanceElectra: number, effectiveBalanceIncrement: number, rounds: number): number
+
+export declare function computeSyncCommitteeIndicesElectra(seed: Uint8Array, activeIndices: Uint32Array, effectiveBalanceIncrements: Uint16Array, syncCommitteeSize: number, maxEffectiveBalanceElectra: number, effectiveBalanceIncrement: number, rounds: number): Uint32Array
+
 export const SHUFFLE_ROUNDS_MAINNET: number
 
 export const SHUFFLE_ROUNDS_MINIMAL: number

index.js

@@ -365,9 +365,12 @@ if (!nativeBinding) {
   throw new Error(`Failed to load native binding`)
 }
 
-const { asyncShuffleList, asyncUnshuffleList, SHUFFLE_ROUNDS_MAINNET, SHUFFLE_ROUNDS_MINIMAL, shuffleList, unshuffleList } = nativeBinding
+const { ComputeShuffledIndex, asyncShuffleList, asyncUnshuffleList, computeProposerIndexElectra, computeSyncCommitteeIndicesElectra, SHUFFLE_ROUNDS_MAINNET, SHUFFLE_ROUNDS_MINIMAL, shuffleList, unshuffleList } = nativeBinding
+export { ComputeShuffledIndex }
 export { asyncShuffleList }
 export { asyncUnshuffleList }
+export { computeProposerIndexElectra }
+export { computeSyncCommitteeIndicesElectra }
 export { SHUFFLE_ROUNDS_MAINNET }
 export { SHUFFLE_ROUNDS_MINIMAL }
 export { shuffleList }

package.json

@@ -34,6 +34,9 @@
     "@biomejs/biome": "^1.9.4",
     "@chainsafe/as-sha256": "^0.5.0",
     "@chainsafe/benchmark": "^1.2.3",
+    "@lodestar/params": "^1.27.0",
+    "@lodestar/state-transition": "^1.27.0",
+    "@lodestar/utils": "^1.27.0",
     "@napi-rs/cli": "^3.0.0-alpha.70",
     "@types/node": "^22.13.2",
     "bigint-buffer": "^1.1.5",
@@ -43,6 +46,7 @@
     "vitest": "^3.0.5"
   },
   "resolutions": {
+    "@chainsafe/pubkey-index-map": "npm:@chainsafe/pubkey-index-map@3.0.0",
     "rollup": "npm:@rollup/wasm-node@*"
   },
   "engines": {

src/lib.rs

@@ -1,6 +1,6 @@
 #![deny(clippy::all)]
 
-use std::mem;
+use std::{cmp::max, collections::HashMap, mem};
 
 use napi::{
   bindgen_prelude::{AsyncTask, Result, Uint32Array, Uint8Array},
@@ -115,12 +115,11 @@ impl ShufflingManager {
 ///  - `list_size > 2**24`
 ///  - `list_size > usize::MAX / 2`
 fn inner_shuffle_list(
-  input: &Uint32Array,
+  mut input: Vec<u32>,
   seed: &[u8],
   rounds: i32,
   forwards: bool,
 ) -> Result<Vec<u32>> {
-  let mut input = input.to_vec();
   if rounds == 0 {
     // no shuffling rounds
     return Ok(input);
@@ -233,7 +232,7 @@ pub fn shuffle_list(
   rounds: i32,
 ) -> Result<Uint32Array> {
   Ok(Uint32Array::new(inner_shuffle_list(
-    &active_indices,
+    active_indices.to_vec(),
     &seed,
     rounds,
     true,
@@ -247,7 +246,7 @@ pub fn unshuffle_list(
   rounds: i32,
 ) -> Result<Uint32Array> {
   Ok(Uint32Array::new(inner_shuffle_list(
-    &active_indices,
+    active_indices.to_vec(),
     &seed,
     rounds,
     false,
@@ -268,7 +267,7 @@ impl Task for AsyncInnerShuffle {
 
   fn compute(&mut self) -> Result<Self::Output> {
     Ok(inner_shuffle_list(
-      &self.input,
+      self.input.to_vec(),
       &self.seed,
       self.rounds,
       self.forwards,
@@ -307,3 +306,174 @@ pub fn async_unshuffle_list(
     forwards: false,
   })
 }
+
+#[napi]
+pub struct ComputeShuffledIndex {
+  /// There are possibly SHUFFLE_ROUND_COUNT (90) values for this cache
+  /// This cache will always hit after the 1st call
+  pivot_by_index: HashMap<u32, u32>,
+  /// Given 2M active validators, there are 2M / 256 = 8k possible position_div
+  /// It means there are at most 8k different sources for each round
+  source_by_position_by_index: HashMap<u32, HashMap<u32, [u8; 32]>>,
+  /// 32 bytes seed + 1 byte i
+  pivot_buffer: [u8; 32 + 1],
+  /// 32 bytes seed + 1 byte i + 4 bytes position_div
+  source_buffer: [u8; 32 + 1 + 4],
+  /// validator count
+  index_count: u32,
+  /// rounds
+  rounds: u32,
+}
+
+fn digest_as_u64(input: &[u8]) -> u64 {
+  u64::from_le_bytes(hash_fixed(input)[0..8].try_into().unwrap())
+}
+
+#[napi]
+impl ComputeShuffledIndex {
+  #[napi(constructor)]
+  pub fn new(seed: &[u8], index_count: u32, rounds: u32) -> Self {
+    // copy seed into the front of pivot_buffer and source_buffer
+    let mut pivot_buffer = [0u8; 32 + 1];
+    pivot_buffer[0..32].copy_from_slice(seed);
+    let mut source_buffer = [0u8; 32 + 1 + 4];
+    source_buffer[0..32].copy_from_slice(seed);
+    Self {
+      pivot_by_index: HashMap::new(),
+      source_by_position_by_index: HashMap::new(),
+      pivot_buffer,
+      source_buffer,
+      index_count,
+      rounds,
+    }
+  }
+
+  #[napi]
+  pub fn get(&mut self, index: u32) -> u32 {
+    let mut permuted = index;
+
+    for i in 0..self.rounds {
+      let pivot = *self.pivot_by_index.entry(i).or_insert_with(|| {
+        self.pivot_buffer[32] = (i % 256) as u8;
+        (digest_as_u64(self.pivot_buffer.as_ref()) % self.index_count as u64)
+          .try_into()
+          .unwrap()
+      });
+
+      let flip = (pivot + self.index_count - permuted) % self.index_count;
+      let position = max(permuted, flip);
+
+      let position_div = position / 256;
+      let source = self
+        .source_by_position_by_index
+        .entry(i)
+        .or_insert(HashMap::new())
+        .entry(position_div)
+        .or_insert_with(|| {
+          self.source_buffer[32] = (i % 256) as u8;
+          self.source_buffer[33..37].copy_from_slice(&position_div.to_le_bytes());
+          hash_fixed(self.source_buffer.as_ref())
+        });
+
+      let byte = source[(position % 256 / 8) as usize];
+      let bit = (byte >> (position % 8)) & 1;
+      permuted = if bit == 1 { flip } else { permuted };
+    }
+
+    permuted
+  }
+}
+
+#[napi]
+pub fn compute_proposer_index_electra(
+  seed: &[u8],
+  active_indices: &[u32],
+  effective_balance_increments: &[u16],
+  max_effective_balance_electra: i64,
+  effective_balance_increment: i64,
+  rounds: u32,
+) -> u32 {
+  get_committee_indices_electra(
+    1,
+    seed,
+    active_indices,
+    effective_balance_increments,
+    max_effective_balance_electra,
+    effective_balance_increment,
+    rounds,
+  )[0]
+}
+
+#[napi]
+pub fn compute_sync_committee_indices_electra(
+  seed: &[u8],
+  active_indices: &[u32],
+  effective_balance_increments: &[u16],
+  sync_committee_size: u32,
+  max_effective_balance_electra: i64,
+  effective_balance_increment: i64,
+  rounds: u32,
+) -> Uint32Array {
+  get_committee_indices_electra(
+    sync_committee_size,
+    seed,
+    active_indices,
+    effective_balance_increments,
+    max_effective_balance_electra,
+    effective_balance_increment,
+    rounds,
+  )
+  .into()
+}
+
+pub fn get_committee_indices_electra(
+  committee_size: u32,
+  seed: &[u8],
+  active_indices: &[u32],
+  effective_balance_increments: &[u16],
+  max_effective_balance_electra: i64,
+  effective_balance_increment: i64,
+  rounds: u32,
+) -> Vec<u32> {
+  let mut committee_indices = Vec::with_capacity(committee_size as usize);
+  let max_random_value = 0xffff;
+  let max_effective_balance_increment = max_effective_balance_electra / effective_balance_increment;
+
+  let mut compute_shuffled_index =
+    ComputeShuffledIndex::new(seed, active_indices.len() as u32, rounds);
+  let mut shuffled_result = HashMap::new();
+
+  let mut i: u32 = 0;
+  let mut cached_hash_input = [0u8; 32 + 8];
+  cached_hash_input[0..32].copy_from_slice(seed);
+  let mut cached_hash = [0u8; 32];
+
+  while (committee_indices.len() as u32) < committee_size {
+    let index = i % active_indices.len() as u32;
+    let shuffled_index = *shuffled_result
+      .entry(index)
+      .or_insert_with(|| compute_shuffled_index.get(index));
+    let candidate_index = active_indices[shuffled_index as usize];
+
+    if i % 16 == 0 {
+      cached_hash_input[32..36].copy_from_slice(&(i / 16).to_le_bytes());
+      cached_hash = hash_fixed(&cached_hash_input);
+    }
+
+    let random_bytes = cached_hash;
+    let offset = ((i % 16) * 2) as usize;
+    let random_value =
+      u16::from_le_bytes(random_bytes[offset..(offset + 2)].try_into().unwrap()) as i64;
+
+    let effective_balance_increment = effective_balance_increments[candidate_index as usize] as i64;
+
+    if effective_balance_increment * max_random_value
+      >= max_effective_balance_increment * random_value
+    {
+      committee_indices.push(candidate_index);
+    }
+
+    i += 1;
+  }
+  committee_indices
+}

test/perf/committeeIndices.test.ts

@@ -0,0 +1,49 @@
+import {randomBytes} from "node:crypto";
+import {bench, describe} from "@chainsafe/benchmark";
+import {
+  EFFECTIVE_BALANCE_INCREMENT,
+  MAX_EFFECTIVE_BALANCE_ELECTRA,
+  SHUFFLE_ROUND_COUNT,
+  SYNC_COMMITTEE_SIZE,
+} from "@lodestar/params";
+import {computeSyncCommitteeIndicesElectra} from "../../index.js";
+import {naiveComputeSyncCommitteeIndicesElectra} from "../referenceImplementation.js";
+
+describe("computeIndices", () => {
+  for (const listSize of [
+    16384, 250_000,
+    1_000_000,
+    // Don't run 4_000_000 since it's very slow and not testnet has gotten there yet
+    // 4e6,
+  ]) {
+    const seed = randomBytes(32);
+    const vc = listSize;
+    const activeIndices = new Uint32Array(Array.from({length: vc}, (_, i) => i));
+    const effectiveBalanceIncrements = new Uint16Array(vc);
+    for (let i = 0; i < vc; i++) {
+      effectiveBalanceIncrements[i] = 32 + 32 * (i % 64);
+    }
+
+    bench({
+      id: `JS   - computeSyncCommitteeIndices - ${listSize} indices`,
+      fn: () => {
+        naiveComputeSyncCommitteeIndicesElectra(seed, activeIndices, effectiveBalanceIncrements);
+      },
+    });
+
+    bench({
+      id: `RS   - computeSyncCommitteeIndices - ${listSize} indices`,
+      fn: () => {
+        computeSyncCommitteeIndicesElectra(
+          seed,
+          activeIndices,
+          effectiveBalanceIncrements,
+          SYNC_COMMITTEE_SIZE,
+          MAX_EFFECTIVE_BALANCE_ELECTRA,
+          EFFECTIVE_BALANCE_INCREMENT,
+          SHUFFLE_ROUND_COUNT
+        );
+      },
+    });
+  }
+});

test/referenceImplementation.ts

@@ -1,5 +1,8 @@
 import {digest} from "@chainsafe/as-sha256";
-import {toBigIntLE, toBigIntBE} from "bigint-buffer";
+import {EFFECTIVE_BALANCE_INCREMENT, MAX_EFFECTIVE_BALANCE_ELECTRA, SYNC_COMMITTEE_SIZE} from "@lodestar/params";
+import {computeShuffledIndex} from "@lodestar/state-transition";
+import {bytesToInt, intToBytes} from "@lodestar/utils";
+import {toBigIntBE, toBigIntLE} from "bigint-buffer";
 
 // ArrayLike<number> but with settable indices
 type Shuffleable = {
@@ -240,3 +243,34 @@ function innerShuffleList(input: Shuffleable, seed: Uint8Array, rounds: number,
     }
   }
 }
+
+/// sync committee computation from lodestar, tweaked to avoid beacon state param
+export function naiveComputeSyncCommitteeIndicesElectra(
+  seed: Uint8Array,
+  activeValidatorIndices: ArrayLike<number>,
+  effectiveBalanceIncrements: Uint16Array
+): number[] {
+  const syncCommitteeIndices = [];
+  const MAX_RANDOM_VALUE = 2 ** 16 - 1;
+  const MAX_EFFECTIVE_BALANCE_INCREMENT = MAX_EFFECTIVE_BALANCE_ELECTRA / EFFECTIVE_BALANCE_INCREMENT;
+
+  const activeValidatorCount = activeValidatorIndices.length;
+
+  let i = 0;
+  while (syncCommitteeIndices.length < SYNC_COMMITTEE_SIZE) {
+    const shuffledIndex = computeShuffledIndex(i % activeValidatorCount, activeValidatorCount, seed);
+    const candidateIndex = activeValidatorIndices[shuffledIndex];
+    const randomBytes = digest(Buffer.concat([seed, intToBytes(Math.floor(i / 16), 8, "le")]));
+    const offset = (i % 16) * 2;
+    const randomValue = bytesToInt(randomBytes.subarray(offset, offset + 2));
+
+    const effectiveBalanceIncrement = effectiveBalanceIncrements[candidateIndex];
+    if (effectiveBalanceIncrement * MAX_RANDOM_VALUE >= MAX_EFFECTIVE_BALANCE_INCREMENT * randomValue) {
+      syncCommitteeIndices.push(candidateIndex);
+    }
+
+    i += 1;
+  }
+
+  return syncCommitteeIndices;
+}