feat: Validators sentinel

yarn-project/aztec-node/package.json

@@ -5,7 +5,8 @@
   "type": "module",
   "exports": {
     ".": "./dest/index.js",
-    "./config": "./dest/aztec-node/config.js"
+    "./config": "./dest/aztec-node/config.js",
+    "./sentinel": "./dest/aztec-node/sentinel.js"
   },
   "bin": "./dest/bin/index.js",
   "typedocOptions": {

yarn-project/aztec-node/src/aztec-node/config.ts

@@ -12,6 +12,8 @@ import { readFileSync } from 'fs';
 import { dirname, resolve } from 'path';
 import { fileURLToPath } from 'url';
 
+import { type SentinelConfig, sentinelConfigMappings } from '../sentinel/config.js';
+
 export { sequencerClientConfigMappings, type SequencerClientConfig };
 
 /**
@@ -24,7 +26,8 @@ export type AztecNodeConfig = ArchiverConfig &
   WorldStateConfig &
   Pick<ProverClientConfig, 'bbBinaryPath' | 'bbWorkingDirectory' | 'realProofs'> &
   P2PConfig &
-  DataStoreConfig & {
+  DataStoreConfig &
+  SentinelConfig & {
     /** Whether the validator is disabled for this node */
     disableValidator: boolean;
     /** Whether to populate the genesis state with initial fee juice for the test accounts */
@@ -41,6 +44,7 @@ export const aztecNodeConfigMappings: ConfigMappingsType<AztecNodeConfig> = {
   ...proverClientConfigMappings,
   ...worldStateConfigMappings,
   ...p2pConfigMappings,
+  ...sentinelConfigMappings,
   l1Contracts: {
     description: 'The deployed L1 contract addresses',
     nested: l1ContractAddressesMapping,

yarn-project/aztec-node/src/aztec-node/server.test.ts

@@ -115,6 +115,7 @@ describe('aztec node', () => {
       l1ToL2MessageSource,
       worldState,
       undefined,
+      undefined,
       12345,
       1,
       globalVariablesBuilder,

yarn-project/aztec-node/src/aztec-node/server.ts

@@ -74,6 +74,7 @@ import {
   TxStatus,
   type TxValidationResult,
 } from '@aztec/stdlib/tx';
+import type { ValidatorsStats } from '@aztec/stdlib/validators';
 import {
   Attributes,
   type TelemetryClient,
@@ -85,6 +86,8 @@ import {
 import { createValidatorClient } from '@aztec/validator-client';
 import { createWorldStateSynchronizer } from '@aztec/world-state';
 
+import { createSentinel } from '../sentinel/factory.js';
+import { Sentinel } from '../sentinel/sentinel.js';
 import { type AztecNodeConfig, getPackageVersion } from './config.js';
 import { NodeMetrics } from './node_metrics.js';
 
@@ -109,6 +112,7 @@ export class AztecNodeService implements AztecNode, AztecNodeAdmin, Traceable {
     protected readonly l1ToL2MessageSource: L1ToL2MessageSource,
     protected readonly worldStateSynchronizer: WorldStateSynchronizer,
     protected readonly sequencer: SequencerClient | undefined,
+    protected readonly validatorsSentinel: Sentinel | undefined,
     protected readonly l1ChainId: number,
     protected readonly version: number,
     protected readonly globalVariableBuilder: GlobalVariableBuilder,
@@ -199,6 +203,9 @@ export class AztecNodeService implements AztecNode, AztecNodeAdmin, Traceable {
 
     const validatorClient = createValidatorClient(config, { p2pClient, telemetry, dateProvider, epochCache });
 
+    const validatorsSentinel = await createSentinel(epochCache, archiver, p2pClient, config);
+    await validatorsSentinel?.start();
+
     // now create the sequencer
     const sequencer = config.disableValidator
       ? undefined
@@ -225,6 +232,7 @@ export class AztecNodeService implements AztecNode, AztecNodeAdmin, Traceable {
       archiver,
       worldStateSynchronizer,
       sequencer,
+      validatorsSentinel,
       ethereumChain.chainInfo.id,
       config.version,
       new GlobalVariableBuilder(config),
@@ -471,6 +479,7 @@ export class AztecNodeService implements AztecNode, AztecNodeAdmin, Traceable {
   public async stop() {
     this.log.info(`Stopping`);
     await this.txQueue.end();
+    await this.validatorsSentinel?.stop();
     await this.sequencer?.stop();
     await this.p2pClient.stop();
     await this.worldStateSynchronizer.stop();
@@ -978,6 +987,10 @@ export class AztecNodeService implements AztecNode, AztecNodeAdmin, Traceable {
     return Promise.resolve();
   }
 
+  public getValidatorsStats(): Promise<ValidatorsStats> {
+    return this.validatorsSentinel?.computeStats() ?? Promise.resolve({ stats: {}, slotWindow: 0 });
+  }
+
   /**
    * Returns an instance of MerkleTreeOperations having first ensured the world state is fully synched
    * @param blockNumber - The block number at which to get the data.

yarn-project/aztec-node/src/sentinel/config.ts

@@ -0,0 +1,19 @@
+import { type ConfigMappingsType, booleanConfigHelper, numberConfigHelper } from '@aztec/foundation/config';
+
+export type SentinelConfig = {
+  sentinelHistoryLengthInEpochs: number;
+  sentinelEnabled: boolean;
+};
+
+export const sentinelConfigMappings: ConfigMappingsType<SentinelConfig> = {
+  sentinelHistoryLengthInEpochs: {
+    description: 'The number of L2 epochs kept of history for each validator for computing their stats.',
+    env: 'SENTINEL_HISTORY_LENGTH_IN_EPOCHS',
+    ...numberConfigHelper(24),
+  },
+  sentinelEnabled: {
+    description: 'Whether the sentinel is enabled or not.',
+    env: 'SENTINEL_ENABLED',
+    ...booleanConfigHelper(true),
+  },
+};

yarn-project/aztec-node/src/sentinel/factory.ts

@@ -0,0 +1,31 @@
+import type { EpochCache } from '@aztec/epoch-cache';
+import { createLogger } from '@aztec/foundation/log';
+import type { DataStoreConfig } from '@aztec/kv-store/config';
+import { createStore } from '@aztec/kv-store/lmdb-v2';
+import type { P2PClient } from '@aztec/p2p';
+import type { L2BlockSource } from '@aztec/stdlib/block';
+
+import type { SentinelConfig } from './config.js';
+import { Sentinel } from './sentinel.js';
+import { SentinelStore } from './store.js';
+
+export async function createSentinel(
+  epochCache: EpochCache,
+  archiver: L2BlockSource,
+  p2p: P2PClient,
+  config: SentinelConfig & DataStoreConfig,
+  logger = createLogger('node:sentinel'),
+): Promise<Sentinel | undefined> {
+  if (!config.sentinelEnabled) {
+    return undefined;
+  }
+  const kvStore = await createStore(
+    'sentinel',
+    SentinelStore.SCHEMA_VERSION,
+    config,
+    createLogger('node:sentinel:lmdb'),
+  );
+  const storeHistoryLength = config.sentinelHistoryLengthInEpochs * epochCache.getL1Constants().epochDuration;
+  const sentinelStore = new SentinelStore(kvStore, { historyLength: storeHistoryLength });
+  return new Sentinel(epochCache, archiver, p2p, sentinelStore, logger);
+}

yarn-project/aztec-node/src/sentinel/index.ts

@@ -0,0 +1,8 @@
+export { Sentinel } from './sentinel.js';
+
+export type {
+  ValidatorsStats,
+  ValidatorStats,
+  ValidatorStatusHistory,
+  ValidatorStatusInSlot,
+} from '@aztec/stdlib/validators';

yarn-project/aztec-node/src/sentinel/sentinel.test.ts

@@ -0,0 +1,228 @@
+import type { EpochCache } from '@aztec/epoch-cache';
+import { times } from '@aztec/foundation/collection';
+import { Secp256k1Signer } from '@aztec/foundation/crypto';
+import { EthAddress } from '@aztec/foundation/eth-address';
+import { AztecLMDBStoreV2, openTmpStore } from '@aztec/kv-store/lmdb-v2';
+import type { P2PClient } from '@aztec/p2p';
+import {
+  type L2BlockSource,
+  type L2BlockStream,
+  type PublishedL2Block,
+  randomPublishedL2Block,
+} from '@aztec/stdlib/block';
+import type { L1RollupConstants } from '@aztec/stdlib/epoch-helpers';
+import type { BlockAttestation } from '@aztec/stdlib/p2p';
+import { makeBlockAttestation } from '@aztec/stdlib/testing';
+import type { ValidatorStats, ValidatorStatusHistory } from '@aztec/stdlib/validators';
+
+import { type MockProxy, mock } from 'jest-mock-extended';
+
+import { Sentinel } from './sentinel.js';
+import { SentinelStore } from './store.js';
+
+describe('sentinel', () => {
+  let epochCache: MockProxy<EpochCache>;
+  let archiver: MockProxy<L2BlockSource>;
+  let p2p: MockProxy<P2PClient>;
+  let blockStream: MockProxy<L2BlockStream>;
+
+  let kvStore: AztecLMDBStoreV2;
+  let store: SentinelStore;
+
+  let sentinel: TestSentinel;
+
+  let slot: bigint;
+  let epoch: bigint;
+  let ts: bigint;
+  let l1Constants: L1RollupConstants;
+
+  beforeEach(async () => {
+    epochCache = mock<EpochCache>();
+    archiver = mock<L2BlockSource>();
+    p2p = mock<P2PClient>();
+    blockStream = mock<L2BlockStream>();
+
+    kvStore = await openTmpStore('sentinel-test');
+    store = new SentinelStore(kvStore, { historyLength: 10 });
+
+    slot = 10n;
+    epoch = 0n;
+    ts = BigInt(Math.ceil(Date.now() / 1000));
+    l1Constants = {
+      l1StartBlock: 1n,
+      l1GenesisTime: ts,
+      slotDuration: 24,
+      epochDuration: 32,
+      ethereumSlotDuration: 12,
+    };
+
+    epochCache.getEpochAndSlotNow.mockReturnValue({ epoch, slot, ts });
+    epochCache.getL1Constants.mockReturnValue(l1Constants);
+
+    sentinel = new TestSentinel(epochCache, archiver, p2p, store, blockStream);
+  });
+
+  afterEach(async () => {
+    await kvStore.close();
+  });
+
+  describe('getSlotActivity', () => {
+    let signers: Secp256k1Signer[];
+    let validators: EthAddress[];
+    let block: PublishedL2Block;
+    let attestations: BlockAttestation[];
+    let proposer: EthAddress;
+    let committee: EthAddress[];
+
+    beforeEach(async () => {
+      signers = times(4, Secp256k1Signer.random);
+      validators = signers.map(signer => signer.address);
+      block = await randomPublishedL2Block(Number(slot));
+      attestations = await Promise.all(
+        signers.map(signer => makeBlockAttestation({ signer, archive: block.block.archive.root })),
+      );
+      proposer = validators[0];
+      committee = [...validators];
+
+      p2p.getAttestationsForSlot.mockResolvedValue(attestations);
+    });
+
+    it('flags block as mined', async () => {
+      await sentinel.handleBlockStreamEvent({ type: 'blocks-added', blocks: [block] });
+
+      const activity = await sentinel.getSlotActivity(slot, epoch, proposer, committee);
+      expect(activity[proposer.toString()]).toEqual('block-mined');
+    });
+
+    it('flags block as proposed when it is not mined but there are attestations', async () => {
+      p2p.getAttestationsForSlot.mockResolvedValue(attestations);
+      const activity = await sentinel.getSlotActivity(slot, epoch, proposer, committee);
+      expect(activity[proposer.toString()]).toEqual('block-proposed');
+    });
+
+    it('flags block as missed when there are no attestations', async () => {
+      p2p.getAttestationsForSlot.mockResolvedValue([]);
+      const activity = await sentinel.getSlotActivity(slot, epoch, proposer, committee);
+      expect(activity[proposer.toString()]).toEqual('block-missed');
+    });
+
+    it('identifies missed attestors if block is mined', async () => {
+      await sentinel.handleBlockStreamEvent({ type: 'blocks-added', blocks: [block] });
+      p2p.getAttestationsForSlot.mockResolvedValue(attestations.slice(0, -1));
+
+      const activity = await sentinel.getSlotActivity(slot, epoch, proposer, committee);
+      expect(activity[committee[1].toString()]).toEqual('attestation-sent');
+      expect(activity[committee[2].toString()]).toEqual('attestation-sent');
+      expect(activity[committee[3].toString()]).toEqual('attestation-missed');
+    });
+
+    it('identifies missed attestors if block is proposed', async () => {
+      p2p.getAttestationsForSlot.mockResolvedValue(attestations.slice(0, -1));
+
+      const activity = await sentinel.getSlotActivity(slot, epoch, proposer, committee);
+      expect(activity[committee[1].toString()]).toEqual('attestation-sent');
+      expect(activity[committee[2].toString()]).toEqual('attestation-sent');
+      expect(activity[committee[3].toString()]).toEqual('attestation-missed');
+    });
+
+    it('does not tag attestors as missed if there was no block and no attestations', async () => {
+      p2p.getAttestationsForSlot.mockResolvedValue([]);
+
+      const activity = await sentinel.getSlotActivity(slot, epoch, proposer, committee);
+      expect(activity[proposer.toString()]).toEqual('block-missed');
+      expect(activity[committee[1].toString()]).not.toBeDefined();
+      expect(activity[committee[2].toString()]).not.toBeDefined();
+      expect(activity[committee[3].toString()]).not.toBeDefined();
+    });
+  });
+
+  describe('computeStatsForValidator', () => {
+    let validator: `0x${string}`;
+
+    beforeEach(() => {
+      validator = EthAddress.random().toString();
+    });
+
+    it('computes stats correctly', () => {
+      const stats = sentinel.computeStatsForValidator(validator, [
+        { slot: 1n, status: 'block-mined' },
+        { slot: 2n, status: 'block-proposed' },
+        { slot: 3n, status: 'block-missed' },
+        { slot: 4n, status: 'block-missed' },
+        { slot: 5n, status: 'attestation-sent' },
+        { slot: 6n, status: 'attestation-missed' },
+      ]);
+
+      expect(stats.address.toString()).toEqual(validator);
+      expect(stats.totalSlots).toEqual(6);
+      expect(stats.missedProposals.count).toEqual(2);
+      expect(stats.missedProposals.currentStreak).toEqual(2);
+      expect(stats.missedProposals.rate).toEqual(0.5);
+      expect(stats.lastProposal?.slot).toEqual(2n);
+      expect(stats.missedAttestations.count).toEqual(1);
+      expect(stats.missedAttestations.currentStreak).toEqual(1);
+      expect(stats.missedAttestations.rate).toEqual(0.5);
+      expect(stats.lastAttestation?.slot).toEqual(5n);
+    });
+
+    it('resets streaks correctly', () => {
+      const stats = sentinel.computeStatsForValidator(validator, [
+        { slot: 1n, status: 'block-mined' },
+        { slot: 2n, status: 'block-missed' },
+        { slot: 3n, status: 'block-mined' },
+        { slot: 4n, status: 'block-missed' },
+        { slot: 5n, status: 'attestation-sent' },
+        { slot: 6n, status: 'attestation-missed' },
+        { slot: 7n, status: 'attestation-sent' },
+        { slot: 8n, status: 'attestation-missed' },
+      ]);
+
+      expect(stats.address.toString()).toEqual(validator);
+      expect(stats.totalSlots).toEqual(8);
+      expect(stats.missedProposals.count).toEqual(2);
+      expect(stats.missedProposals.currentStreak).toEqual(1);
+      expect(stats.missedProposals.rate).toEqual(0.5);
+      expect(stats.missedAttestations.count).toEqual(2);
+      expect(stats.missedAttestations.currentStreak).toEqual(1);
+      expect(stats.missedAttestations.rate).toEqual(0.5);
+    });
+
+    it('considers only latest slots', () => {
+      const history = times(20, i => ({ slot: BigInt(i), status: 'block-missed' } as const));
+      const stats = sentinel.computeStatsForValidator(validator, history, 15n);
+
+      expect(stats.address.toString()).toEqual(validator);
+      expect(stats.totalSlots).toEqual(5);
+      expect(stats.missedProposals.count).toEqual(5);
+    });
+  });
+});
+
+class TestSentinel extends Sentinel {
+  constructor(
+    epochCache: EpochCache,
+    archiver: L2BlockSource,
+    p2p: P2PClient,
+    store: SentinelStore,
+    protected override blockStream: L2BlockStream,
+  ) {
+    super(epochCache, archiver, p2p, store);
+  }
+
+  public override init() {
+    this.initialSlot = this.epochCache.getEpochAndSlotNow().slot;
+    return Promise.resolve();
+  }
+
+  public override getSlotActivity(slot: bigint, epoch: bigint, proposer: EthAddress, committee: EthAddress[]) {
+    return super.getSlotActivity(slot, epoch, proposer, committee);
+  }
+
+  public override computeStatsForValidator(
+    address: `0x${string}`,
+    history: ValidatorStatusHistory,
+    fromSlot?: bigint,
+  ): ValidatorStats {
+    return super.computeStatsForValidator(address, history, fromSlot);
+  }
+}