Track sync committee metrics

packages/beacon-node/src/api/impl/validator/index.ts

@@ -564,12 +564,15 @@ export function getValidatorApi({
         contributionAndProofs.map(async (contributionAndProof, i) => {
           try {
             // TODO: Validate in batch
-            const {syncCommitteeParticipants} = await validateSyncCommitteeGossipContributionAndProof(
+            const {syncCommitteeParticipantIndices} = await validateSyncCommitteeGossipContributionAndProof(
               chain,
               contributionAndProof,
               true // skip known participants check
             );
-            chain.syncContributionAndProofPool.add(contributionAndProof.message, syncCommitteeParticipants);
+            chain.syncContributionAndProofPool.add(
+              contributionAndProof.message,
+              syncCommitteeParticipantIndices.length
+            );
             await network.gossip.publishContributionAndProof(contributionAndProof);
           } catch (e) {
             errors.push(e as Error);
@@ -647,6 +650,12 @@ export function getValidatorApi({
       }
 
       network.prepareSyncCommitteeSubnets(subs);
+
+      if (metrics) {
+        for (const subscription of subscriptions) {
+          metrics.registerLocalValidatorInSyncCommittee(subscription.validatorIndex, subscription.untilEpoch);
+        }
+      }
     },
 
     async prepareBeaconProposer(proposers) {

packages/beacon-node/src/chain/blocks/importBlock.ts

@@ -1,5 +1,5 @@
-import {capella, ssz, allForks} from "@lodestar/types";
-import {MAX_SEED_LOOKAHEAD, SLOTS_PER_EPOCH} from "@lodestar/params";
+import {capella, ssz, allForks, altair} from "@lodestar/types";
+import {ForkSeq, MAX_SEED_LOOKAHEAD, SLOTS_PER_EPOCH} from "@lodestar/params";
 import {toHexString} from "@chainsafe/ssz";
 import {
   CachedBeaconStateAltair,
@@ -384,6 +384,13 @@ export async function importBlock(
   this.metrics?.parentBlockDistance.observe(block.message.slot - parentBlockSlot);
   this.metrics?.proposerBalanceDeltaAny.observe(fullyVerifiedBlock.proposerBalanceDelta);
   this.metrics?.registerImportedBlock(block.message, fullyVerifiedBlock);
+  if (this.config.getForkSeq(block.message.slot) >= ForkSeq.altair) {
+    this.metrics?.registerSyncAggregateInBlock(
+      blockEpoch,
+      (block as altair.SignedBeaconBlock).message.body.syncAggregate,
+      fullyVerifiedBlock.postState.epochCtx.currentSyncCommitteeIndexed.validatorIndices
+    );
+  }
 
   const advancedSlot = this.clock.slotWithFutureTolerance(REPROCESS_MIN_TIME_TO_NEXT_SLOT_SEC);
 

packages/beacon-node/src/chain/validation/syncCommitteeContributionAndProof.ts

@@ -17,7 +17,7 @@ export async function validateSyncCommitteeGossipContributionAndProof(
   chain: IBeaconChain,
   signedContributionAndProof: altair.SignedContributionAndProof,
   skipValidationKnownParticipants = false
-): Promise<{syncCommitteeParticipants: number}> {
+): Promise<{syncCommitteeParticipantIndices: ValidatorIndex[]}> {
   const contributionAndProof = signedContributionAndProof.message;
   const {contribution, aggregatorIndex} = contributionAndProof;
   const {subcommitteeIndex, slot} = contribution;
@@ -53,8 +53,8 @@ export async function validateSyncCommitteeGossipContributionAndProof(
   }
 
   // [REJECT] The contribution has participants -- that is, any(contribution.aggregation_bits)
-  const syncCommitteeIndices = getContributionIndices(headState as CachedBeaconStateAltair, contribution);
-  if (!syncCommitteeIndices.length) {
+  const syncCommitteeParticipantIndices = getContributionIndices(headState as CachedBeaconStateAltair, contribution);
+  if (syncCommitteeParticipantIndices.length === 0) {
     throw new SyncCommitteeError(GossipAction.REJECT, {
       code: SyncCommitteeErrorCode.NO_PARTICIPANT,
     });
@@ -73,7 +73,9 @@ export async function validateSyncCommitteeGossipContributionAndProof(
   // i.e. state.validators[contribution_and_proof.aggregator_index].pubkey in get_sync_subcommittee_pubkeys(state, contribution.subcommittee_index).
   // > Checked in validateGossipSyncCommitteeExceptSig()
 
-  const pubkeys = syncCommitteeIndices.map((validatorIndex) => headState.epochCtx.index2pubkey[validatorIndex]);
+  const participantPubkeys = syncCommitteeParticipantIndices.map(
+    (validatorIndex) => headState.epochCtx.index2pubkey[validatorIndex]
+  );
   const signatureSets = [
     // [REJECT] The contribution_and_proof.selection_proof is a valid signature of the SyncAggregatorSelectionData
     // derived from the contribution by the validator with index contribution_and_proof.aggregator_index.
@@ -84,7 +86,7 @@ export async function validateSyncCommitteeGossipContributionAndProof(
 
     // [REJECT] The aggregate signature is valid for the message beacon_block_root and aggregate pubkey derived from
     // the participation info in aggregation_bits for the subcommittee specified by the contribution.subcommittee_index.
-    getSyncCommitteeContributionSignatureSet(headState as CachedBeaconStateAltair, contribution, pubkeys),
+    getSyncCommitteeContributionSignatureSet(headState as CachedBeaconStateAltair, contribution, participantPubkeys),
   ];
 
   if (!(await chain.bls.verifySignatureSets(signatureSets, {batchable: true}))) {
@@ -94,9 +96,9 @@ export async function validateSyncCommitteeGossipContributionAndProof(
   }
 
   // no need to add to seenSyncCommittteeContributionCache here, gossip handler will do that
-  chain.seenContributionAndProof.add(contributionAndProof, syncCommitteeIndices.length);
+  chain.seenContributionAndProof.add(contributionAndProof, syncCommitteeParticipantIndices.length);
 
-  return {syncCommitteeParticipants: syncCommitteeIndices.length};
+  return {syncCommitteeParticipantIndices};
 }
 
 /**

packages/beacon-node/src/metrics/metrics/lodestar.ts

@@ -746,7 +746,11 @@ export function createLodestarMetrics(
       validatorsConnected: register.gauge({
         name: "validator_monitor_validators",
         help: "Count of validators that are specifically monitored by this beacon node",
-        labelNames: ["index"],
+      }),
+
+      validatorsInSyncCommittee: register.gauge({
+        name: "validator_monitor_validators_in_sync_committee",
+        help: "Count of validators monitored by this beacon node that are part of sync committee",
       }),
 
       // Validator Monitor Metrics (per-epoch summaries)
@@ -850,6 +854,19 @@ export function createLodestarMetrics(
         help: "The min delay between when the validator should send the aggregate and when it was received",
         buckets: [0.1, 0.25, 0.5, 1, 2, 5, 10],
       }),
+      prevEpochSyncCommitteeHits: register.gauge({
+        name: "validator_monitor_prev_epoch_sync_committee_hits",
+        help: "Count of times in prev epoch connected validators participated in imported block's syncAggregate",
+      }),
+      prevEpochSyncCommitteeMisses: register.gauge({
+        name: "validator_monitor_prev_epoch_sync_committee_misses",
+        help: "Count of times in prev epoch connected validators fail to participate in imported block's syncAggregate",
+      }),
+      prevEpochSyncSignatureAggregateInclusions: register.histogram({
+        name: "validator_monitor_prev_epoch_sync_signature_aggregate_inclusions",
+        help: "The count of times a sync signature was seen inside an aggregate",
+        buckets: [0, 1, 2, 3, 5, 10],
+      }),
 
       // Validator Monitor Metrics (real-time)
 
@@ -903,6 +920,10 @@ export function createLodestarMetrics(
         help: "The excess slots (beyond the minimum delay) between the attestation slot and the block slot",
         buckets: [0.1, 0.25, 0.5, 1, 2, 5, 10],
       }),
+      syncSignatureInAggregateTotal: register.gauge({
+        name: "validator_monitor_sync_signature_in_aggregate_total",
+        help: "Number of times a sync signature has been seen in an aggregate",
+      }),
       beaconBlockTotal: register.gauge<"src">({
         name: "validator_monitor_beacon_block_total",
         help: "Total number of beacon blocks seen",

packages/beacon-node/src/metrics/validatorMonitor.ts

@@ -1,6 +1,6 @@
 import {computeEpochAtSlot, IAttesterStatus, parseAttesterFlags} from "@lodestar/state-transition";
 import {ILogger} from "@lodestar/utils";
-import {allForks} from "@lodestar/types";
+import {allForks, altair} from "@lodestar/types";
 import {IChainForkConfig} from "@lodestar/config";
 import {MIN_ATTESTATION_INCLUSION_DELAY, SLOTS_PER_EPOCH} from "@lodestar/params";
 import {Epoch, Slot, ValidatorIndex} from "@lodestar/types";
@@ -20,6 +20,7 @@ export enum OpSource {
 
 export interface IValidatorMonitor {
   registerLocalValidator(index: number): void;
+  registerLocalValidatorInSyncCommittee(index: number, untilEpoch: Epoch): void;
   registerValidatorStatuses(currentEpoch: Epoch, statuses: IAttesterStatus[], balances?: number[]): void;
   registerBeaconBlock(src: OpSource, seenTimestampSec: Seconds, block: allForks.BeaconBlock): void;
   registerImportedBlock(block: allForks.BeaconBlock, data: {proposerBalanceDelta: number}): void;
@@ -41,6 +42,11 @@ export interface IValidatorMonitor {
     indexedAttestation: IndexedAttestation
   ): void;
   registerAttestationInBlock(indexedAttestation: IndexedAttestation, parentSlot: Slot, correctHead: boolean): void;
+  registerGossipSyncContributionAndProof(
+    syncContributionAndProof: altair.ContributionAndProof,
+    syncCommitteeParticipantIndices: ValidatorIndex[]
+  ): void;
+  registerSyncAggregateInBlock(epoch: Epoch, syncAggregate: altair.SyncAggregate, syncCommitteeIndices: number[]): void;
   scrapeMetrics(slotClock: Slot): void;
 }
 
@@ -121,6 +127,12 @@ type EpochSummary = {
   aggregates: number;
   /** The delay between when the aggregate should have been produced and when it was observed. */
   aggregateMinDelay: Seconds | null;
+  /** Count of times validator expected in sync aggregate participated */
+  syncCommitteeHits: number;
+  /** Count of times validator expected in sync aggregate failed to participate */
+  syncCommitteeMisses: number;
+  /** Number of times a validator's sync signature was seen in an aggregate */
+  syncSignatureAggregateInclusions: number;
 };
 
 function withEpochSummary(validator: MonitoredValidator, epoch: Epoch, fn: (summary: EpochSummary) => void): void {
@@ -137,6 +149,9 @@ function withEpochSummary(validator: MonitoredValidator, epoch: Epoch, fn: (summ
       aggregates: 0,
       aggregateMinDelay: null,
       attestationCorrectHead: null,
+      syncCommitteeHits: 0,
+      syncCommitteeMisses: 0,
+      syncSignatureAggregateInclusions: 0,
     };
     validator.summaries.set(epoch, summary);
   }
@@ -160,6 +175,7 @@ type MonitoredValidator = {
   index: number;
   /// A history of the validator over time. */
   summaries: Map<Epoch, EpochSummary>;
+  inSyncCommitteeUntilEpoch: number;
 };
 
 export function createValidatorMonitor(
@@ -176,7 +192,14 @@ export function createValidatorMonitor(
   return {
     registerLocalValidator(index) {
       if (!validators.has(index)) {
-        validators.set(index, {index, summaries: new Map<Epoch, EpochSummary>()});
+        validators.set(index, {index, summaries: new Map<Epoch, EpochSummary>(), inSyncCommitteeUntilEpoch: -1});
+      }
+    },
+
+    registerLocalValidatorInSyncCommittee(index, untilEpoch) {
+      const validator = validators.get(index);
+      if (validator) {
+        validator.inSyncCommitteeUntilEpoch = Math.max(untilEpoch, validator.inSyncCommitteeUntilEpoch ?? -1);
       }
     },
 
@@ -417,6 +440,36 @@ export function createValidatorMonitor(
       }
     },
 
+    registerGossipSyncContributionAndProof(syncContributionAndProof, syncCommitteeParticipantIndices) {
+      const epoch = computeEpochAtSlot(syncContributionAndProof.contribution.slot);
+
+      for (const index of syncCommitteeParticipantIndices) {
+        const validator = validators.get(index);
+        if (validator) {
+          metrics.validatorMonitor.syncSignatureInAggregateTotal.inc();
+
+          withEpochSummary(validator, epoch, (summary) => {
+            summary.syncSignatureAggregateInclusions += 1;
+          });
+        }
+      }
+    },
+
+    registerSyncAggregateInBlock(epoch, syncAggregate, syncCommitteeIndices) {
+      for (let i = 0; i < syncCommitteeIndices.length; i++) {
+        const validator = validators.get(syncCommitteeIndices[i]);
+        if (validator) {
+          withEpochSummary(validator, epoch, (summary) => {
+            if (syncAggregate.syncCommitteeBits.get(i)) {
+              summary.syncCommitteeHits++;
+            } else {
+              summary.syncCommitteeMisses++;
+            }
+          });
+        }
+      }
+    },
+
     /**
      * Scrape `self` for metrics.
      * Should be called whenever Prometheus is scraping.
@@ -444,8 +497,20 @@ export function createValidatorMonitor(
       metrics.validatorMonitor.prevEpochAttestationAggregateInclusions.reset();
       metrics.validatorMonitor.prevEpochAttestationBlockInclusions.reset();
       metrics.validatorMonitor.prevEpochAttestationBlockMinInclusionDistance.reset();
+      metrics.validatorMonitor.prevEpochSyncSignatureAggregateInclusions.reset();
+
+      let validatorsInSyncCommittee = 0;
+      let prevEpochSyncCommitteeHits = 0;
+      let prevEpochSyncCommitteeMisses = 0;
 
       for (const validator of validators.values()) {
+        // Participation in sync committee
+        const validatorInSyncCommittee = validator.inSyncCommitteeUntilEpoch >= epoch;
+        if (validatorInSyncCommittee) {
+          validatorsInSyncCommittee++;
+        }
+
+        // Prev-epoch summary
         const summary = validator.summaries.get(previousEpoch);
         if (!summary) {
           continue;
@@ -472,7 +537,22 @@ export function createValidatorMonitor(
         metrics.validatorMonitor.prevEpochAggregatesTotal.observe(summary.aggregates);
         if (summary.aggregateMinDelay !== null)
           metrics.validatorMonitor.prevEpochAggregatesMinDelaySeconds.observe(summary.aggregateMinDelay);
+
+        // Sync committee
+        prevEpochSyncCommitteeHits += summary.syncCommitteeHits;
+        prevEpochSyncCommitteeMisses += summary.syncCommitteeMisses;
+
+        // Only observe if included in sync committee to prevent distorting metrics
+        if (validatorInSyncCommittee) {
+          metrics.validatorMonitor.prevEpochSyncSignatureAggregateInclusions.observe(
+            summary.syncSignatureAggregateInclusions
+          );
+        }
       }
+
+      metrics.validatorMonitor.validatorsInSyncCommittee.set(validatorsInSyncCommittee);
+      metrics.validatorMonitor.prevEpochSyncCommitteeHits.set(prevEpochSyncCommitteeHits);
+      metrics.validatorMonitor.prevEpochSyncCommitteeMisses.set(prevEpochSyncCommitteeMisses);
     },
   };
 }

packages/beacon-node/src/network/gossip/handlers/index.ts

@@ -301,7 +301,7 @@ export function getGossipHandlers(modules: ValidatorFnsModules, options: GossipH
     },
 
     [GossipType.sync_committee_contribution_and_proof]: async (contributionAndProof) => {
-      const {syncCommitteeParticipants} = await validateSyncCommitteeGossipContributionAndProof(
+      const {syncCommitteeParticipantIndices} = await validateSyncCommitteeGossipContributionAndProof(
         chain,
         contributionAndProof
       ).catch((e) => {
@@ -312,9 +312,10 @@ export function getGossipHandlers(modules: ValidatorFnsModules, options: GossipH
       });
 
       // Handler
+      metrics?.registerGossipSyncContributionAndProof(contributionAndProof.message, syncCommitteeParticipantIndices);
 
       try {
-        chain.syncContributionAndProofPool.add(contributionAndProof.message, syncCommitteeParticipants);
+        chain.syncContributionAndProofPool.add(contributionAndProof.message, syncCommitteeParticipantIndices.length);
       } catch (e) {
         logger.error("Error adding to contributionAndProof pool", {}, e as Error);
       }