Skip to content

chore: reduce severity of errors due to HA node not acquiring signature #21311

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
PhilWindle merged 2 commits into from
Mar 11, 2026
Merged

chore: reduce severity of errors due to HA node not acquiring signature #21311

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
af58c18
chore: reduce severity of errors due to HA node not acquiring signature
spypsy Mar 9, 2026
2d59abe
revert logs that shouldn't hit for HA scenarios
spypsy Mar 11, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion yarn-project/archiver/src/modules/data_store_updater.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -178,7 +178,7 @@ export class ArchiverDataStoreUpdater {
this.log.verbose(`Block number ${blockNumber} already inserted and matches checkpoint`, blockInfos);
lastAlreadyInsertedBlockNumber = blockNumber;
} else {
this.log.warn(`Conflict detected at block ${blockNumber} between checkpointed and local block`, blockInfos);
this.log.info(`Conflict detected at block ${blockNumber} between checkpointed and local block`, blockInfos);
const prunedBlocks = await this.removeBlocksAfter(BlockNumber(blockNumber - 1));
return { prunedBlocks, lastAlreadyInsertedBlockNumber };
}
Expand Down
2 changes: 1 addition & 1 deletion yarn-project/archiver/src/modules/l1_synchronizer.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -828,7 +828,7 @@ export class ArchiverL1Synchronizer implements Traceable {
const prunedCheckpointNumber = result.prunedBlocks[0].checkpointNumber;
const prunedSlotNumber = result.prunedBlocks[0].header.globalVariables.slotNumber;

this.log.warn(
this.log.info(
`Pruned ${result.prunedBlocks.length} mismatching blocks for checkpoint ${prunedCheckpointNumber}`,
{ prunedBlocks: result.prunedBlocks.map(b => b.toBlockInfo()), prunedSlotNumber, prunedCheckpointNumber },
);
Expand Down
2 changes: 1 addition & 1 deletion yarn-project/p2p/src/services/libp2p/libp2p_service.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1192,7 +1192,7 @@ export class LibP2PService extends WithTracer implements P2PService {
// Note: Validators do NOT attest to individual blocks, only to checkpoint proposals.
const isValid = await this.blockReceivedCallback(block, sender);
if (!isValid) {
this.logger.warn(`Block proposal validation failed for block ${block.blockNumber}`, block.toBlockInfo());
this.logger.info(`Block proposal validation failed for block ${block.blockNumber}`, block.toBlockInfo());
}
}

Expand Down
3 changes: 2 additions & 1 deletion yarn-project/validator-client/src/block_proposal_handler.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -178,7 +178,8 @@ export class BlockProposalHandler {
// Check that this block number does not exist already
const existingBlock = await this.blockSource.getBlockHeader(blockNumber);
if (existingBlock) {
this.log.warn(`Block number ${blockNumber} already exists, skipping processing`, proposalInfo);
// Expected in HA setups when the partner node already built and inserted this block

@PhilWindle PhilWindle Mar 11, 2026

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Again, I don't think this is expected in HA setups is it? Proposals from self will have already been detected.

this.log.verbose(`Block number ${blockNumber} already exists, skipping processing`, proposalInfo);
return { isValid: false, blockNumber, reason: 'block_number_already_exists' };
}

Expand Down
2 changes: 1 addition & 1 deletion yarn-project/validator-client/src/key_store/ha_key_store.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -240,7 +240,7 @@ export class HAKeyStore implements ExtendedValidatorKeyStore {
}

if (error instanceof SlashingProtectionError) {
this.log.warn(`Duty already signed by another node with different payload`, {
this.log.info(`Duty already signed by another node with different payload`, {
dutyType: context.dutyType,
slot: context.slot,
existingMessageHash: error.existingMessageHash,
Expand Down
13 changes: 9 additions & 4 deletions yarn-project/validator-client/src/validator.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -380,7 +380,7 @@ export class ValidatorClient extends (EventEmitter as new () => WatcherEmitter)

// Ignore proposals from ourselves (may happen in HA setups)
if (this.getValidatorAddresses().some(addr => addr.equals(proposer))) {
this.log.warn(`Ignoring block proposal from self for slot ${slotNumber}`, {
this.log.debug(`Ignoring block proposal from self for slot ${slotNumber}`, {
proposer: proposer.toString(),
slotNumber,
});
Expand Down Expand Up @@ -416,8 +416,6 @@ export class ValidatorClient extends (EventEmitter as new () => WatcherEmitter)
);

if (!validationResult.isValid) {
this.log.warn(`Block proposal validation failed: ${validationResult.reason}`, proposalInfo);

const reason = validationResult.reason || 'unknown';
// Classify failure reason: bad proposal vs node issue
const badProposalReasons: BlockProposalValidationFailureReason[] = [
Expand All @@ -427,6 +425,13 @@ export class ValidatorClient extends (EventEmitter as new () => WatcherEmitter)
'in_hash_mismatch',
'parent_block_wrong_slot',
];
// block_number_already_exists is expected in HA setups when the partner node already built the block

@PhilWindle PhilWindle Mar 11, 2026

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If the proposal came from our HA counterpart, won't we have already detected this further up?

const expectedReasons = ['block_number_already_exists'];
if (expectedReasons.includes(reason)) {
this.log.verbose(`Block proposal validation failed: ${reason}`, proposalInfo);
} else {
this.log.warn(`Block proposal validation failed: ${reason}`, proposalInfo);
}

if (badProposalReasons.includes(reason as BlockProposalValidationFailureReason)) {
this.metrics.incFailedAttestationsBadProposal(1, reason, partOfCommittee);
Expand Down Expand Up @@ -490,7 +495,7 @@ export class ValidatorClient extends (EventEmitter as new () => WatcherEmitter)

// Ignore proposals from ourselves (may happen in HA setups)
if (this.getValidatorAddresses().some(addr => addr.equals(proposer))) {
this.log.warn(`Ignoring block proposal from self for slot ${slotNumber}`, {
this.log.debug(`Ignoring block proposal from self for slot ${slotNumber}`, {
proposer: proposer.toString(),
slotNumber,
});
Expand Down
4 changes: 2 additions & 2 deletions yarn-project/validator-ha-signer/src/slashing_protection_service.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@ export class SlashingProtectionService {

if (isNew) {
// We successfully acquired the lock
this.log.info(`Acquired lock for duty ${dutyType} at slot ${slot}`, {
this.log.verbose(`Acquired lock for duty ${dutyType} at slot ${slot}`, {
validatorAddress: validatorAddress.toString(),
nodeId,
});
Expand Down Expand Up @@ -177,7 +177,7 @@ export class SlashingProtectionService {
);

if (success) {
this.log.info(`Recorded successful signing for duty ${dutyType} at slot ${slot}`, {
this.log.verbose(`Recorded successful signing for duty ${dutyType} at slot ${slot}`, {
validatorAddress: validatorAddress.toString(),
nodeId,
});
Expand Down
2 changes: 1 addition & 1 deletion yarn-project/world-state/src/synchronizer/server_world_state_synchronizer.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -405,7 +405,7 @@ export class ServerWorldStateSynchronizer
}

private async handleChainPruned(blockNumber: BlockNumber) {
this.log.warn(`Chain pruned to block ${blockNumber}`);
this.log.info(`Chain pruned to block ${blockNumber}`);
const status = await this.merkleTreeDb.unwindBlocks(blockNumber);
this.provenBlockNumber = undefined;
this.instrumentation.updateWorldStateMetrics(status);
Expand Down
Loading