Skip to content

fix: HA e2e test order & retries #20383

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
spypsy merged 2 commits into from
Feb 11, 2026
Merged

fix: HA e2e test order & retries #20383

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c48d966
fix: HA e2e test order & retries
spypsy Feb 11, 2026
541385e
fix cleanup too
spypsy Feb 11, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion ci3/run_compose_test
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,11 @@ shutdown=0
trap 'shutdown=1' TERM INT

down() {
docker compose -p "$name" down --remove-orphans --timeout 2 >/dev/null 2>&1 || true
local down_flags="--remove-orphans --timeout 2"
if [ "${REMOVE_COMPOSE_VOLUMES:-0}" -eq 1 ]; then
down_flags="-v $down_flags"
fi
docker compose -p "$name" down $down_flags >/dev/null 2>&1 || true
}
trap down EXIT

Expand Down
3 changes: 2 additions & 1 deletion yarn-project/end-to-end/scripts/run_test.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ case "$type" in
TEST=$test exec run_compose_test $test end-to-end $PWD/web3signer
;;
"ha")
TEST=$test exec run_compose_test $test end-to-end $PWD/ha
# Remove volumes on cleanup for HA tests to ensure clean database state on retries
TEST=$test REMOVE_COMPOSE_VOLUMES=1 exec run_compose_test $test end-to-end $PWD/ha
;;
esac
230 changes: 118 additions & 112 deletions yarn-project/end-to-end/src/composed/ha/e2e_ha_full.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ const NODE_COUNT = 5;
const VALIDATOR_COUNT = 4;
const COMMITTEE_SIZE = 4;

describe.skip('HA Full Setup', () => {
describe('HA Full Setup', () => {
jest.setTimeout(20 * 60 * 1000); // 20 minutes

let logger: Logger;
Expand Down Expand Up @@ -262,7 +262,12 @@ describe.skip('HA Full Setup', () => {

afterEach(async () => {
// Clean up database state between tests
await mainPool.query('DELETE FROM validator_duties');
try {
await mainPool.query('DELETE FROM validator_duties');
} catch (error) {
// Ignore cleanup errors (table might not exist on first run failure)
logger?.warn(`Failed to clean up validator_duties: ${error}`);
}
});

it('should produce blocks with HA coordination and attestations', async () => {
Expand Down Expand Up @@ -373,6 +378,117 @@ describe.skip('HA Full Setup', () => {
}
});

it('should coordinate governance voting across HA nodes', async () => {
logger.info('Testing real governance voting with HA coordination');

const mockGovernancePayload = deployL1ContractsValues.l1ContractAddresses.governanceAddress;
logger.info(`Setting governance payload: ${mockGovernancePayload.toString()}`);

// Configure all HA nodes to vote for this payload
for (let i = 0; i < NODE_COUNT; i++) {
await haNodeServices[i].setConfig({
governanceProposerPayload: mockGovernancePayload,
});
}
logger.info(`All ${NODE_COUNT} HA nodes configured to vote for governance payload`);

// Send a transaction to trigger block building which will also trigger voting
logger.info('Sending transaction to trigger block building...');
const deployer = new ContractDeployer(StatefulTestContractArtifact, wallet);
const receipt = await deployer.deploy(ownerAddress, ownerAddress, 42).send({
from: ownerAddress,
contractAddressSalt: Fr.random(),
wait: { returnReceipt: true },
});
expect(receipt.blockNumber).toBeDefined();
logger.info(`Transaction mined in block ${receipt.blockNumber}`);

// Get the slot of the block that was just built
const [block] = await aztecNode.getCheckpointedBlocks(receipt.blockNumber!, 1);
if (!block) {
throw new Error(`Block ${receipt.blockNumber} not found`);
}
const blockSlot = block.block.header.globalVariables.slotNumber;
logger.info(`Block was built in slot ${blockSlot}`);

// Compute round for governance voting from the block slot
const round = await governanceProposer.computeRound(blockSlot);
logger.info(`Block slot ${blockSlot}, governance round ${round}`);

// Poll L1 for governance votes
logger.info('Polling L1 for governance votes...');
const l1VoteCount = await retryUntil(
async () => {
const voteCount = Number(
await governanceProposer.getPayloadSignals(
deployL1ContractsValues.l1ContractAddresses.rollupAddress.toString(),
round,
mockGovernancePayload.toString(),
),
);
return voteCount > 0 ? voteCount : undefined;
},
'governance votes to appear on L1',
6, // timeout in seconds (30 attempts * 200ms)
0.2, // interval in seconds (200ms)
);
logger.info(`Found ${l1VoteCount} governance vote(s) on L1 for payload ${mockGovernancePayload.toString()}`);

// Verify votes were actually sent to L1
expect(l1VoteCount).toBeGreaterThan(0);
logger.info(`Verified ${l1VoteCount} governance vote(s) successfully sent to L1`);

// Get L1 round info to determine which slots have actually landed on L1.
// We anchor the comparison on L1's lastSignalSlot since:
// - The DB may have duties for future slots that haven't been published to L1 yet
// - L1 may have signals from earlier slots in the round before the governance payload was set in the DB
const roundInfo = await governanceProposer.getRoundInfo(
deployL1ContractsValues.l1ContractAddresses.rollupAddress.toString(),
round,
);
const lastSignalSlot = Number(roundInfo.lastSignalSlot);
logger.info(
`L1 round ${round} info: lastSignalSlot=${lastSignalSlot}, l1VoteCount=${l1VoteCount}, payloadWithMostSignals=${roundInfo.payloadWithMostSignals}`,
);

// Query governance vote duties only for slots that have actually landed on L1 (up to lastSignalSlot)
const dbResult = await mainPool.query<DutyRow>(
`SELECT * FROM validator_duties WHERE slot::numeric <= $1 AND duty_type = 'GOVERNANCE_VOTE' ORDER BY slot, started_at`,
[lastSignalSlot.toString()],
);
const governanceVoteDuties = dbResult.rows;
logger.info(
`HA database shows ${governanceVoteDuties.length} governance vote duty(ies) up to slot ${lastSignalSlot}`,
);

if (governanceVoteDuties.length > 0) {
// Verify HA coordination: Only one duty per (slot, validator) should exist
const dutyKeys = governanceVoteDuties.map(row => `${row.slot}-${row.validator_address}`);
const uniqueDutyKeys = new Set(dutyKeys);
expect(uniqueDutyKeys.size).toBe(governanceVoteDuties.length); // No duplicate (slot, validator) pairs

// All duties should be completed
for (const duty of governanceVoteDuties) {
logger.info(
` Governance vote duty: slot ${duty.slot}, validator ${duty.validator_address}, node ${duty.node_id}, status ${duty.status}`,
);
expect(duty.status).toBe(DutyStatus.SIGNED);
expect(duty.completed_at).toBeDefined();
}

// L1 votes should match the number of unique slots in the DB that have landed on L1
const uniqueSlots = new Set(governanceVoteDuties.map(row => row.slot));
logger.info(
`L1 vote count: ${l1VoteCount}, unique slots in DB with governance votes up to L1 lastSignalSlot: ${uniqueSlots.size} (slots: ${[...uniqueSlots].join(', ')})`,
);
expect(l1VoteCount).toBe(uniqueSlots.size);
logger.info(`Verified L1 votes (${l1VoteCount}) === unique slots with votes (${uniqueSlots.size})`);
}

logger.info('Governance voting with HA coordination and L1 verification complete');
});

// NOTE: this test needs to run last
it('should distribute work across multiple HA nodes', async () => {
logger.info('Testing HA resilience by killing nodes after they produce blocks');

Expand Down Expand Up @@ -555,114 +671,4 @@ describe.skip('HA Full Setup', () => {
}
}
});

it('should coordinate governance voting across HA nodes', async () => {
logger.info('Testing real governance voting with HA coordination');

const mockGovernancePayload = deployL1ContractsValues.l1ContractAddresses.governanceAddress;
logger.info(`Setting governance payload: ${mockGovernancePayload.toString()}`);

// Configure all HA nodes to vote for this payload
for (let i = 0; i < NODE_COUNT; i++) {
await haNodeServices[i].setConfig({
governanceProposerPayload: mockGovernancePayload,
});
}
logger.info(`All ${NODE_COUNT} HA nodes configured to vote for governance payload`);

// Send a transaction to trigger block building which will also trigger voting
logger.info('Sending transaction to trigger block building...');
const deployer = new ContractDeployer(StatefulTestContractArtifact, wallet);
const receipt = await deployer.deploy(ownerAddress, ownerAddress, 42).send({
from: ownerAddress,
contractAddressSalt: Fr.random(),
wait: { returnReceipt: true },
});
expect(receipt.blockNumber).toBeDefined();
logger.info(`Transaction mined in block ${receipt.blockNumber}`);

// Get the slot of the block that was just built
const [block] = await aztecNode.getCheckpointedBlocks(receipt.blockNumber!, 1);
if (!block) {
throw new Error(`Block ${receipt.blockNumber} not found`);
}
const blockSlot = block.block.header.globalVariables.slotNumber;
logger.info(`Block was built in slot ${blockSlot}`);

// Compute round for governance voting from the block slot
const round = await governanceProposer.computeRound(blockSlot);
logger.info(`Block slot ${blockSlot}, governance round ${round}`);

// Poll L1 for governance votes
logger.info('Polling L1 for governance votes...');
const l1VoteCount = await retryUntil(
async () => {
const voteCount = Number(
await governanceProposer.getPayloadSignals(
deployL1ContractsValues.l1ContractAddresses.rollupAddress.toString(),
round,
mockGovernancePayload.toString(),
),
);
return voteCount > 0 ? voteCount : undefined;
},
'governance votes to appear on L1',
6, // timeout in seconds (30 attempts * 200ms)
0.2, // interval in seconds (200ms)
);
logger.info(`Found ${l1VoteCount} governance vote(s) on L1 for payload ${mockGovernancePayload.toString()}`);

// Verify votes were actually sent to L1
expect(l1VoteCount).toBeGreaterThan(0);
logger.info(`Verified ${l1VoteCount} governance vote(s) successfully sent to L1`);

// Get L1 round info to determine which slots have actually landed on L1.
// We anchor the comparison on L1's lastSignalSlot since:
// - The DB may have duties for future slots that haven't been published to L1 yet
// - L1 may have signals from earlier slots in the round before the governance payload was set in the DB
const roundInfo = await governanceProposer.getRoundInfo(
deployL1ContractsValues.l1ContractAddresses.rollupAddress.toString(),
round,
);
const lastSignalSlot = Number(roundInfo.lastSignalSlot);
logger.info(
`L1 round ${round} info: lastSignalSlot=${lastSignalSlot}, l1VoteCount=${l1VoteCount}, payloadWithMostSignals=${roundInfo.payloadWithMostSignals}`,
);

// Query governance vote duties only for slots that have actually landed on L1 (up to lastSignalSlot)
const dbResult = await mainPool.query<DutyRow>(
`SELECT * FROM validator_duties WHERE slot::numeric <= $1 AND duty_type = 'GOVERNANCE_VOTE' ORDER BY slot, started_at`,
[lastSignalSlot.toString()],
);
const governanceVoteDuties = dbResult.rows;
logger.info(
`HA database shows ${governanceVoteDuties.length} governance vote duty(ies) up to slot ${lastSignalSlot}`,
);

if (governanceVoteDuties.length > 0) {
// Verify HA coordination: Only one duty per (slot, validator) should exist
const dutyKeys = governanceVoteDuties.map(row => `${row.slot}-${row.validator_address}`);
const uniqueDutyKeys = new Set(dutyKeys);
expect(uniqueDutyKeys.size).toBe(governanceVoteDuties.length); // No duplicate (slot, validator) pairs

// All duties should be completed
for (const duty of governanceVoteDuties) {
logger.info(
` Governance vote duty: slot ${duty.slot}, validator ${duty.validator_address}, node ${duty.node_id}, status ${duty.status}`,
);
expect(duty.status).toBe(DutyStatus.SIGNED);
expect(duty.completed_at).toBeDefined();
}

// L1 votes should match the number of unique slots in the DB that have landed on L1
const uniqueSlots = new Set(governanceVoteDuties.map(row => row.slot));
logger.info(
`L1 vote count: ${l1VoteCount}, unique slots in DB with governance votes up to L1 lastSignalSlot: ${uniqueSlots.size} (slots: ${[...uniqueSlots].join(', ')})`,
);
expect(l1VoteCount).toBe(uniqueSlots.size);
logger.info(`Verified L1 votes (${l1VoteCount}) === unique slots with votes (${uniqueSlots.size})`);
}

logger.info('Governance voting with HA coordination and L1 verification complete');
});
});
4 changes: 3 additions & 1 deletion yarn-project/end-to-end/src/fixtures/ha_setup.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,8 +69,10 @@ export async function cleanupHADatabase(pool: Pool, logger?: Logger): Promise<vo
try {
// Drop all HA tables
await pool.query('DROP TABLE IF EXISTS validator_duties CASCADE');
await pool.query('DROP TABLE IF EXISTS slashing_protection CASCADE');
await pool.query('DROP TABLE IF EXISTS schema_version CASCADE');
// Drop migration tracking table (node-pg-migrate uses 'pgmigrations' by default)
// This ensures migrations will run fresh on next startup
await pool.query('DROP TABLE IF EXISTS pgmigrations CASCADE');

logger?.info('HA database cleaned up successfully');
} catch (error) {
Expand Down
Loading