Skip to content
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
207 changes: 199 additions & 8 deletions .github/scripts/__tests__/keepalive-loop.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ const buildGithubStub = ({
failNeedsAttentionLabel = false,
failNeedsAttentionRemoval = false,
failStateCommentWriteAt = 0,
failWorkflowDispatch = false,
} = {}) => {
const actions = [];
let stateCommentWriteCount = 0;
Expand Down Expand Up @@ -85,6 +86,10 @@ const buildGithubStub = ({
return { data: buffer };
},
async createWorkflowDispatch(payload) {
if (failWorkflowDispatch) {
actions.push({ type: 'workflow-dispatch-failed', ...payload });
throw new Error('simulated workflow dispatch failure');
}
actions.push({ type: 'workflow-dispatch', ...payload });
return { data: {} };
},
Expand Down Expand Up @@ -375,6 +380,58 @@ test('evaluateKeepaliveLoop stops when round budget is exhausted', async () => {
assert.equal(result.reason, 'round-budget-exhausted');
});

test('evaluateKeepaliveLoop grants a forced recovery lease across the round budget', async () => {
const pr = {
number: 406,
head: { ref: 'feature/forced-budget-recovery', sha: 'sha-forced-budget' },
labels: [{ name: 'agent:codex' }],
body: '## Tasks\n- [ ] one\n## Acceptance Criteria\n- [ ] a\n<!-- keepalive-config: {"iteration": 5, "max_iterations": 5} -->',
};
const github = buildGithubStub({
pr,
workflowRuns: [{ head_sha: 'sha-forced-budget', conclusion: 'success' }],
});

const result = await evaluateKeepaliveLoop({
github,
context: buildContext(pr.number),
core: buildCore(),
forceRetry: true,
});

assert.equal(result.action, 'run');
assert.equal(result.reason, 'ready');
assert.equal(result.forceRetry, true);
});

test('evaluateKeepaliveLoop grants a forced recovery lease after verification exhaustion', async () => {
const pr = {
number: 408,
head: { ref: 'feature/forced-verification-recovery', sha: 'sha-forced-verification' },
labels: [{ name: 'agent:codex' }],
body: prBodyFixture.replace(/- \[ \]/g, '- [x]'),
};
const existingState = formatStateComment({
trace: 'fixture-trace',
verification: { status: 'failed', iteration: 2, attempt_count: 2 },
});
const github = buildGithubStub({
pr,
comments: [{ id: 24, body: existingState, html_url: 'https://example.com/24' }],
workflowRuns: [{ head_sha: 'sha-forced-verification', conclusion: 'success' }],
});

const result = await evaluateKeepaliveLoop({
github,
context: buildContext(pr.number),
core: buildCore(),
forceRetry: true,
});

assert.equal(result.action, 'run');
assert.equal(result.reason, 'fix-verification-gaps');
});

test('evaluateKeepaliveLoop stops at max iterations even when productive with tasks remaining', async () => {
const pr = {
number: 405,
Expand Down Expand Up @@ -2116,12 +2173,10 @@ test('updateKeepaliveLoopSummary routes repeated actual failures to automation r
const retryLabel = github.actions.find((action) =>
action.type === 'label' && action.labels.includes('agent:retry')
);
assert.ok(retryLabel);
assert.equal(
github.actions.some((action) => action.type === 'workflow-dispatch'),
false,
'a stopped strategy must wait for the scheduled recovery sweep instead of self-recursing',
);
assert.equal(retryLabel, undefined);
const retryDispatch = github.actions.find((action) => action.type === 'workflow-dispatch');
assert.ok(retryDispatch, 'a newly stopped strategy must receive one immediate recovery lease');
assert.deepEqual(retryDispatch.inputs, { pr_number: '457', force_retry: 'true' });

const humanLabel = github.actions.find((action) =>
action.type === 'label' && (
Expand Down Expand Up @@ -3513,7 +3568,7 @@ test('updateKeepaliveLoopSummary routes resource failures to automation retry',
const retryLabel = github.actions.find((action) =>
action.type === 'label' && action.labels.includes('agent:retry')
);
assert.ok(retryLabel);
assert.equal(retryLabel, undefined);
const retryDispatch = github.actions.find((action) => action.type === 'workflow-dispatch');
assert.equal(retryDispatch.workflow_id, 'agents-81-gate-followups.yml');
assert.deepEqual(retryDispatch.inputs, { pr_number: '655', force_retry: 'true' });
Expand All @@ -3523,6 +3578,95 @@ test('updateKeepaliveLoopSummary routes resource failures to automation retry',
);
});

test('automation-owned terminal stops dispatch exactly one forced recovery lease', async () => {
for (const reason of [
'round-budget-exhausted',
'verification-exhausted',
'zero-activity-infrastructure',
]) {
for (const forceRetry of [false, true]) {
const existingState = formatStateComment({
trace: `trace-terminal-${reason}-${forceRetry}`,
iteration: 5,
max_iterations: 5,
failure_threshold: 3,
failure: {},
});
const github = buildGithubStub({
comments: [{ id: 104, body: existingState, html_url: 'https://example.com/104' }],
});

await updateKeepaliveLoopSummary({
github,
context: buildContext(659),
core: buildCore(),
inputs: {
prNumber: 659,
action: 'stop',
reason,
gateConclusion: 'success',
tasksTotal: 3,
tasksUnchecked: reason === 'verification-exhausted' ? 0 : 2,
keepaliveEnabled: true,
autofixEnabled: false,
iteration: 5,
maxIterations: 5,
failureThreshold: 3,
trace: `trace-terminal-${reason}-${forceRetry}`,
forceRetry,
retry_workflow_id: 'agents-81-gate-followups.yml',
},
});

const dispatches = github.actions.filter((action) => action.type === 'workflow-dispatch');
assert.equal(dispatches.length, forceRetry ? 0 : 1, `${reason} forceRetry=${forceRetry}`);
if (!forceRetry) {
assert.deepEqual(dispatches[0].inputs, { pr_number: '659', force_retry: 'true' });
}
}
}
});

test('a failed bounded dispatch adds agent:retry only as a fallback', async () => {
const existingState = formatStateComment({
trace: 'trace-dispatch-fallback',
iteration: 5,
max_iterations: 5,
failure_threshold: 3,
failure: {},
});
const github = buildGithubStub({
comments: [{ id: 105, body: existingState, html_url: 'https://example.com/105' }],
failWorkflowDispatch: true,
});

await updateKeepaliveLoopSummary({
github,
context: buildContext(660),
core: buildCore(),
inputs: {
prNumber: 660,
action: 'stop',
reason: 'round-budget-exhausted',
gateConclusion: 'success',
tasksTotal: 3,
tasksUnchecked: 2,
keepaliveEnabled: true,
autofixEnabled: false,
iteration: 5,
maxIterations: 5,
failureThreshold: 3,
trace: 'trace-dispatch-fallback',
retry_workflow_id: 'agents-81-gate-followups.yml',
},
});

assert.ok(github.actions.some((action) => action.type === 'workflow-dispatch-failed'));
assert.ok(github.actions.some((action) =>
action.type === 'label' && action.labels.includes('agent:retry')
));
});

test('updateKeepaliveLoopSummary routes logic failures to automation retry', async () => {
const existingState = formatStateComment({
trace: 'trace-attention-logic',
Expand Down Expand Up @@ -3559,7 +3703,7 @@ test('updateKeepaliveLoopSummary routes logic failures to automation retry', asy
const retryLabel = github.actions.find((action) =>
action.type === 'label' && action.labels.includes('agent:retry')
);
assert.ok(retryLabel);
assert.equal(retryLabel, undefined);
assert.ok(github.actions.some((action) => action.type === 'workflow-dispatch'));
assert.equal(
github.actions.some((action) => action.type === 'label' && action.labels.includes('needs-human')),
Expand Down Expand Up @@ -3730,6 +3874,53 @@ test('updateKeepaliveLoopSummary clears stale human-blocker labels on tasks-comp
);
});

test('a successful terminal migrates and clears legacy automation attention state', async () => {
const existingState = formatStateComment({
trace: 'trace-legacy-attention',
iteration: 3,
failure_threshold: 3,
failure: { reason: 'agent-run-failed', count: 2 },
attention: {
key: 'legacy-agent-run-failed',
first_seen_at: '2026-07-01T12:00:00Z',
},
});
const github = buildGithubStub({
comments: [{ id: 47, body: existingState, html_url: 'https://example.com/47' }],
labels: ['agent:needs-attention', 'needs-human'],
});

await updateKeepaliveLoopSummary({
github,
context: buildContext(459),
core: buildCore(),
inputs: {
prNumber: 459,
action: 'stop',
reason: 'tasks-complete',
gateConclusion: 'success',
tasksTotal: 3,
tasksUnchecked: 0,
keepaliveEnabled: true,
autofixEnabled: false,
iteration: 3,
maxIterations: 5,
failureThreshold: 3,
trace: 'trace-legacy-attention',
},
});

assert.ok(github.actions.some((action) =>
action.type === 'remove-label' && action.name === 'agent:needs-attention'
));
assert.equal(
github.actions.some((action) => action.type === 'remove-label' && action.name === 'needs-human'),
false,
);
const updateAction = github.actions.find((action) => action.type === 'update');
assert.equal(parseStateComment(updateAction.body).data.attention, undefined);
});

test('evaluateKeepaliveLoop extracts agent type from agent:* labels', async () => {
const pr = {
number: 107,
Expand Down
53 changes: 36 additions & 17 deletions .github/scripts/keepalive_loop.js
Original file line number Diff line number Diff line change
Expand Up @@ -2792,11 +2792,14 @@ async function evaluateKeepaliveLoop({ github: rawGithub, context, core, payload
// An iteration is productive if it has a reasonable productivity score
const isProductive = productivityScore >= 20 && !hasRecentFailures;

// max_iterations is a hard per-PR budget. Once reached, stop dispatching
// and require a human to raise the budget or remove the blocker.
// max_iterations is the ordinary per-PR budget. Once reached, stop the
// current strategy; a single forced recovery lease may cross it once.
const hasMaxIterations = maxIterations > 0;
const reachedMaxIterations = hasMaxIterations && iteration >= maxIterations;
const shouldStopForMaxIterations = reachedMaxIterations;
// A force retry is a single, explicit recovery lease. It may cross the
// persisted round budget once; the summary step prevents a forced run from
// recursively dispatching another forced run.
const shouldStopForMaxIterations = reachedMaxIterations && !forceRetry;

// Build task appendix for the agent prompt (after state load for reconciliation info)
const taskAppendix = buildTaskAppendix(normalisedSections, checkboxCounts, state, { prBody: pr.body });
Expand Down Expand Up @@ -2825,7 +2828,7 @@ async function evaluateKeepaliveLoop({ github: rawGithub, context, core, payload
// This prevents premature stop when the verifier identifies unmet criteria.
const needsVerification = allComplete && !verificationDone && !verificationAttempted;
const needsVerificationRetry = allComplete && verificationFailed
&& verificationAttemptCount < maxVerificationAttempts;
&& (verificationAttemptCount < maxVerificationAttempts || forceRetry);

// Only treat GitHub API conflicts as definitive (mergeable_state === 'dirty')
// CI-log based conflict detection has too many false positives from commit messages
Expand Down Expand Up @@ -4244,9 +4247,14 @@ async function updateKeepaliveLoopSummary({ github: rawGithub, context, core, in
authorityEvidence.fingerprint,
].filter(Boolean).join('|');
const priorAttentionKey = normalise(previousAttention.key);
const previousAttentionHasLegacyOwnership =
Object.keys(previousAttention).length > 0 &&
!normalise(previousAttention.owner) &&
!normalise(previousAttention.disposition);
const previousAttentionAutomationOwned =
previousAttention.owner === 'automation' &&
['automation-retry', 'challenge-due'].includes(previousAttention.disposition);
(previousAttention.owner === 'automation' &&
['automation-retry', 'challenge-due'].includes(previousAttention.disposition)) ||
previousAttentionHasLegacyOwnership;
const challengeDueAt = escalationDisposition === 'challenge-due'
? new Date().toISOString()
: null;
Expand Down Expand Up @@ -4321,7 +4329,9 @@ async function updateKeepaliveLoopSummary({ github: rawGithub, context, core, in
core,
automationOwned: previousAttentionAutomationOwned,
});
if (previousAuthorityChallenge && runResult === 'success') {
if (previousAttentionAutomationOwned && cleanup.complete) {
delete newState.attention;
} else if (previousAuthorityChallenge && runResult === 'success') {
if (cleanup.complete) {
delete newState.attention;
} else {
Expand Down Expand Up @@ -4423,13 +4433,13 @@ async function updateKeepaliveLoopSummary({ github: rawGithub, context, core, in
: escalationDisposition === 'challenge-due'
? 'agent:needs-attention'
: 'agent:retry';
const addRoutingLabel = () => github.rest.issues.addLabels({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: prNumber,
labels: [routingLabel],
});
try {
const addRoutingLabel = () => github.rest.issues.addLabels({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: prNumber,
labels: [routingLabel],
});
const clearAutomationAttention = () => clearStaleHumanBlockerLabels({
github,
owner: context.repo.owner,
Expand All @@ -4447,15 +4457,19 @@ async function updateKeepaliveLoopSummary({ github: rawGithub, context, core, in
// only sweep-routing signal while renewing or replacing a challenge.
await addRoutingLabel();
} else {
// Remove automation-created legacy hard stops before writing the new
// recoverable disposition. A successful terminal also performs this cleanup.
// The direct workflow dispatch below owns the retry lease. Do not
// add agent:retry first: its labeled event could race the dispatch
// and create a second forced run. The label is only a dispatch-
// failure fallback.
await clearAutomationAttention();
await addRoutingLabel();
}
} catch (error) {
if (core) core.warning(`Failed to add ${escalationDisposition} routing label: ${error.message}`);
}
if (escalationDisposition === 'automation-retry' && !stop) {
// Every automation-owned terminal gets one immediate recovery lease.
// A forced recovery never recursively dispatches itself; the next
// ordinary sweep may reassess the exact current state.
if (escalationDisposition === 'automation-retry' && !isForceRetry) {
Comment thread
stranske marked this conversation as resolved.
Outdated
Comment thread
stranske marked this conversation as resolved.
Outdated
try {
const retryWorkflowId = normalise(
inputs.retry_workflow_id ?? inputs.retryWorkflowId,
Expand All @@ -4475,6 +4489,11 @@ async function updateKeepaliveLoopSummary({ github: rawGithub, context, core, in
});
} catch (error) {
core?.warning?.(`Failed to dispatch bounded automation retry: ${error.message}`);
try {
await addRoutingLabel();
Comment thread
stranske marked this conversation as resolved.
Outdated
Comment thread
stranske marked this conversation as resolved.
Outdated
} catch (labelError) {
core?.warning?.(`Failed to add automation retry fallback label: ${labelError.message}`);
}
}
}
}
Expand Down
5 changes: 3 additions & 2 deletions .github/workflows/maint-52-sync-dev-versions.yml
Original file line number Diff line number Diff line change
Expand Up @@ -214,7 +214,7 @@ jobs:

# Run sync check first
if python ../scripts/sync_dev_dependencies.py \
--check 2>&1 \
--check --pre-commit 2>&1 \
| tee /tmp/sync_output.txt; then
if [ "$uv_lock_stale" = "true" ]; then
echo "has_changes=true" >> "$GITHUB_OUTPUT"
Expand All @@ -229,7 +229,7 @@ jobs:

# Apply if not dry run
if [ "${{ inputs.dry_run }}" != "true" ]; then
python ../scripts/sync_dev_dependencies.py --apply
python ../scripts/sync_dev_dependencies.py --apply --pre-commit
if [ -f uv.lock ]; then
uv lock
fi
Expand Down Expand Up @@ -257,6 +257,7 @@ jobs:
# Use --create-if-missing to add dev deps
if python ../scripts/sync_dev_dependencies.py \
--apply \
--pre-commit \
--create-if-missing 2>&1 \
| tee /tmp/sync_output.txt; then
if grep -q "version updates" /tmp/sync_output.txt; then
Expand Down
Loading
Loading