Skip to content
Merged
Show file tree
Hide file tree
Changes from 9 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
543 changes: 535 additions & 8 deletions .github/scripts/__tests__/keepalive-loop.test.js

Large diffs are not rendered by default.

172 changes: 153 additions & 19 deletions .github/scripts/keepalive_loop.js
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,11 @@ const crypto = require('crypto');

const { parseScopeTasksAcceptanceSections } = require('./issue_scope_parser');
const { getGithubApiCache } = require('./github-api-cache-client');
const { loadKeepaliveState, formatStateComment } = require('./keepalive_state');
const {
loadKeepaliveState,
formatStateComment,
upsertStateCommentBody,
} = require('./keepalive_state');
const { resolvePromptMode } = require('./keepalive_prompt_routing');
const { classifyError, ERROR_CATEGORIES } = require('./error_classifier');
const { formatFailureComment } = require('./failure_comment_formatter');
Expand Down Expand Up @@ -2792,11 +2796,14 @@ async function evaluateKeepaliveLoop({ github: rawGithub, context, core, payload
// An iteration is productive if it has a reasonable productivity score
const isProductive = productivityScore >= 20 && !hasRecentFailures;

// max_iterations is a hard per-PR budget. Once reached, stop dispatching
// and require a human to raise the budget or remove the blocker.
// max_iterations is the ordinary per-PR budget. Once reached, stop the
// current strategy; a single forced recovery lease may cross it once.
const hasMaxIterations = maxIterations > 0;
const reachedMaxIterations = hasMaxIterations && iteration >= maxIterations;
const shouldStopForMaxIterations = reachedMaxIterations;
// A force retry is a single, explicit recovery lease. It may cross the
// persisted round budget once; the summary step prevents a forced run from
// recursively dispatching another forced run.
const shouldStopForMaxIterations = reachedMaxIterations && !forceRetry;

// Build task appendix for the agent prompt (after state load for reconciliation info)
const taskAppendix = buildTaskAppendix(normalisedSections, checkboxCounts, state, { prBody: pr.body });
Expand Down Expand Up @@ -2825,7 +2832,7 @@ async function evaluateKeepaliveLoop({ github: rawGithub, context, core, payload
// This prevents premature stop when the verifier identifies unmet criteria.
const needsVerification = allComplete && !verificationDone && !verificationAttempted;
const needsVerificationRetry = allComplete && verificationFailed
&& verificationAttemptCount < maxVerificationAttempts;
&& (verificationAttemptCount < maxVerificationAttempts || forceRetry);

// Only treat GitHub API conflicts as definitive (mergeable_state === 'dirty')
// CI-log based conflict detection has too many false positives from commit messages
Expand Down Expand Up @@ -2897,7 +2904,14 @@ async function evaluateKeepaliveLoop({ github: rawGithub, context, core, payload
// This ensures at least one fix attempt is made before giving up, and
// that transient cancelled rounds don't consume the fix budget.
const gateFailure = await classifyGateFailure({ github, context, pr, core });
if (gateFailure.shouldFixMode && consecutiveFixRounds < fixAttemptMax) {
if (forceRetry) {
// A terminal recovery lease must perform recovery work, even after
// the ordinary complete-Gate/fix budgets are exhausted. The summary
// consumes the lease after this single non-recursive fix attempt.
action = 'fix';
reason = `force-retry-fix-${gateFailure.failureType || 'complete-gate'}`;
if (core) core.info(`Forced recovery: retrying complete Gate failure (${gateFailure.failureType || 'unknown'}).`);
} else if (gateFailure.shouldFixMode && consecutiveFixRounds < fixAttemptMax) {
// Fix is possible and we haven't exhausted fix attempts — try to fix
action = 'fix';
reason = `fix-${gateFailure.failureType}`;
Expand Down Expand Up @@ -3174,6 +3188,12 @@ async function updateKeepaliveLoopSummary({ github: rawGithub, context, core, in
inputs.roundsWithoutTaskCompletion ?? inputs.rounds_without_task_completion;
const agentType = normalise(inputs.agent_type ?? inputs.agentType) || _defaultAgent;
const runResult = normalise(inputs.runResult || inputs.run_result);
const agentExecutionStartedInput =
inputs.agent_execution_started ?? inputs.agentExecutionStarted;
const agentExecutionStarted =
agentExecutionStartedInput === undefined || agentExecutionStartedInput === ''
? null
: toBool(agentExecutionStartedInput, false);
const stateTrace = normalise(inputs.trace || inputs.keepalive_trace || '');

// Delegation policy inputs (from evaluate step when agent:auto is active)
Expand Down Expand Up @@ -3234,6 +3254,10 @@ async function updateKeepaliveLoopSummary({ github: rawGithub, context, core, in
);
// Resolve force_retry early — needed by the live-recount and zero-activity blocks.
const isForceRetry = toBool(inputs.force_retry ?? inputs.forceRetry, false);
const previousRecoveryLease =
previousState?.recovery_lease && typeof previousState.recovery_lease === 'object'
? { ...previousState.recovery_lease }
: {};

let roundsWithoutTaskCompletion = hasRoundsWithoutTaskCompletionInput
? toNumber(roundsWithoutTaskCompletionInput, 0)
Expand Down Expand Up @@ -3557,6 +3581,21 @@ async function updateKeepaliveLoopSummary({ github: rawGithub, context, core, in
summaryReason,
authorityChallengeConfirmed,
});
const recoveryLeaseReason = stop
? normalise(summaryReason).replace(/-repeat$/, '')
: '';
const recoveryLeaseKey = recoveryLeaseReason
? `${recoveryLeaseReason}:max-iterations=${maxIterations}`
: '';
const recoveryLeaseMatches =
Boolean(recoveryLeaseKey) && normalise(previousRecoveryLease.key) === recoveryLeaseKey;
const recoveryLeaseAlreadyIssued =
recoveryLeaseMatches && ['issued', 'consumed'].includes(previousRecoveryLease.status);
const shouldIssueTerminalRecoveryLease =
stop &&
escalationDisposition === 'automation-retry' &&
!isForceRetry &&
!recoveryLeaseAlreadyIssued;
let hardHumanLabelApplied = false;
const tasksComplete = Math.max(0, tasksTotal - tasksUnchecked);
const allTasksComplete = tasksUnchecked === 0 && tasksTotal > 0;
Expand Down Expand Up @@ -4134,6 +4173,64 @@ async function updateKeepaliveLoopSummary({ github: rawGithub, context, core, in
},
};

const ordinaryProgressAfterRecovery =
!isForceRetry &&
actionRunsAgent &&
runResult === 'success' &&
(agentFilesChanged > 0 || tasksCompletedThisRound > 0 || checklistChanged);
const forcedLeaseExecutedAgent =
actionRunsAgent &&
(agentExecutionStarted === null ? true : agentExecutionStarted) &&
Boolean(runResult) &&
!['skipped', 'cancelled'].includes(runResult);
Comment thread
stranske marked this conversation as resolved.
const recoveryLeaseBudgetChanged =
Object.keys(previousRecoveryLease).length > 0 &&
toNumber(previousRecoveryLease.max_iterations, maxIterations) !== maxIterations;
if (
isForceRetry &&
previousRecoveryLease.status === 'issued' &&
!isSuccessStop &&
!isNeutralStop
) {
const forcedLeaseStatus = forcedLeaseExecutedAgent ? 'consumed' : 'deferred';
newState.recovery_lease = {
...previousRecoveryLease,
status: forcedLeaseStatus,
...(forcedLeaseExecutedAgent
? { consumed_at: new Date().toISOString() }
: {
deferred_at: new Date().toISOString(),
deferred_reason: summaryReason,
}),
};
} else if (shouldIssueTerminalRecoveryLease) {
const retryingDeferredLease =
recoveryLeaseMatches && previousRecoveryLease.status === 'deferred';
const dispatchedAt = new Date().toISOString();
newState.recovery_lease = {
key: recoveryLeaseKey,
reason: recoveryLeaseReason,
status: 'issued',
issued_at: retryingDeferredLease
? previousRecoveryLease.issued_at || dispatchedAt
: dispatchedAt,
last_dispatched_at: dispatchedAt,
dispatch_attempt: retryingDeferredLease
? toNumber(previousRecoveryLease.dispatch_attempt, 1) + 1
: 1,
iteration: nextIteration,
max_iterations: maxIterations,
};
} else if (
Object.keys(previousRecoveryLease).length > 0 &&
!recoveryLeaseBudgetChanged &&
!ordinaryProgressAfterRecovery &&
!isSuccessStop &&
!isNeutralStop
) {
newState.recovery_lease = previousRecoveryLease;
}

// Persist agent delegation state when in auto mode
if (agentRoutingMode === 'auto' || previousState?.current_agent) {
const previousDelegationLog = Array.isArray(previousState?.delegation_log)
Expand Down Expand Up @@ -4244,9 +4341,14 @@ async function updateKeepaliveLoopSummary({ github: rawGithub, context, core, in
authorityEvidence.fingerprint,
].filter(Boolean).join('|');
const priorAttentionKey = normalise(previousAttention.key);
const previousAttentionHasLegacyOwnership =
Object.keys(previousAttention).length > 0 &&
!normalise(previousAttention.owner) &&
!normalise(previousAttention.disposition);
const previousAttentionAutomationOwned =
previousAttention.owner === 'automation' &&
['automation-retry', 'challenge-due'].includes(previousAttention.disposition);
(previousAttention.owner === 'automation' &&
['automation-retry', 'challenge-due'].includes(previousAttention.disposition)) ||
previousAttentionHasLegacyOwnership;
const challengeDueAt = escalationDisposition === 'challenge-due'
? new Date().toISOString()
: null;
Expand Down Expand Up @@ -4321,7 +4423,9 @@ async function updateKeepaliveLoopSummary({ github: rawGithub, context, core, in
core,
automationOwned: previousAttentionAutomationOwned,
});
if (previousAuthorityChallenge && runResult === 'success') {
if (previousAttentionAutomationOwned && cleanup.complete) {
delete newState.attention;
} else if (previousAuthorityChallenge && runResult === 'success') {
if (cleanup.complete) {
delete newState.attention;
} else {
Expand Down Expand Up @@ -4423,13 +4527,13 @@ async function updateKeepaliveLoopSummary({ github: rawGithub, context, core, in
: escalationDisposition === 'challenge-due'
? 'agent:needs-attention'
: 'agent:retry';
const addRoutingLabel = () => github.rest.issues.addLabels({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: prNumber,
labels: [routingLabel],
});
try {
const addRoutingLabel = () => github.rest.issues.addLabels({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: prNumber,
labels: [routingLabel],
});
const clearAutomationAttention = () => clearStaleHumanBlockerLabels({
github,
owner: context.repo.owner,
Expand All @@ -4447,15 +4551,23 @@ async function updateKeepaliveLoopSummary({ github: rawGithub, context, core, in
// only sweep-routing signal while renewing or replacing a challenge.
await addRoutingLabel();
} else {
// Remove automation-created legacy hard stops before writing the new
// recoverable disposition. A successful terminal also performs this cleanup.
// The direct workflow dispatch below owns the retry lease. Do not
// add agent:retry first: its labeled event could race the dispatch
// and create a second forced run. The label is only a dispatch-
// failure fallback.
await clearAutomationAttention();
await addRoutingLabel();
}
} catch (error) {
if (core) core.warning(`Failed to add ${escalationDisposition} routing label: ${error.message}`);
}
if (escalationDisposition === 'automation-retry' && !stop) {
// Every automation-owned terminal gets one immediate recovery lease.
// Persist the issued/consumed lease across events so later ordinary
// sweeps cannot mint another lease for the same terminal boundary.
if (
escalationDisposition === 'automation-retry' &&
!isForceRetry &&
(!stop || shouldIssueTerminalRecoveryLease)
Comment thread
stranske marked this conversation as resolved.
) {
try {
const retryWorkflowId = normalise(
inputs.retry_workflow_id ?? inputs.retryWorkflowId,
Expand All @@ -4475,6 +4587,28 @@ async function updateKeepaliveLoopSummary({ github: rawGithub, context, core, in
});
} catch (error) {
core?.warning?.(`Failed to dispatch bounded automation retry: ${error.message}`);
if (newState.recovery_lease?.status === 'issued') {
newState.recovery_lease = {
...newState.recovery_lease,
status: 'deferred',
deferred_at: new Date().toISOString(),
deferred_reason: 'workflow-dispatch-failed',
};
try {
await persistSummary(
upsertStateCommentBody(body, formatStateComment(newState)),
);
} catch (stateError) {
core?.warning?.(
`Failed to persist deferred recovery lease: ${stateError.message}`,
);
}
}
try {
await addRoutingLabel();
Comment thread
stranske marked this conversation as resolved.
Outdated
Comment thread
stranske marked this conversation as resolved.
Outdated
} catch (labelError) {
core?.warning?.(`Failed to add automation retry fallback label: ${labelError.message}`);
}
}
}
}
Expand Down
1 change: 1 addition & 0 deletions .github/workflows/agents-keepalive-loop.yml
Original file line number Diff line number Diff line change
Expand Up @@ -1577,6 +1577,7 @@ jobs:
trace: '${{ needs.evaluate.outputs.trace }}',
// Agent run result - check which agent ran
run_result: runResult,
agent_execution_started: !allRunnersSkipped,
Comment thread
stranske marked this conversation as resolved.
Outdated
// Agent output details for visibility
agent_exit_code: agentExitCode,
agent_changes_made: agentChangesMade,
Expand Down
5 changes: 3 additions & 2 deletions .github/workflows/maint-52-sync-dev-versions.yml
Original file line number Diff line number Diff line change
Expand Up @@ -214,7 +214,7 @@ jobs:

# Run sync check first
if python ../scripts/sync_dev_dependencies.py \
--check 2>&1 \
--check --pre-commit 2>&1 \
| tee /tmp/sync_output.txt; then
if [ "$uv_lock_stale" = "true" ]; then
echo "has_changes=true" >> "$GITHUB_OUTPUT"
Expand All @@ -229,7 +229,7 @@ jobs:

# Apply if not dry run
if [ "${{ inputs.dry_run }}" != "true" ]; then
python ../scripts/sync_dev_dependencies.py --apply
python ../scripts/sync_dev_dependencies.py --apply --pre-commit
if [ -f uv.lock ]; then
uv lock
fi
Expand Down Expand Up @@ -257,6 +257,7 @@ jobs:
# Use --create-if-missing to add dev deps
if python ../scripts/sync_dev_dependencies.py \
--apply \
--pre-commit \
--create-if-missing 2>&1 \
| tee /tmp/sync_output.txt; then
if grep -q "version updates" /tmp/sync_output.txt; then
Expand Down
4 changes: 4 additions & 0 deletions .github/workflows/maint-69-sync-integration-repo.yml
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,7 @@ jobs:
if [ -f "pyproject.toml" ]; then
python "../workflows/scripts/sync_dev_dependencies.py" \
--apply \
--pre-commit \
Comment thread
stranske marked this conversation as resolved.
--pin-file ".github/workflows/autofix-versions.env" \
--pyproject "pyproject.toml"
echo "✅ Synced pyproject.toml dev tool pins"
Expand Down Expand Up @@ -198,6 +199,9 @@ jobs:
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"

git add .github/workflows/ pyproject.toml requirements.lock scripts/
if [ -f .pre-commit-config.yaml ]; then
git add .pre-commit-config.yaml
fi
git commit \
-m "sync: Update workflow templates from stranske/Workflows" \
-m "" \
Expand Down
6 changes: 4 additions & 2 deletions .github/workflows/maint-auto-update-pypi-versions.yml
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ jobs:
run: |
echo "🔄 Syncing pyproject.toml + requirements.lock + templates"
python scripts/sync_tool_versions.py --apply
python scripts/sync_dev_dependencies.py --apply --lockfile
python scripts/sync_dev_dependencies.py --apply --lockfile --pre-commit
Comment thread
stranske marked this conversation as resolved.
Comment thread
coderabbitai[bot] marked this conversation as resolved.

# security_override may bypass the weekly window, but never this gate.
- name: Validate synchronized pins before proposal
Expand All @@ -110,7 +110,7 @@ jobs:
set -euo pipefail
echo "🧪 Validating synchronized pin surfaces before opening a source PR..."
python scripts/sync_tool_versions.py --check
python scripts/sync_dev_dependencies.py --check --lockfile
python scripts/sync_dev_dependencies.py --check --lockfile --pre-commit

- name: Create PR
id: create_pr
Expand All @@ -128,6 +128,7 @@ jobs:
# Check if there are actual changes
if git diff --quiet -- \
.github/workflows/autofix-versions.env \
.pre-commit-config.yaml \
pyproject.toml \
requirements.lock \
templates/consumer-repo/.github/workflows/autofix-versions.env \
Expand All @@ -153,6 +154,7 @@ jobs:
# Commit changes
git add \
.github/workflows/autofix-versions.env \
.pre-commit-config.yaml \
pyproject.toml \
requirements.lock \
templates/consumer-repo/.github/workflows/autofix-versions.env \
Expand Down
5 changes: 4 additions & 1 deletion .github/workflows/maint-sync-env-from-pyproject.yml
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ on:
branches: [main]
paths:
- '.github/workflows/autofix-versions.env'
- '.pre-commit-config.yaml'
- 'templates/consumer-repo/.github/workflows/autofix-versions.env'
- 'templates/integration-repo/.github/workflows/autofix-versions.env'
- 'pyproject.toml'
Expand Down Expand Up @@ -35,7 +36,7 @@ jobs:
run: |
set -euo pipefail
python scripts/sync_tool_versions.py --apply
python scripts/sync_dev_dependencies.py --apply --lockfile
python scripts/sync_dev_dependencies.py --apply --lockfile --pre-commit
Comment thread
stranske marked this conversation as resolved.

- name: Commit and push if changed
run: |
Expand All @@ -45,6 +46,7 @@ jobs:

if git diff --quiet -- \
.github/workflows/autofix-versions.env \
.pre-commit-config.yaml \
Comment thread
stranske marked this conversation as resolved.
pyproject.toml \
requirements.lock \
templates/consumer-repo/.github/workflows/autofix-versions.env \
Expand All @@ -55,6 +57,7 @@ jobs:

git add \
.github/workflows/autofix-versions.env \
.pre-commit-config.yaml \
pyproject.toml \
requirements.lock \
templates/consumer-repo/.github/workflows/autofix-versions.env \
Expand Down
Loading
Loading