Skip to content
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
55 changes: 55 additions & 0 deletions .github/workflows/qwen-autofix.yml
Original file line number Diff line number Diff line change
Expand Up @@ -129,6 +129,18 @@ env:
# invisible to the scan even when it already had unaddressed feedback.
# Names must match job ids in qwen-code-pr-review.yml; a test pins that.
NON_BLOCKING_CHECKS: '["review-pr"]'
# Failed-check annotation patterns that mean the INFRASTRUCTURE died, not the
# code — a self-hosted runner losing the server, the disk filling, a runner
# shutdown, or a git fetch/clone dying mid-transfer. Such a check is red for a
# reason unrelated to the PR and clears on a re-run (observed: #7490's E2E
# "runner lost communication"; #6506's checkout "RPC failed; curl 92" /
# "fetch-pack: invalid index-pack output" — both green on the rerun). The scan
# auto-reruns those failed jobs ONCE, guarded by run_attempt so a persistent
# infra problem cannot loop. Deliberately conservative — only unambiguous
# machine/transport failures, never a bare test-level timeout, which could be
# a real regression (a co-present timeout does not block a match — one
# matching line classifies the run). Case-insensitive, vs the annotations.
INFRA_FAILURE_SIGNATURES: 'lost communication with the server|No space left on device|ENOSPC|received a shutdown signal|The runner has received|Failed to initialize container|runner (was|has been) (lost|terminated)|invalid index-pack output|RPC failed'
# Upper bound on review targets emitted per scan (fan-out defense-in-depth;
# excess is logged and deferred to the next scan).
MAX_TARGETS_PER_SCAN: '10'
Expand Down Expand Up @@ -1809,6 +1821,49 @@ jobs:
ISSUE="${PR}"
fi
CHECKS_JSON="$(jq -c '.statusCheckRollup // []' <<< "${PR_META}")"

# Auto-rerun a check that died on INFRASTRUCTURE, not the code (see
# INFRA_FAILURE_SIGNATURES). Only reached when the PR has a FAILED
# check; then, for each, we read its annotations and — if they carry
# a machine-death signature — rerun that run's failed jobs ONCE. The
# once is enforced by run_attempt: a run already retried to attempt 2
# and still infra-failing is persistent, so we stop and leave it. No
# marker needed; the attempt counter is the guard, and after a rerun
# the attempt increments so the next scan skips it. Any API failure
# here is fail-safe: it just means no rerun.
PR_HEAD_OID="$(jq -r '.headRefOid // ""' <<< "${PR_META}")"
if [[ -n "${PR_HEAD_OID}" ]] && jq -e 'any(.[]; ((.conclusion // .state // "") | IN("FAILURE","FAILED","ERROR","TIMED_OUT","ACTION_REQUIRED")) and (((.workflowName // "") != "Qwen Autofix") or ((.name // "") | startswith("review-address"))))' <<< "${CHECKS_JSON}" > /dev/null 2>&1; then
RERAN_INFRA=false
# Failed check-runs on this head, with their run id and annotation
# count — fetched once. External statuses (no check-run) are absent
# here, which is fine: only Actions runs can be reran.
FAILED_CRS="$(gh api --paginate "repos/${REPO}/commits/${PR_HEAD_OID}/check-runs" \
--jq '.check_runs[] | select(.conclusion == "failure")
| select(((.output.annotations_count // 0) > 0))
| [(.id | tostring), (.details_url // "")] | @tsv' 2> /dev/null | sort -u || true)"
while IFS=$'\t' read -r CR_ID DETAILS_URL; do
[[ -n "${CR_ID}" && "${DETAILS_URL}" == *"/actions/runs/"* ]] || continue
RUN_ID="${DETAILS_URL#*/actions/runs/}"
RUN_ID="${RUN_ID%%/*}"
[[ "${RUN_ID}" =~ ^[0-9]+$ ]] || continue
ANNS="$(gh api "repos/${REPO}/check-runs/${CR_ID}/annotations" \
--jq '[.[].message] | join("\n")' 2> /dev/null || true)"
grep -qiE "${INFRA_FAILURE_SIGNATURES}" <<< "${ANNS}" || continue
ATTEMPT="$(gh api "repos/${REPO}/actions/runs/${RUN_ID}" --jq '.run_attempt // 0' 2> /dev/null || echo 0)"
if [[ "${ATTEMPT}" == '1' ]]; then
if gh api -X POST "repos/${REPO}/actions/runs/${RUN_ID}/rerun-failed-jobs" > /dev/null 2>&1; then
echo "♻️ #${PR}: a check died on infrastructure (run ${RUN_ID}, attempt 1) — reran its failed jobs; not the PR's code"
fleet_row "${PR}" 'infra-reran' "infra failure (run ${RUN_ID}) reran once"
RERAN_INFRA=true
break
fi
else
echo "⚠️ #${PR}: infra failure persisted after a rerun (run ${RUN_ID}, attempt ${ATTEMPT}) — leaving for a human"
fi
done <<< "${FAILED_CRS}"
[[ "${RERAN_INFRA}" == 'true' ]] && continue
fi

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

mark: It needs to be optimized in the future, and it has to be migrated to the skill.

# startedAt is the only staleness clock: a check blocks only if it
# started within the bound; one with no startedAt (queued, not yet
# running) is not blocking (the next scan re-checks once it starts).
Expand Down
183 changes: 183 additions & 0 deletions scripts/tests/qwen-autofix-workflow.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -401,6 +401,189 @@ describe('qwen-autofix workflow', () => {
expect(run([{ ...llm, name: 'resolve-pr' }])).toBe('true');
});

it('auto-reruns a check that died on infrastructure, once, guarded by run_attempt', () => {
// A self-hosted runner losing the server (or the disk filling) reds a check
// for a reason unrelated to the PR; it clears on a rerun (#7490's E2E:
// "runner lost communication" → green on the rerun). The scan reruns such a
// failed job ONCE, and run_attempt is the guard: a run already at attempt 2
// and still infra-failing is persistent and left alone — no infinite loop.
const block = reviewScanJob.match(
/( {12}PR_HEAD_OID="\$\(jq -r '\.headRefOid[\s\S]*?\n {12}fi\n)\n {12}# startedAt is the only staleness/,
)?.[1];
expect(block).toBeTruthy();
const script = block.replace(/^ {12}/gm, '');

const run = ({ checks, annotations, attempt = 1, rerunOk = true }) => {
const dir = mkdtempSync(join(tmpdir(), 'infra-'));
const bin = join(dir, 'bin');
mkdirSync(bin);
// Stubbed gh: check-runs → one failed run-1 check-run with annotations;
// annotations → the given message; runs/{id} → run_attempt; POST
// rerun-failed-jobs → success/fail. Records the rerun POST.
// The workflow calls check-runs with a --jq filter that yields, per
// failed check-run WITH annotations, a `<id>\t<details_url>` line; the
// stub emits what that filter would produce (a single line when there is
// an annotation, nothing otherwise) rather than raw JSON the stub can't
// filter.
const crTsv = annotations
? '42\thttps://github.com/o/r/actions/runs/9001/job/5\n'
: '';
writeFileSync(
join(bin, 'gh'),
[
'#!/usr/bin/env bash',
`echo "$*" >> ${JSON.stringify(join(dir, 'calls.log'))}`,
'args="$*"',
`case "$args" in`,
// %b so the \t/\n in the stubbed tsv become a real tab/newline (the
// filter's @tsv output), which `IFS=$'\\t' read` then splits.
` *"/commits/"*"/check-runs"*) printf '%b' ${JSON.stringify(crTsv)}; exit 0;;`,
` *"/check-runs/42/annotations"*) printf '%s' ${JSON.stringify(annotations || '')}; exit 0;;`,
` *"/actions/runs/9001"*"rerun-failed-jobs"*) exit ${rerunOk ? 0 : 1};;`,
` *"/actions/runs/9001"*) printf '${attempt}'; exit 0;;`,
'esac',
'exit 0',
].join('\n'),
);
chmodSync(join(bin, 'gh'), 0o755);
const out = execFileSync(
'bash',
[
'-c',
`set -uo pipefail\nfleet_row(){ :; }\nfor _ in x; do\n${script}\nprintf 'FELL_THROUGH'\ndone`,
],
{
env: {
...process.env,
REPO: 'o/r',
PR: '1',
PR_META: JSON.stringify({ headRefOid: 'headSHA' }),
CHECKS_JSON: JSON.stringify(checks),
INFRA_FAILURE_SIGNATURES:
'lost communication with the server|No space left on device|ENOSPC|received a shutdown signal|The runner has received|Failed to initialize container|runner (was|has been) (lost|terminated)|invalid index-pack output|RPC failed',
PATH: `${bin}:${process.env.PATH}`,
},
encoding: 'utf8',
},
);
const calls = existsSync(join(dir, 'calls.log'))
? readFileSync(join(dir, 'calls.log'), 'utf8')
: '';
rmSync(dir, { recursive: true, force: true });
return {
reran: /rerun-failed-jobs/.test(calls),
continued: !out.includes('FELL_THROUGH'),
};
};
const FAIL = { name: 'E2E', conclusion: 'FAILURE' };
const OK = { name: 'E2E', conclusion: 'SUCCESS' };

// Infra death (runner lost the server) on attempt 1 → rerun & skip.
expect(
run({
checks: [FAIL],
annotations:
'The self-hosted runner lost communication with the server',
}),
).toEqual({ reran: true, continued: true });
// A REAL failure (no infra signature in the annotation) → never rerun; the
// agent/human handles it. This is the gate that stops masking real bugs.
expect(
run({
checks: [FAIL],
annotations: 'Expected 1 argument but got 2 — src/foo.ts:10',
}),
).toEqual({ reran: false, continued: false });
// Already reran once (attempt 2) and still infra-failing → persistent, do
// not loop.
expect(
run({
checks: [FAIL],
annotations: 'No space left on device',
attempt: 2,
}),
).toEqual({ reran: false, continued: false });
// No failed check at all → the block is skipped entirely.
expect(run({ checks: [OK], annotations: '' })).toEqual({
reran: false,
continued: false,
});
// Infra signature but the rerun POST fails (e.g. PAT lacks actions:write) →
// no crash, falls through to normal processing.
expect(
run({
checks: [FAIL],
annotations: 'No space left on device',
rerunOk: false,
}),
).toEqual({ reran: true, continued: false });
// Each remaining production signature also triggers a rerun.
for (const msg of [
'ENOSPC',
'The runner has received a shutdown signal',
'The runner has received an unexpected signal',
'Failed to initialize container for job',
'The runner was lost',
'The runner was terminated',
'The runner has been lost',
'The runner has been terminated',
'fatal: fetch-pack: invalid index-pack output',
'error: RPC failed; curl 92 HTTP/2 stream 5 was not closed cleanly: CANCEL (err 8)',
]) {
expect(run({ checks: [FAIL], annotations: msg })).toEqual({
reran: true,
continued: true,
});
}
// #6506: a git fetch died mid-checkout, then hung the job into the 20m
// limit. The bare timeout line is deliberately NOT a signature (it can be a
// real regression), but the transport death IS — and one matching line
// classifies the whole run, so the co-present timeout does not block it.
expect(
run({
checks: [FAIL],
annotations: [
'The job has exceeded the maximum execution time of 20m0s',
'fatal: fetch-pack: invalid index-pack output',
'error: RPC failed; curl 92 HTTP/2 stream 5 was not closed cleanly: CANCEL (err 8)',
].join('\n'),
}),
).toEqual({ reran: true, continued: true });
// A BARE job timeout with no transport/infra signature is NOT rerun — it
// can be a real regression (a test hanging on the PR's own code).
expect(
run({
checks: [FAIL],
annotations: 'The job has exceeded the maximum execution time of 20m0s',
}),
).toEqual({ reran: false, continued: false });
// Self-trigger guard: a "Qwen Autofix" workflow's own failed check must NOT
// be rerun (prevents the autofix from re-triggering itself), UNLESS the
// check is a review-address job (the exception carved out in the jq filter).
const AUTOFIX_CHECK = {
name: 'E2E',
conclusion: 'FAILURE',
workflowName: 'Qwen Autofix',
};
expect(
run({ checks: [AUTOFIX_CHECK], annotations: 'No space left on device' }),
).toEqual({ reran: false, continued: false });
expect(
run({
checks: [
{
name: 'review-address issue-123',
conclusion: 'FAILURE',
workflowName: 'Qwen Autofix',
},
],
annotations: 'No space left on device',
}),
).toEqual({ reran: true, continued: true });
// Spawn-heavy: each run() forks bash + a stubbed gh. The default 5s per-test
// budget is tight for this many cases, so give it a comfortable margin.
}, 20000);

it('keeps a still-red check visible, but only once per head', () => {
// A red check is a STATE, not the instant it turned red. Counting only
// "failed since the watermark" made a still-failing PR invisible the
Expand Down
Loading