Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
46 changes: 36 additions & 10 deletions .github/workflows/opencode-review.yml
Original file line number Diff line number Diff line change
Expand Up @@ -1435,10 +1435,10 @@ jobs:
# continue-on-error, so it survives the job failing.
model_pool_outcome: ${{ steps.opencode_review_model_pool.outputs.review_status }}
# Coverage and current-head evidence are prepared before the model pool.
# The model pool has a 60-minute retry budget and its publish gate has an
# 8-minute cap, so 90 minutes releases genuinely hung runners without
# truncating the bounded multi-provider review path.
timeout-minutes: 90
# The model pool has a 60-minute retry budget and its publish gate can wait
# for long same-head package/GPU checks, so 150 minutes releases genuinely
# hung runners without truncating the bounded multi-provider review path.
timeout-minutes: 150
permissions:
actions: read
checks: read
Expand Down Expand Up @@ -3691,9 +3691,9 @@ jobs:
&& steps.opencode_review_model_pool.outputs.review_status == 'success'
&& steps.central_review_process_fallback_scope.outputs.eligible == 'true'
continue-on-error: true
# Keep the normal peer-check hold short, but leave bounded room for the
# dynamic image-validation extension and review publication overhead.
timeout-minutes: 14
# Keep the normal peer-check hold short, but leave bounded room for
# dynamic image/package-build extensions and review publication overhead.
timeout-minutes: 34
env:
GH_TOKEN: ${{ steps.opencode_app_token.outputs.token }}
CHECK_LOOKUP_GH_TOKEN: ${{ github.token }}
Expand All @@ -3713,6 +3713,7 @@ jobs:
CENTRAL_REVIEW_PROCESS_FALLBACK_CHANGED_COUNT: ${{ steps.central_review_process_fallback_scope.outputs.changed_count || '0' }}
CENTRAL_REVIEW_PROCESS_FALLBACK_SCOPE_LABEL: ${{ steps.central_review_process_fallback_scope.outputs.scope_label || 'unsupported' }}
APPROVAL_CHECK_WAIT_ATTEMPTS: "36"
APPROVAL_SLOW_BUILD_CHECK_WAIT_ATTEMPTS: "180"
APPROVAL_SLOW_IMAGE_CHECK_WAIT_ATTEMPTS: "60"
APPROVAL_CHECK_WAIT_SLEEP_SECONDS: "10"
REVIEW_PUBLISH_GH_API_TIMEOUT_SECONDS: "15"
Expand Down Expand Up @@ -3802,8 +3803,14 @@ jobs:
pending_checks_file="$(mktemp)"
failed_checks_file="$(mktemp)"
attempts="${APPROVAL_CHECK_WAIT_ATTEMPTS:-36}"
slow_build_attempts="${APPROVAL_SLOW_BUILD_CHECK_WAIT_ATTEMPTS:-180}"
slow_image_attempts="${APPROVAL_SLOW_IMAGE_CHECK_WAIT_ATTEMPTS:-60}"
attempt=1
pending_checks_need_slow_build_wait() {
local pending_file="$1"
grep -Eiq -- '^- ([^/]+/)?gpu-build([[:space:](]|:)' "$pending_file" ||
grep -Eiq -- '^- ([^/]+/)?build \([^)]*(src-tauri/target/release/bundle|bundle/|\.msi|\.dmg|\.deb|\.appimage|AppImage)' "$pending_file"
}
while [ "$attempt" -le "$attempts" ]; do
curl_api_read "${api_url}/repos/${GH_REPOSITORY}/commits/${HEAD_SHA}/check-runs?per_page=100" >"$check_runs_file"
jq -r "${self_check_filter}
Expand All @@ -3819,6 +3826,11 @@ jobs:
printf '::notice::Extending central fast approval peer-check wait from %s to %s attempts because current-head image validation is still running.\n' "$attempts" "$slow_image_attempts"
attempts="$slow_image_attempts"
fi
if [ "$attempts" -lt "$slow_build_attempts" ] &&
pending_checks_need_slow_build_wait "$pending_checks_file"; then
printf '::notice::Extending central fast approval peer-check wait from %s to %s attempts because current-head package/GPU build checks are still running.\n' "$attempts" "$slow_build_attempts"
attempts="$slow_build_attempts"
fi
if [ "$attempt" -lt "$attempts" ]; then
printf 'Central fast approval waiting for peer checks (%s/%s):\n' "$attempt" "$attempts"
cat "$pending_checks_file"
Expand Down Expand Up @@ -3983,9 +3995,10 @@ jobs:
# keeps GitHub review publication bounded. Failed-check evidence is
# collected from logs/SARIF before this point; central review-process
# self-repair must not run a second model pass from the publish step.
# The approval gate normally waits about six minutes, with a bounded
# extension for image validation plus API and publication overhead.
timeout-minutes: 16
# The approval gate normally waits about six minutes, with bounded
# extensions for image validation or package/GPU builds plus API and
# publication overhead.
timeout-minutes: 36
env:
GH_TOKEN: ${{ steps.opencode_app_token.outputs.token || secrets.PR_REVIEW_MERGE_TOKEN || secrets.OPENCODE_APPROVE_TOKEN || github.token }}
CHECK_LOOKUP_GH_TOKEN: ${{ github.token }}
Expand Down Expand Up @@ -4029,6 +4042,7 @@ jobs:
PR_BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.inputs.pr_base_sha }}
PR_HEAD_SHA: ${{ github.event.pull_request.head.sha || github.event.inputs.pr_head_sha }}
APPROVAL_CHECK_WAIT_ATTEMPTS: "36"
APPROVAL_SLOW_BUILD_CHECK_WAIT_ATTEMPTS: "180"
APPROVAL_SLOW_IMAGE_CHECK_WAIT_ATTEMPTS: "60"
APPROVAL_CHECK_WAIT_SLEEP_SECONDS: "10"
CHECK_LOOKUP_RETRY_ATTEMPTS: "1"
Expand Down Expand Up @@ -6301,9 +6315,16 @@ jobs:
return 1
}

pending_checks_need_slow_build_wait() {
local pending_file="$1"
grep -Eiq -- '^- ([^/]+/)?gpu-build([[:space:](]|:)' "$pending_file" ||
grep -Eiq -- '^- ([^/]+/)?build \([^)]*(src-tauri/target/release/bundle|bundle/|\.msi|\.dmg|\.deb|\.appimage|AppImage)' "$pending_file"
}

wait_for_peer_github_checks() {
local output_file="$1"
local attempts="${APPROVAL_CHECK_WAIT_ATTEMPTS:-36}"
local slow_build_attempts="${APPROVAL_SLOW_BUILD_CHECK_WAIT_ATTEMPTS:-180}"
local slow_image_attempts="${APPROVAL_SLOW_IMAGE_CHECK_WAIT_ATTEMPTS:-60}"
local sleep_seconds="${APPROVAL_CHECK_WAIT_SLEEP_SECONDS:-10}"
local attempt=1
Expand All @@ -6320,6 +6341,11 @@ jobs:
printf '::notice::Extending OpenCode peer-check wait from %s to %s attempts because current-head image validation is still running.\n' "$attempts" "$slow_image_attempts"
attempts="$slow_image_attempts"
fi
if [ "$attempts" -lt "$slow_build_attempts" ] &&
pending_checks_need_slow_build_wait "$output_file"; then
printf '::notice::Extending OpenCode peer-check wait from %s to %s attempts because current-head package/GPU build checks are still running.\n' "$attempts" "$slow_build_attempts"
attempts="$slow_build_attempts"
fi
if [ "$attempt" -lt "$attempts" ]; then
printf 'Waiting for peer GitHub Checks before OpenCode approval (%s/%s):\n' "$attempt" "$attempts"
cat "$output_file"
Expand Down
8 changes: 5 additions & 3 deletions scripts/ci/test_strix_quick_gate.sh
Original file line number Diff line number Diff line change
Expand Up @@ -591,10 +591,10 @@ assert_opencode_review_uses_codegraph_and_gpt5_fallback() {
assert_file_contains "$REPO_ROOT/scripts/ci/run_opencode_review_model_pool.sh" "skipping remaining attempts for this model" "opencode review skips same-model retries after context-window overflow"
assert_file_contains "$REPO_ROOT/.github/workflows/strix.yml" "exceeded your current quota" "strix wrapper neutralizes quota-only provider failures without vulnerability reports"
assert_file_contains "$REPO_ROOT/scripts/ci/strix_quick_gate.sh" "billing details" "strix quick gate classifies provider quota starvation as infrastructure"
assert_file_contains "$workflow_file" 'timeout-minutes: 90' "opencode review target releases stalled review runners within the bounded queue budget"
assert_file_contains "$workflow_file" 'timeout-minutes: 150' "opencode review target releases stalled review runners within the bounded queue budget"
assert_file_contains "$workflow_file" 'timeout-minutes: 12' "opencode evidence preparation fails closed before it ties up the review queue"
assert_file_contains "$workflow_file" 'timeout-minutes: 65' "opencode model pool gives multiple candidates a bounded review window while capping stalled model attempts"
assert_file_contains "$workflow_file" 'timeout-minutes: 14' "opencode fast approval publication is bounded around the dynamic image-check wait"
assert_file_contains "$workflow_file" 'timeout-minutes: 34' "opencode fast approval publication is bounded around the dynamic image and package/GPU check wait"
assert_file_contains "$workflow_file" 'continue-on-error: true' "opencode approval gate still runs after model-pool failure to publish a reason"
assert_file_contains "$workflow_file" 'OPENCODE_RUN_TIMEOUT_SECONDS: "600"' "opencode primary review advances after a bounded stalled provider attempt"
assert_file_contains "$workflow_file" 'OPENCODE_TOTAL_RETRY_BUDGET_SECONDS: "3600"' "opencode model pool exits before the step timeout so the approval gate can publish a reason"
Expand Down Expand Up @@ -670,15 +670,17 @@ assert_opencode_review_uses_codegraph_and_gpt5_fallback() {
assert_file_contains "$workflow_file" 'load_selected_review_output()' "opencode approval step has a direct selected-output fallback when the overview comment is stale or invalid"
assert_file_contains "$workflow_file" "gate result from Review Overview comment" "opencode approval step distinguishes overview-comment gate results"
assert_file_contains "$workflow_file" "gate result from selected OpenCode output" "opencode approval step can recover from an invalid overview by validating the selected successful output"
assert_file_contains "$workflow_file" 'timeout-minutes: 16' "opencode approval step has a bounded wall-clock timeout that covers dynamically extended image checks"
assert_file_contains "$workflow_file" 'timeout-minutes: 36' "opencode approval step has a bounded wall-clock timeout that covers dynamically extended image and package/GPU checks"
assert_file_contains "$workflow_file" 'OPENCODE_RUN_TIMEOUT_SECONDS: "120"' "opencode publish-stage diagnosis is a short best-effort augmentation"
assert_file_not_contains "$workflow_file" "rekick_model_pool_on_exhaustion" "opencode publication must not rerun the exhausted model catalog after the model-pool step"
assert_file_contains "$workflow_file" "publish stage performs no duplicate model-catalog pass" "opencode publication logs that exhausted model retries are delegated to the scheduler"
assert_file_contains "$workflow_file" 'timeout --kill-after=15s "${OPENCODE_EXPORT_TIMEOUT_SECONDS:-120}s"' "opencode failed-check diagnosis bounds export so the publication gate cannot hang silently"
assert_file_contains "$workflow_file" 'APPROVAL_CHECK_WAIT_ATTEMPTS: "36"' "opencode approval gives slow peer checks a bounded six-minute hold window before scheduler retry"
assert_file_contains "$workflow_file" 'APPROVAL_SLOW_BUILD_CHECK_WAIT_ATTEMPTS: "180"' "opencode approval dynamically extends its bounded hold for current-head package and GPU builds"
assert_file_contains "$workflow_file" 'APPROVAL_SLOW_IMAGE_CHECK_WAIT_ATTEMPTS: "60"' "opencode approval dynamically extends its bounded hold only for current-head image validation"
assert_file_contains "$workflow_file" 'APPROVAL_CHECK_WAIT_SLEEP_SECONDS: "10"' "opencode approval poll cadence keeps peer-check API volume bounded"
assert_file_contains "$workflow_file" "current-head image validation is still running" "opencode approval logs why the peer-check wait budget was dynamically extended"
assert_file_contains "$workflow_file" "current-head package/GPU build checks are still running" "opencode approval logs why package/GPU peer-check waits were dynamically extended"
assert_file_not_contains "$workflow_file" 'REVIEW_PUBLISH_STEP_TIMEOUT_SECONDS' "opencode review publication relies on the Actions step timeout instead of a background watchdog"
assert_file_not_contains "$workflow_file" "PUBLISH_STEP_TIMEOUT" "opencode review publication does not leave orphaned watchdog processes"
assert_file_not_contains "$workflow_file" "OPENCODE_PUBLISH_TIMEOUT_WRAPPED" "opencode review publication does not re-exec the runner shell script"
Expand Down
55 changes: 35 additions & 20 deletions tests/test_opencode_agent_contract.py
Original file line number Diff line number Diff line change
Expand Up @@ -859,7 +859,7 @@ def test_workflow_provisions_sandbox_tool_and_reviewer_agent():
r"Prepare bounded OpenCode review evidence[\s\S]{0,120}timeout-minutes: 12",
workflow,
)
assert re.search(r"opencode-review-target:[\s\S]*?timeout-minutes: 90", workflow)
assert re.search(r"opencode-review-target:[\s\S]*?timeout-minutes: 150", workflow)
assert "timeout-minutes: 12" in workflow
assert re.search(
r"Run OpenCode PR Review model pool[\s\S]{0,240}timeout-minutes: 65", workflow
Expand All @@ -879,16 +879,18 @@ def test_workflow_provisions_sandbox_tool_and_reviewer_agent():
workflow,
)
assert re.search(
r"Publish central OpenCode fast approval[\s\S]{0,900}timeout-minutes: 14",
r"Publish central OpenCode fast approval[\s\S]{0,900}timeout-minutes: 34",
workflow,
)
assert re.search(
r"Publish OpenCode review outcome[\s\S]{0,900}timeout-minutes: 16", workflow
r"Publish OpenCode review outcome[\s\S]{0,900}timeout-minutes: 36", workflow
)
assert workflow.count('APPROVAL_CHECK_WAIT_ATTEMPTS: "36"') == 2
assert workflow.count('APPROVAL_SLOW_BUILD_CHECK_WAIT_ATTEMPTS: "180"') == 2
assert workflow.count('APPROVAL_SLOW_IMAGE_CHECK_WAIT_ATTEMPTS: "60"') == 2
assert 'APPROVAL_CHECK_WAIT_SLEEP_SECONDS: "10"' in workflow
assert workflow.count("current-head image validation is still running") == 2
assert workflow.count("current-head package/GPU build checks are still running") == 2
assert 'CHECK_LOOKUP_GH_API_TIMEOUT_SECONDS: "15"' in workflow
assert 'OPENCODE_RUN_TIMEOUT_SECONDS: "120"' in workflow
assert (
Expand Down Expand Up @@ -1640,6 +1642,12 @@ def test_peer_check_wait_budget_fits_publication_step_timeouts():
r'APPROVAL_SLOW_IMAGE_CHECK_WAIT_ATTEMPTS: "(\d+)"', workflow
)
]
slow_build_attempts = [
int(value)
for value in re.findall(
r'APPROVAL_SLOW_BUILD_CHECK_WAIT_ATTEMPTS: "(\d+)"', workflow
)
]
sleeps = [
int(value)
for value in re.findall(r'APPROVAL_CHECK_WAIT_SLEEP_SECONDS: "(\d+)"', workflow)
Expand All @@ -1654,11 +1662,12 @@ def test_peer_check_wait_budget_fits_publication_step_timeouts():
)

assert normal_attempts == [36, 36]
assert slow_build_attempts == [180, 180]
assert slow_image_attempts == [60, 60]
assert sleeps == [10, 10]
assert fast_timeout is not None
assert publish_timeout is not None
wait_seconds = (slow_image_attempts[0] - 1) * sleeps[0]
wait_seconds = (max(slow_build_attempts[0], slow_image_attempts[0]) - 1) * sleeps[0]
assert int(fast_timeout.group(1)) * 60 - wait_seconds >= 120
assert int(publish_timeout.group(1)) * 60 - wait_seconds >= 240

Expand All @@ -1684,23 +1693,29 @@ def test_slow_peer_wait_matches_only_image_validation_checks():
("- docs image validation: in_progress\n", False, False),
)
for candidate, fast_expected, general_expected in probes:
fast_match = (
subprocess.run(
["grep", "-Eiq", "--", fast_pattern],
input=candidate,
text=True,
check=False,
).returncode
== 0
)
fast_match = re.search(fast_pattern, candidate, re.IGNORECASE) is not None
general_match = (
subprocess.run(
["grep", "-Eiq", "--", general_pattern],
input=candidate,
text=True,
check=False,
).returncode
== 0
re.search(general_pattern, candidate, re.IGNORECASE) is not None
)
assert fast_match is fast_expected, candidate
assert general_match is general_expected, candidate

gpu_pattern = r"^- ([^/]+/)?gpu-build([\s(]|:)"
package_build_pattern = (
r"^- ([^/]+/)?build \([^)]*"
r"(src-tauri/target/release/bundle|bundle/|\.msi|\.dmg|\.deb|\.appimage|AppImage)"
)
slow_build_probes = (
("- Release/gpu-build (ubuntu-22.04): IN_PROGRESS\n", True),
("- gpu-build (windows-2022) check run: in_progress\n", True),
("- Release/build (windows-latest, src-tauri/target/release/bundle/msi/*.msi): IN_PROGRESS\n", True),
("- build (macos-latest, src-tauri/target/release/bundle/dmg/*.dmg): IN_PROGRESS\n", True),
("- build (ubuntu-latest, unit tests): IN_PROGRESS\n", False),
("- docs-build: IN_PROGRESS\n", False),
)
for candidate, slow_build_expected in slow_build_probes:
slow_build_match = (
re.search(gpu_pattern, candidate, re.IGNORECASE) is not None
or re.search(package_build_pattern, candidate, re.IGNORECASE) is not None
)
assert slow_build_match is slow_build_expected, candidate
Loading