diff --git a/CHANGELOG.d/20260912-opencode-provider-failure-telemetry.md b/CHANGELOG.d/20260912-opencode-provider-failure-telemetry.md new file mode 100644 index 0000000000..5258f31a1b --- /dev/null +++ b/CHANGELOG.d/20260912-opencode-provider-failure-telemetry.md @@ -0,0 +1 @@ +Preserve bounded phase, reason, HTTP status, and duration evidence for OpenCode gateway failures while suppressing raw provider content and unverified provider/model identifiers and capping failure input. diff --git a/docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md b/docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md index 9b0749f258..027036db95 100644 --- a/docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md +++ b/docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md @@ -282,3 +282,20 @@ all five, and auto-optimize routing by cost. per-agent attempt; it changes only *which* agent gets tried next, never any per-attempt timeout, consistent with the 2026-08-31 amendment above. No other contextual-orchestrator behavior changes with this pin advance. + +- **2026-09-12 proposed amendment: preserve redaction-safe OpenCode failure + provenance.** The OpenCode model-pool adapter must keep the gateway-owned + canonical `error.detail` receipt useful after suppressing raw provider + content. For a bounded structured error it emits only allowlisted phase, + normalized reason, HTTP status, and caller-measured duration. Provider and + served-model identifiers remain `unknown` until a versioned CO-issued + non-secret identifier contract can be validated locally. Unknown, malformed, + and absent fields become fixed `unknown`/`malformed_gateway_envelope` + values; arbitrary + messages, response bodies, headers, credentials, and unbounded identifiers + never reach public Actions logs. The adapter reads at most the final 16 KiB + of the JSONL failure stream, suppresses unverified identifier values, + and fails oversized or deeply nested envelopes closed to the fixed malformed + state. This does not add a retry, timeout, provider choice, or model policy + to `.github`; contextual-orchestrator remains the owner of discovery, + routing, and failover. diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index 1cc9e20313..bd9d799719 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -3353,3 +3353,29 @@ queries the check-runs API at its own time, order-independently. The implementin their change was safe because they had scoped it narrowly, not because they had checked for the name collision — which is the more useful lesson: **a job name is unique only within one workflow file, and the same name in another file can carry the opposite safety property.** + +## 2026-09-12 OpenCode gateway failure provenance — Proposed + +**Observed exact evidence.** `ContextualWisdomLab/.github#2106` OpenCode run +[`34693400612`](https://github.com/ContextualWisdomLab/.github/actions/runs/34693400612) +reached `contextual-orchestrator/orchestrator/free`, failed after five seconds, +and logged only `class=provider-error json-bytes=836 stderr-bytes=0`. Raw-body +suppression worked, but it also discarded the gateway's bounded phase, reason, +provider, status, and served-model receipt, so the control plane could not name +the causal owner or distinguish 429, provider 5xx, request-too-large, queue +admission, and malformed-output failures. + +**Owner boundary and repair.** contextual-orchestrator continues to own +provider discovery, routing, failover, and the canonical `error.detail` +envelope. `.github` owns the OpenCode adapter and public-log sanitizer. The +proposed adapter change parses only the canonical bounded envelope and emits +allowlisted phase, reason, HTTP status, and caller-measured duration; +provider-controlled messages and raw bodies remain suppressed. Provider and +served-model identifiers stay `unknown` until a versioned CO-issued non-secret +identifier contract can be validated locally. Parsing is capped to the final +16 KiB of the failure stream, and oversized or deeply nested envelopes fail +closed. Production-shaped regression fixtures cover 429, provider 502, HTTP +413 request-too-large, queue admission, +malformed model output, missing served-model, malformed JSON, and credential +non-disclosure. This is diagnostic evidence only: it cannot turn provider +failure into approval, retry a model, or relax an exact-head merge gate. diff --git a/scripts/ci/run_opencode_review_model_pool.sh b/scripts/ci/run_opencode_review_model_pool.sh index 80f57d1d43..c3f2f1532f 100644 --- a/scripts/ci/run_opencode_review_model_pool.sh +++ b/scripts/ci/run_opencode_review_model_pool.sh @@ -3,6 +3,8 @@ set -euo pipefail : "${GITHUB_OUTPUT:=/dev/null}" +MAX_OPENCODE_FAILURE_TELEMETRY_BYTES=16384 + record_review_status() { printf 'review_status=%s\n' "$1" >>"$GITHUB_OUTPUT" } @@ -267,7 +269,8 @@ is_credit_exhausted_failure() { emit_sanitized_opencode_failure_detail() { local opencode_json_file="$1" local opencode_stderr_file="$2" - local json_bytes stderr_bytes failure_class + local attempt_duration_seconds="${3:-0}" + local json_bytes stderr_bytes failure_class gateway_telemetry json_bytes=0 stderr_bytes=0 @@ -300,6 +303,61 @@ emit_sanitized_opencode_failure_detail() { fi printf 'OpenCode provider failure metadata: class=%s json-bytes=%s stderr-bytes=%s; provider-controlled content suppressed.\n' \ "$failure_class" "$json_bytes" "$stderr_bytes" + + gateway_telemetry="$( + tail -c "$MAX_OPENCODE_FAILURE_TELEMETRY_BYTES" "$opencode_json_file" 2>/dev/null | + jq -Rrs --arg duration "${attempt_duration_seconds}s" ' + def safe_value($fallback): + if type == "string" and length > 0 and length <= 128 and + test("^[A-Za-z0-9._:/+-]+$") + then . else $fallback end; + def safe_phase: + if . == "connecting" or . == "requesting" or . == "reading" or + . == "decoding" or . == "validating" or + . == "response_error" or . == "queue_admission" + then . else "unknown" end; + def safe_reason: + if . == "rate_limited" or . == "provider_transport" or + . == "request_too_large" or . == "queue_admission_failed" or + . == "malformed_model_output" or + . == "eligible_candidates_exhausted" or + . == "discovery_failure" or . == "model_unavailable" or + . == "quota_exhausted" or . == "authentication_failed" + then . else "unknown" end; + [ + splits("\\n") | fromjson? | + select(.type == "error") | + (.error.data.responseBody? // .error.data.response_body? // empty) | + if type == "string" then fromjson? else . end | + select(type == "object") | + .error.detail? | + select(type == "object") + ] | last // empty | + . as $detail | + (if ($detail.attempts | type) == "array" and + ($detail.attempts | length) > 0 and + ($detail.attempts | length) <= 64 and + ($detail.attempts[-1] | type) == "object" + then $detail.attempts[-1] else {} end) as $attempt | + [ + "phase=" + (($attempt.phase // "unknown") | safe_value("unknown") | safe_phase), + "reason=" + (($attempt.error_code // $detail.terminal_reason // "unknown") | safe_value("unknown") | safe_reason), + "provider=unknown", + "status=" + (if ($attempt.provider_status | type) == "number" and + $attempt.provider_status >= 100 and $attempt.provider_status <= 599 and + ($attempt.provider_status | floor) == $attempt.provider_status + then ($attempt.provider_status | tostring) else "unknown" end), + "duration=" + $duration, + "served_model=unknown" + ] | join(" ") + ' 2>/dev/null || true + )" + if [ -n "$gateway_telemetry" ]; then + printf 'OpenCode gateway failure telemetry: %s; provider-controlled content suppressed.\n' "$gateway_telemetry" + elif [ "$json_bytes" -gt 0 ]; then + printf 'OpenCode gateway failure telemetry: phase=decode_error reason=malformed_gateway_envelope provider=unknown status=unknown duration=%ss served_model=unknown; provider-controlled content suppressed.\n' \ + "$attempt_duration_seconds" + fi } emit_rejected_opencode_artifact_metadata() { @@ -398,6 +456,7 @@ run_one_model_attempt() { local opencode_export_file="$8" local export_timeout_seconds opencode_status session_id opencode_stderr_file local opencode_pid fatal_kill_grace_seconds fatal_poll_seconds + local attempt_started_seconds attempt_duration_seconds export_timeout_seconds="${OPENCODE_EXPORT_TIMEOUT_SECONDS:-120}" fatal_poll_seconds="${OPENCODE_FATAL_ERROR_POLL_SECONDS:-5}" @@ -405,6 +464,7 @@ run_one_model_attempt() { opencode_stderr_file="${opencode_json_file}.stderr" rm -f "$opencode_json_file" "$opencode_stderr_file" "$opencode_export_file" "$candidate_output_file" + attempt_started_seconds="$SECONDS" set +e env -u GH_TOKEN -u GITHUB_TOKEN -u OPENCODE_APP_TOKEN \ -u ACTIONS_ID_TOKEN_REQUEST_TOKEN -u ACTIONS_ID_TOKEN_REQUEST_URL \ @@ -437,10 +497,11 @@ run_one_model_attempt() { done wait "$opencode_pid" opencode_status=$? + attempt_duration_seconds=$((SECONDS - attempt_started_seconds)) set -e if [ "$opencode_status" -ne 0 ]; then printf 'OpenCode %s attempt %s/%s failed with exit %s.\n' "$model_candidate" "$attempt" "$attempts" "$opencode_status" - emit_sanitized_opencode_failure_detail "$opencode_json_file" "$opencode_stderr_file" + emit_sanitized_opencode_failure_detail "$opencode_json_file" "$opencode_stderr_file" "$attempt_duration_seconds" if is_fatal_provider_failure "$opencode_json_file"; then printf 'OpenCode %s attempt %s/%s hit a fatal provider error (context window, token budget, quota, or model unavailable); skipping remaining attempts for this model.\n' "$model_candidate" "$attempt" "$attempts" return 2 diff --git a/tests/test_opencode_model_pool_runner.py b/tests/test_opencode_model_pool_runner.py index 2965d4c55c..f8eb71a083 100644 --- a/tests/test_opencode_model_pool_runner.py +++ b/tests/test_opencode_model_pool_runner.py @@ -408,6 +408,244 @@ def test_failed_provider_without_reason_logs_explicit_absence(tmp_path: Path) -> ) in result.stdout +def gateway_failure_event(detail: dict[str, object]) -> str: + """Return one production-shaped OpenCode event containing a gateway error.""" + response_body = json.dumps({"error": {"detail": detail}}) + return json.dumps( + { + "type": "error", + "error": { + "name": "ProviderError", + "data": { + "message": "provider-controlled message must stay suppressed", + "responseBody": response_body, + }, + }, + } + ) + + +@pytest.mark.parametrize( + ("detail", "expected_telemetry"), + [ + ( + { + "model": "meta-llama/llama-3.3-70b-instruct", + "terminal_reason": "eligible_candidates_exhausted", + "attempts": [ + { + "provider_name": "openrouter", + "phase": "response_error", + "provider_status": 429, + "error_code": "rate_limited", + } + ], + }, + "phase=response_error reason=rate_limited provider=unknown " + "status=429 duration=0s " + "served_model=unknown", + ), + ( + { + "model": "deepseek-ai/deepseek-v4-pro-0813", + "attempts": [ + { + "provider_name": "nvidia_nim", + "phase": "connecting", + "provider_status": 502, + "error_code": "provider_transport", + } + ], + }, + "phase=connecting reason=provider_transport provider=unknown " + "status=502 duration=0s " + "served_model=unknown", + ), + ( + { + "attempts": [ + { + "provider_name": "openrouter", + "phase": "response_error", + "provider_status": 413, + "error_code": "request_too_large", + } + ], + }, + "phase=response_error reason=request_too_large provider=unknown " + "status=413 duration=0s served_model=unknown", + ), + ( + { + "attempts": [ + { + "provider_name": "contextual-orchestrator", + "phase": "queue_admission", + "provider_status": 503, + "error_code": "queue_admission_failed", + } + ], + }, + "phase=queue_admission reason=queue_admission_failed " + "provider=unknown status=503 duration=0s " + "served_model=unknown", + ), + ( + { + "terminal_reason": "eligible_candidates_exhausted", + "attempts": [ + { + "provider_name": "bytez", + "phase": "validating", + "provider_status": 500, + "error_code": "malformed_model_output", + } + ], + }, + "phase=validating reason=malformed_model_output provider=unknown " + "status=500 duration=0s served_model=unknown", + ), + ], +) +def test_gateway_failure_logs_allowlisted_root_cause_fields( + tmp_path: Path, + detail: dict[str, object], + expected_telemetry: str, +) -> None: + """Gateway errors retain bounded routing evidence without raw response text.""" + result = run_failed_model(tmp_path, json_line=gateway_failure_event(detail)) + + assert result.returncode == 1 + telemetry_pattern = re.escape(expected_telemetry).replace( + "duration=0s", r"duration=\d+s" + ) + assert re.search(telemetry_pattern, result.stdout) + assert "provider-controlled message must stay suppressed" not in result.stdout + + +def test_malformed_gateway_failure_logs_only_bounded_decode_state( + tmp_path: Path, +) -> None: + """Malformed provider JSON reports a stable decode state without echoing bytes.""" + secret = "sk" + "-malformed-never-print" + result = run_failed_model( + tmp_path, + json_line=f'{{"type":"error","secret":"{secret}"', + ) + + assert result.returncode == 1 + assert re.search( + r"phase=decode_error reason=malformed_gateway_envelope provider=unknown " + r"status=unknown duration=\d+s served_model=unknown", + result.stdout, + ) + assert secret not in result.stdout + + +def test_gateway_failure_identifier_fields_reject_credential_shapes( + tmp_path: Path, +) -> None: + """Credential-shaped provider and model identifiers never reach public logs.""" + credential = "github" + "_pat_THISMUSTNEVERLEAK123456789" + result = run_failed_model( + tmp_path, + json_line=gateway_failure_event( + { + "model": credential, + "attempts": [ + { + "provider_name": credential, + "phase": "response_error", + "provider_status": 502, + "error_code": "provider_transport", + } + ], + } + ), + ) + + assert result.returncode == 1 + assert re.search( + r"phase=response_error reason=provider_transport provider=unknown " + r"status=502 duration=\d+s served_model=unknown", + result.stdout, + ) + assert credential not in result.stdout + + +def test_gateway_failure_identifier_fields_fail_closed_without_catalog_proof( + tmp_path: Path, +) -> None: + """Unverified identifier-shaped secrets never reach public logs.""" + credential = "BYTEZ" + "_TEST_SECRET_1234567890" + result = run_failed_model( + tmp_path, + json_line=gateway_failure_event( + { + "model": credential, + "attempts": [ + { + "provider_name": credential, + "phase": "response_error", + "provider_status": 502, + "error_code": "provider_transport", + } + ], + } + ), + ) + + assert result.returncode == 1 + assert re.search( + r"phase=response_error reason=provider_transport provider=unknown " + r"status=502 duration=\d+s served_model=unknown", + result.stdout, + ) + assert credential not in result.stdout + + +@pytest.mark.parametrize( + "response_body", + [ + json.dumps( + { + "error": { + "detail": { + "model": "safe/model", + "padding": "x" * (16 * 1024), + } + } + } + ), + "[" * 600 + "]" * 600, + ], +) +def test_gateway_failure_parser_fails_closed_on_oversized_or_deep_body( + tmp_path: Path, + response_body: str, +) -> None: + """Provider-controlled envelopes cannot force unbounded parsing or logging.""" + outer_event = json.dumps( + { + "type": "error", + "error": { + "data": { + "responseBody": response_body, + } + }, + } + ) + result = run_failed_model(tmp_path, json_line=outer_event) + + assert result.returncode == 1 + assert re.search( + r"phase=decode_error reason=malformed_gateway_envelope provider=unknown " + r"status=unknown duration=\d+s served_model=unknown", + result.stdout, + ) + assert "safe/model" not in result.stdout + + def test_backoff_environment_rejects_recursive_arithmetic_injection( tmp_path: Path, ) -> None: