diff --git a/.github/workflows/agent-review-runtime-quality-ci.yml b/.github/workflows/agent-review-runtime-quality-ci.yml index 6c6efc3dd1..7084226ae4 100644 --- a/.github/workflows/agent-review-runtime-quality-ci.yml +++ b/.github/workflows/agent-review-runtime-quality-ci.yml @@ -14,6 +14,11 @@ on: - "docs/doctoring/noema-review-token-lifetime.md" - "docs/product-technical-gap-baseline.md" - ".github/workflows/opencode-review-dispatch.yml" + - "scripts/ci/run_opencode_review_model_pool.sh" + - "scripts/ci/opencode_failure_envelope.py" + - "tests/test_opencode_model_pool_runner.py" + - "tests/test_opencode_failure_envelope.py" + - "docs/doctoring/opencode-provider-failure-envelope.md" - "scripts/ci/ensure_rust_llvm19.sh" - "tests/test_opencode_rust_coverage_toolchain_contract.py" - "scripts/ci/materialize_base_javascript_packages.py" @@ -185,6 +190,11 @@ jobs: noema_suite=true ;; .github/workflows/opencode-review-dispatch.yml|\ + scripts/ci/run_opencode_review_model_pool.sh|\ + scripts/ci/opencode_failure_envelope.py|\ + tests/test_opencode_model_pool_runner.py|\ + tests/test_opencode_failure_envelope.py|\ + docs/doctoring/opencode-provider-failure-envelope.md|\ scripts/ci/ensure_rust_llvm19.sh|\ tests/test_opencode_rust_coverage_toolchain_contract.py|\ scripts/ci/materialize_base_javascript_packages.py|\ @@ -357,6 +367,24 @@ jobs: python -m pytest -q tests/test_javascript_materializer_docstrings.py python -m compileall -q scripts/ci/materialize_base_javascript_packages.py tests/test_javascript_materializer_docstrings.py + - name: Verify OpenCode provider failure envelope + if: steps.affected_suites.outputs.opencode == 'true' + run: | + set -euo pipefail + PYTHONPATH=. python -m pytest -q \ + --cov=scripts.ci.opencode_failure_envelope \ + --cov-branch \ + --cov-fail-under=100 \ + tests/test_opencode_failure_envelope.py \ + tests/test_opencode_model_pool_runner.py + python -m interrogate --fail-under 100 \ + scripts/ci/opencode_failure_envelope.py + python -m compileall -q \ + scripts/ci/opencode_failure_envelope.py \ + tests/test_opencode_failure_envelope.py \ + tests/test_opencode_model_pool_runner.py + bash -n scripts/ci/run_opencode_review_model_pool.sh + - name: Verify exact-head path policy and syntax if: steps.affected_suites.outputs.strix == 'true' env: diff --git a/.github/workflows/opencode-review-dispatch.yml b/.github/workflows/opencode-review-dispatch.yml index d86497b3f4..d8c677b874 100644 --- a/.github/workflows/opencode-review-dispatch.yml +++ b/.github/workflows/opencode-review-dispatch.yml @@ -2641,6 +2641,7 @@ jobs: ContextualWisdomLab/.github:scripts/ci/pr_head_replay_guard.py | \ ContextualWisdomLab/.github:scripts/ci/pr_review_merge_scheduler.py | \ ContextualWisdomLab/.github:scripts/ci/run_opencode_review_model_pool.sh | \ + ContextualWisdomLab/.github:scripts/ci/opencode_failure_envelope.py | \ ContextualWisdomLab/.github:scripts/ci/opencode_review_normalize_output.py | \ ContextualWisdomLab/.github:scripts/ci/strix_quick_gate.sh | \ ContextualWisdomLab/.github:scripts/ci/validate_opencode_failed_check_review.sh | \ @@ -2649,6 +2650,7 @@ jobs: ContextualWisdomLab/.github:tests/test_materialize_base_javascript_packages.py | \ ContextualWisdomLab/.github:tests/test_opencode_agent_contract.py | \ ContextualWisdomLab/.github:tests/test_opencode_model_pool_runner.py | \ + ContextualWisdomLab/.github:tests/test_opencode_failure_envelope.py | \ ContextualWisdomLab/.github:tests/test_pr_head_replay_guard.py | \ ContextualWisdomLab/.github:tests/test_pr_review_fix_scheduler_coverage.py | \ ContextualWisdomLab/.github:tests/test_pr_review_merge_scheduler.py | \ diff --git a/.gitleaks.toml b/.gitleaks.toml index 68256fd97c..b8177b5e3b 100644 --- a/.gitleaks.toml +++ b/.gitleaks.toml @@ -16,3 +16,13 @@ regexes = [ '''gh[pousr]_(?:1234567890abcdef(?:1234)?|abcdef1234567890abcdef1234567890abcdef|installation_token_value|user_token_value|server_token_value|runner_token_value|placeholder_token_with_underscores_123|[a-z]{16}|[a-z]{20}|[a-z]{30}|[a-z]{38})''', '''github_pat_11AAAAA_(?:abcdefg|abcdefg1234567890|[A-Za-z]{7}_[a-z]{7}|[A-Za-z]{7}_[a-z]{17})''', ] + +[[allowlists]] +description = "OpenCode synthetic unknown-provenance token used only by confidentiality regressions." +condition = "AND" +regexTarget = "secret" +paths = [ + '''(^|/)tests/test_opencode_failure_envelope\.py$''', + '''(^|/)tests/test_opencode_model_pool_runner\.py$''', +] +regexes = ['''^BYTEZ_TEST_SECRET_[1]234567890$'''] diff --git a/CHANGELOG.d/20260912-opencode-provider-failure-telemetry.md b/CHANGELOG.d/20260912-opencode-provider-failure-telemetry.md new file mode 100644 index 0000000000..5258f31a1b --- /dev/null +++ b/CHANGELOG.d/20260912-opencode-provider-failure-telemetry.md @@ -0,0 +1 @@ +Preserve bounded phase, reason, HTTP status, and duration evidence for OpenCode gateway failures while suppressing raw provider content and unverified provider/model identifiers and capping failure input. diff --git a/CHANGELOG.md b/CHANGELOG.md index 707c18532e..e5244afeaf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,19 @@ +### OpenCode failures retain bounded causal telemetry + +- `run_opencode_review_model_pool.sh` now measures each failed invocation and + delegates its diagnostic to `opencode_failure_envelope.py`. The parser reads + only the final 16 KiB of the OpenCode error stream and up to 16 KiB of the gateway's + canonical `error.detail` receipt. Failure class comes only from allowlisted + structured status/reason semantics and validated HTTP status. Phase uses a + fixed public enum; provider, exception, and served model remain `unknown` + until an immutable CO receipt/catalog proves their provenance. Raw prose and + lexically valid unknown identifiers cannot influence or enter diagnostics. + Oversized, contradictory, deeply nested, malformed, or missing fields fail + closed; a malformed canonical body also suppresses outer status/reason + authority, and review exhaustion remains nonzero. + The dedicated runtime-quality lane now owns the runner, parser, and fixtures + with 100% statement/branch and public-doc coverage. Refs #2112. + ### Failed-check finding names the Strix sandbox instead of the gateway - `opencode-review-dispatch.yml`'s `emit_strix_provider_failure_finding` rendered one fixed finding for every `STRIX_PROVIDER_UNAVAILABLE` line, whose Root cause read "The contextual-orchestrator gateway or its discovered provider pool was unavailable for this run". `#1953` had just given the Strix sandbox bootstrap failure its own second verdict token (`STRIX_SANDBOX_UNAVAILABLE`) precisely because that attribution is wrong for it -- the sandbox container never reaches its Caido proxy, so the run dies before the gateway serves anything -- and this consumer re-applied the wrong attribution one step downstream, into the review findings and the failure census. The emitter now branches on the second token: a sandbox verdict gets a finding that names Strix's sandbox, says the verdict does not name the gateway, and tells the reader not to change gateway or provider configuration on its strength. A `STRIX_PROVIDER_UNAVAILABLE` line without the token keeps its existing text verbatim, so the gateway class has no regression surface. No test covered this finding text at all before (`gateway or its discovered provider pool` matched nothing under `tests/`); `tests/test_opencode_dispatch_strix_sandbox_finding.py` now runs the production emitter from the published run block and pins both directions plus the no-signal case. Refs #1953, #1935. diff --git a/docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md b/docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md index 9b0749f258..651bb643af 100644 --- a/docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md +++ b/docs/adr/0003-contextual-orchestrator-vendored-free-zdr.md @@ -282,3 +282,21 @@ all five, and auto-optimize routing by cost. per-agent attempt; it changes only *which* agent gets tried next, never any per-attempt timeout, consistent with the 2026-08-31 amendment above. No other contextual-orchestrator behavior changes with this pin advance. + +- **2026-09-12 proposed amendment: preserve redaction-safe OpenCode failure + provenance.** The OpenCode model-pool adapter must keep the gateway-owned + canonical `error.detail` receipt useful after suppressing raw provider + content. For a bounded structured error it emits only allowlisted phase, + normalized reason, HTTP status, and caller-measured duration. Provider and + served-model identifiers remain `unknown` until a versioned CO-issued + non-secret identifier contract can be validated locally. Unknown, malformed, + and absent fields become fixed `unknown`/`malformed_response` + values; arbitrary + messages, response bodies, headers, credentials, and unbounded identifiers + never reach public Actions logs. The adapter reads at most the final 16 KiB + of the JSONL failure stream, suppresses unverified identifier values, and + parses at most 16 KiB from the nested canonical gateway body. Larger gateway + bodies and deeply nested envelopes fail closed to the fixed malformed state. + This does not add a retry, timeout, provider choice, or model policy + to `.github`; contextual-orchestrator remains the owner of discovery, + routing, and failover. diff --git a/docs/doctoring/opencode-provider-failure-envelope.md b/docs/doctoring/opencode-provider-failure-envelope.md new file mode 100644 index 0000000000..b55db52208 --- /dev/null +++ b/docs/doctoring/opencode-provider-failure-envelope.md @@ -0,0 +1,74 @@ +# OpenCode provider-failure envelope + +## Problem and exact evidence + +On 2026-09-12, `.github` PR +[`#2106`](https://github.com/ContextualWisdomLab/.github/pull/2106) at exact +head `24bb6591ab7df23558cb793b4af60c567ff9da97` reached the single required +`contextual-orchestrator/orchestrator/free` model route in OpenCode run +[`34693400612`](https://github.com/ContextualWisdomLab/.github/actions/runs/34693400612). +The request failed after the sidecar and route preflight had succeeded, but the +only surviving causal evidence was `class=provider-error`, two byte counts, +and a statement that provider content was suppressed. That was enough to keep +the review fail-closed, but not enough to distinguish queue admission, HTTP +429/5xx, request size, malformed JSON, route exhaustion, or missing serving +identity. Issue +[`#2112`](https://github.com/ContextualWisdomLab/.github/issues/2112) owns the +repair. + +## Constraints + +- OpenCode, Noema, and Strix keep the single `orchestrator/free` gateway route; + no provider/model/group override or paid fallback is introduced. +- Telemetry is diagnostic only. It cannot produce approval, clean evidence, a + retry, a timeout, or a merge bypass. +- Provider-controlled messages, bodies, prompts, credentials, headers, source + text, and arbitrary nested payloads never reach stdout, status text, or + annotations. +- Only exact allowlisted phase/reason enums and validated HTTP status numbers + may enter causal output. Lexically valid but unproven identifiers are not + evidence of non-secret provenance. Provider, model, and exception identities + remain `unknown` until an immutable CO receipt/catalog proves them. + +## Alternatives and decision + +Keeping the previous byte-count-only line was rejected because it preserves +secrecy at the cost of causal attribution. Printing the raw OpenCode event or +gateway response was rejected because public `pull_request_target` logs cannot +safely carry provider-controlled text. Adding caller-side retries or an +elapsed-time diagnosis was rejected because the gateway owns routing and the +observed five-second failure did not prove a timeout. + +The selected design adds a small standard-library parser at the OpenCode +adapter boundary. It reads at most the final 16 KiB of each failure artifact, +drops an incomplete leading line, and accepts only an OpenCode `type=error` +event within an explicit 64-level structural-depth limit. From the gateway response it parses +at most 16 KiB and reads only the canonical `error.detail`/`error_detail` +receipt and its last bounded attempt. Only allowlisted structured status/reason +pairs determine failure class; contradictory pairs become `provider-error`. +Phase is emitted only when it matches a fixed public enum. Provider, exception, +and served model remain `unknown` because this consumer has no immutable CO +catalog proof that can authenticate dynamic identities. Raw +event/stderr bytes are represented only by presence and byte counts, never +passed to the causal classifier. Oversized gateway bodies, deeply nested or +malformed JSON, and malformed Unicode fail closed to bounded metadata. + +## Executable evidence, risks, and effects + +The production launcher fixtures cover HTTP 429/queue capacity, provider 503, +non-JSON response bodies, HTTP 413 request admission, no eligible route, absent +served-model metadata, contradictory structured causes, raw prose pollution, +unproven identifier provenance, 16 KiB overflow, excessive JSON depth, and +secret-bearing ignored fields. Unit tests cover all parser statements and +branches, and the consolidated runtime-quality workflow selects this suite +whenever the launcher, parser, fixture, or this authority record changes. + +The remaining risk is semantic drift and missing identity provenance in the +gateway receipt. Unknown fields are deliberately not guessed or copied; a +future immutable CO schema/catalog change must add a failing fixture before an +identity or enum enters the allowlist. Operators can now route a +429/queue failure to capacity policy, a 5xx to the gateway/provider boundary, +a 413 to request admission, and malformed JSON to the response adapter without +reading secret-bearing bodies. Until exact-head hosted checks, independent +review, protected-main integration, and an unchanged-head replay of #2106 are +complete, this repair remains Proposed rather than released evidence. diff --git a/docs/product-technical-gap-baseline.md b/docs/product-technical-gap-baseline.md index 1cc9e20313..e74e90c589 100644 --- a/docs/product-technical-gap-baseline.md +++ b/docs/product-technical-gap-baseline.md @@ -3353,3 +3353,35 @@ queries the check-runs API at its own time, order-independently. The implementin their change was safe because they had scoped it narrowly, not because they had checked for the name collision — which is the more useful lesson: **a job name is unique only within one workflow file, and the same name in another file can carry the opposite safety property.** + +## OpenCode provider-failure causal envelope — 2026-09-12 + +**Status: Proposed; owner repair implemented on the issue branch, not yet +protected or released.** `ContextualWisdomLab/.github#2106@24bb6591ab7df23558cb793b4af60c567ff9da97` +had five exact-head security/runtime checks succeed, then OpenCode run +`34693400612` exhausted after emitting only `class=provider-error` and byte +counts. The absence of safe phase/provider/status/model evidence made the +failure causally ambiguous; it did not prove the separate timeout defect. + +Issue `ContextualWisdomLab/.github#2112` now has an executable RED→GREEN owner repair. The OpenCode +adapter parses only the final 16 KiB error-event stream and at most 16 KiB of the canonical +gateway receipt. Only allowlisted structured status/reason pairs determine +failure class; fixed enums bound phase/reason, while provider, exception, and +served model remain `unknown` until an immutable CO receipt/catalog contract +authenticates them. Raw text, lexically valid unknown identifiers, +contradictory evidence, bodies over 16 KiB, and 10,000-level JSON all fail +closed. A malformed canonical body also suppresses outer `data.code` and HTTP +status authority instead of allowing either to override the fixed malformed +state. Production fixtures cover 429, 5xx, malformed JSON with conflicting +outer authority, 413, pool exhaustion, unproven identity, causal pollution, +and credential-shaped fields. +The previously missing CI ownership is also repaired: launcher/parser/test/doc +changes select the dedicated runtime-quality suite, which enforces 100% parser +statement/branch and public-doc coverage. + +**Remaining action:** obtain exact-head hosted checks and independent review, +define and release the versioned CO identity-provenance contract before exposing +provider/model identity, merge normally to protected `main`, then replay #2106 +unchanged. Only that consumer replay can show whether the next real failure +contains enough bounded causal evidence; this Proposed branch is not immutable +release or production proof. diff --git a/scripts/ci/opencode_failure_envelope.py b/scripts/ci/opencode_failure_envelope.py new file mode 100755 index 0000000000..f8f4237a0a --- /dev/null +++ b/scripts/ci/opencode_failure_envelope.py @@ -0,0 +1,394 @@ +#!/usr/bin/env python3 +"""Emit bounded, redaction-safe metadata for one failed OpenCode invocation.""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path +from typing import Any + + +MAX_FAILURE_FILE_BYTES = 16_384 +MAX_GATEWAY_BODY_BYTES = 16_384 +MAX_JSON_DEPTH = 64 +_BODY_ABSENT = object() +SAFE_FAILURE_PHASES = frozenset( + { + "admission", + "authentication", + "connecting", + "queue_admission", + "request_admission", + "response_error", + "route_selection", + "streaming", + } +) +REASON_FAILURE_CLASSES = { + "request_too_large": "request-too-large", + "payload_too_large": "request-too-large", + "context_overflow": "context-window", + "context_window_exceeded": "context-window", + "tokens_limit_reached": "context-window", + "insufficient_credits": "credit-exhausted", + "payment_required": "credit-exhausted", + "budget_limit": "quota-or-budget", + "insufficient_quota": "quota-or-budget", + "quota_exceeded": "quota-or-budget", + "eligible_candidates_exhausted": "model-pool-exhausted", + "no_eligible_route": "model-pool-exhausted", + "model_pool_exhausted": "model-pool-exhausted", + "model_not_found": "model-unavailable", + "no_endpoints": "model-unavailable", + "rate_limit": "rate-limit", + "rate_limited": "rate-limit", + "too_many_requests": "rate-limit", + "queue_capacity": "rate-limit", + "permission_denied": "authentication-or-permission", + "authentication_failed": "authentication-or-permission", + "authorization_failed": "authentication-or-permission", + "timeout": "timeout", + "timed_out": "timeout", + "provider_timeout": "timeout", + "provider_unavailable": "provider-5xx", + "upstream_error": "provider-5xx", +} +_GENERIC_5XX_REFINEMENTS = frozenset( + {"provider-5xx", "model-pool-exhausted", "model-unavailable", "timeout"} +) + + +def _read_bounded(path: Path) -> tuple[bytes, int]: + """Read a bounded final-line tail while retaining the non-secret byte count.""" + try: + byte_count = path.stat().st_size + with path.open("rb") as stream: + truncated = byte_count > MAX_FAILURE_FILE_BYTES + preceding_byte = b"" + if truncated: + stream.seek(-MAX_FAILURE_FILE_BYTES - 1, 2) + preceding_byte = stream.read(1) + raw = stream.read(MAX_FAILURE_FILE_BYTES) + if truncated and preceding_byte != b"\n": + _, separator, raw = raw.partition(b"\n") + if not separator: + raw = b"" + return raw, byte_count + except OSError: + return b"", 0 + + +def _safe_enum(value: Any, allowed_values: frozenset[str] | dict[str, str]) -> str | None: + """Return an exact allowlisted receipt token or no value.""" + return value if isinstance(value, str) and value in allowed_values else None + + +def _safe_http_status(value: Any) -> int | None: + """Return a valid HTTP status while excluding booleans and free text.""" + if type(value) is int and 100 <= value <= 599: + return value + if isinstance(value, str) and len(value) == 3 and value.isascii() and value.isdigit(): + status = int(value) + return status if 100 <= status <= 599 else None + return None + + +def _consistent_authority(values: tuple[Any | None, ...]) -> tuple[Any | None, bool]: + """Return one exact authority value, or flag conflicting validated values.""" + accepted = tuple(value for value in values if value is not None) + if not accepted: + return None, False + return accepted[0], any(value != accepted[0] for value in accepted[1:]) + + +def _status_failure_class(status: int | None) -> str | None: + """Map one validated HTTP status to its conservative public class.""" + if status == 413: + return "request-too-large" + if status == 402: + return "credit-exhausted" + if status == 429: + return "rate-limit" + if status in {401, 403}: + return "authentication-or-permission" + if status is not None and 500 <= status <= 599: + return "provider-5xx" + return "provider-error" if status is not None else None + + +def _failure_classes_compatible( + status_class: str | None, reason_class: str | None +) -> bool: + """Return whether a structured reason may refine one HTTP status class.""" + if status_class is None or reason_class is None or status_class == reason_class: + return True + return ( + status_class == "provider-5xx" + and reason_class in _GENERIC_5XX_REFINEMENTS + ) + + +def _is_within_json_depth(value: Any) -> bool: + """Return whether a decoded provider value stays within the depth invariant.""" + pending = [(value, 1)] + while pending: + current, depth = pending.pop() + if depth > MAX_JSON_DEPTH: + return False + if isinstance(current, dict): + pending.extend((item, depth + 1) for item in current.values()) + elif isinstance(current, list): + pending.extend((item, depth + 1) for item in current) + return True + + +def _last_error_event(raw: bytes) -> dict[str, Any] | None: + """Return the last bounded OpenCode JSON-lines error event.""" + if len(raw) > MAX_FAILURE_FILE_BYTES: + return None + last: dict[str, Any] | None = None + try: + text = raw.decode("utf-8") + except UnicodeDecodeError: + return None + for line in text.splitlines(): + try: + event = json.loads(line) + except (json.JSONDecodeError, RecursionError, TypeError, ValueError): + continue + if not _is_within_json_depth(event): + continue + if isinstance(event, dict) and event.get("type") == "error": + last = event + return last + + +def _gateway_detail(data: dict[str, Any]) -> tuple[dict[str, Any], bool]: + """Extract one canonical gateway error detail and flag malformed bodies.""" + body_value = next( + (data[key] for key in ("responseBody", "response_body", "body") if key in data), + _BODY_ABSENT, + ) + if body_value is _BODY_ABSENT: + payload: Any = data + malformed = False + elif isinstance(body_value, dict): + try: + body_bytes = json.dumps( + body_value, ensure_ascii=False, separators=(",", ":") + ).encode("utf-8") + except (RecursionError, TypeError, UnicodeEncodeError, ValueError): + return {}, True + if ( + len(body_bytes) > MAX_GATEWAY_BODY_BYTES + or not _is_within_json_depth(body_value) + ): + return {}, True + payload = body_value + malformed = False + elif isinstance(body_value, str): + try: + body_bytes = body_value.encode("utf-8") + except UnicodeEncodeError: + return {}, True + if len(body_bytes) > MAX_GATEWAY_BODY_BYTES: + return {}, True + try: + payload = json.loads(body_value) + if not isinstance(payload, dict) or not _is_within_json_depth(payload): + return {}, True + malformed = False + except (json.JSONDecodeError, RecursionError, TypeError, ValueError): + return {}, True + else: + return {}, True + error = payload.get("error") + if isinstance(error, dict) and isinstance(error.get("detail"), dict): + return error["detail"], malformed + detail = payload.get("error_detail") + if isinstance(detail, dict): + return detail, malformed + direct = payload.get("detail") + return (direct, malformed) if isinstance(direct, dict) else ({}, malformed) + + +def _gateway_details(data: dict[str, Any]) -> tuple[tuple[dict[str, Any], ...], bool]: + """Extract every present gateway body alias without precedence selection.""" + body_keys = tuple( + key for key in ("responseBody", "response_body", "body") if key in data + ) + if not body_keys: + detail, malformed = _gateway_detail(data) + return (detail,), malformed + + details = [] + for key in body_keys: + detail, malformed = _gateway_detail({key: data[key]}) + if malformed: + return (), True + details.append(detail) + return tuple(details), False + + +def _failure_class( + *, + status: int | None, + reason: str | None, + malformed_body: bool, + has_json_artifact: bool, + has_stderr_artifact: bool, + has_event: bool, + authority_conflict: bool = False, +) -> str: + """Normalize one failure class from validated structured receipt fields.""" + if authority_conflict: + return "provider-error" + status_class = _status_failure_class(status) + reason_class = REASON_FAILURE_CLASSES.get(reason or "") + if not _failure_classes_compatible(status_class, reason_class): + return "provider-error" + if reason_class is not None: + return reason_class + if status_class is not None: + return status_class + if malformed_body or (has_json_artifact and not has_event): + return "malformed-response" + if has_json_artifact or has_stderr_artifact: + return "provider-error" + return "no-provider-detail" + + +def format_failure_metadata( + json_path: Path, stderr_path: Path, duration_seconds: int +) -> str: + """Format one stable diagnostic line from bounded OpenCode failure artifacts.""" + raw_json, json_bytes = _read_bounded(json_path) + raw_stderr, stderr_bytes = _read_bounded(stderr_path) + event = _last_error_event(raw_json) + + malformed_container = False + if isinstance(event, dict) and "error" in event: + event_error = event["error"] + if isinstance(event_error, dict): + error = event_error + else: + error = {} + malformed_container = True + else: + error = {} + + if "data" in error: + error_data = error["data"] + if isinstance(error_data, dict): + data = error_data + else: + data = {} + malformed_container = True + else: + data = {} + + details, malformed_body = _gateway_details(data) + malformed_body = malformed_container or malformed_body + if malformed_body: + data = {} + + last_attempts = [] + phases = [] + for detail in details: + attempts = detail.get("attempts") + last_attempt: dict[str, Any] = {} + if ( + isinstance(attempts, list) + and attempts + and len(attempts) <= 64 + and isinstance(attempts[-1], dict) + ): + last_attempt = attempts[-1] + last_attempts.append(last_attempt) + phases.append( + _safe_enum(last_attempt.get("phase"), SAFE_FAILURE_PHASES) + or _safe_enum(detail.get("phase"), SAFE_FAILURE_PHASES) + ) + + reason, reason_conflict = _consistent_authority( + tuple( + _safe_enum(detail.get(key), REASON_FAILURE_CLASSES) + for detail in details + for key in ("terminal_reason", "stop_reason", "error_code") + ) + + (_safe_enum(data.get("code"), REASON_FAILURE_CLASSES),) + ) + gateway_status, gateway_status_conflict = _consistent_authority( + ( + _safe_http_status(data.get("statusCode")), + _safe_http_status(data.get("status_code")), + ) + ) + provider_status, provider_status_conflict = _consistent_authority( + tuple( + _safe_http_status(attempt.get("provider_status")) + for attempt in last_attempts + ) + ) + status = gateway_status if gateway_status is not None else provider_status + status_conflict = gateway_status_conflict or provider_status_conflict + phase, phase_conflict = _consistent_authority(tuple(phases)) + + reason_class = REASON_FAILURE_CLASSES.get(reason or "") + status_classes = tuple( + _status_failure_class(candidate) + for candidate in (gateway_status, provider_status) + if candidate is not None + ) + cross_conflict = reason_class is not None and any( + not _failure_classes_compatible(status_class, reason_class) + for status_class in status_classes + ) + authority_conflict = reason_conflict or status_conflict or cross_conflict + failure_class = _failure_class( + status=status, + reason=reason, + malformed_body=malformed_body, + has_json_artifact=json_bytes > 0, + has_stderr_artifact=stderr_bytes > 0, + has_event=event is not None, + authority_conflict=authority_conflict, + ) + normalized_reason = ( + "unknown" + if authority_conflict + else reason or failure_class.replace("-", "_") + ) + fields = { + "class": failure_class, + "json-bytes": str(json_bytes), + "stderr-bytes": str(stderr_bytes), + "phase": ( + str(phase) if phase is not None and not phase_conflict else "unknown" + ), + "reason": normalized_reason, + "provider": "unknown", + "http-status": ( + str(status) if status is not None and not authority_conflict else "unknown" + ), + "exception": "unknown", + "duration-seconds": str(max(0, duration_seconds)), + "served-model": "unknown", + } + rendered = " ".join(f"{key}={value}" for key, value in fields.items()) + return f"OpenCode provider failure metadata: {rendered}; provider-controlled content suppressed." + + +def main(argv: list[str] | None = None) -> int: + """Print one redaction-safe failure line for the shell runner.""" + arguments = sys.argv[1:] if argv is None else argv + if len(arguments) != 3 or not arguments[2].isascii() or not arguments[2].isdigit(): + print("usage: opencode_failure_envelope.py JSON STDERR DURATION_SECONDS", file=sys.stderr) + return 2 + print(format_failure_metadata(Path(arguments[0]), Path(arguments[1]), int(arguments[2]))) + return 0 + + +if __name__ == "__main__": # pragma: no cover - exercised through the shell contract + raise SystemExit(main()) diff --git a/scripts/ci/run_opencode_review_model_pool.sh b/scripts/ci/run_opencode_review_model_pool.sh index 80f57d1d43..c53035806e 100644 --- a/scripts/ci/run_opencode_review_model_pool.sh +++ b/scripts/ci/run_opencode_review_model_pool.sh @@ -267,39 +267,10 @@ is_credit_exhausted_failure() { emit_sanitized_opencode_failure_detail() { local opencode_json_file="$1" local opencode_stderr_file="$2" - local json_bytes stderr_bytes failure_class + local duration_seconds="${3:-0}" - json_bytes=0 - stderr_bytes=0 - if [ -s "$opencode_json_file" ]; then - json_bytes="$(wc -c <"$opencode_json_file" | tr -d ' ')" - fi - if [ -s "$opencode_stderr_file" ]; then - stderr_bytes="$(wc -c <"$opencode_stderr_file" | tr -d ' ')" - fi - - failure_class="unclassified" - if grep -Eiq 'ContextOverflowError|tokens_limit_reached|Request body too large|context window' "$opencode_json_file" "$opencode_stderr_file" 2>/dev/null; then - failure_class="context-window" - elif grep -Eiq 'insufficient credits|payment required|"code"[[:space:]]*:[[:space:]]*402' "$opencode_json_file" "$opencode_stderr_file" 2>/dev/null; then - failure_class="credit-exhausted" - elif grep -Eiq 'budget limit|insufficient_quota|quota exceeded' "$opencode_json_file" "$opencode_stderr_file" 2>/dev/null; then - failure_class="quota-or-budget" - elif grep -Eiq 'model_not_found|model not found|ModelNotFoundError|not a valid model|no endpoints' "$opencode_json_file" "$opencode_stderr_file" 2>/dev/null; then - failure_class="model-unavailable" - elif grep -Eiq 'rate.?limit|too many requests|(^|[^0-9])429([^0-9]|$)' "$opencode_json_file" "$opencode_stderr_file" 2>/dev/null; then - failure_class="rate-limit" - elif grep -Eiq 'permission denied|authentication|authorization|(^|[^0-9])(401|403)([^0-9]|$)' "$opencode_json_file" "$opencode_stderr_file" 2>/dev/null; then - failure_class="authentication-or-permission" - elif grep -Eiq 'timed? ?out|timeout' "$opencode_json_file" "$opencode_stderr_file" 2>/dev/null; then - failure_class="timeout" - elif [ "$json_bytes" -gt 0 ] || [ "$stderr_bytes" -gt 0 ]; then - failure_class="provider-error" - else - failure_class="no-provider-detail" - fi - printf 'OpenCode provider failure metadata: class=%s json-bytes=%s stderr-bytes=%s; provider-controlled content suppressed.\n' \ - "$failure_class" "$json_bytes" "$stderr_bytes" + python3 "$GITHUB_WORKSPACE/scripts/ci/opencode_failure_envelope.py" \ + "$opencode_json_file" "$opencode_stderr_file" "$duration_seconds" } emit_rejected_opencode_artifact_metadata() { @@ -397,12 +368,13 @@ run_one_model_attempt() { local opencode_json_file="$7" local opencode_export_file="$8" local export_timeout_seconds opencode_status session_id opencode_stderr_file - local opencode_pid fatal_kill_grace_seconds fatal_poll_seconds + local opencode_pid fatal_kill_grace_seconds fatal_poll_seconds attempt_started_seconds export_timeout_seconds="${OPENCODE_EXPORT_TIMEOUT_SECONDS:-120}" fatal_poll_seconds="${OPENCODE_FATAL_ERROR_POLL_SECONDS:-5}" fatal_kill_grace_seconds="${OPENCODE_FATAL_KILL_GRACE_SECONDS:-5}" opencode_stderr_file="${opencode_json_file}.stderr" + attempt_started_seconds="$SECONDS" rm -f "$opencode_json_file" "$opencode_stderr_file" "$opencode_export_file" "$candidate_output_file" set +e @@ -440,7 +412,7 @@ run_one_model_attempt() { set -e if [ "$opencode_status" -ne 0 ]; then printf 'OpenCode %s attempt %s/%s failed with exit %s.\n' "$model_candidate" "$attempt" "$attempts" "$opencode_status" - emit_sanitized_opencode_failure_detail "$opencode_json_file" "$opencode_stderr_file" + emit_sanitized_opencode_failure_detail "$opencode_json_file" "$opencode_stderr_file" "$((SECONDS - attempt_started_seconds))" if is_fatal_provider_failure "$opencode_json_file"; then printf 'OpenCode %s attempt %s/%s hit a fatal provider error (context window, token budget, quota, or model unavailable); skipping remaining attempts for this model.\n' "$model_candidate" "$attempt" "$attempts" return 2 diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index 6ea00c099f..1a7e803a49 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -810,8 +810,8 @@ assert_opencode_review_uses_codegraph_and_contextual_orchestrator() { assert_file_contains "$workflow_file" "The publish gate re-runs source-backed validation against PR-head data" "opencode review publish gate validates model output against the PR-head worktree" assert_file_contains "$REPO_ROOT/scripts/ci/run_opencode_review_model_pool.sh" 'OpenCode %s attempt %s/%s failed with exit %s.' "opencode review logs per-model retry attempts" assert_file_contains "$REPO_ROOT/scripts/ci/run_opencode_review_model_pool.sh" "emit_sanitized_opencode_failure_detail" "opencode review logs a bounded provider reason after each failed attempt" - assert_file_contains "$REPO_ROOT/scripts/ci/run_opencode_review_model_pool.sh" "OpenCode provider failure metadata" "opencode review labels provider failure classes in the check log" - assert_file_contains "$REPO_ROOT/scripts/ci/run_opencode_review_model_pool.sh" "provider-controlled content suppressed" "opencode provider failure logging suppresses credential-bearing content" + assert_file_contains "$REPO_ROOT/scripts/ci/opencode_failure_envelope.py" "OpenCode provider failure metadata" "opencode review labels provider failure classes in the check log" + assert_file_contains "$REPO_ROOT/scripts/ci/opencode_failure_envelope.py" "provider-controlled content suppressed" "opencode provider failure logging suppresses credential-bearing content" assert_file_not_contains "$REPO_ROOT/scripts/ci/run_opencode_review_model_pool.sh" 'cat "$opencode_json_file"' "opencode review never replays provider JSON to the check log" assert_file_not_contains "$REPO_ROOT/scripts/ci/run_opencode_review_model_pool.sh" 'cat "$opencode_export_file"' "opencode review never replays provider exports to the check log" assert_file_not_contains "$REPO_ROOT/scripts/ci/run_opencode_review_model_pool.sh" 'cat "$candidate_output_file"' "opencode review never replays rejected assistant output to the check log" diff --git a/tests/test_agent_review_runtime_quality_consolidation.py b/tests/test_agent_review_runtime_quality_consolidation.py index 4592cfd166..9cc7483f04 100644 --- a/tests/test_agent_review_runtime_quality_consolidation.py +++ b/tests/test_agent_review_runtime_quality_consolidation.py @@ -153,6 +153,57 @@ def test_review_repair_suite_is_selected_and_conditionally_executed() -> None: assert workflow.count("runs-on:") == 1 +@pytest.mark.parametrize( + "changed_path", + ( + "scripts/ci/run_opencode_review_model_pool.sh", + "scripts/ci/opencode_failure_envelope.py", + "tests/test_opencode_model_pool_runner.py", + "tests/test_opencode_failure_envelope.py", + "docs/doctoring/opencode-provider-failure-envelope.md", + ), +) +def test_opencode_failure_paths_start_and_select_the_owned_suite( + changed_path: str, +) -> None: + """Every failure-envelope delta must execute its exact owned contract suite.""" + workflow = _workflow_text() + trigger = workflow.split("on:\n", 1)[1].split("\nconcurrency:\n", 1)[0] + assert f' - "{changed_path}"' in trigger + + selector = workflow.split(' case "$changed_path" in\n', 1)[1].split( + " esac", 1 + )[0] + result = subprocess.run( + [ + "bash", + "--noprofile", + "--norc", + "-e", + "-o", + "pipefail", + "-c", + 'IFS= read -r changed_path\nopencode_suite=false\n' + 'case "$changed_path" in\n' + + selector + + 'esac\nprintf "%s" "$opencode_suite"\n', + ], + input=changed_path + "\n", + text=True, + capture_output=True, + check=True, + ) + assert result.stdout == "true" + assert result.stderr == "" + assert "Verify OpenCode provider failure envelope" in workflow + assert "tests/test_opencode_model_pool_runner.py" in workflow + assert "tests/test_opencode_failure_envelope.py" in workflow + assert "--cov=scripts.ci.opencode_failure_envelope" in workflow + assert "--cov-branch" in workflow + assert "--cov-fail-under=100" in workflow + assert "python -m interrogate --fail-under 100" in workflow + + @pytest.mark.parametrize( ("changed_path", "starts_runner", "review_repair", "queue"), ( diff --git a/tests/test_opencode_agent_contract.py b/tests/test_opencode_agent_contract.py index 321d25bd57..aa334e8cc6 100644 --- a/tests/test_opencode_agent_contract.py +++ b/tests/test_opencode_agent_contract.py @@ -1832,6 +1832,9 @@ def test_workflow_provisions_sandbox_tool_and_reviewer_agent(): model_pool_runner = Path("scripts/ci/run_opencode_review_model_pool.sh").read_text( encoding="utf-8" ) + failure_envelope = Path( + "scripts/ci/opencode_failure_envelope.py" + ).read_text(encoding="utf-8") assert "assert_reasoning_effort_for_candidate" in model_pool_runner assert "assert_opencode_reasoning_effort.py" in model_pool_runner assert "--config opencode.jsonc" in model_pool_runner @@ -1856,8 +1859,9 @@ def test_workflow_provisions_sandbox_tool_and_reviewer_agent(): assert "has no model inference timeout" in model_pool_runner assert "timed out after %ss" not in model_pool_runner assert "emit_sanitized_opencode_failure_detail" in model_pool_runner - assert "OpenCode provider failure metadata" in model_pool_runner - assert "provider-controlled content suppressed" in model_pool_runner + assert "opencode_failure_envelope.py" in model_pool_runner + assert "OpenCode provider failure metadata" in failure_envelope + assert "provider-controlled content suppressed" in failure_envelope assert 'cat "$opencode_json_file"' not in model_pool_runner assert 'cat "$opencode_export_file"' not in model_pool_runner assert 'cat "$candidate_output_file"' not in model_pool_runner @@ -1890,6 +1894,8 @@ def test_workflow_provisions_sandbox_tool_and_reviewer_agent(): in workflow ) assert "scripts/ci/run_opencode_review_model_pool.sh | \\" in workflow + assert "scripts/ci/opencode_failure_envelope.py | \\" in workflow + assert "tests/test_opencode_failure_envelope.py | \\" in workflow assert ( "ContextualWisdomLab/.github:tests/test_javascript_coverage_gate.py | \\" in workflow diff --git a/tests/test_opencode_failure_envelope.py b/tests/test_opencode_failure_envelope.py new file mode 100644 index 0000000000..8e9b45169e --- /dev/null +++ b/tests/test_opencode_failure_envelope.py @@ -0,0 +1,756 @@ +"""Unit coverage for bounded OpenCode provider-failure metadata.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from scripts.ci import opencode_failure_envelope as envelope + + +def test_failure_artifact_tail_budget_remains_16_kib() -> None: + """The canonical successor preserves the carried 16 KiB JSONL tail bound.""" + assert envelope.MAX_FAILURE_FILE_BYTES == 16_384 + + +def test_read_bounded_handles_missing_and_oversized_files(tmp_path: Path) -> None: + """Missing artifacts are empty and large artifacts retain their true size.""" + assert envelope._read_bounded(tmp_path / "missing") == (b"", 0) + large = tmp_path / "large" + large.write_bytes(b"x" * (envelope.MAX_FAILURE_FILE_BYTES + 2)) + raw, byte_count = envelope._read_bounded(large) + assert raw == b"" + assert byte_count == envelope.MAX_FAILURE_FILE_BYTES + 2 + assert envelope._last_error_event(raw) is None + assert ( + envelope._last_error_event(b"x" * (envelope.MAX_FAILURE_FILE_BYTES + 1)) + is None + ) + + final_event = b'{"type":"error","error":{"data":{}}}\n' + large.write_bytes(b"x" * envelope.MAX_FAILURE_FILE_BYTES + b"\n" + final_event) + raw, byte_count = envelope._read_bounded(large) + assert raw == final_event + assert byte_count == envelope.MAX_FAILURE_FILE_BYTES + 1 + len(final_event) + + event_prefix = b'{"type":"error","error":{"padding":"' + event_suffix = b'"}}\n' + aligned_event = ( + event_prefix + + b"x" + * (envelope.MAX_FAILURE_FILE_BYTES - len(event_prefix) - len(event_suffix)) + + event_suffix + ) + assert len(aligned_event) == envelope.MAX_FAILURE_FILE_BYTES + large.write_bytes(b"x\n" + aligned_event) + raw, byte_count = envelope._read_bounded(large) + assert raw == aligned_event + assert byte_count == 2 + envelope.MAX_FAILURE_FILE_BYTES + assert envelope._last_error_event(raw) is not None + + +@pytest.mark.parametrize( + ("value", "allowed_values", "expected"), + [ + ("queue_admission", frozenset({"queue_admission"}), "queue_admission"), + ("not_allowlisted", frozenset({"queue_admission"}), None), + (1, frozenset({"queue_admission"}), None), + ], +) +def test_safe_enum_accepts_only_exact_allowlisted_tokens( + value: object, allowed_values: frozenset[str], expected: str | None +) -> None: + """Lexical shape alone cannot make provider data public.""" + assert envelope._safe_enum(value, allowed_values) == expected + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + (429, 429), + ("503", 503), + (True, None), + (99, None), + (600, None), + ("429 ", None), + ("abc", None), + ], +) +def test_safe_http_status_rejects_non_http_values( + value: object, expected: int | None +) -> None: + """Only three-digit HTTP status values survive normalization.""" + assert envelope._safe_http_status(value) == expected + + +def test_last_error_event_uses_last_valid_error_and_rejects_bad_utf8() -> None: + """JSON-lines noise is ignored while invalid UTF-8 fails closed.""" + raw = ( + b"not-json\n" + b'{"type":"text"}\n' + b'{"type":"error","error":{"name":"First"}}\n' + b'{"type":"error","error":{"name":"Last"}}\n' + ) + assert envelope._last_error_event(raw) == { + "type": "error", + "error": {"name": "Last"}, + } + assert envelope._last_error_event(b"\xff") is None + + +@pytest.mark.parametrize( + ("data", "expected", "malformed"), + [ + ({"detail": {"phase": "direct"}}, {"phase": "direct"}, False), + ({"error_detail": {"phase": "legacy"}}, {"phase": "legacy"}, False), + ( + {"body": {"error": {"detail": {"phase": "mapping"}}}}, + {"phase": "mapping"}, + False, + ), + ( + {"response_body": '{"error":{"detail":{"phase":"json"}}}'}, + {"phase": "json"}, + False, + ), + ({"responseBody": "[]"}, {}, True), + ({"responseBody": "not-json"}, {}, True), + ({"responseBody": "\ud800"}, {}, True), + ({"responseBody": "x" * (envelope.MAX_GATEWAY_BODY_BYTES + 1)}, {}, True), + ({"responseBody": None}, {}, True), + ({"responseBody": True}, {}, True), + ({"responseBody": 503}, {}, True), + ({"body": []}, {}, True), + ({"body": {}}, {}, False), + ], +) +def test_gateway_detail_accepts_only_known_bounded_shapes( + data: dict[str, object], expected: dict[str, object], malformed: bool +) -> None: + """Only canonical detail containers are available to the formatter.""" + assert envelope._gateway_detail(data) == (expected, malformed) + + +def test_gateway_detail_fails_closed_on_excessive_json_depth() -> None: + """Deep provider envelopes cannot crash diagnostics with RecursionError.""" + deeply_nested = "[" * 10_000 + "0" + "]" * 10_000 + + assert envelope._gateway_detail({"responseBody": deeply_nested}) == ({}, True) + + +def test_gateway_detail_rejects_response_body_over_16_kib() -> None: + """The canonical gateway response-body parse budget is exactly 16 KiB.""" + body = json.dumps({"detail": {"padding": "x" * 16_384}}) + assert 16_384 < len(body.encode("utf-8")) < 32_768 + + assert envelope._gateway_detail({"responseBody": body}) == ({}, True) + + +def test_gateway_detail_rejects_oversized_mapping_body() -> None: + """Dictionary gateway bodies obey the same 16 KiB input boundary.""" + body = {"detail": {"padding": "x" * envelope.MAX_GATEWAY_BODY_BYTES}} + + assert envelope._gateway_detail({"responseBody": body}) == ({}, True) + + +def test_gateway_detail_rejects_unencodable_mapping_body() -> None: + """Mapping bodies that cannot produce bounded UTF-8 JSON fail closed.""" + body = {"detail": {"value": "\ud800"}} + + assert envelope._gateway_detail({"responseBody": body}) == ({}, True) + + +def test_gateway_details_rejects_any_malformed_alias() -> None: + """One malformed body alias invalidates the combined gateway authority.""" + assert envelope._gateway_details( + {"responseBody": {}, "body": []} + ) == ((), True) + + +@pytest.mark.parametrize( + "body_value", + ["not-json", None, True, 503, []], + ids=["invalid-json", "null", "boolean", "integer", "array"], +) +@pytest.mark.parametrize( + "outer_authority", + [ + {"code": "provider_unavailable"}, + {"statusCode": 503}, + ], + ids=["reason", "status"], +) +def test_malformed_gateway_body_suppresses_outer_causal_authority( + tmp_path: Path, + body_value: object, + outer_authority: dict[str, object], +) -> None: + """A malformed canonical body cannot publish an outer reason or status.""" + json_path = tmp_path / "event.jsonl" + stderr_path = tmp_path / "stderr" + json_path.write_text( + json.dumps( + { + "type": "error", + "error": { + "data": { + "responseBody": body_value, + **outer_authority, + } + }, + } + ) + + "\n", + encoding="utf-8", + ) + stderr_path.write_text("", encoding="utf-8") + + rendered = envelope.format_failure_metadata(json_path, stderr_path, 1) + + assert "class=malformed-response" in rendered + assert "reason=malformed_response" in rendered + assert "http-status=unknown" in rendered + assert "class=provider-5xx" not in rendered + + +def test_format_failure_metadata_rejects_conflicting_gateway_aliases( + tmp_path: Path, +) -> None: + """Conflicting validated causes across body aliases fail closed.""" + json_path = tmp_path / "event.jsonl" + stderr_path = tmp_path / "stderr" + json_path.write_text( + json.dumps( + { + "type": "error", + "error": { + "data": { + "responseBody": { + "error": { + "detail": { + "terminal_reason": "payment_required", + "attempts": [{"provider_status": 402}], + } + } + }, + "body": { + "error": { + "detail": { + "terminal_reason": "provider_unavailable", + "attempts": [{"provider_status": 503}], + } + } + }, + } + }, + } + ) + + "\n", + encoding="utf-8", + ) + stderr_path.write_text("", encoding="utf-8") + + rendered = envelope.format_failure_metadata(json_path, stderr_path, 1) + + assert "class=provider-error" in rendered + assert "reason=unknown" in rendered + assert "http-status=unknown" in rendered + assert "class=credit-exhausted" not in rendered + assert "class=provider-5xx" not in rendered + + +def test_last_error_event_fails_closed_on_excessive_json_depth() -> None: + """Deep top-level JSONL events cannot crash failure diagnostics.""" + deeply_nested = ( + '{"type":"error","error":{"data":' + + "[" * 5_000 + + "0" + + "]" * 5_000 + + "}}\n" + ).encode("utf-8") + + assert len(deeply_nested) < envelope.MAX_FAILURE_FILE_BYTES + assert envelope._last_error_event(deeply_nested) is None + + +@pytest.mark.parametrize( + ("has_json", "has_stderr", "status", "reason", "malformed", "event", "expected"), + [ + (False, False, 413, None, False, True, "request-too-large"), + (False, False, None, "request_too_large", False, True, "request-too-large"), + (False, False, None, "context_overflow", False, True, "context-window"), + (False, False, 402, None, False, True, "credit-exhausted"), + (False, False, None, "insufficient_quota", False, True, "quota-or-budget"), + (False, False, None, "no_eligible_route", False, True, "model-pool-exhausted"), + (False, False, None, "model_not_found", False, True, "model-unavailable"), + (False, False, 429, None, False, True, "rate-limit"), + (False, False, 403, None, False, True, "authentication-or-permission"), + (False, False, None, "timeout", False, True, "timeout"), + (False, False, 502, None, False, True, "provider-5xx"), + (False, False, 502, "payment_required", False, True, "provider-error"), + (True, False, None, None, True, True, "malformed-response"), + (True, False, None, None, False, False, "malformed-response"), + (True, False, None, None, False, True, "provider-error"), + (False, True, None, None, False, False, "provider-error"), + (False, False, None, None, False, False, "no-provider-detail"), + ], +) +def test_failure_class_preserves_distinct_safe_causes( + has_json: bool, + has_stderr: bool, + status: int | None, + reason: str | None, + malformed: bool, + event: bool, + expected: str, +) -> None: + """Each accepted causal category remains distinguishable.""" + assert ( + envelope._failure_class( + status=status, + reason=reason, + malformed_body=malformed, + has_json_artifact=has_json, + has_stderr_artifact=has_stderr, + has_event=event, + ) + == expected + ) + + +def test_format_failure_metadata_handles_direct_detail_and_string_status( + tmp_path: Path, +) -> None: + """Direct gateway detail fields produce one deterministic safe line.""" + json_path = tmp_path / "event.jsonl" + stderr_path = tmp_path / "stderr" + json_path.write_text( + json.dumps( + { + "type": "error", + "error": { + "name": "HTTPError", + "data": { + "status_code": "503", + "detail": { + "phase": "response_error", + "stop_reason": "provider_unavailable", + "model": "nvidia/model:free", + "attempts": [ + { + "provider": "nvidia_nim", + "phase": "connecting", + } + ], + }, + }, + }, + } + ) + + "\n", + encoding="utf-8", + ) + stderr_path.write_text("", encoding="utf-8") + + rendered = envelope.format_failure_metadata(json_path, stderr_path, 5) + + assert "class=provider-5xx" in rendered + assert "phase=connecting" in rendered + assert "reason=provider_unavailable" in rendered + assert "provider=unknown" in rendered + assert "http-status=503" in rendered + assert "exception=unknown" in rendered + assert "duration-seconds=5" in rendered + assert "served-model=unknown" in rendered + + +def test_format_failure_metadata_limits_attempts_and_defaults_fields( + tmp_path: Path, +) -> None: + """Oversized attempt arrays and unsafe scalars degrade to explicit absence.""" + secret = "github" + "_pat_" + "NEVERPRINTTHISVALUE123456" + json_path = tmp_path / "event.jsonl" + stderr_path = tmp_path / "stderr" + json_path.write_text( + json.dumps( + { + "type": "error", + "error": { + "name": f"bad exception {secret}", + "data": { + "code": f"unsafe code {secret}", + "detail": { + "error_code": f"unsafe reason {secret}", + "attempts": [{}] * 65, + "model": f"unsafe model {secret}", + }, + }, + }, + } + ), + encoding="utf-8", + ) + stderr_path.write_text("", encoding="utf-8") + + rendered = envelope.format_failure_metadata(json_path, stderr_path, -4) + + assert "phase=unknown reason=provider_error provider=unknown" in rendered + assert "http-status=unknown exception=unknown duration-seconds=0" in rendered + assert "served-model=unknown" in rendered + assert secret not in rendered + + +def test_format_failure_metadata_rejects_credential_shaped_tokens( + tmp_path: Path, +) -> None: + """Structured identifiers cannot smuggle credential-shaped values into logs.""" + secret = "github" + "_pat_" + "NEVERPRINTTHISVALUE123456" + json_path = tmp_path / "event.jsonl" + stderr_path = tmp_path / "stderr" + json_path.write_text( + json.dumps( + { + "type": "error", + "error": { + "name": secret, + "data": { + "detail": { + "phase": secret, + "terminal_reason": secret, + "model": secret, + "attempts": [{"provider_name": secret}], + } + }, + }, + } + ), + encoding="utf-8", + ) + stderr_path.write_text("", encoding="utf-8") + + rendered = envelope.format_failure_metadata(json_path, stderr_path, 1) + + assert secret not in rendered + assert "phase=unknown" in rendered + assert "reason=provider_error" in rendered + assert "provider=unknown" in rendered + assert "exception=unknown" in rendered + assert "served-model=unknown" in rendered + + +def test_format_failure_metadata_rejects_unproven_identifier_provenance( + tmp_path: Path, +) -> None: + """Lexically safe unknown identifiers cannot become public diagnostics.""" + secret = "BYTEZ" + "_TEST_SECRET_1234567890" + json_path = tmp_path / "event.jsonl" + stderr_path = tmp_path / "stderr" + json_path.write_text( + json.dumps( + { + "type": "error", + "error": { + "name": secret, + "data": { + "detail": { + "phase": secret, + "terminal_reason": secret, + "model": secret, + "attempts": [{"provider_name": secret}], + } + }, + }, + } + ), + encoding="utf-8", + ) + stderr_path.write_text("", encoding="utf-8") + + rendered = envelope.format_failure_metadata(json_path, stderr_path, 1) + + assert secret not in rendered + assert "phase=unknown reason=provider_error provider=unknown" in rendered + assert "exception=unknown" in rendered + assert "served-model=unknown" in rendered + + +def test_format_failure_metadata_ignores_provider_prose_for_causal_class( + tmp_path: Path, +) -> None: + """Untrusted event prose cannot override the structured gateway cause.""" + json_path = tmp_path / "event.jsonl" + stderr_path = tmp_path / "stderr" + json_path.write_text( + json.dumps({"type": "text", "text": "payment required; rate limit; timeout"}) + + "\n" + + json.dumps( + { + "type": "error", + "error": { + "name": "HTTPError", + "data": { + "statusCode": 502, + "detail": {"terminal_reason": "provider_unavailable"}, + "message": "payment required", + }, + }, + } + ) + + "\n", + encoding="utf-8", + ) + stderr_path.write_text("authentication failed", encoding="utf-8") + + rendered = envelope.format_failure_metadata(json_path, stderr_path, 1) + + assert "class=provider-5xx" in rendered + assert "reason=provider_unavailable" in rendered + assert "class=credit-exhausted" not in rendered + assert "class=authentication-or-permission" not in rendered + + +def test_main_prints_metadata_and_rejects_invalid_arguments( + tmp_path: Path, capsys: pytest.CaptureFixture[str], monkeypatch: pytest.MonkeyPatch +) -> None: + """The CLI has one strict invocation shape and delegates to the formatter.""" + json_path = tmp_path / "event.jsonl" + stderr_path = tmp_path / "stderr" + json_path.write_text("", encoding="utf-8") + stderr_path.write_text("", encoding="utf-8") + assert envelope.main([str(json_path), str(stderr_path), "0"]) == 0 + assert "class=no-provider-detail" in capsys.readouterr().out + + assert envelope.main([str(json_path), str(stderr_path), "-1"]) == 2 + assert "usage:" in capsys.readouterr().err + monkeypatch.setattr( + envelope.sys, + "argv", + ["opencode_failure_envelope.py", str(json_path), str(stderr_path), "1"], + ) + assert envelope.main() == 0 + + +def test_format_failure_metadata_rejects_conflicting_status_authorities( + tmp_path: Path, +) -> None: + """Conflicting validated gateway statuses cannot select a public cause.""" + json_path = tmp_path / "event.jsonl" + stderr_path = tmp_path / "stderr" + json_path.write_text( + json.dumps( + { + "type": "error", + "error": { + "data": { + "statusCode": 429, + "status_code": 502, + } + }, + } + ) + + "\n", + encoding="utf-8", + ) + stderr_path.write_text("", encoding="utf-8") + + rendered = envelope.format_failure_metadata(json_path, stderr_path, 1) + + assert "class=provider-error" in rendered + assert "reason=unknown" in rendered + assert "http-status=unknown" in rendered + assert "class=rate-limit" not in rendered + assert "class=provider-5xx" not in rendered + + +def test_format_failure_metadata_rejects_conflicting_reason_authorities( + tmp_path: Path, +) -> None: + """Conflicting validated reason fields cannot select a public cause.""" + json_path = tmp_path / "event.jsonl" + stderr_path = tmp_path / "stderr" + json_path.write_text( + json.dumps( + { + "type": "error", + "error": { + "data": { + "detail": { + "terminal_reason": "payment_required", + "error_code": "provider_unavailable", + } + } + }, + } + ) + + "\n", + encoding="utf-8", + ) + stderr_path.write_text("", encoding="utf-8") + + rendered = envelope.format_failure_metadata(json_path, stderr_path, 1) + + assert "class=provider-error" in rendered + assert "reason=unknown" in rendered + assert "http-status=unknown" in rendered + assert "class=credit-exhausted" not in rendered + assert "class=provider-5xx" not in rendered + + +def test_format_failure_metadata_rejects_cross_family_authority_conflict( + tmp_path: Path, +) -> None: + """A validated status and reason must resolve to compatible causal classes.""" + json_path = tmp_path / "event.jsonl" + stderr_path = tmp_path / "stderr" + json_path.write_text( + json.dumps( + { + "type": "error", + "error": { + "data": { + "statusCode": 502, + "detail": {"terminal_reason": "payment_required"}, + } + }, + } + ) + + "\n", + encoding="utf-8", + ) + stderr_path.write_text("", encoding="utf-8") + + rendered = envelope.format_failure_metadata(json_path, stderr_path, 1) + + assert "class=provider-error" in rendered + assert "reason=unknown" in rendered + assert "http-status=unknown" in rendered + + +def test_gateway_detail_rejects_recursive_mapping_body() -> None: + """A cyclic mapping fails closed before causal fields can be inspected.""" + body: dict[str, object] = {} + body["self"] = body + + assert envelope._gateway_detail({"responseBody": body}) == ({}, True) + + +def test_gateway_details_rejects_malformed_alias() -> None: + """One malformed body alias invalidates the complete authority set.""" + data = { + "responseBody": {"detail": {"phase": "provider_request"}}, + "body": [], + } + + assert envelope._gateway_details(data) == ((), True) + + +@pytest.mark.parametrize( + "error", + [ + None, + "not-an-object", + [], + {"data": None}, + {"data": "not-an-object"}, + {"data": []}, + ], + ids=[ + "null-error", + "string-error", + "list-error", + "null-data", + "string-data", + "list-data", + ], +) +def test_present_malformed_error_containers_fail_closed( + tmp_path: Path, error: object +) -> None: + """Present non-object error containers remain malformed-response authority.""" + json_path = tmp_path / "event.jsonl" + stderr_path = tmp_path / "stderr" + json_path.write_text( + json.dumps({"type": "error", "error": error}) + "\n", + encoding="utf-8", + ) + stderr_path.write_text("", encoding="utf-8") + + rendered = envelope.format_failure_metadata(json_path, stderr_path, 1) + + assert "class=malformed-response" in rendered + assert "reason=malformed_response" in rendered + assert "http-status=unknown" in rendered + + +def test_gateway_status_and_provider_status_keep_separate_layers(tmp_path: Path) -> None: + """A gateway 502 may wrap an upstream 503 without erasing the structured cause.""" + json_path = tmp_path / "event.jsonl" + stderr_path = tmp_path / "stderr" + json_path.write_text( + json.dumps( + { + "type": "error", + "error": { + "data": { + "statusCode": 502, + "detail": { + "terminal_reason": "eligible_candidates_exhausted", + "attempts": [{"provider_status": 503}], + }, + } + }, + } + ) + + "\n", + encoding="utf-8", + ) + stderr_path.write_text("", encoding="utf-8") + + rendered = envelope.format_failure_metadata(json_path, stderr_path, 1) + + assert "class=model-pool-exhausted" in rendered + assert "reason=eligible_candidates_exhausted" in rendered + assert "http-status=502" in rendered + + +@pytest.mark.parametrize( + ("status", "reason", "expected_class"), + [ + (503, "eligible_candidates_exhausted", "model-pool-exhausted"), + (504, "provider_timeout", "timeout"), + (503, "model_not_found", "model-unavailable"), + ], +) +def test_specific_reason_refines_compatible_generic_5xx( + tmp_path: Path, + status: int, + reason: str, + expected_class: str, +) -> None: + """Allowlisted terminal reasons refine only compatible generic 5xx statuses.""" + json_path = tmp_path / "event.jsonl" + stderr_path = tmp_path / "stderr" + json_path.write_text( + json.dumps( + { + "type": "error", + "error": { + "data": { + "statusCode": status, + "detail": {"terminal_reason": reason}, + } + }, + } + ) + + "\n", + encoding="utf-8", + ) + stderr_path.write_text("", encoding="utf-8") + + rendered = envelope.format_failure_metadata(json_path, stderr_path, 1) + + assert f"class={expected_class}" in rendered + assert f"reason={reason}" in rendered + assert f"http-status={status}" in rendered diff --git a/tests/test_opencode_model_pool_runner.py b/tests/test_opencode_model_pool_runner.py index 2965d4c55c..9261a60ccc 100644 --- a/tests/test_opencode_model_pool_runner.py +++ b/tests/test_opencode_model_pool_runner.py @@ -33,6 +33,20 @@ } +def test_confidentiality_fixture_uses_one_exact_scanner_classification() -> None: + """The synthetic credential is classified only on its two regression paths.""" + scanner_secret = "BYTEZ" + "_TEST_SECRET_1234567890" + source = Path(__file__).read_text(encoding="utf-8") + gitleaks_config = (ROOT / ".gitleaks.toml").read_text(encoding="utf-8") + assert scanner_secret not in source + assert scanner_secret not in gitleaks_config + assert gitleaks_config.count("BYTEZ_TEST_SECRET_[1]234567890") == 1 + assert 'condition = "AND"' in gitleaks_config + assert 'regexTarget = "match"' in gitleaks_config + assert "tests/test_opencode_failure_envelope\\.py$" in gitleaks_config + assert "tests/test_opencode_model_pool_runner\\.py$" in gitleaks_config + + def bash_command() -> str: """Return a Bash executable that can run repository shell scripts locally.""" if os.name == "nt": @@ -373,7 +387,7 @@ def test_failed_provider_logs_bounded_reason_and_redacts_credentials( json_line=( '{"type":"error","error":{"name":"ProviderAuthError","data":' f'{{"message":"HTTP 401 authorization Bearer {fake_bearer_token}; ' - f'api_key={fake_openai_token}"' + "}}}" + f'api_key={fake_openai_token}","statusCode":401' + "}}}" ), stderr_line=( f"request failed token={fake_github_token} because provider " @@ -389,7 +403,7 @@ def test_failed_provider_logs_bounded_reason_and_redacts_credentials( assert "json-bytes=" in result.stdout assert "stderr-bytes=" in result.stdout assert "provider-controlled content suppressed" in result.stdout - assert "ProviderAuthError" not in result.stdout + assert "exception=unknown" in result.stdout assert "request failed" not in result.stdout assert fake_bearer_token not in result.stdout assert fake_openai_token not in result.stdout @@ -402,10 +416,302 @@ def test_failed_provider_without_reason_logs_explicit_absence(tmp_path: Path) -> result = run_failed_model(tmp_path) assert result.returncode == 1 - assert ( - "OpenCode provider failure metadata: class=no-provider-detail " - "json-bytes=0 stderr-bytes=0; provider-controlled content suppressed." - ) in result.stdout + assert "class=no-provider-detail json-bytes=0 stderr-bytes=0" in result.stdout + assert "phase=unknown reason=no_provider_detail provider=unknown" in result.stdout + assert "http-status=unknown exception=unknown" in result.stdout + assert "served-model=unknown" in result.stdout + assert "provider-controlled content suppressed" in result.stdout + + +@pytest.mark.parametrize( + ("status", "terminal_reason", "expected_class"), + [ + (429, "queue_capacity", "rate-limit"), + (503, "provider_unavailable", "provider-5xx"), + ], + ids=["queue-capacity-429", "provider-503"], +) +def test_failed_gateway_response_emits_bounded_route_metadata( + tmp_path: Path, + status: int, + terminal_reason: str, + expected_class: str, +) -> None: + """Canonical gateway failures retain only safe causal route fields.""" + secret = "sk" + "-gateway-body-must-not-leak" + response_body = json.dumps( + { + "error": { + "detail": { + "model": "openrouter/deepseek-r1:free", + "terminal_reason": terminal_reason, + "attempts": [ + { + "provider_name": "openrouter", + "phase": "queue_admission", + "provider_status": status, + "secret": secret, + } + ], + }, + "message": secret, + } + } + ) + result = run_failed_model( + tmp_path, + json_line=json.dumps( + { + "type": "error", + "error": { + "name": "AI_APICallError", + "data": { + "statusCode": status, + "responseBody": response_body, + "message": secret, + }, + }, + } + ), + ) + + assert result.returncode == 1 + assert f"class={expected_class}" in result.stdout + assert "phase=queue_admission" in result.stdout + assert f"reason={terminal_reason}" in result.stdout + assert "provider=unknown" in result.stdout + assert f"http-status={status}" in result.stdout + assert "exception=unknown" in result.stdout + assert re.search(r"duration-seconds=\d+", result.stdout) + assert "served-model=unknown" in result.stdout + assert secret not in result.stdout + result.stderr + + +def test_failed_gateway_malformed_body_is_explicit_and_redacted(tmp_path: Path) -> None: + """A non-JSON gateway body reports malformed metadata without echoing it.""" + secret = "malformed-" + "provider-body-secret" + result = run_failed_model( + tmp_path, + json_line=json.dumps( + { + "type": "error", + "error": { + "name": "AI_APICallError", + "data": {"responseBody": f"not-json {secret}"}, + }, + } + ), + ) + + assert result.returncode == 1 + assert "class=malformed-response" in result.stdout + assert "phase=unknown" in result.stdout + assert "reason=malformed_response" in result.stdout + assert "provider=unknown" in result.stdout + assert "http-status=unknown" in result.stdout + assert "exception=unknown" in result.stdout + assert "served-model=unknown" in result.stdout + assert secret not in result.stdout + result.stderr + + +def test_failed_gateway_request_too_large_keeps_admission_cause(tmp_path: Path) -> None: + """HTTP 413 remains distinct from provider transport and model exhaustion.""" + response_body = json.dumps( + { + "error": { + "detail": { + "terminal_reason": "request_too_large", + "attempts": [{"phase": "request_admission", "provider_status": 413}], + } + } + } + ) + result = run_failed_model( + tmp_path, + json_line=json.dumps( + { + "type": "error", + "error": { + "name": "AI_APICallError", + "data": {"statusCode": 413, "responseBody": response_body}, + }, + } + ), + ) + + assert result.returncode == 1 + assert "class=request-too-large" in result.stdout + assert "phase=request_admission" in result.stdout + assert "reason=request_too_large" in result.stdout + assert "http-status=413" in result.stdout + + +def test_failed_gateway_pool_exhaustion_keeps_terminal_reason(tmp_path: Path) -> None: + """No eligible free route is distinguishable from a malformed response.""" + response_body = json.dumps( + { + "error": { + "detail": { + "terminal_reason": "eligible_candidates_exhausted", + "attempts": [{"phase": "route_selection"}], + } + } + } + ) + result = run_failed_model( + tmp_path, + json_line=json.dumps( + { + "type": "error", + "error": { + "name": "ProviderUpstreamError", + "data": {"responseBody": response_body}, + }, + } + ), + ) + + assert result.returncode == 1 + assert "class=model-pool-exhausted" in result.stdout + assert "phase=route_selection" in result.stdout + assert "reason=eligible_candidates_exhausted" in result.stdout + assert "provider=unknown" in result.stdout + assert "http-status=unknown" in result.stdout + + +def test_failed_gateway_missing_model_is_an_explicit_unknown(tmp_path: Path) -> None: + """The adapter never invents a served model when the gateway omits it.""" + response_body = json.dumps( + {"error": {"detail": {"terminal_reason": "provider_unavailable"}}} + ) + result = run_failed_model( + tmp_path, + json_line=json.dumps( + { + "type": "error", + "error": { + "name": "AI_APICallError", + "data": {"statusCode": 502, "responseBody": response_body}, + }, + } + ), + ) + + assert result.returncode == 1 + assert "served-model=unknown" in result.stdout + assert "reason=provider_unavailable" in result.stdout + + +def test_failed_gateway_ignores_unsafe_metadata_tokens(tmp_path: Path) -> None: + """Unsafe nested metadata and provider prose never enter public diagnostics.""" + secret = "github" + "_pat_" + "FAILUREENVELOPESECRET123456" + response_body = json.dumps( + { + "error": { + "detail": { + "model": f"unsafe model {secret}", + "terminal_reason": f"unsafe reason {secret}", + "attempts": [ + { + "provider_name": f"unsafe provider {secret}", + "phase": f"unsafe phase {secret}", + } + ], + }, + "message": secret, + }, + "arbitrary": secret, + } + ) + result = run_failed_model( + tmp_path, + json_line=json.dumps( + { + "type": "error", + "error": { + "name": f"Unsafe Exception {secret}", + "data": {"responseBody": response_body, "message": secret}, + }, + } + ), + ) + + assert result.returncode == 1 + assert "phase=unknown" in result.stdout + assert "reason=provider_error" in result.stdout + assert "provider=unknown" in result.stdout + assert "exception=unknown" in result.stdout + assert "served-model=unknown" in result.stdout + assert secret not in result.stdout + result.stderr + + +def test_failed_gateway_rejects_unproven_identifier_provenance( + tmp_path: Path, +) -> None: + """Lexically safe unknown identifiers cannot become public diagnostics.""" + secret = "BYTEZ" + "_TEST_SECRET_1234567890" + response_body = json.dumps( + { + "error": { + "detail": { + "model": secret, + "terminal_reason": secret, + "attempts": [{"provider_name": secret, "phase": secret}], + } + } + } + ) + result = run_failed_model( + tmp_path, + json_line=json.dumps( + { + "type": "error", + "error": { + "name": secret, + "data": {"responseBody": response_body}, + }, + } + ), + ) + + assert result.returncode == 1 + assert "phase=unknown reason=provider_error provider=unknown" in result.stdout + assert "exception=unknown" in result.stdout + assert "served-model=unknown" in result.stdout + assert secret not in result.stdout + result.stderr + + +def test_failed_gateway_rejects_response_body_over_16_kib(tmp_path: Path) -> None: + """Oversized gateway bodies fail closed through the production launcher.""" + safe_model = "openrouter/model-that-must-not-survive" + response_body = json.dumps( + { + "error": { + "detail": { + "model": safe_model, + "padding": "x" * 16_384, + } + } + } + ) + assert 16_384 < len(response_body.encode("utf-8")) < 32_768 + result = run_failed_model( + tmp_path, + json_line=json.dumps( + { + "type": "error", + "error": { + "name": "AI_APICallError", + "data": {"responseBody": response_body}, + }, + } + ), + ) + + assert result.returncode == 1 + assert "class=malformed-response" in result.stdout + assert "served-model=unknown" in result.stdout + assert safe_model not in result.stdout def test_backoff_environment_rejects_recursive_arithmetic_injection( @@ -622,13 +928,14 @@ def test_model_text_quoting_error_signatures_does_not_kill_run(tmp_path: Path) - def test_delisted_openrouter_model_error_kills_hung_run_early(tmp_path: Path) -> None: - """A delisted pinned OpenRouter model dies seconds after a model-unavailable error.""" + """A delisting signal stops the run and its structured reason names the cause.""" start = time.monotonic() result = run_failed_model( tmp_path, json_line=( '{"type":"error","error":{"name":"ProviderModelNotFoundError","data":' - '{"message":"No endpoints found for nvidia/nemotron-3-ultra-550b-a55b:free."}}}' + '{"message":"No endpoints found for nvidia/nemotron-3-ultra-550b-a55b:free.",' + '"detail":{"terminal_reason":"model_not_found"}}}}' ), model_candidates="openrouter/nvidia/nemotron-3-ultra-550b-a55b:free", extra_env={ diff --git a/tests/test_pr_review_autofix_nvidia_nim_contract.py b/tests/test_pr_review_autofix_nvidia_nim_contract.py index 2e733ac9e9..d69449e2d5 100644 --- a/tests/test_pr_review_autofix_nvidia_nim_contract.py +++ b/tests/test_pr_review_autofix_nvidia_nim_contract.py @@ -17,7 +17,7 @@ DOCTORING_RECORD = Path("docs/doctoring/hourly-nvidia-nim-autofix.md") CHANGELOG = Path("CHANGELOG.md") REVIEW_DISPATCH_WORKFLOW = Path(".github/workflows/opencode-review-dispatch.yml") -REVIEW_DISPATCH_BLOB_SHA = "d86497b3f43bebbabbb4f504eb5132cdf3b7b293" +REVIEW_DISPATCH_BLOB_SHA = "d8c677b874c06181f11527301dfc111c02f80d5b" def _workflow_text(path: Path) -> str: