Skip to content

fix(gate): harden runtime review fallbacks #2928

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
stranske merged 7 commits into from
Aug 4, 2026
Merged

fix(gate): harden runtime review fallbacks #2928

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
84879ce
fix(gate): harden runtime review fallbacks
codex-automation Aug 4, 2026
76e16d6
style: format verifier fallback test
codex-automation Aug 4, 2026
d72163d
fix(gate): normalize pytest runtime before comparison
codex-automation Aug 4, 2026
c9d39ec
fix(gate): normalize wrapped commands in Actions
codex-automation Aug 4, 2026
2efd2e2
fix(gate): reject wrapper-owned dependency repair
codex-automation Aug 4, 2026
57d8acf
fix(gate): bind repairs to active interpreter
codex-automation Aug 4, 2026
66b6106
fix(gate): reject unmanaged YAML tracebacks
codex-automation Aug 4, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
147 changes: 77 additions & 70 deletions scripts/check_deliberate_break.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -203,6 +203,29 @@ def _ensure_pytest_runtime_deps() -> None:
raise error from (import_error or retry_error)


def _pyyaml_runtime_needs_repair() -> bool:
"""Return whether the active PyYAML runtime is missing, stale, or unusable."""
try:
installed_version = metadata.version("PyYAML")
except metadata.PackageNotFoundError:
return True
if installed_version != PYYAML_VERSION:
return True
Comment thread
stranske marked this conversation as resolved.
try:
import_module("yaml")
except Exception:
return True
return False


def _uses_pytest_runtime(command: tuple[str, ...]) -> bool:
"""Return whether a command runs pytest in the active Python environment."""
if len(command) < 3 or command[1:3] != ("-m", "pytest"):
return False
executable = shutil.which(command[0]) or command[0]
return Path(executable).resolve() == Path(sys.executable).resolve()


def _run(
command: tuple[str, ...],
cwd: Path,
Expand All @@ -229,6 +252,17 @@ def _run_with_runtime_deps(
cwd: Path,
) -> subprocess.CompletedProcess[str]:
"""Retry a command after repairing PyYAML only when its output requires it."""
managed_runtime = _uses_pytest_runtime(command)
if managed_runtime and _pyyaml_runtime_needs_repair():
try:
_ensure_pytest_runtime_deps()
except (
subprocess.TimeoutExpired,
subprocess.CalledProcessError,
ImportError,
OSError,
) as exc:
raise RuntimeDependencyError(exc) from exc
try:
completed = _run(command, cwd)
except OSError as exc:
Expand All @@ -248,13 +282,18 @@ def _run_with_runtime_deps(
)
yaml_traceback = bool(re.search(r"(?:^|[/\\])yaml[/\\][^\n]*", output, re.MULTILINE))
if yaml_traceback and not missing_pyyaml:
try:
import_module("yaml")
except Exception:
missing_pyyaml = True
missing_pyyaml = True if not managed_runtime else _pyyaml_runtime_needs_repair()
if not missing_pyyaml:
return completed

if not managed_runtime:
error = ImportError(
"PyYAML failed inside a wrapped or custom deliberate-break command; "
"automatic repair is disabled because the wrapper may use a different "
"Python environment"
)
raise RuntimeDependencyError(error) from error

try:
_ensure_pytest_runtime_deps()
except (subprocess.TimeoutExpired, subprocess.CalledProcessError, ImportError, OSError) as exc:
Expand All @@ -265,6 +304,38 @@ def _run_with_runtime_deps(
raise CommandUnavailableError(exc) from exc


def _runtime_dependency_error_result(error: Exception) -> dict[str, object]:
"""Map dependency-repair failures consistently for head and base runs."""
if isinstance(error, subprocess.TimeoutExpired):
return _json_result(
VERDICT_BROKEN,
reason="command-timeout",
command=list(error.cmd) if isinstance(error.cmd, (tuple, list)) else str(error.cmd),
timeout=error.timeout,
)
if isinstance(error, subprocess.CalledProcessError):
return _json_result(
VERDICT_BROKEN,
reason="dependency-install-failed",
command=list(error.cmd) if isinstance(error.cmd, (tuple, list)) else str(error.cmd),
returncode=error.returncode,
stdout=error.stdout,
stderr=error.stderr,
)
if isinstance(error, ImportError):
return _json_result(
VERDICT_BROKEN,
reason="dependency-import-failed",
detail=str(error),
cause=str(error.__cause__) if error.__cause__ is not None else None,
)
return _json_result(
VERDICT_BROKEN,
reason="dependency-install-unavailable",
detail=str(error),
)


def _git(
args: list[str],
cwd: Path,
Expand Down Expand Up @@ -385,39 +456,7 @@ def verify_spec(
timeout=exc.timeout,
)
except RuntimeDependencyError as wrapped:
error = wrapped.error
if isinstance(error, subprocess.TimeoutExpired):
return _json_result(
VERDICT_BROKEN,
reason="command-timeout",
command=(
list(error.cmd) if isinstance(error.cmd, (tuple, list)) else str(error.cmd)
),
timeout=error.timeout,
)
if isinstance(error, subprocess.CalledProcessError):
return _json_result(
VERDICT_BROKEN,
reason="dependency-install-failed",
command=(
list(error.cmd) if isinstance(error.cmd, (tuple, list)) else str(error.cmd)
),
returncode=error.returncode,
stdout=error.stdout,
stderr=error.stderr,
)
if isinstance(error, ImportError):
return _json_result(
VERDICT_BROKEN,
reason="dependency-import-failed",
detail=str(error),
cause=str(error.__cause__) if error.__cause__ is not None else None,
)
return _json_result(
VERDICT_BROKEN,
reason="dependency-install-unavailable",
detail=str(error),
)
return _runtime_dependency_error_result(wrapped.error)
except CommandUnavailableError as wrapped:
return _json_result(
VERDICT_BROKEN,
Expand Down Expand Up @@ -458,39 +497,7 @@ def verify_spec(
detail=str(exc),
)
except RuntimeDependencyError as wrapped:
error = wrapped.error
if isinstance(error, subprocess.TimeoutExpired):
return _json_result(
VERDICT_BROKEN,
reason="command-timeout",
command=(
list(error.cmd) if isinstance(error.cmd, (tuple, list)) else str(error.cmd)
),
timeout=error.timeout,
)
if isinstance(error, subprocess.CalledProcessError):
return _json_result(
VERDICT_BROKEN,
reason="dependency-install-failed",
command=(
list(error.cmd) if isinstance(error.cmd, (tuple, list)) else str(error.cmd)
),
returncode=error.returncode,
stdout=error.stdout,
stderr=error.stderr,
)
if isinstance(error, ImportError):
return _json_result(
VERDICT_BROKEN,
reason="dependency-import-failed",
detail=str(error),
cause=str(error.__cause__) if error.__cause__ is not None else None,
)
return _json_result(
VERDICT_BROKEN,
reason="dependency-install-unavailable",
detail=str(error),
)
return _runtime_dependency_error_result(wrapped.error)
except CommandUnavailableError as wrapped:
return _json_result(
VERDICT_BROKEN,
Expand Down
7 changes: 5 additions & 2 deletions scripts/langchain/pr_verifier.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -705,8 +705,11 @@ def _coerce_response_content(content: object) -> str:
return text
try:
return json.dumps(content, default=str)
except (TypeError, ValueError):
return str(content)
except Exception:
try:
return str(content)
except Exception:
return f"<unserializable {type(content).__name__}>"


def _parse_llm_response(
Expand Down
147 changes: 77 additions & 70 deletions templates/consumer-repo/scripts/check_deliberate_break.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -203,6 +203,29 @@ def _ensure_pytest_runtime_deps() -> None:
raise error from (import_error or retry_error)


def _pyyaml_runtime_needs_repair() -> bool:
"""Return whether the active PyYAML runtime is missing, stale, or unusable."""
try:
installed_version = metadata.version("PyYAML")
except metadata.PackageNotFoundError:
return True
if installed_version != PYYAML_VERSION:
return True
try:
import_module("yaml")
except Exception:
return True
return False


def _uses_pytest_runtime(command: tuple[str, ...]) -> bool:
"""Return whether a command runs pytest in the active Python environment."""
if len(command) < 3 or command[1:3] != ("-m", "pytest"):
return False
executable = shutil.which(command[0]) or command[0]
return Path(executable).resolve() == Path(sys.executable).resolve()


def _run(
command: tuple[str, ...],
cwd: Path,
Expand All @@ -229,6 +252,17 @@ def _run_with_runtime_deps(
cwd: Path,
) -> subprocess.CompletedProcess[str]:
"""Retry a command after repairing PyYAML only when its output requires it."""
managed_runtime = _uses_pytest_runtime(command)
if managed_runtime and _pyyaml_runtime_needs_repair():
try:
_ensure_pytest_runtime_deps()
except (
subprocess.TimeoutExpired,
subprocess.CalledProcessError,
ImportError,
OSError,
) as exc:
raise RuntimeDependencyError(exc) from exc
try:
completed = _run(command, cwd)
except OSError as exc:
Expand All @@ -248,13 +282,18 @@ def _run_with_runtime_deps(
)
yaml_traceback = bool(re.search(r"(?:^|[/\\])yaml[/\\][^\n]*", output, re.MULTILINE))
if yaml_traceback and not missing_pyyaml:
try:
import_module("yaml")
except Exception:
missing_pyyaml = True
missing_pyyaml = True if not managed_runtime else _pyyaml_runtime_needs_repair()
if not missing_pyyaml:
return completed

if not managed_runtime:
error = ImportError(
"PyYAML failed inside a wrapped or custom deliberate-break command; "
"automatic repair is disabled because the wrapper may use a different "
"Python environment"
)
raise RuntimeDependencyError(error) from error

try:
_ensure_pytest_runtime_deps()
except (subprocess.TimeoutExpired, subprocess.CalledProcessError, ImportError, OSError) as exc:
Expand All @@ -265,6 +304,38 @@ def _run_with_runtime_deps(
raise CommandUnavailableError(exc) from exc


def _runtime_dependency_error_result(error: Exception) -> dict[str, object]:
"""Map dependency-repair failures consistently for head and base runs."""
if isinstance(error, subprocess.TimeoutExpired):
return _json_result(
VERDICT_BROKEN,
reason="command-timeout",
command=list(error.cmd) if isinstance(error.cmd, (tuple, list)) else str(error.cmd),
timeout=error.timeout,
)
if isinstance(error, subprocess.CalledProcessError):
return _json_result(
VERDICT_BROKEN,
reason="dependency-install-failed",
command=list(error.cmd) if isinstance(error.cmd, (tuple, list)) else str(error.cmd),
returncode=error.returncode,
stdout=error.stdout,
stderr=error.stderr,
)
if isinstance(error, ImportError):
return _json_result(
VERDICT_BROKEN,
reason="dependency-import-failed",
detail=str(error),
cause=str(error.__cause__) if error.__cause__ is not None else None,
)
return _json_result(
VERDICT_BROKEN,
reason="dependency-install-unavailable",
detail=str(error),
)


def _git(
args: list[str],
cwd: Path,
Expand Down Expand Up @@ -385,39 +456,7 @@ def verify_spec(
timeout=exc.timeout,
)
except RuntimeDependencyError as wrapped:
error = wrapped.error
if isinstance(error, subprocess.TimeoutExpired):
return _json_result(
VERDICT_BROKEN,
reason="command-timeout",
command=(
list(error.cmd) if isinstance(error.cmd, (tuple, list)) else str(error.cmd)
),
timeout=error.timeout,
)
if isinstance(error, subprocess.CalledProcessError):
return _json_result(
VERDICT_BROKEN,
reason="dependency-install-failed",
command=(
list(error.cmd) if isinstance(error.cmd, (tuple, list)) else str(error.cmd)
),
returncode=error.returncode,
stdout=error.stdout,
stderr=error.stderr,
)
if isinstance(error, ImportError):
return _json_result(
VERDICT_BROKEN,
reason="dependency-import-failed",
detail=str(error),
cause=str(error.__cause__) if error.__cause__ is not None else None,
)
return _json_result(
VERDICT_BROKEN,
reason="dependency-install-unavailable",
detail=str(error),
)
return _runtime_dependency_error_result(wrapped.error)
except CommandUnavailableError as wrapped:
return _json_result(
VERDICT_BROKEN,
Expand Down Expand Up @@ -458,39 +497,7 @@ def verify_spec(
detail=str(exc),
)
except RuntimeDependencyError as wrapped:
error = wrapped.error
if isinstance(error, subprocess.TimeoutExpired):
return _json_result(
VERDICT_BROKEN,
reason="command-timeout",
command=(
list(error.cmd) if isinstance(error.cmd, (tuple, list)) else str(error.cmd)
),
timeout=error.timeout,
)
if isinstance(error, subprocess.CalledProcessError):
return _json_result(
VERDICT_BROKEN,
reason="dependency-install-failed",
command=(
list(error.cmd) if isinstance(error.cmd, (tuple, list)) else str(error.cmd)
),
returncode=error.returncode,
stdout=error.stdout,
stderr=error.stderr,
)
if isinstance(error, ImportError):
return _json_result(
VERDICT_BROKEN,
reason="dependency-import-failed",
detail=str(error),
cause=str(error.__cause__) if error.__cause__ is not None else None,
)
return _json_result(
VERDICT_BROKEN,
reason="dependency-install-unavailable",
detail=str(error),
)
return _runtime_dependency_error_result(wrapped.error)
except CommandUnavailableError as wrapped:
return _json_result(
VERDICT_BROKEN,
Expand Down
Loading
Loading