diff --git a/scripts/check_deliberate_break.py b/scripts/check_deliberate_break.py index fb59c026c..8dc461794 100644 --- a/scripts/check_deliberate_break.py +++ b/scripts/check_deliberate_break.py @@ -203,6 +203,29 @@ def _ensure_pytest_runtime_deps() -> None: raise error from (import_error or retry_error) +def _pyyaml_runtime_needs_repair() -> bool: + """Return whether the active PyYAML runtime is missing, stale, or unusable.""" + try: + installed_version = metadata.version("PyYAML") + except metadata.PackageNotFoundError: + return True + if installed_version != PYYAML_VERSION: + return True + try: + import_module("yaml") + except Exception: + return True + return False + + +def _uses_pytest_runtime(command: tuple[str, ...]) -> bool: + """Return whether a command runs pytest in the active Python environment.""" + if len(command) < 3 or command[1:3] != ("-m", "pytest"): + return False + executable = shutil.which(command[0]) or command[0] + return Path(executable).resolve() == Path(sys.executable).resolve() + + def _run( command: tuple[str, ...], cwd: Path, @@ -229,6 +252,17 @@ def _run_with_runtime_deps( cwd: Path, ) -> subprocess.CompletedProcess[str]: """Retry a command after repairing PyYAML only when its output requires it.""" + managed_runtime = _uses_pytest_runtime(command) + if managed_runtime and _pyyaml_runtime_needs_repair(): + try: + _ensure_pytest_runtime_deps() + except ( + subprocess.TimeoutExpired, + subprocess.CalledProcessError, + ImportError, + OSError, + ) as exc: + raise RuntimeDependencyError(exc) from exc try: completed = _run(command, cwd) except OSError as exc: @@ -248,13 +282,18 @@ def _run_with_runtime_deps( ) yaml_traceback = bool(re.search(r"(?:^|[/\\])yaml[/\\][^\n]*", output, re.MULTILINE)) if yaml_traceback and not missing_pyyaml: - try: - import_module("yaml") - except Exception: - missing_pyyaml = True + missing_pyyaml = True if not managed_runtime else _pyyaml_runtime_needs_repair() if not missing_pyyaml: return completed + if not managed_runtime: + error = ImportError( + "PyYAML failed inside a wrapped or custom deliberate-break command; " + "automatic repair is disabled because the wrapper may use a different " + "Python environment" + ) + raise RuntimeDependencyError(error) from error + try: _ensure_pytest_runtime_deps() except (subprocess.TimeoutExpired, subprocess.CalledProcessError, ImportError, OSError) as exc: @@ -265,6 +304,38 @@ def _run_with_runtime_deps( raise CommandUnavailableError(exc) from exc +def _runtime_dependency_error_result(error: Exception) -> dict[str, object]: + """Map dependency-repair failures consistently for head and base runs.""" + if isinstance(error, subprocess.TimeoutExpired): + return _json_result( + VERDICT_BROKEN, + reason="command-timeout", + command=list(error.cmd) if isinstance(error.cmd, (tuple, list)) else str(error.cmd), + timeout=error.timeout, + ) + if isinstance(error, subprocess.CalledProcessError): + return _json_result( + VERDICT_BROKEN, + reason="dependency-install-failed", + command=list(error.cmd) if isinstance(error.cmd, (tuple, list)) else str(error.cmd), + returncode=error.returncode, + stdout=error.stdout, + stderr=error.stderr, + ) + if isinstance(error, ImportError): + return _json_result( + VERDICT_BROKEN, + reason="dependency-import-failed", + detail=str(error), + cause=str(error.__cause__) if error.__cause__ is not None else None, + ) + return _json_result( + VERDICT_BROKEN, + reason="dependency-install-unavailable", + detail=str(error), + ) + + def _git( args: list[str], cwd: Path, @@ -385,39 +456,7 @@ def verify_spec( timeout=exc.timeout, ) except RuntimeDependencyError as wrapped: - error = wrapped.error - if isinstance(error, subprocess.TimeoutExpired): - return _json_result( - VERDICT_BROKEN, - reason="command-timeout", - command=( - list(error.cmd) if isinstance(error.cmd, (tuple, list)) else str(error.cmd) - ), - timeout=error.timeout, - ) - if isinstance(error, subprocess.CalledProcessError): - return _json_result( - VERDICT_BROKEN, - reason="dependency-install-failed", - command=( - list(error.cmd) if isinstance(error.cmd, (tuple, list)) else str(error.cmd) - ), - returncode=error.returncode, - stdout=error.stdout, - stderr=error.stderr, - ) - if isinstance(error, ImportError): - return _json_result( - VERDICT_BROKEN, - reason="dependency-import-failed", - detail=str(error), - cause=str(error.__cause__) if error.__cause__ is not None else None, - ) - return _json_result( - VERDICT_BROKEN, - reason="dependency-install-unavailable", - detail=str(error), - ) + return _runtime_dependency_error_result(wrapped.error) except CommandUnavailableError as wrapped: return _json_result( VERDICT_BROKEN, @@ -458,39 +497,7 @@ def verify_spec( detail=str(exc), ) except RuntimeDependencyError as wrapped: - error = wrapped.error - if isinstance(error, subprocess.TimeoutExpired): - return _json_result( - VERDICT_BROKEN, - reason="command-timeout", - command=( - list(error.cmd) if isinstance(error.cmd, (tuple, list)) else str(error.cmd) - ), - timeout=error.timeout, - ) - if isinstance(error, subprocess.CalledProcessError): - return _json_result( - VERDICT_BROKEN, - reason="dependency-install-failed", - command=( - list(error.cmd) if isinstance(error.cmd, (tuple, list)) else str(error.cmd) - ), - returncode=error.returncode, - stdout=error.stdout, - stderr=error.stderr, - ) - if isinstance(error, ImportError): - return _json_result( - VERDICT_BROKEN, - reason="dependency-import-failed", - detail=str(error), - cause=str(error.__cause__) if error.__cause__ is not None else None, - ) - return _json_result( - VERDICT_BROKEN, - reason="dependency-install-unavailable", - detail=str(error), - ) + return _runtime_dependency_error_result(wrapped.error) except CommandUnavailableError as wrapped: return _json_result( VERDICT_BROKEN, diff --git a/scripts/langchain/pr_verifier.py b/scripts/langchain/pr_verifier.py index c42a94893..327cc959d 100755 --- a/scripts/langchain/pr_verifier.py +++ b/scripts/langchain/pr_verifier.py @@ -705,8 +705,11 @@ def _coerce_response_content(content: object) -> str: return text try: return json.dumps(content, default=str) - except (TypeError, ValueError): - return str(content) + except Exception: + try: + return str(content) + except Exception: + return f"" def _parse_llm_response( diff --git a/templates/consumer-repo/scripts/check_deliberate_break.py b/templates/consumer-repo/scripts/check_deliberate_break.py index fb59c026c..8dc461794 100644 --- a/templates/consumer-repo/scripts/check_deliberate_break.py +++ b/templates/consumer-repo/scripts/check_deliberate_break.py @@ -203,6 +203,29 @@ def _ensure_pytest_runtime_deps() -> None: raise error from (import_error or retry_error) +def _pyyaml_runtime_needs_repair() -> bool: + """Return whether the active PyYAML runtime is missing, stale, or unusable.""" + try: + installed_version = metadata.version("PyYAML") + except metadata.PackageNotFoundError: + return True + if installed_version != PYYAML_VERSION: + return True + try: + import_module("yaml") + except Exception: + return True + return False + + +def _uses_pytest_runtime(command: tuple[str, ...]) -> bool: + """Return whether a command runs pytest in the active Python environment.""" + if len(command) < 3 or command[1:3] != ("-m", "pytest"): + return False + executable = shutil.which(command[0]) or command[0] + return Path(executable).resolve() == Path(sys.executable).resolve() + + def _run( command: tuple[str, ...], cwd: Path, @@ -229,6 +252,17 @@ def _run_with_runtime_deps( cwd: Path, ) -> subprocess.CompletedProcess[str]: """Retry a command after repairing PyYAML only when its output requires it.""" + managed_runtime = _uses_pytest_runtime(command) + if managed_runtime and _pyyaml_runtime_needs_repair(): + try: + _ensure_pytest_runtime_deps() + except ( + subprocess.TimeoutExpired, + subprocess.CalledProcessError, + ImportError, + OSError, + ) as exc: + raise RuntimeDependencyError(exc) from exc try: completed = _run(command, cwd) except OSError as exc: @@ -248,13 +282,18 @@ def _run_with_runtime_deps( ) yaml_traceback = bool(re.search(r"(?:^|[/\\])yaml[/\\][^\n]*", output, re.MULTILINE)) if yaml_traceback and not missing_pyyaml: - try: - import_module("yaml") - except Exception: - missing_pyyaml = True + missing_pyyaml = True if not managed_runtime else _pyyaml_runtime_needs_repair() if not missing_pyyaml: return completed + if not managed_runtime: + error = ImportError( + "PyYAML failed inside a wrapped or custom deliberate-break command; " + "automatic repair is disabled because the wrapper may use a different " + "Python environment" + ) + raise RuntimeDependencyError(error) from error + try: _ensure_pytest_runtime_deps() except (subprocess.TimeoutExpired, subprocess.CalledProcessError, ImportError, OSError) as exc: @@ -265,6 +304,38 @@ def _run_with_runtime_deps( raise CommandUnavailableError(exc) from exc +def _runtime_dependency_error_result(error: Exception) -> dict[str, object]: + """Map dependency-repair failures consistently for head and base runs.""" + if isinstance(error, subprocess.TimeoutExpired): + return _json_result( + VERDICT_BROKEN, + reason="command-timeout", + command=list(error.cmd) if isinstance(error.cmd, (tuple, list)) else str(error.cmd), + timeout=error.timeout, + ) + if isinstance(error, subprocess.CalledProcessError): + return _json_result( + VERDICT_BROKEN, + reason="dependency-install-failed", + command=list(error.cmd) if isinstance(error.cmd, (tuple, list)) else str(error.cmd), + returncode=error.returncode, + stdout=error.stdout, + stderr=error.stderr, + ) + if isinstance(error, ImportError): + return _json_result( + VERDICT_BROKEN, + reason="dependency-import-failed", + detail=str(error), + cause=str(error.__cause__) if error.__cause__ is not None else None, + ) + return _json_result( + VERDICT_BROKEN, + reason="dependency-install-unavailable", + detail=str(error), + ) + + def _git( args: list[str], cwd: Path, @@ -385,39 +456,7 @@ def verify_spec( timeout=exc.timeout, ) except RuntimeDependencyError as wrapped: - error = wrapped.error - if isinstance(error, subprocess.TimeoutExpired): - return _json_result( - VERDICT_BROKEN, - reason="command-timeout", - command=( - list(error.cmd) if isinstance(error.cmd, (tuple, list)) else str(error.cmd) - ), - timeout=error.timeout, - ) - if isinstance(error, subprocess.CalledProcessError): - return _json_result( - VERDICT_BROKEN, - reason="dependency-install-failed", - command=( - list(error.cmd) if isinstance(error.cmd, (tuple, list)) else str(error.cmd) - ), - returncode=error.returncode, - stdout=error.stdout, - stderr=error.stderr, - ) - if isinstance(error, ImportError): - return _json_result( - VERDICT_BROKEN, - reason="dependency-import-failed", - detail=str(error), - cause=str(error.__cause__) if error.__cause__ is not None else None, - ) - return _json_result( - VERDICT_BROKEN, - reason="dependency-install-unavailable", - detail=str(error), - ) + return _runtime_dependency_error_result(wrapped.error) except CommandUnavailableError as wrapped: return _json_result( VERDICT_BROKEN, @@ -458,39 +497,7 @@ def verify_spec( detail=str(exc), ) except RuntimeDependencyError as wrapped: - error = wrapped.error - if isinstance(error, subprocess.TimeoutExpired): - return _json_result( - VERDICT_BROKEN, - reason="command-timeout", - command=( - list(error.cmd) if isinstance(error.cmd, (tuple, list)) else str(error.cmd) - ), - timeout=error.timeout, - ) - if isinstance(error, subprocess.CalledProcessError): - return _json_result( - VERDICT_BROKEN, - reason="dependency-install-failed", - command=( - list(error.cmd) if isinstance(error.cmd, (tuple, list)) else str(error.cmd) - ), - returncode=error.returncode, - stdout=error.stdout, - stderr=error.stderr, - ) - if isinstance(error, ImportError): - return _json_result( - VERDICT_BROKEN, - reason="dependency-import-failed", - detail=str(error), - cause=str(error.__cause__) if error.__cause__ is not None else None, - ) - return _json_result( - VERDICT_BROKEN, - reason="dependency-install-unavailable", - detail=str(error), - ) + return _runtime_dependency_error_result(wrapped.error) except CommandUnavailableError as wrapped: return _json_result( VERDICT_BROKEN, diff --git a/tests/scripts/test_check_deliberate_break.py b/tests/scripts/test_check_deliberate_break.py index 41684cd66..d7ec6b02c 100644 --- a/tests/scripts/test_check_deliberate_break.py +++ b/tests/scripts/test_check_deliberate_break.py @@ -462,13 +462,18 @@ def test_runtime_dependencies_retry_pyyaml_import_failure(tmp_path, monkeypatch) ] repairs: list[bool] = [] monkeypatch.setattr(deliberate_break, "_run", lambda *_args: attempts.pop(0)) + monkeypatch.setattr( + deliberate_break, + "_pyyaml_runtime_needs_repair", + lambda: False, + ) monkeypatch.setattr( deliberate_break, "_ensure_pytest_runtime_deps", lambda: repairs.append(True), ) - completed = deliberate_break._run_with_runtime_deps(("pytest",), tmp_path) + completed = deliberate_break._run_with_runtime_deps((sys.executable, "-m", "pytest"), tmp_path) assert completed.returncode == 0 assert repairs == [True] @@ -486,7 +491,13 @@ def test_runtime_dependencies_retry_broken_pyyaml_traceback(tmp_path, monkeypatc subprocess.CompletedProcess(["pytest"], 0, "passed", ""), ] repairs: list[bool] = [] + repair_checks = iter((False, True)) monkeypatch.setattr(deliberate_break, "_run", lambda *_args: attempts.pop(0)) + monkeypatch.setattr( + deliberate_break, + "_pyyaml_runtime_needs_repair", + lambda: next(repair_checks), + ) monkeypatch.setattr( deliberate_break, "import_module", @@ -498,13 +509,76 @@ def test_runtime_dependencies_retry_broken_pyyaml_traceback(tmp_path, monkeypatc lambda: repairs.append(True), ) - completed = deliberate_break._run_with_runtime_deps(("pytest",), tmp_path) + completed = deliberate_break._run_with_runtime_deps((sys.executable, "-m", "pytest"), tmp_path) assert completed.returncode == 0 assert repairs == [True] assert attempts == [] +def test_runtime_dependencies_normalize_stale_pyyaml_before_pytest(tmp_path, monkeypatch) -> None: + events: list[str] = [] + completed = subprocess.CompletedProcess(["pytest"], 0, "passed", "") + monkeypatch.setattr( + deliberate_break, + "_run", + lambda *_args: events.append("run") or completed, + ) + monkeypatch.setattr(deliberate_break.metadata, "version", lambda _name: "0.0.0") + monkeypatch.setattr( + deliberate_break, + "_ensure_pytest_runtime_deps", + lambda: events.append("repair"), + ) + + result = deliberate_break._run_with_runtime_deps((sys.executable, "-m", "pytest"), tmp_path) + + assert result is completed + assert events == ["repair", "run"] + + +@pytest.mark.parametrize( + "command", + [ + ("uv", "run", "pytest"), + ("/other/venv/bin/python", "-m", "pytest"), + ], +) +@pytest.mark.parametrize( + "stderr", + [ + "ModuleNotFoundError: No module named 'yaml'", + 'File "/other/venv/lib/site-packages/yaml/__init__.py", line 1\n' + "SyntaxError: broken wheel", + ], +) +def test_runtime_dependencies_do_not_repair_unmanaged_command_environment( + tmp_path, monkeypatch, command, stderr +) -> None: + completed = subprocess.CompletedProcess( + command, + 1, + "", + stderr, + ) + monkeypatch.setattr( + deliberate_break, + "_run", + lambda *_args: completed, + ) + monkeypatch.setattr( + deliberate_break, + "_ensure_pytest_runtime_deps", + lambda: pytest.fail("wrapper-owned environment must not be mutated"), + ) + + with pytest.raises(deliberate_break.RuntimeDependencyError) as caught: + deliberate_break._run_with_runtime_deps(command, tmp_path) + + assert isinstance(caught.value.error, ImportError) + assert "wrapped or custom" in str(caught.value.error) + + def _sound_spec(repo: Path) -> tuple[str, object]: _write_app(repo, 0) base = _commit(repo, "base behavior") diff --git a/tests/scripts/test_pr_verifier_structured_output.py b/tests/scripts/test_pr_verifier_structured_output.py index 038742ffe..a4d30b834 100644 --- a/tests/scripts/test_pr_verifier_structured_output.py +++ b/tests/scripts/test_pr_verifier_structured_output.py @@ -220,6 +220,16 @@ def test_coerce_response_content_falls_back_to_string_when_json_serialization_fa assert pr_verifier._coerce_response_content(payload) == str(payload) +def test_coerce_response_content_survives_failing_string_conversion() -> None: + class Unserializable: + def __str__(self) -> str: + raise RuntimeError("broken string conversion") + + assert ( + pr_verifier._coerce_response_content(Unserializable()) == "" + ) + + def test_text_from_response_content_concatenates_split_text_blocks_without_separator() -> None: payload = _valid_payload() encoded = json.dumps(payload)