Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions scripts/ci/noema_review_gate.py
Original file line number Diff line number Diff line change
Expand Up @@ -1352,6 +1352,7 @@ def _bounded_allowed_locations_json(allowed_locations: Sequence[dict[str, Any]])
total_count = len(allowed_locations)

def render(count: int) -> str:
"""Serialize the first `count` locations, flagged as truncated if fewer than all."""
return json.dumps(
{
"total_count": total_count,
Expand Down
44 changes: 44 additions & 0 deletions tests/test_noema_model_output_edge_coverage.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,13 +2,25 @@

from __future__ import annotations

import io
import json

from scripts.ci.noema_review_gate import (
MAX_HTTP_ERROR_BODY_BYTES,
_extract_http_error_served_model,
_extract_served_model,
_strip_trailing_commas_outside_strings,
extract_json_object,
)


class _UnreadableBody:
"""A response body whose read() fails, like a closed or drained socket."""

def read(self, _size: int) -> bytes:
raise OSError("body already consumed")


def test_trailing_comma_stripper_preserves_escaped_string_content() -> None:
"""Quote/escape state must preserve backslashes and commas inside strings."""
source = '{"value":"x\\\\y,",}'
Expand All @@ -29,3 +41,35 @@ def test_extract_json_object_recovers_only_lossless_trailing_comma() -> None:
def test_extract_served_model_rejects_malformed_json() -> None:
"""Malformed response metadata must never fabricate a serving-model identity."""
assert _extract_served_model("not-json") is None


def test_extract_http_error_served_model_reads_the_canonical_field() -> None:
body = json.dumps({"error": {"detail": {"model": "github_models/deepseek-v3"}}}).encode()
assert _extract_http_error_served_model(io.BytesIO(body)) == "github_models/deepseek-v3"


def test_extract_http_error_served_model_fails_closed_on_unreadable_body() -> None:
assert _extract_http_error_served_model(_UnreadableBody()) is None


def test_extract_http_error_served_model_fails_closed_on_oversized_body() -> None:
oversized = json.dumps({"pad": "x" * MAX_HTTP_ERROR_BODY_BYTES}).encode()
assert _extract_http_error_served_model(io.BytesIO(oversized)) is None


def test_extract_http_error_served_model_fails_closed_on_invalid_json() -> None:
assert _extract_http_error_served_model(io.BytesIO(b"not-json")) is None


def test_extract_http_error_served_model_fails_closed_on_non_dict_payload() -> None:
assert _extract_http_error_served_model(io.BytesIO(b"[]")) is None


def test_extract_http_error_served_model_fails_closed_on_missing_error_object() -> None:
body = json.dumps({"error": "boom"}).encode()
assert _extract_http_error_served_model(io.BytesIO(body)) is None


def test_extract_http_error_served_model_fails_closed_on_missing_detail_object() -> None:
body = json.dumps({"error": {"detail": "boom"}}).encode()
assert _extract_http_error_served_model(io.BytesIO(body)) is None
15 changes: 10 additions & 5 deletions tests/test_noema_repair_attempt_telemetry.py
Original file line number Diff line number Diff line change
Expand Up @@ -79,12 +79,17 @@ def test_malformed_output_fails_closed_without_caller_retry(monkeypatch, capsys)


def test_served_model_is_annotation_safe() -> None:
"""A model id carrying CRLF/GHA-annotation/control characters is rejected
outright, not sanitized and kept -- it prints as ``served_model=unknown``
in the ``::warning::``/``::notice::`` lines, so nothing it contains can
ever reach GitHub Actions' workflow-command parser."""
raw = json.dumps({"model": "bad\r\n::error::boom\u0000\ud800"})
value = gate._extract_served_model(raw)
assert value is not None
assert "\r" not in value and "\n" not in value and "\x00" not in value
assert "\\ud800" in value
assert len(value) <= 200
assert gate._extract_served_model(raw) is None


def test_served_model_accepts_a_real_provider_id() -> None:
raw = json.dumps({"model": "deepseek-ai/deepseek-v4-pro-0813"})
assert gate._extract_served_model(raw) == "deepseek-ai/deepseek-v4-pro-0813"


@pytest.mark.parametrize("text", ["[,]", "{,}", "[1,,]", '{"a":,}'])
Expand Down
Loading