Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions backend/app/analysis_run_worker.py
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,8 @@ async def consume_analysis_run_stream_once(
exc.status_code,
exc.detail,
)
except Exception:
logger.exception("analysis-run %s delivery failed", analysis_run_id)
last_id = str(entry_id)
return last_id

Expand Down
14 changes: 10 additions & 4 deletions lineageweave/adjudication_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
import re
from typing import Protocol

from .http_client import chat_completion_content, post_json
from .http_client import HttpClientError, chat_completion_content, post_json


class AdjudicationClient(Protocol):
Expand Down Expand Up @@ -53,9 +53,11 @@ def judge_prompt(candidate_label: str, record_label: str) -> str:


def parse_confidence(content: str) -> float:
"""Clamp the judge's numeric reply into [0, 1]; no number reads as 0."""
"""Clamp a numeric reply into ``[0, 1]`` or fail without inventing zero."""
parsed = parse_confidence_or_none(content)
return 0.0 if parsed is None else parsed
if parsed is None:
raise HttpClientError("adjudication response had no confidence score")
return parsed


def parse_confidence_or_none(content: str) -> float | None:
Expand Down Expand Up @@ -102,4 +104,8 @@ def judge(self, candidate_label: str, record_label: str) -> float:
headers={"authorization": f"Bearer {self._api_key}"},
timeout=self._timeout,
)
return parse_confidence(chat_completion_content(body))
try:
content = chat_completion_content(body)
except (TypeError, ValueError) as exc:
raise HttpClientError("adjudication response did not contain text") from exc
return parse_confidence(content)
24 changes: 24 additions & 0 deletions tests/test_adjudication_client.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
from __future__ import annotations

import pytest

from lineageweave.adjudication_client import ContextualOrchestratorAdjudicationClient
from lineageweave.http_client import HttpClientError


def test_adjudication_uses_supported_auto_mode_and_long_local_timeout(monkeypatch) -> None:
Expand All @@ -21,3 +24,24 @@ def fake_post_json(url, payload, *, headers, timeout):
assert captured["payload"]["mode"] == "auto"
assert captured["payload"]["reasoning_effort"] == "auto"
assert captured["timeout"] == 180.0


@pytest.mark.parametrize(
"body",
[
{"choices": [{"message": {"content": "not a score"}}]},
{"choices": []},
{"choices": [{"message": {"content": None}}]},
],
)
def test_adjudication_fails_closed_for_unscoreable_responses(monkeypatch, body) -> None:
"""A provider failure must not become a genuine unrelated score of zero."""
monkeypatch.setattr(
"lineageweave.adjudication_client.post_json", lambda *args, **kwargs: body
)
client = ContextualOrchestratorAdjudicationClient(
base_url="http://orchestrator:8000", api_key="synthetic-token"
)

with pytest.raises(HttpClientError):
client.judge("workshop", "follow-up bid")
25 changes: 25 additions & 0 deletions tests/test_analysis_run_worker.py
Original file line number Diff line number Diff line change
Expand Up @@ -130,3 +130,28 @@ async def fake_deliver(conn, **kwargs):

assert last_id == "2-1"
assert delivered == ["00000000-0000-0000-0000-000000000002"]


@pytest.mark.anyio
async def test_one_unexpected_delivery_failure_does_not_end_the_worker(monkeypatch):
"""A malformed provider reply must not stop later durable deliveries."""
delivered = []

async def fake_deliver(conn, **kwargs):
del conn
if kwargs["analysis_run_id"].endswith("1"):
raise RuntimeError("malformed provider reply")
delivered.append(kwargs["analysis_run_id"])

monkeypatch.setattr(analysis_run_worker, "deliver_queued_analysis_run", fake_deliver)

last_id = await analysis_run_worker.consume_analysis_run_stream_once(
_TwoRunsValkey(),
_Pool(),
last_id="0-0",
tepp_client=TeppClient(),
adjudication_client=NullAdjudicationClient(),
)

assert last_id == "2-1"
assert delivered == ["00000000-0000-0000-0000-000000000002"]
6 changes: 5 additions & 1 deletion tests/test_estimate_llm_channel_weights_script.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,10 @@

from __future__ import annotations

import pytest

from lineageweave.adjudication_client import judge_prompt, parse_confidence
from lineageweave.http_client import HttpClientError

import scripts.estimate_llm_channel_weights as script

Expand All @@ -31,7 +34,8 @@ def test_shared_judge_prompt_and_confidence_parse_round_trip() -> None:
assert "Record B: Follow-up record" in prompt
assert parse_confidence("0.85") == 0.85
assert parse_confidence("confidence: 0.4 maybe") == 0.4
assert parse_confidence("no number here") == 0.0
with pytest.raises(HttpClientError):
parse_confidence("no number here")
assert parse_confidence("1.7") == 1.0


Expand Down
Loading