From 1d72a2a4ac18e34f41b0f001af1aecf1dd3859ef Mon Sep 17 00:00:00 2001 From: jonahgabriel Date: Mon, 25 May 2026 17:09:26 -0400 Subject: [PATCH] =?UTF-8?q?feat(OMN-12130):=20add=20node=5Fhandler=5Fcorre?= =?UTF-8?q?ctness=5Fgate=20=E2=80=94=20golden=20eval=20set=20+=20correctne?= =?UTF-8?q?ss=20gate?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pure compute node that scores pre-computed handler outputs against an eval set using exact_match, contains, or starts_with scoring. Returns pass/fail with per-entry failure details. Ships with 12 unit tests and a sentiment-classifier example eval set. --- .../proof-of-life-001/baseline_diff.json | 2 +- .../proof-of-life-001/findings.json | 2 +- .../proof-of-life-001/import_graph.json | 2 +- .../proof-of-life-001/projection_rows.json | 2 +- .../proof-of-life-001/proof_summary.md | 2 +- .../proof-of-life-001/run_manifest.json | 2 +- .../proof-of-life-001/topology_graph.json | 2 +- pyproject.toml | 1 + .../node_handler_correctness_gate/__init__.py | 2 + .../contract.yaml | 71 ++++++ .../contracts/example_eval_set.yaml | 19 ++ .../handlers/__init__.py | 2 + .../handlers/handler_correctness_gate.py | 81 +++++++ .../metadata.yaml | 23 ++ .../models/__init__.py | 2 + .../models/enums.py | 11 + .../models/model_correctness_check_request.py | 18 ++ .../models/model_correctness_check_result.py | 21 ++ .../models/model_eval_entry.py | 17 ++ .../models/model_eval_failure.py | 19 ++ .../models/model_eval_set.py | 18 ++ .../tests/__init__.py | 2 + .../tests/test_handler_correctness_gate.py | 210 ++++++++++++++++++ 23 files changed, 524 insertions(+), 7 deletions(-) create mode 100644 src/omnimarket/nodes/node_handler_correctness_gate/__init__.py create mode 100644 src/omnimarket/nodes/node_handler_correctness_gate/contract.yaml create mode 100644 src/omnimarket/nodes/node_handler_correctness_gate/contracts/example_eval_set.yaml create mode 100644 src/omnimarket/nodes/node_handler_correctness_gate/handlers/__init__.py create mode 100644 src/omnimarket/nodes/node_handler_correctness_gate/handlers/handler_correctness_gate.py create mode 100644 src/omnimarket/nodes/node_handler_correctness_gate/metadata.yaml create mode 100644 src/omnimarket/nodes/node_handler_correctness_gate/models/__init__.py create mode 100644 src/omnimarket/nodes/node_handler_correctness_gate/models/enums.py create mode 100644 src/omnimarket/nodes/node_handler_correctness_gate/models/model_correctness_check_request.py create mode 100644 src/omnimarket/nodes/node_handler_correctness_gate/models/model_correctness_check_result.py create mode 100644 src/omnimarket/nodes/node_handler_correctness_gate/models/model_eval_entry.py create mode 100644 src/omnimarket/nodes/node_handler_correctness_gate/models/model_eval_failure.py create mode 100644 src/omnimarket/nodes/node_handler_correctness_gate/models/model_eval_set.py create mode 100644 src/omnimarket/nodes/node_handler_correctness_gate/tests/__init__.py create mode 100644 src/omnimarket/nodes/node_handler_correctness_gate/tests/test_handler_correctness_gate.py diff --git a/docs/evidence/dep-health-sweep/proof-of-life-001/baseline_diff.json b/docs/evidence/dep-health-sweep/proof-of-life-001/baseline_diff.json index ec747fa47d..19765bd501 100644 --- a/docs/evidence/dep-health-sweep/proof-of-life-001/baseline_diff.json +++ b/docs/evidence/dep-health-sweep/proof-of-life-001/baseline_diff.json @@ -1 +1 @@ -null \ No newline at end of file +null diff --git a/docs/evidence/dep-health-sweep/proof-of-life-001/findings.json b/docs/evidence/dep-health-sweep/proof-of-life-001/findings.json index 4108333567..971ac3be38 100644 --- a/docs/evidence/dep-health-sweep/proof-of-life-001/findings.json +++ b/docs/evidence/dep-health-sweep/proof-of-life-001/findings.json @@ -19,4 +19,4 @@ "rule_id": "MISSING_TOPIC_EDGE", "rule_version": "v1" } -] \ No newline at end of file +] diff --git a/docs/evidence/dep-health-sweep/proof-of-life-001/import_graph.json b/docs/evidence/dep-health-sweep/proof-of-life-001/import_graph.json index 745032b768..07cf053df7 100644 --- a/docs/evidence/dep-health-sweep/proof-of-life-001/import_graph.json +++ b/docs/evidence/dep-health-sweep/proof-of-life-001/import_graph.json @@ -3,4 +3,4 @@ "nodes": [], "edges": [], "orphan_modules": [] -} \ No newline at end of file +} diff --git a/docs/evidence/dep-health-sweep/proof-of-life-001/projection_rows.json b/docs/evidence/dep-health-sweep/proof-of-life-001/projection_rows.json index b99bfdb2c0..f2e4e69d0f 100644 --- a/docs/evidence/dep-health-sweep/proof-of-life-001/projection_rows.json +++ b/docs/evidence/dep-health-sweep/proof-of-life-001/projection_rows.json @@ -23,4 +23,4 @@ "rule_version": "v1", "captured_at": "2026-05-25T19:17:15.256588+00:00" } -] \ No newline at end of file +] diff --git a/docs/evidence/dep-health-sweep/proof-of-life-001/proof_summary.md b/docs/evidence/dep-health-sweep/proof-of-life-001/proof_summary.md index b3cf5c06c8..84fdb73c5c 100644 --- a/docs/evidence/dep-health-sweep/proof-of-life-001/proof_summary.md +++ b/docs/evidence/dep-health-sweep/proof-of-life-001/proof_summary.md @@ -11,4 +11,4 @@ timestamp: 2026-05-25T19:17:15.259389+00:00 - **D: projection**: PASS — rows_upserted=2 ## Findings count: 2 -## Projection rows: 2 \ No newline at end of file +## Projection rows: 2 diff --git a/docs/evidence/dep-health-sweep/proof-of-life-001/run_manifest.json b/docs/evidence/dep-health-sweep/proof-of-life-001/run_manifest.json index abc88b8802..4df31790dd 100644 --- a/docs/evidence/dep-health-sweep/proof-of-life-001/run_manifest.json +++ b/docs/evidence/dep-health-sweep/proof-of-life-001/run_manifest.json @@ -6,4 +6,4 @@ ], "graphify_version": "0.8.2", "rule_version": "v1" -} \ No newline at end of file +} diff --git a/docs/evidence/dep-health-sweep/proof-of-life-001/topology_graph.json b/docs/evidence/dep-health-sweep/proof-of-life-001/topology_graph.json index 67f60c3e8e..f69f7ce11e 100644 --- a/docs/evidence/dep-health-sweep/proof-of-life-001/topology_graph.json +++ b/docs/evidence/dep-health-sweep/proof-of-life-001/topology_graph.json @@ -3,4 +3,4 @@ "pub_edges": [], "sub_edges": [], "orphan_topics": [] -} \ No newline at end of file +} diff --git a/pyproject.toml b/pyproject.toml index db99706590..2f174a3a9d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -309,6 +309,7 @@ node_llm_delegation_projection = "omnimarket.nodes.node_llm_delegation_projectio node_swarm_registry_compute = "omnimarket.nodes.node_swarm_registry_compute" node_knowledge_query_federation_orchestrator = "omnimarket.nodes.node_knowledge_query_federation_orchestrator" node_antipattern_index_effect = "omnimarket.nodes.node_antipattern_index_effect" +node_handler_correctness_gate = "omnimarket.nodes.node_handler_correctness_gate" [project.scripts] onex-integration-test-runner = "omnimarket.nodes.node_integration_test_runner.cli:main" diff --git a/src/omnimarket/nodes/node_handler_correctness_gate/__init__.py b/src/omnimarket/nodes/node_handler_correctness_gate/__init__.py new file mode 100644 index 0000000000..64f06c497f --- /dev/null +++ b/src/omnimarket/nodes/node_handler_correctness_gate/__init__.py @@ -0,0 +1,2 @@ +# SPDX-FileCopyrightText: 2026 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT diff --git a/src/omnimarket/nodes/node_handler_correctness_gate/contract.yaml b/src/omnimarket/nodes/node_handler_correctness_gate/contract.yaml new file mode 100644 index 0000000000..a2ff7d8bc3 --- /dev/null +++ b/src/omnimarket/nodes/node_handler_correctness_gate/contract.yaml @@ -0,0 +1,71 @@ +# SPDX-FileCopyrightText: 2026 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +name: node_handler_correctness_gate +contract_name: node_handler_correctness_gate +node_name: node_handler_correctness_gate +contract_version: {major: 1, minor: 0, patch: 0} +node_version: {major: 1, minor: 0, patch: 0} +node_type: compute +description: > + Pure compute node that scores pre-computed handler outputs against an eval + set and returns pass/fail with a correctness score. The gate itself never + invokes the handler — it only compares outputs. + +input_model: + name: ModelCorrectnessCheckRequest + class: ModelCorrectnessCheckRequest + module: omnimarket.nodes.node_handler_correctness_gate.models.model_correctness_check_request + +output_model: + name: ModelCorrectnessCheckResult + class: ModelCorrectnessCheckResult + module: omnimarket.nodes.node_handler_correctness_gate.models.model_correctness_check_result + +handler: + name: HandlerCorrectnessGate + module: omnimarket.nodes.node_handler_correctness_gate.handlers.handler_correctness_gate + class: HandlerCorrectnessGate + +handler_routing: + routing_strategy: operation_match + handlers: + - handler: + name: HandlerCorrectnessGate + module: omnimarket.nodes.node_handler_correctness_gate.handlers.handler_correctness_gate + event_model: + name: ModelCorrectnessCheckRequest + module: omnimarket.nodes.node_handler_correctness_gate.models.model_correctness_check_request + +descriptor: + node_archetype: compute + purity: pure + idempotent: true + timeout_ms: 60000 + runtime_profiles: [main] + +event_bus: + version: {major: 1, minor: 0, patch: 0} + plugin_managed: false + subscribe_topics: + - onex.cmd.omnimarket.handler-correctness-check.v1 + publish_topics: + - onex.evt.omnimarket.handler-correctness-result.v1 + +terminal_event: onex.evt.omnimarket.handler-correctness-result.v1 + +externally_consumed_topics: + - onex.evt.omnimarket.handler-correctness-result.v1 + +published_events: + - event_type: HandlerCorrectnessResult + topic: onex.evt.omnimarket.handler-correctness-result.v1 + description: Handler correctness gate result with score and failure details. + +metadata: + transport_type: kafka + author: OmniNode Team + license: MIT + created: "2026-05-25" + tags: [compute, eval, correctness, gate, OMN-12130] + related_tickets: + - "OMN-12130" diff --git a/src/omnimarket/nodes/node_handler_correctness_gate/contracts/example_eval_set.yaml b/src/omnimarket/nodes/node_handler_correctness_gate/contracts/example_eval_set.yaml new file mode 100644 index 0000000000..f7fb8fdc64 --- /dev/null +++ b/src/omnimarket/nodes/node_handler_correctness_gate/contracts/example_eval_set.yaml @@ -0,0 +1,19 @@ +name: "sentiment-classifier-v1" +description: "Basic sentiment classification correctness check" +min_score: 0.85 +entries: + - input: "This product is terrible and I want a refund" + expected: "negative" + scoring: "exact_match" + - input: "Best purchase I've ever made" + expected: "positive" + scoring: "exact_match" + - input: "It arrived on time" + expected: "neutral" + scoring: "exact_match" + - input: "Absolutely love it, 5 stars" + expected: "positive" + scoring: "exact_match" + - input: "Worst experience ever, never buying again" + expected: "negative" + scoring: "exact_match" diff --git a/src/omnimarket/nodes/node_handler_correctness_gate/handlers/__init__.py b/src/omnimarket/nodes/node_handler_correctness_gate/handlers/__init__.py new file mode 100644 index 0000000000..64f06c497f --- /dev/null +++ b/src/omnimarket/nodes/node_handler_correctness_gate/handlers/__init__.py @@ -0,0 +1,2 @@ +# SPDX-FileCopyrightText: 2026 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT diff --git a/src/omnimarket/nodes/node_handler_correctness_gate/handlers/handler_correctness_gate.py b/src/omnimarket/nodes/node_handler_correctness_gate/handlers/handler_correctness_gate.py new file mode 100644 index 0000000000..e730d2405e --- /dev/null +++ b/src/omnimarket/nodes/node_handler_correctness_gate/handlers/handler_correctness_gate.py @@ -0,0 +1,81 @@ +# SPDX-FileCopyrightText: 2026 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +from __future__ import annotations + +from omnimarket.nodes.node_handler_correctness_gate.models.enums import ( + EnumScoringMethod, +) +from omnimarket.nodes.node_handler_correctness_gate.models.model_correctness_check_request import ( + ModelCorrectnessCheckRequest, +) +from omnimarket.nodes.node_handler_correctness_gate.models.model_correctness_check_result import ( + ModelCorrectnessCheckResult, +) +from omnimarket.nodes.node_handler_correctness_gate.models.model_eval_entry import ( + ModelEvalEntry, +) +from omnimarket.nodes.node_handler_correctness_gate.models.model_eval_failure import ( + ModelEvalFailure, +) + + +def _score_entry(entry: ModelEvalEntry, actual: str) -> bool: + expected = entry.expected + if entry.scoring == EnumScoringMethod.EXACT_MATCH: + return actual == expected + if entry.scoring == EnumScoringMethod.CONTAINS: + return expected in actual + if entry.scoring == EnumScoringMethod.STARTS_WITH: + return actual.startswith(expected) + return False + + +class HandlerCorrectnessGate: + def handle( + self, request: ModelCorrectnessCheckRequest + ) -> ModelCorrectnessCheckResult: + entries = request.eval_set.entries + total = len(entries) + + if total == 0: + return ModelCorrectnessCheckResult( + handler_id=request.handler_id, + score=0.0, + passed=False, + total_entries=0, + correct_entries=0, + failures=(), + eval_set_name=request.eval_set.name, + ) + + actual_outputs = request.actual_outputs + failures: list[ModelEvalFailure] = [] + correct = 0 + + for idx, entry in enumerate(entries): + actual = actual_outputs[idx] if idx < len(actual_outputs) else "" + if _score_entry(entry, actual): + correct += 1 + else: + failures.append( + ModelEvalFailure( + entry_index=idx, + input=entry.input, + expected=entry.expected, + actual=actual, + scoring=entry.scoring, + ) + ) + + score = correct / total + passed = score >= request.eval_set.min_score + + return ModelCorrectnessCheckResult( + handler_id=request.handler_id, + score=score, + passed=passed, + total_entries=total, + correct_entries=correct, + failures=tuple(failures), + eval_set_name=request.eval_set.name, + ) diff --git a/src/omnimarket/nodes/node_handler_correctness_gate/metadata.yaml b/src/omnimarket/nodes/node_handler_correctness_gate/metadata.yaml new file mode 100644 index 0000000000..59df791458 --- /dev/null +++ b/src/omnimarket/nodes/node_handler_correctness_gate/metadata.yaml @@ -0,0 +1,23 @@ +name: node_handler_correctness_gate +version: "1.0.0" +description: "Pure compute node that scores pre-computed handler outputs against an eval set" +omnibase_core_compat: ">=0.39.0,<1.0.0" +entry_points: + onex.nodes: + node_handler_correctness_gate: "omnimarket.nodes.node_handler_correctness_gate" +capabilities: + standalone: true + full_runtime: false + requires_network: false + requires_repo: false + requires_secrets: false + requires_docker: false + side_effect_class: pure +dependencies: + - "pydantic>=2.0" +authors: ["OmniNode Team"] +license: "MIT" +tags: ["compute", "eval", "correctness", "gate", "OMN-12130"] +pack: "eval" +display_name: "Handler Correctness Gate" +node_role: "compute" diff --git a/src/omnimarket/nodes/node_handler_correctness_gate/models/__init__.py b/src/omnimarket/nodes/node_handler_correctness_gate/models/__init__.py new file mode 100644 index 0000000000..64f06c497f --- /dev/null +++ b/src/omnimarket/nodes/node_handler_correctness_gate/models/__init__.py @@ -0,0 +1,2 @@ +# SPDX-FileCopyrightText: 2026 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT diff --git a/src/omnimarket/nodes/node_handler_correctness_gate/models/enums.py b/src/omnimarket/nodes/node_handler_correctness_gate/models/enums.py new file mode 100644 index 0000000000..87227e3b66 --- /dev/null +++ b/src/omnimarket/nodes/node_handler_correctness_gate/models/enums.py @@ -0,0 +1,11 @@ +# SPDX-FileCopyrightText: 2026 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +from __future__ import annotations + +from enum import StrEnum + + +class EnumScoringMethod(StrEnum): + EXACT_MATCH = "exact_match" + CONTAINS = "contains" + STARTS_WITH = "starts_with" diff --git a/src/omnimarket/nodes/node_handler_correctness_gate/models/model_correctness_check_request.py b/src/omnimarket/nodes/node_handler_correctness_gate/models/model_correctness_check_request.py new file mode 100644 index 0000000000..5dad71dcca --- /dev/null +++ b/src/omnimarket/nodes/node_handler_correctness_gate/models/model_correctness_check_request.py @@ -0,0 +1,18 @@ +# SPDX-FileCopyrightText: 2026 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +from __future__ import annotations + +from pydantic import BaseModel, ConfigDict + +from omnimarket.nodes.node_handler_correctness_gate.models.model_eval_set import ( + ModelEvalSet, +) + + +class ModelCorrectnessCheckRequest(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + handler_id: str + eval_set: ModelEvalSet + actual_outputs: tuple[str, ...] + correlation_id: str diff --git a/src/omnimarket/nodes/node_handler_correctness_gate/models/model_correctness_check_result.py b/src/omnimarket/nodes/node_handler_correctness_gate/models/model_correctness_check_result.py new file mode 100644 index 0000000000..28b63d6bb6 --- /dev/null +++ b/src/omnimarket/nodes/node_handler_correctness_gate/models/model_correctness_check_result.py @@ -0,0 +1,21 @@ +# SPDX-FileCopyrightText: 2026 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +from __future__ import annotations + +from pydantic import BaseModel, ConfigDict + +from omnimarket.nodes.node_handler_correctness_gate.models.model_eval_failure import ( + ModelEvalFailure, +) + + +class ModelCorrectnessCheckResult(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + handler_id: str + score: float + passed: bool + total_entries: int + correct_entries: int + failures: tuple[ModelEvalFailure, ...] = () + eval_set_name: str = "" diff --git a/src/omnimarket/nodes/node_handler_correctness_gate/models/model_eval_entry.py b/src/omnimarket/nodes/node_handler_correctness_gate/models/model_eval_entry.py new file mode 100644 index 0000000000..f0af5ea6cf --- /dev/null +++ b/src/omnimarket/nodes/node_handler_correctness_gate/models/model_eval_entry.py @@ -0,0 +1,17 @@ +# SPDX-FileCopyrightText: 2026 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +from __future__ import annotations + +from pydantic import BaseModel, ConfigDict + +from omnimarket.nodes.node_handler_correctness_gate.models.enums import ( + EnumScoringMethod, +) + + +class ModelEvalEntry(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + input: str + expected: str + scoring: EnumScoringMethod = EnumScoringMethod.EXACT_MATCH diff --git a/src/omnimarket/nodes/node_handler_correctness_gate/models/model_eval_failure.py b/src/omnimarket/nodes/node_handler_correctness_gate/models/model_eval_failure.py new file mode 100644 index 0000000000..0b74cee93d --- /dev/null +++ b/src/omnimarket/nodes/node_handler_correctness_gate/models/model_eval_failure.py @@ -0,0 +1,19 @@ +# SPDX-FileCopyrightText: 2026 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +from __future__ import annotations + +from pydantic import BaseModel, ConfigDict + +from omnimarket.nodes.node_handler_correctness_gate.models.enums import ( + EnumScoringMethod, +) + + +class ModelEvalFailure(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + entry_index: int + input: str + expected: str + actual: str + scoring: EnumScoringMethod diff --git a/src/omnimarket/nodes/node_handler_correctness_gate/models/model_eval_set.py b/src/omnimarket/nodes/node_handler_correctness_gate/models/model_eval_set.py new file mode 100644 index 0000000000..8d4ee13b78 --- /dev/null +++ b/src/omnimarket/nodes/node_handler_correctness_gate/models/model_eval_set.py @@ -0,0 +1,18 @@ +# SPDX-FileCopyrightText: 2026 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +from __future__ import annotations + +from pydantic import BaseModel, ConfigDict + +from omnimarket.nodes.node_handler_correctness_gate.models.model_eval_entry import ( + ModelEvalEntry, +) + + +class ModelEvalSet(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + entries: tuple[ModelEvalEntry, ...] + min_score: float = 0.85 + name: str = "" + description: str = "" diff --git a/src/omnimarket/nodes/node_handler_correctness_gate/tests/__init__.py b/src/omnimarket/nodes/node_handler_correctness_gate/tests/__init__.py new file mode 100644 index 0000000000..64f06c497f --- /dev/null +++ b/src/omnimarket/nodes/node_handler_correctness_gate/tests/__init__.py @@ -0,0 +1,2 @@ +# SPDX-FileCopyrightText: 2026 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT diff --git a/src/omnimarket/nodes/node_handler_correctness_gate/tests/test_handler_correctness_gate.py b/src/omnimarket/nodes/node_handler_correctness_gate/tests/test_handler_correctness_gate.py new file mode 100644 index 0000000000..7c9c7e7200 --- /dev/null +++ b/src/omnimarket/nodes/node_handler_correctness_gate/tests/test_handler_correctness_gate.py @@ -0,0 +1,210 @@ +# SPDX-FileCopyrightText: 2026 OmniNode.ai Inc. +# SPDX-License-Identifier: MIT +from __future__ import annotations + +import pytest + +from omnimarket.nodes.node_handler_correctness_gate.handlers.handler_correctness_gate import ( + HandlerCorrectnessGate, +) +from omnimarket.nodes.node_handler_correctness_gate.models.enums import ( + EnumScoringMethod, +) +from omnimarket.nodes.node_handler_correctness_gate.models.model_correctness_check_request import ( + ModelCorrectnessCheckRequest, +) +from omnimarket.nodes.node_handler_correctness_gate.models.model_eval_entry import ( + ModelEvalEntry, +) +from omnimarket.nodes.node_handler_correctness_gate.models.model_eval_set import ( + ModelEvalSet, +) + + +def _make_request( + entries: list[ModelEvalEntry], + actual_outputs: list[str], + min_score: float = 0.85, + name: str = "test-eval", +) -> ModelCorrectnessCheckRequest: + return ModelCorrectnessCheckRequest( + handler_id="handler-abc", + eval_set=ModelEvalSet( + entries=tuple(entries), + min_score=min_score, + name=name, + ), + actual_outputs=tuple(actual_outputs), + correlation_id="corr-001", + ) + + +def _entry( + expected: str, + scoring: EnumScoringMethod = EnumScoringMethod.EXACT_MATCH, +) -> ModelEvalEntry: + return ModelEvalEntry(input="input", expected=expected, scoring=scoring) + + +# --------------------------------------------------------------------------- +# All correct +# --------------------------------------------------------------------------- + + +def test_all_correct_score_one_passed() -> None: + entries = [_entry("positive"), _entry("negative"), _entry("neutral")] + request = _make_request(entries, ["positive", "negative", "neutral"]) + result = HandlerCorrectnessGate().handle(request) + + assert result.score == 1.0 + assert result.passed is True + assert result.correct_entries == 3 + assert result.total_entries == 3 + assert result.failures == () + + +# --------------------------------------------------------------------------- +# Partial correct +# --------------------------------------------------------------------------- + + +def test_partial_correct_score_and_failures_listed() -> None: + entries = [_entry("positive"), _entry("negative"), _entry("neutral")] + request = _make_request(entries, ["positive", "WRONG", "neutral"], min_score=0.85) + result = HandlerCorrectnessGate().handle(request) + + assert result.score == pytest.approx(2 / 3) + assert result.correct_entries == 2 + assert len(result.failures) == 1 + failure = result.failures[0] + assert failure.entry_index == 1 + assert failure.expected == "negative" + assert failure.actual == "WRONG" + + +# --------------------------------------------------------------------------- +# Below min_score → passed=False +# --------------------------------------------------------------------------- + + +def test_below_min_score_passed_false() -> None: + entries = [_entry("a"), _entry("b"), _entry("c"), _entry("d")] + request = _make_request(entries, ["a", "WRONG", "WRONG", "WRONG"], min_score=0.85) + result = HandlerCorrectnessGate().handle(request) + + assert result.score == pytest.approx(0.25) + assert result.passed is False + + +def test_at_min_score_boundary_passed_true() -> None: + entries = [_entry("a"), _entry("b"), _entry("c"), _entry("d")] + # 3/4 = 0.75; with min_score=0.75, should pass + request = _make_request(entries, ["a", "b", "c", "WRONG"], min_score=0.75) + result = HandlerCorrectnessGate().handle(request) + + assert result.score == pytest.approx(0.75) + assert result.passed is True + + +# --------------------------------------------------------------------------- +# Scoring methods +# --------------------------------------------------------------------------- + + +def test_exact_match_scoring() -> None: + entries = [ + _entry("hello", EnumScoringMethod.EXACT_MATCH), + _entry("hello", EnumScoringMethod.EXACT_MATCH), + ] + request = _make_request(entries, ["hello", "hello world"], min_score=0.5) + result = HandlerCorrectnessGate().handle(request) + + assert result.correct_entries == 1 + assert result.failures[0].actual == "hello world" + + +def test_contains_scoring() -> None: + entries = [ + _entry("positive", EnumScoringMethod.CONTAINS), + _entry("negative", EnumScoringMethod.CONTAINS), + ] + request = _make_request( + entries, ["very positive sentiment", "clearly negative"], min_score=0.5 + ) + result = HandlerCorrectnessGate().handle(request) + + assert result.score == 1.0 + assert result.passed is True + + +def test_contains_scoring_miss() -> None: + entries = [_entry("positive", EnumScoringMethod.CONTAINS)] + request = _make_request(entries, ["negative sentiment"], min_score=0.5) + result = HandlerCorrectnessGate().handle(request) + + assert result.correct_entries == 0 + assert result.passed is False + + +def test_starts_with_scoring() -> None: + entries = [ + _entry("pos", EnumScoringMethod.STARTS_WITH), + _entry("neg", EnumScoringMethod.STARTS_WITH), + ] + request = _make_request(entries, ["positive", "neutral"], min_score=0.5) + result = HandlerCorrectnessGate().handle(request) + + assert result.correct_entries == 1 + assert result.failures[0].entry_index == 1 + + +def test_starts_with_scoring_match() -> None: + entries = [_entry("neg", EnumScoringMethod.STARTS_WITH)] + request = _make_request(entries, ["negative"], min_score=0.5) + result = HandlerCorrectnessGate().handle(request) + + assert result.score == 1.0 + assert result.passed is True + + +# --------------------------------------------------------------------------- +# Empty eval set +# --------------------------------------------------------------------------- + + +def test_empty_eval_set_score_zero_not_passed() -> None: + request = _make_request([], [], min_score=0.0) + result = HandlerCorrectnessGate().handle(request) + + assert result.score == 0.0 + assert result.passed is False + assert result.total_entries == 0 + assert result.correct_entries == 0 + + +# --------------------------------------------------------------------------- +# Metadata propagation +# --------------------------------------------------------------------------- + + +def test_eval_set_name_propagated() -> None: + entries = [_entry("yes")] + request = _make_request(entries, ["yes"], name="my-eval-set") + result = HandlerCorrectnessGate().handle(request) + + assert result.eval_set_name == "my-eval-set" + assert result.handler_id == "handler-abc" + + +# --------------------------------------------------------------------------- +# Fewer actual_outputs than entries (defensive) +# --------------------------------------------------------------------------- + + +def test_missing_actual_output_treated_as_empty_string() -> None: + entries = [_entry("a"), _entry("b")] + request = _make_request(entries, ["a"], min_score=0.5) + result = HandlerCorrectnessGate().handle(request) + + assert result.correct_entries == 1 + assert result.failures[0].actual == ""