diff --git a/docs/value_compare.md b/docs/value_compare.md new file mode 100644 index 00000000..13f31fad --- /dev/null +++ b/docs/value_compare.md @@ -0,0 +1,133 @@ +# Value-comparison contradiction gate (#422) + +`aelfrice.value_compare` is the typed-slot relatedness gate that +replaces the residual-overlap floor in `relationship_detector` for +contradiction detection. Stdlib-only, deterministic, no embeddings. + +## Why it exists + +The R2 detector from #201 used residual-Jaccard token overlap to +decide if two beliefs were "about the same subject" before checking +modality / quantifier disagreement. On the labeled adversarial +corpus this gate caught 2 of 60 (`recall = 0.033`) — the failure +mode is paraphrase: real natural-language contradictions almost +never share enough surface tokens to clear the overlap floor. + +The v3 gate sidesteps this by extracting **typed slots** from each +belief and firing `contradicts` when the two beliefs disagree on a +slot value, regardless of token overlap. Slot match is the +relatedness signal. + +## What gets extracted + +`extract_values(text)` returns a `ValueSlots` with two tuples: + +**Numeric slots** — `NumericSlot(key, value)`. The `key` is the +alphabetic token immediately preceding a number (with optional +``=`` / ``:`` / ``is`` / ``of`` / ``to`` / ``equals`` separator). +``value`` is parsed as float. Examples that match: + +| input | extracted | +|---|---| +| ``alpha = 0.5`` | ``(alpha, 0.5)`` | +| ``timeout: 30`` | ``(timeout, 30.0)`` | +| ``set retries to 3`` | ``(retries, 3.0)`` | +| ``max_depth=4 depth=2`` | ``(max_depth, 4.0)`` and ``(depth, 2.0)`` | + +Filler keys (``is``, ``of``, ``the``, ``a``, …) are dropped — see +`_NUMERIC_KEY_DROP`. Unit-aware comparison is **out of scope** for +v3: the regex's greedy capture of trailing tokens introduced too +many false negatives (e.g. ``alpha = 0.5 prior`` vs ``alpha = 1.0 +in config`` produced different "units" — ``prior`` vs ``in`` — and +silently skipped the conflict). If unit-aware comparison becomes +needed, file a separate issue with a curated unit vocabulary. + +**Enum slots** — `EnumSlot(category, group_id, member)`. Members +come from `ENUM_VOCAB`, a curated taxonomy of 9 categories grouped +by mutual exclusion: + +| category | groups | +|---|---| +| `execution_mode` | `{sync, synchronous}`, `{async, asynchronous}` | +| `default_state` | `{default-on, enabled}`, `{default-off, disabled}` | +| `storage_mode` | `{indexed}`, `{scan, full-scan, table-scan}` | +| `completeness` | `{full}`, `{incremental}`, `{partial}` | +| `strictness` | `{strict}`, `{lax, permissive}` | +| `necessity` | `{required}`, `{optional}` | +| `visibility` | `{public}`, `{private}` | +| `access_mode` | `{readonly, read-only}`, `{writable, read-write}` | +| `determinism` | `{deterministic}`, `{non-deterministic, nondeterministic, stochastic}` | + +Members within a group are aliases — they do **not** conflict with +each other. `group_id` is the alphabetically-first member of the +group, used as a stable cross-belief identifier. Adding a category +extends the contradiction surface; the dict in +``src/aelfrice/value_compare.py`` is the single source of truth. + +## When the comparator fires + +`find_conflicts(slots_a, slots_b)` returns a tuple of `SlotConflict`: + +- **Numeric conflict**: same key, values outside relative tolerance + (default 1%). The 0/0 case is silent (degenerate denominator). +- **Enum conflict**: same category, disjoint group_id sets. Aliases + collapse to one group, so `sync` vs `synchronous` is silent. + +When the comparator returns multiple distinct conflicts on the same +pair, all of them are surfaced — the integration layer decides how +many it takes to fire. + +## Integration + +`relationship_detector.analyze(a, b, use_value_comparison=True)` +runs the gate **before** the residual-overlap floor. If any +conflict is found: + +``` +RelationshipVerdict( + label="contradicts", + score=1.0, + residual_overlap=0.0, + rationale="value_comparison:numeric=alpha", +) +``` + +Score is pinned at 1.0 so the auto-emit policy (#422 acceptance #3) +can use a single threshold for "slot fired" vs "modality-only +fired" verdicts. + +`use_value_comparison=False` (the default) preserves v1 behaviour +byte-for-byte — the v3 path adds zero overhead until flipped. + +## Determinism + +Same `(text_a, text_b)` produces byte-identical slots and identical +conflicts across runs. No embeddings, no learned classifiers, no +random seeds. This is load-bearing for replay-equality (#262 / #403) +and bench-gate reproducibility (per the v2.0 README rationale). + +## Bench gate + +`tests/bench_gate/test_contradiction_v3.py` runs the v3 path against +the labeled adversarial `contradiction` corpus and asserts: + +- recall on `contradicts` ≥ 0.5 +- precision on `contradicts` ≥ 0.7 + +Skip-on-no-corpus on public CI. Below-floor blocks the v3 default-on +flip; the operator decides whether to widen `ENUM_VOCAB`, tune +`DEFAULT_NUMERIC_REL_TOL`, or accept the precision/recall trade-off +under audit-only surface (`aelf resolve` style human-in-loop). + +## Maintenance + +Adding a category: +1. Append the entry to `ENUM_VOCAB` with mutually-exclusive groups. +2. Pin a unit test in `tests/test_value_compare.py` covering the + new contradiction shape. +3. Re-run the bench gate. If recall/precision drop, the new + category is too noisy — narrow it or revert. + +Tuning numeric tolerance: `DEFAULT_NUMERIC_REL_TOL` is pinned at 1%. +Drift on this value silently changes the gate; a unit test in +`test_value_compare.py` pins it explicitly. diff --git a/src/aelfrice/relationship_detector.py b/src/aelfrice/relationship_detector.py index 557b0576..7d770402 100644 --- a/src/aelfrice/relationship_detector.py +++ b/src/aelfrice/relationship_detector.py @@ -234,12 +234,47 @@ def analyze( text_b: str, *, residual_overlap_min: float = DEFAULT_RESIDUAL_OVERLAP_MIN, + use_value_comparison: bool = False, ) -> RelationshipVerdict: """Classify the relationship between two belief texts. Pure function over the two strings. Read by the audit pass and by the bench gate at ``relationship_detector.classify``. + + When ``use_value_comparison=True``, the typed-slot value-comparison + gate (#422) runs **before** the residual-overlap floor. If the + pair has at least one mutual-exclusion slot conflict, the verdict + is ``contradicts`` with a high score regardless of token overlap — + natural-language paraphrase is the failure mode that #201's R2 + gate hit (recall 0.033) and the value-comparison gate + deliberately bypasses it. When the flag is ``False`` (the default) + behaviour is unchanged from the v1 detector. """ + if use_value_comparison: + from aelfrice.value_compare import extract_values, find_conflicts + slots_a = extract_values(text_a) + slots_b = extract_values(text_b) + conflicts = find_conflicts(slots_a, slots_b) + if conflicts: + # Slot-conflict verdict bypasses the residual-overlap + # floor — the slot match itself is the relatedness + # signal. Score is fixed at 1.0 (highest confidence + # this gate emits) so the auto-emit policy in #422 + # acceptance #3 can route on (conflicts present AND + # score >= floor). + kinds = sorted({c.kind for c in conflicts}) + keys = sorted({c.key for c in conflicts}) + rationale = "value_comparison:" + ",".join( + f"{kind}={key}" for kind, key in zip(kinds, keys) + ) if len(kinds) == len(keys) else ( + "value_comparison:" + "+".join(kinds) + ) + return RelationshipVerdict( + label=LABEL_CONTRADICTS, + score=1.0, + residual_overlap=0.0, + rationale=rationale, + ) sa = extract_signals(text_a) sb = extract_signals(text_b) overlap = _residual_jaccard(sa.residual_content, sb.residual_content) @@ -281,9 +316,22 @@ def analyze( ) -def classify(text_a: str, text_b: str) -> str: - """Bench-gate entry point: return the verdict label only.""" - return analyze(text_a, text_b).label +def classify( + text_a: str, + text_b: str, + *, + use_value_comparison: bool = False, +) -> str: + """Bench-gate entry point: return the verdict label only. + + When ``use_value_comparison=True``, the v3 typed-slot gate (#422) + runs ahead of the residual-overlap floor. + """ + return analyze( + text_a, + text_b, + use_value_comparison=use_value_comparison, + ).label # --- Audit report types ------------------------------------------------ diff --git a/src/aelfrice/value_compare.py b/src/aelfrice/value_compare.py new file mode 100644 index 00000000..3e07f92b --- /dev/null +++ b/src/aelfrice/value_compare.py @@ -0,0 +1,342 @@ +"""Typed-slot value-comparison gate for contradiction detection (#422). + +Stdlib-only successor to the residual-overlap relatedness gate in +``relationship_detector``. The R2 regex shape from #201 missed real +natural-language contradictions because adversarial paraphrase +collapses Jaccard token overlap below the floor; this module +sidesteps that by extracting **typed slots** (numerics + enumerated +vocabulary) and firing ``contradicts`` directly on mutual-exclusion +across slot values. No token overlap required. + +Design: + + * Extraction is regex / vocabulary lookup — deterministic, no + embeddings, no learned classifiers. Same run produces same + slots byte-for-byte. + * Numeric slots: ``(key_token, value, unit?)``. The key is the + nearest alphabetic token preceding the number; the value is + parsed as float; the unit is the alphabetic token immediately + after the number, when present. + * Enum slots: ``(category, member)``. The category is the name + of a curated mutual-exclusion group; member is the matching + vocabulary token. Adding a category extends the gate to a new + contradiction surface. + * The comparator fires ``contradicts`` when two beliefs share a + slot key (numeric ``key_token`` or enum ``category``) with + materially different values (numeric: outside relative + tolerance; enum: different members). + +Out of scope: + + * Boolean / negation slots — already covered by the modality + pass in ``relationship_detector``; do not duplicate. + * Subject disambiguation — two beliefs that mention "alpha = 0.5" + and "alpha = 1.0" but refer to different alphas will produce a + false positive. The acceptable surface for the v3 detector is + audit + ``aelf resolve``-style human-in-loop, not auto-emit; + auto-emit policy is decided per #422 acceptance #3. +""" +from __future__ import annotations + +import re +from dataclasses import dataclass +from typing import Final + + +# --- Numeric slot extraction ------------------------------------------ + +# Capture an alphabetic key token immediately before a number, with +# optional ``=``, ``:``, ``of``, or whitespace separator. The number +# admits a leading sign, decimal, exponent. Optional alphabetic unit +# token may follow. +# +# Examples that match (key, value, unit?): +# ``alpha = 0.5`` → (alpha, 0.5, None) +# ``timeout: 30s`` → (timeout, 30, s) +# ``max_depth = 2`` → (max_depth, 2, None) +# ``set retries to 3`` → (retries, 3, None) +# ``budget of 100 nodes`` → (budget, 100, nodes) +# +# Excluded by the key requirement: bare numerics like "0.5" with no +# preceding alphabetic token (insufficient subject anchor). +_NUMERIC_RE: Final[re.Pattern[str]] = re.compile( + r""" + (?P[A-Za-z][A-Za-z0-9_]{0,31}) # key token (≤32 chars) + \s* + (?: + (?:=|:|\bis\b|\bof\b|\bto\b|\bequals?\b) + \s* + )? + (?P[+-]?\d+(?:\.\d+)?(?:[eE][+-]?\d+)?) # number + \b + """, + re.VERBOSE, +) + +# Default relative tolerance for numeric comparison. Two values that +# differ by less than this fraction of max(|a|, |b|) are treated as +# the same — guards against float-format fuzz, not against semantic +# equivalence. +DEFAULT_NUMERIC_REL_TOL: Final[float] = 0.01 + + +# --- Enum vocabulary -------------------------------------------------- + +# Each entry: ``category`` → tuple of mutually-exclusive alias groups, +# where each group is a frozenset of synonymous member tokens. Two +# beliefs contradict on this category when they tag different groups +# within the same category. Members within a single group are +# *aliases* (e.g. ``sync`` ≡ ``synchronous``) and do NOT contradict. +# +# Members must be lowercase; hyphens preserved. Adding a category +# extends the contradiction surface. The taxonomy below was chosen +# for engineering / spec contradiction patterns surfaced in the #201 +# adversarial corpus and SHOULD grow as bench evidence flags new +# patterns. Source-of-truth maintenance: this dict; review on each +# bench-gate failure. +ENUM_VOCAB: Final[dict[str, tuple[frozenset[str], ...]]] = { + "execution_mode": ( + frozenset({"synchronous", "sync"}), + frozenset({"asynchronous", "async"}), + ), + "default_state": ( + frozenset({"default-on", "enabled"}), + frozenset({"default-off", "disabled"}), + ), + "storage_mode": ( + frozenset({"indexed"}), + frozenset({"scan", "full-scan", "table-scan"}), + ), + "completeness": ( + frozenset({"full"}), + frozenset({"incremental"}), + frozenset({"partial"}), + ), + "strictness": ( + frozenset({"strict"}), + frozenset({"lax", "permissive"}), + ), + "necessity": ( + frozenset({"required"}), + frozenset({"optional"}), + ), + "visibility": ( + frozenset({"public"}), + frozenset({"private"}), + ), + "access_mode": ( + frozenset({"readonly", "read-only"}), + frozenset({"writable", "read-write"}), + ), + "determinism": ( + frozenset({"deterministic"}), + frozenset({"non-deterministic", "nondeterministic", "stochastic"}), + ), +} + +# Reverse lookup: member token → (category, group_id). The group_id +# is the alphabetically-first member of its group, used as a stable +# identifier in conflict reporting. Built once at import. +_ENUM_MEMBER_INDEX: Final[dict[str, tuple[str, str]]] = { + member: (category, sorted(group)[0]) + for category, groups in ENUM_VOCAB.items() + for group in groups + for member in group +} + + +# --- Slot dataclasses ------------------------------------------------- + + +@dataclass(frozen=True) +class NumericSlot: + """A ``key = value`` pair extracted from prose. + + ``key`` is the alphabetic token preceding the number (lowercased); + ``value`` is parsed as float. Unit-aware comparison is out of + scope — the regex's greedy capture of trailing tokens as units + introduced false negatives (e.g. ``alpha = 0.5 prior`` vs + ``alpha = 1.0 in config`` produced different units and silently + skipped the conflict). If unit-aware comparison becomes needed, + file a separate issue with a curated unit vocabulary. + """ + + key: str + value: float + + +@dataclass(frozen=True) +class EnumSlot: + """A ``(category, group_id, member)`` triple from the vocabulary. + + ``category`` is the bucket name in ``ENUM_VOCAB``. ``group_id`` + is the alphabetically-first member of the alias group the token + belongs to (stable identifier across alias swaps). ``member`` + is the actual matched token (lowercased, hyphenation preserved). + """ + + category: str + group_id: str + member: str + + +@dataclass(frozen=True) +class ValueSlots: + """All typed slots extracted from a single belief.""" + + numeric: tuple[NumericSlot, ...] + enum: tuple[EnumSlot, ...] + + +# --- Extraction ------------------------------------------------------- + + +def extract_values(text: str) -> ValueSlots: + """Extract numeric + enum slots from a single belief's text. + + Pure function. Same input → byte-identical output. + """ + numerics = _extract_numerics(text) + enums = _extract_enums(text) + return ValueSlots(numeric=numerics, enum=enums) + + +def _extract_numerics(text: str) -> tuple[NumericSlot, ...]: + out: list[NumericSlot] = [] + seen: set[tuple[str, float]] = set() + for m in _NUMERIC_RE.finditer(text): + key = m.group("key").lower() + if key in _NUMERIC_KEY_DROP: + continue + try: + value = float(m.group("value")) + except ValueError: + continue + pair = (key, value) + if pair in seen: + continue + seen.add(pair) + out.append(NumericSlot(key=key, value=value)) + return tuple(out) + + +def _extract_enums(text: str) -> tuple[EnumSlot, ...]: + lowered = text.lower() + out: list[EnumSlot] = [] + seen: set[tuple[str, str]] = set() + for member, (category, group_id) in _ENUM_MEMBER_INDEX.items(): + if re.search(rf"(? tuple[SlotConflict, ...]: + """Return all mutual-exclusion conflicts between two beliefs' slots. + + Numeric conflict: same ``(key, unit)`` with values outside the + relative-tolerance band. Unit-mismatch is silent — different + units mean different scales and the comparator cannot adjudicate + without a unit-conversion table (out of scope). + + Enum conflict: same ``category`` with different ``member`` values. + + Empty tuple means no conflict found, NOT that the pair is + related — the caller decides what no-conflict means. + """ + conflicts: list[SlotConflict] = [] + a_num_by_key: dict[str, list[NumericSlot]] = {} + for s in slots_a.numeric: + a_num_by_key.setdefault(s.key, []).append(s) + for sb in slots_b.numeric: + for sa in a_num_by_key.get(sb.key, ()): + if not _numeric_close(sa.value, sb.value, numeric_rel_tol): + conflicts.append( + SlotConflict( + kind="numeric", + key=sa.key, + value_a=_format_number(sa.value), + value_b=_format_number(sb.value), + ) + ) + + # Conflict on enum is by group_id, not member: ``sync`` and + # ``synchronous`` are aliases (same group_id) and do not conflict + # with each other. Conflict fires only when A and B tag DIFFERENT + # groups within the same category (group_id sets disjoint). + a_groups_by_cat: dict[str, set[str]] = {} + for s in slots_a.enum: + a_groups_by_cat.setdefault(s.category, set()).add(s.group_id) + b_groups_by_cat: dict[str, set[str]] = {} + for s in slots_b.enum: + b_groups_by_cat.setdefault(s.category, set()).add(s.group_id) + for category, a_groups in a_groups_by_cat.items(): + b_groups = b_groups_by_cat.get(category) + if not b_groups: + continue + if a_groups & b_groups: + continue + for ag in sorted(a_groups): + for bg in sorted(b_groups): + conflicts.append( + SlotConflict( + kind="enum", + key=category, + value_a=ag, + value_b=bg, + ) + ) + return tuple(conflicts) + + +def _numeric_close(a: float, b: float, rel_tol: float) -> bool: + if a == b: + return True + denom = max(abs(a), abs(b)) + if denom == 0.0: + return True + return abs(a - b) / denom <= rel_tol + + +def _format_number(x: float) -> str: + if x == int(x): + return str(int(x)) + return f"{x:g}" diff --git a/tests/bench_gate/test_contradiction_v3.py b/tests/bench_gate/test_contradiction_v3.py new file mode 100644 index 00000000..a697d0ac --- /dev/null +++ b/tests/bench_gate/test_contradiction_v3.py @@ -0,0 +1,74 @@ +"""Bench gate for #422 — v3 value-comparison contradiction detector. + +Acceptance #2: re-run against the labeled adversarial corpus from #201. +Target: recall on ``contradicts`` ≥ 0.5 with precision ≥ 0.7, +calibrated against #201's R2 numbers (recall 0.033, precision 0.667). + +Skips cleanly when ``AELFRICE_CORPUS_ROOT`` is unset (public CI), +when the ``contradiction/`` module dir is missing, or when the +corpus has fewer than ``MIN_CONTRADICTS`` ``contradicts``-labeled +rows (the gate requires a row floor before recall is statistically +meaningful). +""" +from __future__ import annotations + +from pathlib import Path + +import pytest + +from tests.conftest import load_corpus_module + +RECALL_FLOOR = 0.5 # per #422 acceptance #2 +PRECISION_FLOOR = 0.7 # per #422 acceptance #2 +MIN_CONTRADICTS = 30 # row floor for stable recall measurement + + +@pytest.mark.bench_gated +def test_v3_value_comparison_recall_and_precision( + aelfrice_corpus_root: Path, +) -> None: + rows = load_corpus_module(aelfrice_corpus_root, "contradiction") + + contradicts_rows = [r for r in rows if r["label"] == "contradicts"] + if len(contradicts_rows) < MIN_CONTRADICTS: + pytest.skip( + f"contradiction corpus has {len(contradicts_rows)} 'contradicts'-" + f"labeled rows; gate requires ≥{MIN_CONTRADICTS} for stable " + f"recall measurement" + ) + + from aelfrice.relationship_detector import classify + + # Confusion matrix on the contradicts vs not-contradicts axis. The + # corpus also has 'refines' / 'unrelated' labels — for precision + # we collapse those into "not-contradicts." + tp = fp = fn = tn = 0 + for r in rows: + actual_contradicts = r["label"] == "contradicts" + predicted = classify( + r["belief_a"], r["belief_b"], use_value_comparison=True + ) + predicted_contradicts = predicted == "contradicts" + if actual_contradicts and predicted_contradicts: + tp += 1 + elif actual_contradicts and not predicted_contradicts: + fn += 1 + elif not actual_contradicts and predicted_contradicts: + fp += 1 + else: + tn += 1 + + recall = tp / (tp + fn) if (tp + fn) > 0 else 0.0 + precision = tp / (tp + fp) if (tp + fp) > 0 else 0.0 + + # Diagnostic shape: surface both numbers in any failure so the + # operator can see which dimension is the gating one. + assert recall >= RECALL_FLOOR and precision >= PRECISION_FLOOR, ( + f"v3 contradiction gate: recall={recall:.3f} (floor {RECALL_FLOOR:.2f}), " + f"precision={precision:.3f} (floor {PRECISION_FLOOR:.2f}). " + f"Confusion: tp={tp} fp={fp} fn={fn} tn={tn}, " + f"n_contradicts={len(contradicts_rows)}, n_total={len(rows)}. " + f"Per #422 acceptance #2, ship requires recall ≥ {RECALL_FLOOR:.2f} " + f"AND precision ≥ {PRECISION_FLOOR:.2f}; below either floor blocks " + f"the v3 default-on flip." + ) diff --git a/tests/test_relationship_detector_v3.py b/tests/test_relationship_detector_v3.py new file mode 100644 index 00000000..8e13d245 --- /dev/null +++ b/tests/test_relationship_detector_v3.py @@ -0,0 +1,100 @@ +"""Integration tests for the v3 value-comparison gate (#422). + +Pins the contract that ``use_value_comparison=True`` activates the +typed-slot gate ahead of the residual-overlap floor, and that +``use_value_comparison=False`` (the default) preserves v1 behaviour +byte-for-byte. +""" +from __future__ import annotations + +import pytest + +from aelfrice.relationship_detector import ( + LABEL_CONTRADICTS, + LABEL_UNRELATED, + analyze, + classify, +) + + +def test_v1_default_unchanged_for_paraphrase_pair() -> None: + """The case that motivated #422: numeric mismatch with low token + overlap. v1 misses it (this is the documented failure mode).""" + a = "alpha = 0.5 prior" + b = "alpha = 1.0 in config" + # v1 (default) — falls below residual_overlap floor → unrelated. + assert classify(a, b) == LABEL_UNRELATED + + +def test_v3_catches_numeric_paraphrase_contradiction() -> None: + a = "alpha = 0.5 prior" + b = "alpha = 1.0 in config" + verdict = analyze(a, b, use_value_comparison=True) + assert verdict.label == LABEL_CONTRADICTS + assert verdict.score == 1.0 + assert "value_comparison" in verdict.rationale + assert "numeric" in verdict.rationale + + +def test_v3_catches_enum_paraphrase_contradiction() -> None: + a = "the pipeline runs synchronous on hot path" + b = "async execution model used here" + verdict = analyze(a, b, use_value_comparison=True) + assert verdict.label == LABEL_CONTRADICTS + assert "execution_mode" in verdict.rationale + + +def test_v3_unrelated_pair_still_unrelated() -> None: + """No slots, no overlap → still ``unrelated`` even with flag on.""" + a = "totally unrelated text here" + b = "something else entirely" + verdict = analyze(a, b, use_value_comparison=True) + assert verdict.label == LABEL_UNRELATED + + +def test_v3_alias_pair_does_not_falsely_contradict() -> None: + """``sync`` and ``synchronous`` are aliases, not opposites.""" + a = "use sync mode" + b = "synchronous everywhere" + # v3 should NOT emit contradicts. With short texts there's also + # too little residual overlap for v1 to call it ``refines``, so + # ``unrelated`` is the expected verdict here. + verdict = analyze(a, b, use_value_comparison=True) + assert verdict.label != LABEL_CONTRADICTS + + +def test_v3_within_tolerance_numeric_does_not_falsely_contradict() -> None: + """0.5 vs 0.502 → within tolerance → no slot conflict → falls + through to v1 modality pass.""" + a = "alpha is 0.5" + b = "alpha is 0.502" + verdict = analyze(a, b, use_value_comparison=True) + assert verdict.label != LABEL_CONTRADICTS + + +def test_classify_passes_flag_through() -> None: + """``classify`` is the bench-gate entry; the kwarg must reach + ``analyze`` unmodified. Use a pair where v1 returns the floor + verdict (``unrelated``) to make the flag-on flip unambiguous.""" + a = "alpha = 0.5 prior" + b = "alpha = 1.0 in config" + assert classify(a, b) == LABEL_UNRELATED + assert classify(a, b, use_value_comparison=True) == LABEL_CONTRADICTS + + +def test_v3_score_pinned_at_one_when_slot_fires() -> None: + """Pinning score=1.0 lets the auto-emit policy in #422 acceptance + #3 use a single threshold for slot-fire vs modality-fire.""" + a = "full backup nightly" + b = "incremental backup nightly" + verdict = analyze(a, b, use_value_comparison=True) + assert verdict.score == 1.0 + + +def test_v3_does_not_run_when_flag_off() -> None: + """Negative test: v1 path stays untouched. We assert the rationale + string never carries the v3 prefix when the flag is off.""" + a = "alpha = 0.5 in synchronous mode" + b = "alpha = 1.0 in async mode" + verdict = analyze(a, b, use_value_comparison=False) + assert "value_comparison" not in verdict.rationale diff --git a/tests/test_value_compare.py b/tests/test_value_compare.py new file mode 100644 index 00000000..da5d21d1 --- /dev/null +++ b/tests/test_value_compare.py @@ -0,0 +1,261 @@ +"""Unit tests for `aelfrice.value_compare` (#422).""" +from __future__ import annotations + +import pytest + +from aelfrice.value_compare import ( + DEFAULT_NUMERIC_REL_TOL, + ENUM_VOCAB, + EnumSlot, + NumericSlot, + SlotConflict, + ValueSlots, + extract_values, + find_conflicts, +) + + +# --------------------------------------------------------------------------- +# Numeric extraction +# --------------------------------------------------------------------------- + + +def test_numeric_simple_assignment() -> None: + s = extract_values("alpha = 0.5 prior") + assert s.numeric == (NumericSlot(key="alpha", value=0.5),) + + +def test_numeric_separator_words_picked_up() -> None: + """``is``, ``of``, ``to``, ``equals`` between key and number all work.""" + s = extract_values("set timeout to 30") + keys = {n.key for n in s.numeric} + assert "timeout" in keys + assert any(n.value == 30 for n in s.numeric) + + +def test_numeric_filler_keys_dropped() -> None: + """Bare ``is 5`` keyed on ``is`` would be a noise slot — filtered.""" + s = extract_values("is 5 the answer") + assert all(n.key != "is" for n in s.numeric) + + +def test_numeric_negative_and_decimal() -> None: + s = extract_values("offset = -1.25e-3 baseline") + assert NumericSlot(key="offset", value=-0.00125) in s.numeric + + +def test_numeric_dedup_within_belief() -> None: + s = extract_values("alpha=0.5 alpha=0.5 alpha=0.5") + assert len([n for n in s.numeric if n.key == "alpha"]) == 1 + + +def test_numeric_multiple_kv_pairs_extracted() -> None: + s = extract_values("depth=2 max_depth=4") + pairs = {(n.key, n.value) for n in s.numeric} + assert ("depth", 2.0) in pairs + assert ("max_depth", 4.0) in pairs + + +# --------------------------------------------------------------------------- +# Enum extraction +# --------------------------------------------------------------------------- + + +def test_enum_simple_match() -> None: + s = extract_values("synchronous on hot path") + assert any( + e.category == "execution_mode" and e.member == "synchronous" + for e in s.enum + ) + + +def test_enum_alias_collapse_to_group_id() -> None: + """``sync`` and ``synchronous`` both belong to the same group_id.""" + a = extract_values("use sync mode") + b = extract_values("synchronous everywhere") + a_gid = next(e.group_id for e in a.enum if e.category == "execution_mode") + b_gid = next(e.group_id for e in b.enum if e.category == "execution_mode") + assert a_gid == b_gid + + +def test_enum_hyphenated_member_match() -> None: + s = extract_values("default-on flag here") + assert any( + e.member == "default-on" for e in s.enum + ) + + +def test_enum_word_boundary_no_substring_match() -> None: + """``async`` should not match inside ``asynchrony`` (a token we don't + enumerate). ``\\b`` boundary handles this; the test pins the contract.""" + s = extract_values("the asynchronous behavior matters") + # ``asynchronous`` IS in the vocab, so it should match. The point is + # we don't ALSO match ``async`` as a substring. Verify by counting + # group_ids for the category — a single group should be tagged. + gids = {e.group_id for e in s.enum if e.category == "execution_mode"} + assert len(gids) == 1 + + +# --------------------------------------------------------------------------- +# Comparator +# --------------------------------------------------------------------------- + + +def test_no_conflict_on_empty_slots() -> None: + a = extract_values("totally unrelated text here") + b = extract_values("something else entirely") + assert find_conflicts(a, b) == () + + +def test_numeric_conflict_fires_on_value_mismatch() -> None: + a = extract_values("alpha = 0.5 prior") + b = extract_values("alpha = 1.0 in config") + conflicts = find_conflicts(a, b) + assert len(conflicts) == 1 + c = conflicts[0] + assert c.kind == "numeric" + assert c.key == "alpha" + assert {c.value_a, c.value_b} == {"0.5", "1"} + + +def test_numeric_within_relative_tolerance_no_conflict() -> None: + a = extract_values("alpha is 0.5") + b = extract_values("alpha is 0.502") + # Within DEFAULT_NUMERIC_REL_TOL (~1%) → silent + assert find_conflicts(a, b) == () + + +def test_numeric_outside_tolerance_conflict() -> None: + a = extract_values("alpha is 0.5") + b = extract_values("alpha is 0.9") + assert any(c.kind == "numeric" for c in find_conflicts(a, b)) + + +def test_numeric_zero_zero_no_conflict() -> None: + """``rel_tol`` denominator guard: 0 vs 0 is not a conflict.""" + a = extract_values("count = 0 items") + b = extract_values("count = 0 entries") + assert find_conflicts(a, b) == () + + +def test_numeric_custom_tolerance_overrides_default() -> None: + a = extract_values("alpha is 0.5") + b = extract_values("alpha is 0.6") + # 20% diff. With default 1% tol this conflicts; with 50% tol it doesn't. + assert find_conflicts(a, b, numeric_rel_tol=0.5) == () + assert find_conflicts(a, b, numeric_rel_tol=0.01) != () + + +def test_enum_conflict_on_distinct_groups() -> None: + a = extract_values("synchronous on hot path") + b = extract_values("async execution model") + conflicts = find_conflicts(a, b) + assert any( + c.kind == "enum" and c.key == "execution_mode" for c in conflicts + ) + + +def test_enum_alias_pair_does_not_conflict() -> None: + a = extract_values("use sync mode") + b = extract_values("synchronous everywhere") + assert find_conflicts(a, b) == () + + +def test_enum_default_state_conflict() -> None: + a = extract_values("default-on flag") + b = extract_values("default-off flag") + conflicts = find_conflicts(a, b) + assert any(c.key == "default_state" for c in conflicts) + + +def test_enum_enabled_disabled_conflict_via_aliases() -> None: + """``enabled``/``disabled`` share groups with default-on/default-off.""" + a = extract_values("enabled by default") + b = extract_values("disabled by default") + conflicts = find_conflicts(a, b) + assert any(c.kind == "enum" for c in conflicts) + + +def test_enum_completeness_full_vs_incremental() -> None: + a = extract_values("full backup nightly") + b = extract_values("incremental backup nightly") + assert any(c.key == "completeness" for c in find_conflicts(a, b)) + + +def test_enum_access_mode_aliases_collapse() -> None: + """``readonly`` and ``read-only`` are the same group.""" + a = extract_values("readonly mode") + b = extract_values("read-only mode") + assert find_conflicts(a, b) == () + + +def test_mixed_numeric_and_enum_conflicts_combine() -> None: + a = extract_values("alpha = 0.5 in synchronous mode") + b = extract_values("alpha = 1.0 in async mode") + conflicts = find_conflicts(a, b) + kinds = {c.kind for c in conflicts} + assert kinds == {"numeric", "enum"} + + +# --------------------------------------------------------------------------- +# Vocab integrity + determinism +# --------------------------------------------------------------------------- + + +def test_enum_vocab_groups_pairwise_disjoint_within_category() -> None: + """A member can belong to AT MOST one group within a category; + cross-category collisions are allowed (a member can mean different + things in different categories) but within-category aliasing must + be resolvable to one group_id.""" + for category, groups in ENUM_VOCAB.items(): + seen: set[str] = set() + for group in groups: + assert not (seen & group), ( + f"category {category!r} has overlapping groups; member " + f"belongs to multiple groups: {seen & group}" + ) + seen |= group + + +def test_default_numeric_rel_tol_pinned() -> None: + """Drift on this value silently changes the bench gate; pin it.""" + assert DEFAULT_NUMERIC_REL_TOL == 0.01 + + +def test_determinism_byte_identical_repeat() -> None: + text_a = "alpha = 0.5 in synchronous full mode" + text_b = "alpha = 1.0 in async incremental mode" + sa = extract_values(text_a) + sb = extract_values(text_b) + a_again = extract_values(text_a) + b_again = extract_values(text_b) + assert sa == a_again + assert sb == b_again + assert find_conflicts(sa, sb) == find_conflicts(a_again, b_again) + + +def test_value_slots_dataclass_is_hashable() -> None: + """Frozen dataclass = hashable + comparable. Future caching layers + rely on this — pin the contract.""" + slots = extract_values("alpha = 1.0") + assert hash(slots) == hash(extract_values("alpha = 1.0")) + + +def test_slot_conflict_dataclass_round_trip() -> None: + c = SlotConflict(kind="numeric", key="alpha", value_a="0.5", value_b="1") + assert c.kind == "numeric" + assert c.key == "alpha" + assert c.value_a == "0.5" + assert c.value_b == "1" + + +def test_extract_returns_value_slots_type() -> None: + s = extract_values("alpha = 0.5") + assert isinstance(s, ValueSlots) + assert isinstance(s.numeric, tuple) + assert isinstance(s.enum, tuple) + + +def test_enum_slot_dataclass_round_trip() -> None: + e = EnumSlot(category="execution_mode", group_id="async", member="async") + assert (e.category, e.group_id, e.member) == ("execution_mode", "async", "async")