From 2a9db9653ca0d86aefc95d75d95f712b7389f27a Mon Sep 17 00:00:00 2001 From: rrs <276464689+robotrocketscience@users.noreply.github.com> Date: Sun, 3 May 2026 08:11:18 -0700 Subject: [PATCH 1/3] feat(dedup): core algorithm + audit pass (#197 R1) Stdlib-only port of the research-line dedup detector. Pairs Jaccard over lowercase Unicode-word tokens (>= 0.8 default) with Levenshtein ratio (>= 0.85 default) as the second-stage gate; both must clear for a pair to count as a near-duplicate. Defaults from the #197 ratification (Jaccard 0.8, Levenshtein 0.85, max-pairs 5000). `dedup_audit(store, ...)` is read-only: walks every belief pair via direct O(n^2) Jaccard prefilter (the FTS5 path misses paraphrases where stemming diverges, e.g. "don't" / "do not"), runs Levenshtein ratio on prefilter survivors, returns a `DedupAuditReport` with pairs + union-find-collapsed clusters. No edges inserted, no beliefs mutated. Sampling is deterministic (sort by `(id_a, id_b)`, truncate at `max_candidate_pairs`) so the same store produces the same report across runs. The CLI surface (`aelf doctor dedup`) and `[dedup]` config block land in the next commit; the write-path SUPERSEDES hook is the bench-gated R2 deferred behind the corpus benchmark. 32 tests: similarity primitives, union-find clustering, audit-pass read-only contract, cluster chain collapse, threshold floor behaviour (paraphrase rejection at default Jaccard, relaxed-mode recovery), invalid-threshold guards, format_audit_report shape. --- src/aelfrice/dedup.py | 422 ++++++++++++++++++++++++++++++++++++++++++ tests/test_dedup.py | 303 ++++++++++++++++++++++++++++++ 2 files changed, 725 insertions(+) create mode 100644 src/aelfrice/dedup.py create mode 100644 tests/test_dedup.py diff --git a/src/aelfrice/dedup.py b/src/aelfrice/dedup.py new file mode 100644 index 000000000..0a91274d8 --- /dev/null +++ b/src/aelfrice/dedup.py @@ -0,0 +1,422 @@ +"""Near-duplicate detection over beliefs (#197). + +Stdlib-only port of the research-line dedup module. Two beliefs with +the same intent and different wording (e.g. "don't push to main" +locked + "never push directly to main" onboard-scraped) both surface +in retrieval today; v1.x has only `INSERT OR IGNORE` on +`(source, sentence)` content_hash, which catches exact matches but +not paraphrases. + +The detector pairs two cheap deterministic similarity signals: + +* **Jaccard** over lowercase Unicode-word tokens. Prefilter — fast, + no allocation per character pair. +* **Levenshtein ratio** (`1 - edit_distance / max(len_a, len_b)`). + Confirmation — guards against shared-vocabulary false positives + that Jaccard alone would accept. + +Both thresholds must hold for a pair to count as a duplicate: Jaccard +>= `jaccard_min` (default 0.8) **and** Levenshtein ratio >= +`levenshtein_min` (default 0.85). Defaults from the research-line +campaign; operator-tunable via `[dedup]` in `.aelfrice.toml`. + +Candidate-pair generation is direct O(n^2) Jaccard prefiltering: for +each belief pair, tokenise once (cached), compute Jaccard, and skip +the pair entirely if Jaccard < `jaccard_min`. Live-store median is +~1.6k beliefs (~1.3M pairs); set-intersection on small token sets +runs at ~1-10 us per pair so the audit pass clears in under a few +seconds. The `max_candidate_pairs` cap (default 5000) bounds the +post-prefilter pair list in case a degenerate corpus produces too +many Jaccard-positive pairs to render. Sampling is deterministic — +sorted by `(belief_id_a, belief_id_b)` and truncated, so the same +store produces the same pair list across runs. + +FTS5 is intentionally not the candidate source here: "don't" and +"do not" share zero indexed tokens after the FTS5 tokenizer's +apostrophe handling, so FTS5 misses the exact paraphrase shape this +detector targets. Jaccard over the lowercase Unicode-word tokenizer +treats them as one shared "don" token plus diverging stop-word +overlap — close enough for the prefilter, with the Levenshtein +ratio as the second-stage confirmation. + +This module is the **algorithm**. The audit-only CLI surface +(`aelf doctor dedup`) lives in `cli.py`; the write-path hook flip is +deferred behind the bench-gate per #197 ratification. +""" +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Final, Iterable + +from aelfrice.bm25 import tokenize +from aelfrice.store import MemoryStore + +# --- Defaults (research-line + #197 ratification) ----------------------- + +DEFAULT_JACCARD_MIN: Final[float] = 0.8 +DEFAULT_LEVENSHTEIN_MIN: Final[float] = 0.85 +DEFAULT_MAX_CANDIDATE_PAIRS: Final[int] = 5000 + +# --- Similarity primitives --------------------------------------------- + + +def jaccard(a: frozenset[str] | set[str], b: frozenset[str] | set[str]) -> float: + """Jaccard similarity over two token sets. + + Empty / empty returns 1.0 by convention (two zero-token strings + are considered identical for dedup purposes — they cannot + differ). One empty + one non-empty returns 0.0. + """ + if not a and not b: + return 1.0 + union = a | b + if not union: + return 1.0 + inter = a & b + return len(inter) / len(union) + + +def levenshtein_distance(a: str, b: str) -> int: + """Two-row dynamic programming edit distance. + + O(len_a * len_b) time, O(min(len_a, len_b)) space. The two-row + formulation avoids allocating a full (n+1) x (m+1) matrix; for + the ~30-200 char belief-content range this is strictly faster + than the textbook full-matrix version. + """ + if a == b: + return 0 + if not a: + return len(b) + if not b: + return len(a) + # Force `b` to be the shorter to bound row size. + if len(a) < len(b): + a, b = b, a + prev: list[int] = list(range(len(b) + 1)) + curr: list[int] = [0] * (len(b) + 1) + for i, ca in enumerate(a, start=1): + curr[0] = i + for j, cb in enumerate(b, start=1): + cost = 0 if ca == cb else 1 + curr[j] = min( + prev[j] + 1, # deletion + curr[j - 1] + 1, # insertion + prev[j - 1] + cost, # substitution + ) + prev, curr = curr, prev + return prev[len(b)] + + +def levenshtein_ratio(a: str, b: str) -> float: + """Length-normalised Levenshtein similarity in [0.0, 1.0]. + + `1 - distance / max(len_a, len_b)`. Both empty returns 1.0; one + empty + one non-empty returns 0.0. This is the + `python-Levenshtein.ratio`-equivalent shape, computed without + the C extension dependency. + """ + if not a and not b: + return 1.0 + longest = max(len(a), len(b)) + if longest == 0: + return 1.0 + return 1.0 - (levenshtein_distance(a, b) / longest) + + +# --- Pair + cluster types ---------------------------------------------- + + +@dataclass(frozen=True) +class DuplicatePair: + """A pair of beliefs that crossed both similarity thresholds. + + `belief_a_id` is the lexicographically smaller id; `belief_b_id` + the larger. Pair ordering is deterministic so two runs over the + same store produce the same pair list. + """ + belief_a_id: str + belief_b_id: str + jaccard_score: float + levenshtein_score: float + + +@dataclass(frozen=True) +class DuplicateCluster: + """A connected component of beliefs reachable via duplicate edges. + + `member_ids` is sorted lexicographically; `representative_id` is + `min(member_ids)` — the deterministic choice the audit uses to + name the cluster. The write-path hook (deferred, bench-gated) + will use the *oldest* member as the SUPERSEDES target instead, + but the audit just picks deterministically. + """ + representative_id: str + member_ids: tuple[str, ...] + + +@dataclass +class DedupAuditReport: + """Summary of one audit pass over the store. + + `pairs` is every above-threshold pair the audit saw; `clusters` + is the union-find collapse of those pairs into connected + components. `n_beliefs_scanned` is the total count of beliefs + walked, before any candidate-pair filtering. `truncated` is + `True` when the candidate-pair sample exceeded + `max_candidate_pairs` and was truncated. + """ + n_beliefs_scanned: int + n_candidate_pairs: int + n_duplicate_pairs: int + n_clusters: int + truncated: bool + pairs: tuple[DuplicatePair, ...] = field(default_factory=tuple) + clusters: tuple[DuplicateCluster, ...] = field(default_factory=tuple) + + +# --- Candidate pair generation ----------------------------------------- + + +def _jaccard_prefiltered_pairs( + beliefs: list[tuple[str, str]], + *, + jaccard_min: float, + max_pairs: int, +) -> tuple[ + list[tuple[str, str, str, str, frozenset[str], frozenset[str], float]], + int, + bool, +]: + """Return `[(id_a, content_a, id_b, content_b, tokens_a, tokens_b, + jaccard)]` for every pair clearing `jaccard_min`, plus the raw + candidate count (all O(n^2) pairs visited) and a `truncated` + boolean. + + Tokens are cached per belief id so each belief tokenises once + even if it participates in many pairs. The pair list is sorted + deterministically by `(id_a, id_b)` and truncated to `max_pairs` + if larger. + """ + n = len(beliefs) + token_cache: list[frozenset[str]] = [ + frozenset(tokenize(content)) for _, content in beliefs + ] + out: list[ + tuple[str, str, str, str, frozenset[str], frozenset[str], float] + ] = [] + raw_count = 0 + for i in range(n): + id_a, content_a = beliefs[i] + ta = token_cache[i] + if not content_a.strip(): + continue + for j in range(i + 1, n): + id_b, content_b = beliefs[j] + tb = token_cache[j] + if not content_b.strip(): + continue + raw_count += 1 + j_score = jaccard(ta, tb) + if j_score < jaccard_min: + continue + # Canonicalise (id_a < id_b) — i < j and ids are sorted + # ASC by list_beliefs_for_indexing, so this holds. + out.append((id_a, content_a, id_b, content_b, ta, tb, j_score)) + out.sort(key=lambda row: (row[0], row[2])) + truncated = len(out) > max_pairs + if truncated: + out = out[:max_pairs] + return out, raw_count, truncated + + +# --- Union-find for cluster collapse ----------------------------------- + + +class _UnionFind: + """Minimal union-find / DSU for clustering duplicate pairs. + + Path compression on `find`; union by size. Operations are + effectively O(alpha(n)) per call. + """ + + def __init__(self) -> None: + self._parent: dict[str, str] = {} + self._size: dict[str, int] = {} + + def make(self, x: str) -> None: + if x not in self._parent: + self._parent[x] = x + self._size[x] = 1 + + def find(self, x: str) -> str: + path: list[str] = [] + while self._parent[x] != x: + path.append(x) + x = self._parent[x] + for p in path: + self._parent[p] = x + return x + + def union(self, a: str, b: str) -> None: + ra, rb = self.find(a), self.find(b) + if ra == rb: + return + if self._size[ra] < self._size[rb]: + ra, rb = rb, ra + self._parent[rb] = ra + self._size[ra] += self._size[rb] + + def groups(self) -> dict[str, list[str]]: + out: dict[str, list[str]] = {} + for x in self._parent: + r = self.find(x) + out.setdefault(r, []).append(x) + return out + + +def cluster_pairs(pairs: Iterable[DuplicatePair]) -> tuple[DuplicateCluster, ...]: + """Collapse duplicate pairs into connected components. + + Each cluster's `representative_id` is `min(member_ids)`; both + cluster lists and the cluster tuple itself are sorted + deterministically. + """ + uf = _UnionFind() + pair_list = list(pairs) + for p in pair_list: + uf.make(p.belief_a_id) + uf.make(p.belief_b_id) + uf.union(p.belief_a_id, p.belief_b_id) + groups = uf.groups() + clusters: list[DuplicateCluster] = [] + for members in groups.values(): + if len(members) < 2: + continue + sorted_members = tuple(sorted(members)) + clusters.append( + DuplicateCluster( + representative_id=sorted_members[0], + member_ids=sorted_members, + ) + ) + clusters.sort(key=lambda c: c.representative_id) + return tuple(clusters) + + +# --- Top-level audit entry point --------------------------------------- + + +def dedup_audit( + store: MemoryStore, + *, + jaccard_min: float = DEFAULT_JACCARD_MIN, + levenshtein_min: float = DEFAULT_LEVENSHTEIN_MIN, + max_candidate_pairs: int = DEFAULT_MAX_CANDIDATE_PAIRS, +) -> DedupAuditReport: + """Walk the store, find near-duplicate belief pairs, return a report. + + Read-only: no edges are inserted, no beliefs are mutated. The + write-path hook (insert SUPERSEDES edges) is deferred behind the + #197 bench gate. + + Raises `ValueError` on malformed thresholds; degrades gracefully + on per-belief FTS5 errors (skips that belief, logs nothing). + """ + if not 0.0 <= jaccard_min <= 1.0: + raise ValueError( + f"jaccard_min must be in [0.0, 1.0], got {jaccard_min}", + ) + if not 0.0 <= levenshtein_min <= 1.0: + raise ValueError( + f"levenshtein_min must be in [0.0, 1.0], got {levenshtein_min}", + ) + if max_candidate_pairs < 1: + raise ValueError( + f"max_candidate_pairs must be >= 1, got {max_candidate_pairs}", + ) + + beliefs = store.list_beliefs_for_indexing() + n_beliefs = len(beliefs) + if n_beliefs < 2: + return DedupAuditReport( + n_beliefs_scanned=n_beliefs, + n_candidate_pairs=0, + n_duplicate_pairs=0, + n_clusters=0, + truncated=False, + ) + + candidates, raw_count, truncated = _jaccard_prefiltered_pairs( + beliefs, + jaccard_min=jaccard_min, + max_pairs=max_candidate_pairs, + ) + + pairs: list[DuplicatePair] = [] + for id_a, content_a, id_b, content_b, _ta, _tb, j_score in candidates: + lr = levenshtein_ratio(content_a, content_b) + if lr < levenshtein_min: + continue + pairs.append( + DuplicatePair( + belief_a_id=id_a, + belief_b_id=id_b, + jaccard_score=j_score, + levenshtein_score=lr, + ) + ) + pairs.sort(key=lambda p: (p.belief_a_id, p.belief_b_id)) + clusters = cluster_pairs(pairs) + return DedupAuditReport( + n_beliefs_scanned=n_beliefs, + n_candidate_pairs=raw_count, + n_duplicate_pairs=len(pairs), + n_clusters=len(clusters), + truncated=truncated, + pairs=tuple(pairs), + clusters=clusters, + ) + + +def format_audit_report(report: DedupAuditReport) -> str: + """Render a `DedupAuditReport` as a human-readable plain-text block. + + Used by `aelf doctor dedup`. The shape mirrors `format_orphan_report` + et al. in `doctor.py` so the doctor surface stays consistent. + """ + lines: list[str] = [] + lines.append("aelf doctor dedup") + lines.append("=" * 40) + lines.append(f"Beliefs scanned : {report.n_beliefs_scanned}") + lines.append(f"Candidate pairs visited : {report.n_candidate_pairs}") + if report.truncated: + lines.append( + f" (truncated to {DEFAULT_MAX_CANDIDATE_PAIRS} — see " + f"[dedup] max_candidate_pairs)" + ) + lines.append(f"Duplicate pairs : {report.n_duplicate_pairs}") + lines.append(f"Duplicate clusters : {report.n_clusters}") + lines.append("") + if report.n_clusters == 0: + lines.append("No near-duplicates above the configured thresholds.") + return "\n".join(lines) + lines.append("Clusters:") + for cluster in report.clusters: + lines.append( + f" {cluster.representative_id} " + f"({len(cluster.member_ids)} members)" + ) + for mid in cluster.member_ids: + marker = "*" if mid == cluster.representative_id else " " + lines.append(f" {marker} {mid}") + lines.append("") + lines.append("Top duplicate pairs (jaccard, levenshtein):") + for p in report.pairs[:25]: + lines.append( + f" {p.belief_a_id} ~ {p.belief_b_id} " + f"(j={p.jaccard_score:.3f}, l={p.levenshtein_score:.3f})" + ) + if len(report.pairs) > 25: + lines.append(f" ... ({len(report.pairs) - 25} more)") + return "\n".join(lines) diff --git a/tests/test_dedup.py b/tests/test_dedup.py new file mode 100644 index 000000000..78538f6fc --- /dev/null +++ b/tests/test_dedup.py @@ -0,0 +1,303 @@ +"""Unit tests for `aelfrice.dedup` (#197 R1). + +Covers the four similarity primitives (jaccard / levenshtein / +levenshtein_ratio / cluster_pairs) plus the top-level `dedup_audit` +entry point against a real `MemoryStore` fixture. + +The audit is read-only — every test asserts that no edges and no +beliefs were inserted/mutated by the audit pass. +""" +from __future__ import annotations + +import pytest + +from aelfrice.dedup import ( + DEFAULT_JACCARD_MIN, + DEFAULT_LEVENSHTEIN_MIN, + DuplicateCluster, + DuplicatePair, + cluster_pairs, + dedup_audit, + format_audit_report, + jaccard, + levenshtein_distance, + levenshtein_ratio, +) +from aelfrice.models import BELIEF_FACTUAL, LOCK_NONE, Belief +from aelfrice.store import MemoryStore + + +# --- Similarity primitives --------------------------------------------- + + +class TestJaccard: + def test_both_empty_is_one(self) -> None: + assert jaccard(frozenset(), frozenset()) == 1.0 + + def test_one_empty_is_zero(self) -> None: + assert jaccard(frozenset({"a"}), frozenset()) == 0.0 + assert jaccard(frozenset(), frozenset({"a"})) == 0.0 + + def test_identical_is_one(self) -> None: + s = frozenset({"the", "cat", "sat"}) + assert jaccard(s, s) == 1.0 + + def test_disjoint_is_zero(self) -> None: + a = frozenset({"x", "y"}) + b = frozenset({"u", "v"}) + assert jaccard(a, b) == 0.0 + + def test_partial_overlap(self) -> None: + a = frozenset({"a", "b", "c"}) + b = frozenset({"b", "c", "d"}) + # |a∩b|=2, |a∪b|=4 + assert jaccard(a, b) == 0.5 + + +class TestLevenshtein: + def test_distance_identical(self) -> None: + assert levenshtein_distance("kitten", "kitten") == 0 + + def test_distance_empty(self) -> None: + assert levenshtein_distance("", "abc") == 3 + assert levenshtein_distance("abc", "") == 3 + assert levenshtein_distance("", "") == 0 + + def test_distance_classic(self) -> None: + # textbook: kitten -> sitting is distance 3 + assert levenshtein_distance("kitten", "sitting") == 3 + + def test_distance_substitution(self) -> None: + assert levenshtein_distance("cat", "bat") == 1 + + def test_distance_insertion(self) -> None: + assert levenshtein_distance("cat", "cats") == 1 + + def test_distance_deletion(self) -> None: + assert levenshtein_distance("cats", "cat") == 1 + + def test_ratio_identical(self) -> None: + assert levenshtein_ratio("hello", "hello") == 1.0 + + def test_ratio_empty_pair(self) -> None: + assert levenshtein_ratio("", "") == 1.0 + assert levenshtein_ratio("", "abc") == 0.0 + assert levenshtein_ratio("abc", "") == 0.0 + + def test_ratio_classic(self) -> None: + # kitten/sitting: dist=3, max_len=7 → ratio = 1 - 3/7 ≈ 0.571 + ratio = levenshtein_ratio("kitten", "sitting") + assert abs(ratio - (1 - 3 / 7)) < 1e-9 + + def test_ratio_close_paraphrase_clears_default(self) -> None: + a = "don't push to main" + b = "do not push to main" + # 18 vs 19 chars; dist 3 (don't → do not). ratio = 1 - 3/19 ≈ 0.842 + assert levenshtein_ratio(a, b) >= 0.84 + + +# --- Union-find + cluster collapse ------------------------------------- + + +class TestClusterPairs: + def test_no_pairs_returns_empty(self) -> None: + assert cluster_pairs([]) == () + + def test_single_pair(self) -> None: + pair = DuplicatePair("a", "b", 1.0, 1.0) + clusters = cluster_pairs([pair]) + assert clusters == ( + DuplicateCluster(representative_id="a", member_ids=("a", "b")), + ) + + def test_chain_collapses(self) -> None: + # a~b, b~c, c~d should produce one cluster {a,b,c,d} + pairs = [ + DuplicatePair("a", "b", 1.0, 1.0), + DuplicatePair("b", "c", 1.0, 1.0), + DuplicatePair("c", "d", 1.0, 1.0), + ] + clusters = cluster_pairs(pairs) + assert len(clusters) == 1 + assert clusters[0].representative_id == "a" + assert clusters[0].member_ids == ("a", "b", "c", "d") + + def test_disjoint_clusters(self) -> None: + pairs = [ + DuplicatePair("a", "b", 1.0, 1.0), + DuplicatePair("c", "d", 1.0, 1.0), + ] + clusters = cluster_pairs(pairs) + assert len(clusters) == 2 + reps = sorted(c.representative_id for c in clusters) + assert reps == ["a", "c"] + + def test_representative_is_min_member(self) -> None: + pair = DuplicatePair("zeta", "alpha", 1.0, 1.0) + clusters = cluster_pairs([pair]) + # member_ids is sorted: alpha < zeta + assert clusters[0].representative_id == "alpha" + assert clusters[0].member_ids == ("alpha", "zeta") + + +# --- Top-level audit against a real store ------------------------------- + + +def _insert(store: MemoryStore, bid: str, content: str) -> None: + """Helper: minimal Belief insertion for audit-pass test fixtures.""" + store.insert_belief( + Belief( + id=bid, + content=content, + content_hash=f"h_{bid}", + alpha=1.0, + beta=1.0, + type=BELIEF_FACTUAL, + lock_level=LOCK_NONE, + locked_at=None, + demotion_pressure=0, + created_at="2026-05-03T00:00:00Z", + last_retrieved_at=None, + ) + ) + + +@pytest.fixture +def store() -> MemoryStore: + return MemoryStore(":memory:") + + +class TestDedupAudit: + def test_empty_store(self, store: MemoryStore) -> None: + report = dedup_audit(store) + assert report.n_beliefs_scanned == 0 + assert report.n_duplicate_pairs == 0 + assert report.n_clusters == 0 + assert report.pairs == () + assert report.clusters == () + + def test_single_belief(self, store: MemoryStore) -> None: + _insert(store, "b1", "alone in the store") + report = dedup_audit(store) + assert report.n_beliefs_scanned == 1 + assert report.n_candidate_pairs == 0 + assert report.n_duplicate_pairs == 0 + + def test_finds_near_duplicate_pair(self, store: MemoryStore) -> None: + # Trailing punctuation only — tokens identical (Jaccard = 1.0), + # 1-char edit distance (Levenshtein ratio ~0.97). + _insert(store, "b1", "deploy via terraform on aws") + _insert(store, "b2", "deploy via terraform on aws.") + _insert(store, "b3", "the cat sat on the mat") + report = dedup_audit(store) + assert report.n_beliefs_scanned == 3 + assert report.n_duplicate_pairs == 1 + p = report.pairs[0] + assert (p.belief_a_id, p.belief_b_id) == ("b1", "b2") + assert p.jaccard_score >= DEFAULT_JACCARD_MIN + assert p.levenshtein_score >= DEFAULT_LEVENSHTEIN_MIN + assert report.n_clusters == 1 + + def test_distinct_beliefs_no_duplicates(self, store: MemoryStore) -> None: + _insert(store, "b1", "deploy via terraform on aws") + _insert(store, "b2", "the database is postgres 15") + _insert(store, "b3", "monitoring dashboards live in grafana") + report = dedup_audit(store) + assert report.n_duplicate_pairs == 0 + assert report.n_clusters == 0 + + def test_threshold_floor_rejects_jaccard_only_match( + self, store: MemoryStore + ) -> None: + # Same words, very different word order → high Jaccard but + # the Levenshtein floor should reject this as too far apart. + _insert( + store, "b1", + "main branch push policy is no direct pushes ever allowed", + ) + _insert( + store, "b2", + "ever allowed pushes direct no is policy push branch main", + ) + report = dedup_audit(store) + # Jaccard would be ~1.0 (same word set) but Levenshtein + # ratio on the strings is far below 0.85. + assert report.n_duplicate_pairs == 0 + + def test_audit_is_read_only(self, store: MemoryStore) -> None: + _insert(store, "b1", "deploy via terraform on aws") + _insert(store, "b2", "deploy via terraform on aws.") + ids_before = sorted(store.list_belief_ids()) + _ = dedup_audit(store) + # No new beliefs, no edges inserted. (#197 audit-only contract.) + assert sorted(store.list_belief_ids()) == ids_before + + def test_cluster_chain(self, store: MemoryStore) -> None: + # Three near-identical (punctuation-only diffs) → one cluster of 3. + _insert(store, "b1", "deploy via terraform on aws") + _insert(store, "b2", "deploy via terraform on aws.") + _insert(store, "b3", "deploy via terraform on aws!") + report = dedup_audit(store) + assert report.n_clusters == 1 + cluster = report.clusters[0] + assert set(cluster.member_ids) == {"b1", "b2", "b3"} + + def test_paraphrase_below_jaccard_floor_is_not_a_duplicate( + self, store: MemoryStore + ) -> None: + # Documents the threshold semantics: "don't" / "do not" + # tokenise into disjoint sets ({don, t} vs {do, not}) so even + # though Levenshtein clears, Jaccard does not. The ratified + # 0.8 Jaccard floor is strict by design; the write-path hook + # (deferred, bench-gated) is the place to reconsider. + _insert(store, "b1", "don't push directly to main") + _insert(store, "b2", "do not push directly to main") + report = dedup_audit(store) + assert report.n_duplicate_pairs == 0 + + def test_invalid_thresholds_raise(self, store: MemoryStore) -> None: + with pytest.raises(ValueError): + dedup_audit(store, jaccard_min=-0.1) + with pytest.raises(ValueError): + dedup_audit(store, jaccard_min=1.5) + with pytest.raises(ValueError): + dedup_audit(store, levenshtein_min=-0.1) + with pytest.raises(ValueError): + dedup_audit(store, max_candidate_pairs=0) + + def test_threshold_relaxation_finds_more(self, store: MemoryStore) -> None: + # Same intent, different conjunction word — tokens diverge enough + # at default 0.8 Jaccard to be rejected; relaxed Jaccard finds it. + _insert(store, "b1", "do not push directly to main branch") + _insert(store, "b2", "never push directly to main branch") + strict = dedup_audit(store) + relaxed = dedup_audit(store, jaccard_min=0.5, levenshtein_min=0.7) + assert strict.n_duplicate_pairs == 0 + assert relaxed.n_duplicate_pairs == 1 + + +class TestFormatAuditReport: + def test_empty_report_renders(self) -> None: + from aelfrice.dedup import DedupAuditReport + r = DedupAuditReport( + n_beliefs_scanned=0, + n_candidate_pairs=0, + n_duplicate_pairs=0, + n_clusters=0, + truncated=False, + ) + out = format_audit_report(r) + assert "aelf doctor dedup" in out + assert "Beliefs scanned" in out + assert "No near-duplicates" in out + + def test_clustered_report_lists_members( + self, store: MemoryStore + ) -> None: + _insert(store, "b1", "deploy via terraform on aws") + _insert(store, "b2", "deploy via terraform on aws.") + report = dedup_audit(store) + out = format_audit_report(report) + assert "b1" in out + assert "b2" in out + assert "Clusters:" in out From 42c8dbd02f989f86c7a22d22ea37a0db1be3bd08 Mon Sep 17 00:00:00 2001 From: rrs <276464689+robotrocketscience@users.noreply.github.com> Date: Sun, 3 May 2026 08:13:46 -0700 Subject: [PATCH 2/3] feat(dedup): aelf doctor dedup CLI + [dedup] config (#197 R1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wires the audit pass behind `aelf doctor --dedup`. New flags `--dedup-jaccard`, `--dedup-levenshtein`, `--dedup-max-pairs` override the per-run thresholds; `[dedup]` block in `.aelfrice.toml` provides project-level defaults. Read-only — no edges inserted, no beliefs mutated; the write-path SUPERSEDES hook is the bench-gated R2 deferred behind the corpus benchmark per #197 ratification. Config loader follows the `[rebuilder]` / `[implicit_feedback]` convention: walk up from cwd looking for `.aelfrice.toml`, malformed values degrade to defaults with a stderr trace, never raises. 8 new tests: TOML loader (default fallthrough, well-formed override, out-of-range fallback, wrong-type fallback, malformed-TOML fallback) + CLI integration (clean store exit 0, pair detection through main(), threshold overrides via flags). Existing 100 cli/doctor tests still pass. --- src/aelfrice/cli.py | 109 ++++++++++++++++++++++++++++++++++ src/aelfrice/dedup.py | 124 +++++++++++++++++++++++++++++++++++++- tests/test_dedup.py | 135 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 367 insertions(+), 1 deletion(-) diff --git a/src/aelfrice/cli.py b/src/aelfrice/cli.py index b23f395e7..7cbfc4300 100644 --- a/src/aelfrice/cli.py +++ b/src/aelfrice/cli.py @@ -2135,6 +2135,8 @@ def _cmd_doctor(args: argparse.Namespace, out: object) -> int: return _cmd_doctor_promote_retention(args, out) if getattr(args, "replay", False): return _cmd_doctor_replay(args, out) + if getattr(args, "dedup", False): + return _cmd_doctor_dedup(args, out) scope = getattr(args, "scope", None) exit_code = 0 if scope in (None, "hooks"): @@ -2165,6 +2167,64 @@ def _cmd_doctor(args: argparse.Namespace, out: object) -> int: return exit_code +def _cmd_doctor_dedup(args: argparse.Namespace, out: object) -> int: + """Run the v2.0 dedup audit (#197 R1). + + Walks the store, finds near-duplicate belief pairs with Jaccard >= + `--dedup-jaccard` AND Levenshtein ratio >= `--dedup-levenshtein`, + and prints a clustered report. Read-only: no edges are inserted, + no beliefs are mutated. The write-path SUPERSEDES hook is the + bench-gated R2 deferred behind the corpus benchmark. + + Exit 0 on success regardless of cluster count — clusters are + diagnostic, not failure conditions. Exit 1 only on store-open + errors. + """ + from aelfrice.dedup import ( + DedupConfig, + dedup_audit, + format_audit_report, + load_dedup_config, + ) + + config = load_dedup_config() + j_override = getattr(args, "dedup_jaccard", None) + l_override = getattr(args, "dedup_levenshtein", None) + mp_override = getattr(args, "dedup_max_pairs", None) + config = DedupConfig( + jaccard_min=( + float(j_override) if j_override is not None else config.jaccard_min + ), + levenshtein_min=( + float(l_override) + if l_override is not None + else config.levenshtein_min + ), + max_candidate_pairs=( + int(mp_override) + if mp_override is not None + else config.max_candidate_pairs + ), + ) + + store = _open_store() + try: + report = dedup_audit( + store, + jaccard_min=config.jaccard_min, + levenshtein_min=config.levenshtein_min, + max_candidate_pairs=config.max_candidate_pairs, + ) + except ValueError as exc: + print(f"aelf doctor dedup: {exc}", file=sys.stderr) + return 1 + finally: + store.close() + + print(format_audit_report(report), file=out) # type: ignore[arg-type] + return 0 + + def _cmd_doctor_replay(args: argparse.Namespace, out: object) -> int: """Run the v2.x full-equality replay probe (#262). @@ -2963,6 +3023,55 @@ def build_parser(*, show_advanced: bool = False) -> argparse.ArgumentParser: "(exists for forward compatibility)." ), ) + p_doctor.add_argument( + "--dedup", + dest="dedup", + action="store_true", + default=False, + help=( + "find near-duplicate beliefs (Jaccard + Levenshtein gate) " + "and print clustered candidates (#197). Read-only: no edges " + "are inserted. Bypasses the hooks/graph checks. Tune via " + "--dedup-jaccard / --dedup-levenshtein / --dedup-max-pairs " + "or [dedup] in .aelfrice.toml." + ), + ) + p_doctor.add_argument( + "--dedup-jaccard", + dest="dedup_jaccard", + type=float, + default=None, + metavar="F", + help=( + "with --dedup: override the Jaccard prefilter threshold " + "(0.0-1.0). Default: [dedup] jaccard_min in .aelfrice.toml > " + "0.8." + ), + ) + p_doctor.add_argument( + "--dedup-levenshtein", + dest="dedup_levenshtein", + type=float, + default=None, + metavar="F", + help=( + "with --dedup: override the Levenshtein-ratio confirmation " + "threshold (0.0-1.0). Default: [dedup] levenshtein_min in " + ".aelfrice.toml > 0.85." + ), + ) + p_doctor.add_argument( + "--dedup-max-pairs", + dest="dedup_max_pairs", + type=int, + default=None, + metavar="N", + help=( + "with --dedup: cap reported duplicate pairs after Jaccard " + "prefilter; deterministic truncation by (id_a, id_b). " + "Default: [dedup] max_candidate_pairs in .aelfrice.toml > 5000." + ), + ) p_doctor.set_defaults(func=_cmd_doctor) p_sweep_feedback = sub.add_parser( diff --git a/src/aelfrice/dedup.py b/src/aelfrice/dedup.py index 0a91274d8..85c4aed04 100644 --- a/src/aelfrice/dedup.py +++ b/src/aelfrice/dedup.py @@ -45,12 +45,21 @@ """ from __future__ import annotations +import sys +import tomllib from dataclasses import dataclass, field -from typing import Final, Iterable +from pathlib import Path +from typing import Any, Final, IO, Iterable, cast from aelfrice.bm25 import tokenize from aelfrice.store import MemoryStore +CONFIG_FILENAME: Final[str] = ".aelfrice.toml" +DEDUP_SECTION: Final[str] = "dedup" +JACCARD_MIN_KEY: Final[str] = "jaccard_min" +LEVENSHTEIN_MIN_KEY: Final[str] = "levenshtein_min" +MAX_CANDIDATE_PAIRS_KEY: Final[str] = "max_candidate_pairs" + # --- Defaults (research-line + #197 ratification) ----------------------- DEFAULT_JACCARD_MIN: Final[float] = 0.8 @@ -379,6 +388,119 @@ def dedup_audit( ) +@dataclass(frozen=True) +class DedupConfig: + """Resolved `[dedup]` section of `.aelfrice.toml`. + + All fields default to the module-level constants; any may be + overridden in a project-local `.aelfrice.toml`. Malformed values + fall back to the default with a stderr trace, matching the + `[rebuilder]` / `[implicit_feedback]` config-resolution + convention. + """ + jaccard_min: float = DEFAULT_JACCARD_MIN + levenshtein_min: float = DEFAULT_LEVENSHTEIN_MIN + max_candidate_pairs: int = DEFAULT_MAX_CANDIDATE_PAIRS + + +def _load_float_in_unit_interval( + section: dict[str, Any], + key: str, + default: float, + candidate: Path, + serr: IO[str], +) -> float: + obj: Any = section.get(key, default) + if isinstance(obj, bool) or not isinstance(obj, (int, float)): + print( + f"aelfrice dedup: ignoring [{DEDUP_SECTION}] {key} in " + f"{candidate} (expected float in [0.0, 1.0])", + file=serr, + ) + return default + val = float(obj) + if not 0.0 <= val <= 1.0: + print( + f"aelfrice dedup: ignoring [{DEDUP_SECTION}] {key} in " + f"{candidate} (expected float in [0.0, 1.0])", + file=serr, + ) + return default + return val + + +def load_dedup_config(start: Path | None = None) -> DedupConfig: + """Walk up from `start` looking for `.aelfrice.toml`. + + Returns the resolved `[dedup]` config. Missing file / missing + section / malformed TOML / wrong-typed values all degrade to + defaults with a stderr trace; never raises. + """ + serr: IO[str] = sys.stderr + current = (start if start is not None else Path.cwd()).resolve() + seen: set[Path] = set() + while current not in seen: + seen.add(current) + candidate = current / CONFIG_FILENAME + if candidate.is_file(): + try: + raw = candidate.read_bytes() + except OSError as exc: + print( + f"aelfrice dedup: cannot read {candidate}: {exc}", + file=serr, + ) + return DedupConfig() + try: + parsed: dict[str, Any] = tomllib.loads( + raw.decode("utf-8", errors="replace"), + ) + except tomllib.TOMLDecodeError as exc: + print( + f"aelfrice dedup: malformed TOML in {candidate}: {exc}", + file=serr, + ) + return DedupConfig() + section_obj: Any = parsed.get(DEDUP_SECTION, {}) + if not isinstance(section_obj, dict): + return DedupConfig() + section = cast(dict[str, Any], section_obj) + j_min = _load_float_in_unit_interval( + section, JACCARD_MIN_KEY, DEFAULT_JACCARD_MIN, + candidate, serr, + ) + l_min = _load_float_in_unit_interval( + section, LEVENSHTEIN_MIN_KEY, DEFAULT_LEVENSHTEIN_MIN, + candidate, serr, + ) + mp_obj: Any = section.get( + MAX_CANDIDATE_PAIRS_KEY, DEFAULT_MAX_CANDIDATE_PAIRS, + ) + if ( + isinstance(mp_obj, bool) + or not isinstance(mp_obj, int) + or mp_obj < 1 + ): + print( + f"aelfrice dedup: ignoring [{DEDUP_SECTION}] " + f"{MAX_CANDIDATE_PAIRS_KEY} in {candidate} " + f"(expected positive int)", + file=serr, + ) + mp_resolved = DEFAULT_MAX_CANDIDATE_PAIRS + else: + mp_resolved = mp_obj + return DedupConfig( + jaccard_min=j_min, + levenshtein_min=l_min, + max_candidate_pairs=mp_resolved, + ) + if current.parent == current: + break + current = current.parent + return DedupConfig() + + def format_audit_report(report: DedupAuditReport) -> str: """Render a `DedupAuditReport` as a human-readable plain-text block. diff --git a/tests/test_dedup.py b/tests/test_dedup.py index 78538f6fc..d9baaf150 100644 --- a/tests/test_dedup.py +++ b/tests/test_dedup.py @@ -301,3 +301,138 @@ def test_clustered_report_lists_members( assert "b1" in out assert "b2" in out assert "Clusters:" in out + + +# --- TOML config loader ------------------------------------------------- + + +class TestLoadDedupConfig: + def test_no_config_returns_defaults(self, tmp_path) -> None: + from aelfrice.dedup import ( + DEFAULT_JACCARD_MIN, + DEFAULT_LEVENSHTEIN_MIN, + DEFAULT_MAX_CANDIDATE_PAIRS, + load_dedup_config, + ) + cfg = load_dedup_config(tmp_path) + assert cfg.jaccard_min == DEFAULT_JACCARD_MIN + assert cfg.levenshtein_min == DEFAULT_LEVENSHTEIN_MIN + assert cfg.max_candidate_pairs == DEFAULT_MAX_CANDIDATE_PAIRS + + def test_well_formed_config_overrides(self, tmp_path) -> None: + from aelfrice.dedup import load_dedup_config + (tmp_path / ".aelfrice.toml").write_text( + "[dedup]\n" + "jaccard_min = 0.7\n" + "levenshtein_min = 0.9\n" + "max_candidate_pairs = 1000\n" + ) + cfg = load_dedup_config(tmp_path) + assert cfg.jaccard_min == 0.7 + assert cfg.levenshtein_min == 0.9 + assert cfg.max_candidate_pairs == 1000 + + def test_out_of_range_falls_back(self, tmp_path) -> None: + from aelfrice.dedup import ( + DEFAULT_JACCARD_MIN, + load_dedup_config, + ) + (tmp_path / ".aelfrice.toml").write_text( + "[dedup]\njaccard_min = 1.5\n" + ) + cfg = load_dedup_config(tmp_path) + assert cfg.jaccard_min == DEFAULT_JACCARD_MIN + + def test_wrong_type_falls_back(self, tmp_path) -> None: + from aelfrice.dedup import ( + DEFAULT_MAX_CANDIDATE_PAIRS, + load_dedup_config, + ) + (tmp_path / ".aelfrice.toml").write_text( + '[dedup]\nmax_candidate_pairs = "lots"\n' + ) + cfg = load_dedup_config(tmp_path) + assert cfg.max_candidate_pairs == DEFAULT_MAX_CANDIDATE_PAIRS + + def test_malformed_toml_falls_back(self, tmp_path) -> None: + from aelfrice.dedup import DedupConfig, load_dedup_config + (tmp_path / ".aelfrice.toml").write_text( + "[dedup\nthis is not valid toml\n" + ) + cfg = load_dedup_config(tmp_path) + assert cfg == DedupConfig() + + +# --- CLI integration ---------------------------------------------------- + + +class TestCLIDoctorDedup: + def test_clean_store_exit_0( + self, tmp_path, monkeypatch: pytest.MonkeyPatch, + ) -> None: + import io + from aelfrice.cli import main + + db = str(tmp_path / "brain.db") + monkeypatch.setenv("AELFRICE_DB", db) + s = MemoryStore(db) + _insert(s, "b1", "the cat sat on the mat") + _insert(s, "b2", "deploy via terraform") + s.close() + + out = io.StringIO() + rc = main(["doctor", "--dedup"], out=out) + assert rc == 0 + assert "aelf doctor dedup" in out.getvalue() + assert "No near-duplicates" in out.getvalue() + + def test_finds_duplicate_via_cli( + self, tmp_path, monkeypatch: pytest.MonkeyPatch, + ) -> None: + import io + from aelfrice.cli import main + + db = str(tmp_path / "brain.db") + monkeypatch.setenv("AELFRICE_DB", db) + s = MemoryStore(db) + _insert(s, "b1", "deploy via terraform on aws") + _insert(s, "b2", "deploy via terraform on aws.") + s.close() + + out = io.StringIO() + rc = main(["doctor", "--dedup"], out=out) + assert rc == 0 + text = out.getvalue() + assert "Duplicate pairs : 1" in text + assert "b1" in text and "b2" in text + + def test_threshold_overrides_via_flags( + self, tmp_path, monkeypatch: pytest.MonkeyPatch, + ) -> None: + import io + from aelfrice.cli import main + + db = str(tmp_path / "brain.db") + monkeypatch.setenv("AELFRICE_DB", db) + s = MemoryStore(db) + # Pair fails at default but passes when both thresholds drop. + _insert(s, "b1", "do not push directly to main branch") + _insert(s, "b2", "never push directly to main branch") + s.close() + + out_strict = io.StringIO() + rc1 = main(["doctor", "--dedup"], out=out_strict) + assert rc1 == 0 + assert "Duplicate pairs : 0" in out_strict.getvalue() + + out_relaxed = io.StringIO() + rc2 = main( + [ + "doctor", "--dedup", + "--dedup-jaccard", "0.5", + "--dedup-levenshtein", "0.7", + ], + out=out_relaxed, + ) + assert rc2 == 0 + assert "Duplicate pairs : 1" in out_relaxed.getvalue() From 931c20e2f99951bc1815a200b55c9b3dafa685ef Mon Sep 17 00:00:00 2001 From: rrs <276464689+robotrocketscience@users.noreply.github.com> Date: Sun, 3 May 2026 08:14:53 -0700 Subject: [PATCH 3/3] docs(dedup): user surface + LIMITATIONS shrink (#197 R1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `docs/dedup.md`: usage, per-run flags, [dedup] config block, defaults table, output-shape sample, explicit "audit-only by design — write-path is bench-gated R2" call-out. LIMITATIONS § Sharp edges: new bullet on near-duplicates from different ingest paths, pointing at the new audit surface. Tightens but does not remove the existing onboard-non-incremental caveat — the audit lists clusters, it does not collapse them. --- docs/LIMITATIONS.md | 1 + docs/dedup.md | 84 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 85 insertions(+) create mode 100644 docs/dedup.md diff --git a/docs/LIMITATIONS.md b/docs/LIMITATIONS.md index 9b01f567f..e2d281680 100644 --- a/docs/LIMITATIONS.md +++ b/docs/LIMITATIONS.md @@ -41,6 +41,7 @@ The default retrieval mode (recall, not audit) is correctly served by latest-ser - **Confidence drops below 0.5 do not auto-flag.** A belief whose posterior drifts under the prior is not surfaced as a warning at v1.x. The only automatic state change driven by negative evidence is locked-belief demotion-pressure (≥5 contradictions → auto-demote). To find drifting beliefs, query `aelf stats` directly. - **Jeffreys prior reads as 0.5.** A belief with no feedback reports posterior mean exactly `0.5`. That means "no evidence yet," not "coin-flip true." - **`aelf onboard` is non-incremental on duplicates.** Re-runs are idempotent; existing beliefs are not re-scored or refreshed. +- **Near-duplicates from different ingest paths persist.** `INSERT OR IGNORE` on `(source, sentence)` content_hash dedupes exact matches but not paraphrases (e.g. "don't push to main" locked + "never push directly to main" scanned both surface in retrieval). v2.0 ships `aelf doctor --dedup` (#197) — read-only audit that lists Jaccard + Levenshtein-confirmed duplicate clusters; the write-path SUPERSEDES hook is bench-gated and deferred behind the corpus benchmark. - **No bulk operations.** No batch lock, no `delete `, no merge. - **No edit.** A wrong belief is corrected by inserting a new one with a `SUPERSEDES` edge; the original stays. - **No graph viz.** Inspect with `sqlite3 "$(python -c 'from aelfrice.cli import db_path; print(db_path())')"`. diff --git a/docs/dedup.md b/docs/dedup.md new file mode 100644 index 000000000..776b3a442 --- /dev/null +++ b/docs/dedup.md @@ -0,0 +1,84 @@ +# Dedup — `aelf doctor --dedup` + +Audit-only near-duplicate detection over the belief store, shipped at v2.0 per [#197](https://github.com/robotrocketscience/aelfrice/issues/197). + +The detector pairs two cheap deterministic signals: + +- **Jaccard** over lowercase Unicode-word tokens — fast prefilter, no allocation per character pair. +- **Levenshtein ratio** (`1 - edit_distance / max(len_a, len_b)`) — second-stage confirmation; guards against shared-vocabulary false positives that Jaccard alone would accept. + +Both thresholds must clear for a pair to count as a near-duplicate. + +## Usage + +```bash +aelf doctor --dedup +``` + +Read-only: walks every belief pair, runs the prefilter, emits a clustered report. No edges are inserted, no beliefs are mutated. + +### Per-run flags + +```bash +aelf doctor --dedup \ + --dedup-jaccard 0.7 \ + --dedup-levenshtein 0.9 \ + --dedup-max-pairs 1000 +``` + +Each `--dedup-*` flag overrides one knob for the current run only. + +### Project defaults via `.aelfrice.toml` + +```toml +[dedup] +jaccard_min = 0.8 +levenshtein_min = 0.85 +max_candidate_pairs = 5000 +``` + +Walk-up resolution: the loader walks from cwd up through ancestor directories looking for `.aelfrice.toml`. Malformed values fall back to the module defaults with a stderr trace; the loader never raises. + +## Defaults + +| knob | default | source | +| --- | --- | --- | +| `jaccard_min` | 0.8 | research-line ratification | +| `levenshtein_min` | 0.85 | research-line ratification | +| `max_candidate_pairs` | 5000 | research-line ratification | + +The 0.8 Jaccard floor is intentionally strict — token-set divergence between e.g. "don't" / "do not" pushes that pair below the floor even though Levenshtein would clear. Lower the threshold via `--dedup-jaccard` for paraphrase-style detection at the cost of more false positives. + +## Output shape + +``` +aelf doctor dedup +======================================== +Beliefs scanned : 1483 +Candidate pairs visited : 1099303 +Duplicate pairs : 12 +Duplicate clusters : 4 + +Clusters: + belief-abc-123 (3 members) + * belief-abc-123 + belief-abc-456 + belief-abc-789 + ... + +Top duplicate pairs (jaccard, levenshtein): + belief-abc-123 ~ belief-abc-456 (j=0.923, l=0.971) + ... +``` + +`Candidate pairs visited` is the raw O(n²) count. `Duplicate pairs` is the post-Jaccard, post-Levenshtein survivor count. Clusters are the union-find collapse of those pairs into connected components; the `*` marks each cluster's deterministic representative (the lexicographically smallest member id). + +## What's not in this command + +The audit is **read-only by design**. The write-path SUPERSEDES hook — collapsing duplicates by inserting `SUPERSEDES` edges from older to newer at every `ingest_turn` / `onboard` / `apply_feedback` write — is the bench-gated R2 deferred behind the v2.0 corpus benchmark per #197 ratification. Until that lands, use this command to inspect candidate clusters and review them by hand. + +## Related + +- Spec memo: [`v2_dedup.md`](v2_dedup.md). +- Issue: [#197](https://github.com/robotrocketscience/aelfrice/issues/197). +- Scope cut: dedup is one of six bench-gated v2.0 modules; corpus contract at [#307](https://github.com/robotrocketscience/aelfrice/issues/307), bench-gate harness at [#319](https://github.com/robotrocketscience/aelfrice/issues/319).