diff --git a/src/aelfrice/classification.py b/src/aelfrice/classification.py index 03da7c6b0..2596f7351 100644 --- a/src/aelfrice/classification.py +++ b/src/aelfrice/classification.py @@ -41,10 +41,7 @@ BELIEF_REQUIREMENT, BELIEF_TYPES, INGEST_SOURCE_FILESYSTEM, - LOCK_NONE, ONBOARD_STATE_PENDING, - ORIGIN_AGENT_INFERRED, - Belief, OnboardSession, ) @@ -485,6 +482,10 @@ def accept_classifications( skipped_existing = 0 skipped_unclassified = 0 + # Lazy import: derivation imports classify_sentence from this module; + # importing at module-load would form a cycle. + from aelfrice.derivation import DerivationInput, derive # noqa: PLC0415 + for sd in sentences_data: idx = int(sd["index"]) text = str(sd["text"]) @@ -496,11 +497,20 @@ def accept_classifications( if not c.persist: skipped_non_persisting += 1 continue - bid = _derive_belief_id(text, source) + out = derive(DerivationInput( + raw_text=text, + source_kind=INGEST_SOURCE_FILESYSTEM, + source_path=source, + ts=timestamp, + override_belief_type=c.belief_type, + )) + # override_belief_type always produces a belief (persist=True + # is the caller's responsibility; checked above). + assert out.belief is not None + bid = out.belief.id if store.get_belief(bid) is not None: skipped_existing += 1 continue - alpha, beta = get_source_adjusted_prior(c.belief_type, source) # v2.0 #205 parallel-write: log the host-classified text. store.record_ingest( source_kind=INGEST_SOURCE_FILESYSTEM, @@ -509,22 +519,7 @@ def accept_classifications( derived_belief_ids=[bid], ts=timestamp, ) - store.insert_belief( - Belief( - id=bid, - content=text, - content_hash=_content_hash(text), - alpha=alpha, - beta=beta, - type=c.belief_type, - lock_level=LOCK_NONE, - locked_at=None, - demotion_pressure=0, - created_at=timestamp, - last_retrieved_at=None, - origin=ORIGIN_AGENT_INFERRED, - ) - ) + store.insert_belief(out.belief) inserted += 1 store.complete_onboard_session(session_id, timestamp) diff --git a/src/aelfrice/cli.py b/src/aelfrice/cli.py index dceb38e54..2289194fc 100644 --- a/src/aelfrice/cli.py +++ b/src/aelfrice/cli.py @@ -32,7 +32,6 @@ from __future__ import annotations import argparse -import hashlib import json import os import subprocess @@ -59,14 +58,12 @@ regime_description, ) from aelfrice.models import ( - BELIEF_FACTUAL, INGEST_SOURCE_CLI_REMEMBER, LOCK_NONE, LOCK_USER, ORIGIN_AGENT_INFERRED, ORIGIN_USER_STATED, ORIGIN_USER_VALIDATED, - Belief, ) from aelfrice import __version__ as _AELFRICE_VERSION from aelfrice.benchmark import run_benchmark, seed_corpus @@ -75,6 +72,7 @@ accept_classifications, start_onboard_session, ) +from aelfrice.derivation import DerivationInput, derive from aelfrice.doctor import ( classify_orphans as _classify_orphans, diagnose, @@ -160,7 +158,6 @@ DEFAULT_HOOK_COMMAND: Final[str] = "aelf-hook" DEFAULT_PRE_COMPACT_HOOK_COMMAND: Final[str] = "aelf-pre-compact-hook" _FEEDBACK_VALENCES: Final[dict[str, float]] = {"used": 1.0, "harmful": -1.0} -_LOCK_ID_LEN: Final[int] = 16 _VALID_SCOPES: Final[tuple[SettingsScope, ...]] = ("user", "project") @@ -262,14 +259,6 @@ def _utc_now_iso() -> str: return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") -def _lock_id_for(content: str) -> str: - return hashlib.sha256(f"lock\x00{content}".encode("utf-8")).hexdigest()[:_LOCK_ID_LEN] - - -def _content_hash(content: str) -> str: - return hashlib.sha256(content.encode("utf-8")).hexdigest() - - def _resolve_corpus_min() -> int: """Read AELFRICE_CORPUS_MIN env var, fall back to auditor default. @@ -797,9 +786,16 @@ def _cmd_rebuild(args: argparse.Namespace, out: object) -> int: def _cmd_lock(args: argparse.Namespace, out: object) -> int: store = _open_store() try: - bid = _lock_id_for(args.statement) - existing = store.get_belief(bid) now = _utc_now_iso() + derived = derive(DerivationInput( + raw_text=args.statement, + source_kind=INGEST_SOURCE_CLI_REMEMBER, + ts=now, + )) + # cli_remember always produces a belief. + assert derived.belief is not None + bid = derived.belief.id + existing = store.get_belief(bid) if existing is None: # v2.0 #205 parallel-write. store.record_ingest( @@ -808,20 +804,7 @@ def _cmd_lock(args: argparse.Namespace, out: object) -> int: derived_belief_ids=[bid], ts=now, ) - store.insert_belief(Belief( - id=bid, - content=args.statement, - content_hash=_content_hash(args.statement), - alpha=9.0, - beta=0.5, - type=BELIEF_FACTUAL, - lock_level=LOCK_USER, - locked_at=now, - demotion_pressure=0, - created_at=now, - last_retrieved_at=None, - origin=ORIGIN_USER_STATED, - )) + store.insert_belief(derived.belief) print(f"locked: {bid}", file=out) # type: ignore[arg-type] else: existing.lock_level = LOCK_USER diff --git a/src/aelfrice/derivation.py b/src/aelfrice/derivation.py new file mode 100644 index 000000000..be6db8d3b --- /dev/null +++ b/src/aelfrice/derivation.py @@ -0,0 +1,269 @@ +"""Pure belief-derivation function. + +Separates the deterministic "raw text -> Belief + edges" computation +from the store I/O so: + +- Every ingest entry point calls `derive()`, then does its own I/O. +- The v2.x replay harness can call `derive()` on rows from `ingest_log` + without touching a live store. + +No `MemoryStore` dependency anywhere in this module. +""" +from __future__ import annotations + +import hashlib +from dataclasses import dataclass, field +from datetime import datetime, timezone +from typing import Final + +from aelfrice.classification import classify_sentence, get_source_adjusted_prior +from aelfrice.models import ( + BELIEF_FACTUAL, + INGEST_SOURCE_CLI_REMEMBER, + INGEST_SOURCE_GIT, + INGEST_SOURCE_MCP_REMEMBER, + LOCK_NONE, + LOCK_USER, + ORIGIN_AGENT_INFERRED, + ORIGIN_USER_STATED, + Belief, + Edge, +) + +_BELIEF_ID_HEX_LEN: Final[int] = 16 + +# Source label used by triple-derived beliefs; matches +# `triple_extractor.TRIPLE_BELIEF_SOURCE`. Defined here to avoid a +# circular import (triple_extractor imports from models, not from +# derivation). Both constants must stay in sync. +_TRIPLE_BELIEF_SOURCE: Final[str] = "triple" + + +# --------------------------------------------------------------------------- +# Input / output dataclasses +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class DerivationInput: + """Everything `derive()` needs; no store references. + + Fields match the columns written to `ingest_log` by each call site + so that a replay harness can reconstruct a `DerivationInput` from a + single log row. + + `source_kind` must be one of `INGEST_SOURCE_KINDS`. + `ts` is an ISO-8601 timestamp string; callers are responsible for + supplying it (so tests can inject a stable clock). Empty string + triggers UTC-now inside `derive()`. + """ + + raw_text: str + source_kind: str # one of INGEST_SOURCE_KINDS + source_path: str | None = None + raw_meta: dict | None = None # type: ignore[type-arg] + session_id: str | None = None + ts: str = "" # ISO-8601; empty string -> utc-now + classifier_version: str | None = None + rule_set_hash: str | None = None + # Optional pre-classified type from a host LLM (polymorphic onboard + # handshake). When set, `derive()` skips `classify_sentence` and uses + # this type to look up the source-adjusted prior. + override_belief_type: str | None = None + + +@dataclass(frozen=True) +class DerivationOutput: + """Result of `derive()`. + + `belief` is None when the classifier sets `persist=False` (questions, + empty text, etc.). Callers should check `belief is not None` before + writing to the store. + + `edges` is the list of edges to insert after the belief lands. + Currently always empty; placeholder for v2.x paths that derive + edges from a single text block. + + `skip_reason` is a short string explaining why `belief` is None, or + None when the belief will be persisted. + """ + + belief: Belief | None + edges: list[Edge] = field(default_factory=list) + skip_reason: str | None = None + + +# --------------------------------------------------------------------------- +# Internal helpers +# --------------------------------------------------------------------------- + + +def _utc_now_iso() -> str: + return datetime.now(timezone.utc).isoformat() + + +def _belief_id(text: str, source: str) -> str: + """Deterministic id from sha256(source + NUL + text)[:16]. + + Shared scheme with `ingest._belief_id` and `scanner._derive_belief_id` + so the same (text, source) pair always resolves to the same id + regardless of call site. + """ + h = hashlib.sha256(f"{source}\x00{text}".encode("utf-8")).hexdigest() + return h[:_BELIEF_ID_HEX_LEN] + + +def _lock_id(text: str) -> str: + """Deterministic id for lock/remember call sites. + + Matches `mcp_server._lock_id_for` and `cli._lock_id_for`. + """ + h = hashlib.sha256(f"lock\x00{text}".encode("utf-8")).hexdigest() + return h[:_BELIEF_ID_HEX_LEN] + + +def _triple_belief_id(phrase: str) -> str: + """Deterministic id for triple-extracted noun-phrase beliefs. + + Matches `triple_extractor._belief_id_for_phrase`. Keyed on + `_TRIPLE_BELIEF_SOURCE` so the same normalised phrase resolves to + the same id across all extraction call sites. + """ + normalized = " ".join(phrase.split()).lower() + h = hashlib.sha256( + f"{_TRIPLE_BELIEF_SOURCE}\x00{normalized}".encode("utf-8") + ).hexdigest() + return h[:_BELIEF_ID_HEX_LEN] + + +def _content_hash(text: str) -> str: + return hashlib.sha256(text.encode("utf-8")).hexdigest() + + +def _triple_content_hash(phrase: str) -> str: + """Content hash for triple-derived beliefs (normalised + lower). + + Matches `triple_extractor._content_hash`. + """ + normalized = " ".join(phrase.split()).lower() + return hashlib.sha256(normalized.encode("utf-8")).hexdigest() + + +# --------------------------------------------------------------------------- +# Public API +# --------------------------------------------------------------------------- + + +def derive(inp: DerivationInput) -> DerivationOutput: + """Pure function: raw text in, belief (or skip) out. No I/O. + + Dispatch rules (in evaluation order): + + 1. Lock / remember paths (`source_kind` in {mcp_remember, + cli_remember}): always persist with a USER lock, no classifier. + 2. Triple-extraction path (`source_kind == git`): always persist as + factual with alpha=1.0 / beta=1.0; id scheme matches + `triple_extractor._belief_id_for_phrase`. + 3. All other paths (filesystem, python_ast, feedback_loop_synthesis, + legacy_unknown): run `classify_sentence`; skip when + `persist=False`. + + The belief `id` is derived deterministically from the input so that + re-deriving the same input yields the same id — replay equality is + id-stable. + """ + ts = inp.ts if inp.ts else _utc_now_iso() + raw = inp.raw_text + + # 1. Lock / remember paths ------------------------------------------- + if inp.source_kind in (INGEST_SOURCE_MCP_REMEMBER, INGEST_SOURCE_CLI_REMEMBER): + bid = _lock_id(raw) + belief = Belief( + id=bid, + content=raw, + content_hash=_content_hash(raw), + alpha=9.0, + beta=0.5, + type=BELIEF_FACTUAL, + lock_level=LOCK_USER, + locked_at=ts, + demotion_pressure=0, + created_at=ts, + last_retrieved_at=None, + session_id=inp.session_id, + origin=ORIGIN_USER_STATED, + ) + return DerivationOutput(belief=belief, edges=[]) + + # 2. Triple-extraction path (git commit-ingest) ----------------------- + if inp.source_kind == INGEST_SOURCE_GIT: + normalized = " ".join(raw.split()) + bid = _triple_belief_id(raw) + belief = Belief( + id=bid, + content=normalized, + content_hash=_triple_content_hash(raw), + alpha=1.0, + beta=1.0, + type=BELIEF_FACTUAL, + lock_level=LOCK_NONE, + locked_at=None, + demotion_pressure=0, + created_at=ts, + last_retrieved_at=None, + session_id=inp.session_id, + origin=ORIGIN_AGENT_INFERRED, + ) + return DerivationOutput(belief=belief, edges=[]) + + # 3. Classifier paths (filesystem, python_ast, etc.) ------------------ + source = inp.source_path or inp.source_kind + + if inp.override_belief_type is not None: + # Host-LLM-classified path (polymorphic onboard handshake): the + # caller has already determined the belief type; skip regex + # classify_sentence and look up the source-adjusted prior directly. + alpha, beta = get_source_adjusted_prior(inp.override_belief_type, source) + bid = _belief_id(raw, source) + belief = Belief( + id=bid, + content=raw, + content_hash=_content_hash(raw), + alpha=alpha, + beta=beta, + type=inp.override_belief_type, + lock_level=LOCK_NONE, + locked_at=None, + demotion_pressure=0, + created_at=ts, + last_retrieved_at=None, + session_id=inp.session_id, + origin=ORIGIN_AGENT_INFERRED, + ) + return DerivationOutput(belief=belief, edges=[]) + + result = classify_sentence(raw, source) + if not result.persist: + return DerivationOutput( + belief=None, + edges=[], + skip_reason="persist=False", + ) + + bid = _belief_id(raw, source) + belief = Belief( + id=bid, + content=raw, + content_hash=_content_hash(raw), + alpha=result.alpha, + beta=result.beta, + type=result.belief_type, + lock_level=LOCK_NONE, + locked_at=None, + demotion_pressure=0, + created_at=ts, + last_retrieved_at=None, + session_id=inp.session_id, + origin=ORIGIN_AGENT_INFERRED, + ) + return DerivationOutput(belief=belief, edges=[]) diff --git a/src/aelfrice/ingest.py b/src/aelfrice/ingest.py index 868f2103e..f99ac004c 100644 --- a/src/aelfrice/ingest.py +++ b/src/aelfrice/ingest.py @@ -16,41 +16,23 @@ """ from __future__ import annotations -import hashlib import json from dataclasses import dataclass from datetime import datetime, timezone from pathlib import Path from typing import cast -from aelfrice.classification import classify_sentence +from aelfrice.derivation import DerivationInput, derive from aelfrice.extraction import extract_sentences from aelfrice.models import ( ANCHOR_TEXT_MAX_LEN, CORROBORATION_SOURCE_TRANSCRIPT_INGEST, EDGE_DERIVED_FROM, INGEST_SOURCE_FILESYSTEM, - LOCK_NONE, - ORIGIN_AGENT_INFERRED, - Belief, Edge, ) from aelfrice.store import MemoryStore -_BELIEF_ID_HEX_LEN: int = 16 - - -def _belief_id(text: str, source: str) -> str: - """Stable id derived from (source, text). Matches the scheme used - by classification._derive_belief_id and scanner._derive_belief_id - so re-ingesting an identical (text, source) pair is idempotent.""" - h = hashlib.sha256(f"{source}\x00{text}".encode("utf-8")).hexdigest() - return h[:_BELIEF_ID_HEX_LEN] - - -def _content_hash(text: str) -> str: - return hashlib.sha256(text.encode("utf-8")).hexdigest() - def _now_utc_iso() -> str: return datetime.now(timezone.utc).isoformat() @@ -114,11 +96,16 @@ def _ingest_turn_ids( ts = created_at or _now_utc_iso() inserted: list[str] = [] for sentence in sentences: - result = classify_sentence(sentence, source) - if not result.persist: + out = derive(DerivationInput( + raw_text=sentence, + source_kind=INGEST_SOURCE_FILESYSTEM, + source_path=source, + session_id=session_id, + ts=ts, + )) + if out.belief is None: continue - belief_id = _belief_id(sentence, source) - ch = _content_hash(sentence) + belief_id = out.belief.id if store.get_belief(belief_id) is not None: # Exact (source, sentence) duplicate: record a corroboration # so re-assertions are observable. Canonical row unchanged. @@ -139,22 +126,7 @@ def _ingest_turn_ids( session_id=session_id, ts=ts, ) - belief = Belief( - id=belief_id, - content=sentence, - content_hash=ch, - alpha=result.alpha, - beta=result.beta, - type=result.belief_type, - lock_level=LOCK_NONE, - locked_at=None, - demotion_pressure=0, - created_at=ts, - last_retrieved_at=None, - session_id=session_id, - origin=ORIGIN_AGENT_INFERRED, - ) - store.insert_belief(belief) + store.insert_belief(out.belief) inserted.append(belief_id) return inserted diff --git a/src/aelfrice/mcp_server.py b/src/aelfrice/mcp_server.py index 3087002f0..160407aa4 100644 --- a/src/aelfrice/mcp_server.py +++ b/src/aelfrice/mcp_server.py @@ -34,7 +34,6 @@ """ from __future__ import annotations -import hashlib from datetime import datetime, timezone from pathlib import Path from typing import Any, Final, Sequence @@ -45,6 +44,7 @@ start_onboard_session, ) from aelfrice.cli import db_path +from aelfrice.derivation import DerivationInput, derive from aelfrice.feedback import apply_feedback from aelfrice.health import ( REGIME_INSUFFICIENT_DATA, @@ -52,21 +52,18 @@ regime_description, ) from aelfrice.models import ( - BELIEF_FACTUAL, CORROBORATION_SOURCE_MCP_REMEMBER, INGEST_SOURCE_MCP_REMEMBER, LOCK_NONE, LOCK_USER, ORIGIN_USER_STATED, ORIGIN_USER_VALIDATED, - Belief, ) from aelfrice.retrieval import DEFAULT_TOKEN_BUDGET, retrieve from aelfrice.scanner import scan_repo from aelfrice.store import MemoryStore _FEEDBACK_VALENCES: Final[dict[str, float]] = {"used": 1.0, "harmful": -1.0} -_LOCK_ID_LEN: Final[int] = 16 # --- Helpers ----------------------------------------------------------- @@ -76,14 +73,6 @@ def _utc_now_iso() -> str: return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") -def _lock_id_for(content: str) -> str: - return hashlib.sha256(f"lock\x00{content}".encode("utf-8")).hexdigest()[:_LOCK_ID_LEN] - - -def _content_hash(content: str) -> str: - return hashlib.sha256(content.encode("utf-8")).hexdigest() - - def _ensure_parent_dir(path: Path) -> None: path.parent.mkdir(parents=True, exist_ok=True) @@ -211,9 +200,16 @@ def tool_search( def tool_lock(store: MemoryStore, *, statement: str) -> dict[str, Any]: - bid = _lock_id_for(statement) - existing = store.get_belief(bid) now = _utc_now_iso() + out = derive(DerivationInput( + raw_text=statement, + source_kind=INGEST_SOURCE_MCP_REMEMBER, + ts=now, + )) + # mcp_remember always produces a belief. + assert out.belief is not None + bid = out.belief.id + existing = store.get_belief(bid) if existing is None: # v2.0 #205 parallel-write: log the user-stated raw text. store.record_ingest( @@ -222,20 +218,7 @@ def tool_lock(store: MemoryStore, *, statement: str) -> dict[str, Any]: derived_belief_ids=[bid], ts=now, ) - store.insert_belief(Belief( - id=bid, - content=statement, - content_hash=_content_hash(statement), - alpha=9.0, - beta=0.5, - type=BELIEF_FACTUAL, - lock_level=LOCK_USER, - locked_at=now, - demotion_pressure=0, - created_at=now, - last_retrieved_at=None, - origin=ORIGIN_USER_STATED, - )) + store.insert_belief(out.belief) return {"kind": "lock.created", "id": bid, "action": "locked"} existing.lock_level = LOCK_USER existing.locked_at = now diff --git a/src/aelfrice/scanner.py b/src/aelfrice/scanner.py index f510a1e77..46cdcf2e9 100644 --- a/src/aelfrice/scanner.py +++ b/src/aelfrice/scanner.py @@ -25,12 +25,11 @@ from pathlib import Path from typing import Final -from aelfrice.classification import classify_sentence +from aelfrice.derivation import DerivationInput, derive from aelfrice.inedible import is_inedible from aelfrice.models import ( INGEST_SOURCE_FILESYSTEM, LOCK_NONE, - ORIGIN_AGENT_INFERRED, Belief, ) from aelfrice.noise_filter import NoiseConfig, is_noise @@ -206,7 +205,7 @@ def scan_repo( # Two-pass when an llm_router is supplied: first collect the # noise-filtered candidates, then route them all through the # batched LLM call. Single-pass when no router (default OFF): - # the regex path is unchanged from v1.0. + # derive() handles the regex-classify path inline. filtered: list[SentenceCandidate] = [] for candidate in candidates: if is_noise(candidate.text, cfg): @@ -214,64 +213,85 @@ def scan_repo( continue filtered.append(candidate) - routes: list[LLMRoute] + routes: list[LLMRoute] | None = None if llm_router is not None: routes = llm_router.classify(filtered) - else: - routes = [_route_from_regex(c) for c in filtered] - - if len(routes) != len(filtered): - # The router contract requires one-route-per-candidate - # in input order. A length mismatch is a programming - # error, not a user-visible state. - raise RuntimeError( - f"llm_router.classify returned {len(routes)} routes for " - f"{len(filtered)} candidates" - ) + if len(routes) != len(filtered): + # The router contract requires one-route-per-candidate + # in input order. A length mismatch is a programming + # error, not a user-visible state. + raise RuntimeError( + f"llm_router.classify returned {len(routes)} routes for " + f"{len(filtered)} candidates" + ) - for candidate, route in zip(filtered, routes): - if not route.persist: - skipped_non_persisting += 1 - continue - belief_id = _derive_belief_id(candidate.text, candidate.source) - if store.get_belief(belief_id) is not None: - skipped_existing += 1 - continue + for idx, candidate in enumerate(filtered): created_at = candidate.commit_date or timestamp - # v2.0 #205 parallel-write: log the raw classifier input - # before materializing the belief. derived_belief_ids is - # known up-front because belief_id is deterministic on - # (source, text). - store.record_ingest( - source_kind=INGEST_SOURCE_FILESYSTEM, - source_path=candidate.source, - raw_text=candidate.text, - derived_belief_ids=[belief_id], - ts=created_at, - ) - store.insert_belief(Belief( - id=belief_id, - content=candidate.text, - content_hash=_content_hash(candidate.text), - alpha=route.alpha, - beta=route.beta, - type=route.belief_type, - lock_level=LOCK_NONE, - locked_at=None, - demotion_pressure=0, - created_at=created_at, - last_retrieved_at=None, - origin=route.origin, - )) - inserted += 1 - # Audit row for fallback insertions (spec § 7.2 step 3). - if route.audit_source is not None: - store.insert_feedback_event( - belief_id=belief_id, - valence=0.0, - source=route.audit_source, - created_at=timestamp, + + if routes is not None: + # LLM-classify path: router already derived type/origin/alpha/beta. + route = routes[idx] + if not route.persist: + skipped_non_persisting += 1 + continue + belief_id = _derive_belief_id(candidate.text, candidate.source) + if store.get_belief(belief_id) is not None: + skipped_existing += 1 + continue + store.record_ingest( + source_kind=INGEST_SOURCE_FILESYSTEM, + source_path=candidate.source, + raw_text=candidate.text, + derived_belief_ids=[belief_id], + ts=created_at, ) + store.insert_belief(Belief( + id=belief_id, + content=candidate.text, + content_hash=_content_hash(candidate.text), + alpha=route.alpha, + beta=route.beta, + type=route.belief_type, + lock_level=LOCK_NONE, + locked_at=None, + demotion_pressure=0, + created_at=created_at, + last_retrieved_at=None, + origin=route.origin, + )) + inserted += 1 + # Audit row for fallback insertions (spec § 7.2 step 3). + if route.audit_source is not None: + store.insert_feedback_event( + belief_id=belief_id, + valence=0.0, + source=route.audit_source, + created_at=timestamp, + ) + else: + # Regex path: delegate belief derivation to pure derive(). + out = derive(DerivationInput( + raw_text=candidate.text, + source_kind=INGEST_SOURCE_FILESYSTEM, + source_path=candidate.source, + ts=created_at, + )) + if out.belief is None: + skipped_non_persisting += 1 + continue + belief_id = out.belief.id + if store.get_belief(belief_id) is not None: + skipped_existing += 1 + continue + store.record_ingest( + source_kind=INGEST_SOURCE_FILESYSTEM, + source_path=candidate.source, + raw_text=candidate.text, + derived_belief_ids=[belief_id], + ts=created_at, + ) + store.insert_belief(out.belief) + inserted += 1 return ScanResult( inserted=inserted, @@ -282,24 +302,6 @@ def scan_repo( ) -def _route_from_regex(candidate: SentenceCandidate) -> LLMRoute: - """Build an LLMRoute from the regex `classify_sentence` output. - - Default-ON path used when no LLM router is supplied. Origin is - fixed at ORIGIN_AGENT_INFERRED (legacy v1.0/v1.2 behaviour); - fixing this is the LLM-classify path's job, not the regex path. - """ - result = classify_sentence(candidate.text, candidate.source) - return LLMRoute( - belief_type=result.belief_type, - origin=ORIGIN_AGENT_INFERRED, - persist=result.persist, - alpha=result.alpha, - beta=result.beta, - audit_source=None, - ) - - def _iter_doc_files(root: Path) -> list[Path]: """Return the doc files under root in deterministic sorted order. diff --git a/src/aelfrice/triple_extractor.py b/src/aelfrice/triple_extractor.py index 8dc8e5df0..82de27716 100644 --- a/src/aelfrice/triple_extractor.py +++ b/src/aelfrice/triple_extractor.py @@ -23,14 +23,13 @@ """ from __future__ import annotations -import hashlib import re from dataclasses import dataclass, field from datetime import datetime, timezone from typing import Final +from aelfrice.derivation import DerivationInput, derive from aelfrice.models import ( - BELIEF_FACTUAL, CORROBORATION_SOURCE_COMMIT_INGEST, EDGE_CITES, EDGE_CONTRADICTS, @@ -39,9 +38,6 @@ EDGE_SUPERSEDES, EDGE_SUPPORTS, INGEST_SOURCE_GIT, - LOCK_NONE, - ORIGIN_AGENT_INFERRED, - Belief, Edge, ) from aelfrice.store import MemoryStore @@ -58,8 +54,6 @@ the same noun phrase always resolves to the same belief id — even across different commits, transcripts, or call sites.""" -_BELIEF_ID_HEX_LEN: Final[int] = 16 - @dataclass(frozen=True) class Triple: @@ -223,25 +217,6 @@ def extract_triples(text: str) -> list[Triple]: # --- Ingest --------------------------------------------------------------- -def _belief_id_for_phrase(phrase: str) -> str: - """Stable id from sha256(TRIPLE_BELIEF_SOURCE \\x00 normalized). - - Same phrase => same id, regardless of the extraction call site. - Different from `ingest._belief_id` (which keys per-sentence per- - source) because triple-derived beliefs need a single canonical - id space across all extraction sites. - """ - normalized = _normalize_phrase(phrase).lower() - h = hashlib.sha256( - f"{TRIPLE_BELIEF_SOURCE}\x00{normalized}".encode("utf-8") - ).hexdigest() - return h[:_BELIEF_ID_HEX_LEN] - - -def _content_hash(content: str) -> str: - return hashlib.sha256(_normalize_phrase(content).lower().encode("utf-8")).hexdigest() - - def _now_iso() -> str: return datetime.now(timezone.utc).isoformat() @@ -258,7 +233,17 @@ def _resolve_or_create_belief( When the belief already exists (same id = same normalised phrase), a corroboration row is recorded so the re-assertion is observable. """ - bid = _belief_id_for_phrase(phrase) + # derive() id-scheme matches _belief_id_for_phrase; compute once. + ts = _now_iso() + out = derive(DerivationInput( + raw_text=phrase, + source_kind=INGEST_SOURCE_GIT, + session_id=session_id, + ts=ts, + )) + # INGEST_SOURCE_GIT always produces a belief (no classifier skip). + assert out.belief is not None + bid = out.belief.id existing = store.get_belief(bid) if existing is not None: store.record_corroboration( @@ -267,7 +252,6 @@ def _resolve_or_create_belief( session_id=session_id, ) return bid - ts = _now_iso() # v2.0 #205 parallel-write: log the raw phrase before materialization. # source_kind=git because the commit-ingest path emits triples from # commit messages; source_path is unknown at this layer (callers @@ -279,22 +263,7 @@ def _resolve_or_create_belief( session_id=session_id, ts=ts, ) - belief = Belief( - id=bid, - content=_normalize_phrase(phrase), - content_hash=_content_hash(phrase), - alpha=1.0, - beta=1.0, - type=BELIEF_FACTUAL, - lock_level=LOCK_NONE, - locked_at=None, - demotion_pressure=0, - created_at=ts, - last_retrieved_at=None, - session_id=session_id, - origin=ORIGIN_AGENT_INFERRED, - ) - store.insert_belief(belief) + store.insert_belief(out.belief) created_ids.append(bid) return bid diff --git a/tests/test_derivation.py b/tests/test_derivation.py new file mode 100644 index 000000000..0bdfb860c --- /dev/null +++ b/tests/test_derivation.py @@ -0,0 +1,475 @@ +"""Unit tests for the pure derive() function in derivation.py. + +Each test states a falsifiable hypothesis and exercises one or more +source_kind values from INGEST_SOURCE_KINDS. No store I/O; all tests +are pure-function calls. +""" +from __future__ import annotations + +import hashlib + +import pytest + +from aelfrice.derivation import ( + DerivationInput, + DerivationOutput, + _belief_id, + _lock_id, + _triple_belief_id, + derive, +) +from aelfrice.models import ( + BELIEF_CORRECTION, + BELIEF_FACTUAL, + BELIEF_PREFERENCE, + BELIEF_REQUIREMENT, + INGEST_SOURCE_CLI_REMEMBER, + INGEST_SOURCE_FEEDBACK_LOOP_SYNTHESIS, + INGEST_SOURCE_FILESYSTEM, + INGEST_SOURCE_GIT, + INGEST_SOURCE_LEGACY_UNKNOWN, + INGEST_SOURCE_MCP_REMEMBER, + INGEST_SOURCE_PYTHON_AST, + LOCK_NONE, + LOCK_USER, + ORIGIN_AGENT_INFERRED, + ORIGIN_USER_STATED, +) + +_TS = "2026-01-01T00:00:00Z" + + +# --------------------------------------------------------------------------- +# DerivationInput / DerivationOutput shape +# --------------------------------------------------------------------------- + + +def test_derivation_input_required_fields_only() -> None: + """Hypothesis: DerivationInput accepts raw_text + source_kind with all + optional fields defaulting to None / empty string. Falsifiable if + construction raises.""" + inp = DerivationInput(raw_text="hello", source_kind=INGEST_SOURCE_FILESYSTEM) + assert inp.source_path is None + assert inp.session_id is None + assert inp.ts == "" + assert inp.override_belief_type is None + + +def test_derivation_output_belief_none_has_skip_reason() -> None: + """Hypothesis: a DerivationOutput with belief=None always carries a + non-empty skip_reason. Falsifiable by any skip output that leaves + skip_reason=None.""" + out = derive(DerivationInput( + raw_text="What is the default port?", + source_kind=INGEST_SOURCE_FILESYSTEM, + source_path="doc:README.md:p0", + ts=_TS, + )) + assert out.belief is None + assert out.skip_reason is not None + assert out.skip_reason != "" + + +# --------------------------------------------------------------------------- +# filesystem (classifier path) +# --------------------------------------------------------------------------- + + +def test_filesystem_factual_belief() -> None: + """Hypothesis: a plain factual sentence via filesystem yields a factual + belief with LOCK_NONE and ORIGIN_AGENT_INFERRED. Falsifiable by any + other type, lock, or origin.""" + out = derive(DerivationInput( + raw_text="The default port is 8080 for the dashboard.", + source_kind=INGEST_SOURCE_FILESYSTEM, + source_path="doc:README.md:p0", + ts=_TS, + )) + assert out.belief is not None + assert out.belief.type == BELIEF_FACTUAL + assert out.belief.lock_level == LOCK_NONE + assert out.belief.origin == ORIGIN_AGENT_INFERRED + assert out.belief.created_at == _TS + + +def test_filesystem_requirement_belief() -> None: + """Hypothesis: a sentence with a requirement keyword yields + belief_type=requirement. Falsifiable by any other type.""" + out = derive(DerivationInput( + raw_text="This project must use uv for environment management.", + source_kind=INGEST_SOURCE_FILESYSTEM, + source_path="doc:README.md:p1", + ts=_TS, + )) + assert out.belief is not None + assert out.belief.type == BELIEF_REQUIREMENT + + +def test_filesystem_preference_belief() -> None: + """Hypothesis: a sentence with a preference keyword yields + belief_type=preference. Falsifiable by any other type.""" + out = derive(DerivationInput( + raw_text="I prefer atomic commits over batched commits.", + source_kind=INGEST_SOURCE_FILESYSTEM, + source_path="doc:README.md:p2", + ts=_TS, + )) + assert out.belief is not None + assert out.belief.type == BELIEF_PREFERENCE + + +def test_filesystem_correction_belief() -> None: + """Hypothesis: a sentence that triggers the correction detector yields + belief_type=correction. Falsifiable by any other type.""" + out = derive(DerivationInput( + raw_text="Actually, the default port is 9090, not 8080.", + source_kind=INGEST_SOURCE_FILESYSTEM, + source_path="doc:README.md:p3", + ts=_TS, + )) + assert out.belief is not None + assert out.belief.type == BELIEF_CORRECTION + + +def test_filesystem_question_skipped() -> None: + """Hypothesis: a question-form sentence via filesystem is skipped + (persist=False). Falsifiable if a belief is returned.""" + out = derive(DerivationInput( + raw_text="What is the default port?", + source_kind=INGEST_SOURCE_FILESYSTEM, + source_path="doc:README.md:p4", + ts=_TS, + )) + assert out.belief is None + assert out.skip_reason == "persist=False" + + +def test_filesystem_empty_text_skipped() -> None: + """Hypothesis: empty raw_text is skipped. Falsifiable if a belief + is returned.""" + out = derive(DerivationInput( + raw_text="", + source_kind=INGEST_SOURCE_FILESYSTEM, + source_path="doc:README.md:p5", + ts=_TS, + )) + assert out.belief is None + + +def test_filesystem_belief_id_stable_and_matches_scheme() -> None: + """Hypothesis: the belief id is sha256(source_path NUL text)[:16], + matching the scheme used by ingest._belief_id. Falsifiable by any + mismatch.""" + text = "The configuration file lives at /etc/aelfrice/conf." + source = "doc:README.md:p0" + expected = hashlib.sha256(f"{source}\x00{text}".encode()).hexdigest()[:16] + out = derive(DerivationInput( + raw_text=text, + source_kind=INGEST_SOURCE_FILESYSTEM, + source_path=source, + ts=_TS, + )) + assert out.belief is not None + assert out.belief.id == expected + + +def test_filesystem_session_id_propagated() -> None: + """Hypothesis: session_id on DerivationInput is stamped onto the + belief. Falsifiable by a None or different session_id on the belief.""" + out = derive(DerivationInput( + raw_text="The deploy uses uv only.", + source_kind=INGEST_SOURCE_FILESYSTEM, + source_path="user", + session_id="test-session-123", + ts=_TS, + )) + assert out.belief is not None + assert out.belief.session_id == "test-session-123" + + +# --------------------------------------------------------------------------- +# python_ast path +# --------------------------------------------------------------------------- + + +def test_python_ast_factual_belief() -> None: + """Hypothesis: source_kind=python_ast with a plain docstring sentence + yields a factual belief. Falsifiable by any other type or a skip.""" + out = derive(DerivationInput( + raw_text="Parse a Python module and extract top-level docstrings.", + source_kind=INGEST_SOURCE_PYTHON_AST, + source_path="ast:src/aelfrice/scanner.py:func:extract_ast", + ts=_TS, + )) + assert out.belief is not None + assert out.belief.type == BELIEF_FACTUAL + + +# --------------------------------------------------------------------------- +# git (triple-extraction path) +# --------------------------------------------------------------------------- + + +def test_git_always_yields_belief() -> None: + """Hypothesis: INGEST_SOURCE_GIT always produces a belief regardless + of text content (no classifier skip). Falsifiable by any None belief.""" + for phrase in ["the index", "What is this?", "", " "]: + out = derive(DerivationInput( + raw_text=phrase, + source_kind=INGEST_SOURCE_GIT, + ts=_TS, + )) + assert out.belief is not None, f"expected belief for phrase {phrase!r}" + + +def test_git_belief_alpha_beta_and_type() -> None: + """Hypothesis: git-path beliefs have alpha=1.0, beta=1.0, type=factual. + Falsifiable by any other value.""" + out = derive(DerivationInput( + raw_text="the new index", + source_kind=INGEST_SOURCE_GIT, + ts=_TS, + )) + assert out.belief is not None + assert out.belief.alpha == 1.0 + assert out.belief.beta == 1.0 + assert out.belief.type == BELIEF_FACTUAL + assert out.belief.lock_level == LOCK_NONE + + +def test_git_belief_id_matches_triple_extractor_scheme() -> None: + """Hypothesis: the belief id from derive() for git source_kind matches + triple_extractor._belief_id_for_phrase (sha256(triple NUL lower)[:16]). + Falsifiable by any mismatch — a mismatch would break idempotency with + the triple-ingest path.""" + phrase = "the new index" + normalized = " ".join(phrase.split()).lower() + expected = hashlib.sha256( + f"triple\x00{normalized}".encode("utf-8") + ).hexdigest()[:16] + out = derive(DerivationInput( + raw_text=phrase, + source_kind=INGEST_SOURCE_GIT, + ts=_TS, + )) + assert out.belief is not None + assert out.belief.id == expected + + +def test_git_belief_content_is_normalized_phrase() -> None: + """Hypothesis: git-path belief content is the whitespace-normalised + phrase (not the raw text with extra spaces). Falsifiable by any + non-normalised content.""" + out = derive(DerivationInput( + raw_text=" the new index ", + source_kind=INGEST_SOURCE_GIT, + ts=_TS, + )) + assert out.belief is not None + assert out.belief.content == "the new index" + + +# --------------------------------------------------------------------------- +# mcp_remember path +# --------------------------------------------------------------------------- + + +def test_mcp_remember_yields_user_locked_belief() -> None: + """Hypothesis: mcp_remember always yields a LOCK_USER belief with + ORIGIN_USER_STATED and alpha=9.0 / beta=0.5. Falsifiable by any + other lock level, origin, or prior.""" + out = derive(DerivationInput( + raw_text="Always use uv for package management.", + source_kind=INGEST_SOURCE_MCP_REMEMBER, + ts=_TS, + )) + assert out.belief is not None + assert out.belief.lock_level == LOCK_USER + assert out.belief.origin == ORIGIN_USER_STATED + assert out.belief.alpha == 9.0 + assert out.belief.beta == 0.5 + assert out.belief.locked_at == _TS + + +def test_mcp_remember_belief_id_matches_lock_scheme() -> None: + """Hypothesis: mcp_remember id is sha256(lock NUL text)[:16], matching + cli._lock_id_for and mcp_server._lock_id_for. Falsifiable by mismatch.""" + stmt = "Always use uv for package management." + expected = hashlib.sha256(f"lock\x00{stmt}".encode()).hexdigest()[:16] + out = derive(DerivationInput( + raw_text=stmt, + source_kind=INGEST_SOURCE_MCP_REMEMBER, + ts=_TS, + )) + assert out.belief is not None + assert out.belief.id == expected + + +# --------------------------------------------------------------------------- +# cli_remember path +# --------------------------------------------------------------------------- + + +def test_cli_remember_yields_user_locked_belief() -> None: + """Hypothesis: cli_remember behaves identically to mcp_remember for + the lock/prior/origin fields. Falsifiable by any difference.""" + out = derive(DerivationInput( + raw_text="The deploy must use the staging environment first.", + source_kind=INGEST_SOURCE_CLI_REMEMBER, + ts=_TS, + )) + assert out.belief is not None + assert out.belief.lock_level == LOCK_USER + assert out.belief.origin == ORIGIN_USER_STATED + assert out.belief.alpha == 9.0 + assert out.belief.beta == 0.5 + + +def test_cli_remember_belief_id_matches_lock_scheme() -> None: + """Hypothesis: cli_remember id uses the same lock scheme as + mcp_remember. Falsifiable by any mismatch with the sha256 formula.""" + stmt = "The deploy must use the staging environment first." + expected = hashlib.sha256(f"lock\x00{stmt}".encode()).hexdigest()[:16] + out = derive(DerivationInput( + raw_text=stmt, + source_kind=INGEST_SOURCE_CLI_REMEMBER, + ts=_TS, + )) + assert out.belief is not None + assert out.belief.id == expected + + +# --------------------------------------------------------------------------- +# override_belief_type (accept_classifications path) +# --------------------------------------------------------------------------- + + +def test_override_belief_type_bypasses_classify_sentence() -> None: + """Hypothesis: when override_belief_type is set, derive() uses that + type directly rather than calling classify_sentence. Concretely: a + question-form sentence (which classify_sentence would reject with + persist=False) should persist when override_belief_type is set. + Falsifiable if the result is skipped.""" + out = derive(DerivationInput( + raw_text="What is the default port?", + source_kind=INGEST_SOURCE_FILESYSTEM, + source_path="doc:README.md:p0", + ts=_TS, + override_belief_type=BELIEF_FACTUAL, + )) + assert out.belief is not None + assert out.belief.type == BELIEF_FACTUAL + + +def test_override_belief_type_all_valid_types() -> None: + """Hypothesis: every belief type in BELIEF_TYPES is accepted by + override_belief_type and produces a belief. Falsifiable by any type + that errors or returns None.""" + for btype in (BELIEF_FACTUAL, BELIEF_CORRECTION, BELIEF_PREFERENCE, BELIEF_REQUIREMENT): + out = derive(DerivationInput( + raw_text="The configuration file is at /etc/aelf/conf.", + source_kind=INGEST_SOURCE_FILESYSTEM, + source_path="doc:README.md:p0", + ts=_TS, + override_belief_type=btype, + )) + assert out.belief is not None, f"expected belief for type {btype!r}" + assert out.belief.type == btype + + +# --------------------------------------------------------------------------- +# feedback_loop_synthesis and legacy_unknown +# --------------------------------------------------------------------------- + + +def test_feedback_loop_synthesis_uses_classifier() -> None: + """Hypothesis: feedback_loop_synthesis goes through classify_sentence + (classifier path, not lock path). Falsifiable if lock_level is USER + or if a plain factual sentence is skipped.""" + out = derive(DerivationInput( + raw_text="The feedback loop synthesis produces factual beliefs.", + source_kind=INGEST_SOURCE_FEEDBACK_LOOP_SYNTHESIS, + ts=_TS, + )) + assert out.belief is not None + assert out.belief.lock_level == LOCK_NONE + assert out.belief.type == BELIEF_FACTUAL + + +def test_legacy_unknown_uses_classifier() -> None: + """Hypothesis: legacy_unknown goes through classify_sentence. + Falsifiable if the result is a lock or if an obvious factual is skipped.""" + out = derive(DerivationInput( + raw_text="Pre-migration belief content from an old session.", + source_kind=INGEST_SOURCE_LEGACY_UNKNOWN, + ts=_TS, + )) + assert out.belief is not None + assert out.belief.lock_level == LOCK_NONE + + +# --------------------------------------------------------------------------- +# ts defaults +# --------------------------------------------------------------------------- + + +def test_empty_ts_triggers_utc_now() -> None: + """Hypothesis: when ts is empty, derive() stamps a non-empty ISO-8601 + string on the belief. Falsifiable by an empty or None created_at.""" + out = derive(DerivationInput( + raw_text="The deploy uses uv only.", + source_kind=INGEST_SOURCE_FILESYSTEM, + source_path="user", + )) + assert out.belief is not None + assert out.belief.created_at + assert "T" in out.belief.created_at # rough ISO-8601 check + + +# --------------------------------------------------------------------------- +# edges field +# --------------------------------------------------------------------------- + + +def test_derive_always_returns_empty_edges() -> None: + """Hypothesis: the edges field is always an empty list in v2.0 + (placeholder for future use). Falsifiable by any non-empty edges list.""" + for source_kind in ( + INGEST_SOURCE_FILESYSTEM, + INGEST_SOURCE_GIT, + INGEST_SOURCE_MCP_REMEMBER, + INGEST_SOURCE_CLI_REMEMBER, + INGEST_SOURCE_PYTHON_AST, + ): + out = derive(DerivationInput( + raw_text="Some sentence.", + source_kind=source_kind, + ts=_TS, + )) + assert out.edges == [], f"expected empty edges for source_kind={source_kind!r}" + + +# --------------------------------------------------------------------------- +# Determinism +# --------------------------------------------------------------------------- + + +def test_derive_is_deterministic() -> None: + """Hypothesis: calling derive() twice with identical inputs produces + identical outputs (same id, type, alpha, beta, content). Falsifiable + by any field that changes between calls.""" + inp = DerivationInput( + raw_text="This project must use uv for environment management.", + source_kind=INGEST_SOURCE_FILESYSTEM, + source_path="doc:README.md:p1", + ts=_TS, + ) + out1 = derive(inp) + out2 = derive(inp) + assert out1.belief is not None + assert out2.belief is not None + assert out1.belief.id == out2.belief.id + assert out1.belief.type == out2.belief.type + assert out1.belief.alpha == out2.belief.alpha + assert out1.belief.beta == out2.belief.beta + assert out1.belief.content == out2.belief.content