diff --git a/src/aelfrice/derivation.py b/src/aelfrice/derivation.py new file mode 100644 index 00000000..d1183bf2 --- /dev/null +++ b/src/aelfrice/derivation.py @@ -0,0 +1,213 @@ +"""Pure derivation layer: raw text in, Belief + Edges out, no store I/O. + +`derive()` is the single place that turns a classified (or fixed-prior) +text input into a `Belief` dataclass and any accompanying `Edge` objects. +All six v2.0 ingest entry points delegate to this function so the +deterministic part of the pipeline is tested and auditable in one place. + +Ingest entry points remain responsible for: + - store.get_belief() duplicate checks + - store.record_ingest() write-log entries + - store.insert_belief() / store.insert_edge() persistence + - store.record_corroboration() for re-assertions + - any store.update_belief() for the lock-upgrade path + +What `derive()` owns: + - belief-id derivation + - content-hash derivation + - classifier dispatch (regex or fixed-prior) + - alpha / beta / type / lock_level / origin selection + - edge list construction (currently empty for most source_kinds; + the DERIVED_FROM wiring in ingest_jsonl is caller-side because it + spans consecutive turns and depends on prior-turn state) +""" +from __future__ import annotations + +import hashlib +from dataclasses import dataclass +from typing import Final + +from aelfrice.classification import classify_sentence +from aelfrice.models import ( + BELIEF_FACTUAL, + INGEST_SOURCE_CLI_REMEMBER, + INGEST_SOURCE_FILESYSTEM, + INGEST_SOURCE_FEEDBACK_LOOP_SYNTHESIS, + INGEST_SOURCE_GIT, + INGEST_SOURCE_KINDS, + INGEST_SOURCE_LEGACY_UNKNOWN, + INGEST_SOURCE_MCP_REMEMBER, + INGEST_SOURCE_PYTHON_AST, + LOCK_NONE, + LOCK_USER, + ORIGIN_AGENT_INFERRED, + ORIGIN_USER_STATED, + Belief, + Edge, +) + +_BELIEF_ID_HEX_LEN: Final[int] = 16 + +# Source-kinds that go through the regex/LLM classify_sentence path. +_CLASSIFY_SOURCE_KINDS: Final[frozenset[str]] = frozenset({ + INGEST_SOURCE_FILESYSTEM, + INGEST_SOURCE_GIT, + INGEST_SOURCE_PYTHON_AST, + INGEST_SOURCE_FEEDBACK_LOOP_SYNTHESIS, + INGEST_SOURCE_LEGACY_UNKNOWN, +}) + +# Source-kinds that create user-locked beliefs with fixed priors. +_LOCK_SOURCE_KINDS: Final[frozenset[str]] = frozenset({ + INGEST_SOURCE_MCP_REMEMBER, + INGEST_SOURCE_CLI_REMEMBER, +}) + + +@dataclass(frozen=True) +class DerivationInput: + """All inputs needed to derive a Belief from raw text. + + Fields: + - raw_text: the sentence / paragraph / phrase to classify. + - source_kind: one of INGEST_SOURCE_KINDS; controls which derivation + path is taken (classify-based vs fixed-prior lock). + - source_path: the provenance label written into the belief-id hash + (e.g. "doc:README.md:p0", "git:commit:abc1234", "user", "triple"). + When None, the belief id is derived from (source_kind, raw_text). + - raw_meta: optional caller-side metadata; not consumed by derive() + itself but threaded through for caller bookkeeping. + - session_id: written to belief.session_id on the output. + - ts: ISO-8601 timestamp written to belief.created_at. + - classifier_version: reserved for future LLM-classifier versioning; + not used in the regex path. None is always valid. + - rule_set_hash: reserved for future deterministic rule-set pinning; + not used by derive() today. + """ + + raw_text: str + source_kind: str # one of INGEST_SOURCE_KINDS + source_path: str | None + raw_meta: dict | None # type: ignore[type-arg] + session_id: str | None + ts: str + classifier_version: str | None + rule_set_hash: str | None + + +@dataclass(frozen=True) +class DerivationOutput: + """Result of a derive() call. + + Fields: + - belief: the derived Belief, or None when classification rejects + the input (persist=False — questions, empty text). + - edges: edges to insert alongside the belief. Currently empty for + all source_kinds; reserved for future intra-turn edge generation. + - skip_reason: human-readable string when belief is None + ('persist=False', 'noise', etc.). None when belief is present. + """ + + belief: Belief | None + edges: list[Edge] + skip_reason: str | None + + +def _belief_id(text: str, source: str) -> str: + """Stable id derived from sha256(source \\x00 text)[:16]. + + Matches the scheme used by ingest._belief_id, scanner._derive_belief_id, + and classification._derive_belief_id so re-ingesting an identical + (source, text) pair is idempotent across all entry points. + """ + h = hashlib.sha256(f"{source}\x00{text}".encode("utf-8")).hexdigest() + return h[:_BELIEF_ID_HEX_LEN] + + +def _content_hash(text: str) -> str: + return hashlib.sha256(text.encode("utf-8")).hexdigest() + + +def derive(inp: DerivationInput) -> DerivationOutput: + """Pure function: raw text in, Belief + Edges out, no store I/O. + + Dispatches by `inp.source_kind`: + + - Classify-based (filesystem, git, python_ast, feedback_loop_synthesis, + legacy_unknown): calls `classify_sentence(raw_text, source_path)`. + Returns belief=None with skip_reason='persist=False' when the + classifier rejects the input (questions, empty text). The + `source_path` parameter is used as the classifier source so + source-prior deflation applies correctly (non-user sources get + deflated alpha). + + - Lock-based (mcp_remember, cli_remember): fixed priors + (alpha=9.0, beta=0.5), type=factual, lock_level=user, + origin=user_stated. Never rejects — the caller is explicitly + asserting a user-locked belief. + + The function is pure: same DerivationInput => identical + DerivationOutput. No global state, no I/O, no randomness. + + Raises ValueError for an unrecognised source_kind. + """ + if inp.source_kind not in INGEST_SOURCE_KINDS: + raise ValueError( + f"unknown source_kind: {inp.source_kind!r}; " + f"expected one of {sorted(INGEST_SOURCE_KINDS)}" + ) + + # The source label for the belief-id hash and classifier source + # adjustment. Falls back to source_kind when source_path is absent + # (e.g. triple-extractor callers that don't thread a path through). + source_label = inp.source_path if inp.source_path is not None else inp.source_kind + bid = _belief_id(inp.raw_text, source_label) + ch = _content_hash(inp.raw_text) + + if inp.source_kind in _LOCK_SOURCE_KINDS: + # Lock path: fixed high-confidence prior matching the requirement + # prior (9.0, 0.5). Matches the hardcoded values in mcp_server.tool_lock + # and cli._cmd_lock — user-locked beliefs carry the same prior as + # hard requirements. + alpha: float = 9.0 + beta: float = 0.5 + belief = Belief( + id=bid, + content=inp.raw_text, + content_hash=ch, + alpha=alpha, + beta=beta, + type=BELIEF_FACTUAL, + lock_level=LOCK_USER, + locked_at=inp.ts, + demotion_pressure=0, + created_at=inp.ts, + last_retrieved_at=None, + session_id=inp.session_id, + origin=ORIGIN_USER_STATED, + ) + return DerivationOutput(belief=belief, edges=[], skip_reason=None) + + # Classify-based path (filesystem, git, python_ast, etc.) + result = classify_sentence(inp.raw_text, source_label) + if not result.persist: + return DerivationOutput( + belief=None, edges=[], skip_reason="persist=False" + ) + + belief = Belief( + id=bid, + content=inp.raw_text, + content_hash=ch, + alpha=result.alpha, + beta=result.beta, + type=result.belief_type, + lock_level=LOCK_NONE, + locked_at=None, + demotion_pressure=0, + created_at=inp.ts, + last_retrieved_at=None, + session_id=inp.session_id, + origin=ORIGIN_AGENT_INFERRED, + ) + return DerivationOutput(belief=belief, edges=[], skip_reason=None) diff --git a/src/aelfrice/ingest.py b/src/aelfrice/ingest.py index 868f2103..1f789549 100644 --- a/src/aelfrice/ingest.py +++ b/src/aelfrice/ingest.py @@ -16,41 +16,23 @@ """ from __future__ import annotations -import hashlib import json from dataclasses import dataclass from datetime import datetime, timezone from pathlib import Path from typing import cast -from aelfrice.classification import classify_sentence +from aelfrice.derivation import DerivationInput, derive from aelfrice.extraction import extract_sentences from aelfrice.models import ( ANCHOR_TEXT_MAX_LEN, CORROBORATION_SOURCE_TRANSCRIPT_INGEST, EDGE_DERIVED_FROM, INGEST_SOURCE_FILESYSTEM, - LOCK_NONE, - ORIGIN_AGENT_INFERRED, - Belief, Edge, ) from aelfrice.store import MemoryStore -_BELIEF_ID_HEX_LEN: int = 16 - - -def _belief_id(text: str, source: str) -> str: - """Stable id derived from (source, text). Matches the scheme used - by classification._derive_belief_id and scanner._derive_belief_id - so re-ingesting an identical (text, source) pair is idempotent.""" - h = hashlib.sha256(f"{source}\x00{text}".encode("utf-8")).hexdigest() - return h[:_BELIEF_ID_HEX_LEN] - - -def _content_hash(text: str) -> str: - return hashlib.sha256(text.encode("utf-8")).hexdigest() - def _now_utc_iso() -> str: return datetime.now(timezone.utc).isoformat() @@ -114,11 +96,20 @@ def _ingest_turn_ids( ts = created_at or _now_utc_iso() inserted: list[str] = [] for sentence in sentences: - result = classify_sentence(sentence, source) - if not result.persist: + inp = DerivationInput( + raw_text=sentence, + source_kind=INGEST_SOURCE_FILESYSTEM, + source_path=source, + raw_meta=None, + session_id=session_id, + ts=ts, + classifier_version=None, + rule_set_hash=None, + ) + out = derive(inp) + if out.belief is None: continue - belief_id = _belief_id(sentence, source) - ch = _content_hash(sentence) + belief_id = out.belief.id if store.get_belief(belief_id) is not None: # Exact (source, sentence) duplicate: record a corroboration # so re-assertions are observable. Canonical row unchanged. @@ -139,22 +130,7 @@ def _ingest_turn_ids( session_id=session_id, ts=ts, ) - belief = Belief( - id=belief_id, - content=sentence, - content_hash=ch, - alpha=result.alpha, - beta=result.beta, - type=result.belief_type, - lock_level=LOCK_NONE, - locked_at=None, - demotion_pressure=0, - created_at=ts, - last_retrieved_at=None, - session_id=session_id, - origin=ORIGIN_AGENT_INFERRED, - ) - store.insert_belief(belief) + store.insert_belief(out.belief) inserted.append(belief_id) return inserted diff --git a/src/aelfrice/scanner.py b/src/aelfrice/scanner.py index f510a1e7..1df69642 100644 --- a/src/aelfrice/scanner.py +++ b/src/aelfrice/scanner.py @@ -26,6 +26,7 @@ from typing import Final from aelfrice.classification import classify_sentence +from aelfrice.derivation import DerivationInput, derive from aelfrice.inedible import is_inedible from aelfrice.models import ( INGEST_SOURCE_FILESYSTEM, @@ -233,36 +234,72 @@ def scan_repo( if not route.persist: skipped_non_persisting += 1 continue - belief_id = _derive_belief_id(candidate.text, candidate.source) - if store.get_belief(belief_id) is not None: - skipped_existing += 1 - continue created_at = candidate.commit_date or timestamp - # v2.0 #205 parallel-write: log the raw classifier input - # before materializing the belief. derived_belief_ids is - # known up-front because belief_id is deterministic on - # (source, text). - store.record_ingest( - source_kind=INGEST_SOURCE_FILESYSTEM, - source_path=candidate.source, - raw_text=candidate.text, - derived_belief_ids=[belief_id], - ts=created_at, - ) - store.insert_belief(Belief( - id=belief_id, - content=candidate.text, - content_hash=_content_hash(candidate.text), - alpha=route.alpha, - beta=route.beta, - type=route.belief_type, - lock_level=LOCK_NONE, - locked_at=None, - demotion_pressure=0, - created_at=created_at, - last_retrieved_at=None, - origin=route.origin, - )) + + if llm_router is None: + # Regex path: delegate fully to derive() — pure, deterministic. + inp = DerivationInput( + raw_text=candidate.text, + source_kind=INGEST_SOURCE_FILESYSTEM, + source_path=candidate.source, + raw_meta=None, + session_id=None, + ts=created_at, + classifier_version=None, + rule_set_hash=None, + ) + out = derive(inp) + # route.persist was already checked above; derive() should agree. + # If it disagrees (edge case: noise filter passed but classify + # rejects), treat as skipped. + if out.belief is None: + skipped_non_persisting += 1 + continue + belief_id = out.belief.id + if store.get_belief(belief_id) is not None: + skipped_existing += 1 + continue + # v2.0 #205 parallel-write. + store.record_ingest( + source_kind=INGEST_SOURCE_FILESYSTEM, + source_path=candidate.source, + raw_text=candidate.text, + derived_belief_ids=[belief_id], + ts=created_at, + ) + store.insert_belief(out.belief) + else: + # LLM-router path: the router supplies origin, alpha, beta + # directly. derive() is not used here because the LLM router + # may return a non-AGENT_INFERRED origin (e.g. DOCUMENT_RECENT) + # that is not representable through the current DerivationInput. + belief_id = _derive_belief_id(candidate.text, candidate.source) + if store.get_belief(belief_id) is not None: + skipped_existing += 1 + continue + # v2.0 #205 parallel-write. + store.record_ingest( + source_kind=INGEST_SOURCE_FILESYSTEM, + source_path=candidate.source, + raw_text=candidate.text, + derived_belief_ids=[belief_id], + ts=created_at, + ) + store.insert_belief(Belief( + id=belief_id, + content=candidate.text, + content_hash=_content_hash(candidate.text), + alpha=route.alpha, + beta=route.beta, + type=route.belief_type, + lock_level=LOCK_NONE, + locked_at=None, + demotion_pressure=0, + created_at=created_at, + last_retrieved_at=None, + origin=route.origin, + )) + inserted += 1 # Audit row for fallback insertions (spec § 7.2 step 3). if route.audit_source is not None: diff --git a/tests/test_derivation.py b/tests/test_derivation.py new file mode 100644 index 00000000..8bfef389 --- /dev/null +++ b/tests/test_derivation.py @@ -0,0 +1,319 @@ +"""Unit tests for derivation.derive() — pure function, no store needed. + +Covers: +- each INGEST_SOURCE_KIND produces a Belief (or rejects correctly) +- classify-based path: persist=False yields belief=None + skip_reason +- lock-based path: yields LOCK_USER + ORIGIN_USER_STATED belief +- purity: identical inputs produce equal outputs +- invalid source_kind raises ValueError +""" +from __future__ import annotations + +import pytest + +from aelfrice.derivation import DerivationInput, DerivationOutput, derive +from aelfrice.models import ( + BELIEF_FACTUAL, + BELIEF_PREFERENCE, + BELIEF_REQUIREMENT, + INGEST_SOURCE_CLI_REMEMBER, + INGEST_SOURCE_FEEDBACK_LOOP_SYNTHESIS, + INGEST_SOURCE_FILESYSTEM, + INGEST_SOURCE_GIT, + INGEST_SOURCE_LEGACY_UNKNOWN, + INGEST_SOURCE_MCP_REMEMBER, + INGEST_SOURCE_PYTHON_AST, + LOCK_NONE, + LOCK_USER, + ORIGIN_AGENT_INFERRED, + ORIGIN_USER_STATED, +) + +_TS = "2026-04-28T00:00:00Z" + + +def _inp( + raw_text: str, + source_kind: str, + source_path: str | None = "doc:README.md:p0", + session_id: str | None = None, +) -> DerivationInput: + return DerivationInput( + raw_text=raw_text, + source_kind=source_kind, + source_path=source_path, + raw_meta=None, + session_id=session_id, + ts=_TS, + classifier_version=None, + rule_set_hash=None, + ) + + +# --- Purity ----------------------------------------------------------------- + + +def test_derive_is_pure_identical_inputs_equal_outputs() -> None: + """Calling derive() twice with the same input yields structurally equal + results — same belief id, same alpha, same everything.""" + inp = _inp( + "The configuration file lives at the default path.", + INGEST_SOURCE_FILESYSTEM, + ) + out1 = derive(inp) + out2 = derive(inp) + assert out1.belief is not None + assert out2.belief is not None + assert out1.belief.id == out2.belief.id + assert out1.belief.alpha == out2.belief.alpha + assert out1.belief.beta == out2.belief.beta + assert out1.belief.type == out2.belief.type + assert out1.skip_reason == out2.skip_reason + + +# --- source_kind: filesystem ------------------------------------------------ + + +def test_filesystem_factual_statement_produces_belief() -> None: + out = derive(_inp( + "The default port is 8080 for the dashboard service.", + INGEST_SOURCE_FILESYSTEM, + )) + assert out.belief is not None + assert out.belief.type == BELIEF_FACTUAL + assert out.belief.lock_level == LOCK_NONE + assert out.belief.origin == ORIGIN_AGENT_INFERRED + assert out.skip_reason is None + + +def test_filesystem_question_rejected() -> None: + out = derive(_inp( + "What is the default port for the dashboard service?", + INGEST_SOURCE_FILESYSTEM, + )) + assert out.belief is None + assert out.skip_reason == "persist=False" + assert out.edges == [] + + +def test_filesystem_empty_text_rejected() -> None: + out = derive(_inp("", INGEST_SOURCE_FILESYSTEM)) + assert out.belief is None + assert out.skip_reason == "persist=False" + + +def test_filesystem_preference_classified_correctly() -> None: + out = derive(_inp( + "I prefer using uv for all Python package management.", + INGEST_SOURCE_FILESYSTEM, + source_path="user", + )) + assert out.belief is not None + assert out.belief.type == BELIEF_PREFERENCE + + +def test_filesystem_requirement_classified_correctly() -> None: + out = derive(_inp( + "You must use SSH key authentication for all deployments.", + INGEST_SOURCE_FILESYSTEM, + source_path="user", + )) + assert out.belief is not None + assert out.belief.type == BELIEF_REQUIREMENT + + +def test_filesystem_belief_id_stable_across_calls() -> None: + inp = _inp("Aelfrice stores beliefs in a local SQLite database.", INGEST_SOURCE_FILESYSTEM) + out1 = derive(inp) + out2 = derive(inp) + assert out1.belief is not None + assert out2.belief is not None + assert out1.belief.id == out2.belief.id + + +def test_filesystem_session_id_stamped_on_belief() -> None: + out = derive(_inp( + "The project uses conventional commits for all changes.", + INGEST_SOURCE_FILESYSTEM, + session_id="session-abc123", + )) + assert out.belief is not None + assert out.belief.session_id == "session-abc123" + + +def test_filesystem_ts_written_to_created_at() -> None: + out = derive(_inp( + "The benchmark harness evaluates retrieval quality.", + INGEST_SOURCE_FILESYSTEM, + )) + assert out.belief is not None + assert out.belief.created_at == _TS + + +# --- source_kind: git ------------------------------------------------------- + + +def test_git_factual_commit_subject_produces_belief() -> None: + out = derive(_inp( + "feat: add BM25 retrieval index for offline search", + INGEST_SOURCE_GIT, + source_path="git:commit:abc1234", + )) + assert out.belief is not None + assert out.belief.type == BELIEF_FACTUAL + assert out.belief.origin == ORIGIN_AGENT_INFERRED + + +def test_git_question_rejected() -> None: + out = derive(_inp( + "What does this commit change in the retrieval layer?", + INGEST_SOURCE_GIT, + source_path="git:commit:abc1234", + )) + assert out.belief is None + assert out.skip_reason == "persist=False" + + +# --- source_kind: python_ast ------------------------------------------------ + + +def test_python_ast_docstring_produces_belief() -> None: + out = derive(_inp( + "Sentence classification: assign one of the four belief types and a " + "source-adjusted Beta prior.", + INGEST_SOURCE_PYTHON_AST, + source_path="ast:src/aelfrice/classification.py:module", + )) + assert out.belief is not None + assert out.belief.origin == ORIGIN_AGENT_INFERRED + assert out.belief.lock_level == LOCK_NONE + + +# --- source_kind: mcp_remember ---------------------------------------------- + + +def test_mcp_remember_produces_locked_belief() -> None: + out = derive(_inp( + "Always use SSH key authentication for production deployments.", + INGEST_SOURCE_MCP_REMEMBER, + source_path=None, + )) + assert out.belief is not None + assert out.belief.lock_level == LOCK_USER + assert out.belief.origin == ORIGIN_USER_STATED + assert out.belief.locked_at == _TS + assert out.skip_reason is None + + +def test_mcp_remember_high_confidence_prior() -> None: + out = derive(_inp( + "Never commit secrets to the repository.", + INGEST_SOURCE_MCP_REMEMBER, + source_path=None, + )) + assert out.belief is not None + # Lock path uses fixed high-confidence priors matching the requirement prior. + assert out.belief.alpha == 9.0 + assert out.belief.beta == 0.5 + + +def test_mcp_remember_question_still_persists() -> None: + """Lock path never rejects — caller is asserting a belief, not classifying.""" + out = derive(_inp( + "What should we use for authentication?", + INGEST_SOURCE_MCP_REMEMBER, + source_path=None, + )) + # Lock path bypasses classify_sentence entirely — always produces a belief. + assert out.belief is not None + assert out.belief.lock_level == LOCK_USER + + +# --- source_kind: cli_remember ---------------------------------------------- + + +def test_cli_remember_produces_locked_belief() -> None: + out = derive(_inp( + "Prefer uv over pip for all Python environment management.", + INGEST_SOURCE_CLI_REMEMBER, + source_path=None, + )) + assert out.belief is not None + assert out.belief.lock_level == LOCK_USER + assert out.belief.origin == ORIGIN_USER_STATED + + +def test_cli_remember_and_mcp_remember_same_text_same_id() -> None: + """mcp_remember and cli_remember with no source_path both fall back to + source_kind as the id-hash key — so they produce different ids for the + same text (different source_kind = different derivation context).""" + out_mcp = derive(_inp("Use uv for Python.", INGEST_SOURCE_MCP_REMEMBER, source_path=None)) + out_cli = derive(_inp("Use uv for Python.", INGEST_SOURCE_CLI_REMEMBER, source_path=None)) + assert out_mcp.belief is not None + assert out_cli.belief is not None + # Different source_kind → different id hash + assert out_mcp.belief.id != out_cli.belief.id + + +# --- source_kind: feedback_loop_synthesis ----------------------------------- + + +def test_feedback_loop_synthesis_produces_belief() -> None: + out = derive(_inp( + "The feedback loop converges after three correction cycles.", + INGEST_SOURCE_FEEDBACK_LOOP_SYNTHESIS, + source_path="synthesis:session-abc", + )) + assert out.belief is not None + assert out.belief.origin == ORIGIN_AGENT_INFERRED + + +# --- source_kind: legacy_unknown -------------------------------------------- + + +def test_legacy_unknown_produces_belief() -> None: + out = derive(_inp( + "Pre-migration belief backfilled from the old schema.", + INGEST_SOURCE_LEGACY_UNKNOWN, + source_path="legacy:row:42", + )) + assert out.belief is not None + assert out.belief.origin == ORIGIN_AGENT_INFERRED + assert out.belief.lock_level == LOCK_NONE + + +# --- Invalid source_kind ---------------------------------------------------- + + +def test_invalid_source_kind_raises_value_error() -> None: + with pytest.raises(ValueError, match="unknown source_kind"): + derive(_inp("Some text.", "bogus_kind")) + + +# --- Edge list -------------------------------------------------------------- + + +def test_derive_returns_empty_edges_list() -> None: + out = derive(_inp( + "The context rebuilder stitches beliefs into a coherent block.", + INGEST_SOURCE_FILESYSTEM, + )) + assert out.edges == [] + + +# --- DerivationOutput invariants ------------------------------------------- + + +def test_belief_none_implies_skip_reason_set() -> None: + out = derive(_inp("What is the answer?", INGEST_SOURCE_FILESYSTEM)) + assert out.belief is None + assert out.skip_reason is not None + + +def test_belief_present_implies_skip_reason_none() -> None: + out = derive(_inp( + "The retrieval layer uses BM25 plus TF-IDF scoring.", + INGEST_SOURCE_FILESYSTEM, + )) + assert out.belief is not None + assert out.skip_reason is None