feat(ingest): transcript-noise filter to stop ingesting tool-output and agent-emit as beliefs (#675)

src/aelfrice/ingest.py

@@ -24,6 +24,7 @@
 
 from aelfrice.derivation_worker import run_worker
 from aelfrice.extraction import extract_sentences
+from aelfrice.noise_filter import is_transcript_noise
 from aelfrice.session_resolution import resolve_session_id
 from aelfrice.models import (
     ANCHOR_TEXT_MAX_LEN,
@@ -117,6 +118,7 @@ def _ingest_turn_ids(
     DERIVED_FROM edges between consecutive turns within a session.
     """
     sentences = extract_sentences(text)
+    sentences = [s for s in sentences if not is_transcript_noise(s)]
     if not sentences:
         return []
 

src/aelfrice/noise_filter.py

@@ -132,6 +132,68 @@
     "headings", "checklists", "fragments", "license",
 })
 
+
+# ---------------------------------------------------------------------------
+# Transcript-noise filter — compiled once at module load
+# ---------------------------------------------------------------------------
+#
+# Five categories of sentences that appear in transcript turns but carry
+# zero belief content. Each category is documented below and tested in
+# tests/test_noise_filter.py.
+#
+# 1. Shell-command shape: starts with a recognised shell prefix at the
+#    leftmost position (case-sensitive). Covers `cd /`, `git `, `gh `,
+#    `uv run`, `pytest`, `python `. The leading space in `git ` and
+#    `gh ` is intentional — it distinguishes the command from prose that
+#    starts with a word that merely contains the token (e.g. "ghosts").
+#
+# 2. Tool-call rendering glyph: ⏺ (U+23FA). Emitted at the start
+#    of tool-call narration lines by some transcript surfaces.
+#
+# 3. Pseudo-XML worktree/task tags: `<worktree`, `<output-file`,
+#    `<task-`, `<summary>Background`. These are structural delimiters
+#    injected by orchestration layers; they are not prose beliefs.
+#
+# 4. Single-word progress emits: matches `^[A-Z][a-z]+ing\.$` — a lone
+#    capitalised gerund followed by a full stop. Examples: "Polling.",
+#    "Running.", "Waiting." Note that "Standing by." does NOT match
+#    this pattern (two words); it is caught by category 5.
+#
+# 5. Agent ack emits: short one-line acknowledgements that convey no
+#    project-specific knowledge. Pattern allows the bare keyword or the
+#    keyword followed by up to 40 characters. Examples: "Yes.",
+#    "Standing by.", "Polling for results.", "Nothing to report.",
+#    "Ready when you are.", "No changes needed."
+
+_TRANSCRIPT_SHELL_PREFIXES: Final[tuple[str, ...]] = (
+    "cd /",
+    "git ",
+    "gh ",
+    "uv run",
+    "pytest",
+    "python ",
+)
+
+# U+23FA — tool-call rendering glyph emitted by some transcript surfaces.
+_TRANSCRIPT_GLYPH_PREFIX: Final[str] = "⏺"
+
+_TRANSCRIPT_XML_PREFIXES: Final[tuple[str, ...]] = (
+    "<worktree",
+    "<output-file",
+    "<task-",
+    "<summary>Background",
+)
+
+# Single-word capitalised gerund followed by a full stop: "Polling.", "Running."
+_TRANSCRIPT_PROGRESS_RE: Final[re.Pattern[str]] = re.compile(
+    r"^[A-Z][a-z]+ing\.$"
+)
+
+# Agent ack emit: bare keyword or keyword + optional short trailing text.
+_TRANSCRIPT_ACK_RE: Final[re.Pattern[str]] = re.compile(
+    r"^(Yes|No|Standing by|Ready|Nothing|Polling)( .{0,40})?\.?$"
+)
+
 CONFIG_FILENAME: Final[str] = ".aelfrice.toml"
 
 
@@ -433,6 +495,54 @@ def is_license_boilerplate(
     return any(p.search(text) is not None for p in _LICENSE_PATTERNS)
 
 
+def is_transcript_noise(sentence: str) -> bool:
+    """Return True if `sentence` is transcript scaffolding, not a belief.
+
+    Checks five categories in order; first match returns True:
+
+    1. **Shell-command shape** — starts with a recognised shell prefix
+       (`cd /`, `git `, `gh `, `uv run`, `pytest`, `python `).
+       Match is case-sensitive and position-anchored at index 0.
+    2. **Tool-call rendering glyph** — starts with ⏺ (U+23FA).
+    3. **Pseudo-XML structural tags** — starts with `<worktree`,
+       `<output-file`, `<task-`, or `<summary>Background`.
+    4. **Single-word progress emit** — matches `^[A-Z][a-z]+ing\\.$`
+       (a lone capitalised gerund and a full stop, nothing else).
+    5. **Agent ack emit** — matches
+       `^(Yes|No|Standing by|Ready|Nothing|Polling)( .{0,40})?\\.?$`;
+       covers bare keywords and short trailing phrases up to 40 chars.
+
+    All patterns are case-sensitive as written. Empty or whitespace-only
+    strings return False (they are handled upstream by `is_noise`).
+    """
+    if not sentence or not sentence.strip():
+        return False
+
+    # Category 1: shell-command shape
+    for prefix in _TRANSCRIPT_SHELL_PREFIXES:
+        if sentence.startswith(prefix):
+            return True
+
+    # Category 2: tool-call rendering glyph (U+23FA)
+    if sentence.startswith(_TRANSCRIPT_GLYPH_PREFIX):
+        return True
+
+    # Category 3: pseudo-XML structural tags
+    for prefix in _TRANSCRIPT_XML_PREFIXES:
+        if sentence.startswith(prefix):
+            return True
+
+    # Category 4: single-word progress emit
+    if _TRANSCRIPT_PROGRESS_RE.match(sentence) is not None:
+        return True
+
+    # Category 5: agent ack emit
+    if _TRANSCRIPT_ACK_RE.match(sentence) is not None:
+        return True
+
+    return False
+
+
 # Punctuation characters stripped during N-gram tokenisation. We remove
 # everything that is not a word character or whitespace so that
 # "features," and "features" are treated as the same token.

tests/test_ingest.py

@@ -6,7 +6,7 @@
 
 import pytest
 
-from aelfrice.ingest import ingest_turn
+from aelfrice.ingest import _ingest_turn_ids, ingest_turn
 from aelfrice.store import MemoryStore
 
 
@@ -112,3 +112,59 @@ def test_complete_session_is_no_op(store: MemoryStore) -> None:
     session = store.create_session()
     # Should not raise even though the session is not in any table.
     store.complete_session(session.id)
+
+
+# --- Transcript-noise filter integration -----------------------------------
+
+
+def test_ingest_turn_ids_filters_transcript_noise_and_keeps_real_sentence(
+    store: MemoryStore,
+) -> None:
+    """A turn containing one sentence from each transcript-noise category
+    plus one real sentence produces exactly one derived belief id, and the
+    persisted belief's content matches the real sentence.
+
+    Noise sentences used (one per category):
+      cat 1 — shell-command shape:       'git checkout main'
+      cat 2 — tool-call rendering glyph: '⏺ Bash(git status)'
+      cat 3 — pseudo-XML tag:            '<worktree id="1">'
+      cat 4 — single-word progress emit: 'Running.'
+      cat 5 — agent ack emit:            'Standing by.'
+      real  — plain factual prose
+
+    The test uses extract_sentences indirectly through _ingest_turn_ids.
+    We build the input as a block of newline-separated strings so that
+    each line reaches is_transcript_noise as an independent sentence.
+    """
+    real_sentence = (
+        "The ingest pipeline stores each classified sentence as a belief "
+        "in the working memory store."
+    )
+    # Build a transcript turn: noise lines first, then the real sentence.
+    # extract_sentences splits on sentence boundaries; using newlines
+    # between the short noise lines ensures they come through as individual
+    # sentences rather than being merged with the prose.
+    turn_text = (
+        "git checkout main\n"
+        "⏺ Bash(git status)\n"
+        "<worktree id='1'>\n"
+        "Running.\n"
+        "Standing by.\n"
+        + real_sentence
+    )
+    ids = _ingest_turn_ids(
+        store=store,
+        text=turn_text,
+        source="test",
+        session_id="sess-675-test",
+    )
+    # Exactly one new belief should have been derived (the real sentence).
+    assert len(ids) == 1, (
+        f"Expected exactly 1 derived belief id, got {len(ids)}: {ids}"
+    )
+    # The persisted belief content must match the real sentence.
+    belief = store.get_belief(ids[0])
+    assert belief is not None, "Derived belief id has no matching belief in store."
+    assert real_sentence in belief.content, (
+        f"Belief content {belief.content!r} does not contain the real sentence."
+    )

tests/test_noise_filter.py

@@ -13,6 +13,7 @@
     is_license_boilerplate,
     is_noise,
     is_three_word_fragment,
+    is_transcript_noise,
     similarity_to_reference,
 )
 
@@ -329,3 +330,177 @@ def test_similarity_excerpt_is_none_when_clean(
     )
     assert over is False
     assert excerpt is None
+
+
+# --- is_transcript_noise: category 1 — shell-command shape ----------------
+
+
+def test_transcript_noise_cd_prefix() -> None:
+    assert is_transcript_noise("cd /home/user/projects") is True
+
+
+def test_transcript_noise_git_prefix() -> None:
+    assert is_transcript_noise("git checkout main") is True
+
+
+def test_transcript_noise_gh_prefix() -> None:
+    assert is_transcript_noise("gh pr view 675") is True
+
+
+def test_transcript_noise_uv_run_prefix() -> None:
+    assert is_transcript_noise("uv run pytest tests/") is True
+
+
+def test_transcript_noise_pytest_prefix() -> None:
+    assert is_transcript_noise("pytest tests/test_ingest.py -v") is True
+
+
+def test_transcript_noise_python_prefix() -> None:
+    assert is_transcript_noise("python script.py --flag") is True
+
+
+def test_transcript_noise_prose_mentioning_git_is_not_noise() -> None:
+    """git token not at position 0 — must not match."""
+    assert is_transcript_noise("The git history shows a clean merge.") is False
+
+
+def test_transcript_noise_prose_mentioning_gh_is_not_noise() -> None:
+    assert is_transcript_noise("The gh CLI wraps the GitHub REST API.") is False
+
+
+# --- is_transcript_noise: category 2 — tool-call rendering glyph (U+23FA) -
+
+
+def test_transcript_noise_glyph_prefix() -> None:
+    assert is_transcript_noise("⏺ Bash(git status)") is True
+
+
+def test_transcript_noise_glyph_alone() -> None:
+    assert is_transcript_noise("⏺") is True
+
+
+def test_transcript_noise_prose_does_not_start_with_glyph() -> None:
+    assert is_transcript_noise("The tool-call rendering glyph is documented.") is False
+
+
+# --- is_transcript_noise: category 3 — pseudo-XML structural tags ----------
+
+
+def test_transcript_noise_worktree_tag() -> None:
+    assert is_transcript_noise("<worktree id='1'>") is True
+
+
+def test_transcript_noise_output_file_tag() -> None:
+    assert is_transcript_noise("<output-file path='x.py'>") is True
+
+
+def test_transcript_noise_task_tag() -> None:
+    assert is_transcript_noise("<task-17>") is True
+
+
+def test_transcript_noise_summary_background_tag() -> None:
+    assert is_transcript_noise("<summary>Background context here.") is True
+
+
+def test_transcript_noise_prose_not_starting_with_xml_tag() -> None:
+    assert is_transcript_noise("The worktree contains three branches.") is False
+
+
+# --- is_transcript_noise: category 4 — single-word progress emit ----------
+
+
+def test_transcript_noise_polling_dot() -> None:
+    assert is_transcript_noise("Polling.") is True
+
+
+def test_transcript_noise_running_dot() -> None:
+    assert is_transcript_noise("Running.") is True
+
+
+def test_transcript_noise_waiting_dot() -> None:
+    assert is_transcript_noise("Waiting.") is True
+
+
+def test_transcript_noise_progress_two_words_does_not_match_progress_re() -> None:
+    """'Polling for results.' is two words — progress_re won't match it.
+    However, the ACK regex (category 5) does catch it. The net result is
+    is_transcript_noise returns True, which is the correct behaviour per spec."""
+    assert is_transcript_noise("Polling for results.") is True
+
+
+def test_transcript_noise_lowercase_gerund_is_not_progress() -> None:
+    """Pattern requires capital first letter."""
+    assert is_transcript_noise("polling.") is False
+
+
+def test_transcript_noise_gerund_without_dot_still_caught_by_ack() -> None:
+    """'Polling' without a trailing dot doesn't match the progress regex
+    (which requires the dot), but the ACK regex (category 5) allows an
+    optional period — so 'Polling' alone is still transcript noise."""
+    assert is_transcript_noise("Polling") is True
+
+
+# --- is_transcript_noise: category 5 — agent ack emit ---------------------
+
+
+def test_transcript_noise_ack_yes() -> None:
+    assert is_transcript_noise("Yes.") is True
+
+
+def test_transcript_noise_ack_no() -> None:
+    assert is_transcript_noise("No.") is True
+
+
+def test_transcript_noise_ack_standing_by() -> None:
+    assert is_transcript_noise("Standing by.") is True
+
+
+def test_transcript_noise_ack_ready() -> None:
+    assert is_transcript_noise("Ready.") is True
+
+
+def test_transcript_noise_ack_nothing() -> None:
+    assert is_transcript_noise("Nothing.") is True
+
+
+def test_transcript_noise_ack_polling() -> None:
+    assert is_transcript_noise("Polling.") is True
+
+
+def test_transcript_noise_ack_with_short_trailing_text() -> None:
+    assert is_transcript_noise("Ready when you are.") is True
+
+
+def test_transcript_noise_ack_nothing_to_report() -> None:
+    assert is_transcript_noise("Nothing to report.") is True
+
+
+def test_transcript_noise_ack_standing_by_for_direction() -> None:
+    assert is_transcript_noise("Standing by for your direction.") is True
+
+
+def test_transcript_noise_ack_polling_for_results() -> None:
+    """'Polling for results.' matches ACK regex (Polling + short trailing)."""
+    assert is_transcript_noise("Polling for results.") is True
+
+
+def test_transcript_noise_ack_no_trailing_punctuation() -> None:
+    assert is_transcript_noise("Ready") is True
+
+
+def test_transcript_noise_prose_is_not_ack() -> None:
+    assert is_transcript_noise(
+        "The retrieval pipeline drops short acks."
+    ) is False
+
+
+def test_transcript_noise_real_sentence_is_not_noise() -> None:
+    assert is_transcript_noise(
+        "The default DB path is keyed by SHA256 of the working directory."
+    ) is False
+
+
+def test_transcript_noise_empty_returns_false() -> None:
+    """Empty strings are handled upstream by is_noise; is_transcript_noise
+    returns False for them rather than True."""
+    assert is_transcript_noise("") is False