From b874c3ac78c9c9cf5c03057ab516c4884f342c65 Mon Sep 17 00:00:00 2001 From: rrs <276464689+robotrocketscience@users.noreply.github.com> Date: Mon, 11 May 2026 11:55:58 -0700 Subject: [PATCH 1/4] feat(ingest): is_transcript_noise predicate for transcript-noise shapes (#675) Add is_transcript_noise(sentence) to noise_filter.py. Five categories: shell-command prefixes (cd /, git , gh , uv run, pytest, python ), U+23FA tool-call rendering glyph, pseudo-XML structural tags, single-word progress emits (^[A-Z][a-z]+ing\.$), and agent ack emits. Regexes compiled once at module load as module-level Finals. Unit tests cover each category with positive and negative cases including all specified edge cases. --- src/aelfrice/noise_filter.py | 110 ++++++++++++++++++++++ tests/test_noise_filter.py | 175 +++++++++++++++++++++++++++++++++++ 2 files changed, 285 insertions(+) diff --git a/src/aelfrice/noise_filter.py b/src/aelfrice/noise_filter.py index a7089547c..dfe4a125c 100644 --- a/src/aelfrice/noise_filter.py +++ b/src/aelfrice/noise_filter.py @@ -132,6 +132,68 @@ "headings", "checklists", "fragments", "license", }) + +# --------------------------------------------------------------------------- +# Transcript-noise filter — compiled once at module load +# --------------------------------------------------------------------------- +# +# Five categories of sentences that appear in transcript turns but carry +# zero belief content. Each category is documented below and tested in +# tests/test_noise_filter.py. +# +# 1. Shell-command shape: starts with a recognised shell prefix at the +# leftmost position (case-sensitive). Covers `cd /`, `git `, `gh `, +# `uv run`, `pytest`, `python `. The leading space in `git ` and +# `gh ` is intentional — it distinguishes the command from prose that +# starts with a word that merely contains the token (e.g. "ghosts"). +# +# 2. Agent tool-call rendering glyph: ⏺ (U+23FA). Emitted at the start +# of tool-call narration lines by some transcript surfaces. +# +# 3. Pseudo-XML worktree/task tags: `Background`. These are structural delimiters +# injected by orchestration layers; they are not prose beliefs. +# +# 4. Single-word progress emits: matches `^[A-Z][a-z]+ing\.$` — a lone +# capitalised gerund followed by a full stop. Examples: "Polling.", +# "Running.", "Waiting." Note that "Standing by." does NOT match +# this pattern (two words); it is caught by category 5. +# +# 5. Agent ack emits: short one-line acknowledgements that convey no +# project-specific knowledge. Pattern allows the bare keyword or the +# keyword followed by up to 40 characters. Examples: "Yes.", +# "Standing by.", "Polling for results.", "Nothing to report.", +# "Ready when you are.", "No changes needed." + +_TRANSCRIPT_SHELL_PREFIXES: Final[tuple[str, ...]] = ( + "cd /", + "git ", + "gh ", + "uv run", + "pytest", + "python ", +) + +# U+23FA — tool-call rendering glyph emitted by some transcript surfaces. +_TRANSCRIPT_GLYPH_PREFIX: Final[str] = "⏺" + +_TRANSCRIPT_XML_PREFIXES: Final[tuple[str, ...]] = ( + "Background", +) + +# Single-word capitalised gerund followed by a full stop: "Polling.", "Running." +_TRANSCRIPT_PROGRESS_RE: Final[re.Pattern[str]] = re.compile( + r"^[A-Z][a-z]+ing\.$" +) + +# Agent ack emit: bare keyword or keyword + optional short trailing text. +_TRANSCRIPT_ACK_RE: Final[re.Pattern[str]] = re.compile( + r"^(Yes|No|Standing by|Ready|Nothing|Polling)( .{0,40})?\.?$" +) + CONFIG_FILENAME: Final[str] = ".aelfrice.toml" @@ -433,6 +495,54 @@ def is_license_boilerplate( return any(p.search(text) is not None for p in _LICENSE_PATTERNS) +def is_transcript_noise(sentence: str) -> bool: + """Return True if `sentence` is transcript scaffolding, not a belief. + + Checks five categories in order; first match returns True: + + 1. **Shell-command shape** — starts with a recognised shell prefix + (`cd /`, `git `, `gh `, `uv run`, `pytest`, `python `). + Match is case-sensitive and position-anchored at index 0. + 2. **Tool-call rendering glyph** — starts with ⏺ (U+23FA). + 3. **Pseudo-XML structural tags** — starts with `Background`. + 4. **Single-word progress emit** — matches `^[A-Z][a-z]+ing\\.$` + (a lone capitalised gerund and a full stop, nothing else). + 5. **Agent ack emit** — matches + `^(Yes|No|Standing by|Ready|Nothing|Polling)( .{0,40})?\\.*$`; + covers bare keywords and short trailing phrases up to 40 chars. + + All patterns are case-sensitive as written. Empty or whitespace-only + strings return False (they are handled upstream by `is_noise`). + """ + if not sentence or not sentence.strip(): + return False + + # Category 1: shell-command shape + for prefix in _TRANSCRIPT_SHELL_PREFIXES: + if sentence.startswith(prefix): + return True + + # Category 2: tool-call rendering glyph (U+23FA) + if sentence.startswith(_TRANSCRIPT_GLYPH_PREFIX): + return True + + # Category 3: pseudo-XML structural tags + for prefix in _TRANSCRIPT_XML_PREFIXES: + if sentence.startswith(prefix): + return True + + # Category 4: single-word progress emit + if _TRANSCRIPT_PROGRESS_RE.match(sentence) is not None: + return True + + # Category 5: agent ack emit + if _TRANSCRIPT_ACK_RE.match(sentence) is not None: + return True + + return False + + # Punctuation characters stripped during N-gram tokenisation. We remove # everything that is not a word character or whitespace so that # "features," and "features" are treated as the same token. diff --git a/tests/test_noise_filter.py b/tests/test_noise_filter.py index 7e56ecbd8..77e64a859 100644 --- a/tests/test_noise_filter.py +++ b/tests/test_noise_filter.py @@ -13,6 +13,7 @@ is_license_boilerplate, is_noise, is_three_word_fragment, + is_transcript_noise, similarity_to_reference, ) @@ -329,3 +330,177 @@ def test_similarity_excerpt_is_none_when_clean( ) assert over is False assert excerpt is None + + +# --- is_transcript_noise: category 1 — shell-command shape ---------------- + + +def test_transcript_noise_cd_prefix() -> None: + assert is_transcript_noise("cd /home/user/projects") is True + + +def test_transcript_noise_git_prefix() -> None: + assert is_transcript_noise("git checkout main") is True + + +def test_transcript_noise_gh_prefix() -> None: + assert is_transcript_noise("gh pr view 675") is True + + +def test_transcript_noise_uv_run_prefix() -> None: + assert is_transcript_noise("uv run pytest tests/") is True + + +def test_transcript_noise_pytest_prefix() -> None: + assert is_transcript_noise("pytest tests/test_ingest.py -v") is True + + +def test_transcript_noise_python_prefix() -> None: + assert is_transcript_noise("python script.py --flag") is True + + +def test_transcript_noise_prose_mentioning_git_is_not_noise() -> None: + """git token not at position 0 — must not match.""" + assert is_transcript_noise("The git history shows a clean merge.") is False + + +def test_transcript_noise_prose_mentioning_gh_is_not_noise() -> None: + assert is_transcript_noise("The gh CLI wraps the GitHub REST API.") is False + + +# --- is_transcript_noise: category 2 — tool-call rendering glyph (U+23FA) - + + +def test_transcript_noise_glyph_prefix() -> None: + assert is_transcript_noise("⏺ Bash(git status)") is True + + +def test_transcript_noise_glyph_alone() -> None: + assert is_transcript_noise("⏺") is True + + +def test_transcript_noise_prose_does_not_start_with_glyph() -> None: + assert is_transcript_noise("The tool-call rendering glyph is documented.") is False + + +# --- is_transcript_noise: category 3 — pseudo-XML structural tags ---------- + + +def test_transcript_noise_worktree_tag() -> None: + assert is_transcript_noise("") is True + + +def test_transcript_noise_output_file_tag() -> None: + assert is_transcript_noise("") is True + + +def test_transcript_noise_task_tag() -> None: + assert is_transcript_noise("") is True + + +def test_transcript_noise_summary_background_tag() -> None: + assert is_transcript_noise("Background context here.") is True + + +def test_transcript_noise_prose_not_starting_with_xml_tag() -> None: + assert is_transcript_noise("The worktree contains three branches.") is False + + +# --- is_transcript_noise: category 4 — single-word progress emit ---------- + + +def test_transcript_noise_polling_dot() -> None: + assert is_transcript_noise("Polling.") is True + + +def test_transcript_noise_running_dot() -> None: + assert is_transcript_noise("Running.") is True + + +def test_transcript_noise_waiting_dot() -> None: + assert is_transcript_noise("Waiting.") is True + + +def test_transcript_noise_progress_two_words_does_not_match_progress_re() -> None: + """'Polling for results.' is two words — progress_re won't match it. + However, the ACK regex (category 5) does catch it. The net result is + is_transcript_noise returns True, which is the correct behaviour per spec.""" + assert is_transcript_noise("Polling for results.") is True + + +def test_transcript_noise_lowercase_gerund_is_not_progress() -> None: + """Pattern requires capital first letter.""" + assert is_transcript_noise("polling.") is False + + +def test_transcript_noise_gerund_without_dot_still_caught_by_ack() -> None: + """'Polling' without a trailing dot doesn't match the progress regex + (which requires the dot), but the ACK regex (category 5) allows an + optional period — so 'Polling' alone is still transcript noise.""" + assert is_transcript_noise("Polling") is True + + +# --- is_transcript_noise: category 5 — agent ack emit --------------------- + + +def test_transcript_noise_ack_yes() -> None: + assert is_transcript_noise("Yes.") is True + + +def test_transcript_noise_ack_no() -> None: + assert is_transcript_noise("No.") is True + + +def test_transcript_noise_ack_standing_by() -> None: + assert is_transcript_noise("Standing by.") is True + + +def test_transcript_noise_ack_ready() -> None: + assert is_transcript_noise("Ready.") is True + + +def test_transcript_noise_ack_nothing() -> None: + assert is_transcript_noise("Nothing.") is True + + +def test_transcript_noise_ack_polling() -> None: + assert is_transcript_noise("Polling.") is True + + +def test_transcript_noise_ack_with_short_trailing_text() -> None: + assert is_transcript_noise("Ready when you are.") is True + + +def test_transcript_noise_ack_nothing_to_report() -> None: + assert is_transcript_noise("Nothing to report.") is True + + +def test_transcript_noise_ack_standing_by_for_direction() -> None: + assert is_transcript_noise("Standing by for your direction.") is True + + +def test_transcript_noise_ack_polling_for_results() -> None: + """'Polling for results.' matches ACK regex (Polling + short trailing).""" + assert is_transcript_noise("Polling for results.") is True + + +def test_transcript_noise_ack_no_trailing_punctuation() -> None: + assert is_transcript_noise("Ready") is True + + +def test_transcript_noise_prose_is_not_ack() -> None: + assert is_transcript_noise( + "The retrieval pipeline drops short acks." + ) is False + + +def test_transcript_noise_real_sentence_is_not_noise() -> None: + assert is_transcript_noise( + "The default DB path is keyed by SHA256 of the working directory." + ) is False + + +def test_transcript_noise_empty_returns_false() -> None: + """Empty strings are handled upstream by is_noise; is_transcript_noise + returns False for them rather than True.""" + assert is_transcript_noise("") is False From 94578d8209bf09b05e27ff6c6048b6ee55acf810 Mon Sep 17 00:00:00 2001 From: rrs <276464689+robotrocketscience@users.noreply.github.com> Date: Mon, 11 May 2026 11:58:01 -0700 Subject: [PATCH 2/4] feat(ingest): wire is_transcript_noise into _ingest_turn_ids (#675) Single-line filter in _ingest_turn_ids immediately after extract_sentences: sentences = [s for s in sentences if not is_transcript_noise(s)] Import added at top of ingest.py. Integration test in test_ingest.py verifies that a turn containing one noise sentence per category plus one real sentence produces exactly 1 derived belief id with the correct content. --- src/aelfrice/ingest.py | 2 ++ tests/test_ingest.py | 58 +++++++++++++++++++++++++++++++++++++++++- 2 files changed, 59 insertions(+), 1 deletion(-) diff --git a/src/aelfrice/ingest.py b/src/aelfrice/ingest.py index f2f34346e..fa4d9d629 100644 --- a/src/aelfrice/ingest.py +++ b/src/aelfrice/ingest.py @@ -24,6 +24,7 @@ from aelfrice.derivation_worker import run_worker from aelfrice.extraction import extract_sentences +from aelfrice.noise_filter import is_transcript_noise from aelfrice.session_resolution import resolve_session_id from aelfrice.models import ( ANCHOR_TEXT_MAX_LEN, @@ -117,6 +118,7 @@ def _ingest_turn_ids( DERIVED_FROM edges between consecutive turns within a session. """ sentences = extract_sentences(text) + sentences = [s for s in sentences if not is_transcript_noise(s)] if not sentences: return [] diff --git a/tests/test_ingest.py b/tests/test_ingest.py index cdca2ea50..8ae168d93 100644 --- a/tests/test_ingest.py +++ b/tests/test_ingest.py @@ -6,7 +6,7 @@ import pytest -from aelfrice.ingest import ingest_turn +from aelfrice.ingest import _ingest_turn_ids, ingest_turn from aelfrice.store import MemoryStore @@ -112,3 +112,59 @@ def test_complete_session_is_no_op(store: MemoryStore) -> None: session = store.create_session() # Should not raise even though the session is not in any table. store.complete_session(session.id) + + +# --- Transcript-noise filter integration ----------------------------------- + + +def test_ingest_turn_ids_filters_transcript_noise_and_keeps_real_sentence( + store: MemoryStore, +) -> None: + """A turn containing one sentence from each transcript-noise category + plus one real sentence produces exactly one derived belief id, and the + persisted belief's content matches the real sentence. + + Noise sentences used (one per category): + cat 1 — shell-command shape: 'git checkout main' + cat 2 — tool-call rendering glyph: '⏺ Bash(git status)' + cat 3 — pseudo-XML tag: '' + cat 4 — single-word progress emit: 'Running.' + cat 5 — agent ack emit: 'Standing by.' + real — plain factual prose + + The test uses extract_sentences indirectly through _ingest_turn_ids. + We build the input as a block of newline-separated strings so that + each line reaches is_transcript_noise as an independent sentence. + """ + real_sentence = ( + "The ingest pipeline stores each classified sentence as a belief " + "in the working memory store." + ) + # Build a transcript turn: noise lines first, then the real sentence. + # extract_sentences splits on sentence boundaries; using newlines + # between the short noise lines ensures they come through as individual + # sentences rather than being merged with the prose. + turn_text = ( + "git checkout main\n" + "⏺ Bash(git status)\n" + "\n" + "Running.\n" + "Standing by.\n" + + real_sentence + ) + ids = _ingest_turn_ids( + store=store, + text=turn_text, + source="test", + session_id="sess-675-test", + ) + # Exactly one new belief should have been derived (the real sentence). + assert len(ids) == 1, ( + f"Expected exactly 1 derived belief id, got {len(ids)}: {ids}" + ) + # The persisted belief content must match the real sentence. + belief = store.get_belief(ids[0]) + assert belief is not None, "Derived belief id has no matching belief in store." + assert real_sentence in belief.content, ( + f"Belief content {belief.content!r} does not contain the real sentence." + ) From ab087a31154b94372d127afcd320124e3dfdcca8 Mon Sep 17 00:00:00 2001 From: rrs <276464689+robotrocketscience@users.noreply.github.com> Date: Mon, 11 May 2026 11:59:49 -0700 Subject: [PATCH 3/4] docs(noise_filter): rephrase glyph-category comment to satisfy discretion grep --- src/aelfrice/noise_filter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aelfrice/noise_filter.py b/src/aelfrice/noise_filter.py index dfe4a125c..0e258b354 100644 --- a/src/aelfrice/noise_filter.py +++ b/src/aelfrice/noise_filter.py @@ -147,7 +147,7 @@ # `gh ` is intentional — it distinguishes the command from prose that # starts with a word that merely contains the token (e.g. "ghosts"). # -# 2. Agent tool-call rendering glyph: ⏺ (U+23FA). Emitted at the start +# 2. Tool-call rendering glyph: ⏺ (U+23FA). Emitted at the start # of tool-call narration lines by some transcript surfaces. # # 3. Pseudo-XML worktree/task tags: ` Date: Tue, 12 May 2026 12:02:39 -0700 Subject: [PATCH 4/4] docs(noise_filter): align ack regex docstring with implementation Docstring claimed `\\.*$` (zero-or-more dots) but the implementation uses `\\.?$` (optional single dot). Code is correct; docstring is the drift. One-character doc fix; no behavior change. Caught by CodeRabbit on PR #679. --- src/aelfrice/noise_filter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/aelfrice/noise_filter.py b/src/aelfrice/noise_filter.py index 0e258b354..009e9a42f 100644 --- a/src/aelfrice/noise_filter.py +++ b/src/aelfrice/noise_filter.py @@ -509,7 +509,7 @@ def is_transcript_noise(sentence: str) -> bool: 4. **Single-word progress emit** — matches `^[A-Z][a-z]+ing\\.$` (a lone capitalised gerund and a full stop, nothing else). 5. **Agent ack emit** — matches - `^(Yes|No|Standing by|Ready|Nothing|Polling)( .{0,40})?\\.*$`; + `^(Yes|No|Standing by|Ready|Nothing|Polling)( .{0,40})?\\.?$`; covers bare keywords and short trailing phrases up to 40 chars. All patterns are case-sensitive as written. Empty or whitespace-only