diff --git a/mempalace/normalize.py b/mempalace/normalize.py index 4252afa4dc..2b4691eb8a 100644 --- a/mempalace/normalize.py +++ b/mempalace/normalize.py @@ -41,9 +41,12 @@ "system-reminder", "command-message", "command-name", + "command-args", "task-notification", "user-prompt-submit-hook", "hook_output", + "local-command-caveat", + "local-command-stdout", ) @@ -89,6 +92,14 @@ def _tag_pattern(name: str) -> "re.Pattern[str]": # "… +N lines" collapsed-output marker, line-anchored. _COLLAPSED_LINES_RE = re.compile(r"(?m)^(?:> )?…\s*\+\d+ lines.*\n?") +# ANSI escape sequences leak into transcripts via Bash tool_result blocks +# (e.g. /context, /help, any colored CLI output). Each pattern is anchored on +# the literal ESC byte (\x1b) — user prose that mentions e.g. "[1m]" by name +# does not start with ESC and therefore stays intact. +_ANSI_CSI_RE = re.compile(r"\x1b\[[0-9;?]*[ -/]*[@-~]") +# OSC sequences (terminal title, hyperlinks) terminated by BEL or ST. +_ANSI_OSC_RE = re.compile(r"\x1b\][^\x07\x1b]*(?:\x07|\x1b\\)") + def strip_noise(text: str) -> str: """Remove system tags, hook output, and Claude Code UI chrome from text. @@ -105,6 +116,10 @@ def strip_noise(text: str) -> str: # Strip the Claude Code collapsed-output chrome "[N tokens] (ctrl+o to expand)". # Narrow shape — a bare "(ctrl+o to expand)" in user prose stays intact. text = re.sub(r"\s*\[\d+\s+tokens?\]\s*\(ctrl\+o to expand\)", "", text) + # Strip ANSI escape sequences. Tag-wrapped ANSI is already gone via the + # tag patterns above; this sweep handles standalone ANSI in tool output. + text = _ANSI_CSI_RE.sub("", text) + text = _ANSI_OSC_RE.sub("", text) # Collapse runs of blank lines created by the removals text = re.sub(r"\n{4,}", "\n\n\n", text) return text.strip() diff --git a/mempalace/palace.py b/mempalace/palace.py index 07efb6a3e3..cb385857a3 100644 --- a/mempalace/palace.py +++ b/mempalace/palace.py @@ -47,7 +47,11 @@ # # v2 (2026-04): introduced strip_noise() for Claude Code JSONL; previous # drawers stored system tags / hook chrome verbatim. -NORMALIZE_VERSION = 2 +# v3 (2026-05): strip_noise() now also removes the rest of the Claude Code +# slash-command envelope (, , +# ) and ANSI CSI/OSC escape sequences +# captured from Bash tool output. +NORMALIZE_VERSION = 3 def get_collection( diff --git a/tests/test_normalize.py b/tests/test_normalize.py index 2b0f180710..540b332ef1 100644 --- a/tests/test_normalize.py +++ b/tests/test_normalize.py @@ -1316,6 +1316,27 @@ def test_plus_n_lines_marker_inline(self): text = "> User:\n> The log showed … +50 lines of stack trace, useful." assert strip_noise(text) == text.strip() + def test_user_documents_ansi_escape_by_name(self): + # User prose that mentions an ANSI sequence by name (e.g. in docs) + # contains no ESC byte, so the ANSI sweep must not touch it. + text = ( + "> User:\n" + "> The bold SGR code is `[1m` and reset is `[22m`. " + "Documenting the format here." + ) + assert strip_noise(text) == text.strip() + + def test_user_mentions_local_command_inline(self): + # Inline mention of a Claude Code envelope tag inside user prose + # (e.g. when documenting the harness) must not be stripped. + text = ( + "> User:\n" + "> Claude Code wraps slash commands in " + "..." + " — that is the tag name." + ) + assert strip_noise(text) == text.strip() + def test_dangling_open_tag_does_not_span_messages(self): # THE span-eating bug: a stray unclosed in one # message must NOT merge with a closing tag in another message and @@ -1391,15 +1412,92 @@ def test_strips_each_known_noise_tag(self): "system-reminder", "command-message", "command-name", + "command-args", "task-notification", "user-prompt-submit-hook", "hook_output", + "local-command-caveat", + "local-command-stdout", ): text = f"> User:\n<{tag}>junk\n> Real." out = strip_noise(text) assert tag not in out, f"{tag} leaked into output" assert "Real." in out + def test_strips_full_claude_code_slash_command_envelope(self): + # Every slash-command invocation in Claude Code emits this 5-tag + # envelope. Before the fix, only command-name and command-message + # were stripped; the other three survived into stored drawers. + text = ( + "Caveat: do not respond.\n" + "/effort\n" + "effort\n" + "max\n" + "Set effort level to max\n" + "Real follow-up content." + ) + out = strip_noise(text) + for tag in ( + "local-command-caveat", + "command-name", + "command-message", + "command-args", + "local-command-stdout", + ): + assert tag not in out, f"{tag} leaked" + assert "Real follow-up content." in out + + def test_strips_empty_command_args_pair(self): + # Claude Code emits with empty body + # when a slash command takes no arguments. The pattern still matches. + text = "> clear\n" + out = strip_noise(text) + assert "command-args" not in out + assert "command-message" not in out + + def test_strips_ansi_color_codes(self): + # SGR color codes from Bash tool output. Surrounding text preserved. + text = "before \x1b[1mbold\x1b[22m after" + out = strip_noise(text) + assert "\x1b" not in out + assert "before" in out + assert "bold" in out + assert "after" in out + + def test_strips_ansi_truecolor_codes(self): + # 24-bit truecolor sequences from the /context renderer. + text = "\x1b[38;2;153;153;153m├\x1b[39m mempalace_add_drawer" + out = strip_noise(text) + assert "\x1b" not in out + assert "├ mempalace_add_drawer" in out + + def test_strips_ansi_cursor_moves(self): + text = "before\x1b[2K\x1b[Hafter" + out = strip_noise(text) + assert "\x1b" not in out + assert "beforeafter" in out + + def test_strips_ansi_osc_terminal_title(self): + # OSC 0 terminator: BEL. + text = "before\x1b]0;window title\x07after" + out = strip_noise(text) + assert "\x1b" not in out + assert "beforeafter" in out + + def test_strips_ansi_osc_hyperlink(self): + # OSC 8 hyperlinks terminated by ST (ESC \). + text = "see \x1b]8;;https://example.com\x1b\\here\x1b]8;;\x1b\\." + out = strip_noise(text) + assert "\x1b" not in out + assert "see here." in out + + def test_strips_ansi_inside_noise_tag_with_tag(self): + # ANSI inside a tag exits with the tag — the dedicated ANSI sweep + # only has to cover standalone ANSI in tool output, not double-strip. + text = "Set mode to \x1b[1mdefault\x1b[22m" + out = strip_noise(text) + assert out == "" + def test_collapses_excessive_blank_lines(self): text = "line one\n\n\n\n\n\nline two" out = strip_noise(text)