fix(normalize): strip full slash-command envelope + ANSI escapes (#1333)

mempalace/normalize.py

@@ -45,6 +45,17 @@
     "system-reminder",
     "command-message",
     "command-name",
+    # Rest of the slash-command envelope Claude Code injects around every
+    # slash-command invocation (#1333). Before this only the middle three tags
+    # were covered, so the caveat, args, and captured stdout/stderr streams
+    # leaked into drawers verbatim. ``local-command-stderr`` is included even
+    # though the #1333 root-cause sketch listed only five tags — it is emitted
+    # for commands that write to stderr and was found in real transcripts
+    # alongside the other five.
+    "command-args",
+    "local-command-caveat",
+    "local-command-stdout",
+    "local-command-stderr",
     "task-notification",
     "user-prompt-submit-hook",
     "hook_output",
@@ -53,11 +64,15 @@
 
 def _tag_pattern(name: str) -> "re.Pattern[str]":
     # Opening tag must begin a line (optionally after a `> ` blockquote marker,
-    # since _messages_to_transcript prefixes lines with `> `). Body is lazy but
-    # forbidden from crossing a blank line, so a dangling open tag can't span
-    # multiple messages. Closing tag eats optional trailing whitespace + newline.
+    # since _messages_to_transcript prefixes lines with `> `). The body is lazy
+    # and halts at the next opening tag of the same name, so a dangling open tag
+    # can't span past the next same-tag block — while still allowing blank lines
+    # *inside* the tag (multi-paragraph <local-command-stdout> output, tables,
+    # stack traces; #1333). Combined with the line-start anchor this keeps a
+    # stray tag from eating neighbouring messages. Closing tag eats optional
+    # trailing whitespace + newline.
     return re.compile(
-        rf"(?m)^(?:> )?<{name}(?:\s[^>]*)?>" rf"(?:(?!\n\s*\n)[\s\S])*?" rf"</{name}>[ \t]*\n?"
+        rf"(?m)^(?:> )?<{name}(?:\s[^>]*)?>" rf"(?:(?!<{name}\b)[\s\S])*?" rf"</{name}>[ \t]*\n?"
     )
 
 
@@ -93,6 +108,23 @@ def _tag_pattern(name: str) -> "re.Pattern[str]":
 # "… +N lines" collapsed-output marker, line-anchored.
 _COLLAPSED_LINES_RE = re.compile(r"(?m)^(?:> )?…\s*\+\d+ lines.*\n?")
 
+# ANSI escape sequences that Claude Code's Bash tool preserves verbatim from
+# terminal output (#1333). Each escape is several BPE tokens, so they bloat
+# embeddings and pollute search. Unlike the patterns above these are NOT
+# line-anchored — they are anchored on the literal ESC byte (0x1B), a control
+# character that never appears in legitimate prose, so text that merely *names*
+# a sequence like "[1m" or "ESC[0m" survives untouched. Both follow ECMA-48 and
+# are ReDoS-safe by construction (disjoint / negated character classes, no
+# overlapping nested quantifiers).
+#
+# CSI (Control Sequence Introducer): ESC [ , parameter bytes (0x30-0x3F),
+# intermediate bytes (0x20-0x2F), one final byte (0x40-0x7E). Covers all SGR
+# color/style codes (the dominant Bash-output noise) plus cursor/erase.
+_ANSI_CSI_RE = re.compile(r"\x1b\[[\x30-\x3f]*[\x20-\x2f]*[\x40-\x7e]")
+# OSC (Operating System Command): ESC ] , string payload, terminated by BEL
+# (0x07) or ST (ESC \). Covers hyperlinks (ESC]8;;URL BEL) and window titles.
+_ANSI_OSC_RE = re.compile(r"\x1b\][^\x07\x1b]*(?:\x07|\x1b\\)")
+
 
 def strip_noise(text: str) -> str:
     """Remove system tags, hook output, and Claude Code UI chrome from text.
@@ -109,6 +141,11 @@ def strip_noise(text: str) -> str:
     # Strip the Claude Code collapsed-output chrome "[N tokens] (ctrl+o to expand)".
     # Narrow shape — a bare "(ctrl+o to expand)" in user prose stays intact.
     text = re.sub(r"\s*\[\d+\s+tokens?\]\s*\(ctrl\+o to expand\)", "", text)
+    # Strip ANSI escape sequences from terminal / Bash-tool output (#1333).
+    # Applied after tag removal so escapes nested inside a stripped envelope
+    # (e.g. <local-command-stdout>) are already gone with the tag.
+    text = _ANSI_OSC_RE.sub("", text)
+    text = _ANSI_CSI_RE.sub("", text)
     # Collapse runs of blank lines created by the removals
     text = re.sub(r"\n{4,}", "\n\n\n", text)
     return text.strip()

mempalace/palace.py

@@ -67,7 +67,12 @@
 #
 # v2 (2026-04): introduced strip_noise() for Claude Code JSONL; previous
 #               drawers stored system tags / hook chrome verbatim.
-NORMALIZE_VERSION = 2
+# v3 (2026-07): strip_noise() now removes the rest of the slash-command
+#               envelope (local-command-caveat / command-args /
+#               local-command-stdout / local-command-stderr) and ANSI escape
+#               sequences from Bash-tool output (#1333). Existing drawers still
+#               hold those envelopes + ANSI bytes verbatim, so bump to rebuild.
+NORMALIZE_VERSION = 3
 
 
 # (palace_id, collection_name, model_name) tuples already validated this

tests/test_normalize.py

@@ -1826,6 +1826,10 @@ def test_strips_each_known_noise_tag(self):
             "system-reminder",
             "command-message",
             "command-name",
+            "command-args",
+            "local-command-caveat",
+            "local-command-stdout",
+            "local-command-stderr",
             "task-notification",
             "user-prompt-submit-hook",
             "hook_output",
@@ -1844,6 +1848,68 @@ def test_collapses_excessive_blank_lines(self):
         assert "\n\n\n\n" not in out
 
 
+class TestStripNoiseClaudeCodeEnvelopeAndAnsi:
+    """#1333: strip the rest of the slash-command envelope and ANSI escapes from
+    Bash-tool output, while prose that merely *names* them survives (verbatim is
+    sacred)."""
+
+    def test_strips_multiparagraph_local_command_stdout(self):
+        # The old blank-line-halt regex stopped at the first blank line and left
+        # the tail + closing tag behind. Real slash-command stdout is often
+        # multi-paragraph — the whole envelope must go.
+        text = (
+            "> User:\n"
+            "> <local-command-stdout>Set defaultPermissionMode to default\n"
+            "\n"
+            "Disabled auto-compact for all sessions</local-command-stdout>\n"
+            "> Real message."
+        )
+        out = strip_noise(text)
+        assert "local-command-stdout" not in out
+        assert "defaultPermissionMode" not in out
+        assert "Disabled auto-compact" not in out
+        assert "Real message." in out
+
+    def test_strips_local_command_stderr(self):
+        text = "> <local-command-stderr>command failed: boom</local-command-stderr>\n> Real."
+        out = strip_noise(text)
+        assert "local-command-stderr" not in out
+        assert "boom" not in out
+        assert "Real." in out
+
+    def test_strips_empty_command_args_remnant(self):
+        text = "> User:\n> <command-args></command-args>\n> Real."
+        out = strip_noise(text)
+        assert "command-args" not in out
+        assert "Real." in out
+
+    def test_strips_local_command_caveat(self):
+        text = "> <local-command-caveat>caveat text</local-command-caveat>\n> Real."
+        out = strip_noise(text)
+        assert "local-command-caveat" not in out
+        assert "caveat text" not in out
+        assert "Real." in out
+
+    def test_strips_ansi_csi_sgr_codes(self):
+        text = "> Assistant: \x1b[38;2;153;153;153m├\x1b[39m mempalace_add_drawer done"
+        out = strip_noise(text)
+        assert "\x1b" not in out
+        assert "[38;2;153" not in out
+        assert "mempalace_add_drawer done" in out
+
+    def test_strips_ansi_osc_hyperlink(self):
+        text = "> Assistant: see \x1b]8;;https://example.com\x07link\x1b]8;;\x07 here"
+        out = strip_noise(text)
+        assert "\x1b" not in out
+        assert "link here" in out
+
+    def test_prose_naming_ansi_survives(self):
+        # No literal ESC byte — text that merely *documents* an escape sequence
+        # must survive verbatim.
+        text = "> User: the SGR reset code is written ESC[0m in the docs."
+        assert strip_noise(text) == text.strip()
+
+
 # ── _try_pi_jsonl ──────────────────────────────────────────────────────
 #
 # Pi agent stores sessions as JSONL under