witness gate: requiring the :: payload silently drops de-marked markers that have no payload, and the changelog claims otherwise

changelog.d/tsk-2k55kq-witness-gate-near-miss-hardening.md

@@ -0,0 +1,8 @@
+### Fixed
+- Tightened the witness-gate near-miss detector so ordinary prose that merely
+  mentions WITNESS (without the ``::`` payload) is no longer flagged, while
+  de-marked (zero-width) and malformed markers still are.
+- Replaced the file-level de-marked-marker exemption in the gate with a
+  line-level one, so a genuine de-marked marker appended to
+  ``scripts/check_witness_token.py`` is reported while its three documented
+  docstring examples are not.

changelog.d/tsk-y54vhk-witness-gate-near-miss-fix.md

@@ -0,0 +1,12 @@
+### Fixed
+- Tightened the witness-gate near-miss detector so ordinary prose that merely
+  mentions WITNESS (without the ``::`` payload) is no longer flagged, while
+  de-marked (zero-width) markers that retain a colon after the broken separator
+  and other malformed markers containing a colon still are.
+- Replaced the file-level de-marked-marker exemption in the gate with a
+  line-level one, so a genuine de-marked marker appended to
+  ``scripts/check_witness_token.py`` is reported while its three documented
+  docstring examples are not.
+- Corrected the near-miss detector to catch de-marked markers whose payload
+  is missing, fixing a silent regression introduced when the detector was
+  tightened to require the ``::`` payload.

scripts/check_witness_token.py

@@ -12,7 +12,9 @@
 test file itself survives. This gate cannot be fooled by that -- it resolves the
 TOKEN, not the path. The gate is non-vacuous in both directions: it fails when a
 declared token is removed (file stays importable), and it does NOT fire on a bare
-test-filename mention in ordinary prose.
+test-filename mention in ordinary prose. A comment that quotes the marker
+syntax (e.g. ``path::token``) without forming a live marker will still be
+flagged by the near-miss detector.
 
 Marker contract (the only thing that triggers the check):
 
@@ -63,11 +65,20 @@
 _REPO_ROOT = Path(__file__).resolve().parent.parent
 REPO_ROOT = Path(os.environ.get("WITNESS_GATE_ROOT") or _REPO_ROOT)
 WITNESS_RE = re.compile(r"#\s*WITNESS:\s*(.+)$")
-_NEAR_MISS_RE = re.compile(r"#\s*WITNESS[^:]")
-# Files whose docstrings contain de-marked illustrative examples. Greppable name.
-_DEMARKED_MARKER_EXEMPTION = frozenset({
-    "scripts/check_witness_token.py",
-})
+# A near-miss resembles a WITNESS marker whose separator is broken -- a
+# zero-width character (e.g. U+200B) lodged between WITNESS and the colon,
+# or the colon replaced. Requiring a colon after the broken separator keeps
+# ordinary prose mentioning WITNESS from being mistaken for a malformed
+# marker, while still catching de-marked markers that omit the ``::``
+# payload. Prose that quotes the marker syntax (e.g. ``path::token``) still
+# trips the near-miss detector; that is a known residual.
+_NEAR_MISS_RE = re.compile(r"#\s*WITNESS[^:](?=[^:]*:)")
+# Documented examples of a de-marked marker in this file's own docstring.
+# They are intentionally de-marked and must not be reported; every other line
+# in the same file (e.g. an appended genuine marker) still is. Greppable name.
+_DEMARKED_MARKER_EXEMPTION = {
+    "scripts/check_witness_token.py": frozenset({4, 21, 38}),
+}
 
 
 @dataclass
@@ -120,14 +131,15 @@ def _extract_claims(file_path: Path, repo_root: Path) -> list[WitnessClaim]:
 
 def _check_near_misses(file_path: Path, repo_root: Path) -> list[Violation]:
     rel = _relative_source(file_path, repo_root)
-    if rel in _DEMARKED_MARKER_EXEMPTION:
-        return []
+    exempt_lines = _DEMARKED_MARKER_EXEMPTION.get(rel, frozenset())
     try:
         source = file_path.read_text(encoding="utf-8")
     except (OSError, UnicodeDecodeError):
         return []
     violations: list[Violation] = []
     for lineno, line in enumerate(source.splitlines(), start=1):
+        if lineno in exempt_lines:
+            continue
         if _NEAR_MISS_RE.search(line):
             claim = WitnessClaim(
                 source_file=rel, line_number=lineno, raw=line.strip()

tests/test_witness_gate.py

@@ -332,6 +332,84 @@ def test_near_miss_in_scripts_detected(self, tmp_path):
         rc, out = _run_main(repo)
         assert rc == 1
 
+    def test_near_miss_regex_spares_prose_arms(self, tmp_path):
+        # WEAKNESS 1: a tightened near-miss regex must not flag ordinary prose
+        # that merely mentions WITNESS, while still catching de-marked (ZWSP)
+        # and malformed markers. All four arms live in one test so a loosening
+        # that silences the prose cannot silence the de-marked arms either.
+        # Arm A -- prose in taosmd/ mentioning WITNESS but no ``::`` is clean.
+        repo = _repo(tmp_path / "a")
+        _write(repo, TEST_TEST, GREEN_TEST)
+        _write(
+            repo,
+            "taosmd/p.py",
+            "# WITNESS markers are validated by the gate.\nVALUE = 1\n",
+        )
+        assert check_witnesses(repo) == []
+        rc, _out = _run_main(repo)
+        assert rc == 0
+
+        # Arm B -- de-marked example (ZWSP) in taosmd/ still exits 1.
+        repo = _repo(tmp_path / "b")
+        _write(repo, TEST_TEST, GREEN_TEST)
+        _write(repo, TEST_SRC, f"# WITNESS\u200b: {TEST_TEST}::{TOKEN}\n")
+        violations = check_witnesses(repo)
+        assert len(violations) == 1
+        assert violations[0].reason == "de-marked or malformed marker"
+        rc, _out = _run_main(repo)
+        assert rc == 1
+
+        # Arm C -- de-marked example (ZWSP) in scripts/ still exits 1.
+        repo = _repo(tmp_path / "c")
+        _write(repo, TEST_TEST, GREEN_TEST)
+        _write(repo, SCRIPTS_SRC, f"# WITNESS\u200b: {TEST_TEST}::{TOKEN}\nVALUE = 1\n")
+        violations = check_witnesses(repo)
+        assert len(violations) == 1
+        assert violations[0].claim.source_file == "scripts/gate_demo.py"
+        assert violations[0].reason == "de-marked or malformed marker"
+        rc, _out = _run_main(repo)
+        assert rc == 1
+
+        # Arm D -- de-marked (ZWSP) marker in taosmd/ with no ``::`` payload
+        # must still exit 1 (regression guard).
+        repo = _repo(tmp_path / "d")
+        _write(repo, TEST_TEST, GREEN_TEST)
+        _write(repo, TEST_SRC, f"# WITNESS\u200b: {TEST_TEST}\n")
+        violations = check_witnesses(repo)
+        assert len(violations) == 1
+        assert violations[0].reason == "de-marked or malformed marker"
+        rc, _out = _run_main(repo)
+        assert rc == 1
+
+    def test_gate_does_not_swallow_appended_near_miss(self, tmp_path):
+        # WEAKNESS 2: the gate's own docstring examples are exempted line by
+        # line, so a genuine de-marked marker appended to a copy of the gate is
+        # still reported. The unmodified gate copy must remain clean.
+        repo = _repo(tmp_path)
+        _write(repo, TEST_TEST, GREEN_TEST)
+        gate_copy = _write(
+            repo,
+            "scripts/check_witness_token.py",
+            GATE_SCRIPT.read_text(encoding="utf-8"),
+        )
+        assert check_witnesses(repo) == []
+        rc, _out = _run_main(repo)
+        assert rc == 0
+
+        appended = gate_copy.read_text(encoding="utf-8")
+        appended += f"# WITNESS\u200b: {TEST_TEST}::{TOKEN}\n"
+        gate_copy.write_text(appended)
+        violations = check_witnesses(repo)
+        near = [
+            v for v in violations
+            if v.reason == "de-marked or malformed marker"
+        ]
+        assert len(near) == 1
+        assert near[0].claim.source_file == "scripts/check_witness_token.py"
+        rc, out = _run_main(repo)
+        assert rc == 1
+        assert "de-marked or malformed marker" in out
+
 
 # ----------------------------------------------------------------------
 # CLI subprocess tests: prove the real script exit codes (Layer A path)