fix: close the tag-stripping gaps CodeQL keeps finding

pipelines/lwe/generate.mjs

@@ -86,7 +86,7 @@ function pageHtml(spec) {
   const titleSuffix = c.title;
   const stat = spec.buddyStat || c.navHint;
   const pets = spec.petsLine || "Learning With Errors";
-  const favFill = c.accent.replace("#", "%23");
+  const favFill = c.accent.replaceAll("#", "%23");
   const favFont = c.glyphFont ? c.glyphFont.replace(/"/g, "'") : "Trebuchet MS,sans-serif";
   const askScript = (spec.hasAsk ?? c.hasAsk) ? `<script src="/lwe/ask.js" defer></script>\n` : "";
   const demoJs = spec.demoJs ? `<script>\n${spec.demoJs}\n</script>\n` : "";
@@ -192,7 +192,7 @@ function injectBetween(file, start, end, content) {
   if (i === -1 || j === -1) { console.log(`  · ${file}: markers not found, skipped (${start})`); return false; }
   const next = src.slice(0, i + start.length) + "\n" + content + "\n" + src.slice(j);
   writeFileSync(path, next);
-  console.log(`  · ${file}: rewrote ${start.replace(/<!--|-->|\/\//g, "").trim()} region`);
+  console.log(`  · ${file}: rewrote ${start.replace(/[^\w:-]+/g, " ").replace(/^[\s-]+|[\s-]+$/g, "")} region`);
   return true;
 }
 

scripts/build.mjs

@@ -107,7 +107,7 @@ async function checkInvariants() {
   const routeKeys = [...routesBlock.matchAll(/\[\s*"([^"]+)"/g)].map((m) => m[1]);
   const allowBlock = (wrangler.match(/"run_worker_first"\s*:\s*\[([\s\S]*?)\]/) || [,""])[1];
   const allow = [...allowBlock.matchAll(/"([^"]+)"/g)].map((m) => m[1]);
-  const globRe = (g) => new RegExp("^" + g.replace(/[.]/g, "\\$&").replace(/\*/g, ".*") + "$");
+  const globRe = (g) => new RegExp("^" + g.replace(/[\\.+?^${}()|[\]]/g, "\\$&").replace(/\*/g, ".*") + "$");
   const covered = (p) => allow.includes(p) || allow.some((a) => a.includes("*") && globRe(a).test(p));
   for (const k of routeKeys) if (!covered(k)) hard.push(`ROUTES key ${k} is not in wrangler run_worker_first (route would silently serve static)`);
 
@@ -1309,7 +1309,7 @@ for (const [file, srcPath, marker] of SHELLS) {
 // the build is allowed past the six shells + luna.css (hard rule 3).
 {
   const hash8 = (buf) => createHash("sha256").update(buf).digest("hex").slice(0, 8);
-  const esc = (s) => s.replace(/[/.]/g, "\\$&");
+  const esc = (s) => s.replace(/[\\/.*+?^${}()|[\]]/g, "\\$&");
   await mkdir(`${OUT}/www/a`, { recursive: true });
 
   const ASSETS = [

scripts/contract-tests.mjs

@@ -5016,6 +5016,59 @@
   assert.deepEqual(lensObservationSummary({}).phases, { page: true, discovery: true, botViews: true });
 });
 
+// ── tag stripping: the closers HTML allows and a regex forgets ───────────
+
+// `</script >` is a legal end tag and `--!>` legally closes a comment, so a stripper
+// spelled `</script>` or `-->` hands the body straight through. Neither one is an
+// injection here (lensMarkdown returns JSON that reaches the DOM through esc(), and an
+// excerpt is stored escaped), so what it costs is fidelity: script source lands in a
+// markdown render, and minified JS lands in a webmention excerpt that a human reads.
+test("a spaced end tag and an --!> comment close their elements", async () => {
+  const { lensMarkdown } = await import("../www/_worker.js/lens.js");
+
+  const md = lensMarkdown("<body><p>keep</p><script >var leak=1;</script >tail</body>", "https://x/");
+  assert.ok(!md.includes("leak"), "a spaced </script > must close the script");
+  assert.ok(md.includes("keep") && md.includes("tail"), "the surrounding prose survives");
+
+  assert.ok(!lensMarkdown("<body><!-- gone --!><p>after</p></body>", "https://x/").includes("gone"),
+    "--!> closes a comment");
+
+  // <head> must not swallow a page that opens with <header>, which is what a
+  // missing word boundary buys: the strip runs to the next literal </head>.
+  assert.ok(lensMarkdown("<body><header><p>nav</p></header><p>body</p></body>", "https://x/").includes("body"),
+    "<header> is not <head>");
+
+  const { contentOf } = await import("../www/_worker.js/webmention-send.js");
+  assert.ok(!contentOf("<html><body><script >var s=1;</script >real</body></html>").includes("var s"),
+    "outbound link extraction skips script bodies closed with a space");
+});
+
+// HTML lets an end tag carry attributes, so `</script bar>` closes a script element
+// as surely as `</script>`. A stripper that only allows whitespace before the `>`
+// hands the whole body through.
+test("an end tag carrying attributes still closes its element", async () => {
+  const { lensMarkdown } = await import("../www/_worker.js/lens.js");
+  const { contentOf } = await import("../www/_worker.js/webmention-send.js");
+
+  assert.ok(!lensMarkdown("<body><script>var leak=1;</script bar>tail</body>", "https://x/").includes("leak"),
+    "lensMarkdown closes on </script bar>");
+  assert.ok(!contentOf("<html><body><script>var s=1;</script bar>real</body></html>").includes("var s"),
+    "contentOf closes on </script bar>");
+});
+
+// Removing a span can RE-FORM the thing being removed out of what is left either
+// side: `<!-` + `<!--x-->` + `-` collapses to `<!--` the moment the middle goes.
+// The webmention strippers are immune for free, since they replace with a SPACE
+// that keeps the two sides apart; lensMarkdown replaces with "" to preserve
+// markdown spacing, so it pays for that with a loop instead.
+test("stripping a comment cannot leave a fresh comment opener behind", async () => {
+  const { lensMarkdown } = await import("../www/_worker.js/lens.js");
+  const once = "<!-<!--x-->-".replace(/<!--[\s\S]*?(?:-->|--!>)/g, "");
+  assert.equal(once, "<!--", "one pass really does re-form the opener");
+  assert.ok(!lensMarkdown("<body><p>a<!-<!--x-->-</p><p>b</p></body>", "https://x/").includes("<!--"),
+    "the fixpoint loop clears what one pass re-formed");
+});
+
 // ── lens parse budget: the CPU-bound half ────────────────────────────────
 
 test("the parse is capped, and a prefix parse says so instead of under-reporting", async () => {

www/_worker.js/lens.js

@@ -2673,14 +2673,36 @@ export function lensText(html) {
   return lensDecode(s).replace(/\s+/g, " ").trim();
 }
 
+// Two things a single pass gets wrong, and only the second one is obvious.
+//
+// An end tag may carry attributes, so `</script bar>` closes a script element as
+// surely as `</script>` does, and a stripper spelled `</script\s*>` hands the body
+// through. That one is a plain fidelity bug: script source lands in the markdown.
+//
+// The subtle one is that removing a span can RE-FORM the very thing being removed
+// out of what is left on either side. `<!-` + `<!--x-->` + `-` collapses to `<!--`
+// the moment the middle goes, so one pass leaves a comment opener behind. Note that
+// the webmention strippers are immune to this for free, because they replace with a
+// SPACE rather than "", which keeps the two sides from touching. These replace with
+// "" to preserve markdown spacing, so they pay for it with a loop instead.
+//
+// Bounded, because this runs on third-party pages up to 2MB and an unbounded
+// fixpoint over someone else's bytes is their budget to spend. Real documents reach
+// a fixpoint on pass two.
+function untilStable(s, re, repl = "") {
+  let out = String(s), prev = null, guard = 0;
+  while (out !== prev && guard++ < 8) { prev = out; out = out.replace(re, repl); }
+  return out;
+}
+
 // best-effort, dependency-free HTML→Markdown — roughly what a basic LLM
 // scraper ingests. High-fidelity Readability/Turndown is a deliberate v2.
 export function lensMarkdown(html, baseUrl) {
   let s = html;
   const b = s.match(/<body[^>]*>([\s\S]*)<\/body>/i); if (b) s = b[1];
-  s = s.replace(/<!--[\s\S]*?-->/g, "");
-  s = s.replace(/<(script|style|noscript|template|svg|head|nav|footer|aside)[\s\S]*?<\/\1>/gi, "");
-  s = s.replace(/<pre[^>]*>([\s\S]*?)<\/pre>/gi, (m, i) => "\n\n```\n" + lensDecode(i.replace(/<[^>]+>/g, "")).replace(/\n+$/g, "") + "\n```\n\n");
+  s = untilStable(s, /<!--[\s\S]*?(?:-->|--!>)/g);
+  s = untilStable(s, /<(script|style|noscript|template|svg|head|nav|footer|aside)\b[\s\S]*?<\/\1\b[^>]*>/gi);
+  s = s.replace(/<pre[^>]*>([\s\S]*?)<\/pre>/gi, (m, i) => "\n\n```\n" + lensDecode(untilStable(i, /<[^>]+>/g)).replace(/\n+$/g, "") + "\n```\n\n");
   s = s.replace(/<code[^>]*>([\s\S]*?)<\/code>/gi, (m, i) => "`" + lensStripInline(i).trim() + "`");
   s = s.replace(/<br\s*\/?>/gi, "\n");
   s = s.replace(/<hr\s*\/?>/gi, "\n\n---\n\n");
@@ -2692,13 +2714,13 @@ export function lensMarkdown(html, baseUrl) {
   s = s.replace(/<(strong|b)\b[^>]*>([\s\S]*?)<\/\1>/gi, (m, t, i) => "**" + lensStripInline(i).trim() + "**");
   s = s.replace(/<(em|i)\b[^>]*>([\s\S]*?)<\/\1>/gi, (m, t, i) => "*" + lensStripInline(i).trim() + "*");
   s = s.replace(/<\/(p|div|section|article|header|main|ul|ol|table|tr|h[1-6])>/gi, "\n\n");
-  s = s.replace(/<[^>]+>/g, "");
+  s = untilStable(s, /<[^>]+>/g);
   s = lensDecode(s);
   s = s.replace(/[ \t]+\n/g, "\n").replace(/\n{3,}/g, "\n\n").replace(/[ \t]{2,}/g, " ");
   return s.trim();
 }
 
-export function lensStripInline(h) { return lensDecode(String(h).replace(/<[^>]+>/g, " ")).replace(/\s+/g, " "); }
+export function lensStripInline(h) { return lensDecode(untilStable(h, /<[^>]+>/g, " ")).replace(/\s+/g, " "); }
 
 export function lensTagAttr(tag, name) { const m = String(tag).match(new RegExp(name + "\\s*=\\s*(\"([^\"]*)\"|'([^']*)'|([^\\s>]+))", "i")); return m ? (m[2] ?? m[3] ?? m[4] ?? "") : ""; }
 

www/_worker.js/webmention-send.js

@@ -190,10 +190,10 @@ export function contentOf(html) {
   return String(html)
     .replace(/<!-- axp:shell -->[\s\S]*?<!-- \/axp:shell -->/g, " ")
     .replace(/<!-- axp:desktop -->[\s\S]*?<!-- \/axp:desktop -->/g, " ")
-    .replace(/<head\b[\s\S]*?<\/head>/i, " ")
-    .replace(/<script\b[\s\S]*?<\/script>/gi, " ")
-    .replace(/<style\b[\s\S]*?<\/style>/gi, " ")
-    .replace(/<!--[\s\S]*?-->/g, " ");
+    .replace(/<head\b[\s\S]*?<\/head\b[^>]*>/i, " ")
+    .replace(/<script\b[\s\S]*?<\/script\b[^>]*>/gi, " ")
+    .replace(/<style\b[\s\S]*?<\/style\b[^>]*>/gi, " ")
+    .replace(/<!--[\s\S]*?(?:-->|--!>)/g, " ");
 }
 
 // Every external link in the page's own content, deduped, minus my own profiles

www/_worker.js/webmention.js

@@ -310,15 +310,15 @@ function mentionKind(html, target) {
 // about my page — far more useful than the post's opening line.
 function excerptAround(html, target) {
   const stripped = String(html)
-    .replace(/<script\b[\s\S]*?<\/script>/gi, " ")
-    .replace(/<style\b[\s\S]*?<\/style>/gi, " ")
+    .replace(/<script\b[\s\S]*?<\/script\b[^>]*>/gi, " ")
+    .replace(/<style\b[\s\S]*?<\/style\b[^>]*>/gi, " ")
     // <svg> matters as much as <script> here: an inline icon carries a d="M9.64a1.998
     // 2 0 0 0 2.83 0l1.25-1.25…" path, and a page like a GitHub gist is full of them.
     // Left in, that geometry is what lands in the excerpt and makes a real mention
     // read like spam.
-    .replace(/<svg\b[\s\S]*?<\/svg>/gi, " ")
-    .replace(/<noscript\b[\s\S]*?<\/noscript>/gi, " ")
-    .replace(/<!--[\s\S]*?-->/g, " ");
+    .replace(/<svg\b[\s\S]*?<\/svg\b[^>]*>/gi, " ")
+    .replace(/<noscript\b[\s\S]*?<\/noscript\b[^>]*>/gi, " ")
+    .replace(/<!--[\s\S]*?(?:-->|--!>)/g, " ");
   const e = firstMatch(stripped, /class="[^"]*\be-content\b[^"]*"[^>]*>([\s\S]{0,1200})</i);
   const hay = e || stripped;
   const idx = hay.indexOf(target);