fix(docs): flatten tab JSX before the LLM plugin reads it, not after

.prettierignore

@@ -10,6 +10,11 @@
 **/coverage/
 pnpm-lock.yaml
 
+# The flattened docs mirror the LLM plugin reads (scripts/prepare-llms-source.mjs).
+# Gitignored, but prettier does not read .gitignore, and `format:check` runs after
+# `build` in CI — so without this the mirror is linted as if it were source.
+docs/.llms-src/
+
 # Generated by scripts/gen-component-catalog.mjs
 skills/bestax-custom-component/references/component-catalog.md
 

docs/.gitignore

@@ -18,3 +18,7 @@
 npm-debug.log*
 yarn-debug.log*
 yarn-error.log*
+
+# Generated by scripts/prepare-llms-source.mjs — the flattened mirror the LLM
+# plugin reads. Rebuilt from scratch on every build; never edit or commit it.
+/.llms-src

docs/CLAUDE.md

@@ -24,13 +24,24 @@ The build runs `docusaurus-plugin-llms` (configured in `docusaurus.config.js`),
 - `/llms.txt` (curated index) and `/llms-full.txt` (full concatenation)
 - a per-page `.md` twin for every doc page (so `llms.txt` links resolve)
 
-The plugin reads **source** markdown and has no transform hook, so MDX tab JSX would land
-verbatim in those artifacts. `scripts/flatten-llms-tabs.mjs` runs after `docusaurus build`
-(chained in the `build` script — it can't be a plugin, since core runs every `postBuild` under
-`Promise.all`) and flattens it: `<PackageManagerTabs command="…" />` collapses to the pnpm
-command, `<Tabs>` linearizes to `####` headings keeping every `TabItem` body. It never touches
-code — fenced _or_ inline — because `docs/api/components/tabs.md` documents bestax-bulma's own
-`<Tabs.List>`/`<Tabs.Item>`. Covered by `scripts/flatten-llms-tabs.test.mjs` (`pnpm test`).
+The plugin reads **source** markdown and has no transform hook, so tab JSX has to be flattened
+out of the way _before_ it runs. `scripts/prepare-llms-source.mjs` (chained ahead of
+`docusaurus build`) mirrors `docs/` into a gitignored `.llms-src/` with the tabs already
+flattened: `<PackageManagerTabs command="…" />` collapses to the pnpm command, and `<Tabs>`
+linearizes to `####` headings keeping every `TabItem` body. The plugin is pointed at that
+mirror via `docsDir: [{ path: '.llms-src', routeBasePath: 'docs' }]` — `path` is what it reads,
+`routeBasePath` is what URLs are built from, so published links are unchanged. Flattening never
+touches code — fenced _or_ inline — because `docs/api/components/tabs.md` documents
+bestax-bulma's own `<Tabs.List>`/`<Tabs.Item>`.
+
+**Order is the whole point, so don't "simplify" it back.** Flattening used to run _after_ the
+build, rewriting the artifacts in place. Plugin 0.5.0 broke that by stripping PascalCase JSX
+tags itself — and content carried in **props** goes with the tag, so
+`<PackageManagerTabs command="…" />` lost its entire command and `<TabItem label="…">` lost its
+label, both silently. Nothing caught it: `verifyArtifact` looks for JSX that _survived_, never
+for content that _vanished_. Flattening first leaves the plugin's strip with nothing to remove.
+Covered by `scripts/flatten-llms-tabs.test.mjs` and `scripts/prepare-llms-source.test.mjs`, the
+latter asserting the plugin's own regex is a no-op against the mirror (`pnpm test`).
 
 `build` then chains `scripts/strip-generated-markers.mjs`, which removes the
 `<!-- bestax:generated -->` markers from the built `.md`/`.txt` only — the plugin does not

docs/docusaurus.config.js

@@ -110,7 +110,15 @@ const config = {
         generateMarkdownFiles: true,
         excludeImports: true,
         removeDuplicateHeadings: true,
-        docsDir: 'docs',
+        // Read the flattened mirror, not docs/ itself. Since 0.5.0 the plugin
+        // strips PascalCase JSX tags — and content carried in props goes with the
+        // tag, so `<PackageManagerTabs command="…" />` and `<TabItem label="…">`
+        // silently lost their command and label. scripts/prepare-llms-source.mjs
+        // (chained ahead of `docusaurus build`) writes .llms-src with the tabs
+        // already flattened, so the plugin's strip has nothing left to remove.
+        // `path` is the directory to read; `routeBasePath` is what URLs are built
+        // from, so published links are identical to reading docs/ directly.
+        docsDir: [{ path: '.llms-src', routeBasePath: 'docs' }],
         title: 'Bestax-Bulma',
         description:
           'A Bulma React component library. LLM-friendly documentation for @allxsmith/bestax-bulma.',

docs/package.json

@@ -6,7 +6,7 @@
     "docusaurus": "docusaurus",
     "docs": "docusaurus start",
     "start": "docusaurus start",
-    "build": "docusaurus build && node scripts/flatten-llms-tabs.mjs && node scripts/strip-generated-markers.mjs",
+    "build": "node scripts/prepare-llms-source.mjs && docusaurus build && node scripts/flatten-llms-tabs.mjs && node scripts/strip-generated-markers.mjs",
     "test": "node --test \"scripts/*.test.mjs\"",
     "swizzle": "docusaurus swizzle",
     "deploy": "docusaurus deploy",

docs/scripts/prepare-llms-source.mjs

@@ -0,0 +1,104 @@
+/**
+ * Build a flattened mirror of the docs tree for `docusaurus-plugin-llms` to read.
+ *
+ * Why this exists, and why the flattening moved here from after the build:
+ *
+ * The plugin reads **source** markdown and has no transform hook, so tab JSX used
+ * to land verbatim in the generated artifacts and `flatten-llms-tabs.mjs` rewrote
+ * them afterwards. That stopped working in plugin 0.5.0, which added
+ *
+ *     // Remove MDX/JSX component tags (PascalCase element names such as <Tabs>,
+ *     // <TabItem>, <Admonition>), keeping their inner text content.
+ *     cleaned = cleaned.replace(/<\/?[A-Z][A-Za-z0-9.]*\b[^>]*\/?>/g, '');
+ *
+ * "Keeping their inner text content" holds for wrapper components, but content
+ * carried in **props** goes with the tag. Two things broke, both silently:
+ *
+ * - `<PackageManagerTabs command="add foo" />` is self-closing and its payload is
+ *   the prop, so the whole command vanished from llms.txt.
+ * - `<TabItem label="App root (global)">` kept its body but lost its label, so the
+ *   `####` headings the flattener would have emitted never existed.
+ *
+ * Neither is detectable after the fact: `verifyArtifact` catches *leftover* JSX,
+ * not silently deleted content, so the build stayed green while the machine-read
+ * copy of the docs quietly lost information.
+ *
+ * The fix is ordering, not content. Flatten the source *first*, into a mirror, and
+ * point the plugin at that (`docsDir: [{ path: '.llms-src', routeBasePath: 'docs' }]`
+ * — a supported shape: `path` is the filesystem directory, `routeBasePath` is what
+ * URLs are built from, so the published links are unchanged). By the time the
+ * plugin runs there is no PascalCase JSX left for it to strip, and its cleanup
+ * becomes a no-op instead of a lossy step.
+ *
+ * The mirror is generated, gitignored, and rebuilt from scratch on every run — it
+ * is never a place to edit anything.
+ */
+
+import { mkdir, readdir, readFile, rm, writeFile } from 'node:fs/promises';
+import path from 'node:path';
+import { pathToFileURL } from 'node:url';
+import { transform, verifyArtifact } from './flatten-llms-tabs.mjs';
+
+/** Only these reach the plugin; it ignores everything else in the tree. */
+const SOURCE_EXTENSIONS = new Set(['.md', '.mdx']);
+
+async function* walk(dir) {
+  for (const entry of await readdir(dir, { withFileTypes: true })) {
+    const full = path.join(dir, entry.name);
+    if (entry.isDirectory()) yield* walk(full);
+    else yield full;
+  }
+}
+
+export async function prepare(sourceDir, targetDir) {
+  // Rebuild from scratch so a renamed or deleted page can't linger in the mirror
+  // and keep appearing in llms.txt after it is gone from the site.
+  await rm(targetDir, { recursive: true, force: true });
+
+  let written = 0;
+  const failures = [];
+
+  for await (const file of walk(sourceDir)) {
+    if (!SOURCE_EXTENSIONS.has(path.extname(file))) continue;
+
+    const relative = path.relative(sourceDir, file);
+    const flattened = transform(await readFile(file, 'utf8'));
+
+    // Verified here rather than post-build so a failure names the source file an
+    // author can actually fix, instead of a generated artifact path.
+    for (const problem of verifyArtifact(flattened)) {
+      failures.push(`${relative}: ${problem}`);
+    }
+
+    const destination = path.join(targetDir, relative);
+    await mkdir(path.dirname(destination), { recursive: true });
+    await writeFile(destination, flattened);
+    written += 1;
+  }
+
+  if (failures.length) {
+    throw new Error(
+      `${failures.length} source file(s) failed verification:\n  ${failures.join('\n  ')}`
+    );
+  }
+
+  return written;
+}
+
+const invokedDirectly =
+  process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href;
+
+if (invokedDirectly) {
+  const sourceDir = process.argv[2] ?? 'docs';
+  const targetDir = process.argv[3] ?? '.llms-src';
+  prepare(sourceDir, targetDir)
+    .then(written => {
+      console.log(
+        `prepare-llms-source: wrote ${written} file(s) to ${targetDir}`
+      );
+    })
+    .catch(error => {
+      console.error(`prepare-llms-source: ${error.message}`);
+      process.exitCode = 1;
+    });
+}

docs/scripts/prepare-llms-source.test.mjs

@@ -0,0 +1,180 @@
+/**
+ * Guard the invariant the LLM pipeline actually depends on: by the time
+ * `docusaurus-plugin-llms` reads the mirror, no tab JSX is left for it to strip.
+ *
+ * This exists because the previous design failed silently. The plugin gained, in
+ * 0.5.0:
+ *
+ *     // Remove MDX/JSX component tags (PascalCase element names such as <Tabs>,
+ *     // <TabItem>, <Admonition>), keeping their inner text content.
+ *     cleaned = cleaned.replace(/<\/?[A-Z][A-Za-z0-9.]*\b[^>]*\/?>/g, '');
+ *
+ * Keeping *inner text* is not keeping *props*. `<PackageManagerTabs command="…" />`
+ * is self-closing, so its whole payload went with the tag; `<TabItem label="…">`
+ * kept its body but lost the label. Both dropped content from llms.txt while the
+ * build stayed green, because the post-build check looked for JSX that *survived*
+ * and never for content that had *vanished*.
+ *
+ * So these tests assert the precondition rather than the symptom: prop-borne
+ * content is materialized into ordinary markdown, and nothing tab-shaped remains.
+ * The last test applies the plugin's own regex to prove the strip is now a no-op.
+ *
+ * Source-only, no build required — same fast `pnpm test` as its siblings.
+ */
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import {
+  mkdtemp,
+  mkdir,
+  readFile,
+  readdir,
+  rm,
+  writeFile,
+} from 'node:fs/promises';
+import os from 'node:os';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { prepare } from './prepare-llms-source.mjs';
+import { transform } from './flatten-llms-tabs.mjs';
+
+/** The exact strip `docusaurus-plugin-llms` applies (content.ts, v0.5.x). */
+const PLUGIN_JSX_STRIP = /<\/?[A-Z][A-Za-z0-9.]*\b[^>]*\/?>/g;
+
+const DOCS_DIR = path.resolve(
+  path.dirname(fileURLToPath(import.meta.url)),
+  '../docs'
+);
+
+async function withTempTree(files, run) {
+  const root = await mkdtemp(path.join(os.tmpdir(), 'llms-src-'));
+  try {
+    const source = path.join(root, 'docs');
+    for (const [relative, body] of Object.entries(files)) {
+      const full = path.join(source, relative);
+      await mkdir(path.dirname(full), { recursive: true });
+      await writeFile(full, body);
+    }
+    const target = path.join(root, '.llms-src');
+    const written = await prepare(source, target);
+    await run(target, written);
+  } finally {
+    await rm(root, { recursive: true, force: true });
+  }
+}
+
+async function* walk(dir) {
+  for (const entry of await readdir(dir, { withFileTypes: true })) {
+    const full = path.join(dir, entry.name);
+    if (entry.isDirectory()) yield* walk(full);
+    else if (/\.mdx?$/.test(entry.name)) yield full;
+  }
+}
+
+test('a self-closing PackageManagerTabs becomes a real pnpm fence', async () => {
+  await withTempTree(
+    {
+      'guides/intro.md':
+        '### Install\n\n<PackageManagerTabs command="add @allxsmith/bestax-bulma" />\n',
+    },
+    async target => {
+      const out = await readFile(path.join(target, 'guides/intro.md'), 'utf8');
+      assert.match(out, /```bash\npnpm add @allxsmith\/bestax-bulma\n```/);
+      assert.doesNotMatch(out, /<PackageManagerTabs/);
+    }
+  );
+});
+
+test('a TabItem label survives as a heading, not just its body', async () => {
+  await withTempTree(
+    {
+      'skills/theming.mdx':
+        '<Tabs>\n<TabItem value="root" label="App root (global)">\n\nbody one\n\n</TabItem>\n</Tabs>\n',
+    },
+    async target => {
+      const out = await readFile(
+        path.join(target, 'skills/theming.mdx'),
+        'utf8'
+      );
+      assert.match(out, /#### App root \(global\)/);
+      assert.match(out, /body one/);
+    }
+  );
+});
+
+test('the mirror is rebuilt from scratch, dropping deleted pages', async () => {
+  const root = await mkdtemp(path.join(os.tmpdir(), 'llms-src-stale-'));
+  try {
+    const source = path.join(root, 'docs');
+    const target = path.join(root, '.llms-src');
+    await mkdir(source, { recursive: true });
+    await writeFile(path.join(source, 'keep.md'), '# Keep\n');
+    await mkdir(target, { recursive: true });
+    await writeFile(path.join(target, 'gone.md'), '# Removed page\n');
+
+    await prepare(source, target);
+
+    const remaining = await readdir(target);
+    assert.deepEqual(remaining, ['keep.md']);
+  } finally {
+    await rm(root, { recursive: true, force: true });
+  }
+});
+
+test('prepare refuses a source file the flattener cannot handle', async () => {
+  // A mid-line tag is the shape `transform` deliberately skips, so it would reach
+  // the plugin verbatim. Better to fail naming the source file than to publish it.
+  await assert.rejects(
+    () =>
+      withTempTree(
+        {
+          'guides/bad.md':
+            'Install it with <PackageManagerTabs command="add foo" /> today.\n',
+        },
+        () => {}
+      ),
+    /bad\.md/
+  );
+});
+
+test('the real corpus leaves the plugin nothing to strip', async () => {
+  const root = await mkdtemp(path.join(os.tmpdir(), 'llms-src-corpus-'));
+  try {
+    const target = path.join(root, '.llms-src');
+    const written = await prepare(DOCS_DIR, target);
+    assert.ok(written > 100, `expected >100 pages, mirrored ${written}`);
+
+    const offenders = [];
+    for await (const file of walk(target)) {
+      const body = await readFile(file, 'utf8');
+      // `transform` is the authority on what counts as code; re-running it is a
+      // no-op on an already-flattened file, so any tab tag it would still act on
+      // is one that reached the mirror outside code.
+      if (transform(body) !== body) {
+        offenders.push(path.relative(target, file));
+      }
+    }
+    assert.deepEqual(offenders, [], 'tab JSX reached the mirror unflattened');
+  } finally {
+    await rm(root, { recursive: true, force: true });
+  }
+});
+
+test("the plugin's own JSX strip cannot delete tab content from the mirror", async () => {
+  await withTempTree(
+    {
+      'a.md': '<PackageManagerTabs command="add foo" />\n',
+      'b.mdx':
+        '<Tabs>\n<TabItem value="x" label="Label X">\n\nbody\n\n</TabItem>\n</Tabs>\n',
+    },
+    async target => {
+      for await (const file of walk(target)) {
+        const body = await readFile(file, 'utf8');
+        assert.equal(
+          body.replace(PLUGIN_JSX_STRIP, ''),
+          body,
+          `${path.basename(file)} still contains PascalCase JSX the plugin would strip`
+        );
+      }
+    }
+  );
+});