chore(docs): make the LLM flattener indentation-aware before the #402 rollout

docs/scripts/flatten-llms-corpus.test.mjs

@@ -0,0 +1,77 @@
+/**
+ * Run the flattener over the real docs corpus.
+ *
+ * The unit tests use hand-written fixtures, so they only ever cover shapes
+ * someone thought to write down. This walks every page we actually ship and
+ * asserts the invariants that matter for the published artifacts — which means a
+ * future page that nests a component somewhere awkward fails here rather than
+ * silently corrupting llms-full.txt.
+ *
+ * Source-only: no build required, so it runs in the same fast `pnpm test`.
+ */
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { readFile, readdir } from 'node:fs/promises';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { transform, verifyArtifact } from './flatten-llms-tabs.mjs';
+
+const DOCS_DIR = path.resolve(
+  path.dirname(fileURLToPath(import.meta.url)),
+  '../docs'
+);
+
+async function* walk(dir) {
+  for (const entry of await readdir(dir, { withFileTypes: true })) {
+    const full = path.join(dir, entry.name);
+    if (entry.isDirectory()) yield* walk(full);
+    else if (/\.mdx?$/.test(entry.name)) yield full;
+  }
+}
+
+const pages = [];
+for await (const file of walk(DOCS_DIR)) pages.push(file);
+
+test('the corpus is non-empty (guards a bad DOCS_DIR)', () => {
+  assert.ok(pages.length > 100, `expected >100 pages, found ${pages.length}`);
+});
+
+test('every page survives the flattener cleanly', async () => {
+  const failures = [];
+
+  for (const file of pages) {
+    const src = await readFile(file, 'utf8');
+    const out = transform(src);
+    const rel = path.relative(DOCS_DIR, file);
+
+    for (const problem of verifyArtifact(out)) {
+      failures.push(`${rel}: ${problem}`);
+    }
+
+    // An empty emitted block means a command was swallowed. Anchored to line
+    // starts and to the indentation we emit — a loose regex false-positives on
+    // the inner fence of a ```` block (docs/tutorial-basics/markdown-features).
+    if (/^([ \t]*)```bash\n\1```$/m.test(out)) {
+      failures.push(`${rel}: emitted an empty bash fence`);
+    }
+
+    // Idempotency on real content, not just fixtures.
+    if (transform(out) !== out) {
+      failures.push(`${rel}: transform is not idempotent`);
+    }
+  }
+
+  assert.deepEqual(failures, []);
+});
+
+test('pages with no tab JSX are passed through untouched', async () => {
+  const changed = [];
+
+  for (const file of pages) {
+    const src = await readFile(file, 'utf8');
+    if (/<Tabs|<TabItem|<PackageManagerTabs/.test(src)) continue;
+    if (transform(src) !== src) changed.push(path.relative(DOCS_DIR, file));
+  }
+
+  assert.deepEqual(changed, []);
+});

docs/scripts/flatten-llms-gate.test.mjs

@@ -0,0 +1,103 @@
+/**
+ * The build gate, exercised through the CLI rather than through `verifyArtifact`.
+ *
+ * `verifyArtifact` had unit coverage already and still shipped a hole: `main()`
+ * skipped verification whenever `transform()` was a no-op, which silently exempted
+ * the one shape the gate exists to catch (a mid-line tag the line-anchored pattern
+ * deliberately ignores). Testing the pure function could not see that — the wiring
+ * is what was wrong. So these run the real script against a temp directory.
+ */
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { mkdtemp, writeFile, readFile, mkdir } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import path from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { execFile } from 'node:child_process';
+import { promisify } from 'node:util';
+
+const run = promisify(execFile);
+const SCRIPT = fileURLToPath(
+  new URL('./flatten-llms-tabs.mjs', import.meta.url)
+);
+
+/** Write `files` into a fresh temp build dir and run the script over it. */
+async function runGate(files) {
+  const dir = await mkdtemp(path.join(tmpdir(), 'flatten-gate-'));
+  for (const [name, body] of Object.entries(files)) {
+    const full = path.join(dir, name);
+    await mkdir(path.dirname(full), { recursive: true });
+    await writeFile(full, body);
+  }
+
+  try {
+    const { stdout } = await run(process.execPath, [SCRIPT, dir]);
+    return { ok: true, stdout, dir };
+  } catch (error) {
+    return { ok: false, stderr: error.stderr ?? '', code: error.code, dir };
+  }
+}
+
+test('a clean artifact passes and is rewritten', async () => {
+  const result = await runGate({
+    'llms-full.txt': '# Docs\n\n<PackageManagerTabs command="add foo" />\n',
+  });
+
+  assert.ok(result.ok, `expected success, got: ${result.stderr}`);
+  assert.match(result.stdout, /rewrote 1 file/);
+
+  const out = await readFile(path.join(result.dir, 'llms-full.txt'), 'utf8');
+  assert.match(out, /```bash\npnpm add foo\n```/);
+});
+
+test('the gate fails on a mid-line tag even though the transform is a no-op', async () => {
+  // The regression. transform() leaves this untouched, so the old `after ===
+  // before` early-continue skipped verification entirely and the build passed.
+  const result = await runGate({
+    'llms-full.txt':
+      'Install it with <PackageManagerTabs command="add foo" /> today.\n',
+  });
+
+  assert.equal(result.ok, false, 'expected the build to fail');
+  assert.match(result.stderr, /failed verification/);
+  assert.match(result.stderr, /unflattened tab JSX/);
+  assert.match(result.stderr, /llms-full\.txt/);
+});
+
+test('the gate fails on a dedented fence body', async () => {
+  const result = await runGate({
+    'docs/guide.md': [
+      '1. **Step:**',
+      '',
+      '   ```bash',
+      'pnpm add foo',
+      '```',
+      '',
+      '<PackageManagerTabs command="install" />',
+    ].join('\n'),
+  });
+
+  assert.equal(result.ok, false);
+  assert.match(result.stderr, /dedented below its opening fence/);
+});
+
+test('artifacts with no tab JSX are not even read for problems', async () => {
+  // The NEEDS_FLATTENING prefilter still applies — an unrelated malformed page
+  // must not start failing the docs build.
+  const result = await runGate({
+    'docs/unrelated.md': '# Title\n\n```bash\nnever closed\n',
+  });
+
+  assert.ok(result.ok, `expected success, got: ${result.stderr}`);
+  assert.match(result.stdout, /rewrote 0 file/);
+});
+
+test('non-artifact files are ignored', async () => {
+  const result = await runGate({
+    'robots.txt': '<PackageManagerTabs command="add foo" /> mid line\n',
+    'assets/app.js': 'const x = "<Tabs>";\n',
+  });
+
+  assert.ok(result.ok, `expected success, got: ${result.stderr}`);
+  assert.match(result.stdout, /rewrote 0 file/);
+});