feat: automate llms.txt and developers/docs/llms.txt generation

app/developers/docs/llms.txt/route.ts

@@ -0,0 +1,48 @@
+import { getTranslations } from "next-intl/server"
+
+import docLinks from "@/data/developer-docs-links.yaml"
+
+import { SITE_URL } from "@/lib/constants"
+
+import {
+  type DocLink,
+  renderDocsNode,
+  type Translator,
+} from "@/lib/llms-txt/render"
+
+export const dynamic = "force-static"
+
+const INTRO = `# Ethereum Developer Documentation
+
+> Technical reference for building on Ethereum: protocol concepts, the Ethereum stack, smart contracts, scaling solutions, and developer tooling.
+
+This file indexes the developer documentation under ${SITE_URL}/developers/docs/. For the full ethereum.org index including learner content, guides, and community resources, see ${SITE_URL}/llms.txt.`
+
+const links = docLinks as unknown as DocLink[]
+
+const renderTopGroup = (group: DocLink, t: Translator): string => {
+  const lines = [`## ${t(group.id)}`, ""]
+  for (const item of group.items ?? []) {
+    lines.push(...renderDocsNode(item, 0, t))
+  }
+  return lines.join("\n")
+}
+
+export const GET = async () => {
+  const t = await getTranslations({
+    locale: "en",
+    namespace: "page-developers-docs",
+  })
+
+  const sections = links.map((entry) =>
+    entry.items
+      ? renderTopGroup(entry, t)
+      : renderDocsNode(entry, 0, t).join("\n")
+  )
+
+  const body = [INTRO, ...sections, ""].join("\n\n")
+
+  return new Response(body, {
+    headers: { "content-type": "text/plain; charset=utf-8" },
+  })
+}

app/llms.txt/route.ts

@@ -0,0 +1,44 @@
+import { getTranslations } from "next-intl/server"
+
+import { SITE_URL } from "@/lib/constants"
+
+import { renderLegalSection, renderNavSection } from "@/lib/llms-txt/render"
+import { buildNavigation } from "@/lib/nav/buildNavigation"
+import {
+  buildFooterDipperLinks,
+  buildFooterLinkSections,
+} from "@/lib/nav/footerLinks"
+
+export const dynamic = "force-static"
+
+const INTRO = `# Ethereum.org
+
+> The official Ethereum website providing comprehensive education, resources, and community information about Ethereum — the decentralized world computer that enables smart contracts and decentralized applications.
+
+Ethereum.org is the primary educational hub for Ethereum, offering beginner-friendly explanations alongside advanced technical documentation. The site covers everything from basic concepts like "What is Ethereum?" to detailed developer guides, staking information, and protocol research. For the developer-documentation-only index, see ${SITE_URL}/developers/docs/llms.txt.`
+
+export const GET = async () => {
+  const t = await getTranslations({ locale: "en", namespace: "common" })
+
+  const nav = buildNavigation(t)
+  const footerSections = buildFooterLinkSections(t)
+  const dipperLinks = buildFooterDipperLinks(t)
+
+  const findFooter = (title: string) =>
+    footerSections.find((s) => s.title === title)
+
+  const body = [
+    INTRO,
+    renderNavSection(nav.learn, findFooter(nav.learn.label)),
+    renderNavSection(nav.use, findFooter(nav.use.label)),
+    renderNavSection(nav.build, findFooter(nav.build.label)),
+    renderNavSection(nav.participate, findFooter(nav.participate.label)),
+    renderNavSection(nav.research, findFooter(nav.research.label)),
+    renderLegalSection(dipperLinks),
+    "",
+  ].join("\n\n")
+
+  return new Response(body, {
+    headers: { "content-type": "text/plain; charset=utf-8" },
+  })
+}

docs/solutions/architecture/llms-txt-automation.md

@@ -0,0 +1,73 @@
+---
+title: "Automated llms.txt and developers/docs/llms.txt Generation"
+date: 2026-05-20
+category: architecture
+module: app/llms.txt, app/developers/docs/llms.txt, src/lib/llms-txt
+tags:
+  - llms-txt
+  - seo
+  - automation
+  - nav
+problem_type: "feature, automation, content-pipeline"
+---
+
+# Automated `llms.txt` Generation
+
+Two `force-static` App Router routes replace the hand-maintained `public/llms.txt`. They regenerate on every deploy; no manual maintenance.
+
+- `ethereum.org/llms.txt` — full site index, organized by main-nav top sections.
+- `ethereum.org/developers/docs/llms.txt` — developer-docs-only index, organized by the docs sidebar.
+
+The split mirrors `nextjs.org/llms.txt` + `nextjs.org/docs/llms.txt`: the root file points at the docs file rather than inlining 100+ lines of deeply nested developer docs.
+
+## Strategy
+
+### What appears, and where it lives
+
+| Decision           | Choice                                                                                         |
+| ------------------ | ---------------------------------------------------------------------------------------------- |
+| Which pages appear | Driven by nav files only (main nav, Footer, developer-docs YAML). Never walks `public/content/`. |
+| Section structure  | Mirrors the main-nav top sections 1:1 (Learn / Use / Build / Participate / Research) + Legal & Policies from Footer's secondary links. |
+| Per-item label     | The nav file's label (resolved via the existing i18n JSON).                                    |
+| Per-item URL       | Always the page's pretty URL (`https://ethereum.org/{href}`). Never `/content/*/index.md`.     |
+| Per-item description | First non-empty of: page's frontmatter `description` → nav's description → label only.       |
+| Locale             | English only at root. Per-locale variants are a later, opt-in extension (no code refactor needed). |
+
+### Per-section layout
+
+```
+## {section.label}                ← top sections from main nav
+
+- {top-level leaf items}          ← e.g. Overview, Quizzes, Videos
+
+### {sub-group label}             ← e.g. Ethereum Explained, How Ethereum Works
+- {sub-group items}
+
+### More                          ← Footer items not already in main nav, dedup'd by href
+- ...
+```
+
+### Root file vs docs file
+
+- **Root file (`/llms.txt`)** treats the developer docs as one pointer (the four top-level Documentation entries from main nav) and links out to `/developers/docs/llms.txt` for the full tree.
+- **Docs file (`/developers/docs/llms.txt`)** renders `developer-docs-links.yaml` directly — top groups as `##`, nested items as indented bullets at the depth they sit in the YAML.
+
+This keeps the root file scannable (~170 lines) and lets crawlers that want depth follow the cross-link.
+
+## Sources of truth
+
+| Source                                        | Provides                                                                                   |
+| --------------------------------------------- | ------------------------------------------------------------------------------------------ |
+| `src/lib/nav/buildNavigation.ts`              | Main-nav top sections + sub-groups + items + their nav descriptions.                       |
+| `src/lib/nav/footerLinks.ts`                  | Footer link sections + dipper links (Legal & Policies). Extracted so Footer and llms.txt share one source. |
+| `src/data/developer-docs-links.yaml`          | The docs sidebar tree, including nested items.                                             |
+| `src/intl/en/common.json` + `page-developers-docs.json` | English labels and descriptions for the i18n keys above (via `getTranslations`). |
+| `public/content/{slug}/index.md` frontmatter  | The richer per-page `description` used preferentially over the nav description.            |
+
+To change what appears in `llms.txt`, edit one of the sources above. The output regenerates on the next deploy. The generated `.txt` files are not in the repo — there is nothing to hand-edit.
+
+## Failure handling
+
+- Missing i18n key → falls back to the key string via the site-wide `getMessageFallback`. Same behavior as anywhere else on the site.
+- Frontmatter `description` missing or unreadable → falls back to the nav description, then to label only. Never throws.
+- Nav `href` points at a page with no `index.md` (JSX-only landings, external URLs) → no frontmatter lookup, nav description carries the entry.