Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
b92b7f1
feat(types): add llms-txt type definitions
lukeocodes Feb 11, 2026
31aeabd
feat(llms-txt): add core utility functions
lukeocodes Feb 11, 2026
557bccd
feat(llms-txt): add API route for llms.txt generation
lukeocodes Feb 11, 2026
634148c
test(llms-txt): add unit tests for utility functions
lukeocodes Feb 11, 2026
165a605
feat(llms-txt): replace API route with file-based server routes
lukeocodes Feb 11, 2026
80487a7
refactor(llms-txt): add handler factory and split llms.txt/llms_full.…
lukeocodes Feb 12, 2026
7a57b2c
feat(llms-txt): add middleware for versioned, org, and root routes
lukeocodes Feb 12, 2026
00566e0
feat(llms-txt): support shorthand URL redirects for llms.txt paths
lukeocodes Feb 12, 2026
b5a17c9
chore(llms-txt): add ISR cache rules and vitest server alias
lukeocodes Feb 12, 2026
e83f60f
test(llms-txt): add generateRootLlmsTxt unit tests
lukeocodes Feb 12, 2026
2374448
fix(llms-txt): move all handling to middleware for Vercel compatibility
lukeocodes Feb 12, 2026
eccca21
fix(llms-txt): resolve type errors in middleware and utils
lukeocodes Feb 12, 2026
b455ff1
refactor(llm-docs): rename llms-txt files to llm-docs
lukeocodes Feb 12, 2026
fd27bc7
test(llm-docs): add failing tests for .md routes in generateRootLlmsTxt
lukeocodes Feb 12, 2026
2567ac3
feat(llm-docs): add handlePackageMd for raw README .md routes
lukeocodes Feb 12, 2026
48dc0da
feat(llm-docs): handle .md routes in middleware
lukeocodes Feb 13, 2026
1adcc2f
feat(llm-docs): add .md shorthand redirects
lukeocodes Feb 13, 2026
8d6efc1
fix: remove no-op replace
lukeocodes Feb 13, 2026
456017b
fix(llm-docs): remove versioned .md routes due to Vercel ISR conflict
lukeocodes Feb 13, 2026
448eac0
fix(llm-docs): rename llms_full.txt to llms-full.txt
lukeocodes Mar 5, 2026
16c42ce
feat(llm-docs): add alternate link tags for llms.txt and .md routes
lukeocodes Mar 5, 2026
b4e40e3
Merge branch 'main' into feat/llms-txt
lukeocodes Mar 5, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions app/pages/index.vue
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,10 @@ async function search() {

const { env } = useAppConfig().buildInfo

useHead({
link: [{ key: 'llms-txt', rel: 'alternate', href: '/llms.txt', title: 'LLMs.txt' }],
})

useSeoMeta({
title: () => $t('seo.home.title'),
ogTitle: () => $t('seo.home.title'),
Expand Down
10 changes: 9 additions & 1 deletion app/pages/org/[org].vue
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,15 @@ const activeTab = shallowRef<'members' | 'teams'>('members')
const canonicalUrl = computed(() => `https://npmx.dev/@${orgName.value}`)

useHead({
link: [{ rel: 'canonical', href: canonicalUrl }],
link: [
{ rel: 'canonical', href: canonicalUrl },
{
key: 'llms-txt',
rel: 'alternate',
href: () => `/package/@${orgName.value}/llms.txt`,
title: 'LLMs.txt',
},
],
})

useSeoMeta({
Expand Down
25 changes: 24 additions & 1 deletion app/pages/package/[[org]]/[name].vue
Original file line number Diff line number Diff line change
Expand Up @@ -669,7 +669,30 @@ const compactNumberFormatter = useCompactNumberFormatter()
const bytesFormatter = useBytesFormatter()

useHead({
link: [{ rel: 'canonical', href: canonicalUrl }],
link: computed(() => {
const base = `/package/${packageName.value}`
const versionSegment = requestedVersion.value ? `/v/${requestedVersion.value}` : ''
const links: Array<{ key?: string; rel: string; href: string; title?: string }> = [
{ rel: 'canonical', href: canonicalUrl.value },
{
key: 'llms-txt',
rel: 'alternate',
href: `${base}${versionSegment}/llms.txt`,
title: 'LLMs.txt',
},
{
key: 'llms-full-txt',
rel: 'alternate',
href: `${base}${versionSegment}/llms-full.txt`,
title: 'LLMs-full.txt',
},
]
// .md routes only support latest version (no /v/ variant)
if (!requestedVersion.value) {
links.push({ key: 'pkg-md', rel: 'alternate', href: `${base}.md`, title: 'Markdown version' })
}
return links
}),
})

useSeoMeta({
Expand Down
69 changes: 69 additions & 0 deletions scripts/smoke-test-llm-docs.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
#!/usr/bin/env bash
#
# Smoke test all llm-docs routes (llms.txt, llms-full.txt, .md)
# Usage: ./scripts/smoke-test-llm-docs.sh http://localhost:3333

set -euo pipefail

BASE="${1:?Usage: $0 <base-url>}"
BASE="${BASE%/}" # strip trailing slash

PASS=0
FAIL=0

check() {
local label="$1"
local url="$2"
local expect_status="${3:-200}"

status=$(curl -s -o /dev/null -w "%{http_code}" -L "$url")

if [ "$status" = "$expect_status" ]; then
echo " PASS GET $url $status $label"
PASS=$((PASS + 1))
else
echo " FAIL GET $url $status $label (expected $expect_status)"
FAIL=$((FAIL + 1))
fi
}

echo "=== Root ==="
check "Root llms.txt" "$BASE/llms.txt"

echo ""
echo "=== Unscoped package (latest) ==="
check "llms.txt" "$BASE/package/nuxt/llms.txt"
check "llms-full.txt" "$BASE/package/nuxt/llms-full.txt"
check ".md" "$BASE/package/nuxt.md"

echo ""
echo "=== Unscoped package (versioned) ==="
check "llms.txt" "$BASE/package/nuxt/v/3.16.2/llms.txt"
check "llms-full.txt" "$BASE/package/nuxt/v/3.16.2/llms-full.txt"

echo ""
echo "=== Scoped package (latest) ==="
check "llms.txt" "$BASE/package/@nuxt/kit/llms.txt"
check "llms-full.txt" "$BASE/package/@nuxt/kit/llms-full.txt"
check ".md" "$BASE/package/@nuxt/kit.md"

echo ""
echo "=== Scoped package (versioned) ==="
check "llms.txt" "$BASE/package/@nuxt/kit/v/4.3.1/llms.txt"
check "llms-full.txt" "$BASE/package/@nuxt/kit/v/4.3.1/llms-full.txt"

echo ""
echo "=== Org-level ==="
check "Org llms.txt" "$BASE/package/@nuxt/llms.txt"

echo ""
echo "=== Shorthand redirects (follow → 200) ==="
check "Unscoped .md redirect" "$BASE/nuxt.md"
check "Scoped .md redirect" "$BASE/@nuxt/kit.md"
check "Unscoped llms.txt redirect" "$BASE/nuxt/llms.txt"
check "Scoped llms.txt redirect" "$BASE/@nuxt/kit/llms.txt"

echo ""
echo "=== Results ==="
echo " $PASS passed, $FAIL failed"
exit $FAIL
26 changes: 21 additions & 5 deletions server/middleware/canonical-redirects.global.ts
Original file line number Diff line number Diff line change
Expand Up @@ -50,26 +50,42 @@ export default defineEventHandler(async event => {
return
}

// /llms.txt at root is handled by the llm-docs middleware
if (path === '/llms.txt') {
return
}

// /@org/pkg or /pkg → /package/org/pkg or /package/pkg
let pkgMatch = path.match(/^\/(?:(?<org>@[^/]+)\/)?(?<name>[^/@]+)$/)
// Also handles trailing /llms.txt or /llms-full.txt suffixes
let pkgMatch = path.match(
/^\/(?:(?<org>@[^/]+)\/)?(?<name>[^/@]+?)(?<suffix>\.md|\/(?:llms\.txt|llms-full\.txt))?$/,
)
if (pkgMatch?.groups) {
const args = [pkgMatch.groups.org, pkgMatch.groups.name].filter(Boolean).join('/')
const suffix = pkgMatch.groups.suffix ?? ''
setHeader(event, 'cache-control', cacheControl)
return sendRedirect(event, `/package/${args}` + (query ? '?' + query : ''), 301)
return sendRedirect(event, `/package/${args}${suffix}` + (query ? '?' + query : ''), 301)
}

// /@org/pkg/v/version or /@org/pkg@version → /package/org/pkg/v/version
// /pkg/v/version or /pkg@version → /package/pkg/v/version
// Also handles trailing /llms.txt or /llms-full.txt suffixes
const pkgVersionMatch =
path.match(/^\/(?:(?<org>@[^/]+)\/)?(?<name>[^/@]+)\/v\/(?<version>[^/]+)$/) ||
path.match(/^\/(?:(?<org>@[^/]+)\/)?(?<name>[^/@]+)@(?<version>[^/]+)$/)
path.match(
/^\/(?:(?<org>@[^/]+)\/)?(?<name>[^/@]+)\/v\/(?<version>[^/]+)(?<suffix>\/(?:llms\.txt|llms-full\.txt))?$/,
) ||
path.match(
/^\/(?:(?<org>@[^/]+)\/)?(?<name>[^/@]+)@(?<version>[^/]+)(?<suffix>\/(?:llms\.txt|llms-full\.txt))?$/,
)

if (pkgVersionMatch?.groups) {
const args = [pkgVersionMatch.groups.org, pkgVersionMatch.groups.name].filter(Boolean).join('/')
const versionSuffix = pkgVersionMatch.groups.suffix ?? ''
setHeader(event, 'cache-control', cacheControl)
return sendRedirect(
event,
`/package/${args}/v/${pkgVersionMatch.groups.version}` + (query ? '?' + query : ''),
`/package/${args}/v/${pkgVersionMatch.groups.version}${versionSuffix}` +
(query ? '?' + query : ''),
301,
)
}
Expand Down
135 changes: 135 additions & 0 deletions server/middleware/llm-docs.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,135 @@
import * as v from 'valibot'
import { PackageRouteParamsSchema } from '#shared/schemas/package'
import { handleApiError } from '#server/utils/error-handler'
import {
handleLlmsTxt,
handleOrgLlmsTxt,
generateRootLlmsTxt,
handlePackageMd,
} from '#server/utils/llm-docs'

const CACHE_HEADER = 's-maxage=3600, stale-while-revalidate=86400'

/**
* Middleware to handle ALL llms.txt / llms-full.txt / .md routes.
*
* All llms.txt handling lives here rather than in file-based routes because
* Vercel's ISR route rules with glob patterns (e.g. `/package/ ** /llms.txt`)
* create catch-all serverless functions that interfere with Nitro's file-based
* route resolution — scoped packages and versioned paths fail to match.
*
* Handles:
* - /llms.txt (root discovery page)
* - /package/:name.md (unscoped, latest, raw README)
* - /package/@:org/:name.md (scoped, latest, raw README)
* - /package/@:org/llms.txt (org package listing)
* - /package/:name/llms.txt (unscoped, latest)
* - /package/:name/llms-full.txt (unscoped, latest, full)
* - /package/@:org/:name/llms.txt (scoped, latest)
* - /package/@:org/:name/llms-full.txt (scoped, latest, full)
* - /package/:name/v/:version/llms.txt (unscoped, versioned)
* - /package/:name/v/:version/llms-full.txt (unscoped, versioned, full)
* - /package/@:org/:name/v/:version/llms.txt (scoped, versioned)
* - /package/@:org/:name/v/:version/llms-full.txt (scoped, versioned, full)
*/
export default defineEventHandler(async event => {
const path = event.path.split('?')[0] ?? '/'

// Handle .md routes — raw README markdown (latest version only)
if (path.startsWith('/package/') && path.endsWith('.md') && !path.includes('/v/')) {
const rawPackageName = path.slice('/package/'.length, -'.md'.length)

if (!rawPackageName) return

try {
const { packageName } = v.parse(PackageRouteParamsSchema, {
packageName: rawPackageName,
})

const content = await handlePackageMd(packageName)
setHeader(event, 'Content-Type', 'text/markdown; charset=utf-8')
setHeader(event, 'Cache-Control', CACHE_HEADER)
return content
} catch (error: unknown) {
handleApiError(error, {
statusCode: 502,
message: 'Failed to generate package markdown.',
})
}
}

if (!path.endsWith('/llms.txt') && !path.endsWith('/llms-full.txt')) return

const full = path.endsWith('/llms-full.txt')
const suffix = full ? '/llms-full.txt' : '/llms.txt'

// Root /llms.txt
Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

a thing that I personally dislike is the amount of comments that explain very simple statements.

If a comment is needed, the code should be made more readable.

if (path === '/llms.txt') {
const url = getRequestURL(event)
const baseUrl = `${url.protocol}//${url.host}`
setHeader(event, 'Content-Type', 'text/markdown; charset=utf-8')
setHeader(event, 'Cache-Control', CACHE_HEADER)
return generateRootLlmsTxt(baseUrl)
}

if (!path.startsWith('/package/')) return

// Strip /package/ prefix and /llms[_full].txt suffix
const inner = path.slice('/package/'.length, -suffix.length)

// Org-level: /package/@org/llms.txt (inner = "@org")
if (!full && inner.startsWith('@') && !inner.includes('/')) {
const orgName = inner.slice(1)
try {
const url = getRequestURL(event)
const baseUrl = `${url.protocol}//${url.host}`
const content = await handleOrgLlmsTxt(orgName, baseUrl)
setHeader(event, 'Content-Type', 'text/markdown; charset=utf-8')
setHeader(event, 'Cache-Control', CACHE_HEADER)
return content
} catch (error: unknown) {
handleApiError(error, { statusCode: 502, message: 'Failed to generate org llms.txt.' })
}
}

// Parse package name and optional version from inner path
let rawPackageName: string
let rawVersion: string | undefined

if (inner.includes('/v/')) {
// Versioned path
if (inner.startsWith('@')) {
const match = inner.match(/^(@[^/]+\/[^/]+)\/v\/(.+)$/)
if (!match?.[1] || !match[2]) return
rawPackageName = match[1]
rawVersion = match[2]
} else {
const match = inner.match(/^([^/]+)\/v\/(.+)$/)
if (!match?.[1] || !match[2]) return
rawPackageName = match[1]
rawVersion = match[2]
}
} else {
// Latest version — inner is just the package name
rawPackageName = inner
}

if (!rawPackageName) return

try {
const { packageName, version } = v.parse(PackageRouteParamsSchema, {
packageName: rawPackageName,
version: rawVersion,
})

const content = await handleLlmsTxt(packageName, version, { includeAgentFiles: full })
setHeader(event, 'Content-Type', 'text/markdown; charset=utf-8')
setHeader(event, 'Cache-Control', CACHE_HEADER)
return content
} catch (error: unknown) {
handleApiError(error, {
statusCode: 502,
message: `Failed to generate ${full ? 'llms-full.txt' : 'llms.txt'}.`,
})
}
})
Loading
Loading