From c6ee998fabb695aee63c57f2cb787dbcfad3fdce Mon Sep 17 00:00:00 2001 From: Zhiyuan Hong Date: Mon, 9 Mar 2026 18:01:17 +0800 Subject: [PATCH 1/3] fix(docs-mcp): recursively crawl and register nested llms.txt resources --- packages/mcp-servers/docs-mcp-server/main.ts | 87 ++++++++++++++++---- 1 file changed, 73 insertions(+), 14 deletions(-) diff --git a/packages/mcp-servers/docs-mcp-server/main.ts b/packages/mcp-servers/docs-mcp-server/main.ts index 41597cb3e4..fb4e83575a 100644 --- a/packages/mcp-servers/docs-mcp-server/main.ts +++ b/packages/mcp-servers/docs-mcp-server/main.ts @@ -54,10 +54,11 @@ const pkg = JSON.parse(await readFile(pkgPath, 'utf-8')) as { const MCP_SERVER_NAME = 'lynx-docs'; -function registerResources( +async function crawlAndRegisterResources( baseURL: string, mcpServer: McpServer, fromMarkdownText: string, + visited: Set = new Set(), ) { const tree = fromMarkdown(fromMarkdownText); // verify markdown is valid @@ -94,13 +95,59 @@ function registerResources( } }); - linkUrls.forEach((link, strippedUrl) => { + for (const [strippedUrl, link] of linkUrls) { // Generate a title for the resource by converting the link node back to markdown // NOTE: The title generation is complex because link titles may contain nested formatting, DON'T just use link.title const title = toMarkdown({ ...link, type: 'root' }).trim(); if (!title) { - return; + continue; + } + + if (strippedUrl.endsWith('llms.txt')) { + if (visited.has(strippedUrl)) { + continue; + } + + debug(`Recursively fetching index: ${link.url}`); + try { + const response = await fetch(link.url); + if (!response.ok) { + debug(`Failed to fetch nested index ${link.url}: ${response.status} ${response.statusText}`); + continue; + } + const nestedMarkdown = await response.text(); + visited.add(strippedUrl); + + mcpServer.registerResource( + title, + `lynx-docs://${strippedUrl}`, + { + title, + description: title, + mimeType: 'text/markdown', + }, + () => ({ + contents: [ + { + uri: `lynx-docs://${strippedUrl}`, + text: nestedMarkdown, + mimeType: 'text/markdown', + }, + ], + }), + ); + + await crawlAndRegisterResources( + baseURL, + mcpServer, + nestedMarkdown, + visited, + ); + } catch (e) { + debug(`Failed to fetch nested index ${link.url}: %o`, e); + } + continue; } debug( @@ -115,17 +162,23 @@ function registerResources( description: title, mimeType: 'text/markdown', }, - async () => ({ - contents: [ - { - uri: `lynx-docs://${strippedUrl}`, - text: await fetch(link.url).then((res) => res.text()), - mimeType: 'text/markdown', - }, - ], - }), + async () => { + const response = await fetch(link.url); + if (!response.ok) { + throw new Error(`Failed to fetch resource ${link.url}: ${response.status} ${response.statusText}`); + } + return { + contents: [ + { + uri: `lynx-docs://${strippedUrl}`, + text: await response.text(), + mimeType: 'text/markdown', + }, + ], + }; + }, ); - }); + } } async function main(baseUrl: string) { @@ -174,7 +227,13 @@ For any questions or requirements regarding Lynx: }), ); - registerResources(baseUrl, mcpServer, ROOT_DOC_MARKDOWN); + const visited = new Set(['llms.txt']); + await crawlAndRegisterResources( + baseUrl, + mcpServer, + ROOT_DOC_MARKDOWN, + visited, + ); const transport = new StdioServerTransport(); await mcpServer.connect(transport); From 0d9b2d9f97f3605cba7a43aef767168d0d03fba6 Mon Sep 17 00:00:00 2001 From: Zhiyuan Hong Date: Mon, 9 Mar 2026 18:01:17 +0800 Subject: [PATCH 2/3] docs(changeset): add changeset for recursive crawling fix --- .changeset/fix-recursive-docs-mcp.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/fix-recursive-docs-mcp.md diff --git a/.changeset/fix-recursive-docs-mcp.md b/.changeset/fix-recursive-docs-mcp.md new file mode 100644 index 0000000000..5b40f313a3 --- /dev/null +++ b/.changeset/fix-recursive-docs-mcp.md @@ -0,0 +1,5 @@ +--- +"@lynx-js/docs-mcp-server": patch +--- + +fix(docs-mcp): recursively crawl and register nested llms.txt resources From 7c2e3a96df1e8efc031504ec8af0661dc89065d9 Mon Sep 17 00:00:00 2001 From: Zhiyuan Hong <28915578+hzy@users.noreply.github.com> Date: Mon, 13 Apr 2026 15:57:03 +0800 Subject: [PATCH 3/3] style: format code in docs-mcp-server --- packages/mcp-servers/docs-mcp-server/main.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/mcp-servers/docs-mcp-server/main.ts b/packages/mcp-servers/docs-mcp-server/main.ts index fb4e83575a..d96aafafdd 100644 --- a/packages/mcp-servers/docs-mcp-server/main.ts +++ b/packages/mcp-servers/docs-mcp-server/main.ts @@ -113,7 +113,9 @@ async function crawlAndRegisterResources( try { const response = await fetch(link.url); if (!response.ok) { - debug(`Failed to fetch nested index ${link.url}: ${response.status} ${response.statusText}`); + debug( + `Failed to fetch nested index ${link.url}: ${response.status} ${response.statusText}`, + ); continue; } const nestedMarkdown = await response.text(); @@ -165,7 +167,9 @@ async function crawlAndRegisterResources( async () => { const response = await fetch(link.url); if (!response.ok) { - throw new Error(`Failed to fetch resource ${link.url}: ${response.status} ${response.statusText}`); + throw new Error( + `Failed to fetch resource ${link.url}: ${response.status} ${response.statusText}`, + ); } return { contents: [