diff --git a/package-lock.json b/package-lock.json index 50c15c36..58d1fb64 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2516,6 +2516,12 @@ "integrity": "sha512-YslZMgtJUyuMbZ+aKvfF3x1f5liK4mWNxghFRv7jqRR9C3R3fAOGTTKvxXDa2Y1s9zSbcpuO0cAxDYsc9SrXoQ==", "license": "Apache-2.0" }, + "node_modules/@chonkiejs/core": { + "version": "0.0.5", + "resolved": "https://registry.npmjs.org/@chonkiejs/core/-/core-0.0.5.tgz", + "integrity": "sha512-Y6geZY+/9AyDy5i+h+aRY1g873d8fe/Xz6vndSo+FrP1hzDDBmAgJXhaZhZqZFQ7RraiNP8ImTiOddNmiiP/hw==", + "license": "MIT" + }, "node_modules/@cloudflare/agents-a2a-example": { "resolved": "examples/a2a", "link": true @@ -5516,6 +5522,15 @@ "node": ">=8.0.0" } }, + "node_modules/@orama/orama": { + "version": "3.1.16", + "resolved": "https://registry.npmjs.org/@orama/orama/-/orama-3.1.16.tgz", + "integrity": "sha512-scSmQBD8eANlMUOglxHrN1JdSW8tDghsPuS83otqealBiIeMukCQMOf/wc0JJjDXomqwNdEQFLXLGHrU6PGxuA==", + "license": "Apache-2.0", + "engines": { + "node": ">= 20.0.0" + } + }, "node_modules/@oslojs/encoding": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@oslojs/encoding/-/encoding-1.1.0.tgz", @@ -14183,6 +14198,16 @@ "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==", "license": "MIT" }, + "node_modules/effect": { + "version": "3.19.4", + "resolved": "https://registry.npmjs.org/effect/-/effect-3.19.4.tgz", + "integrity": "sha512-gApFffMwpDVgmw/FzCaYWt6zw4m0xAnQf5+cPS0+Sl85AxfeovJeEIsiEQVlk+ZvtBYcoPXxi65GIyruzanQ5g==", + "license": "MIT", + "dependencies": { + "@standard-schema/spec": "^1.0.0", + "fast-check": "^3.23.1" + } + }, "node_modules/electron-to-chromium": { "version": "1.5.207", "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.207.tgz", @@ -15004,6 +15029,28 @@ "node": "> 0.1.90" } }, + "node_modules/fast-check": { + "version": "3.23.2", + "resolved": "https://registry.npmjs.org/fast-check/-/fast-check-3.23.2.tgz", + "integrity": "sha512-h5+1OzzfCC3Ef7VbtKdcv7zsstUQwUDlYpUTvjeUsJAssPgLn7QzbboPtL5ro04Mq0rPOsMzl7q5hIbRs2wD1A==", + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/dubzzz" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fast-check" + } + ], + "license": "MIT", + "dependencies": { + "pure-rand": "^6.1.0" + }, + "engines": { + "node": ">=8.0.0" + } + }, "node_modules/fast-decode-uri-component": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/fast-decode-uri-component/-/fast-decode-uri-component-1.0.1.tgz", @@ -20904,6 +20951,22 @@ "once": "^1.3.1" } }, + "node_modules/pure-rand": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/pure-rand/-/pure-rand-6.1.0.tgz", + "integrity": "sha512-bVWawvoZoBYpp6yIoQtQXHZjmz35RSVHnUOTefl8Vcjr8snTPY1wnpSPMWekcFwbxI6gtmT7rSYPFvz71ldiOA==", + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/dubzzz" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fast-check" + } + ], + "license": "MIT" + }, "node_modules/qrcode": { "version": "1.5.3", "resolved": "https://registry.npmjs.org/qrcode/-/qrcode-1.5.3.tgz", @@ -26117,10 +26180,14 @@ "dependencies": { "@astrojs/cloudflare": "^12.6.10", "@astrojs/react": "^4.4.2", + "@cfworker/json-schema": "^4.1.1", + "@chonkiejs/core": "^0.0.5", "@gsap/react": "^2.1.2", + "@orama/orama": "^3.1.16", "@use-it/interval": "^1.0.0", "astro": "^5.15.4", "clsx": "^2.1.1", + "effect": "^3.19.2", "framer-motion": "^12.23.24", "gsap": "^3.13.0", "react": "^19.2.0", diff --git a/site/agents/README.md b/site/agents/README.md index 2fa5ac20..4dcc18a5 100644 --- a/site/agents/README.md +++ b/site/agents/README.md @@ -40,3 +40,68 @@ npm run deploy - **Framer Motion** - Animation library - **GSAP** - Advanced animations - **Cloudflare Workers** - Deployment platform + +# Agents MCP Server + +[![Add to Cursor](https://img.shields.io/badge/Add%20to-Cursor-blue)](https://cursor.com/en-US/install-mcp?name=cloudflare-agents&config=eyJ1cmwiOiJodHBzOi8vYWdlbnRzLm1jcC5jbG91ZGZsYXJlLmNvbS9tY3AifQ%3D%3D) +[![Add to VS Code](https://img.shields.io/badge/Add%20to-VS%20Code-blue)](vscode:mcp/install?%7B%22name%22%3A%22cloudflare-agents%22%2C%22type%22%3A%22http%22%2C%22url%22%3A%22https%3A%2F%2Fagents.mcp.cloudflare.com%2Fmcp%22%7D) + +This is an MCP server for anyone building with Agents SDK. It exposes just 1 tool. + +```json +{ + "name": "search-agent-docs", + "description": "Token efficient search of the Cloudflare Agents SDK documentation", + "inputSchema": { + "query": { + "type": "string", + "description": "query string to search for eg. 'agent hibernate', 'schedule tasks'" + }, + "k": { + "type": "number", + "optional": true, + "default": 5, + "description": "number of results to return" + } + } +} +``` + +## Usage + +Connect to this MCP server to any MCP Client that supports remote MCP servers. + +```txt +https://agents.mcp.cloudflare.com/mcp +``` + +## How it works + +It pulls the docs from Github, chunks them with a recursive chunker, and indexes them with Orama. The index is cached in KV for 1 day. Search is BM25 with stemming enabled for better results. This allows "hibernation" to match with "hibernate" allowing for more natural language queries. + +### Ratelimiting + +To avoid ratelimiting by GitHub, you can set the `GITHUB_TOKEN` environment variable with `wrangler secret put GITHUB_TOKEN` + +## Development + +To run this server locally, you can use the following command: + +```bash +npm install +npm run dev +``` + +You can test this server with the MCP Inspector. + +```bash +npx @modelcontextprotocol/inspector +``` + +## Deployment + +To deploy this server to Cloudflare Workers, you can use the following command: + +```bash +npm run deploy +``` diff --git a/site/agents/env.d.ts b/site/agents/env.d.ts new file mode 100644 index 00000000..ed4b805e --- /dev/null +++ b/site/agents/env.d.ts @@ -0,0 +1,8 @@ +/* eslint-disable */ +// Generated by Wrangler by running `wrangler types env.d.ts --include-runtime false` (hash: 83be826bfd14e0006039e186e1b11835) +declare namespace Cloudflare { + interface Env { + DOCS_KV: KVNamespace; + } +} +interface Env extends Cloudflare.Env {} diff --git a/site/agents/package.json b/site/agents/package.json index 481ba34e..80d0fd15 100644 --- a/site/agents/package.json +++ b/site/agents/package.json @@ -4,16 +4,22 @@ "private": true, "scripts": { "dev": "astro dev", + "start": "wrangler dev", "build": "astro build", - "deploy": "wrangler deploy" + "types": "wrangler types env.d.ts --include-runtime false", + "deploy": "npm run build && wrangler deploy" }, "dependencies": { "@astrojs/cloudflare": "^12.6.10", "@astrojs/react": "^4.4.2", + "@cfworker/json-schema": "^4.1.1", + "@chonkiejs/core": "^0.0.5", "@gsap/react": "^2.1.2", + "@orama/orama": "^3.1.16", "@use-it/interval": "^1.0.0", "astro": "^5.15.4", "clsx": "^2.1.1", + "effect": "^3.19.2", "framer-motion": "^12.23.24", "gsap": "^3.13.0", "react": "^19.2.0", diff --git a/site/agents/src/.env_example b/site/agents/src/.env_example new file mode 100644 index 00000000..7756650d --- /dev/null +++ b/site/agents/src/.env_example @@ -0,0 +1 @@ +GITHUB_TOKEN=optional-to-avoid-rate-limiting \ No newline at end of file diff --git a/site/agents/src/server/index.ts b/site/agents/src/server/index.ts new file mode 100644 index 00000000..e611d132 --- /dev/null +++ b/site/agents/src/server/index.ts @@ -0,0 +1,79 @@ +import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js"; +import { CfWorkerJsonSchemaValidator } from "@modelcontextprotocol/sdk/validation/cfworker"; +import { z } from "zod"; +import { createMcpHandler } from "agents/mcp"; +import { fetchAndBuildIndex, formatResults } from "./utils"; +import { search } from "@orama/orama"; +import { Effect } from "effect"; + +// TODO: instrument this server for observability +const mcpServer = new McpServer( + { + name: "agents-mcp", + version: "0.0.1" + }, + { + capabilities: {}, + jsonSchemaValidator: new CfWorkerJsonSchemaValidator() + } +); + +const inputSchema = { + query: z + .string() + .describe( + "query string to search for eg. 'agent hibernate', 'schedule tasks'" + ), + k: z.number().optional().default(5).describe("number of results to return") +}; + +mcpServer.registerTool( + "search-agent-docs", + { + description: + "Token efficient search of the Cloudflare Agents SDK documentation", + inputSchema + }, + async ({ query, k }) => { + const searchEffect = Effect.gen(function* () { + console.log({ query, k }); + const term = query.trim(); + + const docsDb = yield* fetchAndBuildIndex; + + const result = search(docsDb, { term, limit: k }); + const searchResult = yield* result instanceof Promise + ? Effect.promise(() => result) + : Effect.succeed(result); + + return { + content: [ + { + type: "text" as const, + text: formatResults(searchResult, term, k) + } + ] + }; + }).pipe( + Effect.catchAll((error) => { + console.error(error); + return Effect.succeed({ + content: [ + { + type: "text" as const, + text: `There was an error with the search tool. Please try again later.` + } + ] + }); + }) + ); + + return await Effect.runPromise(searchEffect); + } +); + +export default { + async fetch(request: Request, env: Env, ctx: ExecutionContext) { + return createMcpHandler(mcpServer)(request, env, ctx); + } +}; diff --git a/site/agents/src/server/utils.ts b/site/agents/src/server/utils.ts new file mode 100644 index 00000000..c6a4a997 --- /dev/null +++ b/site/agents/src/server/utils.ts @@ -0,0 +1,202 @@ +import { create, insert, search } from "@orama/orama"; +import { RecursiveChunker } from "@chonkiejs/core"; +import { env } from "cloudflare:workers"; +import { Effect, Schedule } from "effect"; + +interface Document { + fileName: string; + content: string; + url: string; +} + +interface GitHubTreeItem { + path: string; + type: string; + sha: string; + url: string; +} + +const KV_KEY = "docs-v0"; +const DOCS_REPO_API = + "https://api.github.com/repos/cloudflare/agents/git/trees/main?recursive=1"; +const TTL_SECONDS = 24 * 60 * 60; // 1 day +const CHUNK_SIZE = 2000; +const MIN_CHARS_PER_CHUNK = 500; + +const chunker = await RecursiveChunker.create({ + chunkSize: CHUNK_SIZE, + minCharactersPerChunk: MIN_CHARS_PER_CHUNK +}); + +const fetchWithRetry = (url: string) => + Effect.tryPromise({ + try: async () => { + const headers: Record = { + "User-Agent": "Cloudflare-Agents-MCP/1.0", + Accept: "application/vnd.github+json" + }; + + // @ts-expect-error - GITHUB_TOKEN is not defined in the environment variables + const githubToken = env.GITHUB_TOKEN; + if (githubToken) { + headers["Authorization"] = `Bearer ${githubToken}`; + } + + const response = await fetch(url, { headers }); + + if (!response.ok) { + console.error( + `HTTP ${response.status} for ${url}: ${response.statusText}` + ); + throw new Error(`HTTP ${response.status}: ${response.statusText}`); + } + + return response; + }, + catch: (error) => { + console.error(`Fetch error for ${url}:`, error); + return error as Error; + } + }).pipe( + Effect.retry( + Schedule.exponential("100 millis").pipe( + Schedule.intersect(Schedule.recurs(3)) + ) + ), + Effect.tapError((error) => + Effect.sync(() => + console.error(`Failed after retries for ${url}:`, error) + ) + ) + ); + +const fetchDocsFromGitHub = Effect.gen(function* () { + const treeData = yield* fetchWithRetry(DOCS_REPO_API).pipe( + Effect.flatMap((response) => + Effect.tryPromise({ + try: async () => { + const text = await response.text(); + return JSON.parse(text) as { tree: GitHubTreeItem[] }; + }, + catch: (error) => { + console.error("Failed to parse GitHub tree JSON:", error); + return error as Error; + } + }) + ) + ); + + const docFiles = treeData.tree.filter( + (item: GitHubTreeItem) => + item.path.startsWith("docs/") && item.path.endsWith(".md") + ); + + const docs: Document[] = []; + + for (const file of docFiles) { + const contentUrl = `https://raw.githubusercontent.com/cloudflare/agents/main/${file.path}`; + + const contentResult = yield* fetchWithRetry(contentUrl).pipe( + Effect.flatMap((response) => + Effect.tryPromise({ + try: () => response.text(), + catch: (error) => error as Error + }) + ), + Effect.flatMap((content) => Effect.promise(() => chunker.chunk(content))), + Effect.catchAll((error) => { + console.error(`Failed to fetch/chunk ${file.path}:`, error); + return Effect.succeed([]); + }) + ); + + for (const chunk of contentResult) { + docs.push({ + fileName: file.path, + content: chunk.text, + url: contentUrl + }); + } + } + + return docs; +}); + +const getCachedDocs = Effect.tryPromise({ + try: () => env.DOCS_KV.get(KV_KEY, "json") as Promise, + catch: (error) => error as Error +}); + +const cacheDocs = (docs: Document[]) => + Effect.tryPromise({ + try: () => + env.DOCS_KV.put(KV_KEY, JSON.stringify(docs), { + expirationTtl: TTL_SECONDS + }), + catch: (error) => error as Error + }); + +export const fetchAndBuildIndex = Effect.gen(function* () { + const cached = yield* getCachedDocs; + + let docs: Document[]; + + if (!cached) { + docs = yield* fetchDocsFromGitHub; + yield* cacheDocs(docs); + } else { + docs = cached; + } + + const docsDb = yield* Effect.sync(() => + create({ + schema: { + fileName: "string", + content: "string", + url: "string" + } as const, + components: { + tokenizer: { + stemming: true, + language: "english" + } + } + }) + ); + + for (const doc of docs) { + yield* Effect.sync(() => insert(docsDb, doc)); + } + + return docsDb; +}); + +export const formatResults = ( + results: Awaited>, + query: string, + k: number +): string => { + const hitCount = results.count; + const elapsed = results.elapsed.formatted; + + let output = `**Search Results**\n\n`; + output += `Found ${hitCount} result${hitCount !== 1 ? "s" : ""} for "${query}" (${elapsed})\n\n`; + + if (hitCount === 0) { + output += `No results found. Try using different keywords or modify the spelling.`; + return output; + } + + output += `Showing top ${Math.min(k, hitCount)} result${Math.min(k, hitCount) !== 1 ? "s" : ""}:\n\n`; + output += `---\n\n`; + + for (const hit of results.hits) { + const doc = hit.document as Document; + output += `**${doc.fileName}**\n`; + output += `[Full content](${doc.url})\n\n`; + output += `${doc.content}\n\n`; + output += `---\n\n`; + } + + return output; +}; diff --git a/site/agents/tsconfig.json b/site/agents/tsconfig.json index 49bc0968..489ab7af 100644 --- a/site/agents/tsconfig.json +++ b/site/agents/tsconfig.json @@ -1,8 +1,9 @@ { "extends": "astro/tsconfigs/strict", "compilerOptions": { - "target": "ES2017", + "target": "ES2020", "lib": ["dom", "dom.iterable", "esnext"], + "types": ["node", "@cloudflare/workers-types"], "allowJs": true, "skipLibCheck": true, "strict": true, @@ -19,6 +20,6 @@ "@/*": ["./src/*"] } }, - "include": ["src/**/*.ts", "src/**/*.tsx", "src/**/*.astro"], + "include": ["env.d.ts", "src/**/*.ts", "src/**/*.tsx", "src/**/*.astro"], "exclude": ["node_modules", "dist"] } diff --git a/site/agents/wrangler.jsonc b/site/agents/wrangler.jsonc index 9796c158..f97f097c 100644 --- a/site/agents/wrangler.jsonc +++ b/site/agents/wrangler.jsonc @@ -1,7 +1,15 @@ { + "$schema": "../../node_modules/wrangler/config-schema.json", "name": "agents-site", "compatibility_date": "2025-10-25", + "main": "src/server/index.ts", + "compatibility_flags": ["nodejs_compat"], "assets": { "directory": "dist" - } + }, + "kv_namespaces": [ + { + "binding": "DOCS_KV" + } + ] }