feat - tests + evals

README.md

@@ -65,6 +65,15 @@ Get Textin APP_ID and APP_SECRET by following the instructions [here](https://ww
 }
 ```
 
+
+
+## Running evals
+
+The evals package loads an mcp client that then runs the index.ts file, so there is no need to rebuild between tests. You can load environment variables by prefixing the npx command. Full documentation can be found [here](https://www.mcpevals.io/docs).
+
+```bash
+OPENAI_API_KEY=your-key  npx mcp-eval src/evals/evals.ts src/index.ts
+```
 ## License
 
 This MCP server is licensed under the MIT License. This means you are free to use, modify, and distribute the software, subject to the terms and conditions of the MIT License. For more details, please see the LICENSE file in the project repository.

package.json

@@ -1,7 +1,7 @@
 {
   "name": "@intsig/server-textin",
   "version": "0.1.5",
-  "description": "MCP Server for the Textin Robot API. TextIn官方MCP，支持文档OCR识别，文档转Markdown，文档关键信息提取。",
+  "description": "MCP Server for the Textin Robot API. TextIn\u5b98\u65b9MCP\uff0c\u652f\u6301\u6587\u6863OCR\u8bc6\u522b\uff0c\u6587\u6863\u8f6cMarkdown\uff0c\u6587\u6863\u5173\u952e\u4fe1\u606f\u63d0\u53d6\u3002",
   "license": "MIT",
   "author": "Textin(https://www.textin.com)",
   "homepage": "https://www.textin.com/",
@@ -30,10 +30,11 @@
   "dependencies": {
     "@modelcontextprotocol/sdk": "0.6.0",
     "axios": "^1.8.4",
-    "zod-to-json-schema": "^3.24.5"
+    "zod-to-json-schema": "^3.24.5",
+    "mcp-evals": "^1.0.18"
   },
   "devDependencies": {
     "@types/node": "^20.11.24",
     "typescript": "^5.3.3"
   }
-}
+}

src/evals/evals.ts

@@ -0,0 +1,50 @@
+//evals.ts
+
+import { EvalConfig } from 'mcp-evals';
+import { openai } from "@ai-sdk/openai";
+import { grade, EvalFunction } from "mcp-evals";
+
+const recognition_textEval: EvalFunction = {
+    name: "recognition_text Tool Evaluation",
+    description: "Evaluates text recognition from documents",
+    run: async () => {
+        const result = await grade(openai("gpt-4"), "Please extract all readable text from the file located at /files/sample-receipt.pdf.");
+        return JSON.parse(result);
+    }
+};
+
+const general_information_extrationEval: EvalFunction = {
+    name: "general_information_extration Evaluation",
+    description: "Evaluates the tool's ability to automatically and intelligently extract key information from documents",
+    run: async () => {
+        const result = await grade(openai("gpt-4"), "I have a PDF file at '/documents/sample.pdf'. Please extract the key details about the product's specifications, brand, and features, and summarize them clearly.");
+        return JSON.parse(result);
+    }
+};
+
+const doc_to_markdownEval: EvalFunction = {
+    name: "doc_to_markdownEval",
+    description: "Evaluates the doc_to_markdown tool by converting a PDF to Markdown",
+    run: async () => {
+        const result = await grade(openai("gpt-4"), "Please convert the PDF at /path/to/sample.pdf into Markdown.");
+        return JSON.parse(result);
+    }
+};
+
+const read_fileEval: EvalFunction = {
+    name: "read_file Tool Evaluation",
+    description: "Evaluates the read_file tool's ability to read the contents of a file from the file system",
+    run: async () => {
+        const result = await grade(openai("gpt-4"), "Please read the file at path '/tmp/testfile.txt' and return its contents.");
+        return JSON.parse(result);
+    }
+};
+
+const config: EvalConfig = {
+    model: openai("gpt-4"),
+    evals: [recognition_textEval, general_information_extrationEval, doc_to_markdownEval, read_fileEval]
+};
+
+export default config;
+
+export const evals = [recognition_textEval, general_information_extrationEval, doc_to_markdownEval, read_fileEval];