diff --git a/README.md b/README.md index 584df42..0b56586 100644 --- a/README.md +++ b/README.md @@ -102,6 +102,15 @@ This MCP service integrates with EdgeOne Pages Functions to deploy static HTML c For more information, see the [EdgeOne Pages Functions documentation](https://edgeone.ai/document/162227908259442688) and [EdgeOne Pages KV Storage Guide](https://edgeone.ai/document/162227803822321664). + + +## Running evals + +The evals package loads an mcp client that then runs the index.ts file, so there is no need to rebuild between tests. You can load environment variables by prefixing the npx command. Full documentation can be found [here](https://www.mcpevals.io/docs). + +```bash +OPENAI_API_KEY=your-key npx mcp-eval evals.ts index.ts +``` ## License MIT diff --git a/evals.ts b/evals.ts new file mode 100644 index 0000000..882caf7 --- /dev/null +++ b/evals.ts @@ -0,0 +1,32 @@ +//evals.ts + +import { EvalConfig } from 'mcp-evals'; +import { openai } from "@ai-sdk/openai"; +import { grade, EvalFunction } from "mcp-evals"; + +const deploy_htmlEval: EvalFunction = { + name: 'deploy_html Tool Evaluation', + description: 'Tests the deployment of HTML content to EdgeOne Pages, returning the public URL', + run: async () => { + const result = await grade(openai("gpt-4"), "Deploy the following HTML to EdgeOne:

Hello EdgeOne!

Return the public URL."); + return JSON.parse(result); + } +}; + +const deploy_folder_or_zipEval: EvalFunction = { + name: 'deploy_folder_or_zip Tool Evaluation', + description: 'Evaluates the deployment of a folder or zip to EdgeOne Pages and retrieving the public URL', + run: async () => { + const result = await grade(openai("gpt-4"), "Deploy the folder located at './my-website-folder' using the deploy_folder_or_zip tool and share the resulting public URL."); + return JSON.parse(result); + } +}; + +const config: EvalConfig = { + model: openai("gpt-4"), + evals: [deploy_htmlEval, deploy_folder_or_zipEval] +}; + +export default config; + +export const evals = [deploy_htmlEval, deploy_folder_or_zipEval]; \ No newline at end of file diff --git a/package.json b/package.json index 172841e..6815311 100644 --- a/package.json +++ b/package.json @@ -32,10 +32,11 @@ "@modelcontextprotocol/sdk": "^1.7.0", "cos-nodejs-sdk-v5": "^2.14.7", "dotenv": "^16.4.7", - "zod": "^3.24.2" + "zod": "^3.24.2", + "mcp-evals": "^1.0.18" }, "devDependencies": { "@types/node": "^20.11.28", "typescript": "^5.4.2" } -} +} \ No newline at end of file