From df2b8b14d5c4a7b48d637f25e0ea7ea01f410546 Mon Sep 17 00:00:00 2001 From: Matthew Lenhard Date: Mon, 5 May 2025 09:53:49 -0400 Subject: [PATCH 1/3] feat - tests + evals --- package.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 1165530..a4abc9c 100644 --- a/package.json +++ b/package.json @@ -52,7 +52,8 @@ "express": "^5.1.0", "pino": "^9.6.0", "yargs": "^17.7.2", - "zod": "^3.24.3" + "zod": "^3.24.3", + "mcp-evals": "^1.0.18" }, "devDependencies": { "@dtsgenerator/replace-namespace": "^1.7.0", @@ -70,4 +71,4 @@ "typescript": "^5.8.3", "typescript-eslint": "^8.30.1" } -} +} \ No newline at end of file From a46caa24db9643abf1bdc2234273c2bedf607b12 Mon Sep 17 00:00:00 2001 From: Matthew Lenhard Date: Mon, 5 May 2025 11:08:17 -0400 Subject: [PATCH 2/3] [feat] add evals --- README.md | 9 + pnpm-lock.yaml | 495 +++++++++++++++++++++++++++++++++++++++++++++ src/evals/evals.ts | 139 +++++++++++++ 3 files changed, 643 insertions(+) create mode 100644 src/evals/evals.ts diff --git a/README.md b/README.md index 8470249..16fa43c 100644 --- a/README.md +++ b/README.md @@ -149,6 +149,15 @@ pnpm run dev ``` Once the server is running, you can utilize the MCP server within Visual Studio Code or other MCP client. +## Running evals + +The evals package loads an mcp client that then runs the index.ts file, so there is no need to rebuild between tests. You can load environment variables by prefixing the npx command. Full documentation can be found [here](https://www.mcpevals.io/docs). + +```bash +OPENAI_API_KEY=your-key npx mcp-eval src/evals/evals.ts src/index.ts +``` + + ## Upgrading ArgoCD Types To update the TypeScript type definitions based on the latest Argo CD API specification: diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 16655a7..de45088 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -17,6 +17,9 @@ importers: express: specifier: ^5.1.0 version: 5.1.0 + mcp-evals: + specifier: ^1.0.18 + version: 1.0.18(react@19.1.0)(zod@3.24.3) pino: specifier: ^9.6.0 version: 9.6.0 @@ -72,6 +75,53 @@ importers: packages: + '@actions/core@1.11.1': + resolution: {integrity: sha512-hXJCSrkwfA46Vd9Z3q4cpEpHB1rL5NG04+/rbqW9d3+CSvtB1tYe8UTpAlixa1vj0m/ULglfEK2UKxMGxCxv5A==} + + '@actions/exec@1.1.1': + resolution: {integrity: sha512-+sCcHHbVdk93a0XT19ECtO/gIXoxvdsgQLzb2fE2/5sIZmWQuluYyjPQtrtTHdU1YzTZ7bAPN4sITq2xi1679w==} + + '@actions/http-client@2.2.3': + resolution: {integrity: sha512-mx8hyJi/hjFvbPokCg4uRd4ZX78t+YyRPtnKWwIl+RzNaVuFpQHfmlGVfsKEJN8LwTCvL+DfVgAM04XaHkm6bA==} + + '@actions/io@1.1.3': + resolution: {integrity: sha512-wi9JjgKLYS7U/z8PPbco+PvTb/nRWjeoFlJ1Qer83k/3C5PHQi28hiVdeE2kHXmIL99mQFawx8qt/JPjZilJ8Q==} + + '@ai-sdk/openai@1.3.21': + resolution: {integrity: sha512-ipAhkRKUd2YaMmn7DAklX3N7Ywx/rCsJHVyb0V/lKRqPcc612qAFVbjg+Uve8QYJlbPxgfsM4s9JmCFp6PSdYw==} + engines: {node: '>=18'} + peerDependencies: + zod: ^3.0.0 + + '@ai-sdk/provider-utils@2.2.7': + resolution: {integrity: sha512-kM0xS3GWg3aMChh9zfeM+80vEZfXzR3JEUBdycZLtbRZ2TRT8xOj3WodGHPb06sUK5yD7pAXC/P7ctsi2fvUGQ==} + engines: {node: '>=18'} + peerDependencies: + zod: ^3.23.8 + + '@ai-sdk/provider@1.1.3': + resolution: {integrity: sha512-qZMxYJ0qqX/RfnuIaab+zp8UAeJn/ygXXAffR5I4N0n1IrvA6qBsjc8hXLmBiMV2zoXlifkacF7sEFnYnjBcqg==} + engines: {node: '>=18'} + + '@ai-sdk/react@1.2.11': + resolution: {integrity: sha512-+kPqLkJ3TWP6czaJPV+vzAKSUcKQ1598BUrcLHt56sH99+LhmIIW3ylZp0OfC3O6TR3eO1Lt0Yzw4R0mK6g9Gw==} + engines: {node: '>=18'} + peerDependencies: + react: ^18 || ^19 || ^19.0.0-rc + zod: ^3.23.8 + peerDependenciesMeta: + zod: + optional: true + + '@ai-sdk/ui-utils@1.2.10': + resolution: {integrity: sha512-GUj+LBoAlRQF1dL/M49jtufGqtLOMApxTpCmVjoRpIPt/dFALVL9RfqfvxwztyIwbK+IxGzcYjSGRsrWrj+86g==} + engines: {node: '>=18'} + peerDependencies: + zod: ^3.23.8 + + '@anthropic-ai/sdk@0.8.1': + resolution: {integrity: sha512-59etePenCizVx1O8Qhi1T1ruE04ISfNzCnyhZNcsss1QljsLmYS83jttarMNEvGYcsUF7rwxw2lzcC3Zbxao7g==} + '@dtsgenerator/replace-namespace@1.7.0': resolution: {integrity: sha512-fOS8AP/Vz4u/+etAZDFPBpsHK3JASlVAId/cf+SH2Hcv5V96wsuvihXuKUomV08dPEUQhv98pgx/o1NiFUBzmA==} peerDependencies: @@ -266,6 +316,10 @@ packages: resolution: {integrity: sha512-ZAoA40rNMPwSm+AeHpCq8STiNAwzWLJuP8Xv4CHIc9wv/PSuExjMrmjfYNj682vW0OOiZ1HKxzvjQr9XZIisQA==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + '@fastify/busboy@2.1.1': + resolution: {integrity: sha512-vBZP4NlzfOlerQTnba4aqZoMhE/a9HY7HRqoOPaETQcSQuWEIyZMHGfVu6w9wGtGK5fED5qRs2DteVCjOH60sA==} + engines: {node: '>=14'} + '@humanfs/core@0.19.1': resolution: {integrity: sha512-5DyQ4+1JEUzejeK1JGICcideyfUbGixgS9jNgex5nqkW+cY7WZhxBigmieN5Qnw9ZosSNVC9KQKyb+GUaGyKUA==} engines: {node: '>=18.18.0'} @@ -312,6 +366,10 @@ packages: resolution: {integrity: sha512-xNYdFdkJqEfIaTVP1gPKoEvluACHZsHZegIoICX8DM1o6Qf3G5u2BQJHmgd0n4YgRPqqK/u1ujQvrgAxxSJT9w==} engines: {node: '>=18'} + '@modelcontextprotocol/sdk@1.11.0': + resolution: {integrity: sha512-k/1pb70eD638anoi0e8wUGAlbMJXyvdV4p62Ko+EZ7eBe1xMx8Uhak1R5DgfoofsK5IBBnRwsYGTaLZl+6/+RQ==} + engines: {node: '>=18'} + '@nodelib/fs.scandir@2.1.5': resolution: {integrity: sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==} engines: {node: '>= 8'} @@ -324,6 +382,10 @@ packages: resolution: {integrity: sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==} engines: {node: '>= 8'} + '@opentelemetry/api@1.9.0': + resolution: {integrity: sha512-3giAOQvZiH5F9bMlMiv8+GSPMeqg0dbaeo58/0SlA9sxSqZhnUtxzX9/2FzyhS9sWQf5S0GJE0AKBrFqjpeYcg==} + engines: {node: '>=8.0.0'} + '@pkgjs/parseargs@0.11.0': resolution: {integrity: sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==} engines: {node: '>=14'} @@ -438,6 +500,9 @@ packages: '@types/connect@3.4.38': resolution: {integrity: sha512-K6uROf1LD88uDQqJCktA4yzL1YYAK6NgfsI0v/mTgyPKWsX1CnJ0XPSDhViejru1GcRkLWb8RlzFYJRqGUbaug==} + '@types/diff-match-patch@1.0.36': + resolution: {integrity: sha512-xFdR6tkm0MWvBfO8xXCSsinYxHcqkQUlcHeSpMC2ukzOb6lwQAfDmW+Qt0AvlGd8HpsS28qKsB+oPeJn9I39jg==} + '@types/estree@1.0.7': resolution: {integrity: sha512-w28IoSUCJpidD/TGviZwwMJckNESJZXFu7NBZ5YJ4mEUnNraUn9Pm8HSZm/jDF1pDWYKspWE7oVphigUPRakIQ==} @@ -456,6 +521,12 @@ packages: '@types/mime@1.3.5': resolution: {integrity: sha512-/pyBZWSLD2n0dcHE3hq8s8ZvcETHtEuF+3E7XVt0Ig2nvsVQXdghHVcEkIWjy9A0wKfTn97a/PSDYohKIlnP/w==} + '@types/node-fetch@2.6.12': + resolution: {integrity: sha512-8nneRWKCg3rMtF69nLQJnOYUcbafYeFSjqkw3jCRLsqkWFlHaoQrr5mXmofFGOx3DKn7UfmBMyov8ySvLRVldA==} + + '@types/node@18.19.87': + resolution: {integrity: sha512-OIAAu6ypnVZHmsHCeJ+7CCSub38QNBS9uceMQeg7K5Ur0Jr+wG9wEOEvvMbhp09pxD5czIUy/jND7s7Tb6Nw7A==} + '@types/node@22.14.1': resolution: {integrity: sha512-u0HuPQwe/dHrItgHHpmw3N2fYCR6x4ivMNbPHRkBVP4CvN+kiRrKHWk3i8tXiO/joPwXLMYvF9TTF0eqgHIuOw==} @@ -524,6 +595,10 @@ packages: resolution: {integrity: sha512-aEhgas7aJ6vZnNFC7K4/vMGDGyOiqWcYZPpIWrTKuTAlsvDNKy2GFDqh9smL+iq069ZvR0YzEeq0B8NJlLzjFA==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} + abort-controller@3.0.0: + resolution: {integrity: sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==} + engines: {node: '>=6.5'} + accepts@2.0.0: resolution: {integrity: sha512-5cvg6CtKwfgdmVqY1WIiXKc3Q1bkRqGLi+2W/6ao+6Y7gu/RCwRuAhGEzh5B4KlszSuTLgZYuqFqo5bImjNKng==} engines: {node: '>= 0.6'} @@ -542,6 +617,20 @@ packages: resolution: {integrity: sha512-jRR5wdylq8CkOe6hei19GGZnxM6rBGwFl3Bg0YItGDimvjGtAvdZk4Pu6Cl4u4Igsws4a1fd1Vq3ezrhn4KmFw==} engines: {node: '>= 14'} + agentkeepalive@4.6.0: + resolution: {integrity: sha512-kja8j7PjmncONqaTsB8fQ+wE2mSU2DJ9D4XKoJ5PFWIdRMa6SLSN1ff4mOr4jCbfRSsxR4keIiySJU0N9T5hIQ==} + engines: {node: '>= 8.0.0'} + + ai@4.3.13: + resolution: {integrity: sha512-cC5HXItuOwGykSMacCPzNp6+NMTxeuTjOenztVgSJhdC9Z4OrzBxwkyeDAf4h1QP938ZFi7IBdq3u4lxVoVmvw==} + engines: {node: '>=18'} + peerDependencies: + react: ^18 || ^19 || ^19.0.0-rc + zod: ^3.23.8 + peerDependenciesMeta: + react: + optional: true + ajv@6.12.6: resolution: {integrity: sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==} @@ -567,6 +656,9 @@ packages: argparse@2.0.1: resolution: {integrity: sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==} + asynckit@0.4.0: + resolution: {integrity: sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==} + atomic-sleep@1.0.0: resolution: {integrity: sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ==} engines: {node: '>=8.0.0'} @@ -574,6 +666,9 @@ packages: balanced-match@1.0.2: resolution: {integrity: sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==} + base-64@0.1.0: + resolution: {integrity: sha512-Y5gU45svrR5tI2Vt/X9GPd3L0HNIKzGu202EjxrXMpuc2V2CiKgemAbUUsqYmZJvPtCXoUKjNZwBJzsNScUbXA==} + body-parser@2.2.0: resolution: {integrity: sha512-02qvAaxv8tp7fBa/mw1ga98OGm+eCbqzJOKoRt70sLmfEEi+jyBYVTDGfCL/k06/4EMk/z01gCe7HoCH/f2LTg==} engines: {node: '>=18'} @@ -618,6 +713,13 @@ packages: resolution: {integrity: sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA==} engines: {node: '>=10'} + chalk@5.4.1: + resolution: {integrity: sha512-zgVZuo2WcZgfUEmsn6eO3kINexW8RAE4maiQ8QNs8CtpPCSyMiYsULR3HQYkm3w8FIA3SberyMJMSldGsW+U3w==} + engines: {node: ^12.17.0 || ^14.13 || >=16.0.0} + + charenc@0.0.2: + resolution: {integrity: sha512-yrLQ/yVUFXkzg7EDQsPieE/53+0RlaWTs+wBrvW36cyilJ2SaDWfl4Yj7MtLTXleV9uEKefbAGUPv2/iWSooRA==} + chokidar@4.0.3: resolution: {integrity: sha512-Qgzu8kfBvo+cA4962jnP1KkS6Dop5NS6g7R5LFYJr4b8Ub94PPQXUksCw9PvXoeXPRRddRNC5C1JQUR2SMGtnA==} engines: {node: '>= 14.16.0'} @@ -633,6 +735,10 @@ packages: color-name@1.1.4: resolution: {integrity: sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==} + combined-stream@1.0.8: + resolution: {integrity: sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==} + engines: {node: '>= 0.8'} + commander@12.1.0: resolution: {integrity: sha512-Vw8qHK3bZM9y/P10u3Vib8o/DdkvA2OtPtZvD871QKjy74Wj1WSKFILMPRPSdUSx5RFK1arlJzEtA4PkFgnbuA==} engines: {node: '>=18'} @@ -675,6 +781,9 @@ packages: resolution: {integrity: sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA==} engines: {node: '>= 8'} + crypt@0.0.2: + resolution: {integrity: sha512-mCxBlsHFYh9C+HVpiEacem8FEBnMXgU9gy4zmNC+SXAZNB/1idgp/aulFJ4FgCi7GPEVbfyng092GqL2k2rmow==} + debug@4.4.0: resolution: {integrity: sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==} engines: {node: '>=6.0'} @@ -687,10 +796,24 @@ packages: deep-is@0.1.4: resolution: {integrity: sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==} + delayed-stream@1.0.0: + resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==} + engines: {node: '>=0.4.0'} + depd@2.0.0: resolution: {integrity: sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==} engines: {node: '>= 0.8'} + dequal@2.0.3: + resolution: {integrity: sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==} + engines: {node: '>=6'} + + diff-match-patch@1.0.5: + resolution: {integrity: sha512-IayShXAgj/QMXgB0IWmKx+rOPuGMhqm5w6jvFxmVenXKIzRqTAAsbBPT3kWQeGANj3jGgvcvv4yK6SxqYmikgw==} + + digest-fetch@1.3.0: + resolution: {integrity: sha512-CGJuv6iKNM7QyZlM2T3sPAdZWd/p9zQiRNS9G+9COUCwzWFTs0Xp8NF5iePx7wtvhDykReiRRrSeNb4oMmB8lA==} + dotenv@16.5.0: resolution: {integrity: sha512-m/C+AwOAr9/W1UOIZUo232ejMNnJAJtYQjUbHoNTBNTJSvqzzDh7vnrei3o3r3m9blf6ZoDkvcw0VmozNRFJxg==} engines: {node: '>=12'} @@ -732,6 +855,10 @@ packages: resolution: {integrity: sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==} engines: {node: '>= 0.4'} + es-set-tostringtag@2.1.0: + resolution: {integrity: sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==} + engines: {node: '>= 0.4'} + esbuild@0.25.2: resolution: {integrity: sha512-16854zccKPnC+toMywC+uKNeYSv+/eXkevRAfwRD/G9Cleq66m8XFIrigkbvauLLlCfDL45Q2cWegSg53gGBnQ==} engines: {node: '>=18'} @@ -814,6 +941,10 @@ packages: resolution: {integrity: sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==} engines: {node: '>= 0.6'} + event-target-shim@5.0.1: + resolution: {integrity: sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==} + engines: {node: '>=6'} + eventsource-parser@3.0.1: resolution: {integrity: sha512-VARTJ9CYeuQYb0pZEPbzi740OWFgpHe7AYJ2WFZVnUDUQp5Dk2yJUgF36YsZ81cOyxT0QxmXD2EQpapAouzWVA==} engines: {node: '>=18.0.0'} @@ -890,6 +1021,17 @@ packages: resolution: {integrity: sha512-gIXjKqtFuWEgzFRJA9WCQeSJLZDjgJUOMCMzxtvFq/37KojM1BFGufqsCy0r4qSQmYLsZYMeyRqzIWOMup03sw==} engines: {node: '>=14'} + form-data-encoder@1.7.2: + resolution: {integrity: sha512-qfqtYan3rxrnCk1VYaA4H+Ms9xdpPqvLZa6xmMgFvhO32x7/3J/ExcTd6qpxM0vH2GdMI+poehyBZvqfMTto8A==} + + form-data@4.0.2: + resolution: {integrity: sha512-hGfm/slu0ZabnNt4oaRZ6uREyfCj6P4fT/n6A1rGV+Z0VdGXjfOhVUpkn6qVQONHGIFwmveGXyDs75+nr6FM8w==} + engines: {node: '>= 6'} + + formdata-node@4.4.1: + resolution: {integrity: sha512-0iirZp3uVDjVGt9p49aTaqjk84TrglENEDuqfdlZQ1roC9CWlPk6Avf8EEnZNcAqPonwkG35x4n3ww/1THYAeQ==} + engines: {node: '>= 12.20'} + forwarded@0.2.0: resolution: {integrity: sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==} engines: {node: '>= 0.6'} @@ -952,6 +1094,10 @@ packages: resolution: {integrity: sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==} engines: {node: '>= 0.4'} + has-tostringtag@1.0.2: + resolution: {integrity: sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==} + engines: {node: '>= 0.4'} + hasown@2.0.2: resolution: {integrity: sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==} engines: {node: '>= 0.4'} @@ -968,6 +1114,9 @@ packages: resolution: {integrity: sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==} engines: {node: '>= 14'} + humanize-ms@1.2.1: + resolution: {integrity: sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==} + iconv-lite@0.6.3: resolution: {integrity: sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==} engines: {node: '>=0.10.0'} @@ -991,6 +1140,9 @@ packages: resolution: {integrity: sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==} engines: {node: '>= 0.10'} + is-buffer@1.1.6: + resolution: {integrity: sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==} + is-extglob@2.1.1: resolution: {integrity: sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==} engines: {node: '>=0.10.0'} @@ -1030,9 +1182,17 @@ packages: json-schema-traverse@0.4.1: resolution: {integrity: sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==} + json-schema@0.4.0: + resolution: {integrity: sha512-es94M3nTIfsEPisRafak+HDLfHXnKBhV3vU5eqPcS3flIWqcxJWgXHXiey3YrpaNsanY5ei1VoYEbOzijuq9BA==} + json-stable-stringify-without-jsonify@1.0.1: resolution: {integrity: sha512-Bdboy+l7tA3OGW6FjyFHWkP5LuByj1Tk33Ljyq0axyzdk9//JSi2u3fP1QSmd1KNwq6VOKYGlAu87CisVir6Pw==} + jsondiffpatch@0.6.0: + resolution: {integrity: sha512-3QItJOXp2AP1uv7waBkao5nCvhEv+QmJAd38Ybq7wNI74Q+BBmnLn4EDKz6yI9xGAIQoUF87qHt+kc1IVxB4zQ==} + engines: {node: ^18.0.0 || >=20.0.0} + hasBin: true + keyv@4.5.4: resolution: {integrity: sha512-oxVHkHR/EJf2CNXnWxRLW6mg7JyCCUcG0DtEGmL2ctUo1PNTin1PUil+r/+4r5MpVgC/fn1kjsx7mjSujKqIpw==} @@ -1068,6 +1228,15 @@ packages: resolution: {integrity: sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==} engines: {node: '>= 0.4'} + mcp-evals@1.0.18: + resolution: {integrity: sha512-khDcEG0XWshdCRirqLXogNoDLmzFA86QyuKoi5ioXsbeRZ3XQra8Zsg7vD+C0K5vwkFIoB1vTuPjHEHMhdLFtQ==} + hasBin: true + peerDependencies: + react: ^19.1.0 + + md5@2.3.0: + resolution: {integrity: sha512-T1GITYmFaKuO91vxyoQMFETst+O71VUPEU3ze5GNzDm0OWdP8v1ziTaAEPUr/3kLsY3Sftgz242A1SetQiDL7g==} + media-typer@1.1.0: resolution: {integrity: sha512-aisnrDP4GNe06UcKFnV5bfMNPBUw4jsLGaWwWfnH3v02GnBuXX2MCVn5RbrWo0j3pczUilYblq7fQ7Nw2t5XKw==} engines: {node: '>= 0.8'} @@ -1084,10 +1253,18 @@ packages: resolution: {integrity: sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==} engines: {node: '>=8.6'} + mime-db@1.52.0: + resolution: {integrity: sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==} + engines: {node: '>= 0.6'} + mime-db@1.54.0: resolution: {integrity: sha512-aU5EJuIN2WDemCcAp2vFBfp/m4EAhWJnUNSSw0ixs7/kXbd6Pg64EmwJkNdFhB8aWt1sH2CTXrLxo/iAGV3oPQ==} engines: {node: '>= 0.6'} + mime-types@2.1.35: + resolution: {integrity: sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==} + engines: {node: '>= 0.6'} + mime-types@3.0.1: resolution: {integrity: sha512-xRc4oEhT6eaBpU1XF7AjpOFD+xQmXNB5OVKwp4tqCuBpHLS/ZbBDrc07mYTDqVMg6PfxUjjNp85O6Cd2Z/5HWA==} engines: {node: '>= 0.6'} @@ -1109,6 +1286,11 @@ packages: mz@2.7.0: resolution: {integrity: sha512-z81GNO7nnYMEhrGh9LeymoE4+Yr0Wn5McHIZMK5cfQCl+NDX08sCZgUc9/6MHni9IWuFLm1Z3HTCXu2z9fN62Q==} + nanoid@3.3.11: + resolution: {integrity: sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w==} + engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} + hasBin: true + natural-compare@1.4.0: resolution: {integrity: sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==} @@ -1116,6 +1298,11 @@ packages: resolution: {integrity: sha512-8Ofs/AUQh8MaEcrlq5xOX0CQ9ypTF5dl78mjlMNfOK08fzpgTHQRQPBxcPlEtIw0yRpws+Zo/3r+5WRby7u3Gg==} engines: {node: '>= 0.6'} + node-domexception@1.0.0: + resolution: {integrity: sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==} + engines: {node: '>=10.5.0'} + deprecated: Use your platform's native DOMException instead + node-fetch@2.7.0: resolution: {integrity: sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==} engines: {node: 4.x || >=6.0.0} @@ -1144,6 +1331,18 @@ packages: once@1.4.0: resolution: {integrity: sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==} + openai@4.97.0: + resolution: {integrity: sha512-LRoiy0zvEf819ZUEJhgfV8PfsE8G5WpQi4AwA1uCV8SKvvtXQkoWUFkepD6plqyJQRghy2+AEPQ07FrJFKHZ9Q==} + hasBin: true + peerDependencies: + ws: ^8.18.0 + zod: ^3.23.8 + peerDependenciesMeta: + ws: + optional: true + zod: + optional: true + optionator@0.9.4: resolution: {integrity: sha512-6IpQ7mKUxRcZNLIObR0hz7lxsapSSIYNZJwXPGeF0mTVqGKFIXj1DQcMoT22S3ROcLyY/rz0PWaWZ9ayWmad9g==} engines: {node: '>= 0.8.0'} @@ -1272,6 +1471,10 @@ packages: resolution: {integrity: sha512-RmkhL8CAyCRPXCE28MMH0z2PNWQBNk2Q09ZdxM9IOOXwxwZbN+qbWaatPkdkWIKL2ZVDImrN/pK5HTRz2PcS4g==} engines: {node: '>= 0.8'} + react@19.1.0: + resolution: {integrity: sha512-FS+XFBNvn3GTAWq26joslQgWNoFu08F4kl0J4CgdNKADkdSGXQyTCnKteIAJy96Br6YbpEU1LSzV5dYtjMkMDg==} + engines: {node: '>=0.10.0'} + readdirp@4.1.2: resolution: {integrity: sha512-GDhwkLfywWL2s6vEjyhri+eXmfH6j1L7JE27WhqLeYzoh/A3DBaYGEj2H/HFZCn/kMfim73FXxEJTw06WtxQwg==} engines: {node: '>= 14.18.0'} @@ -1321,6 +1524,9 @@ packages: safer-buffer@2.1.2: resolution: {integrity: sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==} + secure-json-parse@2.7.0: + resolution: {integrity: sha512-6aU+Rwsezw7VR8/nyvKTx8QpWH9FrcYiXXlqC4z5d5XQBDRqtbfsRjnwGyqbi3gddNtWHuEk9OANUotL26qKUw==} + semver@7.7.1: resolution: {integrity: sha512-hlq8tAfn0m/61p4BVRcPzIGr6LKiMwo4VM6dGi6pt4qcRkmNzTcWq6eCEjEh+qXjkMDvPlOFFSGwQjoEa6gyMA==} engines: {node: '>=10'} @@ -1409,6 +1615,11 @@ packages: resolution: {integrity: sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw==} engines: {node: '>=8'} + swr@2.3.3: + resolution: {integrity: sha512-dshNvs3ExOqtZ6kJBaAsabhPdHyeY4P2cKwRCniDVifBMoG/SVI7tfLWqPXriVspf2Rg4tPzXJTnwaihIeFw2A==} + peerDependencies: + react: ^16.11.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 + synckit@0.11.4: resolution: {integrity: sha512-Q/XQKRaJiLiFIBNN+mndW7S/RHxvwzuZS6ZwmRzUBqJBv/5QIKCEwkBC8GBf8EQJKYnaFs0wOZbKTXBPj8L9oQ==} engines: {node: ^14.18.0 || >=16.0.0} @@ -1423,6 +1634,10 @@ packages: thread-stream@3.1.0: resolution: {integrity: sha512-OqyPZ9u96VohAyMfJykzmivOrY2wfMSf3C5TtFJVgN+Hm6aj+voFhlK+kZEIv2FBh1X6Xp3DlnCOfEQ3B2J86A==} + throttleit@2.1.0: + resolution: {integrity: sha512-nt6AMGKW1p/70DF/hGBdJB57B8Tspmbp5gfJ8ilhLnt7kkr2ye7hzD6NVG8GGErk2HWF34igrL2CXmNIkzKqKw==} + engines: {node: '>=18'} + tinyexec@0.3.2: resolution: {integrity: sha512-KQQR9yN7R5+OSwaK0XQoj22pwHoTlgYqmUscPYoknOoWCWfj/5/ABTMRi69FrKU5ffPVh5QcFikpWJI/P1ocHA==} @@ -1484,6 +1699,10 @@ packages: engines: {node: '>=18.0.0'} hasBin: true + tunnel@0.0.6: + resolution: {integrity: sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg==} + engines: {node: '>=0.6.11 <=0.7.0 || >=0.7.3'} + type-check@0.4.0: resolution: {integrity: sha512-XleUoc9uwGXqjWwXaUTZAmzMcFZ5858QA2vvx1Ur5xIcixXIP+8LnFDgRplU30us6teqdlskFfu+ae4K79Ooew==} engines: {node: '>= 0.8.0'} @@ -1504,9 +1723,16 @@ packages: engines: {node: '>=14.17'} hasBin: true + undici-types@5.26.5: + resolution: {integrity: sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==} + undici-types@6.21.0: resolution: {integrity: sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ==} + undici@5.29.0: + resolution: {integrity: sha512-raqeBD6NQK4SkWhQzeYKd1KmIG6dllBOTt55Rmkt4HtI9mwdWtJljnrXjAFUBLTSN67HWrOIZ3EPF4kjUw80Bg==} + engines: {node: '>=14.0'} + unpipe@1.0.0: resolution: {integrity: sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==} engines: {node: '>= 0.8'} @@ -1514,10 +1740,23 @@ packages: uri-js@4.4.1: resolution: {integrity: sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==} + use-sync-external-store@1.5.0: + resolution: {integrity: sha512-Rb46I4cGGVBmjamjphe8L/UnvJD+uPPtTkNvX5mZgqdbavhI4EbgIWJiIHXJ8bc/i9EQGPRh4DwEURJ552Do0A==} + peerDependencies: + react: ^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 + vary@1.1.2: resolution: {integrity: sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==} engines: {node: '>= 0.8'} + web-streams-polyfill@3.3.3: + resolution: {integrity: sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==} + engines: {node: '>= 8'} + + web-streams-polyfill@4.0.0-beta.3: + resolution: {integrity: sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==} + engines: {node: '>= 14'} + webidl-conversions@3.0.1: resolution: {integrity: sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==} @@ -1576,6 +1815,70 @@ packages: snapshots: + '@actions/core@1.11.1': + dependencies: + '@actions/exec': 1.1.1 + '@actions/http-client': 2.2.3 + + '@actions/exec@1.1.1': + dependencies: + '@actions/io': 1.1.3 + + '@actions/http-client@2.2.3': + dependencies: + tunnel: 0.0.6 + undici: 5.29.0 + + '@actions/io@1.1.3': {} + + '@ai-sdk/openai@1.3.21(zod@3.24.3)': + dependencies: + '@ai-sdk/provider': 1.1.3 + '@ai-sdk/provider-utils': 2.2.7(zod@3.24.3) + zod: 3.24.3 + + '@ai-sdk/provider-utils@2.2.7(zod@3.24.3)': + dependencies: + '@ai-sdk/provider': 1.1.3 + nanoid: 3.3.11 + secure-json-parse: 2.7.0 + zod: 3.24.3 + + '@ai-sdk/provider@1.1.3': + dependencies: + json-schema: 0.4.0 + + '@ai-sdk/react@1.2.11(react@19.1.0)(zod@3.24.3)': + dependencies: + '@ai-sdk/provider-utils': 2.2.7(zod@3.24.3) + '@ai-sdk/ui-utils': 1.2.10(zod@3.24.3) + react: 19.1.0 + swr: 2.3.3(react@19.1.0) + throttleit: 2.1.0 + optionalDependencies: + zod: 3.24.3 + + '@ai-sdk/ui-utils@1.2.10(zod@3.24.3)': + dependencies: + '@ai-sdk/provider': 1.1.3 + '@ai-sdk/provider-utils': 2.2.7(zod@3.24.3) + zod: 3.24.3 + zod-to-json-schema: 3.24.5(zod@3.24.3) + + '@anthropic-ai/sdk@0.8.1': + dependencies: + '@types/node': 18.19.87 + '@types/node-fetch': 2.6.12 + abort-controller: 3.0.0 + agentkeepalive: 4.6.0 + digest-fetch: 1.3.0 + form-data-encoder: 1.7.2 + formdata-node: 4.4.1 + node-fetch: 2.7.0 + web-streams-polyfill: 3.3.3 + transitivePeerDependencies: + - encoding + '@dtsgenerator/replace-namespace@1.7.0(dtsgenerator@3.19.2)(tslib@2.8.1)': dependencies: dtsgenerator: 3.19.2 @@ -1700,6 +2003,8 @@ snapshots: '@eslint/core': 0.13.0 levn: 0.4.1 + '@fastify/busboy@2.1.1': {} + '@humanfs/core@0.19.1': {} '@humanfs/node@0.16.6': @@ -1754,6 +2059,21 @@ snapshots: transitivePeerDependencies: - supports-color + '@modelcontextprotocol/sdk@1.11.0': + dependencies: + content-type: 1.0.5 + cors: 2.8.5 + cross-spawn: 7.0.6 + eventsource: 3.0.6 + express: 5.1.0 + express-rate-limit: 7.5.0(express@5.1.0) + pkce-challenge: 5.0.0 + raw-body: 3.0.0 + zod: 3.24.3 + zod-to-json-schema: 3.24.5(zod@3.24.3) + transitivePeerDependencies: + - supports-color + '@nodelib/fs.scandir@2.1.5': dependencies: '@nodelib/fs.stat': 2.0.5 @@ -1766,6 +2086,8 @@ snapshots: '@nodelib/fs.scandir': 2.1.5 fastq: 1.19.1 + '@opentelemetry/api@1.9.0': {} + '@pkgjs/parseargs@0.11.0': optional: true @@ -1840,6 +2162,8 @@ snapshots: dependencies: '@types/node': 22.14.1 + '@types/diff-match-patch@1.0.36': {} + '@types/estree@1.0.7': {} '@types/express-serve-static-core@5.0.6': @@ -1861,6 +2185,15 @@ snapshots: '@types/mime@1.3.5': {} + '@types/node-fetch@2.6.12': + dependencies: + '@types/node': 22.14.1 + form-data: 4.0.2 + + '@types/node@18.19.87': + dependencies: + undici-types: 5.26.5 + '@types/node@22.14.1': dependencies: undici-types: 6.21.0 @@ -1963,6 +2296,10 @@ snapshots: '@typescript-eslint/types': 8.30.1 eslint-visitor-keys: 4.2.0 + abort-controller@3.0.0: + dependencies: + event-target-shim: 5.0.1 + accepts@2.0.0: dependencies: mime-types: 3.0.1 @@ -1976,6 +2313,22 @@ snapshots: agent-base@7.1.3: {} + agentkeepalive@4.6.0: + dependencies: + humanize-ms: 1.2.1 + + ai@4.3.13(react@19.1.0)(zod@3.24.3): + dependencies: + '@ai-sdk/provider': 1.1.3 + '@ai-sdk/provider-utils': 2.2.7(zod@3.24.3) + '@ai-sdk/react': 1.2.11(react@19.1.0)(zod@3.24.3) + '@ai-sdk/ui-utils': 1.2.10(zod@3.24.3) + '@opentelemetry/api': 1.9.0 + jsondiffpatch: 0.6.0 + zod: 3.24.3 + optionalDependencies: + react: 19.1.0 + ajv@6.12.6: dependencies: fast-deep-equal: 3.1.3 @@ -1997,10 +2350,14 @@ snapshots: argparse@2.0.1: {} + asynckit@0.4.0: {} + atomic-sleep@1.0.0: {} balanced-match@1.0.2: {} + base-64@0.1.0: {} + body-parser@2.2.0: dependencies: bytes: 3.1.2 @@ -2054,6 +2411,10 @@ snapshots: ansi-styles: 4.3.0 supports-color: 7.2.0 + chalk@5.4.1: {} + + charenc@0.0.2: {} + chokidar@4.0.3: dependencies: readdirp: 4.1.2 @@ -2070,6 +2431,10 @@ snapshots: color-name@1.1.4: {} + combined-stream@1.0.8: + dependencies: + delayed-stream: 1.0.0 + commander@12.1.0: {} commander@4.1.1: {} @@ -2105,14 +2470,27 @@ snapshots: shebang-command: 2.0.0 which: 2.0.2 + crypt@0.0.2: {} + debug@4.4.0: dependencies: ms: 2.1.3 deep-is@0.1.4: {} + delayed-stream@1.0.0: {} + depd@2.0.0: {} + dequal@2.0.3: {} + + diff-match-patch@1.0.5: {} + + digest-fetch@1.3.0: + dependencies: + base-64: 0.1.0 + md5: 2.3.0 + dotenv@16.5.0: {} dtsgenerator@3.19.2: @@ -2154,6 +2532,13 @@ snapshots: dependencies: es-errors: 1.3.0 + es-set-tostringtag@2.1.0: + dependencies: + es-errors: 1.3.0 + get-intrinsic: 1.3.0 + has-tostringtag: 1.0.2 + hasown: 2.0.2 + esbuild@0.25.2: optionalDependencies: '@esbuild/aix-ppc64': 0.25.2 @@ -2270,6 +2655,8 @@ snapshots: etag@1.8.1: {} + event-target-shim@5.0.1: {} + eventsource-parser@3.0.1: {} eventsource@3.0.6: @@ -2374,6 +2761,20 @@ snapshots: cross-spawn: 7.0.6 signal-exit: 4.1.0 + form-data-encoder@1.7.2: {} + + form-data@4.0.2: + dependencies: + asynckit: 0.4.0 + combined-stream: 1.0.8 + es-set-tostringtag: 2.1.0 + mime-types: 2.1.35 + + formdata-node@4.4.1: + dependencies: + node-domexception: 1.0.0 + web-streams-polyfill: 4.0.0-beta.3 + forwarded@0.2.0: {} fresh@2.0.0: {} @@ -2434,6 +2835,10 @@ snapshots: has-symbols@1.1.0: {} + has-tostringtag@1.0.2: + dependencies: + has-symbols: 1.1.0 + hasown@2.0.2: dependencies: function-bind: 1.1.2 @@ -2460,6 +2865,10 @@ snapshots: transitivePeerDependencies: - supports-color + humanize-ms@1.2.1: + dependencies: + ms: 2.1.3 + iconv-lite@0.6.3: dependencies: safer-buffer: 2.1.2 @@ -2477,6 +2886,8 @@ snapshots: ipaddr.js@1.9.1: {} + is-buffer@1.1.6: {} + is-extglob@2.1.1: {} is-fullwidth-code-point@3.0.0: {} @@ -2507,8 +2918,16 @@ snapshots: json-schema-traverse@0.4.1: {} + json-schema@0.4.0: {} + json-stable-stringify-without-jsonify@1.0.1: {} + jsondiffpatch@0.6.0: + dependencies: + '@types/diff-match-patch': 1.0.36 + chalk: 5.4.1 + diff-match-patch: 1.0.5 + keyv@4.5.4: dependencies: json-buffer: 3.0.1 @@ -2536,6 +2955,30 @@ snapshots: math-intrinsics@1.1.0: {} + mcp-evals@1.0.18(react@19.1.0)(zod@3.24.3): + dependencies: + '@actions/core': 1.11.1 + '@ai-sdk/openai': 1.3.21(zod@3.24.3) + '@anthropic-ai/sdk': 0.8.1 + '@modelcontextprotocol/sdk': 1.11.0 + ai: 4.3.13(react@19.1.0)(zod@3.24.3) + chalk: 4.1.2 + dotenv: 16.5.0 + openai: 4.97.0(zod@3.24.3) + react: 19.1.0 + tsx: 4.19.3 + transitivePeerDependencies: + - encoding + - supports-color + - ws + - zod + + md5@2.3.0: + dependencies: + charenc: 0.0.2 + crypt: 0.0.2 + is-buffer: 1.1.6 + media-typer@1.1.0: {} merge-descriptors@2.0.0: {} @@ -2547,8 +2990,14 @@ snapshots: braces: 3.0.3 picomatch: 2.3.1 + mime-db@1.52.0: {} + mime-db@1.54.0: {} + mime-types@2.1.35: + dependencies: + mime-db: 1.52.0 + mime-types@3.0.1: dependencies: mime-db: 1.54.0 @@ -2571,10 +3020,14 @@ snapshots: object-assign: 4.1.1 thenify-all: 1.6.0 + nanoid@3.3.11: {} + natural-compare@1.4.0: {} negotiator@1.0.0: {} + node-domexception@1.0.0: {} + node-fetch@2.7.0: dependencies: whatwg-url: 5.0.0 @@ -2593,6 +3046,20 @@ snapshots: dependencies: wrappy: 1.0.2 + openai@4.97.0(zod@3.24.3): + dependencies: + '@types/node': 18.19.87 + '@types/node-fetch': 2.6.12 + abort-controller: 3.0.0 + agentkeepalive: 4.6.0 + form-data-encoder: 1.7.2 + formdata-node: 4.4.1 + node-fetch: 2.7.0 + optionalDependencies: + zod: 3.24.3 + transitivePeerDependencies: + - encoding + optionator@0.9.4: dependencies: deep-is: 0.1.4 @@ -2699,6 +3166,8 @@ snapshots: iconv-lite: 0.6.3 unpipe: 1.0.0 + react@19.1.0: {} + readdirp@4.1.2: {} real-require@0.2.0: {} @@ -2759,6 +3228,8 @@ snapshots: safer-buffer@2.1.2: {} + secure-json-parse@2.7.0: {} + semver@7.7.1: {} send@1.2.0: @@ -2872,6 +3343,12 @@ snapshots: dependencies: has-flag: 4.0.0 + swr@2.3.3(react@19.1.0): + dependencies: + dequal: 2.0.3 + react: 19.1.0 + use-sync-external-store: 1.5.0(react@19.1.0) + synckit@0.11.4: dependencies: '@pkgr/core': 0.2.4 @@ -2889,6 +3366,8 @@ snapshots: dependencies: real-require: 0.2.0 + throttleit@2.1.0: {} + tinyexec@0.3.2: {} tinyglobby@0.2.12: @@ -2951,6 +3430,8 @@ snapshots: optionalDependencies: fsevents: 2.3.3 + tunnel@0.0.6: {} + type-check@0.4.0: dependencies: prelude-ls: 1.2.1 @@ -2973,16 +3454,30 @@ snapshots: typescript@5.8.3: {} + undici-types@5.26.5: {} + undici-types@6.21.0: {} + undici@5.29.0: + dependencies: + '@fastify/busboy': 2.1.1 + unpipe@1.0.0: {} uri-js@4.4.1: dependencies: punycode: 2.3.1 + use-sync-external-store@1.5.0(react@19.1.0): + dependencies: + react: 19.1.0 + vary@1.1.2: {} + web-streams-polyfill@3.3.3: {} + + web-streams-polyfill@4.0.0-beta.3: {} + webidl-conversions@3.0.1: {} webidl-conversions@4.0.2: {} diff --git a/src/evals/evals.ts b/src/evals/evals.ts new file mode 100644 index 0000000..a607f94 --- /dev/null +++ b/src/evals/evals.ts @@ -0,0 +1,139 @@ +import { EvalConfig } from 'mcp-evals'; +import { openai } from '@ai-sdk/openai'; +import { grade, EvalFunction } from 'mcp-evals'; + +const listApplicationsEval: EvalFunction = { + name: 'List Applications Evaluation', + description: 'Evaluates the model\'s ability to list ArgoCD applications', + run: async () => { + const result = await grade(openai("gpt-4"), "Show me all the ArgoCD applications in the cluster."); + return JSON.parse(result); + } +}; + +const getApplicationEval: EvalFunction = { + name: 'Get Application Evaluation', + description: 'Evaluates the model\'s ability to retrieve a specific ArgoCD application', + run: async () => { + const result = await grade(openai("gpt-4"), "Get details for the 'my-app' application in ArgoCD."); + return JSON.parse(result); + } +}; + +const createApplicationEval: EvalFunction = { + name: 'Create Application Evaluation', + description: 'Evaluates the model\'s ability to create a new ArgoCD application', + run: async () => { + const result = await grade(openai("gpt-4"), "Create a new ArgoCD application named 'test-app' pointing to 'https://github.com/test/repo' in the 'default' namespace."); + return JSON.parse(result); + } +}; + +const updateApplicationEval: EvalFunction = { + name: 'Update Application Evaluation', + description: 'Evaluates the model\'s ability to update an existing ArgoCD application', + run: async () => { + const result = await grade(openai("gpt-4"), "Update the 'test-app' ArgoCD application to point to the 'main' branch instead of 'master'."); + return JSON.parse(result); + } +}; + +const deleteApplicationEval: EvalFunction = { + name: 'Delete Application Evaluation', + description: 'Evaluates the model\'s ability to delete an ArgoCD application', + run: async () => { + const result = await grade(openai("gpt-4"), "Delete the 'test-app' application from ArgoCD."); + return JSON.parse(result); + } +}; + +const syncApplicationEval: EvalFunction = { + name: 'Sync Application Evaluation', + description: 'Evaluates the model\'s ability to sync an ArgoCD application', + run: async () => { + const result = await grade(openai("gpt-4"), "Sync the 'my-app' application in ArgoCD."); + return JSON.parse(result); + } +}; + +const getApplicationResourceTreeEval: EvalFunction = { + name: 'Get Application Resource Tree Evaluation', + description: 'Evaluates the model\'s ability to retrieve the resource tree for an ArgoCD application', + run: async () => { + const result = await grade(openai("gpt-4"), "Show me the resource tree for the 'my-app' application in ArgoCD."); + return JSON.parse(result); + } +}; + +const getApplicationManagedResourcesEval: EvalFunction = { + name: 'Get Application Managed Resources Evaluation', + description: 'Evaluates the model\'s ability to retrieve managed resources for an ArgoCD application', + run: async () => { + const result = await grade(openai("gpt-4"), "What resources are managed by the 'my-app' application in ArgoCD?"); + return JSON.parse(result); + } +}; + +const getApplicationWorkloadLogsEval: EvalFunction = { + name: 'Get Application Workload Logs Evaluation', + description: 'Evaluates the model\'s ability to retrieve logs for an ArgoCD application workload', + run: async () => { + const result = await grade(openai("gpt-4"), "Show me the logs for the 'web' deployment in the 'my-app' application."); + return JSON.parse(result); + } +}; + +const getApplicationEventsEval: EvalFunction = { + name: 'Get Application Events Evaluation', + description: 'Evaluates the model\'s ability to retrieve events for an ArgoCD application', + run: async () => { + const result = await grade(openai("gpt-4"), "What events have occurred for the 'my-app' application in ArgoCD?"); + return JSON.parse(result); + } +}; + +const getResourceEventsEval: EvalFunction = { + name: 'Get Resource Events Evaluation', + description: 'Evaluates the model\'s ability to retrieve events for a resource within an ArgoCD application', + run: async () => { + const result = await grade(openai("gpt-4"), "Show me the events for the 'web' deployment in the 'my-app' application."); + return JSON.parse(result); + } +}; + +const getResourceActionsEval: EvalFunction = { + name: 'Get Resource Actions Evaluation', + description: 'Evaluates the model\'s ability to retrieve available actions for a resource', + run: async () => { + const result = await grade(openai("gpt-4"), "What actions can I perform on the 'database' StatefulSet in the 'my-app' application?"); + return JSON.parse(result); + } +}; + +const runResourceActionEval: EvalFunction = { + name: 'Run Resource Action Evaluation', + description: 'Evaluates the model\'s ability to run an action on a resource', + run: async () => { + const result = await grade(openai("gpt-4"), "Restart the 'web' deployment in the 'my-app' application."); + return JSON.parse(result); + } +}; + +export const evalConfig = { + model: openai('gpt-4'), + evals: [ + listApplicationsEval, + getApplicationEval, + createApplicationEval, + updateApplicationEval, + deleteApplicationEval, + syncApplicationEval, + getApplicationResourceTreeEval, + getApplicationManagedResourcesEval, + getApplicationWorkloadLogsEval, + getApplicationEventsEval, + getResourceEventsEval, + getResourceActionsEval, + runResourceActionEval + ] +}; From 1054aa679b8d61adabe64f3303efe961e79a532a Mon Sep 17 00:00:00 2001 From: Matthew Lenhard Date: Wed, 7 May 2025 08:42:29 -0400 Subject: [PATCH 3/3] [bugfix] fix lint errors --- package.json | 5 +-- pnpm-lock.yaml | 26 +++++++++++--- src/evals/evals.ts | 89 ++++++++++++++++++++++++++++++++-------------- 3 files changed, 86 insertions(+), 34 deletions(-) diff --git a/package.json b/package.json index a4abc9c..a535d41 100644 --- a/package.json +++ b/package.json @@ -50,12 +50,13 @@ "@modelcontextprotocol/sdk": "^1.10.1", "dotenv": "^16.5.0", "express": "^5.1.0", + "mcp-evals": "^1.0.18", "pino": "^9.6.0", "yargs": "^17.7.2", - "zod": "^3.24.3", - "mcp-evals": "^1.0.18" + "zod": "^3.24.3" }, "devDependencies": { + "@ai-sdk/openai": "^1.3.22", "@dtsgenerator/replace-namespace": "^1.7.0", "@eslint/js": "^9.25.0", "@types/express": "^5.0.1", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index de45088..f23c222 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -30,6 +30,9 @@ importers: specifier: ^3.24.3 version: 3.24.3 devDependencies: + '@ai-sdk/openai': + specifier: ^1.3.22 + version: 1.3.22(zod@3.24.3) '@dtsgenerator/replace-namespace': specifier: ^1.7.0 version: 1.7.0(dtsgenerator@3.19.2)(tslib@2.8.1) @@ -87,8 +90,8 @@ packages: '@actions/io@1.1.3': resolution: {integrity: sha512-wi9JjgKLYS7U/z8PPbco+PvTb/nRWjeoFlJ1Qer83k/3C5PHQi28hiVdeE2kHXmIL99mQFawx8qt/JPjZilJ8Q==} - '@ai-sdk/openai@1.3.21': - resolution: {integrity: sha512-ipAhkRKUd2YaMmn7DAklX3N7Ywx/rCsJHVyb0V/lKRqPcc612qAFVbjg+Uve8QYJlbPxgfsM4s9JmCFp6PSdYw==} + '@ai-sdk/openai@1.3.22': + resolution: {integrity: sha512-QwA+2EkG0QyjVR+7h6FE7iOu2ivNqAVMm9UJZkVxxTk5OIq5fFJDTEI/zICEMuHImTTXR2JjsL6EirJ28Jc4cw==} engines: {node: '>=18'} peerDependencies: zod: ^3.0.0 @@ -99,6 +102,12 @@ packages: peerDependencies: zod: ^3.23.8 + '@ai-sdk/provider-utils@2.2.8': + resolution: {integrity: sha512-fqhG+4sCVv8x7nFzYnFo19ryhAa3w096Kmc3hWxMQfW/TubPOmt3A6tYZhl4mUfQWWQMsuSkLrtjlWuXBVSGQA==} + engines: {node: '>=18'} + peerDependencies: + zod: ^3.23.8 + '@ai-sdk/provider@1.1.3': resolution: {integrity: sha512-qZMxYJ0qqX/RfnuIaab+zp8UAeJn/ygXXAffR5I4N0n1IrvA6qBsjc8hXLmBiMV2zoXlifkacF7sEFnYnjBcqg==} engines: {node: '>=18'} @@ -1831,10 +1840,10 @@ snapshots: '@actions/io@1.1.3': {} - '@ai-sdk/openai@1.3.21(zod@3.24.3)': + '@ai-sdk/openai@1.3.22(zod@3.24.3)': dependencies: '@ai-sdk/provider': 1.1.3 - '@ai-sdk/provider-utils': 2.2.7(zod@3.24.3) + '@ai-sdk/provider-utils': 2.2.8(zod@3.24.3) zod: 3.24.3 '@ai-sdk/provider-utils@2.2.7(zod@3.24.3)': @@ -1844,6 +1853,13 @@ snapshots: secure-json-parse: 2.7.0 zod: 3.24.3 + '@ai-sdk/provider-utils@2.2.8(zod@3.24.3)': + dependencies: + '@ai-sdk/provider': 1.1.3 + nanoid: 3.3.11 + secure-json-parse: 2.7.0 + zod: 3.24.3 + '@ai-sdk/provider@1.1.3': dependencies: json-schema: 0.4.0 @@ -2958,7 +2974,7 @@ snapshots: mcp-evals@1.0.18(react@19.1.0)(zod@3.24.3): dependencies: '@actions/core': 1.11.1 - '@ai-sdk/openai': 1.3.21(zod@3.24.3) + '@ai-sdk/openai': 1.3.22(zod@3.24.3) '@anthropic-ai/sdk': 0.8.1 '@modelcontextprotocol/sdk': 1.11.0 ai: 4.3.13(react@19.1.0)(zod@3.24.3) diff --git a/src/evals/evals.ts b/src/evals/evals.ts index a607f94..4b12b4f 100644 --- a/src/evals/evals.ts +++ b/src/evals/evals.ts @@ -1,120 +1,155 @@ -import { EvalConfig } from 'mcp-evals'; import { openai } from '@ai-sdk/openai'; import { grade, EvalFunction } from 'mcp-evals'; const listApplicationsEval: EvalFunction = { name: 'List Applications Evaluation', - description: 'Evaluates the model\'s ability to list ArgoCD applications', + description: "Evaluates the model's ability to list ArgoCD applications", run: async () => { - const result = await grade(openai("gpt-4"), "Show me all the ArgoCD applications in the cluster."); + const result = await grade( + openai('gpt-4'), + 'Show me all the ArgoCD applications in the cluster.' + ); return JSON.parse(result); } }; const getApplicationEval: EvalFunction = { name: 'Get Application Evaluation', - description: 'Evaluates the model\'s ability to retrieve a specific ArgoCD application', + description: "Evaluates the model's ability to retrieve a specific ArgoCD application", run: async () => { - const result = await grade(openai("gpt-4"), "Get details for the 'my-app' application in ArgoCD."); + const result = await grade( + openai('gpt-4'), + "Get details for the 'my-app' application in ArgoCD." + ); return JSON.parse(result); } }; const createApplicationEval: EvalFunction = { name: 'Create Application Evaluation', - description: 'Evaluates the model\'s ability to create a new ArgoCD application', + description: "Evaluates the model's ability to create a new ArgoCD application", run: async () => { - const result = await grade(openai("gpt-4"), "Create a new ArgoCD application named 'test-app' pointing to 'https://github.com/test/repo' in the 'default' namespace."); + const result = await grade( + openai('gpt-4'), + "Create a new ArgoCD application named 'test-app' pointing to 'https://github.com/test/repo' in the 'default' namespace." + ); return JSON.parse(result); } }; const updateApplicationEval: EvalFunction = { name: 'Update Application Evaluation', - description: 'Evaluates the model\'s ability to update an existing ArgoCD application', + description: "Evaluates the model's ability to update an existing ArgoCD application", run: async () => { - const result = await grade(openai("gpt-4"), "Update the 'test-app' ArgoCD application to point to the 'main' branch instead of 'master'."); + const result = await grade( + openai('gpt-4'), + "Update the 'test-app' ArgoCD application to point to the 'main' branch instead of 'master'." + ); return JSON.parse(result); } }; const deleteApplicationEval: EvalFunction = { name: 'Delete Application Evaluation', - description: 'Evaluates the model\'s ability to delete an ArgoCD application', + description: "Evaluates the model's ability to delete an ArgoCD application", run: async () => { - const result = await grade(openai("gpt-4"), "Delete the 'test-app' application from ArgoCD."); + const result = await grade(openai('gpt-4'), "Delete the 'test-app' application from ArgoCD."); return JSON.parse(result); } }; const syncApplicationEval: EvalFunction = { name: 'Sync Application Evaluation', - description: 'Evaluates the model\'s ability to sync an ArgoCD application', + description: "Evaluates the model's ability to sync an ArgoCD application", run: async () => { - const result = await grade(openai("gpt-4"), "Sync the 'my-app' application in ArgoCD."); + const result = await grade(openai('gpt-4'), "Sync the 'my-app' application in ArgoCD."); return JSON.parse(result); } }; const getApplicationResourceTreeEval: EvalFunction = { name: 'Get Application Resource Tree Evaluation', - description: 'Evaluates the model\'s ability to retrieve the resource tree for an ArgoCD application', + description: + "Evaluates the model's ability to retrieve the resource tree for an ArgoCD application", run: async () => { - const result = await grade(openai("gpt-4"), "Show me the resource tree for the 'my-app' application in ArgoCD."); + const result = await grade( + openai('gpt-4'), + "Show me the resource tree for the 'my-app' application in ArgoCD." + ); return JSON.parse(result); } }; const getApplicationManagedResourcesEval: EvalFunction = { name: 'Get Application Managed Resources Evaluation', - description: 'Evaluates the model\'s ability to retrieve managed resources for an ArgoCD application', + description: + "Evaluates the model's ability to retrieve managed resources for an ArgoCD application", run: async () => { - const result = await grade(openai("gpt-4"), "What resources are managed by the 'my-app' application in ArgoCD?"); + const result = await grade( + openai('gpt-4'), + "What resources are managed by the 'my-app' application in ArgoCD?" + ); return JSON.parse(result); } }; const getApplicationWorkloadLogsEval: EvalFunction = { name: 'Get Application Workload Logs Evaluation', - description: 'Evaluates the model\'s ability to retrieve logs for an ArgoCD application workload', + description: "Evaluates the model's ability to retrieve logs for an ArgoCD application workload", run: async () => { - const result = await grade(openai("gpt-4"), "Show me the logs for the 'web' deployment in the 'my-app' application."); + const result = await grade( + openai('gpt-4'), + "Show me the logs for the 'web' deployment in the 'my-app' application." + ); return JSON.parse(result); } }; const getApplicationEventsEval: EvalFunction = { name: 'Get Application Events Evaluation', - description: 'Evaluates the model\'s ability to retrieve events for an ArgoCD application', + description: "Evaluates the model's ability to retrieve events for an ArgoCD application", run: async () => { - const result = await grade(openai("gpt-4"), "What events have occurred for the 'my-app' application in ArgoCD?"); + const result = await grade( + openai('gpt-4'), + "What events have occurred for the 'my-app' application in ArgoCD?" + ); return JSON.parse(result); } }; const getResourceEventsEval: EvalFunction = { name: 'Get Resource Events Evaluation', - description: 'Evaluates the model\'s ability to retrieve events for a resource within an ArgoCD application', + description: + "Evaluates the model's ability to retrieve events for a resource within an ArgoCD application", run: async () => { - const result = await grade(openai("gpt-4"), "Show me the events for the 'web' deployment in the 'my-app' application."); + const result = await grade( + openai('gpt-4'), + "Show me the events for the 'web' deployment in the 'my-app' application." + ); return JSON.parse(result); } }; const getResourceActionsEval: EvalFunction = { name: 'Get Resource Actions Evaluation', - description: 'Evaluates the model\'s ability to retrieve available actions for a resource', + description: "Evaluates the model's ability to retrieve available actions for a resource", run: async () => { - const result = await grade(openai("gpt-4"), "What actions can I perform on the 'database' StatefulSet in the 'my-app' application?"); + const result = await grade( + openai('gpt-4'), + "What actions can I perform on the 'database' StatefulSet in the 'my-app' application?" + ); return JSON.parse(result); } }; const runResourceActionEval: EvalFunction = { name: 'Run Resource Action Evaluation', - description: 'Evaluates the model\'s ability to run an action on a resource', + description: "Evaluates the model's ability to run an action on a resource", run: async () => { - const result = await grade(openai("gpt-4"), "Restart the 'web' deployment in the 'my-app' application."); + const result = await grade( + openai('gpt-4'), + "Restart the 'web' deployment in the 'my-app' application." + ); return JSON.parse(result); } };