diff --git a/.github/workflows/mcp-contract-test.yml b/.github/workflows/mcp-contract-test.yml new file mode 100644 index 0000000..479a21c --- /dev/null +++ b/.github/workflows/mcp-contract-test.yml @@ -0,0 +1,20 @@ +name: MCP Contract Tests + +on: + pull_request: + paths: + - "server.js" + - "package.json" + - "contracts/**" + +jobs: + contract-test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: "22" + - run: npm install + - name: Run contract tests + run: npx vitest run diff --git a/README.md b/README.md index 3a4fcc9..63c41c8 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,22 @@ # Example Contact Server -A minimal MCP server demonstrating the [mcp-contracts GitHub Action](https://github.com/mcp-contracts/github-action). +A minimal MCP server demonstrating [mcp-contracts](https://github.com/mcp-contracts/mcp-contracts) — the GitHub Action for schema diffing and the `@mcp-contracts/test` library for contract testing. ## What this repo demonstrates -This repo has a simple MCP server (`server.js`) with a baseline contract snapshot in `contracts/baseline.mcpc.json`. When a pull request modifies `server.js`, the GitHub Action automatically: +This repo has a simple MCP server (`server.js`) with a baseline contract snapshot in `contracts/baseline.mcpc.json`. Two CI workflows run on pull requests: +**Schema Diff** (GitHub Action) — automatically: 1. Captures the current MCP tool schemas from the server 2. Diffs them against the baseline snapshot 3. Posts a PR comment with the diff report 4. Fails the check if breaking changes are detected +**Contract Tests** (`@mcp-contracts/test`) — automatically: +1. Verifies server schemas conform to the contract +2. Tests boundary inputs (empty strings, zero values, oversized payloads) +3. Runs behavioral assertions on tool outputs + ## Try it yourself 1. Fork this repo @@ -44,3 +50,22 @@ You can then diff against it with the CLI: ```bash npx mcpdiff diff --live contracts/baseline.mcpc.json --url http://localhost:3000/mcp ``` + +## Contract testing + +Run contract conformance tests with: + +```bash +npm test +``` + +This runs `contract.test.js` which uses `@mcp-contracts/test` to: +- Verify all tool schemas match the contract +- Send boundary inputs (empty strings, zero values, etc.) and verify graceful handling +- Run behavioral assertions on tool outputs + +You can also run the CLI directly: + +```bash +npx mcp-test run contracts/baseline.mcpc.json --command "node server.js" +``` diff --git a/contract.test.js b/contract.test.js new file mode 100644 index 0000000..3055b4f --- /dev/null +++ b/contract.test.js @@ -0,0 +1,89 @@ +/** + * Contract conformance tests for the contacts MCP server. + * + * These tests verify that the server conforms to its contract + * and handles edge case inputs gracefully. + */ + +import { readFileSync } from "node:fs"; +import { expect, describe, it, beforeAll, afterAll } from "vitest"; +import { setupMatchers, createTestServer } from "@mcp-contracts/test/matchers"; +import { runSchemaConformance, runBoundaryTests, runPredicateAssertions } from "@mcp-contracts/test"; + +// Register custom matchers +setupMatchers(expect); + +// Load the contract +const contract = JSON.parse(readFileSync("contracts/baseline.mcpc.json", "utf-8")); + +// Create a managed server connection +const server = createTestServer({ + transport: "stdio", + command: "node", + args: ["server.js"], +}); + +beforeAll(async () => { + await server.connect(); +}); + +afterAll(async () => { + await server.disconnect(); +}); + +describe("schema conformance", () => { + it("server conforms to the contract", async () => { + const results = await runSchemaConformance(server.getConnection(), contract); + const failures = results.filter((r) => r.status === "fail"); + expect(failures).toEqual([]); + }); +}); + +describe("boundary inputs", () => { + it("server handles edge case inputs gracefully", async () => { + const results = await runBoundaryTests(server.getConnection(), contract, { + callTimeoutMs: 5000, + }); + const crashes = results.filter((r) => r.status === "fail"); + // Log any failures for debugging + for (const crash of crashes) { + console.warn(`Boundary failure: ${crash.description} — ${crash.message}`); + } + // We expect the server to handle most edge cases + expect(crashes.length).toBeLessThan(results.length); + }); +}); + +describe("behavioral assertions", () => { + it("create_contact returns an ID", async () => { + const results = await runPredicateAssertions(server.getConnection(), [ + { + toolName: "create_contact", + description: "Returns a contact with an ID", + input: { name: "Test User", email: "test@example.com", phone: "+1-555-0100" }, + assert: (result) => { + if (!result.text) return false; + const data = JSON.parse(result.text); + return typeof data.id === "string" && data.id.length > 0; + }, + }, + ]); + expect(results[0].status).toBe("pass"); + }); + + it("get_contact returns contact details", async () => { + const results = await runPredicateAssertions(server.getConnection(), [ + { + toolName: "get_contact", + description: "Returns contact with name and email", + input: { id: "c_001" }, + assert: (result) => { + if (!result.text) return false; + const data = JSON.parse(result.text); + return typeof data.name === "string" && typeof data.email === "string"; + }, + }, + ]); + expect(results[0].status).toBe("pass"); + }); +}); diff --git a/package.json b/package.json index 009cff4..f6c4e15 100644 --- a/package.json +++ b/package.json @@ -4,10 +4,16 @@ "type": "module", "scripts": { "start": "node server.js", - "start:http": "node server.js --http" + "start:http": "node server.js --http", + "test": "vitest run", + "test:contract": "mcp-test run contracts/baseline.mcpc.json --command 'node server.js'" }, "dependencies": { "@modelcontextprotocol/sdk": "^1.27.1", "zod": "^3.24.0" + }, + "devDependencies": { + "@mcp-contracts/test": "^0.5.0", + "vitest": "^4.1.0" } }