wip local mcp

Author: khromov

.env.example

@@ -9,10 +9,15 @@ OPENROUTER_API_KEY=sk-or-v1-your-key-here
 
 # MCP Server Configuration (optional)
 # Leave empty to disable MCP integration
-# Default Svelte MCP server:
+
+# For HTTP MCP servers (use full URL):
 MCP_SERVER_URL=https://mcp.svelte.dev/mcp
 
-# To disable MCP, uncomment the line below:
+
+# For local stdio MCP servers (use command string):
+# MCP_SERVER_URL=npx -y @sveltejs/mcp
+
+# To disable MCP, set mcp to empty string
 # MCP_SERVER_URL=
 
 # To disable the test component tool, uncomment the line below

AGENTS.md

@@ -55,24 +55,46 @@ MODEL=openrouter/meta-llama/llama-3.1-405b-instruct
 
 ### MCP Server Configuration
 
-The `MCP_SERVER_URL` environment variable controls MCP (Model Context Protocol) integration:
+The `MCP_SERVER_URL` environment variable controls MCP (Model Context Protocol) integration. The tool automatically detects whether to use HTTP or StdIO transport based on the value format.
+
+**HTTP MCP Servers (Remote):**
 
 ```bash
 # Enable MCP with Svelte server (default for this benchmark)
 MCP_SERVER_URL=https://mcp.svelte.dev/mcp
 
+# Use a different HTTP MCP server
+MCP_SERVER_URL=https://your-mcp-server.com/mcp
+```
+
+**StdIO MCP Servers (Local):**
+
+For local MCP servers, simply provide the command string (any non-HTTP value):
+
+```bash
+# Use the default Svelte MCP server via npx
+MCP_SERVER_URL=npx -y @sveltejs/mcp
+
+# Use a custom local MCP server
+MCP_SERVER_URL=node path/to/your/mcp-server.js
+
+# Use with Bun runtime
+MCP_SERVER_URL=bun run src/mcp-server.ts --verbose
+```
+
+**Disable MCP:**
+
+```bash
 # Disable MCP integration (run without external tools)
 MCP_SERVER_URL=
-
-# Use a different MCP server
-MCP_SERVER_URL=https://your-mcp-server.com/mcp
 ```
 
 **Behavior:**
 
-- If `MCP_SERVER_URL` is set and not empty: MCP tools are injected into the agent
+- If `MCP_SERVER_URL` starts with `http://` or `https://`: Uses HTTP transport with that URL
+- If `MCP_SERVER_URL` is set but not an HTTP URL: Uses StdIO transport, treating the value as a command string
 - If `MCP_SERVER_URL` is empty or not set: Agent runs without MCP tools (only built-in tools)
-- MCP status is documented in the result JSON and HTML report with a badge
+- MCP transport type (HTTP or StdIO) and configuration are documented in the result JSON and HTML report
 
 ### Required API Keys
 

index.ts

@@ -1,5 +1,6 @@
 import { Experimental_Agent as Agent, hasToolCall, stepCountIs } from "ai";
 import { experimental_createMCPClient as createMCPClient } from "./node_modules/@ai-sdk/mcp/dist/index.mjs";
+import { Experimental_StdioMCPTransport as StdioMCPTransport } from "./node_modules/@ai-sdk/mcp/dist/mcp-stdio/index.mjs";
 import { writeFileSync, mkdirSync, existsSync } from "node:fs";
 import {
   generateReport,
@@ -36,6 +37,27 @@ function getTimestampedFilename(prefix: string, extension: string): string {
   return `${prefix}-${year}-${month}-${day}-${hours}-${minutes}-${seconds}.${extension}`;
 }
 
+/**
+ * Parse a command string into command and args
+ * Example: "npx -y @sveltejs/mcp" -> { command: "npx", args: ["-y", "@sveltejs/mcp"] }
+ */
+function parseCommandString(commandString: string): {
+  command: string;
+  args: string[];
+} {
+  const parts = commandString.trim().split(/\s+/);
+  const command = parts[0] ?? "";
+  const args = parts.slice(1);
+  return { command, args };
+}
+
+/**
+ * Check if a string is an HTTP/HTTPS URL
+ */
+function isHttpUrl(str: string): boolean {
+  return str.startsWith("http://") || str.startsWith("https://");
+}
+
 /**
  * Extract ResultWrite content from agent steps
  */
@@ -167,20 +189,29 @@ async function runSingleTest(
 
 // Main execution
 async function main() {
-  // Get MCP server URL from environment (optional)
+  // Get MCP server URL/command from environment (optional)
   const mcpServerUrl = process.env.MCP_SERVER_URL || "";
   const mcpEnabled = mcpServerUrl.trim() !== "";
 
   // Check if TestComponent tool is disabled
   const testComponentEnabled = process.env.DISABLE_TESTCOMPONENT_TOOL !== "1";
 
+  // Determine MCP transport type
+  const isHttpTransport = mcpEnabled && isHttpUrl(mcpServerUrl);
+  const mcpTransportType = isHttpTransport ? "HTTP" : "StdIO";
+
   console.log("╔════════════════════════════════════════════════════╗");
   console.log("║            SvelteBench 2.0 - Multi-Test            ║");
   console.log("╚════════════════════════════════════════════════════╝");
   console.log(`Model: ${process.env.MODEL}`);
   console.log(`MCP Integration: ${mcpEnabled ? "Enabled" : "Disabled"}`);
   if (mcpEnabled) {
-    console.log(`MCP Server URL: ${mcpServerUrl}`);
+    console.log(`MCP Transport: ${mcpTransportType}`);
+    if (isHttpTransport) {
+      console.log(`MCP Server URL: ${mcpServerUrl}`);
+    } else {
+      console.log(`MCP StdIO Command: ${mcpServerUrl}`);
+    }
   }
   console.log(
     `TestComponent Tool: ${testComponentEnabled ? "Enabled" : "Disabled"}`,
@@ -201,15 +232,28 @@ async function main() {
   // Set up outputs directory
   setupOutputsDirectory();
 
-  // Conditionally create MCP client if URL is provided
-  const mcpClient = mcpEnabled
-    ? await createMCPClient({
+  // Conditionally create MCP client based on transport type
+  let mcpClient: Awaited<ReturnType<typeof createMCPClient>> | null = null;
+  if (mcpEnabled) {
+    if (isHttpTransport) {
+      // HTTP transport
+      mcpClient = await createMCPClient({
         transport: {
           type: "http",
           url: mcpServerUrl,
         },
-      })
-    : null;
+      });
+    } else {
+      // StdIO transport - treat mcpServerUrl as command string
+      const { command, args } = parseCommandString(mcpServerUrl);
+      mcpClient = await createMCPClient({
+        transport: new StdioMCPTransport({
+          command,
+          args,
+        }),
+      });
+    }
+  }
 
   // Load environment configuration and get model provider
   const envConfig = loadEnvConfig();
@@ -286,6 +330,7 @@ async function main() {
     metadata: {
       mcpEnabled,
       mcpServerUrl: mcpEnabled ? mcpServerUrl : null,
+      mcpTransportType: mcpEnabled ? mcpTransportType : null,
       timestamp: new Date().toISOString(),
       model: envConfig.modelString,
     },

lib/report.ts

@@ -69,6 +69,7 @@ interface Step {
 interface Metadata {
   mcpEnabled: boolean;
   mcpServerUrl: string | null;
+  mcpTransportType?: string | null;
   timestamp: string;
   model: string;
 }