diff --git a/.env.example b/.env.example index e76726b..7ff9ce0 100644 --- a/.env.example +++ b/.env.example @@ -1 +1,3 @@ -VERCEL_OIDC_TOKEN="create with vercel cli" \ No newline at end of file +# API keys (set via 'bun run scripts/secrets.ts set VERCEL_OIDC_TOKEN') +# Note: API keys are stored in OS credential manager for security +# No .env file needed - use the secrets CLI instead \ No newline at end of file diff --git a/AGENTS.md b/AGENTS.md index 3e7c825..18de73c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -28,6 +28,11 @@ bun tsc --noEmit # Format code with Prettier bun run prettier + +# Secrets management +bun run secrets # Show token status +bun run secrets set VERCEL_OIDC_TOKEN # Store Vercel token +bun run secrets get VERCEL_OIDC_TOKEN # Get Vercel token ``` ## Environment Variables @@ -45,12 +50,46 @@ Required environment variable: ### MCP Server Configuration -MCP integration is configured via the interactive CLI at runtime. Options: +MCP integration is configured via the interactive CLI at runtime. The CLI presents three options: - **No MCP Integration**: Agent runs with built-in tools only - **MCP over HTTP**: Uses HTTP transport (default: `https://mcp.svelte.dev/mcp`) - **MCP over StdIO**: Uses local command (default: `npx -y @sveltejs/mcp`) +You can provide custom MCP servers when prompted during the interactive setup. + +**Behavior:** + +- If MCP server starts with `http://` or `https://`: Uses HTTP transport with that URL +- If MCP server is set but not an HTTP URL: Uses StdIO transport, treating the value as a command string +- If no MCP is selected: Agent runs without MCP tools (only built-in tools) +- MCP transport type (HTTP or StdIO) and configuration are documented in the result JSON and HTML report + +### Required API Keys + +- `VERCEL_OIDC_TOKEN`: Required for Vercel AI Gateway (stored in bun.secrets) +- Other API keys (Anthropic, OpenAI, OpenRouter) are configured in Vercel dashboard when using AI Gateway + +### Secrets Management + +The tool uses Bun's secure credential storage for the Vercel OIDC token: + +```bash +# Store Vercel OIDC token +bun run secrets set VERCEL_OIDC_TOKEN your_token_here + +# Check if token is stored +bun run secrets + +# Get the stored token +bun run secrets get VERCEL_OIDC_TOKEN +``` + +**Security Benefits:** +- Encrypted storage using OS credential manager (Keychain, libsecret, Windows Credential Manager) +- No plaintext tokens in files +- User-level access control + ## Architecture ### Directory Structure diff --git a/README.md b/README.md index 91a33a7..50e35a4 100644 --- a/README.md +++ b/README.md @@ -13,17 +13,40 @@ bun install ## Setup -Configure your API keys in `.env`: +Configure your Vercel OIDC token using bun.secrets: 1. Install Vercel CLI if you haven't already 2. Run `bun run vercel:link` and link the benchmark to a project that has AI Gateway enabled -3. Run the benchmark with "bun run dev" +3. Store your VERCEL_OIDC_TOKEN securely: + ```bash + # Get your token from Vercel project settings + bun run secrets set VERCEL_OIDC_TOKEN your_token_here + ``` ### Required API Keys -You'll need at least one API key for the providers you want to test: +- `VERCEL_OIDC_TOKEN`: Required for Vercel AI Gateway (stored in bun.secrets) +- Other API keys (Anthropic, OpenAI, OpenRouter) are configured in the Vercel dashboard when using AI Gateway -- `VERCEL_OIDC_TOKEN`: The OIDC token for vercel AI gateway +## Secrets Management + +API keys are stored securely using your OS credential manager: + +```bash +# Check if token is set +bun run secrets + +# Set token +bun run secrets set VERCEL_OIDC_TOKEN your_token_here + +# Get token +bun run secrets get VERCEL_OIDC_TOKEN +``` + +**Security Benefits:** +- Encrypted storage using OS credential manager (Keychain, libsecret, Windows Credential Manager) +- No plaintext API keys in files +- User-level access control ## Usage diff --git a/index.ts b/index.ts index b4a4bb3..98cdf48 100644 --- a/index.ts +++ b/index.ts @@ -45,6 +45,7 @@ import { note, } from "@clack/prompts"; import { gateway } from "ai"; +import { loadTokenToEnv } from "./scripts/secrets.ts"; async function validateAndConfirmPricing( models: string[], @@ -366,6 +367,9 @@ async function runSingleTest( } async function main() { + // Load VERCEL_OIDC_TOKEN from bun.secrets + await loadTokenToEnv(); + const { models, mcp, testingTool, pricing } = await selectOptions(); const mcpServerUrl = mcp; diff --git a/package.json b/package.json index b0d317f..9c6d12b 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,8 @@ "prettier": "prettier --write .", "tsc": "tsc --noEmit", "vercel:link": "vercel link", - "vercel:env:pull": "vercel env pull .env.local --yes" + "vercel:env:pull": "vercel env pull .env.local --yes", + "secrets": "bun run scripts/secrets.ts" }, "devDependencies": { "@sveltejs/vite-plugin-svelte": "^5.0.3", diff --git a/scripts/secrets.ts b/scripts/secrets.ts new file mode 100644 index 0000000..a0a44b7 --- /dev/null +++ b/scripts/secrets.ts @@ -0,0 +1,124 @@ +#!/usr/bin/env bun +/** + * Simple CLI tool for managing VERCEL_OIDC_TOKEN using bun.secrets + * + * Usage: + * bun run secrets set VERCEL_OIDC_TOKEN # Set the token + * bun run secrets get VERCEL_OIDC_TOKEN # Get the token + * bun run secrets # Show status + */ + +import { secrets } from "bun"; + +async function showStatus() { + console.log("\nšŸ” Vercel OIDC Token Status"); + console.log("ā”€".repeat(40)); + + const token = await secrets.get({ + service: "svelte-ai", + name: "VERCEL_OIDC_TOKEN" + }); + + if (token) { + const masked = token.slice(0, 12) + "..." + token.slice(-8); + console.log(`āœ… VERCEL_OIDC_TOKEN: ${masked}`); + } else { + console.log("āŒ VERCEL_OIDC_TOKEN: Not set"); + console.log("\nšŸ’” Run 'bun run secrets set VERCEL_OIDC_TOKEN ' to set it"); + } +} + +async function setToken(value: string) { + if (!value) { + // Empty value means delete the token + await secrets.delete({ + service: "svelte-ai", + name: "VERCEL_OIDC_TOKEN" + }); + console.log("šŸ—‘ļø VERCEL_OIDC_TOKEN deleted from OS credential manager"); + return; + } + + if (value.length < 20) { + console.error("āŒ Invalid token: VERCEL_OIDC_TOKEN appears to be too short"); + process.exit(1); + } + + await secrets.set({ + service: "svelte-ai", + name: "VERCEL_OIDC_TOKEN", + value, + }); + + console.log("āœ… VERCEL_OIDC_TOKEN stored securely in OS credential manager"); +} + +async function getToken() { + const token = await secrets.get({ + service: "svelte-ai", + name: "VERCEL_OIDC_TOKEN" + }); + + if (token) { + console.log(`VERCEL_OIDC_TOKEN: ${token}`); + } else { + console.log("āŒ VERCEL_OIDC_TOKEN not found"); + process.exit(1); + } +} + +async function loadTokenToEnv() { + const token = await secrets.get({ + service: "svelte-ai", + name: "VERCEL_OIDC_TOKEN" + }); + + if (token) { + process.env.VERCEL_OIDC_TOKEN = token; + console.log("āœ… VERCEL_OIDC_TOKEN loaded from bun.secrets"); + } +} + +// Export for use in main application +export { loadTokenToEnv }; + +// CLI logic +if (import.meta.main) { + const command = process.argv[2]; + const arg1 = process.argv[3]; + const arg2 = process.argv[4]; + + switch (command) { + case "set": + if (arg1 !== "VERCEL_OIDC_TOKEN") { + console.error("Usage: bun run secrets set VERCEL_OIDC_TOKEN "); + process.exit(1); + } + // Empty arg2 means delete, otherwise set the value + await setToken(arg2 || ""); + break; + + case "get": + if (arg1 !== "VERCEL_OIDC_TOKEN") { + console.error("Usage: bun run secrets get VERCEL_OIDC_TOKEN"); + process.exit(1); + } + await getToken(); + break; + + case "load": + // Internal command used by main app + await loadTokenToEnv(); + break; + + case undefined: + case "status": + await showStatus(); + break; + + default: + console.error(`Unknown command: ${command}`); + console.error("Available commands: status, set VERCEL_OIDC_TOKEN , get VERCEL_OIDC_TOKEN"); + process.exit(1); + } +} \ No newline at end of file