diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..c7f8e50 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,23 @@ +--- +name: Bug report +description: Report a problem with Primer +labels: [bug] +--- + +## Description + +## Steps to reproduce + +1. +2. +3. + +## Expected behavior + +## Actual behavior + +## Environment + +- OS: +- Node version: +- Primer version: diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..3624ac5 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,5 @@ +blank_issues_enabled: false +contact_links: + - name: Questions + url: https://github.com/pierceboggan/primer/discussions + about: Ask questions and discuss ideas here. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..dabc116 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,13 @@ +--- +name: Feature request +description: Suggest an enhancement for Primer +labels: [enhancement] +--- + +## Summary + +## Problem statement + +## Proposed solution + +## Additional context diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..84350e7 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,10 @@ +## Summary + +- [ ] What changed +- [ ] Why it changed + +## Checklist + +- [ ] Tests added or updated +- [ ] Lint/typecheck pass +- [ ] Docs updated if needed diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..cb9b04c --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,34 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Setup Node + uses: actions/setup-node@v4 + with: + node-version: 20 + cache: npm + - name: Install + run: npm ci + - name: Lint + run: npm run lint + - name: Typecheck + run: npm run typecheck + - name: Test + run: npm run test:coverage + - name: Build + run: npm run build + - name: Upload coverage + uses: actions/upload-artifact@v4 + with: + name: coverage + path: coverage + if-no-files-found: ignore diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml new file mode 100644 index 0000000..3547bac --- /dev/null +++ b/.github/workflows/release-please.yml @@ -0,0 +1,19 @@ +name: Release Please + +on: + push: + branches: [main] + +permissions: + contents: write + pull-requests: write + +jobs: + release-please: + runs-on: ubuntu-latest + steps: + - name: Release + uses: google-github-actions/release-please-action@v4 + with: + config-file: release-please-config.json + manifest-file: release-please-manifest.json diff --git a/.gitignore b/.gitignore index 6b4a136..89da78c 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ node_modules/ # Build output dist/ +coverage/ # IDE .vscode/ diff --git a/.prettierrc.json b/.prettierrc.json new file mode 100644 index 0000000..1ad2e3d --- /dev/null +++ b/.prettierrc.json @@ -0,0 +1,5 @@ +{ + "singleQuote": false, + "trailingComma": "none", + "printWidth": 100 +} diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..e91d5ff --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,12 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +## [Unreleased] + +- Readiness report with monorepo support and fix-first checklist. +- CI workflow with lint, typecheck, tests, and build steps. +- Release automation via release-please. +- ESLint, Prettier, and Vitest tooling plus starter tests. +- Added contributing, security, and issue/PR templates. +- Added examples folder with sample configs. diff --git a/CODEOWNERS b/CODEOWNERS new file mode 100644 index 0000000..77511c7 --- /dev/null +++ b/CODEOWNERS @@ -0,0 +1 @@ +* @pierceboggan diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..e66fc98 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,34 @@ +# Contributing + +Thanks for contributing to Primer. + +## Quick start + +1. Fork and clone the repo. +2. Install dependencies: npm install +3. Build locally: npm run build +4. Run lint/typecheck/tests before opening a PR: + - npm run lint + - npm run typecheck + - npm run test + +## Development workflow + +- Create a feature branch from main. +- Use clear, conventional commit messages (e.g. feat: add readiness report). +- Keep PRs focused and include context in the description. +- Add or update tests when behavior changes. + +## Code style + +- ESLint + Prettier are enforced in CI. +- Prefer small, composable functions with clear types. + +## Reporting issues + +- Use GitHub Issues for bugs and feature requests. +- Provide steps to reproduce and expected behavior. + +## Releasing + +Releases are automated with release-please when changes are merged to main. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..cb6db3e --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Primer + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/PLAN.md b/PLAN.md index 3dcc268..46f5b83 100644 --- a/PLAN.md +++ b/PLAN.md @@ -16,9 +16,14 @@ Make any repository "AI-ready" with a single command — generating optimal conf - Detect language(s), frameworks, and project structure - Identify existing AI configurations - Analyze package managers, build tools, and testing frameworks -- Detect monorepo vs single-project structure +- Detect monorepo vs single-project structure (workspace-aware) -### 2. **Configuration Generation** +### 2. **Readiness Report** +- Score AI readiness across key pillars +- Provide fix-first checklists and maturity levels +- Support monorepos with app-scoped checks + +### 3. **Configuration Generation** | Config Type | Description | |-------------|-------------| @@ -26,19 +31,19 @@ Make any repository "AI-ready" with a single command — generating optimal conf | **MCP Server Config** | `.vscode/mcp.json` for Model Context Protocol servers | | **VS Code Settings** | `.vscode/settings.json` with AI-optimized workspace settings | -### 3. **GitHub Integration** +### 4. **GitHub Integration** - Authenticate via GitHub CLI (`gh auth`) or OAuth device flow - List and select from accessible repositories - Clone repos temporarily for analysis - **Auto-create PRs** with generated configurations - Support for GitHub Enterprise -### 4. **Local Repository Support** +### 5. **Local Repository Support** - Detect local Git repositories - Work offline with local-only mode - Push changes to remote when ready -### 5. **Interactive & Non-Interactive Modes** +### 6. **Interactive & Non-Interactive Modes** - Beautiful TUI with prompts and previews - CI/CD-friendly `--yes` flag for automation - JSON output for scripting @@ -99,6 +104,9 @@ primer pr owner/repo # Analyze repo without making changes primer analyze +# Readiness report +primer readiness + # Update existing configurations primer update @@ -107,6 +115,18 @@ primer templates # Configure CLI settings primer config + +# Generate instructions +primer instructions + +# Run evaluations +primer eval primer.eval.json + +# Run TUI +primer tui + +# Batch processing +primer batch ``` --- @@ -370,23 +390,33 @@ primer/ │ ├── index.ts # Entry point │ ├── cli.ts # Commander setup │ ├── commands/ -│ │ ├── init.ts -│ │ ├── generate.ts │ │ ├── analyze.ts +│ │ ├── batch.tsx +│ │ ├── config.ts +│ │ ├── eval.ts +│ │ ├── generate.ts +│ │ ├── init.ts +│ │ ├── instructions.tsx │ │ ├── pr.ts -│ │ └── config.ts +│ │ ├── readiness.ts +│ │ ├── templates.ts +│ │ ├── tui.tsx +│ │ └── update.ts │ ├── services/ -│ │ ├── github.ts # GitHub API interactions │ │ ├── analyzer.ts # Repo analysis logic +│ │ ├── azureDevops.ts # Azure DevOps integration +│ │ ├── evaluator.ts # Eval runner │ │ ├── generator.ts # Config generation -│ │ └── git.ts # Local git operations +│ │ ├── git.ts # Local git operations +│ │ ├── github.ts # GitHub API interactions +│ │ └── instructions.ts # Copilot SDK integration │ ├── ui/ -│ │ ├── prompts.ts # Inquirer prompts -│ │ ├── spinner.ts # Loading indicators -│ │ └── preview.ts # File previews +│ │ ├── AnimatedBanner.tsx +│ │ ├── BatchTui.tsx +│ │ ├── BatchTuiAzure.tsx +│ │ └── tui.tsx │ └── utils/ │ ├── fs.ts # File system helpers -│ ├── detection.ts # Language/framework detection │ └── logger.ts # Styled console output ├── package.json ├── tsconfig.json @@ -476,24 +506,24 @@ Create example repos for each major stack: # Install dependencies npm install -# Development with watch mode -npm run dev - # Build for production npm run build -# Run tests -npm test - # Lint and format npm run lint npm run format +# Type check +npm run typecheck + +# Run tests +npm run test + +# Coverage +npm run test:coverage + # Link globally for testing npm link - -# Create standalone binaries -npm run package ``` --- diff --git a/README.md b/README.md index 7c5dd6c..ab2da0a 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,9 @@ > Prime your repositories for AI-assisted development. +[![CI](https://github.com/pierceboggan/primer/actions/workflows/ci.yml/badge.svg)](https://github.com/pierceboggan/primer/actions/workflows/ci.yml) +[![License](https://img.shields.io/badge/license-MIT-green.svg)](LICENSE) + Primer is a CLI tool that analyzes your codebase and generates `.github/copilot-instructions.md` files to help AI coding assistants understand your project better. It supports single repos, batch processing across organizations, and includes an evaluation framework to measure instruction effectiveness. ![Primer](primer.png) @@ -12,6 +15,7 @@ Primer is a CLI tool that analyzes your codebase and generates `.github/copilot- - **AI-Powered Generation** - Uses the Copilot SDK to analyze your codebase and generate context-aware instructions - **Batch Processing** - Process multiple repos across organizations with a single command - **Evaluation Framework** - Test and measure how well your instructions improve AI responses +- **Readiness Report** - Score AI readiness across key pillars with a fix-first checklist - **GitHub Integration** - Clone repos, create branches, and open PRs automatically - **Interactive TUI** - Beautiful terminal interface built with Ink - **Config Generation** - Generate MCP and VS Code configurations @@ -22,14 +26,31 @@ Primer is a CLI tool that analyzes your codebase and generates `.github/copilot- 2. **GitHub Copilot CLI** - Installed via VS Code's Copilot Chat extension 3. **Copilot CLI Authentication** - Run `copilot` then `/login` to authenticate 4. **GitHub CLI (optional)** - For batch processing and PR creation: `brew install gh && gh auth login` +5. **Azure DevOps PAT (optional)** - For Azure DevOps batch/PR workflows: set `AZURE_DEVOPS_PAT` ## Installation ```bash -# Clone and install +# Install from npm +npm install -g primer +``` + +### Quick Install + +```bash +primer --help +``` + +### Local Development Install + +```bash +# Clone and install dependencies git clone https://github.com/pierceboggan/primer.git cd primer npm install + +# Build and link the local CLI +npm run build ``` ## Usage @@ -40,26 +61,29 @@ The easiest way to get started is with the `init` command: ```bash # Interactive setup for current directory -npx tsx src/index.ts init +primer init # Accept defaults and generate instructions automatically -npx tsx src/index.ts init --yes +primer init --yes # Work with a GitHub repository -npx tsx src/index.ts init --github +primer init --github + +# Work with an Azure DevOps repository +primer init --provider azure ``` ### Interactive Mode (TUI) ```bash # Run TUI in current directory -npx tsx src/index.ts tui +primer tui # Run on a specific repo -npx tsx src/index.ts tui --repo /path/to/repo +primer tui --repo /path/to/repo # Skip the animated intro -npx tsx src/index.ts tui --no-animation +primer tui --no-animation ``` **Keys:** @@ -73,13 +97,13 @@ npx tsx src/index.ts tui --no-animation ```bash # Generate instructions for current directory -npx tsx src/index.ts instructions +primer instructions # Generate for specific repo with custom output -npx tsx src/index.ts instructions --repo /path/to/repo --output ./instructions.md +primer instructions --repo /path/to/repo --output ./instructions.md # Use a specific model -npx tsx src/index.ts instructions --model gpt-5 +primer instructions --model gpt-5 ``` ### Batch Processing @@ -88,10 +112,13 @@ Process multiple repositories across organizations: ```bash # Launch batch TUI -npx tsx src/index.ts batch +primer batch + +# Launch batch TUI for Azure DevOps +primer batch --provider azure # Save results to file -npx tsx src/index.ts batch --output results.json +primer batch --output results.json ``` **Batch TUI Keys:** @@ -105,31 +132,53 @@ npx tsx src/index.ts batch --output results.json ```bash # Analyze current directory -npx tsx src/index.ts analyze +primer analyze # Analyze specific path with JSON output -npx tsx src/index.ts analyze /path/to/repo --json +primer analyze /path/to/repo --json ``` +### Readiness Report + +Assess how ready a repository is for AI agents and get a prioritized checklist of fixes: + +```bash +# Run readiness report in current directory +primer readiness + +# Run readiness report on a specific repo +primer readiness /path/to/repo + +# Output JSON only +primer readiness --json + +# Write JSON report to a file +primer readiness --output readiness.json +``` + +### Examples + +See [examples/README.md](examples/README.md) for quick usage snippets and a sample eval config. + ### Generate Configs Generate configuration files for your repo: ```bash # Generate MCP config -npx tsx src/index.ts generate mcp +primer generate mcp # Generate VS Code settings -npx tsx src/index.ts generate vscode --force +primer generate vscode --force # Generate custom prompts -npx tsx src/index.ts generate prompts +primer generate prompts # Generate agent configs -npx tsx src/index.ts generate agents +primer generate agents # Generate .aiignore file -npx tsx src/index.ts generate aiignore +primer generate aiignore ``` ### Manage Templates @@ -137,7 +186,7 @@ npx tsx src/index.ts generate aiignore View available instruction templates: ```bash -npx tsx src/index.ts templates +primer templates ``` ### Configuration @@ -145,7 +194,7 @@ npx tsx src/index.ts templates View and manage Primer configuration: ```bash -npx tsx src/index.ts config +primer config ``` ### Update @@ -153,7 +202,7 @@ npx tsx src/index.ts config Check for and apply updates: ```bash -npx tsx src/index.ts update +primer update ``` ### Create Pull Requests @@ -162,10 +211,13 @@ Automatically create a PR to add Primer configs to a repository: ```bash # Create PR for a GitHub repo -npx tsx src/index.ts pr owner/repo-name +primer pr owner/repo-name # Use custom branch name -npx tsx src/index.ts pr owner/repo-name --branch primer/custom-branch +primer pr owner/repo-name --branch primer/custom-branch + +# Create PR for an Azure DevOps repo (org/project/repo) +primer pr my-org/my-project/my-repo --provider azure ``` ### Evaluation Framework @@ -174,19 +226,22 @@ Test how well your instructions improve AI responses: ```bash # Create a starter eval config -npx tsx src/index.ts eval --init +primer eval --init # Run evaluation -npx tsx src/index.ts eval primer.eval.json --repo /path/to/repo +primer eval primer.eval.json --repo /path/to/repo # Save results and use specific models -npx tsx src/index.ts eval --output results.json --model gpt-5 --judge-model gpt-5 +primer eval --output results.json --model gpt-5 --judge-model gpt-5 ``` +When `--output` is provided (or `outputPath` is set in the eval config), Primer writes a JSON report that includes per-case metrics and trajectory events, and also generates a companion HTML trajectory viewer next to the JSON file. + Example `primer.eval.json`: ```json { "instructionFile": ".github/copilot-instructions.md", + "outputPath": "eval-results.json", "cases": [ { "id": "project-overview", @@ -264,8 +319,23 @@ primer/ # Type check npx tsc -p tsconfig.json --noEmit -# Run in dev mode -npx tsx src/index.ts +# Lint +npm run lint + +# Format +npm run format + +# Test +npm run test + +# Coverage +npm run test:coverage + +# Build and link the local CLI +npm run build + +# Run locally +primer ``` ## Troubleshooting diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..0f09971 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,7 @@ +# Security Policy + +## Reporting a Vulnerability + +Please report security issues privately by emailing the maintainer or opening a GitHub security advisory. Avoid filing public issues for vulnerabilities. + +We aim to respond within 72 hours and will work with you on remediation and disclosure. diff --git a/eslint.config.js b/eslint.config.js new file mode 100644 index 0000000..309d504 --- /dev/null +++ b/eslint.config.js @@ -0,0 +1,45 @@ +import js from "@eslint/js"; +import tseslint from "@typescript-eslint/eslint-plugin"; +import tsParser from "@typescript-eslint/parser"; +import eslintConfigPrettier from "eslint-config-prettier"; +import importPlugin from "eslint-plugin-import"; +import nPlugin from "eslint-plugin-n"; +import promisePlugin from "eslint-plugin-promise"; + +const sourceGlobs = ["**/*.{ts,tsx,js,jsx}"]; + +export default [ + { + ignores: ["dist/**", "node_modules/**", "coverage/**"] + }, + js.configs.recommended, + { + files: sourceGlobs, + languageOptions: { + parser: tsParser, + parserOptions: { + project: "./tsconfig.json", + sourceType: "module", + ecmaVersion: "latest" + } + }, + plugins: { + "@typescript-eslint": tseslint, + import: importPlugin, + n: nPlugin, + promise: promisePlugin + }, + rules: { + "@typescript-eslint/no-unused-vars": ["warn", { "argsIgnorePattern": "^_" }], + "@typescript-eslint/consistent-type-imports": ["warn", { "prefer": "type-imports" }], + "import/order": [ + "warn", + { + "newlines-between": "always", + "alphabetize": { "order": "asc", "caseInsensitive": true } + } + ] + } + }, + eslintConfigPrettier +]; diff --git a/examples/README.md b/examples/README.md new file mode 100644 index 0000000..1be29f8 --- /dev/null +++ b/examples/README.md @@ -0,0 +1,14 @@ +# Primer Examples + +This folder includes quick examples to help you get started. + +## CLI usage + +- Analyze a repo: primer analyze /path/to/repo +- Generate instructions: primer instructions --repo /path/to/repo +- Run readiness: primer readiness /path/to/repo +- Run evals: primer eval primer.eval.json --repo /path/to/repo + +## Sample eval config + +See primer.eval.json for a starter eval config you can customize. diff --git a/examples/primer.eval.json b/examples/primer.eval.json new file mode 100644 index 0000000..1294fb1 --- /dev/null +++ b/examples/primer.eval.json @@ -0,0 +1,15 @@ +{ + "instructionFile": ".github/copilot-instructions.md", + "cases": [ + { + "id": "project-overview", + "prompt": "Summarize what this project does and list the main entry points.", + "expectation": "Should mention the primary purpose and key files/directories." + }, + { + "id": "build-commands", + "prompt": "How do I build and test this project?", + "expectation": "Should provide the correct build and test commands from package.json or equivalent." + } + ] +} diff --git a/package.json b/package.json index 2a15771..bf64dc8 100644 --- a/package.json +++ b/package.json @@ -2,9 +2,17 @@ "name": "primer", "version": "1.0.0", "description": "", - "main": "index.js", + "main": "dist/index.js", "scripts": { - "test": "echo \"Error: no test specified\" && exit 1" + "build": "tsc && npm link", + "prepare": "tsc", + "lint": "eslint .", + "format": "prettier --write .", + "format:check": "prettier --check .", + "typecheck": "tsc -p tsconfig.json --noEmit", + "test": "vitest run", + "test:watch": "vitest", + "test:coverage": "vitest run --coverage" }, "keywords": [], "author": "", @@ -23,9 +31,19 @@ "devDependencies": { "@types/node": "^25.1.0", "@types/react": "^19.2.10", + "@typescript-eslint/eslint-plugin": "^8.5.0", + "@typescript-eslint/parser": "^8.5.0", + "@vitest/coverage-v8": "^2.1.4", + "eslint": "^9.7.0", + "eslint-config-prettier": "^9.1.0", + "eslint-plugin-import": "^2.30.0", + "eslint-plugin-n": "^17.10.2", + "eslint-plugin-promise": "^7.1.0", + "prettier": "^3.3.3", "tsup": "^8.5.1", "tsx": "^4.21.0", - "typescript": "^5.9.3" + "typescript": "^5.9.3", + "vitest": "^2.1.4" }, "type": "module", "bin": { diff --git a/primer.eval.json b/primer.eval.json index 3b77578..bc1fa99 100644 --- a/primer.eval.json +++ b/primer.eval.json @@ -7,4 +7,4 @@ "expectation": "Mentions the CLI entrypoint in src/index.ts and that this is the Primer CLI." } ] -} +} \ No newline at end of file diff --git a/release-please-config.json b/release-please-config.json new file mode 100644 index 0000000..0ef4665 --- /dev/null +++ b/release-please-config.json @@ -0,0 +1,9 @@ +{ + "release-type": "node", + "packages": { + ".": { + "package-name": "primer", + "changelog-path": "CHANGELOG.md" + } + } +} diff --git a/release-please-manifest.json b/release-please-manifest.json new file mode 100644 index 0000000..37fcefa --- /dev/null +++ b/release-please-manifest.json @@ -0,0 +1,3 @@ +{ + ".": "1.0.0" +} diff --git a/src/cli.ts b/src/cli.ts index 86bcf74..3a667c1 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -10,6 +10,7 @@ import { evalCommand } from "./commands/eval"; import { tuiCommand } from "./commands/tui"; import { instructionsCommand } from "./commands/instructions"; import { batchCommand } from "./commands/batch"; +import { readinessCommand } from "./commands/readiness"; export function runCli(argv: string[]): void { const program = new Command(); @@ -23,6 +24,7 @@ export function runCli(argv: string[]): void { .command("init") .argument("[path]", "Path to a local repository") .option("--github", "Use a GitHub repository") + .option("--provider ", "Repo provider (github|azure)") .option("--yes", "Accept defaults and skip prompts") .option("--force", "Overwrite existing files") .action(initCommand); @@ -42,8 +44,9 @@ export function runCli(argv: string[]): void { program .command("pr") - .argument("[repo]", "GitHub repo in owner/name form") - .option("--branch ", "Branch name", "primer/add-configs") + .argument("[repo]", "Repo identifier (github: owner/name, azure: org/project/repo)") + .option("--branch ", "Branch name") + .option("--provider ", "Repo provider (github|azure)") .action(prCommand); program @@ -69,10 +72,18 @@ export function runCli(argv: string[]): void { .option("--model ", "Model for instructions generation", "gpt-4.1") .action(instructionsCommand); + program + .command("readiness") + .argument("[path]", "Path to a local repository") + .option("--json", "Output JSON") + .option("--output ", "Write JSON report to file") + .action(readinessCommand); + program .command("batch") .description("Batch process multiple repos across orgs") .option("--output ", "Write results JSON to file") + .option("--provider ", "Repo provider (github|azure)", "github") .action(batchCommand); program.command("templates").action(templatesCommand); diff --git a/src/commands/batch.tsx b/src/commands/batch.tsx index eb69267..bce2c3f 100644 --- a/src/commands/batch.tsx +++ b/src/commands/batch.tsx @@ -2,14 +2,41 @@ import React from "react"; import { render } from "ink"; import { BatchTui } from "../ui/BatchTui"; import { getGitHubToken } from "../services/github"; +import { BatchTuiAzure } from "../ui/BatchTuiAzure"; +import { getAzureDevOpsToken } from "../services/azureDevops"; type BatchOptions = { output?: string; + provider?: string; }; export async function batchCommand(options: BatchOptions): Promise { + const provider = options.provider ?? "github"; + if (provider !== "github" && provider !== "azure") { + console.error("Invalid provider. Use github or azure."); + process.exitCode = 1; + return; + } + + if (provider === "azure") { + const token = getAzureDevOpsToken(); + if (!token) { + console.error("Error: Azure DevOps authentication required."); + console.error(""); + console.error("Set a PAT environment variable:"); + console.error(" export AZURE_DEVOPS_PAT="); + process.exitCode = 1; + return; + } + + const { waitUntilExit } = render( + + ); + await waitUntilExit(); + return; + } + const token = await getGitHubToken(); - if (!token) { console.error("Error: GitHub authentication required."); console.error(""); diff --git a/src/commands/eval.ts b/src/commands/eval.ts index b875f97..bee399d 100644 --- a/src/commands/eval.ts +++ b/src/commands/eval.ts @@ -53,7 +53,7 @@ export async function evalCommand(configPathArg: string | undefined, options: Ev const configPath = path.resolve(configPathArg ?? path.join(repoPath, "primer.eval.json")); - const { summary } = await runEval({ + const { summary, viewerPath } = await runEval({ configPath, repoPath, model: options.model ?? "gpt-5", @@ -62,4 +62,7 @@ export async function evalCommand(configPathArg: string | undefined, options: Ev }); console.log(summary); + if (viewerPath) { + console.log(`Trajectory viewer: ${viewerPath}`); + } } diff --git a/src/commands/init.ts b/src/commands/init.ts index 4f22389..2c6ed94 100644 --- a/src/commands/init.ts +++ b/src/commands/init.ts @@ -4,21 +4,38 @@ import { checkbox, select } from "@inquirer/prompts"; import { analyzeRepo } from "../services/analyzer"; import { generateConfigs } from "../services/generator"; import { GitHubRepo, listAccessibleRepos } from "../services/github"; -import { cloneRepo, isGitRepo } from "../services/git"; +import { + AzureDevOpsOrg, + AzureDevOpsProject, + AzureDevOpsRepo, + getAzureDevOpsToken, + listOrganizations, + listProjects, + listRepos +} from "../services/azureDevops"; +import { buildAuthedUrl, cloneRepo, isGitRepo } from "../services/git"; import { generateCopilotInstructions } from "../services/instructions"; import { ensureDir } from "../utils/fs"; import { prettyPrintSummary } from "../utils/logger"; type InitOptions = { github?: boolean; + provider?: string; yes?: boolean; force?: boolean; }; export async function initCommand(repoPathArg: string | undefined, options: InitOptions): Promise { let repoPath = path.resolve(repoPathArg ?? process.cwd()); + const provider = options.provider ?? (options.github ? "github" : undefined); - if (options.github) { + if (provider && provider !== "github" && provider !== "azure") { + console.error("Invalid provider. Use github or azure."); + process.exitCode = 1; + return; + } + + if (provider === "github") { const token = process.env.GITHUB_TOKEN ?? process.env.GH_TOKEN; if (!token) { console.error("Set GITHUB_TOKEN or GH_TOKEN to use GitHub mode."); @@ -50,6 +67,70 @@ export async function initCommand(repoPathArg: string | undefined, options: Init await cloneRepo(selection.cloneUrl, repoPath); } } + + if (provider === "azure") { + const token = getAzureDevOpsToken(); + if (!token) { + console.error("Set AZURE_DEVOPS_PAT (or AZDO_PAT) to use Azure DevOps mode."); + process.exitCode = 1; + return; + } + + const orgs = await listOrganizations(token); + if (orgs.length === 0) { + console.error("No Azure DevOps organizations found."); + process.exitCode = 1; + return; + } + + const orgSelection = await select({ + message: "Choose an Azure DevOps organization", + choices: orgs.map((org) => ({ + name: org.name, + value: org + })) + }); + + const projects = await listProjects(token, orgSelection.name); + if (projects.length === 0) { + console.error("No Azure DevOps projects found."); + process.exitCode = 1; + return; + } + + const projectSelection = await select({ + message: "Choose an Azure DevOps project", + choices: projects.map((project) => ({ + name: project.name, + value: project + })) + }); + + const repos = await listRepos(token, orgSelection.name, projectSelection.name); + if (repos.length === 0) { + console.error("No Azure DevOps repositories found."); + process.exitCode = 1; + return; + } + + const repoSelection = await select({ + message: "Choose a repository", + choices: repos.map((repo) => ({ + name: `${repo.name}${repo.isPrivate ? " (private)" : ""}`, + value: repo + })) + }); + + const cacheRoot = path.join(process.cwd(), ".primer-cache"); + repoPath = path.join(cacheRoot, orgSelection.name, projectSelection.name, repoSelection.name); + await ensureDir(repoPath); + + const hasGit = await isGitRepo(repoPath); + if (!hasGit) { + const authedUrl = buildAuthedUrl(repoSelection.cloneUrl, token, "azure"); + await cloneRepo(authedUrl, repoPath); + } + } const analysis = await analyzeRepo(repoPath); prettyPrintSummary(analysis); diff --git a/src/commands/pr.ts b/src/commands/pr.ts index d8ce79b..7050840 100644 --- a/src/commands/pr.ts +++ b/src/commands/pr.ts @@ -1,21 +1,88 @@ import path from "path"; +import fs from "fs/promises"; import { analyzeRepo } from "../services/analyzer"; import { generateConfigs } from "../services/generator"; import { createPullRequest, getRepo } from "../services/github"; -import { checkoutBranch, cloneRepo, commitAll, isGitRepo, pushBranch } from "../services/git"; +import { generateCopilotInstructions } from "../services/instructions"; +import { + createPullRequest as createAzurePullRequest, + getAzureDevOpsToken, + getRepo as getAzureRepo +} from "../services/azureDevops"; +import { buildAuthedUrl, checkoutBranch, cloneRepo, commitAll, isGitRepo, pushBranch } from "../services/git"; import { ensureDir } from "../utils/fs"; type PrOptions = { branch?: string; + provider?: string; }; export async function prCommand(repo: string | undefined, options: PrOptions): Promise { + const provider = options.provider ?? "github"; + if (provider !== "github" && provider !== "azure") { + console.error("Invalid provider. Use github or azure."); + process.exitCode = 1; + return; + } + if (!repo) { - console.error("Provide a repo in owner/name form."); + console.error("Provide a repo identifier (github: owner/name, azure: org/project/repo)."); process.exitCode = 1; return; } + if (provider === "azure") { + const token = getAzureDevOpsToken(); + if (!token) { + console.error("Set AZURE_DEVOPS_PAT (or AZDO_PAT) to use Azure DevOps PR automation."); + process.exitCode = 1; + return; + } + + const [organization, project, name] = repo.split("/"); + if (!organization || !project || !name) { + console.error("Invalid Azure DevOps repo format. Use org/project/repo."); + process.exitCode = 1; + return; + } + + const repoInfo = await getAzureRepo(token, organization, project, name); + const cacheRoot = path.join(process.cwd(), ".primer-cache"); + const repoPath = path.join(cacheRoot, organization, project, name); + await ensureDir(repoPath); + + if (!(await isGitRepo(repoPath))) { + const authedUrl = buildAuthedUrl(repoInfo.cloneUrl, token, "azure"); + await cloneRepo(authedUrl, repoPath); + } + + const branch = options.branch ?? "primer/add-instructions"; + await checkoutBranch(repoPath, branch); + + const instructions = await generateCopilotInstructions({ repoPath, model: "gpt-4.1" }); + const instructionsPath = path.join(repoPath, ".github", "copilot-instructions.md"); + await ensureDir(path.dirname(instructionsPath)); + await fs.writeFile(instructionsPath, instructions, "utf8"); + + await commitAll(repoPath, "chore: add copilot instructions via Primer"); + await pushBranch(repoPath, branch, token, "azure"); + + const prUrl = await createAzurePullRequest({ + token, + organization, + project, + repoId: repoInfo.id, + repoName: repoInfo.name, + title: "🤖 Add Copilot instructions via Primer", + body: buildInstructionsPrBody(), + sourceBranch: branch, + targetBranch: repoInfo.defaultBranch + }); + + console.log(`Created PR: ${prUrl}`); + return; + } + const token = process.env.GITHUB_TOKEN ?? process.env.GH_TOKEN; if (!token) { console.error("Set GITHUB_TOKEN or GH_TOKEN to use PR automation."); @@ -89,3 +156,29 @@ function buildPrBody(): string { "*Generated by Primer*" ].join("\n"); } + +function buildInstructionsPrBody(): string { + return [ + "## 🤖 Copilot Instructions Added", + "", + "This PR adds a `.github/copilot-instructions.md` file to help GitHub Copilot understand this codebase better.", + "", + "### What's Included", + "", + "The instructions file contains:", + "- Project overview and architecture", + "- Tech stack and conventions", + "- Build/test commands", + "- Key directories and files", + "", + "### Benefits", + "", + "With these instructions, Copilot will:", + "- Generate more contextually-aware code suggestions", + "- Follow project-specific patterns and conventions", + "- Understand the codebase structure", + "", + "---", + "*Generated by [Primer](https://github.com/pierceboggan/primer) - Prime your repos for AI*" + ].join("\n"); +} diff --git a/src/commands/readiness.ts b/src/commands/readiness.ts new file mode 100644 index 0000000..c2b65fa --- /dev/null +++ b/src/commands/readiness.ts @@ -0,0 +1,120 @@ +import path from "path"; +import fs from "fs/promises"; +import chalk from "chalk"; +import { + ReadinessReport, + ReadinessCriterionResult, + runReadinessReport +} from "../services/readiness"; + +type ReadinessOptions = { + json?: boolean; + output?: string; +}; + +export async function readinessCommand(repoPathArg: string | undefined, options: ReadinessOptions): Promise { + const repoPath = path.resolve(repoPathArg ?? process.cwd()); + const report = await runReadinessReport({ repoPath }); + + if (options.output) { + const outputPath = path.resolve(options.output); + await fs.writeFile(outputPath, JSON.stringify(report, null, 2), "utf8"); + } + + if (options.json) { + console.log(JSON.stringify(report, null, 2)); + return; + } + + printReadinessChecklist(report); +} + +function printReadinessChecklist(report: ReadinessReport): void { + console.log(chalk.bold("Readiness report")); + console.log(`- Repo: ${report.repoPath}`); + console.log(`- Monorepo: ${report.isMonorepo ? "yes" : "no"}${report.apps.length ? ` (${report.apps.length} apps)` : ""}`); + console.log(`- Level: ${report.achievedLevel || 1} (${levelName(report.achievedLevel || 1)})`); + + console.log(chalk.bold("\nPillars")); + for (const pillar of report.pillars) { + const rate = formatPercent(pillar.passRate); + const icon = pillar.passRate >= 0.8 ? chalk.green("●") : chalk.yellow("●"); + console.log(`${icon} ${pillar.name}: ${pillar.passed}/${pillar.total} (${rate})`); + } + + console.log(chalk.bold("\nFix first")); + const fixes = rankFixes(report.criteria); + if (!fixes.length) { + console.log(chalk.green("✔ No failing criteria detected.")); + } else { + for (const fix of fixes) { + const impact = colorImpact(fix.impact); + const effort = colorEffort(fix.effort); + const scope = fix.scope === "app" ? "app" : "repo"; + const detail = fix.appSummary + ? ` (${fix.appSummary.passed}/${fix.appSummary.total} apps)` + : ""; + console.log(`- ${impact} impact / ${effort} effort • ${fix.title}${detail} [${scope}]`); + if (fix.reason) { + console.log(` ${chalk.dim(fix.reason)}`); + } + if (fix.appFailures?.length) { + console.log(` ${chalk.dim(`Apps: ${fix.appFailures.join(", ")}`)}`); + } + } + } + + if (report.extras.length) { + console.log(chalk.bold("\nAI readiness extras")); + for (const extra of report.extras) { + const icon = extra.status === "pass" ? chalk.green("✔") : chalk.red("✖"); + console.log(`${icon} ${extra.title}`); + } + } +} + +function rankFixes(criteria: ReadinessCriterionResult[]): ReadinessCriterionResult[] { + return criteria + .filter((criterion) => criterion.status === "fail") + .sort((a, b) => { + const impactDelta = impactWeight(b.impact) - impactWeight(a.impact); + if (impactDelta !== 0) return impactDelta; + return effortWeight(a.effort) - effortWeight(b.effort); + }); +} + +function impactWeight(value: "high" | "medium" | "low"): number { + if (value === "high") return 3; + if (value === "medium") return 2; + return 1; +} + +function effortWeight(value: "low" | "medium" | "high"): number { + if (value === "low") return 1; + if (value === "medium") return 2; + return 3; +} + +function colorImpact(value: "high" | "medium" | "low"): string { + if (value === "high") return chalk.red("High"); + if (value === "medium") return chalk.yellow("Medium"); + return chalk.green("Low"); +} + +function colorEffort(value: "low" | "medium" | "high"): string { + if (value === "high") return chalk.red("High"); + if (value === "medium") return chalk.yellow("Medium"); + return chalk.green("Low"); +} + +function formatPercent(value: number): string { + return `${Math.round(value * 100)}%`; +} + +function levelName(level: number): string { + if (level === 2) return "Documented"; + if (level === 3) return "Standardized"; + if (level === 4) return "Optimized"; + if (level === 5) return "Autonomous"; + return "Functional"; +} \ No newline at end of file diff --git a/src/services/__tests__/analyzer.test.ts b/src/services/__tests__/analyzer.test.ts new file mode 100644 index 0000000..724678f --- /dev/null +++ b/src/services/__tests__/analyzer.test.ts @@ -0,0 +1,31 @@ +import fs from "fs/promises"; +import os from "os"; +import path from "path"; +import { describe, expect, it } from "vitest"; + +import { analyzeRepo } from "../analyzer"; + +describe("analyzeRepo", () => { + it("detects TypeScript and npm workspace", async () => { + const repoPath = await fs.mkdtemp(path.join(os.tmpdir(), "primer-test-")); + const packageJson = { + name: "demo", + workspaces: ["packages/*"], + dependencies: { react: "^19.0.0" } + }; + + await fs.writeFile(path.join(repoPath, "package.json"), JSON.stringify(packageJson, null, 2)); + await fs.writeFile(path.join(repoPath, "tsconfig.json"), "{}", "utf8"); + await fs.mkdir(path.join(repoPath, "packages", "app"), { recursive: true }); + await fs.writeFile( + path.join(repoPath, "packages", "app", "package.json"), + JSON.stringify({ name: "app", scripts: { build: "tsc" } }, null, 2) + ); + + const result = await analyzeRepo(repoPath); + + expect(result.languages).toContain("TypeScript"); + expect(result.workspaceType).toBe("npm"); + expect(result.apps?.length).toBe(1); + }); +}); diff --git a/src/services/analyzer.ts b/src/services/analyzer.ts index bd1b28a..a9e0566 100644 --- a/src/services/analyzer.ts +++ b/src/services/analyzer.ts @@ -1,13 +1,26 @@ import fs from "fs/promises"; import path from "path"; +import fg from "fast-glob"; import { isGitRepo } from "./git"; +export type RepoApp = { + name: string; + path: string; + packageJsonPath: string; + scripts: Record; + hasTsConfig: boolean; +}; + export type RepoAnalysis = { path: string; isGitRepo: boolean; languages: string[]; frameworks: string[]; packageManager?: string; + isMonorepo?: boolean; + workspaceType?: "npm" | "pnpm" | "yarn"; + workspacePatterns?: string[]; + apps?: RepoApp[]; }; const PACKAGE_MANAGERS: Array<{ file: string; name: string }> = [ @@ -41,15 +54,27 @@ export async function analyzeRepo(repoPath: string): Promise { analysis.packageManager = await detectPackageManager(repoPath, files); + let rootPackageJson: Record | undefined; + if (hasPackageJson) { - const packageJson = await readJson(path.join(repoPath, "package.json")); + rootPackageJson = await readJson(path.join(repoPath, "package.json")); const deps = Object.keys({ - ...(packageJson?.dependencies ?? {}), - ...(packageJson?.devDependencies ?? {}) + ...(rootPackageJson?.dependencies ?? {}), + ...(rootPackageJson?.devDependencies ?? {}) }); analysis.frameworks.push(...detectFrameworks(deps, files)); } + const workspace = await detectWorkspace(repoPath, files, rootPackageJson); + if (workspace) { + analysis.workspaceType = workspace.type; + analysis.workspacePatterns = workspace.patterns; + } + + const apps = await resolveWorkspaceApps(repoPath, workspace?.patterns ?? [], rootPackageJson); + analysis.apps = apps; + analysis.isMonorepo = apps.length > 1; + analysis.languages = unique(analysis.languages); analysis.frameworks = unique(analysis.frameworks); @@ -99,6 +124,117 @@ async function readJson(filePath: string): Promise | und } } +type WorkspaceConfig = { + type: "npm" | "pnpm" | "yarn"; + patterns: string[]; +}; + +async function detectWorkspace( + repoPath: string, + files: string[], + packageJson?: Record +): Promise { + if (files.includes("pnpm-workspace.yaml")) { + const patterns = await readPnpmWorkspace(path.join(repoPath, "pnpm-workspace.yaml")); + if (patterns.length) return { type: "pnpm", patterns }; + } + + const workspaces = packageJson?.workspaces; + if (Array.isArray(workspaces)) { + return { type: files.includes("yarn.lock") ? "yarn" : "npm", patterns: workspaces.map(String) }; + } + + if (workspaces && typeof workspaces === "object") { + const packages = (workspaces as { packages?: unknown }).packages; + if (Array.isArray(packages)) { + return { type: files.includes("yarn.lock") ? "yarn" : "npm", patterns: packages.map(String) }; + } + } + + return undefined; +} + +async function readPnpmWorkspace(filePath: string): Promise { + try { + const raw = await fs.readFile(filePath, "utf8"); + const lines = raw.split(/\r?\n/u); + const patterns: string[] = []; + let inPackages = false; + for (const line of lines) { + if (!inPackages && /^\s*packages\s*:/u.test(line)) { + inPackages = true; + continue; + } + if (inPackages) { + const match = line.match(/^\s*-\s*(.+)$/u); + if (match?.[1]) { + patterns.push(match[1].trim().replace(/^['"]|['"]$/gu, "")); + continue; + } + if (/^\S/u.test(line)) break; + } + } + return patterns; + } catch { + return []; + } +} + +async function resolveWorkspaceApps( + repoPath: string, + patterns: string[], + rootPackageJson?: Record +): Promise { + const workspacePatterns = patterns + .map((pattern) => pattern.replace(/\\/gu, "/")) + .map((pattern) => (pattern.endsWith("package.json") ? pattern : path.posix.join(pattern, "package.json"))); + + const packageJsonPaths = workspacePatterns.length + ? await fg(workspacePatterns, { cwd: repoPath, absolute: true, onlyFiles: true, dot: false }) + : []; + + if (!packageJsonPaths.length && rootPackageJson) { + const rootPath = path.join(repoPath, "package.json"); + return [await buildRepoApp(repoPath, rootPath, rootPackageJson)]; + } + + const apps = await Promise.all( + packageJsonPaths.map(async (pkgPath) => { + const pkg = await readJson(pkgPath); + return buildRepoApp(path.dirname(pkgPath), pkgPath, pkg); + }) + ); + + return apps.filter(Boolean) as RepoApp[]; +} + +async function buildRepoApp( + appPath: string, + packageJsonPath: string, + packageJson?: Record +): Promise { + const scripts = (packageJson?.scripts ?? {}) as Record; + const name = typeof packageJson?.name === "string" ? packageJson.name : path.basename(appPath); + const hasTsConfig = await fileExists(path.join(appPath, "tsconfig.json")); + + return { + name, + path: appPath, + packageJsonPath, + scripts, + hasTsConfig + }; +} + +async function fileExists(filePath: string): Promise { + try { + await fs.access(filePath); + return true; + } catch { + return false; + } +} + function unique(items: T[]): T[] { return Array.from(new Set(items)); } diff --git a/src/services/azureDevops.ts b/src/services/azureDevops.ts new file mode 100644 index 0000000..bebe975 --- /dev/null +++ b/src/services/azureDevops.ts @@ -0,0 +1,235 @@ +type AzureDevOpsProfileResponse = { + id: string; + displayName: string; +}; + +type AzureDevOpsAccountResponse = { + accountId: string; + accountName: string; + accountUri: string; +}; + +type AzureDevOpsListResponse = { + value: T[]; +}; + +type AzureDevOpsProjectResponse = { + id: string; + name: string; + url: string; +}; + +type AzureDevOpsRepoResponse = { + id: string; + name: string; + webUrl: string; + remoteUrl: string; + isPrivate: boolean; + defaultBranch?: string; + project?: { + id: string; + name: string; + }; +}; + +export type AzureDevOpsOrg = { + id: string; + name: string; + url: string; +}; + +export type AzureDevOpsProject = { + id: string; + name: string; + organization: string; + url: string; +}; + +export type AzureDevOpsRepo = { + id: string; + name: string; + organization: string; + project: string; + projectId: string; + webUrl: string; + cloneUrl: string; + isPrivate: boolean; + defaultBranch: string; + hasInstructions?: boolean; +}; + +const PROFILE_URL = "https://app.vssps.visualstudio.com/_apis/profile/profiles/me?api-version=7.1-preview.1"; + +function getAuthHeader(token: string): string { + const encoded = Buffer.from(`:${token}`).toString("base64"); + return `Basic ${encoded}`; +} + +async function adoRequest(url: string, token: string, init?: RequestInit): Promise { + const response = await fetch(url, { + ...init, + headers: { + "Content-Type": "application/json", + Authorization: getAuthHeader(token), + ...(init?.headers ?? {}) + } + }); + + if (!response.ok) { + const text = await response.text(); + throw new Error(`Azure DevOps request failed (${response.status}): ${text}`); + } + + return (await response.json()) as T; +} + +export function getAzureDevOpsToken(): string | null { + return process.env.AZURE_DEVOPS_PAT ?? process.env.AZDO_PAT ?? null; +} + +export async function listOrganizations(token: string): Promise { + const profile = await adoRequest(PROFILE_URL, token); + const accountsUrl = `https://app.vssps.visualstudio.com/_apis/accounts?memberId=${profile.id}&api-version=7.1-preview.1`; + const accounts = await adoRequest>(accountsUrl, token); + + return accounts.value.map((account) => ({ + id: account.accountId, + name: account.accountName, + url: account.accountUri + })); +} + +export async function listProjects(token: string, organization: string): Promise { + const url = `https://dev.azure.com/${organization}/_apis/projects?stateFilter=wellFormed&api-version=7.1-preview.1`; + const response = await adoRequest>(url, token); + + return response.value.map((project) => ({ + id: project.id, + name: project.name, + organization, + url: project.url + })); +} + +export async function listRepos(token: string, organization: string, project: string): Promise { + const url = `https://dev.azure.com/${organization}/${project}/_apis/git/repositories?api-version=7.1-preview.1`; + const response = await adoRequest>(url, token); + + return response.value.map((repo) => ({ + id: repo.id, + name: repo.name, + organization, + project, + projectId: repo.project?.id ?? "", + webUrl: repo.webUrl, + cloneUrl: repo.remoteUrl, + isPrivate: repo.isPrivate, + defaultBranch: repo.defaultBranch ?? "refs/heads/main" + })); +} + +export async function getRepo( + token: string, + organization: string, + project: string, + repo: string +): Promise { + const url = `https://dev.azure.com/${organization}/${project}/_apis/git/repositories/${repo}?api-version=7.1-preview.1`; + const response = await adoRequest(url, token); + + return { + id: response.id, + name: response.name, + organization, + project, + projectId: response.project?.id ?? "", + webUrl: response.webUrl, + cloneUrl: response.remoteUrl, + isPrivate: response.isPrivate, + defaultBranch: response.defaultBranch ?? "refs/heads/main" + }; +} + +function toRefName(branch: string): string { + if (branch.startsWith("refs/")) return branch; + return `refs/heads/${branch}`; +} + +export async function createPullRequest(params: { + token: string; + organization: string; + project: string; + repoId: string; + repoName: string; + title: string; + body: string; + sourceBranch: string; + targetBranch: string; +}): Promise { + const url = `https://dev.azure.com/${params.organization}/${params.project}/_apis/git/repositories/${params.repoId}/pullrequests?api-version=7.1-preview.1`; + const payload = { + title: params.title, + description: params.body, + sourceRefName: toRefName(params.sourceBranch), + targetRefName: toRefName(params.targetBranch) + }; + + const response = await adoRequest<{ pullRequestId: number }>(url, params.token, { + method: "POST", + body: JSON.stringify(payload) + }); + + return `https://dev.azure.com/${params.organization}/${params.project}/_git/${encodeURIComponent( + params.repoName + )}/pullrequest/${response.pullRequestId}`; +} + +export async function checkRepoHasInstructions( + token: string, + organization: string, + project: string, + repoId: string +): Promise { + const url = `https://dev.azure.com/${organization}/${project}/_apis/git/repositories/${repoId}/items?path=/.github/copilot-instructions.md&includeContentMetadata=true&api-version=7.1-preview.1`; + const response = await fetch(url, { + headers: { + Authorization: getAuthHeader(token) + } + }); + + if (response.status === 404) { + return false; + } + + return response.ok; +} + +export async function checkReposForInstructions( + token: string, + repos: AzureDevOpsRepo[], + onProgress?: (checked: number, total: number) => void +): Promise { + const concurrency = 10; + const results: AzureDevOpsRepo[] = []; + let checked = 0; + + for (let i = 0; i < repos.length; i += concurrency) { + const batch = repos.slice(i, i + concurrency); + const checks = await Promise.all( + batch.map(async (repo) => { + const hasInstructions = await checkRepoHasInstructions( + token, + repo.organization, + repo.project, + repo.id + ); + return { ...repo, hasInstructions }; + }) + ); + results.push(...checks); + checked += batch.length; + onProgress?.(checked, repos.length); + } + + return results; +} \ No newline at end of file diff --git a/src/services/evaluator.ts b/src/services/evaluator.ts index 6a30001..adddce6 100644 --- a/src/services/evaluator.ts +++ b/src/services/evaluator.ts @@ -15,6 +15,7 @@ type EvalConfig = { instructionFile?: string; cases: EvalCase[]; systemMessage?: string; + outputPath?: string; }; type EvalRunOptions = { @@ -26,6 +27,40 @@ type EvalRunOptions = { onProgress?: (message: string) => void; }; +type TokenUsage = { + promptTokens?: number; + completionTokens?: number; + totalTokens?: number; +}; + +type ToolCallSummary = { + count: number; + byName: Record; + totalDurationMs: number; +}; + +type AskMetrics = { + durationMs: number; + tokenUsage?: TokenUsage; + toolCalls: ToolCallSummary; +}; + +type EvalMetrics = { + withoutInstructions: AskMetrics; + withInstructions: AskMetrics; + judge: AskMetrics; + totalDurationMs: number; +}; + +type EvalPhase = "withoutInstructions" | "withInstructions" | "judge"; + +type TrajectoryEvent = { + timestampMs: number; + phase: EvalPhase; + type: string; + data?: Record; +}; + export type EvalResult = { id: string; prompt: string; @@ -35,14 +70,18 @@ export type EvalResult = { verdict?: "pass" | "fail" | "unknown"; score?: number; rationale?: string; + metrics?: EvalMetrics; + trajectory?: TrajectoryEvent[]; }; -export async function runEval(options: EvalRunOptions): Promise<{ summary: string; results: EvalResult[] }> { +export async function runEval(options: EvalRunOptions): Promise<{ summary: string; results: EvalResult[]; viewerPath?: string }> { const config = await loadConfig(options.configPath); const instructionFile = config.instructionFile ?? ".github/copilot-instructions.md"; const instructionPath = path.resolve(options.repoPath, instructionFile); const instructionText = await readOptionalFile(instructionPath); const progress = options.onProgress ?? (() => {}); + const outputPath = resolveOutputPath(options.repoPath, options.outputPath, config.outputPath); + const runStartedAt = Date.now(); progress("Starting Copilot SDK..."); const cliPath = await findCopilotCliPath(); @@ -56,19 +95,22 @@ export async function runEval(options: EvalRunOptions): Promise<{ summary: strin for (const [index, testCase] of config.cases.entries()) { const id = testCase.id ?? `case-${index + 1}`; const prompt = buildPrompt(options.repoPath, testCase.prompt); + const caseStartedAt = Date.now(); progress(`Running eval ${index + 1}/${total}: ${id} (without instructions)...`); - const withoutInstructions = await askOnce(client, { + const withoutResult = await askOnce(client, { prompt, model: options.model, - systemMessage: config.systemMessage + systemMessage: config.systemMessage, + phase: "withoutInstructions" }); progress(`Running eval ${index + 1}/${total}: ${id} (with instructions)...`); - const withInstructions = await askOnce(client, { + const withResult = await askOnce(client, { prompt, model: options.model, - systemMessage: [config.systemMessage, instructionText].filter(Boolean).join("\n\n") + systemMessage: [config.systemMessage, instructionText].filter(Boolean).join("\n\n"), + phase: "withInstructions" }); progress(`Running eval ${index + 1}/${total}: ${id} (judging)...`); @@ -76,36 +118,60 @@ export async function runEval(options: EvalRunOptions): Promise<{ summary: strin model: options.judgeModel, prompt: testCase.prompt, expectation: testCase.expectation, - withoutInstructions, - withInstructions + withoutInstructions: withoutResult.content, + withInstructions: withResult.content }); + const metrics: EvalMetrics = { + withoutInstructions: withoutResult.metrics, + withInstructions: withResult.metrics, + judge: judgment.metrics, + totalDurationMs: Date.now() - caseStartedAt + }; + + const trajectory = [ + ...withoutResult.trajectory, + ...withResult.trajectory, + ...judgment.trajectory + ]; + results.push({ id, prompt: testCase.prompt, expectation: testCase.expectation, - withInstructions, - withoutInstructions, - verdict: judgment.verdict, - score: judgment.score, - rationale: judgment.rationale + withInstructions: withResult.content, + withoutInstructions: withoutResult.content, + verdict: judgment.result.verdict, + score: judgment.result.score, + rationale: judgment.result.rationale, + metrics, + trajectory }); - progress(`Eval ${index + 1}/${total}: ${id} → ${judgment.verdict} (score: ${judgment.score})`); + progress(`Eval ${index + 1}/${total}: ${id} → ${judgment.result.verdict} (score: ${judgment.result.score})`); } - if (options.outputPath) { - const output = { - repoPath: options.repoPath, - model: options.model, - judgeModel: options.judgeModel, - results - }; - await fs.writeFile(options.outputPath, JSON.stringify(output, null, 2), "utf8"); + const runFinishedAt = Date.now(); + const output = { + repoPath: options.repoPath, + model: options.model, + judgeModel: options.judgeModel, + runMetrics: { + startedAt: new Date(runStartedAt).toISOString(), + finishedAt: new Date(runFinishedAt).toISOString(), + durationMs: runFinishedAt - runStartedAt + }, + results + }; + let viewerPath: string | undefined; + if (outputPath) { + await fs.writeFile(outputPath, JSON.stringify(output, null, 2), "utf8"); + viewerPath = buildViewerPath(outputPath); + await fs.writeFile(viewerPath, buildTrajectoryViewerHtml(output), "utf8"); } - const summary = formatSummary(results); - return { summary, results }; + const summary = formatSummary(results, runFinishedAt - runStartedAt); + return { summary, results, viewerPath }; } finally { await client.stop(); } @@ -115,12 +181,19 @@ type AskOptions = { prompt: string; model: string; systemMessage?: string; + phase: EvalPhase; +}; + +type AskResult = { + content: string; + metrics: AskMetrics; + trajectory: TrajectoryEvent[]; }; async function askOnce( client: { createSession: (config?: Record) => Promise }, options: AskOptions -): Promise { +): Promise { const session = await client.createSession({ model: options.model, streaming: true, @@ -131,7 +204,10 @@ async function askOnce( }); let content = ""; + const telemetry = createTelemetry(options.phase); + const startedAt = Date.now(); session.on((event: { type: string; data?: Record }) => { + captureTelemetryEvent(event, telemetry); if (event.type === "assistant.message_delta") { const delta = event.data?.deltaContent as string | undefined; if (delta) content += delta; @@ -140,7 +216,16 @@ async function askOnce( await session.sendAndWait({ prompt: options.prompt }, 120000); await session.destroy(); - return content.trim(); + const finishedAt = Date.now(); + return { + content: content.trim(), + metrics: { + durationMs: finishedAt - startedAt, + tokenUsage: normalizeTokenUsage(telemetry.tokenUsage), + toolCalls: telemetry.toolCalls + }, + trajectory: telemetry.trajectory + }; } type JudgeOptions = { @@ -160,7 +245,7 @@ type JudgeResult = { async function judge( client: { createSession: (config?: Record) => Promise }, options: JudgeOptions -): Promise { +): Promise<{ result: JudgeResult; metrics: AskMetrics; trajectory: TrajectoryEvent[] }> { const session = await client.createSession({ model: options.model, streaming: true, @@ -171,7 +256,10 @@ async function judge( }); let content = ""; + const telemetry = createTelemetry("judge"); + const startedAt = Date.now(); session.on((event: { type: string; data?: Record }) => { + captureTelemetryEvent(event, telemetry); if (event.type === "assistant.message_delta") { const delta = event.data?.deltaContent as string | undefined; if (delta) content += delta; @@ -195,7 +283,16 @@ async function judge( await session.sendAndWait({ prompt }, 120000); await session.destroy(); - return parseJudge(content); + const finishedAt = Date.now(); + return { + result: parseJudge(content), + metrics: { + durationMs: finishedAt - startedAt, + tokenUsage: normalizeTokenUsage(telemetry.tokenUsage), + toolCalls: telemetry.toolCalls + }, + trajectory: telemetry.trajectory + }; } function parseJudge(content: string): JudgeResult { @@ -271,14 +368,18 @@ function buildPrompt(repoPath: string, userPrompt: string): string { ].join("\n"); } -function formatSummary(results: EvalResult[]): string { +function formatSummary(results: EvalResult[], runDurationMs: number): string { const total = results.length; const passed = results.filter((r) => r.verdict === "pass").length; const failed = results.filter((r) => r.verdict === "fail").length; const unknown = results.filter((r) => r.verdict === "unknown").length; + const totalUsage = aggregateTokenUsage(results); + const hasUsage = Boolean(totalUsage.promptTokens || totalUsage.completionTokens || totalUsage.totalTokens); const lines = [ - `Eval results: ${passed}/${total} pass, ${failed} fail, ${unknown} unknown.` + `Eval results: ${passed}/${total} pass, ${failed} fail, ${unknown} unknown.`, + `Runtime: ${formatDuration(runDurationMs)}.`, + hasUsage ? `Token usage: ${formatTokenUsage(totalUsage)}.` : "Token usage: unavailable." ]; for (const result of results) { @@ -289,3 +390,305 @@ function formatSummary(results: EvalResult[]): string { return `\n${lines.join("\n")}`; } + +type TelemetryCollector = { + trajectory: TrajectoryEvent[]; + tokenUsage: TokenUsage; + toolCalls: ToolCallSummary; + toolCallMap: Map; + phase: EvalPhase; +}; + +function createTelemetry(phase: EvalPhase): TelemetryCollector { + return { + trajectory: [], + tokenUsage: {}, + toolCalls: { count: 0, byName: {}, totalDurationMs: 0 }, + toolCallMap: new Map(), + phase + }; +} + +function captureTelemetryEvent( + event: { type: string; data?: Record }, + telemetry: TelemetryCollector +): void { + const timestampMs = Date.now(); + telemetry.trajectory.push({ + timestampMs, + phase: telemetry.phase, + type: event.type, + data: sanitizeEventData(event.data) + }); + + if (event.type === "tool.execution_start") { + const toolName = (event.data?.toolName as string | undefined) ?? "unknown"; + const toolId = resolveToolId(event.data, toolName, telemetry.toolCallMap.size); + telemetry.toolCallMap.set(toolId, { name: toolName, startMs: timestampMs }); + telemetry.toolCalls.count += 1; + telemetry.toolCalls.byName[toolName] = (telemetry.toolCalls.byName[toolName] ?? 0) + 1; + } else if (event.type === "tool.execution_finish" || event.type === "tool.execution_error") { + const toolName = (event.data?.toolName as string | undefined) ?? "unknown"; + const toolId = resolveToolId(event.data, toolName, telemetry.toolCallMap.size); + const entry = telemetry.toolCallMap.get(toolId) ?? findLatestToolByName(telemetry.toolCallMap, toolName); + if (entry) { + const durationMs = timestampMs - entry.startMs; + telemetry.toolCalls.totalDurationMs += durationMs; + telemetry.toolCallMap.delete(toolId); + } + } + + const usage = extractTokenUsage(event.data); + if (usage) { + telemetry.tokenUsage = mergeTokenUsage(telemetry.tokenUsage, usage); + } +} + +function resolveToolId( + data: Record | undefined, + toolName: string, + index: number +): string { + const rawId = data?.executionId ?? data?.toolCallId ?? data?.callId ?? data?.id; + if (typeof rawId === "string" || typeof rawId === "number") { + return String(rawId); + } + return `${toolName}-${index + 1}`; +} + +function findLatestToolByName( + map: Map, + toolName: string +): { name?: string; startMs: number } | undefined { + const entries = Array.from(map.values()).filter((entry) => entry.name === toolName); + return entries.at(-1); +} + +function extractTokenUsage(data: Record | undefined): TokenUsage | null { + if (!data) return null; + const usage = (data.usage ?? data.tokenUsage ?? data.tokens) as Record | undefined; + const promptTokens = getNumber(usage?.prompt_tokens ?? usage?.promptTokens ?? data.promptTokens ?? data.inputTokens); + const completionTokens = getNumber(usage?.completion_tokens ?? usage?.completionTokens ?? data.completionTokens ?? data.outputTokens); + const totalTokens = getNumber(usage?.total_tokens ?? usage?.totalTokens ?? data.totalTokens); + + if (promptTokens == null && completionTokens == null && totalTokens == null) { + return null; + } + + return { + promptTokens: promptTokens ?? undefined, + completionTokens: completionTokens ?? undefined, + totalTokens: totalTokens ?? undefined + }; +} + +function getNumber(value: unknown): number | null { + if (typeof value === "number" && Number.isFinite(value)) return value; + if (typeof value === "string" && value.trim()) { + const parsed = Number(value); + return Number.isFinite(parsed) ? parsed : null; + } + return null; +} + +function mergeTokenUsage(existing: TokenUsage, next: TokenUsage): TokenUsage { + return { + promptTokens: Math.max(existing.promptTokens ?? 0, next.promptTokens ?? 0) || undefined, + completionTokens: Math.max(existing.completionTokens ?? 0, next.completionTokens ?? 0) || undefined, + totalTokens: Math.max(existing.totalTokens ?? 0, next.totalTokens ?? 0) || undefined + }; +} + +function normalizeTokenUsage(usage: TokenUsage): TokenUsage | undefined { + if (!usage.promptTokens && !usage.completionTokens && !usage.totalTokens) return undefined; + return usage; +} + +function aggregateTokenUsage(results: EvalResult[]): TokenUsage { + const total: TokenUsage = { promptTokens: 0, completionTokens: 0, totalTokens: 0 }; + for (const result of results) { + const metrics = result.metrics; + if (!metrics) continue; + const usages = [metrics.withoutInstructions.tokenUsage, metrics.withInstructions.tokenUsage, metrics.judge.tokenUsage]; + for (const usage of usages) { + if (!usage) continue; + total.promptTokens = (total.promptTokens ?? 0) + (usage.promptTokens ?? 0); + total.completionTokens = (total.completionTokens ?? 0) + (usage.completionTokens ?? 0); + total.totalTokens = (total.totalTokens ?? 0) + (usage.totalTokens ?? 0); + } + } + return total; +} + +function formatDuration(durationMs: number): string { + const seconds = Math.round(durationMs / 100) / 10; + if (seconds < 60) return `${seconds}s`; + const minutes = Math.floor(seconds / 60); + const remaining = Math.round((seconds % 60) * 10) / 10; + return `${minutes}m ${remaining}s`; +} + +function formatTokenUsage(usage: TokenUsage): string { + const prompt = usage.promptTokens ?? 0; + const completion = usage.completionTokens ?? 0; + const total = usage.totalTokens ?? prompt + completion; + return `prompt ${prompt}, completion ${completion}, total ${total}`; +} + +function resolveOutputPath(repoPath: string, override?: string, configValue?: string): string | undefined { + const chosen = override ?? configValue; + if (!chosen) return undefined; + return path.isAbsolute(chosen) ? chosen : path.resolve(repoPath, chosen); +} + +function buildViewerPath(outputPath: string): string { + if (outputPath.endsWith(".json")) { + return outputPath.replace(/\.json$/u, ".html"); + } + return `${outputPath}.html`; +} + +function buildTrajectoryViewerHtml(data: Record): string { + const serialized = JSON.stringify(data).replace(/ + + + + + Primer Eval Trajectory + + + +

Primer Eval Trajectory

+
+
+
+
+
+ + +`; +} + +function sanitizeEventData(data: Record | undefined): Record | undefined { + if (!data) return undefined; + const sanitized: Record = {}; + for (const [key, value] of Object.entries(data)) { + if (key === "deltaContent" && typeof value === "string") { + sanitized.deltaChars = value.length; + sanitized.deltaPreview = value.slice(0, 120); + continue; + } + sanitized[key] = sanitizeValue(value, 0); + } + return sanitized; +} + +function sanitizeValue(value: unknown, depth: number): unknown { + if (depth > 4) return "[depth-limit]"; + if (typeof value === "string") { + return value.length > 2000 ? `${value.slice(0, 2000)}…` : value; + } + if (Array.isArray(value)) { + return value.slice(0, 50).map((entry) => sanitizeValue(entry, depth + 1)); + } + if (value && typeof value === "object") { + const obj: Record = {}; + for (const [key, entry] of Object.entries(value as Record)) { + obj[key] = sanitizeValue(entry, depth + 1); + } + return obj; + } + return value; +} diff --git a/src/services/git.ts b/src/services/git.ts index a8b1e79..5eceb3a 100644 --- a/src/services/git.ts +++ b/src/services/git.ts @@ -65,6 +65,8 @@ export async function commitAll(repoPath: string, message: string): Promise { +export function buildAuthedUrl(url: string, token: string, provider: AuthProvider): string { + const normalizedUrl = normalizeGitUrl(url); + if (!normalizedUrl.startsWith("https://")) return normalizedUrl; + if (provider === "azure") { + return normalizedUrl.replace("https://", `https://pat:${token}@`); + } + return normalizedUrl.replace("https://", `https://x-access-token:${token}@`); +} + +export async function pushBranch( + repoPath: string, + branch: string, + token?: string, + provider: AuthProvider = "github" +): Promise { const git = simpleGit(repoPath); if (token) { @@ -85,7 +103,7 @@ export async function pushBranch(repoPath: string, branch: string, token?: strin const remoteUrl = (await git.remote(["get-url", "origin"])) ?? ""; const normalizedUrl = normalizeGitUrl(remoteUrl); if (normalizedUrl.startsWith("https://")) { - const authedUrl = normalizedUrl.replace("https://", `https://x-access-token:${token}@`); + const authedUrl = buildAuthedUrl(normalizedUrl, token, provider); await git.remote(["set-url", "origin", authedUrl]); try { await git.push(["-u", "origin", branch]); diff --git a/src/services/readiness.ts b/src/services/readiness.ts new file mode 100644 index 0000000..ea55e77 --- /dev/null +++ b/src/services/readiness.ts @@ -0,0 +1,648 @@ +import fs from "fs/promises"; +import path from "path"; +import { analyzeRepo, RepoApp, RepoAnalysis } from "./analyzer"; + +export type ReadinessPillar = + | "style-validation" + | "build-system" + | "testing" + | "documentation" + | "dev-environment" + | "code-quality" + | "observability" + | "security-governance"; + +export type ReadinessScope = "repo" | "app"; + +export type ReadinessStatus = "pass" | "fail" | "skip"; + +export type ReadinessCriterionResult = { + id: string; + title: string; + pillar: ReadinessPillar; + level: number; + scope: ReadinessScope; + impact: "high" | "medium" | "low"; + effort: "low" | "medium" | "high"; + status: ReadinessStatus; + reason?: string; + evidence?: string[]; + passRate?: number; + appSummary?: { passed: number; total: number }; + appFailures?: string[]; +}; + +export type ReadinessExtraResult = { + id: string; + title: string; + status: ReadinessStatus; + reason?: string; +}; + +export type ReadinessPillarSummary = { + id: ReadinessPillar; + name: string; + passed: number; + total: number; + passRate: number; +}; + +export type ReadinessLevelSummary = { + level: number; + name: string; + passed: number; + total: number; + passRate: number; + achieved: boolean; +}; + +export type ReadinessReport = { + repoPath: string; + generatedAt: string; + isMonorepo: boolean; + apps: Array<{ name: string; path: string }>; + pillars: ReadinessPillarSummary[]; + levels: ReadinessLevelSummary[]; + achievedLevel: number; + criteria: ReadinessCriterionResult[]; + extras: ReadinessExtraResult[]; +}; + +type ReadinessOptions = { + repoPath: string; + includeExtras?: boolean; +}; + +type ReadinessContext = { + repoPath: string; + analysis: RepoAnalysis; + apps: RepoApp[]; + rootFiles: string[]; + rootPackageJson?: Record; +}; + +type ReadinessCriterion = { + id: string; + title: string; + pillar: ReadinessPillar; + level: number; + scope: ReadinessScope; + impact: "high" | "medium" | "low"; + effort: "low" | "medium" | "high"; + check: (context: ReadinessContext, app?: RepoApp) => Promise; +}; + +type CheckResult = { + status: ReadinessStatus; + reason?: string; + evidence?: string[]; +}; + +export async function runReadinessReport(options: ReadinessOptions): Promise { + const repoPath = options.repoPath; + const analysis = await analyzeRepo(repoPath); + const rootFiles = await safeReadDir(repoPath); + const rootPackageJson = await readJson(path.join(repoPath, "package.json")); + const apps = analysis.apps?.length ? analysis.apps : []; + + const context: ReadinessContext = { + repoPath, + analysis, + apps, + rootFiles, + rootPackageJson + }; + + const criteria = buildCriteria(); + const criteriaResults: ReadinessCriterionResult[] = []; + + for (const criterion of criteria) { + if (criterion.scope === "repo") { + const result = await criterion.check(context); + criteriaResults.push({ + id: criterion.id, + title: criterion.title, + pillar: criterion.pillar, + level: criterion.level, + scope: criterion.scope, + impact: criterion.impact, + effort: criterion.effort, + status: result.status, + reason: result.reason, + evidence: result.evidence + }); + continue; + } + + const appResults = await Promise.all( + apps.map(async (app) => ({ + app, + result: await criterion.check(context, app) + })) + ); + + if (!appResults.length) { + criteriaResults.push({ + id: criterion.id, + title: criterion.title, + pillar: criterion.pillar, + level: criterion.level, + scope: criterion.scope, + impact: criterion.impact, + effort: criterion.effort, + status: "skip", + reason: "No application packages detected." + }); + continue; + } + + const passed = appResults.filter((entry) => entry.result.status === "pass").length; + const total = appResults.length; + const passRate = total ? passed / total : 0; + const status: ReadinessStatus = passRate >= 0.8 ? "pass" : "fail"; + const failures = appResults + .filter((entry) => entry.result.status !== "pass") + .map((entry) => entry.app.name); + + criteriaResults.push({ + id: criterion.id, + title: criterion.title, + pillar: criterion.pillar, + level: criterion.level, + scope: criterion.scope, + impact: criterion.impact, + effort: criterion.effort, + status, + reason: status === "pass" ? undefined : `Only ${passed}/${total} apps pass this check.`, + passRate, + appSummary: { passed, total }, + appFailures: failures + }); + } + + const pillars = summarizePillars(criteriaResults); + const levels = summarizeLevels(criteriaResults); + const achievedLevel = levels.filter((level) => level.achieved).reduce((acc, level) => Math.max(acc, level.level), 0); + + const extras = options.includeExtras === false ? [] : await runExtras(context); + + return { + repoPath, + generatedAt: new Date().toISOString(), + isMonorepo: analysis.isMonorepo ?? false, + apps: apps.map((app) => ({ name: app.name, path: app.path })), + pillars, + levels, + achievedLevel, + criteria: criteriaResults, + extras + }; +} + +function buildCriteria(): ReadinessCriterion[] { + return [ + { + id: "lint-config", + title: "Linting configured", + pillar: "style-validation", + level: 1, + scope: "repo", + impact: "high", + effort: "low", + check: async (context) => ({ + status: (await hasLintConfig(context.repoPath)) ? "pass" : "fail", + reason: "Missing ESLint/Biome/Prettier configuration.", + evidence: ["eslint.config.js", ".eslintrc", "biome.json", ".prettierrc"] + }) + }, + { + id: "typecheck-config", + title: "Type checking configured", + pillar: "style-validation", + level: 2, + scope: "repo", + impact: "medium", + effort: "low", + check: async (context) => ({ + status: (await hasTypecheckConfig(context.repoPath)) ? "pass" : "fail", + reason: "Missing type checking config (tsconfig or equivalent).", + evidence: ["tsconfig.json", "pyproject.toml", "mypy.ini"] + }) + }, + { + id: "build-script", + title: "Build script present", + pillar: "build-system", + level: 1, + scope: "app", + impact: "high", + effort: "low", + check: async (_context, app) => ({ + status: app?.scripts?.build ? "pass" : "fail", + reason: "Missing build script in package.json." + }) + }, + { + id: "ci-config", + title: "CI workflow configured", + pillar: "build-system", + level: 2, + scope: "repo", + impact: "high", + effort: "medium", + check: async (context) => ({ + status: (await hasGithubWorkflows(context.repoPath)) ? "pass" : "fail", + reason: "Missing .github/workflows CI configuration.", + evidence: [".github/workflows"] + }) + }, + { + id: "test-script", + title: "Test script present", + pillar: "testing", + level: 1, + scope: "app", + impact: "high", + effort: "low", + check: async (_context, app) => ({ + status: app?.scripts?.test ? "pass" : "fail", + reason: "Missing test script in package.json." + }) + }, + { + id: "readme", + title: "README present", + pillar: "documentation", + level: 1, + scope: "repo", + impact: "high", + effort: "low", + check: async (context) => ({ + status: (await hasReadme(context.repoPath)) ? "pass" : "fail", + reason: "Missing README documentation.", + evidence: ["README.md"] + }) + }, + { + id: "contributing", + title: "CONTRIBUTING guide present", + pillar: "documentation", + level: 2, + scope: "repo", + impact: "medium", + effort: "low", + check: async (context) => ({ + status: (await fileExists(path.join(context.repoPath, "CONTRIBUTING.md"))) ? "pass" : "fail", + reason: "Missing CONTRIBUTING.md for contributor workflows." + }) + }, + { + id: "lockfile", + title: "Lockfile present", + pillar: "dev-environment", + level: 1, + scope: "repo", + impact: "high", + effort: "low", + check: async (context) => ({ + status: hasAnyFile(context.rootFiles, ["pnpm-lock.yaml", "yarn.lock", "package-lock.json", "bun.lockb"]) ? "pass" : "fail", + reason: "Missing package manager lockfile." + }) + }, + { + id: "env-example", + title: "Environment example present", + pillar: "dev-environment", + level: 2, + scope: "repo", + impact: "medium", + effort: "low", + check: async (context) => ({ + status: hasAnyFile(context.rootFiles, [".env.example", ".env.sample"]) ? "pass" : "fail", + reason: "Missing .env.example or .env.sample for setup guidance." + }) + }, + { + id: "format-config", + title: "Formatter configured", + pillar: "code-quality", + level: 2, + scope: "repo", + impact: "medium", + effort: "low", + check: async (context) => ({ + status: (await hasFormatterConfig(context.repoPath)) ? "pass" : "fail", + reason: "Missing Prettier/Biome formatting config." + }) + }, + { + id: "codeowners", + title: "CODEOWNERS present", + pillar: "security-governance", + level: 2, + scope: "repo", + impact: "medium", + effort: "low", + check: async (context) => ({ + status: (await hasCodeowners(context.repoPath)) ? "pass" : "fail", + reason: "Missing CODEOWNERS file." + }) + }, + { + id: "license", + title: "LICENSE present", + pillar: "security-governance", + level: 1, + scope: "repo", + impact: "medium", + effort: "low", + check: async (context) => ({ + status: (await hasLicense(context.repoPath)) ? "pass" : "fail", + reason: "Missing LICENSE file." + }) + }, + { + id: "security-policy", + title: "Security policy present", + pillar: "security-governance", + level: 3, + scope: "repo", + impact: "high", + effort: "low", + check: async (context) => ({ + status: (await fileExists(path.join(context.repoPath, "SECURITY.md"))) ? "pass" : "fail", + reason: "Missing SECURITY.md policy." + }) + }, + { + id: "dependabot", + title: "Dependabot configured", + pillar: "security-governance", + level: 3, + scope: "repo", + impact: "medium", + effort: "medium", + check: async (context) => ({ + status: (await fileExists(path.join(context.repoPath, ".github", "dependabot.yml"))) ? "pass" : "fail", + reason: "Missing .github/dependabot.yml configuration." + }) + }, + { + id: "observability", + title: "Observability tooling present", + pillar: "observability", + level: 3, + scope: "repo", + impact: "medium", + effort: "medium", + check: async (context) => { + const deps = await readAllDependencies(context); + const has = deps.some((dep) => ["@opentelemetry/api", "@opentelemetry/sdk", "pino", "winston", "bunyan"].includes(dep)); + return { + status: has ? "pass" : "fail", + reason: "No observability dependencies detected (OpenTelemetry/logging)." + }; + } + } + ]; +} + +async function runExtras(context: ReadinessContext): Promise { + const results: ReadinessExtraResult[] = []; + + results.push({ + id: "agents-doc", + title: "AGENTS.md present", + status: (await fileExists(path.join(context.repoPath, "AGENTS.md"))) ? "pass" : "fail", + reason: "Missing AGENTS.md to guide coding agents." + }); + + results.push({ + id: "pr-template", + title: "Pull request template present", + status: (await hasPullRequestTemplate(context.repoPath)) ? "pass" : "fail", + reason: "Missing PR template for consistent reviews." + }); + + results.push({ + id: "pre-commit", + title: "Pre-commit hooks configured", + status: (await hasPrecommitConfig(context.repoPath)) ? "pass" : "fail", + reason: "Missing pre-commit or Husky configuration for fast feedback." + }); + + results.push({ + id: "architecture-doc", + title: "Architecture guide present", + status: (await hasArchitectureDoc(context.repoPath)) ? "pass" : "fail", + reason: "Missing architecture documentation." + }); + + return results; +} + +function summarizePillars(criteria: ReadinessCriterionResult[]): ReadinessPillarSummary[] { + const pillarNames: Record = { + "style-validation": "Style & Validation", + "build-system": "Build System", + testing: "Testing", + documentation: "Documentation", + "dev-environment": "Dev Environment", + "code-quality": "Code Quality", + observability: "Observability", + "security-governance": "Security & Governance" + }; + + return (Object.keys(pillarNames) as ReadinessPillar[]).map((pillar) => { + const items = criteria.filter((criterion) => criterion.pillar === pillar); + const { passed, total } = countStatus(items); + return { + id: pillar, + name: pillarNames[pillar], + passed, + total, + passRate: total ? passed / total : 0 + }; + }); +} + +function summarizeLevels(criteria: ReadinessCriterionResult[]): ReadinessLevelSummary[] { + const levelNames: Record = { + 1: "Functional", + 2: "Documented", + 3: "Standardized", + 4: "Optimized", + 5: "Autonomous" + }; + + const summaries: ReadinessLevelSummary[] = []; + for (let level = 1; level <= 5; level += 1) { + const items = criteria.filter((criterion) => criterion.level === level); + const { passed, total } = countStatus(items); + const passRate = total ? passed / total : 0; + summaries.push({ + level, + name: levelNames[level], + passed, + total, + passRate, + achieved: false + }); + } + + for (const summary of summaries) { + const allPrior = summaries.filter((candidate) => candidate.level <= summary.level); + const achieved = allPrior.every((candidate) => candidate.total === 0 || candidate.passRate >= 0.8); + summary.achieved = achieved; + } + + return summaries; +} + +function countStatus(items: ReadinessCriterionResult[]): { passed: number; total: number } { + const relevant = items.filter((item) => item.status !== "skip"); + const passed = relevant.filter((item) => item.status === "pass").length; + return { passed, total: relevant.length }; +} + +async function safeReadDir(dirPath: string): Promise { + try { + return await fs.readdir(dirPath); + } catch { + return []; + } +} + +async function readJson(filePath: string): Promise | undefined> { + try { + const raw = await fs.readFile(filePath, "utf8"); + return JSON.parse(raw) as Record; + } catch { + return undefined; + } +} + +async function fileExists(filePath: string): Promise { + try { + await fs.access(filePath); + return true; + } catch { + return false; + } +} + +function hasAnyFile(files: string[], candidates: string[]): boolean { + return candidates.some((candidate) => files.includes(candidate)); +} + +async function hasReadme(repoPath: string): Promise { + const files = await safeReadDir(repoPath); + return files.some((file) => file.toLowerCase() === "readme.md" || file.toLowerCase() === "readme"); +} + +async function hasLintConfig(repoPath: string): Promise { + return hasAnyFile(await safeReadDir(repoPath), [ + "eslint.config.js", + "eslint.config.mjs", + ".eslintrc", + ".eslintrc.js", + ".eslintrc.cjs", + ".eslintrc.json", + ".eslintrc.yml", + ".eslintrc.yaml", + "biome.json", + "biome.jsonc", + ".prettierrc", + ".prettierrc.json", + ".prettierrc.js", + ".prettierrc.cjs", + "prettier.config.js", + "prettier.config.cjs" + ]); +} + +async function hasFormatterConfig(repoPath: string): Promise { + return hasAnyFile(await safeReadDir(repoPath), [ + "biome.json", + "biome.jsonc", + ".prettierrc", + ".prettierrc.json", + ".prettierrc.js", + ".prettierrc.cjs", + "prettier.config.js", + "prettier.config.cjs" + ]); +} + +async function hasTypecheckConfig(repoPath: string): Promise { + return hasAnyFile(await safeReadDir(repoPath), [ + "tsconfig.json", + "tsconfig.base.json", + "pyproject.toml", + "mypy.ini" + ]); +} + +async function hasGithubWorkflows(repoPath: string): Promise { + return fileExists(path.join(repoPath, ".github", "workflows")); +} + +async function hasCodeowners(repoPath: string): Promise { + const root = await fileExists(path.join(repoPath, "CODEOWNERS")); + const github = await fileExists(path.join(repoPath, ".github", "CODEOWNERS")); + return root || github; +} + +async function hasLicense(repoPath: string): Promise { + const files = await safeReadDir(repoPath); + return files.some((file) => file.toLowerCase().startsWith("license")); +} + +async function hasPullRequestTemplate(repoPath: string): Promise { + const direct = await fileExists(path.join(repoPath, ".github", "PULL_REQUEST_TEMPLATE.md")); + if (direct) return true; + const dir = path.join(repoPath, ".github", "PULL_REQUEST_TEMPLATE"); + try { + const entries = await fs.readdir(dir); + return entries.some((entry) => entry.toLowerCase().endsWith(".md")); + } catch { + return false; + } +} + +async function hasPrecommitConfig(repoPath: string): Promise { + const precommit = await fileExists(path.join(repoPath, ".pre-commit-config.yaml")); + if (precommit) return true; + return fileExists(path.join(repoPath, ".husky")); +} + +async function hasArchitectureDoc(repoPath: string): Promise { + const files = await safeReadDir(repoPath); + if (files.some((file) => file.toLowerCase() === "architecture.md")) return true; + return fileExists(path.join(repoPath, "docs", "architecture.md")); +} + +async function readAllDependencies(context: ReadinessContext): Promise { + const dependencies: string[] = []; + const apps = context.apps.length ? context.apps : []; + for (const app of apps) { + const pkg = await readJson(app.packageJsonPath); + const deps = (pkg?.dependencies ?? {}) as Record; + const devDeps = (pkg?.devDependencies ?? {}) as Record; + dependencies.push(...Object.keys({ + ...deps, + ...devDeps + })); + } + + if (!apps.length && context.rootPackageJson) { + const rootDeps = (context.rootPackageJson.dependencies ?? {}) as Record; + const rootDevDeps = (context.rootPackageJson.devDependencies ?? {}) as Record; + dependencies.push(...Object.keys({ + ...rootDeps, + ...rootDevDeps + })); + } + + return Array.from(new Set(dependencies)); +} \ No newline at end of file diff --git a/src/ui/BatchTuiAzure.tsx b/src/ui/BatchTuiAzure.tsx new file mode 100644 index 0000000..8eb5fea --- /dev/null +++ b/src/ui/BatchTuiAzure.tsx @@ -0,0 +1,526 @@ +import React, { useEffect, useState } from "react"; +import { Box, Text, useApp, useInput } from "ink"; +import path from "path"; +import fs from "fs/promises"; +import { + AzureDevOpsOrg, + AzureDevOpsProject, + AzureDevOpsRepo, + listOrganizations, + listProjects, + listRepos, + checkReposForInstructions, + createPullRequest +} from "../services/azureDevops"; +import { + buildAuthedUrl, + checkoutBranch, + cloneRepo, + commitAll, + isGitRepo, + pushBranch +} from "../services/git"; +import { generateCopilotInstructions } from "../services/instructions"; +import { ensureDir } from "../utils/fs"; +import { StaticBanner } from "./AnimatedBanner"; + +type Props = { + token: string; + outputPath?: string; +}; + +type Status = + | "loading-orgs" + | "select-orgs" + | "loading-projects" + | "select-projects" + | "loading-repos" + | "select-repos" + | "confirm" + | "processing" + | "complete" + | "error"; + +type ProcessResult = { + repo: string; + success: boolean; + prUrl?: string; + error?: string; +}; + +export function BatchTuiAzure({ token, outputPath }: Props): React.JSX.Element { + const app = useApp(); + const [status, setStatus] = useState("loading-orgs"); + const [message, setMessage] = useState("Fetching organizations..."); + const [errorMessage, setErrorMessage] = useState(""); + + const [orgs, setOrgs] = useState([]); + const [projects, setProjects] = useState([]); + const [repos, setRepos] = useState([]); + const [selectedOrgIndices, setSelectedOrgIndices] = useState>(new Set()); + const [selectedProjectIndices, setSelectedProjectIndices] = useState>(new Set()); + const [selectedRepoIndices, setSelectedRepoIndices] = useState>(new Set()); + const [cursorIndex, setCursorIndex] = useState(0); + + const [results, setResults] = useState([]); + const [currentRepoIndex, setCurrentRepoIndex] = useState(0); + const [processingMessage, setProcessingMessage] = useState(""); + + useEffect(() => { + loadOrgs(); + }, []); + + async function loadOrgs() { + try { + const userOrgs = await listOrganizations(token); + setOrgs(userOrgs); + setStatus("select-orgs"); + setMessage("Select organizations (space to toggle, enter to confirm)"); + } catch (error) { + setStatus("error"); + setErrorMessage(error instanceof Error ? error.message : "Failed to fetch organizations"); + } + } + + async function loadProjects() { + setStatus("loading-projects"); + setMessage("Fetching projects..."); + + try { + const selectedOrgs = Array.from(selectedOrgIndices).map(i => orgs[i]); + let allProjects: AzureDevOpsProject[] = []; + + for (let idx = 0; idx < selectedOrgs.length; idx++) { + const org = selectedOrgs[idx]; + setMessage(`Fetching projects from ${org.name} (${idx + 1}/${selectedOrgs.length})...`); + const orgProjects = await listProjects(token, org.name); + allProjects = [...allProjects, ...orgProjects]; + } + + setProjects(allProjects); + setCursorIndex(0); + setSelectedProjectIndices(new Set()); + setStatus("select-projects"); + setMessage("Select projects (space to toggle, enter to confirm)"); + } catch (error) { + setStatus("error"); + setErrorMessage(error instanceof Error ? error.message : "Failed to fetch projects"); + } + } + + async function loadRepos() { + setStatus("loading-repos"); + setMessage("Fetching repositories..."); + + try { + const selectedProjects = Array.from(selectedProjectIndices).map(i => projects[i]); + let allRepos: AzureDevOpsRepo[] = []; + + for (let idx = 0; idx < selectedProjects.length; idx++) { + const project = selectedProjects[idx]; + setMessage( + `Fetching repos from ${project.organization}/${project.name} (${idx + 1}/${selectedProjects.length})...` + ); + const projectRepos = await listRepos(token, project.organization, project.name); + allRepos = [...allRepos, ...projectRepos]; + } + + const seen = new Set(); + const uniqueRepos = allRepos.filter((repo) => { + const key = `${repo.organization}/${repo.project}/${repo.name}`; + if (seen.has(key)) return false; + seen.add(key); + return true; + }); + + setMessage(`Checking ${uniqueRepos.length} repos for existing instructions...`); + const reposWithStatus = await checkReposForInstructions( + token, + uniqueRepos, + (checked, total) => setMessage(`Checking for existing instructions (${checked}/${total})...`) + ); + + reposWithStatus.sort((a, b) => { + if (a.hasInstructions === b.hasInstructions) return 0; + return a.hasInstructions ? 1 : -1; + }); + + const withInstructions = reposWithStatus.filter(r => r.hasInstructions).length; + const withoutInstructions = reposWithStatus.length - withInstructions; + + setRepos(reposWithStatus); + setCursorIndex(0); + setSelectedRepoIndices(new Set()); + setStatus("select-repos"); + setMessage(`Found ${reposWithStatus.length} repos (${withoutInstructions} need instructions, ${withInstructions} already have them)`); + } catch (error) { + setStatus("error"); + setErrorMessage(error instanceof Error ? error.message : "Failed to fetch repositories"); + } + } + + async function processRepos() { + const selectedRepos = Array.from(selectedRepoIndices).map(i => repos[i]); + setStatus("processing"); + setCurrentRepoIndex(0); + setResults([]); + + const nextResults: ProcessResult[] = []; + + for (let i = 0; i < selectedRepos.length; i++) { + const repo = selectedRepos[i]; + setCurrentRepoIndex(i); + setProcessingMessage(`[${i + 1}/${selectedRepos.length}] ${repo.organization}/${repo.project}/${repo.name}: Cloning...`); + + try { + const cacheRoot = path.join(process.cwd(), ".primer-cache"); + const repoPath = path.join(cacheRoot, repo.organization, repo.project, repo.name); + await ensureDir(repoPath); + + if (!(await isGitRepo(repoPath))) { + const authedUrl = buildAuthedUrl(repo.cloneUrl, token, "azure"); + await cloneRepo(authedUrl, repoPath, { + shallow: true, + timeoutMs: 120000, + onProgress: (stage, progress) => { + setProcessingMessage( + `[${i + 1}/${selectedRepos.length}] ${repo.organization}/${repo.project}/${repo.name}: Cloning (${stage} ${progress}%)...` + ); + } + }); + } + + setProcessingMessage(`[${i + 1}/${selectedRepos.length}] ${repo.organization}/${repo.project}/${repo.name}: Creating branch...`); + const branch = "primer/add-instructions"; + await checkoutBranch(repoPath, branch); + + setProcessingMessage(`[${i + 1}/${selectedRepos.length}] ${repo.organization}/${repo.project}/${repo.name}: Generating instructions...`); + const timeoutMs = 120000; + const instructionsPromise = generateCopilotInstructions({ + repoPath, + model: "gpt-4.1", + onProgress: (msg) => { + setProcessingMessage(`[${i + 1}/${selectedRepos.length}] ${repo.organization}/${repo.project}/${repo.name}: ${msg}`); + } + }); + + const timeoutPromise = new Promise((_, reject) => { + setTimeout(() => reject(new Error("Generation timed out after 2 minutes")), timeoutMs); + }); + + const instructions = await Promise.race([instructionsPromise, timeoutPromise]); + + if (!instructions.trim()) { + throw new Error("Generated instructions were empty"); + } + + const instructionsPath = path.join(repoPath, ".github", "copilot-instructions.md"); + await fs.mkdir(path.dirname(instructionsPath), { recursive: true }); + await fs.writeFile(instructionsPath, instructions, "utf8"); + + setProcessingMessage(`[${i + 1}/${selectedRepos.length}] ${repo.organization}/${repo.project}/${repo.name}: Committing...`); + await commitAll(repoPath, "chore: add copilot instructions via Primer"); + + setProcessingMessage(`[${i + 1}/${selectedRepos.length}] ${repo.organization}/${repo.project}/${repo.name}: Pushing...`); + await pushBranch(repoPath, branch, token, "azure"); + + setProcessingMessage(`[${i + 1}/${selectedRepos.length}] ${repo.organization}/${repo.project}/${repo.name}: Creating PR...`); + const prUrl = await createPullRequest({ + token, + organization: repo.organization, + project: repo.project, + repoId: repo.id, + repoName: repo.name, + title: "🤖 Add Copilot instructions via Primer", + body: buildPrBody(), + sourceBranch: branch, + targetBranch: repo.defaultBranch + }); + + nextResults.push({ repo: `${repo.organization}/${repo.project}/${repo.name}`, success: true, prUrl }); + setResults([...nextResults]); + } catch (error) { + const errorMsg = error instanceof Error ? error.message : "Unknown error"; + nextResults.push({ repo: `${repo.organization}/${repo.project}/${repo.name}`, success: false, error: errorMsg }); + setResults([...nextResults]); + } + } + + if (outputPath) { + await fs.writeFile(outputPath, JSON.stringify(nextResults, null, 2), "utf8"); + } + + setStatus("complete"); + setMessage("Batch processing complete!"); + } + + useInput((input, key) => { + if (key.escape || input.toLowerCase() === "q") { + app.exit(); + return; + } + + if (status === "select-orgs") { + if (key.upArrow) { + setCursorIndex(prev => Math.max(0, prev - 1)); + } else if (key.downArrow) { + setCursorIndex(prev => Math.min(orgs.length - 1, prev + 1)); + } else if (input === " ") { + setSelectedOrgIndices(prev => { + const next = new Set(prev); + if (next.has(cursorIndex)) { + next.delete(cursorIndex); + } else { + next.add(cursorIndex); + } + return next; + }); + } else if (key.return && selectedOrgIndices.size > 0) { + loadProjects(); + } + } + + if (status === "select-projects") { + if (key.upArrow) { + setCursorIndex(prev => Math.max(0, prev - 1)); + } else if (key.downArrow) { + setCursorIndex(prev => Math.min(projects.length - 1, prev + 1)); + } else if (input === " ") { + setSelectedProjectIndices(prev => { + const next = new Set(prev); + if (next.has(cursorIndex)) { + next.delete(cursorIndex); + } else { + next.add(cursorIndex); + } + return next; + }); + } else if (key.return && selectedProjectIndices.size > 0) { + loadRepos(); + } + } + + if (status === "select-repos") { + if (key.upArrow) { + setCursorIndex(prev => Math.max(0, prev - 1)); + } else if (key.downArrow) { + setCursorIndex(prev => Math.min(repos.length - 1, prev + 1)); + } else if (input === " ") { + setSelectedRepoIndices(prev => { + const next = new Set(prev); + if (next.has(cursorIndex)) { + next.delete(cursorIndex); + } else { + next.add(cursorIndex); + } + return next; + }); + } else if (input.toLowerCase() === "a") { + const indicesWithoutInstructions = repos + .map((r, i) => ({ r, i })) + .filter(({ r }) => !r.hasInstructions) + .map(({ i }) => i); + setSelectedRepoIndices(new Set(indicesWithoutInstructions)); + } else if (key.return && selectedRepoIndices.size > 0) { + setStatus("confirm"); + setMessage(`Ready to process ${selectedRepoIndices.size} repositories. Press Y to confirm, N to go back.`); + } + } + + if (status === "confirm") { + if (input.toLowerCase() === "y") { + processRepos(); + } else if (input.toLowerCase() === "n") { + setStatus("select-repos"); + setMessage("Select repos (space to toggle, enter to confirm)"); + } + } + }); + + const windowSize = 15; + const getVisibleItems = (items: T[], cursor: number): { items: T[]; startIndex: number } => { + const start = Math.max(0, cursor - Math.floor(windowSize / 2)); + const end = Math.min(items.length, start + windowSize); + const adjustedStart = Math.max(0, end - windowSize); + return { items: items.slice(adjustedStart, end), startIndex: adjustedStart }; + }; + + return ( + + + Batch Processing - Azure DevOps + + {message} + + + {status === "error" && ( + + Error: {errorMessage} + + )} + + {status === "select-orgs" && ( + + {(() => { + const { items: visibleOrgs, startIndex } = getVisibleItems(orgs, cursorIndex); + return visibleOrgs.map((org, i) => { + const realIndex = startIndex + i; + const isSelected = selectedOrgIndices.has(realIndex); + const isCursor = realIndex === cursorIndex; + return ( + + {isCursor ? "❯ " : " "} + {isSelected ? "◉" : "○"} + {org.name} + + ); + }); + })()} + {orgs.length > windowSize && ( + + Showing {Math.min(windowSize, orgs.length)} of {orgs.length} • Use ↑↓ to scroll + + )} + + )} + + {status === "select-projects" && ( + + {(() => { + const { items: visibleProjects, startIndex } = getVisibleItems(projects, cursorIndex); + return visibleProjects.map((project, i) => { + const realIndex = startIndex + i; + const isSelected = selectedProjectIndices.has(realIndex); + const isCursor = realIndex === cursorIndex; + return ( + + {isCursor ? "❯ " : " "} + {isSelected ? "◉" : "○"} + {project.organization}/{project.name} + + ); + }); + })()} + {projects.length > windowSize && ( + + Showing {Math.min(windowSize, projects.length)} of {projects.length} • Use ↑↓ to scroll + + )} + + )} + + {status === "select-repos" && ( + + {(() => { + const { items: visibleRepos, startIndex } = getVisibleItems(repos, cursorIndex); + return visibleRepos.map((repo, i) => { + const realIndex = startIndex + i; + const isSelected = selectedRepoIndices.has(realIndex); + const isCursor = realIndex === cursorIndex; + return ( + + {isCursor ? "❯ " : " "} + {isSelected ? "◉" : "○"} + {repo.hasInstructions ? "✓" : "✗"} + + {repo.organization}/{repo.project}/{repo.name} + + {repo.isPrivate && (private)} + + ); + }); + })()} + {repos.length > windowSize && ( + + Showing {Math.min(windowSize, repos.length)} of {repos.length} • Use ↑↓ to scroll + + )} + + + Selected: {selectedRepoIndices.size} repos + + + + )} + + {status === "processing" && ( + + {processingMessage} + {results.length > 0 && ( + + Completed: + {results.slice(-5).map((r) => ( + + {r.success ? "✓" : "✗"} {r.repo} + {r.success && r.prUrl && → {r.prUrl}} + {!r.success && r.error && ({r.error})} + + ))} + + )} + + )} + + {status === "complete" && ( + + + ✓ Batch complete: {results.filter(r => r.success).length} succeeded, {results.filter(r => !r.success).length} failed + + + {results.map((r) => ( + + {r.success ? "✓" : "✗"} {r.repo} + {r.success && r.prUrl && → {r.prUrl}} + {!r.success && r.error && ({r.error})} + + ))} + + + )} + + + {status === "select-orgs" && ( + Keys: [Space] Toggle [Enter] Confirm [Q] Quit + )} + {status === "select-projects" && ( + Keys: [Space] Toggle [Enter] Confirm [Q] Quit + )} + {status === "select-repos" && ( + Keys: [Space] Toggle [A] Select Missing [Enter] Confirm [Q] Quit + )} + {status === "confirm" && ( + Keys: [Y] Yes, proceed [N] Go back [Q] Quit + )} + {(status === "complete" || status === "error") && ( + Keys: [Q] Quit + )} + + + ); +} + +function buildPrBody(): string { + return [ + "## 🤖 Copilot Instructions Added", + "", + "This PR adds a `.github/copilot-instructions.md` file to help GitHub Copilot understand this codebase better.", + "", + "### What's Included", + "", + "The instructions file contains:", + "- Project overview and architecture", + "- Tech stack and conventions", + "- Build/test commands", + "- Key directories and files", + "", + "### Benefits", + "", + "With these instructions, Copilot will:", + "- Generate more contextually-aware code suggestions", + "- Follow project-specific patterns and conventions", + "- Understand the codebase structure", + "", + "---", + "*Generated by [Primer](https://github.com/pierceboggan/primer) - Prime your repos for AI*" + ].join("\n"); +} \ No newline at end of file diff --git a/src/ui/README.md b/src/ui/README.md index a34f7c0..551d04e 100644 --- a/src/ui/README.md +++ b/src/ui/README.md @@ -1 +1,3 @@ Primer TUI components live here. + +Run the TUI with `primer tui`. diff --git a/src/ui/tui.tsx b/src/ui/tui.tsx index cfa8dbb..7991865 100644 --- a/src/ui/tui.tsx +++ b/src/ui/tui.tsx @@ -8,13 +8,79 @@ import { runEval, type EvalResult } from "../services/evaluator"; import { AnimatedBanner, StaticBanner } from "./AnimatedBanner"; import { BatchTui } from "./BatchTui"; import { getGitHubToken } from "../services/github"; +import { safeWriteFile } from "../utils/fs"; type Props = { repoPath: string; skipAnimation?: boolean; }; -type Status = "intro" | "idle" | "analyzing" | "generating" | "evaluating" | "preview" | "done" | "error" | "batch"; +type Status = + | "intro" + | "idle" + | "analyzing" + | "generating" + | "evaluating" + | "preview" + | "done" + | "error" + | "batch" + | "bootstrapEvalCount" + | "bootstrapEvalConfirm"; + +type EvalCase = { + id: string; + prompt: string; + expectation: string; +}; + +type EvalConfig = { + instructionFile?: string; + cases: EvalCase[]; + systemMessage?: string; + outputPath?: string; +}; + +function buildBootstrapEvalConfig(count: number): EvalConfig { + const templates = [ + { + prompt: "Summarize this repository's purpose and main entry points. Use the README and package.json if available.", + expectation: "A concise summary of repo purpose plus key entry points (CLI or main files) and how to run it." + }, + { + prompt: "Identify the primary languages, frameworks, and package manager used in this repo.", + expectation: "A brief list of languages/frameworks and the detected package manager, with short justification." + }, + { + prompt: "Draft a minimal .github/copilot-instructions.md tailored to this repo's conventions.", + expectation: "A short instruction file referencing observed conventions, avoiding assumptions or secrets." + }, + { + prompt: "Describe how to run the CLI and list its core commands and flags.", + expectation: "Clear usage instructions with command names and key flags derived from docs or source." + }, + { + prompt: "Propose an eval case that checks consistency between CLI and TUI behaviors.", + expectation: "One eval case that validates parity between CLI and TUI outputs or workflows." + } + ]; + + const cases = Array.from({ length: count }, (_, index) => { + const template = templates[index % templates.length]; + const variant = Math.floor(index / templates.length); + const suffix = variant > 0 ? ` (variant ${variant + 1})` : ""; + return { + id: `case-${index + 1}`, + prompt: `${template.prompt}${suffix}`, + expectation: `${template.expectation}${suffix}` + } satisfies EvalCase; + }); + + return { + instructionFile: ".github/copilot-instructions.md", + cases + }; +} export function PrimerTui({ repoPath, skipAnimation = false }: Props): React.JSX.Element { const app = useApp(); @@ -23,7 +89,10 @@ export function PrimerTui({ repoPath, skipAnimation = false }: Props): React.JSX const [message, setMessage] = useState(""); const [generatedContent, setGeneratedContent] = useState(""); const [evalResults, setEvalResults] = useState(null); + const [evalViewerPath, setEvalViewerPath] = useState(null); const [batchToken, setBatchToken] = useState(null); + const [evalCaseCountInput, setEvalCaseCountInput] = useState(""); + const [evalBootstrapCount, setEvalBootstrapCount] = useState(null); const repoLabel = useMemo(() => repoPath, [repoPath]); const handleAnimationComplete = () => { @@ -67,6 +136,78 @@ export function PrimerTui({ repoPath, skipAnimation = false }: Props): React.JSX return; } + if (status === "bootstrapEvalCount") { + if (key.return) { + const trimmed = evalCaseCountInput.trim(); + const count = Number.parseInt(trimmed, 10); + if (!trimmed || !Number.isFinite(count) || count <= 0) { + setMessage("Enter a positive number of eval cases, then press Enter."); + return; + } + + const configPath = path.join(repoPath, "primer.eval.json"); + setEvalBootstrapCount(count); + try { + await fs.access(configPath); + setStatus("bootstrapEvalConfirm"); + setMessage("primer.eval.json exists. Overwrite? (Y/N)"); + } catch { + const config = buildBootstrapEvalConfig(count); + const resultMessage = await safeWriteFile(configPath, JSON.stringify(config, null, 2), false); + setStatus("done"); + setMessage(`Bootstrapped eval: ${resultMessage}`); + setEvalCaseCountInput(""); + setEvalBootstrapCount(null); + } + return; + } + + if (key.backspace || key.delete) { + setEvalCaseCountInput((prev) => prev.slice(0, -1)); + return; + } + + if (/^\d$/.test(input)) { + setEvalCaseCountInput((prev) => prev + input); + return; + } + + return; + } + + if (status === "bootstrapEvalConfirm") { + if (input.toLowerCase() === "y") { + const count = evalBootstrapCount ?? 0; + if (count <= 0) { + setStatus("error"); + setMessage("Missing eval case count. Restart bootstrap."); + return; + } + try { + const configPath = path.join(repoPath, "primer.eval.json"); + const config = buildBootstrapEvalConfig(count); + const resultMessage = await safeWriteFile(configPath, JSON.stringify(config, null, 2), true); + setStatus("done"); + setMessage(`Bootstrapped eval: ${resultMessage}`); + } catch (error) { + setStatus("error"); + setMessage(error instanceof Error ? error.message : "Failed to write eval config."); + } finally { + setEvalCaseCountInput(""); + setEvalBootstrapCount(null); + } + return; + } + + if (input.toLowerCase() === "n") { + setStatus("idle"); + setMessage("Bootstrap cancelled."); + setEvalCaseCountInput(""); + setEvalBootstrapCount(null); + } + return; + } + if (input.toLowerCase() === "a") { setStatus("analyzing"); try { @@ -133,8 +274,9 @@ export function PrimerTui({ repoPath, skipAnimation = false }: Props): React.JSX setStatus("evaluating"); setMessage("Running evals... (this may take a few minutes)"); setEvalResults(null); + setEvalViewerPath(null); try { - const { results } = await runEval({ + const { results, viewerPath } = await runEval({ configPath, repoPath, model: "gpt-4.1", @@ -142,6 +284,7 @@ export function PrimerTui({ repoPath, skipAnimation = false }: Props): React.JSX // Note: onProgress removed - causes issues with SDK in React/Ink context }); setEvalResults(results); + setEvalViewerPath(viewerPath ?? null); const passed = results.filter(r => r.verdict === "pass").length; const failed = results.filter(r => r.verdict === "fail").length; setStatus("done"); @@ -151,6 +294,13 @@ export function PrimerTui({ repoPath, skipAnimation = false }: Props): React.JSX setMessage(error instanceof Error ? error.message : "Eval failed."); } } + + if (input.toLowerCase() === "i") { + setStatus("bootstrapEvalCount"); + setMessage("Enter number of eval cases, then press Enter."); + setEvalCaseCountInput(""); + setEvalBootstrapCount(null); + } }); const statusLabel = status === "intro" ? "..." : status === "idle" ? "ready (awaiting input)" : status; @@ -186,6 +336,11 @@ export function PrimerTui({ repoPath, skipAnimation = false }: Props): React.JSX {message} + {status === "bootstrapEvalCount" && ( + + Eval case count: {evalCaseCountInput || ""} + + )} {status === "preview" && generatedContent && ( Preview (.github/copilot-instructions.md): @@ -200,6 +355,9 @@ export function PrimerTui({ repoPath, skipAnimation = false }: Props): React.JSX {r.verdict === "pass" ? "✓" : r.verdict === "fail" ? "✗" : "?"} {r.id}: {r.verdict} (score: {r.score}) ))} + {evalViewerPath && ( + Trajectory viewer: {evalViewerPath} + )} )} @@ -207,8 +365,10 @@ export function PrimerTui({ repoPath, skipAnimation = false }: Props): React.JSX Press any key to skip animation... ) : status === "preview" ? ( Keys: [S] Save [D] Discard [Q] Quit + ) : status === "bootstrapEvalConfirm" ? ( + Keys: [Y] Overwrite [N] Cancel [Q] Quit ) : ( - Keys: [A] Analyze [G] Generate [E] Eval [B] Batch [Q] Quit + Keys: [A] Analyze [G] Generate [E] Eval [I] Init Eval [B] Batch [Q] Quit )} diff --git a/vitest.config.ts b/vitest.config.ts new file mode 100644 index 0000000..2481bee --- /dev/null +++ b/vitest.config.ts @@ -0,0 +1,12 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + environment: "node", + coverage: { + provider: "v8", + reporter: ["text", "html", "json-summary"], + reportsDirectory: "./coverage" + } + } +});