From 640c7c6b67a8f3c78a893b5239e228d84faa4a92 Mon Sep 17 00:00:00 2001 From: Pierce Boggan Date: Tue, 3 Feb 2026 22:05:56 -0700 Subject: [PATCH] chore: vnext updates --- .github/ISSUE_TEMPLATE/bug_report.md | 23 + .github/ISSUE_TEMPLATE/config.yml | 5 + .github/ISSUE_TEMPLATE/feature_request.md | 13 + .github/PULL_REQUEST_TEMPLATE.md | 10 + .github/workflows/ci.yml | 34 ++ .github/workflows/release-please.yml | 19 + .gitignore | 1 + .prettierrc.json | 5 + CHANGELOG.md | 12 + CODEOWNERS | 1 + CONTRIBUTING.md | 34 ++ LICENSE | 21 + PLAN.md | 76 ++- README.md | 128 ++++- SECURITY.md | 7 + eslint.config.js | 45 ++ examples/README.md | 14 + examples/primer.eval.json | 15 + package.json | 24 +- primer.eval.json | 2 +- release-please-config.json | 9 + release-please-manifest.json | 3 + src/cli.ts | 15 +- src/commands/batch.tsx | 29 +- src/commands/eval.ts | 5 +- src/commands/init.ts | 85 ++- src/commands/pr.ts | 97 +++- src/commands/readiness.ts | 120 ++++ src/services/__tests__/analyzer.test.ts | 31 ++ src/services/analyzer.ts | 142 ++++- src/services/azureDevops.ts | 235 ++++++++ src/services/evaluator.ts | 461 ++++++++++++++- src/services/git.ts | 22 +- src/services/readiness.ts | 648 ++++++++++++++++++++++ src/ui/BatchTuiAzure.tsx | 526 ++++++++++++++++++ src/ui/README.md | 2 + src/ui/tui.tsx | 166 +++++- vitest.config.ts | 12 + 38 files changed, 2996 insertions(+), 101 deletions(-) create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md create mode 100644 .github/ISSUE_TEMPLATE/config.yml create mode 100644 .github/ISSUE_TEMPLATE/feature_request.md create mode 100644 .github/PULL_REQUEST_TEMPLATE.md create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/release-please.yml create mode 100644 .prettierrc.json create mode 100644 CHANGELOG.md create mode 100644 CODEOWNERS create mode 100644 CONTRIBUTING.md create mode 100644 LICENSE create mode 100644 SECURITY.md create mode 100644 eslint.config.js create mode 100644 examples/README.md create mode 100644 examples/primer.eval.json create mode 100644 release-please-config.json create mode 100644 release-please-manifest.json create mode 100644 src/commands/readiness.ts create mode 100644 src/services/__tests__/analyzer.test.ts create mode 100644 src/services/azureDevops.ts create mode 100644 src/services/readiness.ts create mode 100644 src/ui/BatchTuiAzure.tsx create mode 100644 vitest.config.ts diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..c7f8e50 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,23 @@ +--- +name: Bug report +description: Report a problem with Primer +labels: [bug] +--- + +## Description + +## Steps to reproduce + +1. +2. +3. + +## Expected behavior + +## Actual behavior + +## Environment + +- OS: +- Node version: +- Primer version: diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..3624ac5 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,5 @@ +blank_issues_enabled: false +contact_links: + - name: Questions + url: https://github.com/pierceboggan/primer/discussions + about: Ask questions and discuss ideas here. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..dabc116 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,13 @@ +--- +name: Feature request +description: Suggest an enhancement for Primer +labels: [enhancement] +--- + +## Summary + +## Problem statement + +## Proposed solution + +## Additional context diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..84350e7 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,10 @@ +## Summary + +- [ ] What changed +- [ ] Why it changed + +## Checklist + +- [ ] Tests added or updated +- [ ] Lint/typecheck pass +- [ ] Docs updated if needed diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..cb9b04c --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,34 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Setup Node + uses: actions/setup-node@v4 + with: + node-version: 20 + cache: npm + - name: Install + run: npm ci + - name: Lint + run: npm run lint + - name: Typecheck + run: npm run typecheck + - name: Test + run: npm run test:coverage + - name: Build + run: npm run build + - name: Upload coverage + uses: actions/upload-artifact@v4 + with: + name: coverage + path: coverage + if-no-files-found: ignore diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml new file mode 100644 index 0000000..3547bac --- /dev/null +++ b/.github/workflows/release-please.yml @@ -0,0 +1,19 @@ +name: Release Please + +on: + push: + branches: [main] + +permissions: + contents: write + pull-requests: write + +jobs: + release-please: + runs-on: ubuntu-latest + steps: + - name: Release + uses: google-github-actions/release-please-action@v4 + with: + config-file: release-please-config.json + manifest-file: release-please-manifest.json diff --git a/.gitignore b/.gitignore index 6b4a136..89da78c 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ node_modules/ # Build output dist/ +coverage/ # IDE .vscode/ diff --git a/.prettierrc.json b/.prettierrc.json new file mode 100644 index 0000000..1ad2e3d --- /dev/null +++ b/.prettierrc.json @@ -0,0 +1,5 @@ +{ + "singleQuote": false, + "trailingComma": "none", + "printWidth": 100 +} diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..e91d5ff --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,12 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +## [Unreleased] + +- Readiness report with monorepo support and fix-first checklist. +- CI workflow with lint, typecheck, tests, and build steps. +- Release automation via release-please. +- ESLint, Prettier, and Vitest tooling plus starter tests. +- Added contributing, security, and issue/PR templates. +- Added examples folder with sample configs. diff --git a/CODEOWNERS b/CODEOWNERS new file mode 100644 index 0000000..77511c7 --- /dev/null +++ b/CODEOWNERS @@ -0,0 +1 @@ +* @pierceboggan diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..e66fc98 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,34 @@ +# Contributing + +Thanks for contributing to Primer. + +## Quick start + +1. Fork and clone the repo. +2. Install dependencies: npm install +3. Build locally: npm run build +4. Run lint/typecheck/tests before opening a PR: + - npm run lint + - npm run typecheck + - npm run test + +## Development workflow + +- Create a feature branch from main. +- Use clear, conventional commit messages (e.g. feat: add readiness report). +- Keep PRs focused and include context in the description. +- Add or update tests when behavior changes. + +## Code style + +- ESLint + Prettier are enforced in CI. +- Prefer small, composable functions with clear types. + +## Reporting issues + +- Use GitHub Issues for bugs and feature requests. +- Provide steps to reproduce and expected behavior. + +## Releasing + +Releases are automated with release-please when changes are merged to main. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..cb6db3e --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Primer + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/PLAN.md b/PLAN.md index 3dcc268..46f5b83 100644 --- a/PLAN.md +++ b/PLAN.md @@ -16,9 +16,14 @@ Make any repository "AI-ready" with a single command — generating optimal conf - Detect language(s), frameworks, and project structure - Identify existing AI configurations - Analyze package managers, build tools, and testing frameworks -- Detect monorepo vs single-project structure +- Detect monorepo vs single-project structure (workspace-aware) -### 2. **Configuration Generation** +### 2. **Readiness Report** +- Score AI readiness across key pillars +- Provide fix-first checklists and maturity levels +- Support monorepos with app-scoped checks + +### 3. **Configuration Generation** | Config Type | Description | |-------------|-------------| @@ -26,19 +31,19 @@ Make any repository "AI-ready" with a single command — generating optimal conf | **MCP Server Config** | `.vscode/mcp.json` for Model Context Protocol servers | | **VS Code Settings** | `.vscode/settings.json` with AI-optimized workspace settings | -### 3. **GitHub Integration** +### 4. **GitHub Integration** - Authenticate via GitHub CLI (`gh auth`) or OAuth device flow - List and select from accessible repositories - Clone repos temporarily for analysis - **Auto-create PRs** with generated configurations - Support for GitHub Enterprise -### 4. **Local Repository Support** +### 5. **Local Repository Support** - Detect local Git repositories - Work offline with local-only mode - Push changes to remote when ready -### 5. **Interactive & Non-Interactive Modes** +### 6. **Interactive & Non-Interactive Modes** - Beautiful TUI with prompts and previews - CI/CD-friendly `--yes` flag for automation - JSON output for scripting @@ -99,6 +104,9 @@ primer pr owner/repo # Analyze repo without making changes primer analyze +# Readiness report +primer readiness + # Update existing configurations primer update @@ -107,6 +115,18 @@ primer templates # Configure CLI settings primer config + +# Generate instructions +primer instructions + +# Run evaluations +primer eval primer.eval.json + +# Run TUI +primer tui + +# Batch processing +primer batch ``` --- @@ -370,23 +390,33 @@ primer/ │ ├── index.ts # Entry point │ ├── cli.ts # Commander setup │ ├── commands/ -│ │ ├── init.ts -│ │ ├── generate.ts │ │ ├── analyze.ts +│ │ ├── batch.tsx +│ │ ├── config.ts +│ │ ├── eval.ts +│ │ ├── generate.ts +│ │ ├── init.ts +│ │ ├── instructions.tsx │ │ ├── pr.ts -│ │ └── config.ts +│ │ ├── readiness.ts +│ │ ├── templates.ts +│ │ ├── tui.tsx +│ │ └── update.ts │ ├── services/ -│ │ ├── github.ts # GitHub API interactions │ │ ├── analyzer.ts # Repo analysis logic +│ │ ├── azureDevops.ts # Azure DevOps integration +│ │ ├── evaluator.ts # Eval runner │ │ ├── generator.ts # Config generation -│ │ └── git.ts # Local git operations +│ │ ├── git.ts # Local git operations +│ │ ├── github.ts # GitHub API interactions +│ │ └── instructions.ts # Copilot SDK integration │ ├── ui/ -│ │ ├── prompts.ts # Inquirer prompts -│ │ ├── spinner.ts # Loading indicators -│ │ └── preview.ts # File previews +│ │ ├── AnimatedBanner.tsx +│ │ ├── BatchTui.tsx +│ │ ├── BatchTuiAzure.tsx +│ │ └── tui.tsx │ └── utils/ │ ├── fs.ts # File system helpers -│ ├── detection.ts # Language/framework detection │ └── logger.ts # Styled console output ├── package.json ├── tsconfig.json @@ -476,24 +506,24 @@ Create example repos for each major stack: # Install dependencies npm install -# Development with watch mode -npm run dev - # Build for production npm run build -# Run tests -npm test - # Lint and format npm run lint npm run format +# Type check +npm run typecheck + +# Run tests +npm run test + +# Coverage +npm run test:coverage + # Link globally for testing npm link - -# Create standalone binaries -npm run package ``` --- diff --git a/README.md b/README.md index 7c5dd6c..ab2da0a 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,9 @@ > Prime your repositories for AI-assisted development. +[![CI](https://github.com/pierceboggan/primer/actions/workflows/ci.yml/badge.svg)](https://github.com/pierceboggan/primer/actions/workflows/ci.yml) +[![License](https://img.shields.io/badge/license-MIT-green.svg)](LICENSE) + Primer is a CLI tool that analyzes your codebase and generates `.github/copilot-instructions.md` files to help AI coding assistants understand your project better. It supports single repos, batch processing across organizations, and includes an evaluation framework to measure instruction effectiveness. ![Primer](primer.png) @@ -12,6 +15,7 @@ Primer is a CLI tool that analyzes your codebase and generates `.github/copilot- - **AI-Powered Generation** - Uses the Copilot SDK to analyze your codebase and generate context-aware instructions - **Batch Processing** - Process multiple repos across organizations with a single command - **Evaluation Framework** - Test and measure how well your instructions improve AI responses +- **Readiness Report** - Score AI readiness across key pillars with a fix-first checklist - **GitHub Integration** - Clone repos, create branches, and open PRs automatically - **Interactive TUI** - Beautiful terminal interface built with Ink - **Config Generation** - Generate MCP and VS Code configurations @@ -22,14 +26,31 @@ Primer is a CLI tool that analyzes your codebase and generates `.github/copilot- 2. **GitHub Copilot CLI** - Installed via VS Code's Copilot Chat extension 3. **Copilot CLI Authentication** - Run `copilot` then `/login` to authenticate 4. **GitHub CLI (optional)** - For batch processing and PR creation: `brew install gh && gh auth login` +5. **Azure DevOps PAT (optional)** - For Azure DevOps batch/PR workflows: set `AZURE_DEVOPS_PAT` ## Installation ```bash -# Clone and install +# Install from npm +npm install -g primer +``` + +### Quick Install + +```bash +primer --help +``` + +### Local Development Install + +```bash +# Clone and install dependencies git clone https://github.com/pierceboggan/primer.git cd primer npm install + +# Build and link the local CLI +npm run build ``` ## Usage @@ -40,26 +61,29 @@ The easiest way to get started is with the `init` command: ```bash # Interactive setup for current directory -npx tsx src/index.ts init +primer init # Accept defaults and generate instructions automatically -npx tsx src/index.ts init --yes +primer init --yes # Work with a GitHub repository -npx tsx src/index.ts init --github +primer init --github + +# Work with an Azure DevOps repository +primer init --provider azure ``` ### Interactive Mode (TUI) ```bash # Run TUI in current directory -npx tsx src/index.ts tui +primer tui # Run on a specific repo -npx tsx src/index.ts tui --repo /path/to/repo +primer tui --repo /path/to/repo # Skip the animated intro -npx tsx src/index.ts tui --no-animation +primer tui --no-animation ``` **Keys:** @@ -73,13 +97,13 @@ npx tsx src/index.ts tui --no-animation ```bash # Generate instructions for current directory -npx tsx src/index.ts instructions +primer instructions # Generate for specific repo with custom output -npx tsx src/index.ts instructions --repo /path/to/repo --output ./instructions.md +primer instructions --repo /path/to/repo --output ./instructions.md # Use a specific model -npx tsx src/index.ts instructions --model gpt-5 +primer instructions --model gpt-5 ``` ### Batch Processing @@ -88,10 +112,13 @@ Process multiple repositories across organizations: ```bash # Launch batch TUI -npx tsx src/index.ts batch +primer batch + +# Launch batch TUI for Azure DevOps +primer batch --provider azure # Save results to file -npx tsx src/index.ts batch --output results.json +primer batch --output results.json ``` **Batch TUI Keys:** @@ -105,31 +132,53 @@ npx tsx src/index.ts batch --output results.json ```bash # Analyze current directory -npx tsx src/index.ts analyze +primer analyze # Analyze specific path with JSON output -npx tsx src/index.ts analyze /path/to/repo --json +primer analyze /path/to/repo --json ``` +### Readiness Report + +Assess how ready a repository is for AI agents and get a prioritized checklist of fixes: + +```bash +# Run readiness report in current directory +primer readiness + +# Run readiness report on a specific repo +primer readiness /path/to/repo + +# Output JSON only +primer readiness --json + +# Write JSON report to a file +primer readiness --output readiness.json +``` + +### Examples + +See [examples/README.md](examples/README.md) for quick usage snippets and a sample eval config. + ### Generate Configs Generate configuration files for your repo: ```bash # Generate MCP config -npx tsx src/index.ts generate mcp +primer generate mcp # Generate VS Code settings -npx tsx src/index.ts generate vscode --force +primer generate vscode --force # Generate custom prompts -npx tsx src/index.ts generate prompts +primer generate prompts # Generate agent configs -npx tsx src/index.ts generate agents +primer generate agents # Generate .aiignore file -npx tsx src/index.ts generate aiignore +primer generate aiignore ``` ### Manage Templates @@ -137,7 +186,7 @@ npx tsx src/index.ts generate aiignore View available instruction templates: ```bash -npx tsx src/index.ts templates +primer templates ``` ### Configuration @@ -145,7 +194,7 @@ npx tsx src/index.ts templates View and manage Primer configuration: ```bash -npx tsx src/index.ts config +primer config ``` ### Update @@ -153,7 +202,7 @@ npx tsx src/index.ts config Check for and apply updates: ```bash -npx tsx src/index.ts update +primer update ``` ### Create Pull Requests @@ -162,10 +211,13 @@ Automatically create a PR to add Primer configs to a repository: ```bash # Create PR for a GitHub repo -npx tsx src/index.ts pr owner/repo-name +primer pr owner/repo-name # Use custom branch name -npx tsx src/index.ts pr owner/repo-name --branch primer/custom-branch +primer pr owner/repo-name --branch primer/custom-branch + +# Create PR for an Azure DevOps repo (org/project/repo) +primer pr my-org/my-project/my-repo --provider azure ``` ### Evaluation Framework @@ -174,19 +226,22 @@ Test how well your instructions improve AI responses: ```bash # Create a starter eval config -npx tsx src/index.ts eval --init +primer eval --init # Run evaluation -npx tsx src/index.ts eval primer.eval.json --repo /path/to/repo +primer eval primer.eval.json --repo /path/to/repo # Save results and use specific models -npx tsx src/index.ts eval --output results.json --model gpt-5 --judge-model gpt-5 +primer eval --output results.json --model gpt-5 --judge-model gpt-5 ``` +When `--output` is provided (or `outputPath` is set in the eval config), Primer writes a JSON report that includes per-case metrics and trajectory events, and also generates a companion HTML trajectory viewer next to the JSON file. + Example `primer.eval.json`: ```json { "instructionFile": ".github/copilot-instructions.md", + "outputPath": "eval-results.json", "cases": [ { "id": "project-overview", @@ -264,8 +319,23 @@ primer/ # Type check npx tsc -p tsconfig.json --noEmit -# Run in dev mode -npx tsx src/index.ts +# Lint +npm run lint + +# Format +npm run format + +# Test +npm run test + +# Coverage +npm run test:coverage + +# Build and link the local CLI +npm run build + +# Run locally +primer ``` ## Troubleshooting diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..0f09971 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,7 @@ +# Security Policy + +## Reporting a Vulnerability + +Please report security issues privately by emailing the maintainer or opening a GitHub security advisory. Avoid filing public issues for vulnerabilities. + +We aim to respond within 72 hours and will work with you on remediation and disclosure. diff --git a/eslint.config.js b/eslint.config.js new file mode 100644 index 0000000..309d504 --- /dev/null +++ b/eslint.config.js @@ -0,0 +1,45 @@ +import js from "@eslint/js"; +import tseslint from "@typescript-eslint/eslint-plugin"; +import tsParser from "@typescript-eslint/parser"; +import eslintConfigPrettier from "eslint-config-prettier"; +import importPlugin from "eslint-plugin-import"; +import nPlugin from "eslint-plugin-n"; +import promisePlugin from "eslint-plugin-promise"; + +const sourceGlobs = ["**/*.{ts,tsx,js,jsx}"]; + +export default [ + { + ignores: ["dist/**", "node_modules/**", "coverage/**"] + }, + js.configs.recommended, + { + files: sourceGlobs, + languageOptions: { + parser: tsParser, + parserOptions: { + project: "./tsconfig.json", + sourceType: "module", + ecmaVersion: "latest" + } + }, + plugins: { + "@typescript-eslint": tseslint, + import: importPlugin, + n: nPlugin, + promise: promisePlugin + }, + rules: { + "@typescript-eslint/no-unused-vars": ["warn", { "argsIgnorePattern": "^_" }], + "@typescript-eslint/consistent-type-imports": ["warn", { "prefer": "type-imports" }], + "import/order": [ + "warn", + { + "newlines-between": "always", + "alphabetize": { "order": "asc", "caseInsensitive": true } + } + ] + } + }, + eslintConfigPrettier +]; diff --git a/examples/README.md b/examples/README.md new file mode 100644 index 0000000..1be29f8 --- /dev/null +++ b/examples/README.md @@ -0,0 +1,14 @@ +# Primer Examples + +This folder includes quick examples to help you get started. + +## CLI usage + +- Analyze a repo: primer analyze /path/to/repo +- Generate instructions: primer instructions --repo /path/to/repo +- Run readiness: primer readiness /path/to/repo +- Run evals: primer eval primer.eval.json --repo /path/to/repo + +## Sample eval config + +See primer.eval.json for a starter eval config you can customize. diff --git a/examples/primer.eval.json b/examples/primer.eval.json new file mode 100644 index 0000000..1294fb1 --- /dev/null +++ b/examples/primer.eval.json @@ -0,0 +1,15 @@ +{ + "instructionFile": ".github/copilot-instructions.md", + "cases": [ + { + "id": "project-overview", + "prompt": "Summarize what this project does and list the main entry points.", + "expectation": "Should mention the primary purpose and key files/directories." + }, + { + "id": "build-commands", + "prompt": "How do I build and test this project?", + "expectation": "Should provide the correct build and test commands from package.json or equivalent." + } + ] +} diff --git a/package.json b/package.json index 2a15771..bf64dc8 100644 --- a/package.json +++ b/package.json @@ -2,9 +2,17 @@ "name": "primer", "version": "1.0.0", "description": "", - "main": "index.js", + "main": "dist/index.js", "scripts": { - "test": "echo \"Error: no test specified\" && exit 1" + "build": "tsc && npm link", + "prepare": "tsc", + "lint": "eslint .", + "format": "prettier --write .", + "format:check": "prettier --check .", + "typecheck": "tsc -p tsconfig.json --noEmit", + "test": "vitest run", + "test:watch": "vitest", + "test:coverage": "vitest run --coverage" }, "keywords": [], "author": "", @@ -23,9 +31,19 @@ "devDependencies": { "@types/node": "^25.1.0", "@types/react": "^19.2.10", + "@typescript-eslint/eslint-plugin": "^8.5.0", + "@typescript-eslint/parser": "^8.5.0", + "@vitest/coverage-v8": "^2.1.4", + "eslint": "^9.7.0", + "eslint-config-prettier": "^9.1.0", + "eslint-plugin-import": "^2.30.0", + "eslint-plugin-n": "^17.10.2", + "eslint-plugin-promise": "^7.1.0", + "prettier": "^3.3.3", "tsup": "^8.5.1", "tsx": "^4.21.0", - "typescript": "^5.9.3" + "typescript": "^5.9.3", + "vitest": "^2.1.4" }, "type": "module", "bin": { diff --git a/primer.eval.json b/primer.eval.json index 3b77578..bc1fa99 100644 --- a/primer.eval.json +++ b/primer.eval.json @@ -7,4 +7,4 @@ "expectation": "Mentions the CLI entrypoint in src/index.ts and that this is the Primer CLI." } ] -} +} \ No newline at end of file diff --git a/release-please-config.json b/release-please-config.json new file mode 100644 index 0000000..0ef4665 --- /dev/null +++ b/release-please-config.json @@ -0,0 +1,9 @@ +{ + "release-type": "node", + "packages": { + ".": { + "package-name": "primer", + "changelog-path": "CHANGELOG.md" + } + } +} diff --git a/release-please-manifest.json b/release-please-manifest.json new file mode 100644 index 0000000..37fcefa --- /dev/null +++ b/release-please-manifest.json @@ -0,0 +1,3 @@ +{ + ".": "1.0.0" +} diff --git a/src/cli.ts b/src/cli.ts index 86bcf74..3a667c1 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -10,6 +10,7 @@ import { evalCommand } from "./commands/eval"; import { tuiCommand } from "./commands/tui"; import { instructionsCommand } from "./commands/instructions"; import { batchCommand } from "./commands/batch"; +import { readinessCommand } from "./commands/readiness"; export function runCli(argv: string[]): void { const program = new Command(); @@ -23,6 +24,7 @@ export function runCli(argv: string[]): void { .command("init") .argument("[path]", "Path to a local repository") .option("--github", "Use a GitHub repository") + .option("--provider ", "Repo provider (github|azure)") .option("--yes", "Accept defaults and skip prompts") .option("--force", "Overwrite existing files") .action(initCommand); @@ -42,8 +44,9 @@ export function runCli(argv: string[]): void { program .command("pr") - .argument("[repo]", "GitHub repo in owner/name form") - .option("--branch ", "Branch name", "primer/add-configs") + .argument("[repo]", "Repo identifier (github: owner/name, azure: org/project/repo)") + .option("--branch ", "Branch name") + .option("--provider ", "Repo provider (github|azure)") .action(prCommand); program @@ -69,10 +72,18 @@ export function runCli(argv: string[]): void { .option("--model ", "Model for instructions generation", "gpt-4.1") .action(instructionsCommand); + program + .command("readiness") + .argument("[path]", "Path to a local repository") + .option("--json", "Output JSON") + .option("--output ", "Write JSON report to file") + .action(readinessCommand); + program .command("batch") .description("Batch process multiple repos across orgs") .option("--output ", "Write results JSON to file") + .option("--provider ", "Repo provider (github|azure)", "github") .action(batchCommand); program.command("templates").action(templatesCommand); diff --git a/src/commands/batch.tsx b/src/commands/batch.tsx index eb69267..bce2c3f 100644 --- a/src/commands/batch.tsx +++ b/src/commands/batch.tsx @@ -2,14 +2,41 @@ import React from "react"; import { render } from "ink"; import { BatchTui } from "../ui/BatchTui"; import { getGitHubToken } from "../services/github"; +import { BatchTuiAzure } from "../ui/BatchTuiAzure"; +import { getAzureDevOpsToken } from "../services/azureDevops"; type BatchOptions = { output?: string; + provider?: string; }; export async function batchCommand(options: BatchOptions): Promise { + const provider = options.provider ?? "github"; + if (provider !== "github" && provider !== "azure") { + console.error("Invalid provider. Use github or azure."); + process.exitCode = 1; + return; + } + + if (provider === "azure") { + const token = getAzureDevOpsToken(); + if (!token) { + console.error("Error: Azure DevOps authentication required."); + console.error(""); + console.error("Set a PAT environment variable:"); + console.error(" export AZURE_DEVOPS_PAT="); + process.exitCode = 1; + return; + } + + const { waitUntilExit } = render( + + ); + await waitUntilExit(); + return; + } + const token = await getGitHubToken(); - if (!token) { console.error("Error: GitHub authentication required."); console.error(""); diff --git a/src/commands/eval.ts b/src/commands/eval.ts index b875f97..bee399d 100644 --- a/src/commands/eval.ts +++ b/src/commands/eval.ts @@ -53,7 +53,7 @@ export async function evalCommand(configPathArg: string | undefined, options: Ev const configPath = path.resolve(configPathArg ?? path.join(repoPath, "primer.eval.json")); - const { summary } = await runEval({ + const { summary, viewerPath } = await runEval({ configPath, repoPath, model: options.model ?? "gpt-5", @@ -62,4 +62,7 @@ export async function evalCommand(configPathArg: string | undefined, options: Ev }); console.log(summary); + if (viewerPath) { + console.log(`Trajectory viewer: ${viewerPath}`); + } } diff --git a/src/commands/init.ts b/src/commands/init.ts index 4f22389..2c6ed94 100644 --- a/src/commands/init.ts +++ b/src/commands/init.ts @@ -4,21 +4,38 @@ import { checkbox, select } from "@inquirer/prompts"; import { analyzeRepo } from "../services/analyzer"; import { generateConfigs } from "../services/generator"; import { GitHubRepo, listAccessibleRepos } from "../services/github"; -import { cloneRepo, isGitRepo } from "../services/git"; +import { + AzureDevOpsOrg, + AzureDevOpsProject, + AzureDevOpsRepo, + getAzureDevOpsToken, + listOrganizations, + listProjects, + listRepos +} from "../services/azureDevops"; +import { buildAuthedUrl, cloneRepo, isGitRepo } from "../services/git"; import { generateCopilotInstructions } from "../services/instructions"; import { ensureDir } from "../utils/fs"; import { prettyPrintSummary } from "../utils/logger"; type InitOptions = { github?: boolean; + provider?: string; yes?: boolean; force?: boolean; }; export async function initCommand(repoPathArg: string | undefined, options: InitOptions): Promise { let repoPath = path.resolve(repoPathArg ?? process.cwd()); + const provider = options.provider ?? (options.github ? "github" : undefined); - if (options.github) { + if (provider && provider !== "github" && provider !== "azure") { + console.error("Invalid provider. Use github or azure."); + process.exitCode = 1; + return; + } + + if (provider === "github") { const token = process.env.GITHUB_TOKEN ?? process.env.GH_TOKEN; if (!token) { console.error("Set GITHUB_TOKEN or GH_TOKEN to use GitHub mode."); @@ -50,6 +67,70 @@ export async function initCommand(repoPathArg: string | undefined, options: Init await cloneRepo(selection.cloneUrl, repoPath); } } + + if (provider === "azure") { + const token = getAzureDevOpsToken(); + if (!token) { + console.error("Set AZURE_DEVOPS_PAT (or AZDO_PAT) to use Azure DevOps mode."); + process.exitCode = 1; + return; + } + + const orgs = await listOrganizations(token); + if (orgs.length === 0) { + console.error("No Azure DevOps organizations found."); + process.exitCode = 1; + return; + } + + const orgSelection = await select({ + message: "Choose an Azure DevOps organization", + choices: orgs.map((org) => ({ + name: org.name, + value: org + })) + }); + + const projects = await listProjects(token, orgSelection.name); + if (projects.length === 0) { + console.error("No Azure DevOps projects found."); + process.exitCode = 1; + return; + } + + const projectSelection = await select({ + message: "Choose an Azure DevOps project", + choices: projects.map((project) => ({ + name: project.name, + value: project + })) + }); + + const repos = await listRepos(token, orgSelection.name, projectSelection.name); + if (repos.length === 0) { + console.error("No Azure DevOps repositories found."); + process.exitCode = 1; + return; + } + + const repoSelection = await select({ + message: "Choose a repository", + choices: repos.map((repo) => ({ + name: `${repo.name}${repo.isPrivate ? " (private)" : ""}`, + value: repo + })) + }); + + const cacheRoot = path.join(process.cwd(), ".primer-cache"); + repoPath = path.join(cacheRoot, orgSelection.name, projectSelection.name, repoSelection.name); + await ensureDir(repoPath); + + const hasGit = await isGitRepo(repoPath); + if (!hasGit) { + const authedUrl = buildAuthedUrl(repoSelection.cloneUrl, token, "azure"); + await cloneRepo(authedUrl, repoPath); + } + } const analysis = await analyzeRepo(repoPath); prettyPrintSummary(analysis); diff --git a/src/commands/pr.ts b/src/commands/pr.ts index d8ce79b..7050840 100644 --- a/src/commands/pr.ts +++ b/src/commands/pr.ts @@ -1,21 +1,88 @@ import path from "path"; +import fs from "fs/promises"; import { analyzeRepo } from "../services/analyzer"; import { generateConfigs } from "../services/generator"; import { createPullRequest, getRepo } from "../services/github"; -import { checkoutBranch, cloneRepo, commitAll, isGitRepo, pushBranch } from "../services/git"; +import { generateCopilotInstructions } from "../services/instructions"; +import { + createPullRequest as createAzurePullRequest, + getAzureDevOpsToken, + getRepo as getAzureRepo +} from "../services/azureDevops"; +import { buildAuthedUrl, checkoutBranch, cloneRepo, commitAll, isGitRepo, pushBranch } from "../services/git"; import { ensureDir } from "../utils/fs"; type PrOptions = { branch?: string; + provider?: string; }; export async function prCommand(repo: string | undefined, options: PrOptions): Promise { + const provider = options.provider ?? "github"; + if (provider !== "github" && provider !== "azure") { + console.error("Invalid provider. Use github or azure."); + process.exitCode = 1; + return; + } + if (!repo) { - console.error("Provide a repo in owner/name form."); + console.error("Provide a repo identifier (github: owner/name, azure: org/project/repo)."); process.exitCode = 1; return; } + if (provider === "azure") { + const token = getAzureDevOpsToken(); + if (!token) { + console.error("Set AZURE_DEVOPS_PAT (or AZDO_PAT) to use Azure DevOps PR automation."); + process.exitCode = 1; + return; + } + + const [organization, project, name] = repo.split("/"); + if (!organization || !project || !name) { + console.error("Invalid Azure DevOps repo format. Use org/project/repo."); + process.exitCode = 1; + return; + } + + const repoInfo = await getAzureRepo(token, organization, project, name); + const cacheRoot = path.join(process.cwd(), ".primer-cache"); + const repoPath = path.join(cacheRoot, organization, project, name); + await ensureDir(repoPath); + + if (!(await isGitRepo(repoPath))) { + const authedUrl = buildAuthedUrl(repoInfo.cloneUrl, token, "azure"); + await cloneRepo(authedUrl, repoPath); + } + + const branch = options.branch ?? "primer/add-instructions"; + await checkoutBranch(repoPath, branch); + + const instructions = await generateCopilotInstructions({ repoPath, model: "gpt-4.1" }); + const instructionsPath = path.join(repoPath, ".github", "copilot-instructions.md"); + await ensureDir(path.dirname(instructionsPath)); + await fs.writeFile(instructionsPath, instructions, "utf8"); + + await commitAll(repoPath, "chore: add copilot instructions via Primer"); + await pushBranch(repoPath, branch, token, "azure"); + + const prUrl = await createAzurePullRequest({ + token, + organization, + project, + repoId: repoInfo.id, + repoName: repoInfo.name, + title: "🤖 Add Copilot instructions via Primer", + body: buildInstructionsPrBody(), + sourceBranch: branch, + targetBranch: repoInfo.defaultBranch + }); + + console.log(`Created PR: ${prUrl}`); + return; + } + const token = process.env.GITHUB_TOKEN ?? process.env.GH_TOKEN; if (!token) { console.error("Set GITHUB_TOKEN or GH_TOKEN to use PR automation."); @@ -89,3 +156,29 @@ function buildPrBody(): string { "*Generated by Primer*" ].join("\n"); } + +function buildInstructionsPrBody(): string { + return [ + "## 🤖 Copilot Instructions Added", + "", + "This PR adds a `.github/copilot-instructions.md` file to help GitHub Copilot understand this codebase better.", + "", + "### What's Included", + "", + "The instructions file contains:", + "- Project overview and architecture", + "- Tech stack and conventions", + "- Build/test commands", + "- Key directories and files", + "", + "### Benefits", + "", + "With these instructions, Copilot will:", + "- Generate more contextually-aware code suggestions", + "- Follow project-specific patterns and conventions", + "- Understand the codebase structure", + "", + "---", + "*Generated by [Primer](https://github.com/pierceboggan/primer) - Prime your repos for AI*" + ].join("\n"); +} diff --git a/src/commands/readiness.ts b/src/commands/readiness.ts new file mode 100644 index 0000000..c2b65fa --- /dev/null +++ b/src/commands/readiness.ts @@ -0,0 +1,120 @@ +import path from "path"; +import fs from "fs/promises"; +import chalk from "chalk"; +import { + ReadinessReport, + ReadinessCriterionResult, + runReadinessReport +} from "../services/readiness"; + +type ReadinessOptions = { + json?: boolean; + output?: string; +}; + +export async function readinessCommand(repoPathArg: string | undefined, options: ReadinessOptions): Promise { + const repoPath = path.resolve(repoPathArg ?? process.cwd()); + const report = await runReadinessReport({ repoPath }); + + if (options.output) { + const outputPath = path.resolve(options.output); + await fs.writeFile(outputPath, JSON.stringify(report, null, 2), "utf8"); + } + + if (options.json) { + console.log(JSON.stringify(report, null, 2)); + return; + } + + printReadinessChecklist(report); +} + +function printReadinessChecklist(report: ReadinessReport): void { + console.log(chalk.bold("Readiness report")); + console.log(`- Repo: ${report.repoPath}`); + console.log(`- Monorepo: ${report.isMonorepo ? "yes" : "no"}${report.apps.length ? ` (${report.apps.length} apps)` : ""}`); + console.log(`- Level: ${report.achievedLevel || 1} (${levelName(report.achievedLevel || 1)})`); + + console.log(chalk.bold("\nPillars")); + for (const pillar of report.pillars) { + const rate = formatPercent(pillar.passRate); + const icon = pillar.passRate >= 0.8 ? chalk.green("●") : chalk.yellow("●"); + console.log(`${icon} ${pillar.name}: ${pillar.passed}/${pillar.total} (${rate})`); + } + + console.log(chalk.bold("\nFix first")); + const fixes = rankFixes(report.criteria); + if (!fixes.length) { + console.log(chalk.green("✔ No failing criteria detected.")); + } else { + for (const fix of fixes) { + const impact = colorImpact(fix.impact); + const effort = colorEffort(fix.effort); + const scope = fix.scope === "app" ? "app" : "repo"; + const detail = fix.appSummary + ? ` (${fix.appSummary.passed}/${fix.appSummary.total} apps)` + : ""; + console.log(`- ${impact} impact / ${effort} effort • ${fix.title}${detail} [${scope}]`); + if (fix.reason) { + console.log(` ${chalk.dim(fix.reason)}`); + } + if (fix.appFailures?.length) { + console.log(` ${chalk.dim(`Apps: ${fix.appFailures.join(", ")}`)}`); + } + } + } + + if (report.extras.length) { + console.log(chalk.bold("\nAI readiness extras")); + for (const extra of report.extras) { + const icon = extra.status === "pass" ? chalk.green("✔") : chalk.red("✖"); + console.log(`${icon} ${extra.title}`); + } + } +} + +function rankFixes(criteria: ReadinessCriterionResult[]): ReadinessCriterionResult[] { + return criteria + .filter((criterion) => criterion.status === "fail") + .sort((a, b) => { + const impactDelta = impactWeight(b.impact) - impactWeight(a.impact); + if (impactDelta !== 0) return impactDelta; + return effortWeight(a.effort) - effortWeight(b.effort); + }); +} + +function impactWeight(value: "high" | "medium" | "low"): number { + if (value === "high") return 3; + if (value === "medium") return 2; + return 1; +} + +function effortWeight(value: "low" | "medium" | "high"): number { + if (value === "low") return 1; + if (value === "medium") return 2; + return 3; +} + +function colorImpact(value: "high" | "medium" | "low"): string { + if (value === "high") return chalk.red("High"); + if (value === "medium") return chalk.yellow("Medium"); + return chalk.green("Low"); +} + +function colorEffort(value: "low" | "medium" | "high"): string { + if (value === "high") return chalk.red("High"); + if (value === "medium") return chalk.yellow("Medium"); + return chalk.green("Low"); +} + +function formatPercent(value: number): string { + return `${Math.round(value * 100)}%`; +} + +function levelName(level: number): string { + if (level === 2) return "Documented"; + if (level === 3) return "Standardized"; + if (level === 4) return "Optimized"; + if (level === 5) return "Autonomous"; + return "Functional"; +} \ No newline at end of file diff --git a/src/services/__tests__/analyzer.test.ts b/src/services/__tests__/analyzer.test.ts new file mode 100644 index 0000000..724678f --- /dev/null +++ b/src/services/__tests__/analyzer.test.ts @@ -0,0 +1,31 @@ +import fs from "fs/promises"; +import os from "os"; +import path from "path"; +import { describe, expect, it } from "vitest"; + +import { analyzeRepo } from "../analyzer"; + +describe("analyzeRepo", () => { + it("detects TypeScript and npm workspace", async () => { + const repoPath = await fs.mkdtemp(path.join(os.tmpdir(), "primer-test-")); + const packageJson = { + name: "demo", + workspaces: ["packages/*"], + dependencies: { react: "^19.0.0" } + }; + + await fs.writeFile(path.join(repoPath, "package.json"), JSON.stringify(packageJson, null, 2)); + await fs.writeFile(path.join(repoPath, "tsconfig.json"), "{}", "utf8"); + await fs.mkdir(path.join(repoPath, "packages", "app"), { recursive: true }); + await fs.writeFile( + path.join(repoPath, "packages", "app", "package.json"), + JSON.stringify({ name: "app", scripts: { build: "tsc" } }, null, 2) + ); + + const result = await analyzeRepo(repoPath); + + expect(result.languages).toContain("TypeScript"); + expect(result.workspaceType).toBe("npm"); + expect(result.apps?.length).toBe(1); + }); +}); diff --git a/src/services/analyzer.ts b/src/services/analyzer.ts index bd1b28a..a9e0566 100644 --- a/src/services/analyzer.ts +++ b/src/services/analyzer.ts @@ -1,13 +1,26 @@ import fs from "fs/promises"; import path from "path"; +import fg from "fast-glob"; import { isGitRepo } from "./git"; +export type RepoApp = { + name: string; + path: string; + packageJsonPath: string; + scripts: Record; + hasTsConfig: boolean; +}; + export type RepoAnalysis = { path: string; isGitRepo: boolean; languages: string[]; frameworks: string[]; packageManager?: string; + isMonorepo?: boolean; + workspaceType?: "npm" | "pnpm" | "yarn"; + workspacePatterns?: string[]; + apps?: RepoApp[]; }; const PACKAGE_MANAGERS: Array<{ file: string; name: string }> = [ @@ -41,15 +54,27 @@ export async function analyzeRepo(repoPath: string): Promise { analysis.packageManager = await detectPackageManager(repoPath, files); + let rootPackageJson: Record | undefined; + if (hasPackageJson) { - const packageJson = await readJson(path.join(repoPath, "package.json")); + rootPackageJson = await readJson(path.join(repoPath, "package.json")); const deps = Object.keys({ - ...(packageJson?.dependencies ?? {}), - ...(packageJson?.devDependencies ?? {}) + ...(rootPackageJson?.dependencies ?? {}), + ...(rootPackageJson?.devDependencies ?? {}) }); analysis.frameworks.push(...detectFrameworks(deps, files)); } + const workspace = await detectWorkspace(repoPath, files, rootPackageJson); + if (workspace) { + analysis.workspaceType = workspace.type; + analysis.workspacePatterns = workspace.patterns; + } + + const apps = await resolveWorkspaceApps(repoPath, workspace?.patterns ?? [], rootPackageJson); + analysis.apps = apps; + analysis.isMonorepo = apps.length > 1; + analysis.languages = unique(analysis.languages); analysis.frameworks = unique(analysis.frameworks); @@ -99,6 +124,117 @@ async function readJson(filePath: string): Promise | und } } +type WorkspaceConfig = { + type: "npm" | "pnpm" | "yarn"; + patterns: string[]; +}; + +async function detectWorkspace( + repoPath: string, + files: string[], + packageJson?: Record +): Promise { + if (files.includes("pnpm-workspace.yaml")) { + const patterns = await readPnpmWorkspace(path.join(repoPath, "pnpm-workspace.yaml")); + if (patterns.length) return { type: "pnpm", patterns }; + } + + const workspaces = packageJson?.workspaces; + if (Array.isArray(workspaces)) { + return { type: files.includes("yarn.lock") ? "yarn" : "npm", patterns: workspaces.map(String) }; + } + + if (workspaces && typeof workspaces === "object") { + const packages = (workspaces as { packages?: unknown }).packages; + if (Array.isArray(packages)) { + return { type: files.includes("yarn.lock") ? "yarn" : "npm", patterns: packages.map(String) }; + } + } + + return undefined; +} + +async function readPnpmWorkspace(filePath: string): Promise { + try { + const raw = await fs.readFile(filePath, "utf8"); + const lines = raw.split(/\r?\n/u); + const patterns: string[] = []; + let inPackages = false; + for (const line of lines) { + if (!inPackages && /^\s*packages\s*:/u.test(line)) { + inPackages = true; + continue; + } + if (inPackages) { + const match = line.match(/^\s*-\s*(.+)$/u); + if (match?.[1]) { + patterns.push(match[1].trim().replace(/^['"]|['"]$/gu, "")); + continue; + } + if (/^\S/u.test(line)) break; + } + } + return patterns; + } catch { + return []; + } +} + +async function resolveWorkspaceApps( + repoPath: string, + patterns: string[], + rootPackageJson?: Record +): Promise { + const workspacePatterns = patterns + .map((pattern) => pattern.replace(/\\/gu, "/")) + .map((pattern) => (pattern.endsWith("package.json") ? pattern : path.posix.join(pattern, "package.json"))); + + const packageJsonPaths = workspacePatterns.length + ? await fg(workspacePatterns, { cwd: repoPath, absolute: true, onlyFiles: true, dot: false }) + : []; + + if (!packageJsonPaths.length && rootPackageJson) { + const rootPath = path.join(repoPath, "package.json"); + return [await buildRepoApp(repoPath, rootPath, rootPackageJson)]; + } + + const apps = await Promise.all( + packageJsonPaths.map(async (pkgPath) => { + const pkg = await readJson(pkgPath); + return buildRepoApp(path.dirname(pkgPath), pkgPath, pkg); + }) + ); + + return apps.filter(Boolean) as RepoApp[]; +} + +async function buildRepoApp( + appPath: string, + packageJsonPath: string, + packageJson?: Record +): Promise { + const scripts = (packageJson?.scripts ?? {}) as Record; + const name = typeof packageJson?.name === "string" ? packageJson.name : path.basename(appPath); + const hasTsConfig = await fileExists(path.join(appPath, "tsconfig.json")); + + return { + name, + path: appPath, + packageJsonPath, + scripts, + hasTsConfig + }; +} + +async function fileExists(filePath: string): Promise { + try { + await fs.access(filePath); + return true; + } catch { + return false; + } +} + function unique(items: T[]): T[] { return Array.from(new Set(items)); } diff --git a/src/services/azureDevops.ts b/src/services/azureDevops.ts new file mode 100644 index 0000000..bebe975 --- /dev/null +++ b/src/services/azureDevops.ts @@ -0,0 +1,235 @@ +type AzureDevOpsProfileResponse = { + id: string; + displayName: string; +}; + +type AzureDevOpsAccountResponse = { + accountId: string; + accountName: string; + accountUri: string; +}; + +type AzureDevOpsListResponse = { + value: T[]; +}; + +type AzureDevOpsProjectResponse = { + id: string; + name: string; + url: string; +}; + +type AzureDevOpsRepoResponse = { + id: string; + name: string; + webUrl: string; + remoteUrl: string; + isPrivate: boolean; + defaultBranch?: string; + project?: { + id: string; + name: string; + }; +}; + +export type AzureDevOpsOrg = { + id: string; + name: string; + url: string; +}; + +export type AzureDevOpsProject = { + id: string; + name: string; + organization: string; + url: string; +}; + +export type AzureDevOpsRepo = { + id: string; + name: string; + organization: string; + project: string; + projectId: string; + webUrl: string; + cloneUrl: string; + isPrivate: boolean; + defaultBranch: string; + hasInstructions?: boolean; +}; + +const PROFILE_URL = "https://app.vssps.visualstudio.com/_apis/profile/profiles/me?api-version=7.1-preview.1"; + +function getAuthHeader(token: string): string { + const encoded = Buffer.from(`:${token}`).toString("base64"); + return `Basic ${encoded}`; +} + +async function adoRequest(url: string, token: string, init?: RequestInit): Promise { + const response = await fetch(url, { + ...init, + headers: { + "Content-Type": "application/json", + Authorization: getAuthHeader(token), + ...(init?.headers ?? {}) + } + }); + + if (!response.ok) { + const text = await response.text(); + throw new Error(`Azure DevOps request failed (${response.status}): ${text}`); + } + + return (await response.json()) as T; +} + +export function getAzureDevOpsToken(): string | null { + return process.env.AZURE_DEVOPS_PAT ?? process.env.AZDO_PAT ?? null; +} + +export async function listOrganizations(token: string): Promise { + const profile = await adoRequest(PROFILE_URL, token); + const accountsUrl = `https://app.vssps.visualstudio.com/_apis/accounts?memberId=${profile.id}&api-version=7.1-preview.1`; + const accounts = await adoRequest>(accountsUrl, token); + + return accounts.value.map((account) => ({ + id: account.accountId, + name: account.accountName, + url: account.accountUri + })); +} + +export async function listProjects(token: string, organization: string): Promise { + const url = `https://dev.azure.com/${organization}/_apis/projects?stateFilter=wellFormed&api-version=7.1-preview.1`; + const response = await adoRequest>(url, token); + + return response.value.map((project) => ({ + id: project.id, + name: project.name, + organization, + url: project.url + })); +} + +export async function listRepos(token: string, organization: string, project: string): Promise { + const url = `https://dev.azure.com/${organization}/${project}/_apis/git/repositories?api-version=7.1-preview.1`; + const response = await adoRequest>(url, token); + + return response.value.map((repo) => ({ + id: repo.id, + name: repo.name, + organization, + project, + projectId: repo.project?.id ?? "", + webUrl: repo.webUrl, + cloneUrl: repo.remoteUrl, + isPrivate: repo.isPrivate, + defaultBranch: repo.defaultBranch ?? "refs/heads/main" + })); +} + +export async function getRepo( + token: string, + organization: string, + project: string, + repo: string +): Promise { + const url = `https://dev.azure.com/${organization}/${project}/_apis/git/repositories/${repo}?api-version=7.1-preview.1`; + const response = await adoRequest(url, token); + + return { + id: response.id, + name: response.name, + organization, + project, + projectId: response.project?.id ?? "", + webUrl: response.webUrl, + cloneUrl: response.remoteUrl, + isPrivate: response.isPrivate, + defaultBranch: response.defaultBranch ?? "refs/heads/main" + }; +} + +function toRefName(branch: string): string { + if (branch.startsWith("refs/")) return branch; + return `refs/heads/${branch}`; +} + +export async function createPullRequest(params: { + token: string; + organization: string; + project: string; + repoId: string; + repoName: string; + title: string; + body: string; + sourceBranch: string; + targetBranch: string; +}): Promise { + const url = `https://dev.azure.com/${params.organization}/${params.project}/_apis/git/repositories/${params.repoId}/pullrequests?api-version=7.1-preview.1`; + const payload = { + title: params.title, + description: params.body, + sourceRefName: toRefName(params.sourceBranch), + targetRefName: toRefName(params.targetBranch) + }; + + const response = await adoRequest<{ pullRequestId: number }>(url, params.token, { + method: "POST", + body: JSON.stringify(payload) + }); + + return `https://dev.azure.com/${params.organization}/${params.project}/_git/${encodeURIComponent( + params.repoName + )}/pullrequest/${response.pullRequestId}`; +} + +export async function checkRepoHasInstructions( + token: string, + organization: string, + project: string, + repoId: string +): Promise { + const url = `https://dev.azure.com/${organization}/${project}/_apis/git/repositories/${repoId}/items?path=/.github/copilot-instructions.md&includeContentMetadata=true&api-version=7.1-preview.1`; + const response = await fetch(url, { + headers: { + Authorization: getAuthHeader(token) + } + }); + + if (response.status === 404) { + return false; + } + + return response.ok; +} + +export async function checkReposForInstructions( + token: string, + repos: AzureDevOpsRepo[], + onProgress?: (checked: number, total: number) => void +): Promise { + const concurrency = 10; + const results: AzureDevOpsRepo[] = []; + let checked = 0; + + for (let i = 0; i < repos.length; i += concurrency) { + const batch = repos.slice(i, i + concurrency); + const checks = await Promise.all( + batch.map(async (repo) => { + const hasInstructions = await checkRepoHasInstructions( + token, + repo.organization, + repo.project, + repo.id + ); + return { ...repo, hasInstructions }; + }) + ); + results.push(...checks); + checked += batch.length; + onProgress?.(checked, repos.length); + } + + return results; +} \ No newline at end of file diff --git a/src/services/evaluator.ts b/src/services/evaluator.ts index 6a30001..adddce6 100644 --- a/src/services/evaluator.ts +++ b/src/services/evaluator.ts @@ -15,6 +15,7 @@ type EvalConfig = { instructionFile?: string; cases: EvalCase[]; systemMessage?: string; + outputPath?: string; }; type EvalRunOptions = { @@ -26,6 +27,40 @@ type EvalRunOptions = { onProgress?: (message: string) => void; }; +type TokenUsage = { + promptTokens?: number; + completionTokens?: number; + totalTokens?: number; +}; + +type ToolCallSummary = { + count: number; + byName: Record; + totalDurationMs: number; +}; + +type AskMetrics = { + durationMs: number; + tokenUsage?: TokenUsage; + toolCalls: ToolCallSummary; +}; + +type EvalMetrics = { + withoutInstructions: AskMetrics; + withInstructions: AskMetrics; + judge: AskMetrics; + totalDurationMs: number; +}; + +type EvalPhase = "withoutInstructions" | "withInstructions" | "judge"; + +type TrajectoryEvent = { + timestampMs: number; + phase: EvalPhase; + type: string; + data?: Record; +}; + export type EvalResult = { id: string; prompt: string; @@ -35,14 +70,18 @@ export type EvalResult = { verdict?: "pass" | "fail" | "unknown"; score?: number; rationale?: string; + metrics?: EvalMetrics; + trajectory?: TrajectoryEvent[]; }; -export async function runEval(options: EvalRunOptions): Promise<{ summary: string; results: EvalResult[] }> { +export async function runEval(options: EvalRunOptions): Promise<{ summary: string; results: EvalResult[]; viewerPath?: string }> { const config = await loadConfig(options.configPath); const instructionFile = config.instructionFile ?? ".github/copilot-instructions.md"; const instructionPath = path.resolve(options.repoPath, instructionFile); const instructionText = await readOptionalFile(instructionPath); const progress = options.onProgress ?? (() => {}); + const outputPath = resolveOutputPath(options.repoPath, options.outputPath, config.outputPath); + const runStartedAt = Date.now(); progress("Starting Copilot SDK..."); const cliPath = await findCopilotCliPath(); @@ -56,19 +95,22 @@ export async function runEval(options: EvalRunOptions): Promise<{ summary: strin for (const [index, testCase] of config.cases.entries()) { const id = testCase.id ?? `case-${index + 1}`; const prompt = buildPrompt(options.repoPath, testCase.prompt); + const caseStartedAt = Date.now(); progress(`Running eval ${index + 1}/${total}: ${id} (without instructions)...`); - const withoutInstructions = await askOnce(client, { + const withoutResult = await askOnce(client, { prompt, model: options.model, - systemMessage: config.systemMessage + systemMessage: config.systemMessage, + phase: "withoutInstructions" }); progress(`Running eval ${index + 1}/${total}: ${id} (with instructions)...`); - const withInstructions = await askOnce(client, { + const withResult = await askOnce(client, { prompt, model: options.model, - systemMessage: [config.systemMessage, instructionText].filter(Boolean).join("\n\n") + systemMessage: [config.systemMessage, instructionText].filter(Boolean).join("\n\n"), + phase: "withInstructions" }); progress(`Running eval ${index + 1}/${total}: ${id} (judging)...`); @@ -76,36 +118,60 @@ export async function runEval(options: EvalRunOptions): Promise<{ summary: strin model: options.judgeModel, prompt: testCase.prompt, expectation: testCase.expectation, - withoutInstructions, - withInstructions + withoutInstructions: withoutResult.content, + withInstructions: withResult.content }); + const metrics: EvalMetrics = { + withoutInstructions: withoutResult.metrics, + withInstructions: withResult.metrics, + judge: judgment.metrics, + totalDurationMs: Date.now() - caseStartedAt + }; + + const trajectory = [ + ...withoutResult.trajectory, + ...withResult.trajectory, + ...judgment.trajectory + ]; + results.push({ id, prompt: testCase.prompt, expectation: testCase.expectation, - withInstructions, - withoutInstructions, - verdict: judgment.verdict, - score: judgment.score, - rationale: judgment.rationale + withInstructions: withResult.content, + withoutInstructions: withoutResult.content, + verdict: judgment.result.verdict, + score: judgment.result.score, + rationale: judgment.result.rationale, + metrics, + trajectory }); - progress(`Eval ${index + 1}/${total}: ${id} → ${judgment.verdict} (score: ${judgment.score})`); + progress(`Eval ${index + 1}/${total}: ${id} → ${judgment.result.verdict} (score: ${judgment.result.score})`); } - if (options.outputPath) { - const output = { - repoPath: options.repoPath, - model: options.model, - judgeModel: options.judgeModel, - results - }; - await fs.writeFile(options.outputPath, JSON.stringify(output, null, 2), "utf8"); + const runFinishedAt = Date.now(); + const output = { + repoPath: options.repoPath, + model: options.model, + judgeModel: options.judgeModel, + runMetrics: { + startedAt: new Date(runStartedAt).toISOString(), + finishedAt: new Date(runFinishedAt).toISOString(), + durationMs: runFinishedAt - runStartedAt + }, + results + }; + let viewerPath: string | undefined; + if (outputPath) { + await fs.writeFile(outputPath, JSON.stringify(output, null, 2), "utf8"); + viewerPath = buildViewerPath(outputPath); + await fs.writeFile(viewerPath, buildTrajectoryViewerHtml(output), "utf8"); } - const summary = formatSummary(results); - return { summary, results }; + const summary = formatSummary(results, runFinishedAt - runStartedAt); + return { summary, results, viewerPath }; } finally { await client.stop(); } @@ -115,12 +181,19 @@ type AskOptions = { prompt: string; model: string; systemMessage?: string; + phase: EvalPhase; +}; + +type AskResult = { + content: string; + metrics: AskMetrics; + trajectory: TrajectoryEvent[]; }; async function askOnce( client: { createSession: (config?: Record) => Promise }, options: AskOptions -): Promise { +): Promise { const session = await client.createSession({ model: options.model, streaming: true, @@ -131,7 +204,10 @@ async function askOnce( }); let content = ""; + const telemetry = createTelemetry(options.phase); + const startedAt = Date.now(); session.on((event: { type: string; data?: Record }) => { + captureTelemetryEvent(event, telemetry); if (event.type === "assistant.message_delta") { const delta = event.data?.deltaContent as string | undefined; if (delta) content += delta; @@ -140,7 +216,16 @@ async function askOnce( await session.sendAndWait({ prompt: options.prompt }, 120000); await session.destroy(); - return content.trim(); + const finishedAt = Date.now(); + return { + content: content.trim(), + metrics: { + durationMs: finishedAt - startedAt, + tokenUsage: normalizeTokenUsage(telemetry.tokenUsage), + toolCalls: telemetry.toolCalls + }, + trajectory: telemetry.trajectory + }; } type JudgeOptions = { @@ -160,7 +245,7 @@ type JudgeResult = { async function judge( client: { createSession: (config?: Record) => Promise }, options: JudgeOptions -): Promise { +): Promise<{ result: JudgeResult; metrics: AskMetrics; trajectory: TrajectoryEvent[] }> { const session = await client.createSession({ model: options.model, streaming: true, @@ -171,7 +256,10 @@ async function judge( }); let content = ""; + const telemetry = createTelemetry("judge"); + const startedAt = Date.now(); session.on((event: { type: string; data?: Record }) => { + captureTelemetryEvent(event, telemetry); if (event.type === "assistant.message_delta") { const delta = event.data?.deltaContent as string | undefined; if (delta) content += delta; @@ -195,7 +283,16 @@ async function judge( await session.sendAndWait({ prompt }, 120000); await session.destroy(); - return parseJudge(content); + const finishedAt = Date.now(); + return { + result: parseJudge(content), + metrics: { + durationMs: finishedAt - startedAt, + tokenUsage: normalizeTokenUsage(telemetry.tokenUsage), + toolCalls: telemetry.toolCalls + }, + trajectory: telemetry.trajectory + }; } function parseJudge(content: string): JudgeResult { @@ -271,14 +368,18 @@ function buildPrompt(repoPath: string, userPrompt: string): string { ].join("\n"); } -function formatSummary(results: EvalResult[]): string { +function formatSummary(results: EvalResult[], runDurationMs: number): string { const total = results.length; const passed = results.filter((r) => r.verdict === "pass").length; const failed = results.filter((r) => r.verdict === "fail").length; const unknown = results.filter((r) => r.verdict === "unknown").length; + const totalUsage = aggregateTokenUsage(results); + const hasUsage = Boolean(totalUsage.promptTokens || totalUsage.completionTokens || totalUsage.totalTokens); const lines = [ - `Eval results: ${passed}/${total} pass, ${failed} fail, ${unknown} unknown.` + `Eval results: ${passed}/${total} pass, ${failed} fail, ${unknown} unknown.`, + `Runtime: ${formatDuration(runDurationMs)}.`, + hasUsage ? `Token usage: ${formatTokenUsage(totalUsage)}.` : "Token usage: unavailable." ]; for (const result of results) { @@ -289,3 +390,305 @@ function formatSummary(results: EvalResult[]): string { return `\n${lines.join("\n")}`; } + +type TelemetryCollector = { + trajectory: TrajectoryEvent[]; + tokenUsage: TokenUsage; + toolCalls: ToolCallSummary; + toolCallMap: Map; + phase: EvalPhase; +}; + +function createTelemetry(phase: EvalPhase): TelemetryCollector { + return { + trajectory: [], + tokenUsage: {}, + toolCalls: { count: 0, byName: {}, totalDurationMs: 0 }, + toolCallMap: new Map(), + phase + }; +} + +function captureTelemetryEvent( + event: { type: string; data?: Record }, + telemetry: TelemetryCollector +): void { + const timestampMs = Date.now(); + telemetry.trajectory.push({ + timestampMs, + phase: telemetry.phase, + type: event.type, + data: sanitizeEventData(event.data) + }); + + if (event.type === "tool.execution_start") { + const toolName = (event.data?.toolName as string | undefined) ?? "unknown"; + const toolId = resolveToolId(event.data, toolName, telemetry.toolCallMap.size); + telemetry.toolCallMap.set(toolId, { name: toolName, startMs: timestampMs }); + telemetry.toolCalls.count += 1; + telemetry.toolCalls.byName[toolName] = (telemetry.toolCalls.byName[toolName] ?? 0) + 1; + } else if (event.type === "tool.execution_finish" || event.type === "tool.execution_error") { + const toolName = (event.data?.toolName as string | undefined) ?? "unknown"; + const toolId = resolveToolId(event.data, toolName, telemetry.toolCallMap.size); + const entry = telemetry.toolCallMap.get(toolId) ?? findLatestToolByName(telemetry.toolCallMap, toolName); + if (entry) { + const durationMs = timestampMs - entry.startMs; + telemetry.toolCalls.totalDurationMs += durationMs; + telemetry.toolCallMap.delete(toolId); + } + } + + const usage = extractTokenUsage(event.data); + if (usage) { + telemetry.tokenUsage = mergeTokenUsage(telemetry.tokenUsage, usage); + } +} + +function resolveToolId( + data: Record | undefined, + toolName: string, + index: number +): string { + const rawId = data?.executionId ?? data?.toolCallId ?? data?.callId ?? data?.id; + if (typeof rawId === "string" || typeof rawId === "number") { + return String(rawId); + } + return `${toolName}-${index + 1}`; +} + +function findLatestToolByName( + map: Map, + toolName: string +): { name?: string; startMs: number } | undefined { + const entries = Array.from(map.values()).filter((entry) => entry.name === toolName); + return entries.at(-1); +} + +function extractTokenUsage(data: Record | undefined): TokenUsage | null { + if (!data) return null; + const usage = (data.usage ?? data.tokenUsage ?? data.tokens) as Record | undefined; + const promptTokens = getNumber(usage?.prompt_tokens ?? usage?.promptTokens ?? data.promptTokens ?? data.inputTokens); + const completionTokens = getNumber(usage?.completion_tokens ?? usage?.completionTokens ?? data.completionTokens ?? data.outputTokens); + const totalTokens = getNumber(usage?.total_tokens ?? usage?.totalTokens ?? data.totalTokens); + + if (promptTokens == null && completionTokens == null && totalTokens == null) { + return null; + } + + return { + promptTokens: promptTokens ?? undefined, + completionTokens: completionTokens ?? undefined, + totalTokens: totalTokens ?? undefined + }; +} + +function getNumber(value: unknown): number | null { + if (typeof value === "number" && Number.isFinite(value)) return value; + if (typeof value === "string" && value.trim()) { + const parsed = Number(value); + return Number.isFinite(parsed) ? parsed : null; + } + return null; +} + +function mergeTokenUsage(existing: TokenUsage, next: TokenUsage): TokenUsage { + return { + promptTokens: Math.max(existing.promptTokens ?? 0, next.promptTokens ?? 0) || undefined, + completionTokens: Math.max(existing.completionTokens ?? 0, next.completionTokens ?? 0) || undefined, + totalTokens: Math.max(existing.totalTokens ?? 0, next.totalTokens ?? 0) || undefined + }; +} + +function normalizeTokenUsage(usage: TokenUsage): TokenUsage | undefined { + if (!usage.promptTokens && !usage.completionTokens && !usage.totalTokens) return undefined; + return usage; +} + +function aggregateTokenUsage(results: EvalResult[]): TokenUsage { + const total: TokenUsage = { promptTokens: 0, completionTokens: 0, totalTokens: 0 }; + for (const result of results) { + const metrics = result.metrics; + if (!metrics) continue; + const usages = [metrics.withoutInstructions.tokenUsage, metrics.withInstructions.tokenUsage, metrics.judge.tokenUsage]; + for (const usage of usages) { + if (!usage) continue; + total.promptTokens = (total.promptTokens ?? 0) + (usage.promptTokens ?? 0); + total.completionTokens = (total.completionTokens ?? 0) + (usage.completionTokens ?? 0); + total.totalTokens = (total.totalTokens ?? 0) + (usage.totalTokens ?? 0); + } + } + return total; +} + +function formatDuration(durationMs: number): string { + const seconds = Math.round(durationMs / 100) / 10; + if (seconds < 60) return `${seconds}s`; + const minutes = Math.floor(seconds / 60); + const remaining = Math.round((seconds % 60) * 10) / 10; + return `${minutes}m ${remaining}s`; +} + +function formatTokenUsage(usage: TokenUsage): string { + const prompt = usage.promptTokens ?? 0; + const completion = usage.completionTokens ?? 0; + const total = usage.totalTokens ?? prompt + completion; + return `prompt ${prompt}, completion ${completion}, total ${total}`; +} + +function resolveOutputPath(repoPath: string, override?: string, configValue?: string): string | undefined { + const chosen = override ?? configValue; + if (!chosen) return undefined; + return path.isAbsolute(chosen) ? chosen : path.resolve(repoPath, chosen); +} + +function buildViewerPath(outputPath: string): string { + if (outputPath.endsWith(".json")) { + return outputPath.replace(/\.json$/u, ".html"); + } + return `${outputPath}.html`; +} + +function buildTrajectoryViewerHtml(data: Record): string { + const serialized = JSON.stringify(data).replace(/ + + + + + Primer Eval Trajectory + + + +

Primer Eval Trajectory

+
+
+
+
+
+ + +`; +} + +function sanitizeEventData(data: Record | undefined): Record | undefined { + if (!data) return undefined; + const sanitized: Record = {}; + for (const [key, value] of Object.entries(data)) { + if (key === "deltaContent" && typeof value === "string") { + sanitized.deltaChars = value.length; + sanitized.deltaPreview = value.slice(0, 120); + continue; + } + sanitized[key] = sanitizeValue(value, 0); + } + return sanitized; +} + +function sanitizeValue(value: unknown, depth: number): unknown { + if (depth > 4) return "[depth-limit]"; + if (typeof value === "string") { + return value.length > 2000 ? `${value.slice(0, 2000)}…` : value; + } + if (Array.isArray(value)) { + return value.slice(0, 50).map((entry) => sanitizeValue(entry, depth + 1)); + } + if (value && typeof value === "object") { + const obj: Record = {}; + for (const [key, entry] of Object.entries(value as Record)) { + obj[key] = sanitizeValue(entry, depth + 1); + } + return obj; + } + return value; +} diff --git a/src/services/git.ts b/src/services/git.ts index a8b1e79..5eceb3a 100644 --- a/src/services/git.ts +++ b/src/services/git.ts @@ -65,6 +65,8 @@ export async function commitAll(repoPath: string, message: string): Promise { +export function buildAuthedUrl(url: string, token: string, provider: AuthProvider): string { + const normalizedUrl = normalizeGitUrl(url); + if (!normalizedUrl.startsWith("https://")) return normalizedUrl; + if (provider === "azure") { + return normalizedUrl.replace("https://", `https://pat:${token}@`); + } + return normalizedUrl.replace("https://", `https://x-access-token:${token}@`); +} + +export async function pushBranch( + repoPath: string, + branch: string, + token?: string, + provider: AuthProvider = "github" +): Promise { const git = simpleGit(repoPath); if (token) { @@ -85,7 +103,7 @@ export async function pushBranch(repoPath: string, branch: string, token?: strin const remoteUrl = (await git.remote(["get-url", "origin"])) ?? ""; const normalizedUrl = normalizeGitUrl(remoteUrl); if (normalizedUrl.startsWith("https://")) { - const authedUrl = normalizedUrl.replace("https://", `https://x-access-token:${token}@`); + const authedUrl = buildAuthedUrl(normalizedUrl, token, provider); await git.remote(["set-url", "origin", authedUrl]); try { await git.push(["-u", "origin", branch]); diff --git a/src/services/readiness.ts b/src/services/readiness.ts new file mode 100644 index 0000000..ea55e77 --- /dev/null +++ b/src/services/readiness.ts @@ -0,0 +1,648 @@ +import fs from "fs/promises"; +import path from "path"; +import { analyzeRepo, RepoApp, RepoAnalysis } from "./analyzer"; + +export type ReadinessPillar = + | "style-validation" + | "build-system" + | "testing" + | "documentation" + | "dev-environment" + | "code-quality" + | "observability" + | "security-governance"; + +export type ReadinessScope = "repo" | "app"; + +export type ReadinessStatus = "pass" | "fail" | "skip"; + +export type ReadinessCriterionResult = { + id: string; + title: string; + pillar: ReadinessPillar; + level: number; + scope: ReadinessScope; + impact: "high" | "medium" | "low"; + effort: "low" | "medium" | "high"; + status: ReadinessStatus; + reason?: string; + evidence?: string[]; + passRate?: number; + appSummary?: { passed: number; total: number }; + appFailures?: string[]; +}; + +export type ReadinessExtraResult = { + id: string; + title: string; + status: ReadinessStatus; + reason?: string; +}; + +export type ReadinessPillarSummary = { + id: ReadinessPillar; + name: string; + passed: number; + total: number; + passRate: number; +}; + +export type ReadinessLevelSummary = { + level: number; + name: string; + passed: number; + total: number; + passRate: number; + achieved: boolean; +}; + +export type ReadinessReport = { + repoPath: string; + generatedAt: string; + isMonorepo: boolean; + apps: Array<{ name: string; path: string }>; + pillars: ReadinessPillarSummary[]; + levels: ReadinessLevelSummary[]; + achievedLevel: number; + criteria: ReadinessCriterionResult[]; + extras: ReadinessExtraResult[]; +}; + +type ReadinessOptions = { + repoPath: string; + includeExtras?: boolean; +}; + +type ReadinessContext = { + repoPath: string; + analysis: RepoAnalysis; + apps: RepoApp[]; + rootFiles: string[]; + rootPackageJson?: Record; +}; + +type ReadinessCriterion = { + id: string; + title: string; + pillar: ReadinessPillar; + level: number; + scope: ReadinessScope; + impact: "high" | "medium" | "low"; + effort: "low" | "medium" | "high"; + check: (context: ReadinessContext, app?: RepoApp) => Promise; +}; + +type CheckResult = { + status: ReadinessStatus; + reason?: string; + evidence?: string[]; +}; + +export async function runReadinessReport(options: ReadinessOptions): Promise { + const repoPath = options.repoPath; + const analysis = await analyzeRepo(repoPath); + const rootFiles = await safeReadDir(repoPath); + const rootPackageJson = await readJson(path.join(repoPath, "package.json")); + const apps = analysis.apps?.length ? analysis.apps : []; + + const context: ReadinessContext = { + repoPath, + analysis, + apps, + rootFiles, + rootPackageJson + }; + + const criteria = buildCriteria(); + const criteriaResults: ReadinessCriterionResult[] = []; + + for (const criterion of criteria) { + if (criterion.scope === "repo") { + const result = await criterion.check(context); + criteriaResults.push({ + id: criterion.id, + title: criterion.title, + pillar: criterion.pillar, + level: criterion.level, + scope: criterion.scope, + impact: criterion.impact, + effort: criterion.effort, + status: result.status, + reason: result.reason, + evidence: result.evidence + }); + continue; + } + + const appResults = await Promise.all( + apps.map(async (app) => ({ + app, + result: await criterion.check(context, app) + })) + ); + + if (!appResults.length) { + criteriaResults.push({ + id: criterion.id, + title: criterion.title, + pillar: criterion.pillar, + level: criterion.level, + scope: criterion.scope, + impact: criterion.impact, + effort: criterion.effort, + status: "skip", + reason: "No application packages detected." + }); + continue; + } + + const passed = appResults.filter((entry) => entry.result.status === "pass").length; + const total = appResults.length; + const passRate = total ? passed / total : 0; + const status: ReadinessStatus = passRate >= 0.8 ? "pass" : "fail"; + const failures = appResults + .filter((entry) => entry.result.status !== "pass") + .map((entry) => entry.app.name); + + criteriaResults.push({ + id: criterion.id, + title: criterion.title, + pillar: criterion.pillar, + level: criterion.level, + scope: criterion.scope, + impact: criterion.impact, + effort: criterion.effort, + status, + reason: status === "pass" ? undefined : `Only ${passed}/${total} apps pass this check.`, + passRate, + appSummary: { passed, total }, + appFailures: failures + }); + } + + const pillars = summarizePillars(criteriaResults); + const levels = summarizeLevels(criteriaResults); + const achievedLevel = levels.filter((level) => level.achieved).reduce((acc, level) => Math.max(acc, level.level), 0); + + const extras = options.includeExtras === false ? [] : await runExtras(context); + + return { + repoPath, + generatedAt: new Date().toISOString(), + isMonorepo: analysis.isMonorepo ?? false, + apps: apps.map((app) => ({ name: app.name, path: app.path })), + pillars, + levels, + achievedLevel, + criteria: criteriaResults, + extras + }; +} + +function buildCriteria(): ReadinessCriterion[] { + return [ + { + id: "lint-config", + title: "Linting configured", + pillar: "style-validation", + level: 1, + scope: "repo", + impact: "high", + effort: "low", + check: async (context) => ({ + status: (await hasLintConfig(context.repoPath)) ? "pass" : "fail", + reason: "Missing ESLint/Biome/Prettier configuration.", + evidence: ["eslint.config.js", ".eslintrc", "biome.json", ".prettierrc"] + }) + }, + { + id: "typecheck-config", + title: "Type checking configured", + pillar: "style-validation", + level: 2, + scope: "repo", + impact: "medium", + effort: "low", + check: async (context) => ({ + status: (await hasTypecheckConfig(context.repoPath)) ? "pass" : "fail", + reason: "Missing type checking config (tsconfig or equivalent).", + evidence: ["tsconfig.json", "pyproject.toml", "mypy.ini"] + }) + }, + { + id: "build-script", + title: "Build script present", + pillar: "build-system", + level: 1, + scope: "app", + impact: "high", + effort: "low", + check: async (_context, app) => ({ + status: app?.scripts?.build ? "pass" : "fail", + reason: "Missing build script in package.json." + }) + }, + { + id: "ci-config", + title: "CI workflow configured", + pillar: "build-system", + level: 2, + scope: "repo", + impact: "high", + effort: "medium", + check: async (context) => ({ + status: (await hasGithubWorkflows(context.repoPath)) ? "pass" : "fail", + reason: "Missing .github/workflows CI configuration.", + evidence: [".github/workflows"] + }) + }, + { + id: "test-script", + title: "Test script present", + pillar: "testing", + level: 1, + scope: "app", + impact: "high", + effort: "low", + check: async (_context, app) => ({ + status: app?.scripts?.test ? "pass" : "fail", + reason: "Missing test script in package.json." + }) + }, + { + id: "readme", + title: "README present", + pillar: "documentation", + level: 1, + scope: "repo", + impact: "high", + effort: "low", + check: async (context) => ({ + status: (await hasReadme(context.repoPath)) ? "pass" : "fail", + reason: "Missing README documentation.", + evidence: ["README.md"] + }) + }, + { + id: "contributing", + title: "CONTRIBUTING guide present", + pillar: "documentation", + level: 2, + scope: "repo", + impact: "medium", + effort: "low", + check: async (context) => ({ + status: (await fileExists(path.join(context.repoPath, "CONTRIBUTING.md"))) ? "pass" : "fail", + reason: "Missing CONTRIBUTING.md for contributor workflows." + }) + }, + { + id: "lockfile", + title: "Lockfile present", + pillar: "dev-environment", + level: 1, + scope: "repo", + impact: "high", + effort: "low", + check: async (context) => ({ + status: hasAnyFile(context.rootFiles, ["pnpm-lock.yaml", "yarn.lock", "package-lock.json", "bun.lockb"]) ? "pass" : "fail", + reason: "Missing package manager lockfile." + }) + }, + { + id: "env-example", + title: "Environment example present", + pillar: "dev-environment", + level: 2, + scope: "repo", + impact: "medium", + effort: "low", + check: async (context) => ({ + status: hasAnyFile(context.rootFiles, [".env.example", ".env.sample"]) ? "pass" : "fail", + reason: "Missing .env.example or .env.sample for setup guidance." + }) + }, + { + id: "format-config", + title: "Formatter configured", + pillar: "code-quality", + level: 2, + scope: "repo", + impact: "medium", + effort: "low", + check: async (context) => ({ + status: (await hasFormatterConfig(context.repoPath)) ? "pass" : "fail", + reason: "Missing Prettier/Biome formatting config." + }) + }, + { + id: "codeowners", + title: "CODEOWNERS present", + pillar: "security-governance", + level: 2, + scope: "repo", + impact: "medium", + effort: "low", + check: async (context) => ({ + status: (await hasCodeowners(context.repoPath)) ? "pass" : "fail", + reason: "Missing CODEOWNERS file." + }) + }, + { + id: "license", + title: "LICENSE present", + pillar: "security-governance", + level: 1, + scope: "repo", + impact: "medium", + effort: "low", + check: async (context) => ({ + status: (await hasLicense(context.repoPath)) ? "pass" : "fail", + reason: "Missing LICENSE file." + }) + }, + { + id: "security-policy", + title: "Security policy present", + pillar: "security-governance", + level: 3, + scope: "repo", + impact: "high", + effort: "low", + check: async (context) => ({ + status: (await fileExists(path.join(context.repoPath, "SECURITY.md"))) ? "pass" : "fail", + reason: "Missing SECURITY.md policy." + }) + }, + { + id: "dependabot", + title: "Dependabot configured", + pillar: "security-governance", + level: 3, + scope: "repo", + impact: "medium", + effort: "medium", + check: async (context) => ({ + status: (await fileExists(path.join(context.repoPath, ".github", "dependabot.yml"))) ? "pass" : "fail", + reason: "Missing .github/dependabot.yml configuration." + }) + }, + { + id: "observability", + title: "Observability tooling present", + pillar: "observability", + level: 3, + scope: "repo", + impact: "medium", + effort: "medium", + check: async (context) => { + const deps = await readAllDependencies(context); + const has = deps.some((dep) => ["@opentelemetry/api", "@opentelemetry/sdk", "pino", "winston", "bunyan"].includes(dep)); + return { + status: has ? "pass" : "fail", + reason: "No observability dependencies detected (OpenTelemetry/logging)." + }; + } + } + ]; +} + +async function runExtras(context: ReadinessContext): Promise { + const results: ReadinessExtraResult[] = []; + + results.push({ + id: "agents-doc", + title: "AGENTS.md present", + status: (await fileExists(path.join(context.repoPath, "AGENTS.md"))) ? "pass" : "fail", + reason: "Missing AGENTS.md to guide coding agents." + }); + + results.push({ + id: "pr-template", + title: "Pull request template present", + status: (await hasPullRequestTemplate(context.repoPath)) ? "pass" : "fail", + reason: "Missing PR template for consistent reviews." + }); + + results.push({ + id: "pre-commit", + title: "Pre-commit hooks configured", + status: (await hasPrecommitConfig(context.repoPath)) ? "pass" : "fail", + reason: "Missing pre-commit or Husky configuration for fast feedback." + }); + + results.push({ + id: "architecture-doc", + title: "Architecture guide present", + status: (await hasArchitectureDoc(context.repoPath)) ? "pass" : "fail", + reason: "Missing architecture documentation." + }); + + return results; +} + +function summarizePillars(criteria: ReadinessCriterionResult[]): ReadinessPillarSummary[] { + const pillarNames: Record = { + "style-validation": "Style & Validation", + "build-system": "Build System", + testing: "Testing", + documentation: "Documentation", + "dev-environment": "Dev Environment", + "code-quality": "Code Quality", + observability: "Observability", + "security-governance": "Security & Governance" + }; + + return (Object.keys(pillarNames) as ReadinessPillar[]).map((pillar) => { + const items = criteria.filter((criterion) => criterion.pillar === pillar); + const { passed, total } = countStatus(items); + return { + id: pillar, + name: pillarNames[pillar], + passed, + total, + passRate: total ? passed / total : 0 + }; + }); +} + +function summarizeLevels(criteria: ReadinessCriterionResult[]): ReadinessLevelSummary[] { + const levelNames: Record = { + 1: "Functional", + 2: "Documented", + 3: "Standardized", + 4: "Optimized", + 5: "Autonomous" + }; + + const summaries: ReadinessLevelSummary[] = []; + for (let level = 1; level <= 5; level += 1) { + const items = criteria.filter((criterion) => criterion.level === level); + const { passed, total } = countStatus(items); + const passRate = total ? passed / total : 0; + summaries.push({ + level, + name: levelNames[level], + passed, + total, + passRate, + achieved: false + }); + } + + for (const summary of summaries) { + const allPrior = summaries.filter((candidate) => candidate.level <= summary.level); + const achieved = allPrior.every((candidate) => candidate.total === 0 || candidate.passRate >= 0.8); + summary.achieved = achieved; + } + + return summaries; +} + +function countStatus(items: ReadinessCriterionResult[]): { passed: number; total: number } { + const relevant = items.filter((item) => item.status !== "skip"); + const passed = relevant.filter((item) => item.status === "pass").length; + return { passed, total: relevant.length }; +} + +async function safeReadDir(dirPath: string): Promise { + try { + return await fs.readdir(dirPath); + } catch { + return []; + } +} + +async function readJson(filePath: string): Promise | undefined> { + try { + const raw = await fs.readFile(filePath, "utf8"); + return JSON.parse(raw) as Record; + } catch { + return undefined; + } +} + +async function fileExists(filePath: string): Promise { + try { + await fs.access(filePath); + return true; + } catch { + return false; + } +} + +function hasAnyFile(files: string[], candidates: string[]): boolean { + return candidates.some((candidate) => files.includes(candidate)); +} + +async function hasReadme(repoPath: string): Promise { + const files = await safeReadDir(repoPath); + return files.some((file) => file.toLowerCase() === "readme.md" || file.toLowerCase() === "readme"); +} + +async function hasLintConfig(repoPath: string): Promise { + return hasAnyFile(await safeReadDir(repoPath), [ + "eslint.config.js", + "eslint.config.mjs", + ".eslintrc", + ".eslintrc.js", + ".eslintrc.cjs", + ".eslintrc.json", + ".eslintrc.yml", + ".eslintrc.yaml", + "biome.json", + "biome.jsonc", + ".prettierrc", + ".prettierrc.json", + ".prettierrc.js", + ".prettierrc.cjs", + "prettier.config.js", + "prettier.config.cjs" + ]); +} + +async function hasFormatterConfig(repoPath: string): Promise { + return hasAnyFile(await safeReadDir(repoPath), [ + "biome.json", + "biome.jsonc", + ".prettierrc", + ".prettierrc.json", + ".prettierrc.js", + ".prettierrc.cjs", + "prettier.config.js", + "prettier.config.cjs" + ]); +} + +async function hasTypecheckConfig(repoPath: string): Promise { + return hasAnyFile(await safeReadDir(repoPath), [ + "tsconfig.json", + "tsconfig.base.json", + "pyproject.toml", + "mypy.ini" + ]); +} + +async function hasGithubWorkflows(repoPath: string): Promise { + return fileExists(path.join(repoPath, ".github", "workflows")); +} + +async function hasCodeowners(repoPath: string): Promise { + const root = await fileExists(path.join(repoPath, "CODEOWNERS")); + const github = await fileExists(path.join(repoPath, ".github", "CODEOWNERS")); + return root || github; +} + +async function hasLicense(repoPath: string): Promise { + const files = await safeReadDir(repoPath); + return files.some((file) => file.toLowerCase().startsWith("license")); +} + +async function hasPullRequestTemplate(repoPath: string): Promise { + const direct = await fileExists(path.join(repoPath, ".github", "PULL_REQUEST_TEMPLATE.md")); + if (direct) return true; + const dir = path.join(repoPath, ".github", "PULL_REQUEST_TEMPLATE"); + try { + const entries = await fs.readdir(dir); + return entries.some((entry) => entry.toLowerCase().endsWith(".md")); + } catch { + return false; + } +} + +async function hasPrecommitConfig(repoPath: string): Promise { + const precommit = await fileExists(path.join(repoPath, ".pre-commit-config.yaml")); + if (precommit) return true; + return fileExists(path.join(repoPath, ".husky")); +} + +async function hasArchitectureDoc(repoPath: string): Promise { + const files = await safeReadDir(repoPath); + if (files.some((file) => file.toLowerCase() === "architecture.md")) return true; + return fileExists(path.join(repoPath, "docs", "architecture.md")); +} + +async function readAllDependencies(context: ReadinessContext): Promise { + const dependencies: string[] = []; + const apps = context.apps.length ? context.apps : []; + for (const app of apps) { + const pkg = await readJson(app.packageJsonPath); + const deps = (pkg?.dependencies ?? {}) as Record; + const devDeps = (pkg?.devDependencies ?? {}) as Record; + dependencies.push(...Object.keys({ + ...deps, + ...devDeps + })); + } + + if (!apps.length && context.rootPackageJson) { + const rootDeps = (context.rootPackageJson.dependencies ?? {}) as Record; + const rootDevDeps = (context.rootPackageJson.devDependencies ?? {}) as Record; + dependencies.push(...Object.keys({ + ...rootDeps, + ...rootDevDeps + })); + } + + return Array.from(new Set(dependencies)); +} \ No newline at end of file diff --git a/src/ui/BatchTuiAzure.tsx b/src/ui/BatchTuiAzure.tsx new file mode 100644 index 0000000..8eb5fea --- /dev/null +++ b/src/ui/BatchTuiAzure.tsx @@ -0,0 +1,526 @@ +import React, { useEffect, useState } from "react"; +import { Box, Text, useApp, useInput } from "ink"; +import path from "path"; +import fs from "fs/promises"; +import { + AzureDevOpsOrg, + AzureDevOpsProject, + AzureDevOpsRepo, + listOrganizations, + listProjects, + listRepos, + checkReposForInstructions, + createPullRequest +} from "../services/azureDevops"; +import { + buildAuthedUrl, + checkoutBranch, + cloneRepo, + commitAll, + isGitRepo, + pushBranch +} from "../services/git"; +import { generateCopilotInstructions } from "../services/instructions"; +import { ensureDir } from "../utils/fs"; +import { StaticBanner } from "./AnimatedBanner"; + +type Props = { + token: string; + outputPath?: string; +}; + +type Status = + | "loading-orgs" + | "select-orgs" + | "loading-projects" + | "select-projects" + | "loading-repos" + | "select-repos" + | "confirm" + | "processing" + | "complete" + | "error"; + +type ProcessResult = { + repo: string; + success: boolean; + prUrl?: string; + error?: string; +}; + +export function BatchTuiAzure({ token, outputPath }: Props): React.JSX.Element { + const app = useApp(); + const [status, setStatus] = useState("loading-orgs"); + const [message, setMessage] = useState("Fetching organizations..."); + const [errorMessage, setErrorMessage] = useState(""); + + const [orgs, setOrgs] = useState([]); + const [projects, setProjects] = useState([]); + const [repos, setRepos] = useState([]); + const [selectedOrgIndices, setSelectedOrgIndices] = useState>(new Set()); + const [selectedProjectIndices, setSelectedProjectIndices] = useState>(new Set()); + const [selectedRepoIndices, setSelectedRepoIndices] = useState>(new Set()); + const [cursorIndex, setCursorIndex] = useState(0); + + const [results, setResults] = useState([]); + const [currentRepoIndex, setCurrentRepoIndex] = useState(0); + const [processingMessage, setProcessingMessage] = useState(""); + + useEffect(() => { + loadOrgs(); + }, []); + + async function loadOrgs() { + try { + const userOrgs = await listOrganizations(token); + setOrgs(userOrgs); + setStatus("select-orgs"); + setMessage("Select organizations (space to toggle, enter to confirm)"); + } catch (error) { + setStatus("error"); + setErrorMessage(error instanceof Error ? error.message : "Failed to fetch organizations"); + } + } + + async function loadProjects() { + setStatus("loading-projects"); + setMessage("Fetching projects..."); + + try { + const selectedOrgs = Array.from(selectedOrgIndices).map(i => orgs[i]); + let allProjects: AzureDevOpsProject[] = []; + + for (let idx = 0; idx < selectedOrgs.length; idx++) { + const org = selectedOrgs[idx]; + setMessage(`Fetching projects from ${org.name} (${idx + 1}/${selectedOrgs.length})...`); + const orgProjects = await listProjects(token, org.name); + allProjects = [...allProjects, ...orgProjects]; + } + + setProjects(allProjects); + setCursorIndex(0); + setSelectedProjectIndices(new Set()); + setStatus("select-projects"); + setMessage("Select projects (space to toggle, enter to confirm)"); + } catch (error) { + setStatus("error"); + setErrorMessage(error instanceof Error ? error.message : "Failed to fetch projects"); + } + } + + async function loadRepos() { + setStatus("loading-repos"); + setMessage("Fetching repositories..."); + + try { + const selectedProjects = Array.from(selectedProjectIndices).map(i => projects[i]); + let allRepos: AzureDevOpsRepo[] = []; + + for (let idx = 0; idx < selectedProjects.length; idx++) { + const project = selectedProjects[idx]; + setMessage( + `Fetching repos from ${project.organization}/${project.name} (${idx + 1}/${selectedProjects.length})...` + ); + const projectRepos = await listRepos(token, project.organization, project.name); + allRepos = [...allRepos, ...projectRepos]; + } + + const seen = new Set(); + const uniqueRepos = allRepos.filter((repo) => { + const key = `${repo.organization}/${repo.project}/${repo.name}`; + if (seen.has(key)) return false; + seen.add(key); + return true; + }); + + setMessage(`Checking ${uniqueRepos.length} repos for existing instructions...`); + const reposWithStatus = await checkReposForInstructions( + token, + uniqueRepos, + (checked, total) => setMessage(`Checking for existing instructions (${checked}/${total})...`) + ); + + reposWithStatus.sort((a, b) => { + if (a.hasInstructions === b.hasInstructions) return 0; + return a.hasInstructions ? 1 : -1; + }); + + const withInstructions = reposWithStatus.filter(r => r.hasInstructions).length; + const withoutInstructions = reposWithStatus.length - withInstructions; + + setRepos(reposWithStatus); + setCursorIndex(0); + setSelectedRepoIndices(new Set()); + setStatus("select-repos"); + setMessage(`Found ${reposWithStatus.length} repos (${withoutInstructions} need instructions, ${withInstructions} already have them)`); + } catch (error) { + setStatus("error"); + setErrorMessage(error instanceof Error ? error.message : "Failed to fetch repositories"); + } + } + + async function processRepos() { + const selectedRepos = Array.from(selectedRepoIndices).map(i => repos[i]); + setStatus("processing"); + setCurrentRepoIndex(0); + setResults([]); + + const nextResults: ProcessResult[] = []; + + for (let i = 0; i < selectedRepos.length; i++) { + const repo = selectedRepos[i]; + setCurrentRepoIndex(i); + setProcessingMessage(`[${i + 1}/${selectedRepos.length}] ${repo.organization}/${repo.project}/${repo.name}: Cloning...`); + + try { + const cacheRoot = path.join(process.cwd(), ".primer-cache"); + const repoPath = path.join(cacheRoot, repo.organization, repo.project, repo.name); + await ensureDir(repoPath); + + if (!(await isGitRepo(repoPath))) { + const authedUrl = buildAuthedUrl(repo.cloneUrl, token, "azure"); + await cloneRepo(authedUrl, repoPath, { + shallow: true, + timeoutMs: 120000, + onProgress: (stage, progress) => { + setProcessingMessage( + `[${i + 1}/${selectedRepos.length}] ${repo.organization}/${repo.project}/${repo.name}: Cloning (${stage} ${progress}%)...` + ); + } + }); + } + + setProcessingMessage(`[${i + 1}/${selectedRepos.length}] ${repo.organization}/${repo.project}/${repo.name}: Creating branch...`); + const branch = "primer/add-instructions"; + await checkoutBranch(repoPath, branch); + + setProcessingMessage(`[${i + 1}/${selectedRepos.length}] ${repo.organization}/${repo.project}/${repo.name}: Generating instructions...`); + const timeoutMs = 120000; + const instructionsPromise = generateCopilotInstructions({ + repoPath, + model: "gpt-4.1", + onProgress: (msg) => { + setProcessingMessage(`[${i + 1}/${selectedRepos.length}] ${repo.organization}/${repo.project}/${repo.name}: ${msg}`); + } + }); + + const timeoutPromise = new Promise((_, reject) => { + setTimeout(() => reject(new Error("Generation timed out after 2 minutes")), timeoutMs); + }); + + const instructions = await Promise.race([instructionsPromise, timeoutPromise]); + + if (!instructions.trim()) { + throw new Error("Generated instructions were empty"); + } + + const instructionsPath = path.join(repoPath, ".github", "copilot-instructions.md"); + await fs.mkdir(path.dirname(instructionsPath), { recursive: true }); + await fs.writeFile(instructionsPath, instructions, "utf8"); + + setProcessingMessage(`[${i + 1}/${selectedRepos.length}] ${repo.organization}/${repo.project}/${repo.name}: Committing...`); + await commitAll(repoPath, "chore: add copilot instructions via Primer"); + + setProcessingMessage(`[${i + 1}/${selectedRepos.length}] ${repo.organization}/${repo.project}/${repo.name}: Pushing...`); + await pushBranch(repoPath, branch, token, "azure"); + + setProcessingMessage(`[${i + 1}/${selectedRepos.length}] ${repo.organization}/${repo.project}/${repo.name}: Creating PR...`); + const prUrl = await createPullRequest({ + token, + organization: repo.organization, + project: repo.project, + repoId: repo.id, + repoName: repo.name, + title: "🤖 Add Copilot instructions via Primer", + body: buildPrBody(), + sourceBranch: branch, + targetBranch: repo.defaultBranch + }); + + nextResults.push({ repo: `${repo.organization}/${repo.project}/${repo.name}`, success: true, prUrl }); + setResults([...nextResults]); + } catch (error) { + const errorMsg = error instanceof Error ? error.message : "Unknown error"; + nextResults.push({ repo: `${repo.organization}/${repo.project}/${repo.name}`, success: false, error: errorMsg }); + setResults([...nextResults]); + } + } + + if (outputPath) { + await fs.writeFile(outputPath, JSON.stringify(nextResults, null, 2), "utf8"); + } + + setStatus("complete"); + setMessage("Batch processing complete!"); + } + + useInput((input, key) => { + if (key.escape || input.toLowerCase() === "q") { + app.exit(); + return; + } + + if (status === "select-orgs") { + if (key.upArrow) { + setCursorIndex(prev => Math.max(0, prev - 1)); + } else if (key.downArrow) { + setCursorIndex(prev => Math.min(orgs.length - 1, prev + 1)); + } else if (input === " ") { + setSelectedOrgIndices(prev => { + const next = new Set(prev); + if (next.has(cursorIndex)) { + next.delete(cursorIndex); + } else { + next.add(cursorIndex); + } + return next; + }); + } else if (key.return && selectedOrgIndices.size > 0) { + loadProjects(); + } + } + + if (status === "select-projects") { + if (key.upArrow) { + setCursorIndex(prev => Math.max(0, prev - 1)); + } else if (key.downArrow) { + setCursorIndex(prev => Math.min(projects.length - 1, prev + 1)); + } else if (input === " ") { + setSelectedProjectIndices(prev => { + const next = new Set(prev); + if (next.has(cursorIndex)) { + next.delete(cursorIndex); + } else { + next.add(cursorIndex); + } + return next; + }); + } else if (key.return && selectedProjectIndices.size > 0) { + loadRepos(); + } + } + + if (status === "select-repos") { + if (key.upArrow) { + setCursorIndex(prev => Math.max(0, prev - 1)); + } else if (key.downArrow) { + setCursorIndex(prev => Math.min(repos.length - 1, prev + 1)); + } else if (input === " ") { + setSelectedRepoIndices(prev => { + const next = new Set(prev); + if (next.has(cursorIndex)) { + next.delete(cursorIndex); + } else { + next.add(cursorIndex); + } + return next; + }); + } else if (input.toLowerCase() === "a") { + const indicesWithoutInstructions = repos + .map((r, i) => ({ r, i })) + .filter(({ r }) => !r.hasInstructions) + .map(({ i }) => i); + setSelectedRepoIndices(new Set(indicesWithoutInstructions)); + } else if (key.return && selectedRepoIndices.size > 0) { + setStatus("confirm"); + setMessage(`Ready to process ${selectedRepoIndices.size} repositories. Press Y to confirm, N to go back.`); + } + } + + if (status === "confirm") { + if (input.toLowerCase() === "y") { + processRepos(); + } else if (input.toLowerCase() === "n") { + setStatus("select-repos"); + setMessage("Select repos (space to toggle, enter to confirm)"); + } + } + }); + + const windowSize = 15; + const getVisibleItems = (items: T[], cursor: number): { items: T[]; startIndex: number } => { + const start = Math.max(0, cursor - Math.floor(windowSize / 2)); + const end = Math.min(items.length, start + windowSize); + const adjustedStart = Math.max(0, end - windowSize); + return { items: items.slice(adjustedStart, end), startIndex: adjustedStart }; + }; + + return ( + + + Batch Processing - Azure DevOps + + {message} + + + {status === "error" && ( + + Error: {errorMessage} + + )} + + {status === "select-orgs" && ( + + {(() => { + const { items: visibleOrgs, startIndex } = getVisibleItems(orgs, cursorIndex); + return visibleOrgs.map((org, i) => { + const realIndex = startIndex + i; + const isSelected = selectedOrgIndices.has(realIndex); + const isCursor = realIndex === cursorIndex; + return ( + + {isCursor ? "❯ " : " "} + {isSelected ? "◉" : "○"} + {org.name} + + ); + }); + })()} + {orgs.length > windowSize && ( + + Showing {Math.min(windowSize, orgs.length)} of {orgs.length} • Use ↑↓ to scroll + + )} + + )} + + {status === "select-projects" && ( + + {(() => { + const { items: visibleProjects, startIndex } = getVisibleItems(projects, cursorIndex); + return visibleProjects.map((project, i) => { + const realIndex = startIndex + i; + const isSelected = selectedProjectIndices.has(realIndex); + const isCursor = realIndex === cursorIndex; + return ( + + {isCursor ? "❯ " : " "} + {isSelected ? "◉" : "○"} + {project.organization}/{project.name} + + ); + }); + })()} + {projects.length > windowSize && ( + + Showing {Math.min(windowSize, projects.length)} of {projects.length} • Use ↑↓ to scroll + + )} + + )} + + {status === "select-repos" && ( + + {(() => { + const { items: visibleRepos, startIndex } = getVisibleItems(repos, cursorIndex); + return visibleRepos.map((repo, i) => { + const realIndex = startIndex + i; + const isSelected = selectedRepoIndices.has(realIndex); + const isCursor = realIndex === cursorIndex; + return ( + + {isCursor ? "❯ " : " "} + {isSelected ? "◉" : "○"} + {repo.hasInstructions ? "✓" : "✗"} + + {repo.organization}/{repo.project}/{repo.name} + + {repo.isPrivate && (private)} + + ); + }); + })()} + {repos.length > windowSize && ( + + Showing {Math.min(windowSize, repos.length)} of {repos.length} • Use ↑↓ to scroll + + )} + + + Selected: {selectedRepoIndices.size} repos + + + + )} + + {status === "processing" && ( + + {processingMessage} + {results.length > 0 && ( + + Completed: + {results.slice(-5).map((r) => ( + + {r.success ? "✓" : "✗"} {r.repo} + {r.success && r.prUrl && → {r.prUrl}} + {!r.success && r.error && ({r.error})} + + ))} + + )} + + )} + + {status === "complete" && ( + + + ✓ Batch complete: {results.filter(r => r.success).length} succeeded, {results.filter(r => !r.success).length} failed + + + {results.map((r) => ( + + {r.success ? "✓" : "✗"} {r.repo} + {r.success && r.prUrl && → {r.prUrl}} + {!r.success && r.error && ({r.error})} + + ))} + + + )} + + + {status === "select-orgs" && ( + Keys: [Space] Toggle [Enter] Confirm [Q] Quit + )} + {status === "select-projects" && ( + Keys: [Space] Toggle [Enter] Confirm [Q] Quit + )} + {status === "select-repos" && ( + Keys: [Space] Toggle [A] Select Missing [Enter] Confirm [Q] Quit + )} + {status === "confirm" && ( + Keys: [Y] Yes, proceed [N] Go back [Q] Quit + )} + {(status === "complete" || status === "error") && ( + Keys: [Q] Quit + )} + + + ); +} + +function buildPrBody(): string { + return [ + "## 🤖 Copilot Instructions Added", + "", + "This PR adds a `.github/copilot-instructions.md` file to help GitHub Copilot understand this codebase better.", + "", + "### What's Included", + "", + "The instructions file contains:", + "- Project overview and architecture", + "- Tech stack and conventions", + "- Build/test commands", + "- Key directories and files", + "", + "### Benefits", + "", + "With these instructions, Copilot will:", + "- Generate more contextually-aware code suggestions", + "- Follow project-specific patterns and conventions", + "- Understand the codebase structure", + "", + "---", + "*Generated by [Primer](https://github.com/pierceboggan/primer) - Prime your repos for AI*" + ].join("\n"); +} \ No newline at end of file diff --git a/src/ui/README.md b/src/ui/README.md index a34f7c0..551d04e 100644 --- a/src/ui/README.md +++ b/src/ui/README.md @@ -1 +1,3 @@ Primer TUI components live here. + +Run the TUI with `primer tui`. diff --git a/src/ui/tui.tsx b/src/ui/tui.tsx index cfa8dbb..7991865 100644 --- a/src/ui/tui.tsx +++ b/src/ui/tui.tsx @@ -8,13 +8,79 @@ import { runEval, type EvalResult } from "../services/evaluator"; import { AnimatedBanner, StaticBanner } from "./AnimatedBanner"; import { BatchTui } from "./BatchTui"; import { getGitHubToken } from "../services/github"; +import { safeWriteFile } from "../utils/fs"; type Props = { repoPath: string; skipAnimation?: boolean; }; -type Status = "intro" | "idle" | "analyzing" | "generating" | "evaluating" | "preview" | "done" | "error" | "batch"; +type Status = + | "intro" + | "idle" + | "analyzing" + | "generating" + | "evaluating" + | "preview" + | "done" + | "error" + | "batch" + | "bootstrapEvalCount" + | "bootstrapEvalConfirm"; + +type EvalCase = { + id: string; + prompt: string; + expectation: string; +}; + +type EvalConfig = { + instructionFile?: string; + cases: EvalCase[]; + systemMessage?: string; + outputPath?: string; +}; + +function buildBootstrapEvalConfig(count: number): EvalConfig { + const templates = [ + { + prompt: "Summarize this repository's purpose and main entry points. Use the README and package.json if available.", + expectation: "A concise summary of repo purpose plus key entry points (CLI or main files) and how to run it." + }, + { + prompt: "Identify the primary languages, frameworks, and package manager used in this repo.", + expectation: "A brief list of languages/frameworks and the detected package manager, with short justification." + }, + { + prompt: "Draft a minimal .github/copilot-instructions.md tailored to this repo's conventions.", + expectation: "A short instruction file referencing observed conventions, avoiding assumptions or secrets." + }, + { + prompt: "Describe how to run the CLI and list its core commands and flags.", + expectation: "Clear usage instructions with command names and key flags derived from docs or source." + }, + { + prompt: "Propose an eval case that checks consistency between CLI and TUI behaviors.", + expectation: "One eval case that validates parity between CLI and TUI outputs or workflows." + } + ]; + + const cases = Array.from({ length: count }, (_, index) => { + const template = templates[index % templates.length]; + const variant = Math.floor(index / templates.length); + const suffix = variant > 0 ? ` (variant ${variant + 1})` : ""; + return { + id: `case-${index + 1}`, + prompt: `${template.prompt}${suffix}`, + expectation: `${template.expectation}${suffix}` + } satisfies EvalCase; + }); + + return { + instructionFile: ".github/copilot-instructions.md", + cases + }; +} export function PrimerTui({ repoPath, skipAnimation = false }: Props): React.JSX.Element { const app = useApp(); @@ -23,7 +89,10 @@ export function PrimerTui({ repoPath, skipAnimation = false }: Props): React.JSX const [message, setMessage] = useState(""); const [generatedContent, setGeneratedContent] = useState(""); const [evalResults, setEvalResults] = useState(null); + const [evalViewerPath, setEvalViewerPath] = useState(null); const [batchToken, setBatchToken] = useState(null); + const [evalCaseCountInput, setEvalCaseCountInput] = useState(""); + const [evalBootstrapCount, setEvalBootstrapCount] = useState(null); const repoLabel = useMemo(() => repoPath, [repoPath]); const handleAnimationComplete = () => { @@ -67,6 +136,78 @@ export function PrimerTui({ repoPath, skipAnimation = false }: Props): React.JSX return; } + if (status === "bootstrapEvalCount") { + if (key.return) { + const trimmed = evalCaseCountInput.trim(); + const count = Number.parseInt(trimmed, 10); + if (!trimmed || !Number.isFinite(count) || count <= 0) { + setMessage("Enter a positive number of eval cases, then press Enter."); + return; + } + + const configPath = path.join(repoPath, "primer.eval.json"); + setEvalBootstrapCount(count); + try { + await fs.access(configPath); + setStatus("bootstrapEvalConfirm"); + setMessage("primer.eval.json exists. Overwrite? (Y/N)"); + } catch { + const config = buildBootstrapEvalConfig(count); + const resultMessage = await safeWriteFile(configPath, JSON.stringify(config, null, 2), false); + setStatus("done"); + setMessage(`Bootstrapped eval: ${resultMessage}`); + setEvalCaseCountInput(""); + setEvalBootstrapCount(null); + } + return; + } + + if (key.backspace || key.delete) { + setEvalCaseCountInput((prev) => prev.slice(0, -1)); + return; + } + + if (/^\d$/.test(input)) { + setEvalCaseCountInput((prev) => prev + input); + return; + } + + return; + } + + if (status === "bootstrapEvalConfirm") { + if (input.toLowerCase() === "y") { + const count = evalBootstrapCount ?? 0; + if (count <= 0) { + setStatus("error"); + setMessage("Missing eval case count. Restart bootstrap."); + return; + } + try { + const configPath = path.join(repoPath, "primer.eval.json"); + const config = buildBootstrapEvalConfig(count); + const resultMessage = await safeWriteFile(configPath, JSON.stringify(config, null, 2), true); + setStatus("done"); + setMessage(`Bootstrapped eval: ${resultMessage}`); + } catch (error) { + setStatus("error"); + setMessage(error instanceof Error ? error.message : "Failed to write eval config."); + } finally { + setEvalCaseCountInput(""); + setEvalBootstrapCount(null); + } + return; + } + + if (input.toLowerCase() === "n") { + setStatus("idle"); + setMessage("Bootstrap cancelled."); + setEvalCaseCountInput(""); + setEvalBootstrapCount(null); + } + return; + } + if (input.toLowerCase() === "a") { setStatus("analyzing"); try { @@ -133,8 +274,9 @@ export function PrimerTui({ repoPath, skipAnimation = false }: Props): React.JSX setStatus("evaluating"); setMessage("Running evals... (this may take a few minutes)"); setEvalResults(null); + setEvalViewerPath(null); try { - const { results } = await runEval({ + const { results, viewerPath } = await runEval({ configPath, repoPath, model: "gpt-4.1", @@ -142,6 +284,7 @@ export function PrimerTui({ repoPath, skipAnimation = false }: Props): React.JSX // Note: onProgress removed - causes issues with SDK in React/Ink context }); setEvalResults(results); + setEvalViewerPath(viewerPath ?? null); const passed = results.filter(r => r.verdict === "pass").length; const failed = results.filter(r => r.verdict === "fail").length; setStatus("done"); @@ -151,6 +294,13 @@ export function PrimerTui({ repoPath, skipAnimation = false }: Props): React.JSX setMessage(error instanceof Error ? error.message : "Eval failed."); } } + + if (input.toLowerCase() === "i") { + setStatus("bootstrapEvalCount"); + setMessage("Enter number of eval cases, then press Enter."); + setEvalCaseCountInput(""); + setEvalBootstrapCount(null); + } }); const statusLabel = status === "intro" ? "..." : status === "idle" ? "ready (awaiting input)" : status; @@ -186,6 +336,11 @@ export function PrimerTui({ repoPath, skipAnimation = false }: Props): React.JSX {message} + {status === "bootstrapEvalCount" && ( + + Eval case count: {evalCaseCountInput || ""} + + )} {status === "preview" && generatedContent && ( Preview (.github/copilot-instructions.md): @@ -200,6 +355,9 @@ export function PrimerTui({ repoPath, skipAnimation = false }: Props): React.JSX {r.verdict === "pass" ? "✓" : r.verdict === "fail" ? "✗" : "?"} {r.id}: {r.verdict} (score: {r.score}) ))} + {evalViewerPath && ( + Trajectory viewer: {evalViewerPath} + )} )} @@ -207,8 +365,10 @@ export function PrimerTui({ repoPath, skipAnimation = false }: Props): React.JSX Press any key to skip animation... ) : status === "preview" ? ( Keys: [S] Save [D] Discard [Q] Quit + ) : status === "bootstrapEvalConfirm" ? ( + Keys: [Y] Overwrite [N] Cancel [Q] Quit ) : ( - Keys: [A] Analyze [G] Generate [E] Eval [B] Batch [Q] Quit + Keys: [A] Analyze [G] Generate [E] Eval [I] Init Eval [B] Batch [Q] Quit )} diff --git a/vitest.config.ts b/vitest.config.ts new file mode 100644 index 0000000..2481bee --- /dev/null +++ b/vitest.config.ts @@ -0,0 +1,12 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + environment: "node", + coverage: { + provider: "v8", + reporter: ["text", "html", "json-summary"], + reportsDirectory: "./coverage" + } + } +});