agentrc.eval.json

{
  "instructionFile": ".github/copilot-instructions.md",
  "cases": [
    {
      "id": "dry-run-generate",
      "prompt": "Add a --dry-run flag to the generate command",
      "expectation": "The plan must identify that changes are needed in: (1) packages/core/src/services/generator.ts to add a dryRun parameter to the GenerateOptions type and modify generateConfigs() to handle dry-run mode by skipping actual file writes and instead using a helper such as writeOrPreview to preview which files would be written, (2) src/commands/generate.ts to add the --dry-run option and pass it through to generateConfigs(), (3) src/cli.ts to add .option('--dry-run', 'Preview files without writing') to the generate command. The plan must specify that output should show file paths and indicate whether each would be created or skipped, with dry-run handled inside generator.ts rather than by changing safeWriteFile() in packages/core/src/utils/fs.ts. The plan must name the test file src/services/__tests__/generator.test.ts for adding test coverage. The plan must cite the exact function generateConfigs() and type GenerateOptions. The plan must specify running 'npm run typecheck' and 'npm test' to verify changes.",
      "area": "cli"
    },
    {
      "id": "cicd-readiness-criterion",
      "prompt": "Add a readiness criterion that detects CI/CD configuration",
      "expectation": "The plan must identify changes in: (1) packages/core/src/services/readiness/criteria.ts to add a new criterion object with id 'ci-config', pillar 'build-system', level 2, and a check function that calls hasGithubWorkflows() or equivalent, (2) packages/core/src/services/readiness/checkers.ts to implement or verify hasGithubWorkflows() detects .github/workflows/, (3) packages/core/src/services/readiness/types.ts to verify the ReadinessPillar type includes 'build-system' (already exists), (4) src/services/__tests__/readiness.test.ts to add test cases for the new criterion. The plan must cite the buildCriteria() function in criteria.ts and the ReadinessCriterion type. The plan must specify that the check should look for .github/workflows directory. The plan must specify running 'npm test' to verify the new criterion works correctly.",
      "area": "readiness"
    },
    {
      "id": "area-based-eval-generation",
      "prompt": "Make eval scaffold generation area-aware so it generates prompts specific to each area",
      "expectation": "The plan must identify: (1) packages/core/src/services/evalScaffold.ts needs to accept an areas parameter in EvalScaffoldOptions and pass area context to the Copilot SDK session via systemMessage, (2) src/commands/eval.ts needs to analyze the repo to detect areas and optionally filter which areas to generate evals for, (3) The generateEvalScaffold() function must be modified to iterate over areas and generate area-specific prompts with workingDirectory set to each area's path, (4) The EvalCase type already includes area and workingDirectory fields (verify in evalScaffold.ts lines 10-16), so the generation logic must populate these correctly, (5) src/services/__tests__/configScaffold.test.ts or a new test file must verify area-specific eval generation. The plan must cite the exact types EvalScaffoldOptions, EvalCase, and EvalConfig. The plan must specify testing with 'npm test' and verifying that generated cases include area and workingDirectory fields.",
      "area": "evaluator"
    },
    {
      "id": "instruction-diff-command",
      "prompt": "Create a new 'agentrc diff' command that compares generated instructions against existing ones",
      "expectation": "The plan must identify: (1) Create a new file src/commands/diff.ts that implements diffCommand(), (2) Create packages/core/src/services/diff.ts with a compareInstructions() function that uses detectExistingInstructions() from instructions.ts and generates fresh instructions without writing files, then performs a textual diff, (3) Modify src/cli.ts to add a new .command('diff') with options for --model and --strategy, and register it with withGlobalOpts(diffCommand), (4) The implementation must call analyzeRepo() from analyzer.ts and generateCopilotInstructions() from instructions.ts in dry-run mode (no file writes), (5) Output must use outputResult() from packages/core/utils/output.ts following the CommandResult<T> pattern, (6) Add tests in src/services/__tests__/diff.test.ts. The plan must cite exact functions: detectExistingInstructions(), generateCopilotInstructions(), analyzeRepo(), and types CommandResult<T> from output.ts. Verification must include 'npm run typecheck', 'npm run lint', and 'npm test'.",
      "area": "cli"
    },
    {
      "id": "batch-readiness-json-output",
      "prompt": "Add JSON output support to the batch readiness command",
      "expectation": "The plan must identify: (1) src/commands/batchReadiness.tsx uses Ink/React for TUI rendering, so JSON mode must bypass the TUI entirely, (2) Modify batchReadinessCommand() to check if options.json is true and skip rendering <BatchReadinessUI />, instead calling a new non-interactive batch processing function, (3) Create a new function in packages/core/src/services/batch.ts (verify existing batch logic) that processes repos and collects ReadinessReport[] without UI rendering, (4) Use outputResult() from packages/core/utils/output.ts to emit JSON with a structure like CommandResult<{ repos: string[], reports: ReadinessReport[], summary: { passed: number, failed: number } }>, (5) The existing withGlobalOpts() helper in src/cli.ts already merges --json into options, so the command handler just needs to check options.json, (6) Add tests in src/services/__tests__/batch.test.ts. The plan must cite the exact types ReadinessReport from packages/core/src/services/readiness/types.ts, CommandResult from output.ts, and the BatchReadinessUI component. Verification commands: 'npm run typecheck' and 'npm test'.",
      "area": "batch"
    }
  ]
}