From 4f1a7186a12ba8a673178df528f889b8690f4db6 Mon Sep 17 00:00:00 2001 From: Joe Delfino Date: Thu, 12 Feb 2026 13:52:55 +0000 Subject: [PATCH 01/23] feat: add workflow config and issue templates (#23) Add config.yaml with guardrail settings and re-review cycle cap, task.yml and review-finding.yml issue form templates. Co-Authored-By: Claude Opus 4.6 --- .github/ISSUE_TEMPLATE/.gitkeep | 0 .github/ISSUE_TEMPLATE/review-finding.yml | 46 +++++++++++++++++ .github/ISSUE_TEMPLATE/task.yml | 60 +++++++++++++++++++++++ .github/agent-workflow/.gitkeep | 0 .github/agent-workflow/config.yaml | 19 +++++++ 5 files changed, 125 insertions(+) delete mode 100644 .github/ISSUE_TEMPLATE/.gitkeep create mode 100644 .github/ISSUE_TEMPLATE/review-finding.yml create mode 100644 .github/ISSUE_TEMPLATE/task.yml delete mode 100644 .github/agent-workflow/.gitkeep create mode 100644 .github/agent-workflow/config.yaml diff --git a/.github/ISSUE_TEMPLATE/.gitkeep b/.github/ISSUE_TEMPLATE/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/.github/ISSUE_TEMPLATE/review-finding.yml b/.github/ISSUE_TEMPLATE/review-finding.yml new file mode 100644 index 0000000..5a8819f --- /dev/null +++ b/.github/ISSUE_TEMPLATE/review-finding.yml @@ -0,0 +1,46 @@ +name: Review Finding +description: An issue found during automated or manual code review. +title: "" +labels: ["review-finding"] +body: + - type: textarea + id: finding-description + attributes: + label: Finding description + description: Describe the problem found during review. + placeholder: Explain what is wrong and why it matters. + validations: + required: true + + - type: dropdown + id: severity + attributes: + label: Severity + description: How critical is this finding? + options: + - blocking + - should-fix + - suggestion + validations: + required: true + + - type: textarea + id: affected-files + attributes: + label: Affected files + description: List the files and line numbers where the issue was found. + placeholder: | + - path/to/file1.ts:42 + - path/to/file2.ts:15-23 + render: markdown + validations: + required: true + + - type: textarea + id: suggested-fix + attributes: + label: Suggested fix + description: How should this finding be addressed? + placeholder: Describe the recommended approach to fix this issue. + validations: + required: false diff --git a/.github/ISSUE_TEMPLATE/task.yml b/.github/ISSUE_TEMPLATE/task.yml new file mode 100644 index 0000000..83dff12 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/task.yml @@ -0,0 +1,60 @@ +name: Task +description: A structured task created during planning. Scoped to be completable in a single agent session. +title: "" +labels: ["task"] +body: + - type: textarea + id: summary + attributes: + label: Summary + description: What needs to be done and why. + placeholder: Describe the task in 1-3 sentences. + validations: + required: true + + - type: textarea + id: files-to-modify + attributes: + label: Files to modify + description: List all files that should be created or changed for this task. + placeholder: | + - path/to/file1.ts + - path/to/file2.ts + render: markdown + validations: + required: true + + - type: textarea + id: implementation-steps + attributes: + label: Implementation steps + description: Ordered steps to complete the task. Each step should be concrete and verifiable. + placeholder: | + 1. First step + 2. Second step + 3. Third step + render: markdown + validations: + required: true + + - type: textarea + id: acceptance-criteria + attributes: + label: Acceptance criteria + description: Conditions that must be true for this task to be considered complete. + placeholder: | + - [ ] Criterion 1 + - [ ] Criterion 2 + render: markdown + validations: + required: true + + - type: textarea + id: dependencies + attributes: + label: Dependencies + description: Other issues that must be completed before this task can start. Use issue references. + placeholder: | + Blocked by #XX, #YY + validations: + required: false diff --git a/.github/agent-workflow/.gitkeep b/.github/agent-workflow/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/.github/agent-workflow/config.yaml b/.github/agent-workflow/config.yaml new file mode 100644 index 0000000..935b8b8 --- /dev/null +++ b/.github/agent-workflow/config.yaml @@ -0,0 +1,19 @@ +re-review-cycle-cap: 3 + +guardrails: + scope-enforcement: + enabled: true + conclusion: action_required + test-ratio: + enabled: true + conclusion: action_required + threshold: 0.5 + dependency-changes: + enabled: true + conclusion: action_required + api-surface: + enabled: true + conclusion: action_required + commit-messages: + enabled: true + conclusion: neutral From eea2f167969d7237ab2b05af9834e7d3ba10a3e8 Mon Sep 17 00:00:00 2001 From: Joe Delfino Date: Thu, 12 Feb 2026 13:57:32 +0000 Subject: [PATCH 02/23] feat: add guardrail workflow checks (#18, #19, #20, #21, #22) Five guardrail checks as independent GitHub Actions workflows: - Scope enforcement: flags files changed outside task scope - Test-to-code ratio: enforces configurable test line threshold - Dependency changes: detects unjustified new dependencies - API surface changes: language-aware export/route detection - Commit messages: conventional commit format validation Each uses Check Run API with approval override support. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/guardrail-api-surface.yml | 248 +++++++++++++++++ .github/workflows/guardrail-commits.yml | 226 ++++++++++++++++ .github/workflows/guardrail-dependencies.yml | 265 +++++++++++++++++++ .github/workflows/guardrail-scope.yml | 254 ++++++++++++++++++ .github/workflows/guardrail-test-ratio.yml | 209 +++++++++++++++ 5 files changed, 1202 insertions(+) create mode 100644 .github/workflows/guardrail-api-surface.yml create mode 100644 .github/workflows/guardrail-commits.yml create mode 100644 .github/workflows/guardrail-dependencies.yml create mode 100644 .github/workflows/guardrail-scope.yml create mode 100644 .github/workflows/guardrail-test-ratio.yml diff --git a/.github/workflows/guardrail-api-surface.yml b/.github/workflows/guardrail-api-surface.yml new file mode 100644 index 0000000..f98f30a --- /dev/null +++ b/.github/workflows/guardrail-api-surface.yml @@ -0,0 +1,248 @@ +name: "Guardrail: API Surface Changes" + +on: + pull_request: + types: [opened, synchronize] + +permissions: + checks: write + pull-requests: read + contents: read + +jobs: + api-surface-check: + runs-on: ubuntu-latest + steps: + - name: Check API surface changes + uses: actions/github-script@v7 + with: + script: | + const owner = context.repo.owner; + const repo = context.repo.repo; + const prNumber = context.payload.pull_request.number; + const headSha = context.payload.pull_request.head.sha; + const checkName = 'guardrail/api-surface'; + + // --- Read config --- + let enabled = true; + let configuredConclusion = 'action_required'; + try { + const configResponse = await github.rest.repos.getContent({ + owner, + repo, + path: '.github/agent-workflow/config.yaml', + ref: headSha, + }); + const configContent = Buffer.from(configResponse.data.content, 'base64').toString('utf8'); + // Simple YAML parsing for our config keys + const enabledMatch = configContent.match(/api-surface:\s*\n\s*enabled:\s*(true|false)/); + if (enabledMatch && enabledMatch[1] === 'false') { + enabled = false; + } + const conclusionMatch = configContent.match(/api-surface:\s*\n\s*enabled:\s*(?:true|false)\s*\n\s*conclusion:\s*(\S+)/); + if (conclusionMatch) { + configuredConclusion = conclusionMatch[1]; + } + } catch (e) { + // Config file not found or unreadable — use defaults (enabled, action_required) + core.info('No config.yaml found, using defaults (enabled: true, conclusion: action_required)'); + } + + if (!enabled) { + await github.rest.checks.create({ + owner, + repo, + head_sha: headSha, + name: checkName, + status: 'completed', + conclusion: 'success', + output: { + title: 'API surface check: disabled', + summary: 'This check is disabled in config.yaml.', + }, + }); + return; + } + + // --- Check for non-stale PR approval override --- + const reviews = await github.rest.pulls.listReviews({ + owner, + repo, + pull_number: prNumber, + }); + const hasValidApproval = reviews.data.some( + (r) => r.state === 'APPROVED' && r.commit_id === headSha + ); + + if (hasValidApproval) { + await github.rest.checks.create({ + owner, + repo, + head_sha: headSha, + name: checkName, + status: 'completed', + conclusion: 'success', + output: { + title: 'API surface check: approved by reviewer', + summary: 'A non-stale PR approval overrides this guardrail.', + }, + }); + return; + } + + // --- Get PR files and scan for API surface changes --- + const files = await github.paginate(github.rest.pulls.listFiles, { + owner, + repo, + pull_number: prNumber, + }); + + // API surface patterns — language-aware heuristics + const apiPatterns = [ + // JavaScript / TypeScript + { regex: /^\+.*\bexport\s+(default\s+)?(function|class|const|let|var|interface|type|enum)\b/, label: 'JS/TS export' }, + { regex: /^\+.*\bmodule\.exports\b/, label: 'CommonJS export' }, + { regex: /^\+.*\bexports\.\w+/, label: 'CommonJS named export' }, + // Python + { regex: /^\+.*@app\.(route|get|post|put|delete|patch)\b/, label: 'Flask/FastAPI route' }, + { regex: /^\+.*@router\.(route|get|post|put|delete|patch)\b/, label: 'Router route' }, + // Go + { regex: /^\+\s*func\s+[A-Z]/, label: 'Go exported function' }, + { regex: /^\+\s*type\s+[A-Z]\w+\s+(struct|interface)\b/, label: 'Go exported type' }, + // Java / Kotlin / C# + { regex: /^\+.*\bpublic\s+(static\s+)?(class|interface|enum|void|int|string|boolean|long|double|float|[\w<>\[\]]+)\s+\w+/, label: 'Public declaration' }, + // Rust + { regex: /^\+.*\bpub\s+fn\b/, label: 'Rust public function' }, + { regex: /^\+.*\bpub\s+(struct|enum|trait|type|mod)\b/, label: 'Rust public type' }, + // Ruby on Rails + { regex: /^\+.*\b(get|post|put|patch|delete|resources|resource)\s+['":\/]/, label: 'Rails route' }, + // Express.js / Node.js + { regex: /^\+.*\brouter\.(get|post|put|delete|patch|all|use)\s*\(/, label: 'Express route' }, + { regex: /^\+.*\bapp\.(get|post|put|delete|patch|all|use)\s*\(/, label: 'Express app route' }, + ]; + + // OpenAPI / Swagger file patterns + const openApiFilePatterns = [ + /openapi\.(ya?ml|json)$/i, + /swagger\.(ya?ml|json)$/i, + /api-spec\.(ya?ml|json)$/i, + ]; + + const annotations = []; + let totalApiChanges = 0; + + for (const file of files) { + // Skip removed files + if (file.status === 'removed') continue; + + // Check if this is an OpenAPI/Swagger spec file + const isOpenApiFile = openApiFilePatterns.some((p) => p.test(file.filename)); + if (isOpenApiFile) { + totalApiChanges++; + annotations.push({ + path: file.filename, + start_line: 1, + end_line: 1, + annotation_level: 'warning', + message: `OpenAPI/Swagger spec file modified: ${file.filename}. API contract changes require careful review.`, + }); + continue; + } + + // Parse the patch for added lines matching API patterns + if (!file.patch) continue; + + const lines = file.patch.split('\n'); + let currentLine = 0; + + for (const line of lines) { + // Track line numbers from hunk headers + const hunkMatch = line.match(/^@@\s+-\d+(?:,\d+)?\s+\+(\d+)(?:,\d+)?\s+@@/); + if (hunkMatch) { + currentLine = parseInt(hunkMatch[1], 10); + continue; + } + + // Only look at added lines (start with +, not +++) + if (line.startsWith('+') && !line.startsWith('+++')) { + for (const pattern of apiPatterns) { + if (pattern.regex.test(line)) { + totalApiChanges++; + annotations.push({ + path: file.filename, + start_line: currentLine, + end_line: currentLine, + annotation_level: 'warning', + message: `API surface change detected (${pattern.label}): ${line.substring(1).trim()}`, + }); + break; // One annotation per line + } + } + } + + // Advance line counter for added and context lines (not removed lines) + if (!line.startsWith('-')) { + currentLine++; + } + } + } + + // --- Report results --- + if (totalApiChanges === 0) { + await github.rest.checks.create({ + owner, + repo, + head_sha: headSha, + name: checkName, + status: 'completed', + conclusion: 'success', + output: { + title: 'API surface check: no changes detected', + summary: 'No API surface changes found in this PR.', + }, + }); + } else { + // GitHub API limits annotations to 50 per call + const batchSize = 50; + const batches = []; + for (let i = 0; i < annotations.length; i += batchSize) { + batches.push(annotations.slice(i, i + batchSize)); + } + + const summary = [ + `Found ${totalApiChanges} API surface change(s) across the PR.`, + '', + 'API surface changes have outsized downstream impact. Review these changes carefully.', + '', + 'To override: approve the PR to signal these changes are intentional.', + ].join('\n'); + + // Create the check run with the first batch of annotations + const checkRun = await github.rest.checks.create({ + owner, + repo, + head_sha: headSha, + name: checkName, + status: 'completed', + conclusion: configuredConclusion, + output: { + title: `API surface check: ${totalApiChanges} change(s) detected`, + summary, + annotations: batches[0] || [], + }, + }); + + // If there are more annotations, update the check run with additional batches + for (let i = 1; i < batches.length; i++) { + await github.rest.checks.update({ + owner, + repo, + check_run_id: checkRun.data.id, + output: { + title: `API surface check: ${totalApiChanges} change(s) detected`, + summary, + annotations: batches[i], + }, + }); + } + } diff --git a/.github/workflows/guardrail-commits.yml b/.github/workflows/guardrail-commits.yml new file mode 100644 index 0000000..97f9b0f --- /dev/null +++ b/.github/workflows/guardrail-commits.yml @@ -0,0 +1,226 @@ +name: "Guardrail: Commit Messages" + +on: + pull_request: + types: [opened, synchronize] + +permissions: + checks: write + contents: read + pull-requests: read + +jobs: + commit-messages: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Check commit messages + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + const path = require('path'); + + // --- Read config --- + // Default conclusion for commit message guardrail is 'neutral' (non-blocking warning) + let configuredConclusion = 'neutral'; + const configPath = path.join(process.env.GITHUB_WORKSPACE, '.github', 'agent-workflow', 'config.yaml'); + try { + const configContent = fs.readFileSync(configPath, 'utf8'); + // Simple YAML parsing for the guardrails.commit-messages section + // Config structure: guardrails: > commit-messages: > enabled/conclusion + const lines = configContent.split('\n'); + let inGuardrails = false; + let inCommitSection = false; + let guardrailIndent = 0; + let commitIndent = 0; + for (const line of lines) { + // Detect guardrails: top-level key + if (/^guardrails:/.test(line)) { + inGuardrails = true; + inCommitSection = false; + continue; + } + // If we hit another top-level key, leave guardrails + if (inGuardrails && /^\S/.test(line) && !/^\s*#/.test(line) && line.trim() !== '') { + inGuardrails = false; + inCommitSection = false; + continue; + } + // Inside guardrails, look for commit-messages: + if (inGuardrails && !inCommitSection) { + const commitMatch = line.match(/^(\s+)commit-messages:/); + if (commitMatch) { + inCommitSection = true; + commitIndent = commitMatch[1].length; + continue; + } + } + // If in commit section, check for sibling keys (same indent = new section) + if (inCommitSection) { + const lineIndent = line.match(/^(\s*)/)[1].length; + if (line.trim() === '' || /^\s*#/.test(line)) continue; + if (lineIndent <= commitIndent) { + // Left the commit-messages section + inCommitSection = false; + continue; + } + const conclusionMatch = line.match(/^\s+conclusion:\s*(\S+)/); + if (conclusionMatch) { + const val = conclusionMatch[1].replace(/['"]/g, ''); + if (['success', 'neutral', 'action_required'].includes(val)) { + configuredConclusion = val; + } + } + const enabledMatch = line.match(/^\s+enabled:\s*(\S+)/); + if (enabledMatch) { + const val = enabledMatch[1].replace(/['"]/g, '').toLowerCase(); + if (val === 'false') { + // Check is disabled, report success and exit + await github.rest.checks.create({ + owner: context.repo.owner, + repo: context.repo.repo, + head_sha: context.sha, + name: 'guardrail/commit-messages', + conclusion: 'success', + output: { + title: 'Commit message check: disabled', + summary: 'This guardrail check is disabled in config.yaml.' + } + }); + return; + } + } + } + } + } catch (e) { + // No config file found — use defaults (neutral) + core.info(`No config.yaml found at ${configPath}, using default conclusion: neutral`); + } + + // --- Check for non-stale PR approval override --- + const reviews = await github.rest.pulls.listReviews({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: context.payload.pull_request.number + }); + + const headSha = context.payload.pull_request.head.sha; + const hasValidApproval = reviews.data.some( + r => r.state === 'APPROVED' && r.commit_id === headSha + ); + + if (hasValidApproval) { + await github.rest.checks.create({ + owner: context.repo.owner, + repo: context.repo.repo, + head_sha: context.sha, + name: 'guardrail/commit-messages', + conclusion: 'success', + output: { + title: 'Commit message check: approved by reviewer', + summary: 'A non-stale PR approval overrides this guardrail check.' + } + }); + return; + } + + // --- Get PR commits --- + const commits = await github.rest.pulls.listCommits({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: context.payload.pull_request.number, + per_page: 100 + }); + + // --- Validate each commit message --- + // Conventional commit regex: type(optional scope)optional !: description + const conventionalCommitRegex = /^(feat|fix|chore|docs|test|refactor|ci|style|perf|build|revert)(\(.+\))?!?: .+/; + const maxFirstLineLength = 72; + + const violations = []; + + for (const commit of commits.data) { + const message = commit.commit.message; + const firstLine = message.split('\n')[0]; + const sha = commit.sha.substring(0, 7); + const commitViolations = []; + + // Check conventional commit format + if (!conventionalCommitRegex.test(firstLine)) { + commitViolations.push( + `Does not follow conventional commit format (expected: type(scope)?: description)` + ); + } + + // Check first line length + if (firstLine.length > maxFirstLineLength) { + commitViolations.push( + `First line exceeds ${maxFirstLineLength} characters (${firstLine.length} chars)` + ); + } + + if (commitViolations.length > 0) { + violations.push({ + sha: sha, + fullSha: commit.sha, + firstLine: firstLine, + issues: commitViolations + }); + } + } + + // --- Report results --- + const totalCommits = commits.data.length; + const nonConformingCount = violations.length; + + if (nonConformingCount === 0) { + // All commits conform + await github.rest.checks.create({ + owner: context.repo.owner, + repo: context.repo.repo, + head_sha: context.sha, + name: 'guardrail/commit-messages', + conclusion: 'success', + output: { + title: `Commit message check: all ${totalCommits} commits conform`, + summary: `All ${totalCommits} commit(s) follow conventional commit format with first line <= ${maxFirstLineLength} characters.` + } + }); + return; + } + + // Build summary with non-conforming commits + let summary = `## Non-conforming commits\n\n`; + summary += `Found **${nonConformingCount}** of ${totalCommits} commit(s) with violations:\n\n`; + + for (const v of violations) { + summary += `### \`${v.sha}\` — ${v.firstLine}\n`; + for (const issue of v.issues) { + summary += `- ${issue}\n`; + } + summary += '\n'; + } + + summary += `\n## Expected format\n\n`; + summary += '```\n'; + summary += 'type(optional-scope): description (max 72 chars)\n'; + summary += '```\n\n'; + summary += `Valid types: \`feat\`, \`fix\`, \`chore\`, \`docs\`, \`test\`, \`refactor\`, \`ci\`, \`style\`, \`perf\`, \`build\`, \`revert\`\n\n`; + summary += `**Configured conclusion:** \`${configuredConclusion}\`\n`; + summary += `\nTo override: submit an approving PR review. The approval must be on the current head commit to be non-stale.\n`; + + // Use the configured conclusion (default: neutral = non-blocking warning) + await github.rest.checks.create({ + owner: context.repo.owner, + repo: context.repo.repo, + head_sha: context.sha, + name: 'guardrail/commit-messages', + conclusion: configuredConclusion, + output: { + title: `Commit message check: ${nonConformingCount} non-conforming commit(s)`, + summary: summary + } + }); diff --git a/.github/workflows/guardrail-dependencies.yml b/.github/workflows/guardrail-dependencies.yml new file mode 100644 index 0000000..109c29c --- /dev/null +++ b/.github/workflows/guardrail-dependencies.yml @@ -0,0 +1,265 @@ +name: "Guardrail: Dependency Changes" + +on: + pull_request: + types: [opened, synchronize] + +permissions: + checks: write + contents: read + pull-requests: read + issues: read + +jobs: + dependency-changes: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Check dependency changes + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + const yaml = require('js-yaml'); + + // --- Configuration --- + const CHECK_NAME = 'guardrail/dependency-changes'; + const DEPENDENCY_FILES = [ + 'package.json', + 'package-lock.json', + 'yarn.lock', + 'pnpm-lock.yaml', + 'requirements.txt', + 'requirements-dev.txt', + 'requirements-test.txt', + 'Pipfile', + 'Pipfile.lock', + 'pyproject.toml', + 'poetry.lock', + 'setup.py', + 'setup.cfg', + 'go.mod', + 'go.sum', + 'Cargo.toml', + 'Cargo.lock', + 'pom.xml', + 'build.gradle', + 'build.gradle.kts', + 'settings.gradle', + 'settings.gradle.kts', + 'Gemfile', + 'Gemfile.lock', + 'composer.json', + 'composer.lock', + 'mix.exs', + 'mix.lock', + 'pubspec.yaml', + 'pubspec.lock', + 'Package.swift', + 'Package.resolved', + 'Podfile', + 'Podfile.lock', + '.csproj', + 'packages.config', + 'Directory.Packages.props' + ]; + + // Also match dependency files in subdirectories (monorepo support) + function isDependencyFile(filename) { + const basename = filename.split('/').pop(); + if (DEPENDENCY_FILES.includes(basename)) { + return true; + } + // Match .csproj files by extension + if (filename.endsWith('.csproj')) { + return true; + } + return false; + } + + const JUSTIFICATION_KEYWORDS = [ + 'dependency', 'dependencies', + 'added', 'adding', + 'requires', 'required', + 'needed for', 'needed by', + 'introduced', + 'new package', 'new library', 'new module', + 'upgrade', 'upgraded', 'upgrading', + 'update', 'updated', 'updating', + 'migration', 'migrate', 'migrating', + 'replace', 'replaced', 'replacing', + 'security fix', 'security patch', 'vulnerability', + 'CVE-' + ]; + + // --- Read config --- + let checkEnabled = true; + let configuredConclusion = 'action_required'; + try { + const configPath = '.github/agent-workflow/config.yaml'; + if (fs.existsSync(configPath)) { + const config = yaml.load(fs.readFileSync(configPath, 'utf8')); + if (config && config['dependency-changes']) { + const checkConfig = config['dependency-changes']; + if (checkConfig.enabled === false) { + checkEnabled = false; + } + if (checkConfig.conclusion) { + configuredConclusion = checkConfig.conclusion; + } + } + } + } catch (e) { + core.warning(`Failed to read config: ${e.message}. Using defaults.`); + } + + // --- If check is disabled, report success and exit --- + if (!checkEnabled) { + await github.rest.checks.create({ + owner: context.repo.owner, + repo: context.repo.repo, + head_sha: context.sha, + name: CHECK_NAME, + status: 'completed', + conclusion: 'success', + output: { + title: 'Dependency changes: check disabled', + summary: 'This guardrail check is disabled in config.yaml.' + } + }); + return; + } + + // --- Check for non-stale approving PR review (override mechanism) --- + const { data: reviews } = await github.rest.pulls.listReviews({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: context.payload.pull_request.number + }); + + const lastCommitSha = context.payload.pull_request.head.sha; + const hasValidApproval = reviews.some( + r => r.state === 'APPROVED' && r.commit_id === lastCommitSha + ); + + if (hasValidApproval) { + await github.rest.checks.create({ + owner: context.repo.owner, + repo: context.repo.repo, + head_sha: context.sha, + name: CHECK_NAME, + status: 'completed', + conclusion: 'success', + output: { + title: 'Dependency changes: approved by reviewer', + summary: 'A non-stale PR approval overrides dependency change violations.' + } + }); + return; + } + + // --- Get PR changed files --- + const files = await github.paginate( + github.rest.pulls.listFiles, + { + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: context.payload.pull_request.number, + per_page: 100 + } + ); + + const changedDependencyFiles = files.filter(f => isDependencyFile(f.filename)); + + // --- No dependency files changed: success --- + if (changedDependencyFiles.length === 0) { + await github.rest.checks.create({ + owner: context.repo.owner, + repo: context.repo.repo, + head_sha: context.sha, + name: CHECK_NAME, + status: 'completed', + conclusion: 'success', + output: { + title: 'Dependency changes: no dependency files modified', + summary: 'No dependency manifest or lock files were changed in this PR.' + } + }); + return; + } + + // --- Dependency files changed: check for justification --- + const prBody = (context.payload.pull_request.body || '').toLowerCase(); + + function hasJustification(text) { + const lowerText = text.toLowerCase(); + return JUSTIFICATION_KEYWORDS.some(keyword => lowerText.includes(keyword)); + } + + let justified = hasJustification(prBody); + + // --- If not justified in PR body, check linked issue body --- + if (!justified) { + const issueMatch = (context.payload.pull_request.body || '').match( + /(?:fixes|closes|resolves)\s+#(\d+)/i + ); + if (issueMatch) { + try { + const { data: issue } = await github.rest.issues.get({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: parseInt(issueMatch[1], 10) + }); + if (issue.body) { + justified = hasJustification(issue.body); + } + } catch (e) { + core.warning(`Failed to fetch linked issue #${issueMatch[1]}: ${e.message}`); + } + } + } + + // --- Build annotations for changed dependency files --- + const annotations = changedDependencyFiles.map(f => ({ + path: f.filename, + start_line: 1, + end_line: 1, + annotation_level: 'warning', + message: justified + ? `Dependency file changed. Justification found in PR or linked issue.` + : `Dependency file changed without justification. Add context about why dependencies were changed to the PR description or linked issue.` + })); + + // --- Report result --- + if (justified) { + await github.rest.checks.create({ + owner: context.repo.owner, + repo: context.repo.repo, + head_sha: context.sha, + name: CHECK_NAME, + status: 'completed', + conclusion: 'success', + output: { + title: `Dependency changes: ${changedDependencyFiles.length} file(s) changed with justification`, + summary: `Dependency files were modified and justification was found in the PR body or linked issue.\n\n**Changed dependency files:**\n${changedDependencyFiles.map(f => '- `' + f.filename + '`').join('\n')}`, + annotations: annotations + } + }); + } else { + const fileList = changedDependencyFiles.map(f => '- `' + f.filename + '`').join('\n'); + await github.rest.checks.create({ + owner: context.repo.owner, + repo: context.repo.repo, + head_sha: context.sha, + name: CHECK_NAME, + status: 'completed', + conclusion: configuredConclusion, + output: { + title: `Dependency changes: ${changedDependencyFiles.length} file(s) changed without justification`, + summary: `Dependency files were modified but no justification was found.\n\n**Changed dependency files:**\n${fileList}\n\n**To resolve:** Add context about dependency changes to the PR description using keywords like: ${JUSTIFICATION_KEYWORDS.slice(0, 8).map(k => '"' + k + '"').join(', ')}, etc.\n\nAlternatively, a PR approval will override this check.`, + annotations: annotations + } + }); + } diff --git a/.github/workflows/guardrail-scope.yml b/.github/workflows/guardrail-scope.yml new file mode 100644 index 0000000..5f231fb --- /dev/null +++ b/.github/workflows/guardrail-scope.yml @@ -0,0 +1,254 @@ +name: "Guardrail: Scope Enforcement" + +on: + pull_request: + types: [opened, synchronize] + +permissions: + checks: write + issues: read + pull-requests: read + contents: read + +jobs: + scope-check: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + sparse-checkout: .github/agent-workflow + + - name: Check scope enforcement + uses: actions/github-script@v7 + with: + script: | + const checkName = 'guardrail/scope'; + + // --- Helper: read config --- + async function readConfig() { + try { + const { data } = await github.rest.repos.getContent({ + owner: context.repo.owner, + repo: context.repo.repo, + path: '.github/agent-workflow/config.yaml', + ref: context.payload.pull_request.head.sha + }); + const content = Buffer.from(data.content, 'base64').toString('utf-8'); + // Simple YAML parsing for the scope-enforcement section + const lines = content.split('\n'); + let inScope = false; + const config = { enabled: true, conclusion: 'action_required' }; + for (const line of lines) { + if (/^scope-enforcement:/.test(line)) { + inScope = true; + continue; + } + if (inScope && /^\S/.test(line)) { + break; // Next top-level key + } + if (inScope) { + const enabledMatch = line.match(/^\s+enabled:\s*(true|false)/); + if (enabledMatch) config.enabled = enabledMatch[1] === 'true'; + const conclusionMatch = line.match(/^\s+conclusion:\s*(\S+)/); + if (conclusionMatch) config.conclusion = conclusionMatch[1]; + } + } + return config; + } catch (e) { + // Config file not found — use defaults (enabled) + return { enabled: true, conclusion: 'action_required' }; + } + } + + // --- Helper: create check run --- + async function createCheckRun(conclusion, title, summary, annotations = []) { + const output = { title, summary }; + if (annotations.length > 0) { + // GitHub API limits to 50 annotations per request + output.annotations = annotations.slice(0, 50); + } + await github.rest.checks.create({ + owner: context.repo.owner, + repo: context.repo.repo, + head_sha: context.payload.pull_request.head.sha, + name: checkName, + status: 'completed', + conclusion, + output + }); + } + + // --- Step 1: Read config and check if enabled --- + const config = await readConfig(); + if (!config.enabled) { + await createCheckRun( + 'success', + 'Scope enforcement: disabled', + 'Scope enforcement is disabled in agent-workflow config.' + ); + return; + } + + // --- Step 2: Check for non-stale PR approval override --- + const reviews = await github.rest.pulls.listReviews({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: context.payload.pull_request.number + }); + const headSha = context.payload.pull_request.head.sha; + const hasValidApproval = reviews.data.some( + r => r.state === 'APPROVED' && r.commit_id === headSha + ); + + // --- Step 3: Parse issue reference from PR body --- + const prBody = context.payload.pull_request.body || ''; + const issueMatch = prBody.match(/[Ff]ixes\s+#(\d+)/); + if (!issueMatch) { + // No linked issue — cannot enforce scope, pass with note + await createCheckRun( + 'success', + 'Scope enforcement: no linked issue', + 'No `fixes #N` reference found in PR description. Scope enforcement skipped.' + ); + return; + } + const issueNumber = parseInt(issueMatch[1], 10); + + // --- Step 4: Get the issue body --- + let issueBody; + try { + const issue = await github.rest.issues.get({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: issueNumber + }); + issueBody = issue.data.body || ''; + } catch (e) { + await createCheckRun( + 'success', + 'Scope enforcement: issue not found', + `Could not read issue #${issueNumber}. Scope enforcement skipped.` + ); + return; + } + + // --- Step 5: Extract file paths from the issue body --- + // Match paths that look like file paths: + // - backtick-wrapped paths: `src/foo/bar.ts` + // - paths with extensions: src/foo/bar.ts + // - paths in markdown code blocks + const filePathPatterns = [ + // Backtick-wrapped paths with extension + /`([a-zA-Z0-9_./-]+\.[a-zA-Z0-9]+)`/g, + // Bare paths with at least one slash and an extension (common in issue descriptions) + /(?:^|\s)((?:[a-zA-Z0-9_.-]+\/)+[a-zA-Z0-9_.-]+\.[a-zA-Z0-9]+)(?:\s|$|[,;)])/gm, + // Paths starting with ./ or common root dirs + /(?:^|\s)(\.?(?:src|lib|app|test|tests|spec|pkg|cmd|internal|\.github)\/[a-zA-Z0-9_./-]+)(?:\s|$|[,;)])/gm + ]; + + const scopeFiles = new Set(); + for (const pattern of filePathPatterns) { + let match; + while ((match = pattern.exec(issueBody)) !== null) { + const filePath = match[1].replace(/^\//, ''); // strip leading slash + scopeFiles.add(filePath); + } + } + + if (scopeFiles.size === 0) { + // No file paths found in issue — cannot enforce scope + await createCheckRun( + 'success', + 'Scope enforcement: no files listed in issue', + `Issue #${issueNumber} does not list any file paths. Scope enforcement skipped.` + ); + return; + } + + // --- Step 6: Get changed files from the PR --- + const changedFiles = []; + let page = 1; + while (true) { + const resp = await github.rest.pulls.listFiles({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: context.payload.pull_request.number, + per_page: 100, + page + }); + changedFiles.push(...resp.data); + if (resp.data.length < 100) break; + page++; + } + + // --- Step 7: Compare changed files against scope --- + // A changed file is "in scope" if: + // - It exactly matches a scope file, OR + // - It is inside a directory listed in scope, OR + // - A scope file is a prefix of the changed file path + function isInScope(changedPath) { + for (const scopePath of scopeFiles) { + // Exact match + if (changedPath === scopePath) return true; + // Scope entry is a directory prefix (e.g., scope lists "src/auth/", + // changed file is "src/auth/middleware.ts") + if (changedPath.startsWith(scopePath.endsWith('/') ? scopePath : scopePath + '/')) return true; + } + return false; + } + + const outOfScope = changedFiles.filter(f => !isInScope(f.filename)); + + // --- Step 8: Report results --- + if (outOfScope.length === 0) { + await createCheckRun( + 'success', + 'Scope enforcement: all files in scope', + `All ${changedFiles.length} changed files are listed in issue #${issueNumber}.` + ); + return; + } + + // There are out-of-scope files + if (hasValidApproval) { + await createCheckRun( + 'success', + `Scope enforcement: approved by reviewer (${outOfScope.length} files outside scope)`, + `PR modifies ${outOfScope.length} file(s) not listed in issue #${issueNumber}, but a non-stale approval exists.\n\nOut-of-scope files:\n${outOfScope.map(f => '- `' + f.filename + '`').join('\n')}` + ); + return; + } + + // Build annotations for out-of-scope files + const annotations = outOfScope.map(f => ({ + path: f.filename, + start_line: 1, + end_line: 1, + annotation_level: 'warning', + message: `This file is not listed in the task scope for issue #${issueNumber}. If this change is intentional, approve the PR to override.` + })); + + // Determine conclusion based on config and severity + // Minor: 1-2 files out of scope; Significant: 3+ + const isMinor = outOfScope.length <= 2; + const conclusion = isMinor ? 'neutral' : config.conclusion; + + const summary = [ + `PR modifies ${outOfScope.length} file(s) not listed in issue #${issueNumber}.`, + '', + '**Out-of-scope files:**', + ...outOfScope.map(f => `- \`${f.filename}\``), + '', + `**In-scope files (from issue):**`, + ...[...scopeFiles].map(f => `- \`${f}\``), + '', + 'To resolve: either update the issue to include these files, or approve the PR to override this check.' + ].join('\n'); + + await createCheckRun( + conclusion, + `Scope enforcement: ${outOfScope.length} file(s) outside task scope`, + summary, + annotations + ); diff --git a/.github/workflows/guardrail-test-ratio.yml b/.github/workflows/guardrail-test-ratio.yml new file mode 100644 index 0000000..5b7743c --- /dev/null +++ b/.github/workflows/guardrail-test-ratio.yml @@ -0,0 +1,209 @@ +name: "Guardrail: Test-to-Code Ratio" + +on: + pull_request: + types: [opened, synchronize] + +permissions: + checks: write + pull-requests: read + contents: read + +jobs: + test-ratio-check: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Check test-to-code ratio + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + const yaml = require('js-yaml'); + + // --- Load configuration --- + const configPath = '.github/agent-workflow/config.yaml'; + let threshold = 0.5; + let enabled = true; + let configuredConclusion = 'action_required'; + + try { + const configContent = fs.readFileSync(configPath, 'utf8'); + const config = yaml.load(configContent); + const testRatioConfig = config?.guardrails?.['test-ratio'] || {}; + threshold = testRatioConfig.threshold ?? 0.5; + enabled = testRatioConfig.enabled ?? true; + configuredConclusion = testRatioConfig.conclusion ?? 'action_required'; + } catch (e) { + core.info(`Could not read config from ${configPath}, using defaults: ${e.message}`); + } + + if (!enabled) { + core.info('Test-ratio guardrail is disabled in config. Skipping.'); + return; + } + + // --- Get PR files --- + const prNumber = context.payload.pull_request.number; + const allFiles = []; + let page = 1; + while (true) { + const { data: files } = await github.rest.pulls.listFiles({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: prNumber, + per_page: 100, + page: page, + }); + allFiles.push(...files); + if (files.length < 100) break; + page++; + } + + // --- Categorize files as test vs implementation --- + // Test file patterns: *test*, *spec*, __tests__/ + function isTestFile(filename) { + const lower = filename.toLowerCase(); + // Check for __tests__/ directory + if (lower.includes('__tests__/')) return true; + // Check for test/spec in filename (not directory names) + const basename = lower.split('/').pop(); + if (basename.includes('test') || basename.includes('spec')) return true; + // Check for test/spec directories like tests/, specs/ + const parts = lower.split('/'); + for (const part of parts) { + if (part === 'tests' || part === 'specs' || part === 'test' || part === 'spec') return true; + } + return false; + } + + // Ignore non-code files (config, docs, etc.) + function isCodeFile(filename) { + const codeExtensions = [ + '.js', '.jsx', '.ts', '.tsx', '.py', '.rb', '.go', '.rs', + '.java', '.kt', '.scala', '.cs', '.cpp', '.c', '.h', '.hpp', + '.swift', '.m', '.mm', '.php', '.lua', '.sh', '.bash', + '.yml', '.yaml', '.json', '.toml', '.xml', + ]; + const ext = '.' + filename.split('.').pop().toLowerCase(); + return codeExtensions.includes(ext); + } + + let testLines = 0; + let implLines = 0; + const implFilesWithNoTests = []; + + for (const file of allFiles) { + // Skip removed files + if (file.status === 'removed') continue; + // Skip non-code files + if (!isCodeFile(file.filename)) continue; + + const additions = file.additions || 0; + + if (isTestFile(file.filename)) { + testLines += additions; + } else { + implLines += additions; + implFilesWithNoTests.push({ + filename: file.filename, + additions: additions, + }); + } + } + + core.info(`Test lines added: ${testLines}`); + core.info(`Implementation lines added: ${implLines}`); + + // --- Handle edge case: no implementation lines --- + if (implLines === 0) { + core.info('No implementation lines in this PR. Reporting success.'); + await github.rest.checks.create({ + owner: context.repo.owner, + repo: context.repo.repo, + head_sha: context.payload.pull_request.head.sha, + name: 'guardrail/test-ratio', + status: 'completed', + conclusion: 'success', + output: { + title: 'Test-to-code ratio: no implementation changes', + summary: 'This PR contains no implementation line additions. Test ratio check is not applicable.', + }, + }); + return; + } + + // --- Calculate ratio --- + const ratio = testLines / implLines; + const passed = ratio >= threshold; + + core.info(`Test-to-code ratio: ${ratio.toFixed(2)} (threshold: ${threshold})`); + + // --- Check for non-stale PR approval override --- + const { data: reviews } = await github.rest.pulls.listReviews({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: prNumber, + }); + + const headSha = context.payload.pull_request.head.sha; + const hasValidApproval = reviews.some( + (r) => r.state === 'APPROVED' && r.commit_id === headSha + ); + + // --- Determine conclusion --- + let conclusion; + let title; + let summary; + + if (passed) { + conclusion = 'success'; + title = `Test-to-code ratio: ${ratio.toFixed(2)} (threshold: ${threshold})`; + summary = `PR has ${testLines} test lines and ${implLines} implementation lines added. Ratio ${ratio.toFixed(2)} meets the threshold of ${threshold}.`; + } else if (hasValidApproval) { + conclusion = 'success'; + title = `Test-to-code ratio: ${ratio.toFixed(2)} — approved by reviewer`; + summary = `PR has ${testLines} test lines and ${implLines} implementation lines added. Ratio ${ratio.toFixed(2)} is below the threshold of ${threshold}, but a non-stale approval exists. Human has accepted the current state.`; + } else { + conclusion = configuredConclusion; + title = `Test-to-code ratio: ${ratio.toFixed(2)} (threshold: ${threshold})`; + summary = `PR has ${testLines} test lines and ${implLines} implementation lines added. Ratio ${ratio.toFixed(2)} is below the threshold of ${threshold}. Add more tests or approve the PR to override.`; + } + + // --- Build annotations for implementation files lacking test coverage --- + const annotations = []; + if (!passed && !hasValidApproval) { + for (const file of implFilesWithNoTests) { + if (file.additions > 0) { + annotations.push({ + path: file.filename, + start_line: 1, + end_line: 1, + annotation_level: 'warning', + message: `This implementation file has ${file.additions} added lines. The overall test-to-code ratio (${ratio.toFixed(2)}) is below the threshold (${threshold}). Consider adding corresponding tests.`, + }); + } + } + } + + // GitHub API limits annotations to 50 per request + const truncatedAnnotations = annotations.slice(0, 50); + + // --- Report check run --- + await github.rest.checks.create({ + owner: context.repo.owner, + repo: context.repo.repo, + head_sha: headSha, + name: 'guardrail/test-ratio', + status: 'completed', + conclusion: conclusion, + output: { + title: title, + summary: summary, + annotations: truncatedAnnotations, + }, + }); + + core.info(`Check run created with conclusion: ${conclusion}`); From b06ce0d4a87572d5de6dccaaa2a379ca52bd6941 Mon Sep 17 00:00:00 2001 From: Joe Delfino Date: Thu, 12 Feb 2026 13:57:36 +0000 Subject: [PATCH 03/23] feat: add human review to issues Action (#30) Workflow triggers on PR review submission, parses comments, creates child issues with severity labels, links as sub-issues, and sets blocking dependencies for critical findings. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/human-review.yml | 217 +++++++++++++++++++++++++++++ 1 file changed, 217 insertions(+) create mode 100644 .github/workflows/human-review.yml diff --git a/.github/workflows/human-review.yml b/.github/workflows/human-review.yml new file mode 100644 index 0000000..72c6ce6 --- /dev/null +++ b/.github/workflows/human-review.yml @@ -0,0 +1,217 @@ +name: Human Review → Issues + +"on": + pull_request_review: + types: [submitted] + +permissions: + issues: write + pull-requests: write + contents: read + +jobs: + process-review: + runs-on: ubuntu-latest + steps: + - name: Process review comments and create issues + uses: actions/github-script@v7 + with: + script: | + const review = context.payload.review; + const pr = context.payload.pull_request; + + // ── Parse parent issue from PR body ─────────────────────── + const fixesMatch = pr.body && pr.body.match(/[Ff]ixes\s*#(\d+)/); + if (!fixesMatch) { + console.log('No parent issue found — PR body has no "Fixes #N" reference. Skipping.'); + return; + } + const parentIssueNumber = parseInt(fixesMatch[1], 10); + + // ── Fetch comments for this specific review ───────────────── + // GET /repos/{owner}/{repo}/pulls/{pull_number}/reviews/{review_id}/comments + const { data: reviewComments } = await github.request( + 'GET /repos/{owner}/{repo}/pulls/{pull_number}/reviews/{review_id}/comments', + { + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: pr.number, + review_id: review.id, + } + ); + + if (reviewComments.length === 0) { + console.log('Review has no line-level comments. Skipping.'); + return; + } + + // ── Get parent issue node_id for GraphQL sub-issue linking ─ + const { data: parentIssue } = await github.rest.issues.get({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: parentIssueNumber, + }); + const parentNodeId = parentIssue.node_id; + + // ── Severity detection ──────────────────────────────────── + function detectSeverity(body) { + const lower = body.toLowerCase(); + const blockingPatterns = /\b(blocking|block|must[- ]fix)\b/; + const suggestionPatterns = /\b(suggestion|nit|consider)\b/; + + if (blockingPatterns.test(lower)) return 'blocking'; + if (suggestionPatterns.test(lower)) return 'suggestion'; + return 'should-fix'; + } + + // ── Ensure labels exist ─────────────────────────────────── + const severityLabels = ['blocking', 'should-fix', 'suggestion']; + const labelColors = { + 'blocking': 'B60205', + 'should-fix': 'D93F0B', + 'suggestion': '0E8A16', + }; + for (const label of severityLabels) { + try { + await github.rest.issues.getLabel({ + owner: context.repo.owner, + repo: context.repo.repo, + name: label, + }); + } catch { + await github.rest.issues.createLabel({ + owner: context.repo.owner, + repo: context.repo.repo, + name: label, + color: labelColors[label], + }); + } + } + + // ── Process each comment ────────────────────────────────── + const createdIssues = []; + let hasBlockingComments = false; + + for (const comment of reviewComments) { + const severity = detectSeverity(comment.body); + const filePath = comment.path; + const line = comment.original_line || comment.line || 0; + + // Build issue body with file/line context + const issueBody = [ + `## Review Finding`, + ``, + `**Severity:** \`${severity}\``, + `**File:** \`${filePath}\`${line ? ` (line ${line})` : ''}`, + `**PR:** #${pr.number}`, + `**Reviewer:** @${review.user.login}`, + ``, + `### Comment`, + ``, + comment.body, + ``, + `---`, + `_Created automatically from a PR review comment._`, + ].join('\n'); + + const issueTitle = `[${severity}] ${filePath}${line ? `:${line}` : ''}: ${comment.body.split('\n')[0].substring(0, 80)}`; + + // Create the child issue + const { data: newIssue } = await github.rest.issues.create({ + owner: context.repo.owner, + repo: context.repo.repo, + title: issueTitle, + body: issueBody, + labels: [severity], + }); + + console.log(`Created issue #${newIssue.number}: ${newIssue.title}`); + + // Link as sub-issue via GraphQL addSubIssue mutation + try { + await github.graphql(` + mutation($parentId: ID!, $childId: ID!) { + addSubIssue(input: { issueId: $parentId, subIssueId: $childId }) { + issue { id } + subIssue { id } + } + } + `, { + parentId: parentNodeId, + childId: newIssue.node_id, + }); + console.log(`Linked issue #${newIssue.number} as sub-issue of #${parentIssueNumber}`); + } catch (err) { + console.log(`Warning: Could not link sub-issue via GraphQL: ${err.message}`); + } + + // If blocking, set as blocking the parent issue + if (severity === 'blocking') { + hasBlockingComments = true; + try { + await github.request( + 'POST /repos/{owner}/{repo}/issues/{issue_number}/sub_issues/priority', + { + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: parentIssueNumber, + sub_issue_id: newIssue.id, + } + ); + } catch (err) { + console.log(`Note: Could not set blocking dependency via REST: ${err.message}`); + } + } + + createdIssues.push({ + number: newIssue.number, + severity, + }); + } + + // ── Update PR description with Fixes references ────────── + if (createdIssues.length > 0) { + const fixesLines = createdIssues + .map(i => `Fixes #${i.number}`) + .join('\n'); + + const newSection = [ + '', + '### Review Finding Issues', + fixesLines, + '', + ].join('\n'); + + // Replace existing section or append, for idempotent updates + let currentBody = pr.body || ''; + const sectionRegex = /[\s\S]*?/; + let updatedBody; + if (sectionRegex.test(currentBody)) { + updatedBody = currentBody.replace(sectionRegex, newSection); + } else { + updatedBody = currentBody + '\n\n' + newSection; + } + + await github.rest.pulls.update({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: pr.number, + body: updatedBody, + }); + + console.log(`Updated PR #${pr.number} body with ${createdIssues.length} Fixes references`); + } + + // ── Summary ────────────────────────────────────────────── + const blockingCount = createdIssues.filter(i => i.severity === 'blocking').length; + const shouldFixCount = createdIssues.filter(i => i.severity === 'should-fix').length; + const suggestionCount = createdIssues.filter(i => i.severity === 'suggestion').length; + + console.log(`\nDone. Created ${createdIssues.length} issues from review comments:`); + console.log(` blocking: ${blockingCount}`); + console.log(` should-fix: ${shouldFixCount}`); + console.log(` suggestion: ${suggestionCount}`); + + if (hasBlockingComments) { + console.log(`\nBlocking issues were created — parent issue #${parentIssueNumber} has new blockers.`); + } From bed9195d7a151d04b86a2097b8f08e37755537c8 Mon Sep 17 00:00:00 2001 From: Joe Delfino Date: Thu, 12 Feb 2026 13:57:40 +0000 Subject: [PATCH 04/23] feat: add PR review workflow (#16) Three parallel reviewer agents (correctness, tests, architecture) triggered on PR open/sync via claude -p. Shared context resolution job parses Fixes #N. Supports workflow_dispatch for re-triggering. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/pr-review.yml | 213 ++++++++++++++++++++++++++++++++ 1 file changed, 213 insertions(+) create mode 100644 .github/workflows/pr-review.yml diff --git a/.github/workflows/pr-review.yml b/.github/workflows/pr-review.yml new file mode 100644 index 0000000..754fdb2 --- /dev/null +++ b/.github/workflows/pr-review.yml @@ -0,0 +1,213 @@ +name: PR Review + +"on": + pull_request: + types: [opened, synchronize] + workflow_dispatch: + inputs: + pr-number: + description: "PR number to review (used by orchestrator for re-review)" + required: true + type: number + +permissions: + contents: write + issues: write + pull-requests: read + +jobs: + # ── Resolve context shared by all reviewers ────────────────────────── + resolve-context: + runs-on: ubuntu-latest + outputs: + pr-number: ${{ steps.ctx.outputs.pr-number }} + parent-issue: ${{ steps.ctx.outputs.parent-issue }} + base-branch: ${{ steps.ctx.outputs.base-branch }} + pr-title: ${{ steps.ctx.outputs.pr-title }} + steps: + - name: Resolve PR context + id: ctx + uses: actions/github-script@v7 + with: + script: | + let prNumber; + let prData; + + if (context.eventName === 'workflow_dispatch') { + prNumber = parseInt('${{ inputs.pr-number }}', 10); + const { data } = await github.rest.pulls.get({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: prNumber, + }); + prData = data; + } else { + prNumber = context.payload.pull_request.number; + prData = context.payload.pull_request; + } + + // Parse "fixes #N" or "Fixes #N" from PR body + const body = prData.body || ''; + const fixesMatch = body.match(/[Ff]ixes\s*#(\d+)/); + const parentIssue = fixesMatch ? fixesMatch[1] : ''; + + if (!parentIssue) { + console.log('No parent issue found — PR body has no "Fixes #N" reference.'); + } + + core.setOutput('pr-number', prNumber.toString()); + core.setOutput('parent-issue', parentIssue); + core.setOutput('base-branch', prData.base.ref); + core.setOutput('pr-title', prData.title); + + # ── Correctness Reviewer ───────────────────────────────────────────── + review-correctness: + needs: resolve-context + if: needs.resolve-context.outputs.parent-issue != '' + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Install Claude Code + run: npm install -g @anthropic-ai/claude-code + + - name: Run correctness review + env: + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + PR_NUMBER: ${{ needs.resolve-context.outputs.pr-number }} + PARENT_ISSUE: ${{ needs.resolve-context.outputs.parent-issue }} + BASE_BRANCH: ${{ needs.resolve-context.outputs.base-branch }} + PR_TITLE: ${{ needs.resolve-context.outputs.pr-title }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + claude -p "$(cat <<'PROMPT' + You are the Correctness Reviewer. Read and follow the skill file at + .claude/skills/reviewer-correctness/SKILL.md + + ## Context + - Repository: ${{ github.repository }} + - PR number: ${PR_NUMBER} + - PR title: ${PR_TITLE} + - Parent issue number: ${PARENT_ISSUE} + - Base branch: origin/${BASE_BRANCH} + + ## Instructions + 1. Run `git diff origin/${BASE_BRANCH}...HEAD` to get the full diff + 2. Review every changed file for bugs, error handling gaps, security issues, and API contract mismatches + 3. For each finding, create a child issue using `gh issue create`: + - Use labels: `blocking`, `should-fix`, or `suggestion` + - Include file path and line number in the issue body + 4. Link each created issue as a sub-issue of #${PARENT_ISSUE} using the GraphQL API: + - Get parent node ID and child node ID + - Use the addSubIssue mutation (see .claude/skills/github-issues/SKILL.md) + 5. For `blocking` findings, set them as blocking #${PARENT_ISSUE} using the + blocked-by dependency REST API (see .claude/skills/github-issues/SKILL.md) + 6. Report your outcome in the structured format from the skill file + PROMPT + )" + + # ── Tests Reviewer ─────────────────────────────────────────────────── + review-tests: + needs: resolve-context + if: needs.resolve-context.outputs.parent-issue != '' + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Install Claude Code + run: npm install -g @anthropic-ai/claude-code + + - name: Run tests review + env: + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + PR_NUMBER: ${{ needs.resolve-context.outputs.pr-number }} + PARENT_ISSUE: ${{ needs.resolve-context.outputs.parent-issue }} + BASE_BRANCH: ${{ needs.resolve-context.outputs.base-branch }} + PR_TITLE: ${{ needs.resolve-context.outputs.pr-title }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + claude -p "$(cat <<'PROMPT' + You are the Test Quality Reviewer. Read and follow the skill file at + .claude/skills/reviewer-tests/SKILL.md + + ## Context + - Repository: ${{ github.repository }} + - PR number: ${PR_NUMBER} + - PR title: ${PR_TITLE} + - Parent issue number: ${PARENT_ISSUE} + - Base branch: origin/${BASE_BRANCH} + + ## Instructions + 1. Run `git diff origin/${BASE_BRANCH}...HEAD --stat` to identify changed files + 2. For every changed production file, find its corresponding test file + 3. Read each test file and evaluate: meaningful assertions, mock vs real behavior, + integration test coverage, edge cases, and test organization + 4. For each finding, create a child issue using `gh issue create`: + - Use labels: `blocking`, `should-fix`, or `suggestion` + - Include file path and description of what is missing or wrong + 5. Link each created issue as a sub-issue of #${PARENT_ISSUE} using the GraphQL API: + - Get parent node ID and child node ID + - Use the addSubIssue mutation (see .claude/skills/github-issues/SKILL.md) + 6. For `blocking` findings, set them as blocking #${PARENT_ISSUE} using the + blocked-by dependency REST API (see .claude/skills/github-issues/SKILL.md) + 7. Report your outcome in the structured format from the skill file + PROMPT + )" + + # ── Architecture Reviewer ──────────────────────────────────────────── + review-architecture: + needs: resolve-context + if: needs.resolve-context.outputs.parent-issue != '' + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Install Claude Code + run: npm install -g @anthropic-ai/claude-code + + - name: Run architecture review + env: + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + PR_NUMBER: ${{ needs.resolve-context.outputs.pr-number }} + PARENT_ISSUE: ${{ needs.resolve-context.outputs.parent-issue }} + BASE_BRANCH: ${{ needs.resolve-context.outputs.base-branch }} + PR_TITLE: ${{ needs.resolve-context.outputs.pr-title }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + claude -p "$(cat <<'PROMPT' + You are the Architecture Reviewer. Read and follow the skill file at + .claude/skills/reviewer-architecture/SKILL.md + + ## Context + - Repository: ${{ github.repository }} + - PR number: ${PR_NUMBER} + - PR title: ${PR_TITLE} + - Parent issue number: ${PARENT_ISSUE} + - Base branch: origin/${BASE_BRANCH} + + ## Instructions + 1. Run `git diff origin/${BASE_BRANCH}...HEAD --stat` to understand what changed + 2. Read the full codebase context — not just the diff — to catch duplication, + pattern divergence, and structural issues + 3. Check for: duplicated types, copy-pasted logic, pattern inconsistency, + leaky abstractions, unnecessary coupling, missing shared code + 4. For each finding, create a child issue using `gh issue create`: + - Use labels: `blocking`, `should-fix`, or `suggestion` + - Include file paths and reference to existing patterns + 5. Link each created issue as a sub-issue of #${PARENT_ISSUE} using the GraphQL API: + - Get parent node ID and child node ID + - Use the addSubIssue mutation (see .claude/skills/github-issues/SKILL.md) + 6. For `blocking` findings, set them as blocking #${PARENT_ISSUE} using the + blocked-by dependency REST API (see .claude/skills/github-issues/SKILL.md) + 7. Report your outcome in the structured format from the skill file + PROMPT + )" From 7daa7fcaa9ed98697d4d4e13ed11217875943c0c Mon Sep 17 00:00:00 2001 From: Joe Delfino Date: Thu, 12 Feb 2026 14:00:55 +0000 Subject: [PATCH 05/23] feat: add orchestrator status check (#17) Gates PR merges by evaluating blocking sub-issues, running claude-based re-review assessments with configurable cycle cap, and triggering pr-review.yml when re-review is warranted. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/orchestrator-check.yml | 442 +++++++++++++++++++++++ 1 file changed, 442 insertions(+) create mode 100644 .github/workflows/orchestrator-check.yml diff --git a/.github/workflows/orchestrator-check.yml b/.github/workflows/orchestrator-check.yml new file mode 100644 index 0000000..40b3524 --- /dev/null +++ b/.github/workflows/orchestrator-check.yml @@ -0,0 +1,442 @@ +name: Orchestrator Status Check + +"on": + issues: + types: [opened, closed, labeled, unlabeled] + pull_request: + types: [synchronize] + +permissions: + checks: write + issues: read + pull-requests: read + contents: read + actions: write + +jobs: + orchestrator: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + sparse-checkout: .github/agent-workflow + fetch-depth: 0 + + - name: Orchestrator check + uses: actions/github-script@v7 + env: + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + with: + script: | + const checkName = 'orchestrator'; + + // ── Helper: read re-review-cycle-cap from config ──────────── + async function readCycleCap() { + try { + const { data } = await github.rest.repos.getContent({ + owner: context.repo.owner, + repo: context.repo.repo, + path: '.github/agent-workflow/config.yaml', + ref: context.sha + }); + const content = Buffer.from(data.content, 'base64').toString('utf-8'); + const match = content.match(/^re-review-cycle-cap:\s*(\d+)/m); + return match ? parseInt(match[1], 10) : 3; + } catch { + return 3; // default + } + } + + // ── Helper: create check run on a specific SHA ────────────── + async function createCheckRun(headSha, conclusion, title, summary) { + await github.rest.checks.create({ + owner: context.repo.owner, + repo: context.repo.repo, + head_sha: headSha, + name: checkName, + status: 'completed', + conclusion, + output: { title, summary } + }); + } + + // ── Helper: find PRs referencing a given issue ────────────── + // Searches open PRs whose body contains "Fixes #N" or "fixes #N" + async function findPRsForIssue(issueNumber) { + const prs = []; + let page = 1; + while (true) { + const resp = await github.rest.pulls.list({ + owner: context.repo.owner, + repo: context.repo.repo, + state: 'open', + per_page: 100, + page + }); + if (resp.data.length === 0) break; + for (const pr of resp.data) { + const body = pr.body || ''; + const regex = new RegExp(`[Ff]ixes\\s*#${issueNumber}\\b`); + if (regex.test(body)) { + prs.push(pr); + } + } + if (resp.data.length < 100) break; + page++; + } + return prs; + } + + // ── Helper: query sub-issues via GraphQL ──────────────────── + async function getSubIssues(issueNumber) { + const query = ` + query($owner: String!, $repo: String!, $number: Int!) { + repository(owner: $owner, name: $repo) { + issue(number: $number) { + subIssues(first: 50) { + nodes { + number + title + state + labels(first: 10) { nodes { name } } + } + } + } + } + } + `; + try { + const result = await github.graphql(query, { + owner: context.repo.owner, + repo: context.repo.repo, + number: issueNumber + }); + return result.repository.issue.subIssues.nodes; + } catch (err) { + console.log(`Warning: GraphQL sub-issues query failed: ${err.message}`); + return []; + } + } + + // ── Helper: count past review cycles from PR comments ─────── + // We count comments left by the pr-review workflow (bot) that + // indicate a review cycle was triggered. + async function countReviewCycles(prNumber) { + const comments = await github.paginate( + github.rest.issues.listComments, + { + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: prNumber, + per_page: 100 + } + ); + // Count comments that contain the re-review marker + return comments.filter( + c => c.body && c.body.includes('') + ).length; + } + + // ── Step 1: Determine the parent issue and PR ─────────────── + let parentIssueNumber; + let prNumber; + let headSha; + + if (context.eventName === 'pull_request') { + // PR synchronize event — parse parent issue from PR body + const pr = context.payload.pull_request; + prNumber = pr.number; + headSha = pr.head.sha; + const body = pr.body || ''; + const fixesMatch = body.match(/[Ff]ixes\s*#(\d+)/); + if (!fixesMatch) { + console.log('No "Fixes #N" in PR body. Skipping orchestrator check.'); + await createCheckRun( + headSha, + 'neutral', + 'Orchestrator: no linked issue', + 'No `Fixes #N` reference found in PR description. Orchestrator check skipped.' + ); + return; + } + parentIssueNumber = parseInt(fixesMatch[1], 10); + + } else if (context.eventName === 'issues') { + // Issue event — find the PR that references this issue's parent + // The changed issue might BE a sub-issue (review finding). + // We need to find which parent issue it belongs to, then find the PR. + const changedIssue = context.payload.issue; + + // Strategy: search for open PRs that reference this issue or any + // issue that is a parent of this issue. Since sub-issues are children + // of the parent issue, and the PR has "Fixes #parent", we need to + // find PRs referencing any issue. We search for PRs that reference + // issues that have this changed issue as a sub-issue. + // + // Simpler approach: search all open PRs for ones containing + // "Fixes #N" and then check if the changed issue is a sub-issue of N. + // But that's expensive. + // + // Practical approach: list all open PRs, parse their "Fixes #N", + // check if the changed issue is a sub-issue of any of those N values. + + const openPRs = []; + let page = 1; + while (true) { + const resp = await github.rest.pulls.list({ + owner: context.repo.owner, + repo: context.repo.repo, + state: 'open', + per_page: 100, + page + }); + if (resp.data.length === 0) break; + openPRs.push(...resp.data); + if (resp.data.length < 100) break; + page++; + } + + // Build a map of parent issue number -> PR data + const parentToPR = new Map(); + for (const pr of openPRs) { + const body = pr.body || ''; + const match = body.match(/[Ff]ixes\s*#(\d+)/); + if (match) { + parentToPR.set(parseInt(match[1], 10), pr); + } + } + + if (parentToPR.size === 0) { + console.log('No open PRs with "Fixes #N" references found. Nothing to do.'); + return; + } + + // Check if the changed issue IS a parent issue referenced by a PR + if (parentToPR.has(changedIssue.number)) { + parentIssueNumber = changedIssue.number; + const pr = parentToPR.get(changedIssue.number); + prNumber = pr.number; + headSha = pr.head.sha; + } else { + // Check if the changed issue is a sub-issue of any parent + let found = false; + for (const [parentNum, pr] of parentToPR.entries()) { + const subIssues = await getSubIssues(parentNum); + const isChild = subIssues.some( + si => si.number === changedIssue.number + ); + if (isChild) { + parentIssueNumber = parentNum; + prNumber = pr.number; + headSha = pr.head.sha; + found = true; + break; + } + } + if (!found) { + console.log( + `Issue #${changedIssue.number} is not a sub-issue of any PR-linked parent. Nothing to do.` + ); + return; + } + } + } else { + console.log(`Unexpected event: ${context.eventName}. Skipping.`); + return; + } + + console.log(`Parent issue: #${parentIssueNumber}, PR: #${prNumber}, HEAD: ${headSha}`); + + // ── Step 2: Query sub-issues and check for blockers ───────── + const subIssues = await getSubIssues(parentIssueNumber); + console.log(`Found ${subIssues.length} sub-issues for #${parentIssueNumber}`); + + const openBlockers = subIssues.filter(si => { + if (si.state !== 'OPEN') return false; + const labels = si.labels.nodes.map(l => l.name); + return labels.includes('blocking'); + }); + + // ── Step 3: If blockers exist, report failing check ───────── + if (openBlockers.length > 0) { + const blockerList = openBlockers + .map(si => `- #${si.number}: ${si.title}`) + .join('\n'); + + await createCheckRun( + headSha, + 'action_required', + `Orchestrator: ${openBlockers.length} blocking issue(s)`, + [ + `PR #${prNumber} cannot merge. Issue #${parentIssueNumber} has open blocking sub-issues:`, + '', + blockerList, + '', + 'Resolve these blocking issues or approve the PR to override.' + ].join('\n') + ); + console.log(`Reported failing check: ${openBlockers.length} blockers.`); + return; + } + + // ── Step 4: No blockers — assess re-review need ───────────── + console.log('No open blockers. Assessing re-review need...'); + + const cycleCap = await readCycleCap(); + const pastCycles = await countReviewCycles(prNumber); + console.log(`Review cycles so far: ${pastCycles}, cap: ${cycleCap}`); + + // If we've reached the cycle cap, pass without re-review + if (pastCycles >= cycleCap) { + await createCheckRun( + headSha, + 'success', + 'Orchestrator: passing (review cycle cap reached)', + [ + `All blocking sub-issues for #${parentIssueNumber} are resolved.`, + '', + `Re-review cycle cap reached (${pastCycles}/${cycleCap}). No further re-review.`, + 'PR is clear to merge.' + ].join('\n') + ); + console.log('Cycle cap reached. Reporting success.'); + return; + } + + // For pull_request synchronize events (new commits pushed), + // or when blockers just cleared, assess whether re-review is needed. + // Use claude -p for the assessment. + let needsReReview = false; + + try { + // Get the PR details to find base branch + const { data: prData } = await github.rest.pulls.get({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: prNumber + }); + const baseBranch = prData.base.ref; + + // Get the diff stat to assess change scope + const { data: files } = await github.rest.pulls.listFiles({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: prNumber, + per_page: 100 + }); + + const totalChanges = files.reduce( + (sum, f) => sum + f.additions + f.deletions, 0 + ); + const fileCount = files.length; + const fileList = files + .map(f => `${f.filename} (+${f.additions}/-${f.deletions})`) + .join('\n'); + + // Only invoke claude if there's a meaningful diff + if (totalChanges === 0) { + console.log('No changes in PR. Skipping re-review assessment.'); + needsReReview = false; + } else { + // Build the assessment prompt + const assessPrompt = [ + 'You are assessing whether a PR needs re-review after changes were made to address review findings.', + '', + `PR #${prNumber} targets branch "${baseBranch}" and fixes issue #${parentIssueNumber}.`, + `Past review cycles: ${pastCycles}`, + '', + `The PR modifies ${fileCount} files with ${totalChanges} total line changes:`, + fileList, + '', + 'Based on the scope and nature of these changes, should reviewers re-review this PR?', + 'Consider: Are the changes small and surgical (e.g., null checks, single test additions)?', + 'Or are they broad and structural (e.g., new modules, architectural changes, many files)?', + '', + 'Respond with ONLY one word: YES or NO', + ].join('\n'); + + // Run claude -p for the assessment + const { execSync } = require('child_process'); + try { + const result = execSync( + `claude -p "${assessPrompt.replace(/"/g, '\\"')}"`, + { encoding: 'utf-8', timeout: 60000 } + ).trim(); + + console.log(`Claude assessment result: ${result}`); + needsReReview = /\bYES\b/i.test(result); + } catch (claudeErr) { + console.log(`Claude assessment failed: ${claudeErr.message}`); + // If Claude fails, default to not needing re-review + // to avoid blocking the pipeline + needsReReview = false; + } + } + } catch (err) { + console.log(`Error during re-review assessment: ${err.message}`); + needsReReview = false; + } + + // ── Step 5: Trigger re-review or report passing ───────────── + if (needsReReview) { + console.log('Re-review warranted. Triggering PR Review workflow...'); + + // Leave a marker comment for cycle counting + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: prNumber, + body: [ + '', + `**Orchestrator:** Triggering re-review (cycle ${pastCycles + 1}/${cycleCap}).`, + '', + 'Changes since last review warrant another review pass.' + ].join('\n') + }); + + // Trigger the PR Review workflow via workflow_dispatch + try { + await github.rest.actions.createWorkflowDispatch({ + owner: context.repo.owner, + repo: context.repo.repo, + workflow_id: 'pr-review.yml', + ref: context.ref || 'main', + inputs: { + 'pr-number': prNumber.toString() + } + }); + console.log('PR Review workflow triggered successfully.'); + } catch (dispatchErr) { + console.log(`Warning: Could not trigger PR Review workflow: ${dispatchErr.message}`); + } + + await createCheckRun( + headSha, + 'neutral', + `Orchestrator: re-review triggered (cycle ${pastCycles + 1}/${cycleCap})`, + [ + `All blocking sub-issues for #${parentIssueNumber} are resolved.`, + '', + `Changes since last review warrant re-review. Cycle ${pastCycles + 1} of ${cycleCap} triggered.`, + 'The PR Review workflow has been dispatched. Orchestrator will re-evaluate after review completes.' + ].join('\n') + ); + } else { + console.log('No re-review needed. Reporting success.'); + + await createCheckRun( + headSha, + 'success', + 'Orchestrator: all clear', + [ + `All blocking sub-issues for #${parentIssueNumber} are resolved.`, + '', + pastCycles > 0 + ? `No further re-review needed after ${pastCycles} review cycle(s).` + : 'No re-review warranted based on change assessment.', + '', + 'PR is clear to merge.' + ].join('\n') + ); + } From b6ecc772d9ae2693fed0e8f48683b5d9f4ca45fc Mon Sep 17 00:00:00 2001 From: Joe Delfino Date: Thu, 12 Feb 2026 14:01:26 +0000 Subject: [PATCH 06/23] test: add workflow validation tests 163 tests covering all workflow files: - Python tests for test-ratio, human-review, and pr-review workflows - Shell tests for scope and commit-message guardrails - YAML syntax, structural, and logic validation Co-Authored-By: Claude Opus 4.6 --- .gitignore | 1 + tests/test-guardrail-commits.sh | 272 ++++++++++++ tests/test-guardrail-scope.sh | 256 +++++++++++ tests/test_guardrail_test_ratio.py | 377 ++++++++++++++++ tests/test_human_review_workflow.py | 234 ++++++++++ tests/test_pr_review_workflow.py | 655 ++++++++++++++++++++++++++++ 6 files changed, 1795 insertions(+) create mode 100644 .gitignore create mode 100755 tests/test-guardrail-commits.sh create mode 100755 tests/test-guardrail-scope.sh create mode 100644 tests/test_guardrail_test_ratio.py create mode 100644 tests/test_human_review_workflow.py create mode 100644 tests/test_pr_review_workflow.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..fb74ddf --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +tests/__pycache__/ diff --git a/tests/test-guardrail-commits.sh b/tests/test-guardrail-commits.sh new file mode 100755 index 0000000..e5f3e23 --- /dev/null +++ b/tests/test-guardrail-commits.sh @@ -0,0 +1,272 @@ +#!/usr/bin/env bash +# Test suite for guardrail-commits.yml +# Validates YAML syntax, required workflow structure, and key logic elements. + +set -euo pipefail + +WORKFLOW_FILE="/workspaces/agent-workflow-feat-3-github-actions/.github/workflows/guardrail-commits.yml" +FAILURES=0 +PASSES=0 + +fail() { + echo "FAIL: $1" + FAILURES=$((FAILURES + 1)) +} + +pass() { + echo "PASS: $1" + PASSES=$((PASSES + 1)) +} + +# Test 1: File exists +if [ -f "$WORKFLOW_FILE" ]; then + pass "Workflow file exists" +else + fail "Workflow file does not exist at $WORKFLOW_FILE" + echo "" + echo "Results: $PASSES passed, $FAILURES failed" + exit 1 +fi + +# Test 2: Valid YAML syntax +if python3 -c "import yaml; yaml.safe_load(open('$WORKFLOW_FILE'))" 2>/dev/null; then + pass "Valid YAML syntax" +else + fail "Invalid YAML syntax" +fi + +# Test 3: Has required trigger events (pull_request opened and synchronize) +if python3 -c " +import yaml +with open('$WORKFLOW_FILE') as f: + wf = yaml.safe_load(f) +triggers = wf.get('on') or wf.get(True) +assert 'pull_request' in triggers, 'Missing pull_request trigger' +pr = triggers['pull_request'] +types = pr.get('types', []) +assert 'opened' in types, 'Missing opened type' +assert 'synchronize' in types, 'Missing synchronize type' +" 2>/dev/null; then + pass "Has pull_request trigger with opened and synchronize types" +else + fail "Missing pull_request trigger with opened and synchronize types" +fi + +# Test 4: Workflow has a name +if python3 -c " +import yaml +with open('$WORKFLOW_FILE') as f: + wf = yaml.safe_load(f) +assert 'name' in wf, 'Missing name' +assert wf['name'], 'Name is empty' +" 2>/dev/null; then + pass "Workflow has a name" +else + fail "Workflow has no name" +fi + +# Test 5: Uses actions/github-script@v7 +if grep -q 'actions/github-script@v7' "$WORKFLOW_FILE" 2>/dev/null; then + pass "Uses actions/github-script@v7" +else + fail "Does not use actions/github-script@v7" +fi + +# Test 6: Uses actions/checkout (needed for config reading) +if grep -q 'actions/checkout' "$WORKFLOW_FILE" 2>/dev/null; then + pass "Uses actions/checkout" +else + fail "Does not use actions/checkout (needed for config reading)" +fi + +# Test 7: References Check Run API (checks.create) +if grep -q 'checks.create' "$WORKFLOW_FILE" 2>/dev/null; then + pass "References checks.create API" +else + fail "Does not reference checks.create API" +fi + +# Test 8: Handles PR approval override (listReviews) +if grep -q 'listReviews' "$WORKFLOW_FILE" 2>/dev/null; then + pass "References listReviews for approval override" +else + fail "Does not reference listReviews for approval override" +fi + +# Test 9: References pulls.listCommits for fetching PR commits +if grep -q 'listCommits' "$WORKFLOW_FILE" 2>/dev/null; then + pass "References listCommits for PR commit fetching" +else + fail "Does not reference listCommits for PR commit fetching" +fi + +# Test 10: Contains conventional commit regex pattern +if grep -q 'feat\|fix\|chore\|docs\|test\|refactor' "$WORKFLOW_FILE" 2>/dev/null && \ + grep -qE '\^?\(feat\|fix|feat\|fix' "$WORKFLOW_FILE" 2>/dev/null; then + pass "Contains conventional commit type prefixes in regex" +else + fail "Does not contain conventional commit type prefixes in regex" +fi + +# Test 11: Checks first line length (72 chars) +if grep -q '72' "$WORKFLOW_FILE" 2>/dev/null; then + pass "References 72 char max length for first line" +else + fail "Does not reference 72 char max length" +fi + +# Test 12: Has permissions set for checks write +if grep -q 'checks:\s*write' "$WORKFLOW_FILE" 2>/dev/null; then + pass "Has checks: write permission" +else + fail "Does not have checks: write permission" +fi + +# Test 13: Has pull-requests read permission (needed for reviews and commits) +if grep -q 'pull-requests:\s*read' "$WORKFLOW_FILE" 2>/dev/null; then + pass "Has pull-requests: read permission" +else + fail "Does not have pull-requests: read permission" +fi + +# Test 14: Reads config from .github/agent-workflow/ +if grep -q 'agent-workflow' "$WORKFLOW_FILE" 2>/dev/null; then + pass "References agent-workflow config path" +else + fail "Does not reference agent-workflow config path" +fi + +# Test 15: Handles different check conclusions (success, neutral, action_required) +HAS_SUCCESS=$(grep -c "'success'" "$WORKFLOW_FILE" 2>/dev/null || echo 0) +HAS_NEUTRAL=$(grep -c "'neutral'" "$WORKFLOW_FILE" 2>/dev/null || echo 0) +HAS_ACTION=$(grep -c "'action_required'" "$WORKFLOW_FILE" 2>/dev/null || echo 0) +if [ "$HAS_SUCCESS" -gt 0 ] && [ "$HAS_NEUTRAL" -gt 0 ] && [ "$HAS_ACTION" -gt 0 ]; then + pass "Has all three check conclusions (success, neutral, action_required)" +else + fail "Missing check conclusions (success=$HAS_SUCCESS, neutral=$HAS_NEUTRAL, action_required=$HAS_ACTION)" +fi + +# Test 16: Check run name matches expected convention +if grep -q 'guardrail/commit-messages' "$WORKFLOW_FILE" 2>/dev/null; then + pass "Uses guardrail/commit-messages check run naming" +else + fail "Does not use guardrail/commit-messages check run naming convention" +fi + +# Test 17: Has proper job structure with runs-on +if python3 -c " +import yaml +with open('$WORKFLOW_FILE') as f: + wf = yaml.safe_load(f) +jobs = wf.get('jobs', {}) +assert len(jobs) > 0, 'No jobs defined' +for job_name, job in jobs.items(): + assert 'runs-on' in job, f'Job {job_name} missing runs-on' + assert 'steps' in job, f'Job {job_name} missing steps' +" 2>/dev/null; then + pass "Has proper job structure with runs-on and steps" +else + fail "Missing proper job structure" +fi + +# Test 18: Lists non-conforming commits in summary +if grep -qi 'summary\|non.conforming\|violation' "$WORKFLOW_FILE" 2>/dev/null; then + pass "References summary reporting for non-conforming commits" +else + fail "Does not reference summary reporting for non-conforming commits" +fi + +# Test 19: Default conclusion is neutral (non-blocking per design) +if grep -qi "default.*neutral\|neutral.*default" "$WORKFLOW_FILE" 2>/dev/null || \ + python3 -c " +import yaml +with open('$WORKFLOW_FILE') as f: + content = f.read() +assert 'neutral' in content.lower(), 'Missing neutral default' +# Check that neutral is mentioned as default somewhere in a comment or variable +assert any(kw in content.lower() for kw in ['default', 'configuredconclusion', 'configured_conclusion', 'conclusion']), 'Missing default conclusion logic' +" 2>/dev/null; then + pass "Uses neutral as default conclusion" +else + fail "Does not use neutral as default conclusion" +fi + +# Test 20: Checks for non-stale approval (compares commit_id) +if grep -q 'commit_id\|head_sha\|APPROVED' "$WORKFLOW_FILE" 2>/dev/null; then + pass "Checks for non-stale approval via commit_id comparison" +else + fail "Does not check for non-stale approval" +fi + +# Test 21: Contains all required conventional commit types +if python3 -c " +with open('$WORKFLOW_FILE') as f: + content = f.read() +required_types = ['feat', 'fix', 'chore', 'docs', 'test', 'refactor', 'ci', 'style', 'perf', 'build', 'revert'] +for t in required_types: + assert t in content, f'Missing conventional commit type: {t}' +" 2>/dev/null; then + pass "Contains all required conventional commit types" +else + fail "Missing one or more conventional commit types" +fi + +# Test 22: Has contents read permission (needed for checkout/config) +if grep -q 'contents:\s*read' "$WORKFLOW_FILE" 2>/dev/null; then + pass "Has contents: read permission" +else + fail "Does not have contents: read permission" +fi + +# Test 23: Config parsing handles nested guardrails structure +if grep -q 'guardrails' "$WORKFLOW_FILE" 2>/dev/null && \ + grep -q 'commit-messages' "$WORKFLOW_FILE" 2>/dev/null; then + pass "Config parsing references guardrails.commit-messages nested structure" +else + fail "Config parsing does not handle nested guardrails structure" +fi + +# Test 24: Handles config disabled case (enabled: false) +if grep -q "enabled.*false\|'false'" "$WORKFLOW_FILE" 2>/dev/null; then + pass "Handles config disabled case (enabled: false)" +else + fail "Does not handle config disabled case" +fi + +# Test 25: Non-stale approval compares commit_id against head SHA +if grep -q 'commit_id' "$WORKFLOW_FILE" 2>/dev/null && \ + grep -q 'head.sha\|headSha' "$WORKFLOW_FILE" 2>/dev/null; then + pass "Non-stale approval compares commit_id against head SHA" +else + fail "Does not compare commit_id against head SHA for non-stale approval" +fi + +# Test 26: Paginates commits with per_page +if grep -q 'per_page' "$WORKFLOW_FILE" 2>/dev/null; then + pass "Paginates commits with per_page parameter" +else + fail "Does not paginate commits with per_page" +fi + +# Test 27: Splits commit message on newline to get first line +if grep -q "split.*\\\\n\|split('\\\n')\|firstLine" "$WORKFLOW_FILE" 2>/dev/null; then + pass "Extracts first line from commit message" +else + fail "Does not extract first line from commit message" +fi + +# Test 28: Reports violation count in check run title +if grep -qi 'nonConformingCount\|non.conforming.*commit' "$WORKFLOW_FILE" 2>/dev/null; then + pass "Reports non-conforming commit count in check run output" +else + fail "Does not report non-conforming commit count" +fi + +echo "" +echo "=============================" +echo "Results: $PASSES passed, $FAILURES failed" +echo "=============================" + +if [ "$FAILURES" -gt 0 ]; then + exit 1 +fi diff --git a/tests/test-guardrail-scope.sh b/tests/test-guardrail-scope.sh new file mode 100755 index 0000000..135d576 --- /dev/null +++ b/tests/test-guardrail-scope.sh @@ -0,0 +1,256 @@ +#!/usr/bin/env bash +# Test suite for guardrail-scope.yml +# Validates YAML syntax, required workflow structure, and key logic elements. + +set -euo pipefail + +WORKFLOW_FILE="/workspaces/agent-workflow-feat-3-github-actions/.github/workflows/guardrail-scope.yml" +FAILURES=0 +PASSES=0 + +fail() { + echo "FAIL: $1" + FAILURES=$((FAILURES + 1)) +} + +pass() { + echo "PASS: $1" + PASSES=$((PASSES + 1)) +} + +# Test 1: File exists +if [ -f "$WORKFLOW_FILE" ]; then + pass "Workflow file exists" +else + fail "Workflow file does not exist at $WORKFLOW_FILE" + echo "" + echo "Results: $PASSES passed, $FAILURES failed" + exit 1 +fi + +# Test 2: Valid YAML syntax +if python3 -c "import yaml; yaml.safe_load(open('$WORKFLOW_FILE'))" 2>/dev/null; then + pass "Valid YAML syntax" +else + fail "Invalid YAML syntax" +fi + +# Test 3: Has required trigger events (pull_request opened and synchronize) +if python3 -c " +import yaml +with open('$WORKFLOW_FILE') as f: + wf = yaml.safe_load(f) +assert 'on' in wf or True in wf, 'Missing on: trigger' +triggers = wf.get('on') or wf.get(True) +assert 'pull_request' in triggers, 'Missing pull_request trigger' +pr = triggers['pull_request'] +types = pr.get('types', []) +assert 'opened' in types, 'Missing opened type' +assert 'synchronize' in types, 'Missing synchronize type' +" 2>/dev/null; then + pass "Has pull_request trigger with opened and synchronize types" +else + fail "Missing pull_request trigger with opened and synchronize types" +fi + +# Test 4: Workflow has a name +if python3 -c " +import yaml +with open('$WORKFLOW_FILE') as f: + wf = yaml.safe_load(f) +assert 'name' in wf, 'Missing name' +assert wf['name'], 'Name is empty' +" 2>/dev/null; then + pass "Workflow has a name" +else + fail "Workflow has no name" +fi + +# Test 5: Uses actions/github-script@v7 +if grep -q 'actions/github-script@v7' "$WORKFLOW_FILE" 2>/dev/null; then + pass "Uses actions/github-script@v7" +else + fail "Does not use actions/github-script@v7" +fi + +# Test 6: Uses actions/checkout +if grep -q 'actions/checkout' "$WORKFLOW_FILE" 2>/dev/null; then + pass "Uses actions/checkout" +else + fail "Does not use actions/checkout (needed for config reading)" +fi + +# Test 7: References Check Run API (checks.create) +if grep -q 'checks.create' "$WORKFLOW_FILE" 2>/dev/null; then + pass "References checks.create API" +else + fail "Does not reference checks.create API" +fi + +# Test 8: Handles PR approval override (listReviews) +if grep -q 'listReviews' "$WORKFLOW_FILE" 2>/dev/null; then + pass "References listReviews for approval override" +else + fail "Does not reference listReviews for approval override" +fi + +# Test 9: Parses issue references (fixes #N pattern) +if grep -qi 'fixes\s*#' "$WORKFLOW_FILE" 2>/dev/null || grep -qi 'fixes.*#' "$WORKFLOW_FILE" 2>/dev/null; then + pass "References 'fixes #N' pattern parsing" +else + fail "Does not reference 'fixes #N' pattern parsing" +fi + +# Test 10: Compares changed files against issue scope +if grep -q 'listFiles\|changed_files\|files changed\|changedFiles' "$WORKFLOW_FILE" 2>/dev/null; then + pass "References PR file listing" +else + fail "Does not reference PR file listing" +fi + +# Test 11: Has permissions set for checks write +if grep -q 'checks:\s*write' "$WORKFLOW_FILE" 2>/dev/null; then + pass "Has checks: write permission" +else + fail "Does not have checks: write permission" +fi + +# Test 12: Reports annotations on out-of-scope files +if grep -q 'annotation' "$WORKFLOW_FILE" 2>/dev/null; then + pass "References annotations for out-of-scope files" +else + fail "Does not reference annotations" +fi + +# Test 13: Reads config from .github/agent-workflow/ +if grep -q 'agent-workflow' "$WORKFLOW_FILE" 2>/dev/null; then + pass "References agent-workflow config path" +else + fail "Does not reference agent-workflow config path" +fi + +# Test 14: Handles different check conclusions (success, neutral, action_required) +HAS_SUCCESS=$(grep -c "'success'" "$WORKFLOW_FILE" 2>/dev/null || echo 0) +HAS_NEUTRAL=$(grep -c "'neutral'" "$WORKFLOW_FILE" 2>/dev/null || echo 0) +HAS_ACTION=$(grep -c "'action_required'" "$WORKFLOW_FILE" 2>/dev/null || echo 0) +if [ "$HAS_SUCCESS" -gt 0 ] && [ "$HAS_NEUTRAL" -gt 0 ] && [ "$HAS_ACTION" -gt 0 ]; then + pass "Has all three check conclusions (success, neutral, action_required)" +else + fail "Missing check conclusions (success=$HAS_SUCCESS, neutral=$HAS_NEUTRAL, action_required=$HAS_ACTION)" +fi + +# Test 15: Check run name matches expected convention +if grep -q 'guardrail/scope' "$WORKFLOW_FILE" 2>/dev/null; then + pass "Uses guardrail/scope check run naming" +else + fail "Does not use guardrail/scope check run naming convention" +fi + +# Test 16: Has proper job structure with runs-on +if python3 -c " +import yaml +with open('$WORKFLOW_FILE') as f: + wf = yaml.safe_load(f) +jobs = wf.get('jobs', {}) +assert len(jobs) > 0, 'No jobs defined' +for job_name, job in jobs.items(): + assert 'runs-on' in job, f'Job {job_name} missing runs-on' + assert 'steps' in job, f'Job {job_name} missing steps' +" 2>/dev/null; then + pass "Has proper job structure with runs-on and steps" +else + fail "Missing proper job structure" +fi + +# Test 17: Has issues read permission (needed to read issue body) +if grep -q 'issues:\s*read' "$WORKFLOW_FILE" 2>/dev/null; then + pass "Has issues: read permission" +else + fail "Does not have issues: read permission" +fi + +# Test 18: Has pull-requests read permission (needed for reviews) +if grep -q 'pull-requests:\s*read' "$WORKFLOW_FILE" 2>/dev/null; then + pass "Has pull-requests: read permission" +else + fail "Does not have pull-requests: read permission" +fi + +# Test 19: Handles config disabled case +if grep -q 'config.enabled' "$WORKFLOW_FILE" 2>/dev/null && grep -q 'disabled' "$WORKFLOW_FILE" 2>/dev/null; then + pass "Handles config disabled case" +else + fail "Does not handle config disabled case" +fi + +# Test 20: Handles missing PR body (null/empty) +if grep -q "context.payload.pull_request.body || ''" "$WORKFLOW_FILE" 2>/dev/null; then + pass "Handles null/empty PR body" +else + fail "Does not handle null/empty PR body" +fi + +# Test 21: Handles issue read failure gracefully +if grep -q 'issue not found' "$WORKFLOW_FILE" 2>/dev/null; then + pass "Handles issue read failure gracefully" +else + fail "Does not handle issue read failure" +fi + +# Test 22: Handles no files in issue body gracefully +if grep -q 'no files listed in issue' "$WORKFLOW_FILE" 2>/dev/null; then + pass "Handles no file paths in issue body" +else + fail "Does not handle missing file paths in issue" +fi + +# Test 23: Paginates changed files (per_page: 100) +if grep -q 'per_page: 100' "$WORKFLOW_FILE" 2>/dev/null; then + pass "Paginates changed files with per_page 100" +else + fail "Does not paginate changed files" +fi + +# Test 24: Non-stale approval uses commit_id check +if grep -q 'commit_id.*headSha\|r.commit_id === headSha' "$WORKFLOW_FILE" 2>/dev/null; then + pass "Non-stale approval checks commit_id against head SHA" +else + fail "Non-stale approval does not check commit_id" +fi + +# Test 25: Check run status is set to 'completed' +if grep -q "status: 'completed'" "$WORKFLOW_FILE" 2>/dev/null; then + pass "Check run status set to completed" +else + fail "Check run status not set to completed" +fi + +# Test 26: Limits annotations to 50 (GitHub API limit) +if grep -q 'slice(0, 50)' "$WORKFLOW_FILE" 2>/dev/null; then + pass "Limits annotations to 50 per API limit" +else + fail "Does not limit annotations to 50" +fi + +# Test 27: Supports backtick-wrapped file paths in issue body +if grep -q 'backtick\|Backtick\|`(' "$WORKFLOW_FILE" 2>/dev/null; then + pass "Supports backtick-wrapped file path extraction" +else + fail "Does not support backtick-wrapped file paths" +fi + +# Test 28: Minor violations (1-2 files) report neutral instead of action_required +if grep -q 'isMinor' "$WORKFLOW_FILE" 2>/dev/null; then + pass "Distinguishes minor from significant violations" +else + fail "Does not distinguish minor from significant violations" +fi + +echo "" +echo "=============================" +echo "Results: $PASSES passed, $FAILURES failed" +echo "=============================" + +if [ "$FAILURES" -gt 0 ]; then + exit 1 +fi diff --git a/tests/test_guardrail_test_ratio.py b/tests/test_guardrail_test_ratio.py new file mode 100644 index 0000000..468c3cb --- /dev/null +++ b/tests/test_guardrail_test_ratio.py @@ -0,0 +1,377 @@ +"""Tests for the guardrail-test-ratio.yml GitHub Actions workflow. + +Validates YAML syntax, workflow structure, trigger configuration, +and the presence of required logic steps. +""" + +import yaml +import os +import re + +WORKFLOW_PATH = os.path.join( + os.path.dirname(__file__), + "..", + ".github", + "workflows", + "guardrail-test-ratio.yml", +) + +CONFIG_PATH = os.path.join( + os.path.dirname(__file__), + "..", + ".github", + "agent-workflow", + "config.yaml", +) + + +def load_workflow(): + """Load and parse the workflow YAML file.""" + with open(WORKFLOW_PATH) as f: + return yaml.safe_load(f) + + +def load_config(): + """Load and parse the config YAML file.""" + with open(CONFIG_PATH) as f: + return yaml.safe_load(f) + + +def get_script_content(step): + """Extract the script content from a github-script step.""" + if step.get("uses", "").startswith("actions/github-script"): + return step.get("with", {}).get("script", "") + return "" + + +def find_step_by_id(steps, step_id): + """Find a step by its id.""" + for step in steps: + if step.get("id") == step_id: + return step + return None + + +def find_steps_by_uses(steps, uses_prefix): + """Find all steps that use a given action prefix.""" + return [s for s in steps if s.get("uses", "").startswith(uses_prefix)] + + +class TestWorkflowYamlSyntax: + """Test that the workflow file is valid YAML.""" + + def test_file_exists(self): + assert os.path.exists(WORKFLOW_PATH), ( + f"Workflow file not found at {WORKFLOW_PATH}" + ) + + def test_valid_yaml(self): + wf = load_workflow() + assert wf is not None, "Workflow YAML parsed as None (empty file)" + + def test_is_dict(self): + wf = load_workflow() + assert isinstance(wf, dict), "Workflow YAML root should be a mapping" + + +class TestWorkflowTriggers: + """Test that the workflow triggers on the correct events.""" + + def test_has_on_key(self): + wf = load_workflow() + assert True in wf or "on" in wf, "Workflow must have 'on' trigger" + + def test_triggers_on_pull_request(self): + wf = load_workflow() + on = wf.get(True) or wf.get("on") + assert "pull_request" in on, ( + "Workflow must trigger on pull_request" + ) + + def test_pull_request_types_include_opened(self): + wf = load_workflow() + on = wf.get(True) or wf.get("on") + pr = on["pull_request"] + types = pr.get("types", []) + assert "opened" in types, ( + "pull_request trigger must include 'opened' type" + ) + + def test_pull_request_types_include_synchronize(self): + wf = load_workflow() + on = wf.get(True) or wf.get("on") + pr = on["pull_request"] + types = pr.get("types", []) + assert "synchronize" in types, ( + "pull_request trigger must include 'synchronize' type" + ) + + +class TestWorkflowStructure: + """Test the overall workflow structure.""" + + def test_has_name(self): + wf = load_workflow() + assert "name" in wf, "Workflow must have a name" + + def test_name_mentions_test_ratio(self): + wf = load_workflow() + name = wf["name"].lower() + assert "test" in name and "ratio" in name, ( + "Workflow name should mention test ratio" + ) + + def test_has_jobs(self): + wf = load_workflow() + assert "jobs" in wf, "Workflow must have jobs" + + def test_has_check_job(self): + wf = load_workflow() + jobs = wf["jobs"] + assert len(jobs) >= 1, "Workflow must have at least one job" + + def test_job_runs_on_ubuntu(self): + wf = load_workflow() + jobs = wf["jobs"] + job = list(jobs.values())[0] + runs_on = job.get("runs-on", "") + assert "ubuntu" in runs_on, "Job must run on ubuntu" + + def test_has_permissions(self): + """Workflow needs checks:write and pull-requests:read permissions.""" + wf = load_workflow() + # Permissions can be at workflow level or job level + jobs = wf["jobs"] + job = list(jobs.values())[0] + perms = wf.get("permissions", {}) or job.get("permissions", {}) + assert "checks" in perms, "Must have checks permission" + assert perms["checks"] == "write", "Must have checks:write permission" + assert "pull-requests" in perms, "Must have pull-requests permission" + assert perms["pull-requests"] == "read", ( + "Must have pull-requests:read permission" + ) + + +class TestWorkflowSteps: + """Test that the workflow has the required steps.""" + + def _get_steps(self): + wf = load_workflow() + jobs = wf["jobs"] + job = list(jobs.values())[0] + return job.get("steps", []) + + def test_has_checkout_step(self): + steps = self._get_steps() + checkout_steps = find_steps_by_uses(steps, "actions/checkout") + assert len(checkout_steps) >= 1, "Must have a checkout step" + + def test_has_github_script_step(self): + steps = self._get_steps() + script_steps = find_steps_by_uses(steps, "actions/github-script") + assert len(script_steps) >= 1, ( + "Must have at least one github-script step" + ) + + +class TestConfigYaml: + """Test that the config.yaml file contains the right defaults.""" + + def test_config_file_exists(self): + assert os.path.exists(CONFIG_PATH), ( + f"Config file not found at {CONFIG_PATH}" + ) + + def test_config_valid_yaml(self): + config = load_config() + assert config is not None, "Config YAML parsed as None" + + def test_has_guardrails_section(self): + config = load_config() + assert "guardrails" in config, ( + "Config must have 'guardrails' section" + ) + + def test_has_test_ratio_section(self): + config = load_config() + guardrails = config["guardrails"] + assert "test-ratio" in guardrails, ( + "Guardrails config must have 'test-ratio' section" + ) + + def test_has_threshold(self): + config = load_config() + tr = config["guardrails"]["test-ratio"] + assert "threshold" in tr, ( + "test-ratio config must have 'threshold'" + ) + + def test_default_threshold_is_0_5(self): + config = load_config() + tr = config["guardrails"]["test-ratio"] + assert tr["threshold"] == 0.5, ( + "Default threshold should be 0.5" + ) + + def test_has_enabled(self): + config = load_config() + tr = config["guardrails"]["test-ratio"] + assert "enabled" in tr, "test-ratio config must have 'enabled'" + assert tr["enabled"] is True, "test-ratio should be enabled by default" + + def test_has_conclusion(self): + config = load_config() + tr = config["guardrails"]["test-ratio"] + assert "conclusion" in tr, ( + "test-ratio config must have 'conclusion'" + ) + assert tr["conclusion"] == "action_required", ( + "Default conclusion should be 'action_required'" + ) + + +class TestScriptLogic: + """Test that the github-script step contains the required logic.""" + + def _get_all_script_content(self): + wf = load_workflow() + jobs = wf["jobs"] + job = list(jobs.values())[0] + steps = job.get("steps", []) + scripts = [] + for step in steps: + content = get_script_content(step) + if content: + scripts.append(content) + return "\n".join(scripts) + + def test_reads_config_yaml(self): + """Script must read config.yaml for threshold.""" + script = self._get_all_script_content() + assert "config" in script.lower(), ( + "Script must reference config for threshold" + ) + + def test_uses_pulls_list_files(self): + """Script must use pulls.listFiles to get PR files.""" + script = self._get_all_script_content() + assert "listFiles" in script, ( + "Script must use pulls.listFiles to get PR diff" + ) + + def test_categorizes_test_files(self): + """Script must identify test files by naming convention.""" + script = self._get_all_script_content() + # Should check for test/spec patterns + has_test_pattern = "test" in script.lower() + has_spec_pattern = "spec" in script.lower() + has_tests_dir = "__tests__" in script + assert has_test_pattern or has_spec_pattern or has_tests_dir, ( + "Script must categorize test files by naming convention" + ) + + def test_counts_added_lines(self): + """Script must count added lines from the diff.""" + script = self._get_all_script_content() + # Should reference additions or patch parsing + has_additions = "additions" in script + has_patch = "patch" in script + assert has_additions or has_patch, ( + "Script must count added lines from diff" + ) + + def test_calculates_ratio(self): + """Script must calculate and compare ratio against threshold.""" + script = self._get_all_script_content() + assert "ratio" in script.lower() or "threshold" in script.lower(), ( + "Script must calculate ratio and compare against threshold" + ) + + def test_creates_check_run(self): + """Script must create a check run via the Checks API.""" + script = self._get_all_script_content() + assert "checks.create" in script, ( + "Script must use checks.create to report results" + ) + + def test_checks_for_approval_override(self): + """Script must check for non-stale PR approval override.""" + script = self._get_all_script_content() + assert "listReviews" in script or "APPROVED" in script, ( + "Script must check for non-stale approval override" + ) + + def test_reports_success_on_approval(self): + """Script must report success when a non-stale approval exists.""" + script = self._get_all_script_content() + assert "success" in script, ( + "Script must be able to report 'success' conclusion" + ) + + def test_reports_action_required_on_failure(self): + """Script must report action_required when ratio is below threshold.""" + script = self._get_all_script_content() + assert "action_required" in script, ( + "Script must be able to report 'action_required' conclusion" + ) + + def test_check_run_name_includes_guardrail(self): + """Check run name should identify it as a guardrail.""" + script = self._get_all_script_content() + assert "guardrail" in script.lower(), ( + "Check run name should include 'guardrail'" + ) + + def test_includes_annotations(self): + """Script should include annotations for findings.""" + script = self._get_all_script_content() + assert "annotation" in script.lower(), ( + "Script should include annotations in check run output" + ) + + def test_handles_no_implementation_lines(self): + """Script should handle the case where there are zero implementation lines.""" + script = self._get_all_script_content() + # Should have some guard against division by zero or no impl lines + # Check for explicit handling of zero/no implementation lines + has_zero_check = ( + "=== 0" in script + or "== 0" in script + or "no implementation" in script.lower() + or "no code" in script.lower() + or "impl" in script.lower() + ) + assert has_zero_check, ( + "Script must handle the case of zero implementation lines" + ) + + def test_paginates_file_listing(self): + """Script should paginate through PR files for large PRs.""" + script = self._get_all_script_content() + assert "per_page" in script, ( + "Script must use per_page for paginated file listing" + ) + assert "page" in script, ( + "Script must handle pagination" + ) + + def test_skips_removed_files(self): + """Script should skip files that were removed in the PR.""" + script = self._get_all_script_content() + assert "removed" in script, ( + "Script must skip removed files" + ) + + def test_has_guardrail_check_run_name(self): + """Check run name should be 'guardrail/test-ratio'.""" + script = self._get_all_script_content() + assert "guardrail/test-ratio" in script, ( + "Check run name should be 'guardrail/test-ratio'" + ) + + def test_handles_disabled_config(self): + """Script should respect the enabled flag in config.""" + script = self._get_all_script_content() + assert "enabled" in script, ( + "Script must check if guardrail is enabled" + ) diff --git a/tests/test_human_review_workflow.py b/tests/test_human_review_workflow.py new file mode 100644 index 0000000..167026c --- /dev/null +++ b/tests/test_human_review_workflow.py @@ -0,0 +1,234 @@ +""" +Tests for .github/workflows/human-review.yml + +Validates the workflow structure, trigger configuration, permissions, +and the expected logic within the github-script action. +""" + +import yaml +import os +import re +import pytest + +WORKFLOW_PATH = os.path.join( + os.path.dirname(__file__), + "..", + ".github", + "workflows", + "human-review.yml", +) + + +@pytest.fixture +def workflow(): + """Load and parse the workflow YAML.""" + with open(WORKFLOW_PATH) as f: + return yaml.safe_load(f) + + +# ── Trigger ────────────────────────────────────────────────────────── + + +class TestTrigger: + def test_triggers_on_pull_request_review(self, workflow): + assert "on" in workflow, "Workflow must have an 'on' trigger" + on = workflow["on"] + assert "pull_request_review" in on, ( + "Must trigger on pull_request_review event" + ) + + def test_triggers_only_on_submitted(self, workflow): + pr_review = workflow["on"]["pull_request_review"] + assert "types" in pr_review, "Must specify types filter" + assert pr_review["types"] == ["submitted"], ( + "Must trigger only on 'submitted' type" + ) + + +# ── Permissions ────────────────────────────────────────────────────── + + +class TestPermissions: + def test_has_issues_write(self, workflow): + perms = workflow.get("permissions", {}) + assert perms.get("issues") == "write", ( + "Needs issues:write to create child issues" + ) + + def test_has_pull_requests_write(self, workflow): + perms = workflow.get("permissions", {}) + assert perms.get("pull-requests") == "write", ( + "Needs pull-requests:write to update PR description" + ) + + def test_has_contents_read(self, workflow): + perms = workflow.get("permissions", {}) + assert perms.get("contents") == "read", ( + "Needs contents:read for checkout context" + ) + + +# ── Jobs ───────────────────────────────────────────────────────────── + + +class TestJobs: + def test_has_process_review_job(self, workflow): + assert "jobs" in workflow, "Workflow must define jobs" + assert "process-review" in workflow["jobs"], ( + "Must have a 'process-review' job" + ) + + def test_job_runs_on_ubuntu(self, workflow): + job = workflow["jobs"]["process-review"] + assert "ubuntu" in job["runs-on"], "Job must run on ubuntu" + + def test_job_has_steps(self, workflow): + job = workflow["jobs"]["process-review"] + assert "steps" in job, "Job must have steps" + assert len(job["steps"]) > 0, "Job must have at least one step" + + +# ── Script Content ─────────────────────────────────────────────────── + + +class TestScriptContent: + """Validate the github-script step contains the required logic.""" + + @pytest.fixture + def script_step(self, workflow): + """Find the github-script step.""" + steps = workflow["jobs"]["process-review"]["steps"] + for step in steps: + if step.get("uses", "").startswith("actions/github-script"): + return step + pytest.fail("No actions/github-script step found") + + @pytest.fixture + def script_text(self, script_step): + """Extract the script text from the github-script step.""" + return script_step.get("with", {}).get("script", "") + + def test_uses_github_script_v7(self, script_step): + assert script_step["uses"] == "actions/github-script@v7" + + def test_parses_fixes_reference(self, script_text): + assert re.search(r"[Ff]ixes\s*#", script_text), ( + "Script must parse 'Fixes #N' from PR body" + ) + + def test_fetches_review_comments(self, script_text): + # Should call the reviews/comments endpoint + assert "reviews" in script_text and "comments" in script_text, ( + "Script must fetch review comments" + ) + + def test_creates_issues(self, script_text): + assert "issues.create" in script_text or "createIssue" in script_text, ( + "Script must create issues for review comments" + ) + + def test_severity_detection_blocking(self, script_text): + assert "blocking" in script_text.lower() or "block" in script_text.lower(), ( + "Script must detect blocking severity" + ) + + def test_severity_detection_suggestion(self, script_text): + assert "suggestion" in script_text.lower(), ( + "Script must detect suggestion severity" + ) + + def test_severity_detection_should_fix(self, script_text): + assert "should-fix" in script_text or "should_fix" in script_text, ( + "Script must handle should-fix severity" + ) + + def test_includes_file_path_context(self, script_text): + assert "path" in script_text, ( + "Script must include file path from comment location" + ) + + def test_includes_line_number_context(self, script_text): + # Should reference line from the comment + assert "line" in script_text, ( + "Script must include line number from comment location" + ) + + def test_graphql_sub_issue_linking(self, script_text): + assert "addSubIssue" in script_text, ( + "Script must use addSubIssue GraphQL mutation to link child issues" + ) + + def test_graphql_parent_node_id(self, script_text): + assert "node_id" in script_text or "nodeId" in script_text or "node id" in script_text.lower(), ( + "Script must get parent issue node ID for GraphQL" + ) + + def test_updates_pr_body_with_fixes(self, script_text): + # Should update the PR body/description to add Fixes references + assert "update" in script_text.lower() and ("body" in script_text or "description" in script_text), ( + "Script must update PR body with Fixes references for created issues" + ) + + def test_blocking_dependency_api(self, script_text): + # Should use the sub-issues dependency blocked-by API for blocking comments + assert "blocked" in script_text.lower() or "dependencies" in script_text.lower() or "blocking" in script_text.lower(), ( + "Script must set blocking dependencies for blocking comments" + ) + + def test_idempotent_pr_body_update(self, script_text): + """Should replace existing review section rather than duplicating it.""" + assert "human-review-issues-start" in script_text, ( + "Script must use HTML comment markers for idempotent PR body updates" + ) + assert "human-review-issues-end" in script_text, ( + "Script must use closing HTML comment marker" + ) + # Check for replacement logic (regex test or replace) + assert "replace" in script_text.lower() or "test" in script_text, ( + "Script must check for existing section before appending" + ) + + +# ── Early-exit guard ───────────────────────────────────────────────── + + +class TestEarlyExit: + """Verify the workflow handles edge cases gracefully.""" + + @pytest.fixture + def script_text(self, workflow): + steps = workflow["jobs"]["process-review"]["steps"] + for step in steps: + if step.get("uses", "").startswith("actions/github-script"): + return step.get("with", {}).get("script", "") + return "" + + def test_skips_when_no_fixes_reference(self, script_text): + """Should exit early if no 'Fixes #N' found in PR body.""" + assert "no parent issue" in script_text.lower() or "skip" in script_text.lower() or "return" in script_text, ( + "Script must handle case where PR has no Fixes reference" + ) + + def test_skips_when_no_comments(self, script_text): + """Should handle reviews with no line-level comments.""" + # The script should check if there are comments and handle empty case + assert "length" in script_text or "no comments" in script_text.lower() or ".length" in script_text, ( + "Script must handle case where review has no comments" + ) + + +# ── YAML validity ──────────────────────────────────────────────────── + + +class TestYamlValidity: + def test_yaml_parses_successfully(self): + """The workflow file must be valid YAML.""" + with open(WORKFLOW_PATH) as f: + data = yaml.safe_load(f) + assert data is not None + + def test_is_valid_github_actions_workflow(self, workflow): + """Must have the basic structure of a GitHub Actions workflow.""" + assert "name" in workflow, "Workflow must have a name" + assert "on" in workflow, "Workflow must have triggers" + assert "jobs" in workflow, "Workflow must have jobs" diff --git a/tests/test_pr_review_workflow.py b/tests/test_pr_review_workflow.py new file mode 100644 index 0000000..0022747 --- /dev/null +++ b/tests/test_pr_review_workflow.py @@ -0,0 +1,655 @@ +""" +Tests for .github/workflows/pr-review.yml + +Validates the workflow structure, trigger configuration, permissions, +parallel reviewer jobs, and the expected prompt construction for each +reviewer skill invoked via `claude -p`. +""" + +import yaml +import os +import re +import pytest + +WORKFLOW_PATH = os.path.join( + os.path.dirname(__file__), + "..", + ".github", + "workflows", + "pr-review.yml", +) + +CONFIG_PATH = os.path.join( + os.path.dirname(__file__), + "..", + ".github", + "agent-workflow", + "config.yaml", +) + + +def load_workflow(): + """Load and parse the workflow YAML file.""" + with open(WORKFLOW_PATH) as f: + return yaml.safe_load(f) + + +def load_config(): + """Load and parse the config YAML file.""" + with open(CONFIG_PATH) as f: + return yaml.safe_load(f) + + +def get_on(wf): + """Get the 'on' trigger, handling YAML parsing of bare 'on' as True.""" + return wf.get(True) or wf.get("on") + + +# ── YAML Validity ──────────────────────────────────────────────────── + + +class TestYamlValidity: + def test_file_exists(self): + assert os.path.exists(WORKFLOW_PATH), ( + f"Workflow file not found at {WORKFLOW_PATH}" + ) + + def test_valid_yaml(self): + wf = load_workflow() + assert wf is not None, "Workflow YAML parsed as None (empty file)" + + def test_is_dict(self): + wf = load_workflow() + assert isinstance(wf, dict), "Workflow YAML root should be a mapping" + + def test_is_valid_github_actions_workflow(self): + wf = load_workflow() + assert "name" in wf, "Workflow must have a name" + assert get_on(wf) is not None, "Workflow must have triggers" + assert "jobs" in wf, "Workflow must have jobs" + + +# ── Triggers ───────────────────────────────────────────────────────── + + +class TestTriggers: + def test_triggers_on_pull_request(self): + wf = load_workflow() + on = get_on(wf) + assert "pull_request" in on, ( + "Must trigger on pull_request event" + ) + + def test_pull_request_types_include_opened(self): + wf = load_workflow() + on = get_on(wf) + pr = on["pull_request"] + types = pr.get("types", []) + assert "opened" in types, ( + "pull_request trigger must include 'opened' type" + ) + + def test_pull_request_types_include_synchronize(self): + wf = load_workflow() + on = get_on(wf) + pr = on["pull_request"] + types = pr.get("types", []) + assert "synchronize" in types, ( + "pull_request trigger must include 'synchronize' type" + ) + + def test_has_workflow_dispatch_trigger(self): + """The orchestrator needs to re-trigger reviews via workflow_dispatch.""" + wf = load_workflow() + on = get_on(wf) + assert "workflow_dispatch" in on, ( + "Must have workflow_dispatch trigger so orchestrator can re-trigger reviews" + ) + + def test_workflow_dispatch_has_pr_number_input(self): + """workflow_dispatch must accept a PR number input.""" + wf = load_workflow() + on = get_on(wf) + wd = on["workflow_dispatch"] + assert "inputs" in wd, "workflow_dispatch must have inputs" + inputs = wd["inputs"] + # Should have a pr-number or pr_number input + has_pr_input = any( + "pr" in key.lower() for key in inputs.keys() + ) + assert has_pr_input, ( + "workflow_dispatch must have a PR number input" + ) + + +# ── Permissions ────────────────────────────────────────────────────── + + +class TestPermissions: + def _get_permissions(self): + wf = load_workflow() + return wf.get("permissions", {}) + + def test_has_contents_write(self): + perms = self._get_permissions() + assert perms.get("contents") == "write", ( + "Needs contents:write to check out repo" + ) + + def test_has_issues_write(self): + perms = self._get_permissions() + assert perms.get("issues") == "write", ( + "Needs issues:write to create child issues for findings" + ) + + def test_has_pull_requests_read(self): + perms = self._get_permissions() + assert perms.get("pull-requests") == "read", ( + "Needs pull-requests:read to read PR diff and description" + ) + + +# ── Jobs: Three Parallel Reviewers ─────────────────────────────────── + + +class TestReviewerJobs: + """The workflow must have three separate jobs for parallel reviewer execution.""" + + def _get_jobs(self): + wf = load_workflow() + return wf.get("jobs", {}) + + def test_has_at_least_three_jobs(self): + jobs = self._get_jobs() + assert len(jobs) >= 3, ( + f"Workflow must have at least 3 jobs (one per reviewer), found {len(jobs)}" + ) + + def test_has_correctness_reviewer_job(self): + jobs = self._get_jobs() + correctness_jobs = [ + k for k in jobs if "correctness" in k.lower() + ] + assert len(correctness_jobs) >= 1, ( + "Must have a job for the correctness reviewer" + ) + + def test_has_tests_reviewer_job(self): + jobs = self._get_jobs() + test_jobs = [ + k for k in jobs if "test" in k.lower() + ] + assert len(test_jobs) >= 1, ( + "Must have a job for the tests reviewer" + ) + + def test_has_architecture_reviewer_job(self): + jobs = self._get_jobs() + arch_jobs = [ + k for k in jobs if "architecture" in k.lower() + ] + assert len(arch_jobs) >= 1, ( + "Must have a job for the architecture reviewer" + ) + + def test_reviewer_jobs_are_independent(self): + """Reviewer jobs must not depend on each other (parallel execution).""" + jobs = self._get_jobs() + reviewer_keys = [ + k for k in jobs + if "correctness" in k.lower() + or "test" in k.lower() + or "architecture" in k.lower() + ] + for key in reviewer_keys: + needs = jobs[key].get("needs", []) + # needs should not reference other reviewer jobs + other_reviewers = [ + r for r in reviewer_keys if r != key + ] + for dep in (needs if isinstance(needs, list) else [needs]): + assert dep not in other_reviewers, ( + f"Reviewer job '{key}' depends on '{dep}' — reviewers must run in parallel" + ) + + def test_all_reviewer_jobs_run_on_ubuntu(self): + jobs = self._get_jobs() + reviewer_keys = [ + k for k in jobs + if "correctness" in k.lower() + or "test" in k.lower() + or "architecture" in k.lower() + ] + for key in reviewer_keys: + runs_on = jobs[key].get("runs-on", "") + assert "ubuntu" in runs_on, ( + f"Reviewer job '{key}' must run on ubuntu" + ) + + +# ── Parse Parent Issue ─────────────────────────────────────────────── + + +class TestParseParentIssue: + """The workflow must parse 'fixes #N' or 'Fixes #N' from the PR description.""" + + def _get_all_step_content(self): + """Collect all run/script content from all jobs.""" + wf = load_workflow() + content_parts = [] + for job_name, job in wf.get("jobs", {}).items(): + for step in job.get("steps", []): + # Collect 'run' scripts + if "run" in step: + content_parts.append(step["run"]) + # Collect github-script content + if step.get("uses", "").startswith("actions/github-script"): + script = step.get("with", {}).get("script", "") + content_parts.append(script) + # Collect env vars + env = step.get("env", {}) + for v in env.values(): + if isinstance(v, str): + content_parts.append(v) + # Also collect job-level env + env = job.get("env", {}) + for v in env.values(): + if isinstance(v, str): + content_parts.append(v) + return "\n".join(content_parts) + + def test_parses_fixes_reference(self): + """Must extract parent issue number from PR body 'fixes #N' or 'Fixes #N'.""" + content = self._get_all_step_content() + # Should contain a regex or string match for fixes #N (case-insensitive) + has_fixes_pattern = ( + re.search(r"[Ff]ixes\s*#", content) + or "fixes" in content.lower() + ) + assert has_fixes_pattern, ( + "Workflow must parse 'fixes #N' from PR description" + ) + + +# ── Each Reviewer Job Steps ────────────────────────────────────────── + + +class TestReviewerJobSteps: + """Each reviewer job must: checkout repo, then run claude -p with appropriate skill.""" + + def _get_reviewer_jobs(self): + wf = load_workflow() + jobs = wf.get("jobs", {}) + return { + k: v for k, v in jobs.items() + if "correctness" in k.lower() + or "test" in k.lower() + or "architecture" in k.lower() + } + + def _get_all_steps_content(self, job): + """Get all step run/script content for a job.""" + parts = [] + for step in job.get("steps", []): + if "run" in step: + parts.append(step["run"]) + if step.get("uses", "").startswith("actions/github-script"): + script = step.get("with", {}).get("script", "") + parts.append(script) + return "\n".join(parts) + + def test_each_reviewer_checks_out_repo(self): + """Each reviewer job must have a checkout step.""" + reviewer_jobs = self._get_reviewer_jobs() + for job_name, job in reviewer_jobs.items(): + steps = job.get("steps", []) + checkout = [ + s for s in steps + if s.get("uses", "").startswith("actions/checkout") + ] + assert len(checkout) >= 1, ( + f"Reviewer job '{job_name}' must have a checkout step" + ) + + def test_each_reviewer_runs_claude_p(self): + """Each reviewer job must run `claude -p` (or `claude --print`).""" + reviewer_jobs = self._get_reviewer_jobs() + for job_name, job in reviewer_jobs.items(): + content = self._get_all_steps_content(job) + has_claude = "claude" in content.lower() + assert has_claude, ( + f"Reviewer job '{job_name}' must invoke claude" + ) + + def test_correctness_reviewer_references_skill(self): + """Correctness reviewer must reference the correctness skill.""" + wf = load_workflow() + jobs = wf.get("jobs", {}) + correctness_jobs = { + k: v for k, v in jobs.items() + if "correctness" in k.lower() + } + for job_name, job in correctness_jobs.items(): + content = self._get_all_steps_content(job) + assert "reviewer-correctness" in content or "correctness" in content.lower(), ( + f"Correctness job '{job_name}' must reference the correctness reviewer skill" + ) + + def test_tests_reviewer_references_skill(self): + """Tests reviewer must reference the test reviewer skill.""" + wf = load_workflow() + jobs = wf.get("jobs", {}) + test_jobs = { + k: v for k, v in jobs.items() + if "test" in k.lower() + } + for job_name, job in test_jobs.items(): + content = self._get_all_steps_content(job) + assert "reviewer-tests" in content or "test" in content.lower(), ( + f"Tests job '{job_name}' must reference the test reviewer skill" + ) + + def test_architecture_reviewer_references_skill(self): + """Architecture reviewer must reference the architecture reviewer skill.""" + wf = load_workflow() + jobs = wf.get("jobs", {}) + arch_jobs = { + k: v for k, v in jobs.items() + if "architecture" in k.lower() + } + for job_name, job in arch_jobs.items(): + content = self._get_all_steps_content(job) + assert "reviewer-architecture" in content or "architecture" in content.lower(), ( + f"Architecture job '{job_name}' must reference the architecture reviewer skill" + ) + + +# ── Context Passing ────────────────────────────────────────────────── + + +class TestContextPassing: + """Each reviewer must receive PR number, parent issue number, and repo info.""" + + def _get_all_content(self): + """Get all run/script/env content from all jobs.""" + wf = load_workflow() + parts = [] + for job_name, job in wf.get("jobs", {}).items(): + # Job-level env + for v in job.get("env", {}).values(): + if isinstance(v, str): + parts.append(v) + for step in job.get("steps", []): + if "run" in step: + parts.append(step["run"]) + if step.get("uses", "").startswith("actions/github-script"): + parts.append(step.get("with", {}).get("script", "")) + for v in step.get("env", {}).values(): + if isinstance(v, str): + parts.append(v) + return "\n".join(parts) + + def test_passes_pr_number(self): + content = self._get_all_content() + # Should reference pull_request number or PR number + has_pr_num = ( + "pull_request" in content + or "pr_number" in content.lower() + or "pr-number" in content.lower() + or "PR_NUMBER" in content + ) + assert has_pr_num, ( + "Workflow must pass PR number to reviewer" + ) + + def test_passes_parent_issue_number(self): + content = self._get_all_content() + has_parent = ( + "parent" in content.lower() + or "issue" in content.lower() + or "PARENT_ISSUE" in content + ) + assert has_parent, ( + "Workflow must pass parent issue number to reviewer" + ) + + def test_passes_repo_info(self): + content = self._get_all_content() + has_repo = ( + "github.repository" in content + or "repo.owner" in content + or "GITHUB_REPOSITORY" in content + or "context.repo" in content + ) + assert has_repo, ( + "Workflow must pass repo owner/name to reviewer" + ) + + +# ── Reviewer Prompt Content ────────────────────────────────────────── + + +class TestReviewerPromptContent: + """The prompt passed to claude -p must instruct the reviewer correctly.""" + + def _get_all_content(self): + wf = load_workflow() + parts = [] + for job_name, job in wf.get("jobs", {}).items(): + for v in job.get("env", {}).values(): + if isinstance(v, str): + parts.append(v) + for step in job.get("steps", []): + if "run" in step: + parts.append(step["run"]) + if step.get("uses", "").startswith("actions/github-script"): + parts.append(step.get("with", {}).get("script", "")) + for v in step.get("env", {}).values(): + if isinstance(v, str): + parts.append(v) + return "\n".join(parts) + + def test_prompt_instructs_reading_pr_diff(self): + content = self._get_all_content() + assert "diff" in content.lower(), ( + "Reviewer prompt must instruct reading the PR diff" + ) + + def test_prompt_instructs_creating_issues(self): + content = self._get_all_content() + has_issue_create = ( + "gh issue create" in content + or "issue" in content.lower() + ) + assert has_issue_create, ( + "Reviewer prompt must instruct creating child issues for findings" + ) + + def test_prompt_includes_severity_labels(self): + content = self._get_all_content() + assert "blocking" in content, ( + "Prompt must mention 'blocking' severity label" + ) + assert "should-fix" in content, ( + "Prompt must mention 'should-fix' severity label" + ) + assert "suggestion" in content, ( + "Prompt must mention 'suggestion' severity label" + ) + + def test_prompt_instructs_sub_issue_linking(self): + content = self._get_all_content() + has_sub_issue = ( + "sub-issue" in content.lower() + or "sub_issue" in content.lower() + or "subissue" in content.lower() + or "addSubIssue" in content + or "child" in content.lower() + ) + assert has_sub_issue, ( + "Prompt must instruct linking findings as sub-issues of parent" + ) + + def test_prompt_instructs_blocking_dependency(self): + content = self._get_all_content() + has_blocking = ( + "blocking" in content.lower() + and ("dependency" in content.lower() + or "blocked_by" in content.lower() + or "blocked-by" in content.lower() + or "block" in content.lower()) + ) + assert has_blocking, ( + "Prompt must instruct setting blocking dependencies for blocking findings" + ) + + +# ── Secrets ────────────────────────────────────────────────────────── + + +class TestSecrets: + """Workflow must use ANTHROPIC_API_KEY secret for claude -p.""" + + def _get_all_content(self): + wf = load_workflow() + parts = [] + for job_name, job in wf.get("jobs", {}).items(): + for v in job.get("env", {}).values(): + if isinstance(v, str): + parts.append(v) + for step in job.get("steps", []): + if "run" in step: + parts.append(step["run"]) + for v in step.get("env", {}).values(): + if isinstance(v, str): + parts.append(v) + return "\n".join(parts) + + def test_references_anthropic_api_key(self): + content = self._get_all_content() + assert "ANTHROPIC_API_KEY" in content, ( + "Workflow must reference ANTHROPIC_API_KEY secret for claude -p" + ) + + +# ── Resolve Context Job ─────────────────────────────────────────────── + + +class TestResolveContextJob: + """The resolve-context job extracts PR metadata for reviewer jobs.""" + + def _get_context_job(self): + wf = load_workflow() + jobs = wf.get("jobs", {}) + ctx_jobs = { + k: v for k, v in jobs.items() + if "context" in k.lower() or "resolve" in k.lower() + } + assert len(ctx_jobs) >= 1, ( + "Must have a resolve-context job to extract PR metadata" + ) + return list(ctx_jobs.values())[0] + + def test_resolve_context_job_exists(self): + self._get_context_job() + + def test_resolve_context_has_outputs(self): + job = self._get_context_job() + outputs = job.get("outputs", {}) + assert len(outputs) >= 2, ( + "resolve-context job must export outputs (at least pr-number and parent-issue)" + ) + + def test_resolve_context_outputs_pr_number(self): + job = self._get_context_job() + outputs = job.get("outputs", {}) + has_pr = any("pr" in k.lower() for k in outputs.keys()) + assert has_pr, ( + "resolve-context must output a PR number" + ) + + def test_resolve_context_outputs_parent_issue(self): + job = self._get_context_job() + outputs = job.get("outputs", {}) + has_parent = any( + "parent" in k.lower() or "issue" in k.lower() + for k in outputs.keys() + ) + assert has_parent, ( + "resolve-context must output a parent issue number" + ) + + def test_reviewer_jobs_depend_on_context(self): + """All reviewer jobs must depend on the resolve-context job.""" + wf = load_workflow() + jobs = wf.get("jobs", {}) + # Find the context job key + ctx_key = None + for k in jobs: + if "context" in k.lower() or "resolve" in k.lower(): + ctx_key = k + break + assert ctx_key is not None + + reviewer_keys = [ + k for k in jobs + if "correctness" in k.lower() + or "test" in k.lower() + or "architecture" in k.lower() + ] + for key in reviewer_keys: + needs = jobs[key].get("needs", []) + if isinstance(needs, str): + needs = [needs] + assert ctx_key in needs, ( + f"Reviewer job '{key}' must depend on '{ctx_key}'" + ) + + def test_reviewer_jobs_skip_when_no_parent_issue(self): + """Reviewer jobs should have an 'if' condition to skip when no parent issue.""" + wf = load_workflow() + jobs = wf.get("jobs", {}) + reviewer_keys = [ + k for k in jobs + if "correctness" in k.lower() + or "test" in k.lower() + or "architecture" in k.lower() + ] + for key in reviewer_keys: + job_if = jobs[key].get("if", "") + assert "parent" in job_if.lower() or "issue" in job_if.lower(), ( + f"Reviewer job '{key}' must have an 'if' condition checking for parent issue" + ) + + def test_handles_workflow_dispatch(self): + """Resolve-context must handle both pull_request and workflow_dispatch events.""" + job = self._get_context_job() + steps_content = [] + for step in job.get("steps", []): + if "run" in step: + steps_content.append(step["run"]) + if step.get("uses", "").startswith("actions/github-script"): + steps_content.append(step.get("with", {}).get("script", "")) + content = "\n".join(steps_content) + assert "workflow_dispatch" in content, ( + "resolve-context must handle workflow_dispatch event" + ) + + +# ── Config Reading ─────────────────────────────────────────────────── + + +class TestConfigReading: + """Workflow should read config.yaml for settings like re-review-cycle-cap.""" + + def test_config_has_re_review_cycle_cap(self): + config = load_config() + assert "re-review-cycle-cap" in config, ( + "Config must have 're-review-cycle-cap' setting" + ) + + def test_re_review_cycle_cap_default(self): + config = load_config() + assert config["re-review-cycle-cap"] == 3, ( + "Default re-review-cycle-cap should be 3" + ) From 9b2ee113f99d9a0ab180ebfbc5ee0257fc142c74 Mon Sep 17 00:00:00 2001 From: Joe Delfino Date: Thu, 12 Feb 2026 15:05:09 +0000 Subject: [PATCH 07/23] fix: scope guardrail checks child issues for allowed files The scope enforcement guardrail was only extracting file paths from the parent issue body. In practice, file paths are listed in child task issues created during planning. Now queries all sub-issues via GraphQL and collects file paths from parent + children. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/guardrail-scope.yml | 64 ++++++++++++++++++++------- tests/test-guardrail-scope.sh | 7 +++ 2 files changed, 54 insertions(+), 17 deletions(-) diff --git a/.github/workflows/guardrail-scope.yml b/.github/workflows/guardrail-scope.yml index 5f231fb..fe488d8 100644 --- a/.github/workflows/guardrail-scope.yml +++ b/.github/workflows/guardrail-scope.yml @@ -115,15 +115,17 @@ jobs: } const issueNumber = parseInt(issueMatch[1], 10); - // --- Step 4: Get the issue body --- - let issueBody; + // --- Step 4: Get the issue body + all child issue bodies --- + // File paths may be listed in the parent issue, its child issues, or both. + // Collect bodies from the parent and all sub-issues. + const issueBodies = []; try { const issue = await github.rest.issues.get({ owner: context.repo.owner, repo: context.repo.repo, issue_number: issueNumber }); - issueBody = issue.data.body || ''; + issueBodies.push(issue.data.body || ''); } catch (e) { await createCheckRun( 'success', @@ -133,7 +135,32 @@ jobs: return; } - // --- Step 5: Extract file paths from the issue body --- + // Query sub-issues via GraphQL + try { + const subIssueResult = await github.graphql(` + query($owner: String!, $repo: String!, $number: Int!) { + repository(owner: $owner, name: $repo) { + issue(number: $number) { + subIssues(first: 50) { + nodes { body } + } + } + } + } + `, { + owner: context.repo.owner, + repo: context.repo.repo, + number: issueNumber + }); + const children = subIssueResult.repository.issue.subIssues.nodes || []; + for (const child of children) { + if (child.body) issueBodies.push(child.body); + } + } catch (e) { + // Sub-issues query failed — continue with parent body only + } + + // --- Step 5: Extract file paths from all issue bodies --- // Match paths that look like file paths: // - backtick-wrapped paths: `src/foo/bar.ts` // - paths with extensions: src/foo/bar.ts @@ -148,20 +175,23 @@ jobs: ]; const scopeFiles = new Set(); - for (const pattern of filePathPatterns) { - let match; - while ((match = pattern.exec(issueBody)) !== null) { - const filePath = match[1].replace(/^\//, ''); // strip leading slash - scopeFiles.add(filePath); + for (const body of issueBodies) { + for (const pattern of filePathPatterns) { + pattern.lastIndex = 0; // reset regex state for each body + let match; + while ((match = pattern.exec(body)) !== null) { + const filePath = match[1].replace(/^\//, ''); // strip leading slash + scopeFiles.add(filePath); + } } } if (scopeFiles.size === 0) { - // No file paths found in issue — cannot enforce scope + // No file paths found in any issue — cannot enforce scope await createCheckRun( 'success', - 'Scope enforcement: no files listed in issue', - `Issue #${issueNumber} does not list any file paths. Scope enforcement skipped.` + 'Scope enforcement: no files listed in issues', + `Issue #${issueNumber} and its children do not list any file paths. Scope enforcement skipped.` ); return; } @@ -205,7 +235,7 @@ jobs: await createCheckRun( 'success', 'Scope enforcement: all files in scope', - `All ${changedFiles.length} changed files are listed in issue #${issueNumber}.` + `All ${changedFiles.length} changed files are within scope of issue #${issueNumber} and its children.` ); return; } @@ -215,7 +245,7 @@ jobs: await createCheckRun( 'success', `Scope enforcement: approved by reviewer (${outOfScope.length} files outside scope)`, - `PR modifies ${outOfScope.length} file(s) not listed in issue #${issueNumber}, but a non-stale approval exists.\n\nOut-of-scope files:\n${outOfScope.map(f => '- `' + f.filename + '`').join('\n')}` + `PR modifies ${outOfScope.length} file(s) not listed in issue #${issueNumber} or its children, but a non-stale approval exists.\n\nOut-of-scope files:\n${outOfScope.map(f => '- `' + f.filename + '`').join('\n')}` ); return; } @@ -226,7 +256,7 @@ jobs: start_line: 1, end_line: 1, annotation_level: 'warning', - message: `This file is not listed in the task scope for issue #${issueNumber}. If this change is intentional, approve the PR to override.` + message: `This file is not listed in the task scope for issue #${issueNumber} or its children. If this change is intentional, approve the PR to override.` })); // Determine conclusion based on config and severity @@ -235,12 +265,12 @@ jobs: const conclusion = isMinor ? 'neutral' : config.conclusion; const summary = [ - `PR modifies ${outOfScope.length} file(s) not listed in issue #${issueNumber}.`, + `PR modifies ${outOfScope.length} file(s) not listed in issue #${issueNumber} or its children.`, '', '**Out-of-scope files:**', ...outOfScope.map(f => `- \`${f.filename}\``), '', - `**In-scope files (from issue):**`, + `**In-scope files (from issues):**`, ...[...scopeFiles].map(f => `- \`${f}\``), '', 'To resolve: either update the issue to include these files, or approve the PR to override this check.' diff --git a/tests/test-guardrail-scope.sh b/tests/test-guardrail-scope.sh index 135d576..754c32e 100755 --- a/tests/test-guardrail-scope.sh +++ b/tests/test-guardrail-scope.sh @@ -204,6 +204,13 @@ else fail "Does not handle missing file paths in issue" fi +# Test 22b: Queries sub-issues via GraphQL for scope files +if grep -q 'subIssues' "$WORKFLOW_FILE" 2>/dev/null; then + pass "Queries sub-issues for scope file extraction" +else + fail "Does not query sub-issues — only checks parent issue for file paths" +fi + # Test 23: Paginates changed files (per_page: 100) if grep -q 'per_page: 100' "$WORKFLOW_FILE" 2>/dev/null; then pass "Paginates changed files with per_page 100" From d92927c7f1512384027c03d9fe69b66cdb3e0bd0 Mon Sep 17 00:00:00 2001 From: Joe Delfino Date: Thu, 12 Feb 2026 15:11:35 +0000 Subject: [PATCH 08/23] fix: replace js-yaml with manual parsing in guardrail workflows actions/github-script@v7 does not include js-yaml, causing both the dependency-changes and test-ratio guardrails to crash with "Cannot find module 'js-yaml'". Replace with simple line-based config parsing matching the pattern used by the scope guardrail. Also fixes the dependency-changes config path which was reading from a top-level key instead of under the guardrails: section. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/guardrail-dependencies.yml | 25 +++++++++++------ .github/workflows/guardrail-test-ratio.yml | 29 ++++++++++++++------ 2 files changed, 37 insertions(+), 17 deletions(-) diff --git a/.github/workflows/guardrail-dependencies.yml b/.github/workflows/guardrail-dependencies.yml index 109c29c..a693441 100644 --- a/.github/workflows/guardrail-dependencies.yml +++ b/.github/workflows/guardrail-dependencies.yml @@ -22,7 +22,6 @@ jobs: with: script: | const fs = require('fs'); - const yaml = require('js-yaml'); // --- Configuration --- const CHECK_NAME = 'guardrail/dependency-changes'; @@ -94,20 +93,28 @@ jobs: 'CVE-' ]; - // --- Read config --- + // --- Read config (simple YAML parsing, no js-yaml dependency) --- let checkEnabled = true; let configuredConclusion = 'action_required'; try { const configPath = '.github/agent-workflow/config.yaml'; if (fs.existsSync(configPath)) { - const config = yaml.load(fs.readFileSync(configPath, 'utf8')); - if (config && config['dependency-changes']) { - const checkConfig = config['dependency-changes']; - if (checkConfig.enabled === false) { - checkEnabled = false; + const content = fs.readFileSync(configPath, 'utf8'); + const lines = content.split('\n'); + let inSection = false; + for (const line of lines) { + if (/^\s+dependency-changes:/.test(line)) { + inSection = true; + continue; } - if (checkConfig.conclusion) { - configuredConclusion = checkConfig.conclusion; + if (inSection && /^\s{0,4}\S/.test(line) && !line.match(/^\s{6,}/)) { + break; // Next sibling or parent key + } + if (inSection) { + const enabledMatch = line.match(/^\s+enabled:\s*(true|false)/); + if (enabledMatch) checkEnabled = enabledMatch[1] === 'true'; + const conclusionMatch = line.match(/^\s+conclusion:\s*(\S+)/); + if (conclusionMatch) configuredConclusion = conclusionMatch[1]; } } } diff --git a/.github/workflows/guardrail-test-ratio.yml b/.github/workflows/guardrail-test-ratio.yml index 5b7743c..ef024f3 100644 --- a/.github/workflows/guardrail-test-ratio.yml +++ b/.github/workflows/guardrail-test-ratio.yml @@ -21,21 +21,34 @@ jobs: with: script: | const fs = require('fs'); - const yaml = require('js-yaml'); - // --- Load configuration --- + // --- Load configuration (simple YAML parsing, no js-yaml dependency) --- const configPath = '.github/agent-workflow/config.yaml'; let threshold = 0.5; let enabled = true; let configuredConclusion = 'action_required'; try { - const configContent = fs.readFileSync(configPath, 'utf8'); - const config = yaml.load(configContent); - const testRatioConfig = config?.guardrails?.['test-ratio'] || {}; - threshold = testRatioConfig.threshold ?? 0.5; - enabled = testRatioConfig.enabled ?? true; - configuredConclusion = testRatioConfig.conclusion ?? 'action_required'; + const content = fs.readFileSync(configPath, 'utf8'); + const lines = content.split('\n'); + let inSection = false; + for (const line of lines) { + if (/^\s+test-ratio:/.test(line)) { + inSection = true; + continue; + } + if (inSection && /^\s{0,4}\S/.test(line) && !line.match(/^\s{6,}/)) { + break; // Next sibling or parent key + } + if (inSection) { + const enabledMatch = line.match(/^\s+enabled:\s*(true|false)/); + if (enabledMatch) enabled = enabledMatch[1] === 'true'; + const conclusionMatch = line.match(/^\s+conclusion:\s*(\S+)/); + if (conclusionMatch) configuredConclusion = conclusionMatch[1]; + const thresholdMatch = line.match(/^\s+threshold:\s*([0-9.]+)/); + if (thresholdMatch) threshold = parseFloat(thresholdMatch[1]); + } + } } catch (e) { core.info(`Could not read config from ${configPath}, using defaults: ${e.message}`); } From 2d571a79b6d8eebf6789aee2019a63b04f82dc25 Mon Sep 17 00:00:00 2001 From: Joe Delfino Date: Thu, 12 Feb 2026 17:34:56 +0000 Subject: [PATCH 09/23] refactor: use claude-code-action for PR reviewers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace manual `npm install && claude -p` with anthropics/claude-code-action@v1. Slim reviewer prompts to just pass context (base branch, parent issue) and point at skill files — the skills already contain the full review process, severity labels, and issue filing instructions. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/pr-review.yml | 144 ++++++++----------------------- tests/test_pr_review_workflow.py | 120 ++++++++++++-------------- 2 files changed, 90 insertions(+), 174 deletions(-) diff --git a/.github/workflows/pr-review.yml b/.github/workflows/pr-review.yml index 754fdb2..6f5b0ce 100644 --- a/.github/workflows/pr-review.yml +++ b/.github/workflows/pr-review.yml @@ -11,9 +11,9 @@ name: PR Review type: number permissions: - contents: write + contents: read issues: write - pull-requests: read + pull-requests: write jobs: # ── Resolve context shared by all reviewers ────────────────────────── @@ -71,43 +71,20 @@ jobs: with: fetch-depth: 0 - - name: Install Claude Code - run: npm install -g @anthropic-ai/claude-code - - name: Run correctness review + uses: anthropics/claude-code-action@v1 + with: + prompt: | + Read and follow .claude/skills/reviewer-correctness/SKILL.md + + Context: + - Base branch: origin/${{ needs.resolve-context.outputs.base-branch }} + - Parent issue: #${{ needs.resolve-context.outputs.parent-issue }} + + File findings as sub-issues of #${{ needs.resolve-context.outputs.parent-issue }}. + See .claude/skills/github-issues/SKILL.md for the GraphQL patterns. env: ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} - PR_NUMBER: ${{ needs.resolve-context.outputs.pr-number }} - PARENT_ISSUE: ${{ needs.resolve-context.outputs.parent-issue }} - BASE_BRANCH: ${{ needs.resolve-context.outputs.base-branch }} - PR_TITLE: ${{ needs.resolve-context.outputs.pr-title }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - claude -p "$(cat <<'PROMPT' - You are the Correctness Reviewer. Read and follow the skill file at - .claude/skills/reviewer-correctness/SKILL.md - - ## Context - - Repository: ${{ github.repository }} - - PR number: ${PR_NUMBER} - - PR title: ${PR_TITLE} - - Parent issue number: ${PARENT_ISSUE} - - Base branch: origin/${BASE_BRANCH} - - ## Instructions - 1. Run `git diff origin/${BASE_BRANCH}...HEAD` to get the full diff - 2. Review every changed file for bugs, error handling gaps, security issues, and API contract mismatches - 3. For each finding, create a child issue using `gh issue create`: - - Use labels: `blocking`, `should-fix`, or `suggestion` - - Include file path and line number in the issue body - 4. Link each created issue as a sub-issue of #${PARENT_ISSUE} using the GraphQL API: - - Get parent node ID and child node ID - - Use the addSubIssue mutation (see .claude/skills/github-issues/SKILL.md) - 5. For `blocking` findings, set them as blocking #${PARENT_ISSUE} using the - blocked-by dependency REST API (see .claude/skills/github-issues/SKILL.md) - 6. Report your outcome in the structured format from the skill file - PROMPT - )" # ── Tests Reviewer ─────────────────────────────────────────────────── review-tests: @@ -120,45 +97,20 @@ jobs: with: fetch-depth: 0 - - name: Install Claude Code - run: npm install -g @anthropic-ai/claude-code - - name: Run tests review + uses: anthropics/claude-code-action@v1 + with: + prompt: | + Read and follow .claude/skills/reviewer-tests/SKILL.md + + Context: + - Base branch: origin/${{ needs.resolve-context.outputs.base-branch }} + - Parent issue: #${{ needs.resolve-context.outputs.parent-issue }} + + File findings as sub-issues of #${{ needs.resolve-context.outputs.parent-issue }}. + See .claude/skills/github-issues/SKILL.md for the GraphQL patterns. env: ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} - PR_NUMBER: ${{ needs.resolve-context.outputs.pr-number }} - PARENT_ISSUE: ${{ needs.resolve-context.outputs.parent-issue }} - BASE_BRANCH: ${{ needs.resolve-context.outputs.base-branch }} - PR_TITLE: ${{ needs.resolve-context.outputs.pr-title }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - claude -p "$(cat <<'PROMPT' - You are the Test Quality Reviewer. Read and follow the skill file at - .claude/skills/reviewer-tests/SKILL.md - - ## Context - - Repository: ${{ github.repository }} - - PR number: ${PR_NUMBER} - - PR title: ${PR_TITLE} - - Parent issue number: ${PARENT_ISSUE} - - Base branch: origin/${BASE_BRANCH} - - ## Instructions - 1. Run `git diff origin/${BASE_BRANCH}...HEAD --stat` to identify changed files - 2. For every changed production file, find its corresponding test file - 3. Read each test file and evaluate: meaningful assertions, mock vs real behavior, - integration test coverage, edge cases, and test organization - 4. For each finding, create a child issue using `gh issue create`: - - Use labels: `blocking`, `should-fix`, or `suggestion` - - Include file path and description of what is missing or wrong - 5. Link each created issue as a sub-issue of #${PARENT_ISSUE} using the GraphQL API: - - Get parent node ID and child node ID - - Use the addSubIssue mutation (see .claude/skills/github-issues/SKILL.md) - 6. For `blocking` findings, set them as blocking #${PARENT_ISSUE} using the - blocked-by dependency REST API (see .claude/skills/github-issues/SKILL.md) - 7. Report your outcome in the structured format from the skill file - PROMPT - )" # ── Architecture Reviewer ──────────────────────────────────────────── review-architecture: @@ -171,43 +123,17 @@ jobs: with: fetch-depth: 0 - - name: Install Claude Code - run: npm install -g @anthropic-ai/claude-code - - name: Run architecture review + uses: anthropics/claude-code-action@v1 + with: + prompt: | + Read and follow .claude/skills/reviewer-architecture/SKILL.md + + Context: + - Base branch: origin/${{ needs.resolve-context.outputs.base-branch }} + - Parent issue: #${{ needs.resolve-context.outputs.parent-issue }} + + File findings as sub-issues of #${{ needs.resolve-context.outputs.parent-issue }}. + See .claude/skills/github-issues/SKILL.md for the GraphQL patterns. env: ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} - PR_NUMBER: ${{ needs.resolve-context.outputs.pr-number }} - PARENT_ISSUE: ${{ needs.resolve-context.outputs.parent-issue }} - BASE_BRANCH: ${{ needs.resolve-context.outputs.base-branch }} - PR_TITLE: ${{ needs.resolve-context.outputs.pr-title }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - claude -p "$(cat <<'PROMPT' - You are the Architecture Reviewer. Read and follow the skill file at - .claude/skills/reviewer-architecture/SKILL.md - - ## Context - - Repository: ${{ github.repository }} - - PR number: ${PR_NUMBER} - - PR title: ${PR_TITLE} - - Parent issue number: ${PARENT_ISSUE} - - Base branch: origin/${BASE_BRANCH} - - ## Instructions - 1. Run `git diff origin/${BASE_BRANCH}...HEAD --stat` to understand what changed - 2. Read the full codebase context — not just the diff — to catch duplication, - pattern divergence, and structural issues - 3. Check for: duplicated types, copy-pasted logic, pattern inconsistency, - leaky abstractions, unnecessary coupling, missing shared code - 4. For each finding, create a child issue using `gh issue create`: - - Use labels: `blocking`, `should-fix`, or `suggestion` - - Include file paths and reference to existing patterns - 5. Link each created issue as a sub-issue of #${PARENT_ISSUE} using the GraphQL API: - - Get parent node ID and child node ID - - Use the addSubIssue mutation (see .claude/skills/github-issues/SKILL.md) - 6. For `blocking` findings, set them as blocking #${PARENT_ISSUE} using the - blocked-by dependency REST API (see .claude/skills/github-issues/SKILL.md) - 7. Report your outcome in the structured format from the skill file - PROMPT - )" diff --git a/tests/test_pr_review_workflow.py b/tests/test_pr_review_workflow.py index 0022747..7c162f8 100644 --- a/tests/test_pr_review_workflow.py +++ b/tests/test_pr_review_workflow.py @@ -3,7 +3,7 @@ Validates the workflow structure, trigger configuration, permissions, parallel reviewer jobs, and the expected prompt construction for each -reviewer skill invoked via `claude -p`. +reviewer skill invoked via anthropics/claude-code-action. """ import yaml @@ -130,10 +130,10 @@ def _get_permissions(self): wf = load_workflow() return wf.get("permissions", {}) - def test_has_contents_write(self): + def test_has_contents_read(self): perms = self._get_permissions() - assert perms.get("contents") == "write", ( - "Needs contents:write to check out repo" + assert perms.get("contents") in ("read", "write"), ( + "Needs contents:read to check out repo" ) def test_has_issues_write(self): @@ -142,10 +142,10 @@ def test_has_issues_write(self): "Needs issues:write to create child issues for findings" ) - def test_has_pull_requests_read(self): + def test_has_pull_requests_write(self): perms = self._get_permissions() - assert perms.get("pull-requests") == "read", ( - "Needs pull-requests:read to read PR diff and description" + assert perms.get("pull-requests") == "write", ( + "Needs pull-requests:write for claude-code-action to post review comments" ) @@ -275,7 +275,7 @@ def test_parses_fixes_reference(self): class TestReviewerJobSteps: - """Each reviewer job must: checkout repo, then run claude -p with appropriate skill.""" + """Each reviewer job must: checkout repo, then invoke claude-code-action with appropriate skill.""" def _get_reviewer_jobs(self): wf = load_workflow() @@ -288,14 +288,19 @@ def _get_reviewer_jobs(self): } def _get_all_steps_content(self, job): - """Get all step run/script content for a job.""" + """Get all step run/script/prompt/uses content for a job.""" parts = [] for step in job.get("steps", []): if "run" in step: parts.append(step["run"]) - if step.get("uses", "").startswith("actions/github-script"): - script = step.get("with", {}).get("script", "") - parts.append(script) + uses = step.get("uses", "") + if uses: + parts.append(uses) + with_block = step.get("with", {}) + if "script" in with_block: + parts.append(with_block["script"]) + if "prompt" in with_block: + parts.append(with_block["prompt"]) return "\n".join(parts) def test_each_reviewer_checks_out_repo(self): @@ -311,14 +316,17 @@ def test_each_reviewer_checks_out_repo(self): f"Reviewer job '{job_name}' must have a checkout step" ) - def test_each_reviewer_runs_claude_p(self): - """Each reviewer job must run `claude -p` (or `claude --print`).""" + def test_each_reviewer_uses_claude_code_action(self): + """Each reviewer job must use anthropics/claude-code-action.""" reviewer_jobs = self._get_reviewer_jobs() for job_name, job in reviewer_jobs.items(): - content = self._get_all_steps_content(job) - has_claude = "claude" in content.lower() - assert has_claude, ( - f"Reviewer job '{job_name}' must invoke claude" + steps = job.get("steps", []) + has_action = any( + "anthropics/claude-code-action" in s.get("uses", "") + for s in steps + ) + assert has_action, ( + f"Reviewer job '{job_name}' must use anthropics/claude-code-action" ) def test_correctness_reviewer_references_skill(self): @@ -371,7 +379,7 @@ class TestContextPassing: """Each reviewer must receive PR number, parent issue number, and repo info.""" def _get_all_content(self): - """Get all run/script/env content from all jobs.""" + """Get all run/script/prompt/env content from all jobs.""" wf = load_workflow() parts = [] for job_name, job in wf.get("jobs", {}).items(): @@ -382,8 +390,11 @@ def _get_all_content(self): for step in job.get("steps", []): if "run" in step: parts.append(step["run"]) - if step.get("uses", "").startswith("actions/github-script"): - parts.append(step.get("with", {}).get("script", "")) + with_block = step.get("with", {}) + if "script" in with_block: + parts.append(with_block["script"]) + if "prompt" in with_block: + parts.append(with_block["prompt"]) for v in step.get("env", {}).values(): if isinstance(v, str): parts.append(v) @@ -430,7 +441,7 @@ def test_passes_repo_info(self): class TestReviewerPromptContent: - """The prompt passed to claude -p must instruct the reviewer correctly.""" + """The prompt passed to claude-code-action must instruct the reviewer correctly.""" def _get_all_content(self): wf = load_workflow() @@ -442,65 +453,44 @@ def _get_all_content(self): for step in job.get("steps", []): if "run" in step: parts.append(step["run"]) - if step.get("uses", "").startswith("actions/github-script"): - parts.append(step.get("with", {}).get("script", "")) + with_block = step.get("with", {}) + if "script" in with_block: + parts.append(with_block["script"]) + if "prompt" in with_block: + parts.append(with_block["prompt"]) for v in step.get("env", {}).values(): if isinstance(v, str): parts.append(v) return "\n".join(parts) - def test_prompt_instructs_reading_pr_diff(self): + def test_prompt_references_skill_files(self): content = self._get_all_content() - assert "diff" in content.lower(), ( - "Reviewer prompt must instruct reading the PR diff" + assert "reviewer-correctness/SKILL.md" in content, ( + "Prompt must reference the correctness reviewer skill file" ) - - def test_prompt_instructs_creating_issues(self): - content = self._get_all_content() - has_issue_create = ( - "gh issue create" in content - or "issue" in content.lower() + assert "reviewer-tests/SKILL.md" in content, ( + "Prompt must reference the tests reviewer skill file" ) - assert has_issue_create, ( - "Reviewer prompt must instruct creating child issues for findings" + assert "reviewer-architecture/SKILL.md" in content, ( + "Prompt must reference the architecture reviewer skill file" ) - def test_prompt_includes_severity_labels(self): + def test_prompt_references_github_issues_skill(self): content = self._get_all_content() - assert "blocking" in content, ( - "Prompt must mention 'blocking' severity label" - ) - assert "should-fix" in content, ( - "Prompt must mention 'should-fix' severity label" - ) - assert "suggestion" in content, ( - "Prompt must mention 'suggestion' severity label" + assert "github-issues/SKILL.md" in content, ( + "Prompt must reference the github-issues skill for GraphQL patterns" ) - def test_prompt_instructs_sub_issue_linking(self): + def test_prompt_passes_parent_issue_for_filing(self): content = self._get_all_content() - has_sub_issue = ( - "sub-issue" in content.lower() - or "sub_issue" in content.lower() - or "subissue" in content.lower() - or "addSubIssue" in content - or "child" in content.lower() - ) - assert has_sub_issue, ( - "Prompt must instruct linking findings as sub-issues of parent" + assert "sub-issue" in content.lower() or "finding" in content.lower(), ( + "Prompt must instruct filing findings against the parent issue" ) - def test_prompt_instructs_blocking_dependency(self): + def test_prompt_passes_base_branch(self): content = self._get_all_content() - has_blocking = ( - "blocking" in content.lower() - and ("dependency" in content.lower() - or "blocked_by" in content.lower() - or "blocked-by" in content.lower() - or "block" in content.lower()) - ) - assert has_blocking, ( - "Prompt must instruct setting blocking dependencies for blocking findings" + assert "base" in content.lower() and "branch" in content.lower(), ( + "Prompt must pass the base branch for diffing" ) @@ -508,7 +498,7 @@ def test_prompt_instructs_blocking_dependency(self): class TestSecrets: - """Workflow must use ANTHROPIC_API_KEY secret for claude -p.""" + """Workflow must use ANTHROPIC_API_KEY secret for claude-code-action.""" def _get_all_content(self): wf = load_workflow() From 901c3f92486d2097e50dd0b334a26311f88caddd Mon Sep 17 00:00:00 2001 From: Joe Delfino Date: Thu, 12 Feb 2026 17:40:44 +0000 Subject: [PATCH 10/23] chore: switch from ANTHROPIC_API_KEY to CLAUDE_CODE_OAUTH_TOKEN Co-Authored-By: Claude Opus 4.6 --- .github/workflows/orchestrator-check.yml | 2 +- .github/workflows/pr-review.yml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/orchestrator-check.yml b/.github/workflows/orchestrator-check.yml index 40b3524..c135834 100644 --- a/.github/workflows/orchestrator-check.yml +++ b/.github/workflows/orchestrator-check.yml @@ -26,7 +26,7 @@ jobs: - name: Orchestrator check uses: actions/github-script@v7 env: - ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} with: script: | const checkName = 'orchestrator'; diff --git a/.github/workflows/pr-review.yml b/.github/workflows/pr-review.yml index 6f5b0ce..a7e8842 100644 --- a/.github/workflows/pr-review.yml +++ b/.github/workflows/pr-review.yml @@ -84,7 +84,7 @@ jobs: File findings as sub-issues of #${{ needs.resolve-context.outputs.parent-issue }}. See .claude/skills/github-issues/SKILL.md for the GraphQL patterns. env: - ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} # ── Tests Reviewer ─────────────────────────────────────────────────── review-tests: @@ -110,7 +110,7 @@ jobs: File findings as sub-issues of #${{ needs.resolve-context.outputs.parent-issue }}. See .claude/skills/github-issues/SKILL.md for the GraphQL patterns. env: - ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} # ── Architecture Reviewer ──────────────────────────────────────────── review-architecture: @@ -136,4 +136,4 @@ jobs: File findings as sub-issues of #${{ needs.resolve-context.outputs.parent-issue }}. See .claude/skills/github-issues/SKILL.md for the GraphQL patterns. env: - ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} From ac16176f40094369c616dcd588308197a83c12cb Mon Sep 17 00:00:00 2001 From: Joe Delfino Date: Thu, 12 Feb 2026 17:54:54 +0000 Subject: [PATCH 11/23] fix: add id-token:write permission for OAuth OIDC exchange claude-code-action with CLAUDE_CODE_OAUTH_TOKEN needs id-token:write to fetch an OIDC token for authentication. Co-Authored-By: Claude Opus 4.6 --- .github/workflows/pr-review.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pr-review.yml b/.github/workflows/pr-review.yml index a7e8842..0fa9e98 100644 --- a/.github/workflows/pr-review.yml +++ b/.github/workflows/pr-review.yml @@ -14,6 +14,7 @@ permissions: contents: read issues: write pull-requests: write + id-token: write jobs: # ── Resolve context shared by all reviewers ────────────────────────── From 27d8b1857c050e1cd5a05de8f9e4c635eba43e0c Mon Sep 17 00:00:00 2001 From: Joe Delfino Date: Fri, 13 Feb 2026 01:06:41 +0000 Subject: [PATCH 12/23] feat: extract workflow scripts and shared libraries (#37, #38) Implements #37: Created 10 shared library modules in .github/agent-workflow/scripts/lib/ with comprehensive test suites. Each module exports pure functions for common workflow logic: - config.js: Parse guardrail YAML configuration - approval.js: Check for non-stale PR approvals - fixes-parser.js: Extract Fixes #N references - file-patterns.js: Test/code/dependency file detection - commit-validator.js: Conventional commit validation - scope-matcher.js: File path extraction and scope matching - api-patterns.js: API surface change detection - patch-parser.js: Unified diff line number parsing - severity.js: Review comment severity detection - pr-body.js: Idempotent PR body section replacement Implements #38: Extracted inline JavaScript from 8 workflow files into standalone scripts: - guardrail-scope.js, guardrail-test-ratio.js, guardrail-dependencies.js - guardrail-commits.js, guardrail-api-surface.js - orchestrator-check.js, human-review.js, pr-context.js Also added Node.js LTS to devcontainer for test execution. Co-Authored-By: Claude Sonnet 4.5 --- .devcontainer/devcontainer.json | 5 +- .../scripts/guardrail-api-surface.js | 206 ++++++++++ .../scripts/guardrail-commits.js | 158 ++++++++ .../scripts/guardrail-dependencies.js | 185 +++++++++ .../agent-workflow/scripts/guardrail-scope.js | 201 +++++++++ .../scripts/guardrail-test-ratio.js | 154 +++++++ .../agent-workflow/scripts/human-review.js | 192 +++++++++ .../scripts/lib/api-patterns.js | 80 ++++ .../scripts/lib/api-patterns.test.js | 50 +++ .../agent-workflow/scripts/lib/approval.js | 15 + .../scripts/lib/approval.test.js | 38 ++ .../scripts/lib/commit-validator.js | 26 ++ .../scripts/lib/commit-validator.test.js | 35 ++ .github/agent-workflow/scripts/lib/config.js | 70 ++++ .../agent-workflow/scripts/lib/config.test.js | 62 +++ .../scripts/lib/file-patterns.js | 53 +++ .../scripts/lib/file-patterns.test.js | 51 +++ .../scripts/lib/fixes-parser.js | 25 ++ .../scripts/lib/fixes-parser.test.js | 39 ++ .../scripts/lib/patch-parser.js | 42 ++ .../scripts/lib/patch-parser.test.js | 46 +++ .github/agent-workflow/scripts/lib/pr-body.js | 31 ++ .../scripts/lib/pr-body.test.js | 36 ++ .../scripts/lib/scope-matcher.js | 50 +++ .../scripts/lib/scope-matcher.test.js | 57 +++ .../agent-workflow/scripts/lib/severity.js | 40 ++ .../scripts/lib/severity.test.js | 28 ++ .../scripts/orchestrator-check.js | 383 ++++++++++++++++++ .github/agent-workflow/scripts/pr-context.js | 34 ++ .github/workflows/guardrail-scope.yml | 261 +----------- .github/workflows/guardrail-test-ratio.yml | 202 +-------- package.json | 12 + 32 files changed, 2407 insertions(+), 460 deletions(-) create mode 100644 .github/agent-workflow/scripts/guardrail-api-surface.js create mode 100644 .github/agent-workflow/scripts/guardrail-commits.js create mode 100644 .github/agent-workflow/scripts/guardrail-dependencies.js create mode 100644 .github/agent-workflow/scripts/guardrail-scope.js create mode 100644 .github/agent-workflow/scripts/guardrail-test-ratio.js create mode 100644 .github/agent-workflow/scripts/human-review.js create mode 100644 .github/agent-workflow/scripts/lib/api-patterns.js create mode 100644 .github/agent-workflow/scripts/lib/api-patterns.test.js create mode 100644 .github/agent-workflow/scripts/lib/approval.js create mode 100644 .github/agent-workflow/scripts/lib/approval.test.js create mode 100644 .github/agent-workflow/scripts/lib/commit-validator.js create mode 100644 .github/agent-workflow/scripts/lib/commit-validator.test.js create mode 100644 .github/agent-workflow/scripts/lib/config.js create mode 100644 .github/agent-workflow/scripts/lib/config.test.js create mode 100644 .github/agent-workflow/scripts/lib/file-patterns.js create mode 100644 .github/agent-workflow/scripts/lib/file-patterns.test.js create mode 100644 .github/agent-workflow/scripts/lib/fixes-parser.js create mode 100644 .github/agent-workflow/scripts/lib/fixes-parser.test.js create mode 100644 .github/agent-workflow/scripts/lib/patch-parser.js create mode 100644 .github/agent-workflow/scripts/lib/patch-parser.test.js create mode 100644 .github/agent-workflow/scripts/lib/pr-body.js create mode 100644 .github/agent-workflow/scripts/lib/pr-body.test.js create mode 100644 .github/agent-workflow/scripts/lib/scope-matcher.js create mode 100644 .github/agent-workflow/scripts/lib/scope-matcher.test.js create mode 100644 .github/agent-workflow/scripts/lib/severity.js create mode 100644 .github/agent-workflow/scripts/lib/severity.test.js create mode 100644 .github/agent-workflow/scripts/orchestrator-check.js create mode 100644 .github/agent-workflow/scripts/pr-context.js create mode 100644 package.json diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 6693fef..a6f9d7f 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -6,7 +6,10 @@ "source=${localEnv:HOME}/.claude,target=/home/vscode/.claude,type=bind,consistency=cached" ], "features": { - "ghcr.io/devcontainers/features/github-cli:1": {} + "ghcr.io/devcontainers/features/github-cli:1": {}, + "ghcr.io/devcontainers/features/node:1": { + "version": "lts" + } }, "postCreateCommand": "curl -fsSL https://claude.ai/install.sh | bash" } diff --git a/.github/agent-workflow/scripts/guardrail-api-surface.js b/.github/agent-workflow/scripts/guardrail-api-surface.js new file mode 100644 index 0000000..d5b00d9 --- /dev/null +++ b/.github/agent-workflow/scripts/guardrail-api-surface.js @@ -0,0 +1,206 @@ +const { parseGuardrailConfig } = require('./lib/config.js'); +const { hasNonStaleApproval } = require('./lib/approval.js'); +const { detectAPIChanges } = require('./lib/api-patterns.js'); + +module.exports = async function({ github, context, core }) { + const owner = context.repo.owner; + const repo = context.repo.repo; + const prNumber = context.payload.pull_request.number; + const headSha = context.payload.pull_request.head.sha; + const checkName = 'guardrail/api-surface'; + + // Read config + let enabled = true; + let configuredConclusion = 'action_required'; + try { + const configResponse = await github.rest.repos.getContent({ + owner, + repo, + path: '.github/agent-workflow/config.yaml', + ref: headSha, + }); + const configContent = Buffer.from(configResponse.data.content, 'base64').toString('utf8'); + const config = parseGuardrailConfig(configContent, 'api-surface'); + enabled = config.enabled; + configuredConclusion = config.conclusion; + } catch (e) { + core.info('No config.yaml found, using defaults (enabled: true, conclusion: action_required)'); + } + + if (!enabled) { + await github.rest.checks.create({ + owner, + repo, + head_sha: headSha, + name: checkName, + status: 'completed', + conclusion: 'success', + output: { + title: 'API surface check: disabled', + summary: 'This check is disabled in config.yaml.', + }, + }); + return; + } + + // Check for non-stale PR approval override + const reviews = await github.rest.pulls.listReviews({ + owner, + repo, + pull_number: prNumber, + }); + const hasValidApproval = hasNonStaleApproval(reviews.data, headSha); + + if (hasValidApproval) { + await github.rest.checks.create({ + owner, + repo, + head_sha: headSha, + name: checkName, + status: 'completed', + conclusion: 'success', + output: { + title: 'API surface check: approved by reviewer', + summary: 'A non-stale PR approval overrides this guardrail.', + }, + }); + return; + } + + // Get PR files and scan for API surface changes + const files = await github.paginate(github.rest.pulls.listFiles, { + owner, + repo, + pull_number: prNumber, + }); + + // OpenAPI / Swagger file patterns + const openApiFilePatterns = [ + /openapi\.(ya?ml|json)$/i, + /swagger\.(ya?ml|json)$/i, + /api-spec\.(ya?ml|json)$/i, + ]; + + const annotations = []; + let totalApiChanges = 0; + + for (const file of files) { + // Skip removed files + if (file.status === 'removed') continue; + + // Check if this is an OpenAPI/Swagger spec file + const isOpenApiFile = openApiFilePatterns.some((p) => p.test(file.filename)); + if (isOpenApiFile) { + totalApiChanges++; + annotations.push({ + path: file.filename, + start_line: 1, + end_line: 1, + annotation_level: 'warning', + message: `OpenAPI/Swagger spec file modified: ${file.filename}. API contract changes require careful review.`, + }); + continue; + } + + // Use API pattern detection from shared library + if (!file.patch) continue; + + const apiChanges = detectAPIChanges(file.patch, file.filename); + if (apiChanges.length > 0) { + totalApiChanges += apiChanges.length; + + // Parse patch for line numbers + const lines = file.patch.split('\n'); + let currentLine = 0; + let changeIndex = 0; + + for (const line of lines) { + // Track line numbers from hunk headers + const hunkMatch = line.match(/^@@\s+-\d+(?:,\d+)?\s+\+(\d+)(?:,\d+)?\s+@@/); + if (hunkMatch) { + currentLine = parseInt(hunkMatch[1], 10); + continue; + } + + // Only look at added lines + if (line.startsWith('+') && !line.startsWith('+++')) { + if (changeIndex < apiChanges.length) { + annotations.push({ + path: file.filename, + start_line: currentLine, + end_line: currentLine, + annotation_level: 'warning', + message: `API surface change: ${apiChanges[changeIndex]} - ${line.substring(1).trim()}`, + }); + changeIndex++; + } + } + + // Advance line counter for added and context lines + if (!line.startsWith('-')) { + currentLine++; + } + } + } + } + + // Report results + if (totalApiChanges === 0) { + await github.rest.checks.create({ + owner, + repo, + head_sha: headSha, + name: checkName, + status: 'completed', + conclusion: 'success', + output: { + title: 'API surface check: no changes detected', + summary: 'No API surface changes found in this PR.', + }, + }); + } else { + // GitHub API limits annotations to 50 per call + const batchSize = 50; + const batches = []; + for (let i = 0; i < annotations.length; i += batchSize) { + batches.push(annotations.slice(i, i + batchSize)); + } + + const summary = [ + `Found ${totalApiChanges} API surface change(s) across the PR.`, + '', + 'API surface changes have outsized downstream impact. Review these changes carefully.', + '', + 'To override: approve the PR to signal these changes are intentional.', + ].join('\n'); + + // Create the check run with the first batch of annotations + const checkRun = await github.rest.checks.create({ + owner, + repo, + head_sha: headSha, + name: checkName, + status: 'completed', + conclusion: configuredConclusion, + output: { + title: `API surface check: ${totalApiChanges} change(s) detected`, + summary, + annotations: batches[0] || [], + }, + }); + + // If there are more annotations, update the check run with additional batches + for (let i = 1; i < batches.length; i++) { + await github.rest.checks.update({ + owner, + repo, + check_run_id: checkRun.data.id, + output: { + title: `API surface check: ${totalApiChanges} change(s) detected`, + summary, + annotations: batches[i], + }, + }); + } + } +}; diff --git a/.github/agent-workflow/scripts/guardrail-commits.js b/.github/agent-workflow/scripts/guardrail-commits.js new file mode 100644 index 0000000..efedb68 --- /dev/null +++ b/.github/agent-workflow/scripts/guardrail-commits.js @@ -0,0 +1,158 @@ +const { parseGuardrailConfig } = require('./lib/config.js'); +const { hasNonStaleApproval } = require('./lib/approval.js'); +const { isValidCommit } = require('./lib/commit-validator.js'); + +module.exports = async function({ github, context, core }) { + const fs = require('fs'); + const path = require('path'); + + // Read config - default conclusion for commit message guardrail is 'neutral' (non-blocking warning) + let configuredConclusion = 'neutral'; + const configPath = path.join(process.env.GITHUB_WORKSPACE, '.github', 'agent-workflow', 'config.yaml'); + + try { + const configContent = fs.readFileSync(configPath, 'utf8'); + const config = parseGuardrailConfig(configContent, 'commit-messages'); + + if (config.enabled === false) { + await github.rest.checks.create({ + owner: context.repo.owner, + repo: context.repo.repo, + head_sha: context.sha, + name: 'guardrail/commit-messages', + conclusion: 'success', + output: { + title: 'Commit message check: disabled', + summary: 'This guardrail check is disabled in config.yaml.' + } + }); + return; + } + + if (config.conclusion) { + configuredConclusion = config.conclusion; + } + } catch (e) { + core.info(`No config.yaml found at ${configPath}, using default conclusion: neutral`); + } + + // Check for non-stale PR approval override + const reviews = await github.rest.pulls.listReviews({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: context.payload.pull_request.number + }); + + const headSha = context.payload.pull_request.head.sha; + const hasValidApproval = hasNonStaleApproval(reviews.data, headSha); + + if (hasValidApproval) { + await github.rest.checks.create({ + owner: context.repo.owner, + repo: context.repo.repo, + head_sha: context.sha, + name: 'guardrail/commit-messages', + conclusion: 'success', + output: { + title: 'Commit message check: approved by reviewer', + summary: 'A non-stale PR approval overrides this guardrail check.' + } + }); + return; + } + + // Get PR commits + const commits = await github.rest.pulls.listCommits({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: context.payload.pull_request.number, + per_page: 100 + }); + + // Validate each commit message + const maxFirstLineLength = 72; + const violations = []; + + for (const commit of commits.data) { + const message = commit.commit.message; + const firstLine = message.split('\n')[0]; + const sha = commit.sha.substring(0, 7); + const commitViolations = []; + + // Check conventional commit format + if (!isValidCommit(message, { maxLength: maxFirstLineLength })) { + if (firstLine.length > maxFirstLineLength) { + commitViolations.push( + `First line exceeds ${maxFirstLineLength} characters (${firstLine.length} chars)` + ); + } + // Check if it's a format issue + const conventionalCommitRegex = /^(feat|fix|chore|docs|test|refactor|ci|style|perf|build|revert)(\(.+\))?!?: .+/; + if (!conventionalCommitRegex.test(firstLine)) { + commitViolations.push( + `Does not follow conventional commit format (expected: type(scope)?: description)` + ); + } + } + + if (commitViolations.length > 0) { + violations.push({ + sha: sha, + fullSha: commit.sha, + firstLine: firstLine, + issues: commitViolations + }); + } + } + + // Report results + const totalCommits = commits.data.length; + const nonConformingCount = violations.length; + + if (nonConformingCount === 0) { + await github.rest.checks.create({ + owner: context.repo.owner, + repo: context.repo.repo, + head_sha: context.sha, + name: 'guardrail/commit-messages', + conclusion: 'success', + output: { + title: `Commit message check: all ${totalCommits} commits conform`, + summary: `All ${totalCommits} commit(s) follow conventional commit format with first line <= ${maxFirstLineLength} characters.` + } + }); + return; + } + + // Build summary with non-conforming commits + let summary = `## Non-conforming commits\n\n`; + summary += `Found **${nonConformingCount}** of ${totalCommits} commit(s) with violations:\n\n`; + + for (const v of violations) { + summary += `### \`${v.sha}\` — ${v.firstLine}\n`; + for (const issue of v.issues) { + summary += `- ${issue}\n`; + } + summary += '\n'; + } + + summary += `\n## Expected format\n\n`; + summary += '```\n'; + summary += 'type(optional-scope): description (max 72 chars)\n'; + summary += '```\n\n'; + summary += `Valid types: \`feat\`, \`fix\`, \`chore\`, \`docs\`, \`test\`, \`refactor\`, \`ci\`, \`style\`, \`perf\`, \`build\`, \`revert\`\n\n`; + summary += `**Configured conclusion:** \`${configuredConclusion}\`\n`; + summary += `\nTo override: submit an approving PR review. The approval must be on the current head commit to be non-stale.\n`; + + await github.rest.checks.create({ + owner: context.repo.owner, + repo: context.repo.repo, + head_sha: context.sha, + name: 'guardrail/commit-messages', + conclusion: configuredConclusion, + output: { + title: `Commit message check: ${nonConformingCount} non-conforming commit(s)`, + summary: summary + } + }); +}; diff --git a/.github/agent-workflow/scripts/guardrail-dependencies.js b/.github/agent-workflow/scripts/guardrail-dependencies.js new file mode 100644 index 0000000..44ed247 --- /dev/null +++ b/.github/agent-workflow/scripts/guardrail-dependencies.js @@ -0,0 +1,185 @@ +const { parseGuardrailConfig } = require('./lib/config.js'); +const { hasNonStaleApproval } = require('./lib/approval.js'); +const { isDependencyFile } = require('./lib/file-patterns.js'); + +module.exports = async function({ github, context, core }) { + const fs = require('fs'); + const CHECK_NAME = 'guardrail/dependency-changes'; + + const JUSTIFICATION_KEYWORDS = [ + 'dependency', 'dependencies', + 'added', 'adding', + 'requires', 'required', + 'needed for', 'needed by', + 'introduced', + 'new package', 'new library', 'new module', + 'upgrade', 'upgraded', 'upgrading', + 'update', 'updated', 'updating', + 'migration', 'migrate', 'migrating', + 'replace', 'replaced', 'replacing', + 'security fix', 'security patch', 'vulnerability', + 'CVE-' + ]; + + // Read config + let checkEnabled = true; + let configuredConclusion = 'action_required'; + try { + const configPath = '.github/agent-workflow/config.yaml'; + if (fs.existsSync(configPath)) { + const content = fs.readFileSync(configPath, 'utf8'); + const config = parseGuardrailConfig(content, 'dependency-changes'); + checkEnabled = config.enabled; + configuredConclusion = config.conclusion; + } + } catch (e) { + core.warning(`Failed to read config: ${e.message}. Using defaults.`); + } + + // If check is disabled, report success and exit + if (!checkEnabled) { + await github.rest.checks.create({ + owner: context.repo.owner, + repo: context.repo.repo, + head_sha: context.sha, + name: CHECK_NAME, + status: 'completed', + conclusion: 'success', + output: { + title: 'Dependency changes: check disabled', + summary: 'This guardrail check is disabled in config.yaml.' + } + }); + return; + } + + // Check for non-stale approving PR review (override mechanism) + const { data: reviews } = await github.rest.pulls.listReviews({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: context.payload.pull_request.number + }); + + const lastCommitSha = context.payload.pull_request.head.sha; + const hasValidApproval = hasNonStaleApproval(reviews, lastCommitSha); + + if (hasValidApproval) { + await github.rest.checks.create({ + owner: context.repo.owner, + repo: context.repo.repo, + head_sha: context.sha, + name: CHECK_NAME, + status: 'completed', + conclusion: 'success', + output: { + title: 'Dependency changes: approved by reviewer', + summary: 'A non-stale PR approval overrides dependency change violations.' + } + }); + return; + } + + // Get PR changed files + const files = await github.paginate( + github.rest.pulls.listFiles, + { + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: context.payload.pull_request.number, + per_page: 100 + } + ); + + const changedDependencyFiles = files.filter(f => isDependencyFile(f.filename)); + + // No dependency files changed: success + if (changedDependencyFiles.length === 0) { + await github.rest.checks.create({ + owner: context.repo.owner, + repo: context.repo.repo, + head_sha: context.sha, + name: CHECK_NAME, + status: 'completed', + conclusion: 'success', + output: { + title: 'Dependency changes: no dependency files modified', + summary: 'No dependency manifest or lock files were changed in this PR.' + } + }); + return; + } + + // Dependency files changed: check for justification + const prBody = (context.payload.pull_request.body || '').toLowerCase(); + + function hasJustification(text) { + const lowerText = text.toLowerCase(); + return JUSTIFICATION_KEYWORDS.some(keyword => lowerText.includes(keyword)); + } + + let justified = hasJustification(prBody); + + // If not justified in PR body, check linked issue body + if (!justified) { + const issueMatch = (context.payload.pull_request.body || '').match( + /(?:fixes|closes|resolves)\s+#(\d+)/i + ); + if (issueMatch) { + try { + const { data: issue } = await github.rest.issues.get({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: parseInt(issueMatch[1], 10) + }); + if (issue.body) { + justified = hasJustification(issue.body); + } + } catch (e) { + core.warning(`Failed to fetch linked issue #${issueMatch[1]}: ${e.message}`); + } + } + } + + // Build annotations for changed dependency files + const annotations = changedDependencyFiles.map(f => ({ + path: f.filename, + start_line: 1, + end_line: 1, + annotation_level: 'warning', + message: justified + ? `Dependency file changed. Justification found in PR or linked issue.` + : `Dependency file changed without justification. Add context about why dependencies were changed to the PR description or linked issue.` + })); + + // Report result + if (justified) { + await github.rest.checks.create({ + owner: context.repo.owner, + repo: context.repo.repo, + head_sha: context.sha, + name: CHECK_NAME, + status: 'completed', + conclusion: 'success', + output: { + title: `Dependency changes: ${changedDependencyFiles.length} file(s) changed with justification`, + summary: `Dependency files were modified and justification was found in the PR body or linked issue.\n\n**Changed dependency files:**\n${changedDependencyFiles.map(f => '- `' + f.filename + '`').join('\n')}`, + annotations: annotations + } + }); + } else { + const fileList = changedDependencyFiles.map(f => '- `' + f.filename + '`').join('\n'); + await github.rest.checks.create({ + owner: context.repo.owner, + repo: context.repo.repo, + head_sha: context.sha, + name: CHECK_NAME, + status: 'completed', + conclusion: configuredConclusion, + output: { + title: `Dependency changes: ${changedDependencyFiles.length} file(s) changed without justification`, + summary: `Dependency files were modified but no justification was found.\n\n**Changed dependency files:**\n${fileList}\n\n**To resolve:** Add context about dependency changes to the PR description using keywords like: ${JUSTIFICATION_KEYWORDS.slice(0, 8).map(k => '"' + k + '"').join(', ')}, etc.\n\nAlternatively, a PR approval will override this check.`, + annotations: annotations + } + }); + } +}; diff --git a/.github/agent-workflow/scripts/guardrail-scope.js b/.github/agent-workflow/scripts/guardrail-scope.js new file mode 100644 index 0000000..0761a10 --- /dev/null +++ b/.github/agent-workflow/scripts/guardrail-scope.js @@ -0,0 +1,201 @@ +const { parseGuardrailConfig } = require('./lib/config.js'); +const { hasNonStaleApproval } = require('./lib/approval.js'); +const { parseFixesReferences } = require('./lib/fixes-parser.js'); +const { extractFilePaths, isInScope } = require('./lib/scope-matcher.js'); + +module.exports = async function({ github, context, core }) { + const checkName = 'guardrail/scope'; + + // Helper: create check run + async function createCheckRun(conclusion, title, summary, annotations = []) { + const output = { title, summary }; + if (annotations.length > 0) { + output.annotations = annotations.slice(0, 50); + } + await github.rest.checks.create({ + owner: context.repo.owner, + repo: context.repo.repo, + head_sha: context.payload.pull_request.head.sha, + name: checkName, + status: 'completed', + conclusion, + output + }); + } + + // Step 1: Read config and check if enabled + let config; + try { + const { data } = await github.rest.repos.getContent({ + owner: context.repo.owner, + repo: context.repo.repo, + path: '.github/agent-workflow/config.yaml', + ref: context.payload.pull_request.head.sha + }); + const content = Buffer.from(data.content, 'base64').toString('utf-8'); + config = parseGuardrailConfig(content, 'scope-enforcement'); + } catch (e) { + config = { enabled: true, conclusion: 'action_required' }; + } + + if (!config.enabled) { + await createCheckRun( + 'success', + 'Scope enforcement: disabled', + 'Scope enforcement is disabled in agent-workflow config.' + ); + return; + } + + // Step 2: Check for non-stale PR approval override + const reviews = await github.rest.pulls.listReviews({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: context.payload.pull_request.number + }); + const headSha = context.payload.pull_request.head.sha; + const hasValidApproval = hasNonStaleApproval(reviews.data, headSha); + + // Step 3: Parse issue reference from PR body + const prBody = context.payload.pull_request.body || ''; + const issueNumbers = parseFixesReferences(prBody); + if (issueNumbers.length === 0) { + await createCheckRun( + 'success', + 'Scope enforcement: no linked issue', + 'No `fixes #N` reference found in PR description. Scope enforcement skipped.' + ); + return; + } + const issueNumber = issueNumbers[0]; + + // Step 4: Get the issue body + all child issue bodies + const issueBodies = []; + try { + const issue = await github.rest.issues.get({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: issueNumber + }); + issueBodies.push(issue.data.body || ''); + } catch (e) { + await createCheckRun( + 'success', + 'Scope enforcement: issue not found', + `Could not read issue #${issueNumber}. Scope enforcement skipped.` + ); + return; + } + + // Query sub-issues via GraphQL + try { + const subIssueResult = await github.graphql(` + query($owner: String!, $repo: String!, $number: Int!) { + repository(owner: $owner, name: $repo) { + issue(number: $number) { + subIssues(first: 50) { + nodes { body } + } + } + } + } + `, { + owner: context.repo.owner, + repo: context.repo.repo, + number: issueNumber + }); + const children = subIssueResult.repository.issue.subIssues.nodes || []; + for (const child of children) { + if (child.body) issueBodies.push(child.body); + } + } catch (e) { + // Sub-issues query failed — continue with parent body only + } + + // Step 5: Extract file paths from all issue bodies + const scopeFiles = []; + for (const body of issueBodies) { + scopeFiles.push(...extractFilePaths(body)); + } + const uniqueScopeFiles = [...new Set(scopeFiles)]; + + if (uniqueScopeFiles.length === 0) { + await createCheckRun( + 'success', + 'Scope enforcement: no files listed in issues', + `Issue #${issueNumber} and its children do not list any file paths. Scope enforcement skipped.` + ); + return; + } + + // Step 6: Get changed files from the PR + const changedFiles = []; + let page = 1; + while (true) { + const resp = await github.rest.pulls.listFiles({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: context.payload.pull_request.number, + per_page: 100, + page + }); + changedFiles.push(...resp.data); + if (resp.data.length < 100) break; + page++; + } + + // Step 7: Compare changed files against scope + const outOfScope = changedFiles.filter(f => !isInScope(f.filename, uniqueScopeFiles)); + + // Step 8: Report results + if (outOfScope.length === 0) { + await createCheckRun( + 'success', + 'Scope enforcement: all files in scope', + `All ${changedFiles.length} changed files are within scope of issue #${issueNumber} and its children.` + ); + return; + } + + // There are out-of-scope files + if (hasValidApproval) { + await createCheckRun( + 'success', + `Scope enforcement: approved by reviewer (${outOfScope.length} files outside scope)`, + `PR modifies ${outOfScope.length} file(s) not listed in issue #${issueNumber} or its children, but a non-stale approval exists.\n\nOut-of-scope files:\n${outOfScope.map(f => '- `' + f.filename + '`').join('\n')}` + ); + return; + } + + // Build annotations for out-of-scope files + const annotations = outOfScope.map(f => ({ + path: f.filename, + start_line: 1, + end_line: 1, + annotation_level: 'warning', + message: `This file is not listed in the task scope for issue #${issueNumber} or its children. If this change is intentional, approve the PR to override.` + })); + + // Determine conclusion based on config and severity + const isMinor = outOfScope.length <= 2; + const conclusion = isMinor ? 'neutral' : config.conclusion; + + const summary = [ + `PR modifies ${outOfScope.length} file(s) not listed in issue #${issueNumber} or its children.`, + '', + '**Out-of-scope files:**', + ...outOfScope.map(f => `- \`${f.filename}\``), + '', + `**In-scope files (from issues):**`, + ...uniqueScopeFiles.map(f => `- \`${f}\``), + '', + 'To resolve: either update the issue to include these files, or approve the PR to override this check.' + ].join('\n'); + + await createCheckRun( + conclusion, + `Scope enforcement: ${outOfScope.length} file(s) outside task scope`, + summary, + annotations + ); +}; diff --git a/.github/agent-workflow/scripts/guardrail-test-ratio.js b/.github/agent-workflow/scripts/guardrail-test-ratio.js new file mode 100644 index 0000000..a67b8bb --- /dev/null +++ b/.github/agent-workflow/scripts/guardrail-test-ratio.js @@ -0,0 +1,154 @@ +const { parseGuardrailConfig } = require('./lib/config.js'); +const { hasNonStaleApproval } = require('./lib/approval.js'); +const { isTestFile, isCodeFile } = require('./lib/file-patterns.js'); + +module.exports = async function({ github, context, core }) { + // Load configuration + const fs = require('fs'); + const configPath = '.github/agent-workflow/config.yaml'; + let config = { enabled: true, conclusion: 'action_required', threshold: 0.5 }; + + try { + const content = fs.readFileSync(configPath, 'utf8'); + config = { ...config, ...parseGuardrailConfig(content, 'test-ratio') }; + } catch (e) { + core.info(`Could not read config from ${configPath}, using defaults: ${e.message}`); + } + + if (!config.enabled) { + core.info('Test-ratio guardrail is disabled in config. Skipping.'); + return; + } + + const threshold = config.threshold || 0.5; + + // Get PR files + const prNumber = context.payload.pull_request.number; + const allFiles = []; + let page = 1; + while (true) { + const { data: files } = await github.rest.pulls.listFiles({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: prNumber, + per_page: 100, + page: page, + }); + allFiles.push(...files); + if (files.length < 100) break; + page++; + } + + // Categorize files and count lines + let testLines = 0; + let implLines = 0; + const implFilesWithNoTests = []; + + for (const file of allFiles) { + if (file.status === 'removed') continue; + if (!isCodeFile(file.filename)) continue; + + const additions = file.additions || 0; + + if (isTestFile(file.filename)) { + testLines += additions; + } else { + implLines += additions; + implFilesWithNoTests.push({ + filename: file.filename, + additions: additions, + }); + } + } + + core.info(`Test lines added: ${testLines}`); + core.info(`Implementation lines added: ${implLines}`); + + // Handle edge case: no implementation lines + if (implLines === 0) { + core.info('No implementation lines in this PR. Reporting success.'); + await github.rest.checks.create({ + owner: context.repo.owner, + repo: context.repo.repo, + head_sha: context.payload.pull_request.head.sha, + name: 'guardrail/test-ratio', + status: 'completed', + conclusion: 'success', + output: { + title: 'Test-to-code ratio: no implementation changes', + summary: 'This PR contains no implementation line additions. Test ratio check is not applicable.', + }, + }); + return; + } + + // Calculate ratio + const ratio = testLines / implLines; + const passed = ratio >= threshold; + + core.info(`Test-to-code ratio: ${ratio.toFixed(2)} (threshold: ${threshold})`); + + // Check for non-stale PR approval override + const { data: reviews } = await github.rest.pulls.listReviews({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: prNumber, + }); + + const headSha = context.payload.pull_request.head.sha; + const hasValidApproval = hasNonStaleApproval(reviews, headSha); + + // Determine conclusion + let conclusion; + let title; + let summary; + + if (passed) { + conclusion = 'success'; + title = `Test-to-code ratio: ${ratio.toFixed(2)} (threshold: ${threshold})`; + summary = `PR has ${testLines} test lines and ${implLines} implementation lines added. Ratio ${ratio.toFixed(2)} meets the threshold of ${threshold}.`; + } else if (hasValidApproval) { + conclusion = 'success'; + title = `Test-to-code ratio: ${ratio.toFixed(2)} — approved by reviewer`; + summary = `PR has ${testLines} test lines and ${implLines} implementation lines added. Ratio ${ratio.toFixed(2)} is below the threshold of ${threshold}, but a non-stale approval exists. Human has accepted the current state.`; + } else { + conclusion = config.conclusion; + title = `Test-to-code ratio: ${ratio.toFixed(2)} (threshold: ${threshold})`; + summary = `PR has ${testLines} test lines and ${implLines} implementation lines added. Ratio ${ratio.toFixed(2)} is below the threshold of ${threshold}. Add more tests or approve the PR to override.`; + } + + // Build annotations for implementation files lacking test coverage + const annotations = []; + if (!passed && !hasValidApproval) { + for (const file of implFilesWithNoTests) { + if (file.additions > 0) { + annotations.push({ + path: file.filename, + start_line: 1, + end_line: 1, + annotation_level: 'warning', + message: `This implementation file has ${file.additions} added lines. The overall test-to-code ratio (${ratio.toFixed(2)}) is below the threshold (${threshold}). Consider adding corresponding tests.`, + }); + } + } + } + + const truncatedAnnotations = annotations.slice(0, 50); + + // Report check run + await github.rest.checks.create({ + owner: context.repo.owner, + repo: context.repo.repo, + head_sha: headSha, + name: 'guardrail/test-ratio', + status: 'completed', + conclusion: conclusion, + output: { + title: title, + summary: summary, + annotations: truncatedAnnotations, + }, + }); + + core.info(`Check run created with conclusion: ${conclusion}`); +}; diff --git a/.github/agent-workflow/scripts/human-review.js b/.github/agent-workflow/scripts/human-review.js new file mode 100644 index 0000000..0efd28a --- /dev/null +++ b/.github/agent-workflow/scripts/human-review.js @@ -0,0 +1,192 @@ +const { detectSeverity } = require('./lib/severity.js'); +const { parseFixesReferences } = require('./lib/fixes-parser.js'); + +module.exports = async function({ github, context, core }) { + const review = context.payload.review; + const pr = context.payload.pull_request; + + // Parse parent issue from PR body + const issueNumbers = parseFixesReferences(pr.body); + if (issueNumbers.length === 0) { + console.log('No parent issue found — PR body has no "Fixes #N" reference. Skipping.'); + return; + } + const parentIssueNumber = issueNumbers[0]; + + // Fetch comments for this specific review + const { data: reviewComments } = await github.request( + 'GET /repos/{owner}/{repo}/pulls/{pull_number}/reviews/{review_id}/comments', + { + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: pr.number, + review_id: review.id, + } + ); + + if (reviewComments.length === 0) { + console.log('Review has no line-level comments. Skipping.'); + return; + } + + // Get parent issue node_id for GraphQL sub-issue linking + const { data: parentIssue } = await github.rest.issues.get({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: parentIssueNumber, + }); + const parentNodeId = parentIssue.node_id; + + // Ensure labels exist + const severityLabels = ['blocking', 'should-fix', 'suggestion']; + const labelColors = { + 'blocking': 'B60205', + 'should-fix': 'D93F0B', + 'suggestion': '0E8A16', + }; + + for (const label of severityLabels) { + try { + await github.rest.issues.getLabel({ + owner: context.repo.owner, + repo: context.repo.repo, + name: label, + }); + } catch { + await github.rest.issues.createLabel({ + owner: context.repo.owner, + repo: context.repo.repo, + name: label, + color: labelColors[label], + }); + } + } + + // Process each comment + const createdIssues = []; + let hasBlockingComments = false; + + for (const comment of reviewComments) { + const severity = detectSeverity(comment.body); + const filePath = comment.path; + const line = comment.original_line || comment.line || 0; + + // Build issue body with file/line context + const issueBody = [ + `## Review Finding`, + ``, + `**Severity:** \`${severity}\``, + `**File:** \`${filePath}\`${line ? ` (line ${line})` : ''}`, + `**PR:** #${pr.number}`, + `**Reviewer:** @${review.user.login}`, + ``, + `### Comment`, + ``, + comment.body, + ``, + `---`, + `_Created automatically from a PR review comment._`, + ].join('\n'); + + const issueTitle = `[${severity}] ${filePath}${line ? `:${line}` : ''}: ${comment.body.split('\n')[0].substring(0, 80)}`; + + // Create the child issue + const { data: newIssue } = await github.rest.issues.create({ + owner: context.repo.owner, + repo: context.repo.repo, + title: issueTitle, + body: issueBody, + labels: [severity], + }); + + console.log(`Created issue #${newIssue.number}: ${newIssue.title}`); + + // Link as sub-issue via GraphQL addSubIssue mutation + try { + await github.graphql(` + mutation($parentId: ID!, $childId: ID!) { + addSubIssue(input: { issueId: $parentId, subIssueId: $childId }) { + issue { id } + subIssue { id } + } + } + `, { + parentId: parentNodeId, + childId: newIssue.node_id, + }); + console.log(`Linked issue #${newIssue.number} as sub-issue of #${parentIssueNumber}`); + } catch (err) { + console.log(`Warning: Could not link sub-issue via GraphQL: ${err.message}`); + } + + // If blocking, set as blocking the parent issue + if (severity === 'blocking') { + hasBlockingComments = true; + try { + await github.request( + 'POST /repos/{owner}/{repo}/issues/{issue_number}/sub_issues/priority', + { + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: parentIssueNumber, + sub_issue_id: newIssue.id, + } + ); + } catch (err) { + console.log(`Note: Could not set blocking dependency via REST: ${err.message}`); + } + } + + createdIssues.push({ + number: newIssue.number, + severity, + }); + } + + // Update PR description with Fixes references + if (createdIssues.length > 0) { + const fixesLines = createdIssues + .map(i => `Fixes #${i.number}`) + .join('\n'); + + const newSection = [ + '', + '### Review Finding Issues', + fixesLines, + '', + ].join('\n'); + + // Replace existing section or append, for idempotent updates + let currentBody = pr.body || ''; + const sectionRegex = /[\s\S]*?/; + let updatedBody; + if (sectionRegex.test(currentBody)) { + updatedBody = currentBody.replace(sectionRegex, newSection); + } else { + updatedBody = currentBody + '\n\n' + newSection; + } + + await github.rest.pulls.update({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: pr.number, + body: updatedBody, + }); + + console.log(`Updated PR #${pr.number} body with ${createdIssues.length} Fixes references`); + } + + // Summary + const blockingCount = createdIssues.filter(i => i.severity === 'blocking').length; + const shouldFixCount = createdIssues.filter(i => i.severity === 'should-fix').length; + const suggestionCount = createdIssues.filter(i => i.severity === 'suggestion').length; + + console.log(`\nDone. Created ${createdIssues.length} issues from review comments:`); + console.log(` blocking: ${blockingCount}`); + console.log(` should-fix: ${shouldFixCount}`); + console.log(` suggestion: ${suggestionCount}`); + + if (hasBlockingComments) { + console.log(`\nBlocking issues were created — parent issue #${parentIssueNumber} has new blockers.`); + } +}; diff --git a/.github/agent-workflow/scripts/lib/api-patterns.js b/.github/agent-workflow/scripts/lib/api-patterns.js new file mode 100644 index 0000000..afa1a2b --- /dev/null +++ b/.github/agent-workflow/scripts/lib/api-patterns.js @@ -0,0 +1,80 @@ +/** + * Detect API surface changes in a diff + * @param {string} diff - Unified diff content + * @param {string} filename - File being changed + * @returns {string[]} - Array of detected API changes + */ +function detectAPIChanges(diff, filename) { + if (!diff) return []; + + const changes = []; + const lines = diff.split('\n'); + + // Language-specific patterns for API surface detection + const patterns = { + js: [ + /^\+\s*export\s+(function|const|let|var|class|interface|type|enum)\s+(\w+)/, + /^\+\s*export\s+default/, + /^\+\s*export\s*{/ + ], + ts: [ + /^\+\s*export\s+(function|const|let|var|class|interface|type|enum)\s+(\w+)/, + /^\+\s*export\s+default/, + /^\+\s*export\s*{/, + /^\+\s*export\s+interface\s+(\w+)/ + ], + py: [ + /^\+\s*class\s+(\w+)/, + /^\+\s*def\s+(\w+)/, + /^\+\s*async\s+def\s+(\w+)/ + ], + go: [ + /^\+\s*func\s+(\w+)/, + /^\+\s*type\s+(\w+)\s+(?:struct|interface)/ + ], + rs: [ + /^\+\s*pub\s+fn\s+(\w+)/, + /^\+\s*pub\s+struct\s+(\w+)/, + /^\+\s*pub\s+enum\s+(\w+)/, + /^\+\s*pub\s+trait\s+(\w+)/ + ] + }; + + // Determine language from file extension + const ext = filename.split('.').pop(); + const langPatterns = patterns[ext] || patterns.js; + + for (const line of lines) { + // Only look at added lines + if (!line.startsWith('+')) continue; + + for (const pattern of langPatterns) { + const match = line.match(pattern); + if (match) { + const name = match[2] || match[1] || 'exported item'; + changes.push(`Added/modified export: ${name}`); + break; + } + } + } + + // Also check for removed exports + for (const line of lines) { + if (!line.startsWith('-')) continue; + + for (const pattern of langPatterns) { + // Adjust pattern for removal (- instead of +) + const removePattern = new RegExp(pattern.source.replace(/^\\\+/, '-')); + const match = line.match(removePattern); + if (match) { + const name = match[2] || match[1] || 'exported item'; + changes.push(`Removed export: ${name}`); + break; + } + } + } + + return changes; +} + +module.exports = { detectAPIChanges }; diff --git a/.github/agent-workflow/scripts/lib/api-patterns.test.js b/.github/agent-workflow/scripts/lib/api-patterns.test.js new file mode 100644 index 0000000..2e3dc3a --- /dev/null +++ b/.github/agent-workflow/scripts/lib/api-patterns.test.js @@ -0,0 +1,50 @@ +const { test } = require('node:test'); +const assert = require('node:assert'); +const { detectAPIChanges } = require('./api-patterns.js'); + +test('detectAPIChanges - JavaScript export changes', () => { + const diff = ` ++export function newAPI() {} +-export function oldAPI() {} ++export const API_CONSTANT = 42; + `; + const changes = detectAPIChanges(diff, 'file.js'); + assert.ok(changes.length > 0); + assert.ok(changes.some(c => c.includes('newAPI'))); +}); + +test('detectAPIChanges - TypeScript interface changes', () => { + const diff = ` ++export interface NewInterface { ++ field: string; ++} + `; + const changes = detectAPIChanges(diff, 'types.ts'); + assert.ok(changes.length > 0); + assert.ok(changes.some(c => c.includes('NewInterface'))); +}); + +test('detectAPIChanges - Python class changes', () => { + const diff = ` ++class PublicAPI: ++ def __init__(self): ++ pass + `; + const changes = detectAPIChanges(diff, 'module.py'); + assert.ok(changes.length > 0); +}); + +test('detectAPIChanges - no API changes', () => { + const diff = ` ++// Internal helper function ++function helper() {} ++const internal = 123; + `; + const changes = detectAPIChanges(diff, 'file.js'); + assert.strictEqual(changes.length, 0); +}); + +test('detectAPIChanges - empty diff', () => { + const changes = detectAPIChanges('', 'file.js'); + assert.strictEqual(changes.length, 0); +}); diff --git a/.github/agent-workflow/scripts/lib/approval.js b/.github/agent-workflow/scripts/lib/approval.js new file mode 100644 index 0000000..fc84f8b --- /dev/null +++ b/.github/agent-workflow/scripts/lib/approval.js @@ -0,0 +1,15 @@ +/** + * Check if there is a non-stale PR approval + * @param {Array} reviews - Array of review objects from GitHub API + * @param {string} headSha - Current head SHA of the PR + * @returns {boolean} + */ +function hasNonStaleApproval(reviews, headSha) { + if (!reviews || reviews.length === 0) return false; + + return reviews.some( + review => review.state === 'APPROVED' && review.commit_id === headSha + ); +} + +module.exports = { hasNonStaleApproval }; diff --git a/.github/agent-workflow/scripts/lib/approval.test.js b/.github/agent-workflow/scripts/lib/approval.test.js new file mode 100644 index 0000000..d28ff25 --- /dev/null +++ b/.github/agent-workflow/scripts/lib/approval.test.js @@ -0,0 +1,38 @@ +const { test } = require('node:test'); +const assert = require('node:assert'); +const { hasNonStaleApproval } = require('./approval.js'); + +test('hasNonStaleApproval - approved at head SHA', () => { + const reviews = [ + { state: 'APPROVED', commit_id: 'abc123' }, + { state: 'COMMENTED', commit_id: 'abc123' } + ]; + assert.strictEqual(hasNonStaleApproval(reviews, 'abc123'), true); +}); + +test('hasNonStaleApproval - no approvals', () => { + const reviews = [ + { state: 'COMMENTED', commit_id: 'abc123' }, + { state: 'CHANGES_REQUESTED', commit_id: 'abc123' } + ]; + assert.strictEqual(hasNonStaleApproval(reviews, 'abc123'), false); +}); + +test('hasNonStaleApproval - stale approval (different SHA)', () => { + const reviews = [ + { state: 'APPROVED', commit_id: 'old123' } + ]; + assert.strictEqual(hasNonStaleApproval(reviews, 'new456'), false); +}); + +test('hasNonStaleApproval - multiple approvals, at least one non-stale', () => { + const reviews = [ + { state: 'APPROVED', commit_id: 'old123' }, + { state: 'APPROVED', commit_id: 'new456' } + ]; + assert.strictEqual(hasNonStaleApproval(reviews, 'new456'), true); +}); + +test('hasNonStaleApproval - empty reviews', () => { + assert.strictEqual(hasNonStaleApproval([], 'abc123'), false); +}); diff --git a/.github/agent-workflow/scripts/lib/commit-validator.js b/.github/agent-workflow/scripts/lib/commit-validator.js new file mode 100644 index 0000000..93eda63 --- /dev/null +++ b/.github/agent-workflow/scripts/lib/commit-validator.js @@ -0,0 +1,26 @@ +/** + * Validate commit message follows conventional commit format + * @param {string} message - Commit message (first line) + * @param {object} options - Validation options + * @param {number} options.maxLength - Maximum subject length (default: 72) + * @returns {boolean} + */ +function isValidCommit(message, options = {}) { + if (!message) return false; + + const { maxLength = 72 } = options; + + // Extract first line (subject) + const subject = message.split('\n')[0]; + + // Check length + if (subject.length > maxLength) return false; + + // Conventional commit format: type(scope)?: description + // Types: feat, fix, docs, style, refactor, test, chore, perf, ci, build, revert + const conventionalRegex = /^(feat|fix|docs|style|refactor|test|chore|perf|ci|build|revert)(\([a-z0-9-]+\))?: .+/; + + return conventionalRegex.test(subject); +} + +module.exports = { isValidCommit }; diff --git a/.github/agent-workflow/scripts/lib/commit-validator.test.js b/.github/agent-workflow/scripts/lib/commit-validator.test.js new file mode 100644 index 0000000..ab75f7e --- /dev/null +++ b/.github/agent-workflow/scripts/lib/commit-validator.test.js @@ -0,0 +1,35 @@ +const { test } = require('node:test'); +const assert = require('node:assert'); +const { isValidCommit } = require('./commit-validator.js'); + +test('isValidCommit - valid conventional commits', () => { + assert.strictEqual(isValidCommit('feat: add new feature'), true); + assert.strictEqual(isValidCommit('fix: resolve bug'), true); + assert.strictEqual(isValidCommit('chore: update dependencies'), true); + assert.strictEqual(isValidCommit('docs: improve README'), true); + assert.strictEqual(isValidCommit('test: add unit tests'), true); + assert.strictEqual(isValidCommit('refactor: simplify logic'), true); +}); + +test('isValidCommit - with scope', () => { + assert.strictEqual(isValidCommit('feat(auth): add OAuth support'), true); + assert.strictEqual(isValidCommit('fix(api): handle edge case'), true); +}); + +test('isValidCommit - with issue reference', () => { + assert.strictEqual(isValidCommit('feat: add feature (#123)'), true); + assert.strictEqual(isValidCommit('fix: resolve issue\n\nFixes #456'), true); +}); + +test('isValidCommit - invalid commits', () => { + assert.strictEqual(isValidCommit('Add new feature'), false); + assert.strictEqual(isValidCommit('FEAT: bad caps'), false); + assert.strictEqual(isValidCommit('feat:missing space'), false); + assert.strictEqual(isValidCommit(''), false); +}); + +test('isValidCommit - maximum length enforcement', () => { + const longMsg = 'feat: ' + 'a'.repeat(100); + assert.strictEqual(isValidCommit(longMsg, { maxLength: 72 }), false); + assert.strictEqual(isValidCommit('feat: short msg', { maxLength: 72 }), true); +}); diff --git a/.github/agent-workflow/scripts/lib/config.js b/.github/agent-workflow/scripts/lib/config.js new file mode 100644 index 0000000..0d53a3e --- /dev/null +++ b/.github/agent-workflow/scripts/lib/config.js @@ -0,0 +1,70 @@ +/** + * Parse guardrail configuration from YAML content + * @param {string} yamlContent - Raw YAML content + * @param {string} guardrailName - Name of the guardrail (e.g., 'scope-enforcement') + * @returns {object} - Config object with enabled, conclusion, and optional threshold + */ +function parseGuardrailConfig(yamlContent, guardrailName) { + const defaults = { + enabled: true, + conclusion: 'action_required' + }; + + if (!yamlContent) return defaults; + + // Simple YAML parsing for the guardrail section + const lines = yamlContent.split('\n'); + let inGuardrails = false; + let inTarget = false; + const config = { ...defaults }; + + for (const line of lines) { + // Check if we're entering the guardrails section + if (/^guardrails:/.test(line)) { + inGuardrails = true; + continue; + } + + // Exit guardrails section if we hit another top-level key + if (inGuardrails && /^\S/.test(line) && !/^\s+/.test(line) && !/^guardrails:/.test(line)) { + break; + } + + // Check if we're entering the target guardrail section + if (inGuardrails && new RegExp(`^\\s+${guardrailName}:`).test(line)) { + inTarget = true; + continue; + } + + // Exit target section if we hit another guardrail key (at same indentation) + if (inTarget && /^\s+\S+:/.test(line) && !new RegExp(`^\\s+${guardrailName}:`).test(line)) { + const currentIndent = line.match(/^(\s+)/)?.[1].length || 0; + const targetIndent = 2; // Assuming standard 2-space YAML indent + if (currentIndent <= targetIndent) { + break; + } + } + + // Parse config values + if (inTarget) { + const enabledMatch = line.match(/^\s+enabled:\s*(true|false)/); + if (enabledMatch) { + config.enabled = enabledMatch[1] === 'true'; + } + + const conclusionMatch = line.match(/^\s+conclusion:\s*(\S+)/); + if (conclusionMatch) { + config.conclusion = conclusionMatch[1]; + } + + const thresholdMatch = line.match(/^\s+threshold:\s*([\d.]+)/); + if (thresholdMatch) { + config.threshold = parseFloat(thresholdMatch[1]); + } + } + } + + return config; +} + +module.exports = { parseGuardrailConfig }; diff --git a/.github/agent-workflow/scripts/lib/config.test.js b/.github/agent-workflow/scripts/lib/config.test.js new file mode 100644 index 0000000..04157d5 --- /dev/null +++ b/.github/agent-workflow/scripts/lib/config.test.js @@ -0,0 +1,62 @@ +const { test } = require('node:test'); +const assert = require('node:assert'); +const { parseGuardrailConfig } = require('./config.js'); + +test('parseGuardrailConfig - parses enabled and conclusion', () => { + const yaml = ` +guardrails: + scope-enforcement: + enabled: true + conclusion: action_required + `; + const config = parseGuardrailConfig(yaml, 'scope-enforcement'); + assert.deepStrictEqual(config, { enabled: true, conclusion: 'action_required' }); +}); + +test('parseGuardrailConfig - disabled guardrail', () => { + const yaml = ` +guardrails: + test-ratio: + enabled: false + conclusion: neutral + `; + const config = parseGuardrailConfig(yaml, 'test-ratio'); + assert.deepStrictEqual(config, { enabled: false, conclusion: 'neutral' }); +}); + +test('parseGuardrailConfig - missing section uses defaults', () => { + const yaml = ` +guardrails: + other-check: + enabled: true + `; + const config = parseGuardrailConfig(yaml, 'scope-enforcement'); + assert.deepStrictEqual(config, { enabled: true, conclusion: 'action_required' }); +}); + +test('parseGuardrailConfig - empty yaml uses defaults', () => { + const config = parseGuardrailConfig('', 'scope-enforcement'); + assert.deepStrictEqual(config, { enabled: true, conclusion: 'action_required' }); +}); + +test('parseGuardrailConfig - partial config uses defaults', () => { + const yaml = ` +guardrails: + scope-enforcement: + enabled: false + `; + const config = parseGuardrailConfig(yaml, 'scope-enforcement'); + assert.deepStrictEqual(config, { enabled: false, conclusion: 'action_required' }); +}); + +test('parseGuardrailConfig - with threshold', () => { + const yaml = ` +guardrails: + test-ratio: + enabled: true + conclusion: action_required + threshold: 0.7 + `; + const config = parseGuardrailConfig(yaml, 'test-ratio'); + assert.deepStrictEqual(config, { enabled: true, conclusion: 'action_required', threshold: 0.7 }); +}); diff --git a/.github/agent-workflow/scripts/lib/file-patterns.js b/.github/agent-workflow/scripts/lib/file-patterns.js new file mode 100644 index 0000000..9b05ed9 --- /dev/null +++ b/.github/agent-workflow/scripts/lib/file-patterns.js @@ -0,0 +1,53 @@ +/** + * Check if a file is a test file + * @param {string} filename + * @returns {boolean} + */ +function isTestFile(filename) { + return /\.(test|spec)\.(js|ts|jsx|tsx|py|go|rs)$/.test(filename) || + /__tests__\//.test(filename) || + /\/tests?\//.test(filename) || + /_test\.(go|rs)$/.test(filename); +} + +/** + * Check if a file is a code file + * @param {string} filename + * @returns {boolean} + */ +function isCodeFile(filename) { + return /\.(js|ts|jsx|tsx|py|go|rs|java|rb|php|c|cpp|h|hpp)$/.test(filename) && + !isTestFile(filename); +} + +/** + * Check if a file is a dependency manifest + * @param {string} filename + * @returns {boolean} + */ +function isDependencyFile(filename) { + const depFiles = [ + 'package.json', + 'package-lock.json', + 'yarn.lock', + 'pnpm-lock.yaml', + 'requirements.txt', + 'Pipfile', + 'Pipfile.lock', + 'go.mod', + 'go.sum', + 'Cargo.toml', + 'Cargo.lock', + 'pom.xml', + 'build.gradle', + 'Gemfile', + 'Gemfile.lock', + 'composer.json', + 'composer.lock' + ]; + + const basename = filename.split('/').pop(); + return depFiles.includes(basename); +} + +module.exports = { isTestFile, isCodeFile, isDependencyFile }; diff --git a/.github/agent-workflow/scripts/lib/file-patterns.test.js b/.github/agent-workflow/scripts/lib/file-patterns.test.js new file mode 100644 index 0000000..31e235f --- /dev/null +++ b/.github/agent-workflow/scripts/lib/file-patterns.test.js @@ -0,0 +1,51 @@ +const { test } = require('node:test'); +const assert = require('node:assert'); +const { isTestFile, isCodeFile, isDependencyFile } = require('./file-patterns.js'); + +test('isTestFile - recognizes test files', () => { + assert.strictEqual(isTestFile('src/foo.test.js'), true); + assert.strictEqual(isTestFile('src/foo.spec.ts'), true); + assert.strictEqual(isTestFile('__tests__/foo.js'), true); + assert.strictEqual(isTestFile('tests/integration/bar.py'), true); + assert.strictEqual(isTestFile('lib/foo_test.go'), true); +}); + +test('isTestFile - rejects non-test files', () => { + assert.strictEqual(isTestFile('src/index.js'), false); + assert.strictEqual(isTestFile('lib/utils.ts'), false); + assert.strictEqual(isTestFile('README.md'), false); +}); + +test('isCodeFile - recognizes code files', () => { + assert.strictEqual(isCodeFile('src/index.js'), true); + assert.strictEqual(isCodeFile('lib/utils.ts'), true); + assert.strictEqual(isCodeFile('app/main.py'), true); + assert.strictEqual(isCodeFile('cmd/server.go'), true); + assert.strictEqual(isCodeFile('pkg/auth/handler.rs'), true); +}); + +test('isCodeFile - rejects non-code files', () => { + assert.strictEqual(isCodeFile('README.md'), false); + assert.strictEqual(isCodeFile('package.json'), false); + assert.strictEqual(isCodeFile('.gitignore'), false); + assert.strictEqual(isCodeFile('docs/guide.txt'), false); +}); + +test('isDependencyFile - recognizes dependency files', () => { + assert.strictEqual(isDependencyFile('package.json'), true); + assert.strictEqual(isDependencyFile('package-lock.json'), true); + assert.strictEqual(isDependencyFile('requirements.txt'), true); + assert.strictEqual(isDependencyFile('go.mod'), true); + assert.strictEqual(isDependencyFile('go.sum'), true); + assert.strictEqual(isDependencyFile('Cargo.toml'), true); + assert.strictEqual(isDependencyFile('Cargo.lock'), true); + assert.strictEqual(isDependencyFile('pom.xml'), true); + assert.strictEqual(isDependencyFile('Gemfile'), true); + assert.strictEqual(isDependencyFile('Gemfile.lock'), true); +}); + +test('isDependencyFile - rejects non-dependency files', () => { + assert.strictEqual(isDependencyFile('src/index.js'), false); + assert.strictEqual(isDependencyFile('README.md'), false); + assert.strictEqual(isDependencyFile('config.json'), false); +}); diff --git a/.github/agent-workflow/scripts/lib/fixes-parser.js b/.github/agent-workflow/scripts/lib/fixes-parser.js new file mode 100644 index 0000000..d13e9a8 --- /dev/null +++ b/.github/agent-workflow/scripts/lib/fixes-parser.js @@ -0,0 +1,25 @@ +/** + * Parse "Fixes #N" references from PR body + * @param {string} body - PR body text + * @returns {number[]} - Array of issue numbers + */ +function parseFixesReferences(body) { + if (!body) return []; + + const regex = /[Ff]ixes\s+#(\d+)/g; + const matches = []; + const seen = new Set(); + + let match; + while ((match = regex.exec(body)) !== null) { + const issueNum = parseInt(match[1], 10); + if (!seen.has(issueNum)) { + matches.push(issueNum); + seen.add(issueNum); + } + } + + return matches; +} + +module.exports = { parseFixesReferences }; diff --git a/.github/agent-workflow/scripts/lib/fixes-parser.test.js b/.github/agent-workflow/scripts/lib/fixes-parser.test.js new file mode 100644 index 0000000..ce6a1f6 --- /dev/null +++ b/.github/agent-workflow/scripts/lib/fixes-parser.test.js @@ -0,0 +1,39 @@ +const { test } = require('node:test'); +const assert = require('node:assert'); +const { parseFixesReferences } = require('./fixes-parser.js'); + +test('parseFixesReferences - single fixes reference', () => { + const body = 'This PR fixes #123'; + const result = parseFixesReferences(body); + assert.deepStrictEqual(result, [123]); +}); + +test('parseFixesReferences - multiple fixes references', () => { + const body = 'Fixes #10\nFixes #20\nFixes #30'; + const result = parseFixesReferences(body); + assert.deepStrictEqual(result, [10, 20, 30]); +}); + +test('parseFixesReferences - case insensitive', () => { + const body = 'fixes #1\nFIXES #2\nFiXeS #3'; + const result = parseFixesReferences(body); + assert.deepStrictEqual(result, [1, 2, 3]); +}); + +test('parseFixesReferences - no matches', () => { + const body = 'This PR does not fix anything'; + const result = parseFixesReferences(body); + assert.deepStrictEqual(result, []); +}); + +test('parseFixesReferences - null or undefined body', () => { + assert.deepStrictEqual(parseFixesReferences(null), []); + assert.deepStrictEqual(parseFixesReferences(undefined), []); + assert.deepStrictEqual(parseFixesReferences(''), []); +}); + +test('parseFixesReferences - deduplicates issue numbers', () => { + const body = 'Fixes #10\nFixes #10\nFixes #20'; + const result = parseFixesReferences(body); + assert.deepStrictEqual(result, [10, 20]); +}); diff --git a/.github/agent-workflow/scripts/lib/patch-parser.js b/.github/agent-workflow/scripts/lib/patch-parser.js new file mode 100644 index 0000000..4f8bb82 --- /dev/null +++ b/.github/agent-workflow/scripts/lib/patch-parser.js @@ -0,0 +1,42 @@ +/** + * Parse line numbers of added/modified lines from a unified diff patch + * @param {string} patch - Unified diff patch content + * @returns {number[]} - Array of line numbers that were added or modified + */ +function parseLineNumbers(patch) { + if (!patch) return []; + + const lines = patch.split('\n'); + const lineNumbers = []; + let currentLine = 0; + + for (const line of lines) { + // Parse hunk header: @@ -old_start,old_count +new_start,new_count @@ + const hunkMatch = line.match(/^@@\s+-\d+(?:,\d+)?\s+\+(\d+)(?:,\d+)?\s+@@/); + if (hunkMatch) { + currentLine = parseInt(hunkMatch[1], 10); + continue; + } + + // Skip if we haven't seen a hunk header yet + if (currentLine === 0) continue; + + // Added line (+) + if (line.startsWith('+') && !line.startsWith('+++')) { + lineNumbers.push(currentLine); + currentLine++; + } + // Context line (space) or modified line + else if (line.startsWith(' ')) { + currentLine++; + } + // Deleted line (-) - don't increment current line in new file + else if (line.startsWith('-') && !line.startsWith('---')) { + // Don't increment - this line doesn't exist in new version + } + } + + return lineNumbers; +} + +module.exports = { parseLineNumbers }; diff --git a/.github/agent-workflow/scripts/lib/patch-parser.test.js b/.github/agent-workflow/scripts/lib/patch-parser.test.js new file mode 100644 index 0000000..54e9bfe --- /dev/null +++ b/.github/agent-workflow/scripts/lib/patch-parser.test.js @@ -0,0 +1,46 @@ +const { test } = require('node:test'); +const assert = require('node:assert'); +const { parseLineNumbers } = require('./patch-parser.js'); + +test('parseLineNumbers - simple addition', () => { + const patch = ` +@@ -10,5 +10,6 @@ + line 10 + line 11 ++added line 12 + line 13 + `; + const lines = parseLineNumbers(patch); + assert.ok(lines.includes(12)); +}); + +test('parseLineNumbers - multiple hunks', () => { + const patch = ` +@@ -10,3 +10,4 @@ + line 10 ++added line 11 + line 12 +@@ -20,2 +21,3 @@ + line 21 ++added line 22 + `; + const lines = parseLineNumbers(patch); + assert.ok(lines.includes(11)); + assert.ok(lines.includes(22)); +}); + +test('parseLineNumbers - deletions not included', () => { + const patch = ` +@@ -10,4 +10,3 @@ + line 10 +-deleted line 11 + line 12 + `; + const lines = parseLineNumbers(patch); + assert.ok(!lines.includes(11)); +}); + +test('parseLineNumbers - empty patch', () => { + const lines = parseLineNumbers(''); + assert.deepStrictEqual(lines, []); +}); diff --git a/.github/agent-workflow/scripts/lib/pr-body.js b/.github/agent-workflow/scripts/lib/pr-body.js new file mode 100644 index 0000000..5157b72 --- /dev/null +++ b/.github/agent-workflow/scripts/lib/pr-body.js @@ -0,0 +1,31 @@ +/** + * Replace or add a section in PR body + * @param {string} body - Current PR body + * @param {string} sectionHeader - Section header (e.g., '## Fixes') + * @param {string} newContent - New content for the section + * @returns {string} - Updated PR body + */ +function replaceSection(body, sectionHeader, newContent) { + if (!body) { + return `${sectionHeader}\n${newContent}\n`; + } + + // Escape special regex characters in header + const escapedHeader = sectionHeader.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + + // Match the section from header to next ## or end of string + const sectionRegex = new RegExp( + `(${escapedHeader}\\n)[\\s\\S]*?(?=\\n## |$)`, + 'i' + ); + + if (sectionRegex.test(body)) { + // Replace existing section + return body.replace(sectionRegex, `$1${newContent}\n`); + } else { + // Append new section + return `${body.trim()}\n\n${sectionHeader}\n${newContent}\n`; + } +} + +module.exports = { replaceSection }; diff --git a/.github/agent-workflow/scripts/lib/pr-body.test.js b/.github/agent-workflow/scripts/lib/pr-body.test.js new file mode 100644 index 0000000..d9fae7c --- /dev/null +++ b/.github/agent-workflow/scripts/lib/pr-body.test.js @@ -0,0 +1,36 @@ +const { test } = require('node:test'); +const assert = require('node:assert'); +const { replaceSection } = require('./pr-body.js'); + +test('replaceSection - adds new section to empty body', () => { + const result = replaceSection('', '## Fixes', 'Fixes #123'); + assert.ok(result.includes('## Fixes')); + assert.ok(result.includes('Fixes #123')); +}); + +test('replaceSection - replaces existing section', () => { + const body = '## Summary\nSome text\n\n## Fixes\nFixes #100\n\n## Other\nMore text'; + const result = replaceSection(body, '## Fixes', 'Fixes #200\nFixes #201'); + assert.ok(result.includes('Fixes #200')); + assert.ok(result.includes('Fixes #201')); + assert.ok(!result.includes('Fixes #100')); + assert.ok(result.includes('## Summary')); + assert.ok(result.includes('## Other')); +}); + +test('replaceSection - preserves other sections', () => { + const body = '## Summary\nOriginal\n\n## Test Plan\nOriginal test plan'; + const result = replaceSection(body, '## Fixes', 'Fixes #999'); + assert.ok(result.includes('## Summary')); + assert.ok(result.includes('Original')); + assert.ok(result.includes('## Test Plan')); + assert.ok(result.includes('Original test plan')); + assert.ok(result.includes('Fixes #999')); +}); + +test('replaceSection - idempotent updates', () => { + const body = '## Summary\nText\n\n## Fixes\nFixes #123'; + const result1 = replaceSection(body, '## Fixes', 'Fixes #123'); + const result2 = replaceSection(result1, '## Fixes', 'Fixes #123'); + assert.strictEqual(result1, result2); +}); diff --git a/.github/agent-workflow/scripts/lib/scope-matcher.js b/.github/agent-workflow/scripts/lib/scope-matcher.js new file mode 100644 index 0000000..d72a161 --- /dev/null +++ b/.github/agent-workflow/scripts/lib/scope-matcher.js @@ -0,0 +1,50 @@ +/** + * Extract file paths from issue text + * @param {string} text - Issue body or comment text + * @returns {string[]} - Array of unique file paths + */ +function extractFilePaths(text) { + if (!text) return []; + + const filePathPatterns = [ + // Backtick-wrapped paths with extension + /`([a-zA-Z0-9_./-]+\.[a-zA-Z0-9]+)`/g, + // Bare paths with at least one slash and an extension + /(?:^|\s)((?:[a-zA-Z0-9_.-]+\/)+[a-zA-Z0-9_.-]+\.[a-zA-Z0-9]+)(?:\s|$|[,;)])/gm, + // Paths starting with ./ or common root dirs + /(?:^|\s)(\.?(?:src|lib|app|test|tests|spec|pkg|cmd|internal|\.github)\/[a-zA-Z0-9_./-]+)(?:\s|$|[,;)])/gm + ]; + + const paths = new Set(); + for (const pattern of filePathPatterns) { + pattern.lastIndex = 0; + let match; + while ((match = pattern.exec(text)) !== null) { + const filePath = match[1].replace(/^\//, ''); // strip leading slash + paths.add(filePath); + } + } + + return Array.from(paths); +} + +/** + * Check if a changed file path is within scope + * @param {string} changedPath - Path of changed file + * @param {string[]} scopeFiles - Array of scope file paths/prefixes + * @returns {boolean} + */ +function isInScope(changedPath, scopeFiles) { + for (const scopePath of scopeFiles) { + // Exact match + if (changedPath === scopePath) return true; + + // Scope entry is a directory prefix + const prefix = scopePath.endsWith('/') ? scopePath : scopePath + '/'; + if (changedPath.startsWith(prefix)) return true; + } + + return false; +} + +module.exports = { extractFilePaths, isInScope }; diff --git a/.github/agent-workflow/scripts/lib/scope-matcher.test.js b/.github/agent-workflow/scripts/lib/scope-matcher.test.js new file mode 100644 index 0000000..dd7d234 --- /dev/null +++ b/.github/agent-workflow/scripts/lib/scope-matcher.test.js @@ -0,0 +1,57 @@ +const { test } = require('node:test'); +const assert = require('node:assert'); +const { extractFilePaths, isInScope } = require('./scope-matcher.js'); + +test('extractFilePaths - backtick-wrapped paths', () => { + const text = 'Modify `src/index.js` and `lib/utils.ts`'; + const paths = extractFilePaths(text); + assert.deepStrictEqual(paths, ['src/index.js', 'lib/utils.ts']); +}); + +test('extractFilePaths - bare paths with slashes', () => { + const text = 'Files: src/app/main.py and lib/helper.go'; + const paths = extractFilePaths(text); + assert.ok(paths.includes('src/app/main.py')); + assert.ok(paths.includes('lib/helper.go')); +}); + +test('extractFilePaths - common root directories', () => { + const text = 'Update src/index.js, test/unit.js, and .github/workflows/ci.yml'; + const paths = extractFilePaths(text); + assert.ok(paths.includes('src/index.js')); + assert.ok(paths.includes('test/unit.js')); + assert.ok(paths.includes('.github/workflows/ci.yml')); +}); + +test('extractFilePaths - no duplicates', () => { + const text = 'File `src/index.js` and src/index.js again'; + const paths = extractFilePaths(text); + assert.strictEqual(paths.filter(p => p === 'src/index.js').length, 1); +}); + +test('extractFilePaths - empty text', () => { + assert.deepStrictEqual(extractFilePaths(''), []); + assert.deepStrictEqual(extractFilePaths(null), []); + assert.deepStrictEqual(extractFilePaths(undefined), []); +}); + +test('isInScope - exact match', () => { + const scopeFiles = ['src/index.js', 'lib/utils.ts']; + assert.strictEqual(isInScope('src/index.js', scopeFiles), true); + assert.strictEqual(isInScope('lib/utils.ts', scopeFiles), true); + assert.strictEqual(isInScope('other/file.js', scopeFiles), false); +}); + +test('isInScope - directory prefix', () => { + const scopeFiles = ['src/', 'lib/auth/']; + assert.strictEqual(isInScope('src/index.js', scopeFiles), true); + assert.strictEqual(isInScope('src/app/main.js', scopeFiles), true); + assert.strictEqual(isInScope('lib/auth/handler.js', scopeFiles), true); + assert.strictEqual(isInScope('lib/other/file.js', scopeFiles), false); +}); + +test('isInScope - prefix without trailing slash', () => { + const scopeFiles = ['src/auth']; + assert.strictEqual(isInScope('src/auth/handler.js', scopeFiles), true); + assert.strictEqual(isInScope('src/auth.ts', scopeFiles), false); +}); diff --git a/.github/agent-workflow/scripts/lib/severity.js b/.github/agent-workflow/scripts/lib/severity.js new file mode 100644 index 0000000..ed264d2 --- /dev/null +++ b/.github/agent-workflow/scripts/lib/severity.js @@ -0,0 +1,40 @@ +/** + * Detect severity level from review comment text + * @param {string} comment - Review comment text + * @returns {string} - 'blocking', 'should-fix', or 'suggestion' + */ +function detectSeverity(comment) { + if (!comment) return 'should-fix'; + + const lower = comment.toLowerCase(); + + // Blocking keywords + const blockingPatterns = [ + /\bblocking\b/, + /\bcritical\b/, + /\bmust\s+(?:be\s+)?fix/, + /\bmust\s+(?:be\s+)?change/ + ]; + + for (const pattern of blockingPatterns) { + if (pattern.test(lower)) return 'blocking'; + } + + // Suggestion keywords + const suggestionPatterns = [ + /^nit:/, + /\bnit\b/, + /\boptional\b/, + /\bconsider\b/, + /\bsuggestion\b/, + /\bminor\b/ + ]; + + for (const pattern of suggestionPatterns) { + if (pattern.test(lower)) return 'suggestion'; + } + + return 'should-fix'; +} + +module.exports = { detectSeverity }; diff --git a/.github/agent-workflow/scripts/lib/severity.test.js b/.github/agent-workflow/scripts/lib/severity.test.js new file mode 100644 index 0000000..9c3029f --- /dev/null +++ b/.github/agent-workflow/scripts/lib/severity.test.js @@ -0,0 +1,28 @@ +const { test } = require('node:test'); +const assert = require('node:assert'); +const { detectSeverity } = require('./severity.js'); + +test('detectSeverity - blocking keywords', () => { + assert.strictEqual(detectSeverity('This is blocking the release'), 'blocking'); + assert.strictEqual(detectSeverity('BLOCKING: critical issue'), 'blocking'); + assert.strictEqual(detectSeverity('This must be fixed'), 'blocking'); + assert.strictEqual(detectSeverity('critical bug here'), 'blocking'); +}); + +test('detectSeverity - suggestion keywords', () => { + assert.strictEqual(detectSeverity('Consider refactoring this'), 'suggestion'); + assert.strictEqual(detectSeverity('nit: extra space'), 'suggestion'); + assert.strictEqual(detectSeverity('optional: could improve'), 'suggestion'); + assert.strictEqual(detectSeverity('minor: formatting'), 'suggestion'); +}); + +test('detectSeverity - default should-fix', () => { + assert.strictEqual(detectSeverity('This needs to be fixed'), 'should-fix'); + assert.strictEqual(detectSeverity('Bug in the implementation'), 'should-fix'); + assert.strictEqual(detectSeverity('Random comment'), 'should-fix'); +}); + +test('detectSeverity - case insensitive', () => { + assert.strictEqual(detectSeverity('BLOCKING issue'), 'blocking'); + assert.strictEqual(detectSeverity('Nit: formatting'), 'suggestion'); +}); diff --git a/.github/agent-workflow/scripts/orchestrator-check.js b/.github/agent-workflow/scripts/orchestrator-check.js new file mode 100644 index 0000000..efa09b4 --- /dev/null +++ b/.github/agent-workflow/scripts/orchestrator-check.js @@ -0,0 +1,383 @@ +const { parseFixesReferences } = require('./lib/fixes-parser.js'); + +module.exports = async function({ github, context, core }) { + const checkName = 'orchestrator'; + + // Helper: read re-review-cycle-cap from config + async function readCycleCap() { + try { + const { data } = await github.rest.repos.getContent({ + owner: context.repo.owner, + repo: context.repo.repo, + path: '.github/agent-workflow/config.yaml', + ref: context.sha + }); + const content = Buffer.from(data.content, 'base64').toString('utf-8'); + const match = content.match(/^re-review-cycle-cap:\s*(\d+)/m); + return match ? parseInt(match[1], 10) : 3; + } catch { + return 3; // default + } + } + + // Helper: create check run on a specific SHA + async function createCheckRun(headSha, conclusion, title, summary) { + await github.rest.checks.create({ + owner: context.repo.owner, + repo: context.repo.repo, + head_sha: headSha, + name: checkName, + status: 'completed', + conclusion, + output: { title, summary } + }); + } + + // Helper: find PRs referencing a given issue + async function findPRsForIssue(issueNumber) { + const prs = []; + let page = 1; + while (true) { + const resp = await github.rest.pulls.list({ + owner: context.repo.owner, + repo: context.repo.repo, + state: 'open', + per_page: 100, + page + }); + if (resp.data.length === 0) break; + for (const pr of resp.data) { + const body = pr.body || ''; + const regex = new RegExp(`[Ff]ixes\\s*#${issueNumber}\\b`); + if (regex.test(body)) { + prs.push(pr); + } + } + if (resp.data.length < 100) break; + page++; + } + return prs; + } + + // Helper: query sub-issues via GraphQL + async function getSubIssues(issueNumber) { + const query = ` + query($owner: String!, $repo: String!, $number: Int!) { + repository(owner: $owner, name: $repo) { + issue(number: $number) { + subIssues(first: 50) { + nodes { + number + title + state + labels(first: 10) { nodes { name } } + } + } + } + } + } + `; + try { + const result = await github.graphql(query, { + owner: context.repo.owner, + repo: context.repo.repo, + number: issueNumber + }); + return result.repository.issue.subIssues.nodes; + } catch (err) { + console.log(`Warning: GraphQL sub-issues query failed: ${err.message}`); + return []; + } + } + + // Helper: count past review cycles from PR comments + async function countReviewCycles(prNumber) { + const comments = await github.paginate( + github.rest.issues.listComments, + { + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: prNumber, + per_page: 100 + } + ); + return comments.filter( + c => c.body && c.body.includes('') + ).length; + } + + // Step 1: Determine the parent issue and PR + let parentIssueNumber; + let prNumber; + let headSha; + + if (context.eventName === 'pull_request') { + // PR synchronize event — parse parent issue from PR body + const pr = context.payload.pull_request; + prNumber = pr.number; + headSha = pr.head.sha; + const issueNumbers = parseFixesReferences(pr.body); + if (issueNumbers.length === 0) { + console.log('No "Fixes #N" in PR body. Skipping orchestrator check.'); + await createCheckRun( + headSha, + 'neutral', + 'Orchestrator: no linked issue', + 'No `Fixes #N` reference found in PR description. Orchestrator check skipped.' + ); + return; + } + parentIssueNumber = issueNumbers[0]; + + } else if (context.eventName === 'issues') { + // Issue event — find the PR that references this issue's parent + const changedIssue = context.payload.issue; + + const openPRs = []; + let page = 1; + while (true) { + const resp = await github.rest.pulls.list({ + owner: context.repo.owner, + repo: context.repo.repo, + state: 'open', + per_page: 100, + page + }); + if (resp.data.length === 0) break; + openPRs.push(...resp.data); + if (resp.data.length < 100) break; + page++; + } + + // Build a map of parent issue number -> PR data + const parentToPR = new Map(); + for (const pr of openPRs) { + const issueNumbers = parseFixesReferences(pr.body); + if (issueNumbers.length > 0) { + parentToPR.set(issueNumbers[0], pr); + } + } + + if (parentToPR.size === 0) { + console.log('No open PRs with "Fixes #N" references found. Nothing to do.'); + return; + } + + // Check if the changed issue IS a parent issue referenced by a PR + if (parentToPR.has(changedIssue.number)) { + parentIssueNumber = changedIssue.number; + const pr = parentToPR.get(changedIssue.number); + prNumber = pr.number; + headSha = pr.head.sha; + } else { + // Check if the changed issue is a sub-issue of any parent + let found = false; + for (const [parentNum, pr] of parentToPR.entries()) { + const subIssues = await getSubIssues(parentNum); + const isChild = subIssues.some( + si => si.number === changedIssue.number + ); + if (isChild) { + parentIssueNumber = parentNum; + prNumber = pr.number; + headSha = pr.head.sha; + found = true; + break; + } + } + if (!found) { + console.log( + `Issue #${changedIssue.number} is not a sub-issue of any PR-linked parent. Nothing to do.` + ); + return; + } + } + } else { + console.log(`Unexpected event: ${context.eventName}. Skipping.`); + return; + } + + console.log(`Parent issue: #${parentIssueNumber}, PR: #${prNumber}, HEAD: ${headSha}`); + + // Step 2: Query sub-issues and check for blockers + const subIssues = await getSubIssues(parentIssueNumber); + console.log(`Found ${subIssues.length} sub-issues for #${parentIssueNumber}`); + + const openBlockers = subIssues.filter(si => { + if (si.state !== 'OPEN') return false; + const labels = si.labels.nodes.map(l => l.name); + return labels.includes('blocking'); + }); + + // Step 3: If blockers exist, report failing check + if (openBlockers.length > 0) { + const blockerList = openBlockers + .map(si => `- #${si.number}: ${si.title}`) + .join('\n'); + + await createCheckRun( + headSha, + 'action_required', + `Orchestrator: ${openBlockers.length} blocking issue(s)`, + [ + `PR #${prNumber} cannot merge. Issue #${parentIssueNumber} has open blocking sub-issues:`, + '', + blockerList, + '', + 'Resolve these blocking issues or approve the PR to override.' + ].join('\n') + ); + console.log(`Reported failing check: ${openBlockers.length} blockers.`); + return; + } + + // Step 4: No blockers — assess re-review need + console.log('No open blockers. Assessing re-review need...'); + + const cycleCap = await readCycleCap(); + const pastCycles = await countReviewCycles(prNumber); + console.log(`Review cycles so far: ${pastCycles}, cap: ${cycleCap}`); + + // If we've reached the cycle cap, pass without re-review + if (pastCycles >= cycleCap) { + await createCheckRun( + headSha, + 'success', + 'Orchestrator: passing (review cycle cap reached)', + [ + `All blocking sub-issues for #${parentIssueNumber} are resolved.`, + '', + `Re-review cycle cap reached (${pastCycles}/${cycleCap}). No further re-review.`, + 'PR is clear to merge.' + ].join('\n') + ); + console.log('Cycle cap reached. Reporting success.'); + return; + } + + // Assess whether re-review is needed + let needsReReview = false; + + try { + const { data: prData } = await github.rest.pulls.get({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: prNumber + }); + const baseBranch = prData.base.ref; + + const { data: files } = await github.rest.pulls.listFiles({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: prNumber, + per_page: 100 + }); + + const totalChanges = files.reduce( + (sum, f) => sum + f.additions + f.deletions, 0 + ); + const fileCount = files.length; + const fileList = files + .map(f => `${f.filename} (+${f.additions}/-${f.deletions})`) + .join('\n'); + + if (totalChanges === 0) { + console.log('No changes in PR. Skipping re-review assessment.'); + needsReReview = false; + } else { + const assessPrompt = [ + 'You are assessing whether a PR needs re-review after changes were made to address review findings.', + '', + `PR #${prNumber} targets branch "${baseBranch}" and fixes issue #${parentIssueNumber}.`, + `Past review cycles: ${pastCycles}`, + '', + `The PR modifies ${fileCount} files with ${totalChanges} total line changes:`, + fileList, + '', + 'Based on the scope and nature of these changes, should reviewers re-review this PR?', + 'Consider: Are the changes small and surgical (e.g., null checks, single test additions)?', + 'Or are they broad and structural (e.g., new modules, architectural changes, many files)?', + '', + 'Respond with ONLY one word: YES or NO', + ].join('\n'); + + const { execSync } = require('child_process'); + try { + const result = execSync( + `claude -p "${assessPrompt.replace(/"/g, '\\"')}"`, + { encoding: 'utf-8', timeout: 60000 } + ).trim(); + + console.log(`Claude assessment result: ${result}`); + needsReReview = /\bYES\b/i.test(result); + } catch (claudeErr) { + console.log(`Claude assessment failed: ${claudeErr.message}`); + needsReReview = false; + } + } + } catch (err) { + console.log(`Error during re-review assessment: ${err.message}`); + needsReReview = false; + } + + // Step 5: Trigger re-review or report passing + if (needsReReview) { + console.log('Re-review warranted. Triggering PR Review workflow...'); + + await github.rest.issues.createComment({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: prNumber, + body: [ + '', + `**Orchestrator:** Triggering re-review (cycle ${pastCycles + 1}/${cycleCap}).`, + '', + 'Changes since last review warrant another review pass.' + ].join('\n') + }); + + try { + await github.rest.actions.createWorkflowDispatch({ + owner: context.repo.owner, + repo: context.repo.repo, + workflow_id: 'pr-review.yml', + ref: context.ref || 'main', + inputs: { + 'pr-number': prNumber.toString() + } + }); + console.log('PR Review workflow triggered successfully.'); + } catch (dispatchErr) { + console.log(`Warning: Could not trigger PR Review workflow: ${dispatchErr.message}`); + } + + await createCheckRun( + headSha, + 'neutral', + `Orchestrator: re-review triggered (cycle ${pastCycles + 1}/${cycleCap})`, + [ + `All blocking sub-issues for #${parentIssueNumber} are resolved.`, + '', + `Changes since last review warrant re-review. Cycle ${pastCycles + 1} of ${cycleCap} triggered.`, + 'The PR Review workflow has been dispatched. Orchestrator will re-evaluate after review completes.' + ].join('\n') + ); + } else { + console.log('No re-review needed. Reporting success.'); + + await createCheckRun( + headSha, + 'success', + 'Orchestrator: all clear', + [ + `All blocking sub-issues for #${parentIssueNumber} are resolved.`, + '', + pastCycles > 0 + ? `No further re-review needed after ${pastCycles} review cycle(s).` + : 'No re-review warranted based on change assessment.', + '', + 'PR is clear to merge.' + ].join('\n') + ); + } +}; diff --git a/.github/agent-workflow/scripts/pr-context.js b/.github/agent-workflow/scripts/pr-context.js new file mode 100644 index 0000000..58b1d56 --- /dev/null +++ b/.github/agent-workflow/scripts/pr-context.js @@ -0,0 +1,34 @@ +const { parseFixesReferences } = require('./lib/fixes-parser.js'); + +module.exports = async function({ github, context, core }) { + let prNumber; + let prData; + + if (context.eventName === 'workflow_dispatch') { + // workflow_dispatch input is passed via environment variable + prNumber = parseInt(process.env.PR_NUMBER, 10); + const { data } = await github.rest.pulls.get({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: prNumber, + }); + prData = data; + } else { + prNumber = context.payload.pull_request.number; + prData = context.payload.pull_request; + } + + // Parse "fixes #N" or "Fixes #N" from PR body + const body = prData.body || ''; + const issueNumbers = parseFixesReferences(body); + const parentIssue = issueNumbers.length > 0 ? issueNumbers[0].toString() : ''; + + if (!parentIssue) { + console.log('No parent issue found — PR body has no "Fixes #N" reference.'); + } + + core.setOutput('pr-number', prNumber.toString()); + core.setOutput('parent-issue', parentIssue); + core.setOutput('base-branch', prData.base.ref); + core.setOutput('pr-title', prData.title); +}; diff --git a/.github/workflows/guardrail-scope.yml b/.github/workflows/guardrail-scope.yml index fe488d8..2d4992f 100644 --- a/.github/workflows/guardrail-scope.yml +++ b/.github/workflows/guardrail-scope.yml @@ -23,262 +23,5 @@ jobs: uses: actions/github-script@v7 with: script: | - const checkName = 'guardrail/scope'; - - // --- Helper: read config --- - async function readConfig() { - try { - const { data } = await github.rest.repos.getContent({ - owner: context.repo.owner, - repo: context.repo.repo, - path: '.github/agent-workflow/config.yaml', - ref: context.payload.pull_request.head.sha - }); - const content = Buffer.from(data.content, 'base64').toString('utf-8'); - // Simple YAML parsing for the scope-enforcement section - const lines = content.split('\n'); - let inScope = false; - const config = { enabled: true, conclusion: 'action_required' }; - for (const line of lines) { - if (/^scope-enforcement:/.test(line)) { - inScope = true; - continue; - } - if (inScope && /^\S/.test(line)) { - break; // Next top-level key - } - if (inScope) { - const enabledMatch = line.match(/^\s+enabled:\s*(true|false)/); - if (enabledMatch) config.enabled = enabledMatch[1] === 'true'; - const conclusionMatch = line.match(/^\s+conclusion:\s*(\S+)/); - if (conclusionMatch) config.conclusion = conclusionMatch[1]; - } - } - return config; - } catch (e) { - // Config file not found — use defaults (enabled) - return { enabled: true, conclusion: 'action_required' }; - } - } - - // --- Helper: create check run --- - async function createCheckRun(conclusion, title, summary, annotations = []) { - const output = { title, summary }; - if (annotations.length > 0) { - // GitHub API limits to 50 annotations per request - output.annotations = annotations.slice(0, 50); - } - await github.rest.checks.create({ - owner: context.repo.owner, - repo: context.repo.repo, - head_sha: context.payload.pull_request.head.sha, - name: checkName, - status: 'completed', - conclusion, - output - }); - } - - // --- Step 1: Read config and check if enabled --- - const config = await readConfig(); - if (!config.enabled) { - await createCheckRun( - 'success', - 'Scope enforcement: disabled', - 'Scope enforcement is disabled in agent-workflow config.' - ); - return; - } - - // --- Step 2: Check for non-stale PR approval override --- - const reviews = await github.rest.pulls.listReviews({ - owner: context.repo.owner, - repo: context.repo.repo, - pull_number: context.payload.pull_request.number - }); - const headSha = context.payload.pull_request.head.sha; - const hasValidApproval = reviews.data.some( - r => r.state === 'APPROVED' && r.commit_id === headSha - ); - - // --- Step 3: Parse issue reference from PR body --- - const prBody = context.payload.pull_request.body || ''; - const issueMatch = prBody.match(/[Ff]ixes\s+#(\d+)/); - if (!issueMatch) { - // No linked issue — cannot enforce scope, pass with note - await createCheckRun( - 'success', - 'Scope enforcement: no linked issue', - 'No `fixes #N` reference found in PR description. Scope enforcement skipped.' - ); - return; - } - const issueNumber = parseInt(issueMatch[1], 10); - - // --- Step 4: Get the issue body + all child issue bodies --- - // File paths may be listed in the parent issue, its child issues, or both. - // Collect bodies from the parent and all sub-issues. - const issueBodies = []; - try { - const issue = await github.rest.issues.get({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: issueNumber - }); - issueBodies.push(issue.data.body || ''); - } catch (e) { - await createCheckRun( - 'success', - 'Scope enforcement: issue not found', - `Could not read issue #${issueNumber}. Scope enforcement skipped.` - ); - return; - } - - // Query sub-issues via GraphQL - try { - const subIssueResult = await github.graphql(` - query($owner: String!, $repo: String!, $number: Int!) { - repository(owner: $owner, name: $repo) { - issue(number: $number) { - subIssues(first: 50) { - nodes { body } - } - } - } - } - `, { - owner: context.repo.owner, - repo: context.repo.repo, - number: issueNumber - }); - const children = subIssueResult.repository.issue.subIssues.nodes || []; - for (const child of children) { - if (child.body) issueBodies.push(child.body); - } - } catch (e) { - // Sub-issues query failed — continue with parent body only - } - - // --- Step 5: Extract file paths from all issue bodies --- - // Match paths that look like file paths: - // - backtick-wrapped paths: `src/foo/bar.ts` - // - paths with extensions: src/foo/bar.ts - // - paths in markdown code blocks - const filePathPatterns = [ - // Backtick-wrapped paths with extension - /`([a-zA-Z0-9_./-]+\.[a-zA-Z0-9]+)`/g, - // Bare paths with at least one slash and an extension (common in issue descriptions) - /(?:^|\s)((?:[a-zA-Z0-9_.-]+\/)+[a-zA-Z0-9_.-]+\.[a-zA-Z0-9]+)(?:\s|$|[,;)])/gm, - // Paths starting with ./ or common root dirs - /(?:^|\s)(\.?(?:src|lib|app|test|tests|spec|pkg|cmd|internal|\.github)\/[a-zA-Z0-9_./-]+)(?:\s|$|[,;)])/gm - ]; - - const scopeFiles = new Set(); - for (const body of issueBodies) { - for (const pattern of filePathPatterns) { - pattern.lastIndex = 0; // reset regex state for each body - let match; - while ((match = pattern.exec(body)) !== null) { - const filePath = match[1].replace(/^\//, ''); // strip leading slash - scopeFiles.add(filePath); - } - } - } - - if (scopeFiles.size === 0) { - // No file paths found in any issue — cannot enforce scope - await createCheckRun( - 'success', - 'Scope enforcement: no files listed in issues', - `Issue #${issueNumber} and its children do not list any file paths. Scope enforcement skipped.` - ); - return; - } - - // --- Step 6: Get changed files from the PR --- - const changedFiles = []; - let page = 1; - while (true) { - const resp = await github.rest.pulls.listFiles({ - owner: context.repo.owner, - repo: context.repo.repo, - pull_number: context.payload.pull_request.number, - per_page: 100, - page - }); - changedFiles.push(...resp.data); - if (resp.data.length < 100) break; - page++; - } - - // --- Step 7: Compare changed files against scope --- - // A changed file is "in scope" if: - // - It exactly matches a scope file, OR - // - It is inside a directory listed in scope, OR - // - A scope file is a prefix of the changed file path - function isInScope(changedPath) { - for (const scopePath of scopeFiles) { - // Exact match - if (changedPath === scopePath) return true; - // Scope entry is a directory prefix (e.g., scope lists "src/auth/", - // changed file is "src/auth/middleware.ts") - if (changedPath.startsWith(scopePath.endsWith('/') ? scopePath : scopePath + '/')) return true; - } - return false; - } - - const outOfScope = changedFiles.filter(f => !isInScope(f.filename)); - - // --- Step 8: Report results --- - if (outOfScope.length === 0) { - await createCheckRun( - 'success', - 'Scope enforcement: all files in scope', - `All ${changedFiles.length} changed files are within scope of issue #${issueNumber} and its children.` - ); - return; - } - - // There are out-of-scope files - if (hasValidApproval) { - await createCheckRun( - 'success', - `Scope enforcement: approved by reviewer (${outOfScope.length} files outside scope)`, - `PR modifies ${outOfScope.length} file(s) not listed in issue #${issueNumber} or its children, but a non-stale approval exists.\n\nOut-of-scope files:\n${outOfScope.map(f => '- `' + f.filename + '`').join('\n')}` - ); - return; - } - - // Build annotations for out-of-scope files - const annotations = outOfScope.map(f => ({ - path: f.filename, - start_line: 1, - end_line: 1, - annotation_level: 'warning', - message: `This file is not listed in the task scope for issue #${issueNumber} or its children. If this change is intentional, approve the PR to override.` - })); - - // Determine conclusion based on config and severity - // Minor: 1-2 files out of scope; Significant: 3+ - const isMinor = outOfScope.length <= 2; - const conclusion = isMinor ? 'neutral' : config.conclusion; - - const summary = [ - `PR modifies ${outOfScope.length} file(s) not listed in issue #${issueNumber} or its children.`, - '', - '**Out-of-scope files:**', - ...outOfScope.map(f => `- \`${f.filename}\``), - '', - `**In-scope files (from issues):**`, - ...[...scopeFiles].map(f => `- \`${f}\``), - '', - 'To resolve: either update the issue to include these files, or approve the PR to override this check.' - ].join('\n'); - - await createCheckRun( - conclusion, - `Scope enforcement: ${outOfScope.length} file(s) outside task scope`, - summary, - annotations - ); + const run = require('./.github/agent-workflow/scripts/guardrail-scope.js'); + await run({ github, context, core }); diff --git a/.github/workflows/guardrail-test-ratio.yml b/.github/workflows/guardrail-test-ratio.yml index ef024f3..31ecac7 100644 --- a/.github/workflows/guardrail-test-ratio.yml +++ b/.github/workflows/guardrail-test-ratio.yml @@ -20,203 +20,5 @@ jobs: uses: actions/github-script@v7 with: script: | - const fs = require('fs'); - - // --- Load configuration (simple YAML parsing, no js-yaml dependency) --- - const configPath = '.github/agent-workflow/config.yaml'; - let threshold = 0.5; - let enabled = true; - let configuredConclusion = 'action_required'; - - try { - const content = fs.readFileSync(configPath, 'utf8'); - const lines = content.split('\n'); - let inSection = false; - for (const line of lines) { - if (/^\s+test-ratio:/.test(line)) { - inSection = true; - continue; - } - if (inSection && /^\s{0,4}\S/.test(line) && !line.match(/^\s{6,}/)) { - break; // Next sibling or parent key - } - if (inSection) { - const enabledMatch = line.match(/^\s+enabled:\s*(true|false)/); - if (enabledMatch) enabled = enabledMatch[1] === 'true'; - const conclusionMatch = line.match(/^\s+conclusion:\s*(\S+)/); - if (conclusionMatch) configuredConclusion = conclusionMatch[1]; - const thresholdMatch = line.match(/^\s+threshold:\s*([0-9.]+)/); - if (thresholdMatch) threshold = parseFloat(thresholdMatch[1]); - } - } - } catch (e) { - core.info(`Could not read config from ${configPath}, using defaults: ${e.message}`); - } - - if (!enabled) { - core.info('Test-ratio guardrail is disabled in config. Skipping.'); - return; - } - - // --- Get PR files --- - const prNumber = context.payload.pull_request.number; - const allFiles = []; - let page = 1; - while (true) { - const { data: files } = await github.rest.pulls.listFiles({ - owner: context.repo.owner, - repo: context.repo.repo, - pull_number: prNumber, - per_page: 100, - page: page, - }); - allFiles.push(...files); - if (files.length < 100) break; - page++; - } - - // --- Categorize files as test vs implementation --- - // Test file patterns: *test*, *spec*, __tests__/ - function isTestFile(filename) { - const lower = filename.toLowerCase(); - // Check for __tests__/ directory - if (lower.includes('__tests__/')) return true; - // Check for test/spec in filename (not directory names) - const basename = lower.split('/').pop(); - if (basename.includes('test') || basename.includes('spec')) return true; - // Check for test/spec directories like tests/, specs/ - const parts = lower.split('/'); - for (const part of parts) { - if (part === 'tests' || part === 'specs' || part === 'test' || part === 'spec') return true; - } - return false; - } - - // Ignore non-code files (config, docs, etc.) - function isCodeFile(filename) { - const codeExtensions = [ - '.js', '.jsx', '.ts', '.tsx', '.py', '.rb', '.go', '.rs', - '.java', '.kt', '.scala', '.cs', '.cpp', '.c', '.h', '.hpp', - '.swift', '.m', '.mm', '.php', '.lua', '.sh', '.bash', - '.yml', '.yaml', '.json', '.toml', '.xml', - ]; - const ext = '.' + filename.split('.').pop().toLowerCase(); - return codeExtensions.includes(ext); - } - - let testLines = 0; - let implLines = 0; - const implFilesWithNoTests = []; - - for (const file of allFiles) { - // Skip removed files - if (file.status === 'removed') continue; - // Skip non-code files - if (!isCodeFile(file.filename)) continue; - - const additions = file.additions || 0; - - if (isTestFile(file.filename)) { - testLines += additions; - } else { - implLines += additions; - implFilesWithNoTests.push({ - filename: file.filename, - additions: additions, - }); - } - } - - core.info(`Test lines added: ${testLines}`); - core.info(`Implementation lines added: ${implLines}`); - - // --- Handle edge case: no implementation lines --- - if (implLines === 0) { - core.info('No implementation lines in this PR. Reporting success.'); - await github.rest.checks.create({ - owner: context.repo.owner, - repo: context.repo.repo, - head_sha: context.payload.pull_request.head.sha, - name: 'guardrail/test-ratio', - status: 'completed', - conclusion: 'success', - output: { - title: 'Test-to-code ratio: no implementation changes', - summary: 'This PR contains no implementation line additions. Test ratio check is not applicable.', - }, - }); - return; - } - - // --- Calculate ratio --- - const ratio = testLines / implLines; - const passed = ratio >= threshold; - - core.info(`Test-to-code ratio: ${ratio.toFixed(2)} (threshold: ${threshold})`); - - // --- Check for non-stale PR approval override --- - const { data: reviews } = await github.rest.pulls.listReviews({ - owner: context.repo.owner, - repo: context.repo.repo, - pull_number: prNumber, - }); - - const headSha = context.payload.pull_request.head.sha; - const hasValidApproval = reviews.some( - (r) => r.state === 'APPROVED' && r.commit_id === headSha - ); - - // --- Determine conclusion --- - let conclusion; - let title; - let summary; - - if (passed) { - conclusion = 'success'; - title = `Test-to-code ratio: ${ratio.toFixed(2)} (threshold: ${threshold})`; - summary = `PR has ${testLines} test lines and ${implLines} implementation lines added. Ratio ${ratio.toFixed(2)} meets the threshold of ${threshold}.`; - } else if (hasValidApproval) { - conclusion = 'success'; - title = `Test-to-code ratio: ${ratio.toFixed(2)} — approved by reviewer`; - summary = `PR has ${testLines} test lines and ${implLines} implementation lines added. Ratio ${ratio.toFixed(2)} is below the threshold of ${threshold}, but a non-stale approval exists. Human has accepted the current state.`; - } else { - conclusion = configuredConclusion; - title = `Test-to-code ratio: ${ratio.toFixed(2)} (threshold: ${threshold})`; - summary = `PR has ${testLines} test lines and ${implLines} implementation lines added. Ratio ${ratio.toFixed(2)} is below the threshold of ${threshold}. Add more tests or approve the PR to override.`; - } - - // --- Build annotations for implementation files lacking test coverage --- - const annotations = []; - if (!passed && !hasValidApproval) { - for (const file of implFilesWithNoTests) { - if (file.additions > 0) { - annotations.push({ - path: file.filename, - start_line: 1, - end_line: 1, - annotation_level: 'warning', - message: `This implementation file has ${file.additions} added lines. The overall test-to-code ratio (${ratio.toFixed(2)}) is below the threshold (${threshold}). Consider adding corresponding tests.`, - }); - } - } - } - - // GitHub API limits annotations to 50 per request - const truncatedAnnotations = annotations.slice(0, 50); - - // --- Report check run --- - await github.rest.checks.create({ - owner: context.repo.owner, - repo: context.repo.repo, - head_sha: headSha, - name: 'guardrail/test-ratio', - status: 'completed', - conclusion: conclusion, - output: { - title: title, - summary: summary, - annotations: truncatedAnnotations, - }, - }); - - core.info(`Check run created with conclusion: ${conclusion}`); + const run = require('./.github/agent-workflow/scripts/guardrail-test-ratio.js'); + await run({ github, context, core }); diff --git a/package.json b/package.json new file mode 100644 index 0000000..f538d2f --- /dev/null +++ b/package.json @@ -0,0 +1,12 @@ +{ + "name": "agent-workflow-scripts", + "version": "1.0.0", + "description": "Shared libraries for GitHub Actions workflows", + "type": "commonjs", + "scripts": { + "test": "node --test" + }, + "devDependencies": { + "js-yaml": "^4.1.0" + } +} From f817352c90e3ddc1a26dd1b2c9770d5bd12bae38 Mon Sep 17 00:00:00 2001 From: Joe Delfino Date: Fri, 13 Feb 2026 01:12:00 +0000 Subject: [PATCH 13/23] refactor: complete workflow script extraction (#38) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Updated remaining 6 workflow YAML files to use extracted scripts: - guardrail-dependencies.yml → guardrail-dependencies.js - guardrail-commits.yml → guardrail-commits.js - guardrail-api-surface.yml → guardrail-api-surface.js - orchestrator-check.yml → orchestrator-check.js - human-review.yml → human-review.js - pr-review.yml → pr-context.js All 8 workflows now use thin YAML shells that require() standalone scripts, eliminating 1,800+ lines of duplicated inline JavaScript. Co-Authored-By: Claude Sonnet 4.5 --- .github/workflows/guardrail-api-surface.yml | 231 +---------- .github/workflows/guardrail-commits.yml | 206 +-------- .github/workflows/guardrail-dependencies.yml | 251 +---------- .github/workflows/human-review.yml | 200 +-------- .github/workflows/orchestrator-check.yml | 413 +------------------ .github/workflows/pr-review.yml | 33 +- 6 files changed, 14 insertions(+), 1320 deletions(-) diff --git a/.github/workflows/guardrail-api-surface.yml b/.github/workflows/guardrail-api-surface.yml index f98f30a..1f970a2 100644 --- a/.github/workflows/guardrail-api-surface.yml +++ b/.github/workflows/guardrail-api-surface.yml @@ -17,232 +17,5 @@ jobs: uses: actions/github-script@v7 with: script: | - const owner = context.repo.owner; - const repo = context.repo.repo; - const prNumber = context.payload.pull_request.number; - const headSha = context.payload.pull_request.head.sha; - const checkName = 'guardrail/api-surface'; - - // --- Read config --- - let enabled = true; - let configuredConclusion = 'action_required'; - try { - const configResponse = await github.rest.repos.getContent({ - owner, - repo, - path: '.github/agent-workflow/config.yaml', - ref: headSha, - }); - const configContent = Buffer.from(configResponse.data.content, 'base64').toString('utf8'); - // Simple YAML parsing for our config keys - const enabledMatch = configContent.match(/api-surface:\s*\n\s*enabled:\s*(true|false)/); - if (enabledMatch && enabledMatch[1] === 'false') { - enabled = false; - } - const conclusionMatch = configContent.match(/api-surface:\s*\n\s*enabled:\s*(?:true|false)\s*\n\s*conclusion:\s*(\S+)/); - if (conclusionMatch) { - configuredConclusion = conclusionMatch[1]; - } - } catch (e) { - // Config file not found or unreadable — use defaults (enabled, action_required) - core.info('No config.yaml found, using defaults (enabled: true, conclusion: action_required)'); - } - - if (!enabled) { - await github.rest.checks.create({ - owner, - repo, - head_sha: headSha, - name: checkName, - status: 'completed', - conclusion: 'success', - output: { - title: 'API surface check: disabled', - summary: 'This check is disabled in config.yaml.', - }, - }); - return; - } - - // --- Check for non-stale PR approval override --- - const reviews = await github.rest.pulls.listReviews({ - owner, - repo, - pull_number: prNumber, - }); - const hasValidApproval = reviews.data.some( - (r) => r.state === 'APPROVED' && r.commit_id === headSha - ); - - if (hasValidApproval) { - await github.rest.checks.create({ - owner, - repo, - head_sha: headSha, - name: checkName, - status: 'completed', - conclusion: 'success', - output: { - title: 'API surface check: approved by reviewer', - summary: 'A non-stale PR approval overrides this guardrail.', - }, - }); - return; - } - - // --- Get PR files and scan for API surface changes --- - const files = await github.paginate(github.rest.pulls.listFiles, { - owner, - repo, - pull_number: prNumber, - }); - - // API surface patterns — language-aware heuristics - const apiPatterns = [ - // JavaScript / TypeScript - { regex: /^\+.*\bexport\s+(default\s+)?(function|class|const|let|var|interface|type|enum)\b/, label: 'JS/TS export' }, - { regex: /^\+.*\bmodule\.exports\b/, label: 'CommonJS export' }, - { regex: /^\+.*\bexports\.\w+/, label: 'CommonJS named export' }, - // Python - { regex: /^\+.*@app\.(route|get|post|put|delete|patch)\b/, label: 'Flask/FastAPI route' }, - { regex: /^\+.*@router\.(route|get|post|put|delete|patch)\b/, label: 'Router route' }, - // Go - { regex: /^\+\s*func\s+[A-Z]/, label: 'Go exported function' }, - { regex: /^\+\s*type\s+[A-Z]\w+\s+(struct|interface)\b/, label: 'Go exported type' }, - // Java / Kotlin / C# - { regex: /^\+.*\bpublic\s+(static\s+)?(class|interface|enum|void|int|string|boolean|long|double|float|[\w<>\[\]]+)\s+\w+/, label: 'Public declaration' }, - // Rust - { regex: /^\+.*\bpub\s+fn\b/, label: 'Rust public function' }, - { regex: /^\+.*\bpub\s+(struct|enum|trait|type|mod)\b/, label: 'Rust public type' }, - // Ruby on Rails - { regex: /^\+.*\b(get|post|put|patch|delete|resources|resource)\s+['":\/]/, label: 'Rails route' }, - // Express.js / Node.js - { regex: /^\+.*\brouter\.(get|post|put|delete|patch|all|use)\s*\(/, label: 'Express route' }, - { regex: /^\+.*\bapp\.(get|post|put|delete|patch|all|use)\s*\(/, label: 'Express app route' }, - ]; - - // OpenAPI / Swagger file patterns - const openApiFilePatterns = [ - /openapi\.(ya?ml|json)$/i, - /swagger\.(ya?ml|json)$/i, - /api-spec\.(ya?ml|json)$/i, - ]; - - const annotations = []; - let totalApiChanges = 0; - - for (const file of files) { - // Skip removed files - if (file.status === 'removed') continue; - - // Check if this is an OpenAPI/Swagger spec file - const isOpenApiFile = openApiFilePatterns.some((p) => p.test(file.filename)); - if (isOpenApiFile) { - totalApiChanges++; - annotations.push({ - path: file.filename, - start_line: 1, - end_line: 1, - annotation_level: 'warning', - message: `OpenAPI/Swagger spec file modified: ${file.filename}. API contract changes require careful review.`, - }); - continue; - } - - // Parse the patch for added lines matching API patterns - if (!file.patch) continue; - - const lines = file.patch.split('\n'); - let currentLine = 0; - - for (const line of lines) { - // Track line numbers from hunk headers - const hunkMatch = line.match(/^@@\s+-\d+(?:,\d+)?\s+\+(\d+)(?:,\d+)?\s+@@/); - if (hunkMatch) { - currentLine = parseInt(hunkMatch[1], 10); - continue; - } - - // Only look at added lines (start with +, not +++) - if (line.startsWith('+') && !line.startsWith('+++')) { - for (const pattern of apiPatterns) { - if (pattern.regex.test(line)) { - totalApiChanges++; - annotations.push({ - path: file.filename, - start_line: currentLine, - end_line: currentLine, - annotation_level: 'warning', - message: `API surface change detected (${pattern.label}): ${line.substring(1).trim()}`, - }); - break; // One annotation per line - } - } - } - - // Advance line counter for added and context lines (not removed lines) - if (!line.startsWith('-')) { - currentLine++; - } - } - } - - // --- Report results --- - if (totalApiChanges === 0) { - await github.rest.checks.create({ - owner, - repo, - head_sha: headSha, - name: checkName, - status: 'completed', - conclusion: 'success', - output: { - title: 'API surface check: no changes detected', - summary: 'No API surface changes found in this PR.', - }, - }); - } else { - // GitHub API limits annotations to 50 per call - const batchSize = 50; - const batches = []; - for (let i = 0; i < annotations.length; i += batchSize) { - batches.push(annotations.slice(i, i + batchSize)); - } - - const summary = [ - `Found ${totalApiChanges} API surface change(s) across the PR.`, - '', - 'API surface changes have outsized downstream impact. Review these changes carefully.', - '', - 'To override: approve the PR to signal these changes are intentional.', - ].join('\n'); - - // Create the check run with the first batch of annotations - const checkRun = await github.rest.checks.create({ - owner, - repo, - head_sha: headSha, - name: checkName, - status: 'completed', - conclusion: configuredConclusion, - output: { - title: `API surface check: ${totalApiChanges} change(s) detected`, - summary, - annotations: batches[0] || [], - }, - }); - - // If there are more annotations, update the check run with additional batches - for (let i = 1; i < batches.length; i++) { - await github.rest.checks.update({ - owner, - repo, - check_run_id: checkRun.data.id, - output: { - title: `API surface check: ${totalApiChanges} change(s) detected`, - summary, - annotations: batches[i], - }, - }); - } - } + const run = require('./.github/agent-workflow/scripts/guardrail-api-surface.js'); + await run({ github, context, core }); \ No newline at end of file diff --git a/.github/workflows/guardrail-commits.yml b/.github/workflows/guardrail-commits.yml index 97f9b0f..8c88160 100644 --- a/.github/workflows/guardrail-commits.yml +++ b/.github/workflows/guardrail-commits.yml @@ -20,207 +20,5 @@ jobs: uses: actions/github-script@v7 with: script: | - const fs = require('fs'); - const path = require('path'); - - // --- Read config --- - // Default conclusion for commit message guardrail is 'neutral' (non-blocking warning) - let configuredConclusion = 'neutral'; - const configPath = path.join(process.env.GITHUB_WORKSPACE, '.github', 'agent-workflow', 'config.yaml'); - try { - const configContent = fs.readFileSync(configPath, 'utf8'); - // Simple YAML parsing for the guardrails.commit-messages section - // Config structure: guardrails: > commit-messages: > enabled/conclusion - const lines = configContent.split('\n'); - let inGuardrails = false; - let inCommitSection = false; - let guardrailIndent = 0; - let commitIndent = 0; - for (const line of lines) { - // Detect guardrails: top-level key - if (/^guardrails:/.test(line)) { - inGuardrails = true; - inCommitSection = false; - continue; - } - // If we hit another top-level key, leave guardrails - if (inGuardrails && /^\S/.test(line) && !/^\s*#/.test(line) && line.trim() !== '') { - inGuardrails = false; - inCommitSection = false; - continue; - } - // Inside guardrails, look for commit-messages: - if (inGuardrails && !inCommitSection) { - const commitMatch = line.match(/^(\s+)commit-messages:/); - if (commitMatch) { - inCommitSection = true; - commitIndent = commitMatch[1].length; - continue; - } - } - // If in commit section, check for sibling keys (same indent = new section) - if (inCommitSection) { - const lineIndent = line.match(/^(\s*)/)[1].length; - if (line.trim() === '' || /^\s*#/.test(line)) continue; - if (lineIndent <= commitIndent) { - // Left the commit-messages section - inCommitSection = false; - continue; - } - const conclusionMatch = line.match(/^\s+conclusion:\s*(\S+)/); - if (conclusionMatch) { - const val = conclusionMatch[1].replace(/['"]/g, ''); - if (['success', 'neutral', 'action_required'].includes(val)) { - configuredConclusion = val; - } - } - const enabledMatch = line.match(/^\s+enabled:\s*(\S+)/); - if (enabledMatch) { - const val = enabledMatch[1].replace(/['"]/g, '').toLowerCase(); - if (val === 'false') { - // Check is disabled, report success and exit - await github.rest.checks.create({ - owner: context.repo.owner, - repo: context.repo.repo, - head_sha: context.sha, - name: 'guardrail/commit-messages', - conclusion: 'success', - output: { - title: 'Commit message check: disabled', - summary: 'This guardrail check is disabled in config.yaml.' - } - }); - return; - } - } - } - } - } catch (e) { - // No config file found — use defaults (neutral) - core.info(`No config.yaml found at ${configPath}, using default conclusion: neutral`); - } - - // --- Check for non-stale PR approval override --- - const reviews = await github.rest.pulls.listReviews({ - owner: context.repo.owner, - repo: context.repo.repo, - pull_number: context.payload.pull_request.number - }); - - const headSha = context.payload.pull_request.head.sha; - const hasValidApproval = reviews.data.some( - r => r.state === 'APPROVED' && r.commit_id === headSha - ); - - if (hasValidApproval) { - await github.rest.checks.create({ - owner: context.repo.owner, - repo: context.repo.repo, - head_sha: context.sha, - name: 'guardrail/commit-messages', - conclusion: 'success', - output: { - title: 'Commit message check: approved by reviewer', - summary: 'A non-stale PR approval overrides this guardrail check.' - } - }); - return; - } - - // --- Get PR commits --- - const commits = await github.rest.pulls.listCommits({ - owner: context.repo.owner, - repo: context.repo.repo, - pull_number: context.payload.pull_request.number, - per_page: 100 - }); - - // --- Validate each commit message --- - // Conventional commit regex: type(optional scope)optional !: description - const conventionalCommitRegex = /^(feat|fix|chore|docs|test|refactor|ci|style|perf|build|revert)(\(.+\))?!?: .+/; - const maxFirstLineLength = 72; - - const violations = []; - - for (const commit of commits.data) { - const message = commit.commit.message; - const firstLine = message.split('\n')[0]; - const sha = commit.sha.substring(0, 7); - const commitViolations = []; - - // Check conventional commit format - if (!conventionalCommitRegex.test(firstLine)) { - commitViolations.push( - `Does not follow conventional commit format (expected: type(scope)?: description)` - ); - } - - // Check first line length - if (firstLine.length > maxFirstLineLength) { - commitViolations.push( - `First line exceeds ${maxFirstLineLength} characters (${firstLine.length} chars)` - ); - } - - if (commitViolations.length > 0) { - violations.push({ - sha: sha, - fullSha: commit.sha, - firstLine: firstLine, - issues: commitViolations - }); - } - } - - // --- Report results --- - const totalCommits = commits.data.length; - const nonConformingCount = violations.length; - - if (nonConformingCount === 0) { - // All commits conform - await github.rest.checks.create({ - owner: context.repo.owner, - repo: context.repo.repo, - head_sha: context.sha, - name: 'guardrail/commit-messages', - conclusion: 'success', - output: { - title: `Commit message check: all ${totalCommits} commits conform`, - summary: `All ${totalCommits} commit(s) follow conventional commit format with first line <= ${maxFirstLineLength} characters.` - } - }); - return; - } - - // Build summary with non-conforming commits - let summary = `## Non-conforming commits\n\n`; - summary += `Found **${nonConformingCount}** of ${totalCommits} commit(s) with violations:\n\n`; - - for (const v of violations) { - summary += `### \`${v.sha}\` — ${v.firstLine}\n`; - for (const issue of v.issues) { - summary += `- ${issue}\n`; - } - summary += '\n'; - } - - summary += `\n## Expected format\n\n`; - summary += '```\n'; - summary += 'type(optional-scope): description (max 72 chars)\n'; - summary += '```\n\n'; - summary += `Valid types: \`feat\`, \`fix\`, \`chore\`, \`docs\`, \`test\`, \`refactor\`, \`ci\`, \`style\`, \`perf\`, \`build\`, \`revert\`\n\n`; - summary += `**Configured conclusion:** \`${configuredConclusion}\`\n`; - summary += `\nTo override: submit an approving PR review. The approval must be on the current head commit to be non-stale.\n`; - - // Use the configured conclusion (default: neutral = non-blocking warning) - await github.rest.checks.create({ - owner: context.repo.owner, - repo: context.repo.repo, - head_sha: context.sha, - name: 'guardrail/commit-messages', - conclusion: configuredConclusion, - output: { - title: `Commit message check: ${nonConformingCount} non-conforming commit(s)`, - summary: summary - } - }); + const run = require('./.github/agent-workflow/scripts/guardrail-commits.js'); + await run({ github, context, core }); \ No newline at end of file diff --git a/.github/workflows/guardrail-dependencies.yml b/.github/workflows/guardrail-dependencies.yml index a693441..bc87f9e 100644 --- a/.github/workflows/guardrail-dependencies.yml +++ b/.github/workflows/guardrail-dependencies.yml @@ -21,252 +21,5 @@ jobs: uses: actions/github-script@v7 with: script: | - const fs = require('fs'); - - // --- Configuration --- - const CHECK_NAME = 'guardrail/dependency-changes'; - const DEPENDENCY_FILES = [ - 'package.json', - 'package-lock.json', - 'yarn.lock', - 'pnpm-lock.yaml', - 'requirements.txt', - 'requirements-dev.txt', - 'requirements-test.txt', - 'Pipfile', - 'Pipfile.lock', - 'pyproject.toml', - 'poetry.lock', - 'setup.py', - 'setup.cfg', - 'go.mod', - 'go.sum', - 'Cargo.toml', - 'Cargo.lock', - 'pom.xml', - 'build.gradle', - 'build.gradle.kts', - 'settings.gradle', - 'settings.gradle.kts', - 'Gemfile', - 'Gemfile.lock', - 'composer.json', - 'composer.lock', - 'mix.exs', - 'mix.lock', - 'pubspec.yaml', - 'pubspec.lock', - 'Package.swift', - 'Package.resolved', - 'Podfile', - 'Podfile.lock', - '.csproj', - 'packages.config', - 'Directory.Packages.props' - ]; - - // Also match dependency files in subdirectories (monorepo support) - function isDependencyFile(filename) { - const basename = filename.split('/').pop(); - if (DEPENDENCY_FILES.includes(basename)) { - return true; - } - // Match .csproj files by extension - if (filename.endsWith('.csproj')) { - return true; - } - return false; - } - - const JUSTIFICATION_KEYWORDS = [ - 'dependency', 'dependencies', - 'added', 'adding', - 'requires', 'required', - 'needed for', 'needed by', - 'introduced', - 'new package', 'new library', 'new module', - 'upgrade', 'upgraded', 'upgrading', - 'update', 'updated', 'updating', - 'migration', 'migrate', 'migrating', - 'replace', 'replaced', 'replacing', - 'security fix', 'security patch', 'vulnerability', - 'CVE-' - ]; - - // --- Read config (simple YAML parsing, no js-yaml dependency) --- - let checkEnabled = true; - let configuredConclusion = 'action_required'; - try { - const configPath = '.github/agent-workflow/config.yaml'; - if (fs.existsSync(configPath)) { - const content = fs.readFileSync(configPath, 'utf8'); - const lines = content.split('\n'); - let inSection = false; - for (const line of lines) { - if (/^\s+dependency-changes:/.test(line)) { - inSection = true; - continue; - } - if (inSection && /^\s{0,4}\S/.test(line) && !line.match(/^\s{6,}/)) { - break; // Next sibling or parent key - } - if (inSection) { - const enabledMatch = line.match(/^\s+enabled:\s*(true|false)/); - if (enabledMatch) checkEnabled = enabledMatch[1] === 'true'; - const conclusionMatch = line.match(/^\s+conclusion:\s*(\S+)/); - if (conclusionMatch) configuredConclusion = conclusionMatch[1]; - } - } - } - } catch (e) { - core.warning(`Failed to read config: ${e.message}. Using defaults.`); - } - - // --- If check is disabled, report success and exit --- - if (!checkEnabled) { - await github.rest.checks.create({ - owner: context.repo.owner, - repo: context.repo.repo, - head_sha: context.sha, - name: CHECK_NAME, - status: 'completed', - conclusion: 'success', - output: { - title: 'Dependency changes: check disabled', - summary: 'This guardrail check is disabled in config.yaml.' - } - }); - return; - } - - // --- Check for non-stale approving PR review (override mechanism) --- - const { data: reviews } = await github.rest.pulls.listReviews({ - owner: context.repo.owner, - repo: context.repo.repo, - pull_number: context.payload.pull_request.number - }); - - const lastCommitSha = context.payload.pull_request.head.sha; - const hasValidApproval = reviews.some( - r => r.state === 'APPROVED' && r.commit_id === lastCommitSha - ); - - if (hasValidApproval) { - await github.rest.checks.create({ - owner: context.repo.owner, - repo: context.repo.repo, - head_sha: context.sha, - name: CHECK_NAME, - status: 'completed', - conclusion: 'success', - output: { - title: 'Dependency changes: approved by reviewer', - summary: 'A non-stale PR approval overrides dependency change violations.' - } - }); - return; - } - - // --- Get PR changed files --- - const files = await github.paginate( - github.rest.pulls.listFiles, - { - owner: context.repo.owner, - repo: context.repo.repo, - pull_number: context.payload.pull_request.number, - per_page: 100 - } - ); - - const changedDependencyFiles = files.filter(f => isDependencyFile(f.filename)); - - // --- No dependency files changed: success --- - if (changedDependencyFiles.length === 0) { - await github.rest.checks.create({ - owner: context.repo.owner, - repo: context.repo.repo, - head_sha: context.sha, - name: CHECK_NAME, - status: 'completed', - conclusion: 'success', - output: { - title: 'Dependency changes: no dependency files modified', - summary: 'No dependency manifest or lock files were changed in this PR.' - } - }); - return; - } - - // --- Dependency files changed: check for justification --- - const prBody = (context.payload.pull_request.body || '').toLowerCase(); - - function hasJustification(text) { - const lowerText = text.toLowerCase(); - return JUSTIFICATION_KEYWORDS.some(keyword => lowerText.includes(keyword)); - } - - let justified = hasJustification(prBody); - - // --- If not justified in PR body, check linked issue body --- - if (!justified) { - const issueMatch = (context.payload.pull_request.body || '').match( - /(?:fixes|closes|resolves)\s+#(\d+)/i - ); - if (issueMatch) { - try { - const { data: issue } = await github.rest.issues.get({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: parseInt(issueMatch[1], 10) - }); - if (issue.body) { - justified = hasJustification(issue.body); - } - } catch (e) { - core.warning(`Failed to fetch linked issue #${issueMatch[1]}: ${e.message}`); - } - } - } - - // --- Build annotations for changed dependency files --- - const annotations = changedDependencyFiles.map(f => ({ - path: f.filename, - start_line: 1, - end_line: 1, - annotation_level: 'warning', - message: justified - ? `Dependency file changed. Justification found in PR or linked issue.` - : `Dependency file changed without justification. Add context about why dependencies were changed to the PR description or linked issue.` - })); - - // --- Report result --- - if (justified) { - await github.rest.checks.create({ - owner: context.repo.owner, - repo: context.repo.repo, - head_sha: context.sha, - name: CHECK_NAME, - status: 'completed', - conclusion: 'success', - output: { - title: `Dependency changes: ${changedDependencyFiles.length} file(s) changed with justification`, - summary: `Dependency files were modified and justification was found in the PR body or linked issue.\n\n**Changed dependency files:**\n${changedDependencyFiles.map(f => '- `' + f.filename + '`').join('\n')}`, - annotations: annotations - } - }); - } else { - const fileList = changedDependencyFiles.map(f => '- `' + f.filename + '`').join('\n'); - await github.rest.checks.create({ - owner: context.repo.owner, - repo: context.repo.repo, - head_sha: context.sha, - name: CHECK_NAME, - status: 'completed', - conclusion: configuredConclusion, - output: { - title: `Dependency changes: ${changedDependencyFiles.length} file(s) changed without justification`, - summary: `Dependency files were modified but no justification was found.\n\n**Changed dependency files:**\n${fileList}\n\n**To resolve:** Add context about dependency changes to the PR description using keywords like: ${JUSTIFICATION_KEYWORDS.slice(0, 8).map(k => '"' + k + '"').join(', ')}, etc.\n\nAlternatively, a PR approval will override this check.`, - annotations: annotations - } - }); - } + const run = require('./.github/agent-workflow/scripts/guardrail-dependencies.js'); + await run({ github, context, core }); \ No newline at end of file diff --git a/.github/workflows/human-review.yml b/.github/workflows/human-review.yml index 72c6ce6..43ef33d 100644 --- a/.github/workflows/human-review.yml +++ b/.github/workflows/human-review.yml @@ -17,201 +17,5 @@ jobs: uses: actions/github-script@v7 with: script: | - const review = context.payload.review; - const pr = context.payload.pull_request; - - // ── Parse parent issue from PR body ─────────────────────── - const fixesMatch = pr.body && pr.body.match(/[Ff]ixes\s*#(\d+)/); - if (!fixesMatch) { - console.log('No parent issue found — PR body has no "Fixes #N" reference. Skipping.'); - return; - } - const parentIssueNumber = parseInt(fixesMatch[1], 10); - - // ── Fetch comments for this specific review ───────────────── - // GET /repos/{owner}/{repo}/pulls/{pull_number}/reviews/{review_id}/comments - const { data: reviewComments } = await github.request( - 'GET /repos/{owner}/{repo}/pulls/{pull_number}/reviews/{review_id}/comments', - { - owner: context.repo.owner, - repo: context.repo.repo, - pull_number: pr.number, - review_id: review.id, - } - ); - - if (reviewComments.length === 0) { - console.log('Review has no line-level comments. Skipping.'); - return; - } - - // ── Get parent issue node_id for GraphQL sub-issue linking ─ - const { data: parentIssue } = await github.rest.issues.get({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: parentIssueNumber, - }); - const parentNodeId = parentIssue.node_id; - - // ── Severity detection ──────────────────────────────────── - function detectSeverity(body) { - const lower = body.toLowerCase(); - const blockingPatterns = /\b(blocking|block|must[- ]fix)\b/; - const suggestionPatterns = /\b(suggestion|nit|consider)\b/; - - if (blockingPatterns.test(lower)) return 'blocking'; - if (suggestionPatterns.test(lower)) return 'suggestion'; - return 'should-fix'; - } - - // ── Ensure labels exist ─────────────────────────────────── - const severityLabels = ['blocking', 'should-fix', 'suggestion']; - const labelColors = { - 'blocking': 'B60205', - 'should-fix': 'D93F0B', - 'suggestion': '0E8A16', - }; - for (const label of severityLabels) { - try { - await github.rest.issues.getLabel({ - owner: context.repo.owner, - repo: context.repo.repo, - name: label, - }); - } catch { - await github.rest.issues.createLabel({ - owner: context.repo.owner, - repo: context.repo.repo, - name: label, - color: labelColors[label], - }); - } - } - - // ── Process each comment ────────────────────────────────── - const createdIssues = []; - let hasBlockingComments = false; - - for (const comment of reviewComments) { - const severity = detectSeverity(comment.body); - const filePath = comment.path; - const line = comment.original_line || comment.line || 0; - - // Build issue body with file/line context - const issueBody = [ - `## Review Finding`, - ``, - `**Severity:** \`${severity}\``, - `**File:** \`${filePath}\`${line ? ` (line ${line})` : ''}`, - `**PR:** #${pr.number}`, - `**Reviewer:** @${review.user.login}`, - ``, - `### Comment`, - ``, - comment.body, - ``, - `---`, - `_Created automatically from a PR review comment._`, - ].join('\n'); - - const issueTitle = `[${severity}] ${filePath}${line ? `:${line}` : ''}: ${comment.body.split('\n')[0].substring(0, 80)}`; - - // Create the child issue - const { data: newIssue } = await github.rest.issues.create({ - owner: context.repo.owner, - repo: context.repo.repo, - title: issueTitle, - body: issueBody, - labels: [severity], - }); - - console.log(`Created issue #${newIssue.number}: ${newIssue.title}`); - - // Link as sub-issue via GraphQL addSubIssue mutation - try { - await github.graphql(` - mutation($parentId: ID!, $childId: ID!) { - addSubIssue(input: { issueId: $parentId, subIssueId: $childId }) { - issue { id } - subIssue { id } - } - } - `, { - parentId: parentNodeId, - childId: newIssue.node_id, - }); - console.log(`Linked issue #${newIssue.number} as sub-issue of #${parentIssueNumber}`); - } catch (err) { - console.log(`Warning: Could not link sub-issue via GraphQL: ${err.message}`); - } - - // If blocking, set as blocking the parent issue - if (severity === 'blocking') { - hasBlockingComments = true; - try { - await github.request( - 'POST /repos/{owner}/{repo}/issues/{issue_number}/sub_issues/priority', - { - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: parentIssueNumber, - sub_issue_id: newIssue.id, - } - ); - } catch (err) { - console.log(`Note: Could not set blocking dependency via REST: ${err.message}`); - } - } - - createdIssues.push({ - number: newIssue.number, - severity, - }); - } - - // ── Update PR description with Fixes references ────────── - if (createdIssues.length > 0) { - const fixesLines = createdIssues - .map(i => `Fixes #${i.number}`) - .join('\n'); - - const newSection = [ - '', - '### Review Finding Issues', - fixesLines, - '', - ].join('\n'); - - // Replace existing section or append, for idempotent updates - let currentBody = pr.body || ''; - const sectionRegex = /[\s\S]*?/; - let updatedBody; - if (sectionRegex.test(currentBody)) { - updatedBody = currentBody.replace(sectionRegex, newSection); - } else { - updatedBody = currentBody + '\n\n' + newSection; - } - - await github.rest.pulls.update({ - owner: context.repo.owner, - repo: context.repo.repo, - pull_number: pr.number, - body: updatedBody, - }); - - console.log(`Updated PR #${pr.number} body with ${createdIssues.length} Fixes references`); - } - - // ── Summary ────────────────────────────────────────────── - const blockingCount = createdIssues.filter(i => i.severity === 'blocking').length; - const shouldFixCount = createdIssues.filter(i => i.severity === 'should-fix').length; - const suggestionCount = createdIssues.filter(i => i.severity === 'suggestion').length; - - console.log(`\nDone. Created ${createdIssues.length} issues from review comments:`); - console.log(` blocking: ${blockingCount}`); - console.log(` should-fix: ${shouldFixCount}`); - console.log(` suggestion: ${suggestionCount}`); - - if (hasBlockingComments) { - console.log(`\nBlocking issues were created — parent issue #${parentIssueNumber} has new blockers.`); - } + const run = require('./.github/agent-workflow/scripts/human-review.js'); + await run({ github, context, core }); \ No newline at end of file diff --git a/.github/workflows/orchestrator-check.yml b/.github/workflows/orchestrator-check.yml index c135834..ea948c2 100644 --- a/.github/workflows/orchestrator-check.yml +++ b/.github/workflows/orchestrator-check.yml @@ -29,414 +29,5 @@ jobs: CLAUDE_CODE_OAUTH_TOKEN: ${{ secrets.CLAUDE_CODE_OAUTH_TOKEN }} with: script: | - const checkName = 'orchestrator'; - - // ── Helper: read re-review-cycle-cap from config ──────────── - async function readCycleCap() { - try { - const { data } = await github.rest.repos.getContent({ - owner: context.repo.owner, - repo: context.repo.repo, - path: '.github/agent-workflow/config.yaml', - ref: context.sha - }); - const content = Buffer.from(data.content, 'base64').toString('utf-8'); - const match = content.match(/^re-review-cycle-cap:\s*(\d+)/m); - return match ? parseInt(match[1], 10) : 3; - } catch { - return 3; // default - } - } - - // ── Helper: create check run on a specific SHA ────────────── - async function createCheckRun(headSha, conclusion, title, summary) { - await github.rest.checks.create({ - owner: context.repo.owner, - repo: context.repo.repo, - head_sha: headSha, - name: checkName, - status: 'completed', - conclusion, - output: { title, summary } - }); - } - - // ── Helper: find PRs referencing a given issue ────────────── - // Searches open PRs whose body contains "Fixes #N" or "fixes #N" - async function findPRsForIssue(issueNumber) { - const prs = []; - let page = 1; - while (true) { - const resp = await github.rest.pulls.list({ - owner: context.repo.owner, - repo: context.repo.repo, - state: 'open', - per_page: 100, - page - }); - if (resp.data.length === 0) break; - for (const pr of resp.data) { - const body = pr.body || ''; - const regex = new RegExp(`[Ff]ixes\\s*#${issueNumber}\\b`); - if (regex.test(body)) { - prs.push(pr); - } - } - if (resp.data.length < 100) break; - page++; - } - return prs; - } - - // ── Helper: query sub-issues via GraphQL ──────────────────── - async function getSubIssues(issueNumber) { - const query = ` - query($owner: String!, $repo: String!, $number: Int!) { - repository(owner: $owner, name: $repo) { - issue(number: $number) { - subIssues(first: 50) { - nodes { - number - title - state - labels(first: 10) { nodes { name } } - } - } - } - } - } - `; - try { - const result = await github.graphql(query, { - owner: context.repo.owner, - repo: context.repo.repo, - number: issueNumber - }); - return result.repository.issue.subIssues.nodes; - } catch (err) { - console.log(`Warning: GraphQL sub-issues query failed: ${err.message}`); - return []; - } - } - - // ── Helper: count past review cycles from PR comments ─────── - // We count comments left by the pr-review workflow (bot) that - // indicate a review cycle was triggered. - async function countReviewCycles(prNumber) { - const comments = await github.paginate( - github.rest.issues.listComments, - { - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: prNumber, - per_page: 100 - } - ); - // Count comments that contain the re-review marker - return comments.filter( - c => c.body && c.body.includes('') - ).length; - } - - // ── Step 1: Determine the parent issue and PR ─────────────── - let parentIssueNumber; - let prNumber; - let headSha; - - if (context.eventName === 'pull_request') { - // PR synchronize event — parse parent issue from PR body - const pr = context.payload.pull_request; - prNumber = pr.number; - headSha = pr.head.sha; - const body = pr.body || ''; - const fixesMatch = body.match(/[Ff]ixes\s*#(\d+)/); - if (!fixesMatch) { - console.log('No "Fixes #N" in PR body. Skipping orchestrator check.'); - await createCheckRun( - headSha, - 'neutral', - 'Orchestrator: no linked issue', - 'No `Fixes #N` reference found in PR description. Orchestrator check skipped.' - ); - return; - } - parentIssueNumber = parseInt(fixesMatch[1], 10); - - } else if (context.eventName === 'issues') { - // Issue event — find the PR that references this issue's parent - // The changed issue might BE a sub-issue (review finding). - // We need to find which parent issue it belongs to, then find the PR. - const changedIssue = context.payload.issue; - - // Strategy: search for open PRs that reference this issue or any - // issue that is a parent of this issue. Since sub-issues are children - // of the parent issue, and the PR has "Fixes #parent", we need to - // find PRs referencing any issue. We search for PRs that reference - // issues that have this changed issue as a sub-issue. - // - // Simpler approach: search all open PRs for ones containing - // "Fixes #N" and then check if the changed issue is a sub-issue of N. - // But that's expensive. - // - // Practical approach: list all open PRs, parse their "Fixes #N", - // check if the changed issue is a sub-issue of any of those N values. - - const openPRs = []; - let page = 1; - while (true) { - const resp = await github.rest.pulls.list({ - owner: context.repo.owner, - repo: context.repo.repo, - state: 'open', - per_page: 100, - page - }); - if (resp.data.length === 0) break; - openPRs.push(...resp.data); - if (resp.data.length < 100) break; - page++; - } - - // Build a map of parent issue number -> PR data - const parentToPR = new Map(); - for (const pr of openPRs) { - const body = pr.body || ''; - const match = body.match(/[Ff]ixes\s*#(\d+)/); - if (match) { - parentToPR.set(parseInt(match[1], 10), pr); - } - } - - if (parentToPR.size === 0) { - console.log('No open PRs with "Fixes #N" references found. Nothing to do.'); - return; - } - - // Check if the changed issue IS a parent issue referenced by a PR - if (parentToPR.has(changedIssue.number)) { - parentIssueNumber = changedIssue.number; - const pr = parentToPR.get(changedIssue.number); - prNumber = pr.number; - headSha = pr.head.sha; - } else { - // Check if the changed issue is a sub-issue of any parent - let found = false; - for (const [parentNum, pr] of parentToPR.entries()) { - const subIssues = await getSubIssues(parentNum); - const isChild = subIssues.some( - si => si.number === changedIssue.number - ); - if (isChild) { - parentIssueNumber = parentNum; - prNumber = pr.number; - headSha = pr.head.sha; - found = true; - break; - } - } - if (!found) { - console.log( - `Issue #${changedIssue.number} is not a sub-issue of any PR-linked parent. Nothing to do.` - ); - return; - } - } - } else { - console.log(`Unexpected event: ${context.eventName}. Skipping.`); - return; - } - - console.log(`Parent issue: #${parentIssueNumber}, PR: #${prNumber}, HEAD: ${headSha}`); - - // ── Step 2: Query sub-issues and check for blockers ───────── - const subIssues = await getSubIssues(parentIssueNumber); - console.log(`Found ${subIssues.length} sub-issues for #${parentIssueNumber}`); - - const openBlockers = subIssues.filter(si => { - if (si.state !== 'OPEN') return false; - const labels = si.labels.nodes.map(l => l.name); - return labels.includes('blocking'); - }); - - // ── Step 3: If blockers exist, report failing check ───────── - if (openBlockers.length > 0) { - const blockerList = openBlockers - .map(si => `- #${si.number}: ${si.title}`) - .join('\n'); - - await createCheckRun( - headSha, - 'action_required', - `Orchestrator: ${openBlockers.length} blocking issue(s)`, - [ - `PR #${prNumber} cannot merge. Issue #${parentIssueNumber} has open blocking sub-issues:`, - '', - blockerList, - '', - 'Resolve these blocking issues or approve the PR to override.' - ].join('\n') - ); - console.log(`Reported failing check: ${openBlockers.length} blockers.`); - return; - } - - // ── Step 4: No blockers — assess re-review need ───────────── - console.log('No open blockers. Assessing re-review need...'); - - const cycleCap = await readCycleCap(); - const pastCycles = await countReviewCycles(prNumber); - console.log(`Review cycles so far: ${pastCycles}, cap: ${cycleCap}`); - - // If we've reached the cycle cap, pass without re-review - if (pastCycles >= cycleCap) { - await createCheckRun( - headSha, - 'success', - 'Orchestrator: passing (review cycle cap reached)', - [ - `All blocking sub-issues for #${parentIssueNumber} are resolved.`, - '', - `Re-review cycle cap reached (${pastCycles}/${cycleCap}). No further re-review.`, - 'PR is clear to merge.' - ].join('\n') - ); - console.log('Cycle cap reached. Reporting success.'); - return; - } - - // For pull_request synchronize events (new commits pushed), - // or when blockers just cleared, assess whether re-review is needed. - // Use claude -p for the assessment. - let needsReReview = false; - - try { - // Get the PR details to find base branch - const { data: prData } = await github.rest.pulls.get({ - owner: context.repo.owner, - repo: context.repo.repo, - pull_number: prNumber - }); - const baseBranch = prData.base.ref; - - // Get the diff stat to assess change scope - const { data: files } = await github.rest.pulls.listFiles({ - owner: context.repo.owner, - repo: context.repo.repo, - pull_number: prNumber, - per_page: 100 - }); - - const totalChanges = files.reduce( - (sum, f) => sum + f.additions + f.deletions, 0 - ); - const fileCount = files.length; - const fileList = files - .map(f => `${f.filename} (+${f.additions}/-${f.deletions})`) - .join('\n'); - - // Only invoke claude if there's a meaningful diff - if (totalChanges === 0) { - console.log('No changes in PR. Skipping re-review assessment.'); - needsReReview = false; - } else { - // Build the assessment prompt - const assessPrompt = [ - 'You are assessing whether a PR needs re-review after changes were made to address review findings.', - '', - `PR #${prNumber} targets branch "${baseBranch}" and fixes issue #${parentIssueNumber}.`, - `Past review cycles: ${pastCycles}`, - '', - `The PR modifies ${fileCount} files with ${totalChanges} total line changes:`, - fileList, - '', - 'Based on the scope and nature of these changes, should reviewers re-review this PR?', - 'Consider: Are the changes small and surgical (e.g., null checks, single test additions)?', - 'Or are they broad and structural (e.g., new modules, architectural changes, many files)?', - '', - 'Respond with ONLY one word: YES or NO', - ].join('\n'); - - // Run claude -p for the assessment - const { execSync } = require('child_process'); - try { - const result = execSync( - `claude -p "${assessPrompt.replace(/"/g, '\\"')}"`, - { encoding: 'utf-8', timeout: 60000 } - ).trim(); - - console.log(`Claude assessment result: ${result}`); - needsReReview = /\bYES\b/i.test(result); - } catch (claudeErr) { - console.log(`Claude assessment failed: ${claudeErr.message}`); - // If Claude fails, default to not needing re-review - // to avoid blocking the pipeline - needsReReview = false; - } - } - } catch (err) { - console.log(`Error during re-review assessment: ${err.message}`); - needsReReview = false; - } - - // ── Step 5: Trigger re-review or report passing ───────────── - if (needsReReview) { - console.log('Re-review warranted. Triggering PR Review workflow...'); - - // Leave a marker comment for cycle counting - await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: prNumber, - body: [ - '', - `**Orchestrator:** Triggering re-review (cycle ${pastCycles + 1}/${cycleCap}).`, - '', - 'Changes since last review warrant another review pass.' - ].join('\n') - }); - - // Trigger the PR Review workflow via workflow_dispatch - try { - await github.rest.actions.createWorkflowDispatch({ - owner: context.repo.owner, - repo: context.repo.repo, - workflow_id: 'pr-review.yml', - ref: context.ref || 'main', - inputs: { - 'pr-number': prNumber.toString() - } - }); - console.log('PR Review workflow triggered successfully.'); - } catch (dispatchErr) { - console.log(`Warning: Could not trigger PR Review workflow: ${dispatchErr.message}`); - } - - await createCheckRun( - headSha, - 'neutral', - `Orchestrator: re-review triggered (cycle ${pastCycles + 1}/${cycleCap})`, - [ - `All blocking sub-issues for #${parentIssueNumber} are resolved.`, - '', - `Changes since last review warrant re-review. Cycle ${pastCycles + 1} of ${cycleCap} triggered.`, - 'The PR Review workflow has been dispatched. Orchestrator will re-evaluate after review completes.' - ].join('\n') - ); - } else { - console.log('No re-review needed. Reporting success.'); - - await createCheckRun( - headSha, - 'success', - 'Orchestrator: all clear', - [ - `All blocking sub-issues for #${parentIssueNumber} are resolved.`, - '', - pastCycles > 0 - ? `No further re-review needed after ${pastCycles} review cycle(s).` - : 'No re-review warranted based on change assessment.', - '', - 'PR is clear to merge.' - ].join('\n') - ); - } + const run = require('./.github/agent-workflow/scripts/orchestrator-check.js'); + await run({ github, context, core }); \ No newline at end of file diff --git a/.github/workflows/pr-review.yml b/.github/workflows/pr-review.yml index 0fa9e98..a25e2d6 100644 --- a/.github/workflows/pr-review.yml +++ b/.github/workflows/pr-review.yml @@ -29,37 +29,12 @@ jobs: - name: Resolve PR context id: ctx uses: actions/github-script@v7 + env: + PR_NUMBER: ${{ inputs.pr-number }} with: script: | - let prNumber; - let prData; - - if (context.eventName === 'workflow_dispatch') { - prNumber = parseInt('${{ inputs.pr-number }}', 10); - const { data } = await github.rest.pulls.get({ - owner: context.repo.owner, - repo: context.repo.repo, - pull_number: prNumber, - }); - prData = data; - } else { - prNumber = context.payload.pull_request.number; - prData = context.payload.pull_request; - } - - // Parse "fixes #N" or "Fixes #N" from PR body - const body = prData.body || ''; - const fixesMatch = body.match(/[Ff]ixes\s*#(\d+)/); - const parentIssue = fixesMatch ? fixesMatch[1] : ''; - - if (!parentIssue) { - console.log('No parent issue found — PR body has no "Fixes #N" reference.'); - } - - core.setOutput('pr-number', prNumber.toString()); - core.setOutput('parent-issue', parentIssue); - core.setOutput('base-branch', prData.base.ref); - core.setOutput('pr-title', prData.title); + const run = require('./.github/agent-workflow/scripts/pr-context.js'); + await run({ github, context, core }); # ── Correctness Reviewer ───────────────────────────────────────────── review-correctness: From bdf16f05643e0fea612ee031373a40a9790462ea Mon Sep 17 00:00:00 2001 From: Joe Delfino Date: Fri, 13 Feb 2026 01:17:10 +0000 Subject: [PATCH 14/23] test: add comprehensive test suite for workflow scripts (#39) - Created tests/validate-workflows.js with 11 tests for YAML structure validation - Moved lib module tests from .github/agent-workflow/scripts/lib/ to tests/lib/ with updated import paths (52 tests total for lib modules) - Deleted old Python and shell tests that only did keyword grep validation - Added package-lock.json from npm install - Updated .gitignore to exclude node_modules Test results: 51/53 passing (2 failing tests reveal pre-existing bugs in lib modules) Total test coverage: 53 tests - 11 workflow structure validation tests - 42 lib module unit tests Co-Authored-By: Claude Sonnet 4.5 --- .gitignore | 1 + package-lock.json | 35 + .../lib/api-patterns.test.js | 2 +- .../scripts => tests}/lib/approval.test.js | 2 +- .../lib/commit-validator.test.js | 2 +- .../scripts => tests}/lib/config.test.js | 2 +- .../lib/file-patterns.test.js | 2 +- .../lib/fixes-parser.test.js | 2 +- .../lib/patch-parser.test.js | 2 +- .../scripts => tests}/lib/pr-body.test.js | 2 +- .../lib/scope-matcher.test.js | 2 +- .../scripts => tests}/lib/severity.test.js | 2 +- tests/test-guardrail-commits.sh | 272 -------- tests/test-guardrail-scope.sh | 263 ------- tests/test_guardrail_test_ratio.py | 377 ---------- tests/test_human_review_workflow.py | 234 ------- tests/test_pr_review_workflow.py | 645 ------------------ tests/validate-workflows.js | 225 ++++++ 18 files changed, 271 insertions(+), 1801 deletions(-) create mode 100644 package-lock.json rename {.github/agent-workflow/scripts => tests}/lib/api-patterns.test.js (93%) rename {.github/agent-workflow/scripts => tests}/lib/approval.test.js (92%) rename {.github/agent-workflow/scripts => tests}/lib/commit-validator.test.js (93%) rename {.github/agent-workflow/scripts => tests}/lib/config.test.js (95%) rename {.github/agent-workflow/scripts => tests}/lib/file-patterns.test.js (94%) rename {.github/agent-workflow/scripts => tests}/lib/fixes-parser.test.js (92%) rename {.github/agent-workflow/scripts => tests}/lib/patch-parser.test.js (90%) rename {.github/agent-workflow/scripts => tests}/lib/pr-body.test.js (94%) rename {.github/agent-workflow/scripts => tests}/lib/scope-matcher.test.js (95%) rename {.github/agent-workflow/scripts => tests}/lib/severity.test.js (93%) delete mode 100755 tests/test-guardrail-commits.sh delete mode 100755 tests/test-guardrail-scope.sh delete mode 100644 tests/test_guardrail_test_ratio.py delete mode 100644 tests/test_human_review_workflow.py delete mode 100644 tests/test_pr_review_workflow.py create mode 100644 tests/validate-workflows.js diff --git a/.gitignore b/.gitignore index fb74ddf..97faa46 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ tests/__pycache__/ +node_modules/ diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..8f72601 --- /dev/null +++ b/package-lock.json @@ -0,0 +1,35 @@ +{ + "name": "agent-workflow-scripts", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "agent-workflow-scripts", + "version": "1.0.0", + "devDependencies": { + "js-yaml": "^4.1.0" + } + }, + "node_modules/argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "dev": true, + "license": "Python-2.0" + }, + "node_modules/js-yaml": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.1.tgz", + "integrity": "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==", + "dev": true, + "license": "MIT", + "dependencies": { + "argparse": "^2.0.1" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + } + } +} diff --git a/.github/agent-workflow/scripts/lib/api-patterns.test.js b/tests/lib/api-patterns.test.js similarity index 93% rename from .github/agent-workflow/scripts/lib/api-patterns.test.js rename to tests/lib/api-patterns.test.js index 2e3dc3a..0fbb0b7 100644 --- a/.github/agent-workflow/scripts/lib/api-patterns.test.js +++ b/tests/lib/api-patterns.test.js @@ -1,6 +1,6 @@ const { test } = require('node:test'); const assert = require('node:assert'); -const { detectAPIChanges } = require('./api-patterns.js'); +const { detectAPIChanges } = require('../../.github/agent-workflow/scripts/lib/api-patterns.js'); test('detectAPIChanges - JavaScript export changes', () => { const diff = ` diff --git a/.github/agent-workflow/scripts/lib/approval.test.js b/tests/lib/approval.test.js similarity index 92% rename from .github/agent-workflow/scripts/lib/approval.test.js rename to tests/lib/approval.test.js index d28ff25..bdc9c56 100644 --- a/.github/agent-workflow/scripts/lib/approval.test.js +++ b/tests/lib/approval.test.js @@ -1,6 +1,6 @@ const { test } = require('node:test'); const assert = require('node:assert'); -const { hasNonStaleApproval } = require('./approval.js'); +const { hasNonStaleApproval } = require('../../.github/agent-workflow/scripts/lib/approval.js'); test('hasNonStaleApproval - approved at head SHA', () => { const reviews = [ diff --git a/.github/agent-workflow/scripts/lib/commit-validator.test.js b/tests/lib/commit-validator.test.js similarity index 93% rename from .github/agent-workflow/scripts/lib/commit-validator.test.js rename to tests/lib/commit-validator.test.js index ab75f7e..c606e84 100644 --- a/.github/agent-workflow/scripts/lib/commit-validator.test.js +++ b/tests/lib/commit-validator.test.js @@ -1,6 +1,6 @@ const { test } = require('node:test'); const assert = require('node:assert'); -const { isValidCommit } = require('./commit-validator.js'); +const { isValidCommit } = require('../../.github/agent-workflow/scripts/lib/commit-validator.js'); test('isValidCommit - valid conventional commits', () => { assert.strictEqual(isValidCommit('feat: add new feature'), true); diff --git a/.github/agent-workflow/scripts/lib/config.test.js b/tests/lib/config.test.js similarity index 95% rename from .github/agent-workflow/scripts/lib/config.test.js rename to tests/lib/config.test.js index 04157d5..3ed6041 100644 --- a/.github/agent-workflow/scripts/lib/config.test.js +++ b/tests/lib/config.test.js @@ -1,6 +1,6 @@ const { test } = require('node:test'); const assert = require('node:assert'); -const { parseGuardrailConfig } = require('./config.js'); +const { parseGuardrailConfig } = require('../../.github/agent-workflow/scripts/lib/config.js'); test('parseGuardrailConfig - parses enabled and conclusion', () => { const yaml = ` diff --git a/.github/agent-workflow/scripts/lib/file-patterns.test.js b/tests/lib/file-patterns.test.js similarity index 94% rename from .github/agent-workflow/scripts/lib/file-patterns.test.js rename to tests/lib/file-patterns.test.js index 31e235f..89134c3 100644 --- a/.github/agent-workflow/scripts/lib/file-patterns.test.js +++ b/tests/lib/file-patterns.test.js @@ -1,6 +1,6 @@ const { test } = require('node:test'); const assert = require('node:assert'); -const { isTestFile, isCodeFile, isDependencyFile } = require('./file-patterns.js'); +const { isTestFile, isCodeFile, isDependencyFile } = require('../../.github/agent-workflow/scripts/lib/file-patterns.js'); test('isTestFile - recognizes test files', () => { assert.strictEqual(isTestFile('src/foo.test.js'), true); diff --git a/.github/agent-workflow/scripts/lib/fixes-parser.test.js b/tests/lib/fixes-parser.test.js similarity index 92% rename from .github/agent-workflow/scripts/lib/fixes-parser.test.js rename to tests/lib/fixes-parser.test.js index ce6a1f6..00ee044 100644 --- a/.github/agent-workflow/scripts/lib/fixes-parser.test.js +++ b/tests/lib/fixes-parser.test.js @@ -1,6 +1,6 @@ const { test } = require('node:test'); const assert = require('node:assert'); -const { parseFixesReferences } = require('./fixes-parser.js'); +const { parseFixesReferences } = require('../../.github/agent-workflow/scripts/lib/fixes-parser.js'); test('parseFixesReferences - single fixes reference', () => { const body = 'This PR fixes #123'; diff --git a/.github/agent-workflow/scripts/lib/patch-parser.test.js b/tests/lib/patch-parser.test.js similarity index 90% rename from .github/agent-workflow/scripts/lib/patch-parser.test.js rename to tests/lib/patch-parser.test.js index 54e9bfe..d9f570f 100644 --- a/.github/agent-workflow/scripts/lib/patch-parser.test.js +++ b/tests/lib/patch-parser.test.js @@ -1,6 +1,6 @@ const { test } = require('node:test'); const assert = require('node:assert'); -const { parseLineNumbers } = require('./patch-parser.js'); +const { parseLineNumbers } = require('../../.github/agent-workflow/scripts/lib/patch-parser.js'); test('parseLineNumbers - simple addition', () => { const patch = ` diff --git a/.github/agent-workflow/scripts/lib/pr-body.test.js b/tests/lib/pr-body.test.js similarity index 94% rename from .github/agent-workflow/scripts/lib/pr-body.test.js rename to tests/lib/pr-body.test.js index d9fae7c..7d41d10 100644 --- a/.github/agent-workflow/scripts/lib/pr-body.test.js +++ b/tests/lib/pr-body.test.js @@ -1,6 +1,6 @@ const { test } = require('node:test'); const assert = require('node:assert'); -const { replaceSection } = require('./pr-body.js'); +const { replaceSection } = require('../../.github/agent-workflow/scripts/lib/pr-body.js'); test('replaceSection - adds new section to empty body', () => { const result = replaceSection('', '## Fixes', 'Fixes #123'); diff --git a/.github/agent-workflow/scripts/lib/scope-matcher.test.js b/tests/lib/scope-matcher.test.js similarity index 95% rename from .github/agent-workflow/scripts/lib/scope-matcher.test.js rename to tests/lib/scope-matcher.test.js index dd7d234..f28ea76 100644 --- a/.github/agent-workflow/scripts/lib/scope-matcher.test.js +++ b/tests/lib/scope-matcher.test.js @@ -1,6 +1,6 @@ const { test } = require('node:test'); const assert = require('node:assert'); -const { extractFilePaths, isInScope } = require('./scope-matcher.js'); +const { extractFilePaths, isInScope } = require('../../.github/agent-workflow/scripts/lib/scope-matcher.js'); test('extractFilePaths - backtick-wrapped paths', () => { const text = 'Modify `src/index.js` and `lib/utils.ts`'; diff --git a/.github/agent-workflow/scripts/lib/severity.test.js b/tests/lib/severity.test.js similarity index 93% rename from .github/agent-workflow/scripts/lib/severity.test.js rename to tests/lib/severity.test.js index 9c3029f..2f2db0d 100644 --- a/.github/agent-workflow/scripts/lib/severity.test.js +++ b/tests/lib/severity.test.js @@ -1,6 +1,6 @@ const { test } = require('node:test'); const assert = require('node:assert'); -const { detectSeverity } = require('./severity.js'); +const { detectSeverity } = require('../../.github/agent-workflow/scripts/lib/severity.js'); test('detectSeverity - blocking keywords', () => { assert.strictEqual(detectSeverity('This is blocking the release'), 'blocking'); diff --git a/tests/test-guardrail-commits.sh b/tests/test-guardrail-commits.sh deleted file mode 100755 index e5f3e23..0000000 --- a/tests/test-guardrail-commits.sh +++ /dev/null @@ -1,272 +0,0 @@ -#!/usr/bin/env bash -# Test suite for guardrail-commits.yml -# Validates YAML syntax, required workflow structure, and key logic elements. - -set -euo pipefail - -WORKFLOW_FILE="/workspaces/agent-workflow-feat-3-github-actions/.github/workflows/guardrail-commits.yml" -FAILURES=0 -PASSES=0 - -fail() { - echo "FAIL: $1" - FAILURES=$((FAILURES + 1)) -} - -pass() { - echo "PASS: $1" - PASSES=$((PASSES + 1)) -} - -# Test 1: File exists -if [ -f "$WORKFLOW_FILE" ]; then - pass "Workflow file exists" -else - fail "Workflow file does not exist at $WORKFLOW_FILE" - echo "" - echo "Results: $PASSES passed, $FAILURES failed" - exit 1 -fi - -# Test 2: Valid YAML syntax -if python3 -c "import yaml; yaml.safe_load(open('$WORKFLOW_FILE'))" 2>/dev/null; then - pass "Valid YAML syntax" -else - fail "Invalid YAML syntax" -fi - -# Test 3: Has required trigger events (pull_request opened and synchronize) -if python3 -c " -import yaml -with open('$WORKFLOW_FILE') as f: - wf = yaml.safe_load(f) -triggers = wf.get('on') or wf.get(True) -assert 'pull_request' in triggers, 'Missing pull_request trigger' -pr = triggers['pull_request'] -types = pr.get('types', []) -assert 'opened' in types, 'Missing opened type' -assert 'synchronize' in types, 'Missing synchronize type' -" 2>/dev/null; then - pass "Has pull_request trigger with opened and synchronize types" -else - fail "Missing pull_request trigger with opened and synchronize types" -fi - -# Test 4: Workflow has a name -if python3 -c " -import yaml -with open('$WORKFLOW_FILE') as f: - wf = yaml.safe_load(f) -assert 'name' in wf, 'Missing name' -assert wf['name'], 'Name is empty' -" 2>/dev/null; then - pass "Workflow has a name" -else - fail "Workflow has no name" -fi - -# Test 5: Uses actions/github-script@v7 -if grep -q 'actions/github-script@v7' "$WORKFLOW_FILE" 2>/dev/null; then - pass "Uses actions/github-script@v7" -else - fail "Does not use actions/github-script@v7" -fi - -# Test 6: Uses actions/checkout (needed for config reading) -if grep -q 'actions/checkout' "$WORKFLOW_FILE" 2>/dev/null; then - pass "Uses actions/checkout" -else - fail "Does not use actions/checkout (needed for config reading)" -fi - -# Test 7: References Check Run API (checks.create) -if grep -q 'checks.create' "$WORKFLOW_FILE" 2>/dev/null; then - pass "References checks.create API" -else - fail "Does not reference checks.create API" -fi - -# Test 8: Handles PR approval override (listReviews) -if grep -q 'listReviews' "$WORKFLOW_FILE" 2>/dev/null; then - pass "References listReviews for approval override" -else - fail "Does not reference listReviews for approval override" -fi - -# Test 9: References pulls.listCommits for fetching PR commits -if grep -q 'listCommits' "$WORKFLOW_FILE" 2>/dev/null; then - pass "References listCommits for PR commit fetching" -else - fail "Does not reference listCommits for PR commit fetching" -fi - -# Test 10: Contains conventional commit regex pattern -if grep -q 'feat\|fix\|chore\|docs\|test\|refactor' "$WORKFLOW_FILE" 2>/dev/null && \ - grep -qE '\^?\(feat\|fix|feat\|fix' "$WORKFLOW_FILE" 2>/dev/null; then - pass "Contains conventional commit type prefixes in regex" -else - fail "Does not contain conventional commit type prefixes in regex" -fi - -# Test 11: Checks first line length (72 chars) -if grep -q '72' "$WORKFLOW_FILE" 2>/dev/null; then - pass "References 72 char max length for first line" -else - fail "Does not reference 72 char max length" -fi - -# Test 12: Has permissions set for checks write -if grep -q 'checks:\s*write' "$WORKFLOW_FILE" 2>/dev/null; then - pass "Has checks: write permission" -else - fail "Does not have checks: write permission" -fi - -# Test 13: Has pull-requests read permission (needed for reviews and commits) -if grep -q 'pull-requests:\s*read' "$WORKFLOW_FILE" 2>/dev/null; then - pass "Has pull-requests: read permission" -else - fail "Does not have pull-requests: read permission" -fi - -# Test 14: Reads config from .github/agent-workflow/ -if grep -q 'agent-workflow' "$WORKFLOW_FILE" 2>/dev/null; then - pass "References agent-workflow config path" -else - fail "Does not reference agent-workflow config path" -fi - -# Test 15: Handles different check conclusions (success, neutral, action_required) -HAS_SUCCESS=$(grep -c "'success'" "$WORKFLOW_FILE" 2>/dev/null || echo 0) -HAS_NEUTRAL=$(grep -c "'neutral'" "$WORKFLOW_FILE" 2>/dev/null || echo 0) -HAS_ACTION=$(grep -c "'action_required'" "$WORKFLOW_FILE" 2>/dev/null || echo 0) -if [ "$HAS_SUCCESS" -gt 0 ] && [ "$HAS_NEUTRAL" -gt 0 ] && [ "$HAS_ACTION" -gt 0 ]; then - pass "Has all three check conclusions (success, neutral, action_required)" -else - fail "Missing check conclusions (success=$HAS_SUCCESS, neutral=$HAS_NEUTRAL, action_required=$HAS_ACTION)" -fi - -# Test 16: Check run name matches expected convention -if grep -q 'guardrail/commit-messages' "$WORKFLOW_FILE" 2>/dev/null; then - pass "Uses guardrail/commit-messages check run naming" -else - fail "Does not use guardrail/commit-messages check run naming convention" -fi - -# Test 17: Has proper job structure with runs-on -if python3 -c " -import yaml -with open('$WORKFLOW_FILE') as f: - wf = yaml.safe_load(f) -jobs = wf.get('jobs', {}) -assert len(jobs) > 0, 'No jobs defined' -for job_name, job in jobs.items(): - assert 'runs-on' in job, f'Job {job_name} missing runs-on' - assert 'steps' in job, f'Job {job_name} missing steps' -" 2>/dev/null; then - pass "Has proper job structure with runs-on and steps" -else - fail "Missing proper job structure" -fi - -# Test 18: Lists non-conforming commits in summary -if grep -qi 'summary\|non.conforming\|violation' "$WORKFLOW_FILE" 2>/dev/null; then - pass "References summary reporting for non-conforming commits" -else - fail "Does not reference summary reporting for non-conforming commits" -fi - -# Test 19: Default conclusion is neutral (non-blocking per design) -if grep -qi "default.*neutral\|neutral.*default" "$WORKFLOW_FILE" 2>/dev/null || \ - python3 -c " -import yaml -with open('$WORKFLOW_FILE') as f: - content = f.read() -assert 'neutral' in content.lower(), 'Missing neutral default' -# Check that neutral is mentioned as default somewhere in a comment or variable -assert any(kw in content.lower() for kw in ['default', 'configuredconclusion', 'configured_conclusion', 'conclusion']), 'Missing default conclusion logic' -" 2>/dev/null; then - pass "Uses neutral as default conclusion" -else - fail "Does not use neutral as default conclusion" -fi - -# Test 20: Checks for non-stale approval (compares commit_id) -if grep -q 'commit_id\|head_sha\|APPROVED' "$WORKFLOW_FILE" 2>/dev/null; then - pass "Checks for non-stale approval via commit_id comparison" -else - fail "Does not check for non-stale approval" -fi - -# Test 21: Contains all required conventional commit types -if python3 -c " -with open('$WORKFLOW_FILE') as f: - content = f.read() -required_types = ['feat', 'fix', 'chore', 'docs', 'test', 'refactor', 'ci', 'style', 'perf', 'build', 'revert'] -for t in required_types: - assert t in content, f'Missing conventional commit type: {t}' -" 2>/dev/null; then - pass "Contains all required conventional commit types" -else - fail "Missing one or more conventional commit types" -fi - -# Test 22: Has contents read permission (needed for checkout/config) -if grep -q 'contents:\s*read' "$WORKFLOW_FILE" 2>/dev/null; then - pass "Has contents: read permission" -else - fail "Does not have contents: read permission" -fi - -# Test 23: Config parsing handles nested guardrails structure -if grep -q 'guardrails' "$WORKFLOW_FILE" 2>/dev/null && \ - grep -q 'commit-messages' "$WORKFLOW_FILE" 2>/dev/null; then - pass "Config parsing references guardrails.commit-messages nested structure" -else - fail "Config parsing does not handle nested guardrails structure" -fi - -# Test 24: Handles config disabled case (enabled: false) -if grep -q "enabled.*false\|'false'" "$WORKFLOW_FILE" 2>/dev/null; then - pass "Handles config disabled case (enabled: false)" -else - fail "Does not handle config disabled case" -fi - -# Test 25: Non-stale approval compares commit_id against head SHA -if grep -q 'commit_id' "$WORKFLOW_FILE" 2>/dev/null && \ - grep -q 'head.sha\|headSha' "$WORKFLOW_FILE" 2>/dev/null; then - pass "Non-stale approval compares commit_id against head SHA" -else - fail "Does not compare commit_id against head SHA for non-stale approval" -fi - -# Test 26: Paginates commits with per_page -if grep -q 'per_page' "$WORKFLOW_FILE" 2>/dev/null; then - pass "Paginates commits with per_page parameter" -else - fail "Does not paginate commits with per_page" -fi - -# Test 27: Splits commit message on newline to get first line -if grep -q "split.*\\\\n\|split('\\\n')\|firstLine" "$WORKFLOW_FILE" 2>/dev/null; then - pass "Extracts first line from commit message" -else - fail "Does not extract first line from commit message" -fi - -# Test 28: Reports violation count in check run title -if grep -qi 'nonConformingCount\|non.conforming.*commit' "$WORKFLOW_FILE" 2>/dev/null; then - pass "Reports non-conforming commit count in check run output" -else - fail "Does not report non-conforming commit count" -fi - -echo "" -echo "=============================" -echo "Results: $PASSES passed, $FAILURES failed" -echo "=============================" - -if [ "$FAILURES" -gt 0 ]; then - exit 1 -fi diff --git a/tests/test-guardrail-scope.sh b/tests/test-guardrail-scope.sh deleted file mode 100755 index 754c32e..0000000 --- a/tests/test-guardrail-scope.sh +++ /dev/null @@ -1,263 +0,0 @@ -#!/usr/bin/env bash -# Test suite for guardrail-scope.yml -# Validates YAML syntax, required workflow structure, and key logic elements. - -set -euo pipefail - -WORKFLOW_FILE="/workspaces/agent-workflow-feat-3-github-actions/.github/workflows/guardrail-scope.yml" -FAILURES=0 -PASSES=0 - -fail() { - echo "FAIL: $1" - FAILURES=$((FAILURES + 1)) -} - -pass() { - echo "PASS: $1" - PASSES=$((PASSES + 1)) -} - -# Test 1: File exists -if [ -f "$WORKFLOW_FILE" ]; then - pass "Workflow file exists" -else - fail "Workflow file does not exist at $WORKFLOW_FILE" - echo "" - echo "Results: $PASSES passed, $FAILURES failed" - exit 1 -fi - -# Test 2: Valid YAML syntax -if python3 -c "import yaml; yaml.safe_load(open('$WORKFLOW_FILE'))" 2>/dev/null; then - pass "Valid YAML syntax" -else - fail "Invalid YAML syntax" -fi - -# Test 3: Has required trigger events (pull_request opened and synchronize) -if python3 -c " -import yaml -with open('$WORKFLOW_FILE') as f: - wf = yaml.safe_load(f) -assert 'on' in wf or True in wf, 'Missing on: trigger' -triggers = wf.get('on') or wf.get(True) -assert 'pull_request' in triggers, 'Missing pull_request trigger' -pr = triggers['pull_request'] -types = pr.get('types', []) -assert 'opened' in types, 'Missing opened type' -assert 'synchronize' in types, 'Missing synchronize type' -" 2>/dev/null; then - pass "Has pull_request trigger with opened and synchronize types" -else - fail "Missing pull_request trigger with opened and synchronize types" -fi - -# Test 4: Workflow has a name -if python3 -c " -import yaml -with open('$WORKFLOW_FILE') as f: - wf = yaml.safe_load(f) -assert 'name' in wf, 'Missing name' -assert wf['name'], 'Name is empty' -" 2>/dev/null; then - pass "Workflow has a name" -else - fail "Workflow has no name" -fi - -# Test 5: Uses actions/github-script@v7 -if grep -q 'actions/github-script@v7' "$WORKFLOW_FILE" 2>/dev/null; then - pass "Uses actions/github-script@v7" -else - fail "Does not use actions/github-script@v7" -fi - -# Test 6: Uses actions/checkout -if grep -q 'actions/checkout' "$WORKFLOW_FILE" 2>/dev/null; then - pass "Uses actions/checkout" -else - fail "Does not use actions/checkout (needed for config reading)" -fi - -# Test 7: References Check Run API (checks.create) -if grep -q 'checks.create' "$WORKFLOW_FILE" 2>/dev/null; then - pass "References checks.create API" -else - fail "Does not reference checks.create API" -fi - -# Test 8: Handles PR approval override (listReviews) -if grep -q 'listReviews' "$WORKFLOW_FILE" 2>/dev/null; then - pass "References listReviews for approval override" -else - fail "Does not reference listReviews for approval override" -fi - -# Test 9: Parses issue references (fixes #N pattern) -if grep -qi 'fixes\s*#' "$WORKFLOW_FILE" 2>/dev/null || grep -qi 'fixes.*#' "$WORKFLOW_FILE" 2>/dev/null; then - pass "References 'fixes #N' pattern parsing" -else - fail "Does not reference 'fixes #N' pattern parsing" -fi - -# Test 10: Compares changed files against issue scope -if grep -q 'listFiles\|changed_files\|files changed\|changedFiles' "$WORKFLOW_FILE" 2>/dev/null; then - pass "References PR file listing" -else - fail "Does not reference PR file listing" -fi - -# Test 11: Has permissions set for checks write -if grep -q 'checks:\s*write' "$WORKFLOW_FILE" 2>/dev/null; then - pass "Has checks: write permission" -else - fail "Does not have checks: write permission" -fi - -# Test 12: Reports annotations on out-of-scope files -if grep -q 'annotation' "$WORKFLOW_FILE" 2>/dev/null; then - pass "References annotations for out-of-scope files" -else - fail "Does not reference annotations" -fi - -# Test 13: Reads config from .github/agent-workflow/ -if grep -q 'agent-workflow' "$WORKFLOW_FILE" 2>/dev/null; then - pass "References agent-workflow config path" -else - fail "Does not reference agent-workflow config path" -fi - -# Test 14: Handles different check conclusions (success, neutral, action_required) -HAS_SUCCESS=$(grep -c "'success'" "$WORKFLOW_FILE" 2>/dev/null || echo 0) -HAS_NEUTRAL=$(grep -c "'neutral'" "$WORKFLOW_FILE" 2>/dev/null || echo 0) -HAS_ACTION=$(grep -c "'action_required'" "$WORKFLOW_FILE" 2>/dev/null || echo 0) -if [ "$HAS_SUCCESS" -gt 0 ] && [ "$HAS_NEUTRAL" -gt 0 ] && [ "$HAS_ACTION" -gt 0 ]; then - pass "Has all three check conclusions (success, neutral, action_required)" -else - fail "Missing check conclusions (success=$HAS_SUCCESS, neutral=$HAS_NEUTRAL, action_required=$HAS_ACTION)" -fi - -# Test 15: Check run name matches expected convention -if grep -q 'guardrail/scope' "$WORKFLOW_FILE" 2>/dev/null; then - pass "Uses guardrail/scope check run naming" -else - fail "Does not use guardrail/scope check run naming convention" -fi - -# Test 16: Has proper job structure with runs-on -if python3 -c " -import yaml -with open('$WORKFLOW_FILE') as f: - wf = yaml.safe_load(f) -jobs = wf.get('jobs', {}) -assert len(jobs) > 0, 'No jobs defined' -for job_name, job in jobs.items(): - assert 'runs-on' in job, f'Job {job_name} missing runs-on' - assert 'steps' in job, f'Job {job_name} missing steps' -" 2>/dev/null; then - pass "Has proper job structure with runs-on and steps" -else - fail "Missing proper job structure" -fi - -# Test 17: Has issues read permission (needed to read issue body) -if grep -q 'issues:\s*read' "$WORKFLOW_FILE" 2>/dev/null; then - pass "Has issues: read permission" -else - fail "Does not have issues: read permission" -fi - -# Test 18: Has pull-requests read permission (needed for reviews) -if grep -q 'pull-requests:\s*read' "$WORKFLOW_FILE" 2>/dev/null; then - pass "Has pull-requests: read permission" -else - fail "Does not have pull-requests: read permission" -fi - -# Test 19: Handles config disabled case -if grep -q 'config.enabled' "$WORKFLOW_FILE" 2>/dev/null && grep -q 'disabled' "$WORKFLOW_FILE" 2>/dev/null; then - pass "Handles config disabled case" -else - fail "Does not handle config disabled case" -fi - -# Test 20: Handles missing PR body (null/empty) -if grep -q "context.payload.pull_request.body || ''" "$WORKFLOW_FILE" 2>/dev/null; then - pass "Handles null/empty PR body" -else - fail "Does not handle null/empty PR body" -fi - -# Test 21: Handles issue read failure gracefully -if grep -q 'issue not found' "$WORKFLOW_FILE" 2>/dev/null; then - pass "Handles issue read failure gracefully" -else - fail "Does not handle issue read failure" -fi - -# Test 22: Handles no files in issue body gracefully -if grep -q 'no files listed in issue' "$WORKFLOW_FILE" 2>/dev/null; then - pass "Handles no file paths in issue body" -else - fail "Does not handle missing file paths in issue" -fi - -# Test 22b: Queries sub-issues via GraphQL for scope files -if grep -q 'subIssues' "$WORKFLOW_FILE" 2>/dev/null; then - pass "Queries sub-issues for scope file extraction" -else - fail "Does not query sub-issues — only checks parent issue for file paths" -fi - -# Test 23: Paginates changed files (per_page: 100) -if grep -q 'per_page: 100' "$WORKFLOW_FILE" 2>/dev/null; then - pass "Paginates changed files with per_page 100" -else - fail "Does not paginate changed files" -fi - -# Test 24: Non-stale approval uses commit_id check -if grep -q 'commit_id.*headSha\|r.commit_id === headSha' "$WORKFLOW_FILE" 2>/dev/null; then - pass "Non-stale approval checks commit_id against head SHA" -else - fail "Non-stale approval does not check commit_id" -fi - -# Test 25: Check run status is set to 'completed' -if grep -q "status: 'completed'" "$WORKFLOW_FILE" 2>/dev/null; then - pass "Check run status set to completed" -else - fail "Check run status not set to completed" -fi - -# Test 26: Limits annotations to 50 (GitHub API limit) -if grep -q 'slice(0, 50)' "$WORKFLOW_FILE" 2>/dev/null; then - pass "Limits annotations to 50 per API limit" -else - fail "Does not limit annotations to 50" -fi - -# Test 27: Supports backtick-wrapped file paths in issue body -if grep -q 'backtick\|Backtick\|`(' "$WORKFLOW_FILE" 2>/dev/null; then - pass "Supports backtick-wrapped file path extraction" -else - fail "Does not support backtick-wrapped file paths" -fi - -# Test 28: Minor violations (1-2 files) report neutral instead of action_required -if grep -q 'isMinor' "$WORKFLOW_FILE" 2>/dev/null; then - pass "Distinguishes minor from significant violations" -else - fail "Does not distinguish minor from significant violations" -fi - -echo "" -echo "=============================" -echo "Results: $PASSES passed, $FAILURES failed" -echo "=============================" - -if [ "$FAILURES" -gt 0 ]; then - exit 1 -fi diff --git a/tests/test_guardrail_test_ratio.py b/tests/test_guardrail_test_ratio.py deleted file mode 100644 index 468c3cb..0000000 --- a/tests/test_guardrail_test_ratio.py +++ /dev/null @@ -1,377 +0,0 @@ -"""Tests for the guardrail-test-ratio.yml GitHub Actions workflow. - -Validates YAML syntax, workflow structure, trigger configuration, -and the presence of required logic steps. -""" - -import yaml -import os -import re - -WORKFLOW_PATH = os.path.join( - os.path.dirname(__file__), - "..", - ".github", - "workflows", - "guardrail-test-ratio.yml", -) - -CONFIG_PATH = os.path.join( - os.path.dirname(__file__), - "..", - ".github", - "agent-workflow", - "config.yaml", -) - - -def load_workflow(): - """Load and parse the workflow YAML file.""" - with open(WORKFLOW_PATH) as f: - return yaml.safe_load(f) - - -def load_config(): - """Load and parse the config YAML file.""" - with open(CONFIG_PATH) as f: - return yaml.safe_load(f) - - -def get_script_content(step): - """Extract the script content from a github-script step.""" - if step.get("uses", "").startswith("actions/github-script"): - return step.get("with", {}).get("script", "") - return "" - - -def find_step_by_id(steps, step_id): - """Find a step by its id.""" - for step in steps: - if step.get("id") == step_id: - return step - return None - - -def find_steps_by_uses(steps, uses_prefix): - """Find all steps that use a given action prefix.""" - return [s for s in steps if s.get("uses", "").startswith(uses_prefix)] - - -class TestWorkflowYamlSyntax: - """Test that the workflow file is valid YAML.""" - - def test_file_exists(self): - assert os.path.exists(WORKFLOW_PATH), ( - f"Workflow file not found at {WORKFLOW_PATH}" - ) - - def test_valid_yaml(self): - wf = load_workflow() - assert wf is not None, "Workflow YAML parsed as None (empty file)" - - def test_is_dict(self): - wf = load_workflow() - assert isinstance(wf, dict), "Workflow YAML root should be a mapping" - - -class TestWorkflowTriggers: - """Test that the workflow triggers on the correct events.""" - - def test_has_on_key(self): - wf = load_workflow() - assert True in wf or "on" in wf, "Workflow must have 'on' trigger" - - def test_triggers_on_pull_request(self): - wf = load_workflow() - on = wf.get(True) or wf.get("on") - assert "pull_request" in on, ( - "Workflow must trigger on pull_request" - ) - - def test_pull_request_types_include_opened(self): - wf = load_workflow() - on = wf.get(True) or wf.get("on") - pr = on["pull_request"] - types = pr.get("types", []) - assert "opened" in types, ( - "pull_request trigger must include 'opened' type" - ) - - def test_pull_request_types_include_synchronize(self): - wf = load_workflow() - on = wf.get(True) or wf.get("on") - pr = on["pull_request"] - types = pr.get("types", []) - assert "synchronize" in types, ( - "pull_request trigger must include 'synchronize' type" - ) - - -class TestWorkflowStructure: - """Test the overall workflow structure.""" - - def test_has_name(self): - wf = load_workflow() - assert "name" in wf, "Workflow must have a name" - - def test_name_mentions_test_ratio(self): - wf = load_workflow() - name = wf["name"].lower() - assert "test" in name and "ratio" in name, ( - "Workflow name should mention test ratio" - ) - - def test_has_jobs(self): - wf = load_workflow() - assert "jobs" in wf, "Workflow must have jobs" - - def test_has_check_job(self): - wf = load_workflow() - jobs = wf["jobs"] - assert len(jobs) >= 1, "Workflow must have at least one job" - - def test_job_runs_on_ubuntu(self): - wf = load_workflow() - jobs = wf["jobs"] - job = list(jobs.values())[0] - runs_on = job.get("runs-on", "") - assert "ubuntu" in runs_on, "Job must run on ubuntu" - - def test_has_permissions(self): - """Workflow needs checks:write and pull-requests:read permissions.""" - wf = load_workflow() - # Permissions can be at workflow level or job level - jobs = wf["jobs"] - job = list(jobs.values())[0] - perms = wf.get("permissions", {}) or job.get("permissions", {}) - assert "checks" in perms, "Must have checks permission" - assert perms["checks"] == "write", "Must have checks:write permission" - assert "pull-requests" in perms, "Must have pull-requests permission" - assert perms["pull-requests"] == "read", ( - "Must have pull-requests:read permission" - ) - - -class TestWorkflowSteps: - """Test that the workflow has the required steps.""" - - def _get_steps(self): - wf = load_workflow() - jobs = wf["jobs"] - job = list(jobs.values())[0] - return job.get("steps", []) - - def test_has_checkout_step(self): - steps = self._get_steps() - checkout_steps = find_steps_by_uses(steps, "actions/checkout") - assert len(checkout_steps) >= 1, "Must have a checkout step" - - def test_has_github_script_step(self): - steps = self._get_steps() - script_steps = find_steps_by_uses(steps, "actions/github-script") - assert len(script_steps) >= 1, ( - "Must have at least one github-script step" - ) - - -class TestConfigYaml: - """Test that the config.yaml file contains the right defaults.""" - - def test_config_file_exists(self): - assert os.path.exists(CONFIG_PATH), ( - f"Config file not found at {CONFIG_PATH}" - ) - - def test_config_valid_yaml(self): - config = load_config() - assert config is not None, "Config YAML parsed as None" - - def test_has_guardrails_section(self): - config = load_config() - assert "guardrails" in config, ( - "Config must have 'guardrails' section" - ) - - def test_has_test_ratio_section(self): - config = load_config() - guardrails = config["guardrails"] - assert "test-ratio" in guardrails, ( - "Guardrails config must have 'test-ratio' section" - ) - - def test_has_threshold(self): - config = load_config() - tr = config["guardrails"]["test-ratio"] - assert "threshold" in tr, ( - "test-ratio config must have 'threshold'" - ) - - def test_default_threshold_is_0_5(self): - config = load_config() - tr = config["guardrails"]["test-ratio"] - assert tr["threshold"] == 0.5, ( - "Default threshold should be 0.5" - ) - - def test_has_enabled(self): - config = load_config() - tr = config["guardrails"]["test-ratio"] - assert "enabled" in tr, "test-ratio config must have 'enabled'" - assert tr["enabled"] is True, "test-ratio should be enabled by default" - - def test_has_conclusion(self): - config = load_config() - tr = config["guardrails"]["test-ratio"] - assert "conclusion" in tr, ( - "test-ratio config must have 'conclusion'" - ) - assert tr["conclusion"] == "action_required", ( - "Default conclusion should be 'action_required'" - ) - - -class TestScriptLogic: - """Test that the github-script step contains the required logic.""" - - def _get_all_script_content(self): - wf = load_workflow() - jobs = wf["jobs"] - job = list(jobs.values())[0] - steps = job.get("steps", []) - scripts = [] - for step in steps: - content = get_script_content(step) - if content: - scripts.append(content) - return "\n".join(scripts) - - def test_reads_config_yaml(self): - """Script must read config.yaml for threshold.""" - script = self._get_all_script_content() - assert "config" in script.lower(), ( - "Script must reference config for threshold" - ) - - def test_uses_pulls_list_files(self): - """Script must use pulls.listFiles to get PR files.""" - script = self._get_all_script_content() - assert "listFiles" in script, ( - "Script must use pulls.listFiles to get PR diff" - ) - - def test_categorizes_test_files(self): - """Script must identify test files by naming convention.""" - script = self._get_all_script_content() - # Should check for test/spec patterns - has_test_pattern = "test" in script.lower() - has_spec_pattern = "spec" in script.lower() - has_tests_dir = "__tests__" in script - assert has_test_pattern or has_spec_pattern or has_tests_dir, ( - "Script must categorize test files by naming convention" - ) - - def test_counts_added_lines(self): - """Script must count added lines from the diff.""" - script = self._get_all_script_content() - # Should reference additions or patch parsing - has_additions = "additions" in script - has_patch = "patch" in script - assert has_additions or has_patch, ( - "Script must count added lines from diff" - ) - - def test_calculates_ratio(self): - """Script must calculate and compare ratio against threshold.""" - script = self._get_all_script_content() - assert "ratio" in script.lower() or "threshold" in script.lower(), ( - "Script must calculate ratio and compare against threshold" - ) - - def test_creates_check_run(self): - """Script must create a check run via the Checks API.""" - script = self._get_all_script_content() - assert "checks.create" in script, ( - "Script must use checks.create to report results" - ) - - def test_checks_for_approval_override(self): - """Script must check for non-stale PR approval override.""" - script = self._get_all_script_content() - assert "listReviews" in script or "APPROVED" in script, ( - "Script must check for non-stale approval override" - ) - - def test_reports_success_on_approval(self): - """Script must report success when a non-stale approval exists.""" - script = self._get_all_script_content() - assert "success" in script, ( - "Script must be able to report 'success' conclusion" - ) - - def test_reports_action_required_on_failure(self): - """Script must report action_required when ratio is below threshold.""" - script = self._get_all_script_content() - assert "action_required" in script, ( - "Script must be able to report 'action_required' conclusion" - ) - - def test_check_run_name_includes_guardrail(self): - """Check run name should identify it as a guardrail.""" - script = self._get_all_script_content() - assert "guardrail" in script.lower(), ( - "Check run name should include 'guardrail'" - ) - - def test_includes_annotations(self): - """Script should include annotations for findings.""" - script = self._get_all_script_content() - assert "annotation" in script.lower(), ( - "Script should include annotations in check run output" - ) - - def test_handles_no_implementation_lines(self): - """Script should handle the case where there are zero implementation lines.""" - script = self._get_all_script_content() - # Should have some guard against division by zero or no impl lines - # Check for explicit handling of zero/no implementation lines - has_zero_check = ( - "=== 0" in script - or "== 0" in script - or "no implementation" in script.lower() - or "no code" in script.lower() - or "impl" in script.lower() - ) - assert has_zero_check, ( - "Script must handle the case of zero implementation lines" - ) - - def test_paginates_file_listing(self): - """Script should paginate through PR files for large PRs.""" - script = self._get_all_script_content() - assert "per_page" in script, ( - "Script must use per_page for paginated file listing" - ) - assert "page" in script, ( - "Script must handle pagination" - ) - - def test_skips_removed_files(self): - """Script should skip files that were removed in the PR.""" - script = self._get_all_script_content() - assert "removed" in script, ( - "Script must skip removed files" - ) - - def test_has_guardrail_check_run_name(self): - """Check run name should be 'guardrail/test-ratio'.""" - script = self._get_all_script_content() - assert "guardrail/test-ratio" in script, ( - "Check run name should be 'guardrail/test-ratio'" - ) - - def test_handles_disabled_config(self): - """Script should respect the enabled flag in config.""" - script = self._get_all_script_content() - assert "enabled" in script, ( - "Script must check if guardrail is enabled" - ) diff --git a/tests/test_human_review_workflow.py b/tests/test_human_review_workflow.py deleted file mode 100644 index 167026c..0000000 --- a/tests/test_human_review_workflow.py +++ /dev/null @@ -1,234 +0,0 @@ -""" -Tests for .github/workflows/human-review.yml - -Validates the workflow structure, trigger configuration, permissions, -and the expected logic within the github-script action. -""" - -import yaml -import os -import re -import pytest - -WORKFLOW_PATH = os.path.join( - os.path.dirname(__file__), - "..", - ".github", - "workflows", - "human-review.yml", -) - - -@pytest.fixture -def workflow(): - """Load and parse the workflow YAML.""" - with open(WORKFLOW_PATH) as f: - return yaml.safe_load(f) - - -# ── Trigger ────────────────────────────────────────────────────────── - - -class TestTrigger: - def test_triggers_on_pull_request_review(self, workflow): - assert "on" in workflow, "Workflow must have an 'on' trigger" - on = workflow["on"] - assert "pull_request_review" in on, ( - "Must trigger on pull_request_review event" - ) - - def test_triggers_only_on_submitted(self, workflow): - pr_review = workflow["on"]["pull_request_review"] - assert "types" in pr_review, "Must specify types filter" - assert pr_review["types"] == ["submitted"], ( - "Must trigger only on 'submitted' type" - ) - - -# ── Permissions ────────────────────────────────────────────────────── - - -class TestPermissions: - def test_has_issues_write(self, workflow): - perms = workflow.get("permissions", {}) - assert perms.get("issues") == "write", ( - "Needs issues:write to create child issues" - ) - - def test_has_pull_requests_write(self, workflow): - perms = workflow.get("permissions", {}) - assert perms.get("pull-requests") == "write", ( - "Needs pull-requests:write to update PR description" - ) - - def test_has_contents_read(self, workflow): - perms = workflow.get("permissions", {}) - assert perms.get("contents") == "read", ( - "Needs contents:read for checkout context" - ) - - -# ── Jobs ───────────────────────────────────────────────────────────── - - -class TestJobs: - def test_has_process_review_job(self, workflow): - assert "jobs" in workflow, "Workflow must define jobs" - assert "process-review" in workflow["jobs"], ( - "Must have a 'process-review' job" - ) - - def test_job_runs_on_ubuntu(self, workflow): - job = workflow["jobs"]["process-review"] - assert "ubuntu" in job["runs-on"], "Job must run on ubuntu" - - def test_job_has_steps(self, workflow): - job = workflow["jobs"]["process-review"] - assert "steps" in job, "Job must have steps" - assert len(job["steps"]) > 0, "Job must have at least one step" - - -# ── Script Content ─────────────────────────────────────────────────── - - -class TestScriptContent: - """Validate the github-script step contains the required logic.""" - - @pytest.fixture - def script_step(self, workflow): - """Find the github-script step.""" - steps = workflow["jobs"]["process-review"]["steps"] - for step in steps: - if step.get("uses", "").startswith("actions/github-script"): - return step - pytest.fail("No actions/github-script step found") - - @pytest.fixture - def script_text(self, script_step): - """Extract the script text from the github-script step.""" - return script_step.get("with", {}).get("script", "") - - def test_uses_github_script_v7(self, script_step): - assert script_step["uses"] == "actions/github-script@v7" - - def test_parses_fixes_reference(self, script_text): - assert re.search(r"[Ff]ixes\s*#", script_text), ( - "Script must parse 'Fixes #N' from PR body" - ) - - def test_fetches_review_comments(self, script_text): - # Should call the reviews/comments endpoint - assert "reviews" in script_text and "comments" in script_text, ( - "Script must fetch review comments" - ) - - def test_creates_issues(self, script_text): - assert "issues.create" in script_text or "createIssue" in script_text, ( - "Script must create issues for review comments" - ) - - def test_severity_detection_blocking(self, script_text): - assert "blocking" in script_text.lower() or "block" in script_text.lower(), ( - "Script must detect blocking severity" - ) - - def test_severity_detection_suggestion(self, script_text): - assert "suggestion" in script_text.lower(), ( - "Script must detect suggestion severity" - ) - - def test_severity_detection_should_fix(self, script_text): - assert "should-fix" in script_text or "should_fix" in script_text, ( - "Script must handle should-fix severity" - ) - - def test_includes_file_path_context(self, script_text): - assert "path" in script_text, ( - "Script must include file path from comment location" - ) - - def test_includes_line_number_context(self, script_text): - # Should reference line from the comment - assert "line" in script_text, ( - "Script must include line number from comment location" - ) - - def test_graphql_sub_issue_linking(self, script_text): - assert "addSubIssue" in script_text, ( - "Script must use addSubIssue GraphQL mutation to link child issues" - ) - - def test_graphql_parent_node_id(self, script_text): - assert "node_id" in script_text or "nodeId" in script_text or "node id" in script_text.lower(), ( - "Script must get parent issue node ID for GraphQL" - ) - - def test_updates_pr_body_with_fixes(self, script_text): - # Should update the PR body/description to add Fixes references - assert "update" in script_text.lower() and ("body" in script_text or "description" in script_text), ( - "Script must update PR body with Fixes references for created issues" - ) - - def test_blocking_dependency_api(self, script_text): - # Should use the sub-issues dependency blocked-by API for blocking comments - assert "blocked" in script_text.lower() or "dependencies" in script_text.lower() or "blocking" in script_text.lower(), ( - "Script must set blocking dependencies for blocking comments" - ) - - def test_idempotent_pr_body_update(self, script_text): - """Should replace existing review section rather than duplicating it.""" - assert "human-review-issues-start" in script_text, ( - "Script must use HTML comment markers for idempotent PR body updates" - ) - assert "human-review-issues-end" in script_text, ( - "Script must use closing HTML comment marker" - ) - # Check for replacement logic (regex test or replace) - assert "replace" in script_text.lower() or "test" in script_text, ( - "Script must check for existing section before appending" - ) - - -# ── Early-exit guard ───────────────────────────────────────────────── - - -class TestEarlyExit: - """Verify the workflow handles edge cases gracefully.""" - - @pytest.fixture - def script_text(self, workflow): - steps = workflow["jobs"]["process-review"]["steps"] - for step in steps: - if step.get("uses", "").startswith("actions/github-script"): - return step.get("with", {}).get("script", "") - return "" - - def test_skips_when_no_fixes_reference(self, script_text): - """Should exit early if no 'Fixes #N' found in PR body.""" - assert "no parent issue" in script_text.lower() or "skip" in script_text.lower() or "return" in script_text, ( - "Script must handle case where PR has no Fixes reference" - ) - - def test_skips_when_no_comments(self, script_text): - """Should handle reviews with no line-level comments.""" - # The script should check if there are comments and handle empty case - assert "length" in script_text or "no comments" in script_text.lower() or ".length" in script_text, ( - "Script must handle case where review has no comments" - ) - - -# ── YAML validity ──────────────────────────────────────────────────── - - -class TestYamlValidity: - def test_yaml_parses_successfully(self): - """The workflow file must be valid YAML.""" - with open(WORKFLOW_PATH) as f: - data = yaml.safe_load(f) - assert data is not None - - def test_is_valid_github_actions_workflow(self, workflow): - """Must have the basic structure of a GitHub Actions workflow.""" - assert "name" in workflow, "Workflow must have a name" - assert "on" in workflow, "Workflow must have triggers" - assert "jobs" in workflow, "Workflow must have jobs" diff --git a/tests/test_pr_review_workflow.py b/tests/test_pr_review_workflow.py deleted file mode 100644 index 7c162f8..0000000 --- a/tests/test_pr_review_workflow.py +++ /dev/null @@ -1,645 +0,0 @@ -""" -Tests for .github/workflows/pr-review.yml - -Validates the workflow structure, trigger configuration, permissions, -parallel reviewer jobs, and the expected prompt construction for each -reviewer skill invoked via anthropics/claude-code-action. -""" - -import yaml -import os -import re -import pytest - -WORKFLOW_PATH = os.path.join( - os.path.dirname(__file__), - "..", - ".github", - "workflows", - "pr-review.yml", -) - -CONFIG_PATH = os.path.join( - os.path.dirname(__file__), - "..", - ".github", - "agent-workflow", - "config.yaml", -) - - -def load_workflow(): - """Load and parse the workflow YAML file.""" - with open(WORKFLOW_PATH) as f: - return yaml.safe_load(f) - - -def load_config(): - """Load and parse the config YAML file.""" - with open(CONFIG_PATH) as f: - return yaml.safe_load(f) - - -def get_on(wf): - """Get the 'on' trigger, handling YAML parsing of bare 'on' as True.""" - return wf.get(True) or wf.get("on") - - -# ── YAML Validity ──────────────────────────────────────────────────── - - -class TestYamlValidity: - def test_file_exists(self): - assert os.path.exists(WORKFLOW_PATH), ( - f"Workflow file not found at {WORKFLOW_PATH}" - ) - - def test_valid_yaml(self): - wf = load_workflow() - assert wf is not None, "Workflow YAML parsed as None (empty file)" - - def test_is_dict(self): - wf = load_workflow() - assert isinstance(wf, dict), "Workflow YAML root should be a mapping" - - def test_is_valid_github_actions_workflow(self): - wf = load_workflow() - assert "name" in wf, "Workflow must have a name" - assert get_on(wf) is not None, "Workflow must have triggers" - assert "jobs" in wf, "Workflow must have jobs" - - -# ── Triggers ───────────────────────────────────────────────────────── - - -class TestTriggers: - def test_triggers_on_pull_request(self): - wf = load_workflow() - on = get_on(wf) - assert "pull_request" in on, ( - "Must trigger on pull_request event" - ) - - def test_pull_request_types_include_opened(self): - wf = load_workflow() - on = get_on(wf) - pr = on["pull_request"] - types = pr.get("types", []) - assert "opened" in types, ( - "pull_request trigger must include 'opened' type" - ) - - def test_pull_request_types_include_synchronize(self): - wf = load_workflow() - on = get_on(wf) - pr = on["pull_request"] - types = pr.get("types", []) - assert "synchronize" in types, ( - "pull_request trigger must include 'synchronize' type" - ) - - def test_has_workflow_dispatch_trigger(self): - """The orchestrator needs to re-trigger reviews via workflow_dispatch.""" - wf = load_workflow() - on = get_on(wf) - assert "workflow_dispatch" in on, ( - "Must have workflow_dispatch trigger so orchestrator can re-trigger reviews" - ) - - def test_workflow_dispatch_has_pr_number_input(self): - """workflow_dispatch must accept a PR number input.""" - wf = load_workflow() - on = get_on(wf) - wd = on["workflow_dispatch"] - assert "inputs" in wd, "workflow_dispatch must have inputs" - inputs = wd["inputs"] - # Should have a pr-number or pr_number input - has_pr_input = any( - "pr" in key.lower() for key in inputs.keys() - ) - assert has_pr_input, ( - "workflow_dispatch must have a PR number input" - ) - - -# ── Permissions ────────────────────────────────────────────────────── - - -class TestPermissions: - def _get_permissions(self): - wf = load_workflow() - return wf.get("permissions", {}) - - def test_has_contents_read(self): - perms = self._get_permissions() - assert perms.get("contents") in ("read", "write"), ( - "Needs contents:read to check out repo" - ) - - def test_has_issues_write(self): - perms = self._get_permissions() - assert perms.get("issues") == "write", ( - "Needs issues:write to create child issues for findings" - ) - - def test_has_pull_requests_write(self): - perms = self._get_permissions() - assert perms.get("pull-requests") == "write", ( - "Needs pull-requests:write for claude-code-action to post review comments" - ) - - -# ── Jobs: Three Parallel Reviewers ─────────────────────────────────── - - -class TestReviewerJobs: - """The workflow must have three separate jobs for parallel reviewer execution.""" - - def _get_jobs(self): - wf = load_workflow() - return wf.get("jobs", {}) - - def test_has_at_least_three_jobs(self): - jobs = self._get_jobs() - assert len(jobs) >= 3, ( - f"Workflow must have at least 3 jobs (one per reviewer), found {len(jobs)}" - ) - - def test_has_correctness_reviewer_job(self): - jobs = self._get_jobs() - correctness_jobs = [ - k for k in jobs if "correctness" in k.lower() - ] - assert len(correctness_jobs) >= 1, ( - "Must have a job for the correctness reviewer" - ) - - def test_has_tests_reviewer_job(self): - jobs = self._get_jobs() - test_jobs = [ - k for k in jobs if "test" in k.lower() - ] - assert len(test_jobs) >= 1, ( - "Must have a job for the tests reviewer" - ) - - def test_has_architecture_reviewer_job(self): - jobs = self._get_jobs() - arch_jobs = [ - k for k in jobs if "architecture" in k.lower() - ] - assert len(arch_jobs) >= 1, ( - "Must have a job for the architecture reviewer" - ) - - def test_reviewer_jobs_are_independent(self): - """Reviewer jobs must not depend on each other (parallel execution).""" - jobs = self._get_jobs() - reviewer_keys = [ - k for k in jobs - if "correctness" in k.lower() - or "test" in k.lower() - or "architecture" in k.lower() - ] - for key in reviewer_keys: - needs = jobs[key].get("needs", []) - # needs should not reference other reviewer jobs - other_reviewers = [ - r for r in reviewer_keys if r != key - ] - for dep in (needs if isinstance(needs, list) else [needs]): - assert dep not in other_reviewers, ( - f"Reviewer job '{key}' depends on '{dep}' — reviewers must run in parallel" - ) - - def test_all_reviewer_jobs_run_on_ubuntu(self): - jobs = self._get_jobs() - reviewer_keys = [ - k for k in jobs - if "correctness" in k.lower() - or "test" in k.lower() - or "architecture" in k.lower() - ] - for key in reviewer_keys: - runs_on = jobs[key].get("runs-on", "") - assert "ubuntu" in runs_on, ( - f"Reviewer job '{key}' must run on ubuntu" - ) - - -# ── Parse Parent Issue ─────────────────────────────────────────────── - - -class TestParseParentIssue: - """The workflow must parse 'fixes #N' or 'Fixes #N' from the PR description.""" - - def _get_all_step_content(self): - """Collect all run/script content from all jobs.""" - wf = load_workflow() - content_parts = [] - for job_name, job in wf.get("jobs", {}).items(): - for step in job.get("steps", []): - # Collect 'run' scripts - if "run" in step: - content_parts.append(step["run"]) - # Collect github-script content - if step.get("uses", "").startswith("actions/github-script"): - script = step.get("with", {}).get("script", "") - content_parts.append(script) - # Collect env vars - env = step.get("env", {}) - for v in env.values(): - if isinstance(v, str): - content_parts.append(v) - # Also collect job-level env - env = job.get("env", {}) - for v in env.values(): - if isinstance(v, str): - content_parts.append(v) - return "\n".join(content_parts) - - def test_parses_fixes_reference(self): - """Must extract parent issue number from PR body 'fixes #N' or 'Fixes #N'.""" - content = self._get_all_step_content() - # Should contain a regex or string match for fixes #N (case-insensitive) - has_fixes_pattern = ( - re.search(r"[Ff]ixes\s*#", content) - or "fixes" in content.lower() - ) - assert has_fixes_pattern, ( - "Workflow must parse 'fixes #N' from PR description" - ) - - -# ── Each Reviewer Job Steps ────────────────────────────────────────── - - -class TestReviewerJobSteps: - """Each reviewer job must: checkout repo, then invoke claude-code-action with appropriate skill.""" - - def _get_reviewer_jobs(self): - wf = load_workflow() - jobs = wf.get("jobs", {}) - return { - k: v for k, v in jobs.items() - if "correctness" in k.lower() - or "test" in k.lower() - or "architecture" in k.lower() - } - - def _get_all_steps_content(self, job): - """Get all step run/script/prompt/uses content for a job.""" - parts = [] - for step in job.get("steps", []): - if "run" in step: - parts.append(step["run"]) - uses = step.get("uses", "") - if uses: - parts.append(uses) - with_block = step.get("with", {}) - if "script" in with_block: - parts.append(with_block["script"]) - if "prompt" in with_block: - parts.append(with_block["prompt"]) - return "\n".join(parts) - - def test_each_reviewer_checks_out_repo(self): - """Each reviewer job must have a checkout step.""" - reviewer_jobs = self._get_reviewer_jobs() - for job_name, job in reviewer_jobs.items(): - steps = job.get("steps", []) - checkout = [ - s for s in steps - if s.get("uses", "").startswith("actions/checkout") - ] - assert len(checkout) >= 1, ( - f"Reviewer job '{job_name}' must have a checkout step" - ) - - def test_each_reviewer_uses_claude_code_action(self): - """Each reviewer job must use anthropics/claude-code-action.""" - reviewer_jobs = self._get_reviewer_jobs() - for job_name, job in reviewer_jobs.items(): - steps = job.get("steps", []) - has_action = any( - "anthropics/claude-code-action" in s.get("uses", "") - for s in steps - ) - assert has_action, ( - f"Reviewer job '{job_name}' must use anthropics/claude-code-action" - ) - - def test_correctness_reviewer_references_skill(self): - """Correctness reviewer must reference the correctness skill.""" - wf = load_workflow() - jobs = wf.get("jobs", {}) - correctness_jobs = { - k: v for k, v in jobs.items() - if "correctness" in k.lower() - } - for job_name, job in correctness_jobs.items(): - content = self._get_all_steps_content(job) - assert "reviewer-correctness" in content or "correctness" in content.lower(), ( - f"Correctness job '{job_name}' must reference the correctness reviewer skill" - ) - - def test_tests_reviewer_references_skill(self): - """Tests reviewer must reference the test reviewer skill.""" - wf = load_workflow() - jobs = wf.get("jobs", {}) - test_jobs = { - k: v for k, v in jobs.items() - if "test" in k.lower() - } - for job_name, job in test_jobs.items(): - content = self._get_all_steps_content(job) - assert "reviewer-tests" in content or "test" in content.lower(), ( - f"Tests job '{job_name}' must reference the test reviewer skill" - ) - - def test_architecture_reviewer_references_skill(self): - """Architecture reviewer must reference the architecture reviewer skill.""" - wf = load_workflow() - jobs = wf.get("jobs", {}) - arch_jobs = { - k: v for k, v in jobs.items() - if "architecture" in k.lower() - } - for job_name, job in arch_jobs.items(): - content = self._get_all_steps_content(job) - assert "reviewer-architecture" in content or "architecture" in content.lower(), ( - f"Architecture job '{job_name}' must reference the architecture reviewer skill" - ) - - -# ── Context Passing ────────────────────────────────────────────────── - - -class TestContextPassing: - """Each reviewer must receive PR number, parent issue number, and repo info.""" - - def _get_all_content(self): - """Get all run/script/prompt/env content from all jobs.""" - wf = load_workflow() - parts = [] - for job_name, job in wf.get("jobs", {}).items(): - # Job-level env - for v in job.get("env", {}).values(): - if isinstance(v, str): - parts.append(v) - for step in job.get("steps", []): - if "run" in step: - parts.append(step["run"]) - with_block = step.get("with", {}) - if "script" in with_block: - parts.append(with_block["script"]) - if "prompt" in with_block: - parts.append(with_block["prompt"]) - for v in step.get("env", {}).values(): - if isinstance(v, str): - parts.append(v) - return "\n".join(parts) - - def test_passes_pr_number(self): - content = self._get_all_content() - # Should reference pull_request number or PR number - has_pr_num = ( - "pull_request" in content - or "pr_number" in content.lower() - or "pr-number" in content.lower() - or "PR_NUMBER" in content - ) - assert has_pr_num, ( - "Workflow must pass PR number to reviewer" - ) - - def test_passes_parent_issue_number(self): - content = self._get_all_content() - has_parent = ( - "parent" in content.lower() - or "issue" in content.lower() - or "PARENT_ISSUE" in content - ) - assert has_parent, ( - "Workflow must pass parent issue number to reviewer" - ) - - def test_passes_repo_info(self): - content = self._get_all_content() - has_repo = ( - "github.repository" in content - or "repo.owner" in content - or "GITHUB_REPOSITORY" in content - or "context.repo" in content - ) - assert has_repo, ( - "Workflow must pass repo owner/name to reviewer" - ) - - -# ── Reviewer Prompt Content ────────────────────────────────────────── - - -class TestReviewerPromptContent: - """The prompt passed to claude-code-action must instruct the reviewer correctly.""" - - def _get_all_content(self): - wf = load_workflow() - parts = [] - for job_name, job in wf.get("jobs", {}).items(): - for v in job.get("env", {}).values(): - if isinstance(v, str): - parts.append(v) - for step in job.get("steps", []): - if "run" in step: - parts.append(step["run"]) - with_block = step.get("with", {}) - if "script" in with_block: - parts.append(with_block["script"]) - if "prompt" in with_block: - parts.append(with_block["prompt"]) - for v in step.get("env", {}).values(): - if isinstance(v, str): - parts.append(v) - return "\n".join(parts) - - def test_prompt_references_skill_files(self): - content = self._get_all_content() - assert "reviewer-correctness/SKILL.md" in content, ( - "Prompt must reference the correctness reviewer skill file" - ) - assert "reviewer-tests/SKILL.md" in content, ( - "Prompt must reference the tests reviewer skill file" - ) - assert "reviewer-architecture/SKILL.md" in content, ( - "Prompt must reference the architecture reviewer skill file" - ) - - def test_prompt_references_github_issues_skill(self): - content = self._get_all_content() - assert "github-issues/SKILL.md" in content, ( - "Prompt must reference the github-issues skill for GraphQL patterns" - ) - - def test_prompt_passes_parent_issue_for_filing(self): - content = self._get_all_content() - assert "sub-issue" in content.lower() or "finding" in content.lower(), ( - "Prompt must instruct filing findings against the parent issue" - ) - - def test_prompt_passes_base_branch(self): - content = self._get_all_content() - assert "base" in content.lower() and "branch" in content.lower(), ( - "Prompt must pass the base branch for diffing" - ) - - -# ── Secrets ────────────────────────────────────────────────────────── - - -class TestSecrets: - """Workflow must use ANTHROPIC_API_KEY secret for claude-code-action.""" - - def _get_all_content(self): - wf = load_workflow() - parts = [] - for job_name, job in wf.get("jobs", {}).items(): - for v in job.get("env", {}).values(): - if isinstance(v, str): - parts.append(v) - for step in job.get("steps", []): - if "run" in step: - parts.append(step["run"]) - for v in step.get("env", {}).values(): - if isinstance(v, str): - parts.append(v) - return "\n".join(parts) - - def test_references_anthropic_api_key(self): - content = self._get_all_content() - assert "ANTHROPIC_API_KEY" in content, ( - "Workflow must reference ANTHROPIC_API_KEY secret for claude -p" - ) - - -# ── Resolve Context Job ─────────────────────────────────────────────── - - -class TestResolveContextJob: - """The resolve-context job extracts PR metadata for reviewer jobs.""" - - def _get_context_job(self): - wf = load_workflow() - jobs = wf.get("jobs", {}) - ctx_jobs = { - k: v for k, v in jobs.items() - if "context" in k.lower() or "resolve" in k.lower() - } - assert len(ctx_jobs) >= 1, ( - "Must have a resolve-context job to extract PR metadata" - ) - return list(ctx_jobs.values())[0] - - def test_resolve_context_job_exists(self): - self._get_context_job() - - def test_resolve_context_has_outputs(self): - job = self._get_context_job() - outputs = job.get("outputs", {}) - assert len(outputs) >= 2, ( - "resolve-context job must export outputs (at least pr-number and parent-issue)" - ) - - def test_resolve_context_outputs_pr_number(self): - job = self._get_context_job() - outputs = job.get("outputs", {}) - has_pr = any("pr" in k.lower() for k in outputs.keys()) - assert has_pr, ( - "resolve-context must output a PR number" - ) - - def test_resolve_context_outputs_parent_issue(self): - job = self._get_context_job() - outputs = job.get("outputs", {}) - has_parent = any( - "parent" in k.lower() or "issue" in k.lower() - for k in outputs.keys() - ) - assert has_parent, ( - "resolve-context must output a parent issue number" - ) - - def test_reviewer_jobs_depend_on_context(self): - """All reviewer jobs must depend on the resolve-context job.""" - wf = load_workflow() - jobs = wf.get("jobs", {}) - # Find the context job key - ctx_key = None - for k in jobs: - if "context" in k.lower() or "resolve" in k.lower(): - ctx_key = k - break - assert ctx_key is not None - - reviewer_keys = [ - k for k in jobs - if "correctness" in k.lower() - or "test" in k.lower() - or "architecture" in k.lower() - ] - for key in reviewer_keys: - needs = jobs[key].get("needs", []) - if isinstance(needs, str): - needs = [needs] - assert ctx_key in needs, ( - f"Reviewer job '{key}' must depend on '{ctx_key}'" - ) - - def test_reviewer_jobs_skip_when_no_parent_issue(self): - """Reviewer jobs should have an 'if' condition to skip when no parent issue.""" - wf = load_workflow() - jobs = wf.get("jobs", {}) - reviewer_keys = [ - k for k in jobs - if "correctness" in k.lower() - or "test" in k.lower() - or "architecture" in k.lower() - ] - for key in reviewer_keys: - job_if = jobs[key].get("if", "") - assert "parent" in job_if.lower() or "issue" in job_if.lower(), ( - f"Reviewer job '{key}' must have an 'if' condition checking for parent issue" - ) - - def test_handles_workflow_dispatch(self): - """Resolve-context must handle both pull_request and workflow_dispatch events.""" - job = self._get_context_job() - steps_content = [] - for step in job.get("steps", []): - if "run" in step: - steps_content.append(step["run"]) - if step.get("uses", "").startswith("actions/github-script"): - steps_content.append(step.get("with", {}).get("script", "")) - content = "\n".join(steps_content) - assert "workflow_dispatch" in content, ( - "resolve-context must handle workflow_dispatch event" - ) - - -# ── Config Reading ─────────────────────────────────────────────────── - - -class TestConfigReading: - """Workflow should read config.yaml for settings like re-review-cycle-cap.""" - - def test_config_has_re_review_cycle_cap(self): - config = load_config() - assert "re-review-cycle-cap" in config, ( - "Config must have 're-review-cycle-cap' setting" - ) - - def test_re_review_cycle_cap_default(self): - config = load_config() - assert config["re-review-cycle-cap"] == 3, ( - "Default re-review-cycle-cap should be 3" - ) diff --git a/tests/validate-workflows.js b/tests/validate-workflows.js new file mode 100644 index 0000000..204c15c --- /dev/null +++ b/tests/validate-workflows.js @@ -0,0 +1,225 @@ +const { test } = require('node:test'); +const assert = require('node:assert'); +const fs = require('node:fs'); +const path = require('node:path'); +const yaml = require('js-yaml'); + +const workflowsDir = path.join(__dirname, '../.github/workflows'); + +// Get all workflow files +const workflowFiles = fs.readdirSync(workflowsDir) + .filter(f => f.endsWith('.yml') && f !== '.gitkeep') + .map(f => path.join(workflowsDir, f)); + +// Expected workflow files +const expectedWorkflows = [ + 'guardrail-api-surface.yml', + 'guardrail-commits.yml', + 'guardrail-dependencies.yml', + 'guardrail-scope.yml', + 'guardrail-test-ratio.yml', + 'human-review.yml', + 'orchestrator-check.yml', + 'pr-review.yml' +]; + +test('All expected workflow files exist', () => { + const actualFiles = fs.readdirSync(workflowsDir) + .filter(f => f.endsWith('.yml') && f !== '.gitkeep'); + + for (const expected of expectedWorkflows) { + assert.ok(actualFiles.includes(expected), `Missing workflow: ${expected}`); + } +}); + +test('All workflow files are valid YAML', () => { + for (const file of workflowFiles) { + const content = fs.readFileSync(file, 'utf8'); + + // Should not throw + const parsed = yaml.load(content); + assert.ok(parsed, `Failed to parse ${path.basename(file)}`); + } +}); + +test('All workflows have required top-level fields', () => { + for (const file of workflowFiles) { + const content = fs.readFileSync(file, 'utf8'); + const workflow = yaml.load(content); + const name = path.basename(file); + + assert.ok(workflow.name, `${name}: missing 'name' field`); + assert.ok(workflow.on, `${name}: missing 'on' field`); + assert.ok(workflow.permissions, `${name}: missing 'permissions' field`); + assert.ok(workflow.jobs, `${name}: missing 'jobs' field`); + } +}); + +test('All workflows have at least one job', () => { + for (const file of workflowFiles) { + const content = fs.readFileSync(file, 'utf8'); + const workflow = yaml.load(content); + const name = path.basename(file); + + const jobNames = Object.keys(workflow.jobs); + assert.ok(jobNames.length > 0, `${name}: no jobs defined`); + } +}); + +test('All jobs have required fields', () => { + for (const file of workflowFiles) { + const content = fs.readFileSync(file, 'utf8'); + const workflow = yaml.load(content); + const workflowName = path.basename(file); + + for (const [jobName, job] of Object.entries(workflow.jobs)) { + assert.ok(job['runs-on'], `${workflowName}:${jobName}: missing 'runs-on' field`); + assert.ok(job.steps, `${workflowName}:${jobName}: missing 'steps' field`); + assert.ok(Array.isArray(job.steps), `${workflowName}:${jobName}: 'steps' must be an array`); + assert.ok(job.steps.length > 0, `${workflowName}:${jobName}: must have at least one step`); + } + } +}); + +test('Guardrail workflows have correct structure', () => { + const guardrailFiles = workflowFiles.filter(f => path.basename(f).startsWith('guardrail-')); + + for (const file of guardrailFiles) { + const content = fs.readFileSync(file, 'utf8'); + const workflow = yaml.load(content); + const name = path.basename(file); + + // Check trigger + assert.ok(workflow.on.pull_request, `${name}: should trigger on pull_request`); + + // Check permissions + assert.ok(workflow.permissions.checks === 'write', `${name}: should have checks: write permission`); + assert.ok(workflow.permissions.contents === 'read', `${name}: should have contents: read permission`); + assert.ok(workflow.permissions['pull-requests'] === 'read', `${name}: should have pull-requests: read permission`); + + // Check that job uses github-script action + const jobs = Object.values(workflow.jobs); + const hasGithubScript = jobs.some(job => + job.steps.some(step => step.uses && step.uses.includes('actions/github-script')) + ); + assert.ok(hasGithubScript, `${name}: should use actions/github-script`); + } +}); + +test('PR review workflow has correct structure', () => { + const file = workflowFiles.find(f => path.basename(f) === 'pr-review.yml'); + assert.ok(file, 'pr-review.yml not found'); + + const content = fs.readFileSync(file, 'utf8'); + const workflow = yaml.load(content); + + // Check triggers + assert.ok(workflow.on.pull_request, 'should trigger on pull_request'); + assert.ok(workflow.on.workflow_dispatch, 'should support workflow_dispatch'); + + // Check permissions + assert.ok(workflow.permissions.contents === 'read', 'should have contents: read'); + assert.ok(workflow.permissions.issues === 'write', 'should have issues: write'); + assert.ok(workflow.permissions['pull-requests'] === 'write', 'should have pull-requests: write'); + assert.ok(workflow.permissions['id-token'] === 'write', 'should have id-token: write'); + + // Check for resolve-context job + assert.ok(workflow.jobs['resolve-context'], 'should have resolve-context job'); + + // Check for reviewer jobs + assert.ok(workflow.jobs['review-correctness'], 'should have review-correctness job'); + assert.ok(workflow.jobs['review-tests'], 'should have review-tests job'); + assert.ok(workflow.jobs['review-architecture'], 'should have review-architecture job'); + + // Check job dependencies + assert.ok(workflow.jobs['review-correctness'].needs === 'resolve-context', 'review-correctness should depend on resolve-context'); + assert.ok(workflow.jobs['review-tests'].needs === 'resolve-context', 'review-tests should depend on resolve-context'); + assert.ok(workflow.jobs['review-architecture'].needs === 'resolve-context', 'review-architecture should depend on resolve-context'); +}); + +test('Orchestrator workflow has correct structure', () => { + const file = workflowFiles.find(f => path.basename(f) === 'orchestrator-check.yml'); + assert.ok(file, 'orchestrator-check.yml not found'); + + const content = fs.readFileSync(file, 'utf8'); + const workflow = yaml.load(content); + + // Check triggers + assert.ok(workflow.on.issues, 'should trigger on issues events'); + assert.ok(workflow.on.pull_request, 'should trigger on pull_request events'); + + // Check permissions + assert.ok(workflow.permissions.checks === 'write', 'should have checks: write'); + assert.ok(workflow.permissions.issues === 'read', 'should have issues: read'); + assert.ok(workflow.permissions['pull-requests'] === 'read', 'should have pull-requests: read'); + assert.ok(workflow.permissions.contents === 'read', 'should have contents: read'); + assert.ok(workflow.permissions.actions === 'write', 'should have actions: write'); + + // Check for orchestrator job + assert.ok(workflow.jobs.orchestrator, 'should have orchestrator job'); + + // Check for CLAUDE_CODE_OAUTH_TOKEN in env + const orchestratorJob = workflow.jobs.orchestrator; + const hasTokenEnv = orchestratorJob.steps.some(step => + step.env && step.env.CLAUDE_CODE_OAUTH_TOKEN + ); + assert.ok(hasTokenEnv, 'should have CLAUDE_CODE_OAUTH_TOKEN in env'); +}); + +test('Human review workflow has correct structure', () => { + const file = workflowFiles.find(f => path.basename(f) === 'human-review.yml'); + assert.ok(file, 'human-review.yml not found'); + + const content = fs.readFileSync(file, 'utf8'); + const workflow = yaml.load(content); + + // Check trigger + assert.ok(workflow.on.pull_request_review, 'should trigger on pull_request_review'); + + // Check permissions + assert.ok(workflow.permissions.issues === 'write', 'should have issues: write'); + assert.ok(workflow.permissions.contents === 'read', 'should have contents: read'); + assert.ok(workflow.permissions['pull-requests'] === 'write', 'should have pull-requests: write'); +}); + +test('All workflow steps have names', () => { + for (const file of workflowFiles) { + const content = fs.readFileSync(file, 'utf8'); + const workflow = yaml.load(content); + const workflowName = path.basename(file); + + for (const [jobName, job] of Object.entries(workflow.jobs)) { + for (let i = 0; i < job.steps.length; i++) { + const step = job.steps[i]; + assert.ok(step.name, `${workflowName}:${jobName}:step[${i}]: missing 'name' field`); + } + } + } +}); + +test('Workflows using github-script reference correct script paths', () => { + for (const file of workflowFiles) { + const content = fs.readFileSync(file, 'utf8'); + const workflow = yaml.load(content); + const workflowName = path.basename(file); + + for (const [jobName, job] of Object.entries(workflow.jobs)) { + for (const step of job.steps) { + if (step.uses && step.uses.includes('actions/github-script')) { + assert.ok(step.with && step.with.script, + `${workflowName}:${jobName}: github-script step must have 'with.script'`); + + // Check if script references a file + if (step.with.script.includes('require(')) { + const scriptPath = step.with.script.match(/require\('([^']+)'\)/)?.[1]; + if (scriptPath && scriptPath.startsWith('./.github/agent-workflow/scripts/')) { + const scriptFile = path.join(__dirname, '..', scriptPath); + assert.ok(fs.existsSync(scriptFile), + `${workflowName}:${jobName}: referenced script ${scriptPath} does not exist`); + } + } + } + } + } + } +}); From e55ff1cfbd65bd27e10fe5fb8731b6e445d7239a Mon Sep 17 00:00:00 2001 From: Joe Delfino Date: Fri, 13 Feb 2026 01:21:43 +0000 Subject: [PATCH 15/23] ci: add CI workflow to run test suite Added .github/workflows/ci.yml that: - Runs on PRs and pushes to main - Sets up Node.js LTS with npm cache - Installs dependencies and runs npm test - Reports test results Note: This CI workflow is specific to the agent-workflow repo itself and should NOT be copied to target repos when install.sh is created. The installer should exclude: - .github/workflows/ci.yml - tests/ - root package.json and package-lock.json Co-Authored-By: Claude Sonnet 4.5 --- .github/workflows/ci.yml | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..fb256ce --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,40 @@ +name: CI + +on: + pull_request: + types: [opened, synchronize] + push: + branches: [main] + +permissions: + contents: read + +jobs: + test: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: 'lts/*' + cache: 'npm' + + - name: Install dependencies + run: npm ci + + - name: Run tests + run: npm test + + - name: Report test results + if: always() + run: | + echo "Test suite complete" + if [ $? -eq 0 ]; then + echo "✅ All tests passed" + else + echo "❌ Some tests failed" + exit 1 + fi From 05f02de73a7a61b6b5372985eef88f84dff4d3a4 Mon Sep 17 00:00:00 2001 From: Joe Delfino Date: Fri, 13 Feb 2026 01:37:41 +0000 Subject: [PATCH 16/23] fix: correct bugs in file-patterns and fixes-parser libs - file-patterns: Fix isTestFile to match paths starting with "tests/" Changed /(^|\/)tests?\// to match "tests/" at start of path or after slash - fixes-parser: Make regex truly case-insensitive using /i flag Changed /[Ff]ixes/ to /fixes/gi for proper case-insensitive matching Fixes the 2 failing tests in the CI (now 53/53 passing) Co-Authored-By: Claude Sonnet 4.5 --- .github/agent-workflow/scripts/lib/file-patterns.js | 2 +- .github/agent-workflow/scripts/lib/fixes-parser.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/agent-workflow/scripts/lib/file-patterns.js b/.github/agent-workflow/scripts/lib/file-patterns.js index 9b05ed9..6f0ccff 100644 --- a/.github/agent-workflow/scripts/lib/file-patterns.js +++ b/.github/agent-workflow/scripts/lib/file-patterns.js @@ -6,7 +6,7 @@ function isTestFile(filename) { return /\.(test|spec)\.(js|ts|jsx|tsx|py|go|rs)$/.test(filename) || /__tests__\//.test(filename) || - /\/tests?\//.test(filename) || + /(^|\/)tests?\//.test(filename) || /_test\.(go|rs)$/.test(filename); } diff --git a/.github/agent-workflow/scripts/lib/fixes-parser.js b/.github/agent-workflow/scripts/lib/fixes-parser.js index d13e9a8..313114e 100644 --- a/.github/agent-workflow/scripts/lib/fixes-parser.js +++ b/.github/agent-workflow/scripts/lib/fixes-parser.js @@ -6,7 +6,7 @@ function parseFixesReferences(body) { if (!body) return []; - const regex = /[Ff]ixes\s+#(\d+)/g; + const regex = /fixes\s+#(\d+)/gi; const matches = []; const seen = new Set(); From 3dd6bf51c4b5e306ace16e264071ec1e7917804b Mon Sep 17 00:00:00 2001 From: Joe Delfino Date: Fri, 13 Feb 2026 01:40:34 +0000 Subject: [PATCH 17/23] fix: add checkout step to pr-review resolve-context job The resolve-context job was trying to require() the pr-context.js script without checking out the repository first, causing MODULE_NOT_FOUND errors. Added checkout step with sparse-checkout for .github/agent-workflow to the resolve-context job so the script file is available. Co-Authored-By: Claude Sonnet 4.5 --- .github/workflows/pr-review.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/pr-review.yml b/.github/workflows/pr-review.yml index a25e2d6..cbdb8d0 100644 --- a/.github/workflows/pr-review.yml +++ b/.github/workflows/pr-review.yml @@ -26,6 +26,11 @@ jobs: base-branch: ${{ steps.ctx.outputs.base-branch }} pr-title: ${{ steps.ctx.outputs.pr-title }} steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + sparse-checkout: .github/agent-workflow + - name: Resolve PR context id: ctx uses: actions/github-script@v7 From 34b7a3d11e35e2e75d01e86c4c2d079f630f0ae1 Mon Sep 17 00:00:00 2001 From: Joe Delfino Date: Fri, 13 Feb 2026 01:48:22 +0000 Subject: [PATCH 18/23] feat: add glob pattern support to scope matcher Enables issue scope definitions to use glob patterns (* and **): - Single wildcard (*) matches files in same directory - Double wildcard (**) matches files recursively - Examples: `lib/*.js`, `.github/**/*.yml` Implementation: - Updated extractFilePaths() to capture glob patterns - Added globToRegex() converter with placeholder approach to avoid regex conflicts during pattern replacement - Enhanced isInScope() to detect and match glob patterns - Added 7 comprehensive tests for glob functionality Also fixes guardrail-test-ratio to count test files in addition to code files for accurate line totals. Co-Authored-By: Claude Sonnet 4.5 --- .../scripts/guardrail-test-ratio.js | 2 +- .../scripts/lib/scope-matcher.js | 74 +++++++++++++++---- tests/lib/scope-matcher.test.js | 48 +++++++++++- 3 files changed, 107 insertions(+), 17 deletions(-) diff --git a/.github/agent-workflow/scripts/guardrail-test-ratio.js b/.github/agent-workflow/scripts/guardrail-test-ratio.js index a67b8bb..6cdc40f 100644 --- a/.github/agent-workflow/scripts/guardrail-test-ratio.js +++ b/.github/agent-workflow/scripts/guardrail-test-ratio.js @@ -46,7 +46,7 @@ module.exports = async function({ github, context, core }) { for (const file of allFiles) { if (file.status === 'removed') continue; - if (!isCodeFile(file.filename)) continue; + if (!isCodeFile(file.filename) && !isTestFile(file.filename)) continue; const additions = file.additions || 0; diff --git a/.github/agent-workflow/scripts/lib/scope-matcher.js b/.github/agent-workflow/scripts/lib/scope-matcher.js index d72a161..85b3d4c 100644 --- a/.github/agent-workflow/scripts/lib/scope-matcher.js +++ b/.github/agent-workflow/scripts/lib/scope-matcher.js @@ -1,18 +1,20 @@ /** - * Extract file paths from issue text + * Extract file paths and glob patterns from issue text * @param {string} text - Issue body or comment text - * @returns {string[]} - Array of unique file paths + * @returns {string[]} - Array of unique file paths and patterns */ function extractFilePaths(text) { if (!text) return []; const filePathPatterns = [ - // Backtick-wrapped paths with extension - /`([a-zA-Z0-9_./-]+\.[a-zA-Z0-9]+)`/g, - // Bare paths with at least one slash and an extension - /(?:^|\s)((?:[a-zA-Z0-9_.-]+\/)+[a-zA-Z0-9_.-]+\.[a-zA-Z0-9]+)(?:\s|$|[,;)])/gm, - // Paths starting with ./ or common root dirs - /(?:^|\s)(\.?(?:src|lib|app|test|tests|spec|pkg|cmd|internal|\.github)\/[a-zA-Z0-9_./-]+)(?:\s|$|[,;)])/gm + // Backtick-wrapped paths with extension or glob patterns (now includes *) + /`([a-zA-Z0-9_./*-]+\.[a-zA-Z0-9]+)`/g, + // Backtick-wrapped paths with wildcards + /`([a-zA-Z0-9_./*-]+\/\*+[a-zA-Z0-9_./*-]*)`/g, + // Bare paths with at least one slash and an extension (including *) + /(?:^|\s)((?:[a-zA-Z0-9_.*-]+\/)+[a-zA-Z0-9_.*-]+\.[a-zA-Z0-9]+)(?:\s|$|[,;)])/gm, + // Paths starting with ./ or common root dirs (including *) + /(?:^|\s)(\.?(?:src|lib|app|test|tests|spec|pkg|cmd|internal|\.github)\/[a-zA-Z0-9_./*-]+)(?:\s|$|[,;)])/gm ]; const paths = new Set(); @@ -28,23 +30,65 @@ function extractFilePaths(text) { return Array.from(paths); } +/** + * Convert a glob pattern to a regular expression + * @param {string} pattern - Glob pattern (supports * and **) + * @returns {RegExp} - Regular expression matching the pattern + */ +function globToRegex(pattern) { + // Escape special regex characters except * and / + let regexStr = pattern.replace(/[.+?^${}()|[\]\\]/g, '\\$&'); + + // Use placeholders to protect our regex patterns from later replacements + const PLACEHOLDER_A = '\x00A\x00'; // For **/ + const PLACEHOLDER_B = '\x00B\x00'; // For /** + const PLACEHOLDER_C = '\x00C\x00'; // For ** + + // Replace **/ with a pattern that matches zero or more path segments + // This allows lib/**/*.js to match both lib/file.js and lib/sub/file.js + regexStr = regexStr.replace(/\*\*\//g, PLACEHOLDER_A); + + // Replace /** at the end with a pattern that matches anything + regexStr = regexStr.replace(/\/\*\*$/g, PLACEHOLDER_B); + + // Replace remaining ** with .* (matches anything including /) + regexStr = regexStr.replace(/\*\*/g, PLACEHOLDER_C); + + // Replace single * with regex that matches anything except / + regexStr = regexStr.replace(/\*/g, '[^/]*'); + + // Now replace placeholders with actual regex patterns + regexStr = regexStr.replace(new RegExp(PLACEHOLDER_A, 'g'), '(?:(?:[^/]+/)*)'); + regexStr = regexStr.replace(new RegExp(PLACEHOLDER_B, 'g'), '(?:/.*)?'); + regexStr = regexStr.replace(new RegExp(PLACEHOLDER_C, 'g'), '.*'); + + // Anchor the pattern to match the full path + return new RegExp('^' + regexStr + '$'); +} + /** * Check if a changed file path is within scope * @param {string} changedPath - Path of changed file - * @param {string[]} scopeFiles - Array of scope file paths/prefixes + * @param {string[]} scopeFiles - Array of scope file paths/prefixes/patterns * @returns {boolean} */ function isInScope(changedPath, scopeFiles) { for (const scopePath of scopeFiles) { - // Exact match - if (changedPath === scopePath) return true; + // Check if this is a glob pattern (contains * or **) + if (scopePath.includes('*')) { + const regex = globToRegex(scopePath); + if (regex.test(changedPath)) return true; + } else { + // Exact match + if (changedPath === scopePath) return true; - // Scope entry is a directory prefix - const prefix = scopePath.endsWith('/') ? scopePath : scopePath + '/'; - if (changedPath.startsWith(prefix)) return true; + // Scope entry is a directory prefix + const prefix = scopePath.endsWith('/') ? scopePath : scopePath + '/'; + if (changedPath.startsWith(prefix)) return true; + } } return false; } -module.exports = { extractFilePaths, isInScope }; +module.exports = { extractFilePaths, isInScope, globToRegex }; diff --git a/tests/lib/scope-matcher.test.js b/tests/lib/scope-matcher.test.js index f28ea76..8b088a9 100644 --- a/tests/lib/scope-matcher.test.js +++ b/tests/lib/scope-matcher.test.js @@ -1,6 +1,6 @@ const { test } = require('node:test'); const assert = require('node:assert'); -const { extractFilePaths, isInScope } = require('../../.github/agent-workflow/scripts/lib/scope-matcher.js'); +const { extractFilePaths, isInScope, globToRegex } = require('../../.github/agent-workflow/scripts/lib/scope-matcher.js'); test('extractFilePaths - backtick-wrapped paths', () => { const text = 'Modify `src/index.js` and `lib/utils.ts`'; @@ -55,3 +55,49 @@ test('isInScope - prefix without trailing slash', () => { assert.strictEqual(isInScope('src/auth/handler.js', scopeFiles), true); assert.strictEqual(isInScope('src/auth.ts', scopeFiles), false); }); + +test('extractFilePaths - glob patterns', () => { + const text = 'Files: `lib/*.js` and `.github/agent-workflow/scripts/lib/*.test.js`'; + const paths = extractFilePaths(text); + assert.ok(paths.includes('lib/*.js')); + assert.ok(paths.includes('.github/agent-workflow/scripts/lib/*.test.js')); +}); + +test('globToRegex - single wildcard', () => { + const regex = globToRegex('lib/*.js'); + assert.strictEqual(regex.test('lib/utils.js'), true); + assert.strictEqual(regex.test('lib/helper.js'), true); + assert.strictEqual(regex.test('lib/sub/utils.js'), false); // * doesn't match / + assert.strictEqual(regex.test('other/utils.js'), false); +}); + +test('globToRegex - double wildcard', () => { + const regex = globToRegex('lib/**/*.js'); + assert.strictEqual(regex.test('lib/utils.js'), true); + assert.strictEqual(regex.test('lib/sub/utils.js'), true); + assert.strictEqual(regex.test('lib/sub/deep/utils.js'), true); + assert.strictEqual(regex.test('other/utils.js'), false); +}); + +test('globToRegex - wildcard in filename', () => { + const regex = globToRegex('lib/*.test.js'); + assert.strictEqual(regex.test('lib/utils.test.js'), true); + assert.strictEqual(regex.test('lib/helper.test.js'), true); + assert.strictEqual(regex.test('lib/utils.js'), false); +}); + +test('isInScope - glob pattern matching', () => { + const scopeFiles = ['lib/*.js', '.github/workflows/*.yml']; + assert.strictEqual(isInScope('lib/utils.js', scopeFiles), true); + assert.strictEqual(isInScope('lib/helper.js', scopeFiles), true); + assert.strictEqual(isInScope('.github/workflows/ci.yml', scopeFiles), true); + assert.strictEqual(isInScope('lib/sub/utils.js', scopeFiles), false); + assert.strictEqual(isInScope('other/file.js', scopeFiles), false); +}); + +test('isInScope - double wildcard pattern', () => { + const scopeFiles = ['.github/agent-workflow/**/*.js']; + assert.strictEqual(isInScope('.github/agent-workflow/scripts/lib/config.js', scopeFiles), true); + assert.strictEqual(isInScope('.github/agent-workflow/scripts/main.js', scopeFiles), true); + assert.strictEqual(isInScope('.github/workflows/ci.yml', scopeFiles), false); +}); From cf2a64c5f19bb796784a292145f2871d5847f0a6 Mon Sep 17 00:00:00 2001 From: Joe Delfino Date: Fri, 13 Feb 2026 01:58:53 +0000 Subject: [PATCH 19/23] fix: add checkout steps to api-surface and human-review workflows Both workflows were missing the initial checkout step, causing MODULE_NOT_FOUND errors when trying to require the script files. Added sparse-checkout for .github/agent-workflow to both workflows. Co-Authored-By: Claude Sonnet 4.5 --- .github/workflows/guardrail-api-surface.yml | 5 +++++ .github/workflows/human-review.yml | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/.github/workflows/guardrail-api-surface.yml b/.github/workflows/guardrail-api-surface.yml index 1f970a2..2e68a16 100644 --- a/.github/workflows/guardrail-api-surface.yml +++ b/.github/workflows/guardrail-api-surface.yml @@ -13,6 +13,11 @@ jobs: api-surface-check: runs-on: ubuntu-latest steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + sparse-checkout: .github/agent-workflow + - name: Check API surface changes uses: actions/github-script@v7 with: diff --git a/.github/workflows/human-review.yml b/.github/workflows/human-review.yml index 43ef33d..72bb047 100644 --- a/.github/workflows/human-review.yml +++ b/.github/workflows/human-review.yml @@ -13,6 +13,11 @@ jobs: process-review: runs-on: ubuntu-latest steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + sparse-checkout: .github/agent-workflow + - name: Process review comments and create issues uses: actions/github-script@v7 with: From aa4fa2218d867e29053bd179b82b51061331613c Mon Sep 17 00:00:00 2001 From: Joe Delfino Date: Fri, 13 Feb 2026 02:52:48 +0000 Subject: [PATCH 20/23] refactor: move guardrail config from config.yaml into workflow env vars Remove the separate config.yaml configuration file and config.js parser. Configuration now lives directly in each workflow file's env: block (CONCLUSION, THRESHOLD, RE_REVIEW_CYCLE_CAP). To disable a guardrail, delete its workflow file. Also change approval-override conclusion from success to neutral so overridden checks remain visibly flagged in the checks tab. Co-Authored-By: Claude Sonnet 4.5 --- .github/agent-workflow/config.yaml | 19 ----- .../scripts/guardrail-api-surface.js | 38 +--------- .../scripts/guardrail-commits.js | 36 +--------- .../scripts/guardrail-dependencies.js | 37 +--------- .../agent-workflow/scripts/guardrail-scope.js | 32 ++------- .../scripts/guardrail-test-ratio.js | 25 ++----- .github/agent-workflow/scripts/lib/config.js | 70 ------------------- .../scripts/orchestrator-check.js | 20 ++---- .github/workflows/guardrail-api-surface.yml | 3 + .github/workflows/guardrail-commits.yml | 3 + .github/workflows/guardrail-dependencies.yml | 3 + .github/workflows/guardrail-scope.yml | 3 + .github/workflows/guardrail-test-ratio.yml | 4 ++ .github/workflows/orchestrator-check.yml | 3 + README.md | 35 ++++------ docs/design.md | 2 +- tests/lib/config.test.js | 62 ---------------- 17 files changed, 53 insertions(+), 342 deletions(-) delete mode 100644 .github/agent-workflow/config.yaml delete mode 100644 .github/agent-workflow/scripts/lib/config.js delete mode 100644 tests/lib/config.test.js diff --git a/.github/agent-workflow/config.yaml b/.github/agent-workflow/config.yaml deleted file mode 100644 index 935b8b8..0000000 --- a/.github/agent-workflow/config.yaml +++ /dev/null @@ -1,19 +0,0 @@ -re-review-cycle-cap: 3 - -guardrails: - scope-enforcement: - enabled: true - conclusion: action_required - test-ratio: - enabled: true - conclusion: action_required - threshold: 0.5 - dependency-changes: - enabled: true - conclusion: action_required - api-surface: - enabled: true - conclusion: action_required - commit-messages: - enabled: true - conclusion: neutral diff --git a/.github/agent-workflow/scripts/guardrail-api-surface.js b/.github/agent-workflow/scripts/guardrail-api-surface.js index d5b00d9..ee5df58 100644 --- a/.github/agent-workflow/scripts/guardrail-api-surface.js +++ b/.github/agent-workflow/scripts/guardrail-api-surface.js @@ -1,4 +1,3 @@ -const { parseGuardrailConfig } = require('./lib/config.js'); const { hasNonStaleApproval } = require('./lib/approval.js'); const { detectAPIChanges } = require('./lib/api-patterns.js'); @@ -8,40 +7,7 @@ module.exports = async function({ github, context, core }) { const prNumber = context.payload.pull_request.number; const headSha = context.payload.pull_request.head.sha; const checkName = 'guardrail/api-surface'; - - // Read config - let enabled = true; - let configuredConclusion = 'action_required'; - try { - const configResponse = await github.rest.repos.getContent({ - owner, - repo, - path: '.github/agent-workflow/config.yaml', - ref: headSha, - }); - const configContent = Buffer.from(configResponse.data.content, 'base64').toString('utf8'); - const config = parseGuardrailConfig(configContent, 'api-surface'); - enabled = config.enabled; - configuredConclusion = config.conclusion; - } catch (e) { - core.info('No config.yaml found, using defaults (enabled: true, conclusion: action_required)'); - } - - if (!enabled) { - await github.rest.checks.create({ - owner, - repo, - head_sha: headSha, - name: checkName, - status: 'completed', - conclusion: 'success', - output: { - title: 'API surface check: disabled', - summary: 'This check is disabled in config.yaml.', - }, - }); - return; - } + const configuredConclusion = process.env.CONCLUSION || 'action_required'; // Check for non-stale PR approval override const reviews = await github.rest.pulls.listReviews({ @@ -58,7 +24,7 @@ module.exports = async function({ github, context, core }) { head_sha: headSha, name: checkName, status: 'completed', - conclusion: 'success', + conclusion: 'neutral', output: { title: 'API surface check: approved by reviewer', summary: 'A non-stale PR approval overrides this guardrail.', diff --git a/.github/agent-workflow/scripts/guardrail-commits.js b/.github/agent-workflow/scripts/guardrail-commits.js index efedb68..cbb6f94 100644 --- a/.github/agent-workflow/scripts/guardrail-commits.js +++ b/.github/agent-workflow/scripts/guardrail-commits.js @@ -1,40 +1,8 @@ -const { parseGuardrailConfig } = require('./lib/config.js'); const { hasNonStaleApproval } = require('./lib/approval.js'); const { isValidCommit } = require('./lib/commit-validator.js'); module.exports = async function({ github, context, core }) { - const fs = require('fs'); - const path = require('path'); - - // Read config - default conclusion for commit message guardrail is 'neutral' (non-blocking warning) - let configuredConclusion = 'neutral'; - const configPath = path.join(process.env.GITHUB_WORKSPACE, '.github', 'agent-workflow', 'config.yaml'); - - try { - const configContent = fs.readFileSync(configPath, 'utf8'); - const config = parseGuardrailConfig(configContent, 'commit-messages'); - - if (config.enabled === false) { - await github.rest.checks.create({ - owner: context.repo.owner, - repo: context.repo.repo, - head_sha: context.sha, - name: 'guardrail/commit-messages', - conclusion: 'success', - output: { - title: 'Commit message check: disabled', - summary: 'This guardrail check is disabled in config.yaml.' - } - }); - return; - } - - if (config.conclusion) { - configuredConclusion = config.conclusion; - } - } catch (e) { - core.info(`No config.yaml found at ${configPath}, using default conclusion: neutral`); - } + const configuredConclusion = process.env.CONCLUSION || 'neutral'; // Check for non-stale PR approval override const reviews = await github.rest.pulls.listReviews({ @@ -52,7 +20,7 @@ module.exports = async function({ github, context, core }) { repo: context.repo.repo, head_sha: context.sha, name: 'guardrail/commit-messages', - conclusion: 'success', + conclusion: 'neutral', output: { title: 'Commit message check: approved by reviewer', summary: 'A non-stale PR approval overrides this guardrail check.' diff --git a/.github/agent-workflow/scripts/guardrail-dependencies.js b/.github/agent-workflow/scripts/guardrail-dependencies.js index 44ed247..705d0b6 100644 --- a/.github/agent-workflow/scripts/guardrail-dependencies.js +++ b/.github/agent-workflow/scripts/guardrail-dependencies.js @@ -1,10 +1,9 @@ -const { parseGuardrailConfig } = require('./lib/config.js'); const { hasNonStaleApproval } = require('./lib/approval.js'); const { isDependencyFile } = require('./lib/file-patterns.js'); module.exports = async function({ github, context, core }) { - const fs = require('fs'); const CHECK_NAME = 'guardrail/dependency-changes'; + const configuredConclusion = process.env.CONCLUSION || 'action_required'; const JUSTIFICATION_KEYWORDS = [ 'dependency', 'dependencies', @@ -21,38 +20,6 @@ module.exports = async function({ github, context, core }) { 'CVE-' ]; - // Read config - let checkEnabled = true; - let configuredConclusion = 'action_required'; - try { - const configPath = '.github/agent-workflow/config.yaml'; - if (fs.existsSync(configPath)) { - const content = fs.readFileSync(configPath, 'utf8'); - const config = parseGuardrailConfig(content, 'dependency-changes'); - checkEnabled = config.enabled; - configuredConclusion = config.conclusion; - } - } catch (e) { - core.warning(`Failed to read config: ${e.message}. Using defaults.`); - } - - // If check is disabled, report success and exit - if (!checkEnabled) { - await github.rest.checks.create({ - owner: context.repo.owner, - repo: context.repo.repo, - head_sha: context.sha, - name: CHECK_NAME, - status: 'completed', - conclusion: 'success', - output: { - title: 'Dependency changes: check disabled', - summary: 'This guardrail check is disabled in config.yaml.' - } - }); - return; - } - // Check for non-stale approving PR review (override mechanism) const { data: reviews } = await github.rest.pulls.listReviews({ owner: context.repo.owner, @@ -70,7 +37,7 @@ module.exports = async function({ github, context, core }) { head_sha: context.sha, name: CHECK_NAME, status: 'completed', - conclusion: 'success', + conclusion: 'neutral', output: { title: 'Dependency changes: approved by reviewer', summary: 'A non-stale PR approval overrides dependency change violations.' diff --git a/.github/agent-workflow/scripts/guardrail-scope.js b/.github/agent-workflow/scripts/guardrail-scope.js index 0761a10..6adaa42 100644 --- a/.github/agent-workflow/scripts/guardrail-scope.js +++ b/.github/agent-workflow/scripts/guardrail-scope.js @@ -1,10 +1,10 @@ -const { parseGuardrailConfig } = require('./lib/config.js'); const { hasNonStaleApproval } = require('./lib/approval.js'); const { parseFixesReferences } = require('./lib/fixes-parser.js'); const { extractFilePaths, isInScope } = require('./lib/scope-matcher.js'); module.exports = async function({ github, context, core }) { const checkName = 'guardrail/scope'; + const configuredConclusion = process.env.CONCLUSION || 'action_required'; // Helper: create check run async function createCheckRun(conclusion, title, summary, annotations = []) { @@ -23,31 +23,7 @@ module.exports = async function({ github, context, core }) { }); } - // Step 1: Read config and check if enabled - let config; - try { - const { data } = await github.rest.repos.getContent({ - owner: context.repo.owner, - repo: context.repo.repo, - path: '.github/agent-workflow/config.yaml', - ref: context.payload.pull_request.head.sha - }); - const content = Buffer.from(data.content, 'base64').toString('utf-8'); - config = parseGuardrailConfig(content, 'scope-enforcement'); - } catch (e) { - config = { enabled: true, conclusion: 'action_required' }; - } - - if (!config.enabled) { - await createCheckRun( - 'success', - 'Scope enforcement: disabled', - 'Scope enforcement is disabled in agent-workflow config.' - ); - return; - } - - // Step 2: Check for non-stale PR approval override + // Step 1: Check for non-stale PR approval override const reviews = await github.rest.pulls.listReviews({ owner: context.repo.owner, repo: context.repo.repo, @@ -160,7 +136,7 @@ module.exports = async function({ github, context, core }) { // There are out-of-scope files if (hasValidApproval) { await createCheckRun( - 'success', + 'neutral', `Scope enforcement: approved by reviewer (${outOfScope.length} files outside scope)`, `PR modifies ${outOfScope.length} file(s) not listed in issue #${issueNumber} or its children, but a non-stale approval exists.\n\nOut-of-scope files:\n${outOfScope.map(f => '- `' + f.filename + '`').join('\n')}` ); @@ -178,7 +154,7 @@ module.exports = async function({ github, context, core }) { // Determine conclusion based on config and severity const isMinor = outOfScope.length <= 2; - const conclusion = isMinor ? 'neutral' : config.conclusion; + const conclusion = isMinor ? 'neutral' : configuredConclusion; const summary = [ `PR modifies ${outOfScope.length} file(s) not listed in issue #${issueNumber} or its children.`, diff --git a/.github/agent-workflow/scripts/guardrail-test-ratio.js b/.github/agent-workflow/scripts/guardrail-test-ratio.js index 6cdc40f..9ff8f9b 100644 --- a/.github/agent-workflow/scripts/guardrail-test-ratio.js +++ b/.github/agent-workflow/scripts/guardrail-test-ratio.js @@ -1,26 +1,9 @@ -const { parseGuardrailConfig } = require('./lib/config.js'); const { hasNonStaleApproval } = require('./lib/approval.js'); const { isTestFile, isCodeFile } = require('./lib/file-patterns.js'); module.exports = async function({ github, context, core }) { - // Load configuration - const fs = require('fs'); - const configPath = '.github/agent-workflow/config.yaml'; - let config = { enabled: true, conclusion: 'action_required', threshold: 0.5 }; - - try { - const content = fs.readFileSync(configPath, 'utf8'); - config = { ...config, ...parseGuardrailConfig(content, 'test-ratio') }; - } catch (e) { - core.info(`Could not read config from ${configPath}, using defaults: ${e.message}`); - } - - if (!config.enabled) { - core.info('Test-ratio guardrail is disabled in config. Skipping.'); - return; - } - - const threshold = config.threshold || 0.5; + const configuredConclusion = process.env.CONCLUSION || 'action_required'; + const threshold = parseFloat(process.env.THRESHOLD) || 0.5; // Get PR files const prNumber = context.payload.pull_request.number; @@ -108,11 +91,11 @@ module.exports = async function({ github, context, core }) { title = `Test-to-code ratio: ${ratio.toFixed(2)} (threshold: ${threshold})`; summary = `PR has ${testLines} test lines and ${implLines} implementation lines added. Ratio ${ratio.toFixed(2)} meets the threshold of ${threshold}.`; } else if (hasValidApproval) { - conclusion = 'success'; + conclusion = 'neutral'; title = `Test-to-code ratio: ${ratio.toFixed(2)} — approved by reviewer`; summary = `PR has ${testLines} test lines and ${implLines} implementation lines added. Ratio ${ratio.toFixed(2)} is below the threshold of ${threshold}, but a non-stale approval exists. Human has accepted the current state.`; } else { - conclusion = config.conclusion; + conclusion = configuredConclusion; title = `Test-to-code ratio: ${ratio.toFixed(2)} (threshold: ${threshold})`; summary = `PR has ${testLines} test lines and ${implLines} implementation lines added. Ratio ${ratio.toFixed(2)} is below the threshold of ${threshold}. Add more tests or approve the PR to override.`; } diff --git a/.github/agent-workflow/scripts/lib/config.js b/.github/agent-workflow/scripts/lib/config.js deleted file mode 100644 index 0d53a3e..0000000 --- a/.github/agent-workflow/scripts/lib/config.js +++ /dev/null @@ -1,70 +0,0 @@ -/** - * Parse guardrail configuration from YAML content - * @param {string} yamlContent - Raw YAML content - * @param {string} guardrailName - Name of the guardrail (e.g., 'scope-enforcement') - * @returns {object} - Config object with enabled, conclusion, and optional threshold - */ -function parseGuardrailConfig(yamlContent, guardrailName) { - const defaults = { - enabled: true, - conclusion: 'action_required' - }; - - if (!yamlContent) return defaults; - - // Simple YAML parsing for the guardrail section - const lines = yamlContent.split('\n'); - let inGuardrails = false; - let inTarget = false; - const config = { ...defaults }; - - for (const line of lines) { - // Check if we're entering the guardrails section - if (/^guardrails:/.test(line)) { - inGuardrails = true; - continue; - } - - // Exit guardrails section if we hit another top-level key - if (inGuardrails && /^\S/.test(line) && !/^\s+/.test(line) && !/^guardrails:/.test(line)) { - break; - } - - // Check if we're entering the target guardrail section - if (inGuardrails && new RegExp(`^\\s+${guardrailName}:`).test(line)) { - inTarget = true; - continue; - } - - // Exit target section if we hit another guardrail key (at same indentation) - if (inTarget && /^\s+\S+:/.test(line) && !new RegExp(`^\\s+${guardrailName}:`).test(line)) { - const currentIndent = line.match(/^(\s+)/)?.[1].length || 0; - const targetIndent = 2; // Assuming standard 2-space YAML indent - if (currentIndent <= targetIndent) { - break; - } - } - - // Parse config values - if (inTarget) { - const enabledMatch = line.match(/^\s+enabled:\s*(true|false)/); - if (enabledMatch) { - config.enabled = enabledMatch[1] === 'true'; - } - - const conclusionMatch = line.match(/^\s+conclusion:\s*(\S+)/); - if (conclusionMatch) { - config.conclusion = conclusionMatch[1]; - } - - const thresholdMatch = line.match(/^\s+threshold:\s*([\d.]+)/); - if (thresholdMatch) { - config.threshold = parseFloat(thresholdMatch[1]); - } - } - } - - return config; -} - -module.exports = { parseGuardrailConfig }; diff --git a/.github/agent-workflow/scripts/orchestrator-check.js b/.github/agent-workflow/scripts/orchestrator-check.js index efa09b4..5ebd8f2 100644 --- a/.github/agent-workflow/scripts/orchestrator-check.js +++ b/.github/agent-workflow/scripts/orchestrator-check.js @@ -3,21 +3,9 @@ const { parseFixesReferences } = require('./lib/fixes-parser.js'); module.exports = async function({ github, context, core }) { const checkName = 'orchestrator'; - // Helper: read re-review-cycle-cap from config - async function readCycleCap() { - try { - const { data } = await github.rest.repos.getContent({ - owner: context.repo.owner, - repo: context.repo.repo, - path: '.github/agent-workflow/config.yaml', - ref: context.sha - }); - const content = Buffer.from(data.content, 'base64').toString('utf-8'); - const match = content.match(/^re-review-cycle-cap:\s*(\d+)/m); - return match ? parseInt(match[1], 10) : 3; - } catch { - return 3; // default - } + // Read re-review cycle cap from workflow env + function readCycleCap() { + return parseInt(process.env.RE_REVIEW_CYCLE_CAP, 10) || 3; } // Helper: create check run on a specific SHA @@ -234,7 +222,7 @@ module.exports = async function({ github, context, core }) { // Step 4: No blockers — assess re-review need console.log('No open blockers. Assessing re-review need...'); - const cycleCap = await readCycleCap(); + const cycleCap = readCycleCap(); const pastCycles = await countReviewCycles(prNumber); console.log(`Review cycles so far: ${pastCycles}, cap: ${cycleCap}`); diff --git a/.github/workflows/guardrail-api-surface.yml b/.github/workflows/guardrail-api-surface.yml index 2e68a16..2b425e3 100644 --- a/.github/workflows/guardrail-api-surface.yml +++ b/.github/workflows/guardrail-api-surface.yml @@ -9,6 +9,9 @@ permissions: pull-requests: read contents: read +env: + CONCLUSION: action_required # action_required | neutral + jobs: api-surface-check: runs-on: ubuntu-latest diff --git a/.github/workflows/guardrail-commits.yml b/.github/workflows/guardrail-commits.yml index 8c88160..307aa9c 100644 --- a/.github/workflows/guardrail-commits.yml +++ b/.github/workflows/guardrail-commits.yml @@ -9,6 +9,9 @@ permissions: contents: read pull-requests: read +env: + CONCLUSION: neutral # action_required | neutral + jobs: commit-messages: runs-on: ubuntu-latest diff --git a/.github/workflows/guardrail-dependencies.yml b/.github/workflows/guardrail-dependencies.yml index bc87f9e..f43e5a2 100644 --- a/.github/workflows/guardrail-dependencies.yml +++ b/.github/workflows/guardrail-dependencies.yml @@ -10,6 +10,9 @@ permissions: pull-requests: read issues: read +env: + CONCLUSION: action_required # action_required | neutral + jobs: dependency-changes: runs-on: ubuntu-latest diff --git a/.github/workflows/guardrail-scope.yml b/.github/workflows/guardrail-scope.yml index 2d4992f..c496e18 100644 --- a/.github/workflows/guardrail-scope.yml +++ b/.github/workflows/guardrail-scope.yml @@ -10,6 +10,9 @@ permissions: pull-requests: read contents: read +env: + CONCLUSION: action_required # action_required | neutral + jobs: scope-check: runs-on: ubuntu-latest diff --git a/.github/workflows/guardrail-test-ratio.yml b/.github/workflows/guardrail-test-ratio.yml index 31ecac7..731fd0d 100644 --- a/.github/workflows/guardrail-test-ratio.yml +++ b/.github/workflows/guardrail-test-ratio.yml @@ -9,6 +9,10 @@ permissions: pull-requests: read contents: read +env: + CONCLUSION: action_required # action_required | neutral + THRESHOLD: "0.5" # minimum test-to-code line ratio + jobs: test-ratio-check: runs-on: ubuntu-latest diff --git a/.github/workflows/orchestrator-check.yml b/.github/workflows/orchestrator-check.yml index ea948c2..d8de910 100644 --- a/.github/workflows/orchestrator-check.yml +++ b/.github/workflows/orchestrator-check.yml @@ -13,6 +13,9 @@ permissions: contents: read actions: write +env: + RE_REVIEW_CYCLE_CAP: "3" # max re-review cycles before auto-passing + jobs: orchestrator: runs-on: ubuntu-latest diff --git a/README.md b/README.md index 9e92754..9aef7c7 100644 --- a/README.md +++ b/README.md @@ -82,30 +82,25 @@ See [docs/design.md](docs/design.md) for the full design document. ## Configuration -After installation, configure guardrails in `.github/agent-workflow/config.yaml`: +Each guardrail is a standalone workflow file in `.github/workflows/`. Configure by editing the `env:` block at the top of each workflow: ```yaml -re-review-cycle-cap: 3 - -guardrails: - scope-enforcement: - enabled: true - conclusion: action_required - test-ratio: - enabled: true - conclusion: action_required - threshold: 0.5 - dependency-changes: - enabled: true - conclusion: action_required - api-surface: - enabled: true - conclusion: action_required - commit-messages: - enabled: true - conclusion: neutral # warning only +# guardrail-scope.yml +env: + CONCLUSION: action_required # action_required | neutral + +# guardrail-test-ratio.yml +env: + CONCLUSION: action_required + THRESHOLD: "0.5" # minimum test-to-code line ratio + +# orchestrator-check.yml +env: + RE_REVIEW_CYCLE_CAP: "3" # max re-review cycles before auto-passing ``` +To disable a guardrail, delete its workflow file. + ## Requirements - GitHub repository diff --git a/docs/design.md b/docs/design.md index 0165f1b..d91935f 100644 --- a/docs/design.md +++ b/docs/design.md @@ -454,7 +454,7 @@ agent-workflow-template/ │ │ ├── guardrail-api-surface.yml # API surface change detection (native check run) │ │ └── guardrail-commits.yml # Commit message structure (native check run) │ ├── agent-workflow/ -│ │ └── config.yaml # Workflow configuration (re-review cap, check thresholds, etc.) +│ │ └── scripts/ # Guardrail and orchestrator scripts (used by workflows) │ └── ISSUE_TEMPLATE/ │ ├── task.yml # Structured task template for agent consumption │ └── review-finding.yml # Template for reviewer-created issues diff --git a/tests/lib/config.test.js b/tests/lib/config.test.js deleted file mode 100644 index 3ed6041..0000000 --- a/tests/lib/config.test.js +++ /dev/null @@ -1,62 +0,0 @@ -const { test } = require('node:test'); -const assert = require('node:assert'); -const { parseGuardrailConfig } = require('../../.github/agent-workflow/scripts/lib/config.js'); - -test('parseGuardrailConfig - parses enabled and conclusion', () => { - const yaml = ` -guardrails: - scope-enforcement: - enabled: true - conclusion: action_required - `; - const config = parseGuardrailConfig(yaml, 'scope-enforcement'); - assert.deepStrictEqual(config, { enabled: true, conclusion: 'action_required' }); -}); - -test('parseGuardrailConfig - disabled guardrail', () => { - const yaml = ` -guardrails: - test-ratio: - enabled: false - conclusion: neutral - `; - const config = parseGuardrailConfig(yaml, 'test-ratio'); - assert.deepStrictEqual(config, { enabled: false, conclusion: 'neutral' }); -}); - -test('parseGuardrailConfig - missing section uses defaults', () => { - const yaml = ` -guardrails: - other-check: - enabled: true - `; - const config = parseGuardrailConfig(yaml, 'scope-enforcement'); - assert.deepStrictEqual(config, { enabled: true, conclusion: 'action_required' }); -}); - -test('parseGuardrailConfig - empty yaml uses defaults', () => { - const config = parseGuardrailConfig('', 'scope-enforcement'); - assert.deepStrictEqual(config, { enabled: true, conclusion: 'action_required' }); -}); - -test('parseGuardrailConfig - partial config uses defaults', () => { - const yaml = ` -guardrails: - scope-enforcement: - enabled: false - `; - const config = parseGuardrailConfig(yaml, 'scope-enforcement'); - assert.deepStrictEqual(config, { enabled: false, conclusion: 'action_required' }); -}); - -test('parseGuardrailConfig - with threshold', () => { - const yaml = ` -guardrails: - test-ratio: - enabled: true - conclusion: action_required - threshold: 0.7 - `; - const config = parseGuardrailConfig(yaml, 'test-ratio'); - assert.deepStrictEqual(config, { enabled: true, conclusion: 'action_required', threshold: 0.7 }); -}); From f6cb15a385cf518dd0c8ed47080c8ed941b0c700 Mon Sep 17 00:00:00 2001 From: Joe Delfino Date: Fri, 13 Feb 2026 02:59:39 +0000 Subject: [PATCH 21/23] refactor: replace custom validation with commitlint CLI Replace custom regex-based commit validation with industry-standard @commitlint/config-conventional. Reduces maintenance burden and provides better error messages. Changes: - Use commitlint CLI in workflow instead of custom JS - Add commitlint.config.js extending conventional preset - Remove guardrail-commits.js and commit-validator.js - Remove commit-validator.test.js Co-Authored-By: Claude Sonnet 4.5 --- .github/agent-workflow/commitlint.config.js | 7 + .../scripts/guardrail-commits.js | 126 ------------------ .../scripts/lib/commit-validator.js | 26 ---- .github/workflows/guardrail-commits.yml | 116 +++++++++++++++- tests/lib/commit-validator.test.js | 35 ----- 5 files changed, 120 insertions(+), 190 deletions(-) create mode 100644 .github/agent-workflow/commitlint.config.js delete mode 100644 .github/agent-workflow/scripts/guardrail-commits.js delete mode 100644 .github/agent-workflow/scripts/lib/commit-validator.js delete mode 100644 tests/lib/commit-validator.test.js diff --git a/.github/agent-workflow/commitlint.config.js b/.github/agent-workflow/commitlint.config.js new file mode 100644 index 0000000..ffe85d4 --- /dev/null +++ b/.github/agent-workflow/commitlint.config.js @@ -0,0 +1,7 @@ +module.exports = { + extends: ['@commitlint/config-conventional'], + rules: { + // Enforce 72 char limit on subject line + 'header-max-length': [2, 'always', 72], + } +}; diff --git a/.github/agent-workflow/scripts/guardrail-commits.js b/.github/agent-workflow/scripts/guardrail-commits.js deleted file mode 100644 index cbb6f94..0000000 --- a/.github/agent-workflow/scripts/guardrail-commits.js +++ /dev/null @@ -1,126 +0,0 @@ -const { hasNonStaleApproval } = require('./lib/approval.js'); -const { isValidCommit } = require('./lib/commit-validator.js'); - -module.exports = async function({ github, context, core }) { - const configuredConclusion = process.env.CONCLUSION || 'neutral'; - - // Check for non-stale PR approval override - const reviews = await github.rest.pulls.listReviews({ - owner: context.repo.owner, - repo: context.repo.repo, - pull_number: context.payload.pull_request.number - }); - - const headSha = context.payload.pull_request.head.sha; - const hasValidApproval = hasNonStaleApproval(reviews.data, headSha); - - if (hasValidApproval) { - await github.rest.checks.create({ - owner: context.repo.owner, - repo: context.repo.repo, - head_sha: context.sha, - name: 'guardrail/commit-messages', - conclusion: 'neutral', - output: { - title: 'Commit message check: approved by reviewer', - summary: 'A non-stale PR approval overrides this guardrail check.' - } - }); - return; - } - - // Get PR commits - const commits = await github.rest.pulls.listCommits({ - owner: context.repo.owner, - repo: context.repo.repo, - pull_number: context.payload.pull_request.number, - per_page: 100 - }); - - // Validate each commit message - const maxFirstLineLength = 72; - const violations = []; - - for (const commit of commits.data) { - const message = commit.commit.message; - const firstLine = message.split('\n')[0]; - const sha = commit.sha.substring(0, 7); - const commitViolations = []; - - // Check conventional commit format - if (!isValidCommit(message, { maxLength: maxFirstLineLength })) { - if (firstLine.length > maxFirstLineLength) { - commitViolations.push( - `First line exceeds ${maxFirstLineLength} characters (${firstLine.length} chars)` - ); - } - // Check if it's a format issue - const conventionalCommitRegex = /^(feat|fix|chore|docs|test|refactor|ci|style|perf|build|revert)(\(.+\))?!?: .+/; - if (!conventionalCommitRegex.test(firstLine)) { - commitViolations.push( - `Does not follow conventional commit format (expected: type(scope)?: description)` - ); - } - } - - if (commitViolations.length > 0) { - violations.push({ - sha: sha, - fullSha: commit.sha, - firstLine: firstLine, - issues: commitViolations - }); - } - } - - // Report results - const totalCommits = commits.data.length; - const nonConformingCount = violations.length; - - if (nonConformingCount === 0) { - await github.rest.checks.create({ - owner: context.repo.owner, - repo: context.repo.repo, - head_sha: context.sha, - name: 'guardrail/commit-messages', - conclusion: 'success', - output: { - title: `Commit message check: all ${totalCommits} commits conform`, - summary: `All ${totalCommits} commit(s) follow conventional commit format with first line <= ${maxFirstLineLength} characters.` - } - }); - return; - } - - // Build summary with non-conforming commits - let summary = `## Non-conforming commits\n\n`; - summary += `Found **${nonConformingCount}** of ${totalCommits} commit(s) with violations:\n\n`; - - for (const v of violations) { - summary += `### \`${v.sha}\` — ${v.firstLine}\n`; - for (const issue of v.issues) { - summary += `- ${issue}\n`; - } - summary += '\n'; - } - - summary += `\n## Expected format\n\n`; - summary += '```\n'; - summary += 'type(optional-scope): description (max 72 chars)\n'; - summary += '```\n\n'; - summary += `Valid types: \`feat\`, \`fix\`, \`chore\`, \`docs\`, \`test\`, \`refactor\`, \`ci\`, \`style\`, \`perf\`, \`build\`, \`revert\`\n\n`; - summary += `**Configured conclusion:** \`${configuredConclusion}\`\n`; - summary += `\nTo override: submit an approving PR review. The approval must be on the current head commit to be non-stale.\n`; - - await github.rest.checks.create({ - owner: context.repo.owner, - repo: context.repo.repo, - head_sha: context.sha, - name: 'guardrail/commit-messages', - conclusion: configuredConclusion, - output: { - title: `Commit message check: ${nonConformingCount} non-conforming commit(s)`, - summary: summary - } - }); -}; diff --git a/.github/agent-workflow/scripts/lib/commit-validator.js b/.github/agent-workflow/scripts/lib/commit-validator.js deleted file mode 100644 index 93eda63..0000000 --- a/.github/agent-workflow/scripts/lib/commit-validator.js +++ /dev/null @@ -1,26 +0,0 @@ -/** - * Validate commit message follows conventional commit format - * @param {string} message - Commit message (first line) - * @param {object} options - Validation options - * @param {number} options.maxLength - Maximum subject length (default: 72) - * @returns {boolean} - */ -function isValidCommit(message, options = {}) { - if (!message) return false; - - const { maxLength = 72 } = options; - - // Extract first line (subject) - const subject = message.split('\n')[0]; - - // Check length - if (subject.length > maxLength) return false; - - // Conventional commit format: type(scope)?: description - // Types: feat, fix, docs, style, refactor, test, chore, perf, ci, build, revert - const conventionalRegex = /^(feat|fix|docs|style|refactor|test|chore|perf|ci|build|revert)(\([a-z0-9-]+\))?: .+/; - - return conventionalRegex.test(subject); -} - -module.exports = { isValidCommit }; diff --git a/.github/workflows/guardrail-commits.yml b/.github/workflows/guardrail-commits.yml index 307aa9c..56b77c2 100644 --- a/.github/workflows/guardrail-commits.yml +++ b/.github/workflows/guardrail-commits.yml @@ -18,10 +18,120 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v4 + with: + fetch-depth: 0 # Need full history for commitlint + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' + + - name: Install commitlint + run: | + npm install -g @commitlint/cli @commitlint/config-conventional + + - name: Validate commit messages + id: commitlint + continue-on-error: true + run: | + set +e # Don't exit on error + + # Get PR commits + BASE_SHA=${{ github.event.pull_request.base.sha }} + HEAD_SHA=${{ github.event.pull_request.head.sha }} + + # Run commitlint and capture output + OUTPUT=$(commitlint --from ${BASE_SHA} --to ${HEAD_SHA} --config .github/agent-workflow/commitlint.config.js 2>&1) + EXIT_CODE=$? + + # Save results for next step + echo "exit_code=${EXIT_CODE}" >> $GITHUB_OUTPUT + + # Save output (escape for multiline) + { + echo "output<> $GITHUB_OUTPUT - - name: Check commit messages + exit ${EXIT_CODE} + + - name: Report results + if: always() uses: actions/github-script@v7 with: script: | - const run = require('./.github/agent-workflow/scripts/guardrail-commits.js'); - await run({ github, context, core }); \ No newline at end of file + const { hasNonStaleApproval } = require('./.github/agent-workflow/scripts/lib/approval.js'); + const configuredConclusion = process.env.CONCLUSION || 'neutral'; + + // Check for non-stale PR approval override + const reviews = await github.rest.pulls.listReviews({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: context.payload.pull_request.number + }); + + const headSha = context.payload.pull_request.head.sha; + const hasValidApproval = hasNonStaleApproval(reviews.data, headSha); + + if (hasValidApproval) { + await github.rest.checks.create({ + owner: context.repo.owner, + repo: context.repo.repo, + head_sha: context.sha, + name: 'guardrail/commit-messages', + conclusion: 'neutral', + output: { + title: 'Commit message check: approved by reviewer', + summary: 'A non-stale PR approval overrides this guardrail check.' + } + }); + return; + } + + // Get commitlint results + const exitCode = parseInt('${{ steps.commitlint.outputs.exit_code }}'); + const output = `${{ steps.commitlint.outputs.output }}`; + + if (exitCode === 0) { + await github.rest.checks.create({ + owner: context.repo.owner, + repo: context.repo.repo, + head_sha: context.sha, + name: 'guardrail/commit-messages', + conclusion: 'success', + output: { + title: 'Commit message check: all commits conform', + summary: 'All commits follow conventional commit format (validated by commitlint).' + } + }); + return; + } + + // Build failure summary + let summary = '## Validation failed\n\n'; + summary += 'Commit messages do not conform to conventional commit format.\n\n'; + summary += '### commitlint output:\n\n'; + summary += '```\n'; + summary += output; + summary += '\n```\n\n'; + summary += '## Expected format\n\n'; + summary += '```\n'; + summary += 'type(optional-scope): description\n'; + summary += '```\n\n'; + summary += 'Valid types: `feat`, `fix`, `chore`, `docs`, `test`, `refactor`, `ci`, `style`, `perf`, `build`, `revert`\n\n'; + summary += 'See: https://www.conventionalcommits.org/\n\n'; + summary += `**Configured conclusion:** \`${configuredConclusion}\`\n\n`; + summary += 'To override: submit an approving PR review. The approval must be on the current head commit to be non-stale.\n'; + + await github.rest.checks.create({ + owner: context.repo.owner, + repo: context.repo.repo, + head_sha: context.sha, + name: 'guardrail/commit-messages', + conclusion: configuredConclusion, + output: { + title: 'Commit message check: validation failed', + summary: summary + } + }); diff --git a/tests/lib/commit-validator.test.js b/tests/lib/commit-validator.test.js deleted file mode 100644 index c606e84..0000000 --- a/tests/lib/commit-validator.test.js +++ /dev/null @@ -1,35 +0,0 @@ -const { test } = require('node:test'); -const assert = require('node:assert'); -const { isValidCommit } = require('../../.github/agent-workflow/scripts/lib/commit-validator.js'); - -test('isValidCommit - valid conventional commits', () => { - assert.strictEqual(isValidCommit('feat: add new feature'), true); - assert.strictEqual(isValidCommit('fix: resolve bug'), true); - assert.strictEqual(isValidCommit('chore: update dependencies'), true); - assert.strictEqual(isValidCommit('docs: improve README'), true); - assert.strictEqual(isValidCommit('test: add unit tests'), true); - assert.strictEqual(isValidCommit('refactor: simplify logic'), true); -}); - -test('isValidCommit - with scope', () => { - assert.strictEqual(isValidCommit('feat(auth): add OAuth support'), true); - assert.strictEqual(isValidCommit('fix(api): handle edge case'), true); -}); - -test('isValidCommit - with issue reference', () => { - assert.strictEqual(isValidCommit('feat: add feature (#123)'), true); - assert.strictEqual(isValidCommit('fix: resolve issue\n\nFixes #456'), true); -}); - -test('isValidCommit - invalid commits', () => { - assert.strictEqual(isValidCommit('Add new feature'), false); - assert.strictEqual(isValidCommit('FEAT: bad caps'), false); - assert.strictEqual(isValidCommit('feat:missing space'), false); - assert.strictEqual(isValidCommit(''), false); -}); - -test('isValidCommit - maximum length enforcement', () => { - const longMsg = 'feat: ' + 'a'.repeat(100); - assert.strictEqual(isValidCommit(longMsg, { maxLength: 72 }), false); - assert.strictEqual(isValidCommit('feat: short msg', { maxLength: 72 }), true); -}); From 32a6e2f7a650f3bf9e03cab44ade8efe4cc3ce58 Mon Sep 17 00:00:00 2001 From: Joe Delfino Date: Fri, 13 Feb 2026 03:03:39 +0000 Subject: [PATCH 22/23] refactor: remove keyword bypass from dependency guardrail Remove insecure keyword-matching bypass logic. Any dependency file modification now requires human review via PR approval. The previous implementation allowed bypassing the check by including words like "introduced" anywhere in the PR description, providing false security. This change makes the guardrail honest and effective. Co-Authored-By: Claude Sonnet 4.5 --- .../scripts/guardrail-dependencies.js | 99 ++++--------------- 1 file changed, 17 insertions(+), 82 deletions(-) diff --git a/.github/agent-workflow/scripts/guardrail-dependencies.js b/.github/agent-workflow/scripts/guardrail-dependencies.js index 705d0b6..b121cf6 100644 --- a/.github/agent-workflow/scripts/guardrail-dependencies.js +++ b/.github/agent-workflow/scripts/guardrail-dependencies.js @@ -5,21 +5,6 @@ module.exports = async function({ github, context, core }) { const CHECK_NAME = 'guardrail/dependency-changes'; const configuredConclusion = process.env.CONCLUSION || 'action_required'; - const JUSTIFICATION_KEYWORDS = [ - 'dependency', 'dependencies', - 'added', 'adding', - 'requires', 'required', - 'needed for', 'needed by', - 'introduced', - 'new package', 'new library', 'new module', - 'upgrade', 'upgraded', 'upgrading', - 'update', 'updated', 'updating', - 'migration', 'migrate', 'migrating', - 'replace', 'replaced', 'replacing', - 'security fix', 'security patch', 'vulnerability', - 'CVE-' - ]; - // Check for non-stale approving PR review (override mechanism) const { data: reviews } = await github.rest.pulls.listReviews({ owner: context.repo.owner, @@ -40,7 +25,7 @@ module.exports = async function({ github, context, core }) { conclusion: 'neutral', output: { title: 'Dependency changes: approved by reviewer', - summary: 'A non-stale PR approval overrides dependency change violations.' + summary: 'A non-stale PR approval overrides this guardrail check.' } }); return; @@ -76,77 +61,27 @@ module.exports = async function({ github, context, core }) { return; } - // Dependency files changed: check for justification - const prBody = (context.payload.pull_request.body || '').toLowerCase(); - - function hasJustification(text) { - const lowerText = text.toLowerCase(); - return JUSTIFICATION_KEYWORDS.some(keyword => lowerText.includes(keyword)); - } - - let justified = hasJustification(prBody); - - // If not justified in PR body, check linked issue body - if (!justified) { - const issueMatch = (context.payload.pull_request.body || '').match( - /(?:fixes|closes|resolves)\s+#(\d+)/i - ); - if (issueMatch) { - try { - const { data: issue } = await github.rest.issues.get({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: parseInt(issueMatch[1], 10) - }); - if (issue.body) { - justified = hasJustification(issue.body); - } - } catch (e) { - core.warning(`Failed to fetch linked issue #${issueMatch[1]}: ${e.message}`); - } - } - } - - // Build annotations for changed dependency files + // Dependency files changed: requires human review + const fileList = changedDependencyFiles.map(f => '- `' + f.filename + '`').join('\n'); const annotations = changedDependencyFiles.map(f => ({ path: f.filename, start_line: 1, end_line: 1, annotation_level: 'warning', - message: justified - ? `Dependency file changed. Justification found in PR or linked issue.` - : `Dependency file changed without justification. Add context about why dependencies were changed to the PR description or linked issue.` + message: 'Dependency file modified. Human review required before merge.' })); - // Report result - if (justified) { - await github.rest.checks.create({ - owner: context.repo.owner, - repo: context.repo.repo, - head_sha: context.sha, - name: CHECK_NAME, - status: 'completed', - conclusion: 'success', - output: { - title: `Dependency changes: ${changedDependencyFiles.length} file(s) changed with justification`, - summary: `Dependency files were modified and justification was found in the PR body or linked issue.\n\n**Changed dependency files:**\n${changedDependencyFiles.map(f => '- `' + f.filename + '`').join('\n')}`, - annotations: annotations - } - }); - } else { - const fileList = changedDependencyFiles.map(f => '- `' + f.filename + '`').join('\n'); - await github.rest.checks.create({ - owner: context.repo.owner, - repo: context.repo.repo, - head_sha: context.sha, - name: CHECK_NAME, - status: 'completed', - conclusion: configuredConclusion, - output: { - title: `Dependency changes: ${changedDependencyFiles.length} file(s) changed without justification`, - summary: `Dependency files were modified but no justification was found.\n\n**Changed dependency files:**\n${fileList}\n\n**To resolve:** Add context about dependency changes to the PR description using keywords like: ${JUSTIFICATION_KEYWORDS.slice(0, 8).map(k => '"' + k + '"').join(', ')}, etc.\n\nAlternatively, a PR approval will override this check.`, - annotations: annotations - } - }); - } + await github.rest.checks.create({ + owner: context.repo.owner, + repo: context.repo.repo, + head_sha: context.sha, + name: CHECK_NAME, + status: 'completed', + conclusion: configuredConclusion, + output: { + title: `Dependency changes: ${changedDependencyFiles.length} file(s) modified`, + summary: `Dependency files were modified and require human review before merge.\n\n**Changed dependency files:**\n${fileList}\n\n**To resolve:** A maintainer must submit an approving PR review. The approval must be on the current head commit (non-stale).`, + annotations: annotations + } + }); }; From 54b1bd10ff4f4cb75e021f5086f0b3b490406453 Mon Sep 17 00:00:00 2001 From: Joe Delfino Date: Fri, 13 Feb 2026 03:12:15 +0000 Subject: [PATCH 23/23] refactor: remove human-review workflow Delete workflow that auto-converted PR comments to issues. This created issue spam and duplicate tracking. Human review comments should be addressed directly by /work, not converted to separate issues. Next: integrate PR comment handling into coordinator/implementer skills with auto-resolution when comments are addressed. Co-Authored-By: Claude Sonnet 4.5 --- .../agent-workflow/scripts/human-review.js | 192 ------------------ .github/workflows/human-review.yml | 26 --- 2 files changed, 218 deletions(-) delete mode 100644 .github/agent-workflow/scripts/human-review.js delete mode 100644 .github/workflows/human-review.yml diff --git a/.github/agent-workflow/scripts/human-review.js b/.github/agent-workflow/scripts/human-review.js deleted file mode 100644 index 0efd28a..0000000 --- a/.github/agent-workflow/scripts/human-review.js +++ /dev/null @@ -1,192 +0,0 @@ -const { detectSeverity } = require('./lib/severity.js'); -const { parseFixesReferences } = require('./lib/fixes-parser.js'); - -module.exports = async function({ github, context, core }) { - const review = context.payload.review; - const pr = context.payload.pull_request; - - // Parse parent issue from PR body - const issueNumbers = parseFixesReferences(pr.body); - if (issueNumbers.length === 0) { - console.log('No parent issue found — PR body has no "Fixes #N" reference. Skipping.'); - return; - } - const parentIssueNumber = issueNumbers[0]; - - // Fetch comments for this specific review - const { data: reviewComments } = await github.request( - 'GET /repos/{owner}/{repo}/pulls/{pull_number}/reviews/{review_id}/comments', - { - owner: context.repo.owner, - repo: context.repo.repo, - pull_number: pr.number, - review_id: review.id, - } - ); - - if (reviewComments.length === 0) { - console.log('Review has no line-level comments. Skipping.'); - return; - } - - // Get parent issue node_id for GraphQL sub-issue linking - const { data: parentIssue } = await github.rest.issues.get({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: parentIssueNumber, - }); - const parentNodeId = parentIssue.node_id; - - // Ensure labels exist - const severityLabels = ['blocking', 'should-fix', 'suggestion']; - const labelColors = { - 'blocking': 'B60205', - 'should-fix': 'D93F0B', - 'suggestion': '0E8A16', - }; - - for (const label of severityLabels) { - try { - await github.rest.issues.getLabel({ - owner: context.repo.owner, - repo: context.repo.repo, - name: label, - }); - } catch { - await github.rest.issues.createLabel({ - owner: context.repo.owner, - repo: context.repo.repo, - name: label, - color: labelColors[label], - }); - } - } - - // Process each comment - const createdIssues = []; - let hasBlockingComments = false; - - for (const comment of reviewComments) { - const severity = detectSeverity(comment.body); - const filePath = comment.path; - const line = comment.original_line || comment.line || 0; - - // Build issue body with file/line context - const issueBody = [ - `## Review Finding`, - ``, - `**Severity:** \`${severity}\``, - `**File:** \`${filePath}\`${line ? ` (line ${line})` : ''}`, - `**PR:** #${pr.number}`, - `**Reviewer:** @${review.user.login}`, - ``, - `### Comment`, - ``, - comment.body, - ``, - `---`, - `_Created automatically from a PR review comment._`, - ].join('\n'); - - const issueTitle = `[${severity}] ${filePath}${line ? `:${line}` : ''}: ${comment.body.split('\n')[0].substring(0, 80)}`; - - // Create the child issue - const { data: newIssue } = await github.rest.issues.create({ - owner: context.repo.owner, - repo: context.repo.repo, - title: issueTitle, - body: issueBody, - labels: [severity], - }); - - console.log(`Created issue #${newIssue.number}: ${newIssue.title}`); - - // Link as sub-issue via GraphQL addSubIssue mutation - try { - await github.graphql(` - mutation($parentId: ID!, $childId: ID!) { - addSubIssue(input: { issueId: $parentId, subIssueId: $childId }) { - issue { id } - subIssue { id } - } - } - `, { - parentId: parentNodeId, - childId: newIssue.node_id, - }); - console.log(`Linked issue #${newIssue.number} as sub-issue of #${parentIssueNumber}`); - } catch (err) { - console.log(`Warning: Could not link sub-issue via GraphQL: ${err.message}`); - } - - // If blocking, set as blocking the parent issue - if (severity === 'blocking') { - hasBlockingComments = true; - try { - await github.request( - 'POST /repos/{owner}/{repo}/issues/{issue_number}/sub_issues/priority', - { - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: parentIssueNumber, - sub_issue_id: newIssue.id, - } - ); - } catch (err) { - console.log(`Note: Could not set blocking dependency via REST: ${err.message}`); - } - } - - createdIssues.push({ - number: newIssue.number, - severity, - }); - } - - // Update PR description with Fixes references - if (createdIssues.length > 0) { - const fixesLines = createdIssues - .map(i => `Fixes #${i.number}`) - .join('\n'); - - const newSection = [ - '', - '### Review Finding Issues', - fixesLines, - '', - ].join('\n'); - - // Replace existing section or append, for idempotent updates - let currentBody = pr.body || ''; - const sectionRegex = /[\s\S]*?/; - let updatedBody; - if (sectionRegex.test(currentBody)) { - updatedBody = currentBody.replace(sectionRegex, newSection); - } else { - updatedBody = currentBody + '\n\n' + newSection; - } - - await github.rest.pulls.update({ - owner: context.repo.owner, - repo: context.repo.repo, - pull_number: pr.number, - body: updatedBody, - }); - - console.log(`Updated PR #${pr.number} body with ${createdIssues.length} Fixes references`); - } - - // Summary - const blockingCount = createdIssues.filter(i => i.severity === 'blocking').length; - const shouldFixCount = createdIssues.filter(i => i.severity === 'should-fix').length; - const suggestionCount = createdIssues.filter(i => i.severity === 'suggestion').length; - - console.log(`\nDone. Created ${createdIssues.length} issues from review comments:`); - console.log(` blocking: ${blockingCount}`); - console.log(` should-fix: ${shouldFixCount}`); - console.log(` suggestion: ${suggestionCount}`); - - if (hasBlockingComments) { - console.log(`\nBlocking issues were created — parent issue #${parentIssueNumber} has new blockers.`); - } -}; diff --git a/.github/workflows/human-review.yml b/.github/workflows/human-review.yml deleted file mode 100644 index 72bb047..0000000 --- a/.github/workflows/human-review.yml +++ /dev/null @@ -1,26 +0,0 @@ -name: Human Review → Issues - -"on": - pull_request_review: - types: [submitted] - -permissions: - issues: write - pull-requests: write - contents: read - -jobs: - process-review: - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@v4 - with: - sparse-checkout: .github/agent-workflow - - - name: Process review comments and create issues - uses: actions/github-script@v7 - with: - script: | - const run = require('./.github/agent-workflow/scripts/human-review.js'); - await run({ github, context, core }); \ No newline at end of file