From 80ad62ec0f53bb05bd702e5577ceb67a06a710db Mon Sep 17 00:00:00 2001 From: Tobias Wilken Date: Sat, 11 Oct 2025 11:56:43 +0200 Subject: [PATCH] feat: implement automatic repository synchronization Add automated sync system to apply REPOSITORIES.md changes to GitHub organization. Changes: - Add sync-repositories.js script with dry-run and apply modes - Create sync-repositories.yml workflow for push to main - Enhance drift-detection.yml workflow with sync preview - Add npm scripts for convenient sync operations The sync system creates missing repositories, updates descriptions and topics, and ensures REPOSITORIES.md remains the single source of truth. It runs on every push to main to catch and correct manual GitHub changes. On pull requests, the system shows both current drift and planned actions. On merge to main, changes are automatically applied with full audit trail via commit comments. --- .github/workflows/drift-detection.yml | 19 +- .github/workflows/sync-repositories.yml | 86 ++++++ package.json | 4 +- scripts/sync-repositories.js | 370 ++++++++++++++++++++++++ 4 files changed, 475 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/sync-repositories.yml create mode 100755 scripts/sync-repositories.js diff --git a/.github/workflows/drift-detection.yml b/.github/workflows/drift-detection.yml index 58187cf..6d28ff2 100644 --- a/.github/workflows/drift-detection.yml +++ b/.github/workflows/drift-detection.yml @@ -38,12 +38,25 @@ jobs: set -e continue-on-error: true - - name: Comment PR with drift report + - name: Preview sync actions (dry-run) + id: sync + env: + WORLDDRIVEN_GITHUB_TOKEN: ${{ secrets.WORLDDRIVEN_GITHUB_TOKEN }} + run: | + set +e + node scripts/sync-repositories.js > sync-preview.md 2>&1 + EXIT_CODE=$? + echo "exit_code=$EXIT_CODE" >> $GITHUB_OUTPUT + set -e + continue-on-error: true + + - name: Comment PR with drift report and sync preview uses: actions/github-script@v7 with: script: | const fs = require('fs'); - const report = fs.readFileSync('drift-report.md', 'utf8'); + const driftReport = fs.readFileSync('drift-report.md', 'utf8'); + const syncPreview = fs.readFileSync('sync-preview.md', 'utf8'); // Find existing comment const { data: comments } = await github.rest.issues.listComments({ @@ -57,7 +70,7 @@ jobs: comment.body.includes('Repository Drift Report') ); - const commentBody = `${report}\n\n---\n*🤖 This report is automatically generated on every PR that modifies REPOSITORIES.md*`; + const commentBody = `${driftReport}\n\n---\n\n${syncPreview}\n\n---\n*🤖 This report is automatically generated on every PR that modifies REPOSITORIES.md*\n*The sync preview shows what actions will be applied when this PR is merged to main.*`; if (botComment) { // Update existing comment diff --git a/.github/workflows/sync-repositories.yml b/.github/workflows/sync-repositories.yml new file mode 100644 index 0000000..6396084 --- /dev/null +++ b/.github/workflows/sync-repositories.yml @@ -0,0 +1,86 @@ +name: Sync Repositories + +on: + push: + branches: + - main + +jobs: + sync: + runs-on: ubuntu-latest + permissions: + contents: write + issues: write + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: '20' + + - name: Make scripts executable + run: chmod +x scripts/*.js + + - name: Sync repositories to GitHub + id: sync + env: + WORLDDRIVEN_GITHUB_TOKEN: ${{ secrets.WORLDDRIVEN_GITHUB_TOKEN }} + run: | + set +e + node scripts/sync-repositories.js --apply > sync-report.md 2>&1 + EXIT_CODE=$? + echo "exit_code=$EXIT_CODE" >> $GITHUB_OUTPUT + set -e + continue-on-error: true + + - name: Comment on commit with results + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + const report = fs.readFileSync('sync-report.md', 'utf8'); + + // Create a comment on the commit + await github.rest.repos.createCommitComment({ + owner: context.repo.owner, + repo: context.repo.repo, + commit_sha: context.sha, + body: `${report}\n\n---\n*🤖 Automated sync of GitHub organization with REPOSITORIES.md*`, + }); + + - name: Create issue if failures occurred + if: steps.sync.outputs.exit_code != '0' + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + const report = fs.readFileSync('sync-report.md', 'utf8'); + + const title = '⚠️ Repository sync failed'; + const body = `Repository synchronization encountered errors on commit ${context.sha.substring(0, 7)}. + + ${report} + + --- + **Commit**: ${context.sha} + **Triggered by**: @${context.actor} + **Action**: ${context.payload.head_commit?.message || 'Push to main'} + + Please review the errors above and take appropriate action.`; + + await github.rest.issues.create({ + owner: context.repo.owner, + repo: context.repo.repo, + title: title, + body: body, + labels: ['automation', 'sync-failure'], + }); + + - name: Report sync status + if: steps.sync.outputs.exit_code == '0' + run: | + echo "✅ Repository sync completed successfully" + echo "All changes have been applied to the GitHub organization" diff --git a/package.json b/package.json index 6c96a03..9227e49 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,9 @@ "scripts": { "parse": "node scripts/parse-repositories.js", "fetch-github": "node scripts/fetch-github-state.js", - "detect-drift": "node scripts/detect-drift.js" + "detect-drift": "node scripts/detect-drift.js", + "sync": "node scripts/sync-repositories.js", + "sync-apply": "node scripts/sync-repositories.js --apply" }, "keywords": [ "worlddriven", diff --git a/scripts/sync-repositories.js b/scripts/sync-repositories.js new file mode 100755 index 0000000..08023a9 --- /dev/null +++ b/scripts/sync-repositories.js @@ -0,0 +1,370 @@ +#!/usr/bin/env node + +/** + * Sync GitHub organization repositories to match REPOSITORIES.md + * + * Modes: + * - Dry-run (default): Show what would be applied without making changes + * - Apply (--apply): Actually make changes to GitHub organization + */ + +import { parseRepositoriesFile } from './parse-repositories.js'; +import { fetchGitHubRepositories } from './fetch-github-state.js'; +import { detectDrift } from './detect-drift.js'; + +const GITHUB_API_BASE = 'https://api.github.com'; +const ORG_NAME = 'worlddriven'; + +/** + * Create a repository in the GitHub organization + */ +async function createRepository(token, repoData) { + const url = `${GITHUB_API_BASE}/orgs/${ORG_NAME}/repos`; + + const body = { + name: repoData.name, + description: repoData.description, + private: false, + has_issues: true, + has_projects: true, + has_wiki: true, + }; + + const response = await fetch(url, { + method: 'POST', + headers: { + 'Authorization': `Bearer ${token}`, + 'Accept': 'application/vnd.github+json', + 'X-GitHub-Api-Version': '2022-11-28', + 'Content-Type': 'application/json', + }, + body: JSON.stringify(body), + }); + + if (!response.ok) { + const error = await response.text(); + throw new Error(`GitHub API error (${response.status}): ${error}`); + } + + return await response.json(); +} + +/** + * Update repository description + */ +async function updateRepositoryDescription(token, repoName, description) { + const url = `${GITHUB_API_BASE}/repos/${ORG_NAME}/${repoName}`; + + const response = await fetch(url, { + method: 'PATCH', + headers: { + 'Authorization': `Bearer ${token}`, + 'Accept': 'application/vnd.github+json', + 'X-GitHub-Api-Version': '2022-11-28', + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ description }), + }); + + if (!response.ok) { + const error = await response.text(); + throw new Error(`GitHub API error (${response.status}): ${error}`); + } + + return await response.json(); +} + +/** + * Update repository topics (replaces all topics) + */ +async function updateRepositoryTopics(token, repoName, topics) { + const url = `${GITHUB_API_BASE}/repos/${ORG_NAME}/${repoName}/topics`; + + const response = await fetch(url, { + method: 'PUT', + headers: { + 'Authorization': `Bearer ${token}`, + 'Accept': 'application/vnd.github.mercy-preview+json', + 'X-GitHub-Api-Version': '2022-11-28', + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ names: topics || [] }), + }); + + if (!response.ok) { + const error = await response.text(); + throw new Error(`GitHub API error (${response.status}): ${error}`); + } + + return await response.json(); +} + +/** + * Generate sync plan from drift + */ +function generateSyncPlan(drift) { + const plan = { + actions: [], + summary: { + create: 0, + updateDescription: 0, + updateTopics: 0, + skip: 0, + }, + }; + + // Create missing repositories + for (const repo of drift.missing) { + plan.actions.push({ + type: 'create', + repo: repo.name, + data: repo, + }); + plan.summary.create++; + } + + // Update descriptions + for (const diff of drift.descriptionDiff) { + plan.actions.push({ + type: 'update-description', + repo: diff.name, + from: diff.actual, + to: diff.desired, + }); + plan.summary.updateDescription++; + } + + // Update topics + for (const diff of drift.topicsDiff) { + plan.actions.push({ + type: 'update-topics', + repo: diff.name, + from: diff.actual, + to: diff.desired, + }); + plan.summary.updateTopics++; + } + + // Report extra repos (but don't delete) + for (const repo of drift.extra) { + plan.actions.push({ + type: 'skip', + repo: repo.name, + reason: 'Extra repository in GitHub - not in REPOSITORIES.md (manual deletion required)', + }); + plan.summary.skip++; + } + + return plan; +} + +/** + * Execute sync plan + */ +async function executeSyncPlan(token, plan, dryRun) { + const results = { + success: [], + failures: [], + skipped: [], + }; + + for (const action of plan.actions) { + try { + if (action.type === 'skip') { + results.skipped.push({ + action, + reason: action.reason, + }); + continue; + } + + if (dryRun) { + results.success.push({ + action, + result: 'DRY-RUN: Would be applied', + }); + continue; + } + + // Apply the action + let result; + switch (action.type) { + case 'create': + result = await createRepository(token, action.data); + // After creating, set topics if they exist + if (action.data.topics && action.data.topics.length > 0) { + await updateRepositoryTopics(token, action.data.name, action.data.topics); + } + break; + + case 'update-description': + result = await updateRepositoryDescription(token, action.repo, action.to); + break; + + case 'update-topics': + result = await updateRepositoryTopics(token, action.repo, action.to); + break; + + default: + throw new Error(`Unknown action type: ${action.type}`); + } + + results.success.push({ + action, + result: 'Applied successfully', + }); + + } catch (error) { + results.failures.push({ + action, + error: error.message, + }); + } + } + + return results; +} + +/** + * Format sync results as markdown + */ +function formatSyncReport(plan, results, dryRun) { + const lines = []; + + const mode = dryRun ? '🔍 DRY-RUN' : '✅ APPLY'; + lines.push(`# ${mode} Repository Sync Report`); + lines.push(''); + + if (plan.actions.length === 0) { + lines.push('✅ **No changes needed** - GitHub organization matches REPOSITORIES.md'); + return lines.join('\n'); + } + + lines.push(`**Summary**: ${plan.actions.length} total actions`); + lines.push(`- Create: ${plan.summary.create}`); + lines.push(`- Update descriptions: ${plan.summary.updateDescription}`); + lines.push(`- Update topics: ${plan.summary.updateTopics}`); + lines.push(`- Skip (manual action needed): ${plan.summary.skip}`); + lines.push(''); + + // Success + if (results.success.length > 0) { + const header = dryRun ? '📋 Actions to Apply' : '✅ Successfully Applied'; + lines.push(`## ${header} (${results.success.length})`); + lines.push(''); + for (const item of results.success) { + const action = item.action; + switch (action.type) { + case 'create': + lines.push(`- **Create** \`${action.repo}\``); + lines.push(` - Description: ${action.data.description}`); + if (action.data.topics && action.data.topics.length > 0) { + lines.push(` - Topics: ${action.data.topics.join(', ')}`); + } + break; + + case 'update-description': + lines.push(`- **Update description** for \`${action.repo}\``); + lines.push(` - From: ${action.from || '(empty)'}`); + lines.push(` - To: ${action.to || '(empty)'}`); + break; + + case 'update-topics': + lines.push(`- **Update topics** for \`${action.repo}\``); + lines.push(` - From: ${action.from.join(', ') || '(none)'}`); + lines.push(` - To: ${action.to.join(', ') || '(none)'}`); + break; + } + lines.push(''); + } + } + + // Failures + if (results.failures.length > 0) { + lines.push(`## ❌ Failed (${results.failures.length})`); + lines.push(''); + for (const item of results.failures) { + const action = item.action; + lines.push(`- **${action.type}** for \`${action.repo}\``); + lines.push(` - Error: ${item.error}`); + lines.push(''); + } + } + + // Skipped + if (results.skipped.length > 0) { + lines.push(`## ⚠️ Skipped - Manual Action Required (${results.skipped.length})`); + lines.push(''); + for (const item of results.skipped) { + lines.push(`- \`${item.action.repo}\`: ${item.reason}`); + } + lines.push(''); + } + + return lines.join('\n'); +} + +/** + * Main function + */ +async function main() { + const args = process.argv.slice(2); + const dryRun = !args.includes('--apply'); + const token = process.env.WORLDDRIVEN_GITHUB_TOKEN; + + if (!token) { + console.error('❌ Error: WORLDDRIVEN_GITHUB_TOKEN environment variable is not set'); + process.exit(1); + } + + try { + // Determine mode + if (dryRun) { + console.error('🔍 Running in DRY-RUN mode (no changes will be made)'); + console.error(' Use --apply to actually apply changes'); + } else { + console.error('✅ Running in APPLY mode (changes will be made to GitHub)'); + } + console.error(''); + + // Load desired state + console.error('📖 Parsing REPOSITORIES.md...'); + const desiredRepos = await parseRepositoriesFile(); + + // Fetch actual state + console.error('🌐 Fetching GitHub organization state...'); + const actualRepos = await fetchGitHubRepositories(token); + + // Detect drift + console.error('🔍 Detecting drift...'); + const drift = detectDrift(desiredRepos, actualRepos); + + // Generate sync plan + console.error('📋 Generating sync plan...'); + const plan = generateSyncPlan(drift); + + // Execute plan + console.error(`${dryRun ? '🔍' : '⚡'} ${dryRun ? 'Simulating' : 'Executing'} sync plan...`); + console.error(''); + const results = await executeSyncPlan(token, plan, dryRun); + + // Format and output report + const report = formatSyncReport(plan, results, dryRun); + console.log(report); + + // Exit with error if there were failures + process.exit(results.failures.length > 0 ? 1 : 0); + + } catch (error) { + console.error(`❌ Error: ${error.message}`); + process.exit(1); + } +} + +// CLI usage +if (import.meta.url === `file://${process.argv[1]}`) { + main(); +} + +// Export for use as module +export { generateSyncPlan, executeSyncPlan, formatSyncReport };