From c1050582fa034b887f2c10edb42e9e301ff0bc34 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 12 Nov 2025 13:33:44 +0000 Subject: [PATCH] docs(investigation): document build failure root cause and solution MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Investigated why "Build documentation" succeeds on PRs but fails on main. Root cause: - PR #75 fixed the Notion fetch scripts to quote YAML special characters - BUT the content branch still contains old files with unquoted frontmatter - PRs regenerate content (using fixed scripts) → success ✅ - Main uses existing content branch → failure ❌ Solution: - Created automated fix script (scripts/fix-yaml-frontmatter.ts) - Identified 10 files needing fixes in content branch - All have unquoted colons (:) and ampersands (&) in titles Files added: - INVESTIGATION-REPORT.md: Detailed analysis and recommendations - SOLUTION.md: Step-by-step fix instructions - scripts/fix-yaml-frontmatter.ts: Automated YAML frontmatter validator/fixer Next steps: 1. Apply fix script to content branch 2. Commit and push changes to content branch 3. Verify main branch build succeeds Related: PR #75, main branch build failures --- INVESTIGATION-REPORT.md | 148 +++++++++++++++++++++++ SOLUTION.md | 162 +++++++++++++++++++++++++ scripts/fix-yaml-frontmatter.ts | 205 ++++++++++++++++++++++++++++++++ 3 files changed, 515 insertions(+) create mode 100644 INVESTIGATION-REPORT.md create mode 100644 SOLUTION.md create mode 100644 scripts/fix-yaml-frontmatter.ts diff --git a/INVESTIGATION-REPORT.md b/INVESTIGATION-REPORT.md new file mode 100644 index 00000000..1671b39d --- /dev/null +++ b/INVESTIGATION-REPORT.md @@ -0,0 +1,148 @@ +# Investigation Report: Build Failures on Main Branch + +## Issue Summary + +The `bun run build` command succeeds on PR branches but fails on the main branch with YAML frontmatter parsing errors: + +``` +YAMLException: incomplete explicit mapping pair; a key node is missed; or followed by a non-tabulated empty line +``` + +## Root Cause Analysis + +### The Problem + +The build failures are caused by **two separate but related issues**: + +1. **Script Issue (Fixed in PR #75)**: The Notion fetch scripts were generating markdown files with unquoted special characters (`:` and `&`) in YAML frontmatter fields +2. **Content Branch Issue (Not Fixed)**: The `content` branch contains pre-existing markdown files with the same unquoted special characters + +### Why PRs Succeed But Main Fails + +**PR Builds**: +- Preview workflow regenerates content from Notion using the **fixed scripts** (from PR #75) +- New content has properly quoted YAML frontmatter +- Build succeeds ✅ + +**Main Branch Builds**: +- Uses the existing `content` branch for speed (when not modifying scripts) +- Content branch still has **old files** with unquoted frontmatter +- Build fails ❌ + +### Technical Details + +YAML parsers (gray-matter/js-yaml) treat special characters differently: + +```yaml +# ❌ FAILS - colon interpreted as key-value separator +title: Troubleshooting: Data Privacy & Security + +# ✅ WORKS - quoted string is treated as a single value +title: "Troubleshooting: Data Privacy & Security" +``` + +Characters requiring quoting: `& : [ ] { } , | > * ! % @ \` # -` + +## Affected Files + +10 markdown files in the `content` branch have unquoted special characters: + +1. `docs/troubleshooting/troubleshooting-data-privacy--security.md` +2. `docs/encryption--security.md` +3. `docs/understanding-comapeos-core-concepts-and-functions.md` +4. `docs/getting-started-essentials/installing-comapeo--onboarding.md` +5. `docs/managing-data-privacy--security/adjusting-data-sharing--privacy.md` +6. `docs/sharing-a-single-observation--metadata.md` +7. `docs/troubleshooting/troubleshooting-setup--customization.md` +8. `docs/troubleshooting/troubleshooting-mapping-with-collaborators.md` +9. `docs/troubleshooting/troubleshooting-observations--tracks.md` +10. `docs/troubleshooting/troubleshooting-moving-observations--tracks-outside-of-comapeo.md` + +Each file has unquoted values in `title`, `sidebar_label`, and `pagination_label` fields. + +## Solution + +### Option 1: Manual Fix (Immediate) + +Manually update the 10 files in the `content` branch to quote all frontmatter values containing special characters. + +**Pros**: +- Immediate fix +- Minimal risk + +**Cons**: +- Manual process +- Could miss files + +### Option 2: Automated Script (Recommended) + +Create a script to: +1. Scan all markdown files in `content` branch +2. Parse frontmatter +3. Quote values containing special characters +4. Commit and push changes + +**Pros**: +- Comprehensive +- Reusable for future issues +- Catches all affected files + +**Cons**: +- Requires more development time + +### Option 3: Regenerate Content (Nuclear Option) + +Delete the `content` branch and regenerate all content from Notion using the fixed scripts. + +**Pros**: +- Guaranteed to fix all issues +- Fresh content + +**Cons**: +- Time-consuming (~8 minutes for full regeneration) +- Requires Notion API access +- May introduce other changes + +## Recommended Action Plan + +1. **Immediate**: Apply the manual fixes to the 10 identified files in the `content` branch +2. **Short-term**: Create an automated script to validate and fix YAML frontmatter +3. **Long-term**: Add CI checks to prevent unquoted special characters in generated content + +## Files Changed + +A commit has been prepared (but not pushed) to the `content` branch with all necessary fixes: +- 10 files modified +- 31 insertions, 31 deletions +- All titles, sidebar_labels, and pagination_labels properly quoted + +## Testing + +To verify the fix: +```bash +# Switch to content branch +git checkout content + +# Apply the fixes (already committed locally) +git log -1 --stat + +# Build the documentation +bun run build + +# Expected: Build succeeds without YAML parsing errors +``` + +## Prevention + +PR #75 already prevents this issue going forward by: +1. Adding `quoteYamlValue()` helper function +2. Automatically quoting all frontmatter values with special characters +3. Applying to all generated markdown files + +The `content` branch just needs a one-time update to apply these fixes retroactively. + +## Related + +- PR #75: "fix(docs,ci): YAML frontmatter quoting and sharp module installation" +- Main branch commit: `aeabd1e` +- Session: `011CV44zE2CUY21DYsDn1EWn` diff --git a/SOLUTION.md b/SOLUTION.md new file mode 100644 index 00000000..2ae494c2 --- /dev/null +++ b/SOLUTION.md @@ -0,0 +1,162 @@ +# Solution: Fix Build Failures on Main Branch + +## Quick Fix (Immediate Action) + +The build failures on main branch are caused by unquoted special characters in YAML frontmatter on the `content` branch. Here's how to fix it: + +### Step 1: Checkout the content branch + +```bash +git fetch origin content:content +git checkout content +``` + +### Step 2: Run the fix script + +```bash +# Option A: Use the automated script (recommended) +bun scripts/fix-yaml-frontmatter.ts docs/ + +# Option B: Regenerate all content from Notion (slower but comprehensive) +bun run notion:fetch:all +``` + +### Step 3: Review and commit changes + +```bash +git diff +git add -A +git commit -m "fix(content): quote YAML frontmatter values with special characters" +``` + +### Step 4: Push to content branch + +```bash +git push origin content +``` + +### Step 5: Verify the fix + +```bash +git checkout main +git pull +bun run build +# Should succeed without errors +``` + +## What Was Fixed + +### Files Modified (10 total) + +All files had their `title`, `sidebar_label`, and `pagination_label` fields quoted to handle special characters (`:` and `&`): + +1. `docs/troubleshooting/troubleshooting-data-privacy--security.md` + - Before: `title: Troubleshooting: Data Privacy & Security` + - After: `title: "Troubleshooting: Data Privacy & Security"` + +2. `docs/encryption--security.md` + - Before: `title: Encryption & Security` + - After: `title: "Encryption & Security"` + - Also fixed nested `sidebar_custom_props.title` field + +3. `docs/understanding-comapeos-core-concepts-and-functions.md` + - Before: `title: Understanding CoMapeo's Core Concepts & Functions` + - After: `title: "Understanding CoMapeo's Core Concepts & Functions"` + +4. `docs/getting-started-essentials/installing-comapeo--onboarding.md` + - Before: `title: Installing CoMapeo & Onboarding` + - After: `title: "Installing CoMapeo & Onboarding"` + +5. `docs/managing-data-privacy--security/adjusting-data-sharing--privacy.md` + - Before: `title: Adjusting Data Sharing & Privacy` + - After: `title: "Adjusting Data Sharing & Privacy"` + +6. `docs/sharing-a-single-observation--metadata.md` + - Before: `title: Sharing a Single Observation & Metadata` + - After: `title: "Sharing a Single Observation & Metadata"` + +7. `docs/troubleshooting/troubleshooting-setup--customization.md` + - Before: `title: Troubleshooting: Setup & Customization` + - After: `title: "Troubleshooting: Setup & Customization"` + +8. `docs/troubleshooting/troubleshooting-mapping-with-collaborators.md` + - Before: `title: Troubleshooting: Mapping with Collaborators` + - After: `title: "Troubleshooting: Mapping with Collaborators"` + +9. `docs/troubleshooting/troubleshooting-observations--tracks.md` + - Before: `title: Troubleshooting: Observations & Tracks` + - After: `title: "Troubleshooting: Observations & Tracks"` + +10. `docs/troubleshooting/troubleshooting-moving-observations--tracks-outside-of-comapeo.md` + - Before: `title: Troubleshooting: Moving Observations & Tracks outside of CoMapeo` + - After: `title: "Troubleshooting: Moving Observations & Tracks outside of CoMapeo"` + +## Why This Happened + +1. **PR #75 fixed the generation script** but didn't update existing files +2. **PRs regenerate content** from Notion using the fixed script (success ✅) +3. **Main branch uses the content branch** which has old files (failure ❌) + +## Prevention + +### Already Implemented (PR #75) + +The `scripts/notion-fetch/generateBlocks.ts` now has a `quoteYamlValue()` function that automatically quotes special characters in all generated frontmatter. This prevents new files from having this issue. + +### Additional Recommendations + +1. **Add CI validation**: Create a pre-commit or CI check to validate YAML frontmatter +2. **Document the requirement**: Add to CLAUDE.md that all YAML values with special characters must be quoted +3. **Regular audits**: Run `scripts/fix-yaml-frontmatter.ts` periodically on the content branch + +## Using the Fix Script + +The `scripts/fix-yaml-frontmatter.ts` script can be used anytime to validate and fix YAML frontmatter: + +```bash +# Fix all markdown files in docs/ +bun scripts/fix-yaml-frontmatter.ts docs/ + +# Fix files in a specific subdirectory +bun scripts/fix-yaml-frontmatter.ts docs/troubleshooting/ + +# Check without modifying (dry run - not implemented yet) +# Can be added as a feature if needed +``` + +The script: +- ✅ Scans all `.md` and `.mdx` files recursively +- ✅ Identifies frontmatter fields needing quotes +- ✅ Quotes fields with special characters: `& : [ ] { } , | > * ! % @ \` # -` +- ✅ Preserves already-quoted values +- ✅ Reports summary of changes + +## Testing + +After applying the fix, verify with: + +```bash +# Test that YAML parses correctly +bunx prettier --check docs/**/*.md + +# Test that build succeeds +bun run build + +# Expected output: +# [SUCCESS] Generated static files in "build" +``` + +## Need Help? + +If issues persist: +1. Check `INVESTIGATION-REPORT.md` for detailed analysis +2. Verify PR #75 changes are merged to main +3. Ensure Notion API credentials are configured (if regenerating content) +4. Check CI logs for specific error messages + +## Related Files + +- `INVESTIGATION-REPORT.md` - Detailed analysis of the issue +- `scripts/fix-yaml-frontmatter.ts` - Automated fix script +- `scripts/notion-fetch/generateBlocks.ts:998-1016` - Prevention code +- PR #75 - Original fix for generation scripts diff --git a/scripts/fix-yaml-frontmatter.ts b/scripts/fix-yaml-frontmatter.ts new file mode 100644 index 00000000..da1599d8 --- /dev/null +++ b/scripts/fix-yaml-frontmatter.ts @@ -0,0 +1,205 @@ +#!/usr/bin/env bun + +/** + * Script to fix YAML frontmatter in markdown files by quoting special characters + * + * This script scans markdown files and quotes YAML frontmatter values that contain + * special characters which can break YAML parsing (& : [ ] { } , | > * ! % @ ` # -) + * + * Usage: + * bun scripts/fix-yaml-frontmatter.ts [directory] + * + * Example: + * bun scripts/fix-yaml-frontmatter.ts docs/ + */ + +import { readFileSync, writeFileSync, readdirSync, statSync } from "fs"; +import { join } from "path"; + +/** + * Check if a YAML value needs quoting + * YAML special characters: & : [ ] { } , | > * ! % @ ` # - and quotes + */ +function needsQuoting(value: string): boolean { + if (!value || typeof value !== "string") { + return false; + } + + // Already quoted + if ( + (value.startsWith('"') && value.endsWith('"')) || + (value.startsWith("'") && value.endsWith("'")) + ) { + return false; + } + + // Check for special characters + return /[&:[\]{}|>*!%@`#-]|^\s|^['"]|['"]$/.test(value); +} + +/** + * Quote a YAML value if needed + */ +function quoteValue(value: string): string { + if (!needsQuoting(value)) { + return value; + } + + // Use double quotes and escape any existing double quotes + return `"${value.replace(/"/g, '\\"')}"`; +} + +/** + * Process a single markdown file + */ +function processMarkdownFile(filePath: string): { + modified: boolean; + error?: string; +} { + try { + const content = readFileSync(filePath, "utf-8"); + + // Check if file has frontmatter + if (!content.startsWith("---")) { + return { modified: false }; + } + + // Extract frontmatter + const endOfFrontmatter = content.indexOf("---", 3); + if (endOfFrontmatter === -1) { + return { modified: false }; + } + + const frontmatter = content.substring(3, endOfFrontmatter); + const bodyContent = content.substring(endOfFrontmatter); + + // Process frontmatter line by line + const lines = frontmatter.split("\n"); + let modified = false; + const newLines = lines.map((line) => { + // Match key: value patterns + const match = line.match(/^(\s*)([\w_-]+):\s*(.+?)(\s*)$/); + if (!match) { + return line; + } + + const [, indent, key, value, trailing] = match; + + // Fields that should be quoted + const fieldsToCheck = [ + "title", + "sidebar_label", + "pagination_label", + "description", + ]; + + // Check if this is a field we want to quote + if (!fieldsToCheck.includes(key)) { + // Check for nested fields like "title: ..." in sidebar_custom_props + if (key === "title" && indent.length > 0) { + // This is a nested title field + } else { + return line; + } + } + + const quotedValue = quoteValue(value.trim()); + if (quotedValue !== value.trim()) { + modified = true; + return `${indent}${key}: ${quotedValue}${trailing}`; + } + + return line; + }); + + if (!modified) { + return { modified: false }; + } + + // Write back the file + const newContent = "---" + newLines.join("\n") + bodyContent; + writeFileSync(filePath, newContent, "utf-8"); + + return { modified: true }; + } catch (error) { + return { + modified: false, + error: error instanceof Error ? error.message : String(error), + }; + } +} + +/** + * Recursively process all markdown files in a directory + */ +function processDirectory( + dirPath: string +): { total: number; modified: number; errors: string[] } { + const results = { total: 0, modified: 0, errors: [] as string[] }; + + function walk(dir: string) { + const entries = readdirSync(dir); + + for (const entry of entries) { + const fullPath = join(dir, entry); + const stat = statSync(fullPath); + + if (stat.isDirectory()) { + walk(fullPath); + } else if (entry.endsWith(".md") || entry.endsWith(".mdx")) { + results.total++; + console.log(`Processing: ${fullPath}`); + + const result = processMarkdownFile(fullPath); + + if (result.error) { + console.error(` ❌ Error: ${result.error}`); + results.errors.push(`${fullPath}: ${result.error}`); + } else if (result.modified) { + console.log(` ✅ Fixed frontmatter`); + results.modified++; + } else { + console.log(` ⏭️ No changes needed`); + } + } + } + } + + walk(dirPath); + return results; +} + +/** + * Main function + */ +function main() { + const args = process.argv.slice(2); + const targetDir = args[0] || "docs"; + + console.log(`\n🔍 Scanning markdown files in: ${targetDir}\n`); + + const results = processDirectory(targetDir); + + console.log(`\n📊 Summary:`); + console.log(` Total files: ${results.total}`); + console.log(` Modified: ${results.modified}`); + console.log(` Errors: ${results.errors.length}`); + + if (results.errors.length > 0) { + console.log(`\n❌ Errors encountered:`); + results.errors.forEach((error) => console.log(` ${error}`)); + process.exit(1); + } + + if (results.modified > 0) { + console.log( + `\n✅ Fixed ${results.modified} file(s) with frontmatter issues` + ); + console.log(` Review changes with: git diff`); + console.log(` Commit changes with: git add -A && git commit`); + } else { + console.log(`\n✨ All files have properly formatted frontmatter!`); + } +} + +main();