Add CI for Sphinx/ReadTheDocs build

.github/workflows/sphinx-build-test.yml

@@ -0,0 +1,40 @@
+name: Build Sphinx Documentation
+
+on:
+  pull_request:
+    branches: [main]
+
+jobs:
+  sphinx-build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          submodules: 'recursive'
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+          cache: 'pip'
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+
+      - name: Build Sphinx Documentation
+        run: |
+          cd docs
+          # Run Sphinx build to catch issues
+          # -D build_toctree=True enables toctree processing to verify all docs are included
+          python -m sphinx -b html . _build/html
+
+      - name: Check for broken links to external sites
+        run: |
+          cd docs
+          # Run linkcheck builder to find broken links
+          python -m sphinx -b linkcheck . _build/linkcheck
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

docs/_ext/generate_toc_html.py

@@ -93,7 +93,6 @@ def process_document(env, docname, parent_maxdepth=1, processed_docs=None):
 
     # First check for commented toctree
     doc_path = env.doc2path(docname)
-    logger.info(f"Checking for commented toctree in {doc_path}")
     with open(doc_path, 'r', encoding='utf-8') as f:
         content = f.read()
 
@@ -137,7 +136,6 @@ def process_document(env, docname, parent_maxdepth=1, processed_docs=None):
 
     # Then process uncommented toctrees
     uncommented_toctrees = list(doctree.traverse(addnodes.toctree))
-    logger.info(f"Found {len(uncommented_toctrees)} uncommented toctrees in {docname}")
     for node in uncommented_toctrees:
         caption = node.get('caption')
         maxdepth = node.get('maxdepth', parent_maxdepth)

docs/conf.py

@@ -14,24 +14,25 @@
 sys.path.insert(0, os.path.abspath('.'))  # For finding _ext
 sys.path.insert(0, os.path.abspath('..'))
 
-def source_read_handler(app, docname, source):
+def uncomment_toctrees(app, docname, source):
     content = source[0]
     # Regex to find the comment block and extract its content
     pattern = re.compile(r'<!-- RTD-TOC-START\s*(.*?)\s*RTD-TOC-END -->', re.DOTALL)
-    
+
     def uncomment_toc(match):
         return match.group(1) # Return only the content inside the comments
 
     # Replace the comment block with its uncommented content
     new_content = pattern.sub(uncomment_toc, content)
+
     source[0] = new_content
 
 def handle_utf16le_files(app, docname, source):
     doc_path = Path(app.env.doc2path(docname))
 
     with open(doc_path, 'rb') as f:
         content_bytes = f.read()
-        
+
         # Check for UTF-16LE BOM (FF FE)
         if content_bytes.startswith(b'\xff\xfe'):
             # Decode from UTF-16LE
@@ -41,23 +42,47 @@ def handle_utf16le_files(app, docname, source):
             # Set the source content
             source[0] = content
 
+def add_orphan_directive(app, docname, source):
+    content = source[0]
+    # Check if the document already has "orphan: true"
+    if 'orphan: true' in content:
+        return
+
+    # Check if the document starts with YAML frontmatter
+    if content.strip().startswith('---'):
+        # Find the end of frontmatter
+        lines = content.split('\n')
+        frontmatter_end = -1
+        for i in range(1, len(lines)):
+            if lines[i].strip() == '---':
+                frontmatter_end = i
+                break
+
+        # Insert orphan: true at the end of the frontmatter
+        if frontmatter_end != -1:
+            lines.insert(frontmatter_end - 1, 'orphan: true')
+            source[0] = '\n'.join(lines)
+    else:
+        # No frontmatter, add frontmatter with "orphan: true"
+        source[0] = '---\norphan: true\n---\n' + content
+
 def latex_block_to_inline(app, docname, source):
     content = source[0]
     # Replace $$ with $ for inline math, but only when not part of a block
     # First find all block math ($$...$$) on their own lines
     block_matches = re.finditer(r'^\s*\$\$(.*?)\$\$\s*$', content, re.MULTILINE | re.DOTALL)
     block_positions = [(m.start(), m.end()) for m in block_matches]
-    
+
     # Now find all $$ pairs
     all_matches = list(re.finditer(r'\$\$(.*?)\$\$', content, re.DOTALL))
-    
+
     # Filter to only inline matches by checking if they overlap with any block matches
     def is_inline(match):
         pos = match.span()
         return not any(block_start <= pos[0] <= block_end for block_start, block_end in block_positions)
-    
+
     inline_matches = [m for m in all_matches if is_inline(m)]
-    
+
     # Replace inline $$ with $ working backwards to preserve positions
     for match in reversed(inline_matches):
         start, end = match.span()
@@ -66,8 +91,22 @@ def is_inline(match):
     source[0] = content
 
 def setup(app):
-    # Processing toctrees is really slow, O(n^2), so we will leave them commented out
-    # app.connect('source-read', source_read_handler)
+    # Register custom configuration value
+    app.add_config_value('build_toctree', False, 'env', [bool])
+
+    # Check if we want to build toctrees for validation
+    # This can be set via: sphinx-build -D build_toctree=true
+    build_toctree = getattr(app.config, 'build_toctree', False)
+
+    if build_toctree:
+        # Enable toctree processing to verify all docs are included
+        app.connect('source-read', uncomment_toctrees)
+    else:
+        # When not building toctrees, add :orphan: to documents to suppress toctree warnings
+        # Processing toctrees is really slow, so we leave them commented out in normal builds
+        # and build the TOC a different way
+        app.connect('source-read', add_orphan_directive)
+
     app.connect('source-read', latex_block_to_inline)
     app.connect('source-read', handle_utf16le_files)
 
@@ -98,25 +137,24 @@ def setup(app):
 # Debugging flag for verbose output
 verbose = True
 
-intersphinx_mapping = {
-    'python': ('https://docs.python.org/3/', None),
-    'sphinx': ('https://www.sphinx-doc.org/en/master/', None),
-}
-intersphinx_disabled_domains = ['std']
-
 templates_path = ['_templates']
 exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store', 'index.md',
+                    'external/slang/CONTRIBUTING.md',
+                    'external/slang/README.md',
+                    'external/slang/docs/README.md',
                     'external/slang/docs/stdlib-doc.md',
+                    'external/slang/examples/README.md',
                     'external/slang/external',
+                    'external/slangpy',
+                    'getting-started.md',
 ]
-include_patterns = ['index.rst', '*.md',
-                    "external/slang/docs/user-guide/*.md",
-                    "external/slang/docs/command-line-slangc-reference.md",
-                    "external/core-module-reference/index.md",
-                    "external/core-module-reference/attributes/**",
-                    "external/core-module-reference/global-decls/**",
-                    "external/core-module-reference/interfaces/**",
-                    "external/core-module-reference/types/**",
+
+include_patterns = [
+    'index.rst',
+    '*.md',
+    '*.rst',
+    '**/*.md',
+    '**/*.rst',
 ]
 
 # Configure myst-parser for markdown files
@@ -133,6 +171,27 @@ def setup(app):
 myst_heading_anchors = 3
 myst_title_to_header = True
 
+# Suppress specific warnings
+suppress_warnings = ["myst.header", "myst.xref_missing", "myst.xref_ambiguous"]
+
+linkcheck_anchors = False
+linkcheck_ignore = [
+    r"https://github.com/your-name/.*",
+    r"http://claude.ai/code",
+    r"http://libllvm\.so.*",
+    r"http://libLLVM\.so.*", 
+    r"http://slang\.so.*",
+]
+linkcheck_report_timeouts_as_broken = True
+
+# Configure request headers for authentication
+linkcheck_request_headers = {
+    "https://github.com/*": {
+        "Authorization": f"token {os.environ.get('GITHUB_TOKEN', '')}",
+        "User-Agent": "Slang-Documentation-Linkcheck/1.0"
+    }
+}
+
 # -- Options for HTML output -------------------------------------------------
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output