Notebook parity: strict section/cell + one-to-one figure ordering

.github/workflows/ci.yml

@@ -55,10 +55,35 @@ jobs:
           python -m pip install -e .[dev,docs,notebooks]
           python -m pip install reportlab pillow
 
+      - name: Resolve nSTAT data cache directory
+        id: nstat-data-dir
+        run: |
+          echo "dir=$(python tools/data/print_data_dir.py)" >> "$GITHUB_OUTPUT"
+
+      - name: Cache nSTAT example data
+        uses: actions/cache@v4
+        with:
+          path: ${{ steps.nstat-data-dir.outputs.dir }}
+          key: nstat-example-data-${{ runner.os }}-doi-10.6084-m9.figshare.4834640-v1
+          restore-keys: |
+            nstat-example-data-${{ runner.os }}-doi-10.6084-m9.figshare.4834640-
+
+      - name: Download nSTAT example data
+        run: |
+          python tools/data/download_example_data.py
+
+      - name: Checkout pinned MATLAB nSTAT reference
+        run: |
+          python tools/parity/checkout_matlab_reference.py \
+            --config parity/matlab_reference.yml \
+            --dest /tmp/upstream-nstat \
+            --metadata-out parity/matlab_reference_checkout_docs_smoke.json
+
       - name: Rebuild generated docs/notebooks
         run: |
           python tools/docs/generate_help_pages.py
           python tools/notebooks/generate_notebooks.py
+          python tools/notebooks/clean_notebooks.py --check
 
       - name: Ensure generated artifacts are committed
         run: |
@@ -71,7 +96,11 @@ jobs:
         run: python tools/docs/verify_search_index.py
 
       - name: Run smoke notebooks
-        run: python tools/notebooks/run_notebooks.py --group smoke --timeout 600
+        run: |
+          python tools/notebooks/execute_notebooks.py \
+            --group smoke \
+            --timeout 600 \
+            --out-report output/notebooks/notebook_execution_report.json
 
       - name: Generate smoke validation PDF
         run: |
@@ -93,6 +122,16 @@ jobs:
           path: output/pdf/*.pdf
           if-no-files-found: warn
 
+      - name: Upload smoke notebook execution artifact
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: ci-smoke-notebook-execution
+          path: |
+            output/notebooks/notebook_execution_report.json
+            output/notebooks/executed/**
+          if-no-files-found: warn
+
       - name: Run release gate checks
         run: python tools/release/check_release_gate.py
 

.github/workflows/full-parity-nightly.yml

@@ -32,6 +32,23 @@ jobs:
           python -m pip install -e .[dev,docs,notebooks]
           python -m pip install reportlab pillow
 
+      - name: Resolve nSTAT data cache directory
+        id: nstat-data-dir
+        run: |
+          echo "dir=$(python tools/data/print_data_dir.py)" >> "$GITHUB_OUTPUT"
+
+      - name: Cache nSTAT example data
+        uses: actions/cache@v4
+        with:
+          path: ${{ steps.nstat-data-dir.outputs.dir }}
+          key: nstat-example-data-${{ runner.os }}-doi-10.6084-m9.figshare.4834640-v1
+          restore-keys: |
+            nstat-example-data-${{ runner.os }}-doi-10.6084-m9.figshare.4834640-
+
+      - name: Download nSTAT example data
+        run: |
+          python tools/data/download_example_data.py
+
       - name: Checkout pinned MATLAB nSTAT reference
         run: |
           python tools/parity/checkout_matlab_reference.py \
@@ -43,6 +60,10 @@ jobs:
         run: |
           python tools/parity/prepare_validation_images.py
 
+      - name: Validate notebook cleanliness
+        run: |
+          python tools/notebooks/clean_notebooks.py --check
+
       - name: Run full parity and notebook gates
         run: |
           python tools/parity/build_parity_snapshot.py \
@@ -59,7 +80,10 @@ jobs:
           python tools/parity/check_example_output_spec.py \
             --report parity/function_example_alignment_report.json \
             --spec parity/example_output_spec.yml
-          python tools/notebooks/run_notebooks.py --group all --timeout 900
+          python tools/notebooks/execute_notebooks.py \
+            --group all \
+            --timeout 900 \
+            --out-report output/notebooks/notebook_execution_report.json
 
       - name: Generate class equivalence inventory/report artifacts
         run: |
@@ -131,3 +155,13 @@ jobs:
           name: nightly-validation-images
           path: tmp/pdfs/validation_report/notebook_images
           if-no-files-found: warn
+
+      - name: Upload notebook execution artifact
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: nightly-notebook-execution
+          path: |
+            output/notebooks/notebook_execution_report.json
+            output/notebooks/executed/**
+          if-no-files-found: warn

.github/workflows/image-mode-parity.yml

@@ -32,6 +32,23 @@ jobs:
           python -m pip install -e .[dev,notebooks]
           python -m pip install reportlab pillow
 
+      - name: Resolve nSTAT data cache directory
+        id: nstat-data-dir
+        run: |
+          echo "dir=$(python tools/data/print_data_dir.py)" >> "$GITHUB_OUTPUT"
+
+      - name: Cache nSTAT example data
+        uses: actions/cache@v4
+        with:
+          path: ${{ steps.nstat-data-dir.outputs.dir }}
+          key: nstat-example-data-${{ runner.os }}-doi-10.6084-m9.figshare.4834640-v1
+          restore-keys: |
+            nstat-example-data-${{ runner.os }}-doi-10.6084-m9.figshare.4834640-
+
+      - name: Download nSTAT example data
+        run: |
+          python tools/data/download_example_data.py
+
       - name: Checkout pinned MATLAB nSTAT reference
         run: |
           python tools/parity/checkout_matlab_reference.py \
@@ -43,6 +60,10 @@ jobs:
         run: |
           python tools/parity/prepare_validation_images.py
 
+      - name: Validate notebook cleanliness
+        run: |
+          python tools/notebooks/clean_notebooks.py --check
+
       - name: Generate Python validation PDF (image mode)
         run: |
           python tools/reports/generate_validation_pdf.py \
@@ -73,6 +94,7 @@ jobs:
           python tools/reports/check_pdf_image_parity.py \
             --python-pdf output/pdf/image_mode_parity/python_pages.pdf \
             --matlab-pdf output/pdf/image_mode_parity/matlab_pages.pdf \
+            --pairs-json output/pdf/image_mode_parity/pairs.json \
             --out-dir output/pdf/image_mode_parity \
             --dpi 150 \
             --ssim-threshold 0.70 \

.github/workflows/notebooks-full.yml

@@ -21,8 +21,41 @@ jobs:
           python -m pip install --upgrade pip
           python -m pip install -e .[notebooks]
 
+      - name: Resolve nSTAT data cache directory
+        id: nstat-data-dir
+        run: |
+          echo "dir=$(python tools/data/print_data_dir.py)" >> "$GITHUB_OUTPUT"
+
+      - name: Cache nSTAT example data
+        uses: actions/cache@v4
+        with:
+          path: ${{ steps.nstat-data-dir.outputs.dir }}
+          key: nstat-example-data-${{ runner.os }}-doi-10.6084-m9.figshare.4834640-v1
+          restore-keys: |
+            nstat-example-data-${{ runner.os }}-doi-10.6084-m9.figshare.4834640-
+
+      - name: Download nSTAT example data
+        run: |
+          python tools/data/download_example_data.py
+
       - name: Generate notebooks
-        run: python tools/notebooks/generate_notebooks.py
+        run: |
+          python tools/notebooks/generate_notebooks.py
+          python tools/notebooks/clean_notebooks.py --check
 
       - name: Execute full notebook suite
-        run: python tools/notebooks/run_notebooks.py --group full --timeout 900
+        run: |
+          python tools/notebooks/execute_notebooks.py \
+            --group full \
+            --timeout 900 \
+            --out-report output/notebooks/notebook_execution_report.json
+
+      - name: Upload full notebook execution artifact
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: notebooks-full-execution
+          path: |
+            output/notebooks/notebook_execution_report.json
+            output/notebooks/executed/**
+          if-no-files-found: warn

.github/workflows/parity-gate.yml

@@ -31,6 +31,23 @@ jobs:
           python -m pip install -e .[dev,notebooks]
           python -m pip install pyyaml
 
+      - name: Resolve nSTAT data cache directory
+        id: nstat-data-dir
+        run: |
+          echo "dir=$(python tools/data/print_data_dir.py)" >> "$GITHUB_OUTPUT"
+
+      - name: Cache nSTAT example data
+        uses: actions/cache@v4
+        with:
+          path: ${{ steps.nstat-data-dir.outputs.dir }}
+          key: nstat-example-data-${{ runner.os }}-doi-10.6084-m9.figshare.4834640-v1
+          restore-keys: |
+            nstat-example-data-${{ runner.os }}-doi-10.6084-m9.figshare.4834640-
+
+      - name: Download nSTAT example data
+        run: |
+          python tools/data/download_example_data.py
+
       - name: Checkout pinned MATLAB nSTAT reference
         run: |
           python tools/parity/checkout_matlab_reference.py \
@@ -42,6 +59,10 @@ jobs:
         run: |
           python tools/parity/prepare_validation_images.py
 
+      - name: Validate notebook cleanliness
+        run: |
+          python tools/notebooks/clean_notebooks.py --check
+
       - name: Build parity snapshot and enforce gates
         run: |
           python tools/parity/build_parity_snapshot.py \

.github/workflows/release-rc.yml

@@ -47,6 +47,7 @@ jobs:
       - name: Regenerate notebooks
         run: |
           python tools/notebooks/generate_notebooks.py
+          python tools/notebooks/clean_notebooks.py --check
           git diff --exit-code
 
       - name: Checkout pinned MATLAB nSTAT reference

.github/workflows/release-stable.yml

@@ -56,6 +56,7 @@ jobs:
       - name: Rebuild notebooks and verify clean state
         run: |
           python tools/notebooks/generate_notebooks.py
+          python tools/notebooks/clean_notebooks.py --check
           git diff --exit-code
 
       - name: Hard checks (lint, typing, unit tests, docs)

.github/workflows/validation-pdf.yml

@@ -33,10 +33,22 @@ jobs:
           python -m pip install -e .[dev,notebooks]
           python -m pip install reportlab pillow
 
-      - name: Regenerate notebooks
+      - name: Resolve nSTAT data cache directory
+        id: nstat-data-dir
         run: |
-          python tools/notebooks/generate_notebooks.py
-          git diff --exit-code
+          echo "dir=$(python tools/data/print_data_dir.py)" >> "$GITHUB_OUTPUT"
+
+      - name: Cache nSTAT example data
+        uses: actions/cache@v4
+        with:
+          path: ${{ steps.nstat-data-dir.outputs.dir }}
+          key: nstat-example-data-${{ runner.os }}-doi-10.6084-m9.figshare.4834640-v1
+          restore-keys: |
+            nstat-example-data-${{ runner.os }}-doi-10.6084-m9.figshare.4834640-
+
+      - name: Download nSTAT example data
+        run: |
+          python tools/data/download_example_data.py
 
       - name: Checkout pinned MATLAB nSTAT reference
         run: |
@@ -45,6 +57,12 @@ jobs:
             --dest /tmp/upstream-nstat \
             --metadata-out parity/matlab_reference_checkout.json
 
+      - name: Regenerate notebooks
+        run: |
+          python tools/notebooks/generate_notebooks.py
+          python tools/notebooks/clean_notebooks.py --check
+          git diff --exit-code
+
       - name: Prepare deterministic validation images
         run: |
           python tools/parity/prepare_validation_images.py

.gitignore

@@ -2,6 +2,7 @@
 __pycache__/
 *.py[cod]
 *.so
+.DS_Store
 
 # Packaging
 build/
@@ -28,6 +29,9 @@ docs/_build/
 
 # Local data cache
 .data_cache/
+data_cache/
+output/data_download/
+*.zip
 
 # Local report artifacts
 output/