Staging

.github/actions/setup-ci-tools/action.yml

@@ -35,13 +35,22 @@ runs:
         echo "branch=$BRANCH" >> $GITHUB_OUTPUT
         echo "Using nmaci branch: $BRANCH"
 
+    - name: Get nmaci latest commit
+      id: nmaci-sha
+      shell: bash
+      run: |
+        BRANCH=${{ steps.detect-branch.outputs.branch }}
+        SHA=$(git ls-remote https://github.com/neuromatch/nmaci.git refs/heads/$BRANCH | cut -c1-8)
+        echo "sha=$SHA" >> $GITHUB_OUTPUT
+        echo "nmaci $BRANCH is at commit $SHA"
+
     - name: Cache nmaci tools
       id: cache-nmaci
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       with:
         path: ci/
-        key: nmaci-${{ runner.os }}-${{ steps.detect-branch.outputs.branch }}-v2
-        restore-keys: nmaci-${{ runner.os }}-${{ steps.detect-branch.outputs.branch }}-
+        key: nmaci-v2-${{ runner.os }}-${{ steps.detect-branch.outputs.branch }}-${{ steps.nmaci-sha.outputs.sha }}
+        restore-keys: nmaci-v2-${{ runner.os }}-${{ steps.detect-branch.outputs.branch }}-
 
     - name: Download nmaci tools
       if: steps.cache-nmaci.outputs.cache-hit != 'true'
@@ -50,6 +59,7 @@ runs:
         BRANCH=${{ steps.detect-branch.outputs.branch }}
         wget -q https://github.com/neuromatch/nmaci/archive/refs/heads/$BRANCH.tar.gz
         tar -xzf $BRANCH.tar.gz
+        rm -rf ci/scripts
         mv nmaci-$BRANCH/scripts/ ci/
         # Preserve requirements for caching
         mv nmaci-$BRANCH/requirements.txt ci/requirements.txt

.github/workflows/notebook-pr.yaml

@@ -103,6 +103,7 @@ jobs:
     needs: setup
     if: needs.setup.outputs.skip_ci != 'true' && needs.setup.outputs.has_notebooks == 'true'
     runs-on: ubuntu-latest
+    timeout-minutes: 120
     strategy:
       fail-fast: false
       matrix:
@@ -134,14 +135,28 @@ jobs:
       - name: Process notebook
         env:
           COMMIT_MESSAGE: ${{ needs.setup.outputs.commit_message }}
+          PYTHONUNBUFFERED: "1"
         run: |
-          echo "Processing: ${{ matrix.notebook }}"
+          echo "::group::Processing ${{ matrix.notebook }}"
+          echo "Started at: $(date -u '+%Y-%m-%d %H:%M:%S UTC')"
+          echo "Job timeout: 20 minutes"
+
           if [[ "$COMMIT_MESSAGE" == *"ci:check"* ]]; then
             execflag="--check-execution"
+            echo "Mode: Check execution (no changes)"
           else
             execflag="--execute"
+            echo "Mode: Execute and process"
           fi
+          echo "::endgroup::"
+
+          echo "::group::Notebook execution output"
           python ci/process_notebooks.py "${{ matrix.notebook }}" $execflag
+          exit_code=$?
+          echo "::endgroup::"
+
+          echo "Completed at: $(date -u '+%Y-%m-%d %H:%M:%S UTC')"
+          exit $exit_code
 
       - name: Get tutorial directory
         id: get-dir
@@ -198,6 +213,85 @@ jobs:
         with:
           path: artifacts/
 
+      - name: Detect failed notebooks
+        id: detect-failures
+        run: |
+          expected='${{ needs.setup.outputs.notebooks }}'
+          failed_nbs=""
+          success_nbs=""
+
+          echo "Checking which notebooks were successfully processed..."
+          for nb in $(echo "$expected" | jq -r '.[]'); do
+            nb_name=$(basename "$nb" .ipynb)
+            # Check if artifact exists for this notebook
+            if ls artifacts/tutorial-*"___${nb_name}" 1>/dev/null 2>&1; then
+              echo "  ✓ $nb - SUCCESS"
+              success_nbs="$success_nbs $nb"
+            else
+              echo "  ✗ $nb - FAILED (no artifact)"
+              failed_nbs="$failed_nbs $nb"
+            fi
+          done
+
+          # Trim leading spaces
+          success_nbs="${success_nbs# }"
+          failed_nbs="${failed_nbs# }"
+
+          echo "success_notebooks=$success_nbs" >> $GITHUB_OUTPUT
+          echo "failed_notebooks=$failed_nbs" >> $GITHUB_OUTPUT
+
+          if [ -n "$failed_nbs" ]; then
+            echo "has_failures=true" >> $GITHUB_OUTPUT
+            echo ""
+            echo "⚠️  Some notebooks failed to process: $failed_nbs"
+          else
+            echo "has_failures=false" >> $GITHUB_OUTPUT
+            echo ""
+            echo "✅ All notebooks processed successfully"
+          fi
+
+      - name: Collect failure context
+        if: steps.detect-failures.outputs.has_failures == 'true'
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          run_id=${{ github.run_id }}
+          echo "## Notebook Processing Failure Context" > failure_context.md
+          echo "" >> failure_context.md
+          jobs_json=$(gh api repos/${{ github.repository }}/actions/runs/${run_id}/jobs)
+          for nb in ${{ steps.detect-failures.outputs.failed_notebooks }}; do
+            nb_name=$(basename "$nb" .ipynb)
+            echo "### $nb" >> failure_context.md
+            echo '```' >> failure_context.md
+            job_id=$(echo "$jobs_json" | jq -r --arg nb "$nb" '.jobs[] | select(.name | contains($nb)) | .id')
+            if [ -n "$job_id" ] && [ "$job_id" != "null" ]; then
+              # Get logs from the "Notebook execution output" group
+              gh api repos/${{ github.repository }}/actions/jobs/${job_id}/logs 2>/dev/null | \
+                awk '
+                  /##\[group\]Notebook execution output/{capturing=1; next}
+                  capturing && /##\[endgroup\]/{exit}
+                  capturing && /##\[error\]/{print; exit}
+                  capturing && /##\[group\]/{exit}
+                  capturing && /Post job cleanup/{exit}
+                  capturing{print}
+                ' | \
+                tail -100 >> failure_context.md || echo "Could not retrieve logs" >> failure_context.md
+            else
+              echo "Job not found for $nb" >> failure_context.md
+            fi
+            echo '```' >> failure_context.md
+            echo "" >> failure_context.md
+          done
+
+      - name: Post failure context comment
+        if: steps.detect-failures.outputs.has_failures == 'true'
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          if [ -f failure_context.md ]; then
+            gh pr comment ${{ github.event.pull_request.number }} --body-file failure_context.md
+          fi
+
       - name: Restore processed files
         run: |
           echo "Restoring processed files from artifacts..."
@@ -251,7 +345,98 @@ jobs:
       - name: Clean up artifacts directory
         run: rm -rf artifacts/
 
+      - name: Revert failed notebooks to main
+        if: steps.detect-failures.outputs.has_failures == 'true' && steps.detect-failures.outputs.success_notebooks != ''
+        run: |
+          failed_nbs="${{ steps.detect-failures.outputs.failed_notebooks }}"
+          echo "Reverting failed notebooks to main branch version: $failed_nbs"
+
+          # Revert each failed notebook to the main branch version
+          for nb in $failed_nbs; do
+            if git show origin/main:"$nb" > /dev/null 2>&1; then
+              git checkout origin/main -- "$nb"
+              echo "  Reverted: $nb"
+            else
+              # Notebook is new in this PR, remove it
+              git rm -f "$nb" 2>/dev/null || rm -f "$nb"
+              echo "  Removed (new file): $nb"
+            fi
+          done
+
+      - name: Create follow-up PR for failed notebooks
+        if: steps.detect-failures.outputs.has_failures == 'true' && steps.detect-failures.outputs.success_notebooks != ''
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          git config --local user.email "action@github.com"
+          git config --local user.name "GitHub Action"
+
+          failed_nbs="${{ steps.detect-failures.outputs.failed_notebooks }}"
+          pr_number=${{ github.event.pull_request.number }}
+          original_branch=${{ github.head_ref }}
+          new_branch="reprocess-pr${pr_number}-$(date +%Y%m%d%H%M%S)"
+
+          echo "Creating follow-up PR for failed notebooks: $failed_nbs"
+
+          # Stash all local changes to keep new branch clean
+          git stash --include-untracked
+
+          # Create new branch from origin/main (has main's version of notebooks)
+          git fetch origin main
+          git checkout -b "$new_branch" origin/main
+
+          # Copy failed notebooks from the original PR branch (unprocessed version)
+          for nb in $failed_nbs; do
+            git checkout "origin/$original_branch" -- "$nb" 2>/dev/null || echo "Note: $nb may be new in this PR"
+          done
+
+          # Commit the notebooks
+          git add $failed_nbs
+          git commit -m "Reprocess notebooks from PR #${pr_number}
+
+          These notebooks failed to process in the original PR and need reprocessing:
+          $(echo "$failed_nbs" | tr ' ' '\n' | sed 's/^/- /')"
+
+          # Configure git to use GITHUB_TOKEN for authentication
+          gh auth setup-git
+
+          # Push the new branch
+          git push origin "$new_branch"
+
+          # Create the PR targeting the original branch and capture the URL
+          new_pr_url=$(gh pr create \
+            --title "Reprocess: Failed notebooks from PR #${pr_number}" \
+            --body "## Auto-generated PR
+
+          The following notebooks failed to process in PR #${pr_number} and need reprocessing:
+
+          $(echo "$failed_nbs" | tr ' ' '\n' | sed 's/^/- /')
+
+          This PR was automatically created to allow these notebooks to be processed independently.
+
+          Original PR: #${pr_number}" \
+            --base "$original_branch" \
+            --head "$new_branch")
+
+          echo "Created follow-up PR: $new_pr_url"
+
+          # Add comment to original PR referencing the follow-up PR
+          gh pr comment "$pr_number" --body "## Follow-up PR Created
+
+          A follow-up PR has been created for the notebooks that failed to process:
+
+          **Follow-up PR:** $new_pr_url
+          **Branch:** \`$new_branch\`
+
+          The failed notebooks have been reverted to their \`main\` branch versions in this PR.
+          Once this PR is merged, the follow-up PR can be used to reprocess the failed notebooks independently."
+
+          # Return to original branch and restore state
+          git checkout "$original_branch"
+          git stash pop || true
+
       - name: Commit post-processed files
+        if: steps.detect-failures.outputs.success_notebooks != ''
         run: |
           git config --local user.email "action@github.com"
           git config --local user.name "GitHub Action"
@@ -261,9 +446,18 @@ jobs:
           git add '**/student/*.ipynb'
           git add '**/instructor/*.ipynb'
           git add '**/README.md'
-          git diff-index --quiet HEAD || git commit -m "Process tutorial notebooks"
+
+          # Build commit message based on whether there were failures
+          if [ "${{ steps.detect-failures.outputs.has_failures }}" == "true" ]; then
+            msg="Process tutorial notebooks (some notebooks moved to follow-up PR)"
+          else
+            msg="Process tutorial notebooks"
+          fi
+
+          git diff-index --quiet HEAD || git commit -m "$msg"
 
       - name: Push post-processed files
+        if: steps.detect-failures.outputs.success_notebooks != ''
         uses: ad-m/github-push-action@v0.6.0
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}