Add E2E Testing #117

.github/docs/NIGHTLY-BUILD.md

@@ -17,8 +17,9 @@ Tracks are specified as a comma-separated string in the `NIGHTLY_TRACKS` repo va
 | `test-backend-<branch>` | Run backend tests against `<branch>` |
 | `test-frontend-<branch>` | Run frontend tests against `<branch>` |
 | `deploy-<branch>` | Deploy full stack from `<branch>` with automatic teardown |
+| `e2e-<branch>` | Deploy full stack from `<branch>`, run Playwright E2E tests, then teardown |
 | `merge-validation:<base>:<overlay>` | Deploy `<base>`, then overlay `<overlay>` on top (colons delimit to avoid branch name ambiguity) |
-| `all` | Run all tracks with defaults: tests + deploy on `develop`, MV `main`→`develop` |
+| `all` | Run all tracks with defaults: tests + deploy + e2e on `develop`, MV `main`→`develop` |
 
 ### Examples
 
@@ -27,9 +28,10 @@ test-backend-develop
 test-frontend-main,test-backend-main
 deploy-develop
 deploy-main,deploy-develop
+e2e-develop
 merge-validation:main:develop
 merge-validation:main:feature/my-branch
-test-backend-develop,deploy-develop,merge-validation:main:develop
+test-backend-develop,deploy-develop,e2e-develop,merge-validation:main:develop
 all
 ```
 
@@ -44,7 +46,7 @@ The `resolve-tracks` job parses the tracks string into boolean flags and branch
 ## Workflow Jobs
 
 ### resolve-tracks
-Parses the tracks string and outputs boolean flags (`run_test_backend`, `run_test_frontend`, `run_deploy`, `run_mv`) and branch refs for each enabled track.
+Parses the tracks string and outputs boolean flags (`run_test_backend`, `run_test_frontend`, `run_deploy`, `run_e2e`, `run_mv`) and branch refs for each enabled track.
 
 ### Test Tracks
 
@@ -62,7 +64,23 @@ When `deploy-<branch>` is specified, calls the reusable `nightly-deploy-pipeline
 - `alb-subdomain`: `nightly-<branch>-api`
 - Automatic teardown (unless `skip_teardown` is set)
 
-The deploy pipeline runs: install-infra → check-stack-deps → deploy-infra → deploy-rag → deploy-inference → deploy-app → deploy-frontend + deploy-gateway → smoke-test → teardown.
+The deploy pipeline runs: install-infra → check-stack-deps → deploy-infra → deploy-rag → deploy-inference → deploy-app → deploy-frontend + deploy-gateway → smoke-test → e2e-tests (if enabled) → teardown.
+
+### E2E Test Track
+
+When `e2e-<branch>` is specified, calls the reusable `nightly-deploy-pipeline.yml` with:
+- `project-prefix`: `nightly-e2e-<branch>`
+- `alb-subdomain`: `nightly-e2e-<branch>-api`
+- `run-e2e`: `true`
+- Automatic teardown (unless `skip_teardown` is set)
+
+This deploys a full stack and runs Playwright E2E tests against it. The E2E job runs after the smoke test confirms the stack is healthy. It uses a separate CI Playwright config (`playwright.ci.config.ts`) that points at the deployed stack URL instead of starting local servers.
+
+E2E test failures are **informational** — they mark the nightly summary as "partial" rather than "failed". This allows the track to stabilize without blocking other tracks.
+
+**Required secrets** (in the `development` environment):
+- `E2E_ADMIN_USERNAME` / `E2E_ADMIN_PASSWORD` — Cognito admin test account
+- `E2E_USER_USERNAME` / `E2E_USER_PASSWORD` — Cognito regular user test account
 
 ### Merge Validation Track
 
@@ -91,6 +109,7 @@ Reusable workflow (`workflow_call`) containing the full deploy pipeline.
 | `skip-teardown` | no | Skip teardown (default: `false`) |
 | `label` | no | Label for job names |
 | `source-project-prefix` | no | If set, Docker jobs try ECR image promotion before building |
+| `run-e2e` | no | Run Playwright E2E tests after smoke test (default: `false`) |
 
 ### Promote-or-Build Pattern
 
@@ -126,6 +145,8 @@ Deploy and MV tracks use the `development` GitHub environment with overrides:
 ### Required Secrets
 - `AWS_ROLE_ARN` or `AWS_ACCESS_KEY_ID` + `AWS_SECRET_ACCESS_KEY`
 - `GITHUB_TOKEN` (automatically provided)
+- `E2E_ADMIN_USERNAME` / `E2E_ADMIN_PASSWORD` — Cognito admin test account (required for e2e track)
+- `E2E_USER_USERNAME` / `E2E_USER_PASSWORD` — Cognito regular user test account (required for e2e track)
 
 ### Required Variables
 - `AWS_REGION`, `CDK_AWS_ACCOUNT`
@@ -138,6 +159,7 @@ Deploy and MV tracks use the `development` GitHub environment with overrides:
 | `backend-coverage` | 30 days | Backend test track enabled |
 | `frontend-coverage` | 30 days | Frontend test track enabled |
 | `coverage-comparison` | 30 days | Any test track succeeded |
+| `playwright-report-<label>` | 30 days | E2E track enabled (includes HTML report, screenshots, traces) |
 
 ## Troubleshooting
 
@@ -159,3 +181,10 @@ Check that `NIGHTLY_TRACKS` is set as a repository variable (not a secret). Empt
 
 ### Merge validation fails on overlay
 This is the intended signal — it means the overlay branch has CDK/infra incompatibilities with the base branch that need to be resolved before merging.
+
+### E2E tests fail
+- Download the `playwright-report-<label>` artifact for screenshots and traces
+- Check that `E2E_ADMIN_USERNAME`, `E2E_ADMIN_PASSWORD`, `E2E_USER_USERNAME`, `E2E_USER_PASSWORD` secrets are set in the `development` environment
+- Verify the Cognito test users exist and have the correct roles
+- If auth setup fails, the Cognito managed login UI may have changed — check the `auth-admin.setup.ts` / `auth-user.setup.ts` selectors
+- E2E failures are informational and do not block other tracks

.github/workflows/nightly-deploy-pipeline.yml

@@ -34,6 +34,11 @@ on:
         required: false
         default: ''
         type: string
+      run-e2e:
+        description: 'Run Playwright E2E tests after smoke test (requires Cognito test user secrets)'
+        required: false
+        default: 'false'
+        type: string
 
 permissions:
   contents: read
@@ -584,10 +589,80 @@ jobs:
           echo "status=$STATUS" >> "$GITHUB_OUTPUT"
           echo "duration=$SECONDS" >> "$GITHUB_OUTPUT"
 
+  e2e-tests:
+    name: "[${{ inputs.label }}] E2E Tests"
+    runs-on: ubuntu-24.04
+    needs: smoke-test
+    if: ${{ inputs.run-e2e == 'true' && needs.smoke-test.result == 'success' }}
+    environment: development
+    outputs:
+      status: ${{ steps.summary.outputs.status }}
+      duration: ${{ steps.summary.outputs.duration }}
+    permissions:
+      id-token: write
+      contents: read
+    env:
+      CDK_AWS_REGION: ${{ vars.AWS_REGION }}
+      CDK_PROJECT_PREFIX: ${{ inputs.project-prefix }}
+      CDK_ALB_SUBDOMAIN: ${{ inputs.alb-subdomain }}
+      CDK_HOSTED_ZONE_DOMAIN: ${{ vars.CDK_HOSTED_ZONE_DOMAIN }}
+      ADMIN_USERNAME: ${{ secrets.E2E_ADMIN_USERNAME }}
+      ADMIN_PASSWORD: ${{ secrets.E2E_ADMIN_PASSWORD }}
+      USER_USERNAME: ${{ secrets.E2E_USER_USERNAME }}
+      USER_PASSWORD: ${{ secrets.E2E_USER_PASSWORD }}
+      AWS_ROLE_ARN: ${{ secrets.AWS_ROLE_ARN }}
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          ref: ${{ inputs.ref }}
+
+      - name: Restore frontend node_modules cache
+        id: cache-frontend
+        uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
+        with:
+          path: frontend/ai.client/node_modules
+          key: frontend-node-modules-${{ hashFiles('frontend/ai.client/package-lock.json') }}
+
+      - name: Install frontend dependencies
+        if: steps.cache-frontend.outputs.cache-hit != 'true'
+        run: bash scripts/stack-frontend/install.sh
+
+      - name: Configure AWS credentials
+        uses: ./.github/actions/configure-aws-credentials
+        with:
+          aws-region: ${{ env.CDK_AWS_REGION }}
+          role-session-name: GitHubActions-${{ inputs.label }}-E2E
+          aws-role-arn: ${{ env.AWS_ROLE_ARN }}
+          aws-access-key-id: ${{ env.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ env.AWS_SECRET_ACCESS_KEY }}
+
+      - name: Run E2E tests
+        id: run-tests
+        run: bash scripts/nightly/e2e-test.sh
+
+      - name: Upload Playwright report
+        if: always()
+        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
+        with:
+          name: playwright-report-${{ inputs.label }}
+          path: frontend/ai.client/playwright-report/
+          retention-days: 30
+
+      - name: Generate summary
+        id: summary
+        if: always()
+        run: |
+          if [ "${{ steps.run-tests.outcome }}" = "success" ]; then STATUS="success"; else STATUS="failure"; fi
+          echo "status=$STATUS" >> "$GITHUB_OUTPUT"
+          echo "duration=$SECONDS" >> "$GITHUB_OUTPUT"
+
   teardown:
     name: "[${{ inputs.label }}] Teardown"
     runs-on: ubuntu-24.04
-    needs: smoke-test
+    needs: [smoke-test, e2e-tests]
     if: ${{ always() && inputs.skip-teardown != 'true' }}
     environment: development
     outputs:

.github/workflows/nightly.yml

@@ -5,10 +5,11 @@
 #   test-backend-<branch>              Run backend tests against <branch>
 #   test-frontend-<branch>             Run frontend tests against <branch>
 #   deploy-<branch>                    Deploy full stack from <branch>
+#   e2e-<branch>                       Deploy full stack, run Playwright E2E tests, teardown
 #   merge-validation:<base>:<overlay>  Deploy <base>, then overlay <overlay>
 #   scan-images-<branch>               Scan Docker images from <branch> for vulnerabilities
 #   all                                Run all tracks with default branches
-#                                      (tests + deploy on develop, MV main→develop)
+#                                      (tests + deploy + e2e on develop, MV main→develop)
 #
 # Examples:
 #   test-backend-develop
@@ -79,6 +80,8 @@ jobs:
       mv_overlay_ref: ${{ steps.resolve.outputs.mv_overlay_ref }}
       run_scan_images: ${{ steps.resolve.outputs.run_scan_images }}
       scan_images_ref: ${{ steps.resolve.outputs.scan_images_ref }}
+      run_e2e: ${{ steps.resolve.outputs.run_e2e }}
+      e2e_ref: ${{ steps.resolve.outputs.e2e_ref }}
       any_track: ${{ steps.resolve.outputs.any_track }}
     steps:
       - name: Parse tracks
@@ -103,6 +106,8 @@ jobs:
           mv_overlay_ref=""
           run_scan_images=false
           scan_images_ref=""
+          run_e2e=false
+          e2e_ref=""
 
           # Empty tracks = nothing runs (fork-safe default)
           if [ -z "$TRACKS" ]; then
@@ -125,6 +130,8 @@ jobs:
                   mv_overlay_ref="develop"
                   run_scan_images=true
                   scan_images_ref="develop"
+                  run_e2e=true
+                  e2e_ref="develop"
                   ;;
                 test-backend-*)
                   run_test_backend=true
@@ -147,6 +154,10 @@ jobs:
                   run_scan_images=true
                   scan_images_ref="${token#scan-images-}"
                   ;;
+                e2e-*)
+                  run_e2e=true
+                  e2e_ref="${token#e2e-}"
+                  ;;
                 *)
                   echo "::warning::Unknown track token: $token"
                   ;;
@@ -158,7 +169,7 @@ jobs:
           any_track=false
           if [ "$run_test_backend" = "true" ] || [ "$run_test_frontend" = "true" ] || \
              [ "$run_deploy" = "true" ] || [ "$run_mv" = "true" ] || \
-             [ "$run_scan_images" = "true" ]; then
+             [ "$run_scan_images" = "true" ] || [ "$run_e2e" = "true" ]; then
             any_track=true
           fi
 
@@ -175,6 +186,8 @@ jobs:
             echo "mv_overlay_ref=$mv_overlay_ref"
             echo "run_scan_images=$run_scan_images"
             echo "scan_images_ref=$scan_images_ref"
+            echo "run_e2e=$run_e2e"
+            echo "e2e_ref=$e2e_ref"
             echo "any_track=$any_track"
           } >> "$GITHUB_OUTPUT"
 
@@ -185,6 +198,7 @@ jobs:
           echo "  deploy:        $run_deploy (ref: $deploy_ref)"
           echo "  mv:            $run_mv (base: $mv_base_ref, overlay: $mv_overlay_ref)"
           echo "  scan-images:   $run_scan_images (ref: $scan_images_ref)"
+          echo "  e2e:           $run_e2e (ref: $e2e_ref)"
           echo "  any_track:     $any_track"
 
   # ── Test Tracks ───────────────────────────────────────────────────────────
@@ -471,6 +485,27 @@ jobs:
       label: deploy-${{ needs.resolve-tracks.outputs.deploy_ref }}
     secrets: inherit
 
+  # ── E2E Test Track ────────────────────────────────────────────────────────
+  # Deploys the full stack and runs Playwright E2E tests against it.
+  # Uses the deploy pipeline with run-e2e enabled. Separate from the deploy
+  # track so you can deploy without E2E or run E2E independently.
+
+  e2e:
+    needs: resolve-tracks
+    if: needs.resolve-tracks.outputs.run_e2e == 'true'
+    permissions:
+      id-token: write
+      contents: read
+    uses: ./.github/workflows/nightly-deploy-pipeline.yml
+    with:
+      ref: ${{ needs.resolve-tracks.outputs.e2e_ref }}
+      project-prefix: nightly-e2e-${{ needs.resolve-tracks.outputs.e2e_ref }}
+      alb-subdomain: nightly-e2e-${{ needs.resolve-tracks.outputs.e2e_ref }}-api
+      skip-teardown: ${{ github.event.inputs.skip_teardown || 'false' }}
+      label: e2e-${{ needs.resolve-tracks.outputs.e2e_ref }}
+      run-e2e: 'true'
+    secrets: inherit
+
   # ── Merge Validation Track ────────────────────────────────────────────────
   # Deploys base branch first, then overlays the overlay branch on top —
   # simulating a real merge to catch CDK/infra incompatibilities.
@@ -593,7 +628,7 @@ jobs:
   summary:
     name: Nightly Summary
     runs-on: ubuntu-24.04
-    needs: [resolve-tracks, test-backend, test-frontend, analyze-coverage, ai-coverage-analysis, deploy, mv-base, mv-overlay, scan-images]
+    needs: [resolve-tracks, test-backend, test-frontend, analyze-coverage, ai-coverage-analysis, deploy, e2e, mv-base, mv-overlay, scan-images]
     if: always() && needs.resolve-tracks.outputs.any_track == 'true'
     steps:
       - name: Checkout code
@@ -606,9 +641,11 @@ jobs:
           RUN_TEST_BACKEND: ${{ needs.resolve-tracks.outputs.run_test_backend }}
           RUN_TEST_FRONTEND: ${{ needs.resolve-tracks.outputs.run_test_frontend }}
           RUN_DEPLOY: ${{ needs.resolve-tracks.outputs.run_deploy }}
+          RUN_E2E: ${{ needs.resolve-tracks.outputs.run_e2e }}
           RUN_MV: ${{ needs.resolve-tracks.outputs.run_mv }}
           RUN_SCAN_IMAGES: ${{ needs.resolve-tracks.outputs.run_scan_images }}
           DEPLOY_REF: ${{ needs.resolve-tracks.outputs.deploy_ref }}
+          E2E_REF: ${{ needs.resolve-tracks.outputs.e2e_ref }}
           MV_BASE_REF: ${{ needs.resolve-tracks.outputs.mv_base_ref }}
           MV_OVERLAY_REF: ${{ needs.resolve-tracks.outputs.mv_overlay_ref }}
         run: |
@@ -657,6 +694,13 @@ jobs:
             [ "$status" = "failure" ] && AGGREGATE="failure"
           fi
 
+          # E2E track (reusable workflow — informational, does not fail aggregate)
+          if [ "$RUN_E2E" = "true" ]; then
+            status="$(map_status "${{ needs.e2e.result }}")"
+            ROWS+=("E2E Tests (${E2E_REF})|${status}|0")
+            [ "$status" = "failure" ] && [ "$AGGREGATE" != "failure" ] && AGGREGATE="partial"
+          fi
+
           # MV tracks
           if [ "$RUN_MV" = "true" ]; then
             status="$(map_status "${{ needs.mv-base.result }}")"