From 81e993328e800daad5d521c045b7ddb7a180d81d Mon Sep 17 00:00:00 2001 From: Rodrigo Espinoza Date: Fri, 16 Jan 2026 15:18:50 -0600 Subject: [PATCH 1/8] feat(POTS-000): initial commit for playwright implementation over POTs --- config.yml | 54 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/config.yml b/config.yml index de4208a..9ad871a 100644 --- a/config.yml +++ b/config.yml @@ -355,6 +355,57 @@ jobs: fi echo "Script done" + playwright-test: + <<: *defaults + docker: + - image: quay.io/pantheon-public/build-tools-ci:8.x-php<< pipeline.parameters.php-version >> + executor: playwright-executor + # parallelism: 4 + environment: + <<: *env + steps: + - restore-project: + workspace_root: << pipeline.parameters.artifact_workspace >> + + - restore_cache: + keys: + - composer-cache-{{ checksum "composer.lock" }} + - composer-cache- + + - run: + name: setup-environment-vars + command: | + if [ -f "./.circleci/scripts/set-environment" ]; then + ./.circleci/scripts/set-environment + else + ./vendor/fourkitchens/pots/scripts/set-environment + fi + - run: | + echo "export REMOTE_ENV_BASE_URL=https://${BASIC_USER}:${BASIC_PASS}@${TERMINUS_ENV}-${TERMINUS_SITE}.pantheonsite.io" >> $BASH_ENV + - run: + name: Install npm dependencies + command: cd test/playwright && npm ci + - run: + name: Install Playwright browsers + command: cd test/playwright && npx playwright install --with-deps + - run: + name: Waking up pantheon environment. + command: | + terminus -n auth:login --machine-token="$TERMINUS_TOKEN" + terminus -n env:wake $TERMINUS_SITE.$TERMINUS_ENV + - run: + name: Running playwright tests + command: | + cd test/playwright + npx playwright test + - store_test_results: + path: ./test/playwright/playwright-report + + - store_artifacts: + path: ./test/playwright/playwright-report + destination: playwright-report-first + - store_artifacts: + path: /tmp/artifacts # Run this on one of the main branches to push a release back to github. # # This usually will do things like create a change log and bump the version @@ -399,6 +450,9 @@ workflows: requires: - static_tests - build + - playwright-test: + requires: + - deploy_to_pantheon - approve_deploy_to_test: type: approval requires: From 53c1a23b3dc50091cd62bd184569bfd795dceccd Mon Sep 17 00:00:00 2001 From: Rodrigo Espinoza Date: Fri, 23 Jan 2026 11:57:53 -0600 Subject: [PATCH 2/8] feat(4K_INT-43): making playwright testing optional --- config.yml | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/config.yml b/config.yml index 9ad871a..a2e325c 100644 --- a/config.yml +++ b/config.yml @@ -12,6 +12,12 @@ parameters: default: false # The value for this variable will be set using a External POST request # To trigger the Automatic security updates check. + run-playwright-tests: + description: "Enables playwright tests" + type: boolean + default: false + # Enables playwright tests. + php-version: description: major.minor version number for php version to use. type: string @@ -436,6 +442,13 @@ workflows: - 'main' - 'master' + playwright_test: + when: << pipeline.parameters.run-playwright-tests >> + jobs: + - playwright-test: + requires: + - deploy_to_pantheon + build-deploy-pantheon: when: and: @@ -450,9 +463,6 @@ workflows: requires: - static_tests - build - - playwright-test: - requires: - - deploy_to_pantheon - approve_deploy_to_test: type: approval requires: From f08d1d4c24afa7f4fc613a87ffb760584b0f0e6b Mon Sep 17 00:00:00 2001 From: Rodrigo Espinoza Date: Fri, 23 Jan 2026 13:58:55 -0600 Subject: [PATCH 3/8] feat(4K_INT-43): Improving optional job for playwright --- config.yml | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/config.yml b/config.yml index a2e325c..5839680 100644 --- a/config.yml +++ b/config.yml @@ -17,6 +17,11 @@ parameters: type: boolean default: false # Enables playwright tests. + playwright-path: + description: "Path to playwright tests" + type: string + default: "test/playwright" + # Path where the playwright test are located. php-version: description: major.minor version number for php version to use. @@ -370,6 +375,13 @@ jobs: environment: <<: *env steps: + - run: + name: Skip Playwright when disabled + command: | + if [ "<< pipeline.parameters.run-playwright-tests >>" != "true" ]; then + echo "Playwright tests disabled via run-playwright-tests." + circleci step halt + fi - restore-project: workspace_root: << pipeline.parameters.artifact_workspace >> @@ -387,13 +399,17 @@ jobs: ./vendor/fourkitchens/pots/scripts/set-environment fi - run: | - echo "export REMOTE_ENV_BASE_URL=https://${BASIC_USER}:${BASIC_PASS}@${TERMINUS_ENV}-${TERMINUS_SITE}.pantheonsite.io" >> $BASH_ENV + if [ -n "$BASIC_USER" ] && [ -n "$BASIC_PASS" ]; then + echo "export REMOTE_ENV_BASE_URL=https://${BASIC_USER}:${BASIC_PASS}@${TERMINUS_ENV}-${TERMINUS_SITE}.pantheonsite.io" >> $BASH_ENV + else + echo "export REMOTE_ENV_BASE_URL=https://${TERMINUS_ENV}-${TERMINUS_SITE}.pantheonsite.io" >> $BASH_ENV + fi - run: name: Install npm dependencies - command: cd test/playwright && npm ci + command: cd << pipeline.parameters.playwright-path >> && npm ci - run: name: Install Playwright browsers - command: cd test/playwright && npx playwright install --with-deps + command: cd << pipeline.parameters.playwright-path >> && npx playwright install --with-deps - run: name: Waking up pantheon environment. command: | @@ -402,13 +418,13 @@ jobs: - run: name: Running playwright tests command: | - cd test/playwright + cd << pipeline.parameters.playwright-path >> npx playwright test - store_test_results: - path: ./test/playwright/playwright-report + path: ./<< pipeline.parameters.playwright-path >>/playwright-report - store_artifacts: - path: ./test/playwright/playwright-report + path: ./<< pipeline.parameters.playwright-path >>/playwright-report destination: playwright-report-first - store_artifacts: path: /tmp/artifacts From 2e362ba318543bde65c634ce588a847788021135 Mon Sep 17 00:00:00 2001 From: Rodrigo Espinoza Date: Fri, 23 Jan 2026 14:18:38 -0600 Subject: [PATCH 4/8] feat(4K_INT-43): adding missing code --- config.yml | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/config.yml b/config.yml index 5839680..ff2cc81 100644 --- a/config.yml +++ b/config.yml @@ -1,5 +1,11 @@ version: 2.1 +# Executors define the environment in which the jobs will run. +executors: + playwright-executor: + docker: + - image: mcr.microsoft.com/playwright:v1.55.0-noble + # To easily configure jobs here, parameters are set up below. Since we are # typically not calling this workflow via pipelines, consider changing the # defaults here for any permanent configuration you need to do on a project @@ -458,13 +464,6 @@ workflows: - 'main' - 'master' - playwright_test: - when: << pipeline.parameters.run-playwright-tests >> - jobs: - - playwright-test: - requires: - - deploy_to_pantheon - build-deploy-pantheon: when: and: @@ -479,6 +478,9 @@ workflows: requires: - static_tests - build + - playwright-test: + requires: + - deploy_to_pantheon - approve_deploy_to_test: type: approval requires: From dba3fd7548a2f626613d5997f3e3cd3263e42b35 Mon Sep 17 00:00:00 2001 From: Rodrigo Espinoza Date: Fri, 20 Mar 2026 15:17:46 -0600 Subject: [PATCH 5/8] feat(4K_INT-43): adding job for VRT testing --- config.yml | 109 ++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 95 insertions(+), 14 deletions(-) diff --git a/config.yml b/config.yml index ff2cc81..d84b45d 100644 --- a/config.yml +++ b/config.yml @@ -18,17 +18,23 @@ parameters: default: false # The value for this variable will be set using a External POST request # To trigger the Automatic security updates check. - run-playwright-tests: - description: "Enables playwright tests" + run-playwright-e2e-tests: + description: "Enables playwright E2E tests" type: boolean - default: false - # Enables playwright tests. + default: true + # Enables playwright VRT tests. + run-playwright-vrt-tests: + description: "Enables playwright VRT tests" + type: boolean + default: true + playwright-vrt-target-env: + description: "Pantheon environment to compare the deployed multidev against during VRT" + type: string + default: "test" playwright-path: description: "Path to playwright tests" type: string default: "test/playwright" - # Path where the playwright test are located. - php-version: description: major.minor version number for php version to use. type: string @@ -372,19 +378,19 @@ jobs: fi echo "Script done" - playwright-test: + playwright-e2e-test: <<: *defaults docker: - image: quay.io/pantheon-public/build-tools-ci:8.x-php<< pipeline.parameters.php-version >> executor: playwright-executor - # parallelism: 4 + parallelism: 4 environment: <<: *env steps: - run: name: Skip Playwright when disabled command: | - if [ "<< pipeline.parameters.run-playwright-tests >>" != "true" ]; then + if [ "<< pipeline.parameters.run-playwright-e2e-tests >>" != "true" ]; then echo "Playwright tests disabled via run-playwright-tests." circleci step halt fi @@ -406,9 +412,9 @@ jobs: fi - run: | if [ -n "$BASIC_USER" ] && [ -n "$BASIC_PASS" ]; then - echo "export REMOTE_ENV_BASE_URL=https://${BASIC_USER}:${BASIC_PASS}@${TERMINUS_ENV}-${TERMINUS_SITE}.pantheonsite.io" >> $BASH_ENV + echo "export REMOTE_ENV_BASE_URL=https://${BASIC_USER}:${BASIC_PASS}@${TERMINUS_ENV}-${TERMINUS_SITE}.pantheonsite.io/" >> $BASH_ENV else - echo "export REMOTE_ENV_BASE_URL=https://${TERMINUS_ENV}-${TERMINUS_SITE}.pantheonsite.io" >> $BASH_ENV + echo "export REMOTE_ENV_BASE_URL=https://${TERMINUS_ENV}-${TERMINUS_SITE}.pantheonsite.io/" >> $BASH_ENV fi - run: name: Install npm dependencies @@ -422,10 +428,78 @@ jobs: terminus -n auth:login --machine-token="$TERMINUS_TOKEN" terminus -n env:wake $TERMINUS_SITE.$TERMINUS_ENV - run: - name: Running playwright tests + name: Wake pantheon environment via HTTP + command: | + curl -fsS --retry 5 --retry-delay 5 "$REMOTE_ENV_BASE_URL" > /dev/null + - run: + name: Running playwright E2E tests command: | cd << pipeline.parameters.playwright-path >> - npx playwright test + npx playwright test --grep-invert "@vrt" + - store_test_results: + path: ./<< pipeline.parameters.playwright-path >>/playwright-report + + - store_artifacts: + path: ./<< pipeline.parameters.playwright-path >>/playwright-report + destination: playwright-report-first + - store_artifacts: + path: /tmp/artifacts + + + playwright-vrt-test: + <<: *defaults + docker: + - image: quay.io/pantheon-public/build-tools-ci:8.x-php<< pipeline.parameters.php-version >> + executor: playwright-executor + parallelism: 4 + environment: + <<: *env + steps: + - run: + name: Skip Playwright when disabled + command: | + if [ "<< pipeline.parameters.run-playwright-vrt-tests >>" != "true" ]; then + echo "Playwright tests disabled via run-playwright-vrt-tests." + circleci step halt + fi + - restore-project: + workspace_root: << pipeline.parameters.artifact_workspace >> + + - restore_cache: + keys: + - composer-cache-{{ checksum "composer.lock" }} + - composer-cache- + + - run: + name: setup-environment-vars + command: | + if [ -f "./.circleci/scripts/set-environment" ]; then + ./.circleci/scripts/set-environment + else + ./vendor/fourkitchens/pots/scripts/set-environment + fi + - run: + name: Install npm dependencies + command: cd << pipeline.parameters.playwright-path >> && npm ci + - run: + name: Install Playwright browsers + command: cd << pipeline.parameters.playwright-path >> && npx playwright install --with-deps + - run: + name: Waking up source pantheon environment. + command: | + terminus -n auth:login --machine-token="$TERMINUS_TOKEN" + terminus -n env:wake $TERMINUS_SITE.$TERMINUS_ENV + - run: + name: Configure VRT source and target environments + command: | + echo "export VRT_PANTHEON_SITE=${TERMINUS_SITE}" >> $BASH_ENV + echo "export VRT_SOURCE_ENV=${TERMINUS_ENV}" >> $BASH_ENV + echo "export VRT_TARGET_ENV=<< pipeline.parameters.playwright-vrt-target-env >>" >> $BASH_ENV + - run: + name: Running playwright VRT tests + command: | + cd << pipeline.parameters.playwright-path >> + npm run vrt:ci - store_test_results: path: ./<< pipeline.parameters.playwright-path >>/playwright-report @@ -478,9 +552,16 @@ workflows: requires: - static_tests - build - - playwright-test: + - playwright-e2e-test: requires: - deploy_to_pantheon + - approve-playwright-vrt-test: + type: approval + requires: + - deploy_to_pantheon + - playwright-vrt-test: + requires: + - approve-playwright-vrt-test - approve_deploy_to_test: type: approval requires: From 61e7ca316d640b05b1e28cbc5709d7a7914f5249 Mon Sep 17 00:00:00 2001 From: Rodrigo Espinoza Date: Fri, 20 Mar 2026 15:28:08 -0600 Subject: [PATCH 6/8] feat(4K_INT-43): Adding Documentation about playwright into pots --- README.MD | 101 +++++++++++++++++++++++++++++++++++++++++++++++++++++ config.yml | 4 +-- 2 files changed, 103 insertions(+), 2 deletions(-) diff --git a/README.MD b/README.MD index 8b3b239..32698c0 100644 --- a/README.MD +++ b/README.MD @@ -34,6 +34,10 @@ Whatever you want it to stand for. Here's some options: - [Configure More CircleCi Environment Variables](#configure-more-circleci-environment-variables-1) * [Configure CircleCI config.yml](#configure-circleci-configyml) + [Pipeline Parameter Defaults](#pipeline-parameter-defaults) + - [run-playwright-e2e-tests](#run-playwright-e2e-tests) + - [run-playwright-vrt-tests](#run-playwright-vrt-tests) + - [playwright-vrt-target-env](#playwright-vrt-target-env) + - [playwright-path](#playwright-path) - [php-version](#php-version) - [tz](#tz) - [host-variant](#host-variant) @@ -44,6 +48,7 @@ Whatever you want it to stand for. Here's some options: - [artifact_workspace](#artifact_workspace) + [Other Configuration](#other-configuration) - [persisting_dirs](#persisting_dirs) + - [Playwright Jobs](#playwright-jobs) * [Configurable Environment Variables](#configurable-environment-variables) + [Shared](#shared-1) + [Pantheon Specific](#pantheon-specific) @@ -351,6 +356,63 @@ to ``` The following parameters exist: +#### run-playwright-e2e-tests +This boolean controls whether the Playwright end-to-end test job runs after the +Pantheon multidev is created. + +Default: `true` + +When enabled, the `playwright-e2e-test` job will: + +- restore the built project from the artifact workspace +- install dependencies from the Playwright test directory +- install Playwright browsers +- wake the Pantheon environment +- run `npx playwright test --grep-invert "@vrt"` + +Use this when you want standard browser coverage on the deployed multidev but +you do not want visual regression tests mixed into that run. + +#### run-playwright-vrt-tests +This boolean controls whether the Playwright visual regression test job is +available in the Pantheon workflow. + +Default: `true` + +This job is still behind a manual CircleCI approval step named +`approve-playwright-vrt-test`. Once approved, the job will halt immediately if +this parameter is set to `false`. + +The VRT job is intended for screenshot comparison checks and runs separately +from the standard E2E suite. + +#### playwright-vrt-target-env +This string defines which Pantheon environment should be used as the visual +baseline for VRT comparisons. + +Default: `"test"` + +During the `playwright-vrt-test` job, this value is exported as +`VRT_TARGET_ENV`. The currently deployed multidev becomes the source +environment, and the target environment becomes the baseline for comparison. + +#### playwright-path +This string points to the directory that contains your Playwright project. + +Default: `"test/playwright"` + +This directory is expected to contain the files needed for Playwright CI runs, +including `package.json` and your Playwright configuration. The current jobs +use this path for: + +- `npm ci` +- `npx playwright install --with-deps` +- `npx playwright test --grep-invert "@vrt"` +- `npm run vrt:ci` + +If your tests live somewhere else, update this parameter instead of changing +each job manually. + #### php-version This is a string that represents the version number for PHP version to use across the entire build. @@ -430,6 +492,29 @@ sometimes want to include other root directories that aren't included like `node_modules`, `simplesaml`, or `private`. We exclude most top level directories because they are unneeded on the hosting system itself. +#### Playwright Jobs +The Playwright jobs are currently part of the Pantheon workflow only. Their +purpose is to validate the freshly deployed multidev before any later manual +deployment steps. + +The workflow order is: + +- `deploy_to_pantheon` deploys the multidev +- `playwright-e2e-test` runs automatically after that deploy +- `approve-playwright-vrt-test` lets a user decide whether to run VRT +- `playwright-vrt-test` runs after approval + +Some practical details: + +- The E2E job excludes any test tagged with `@vrt` +- The VRT job expects an `npm run vrt:ci` script in your Playwright project +- Both jobs store the Playwright HTML report as a CircleCI artifact +- The deploy approvals for Pantheon test and live remain separate from the + Playwright approval step + +This makes the default flow easier to manage: regular browser coverage can run +automatically, while the slower visual comparison step stays opt-in. + ## Configurable Environment Variables Environment variables can be configured using the CircleCI Environment Variables @@ -460,6 +545,12 @@ mandatory environment variables in this way as well. `system.site.yml`. Ex: YES if `./config/system.site.yml` or `./config/default/system.site.yml` or `./config/sync/system.site.yml` exists. Possible values are `YES`, or `NO`. +* `BASIC_USER`: Optional username used when the deployed Pantheon environment is + protected by HTTP basic authentication. If set together with `BASIC_PASS`, + the Playwright E2E job will use those credentials when waking the site over + HTTP. +* `BASIC_PASS`: Optional password paired with `BASIC_USER` for HTTP basic + authentication during Playwright E2E runs. ### Pantheon Specific @@ -539,6 +630,16 @@ The scripts available for overload are as follows: multidev. On Acquia and General hosting, this just gets the Artifact code to the repo it should live in. +For Playwright support, your project should also provide a Playwright workspace +at the path configured by `playwright-path`. In practice that means: + +- a `package.json` that supports `npm ci` +- standard E2E tests that can run with `npx playwright test --grep-invert "@vrt"` +- a VRT script named `vrt:ci` for the visual regression job + +If you want a spec to run only as part of VRT, tag it with `@vrt` so it is +excluded from the standard E2E job. + ### Pantheon Specific Scripts - `drush-commands`: Provides the basic deployment drush commands to run update diff --git a/config.yml b/config.yml index d84b45d..938a487 100644 --- a/config.yml +++ b/config.yml @@ -21,12 +21,12 @@ parameters: run-playwright-e2e-tests: description: "Enables playwright E2E tests" type: boolean - default: true + default: false # Enables playwright VRT tests. run-playwright-vrt-tests: description: "Enables playwright VRT tests" type: boolean - default: true + default: false playwright-vrt-target-env: description: "Pantheon environment to compare the deployed multidev against during VRT" type: string From e0caebdacb151a4491c01b0317d42025e2e60190 Mon Sep 17 00:00:00 2001 From: Rodrigo Espinoza Date: Wed, 25 Mar 2026 17:34:14 -0600 Subject: [PATCH 7/8] feat(4K_INT-43): Making playwright test to run in parallel --- config.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/config.yml b/config.yml index 938a487..2061fd5 100644 --- a/config.yml +++ b/config.yml @@ -41,7 +41,7 @@ parameters: # This value must be only a major.minor version if pantheon is the # host-variant as the build-tool images do not support major.minor.patch # version numbers. - default: "8.2" + default: "8.3" tz: description: The timezone definition. type: string @@ -416,6 +416,8 @@ jobs: else echo "export REMOTE_ENV_BASE_URL=https://${TERMINUS_ENV}-${TERMINUS_SITE}.pantheonsite.io/" >> $BASH_ENV fi + echo "export PLAYWRIGHT_SHARD=$((${CIRCLE_NODE_INDEX}+1))" >> $BASH_ENV + echo "export PLAYWRIGHT_TOTAL_SHARDS=${CIRCLE_NODE_TOTAL}" >> $BASH_ENV - run: name: Install npm dependencies command: cd << pipeline.parameters.playwright-path >> && npm ci @@ -435,7 +437,7 @@ jobs: name: Running playwright E2E tests command: | cd << pipeline.parameters.playwright-path >> - npx playwright test --grep-invert "@vrt" + npx playwright test --grep-invert "@vrt" --shard="${PLAYWRIGHT_SHARD}/${PLAYWRIGHT_TOTAL_SHARDS}" - store_test_results: path: ./<< pipeline.parameters.playwright-path >>/playwright-report @@ -495,6 +497,8 @@ jobs: echo "export VRT_PANTHEON_SITE=${TERMINUS_SITE}" >> $BASH_ENV echo "export VRT_SOURCE_ENV=${TERMINUS_ENV}" >> $BASH_ENV echo "export VRT_TARGET_ENV=<< pipeline.parameters.playwright-vrt-target-env >>" >> $BASH_ENV + echo "export PLAYWRIGHT_SHARD=$((${CIRCLE_NODE_INDEX}+1))" >> $BASH_ENV + echo "export PLAYWRIGHT_TOTAL_SHARDS=${CIRCLE_NODE_TOTAL}" >> $BASH_ENV - run: name: Running playwright VRT tests command: | From 12814548ab423fa0a4bcd96badc5d04d99744e8f Mon Sep 17 00:00:00 2001 From: Rodrigo Espinoza Date: Fri, 27 Mar 2026 09:22:08 -0600 Subject: [PATCH 8/8] feat(4K_INT-43): Making live the default env --- config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config.yml b/config.yml index 2061fd5..3fc12f4 100644 --- a/config.yml +++ b/config.yml @@ -30,7 +30,7 @@ parameters: playwright-vrt-target-env: description: "Pantheon environment to compare the deployed multidev against during VRT" type: string - default: "test" + default: "live" playwright-path: description: "Path to playwright tests" type: string