test: set up Cypress visual diff smoke tests

.gitignore

@@ -84,6 +84,7 @@ local_keycloak.json
 cypress.env.json
 cypress/videos/
 cypress/screenshots/
+cypress/snapshots/
 cypress/downloads/
 package-lock.json
 ## Documentation Markdown files served locally for testing during development 

cypress.config.js

@@ -1,5 +1,6 @@
 const { defineConfig } = require('cypress')
 const { plugin: cypressGrepPlugin } = require('@cypress/grep/plugin')
+const { configureVisualRegression } = require('cypress-visual-regression')
 const fs = require('fs')
 
 module.exports = defineConfig({
@@ -16,6 +17,9 @@ module.exports = defineConfig({
     AUTH_REALM: 'hmda2',
     AUTH_CLIENT_ID: 'hmda2-api',
     preserveCookies: ['_login_gov_session'],
+    visualRegressionBaseDirectory: 'cypress/snapshots/base',
+    visualRegressionDiffDirectory: 'cypress/snapshots/diff',
+    visualRegressionGenerateDiff: 'fail',
     // Always enable spec filtering
     grepFilterSpecs: true,
     // Always omit filtered tests
@@ -35,10 +39,8 @@ module.exports = defineConfig({
   e2e: {
     experimentalRunAllSpecs: true,
     testIsolation: true,
-    setupNodeEvents(on, config) {
-      return require('./cypress/plugins/index.js')(on, config)
-    },
     specPattern: 'cypress/e2e/**/*.{js,jsx,ts,tsx}',
+    screenshotsFolder: 'cypress/snapshots/actual',
     experimentalOriginDependencies: true,
     chromeWebSecurity: false,
     defaultCommandTimeout: 10000,
@@ -50,6 +52,9 @@ module.exports = defineConfig({
     // Delete videos for specs without failing or retried tests, see docs:
     // https://docs.cypress.io/app/guides/screenshots-and-videos#Delete-videos-for-specs-without-failing-or-retried-tests
     setupNodeEvents(on, config) {
+      require('./cypress/plugins/index.js')(on, config)
+      configureVisualRegression(on)
+
       on('after:spec', (spec, results) => {
         if (results && results.video) {
           // Do we have failures for any retry attempts?

cypress/README.md

@@ -67,6 +67,8 @@ We use [`@cypress/grep`](https://www.npmjs.com/package/@cypress/grep) to organiz
 - `yarn test-smoke`: Executes much quicker than the full suite of tests by running a subset of end-to-end tests that represent core functionality.
 - `yarn test-unauthenticated`: Runs all tests that don't require Keycloak authentication.
 - `yarn test-large`: Runs our large filer test which uploads a very large file and takes awhile.
+- `yarn test-visual:baseline`: Generates baseline snapshots for [visual regression testing](https://github.com/cypress-visual-regression/cypress-visual-regression). Can be run against any environment but to test local changes you'll want to do `CYPRESS_HOST=http://localhost:3000 yarn test-visual:baseline --env grepTags='@localhost'`.
+- `yarn test-visual`: Takes snapshots and compares them to the baseline images. Can be run against any environment but to test local changes you'll want to do `CYPRESS_HOST=http://localhost:3000 yarn test-visual --env grepTags='@localhost'`.
 
 When you add new tests, make sure to tag them appropriately so that they run with the above commands. You can view the commands and their tags in [`package.json`](https://github.com/cfpb/hmda-frontend/blob/master/package.json). For example, localhost tests have a `@localhost` tag. Tests that require Keycloak authentication have a `@auth-required` tag and the above `yarn test-unauthenticated` command runs all tests *without* this tag.
 

cypress/support/e2e.js

@@ -2,9 +2,39 @@ import { register as registerCypressGrep } from '@cypress/grep'
 import '@testing-library/cypress/add-commands'
 import 'cypress-file-upload'
 import 'cypress-keycloak'
+import { addCompareSnapshotCommand } from 'cypress-visual-regression/dist/command'
 import { logEnv, urlExists } from './helpers'
 
 registerCypressGrep()
+addCompareSnapshotCommand({ errorThreshold: 0.1 })
+
+// Create a name for the screenshot based on the test title
+const getScreenshotName = title => title.replace(/[^a-zA-Z0-9-_]/g, '-').replace(/-{2,}/g, '-').slice(0, 120)
+
+// When CYPRESS_visualRegressionType env var is set (to either `base` or `regression`),
+// a screenshot will be taken after each test.
+// See https://github.com/cypress-visual-regression/cypress-visual-regression#plugin-options
+afterEach(function () {
+  if (!Cypress.env('visualRegressionType')) return
+  if (this.currentTest?.state !== 'passed') return
+
+  const name = getScreenshotName(this.currentTest.fullTitle())
+
+  cy.get('body', { log: false }).then((body) => {
+    // Make sure the page is done loading
+    const isLoading = body.find('.LoadingIconWrapper').length > 0
+    if (isLoading) return
+
+    // Every page (I think?) has a #mainWrapper div that contains the content
+    const hasMainWrapper = body.find('#mainWrapper').length > 0
+    const screenshotWrapper = hasMainWrapper ? '#mainWrapper' : 'body'
+    const screenshotElement = body.find(screenshotWrapper).first()
+
+    if (!screenshotElement.length || !screenshotElement.is(':visible')) return
+
+    cy.get(screenshotWrapper, { log: false }).compareSnapshot(name)
+  })
+})
 
 // ***********************************************************
 // This example support/index.js is processed and

cypress/support/helpers.js

@@ -24,17 +24,8 @@ export function withFormData(method, url, formData, done) {
 }
 
 export function urlExists(url) {
-  return new Promise((resolve) => {
-    const xhr = new XMLHttpRequest()
-
-    xhr.onreadystatechange = () => {
-      if (xhr.readyState === 4)
-        resolve({ url, status: xhr.status < 400, statusCode: xhr.status })
-    }
-
-    xhr.open('HEAD', url)
-    xhr.send()
-  })
+  return cy.request({ url, method: 'HEAD', failOnStatusCode: false, timeout: 30000 })
+           .then((response) => ({ url, status: response.status < 400, statusCode: response.status }))
 }
 
 /* Data Browser Helpers */

package.json

@@ -11,6 +11,8 @@
     "build": "vite build",
     "preview": "vite preview",
     "test": "yarn run cypress run",
+    "test-visual:baseline": "CYPRESS_visualRegressionType=base yarn run cypress run",
+    "test-visual": "CYPRESS_visualRegressionType=regression yarn run cypress run",
     "test-localhost": "CYPRESS_HOST=http://localhost:3000 yarn run cypress run --env grepTags='@localhost'",
     "test-unauthenticated": "yarn run cypress run --env grepTags='-@auth-required'",
     "test-smoke": "CYPRESS_YEARS=2024 yarn run cypress run --env grepTags='@smoke'",
@@ -96,6 +98,7 @@
     "cypress": "^14.0",
     "cypress-file-upload": "5.0.8",
     "cypress-keycloak": "2.0.2",
+    "cypress-visual-regression": "^5.3.0",
     "dotenv": "^16.3.1",
     "ejs": "3.1.10",
     "enzyme": "^3.11.0",

yarn.lock

@@ -4941,7 +4941,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"chalk@npm:^4.0.0, chalk@npm:^4.1.0, chalk@npm:^4.1.2":
+"chalk@npm:4.1.2, chalk@npm:^4.0.0, chalk@npm:^4.1.0, chalk@npm:^4.1.2":
   version: 4.1.2
   resolution: "chalk@npm:4.1.2"
   dependencies:
@@ -5621,6 +5621,20 @@ __metadata:
   languageName: node
   linkType: hard
 
+"cypress-visual-regression@npm:^5.3.0":
+  version: 5.3.0
+  resolution: "cypress-visual-regression@npm:5.3.0"
+  dependencies:
+    chalk: "npm:4.1.2"
+    pixelmatch: "npm:5.3.0"
+    pngjs: "npm:7.0.0"
+    sanitize-filename: "npm:1.6.3"
+  peerDependencies:
+    cypress: ">=12"
+  checksum: 10c0/a38ee912f85dd444fcd327884449dbf2079216d6aa2e8e953b7dd389ebc9a6d7c6ee03a3dccb20b93957976bd536299b314c74d3c98aab9b8fb59d416afeacd0
+  languageName: node
+  linkType: hard
+
 "cypress@npm:^14.0":
   version: 14.5.4
   resolution: "cypress@npm:14.5.4"
@@ -8107,6 +8121,7 @@ __metadata:
     cypress: "npm:^14.0"
     cypress-file-upload: "npm:5.0.8"
     cypress-keycloak: "npm:2.0.2"
+    cypress-visual-regression: "npm:^5.3.0"
     date-fns: "npm:^4.1.0"
     detect-browser: "npm:4.8.0"
     dotenv: "npm:^16.3.1"
@@ -11541,6 +11556,17 @@ __metadata:
   languageName: node
   linkType: hard
 
+"pixelmatch@npm:5.3.0":
+  version: 5.3.0
+  resolution: "pixelmatch@npm:5.3.0"
+  dependencies:
+    pngjs: "npm:^6.0.0"
+  bin:
+    pixelmatch: bin/pixelmatch
+  checksum: 10c0/30850661db29b57cefbe6cf36e930b7517aea4e0ed129e85fcc8ec04a7e6e7648a822a972f8e01d2d3db268ca3c735555caf6b8099a164d8b64d105986d682d2
+  languageName: node
+  linkType: hard
+
 "pkg-dir@npm:^4.2.0":
   version: 4.2.0
   resolution: "pkg-dir@npm:4.2.0"
@@ -11578,6 +11604,20 @@ __metadata:
   languageName: node
   linkType: hard
 
+"pngjs@npm:7.0.0":
+  version: 7.0.0
+  resolution: "pngjs@npm:7.0.0"
+  checksum: 10c0/0d4c7a0fd476a9c33df7d0a2a73e1d56537628a668841f6995c2bca070cf30819f9254a64363266bc14ef2fee47659dd3b4f2b18eec7ab65143015139f497b38
+  languageName: node
+  linkType: hard
+
+"pngjs@npm:^6.0.0":
+  version: 6.0.0
+  resolution: "pngjs@npm:6.0.0"
+  checksum: 10c0/ac23ea329b1881d1a10575aff58116dc27b894ec3f5b84ba15c7f527d21e609fbce7ba16d48f8ccb86c7ce45ceed622472765476ab2875949d4bec55e153f87a
+  languageName: node
+  linkType: hard
+
 "possible-typed-array-names@npm:^1.0.0":
   version: 1.1.0
   resolution: "possible-typed-array-names@npm:1.1.0"
@@ -12993,6 +13033,15 @@ __metadata:
   languageName: node
   linkType: hard
 
+"sanitize-filename@npm:1.6.3":
+  version: 1.6.3
+  resolution: "sanitize-filename@npm:1.6.3"
+  dependencies:
+    truncate-utf8-bytes: "npm:^1.0.0"
+  checksum: 10c0/16ff47556a6e54e228c28db096bedd303da67b030d4bea4925fd71324932d6b02c7b0446f00ad33987b25b6414f24ae968e01a1a1679ce599542e82c4b07eb1f
+  languageName: node
+  linkType: hard
+
 "sass-embedded-android-arm64@npm:1.83.4":
   version: 1.83.4
   resolution: "sass-embedded-android-arm64@npm:1.83.4"
@@ -14269,6 +14318,15 @@ __metadata:
   languageName: node
   linkType: hard
 
+"truncate-utf8-bytes@npm:^1.0.0":
+  version: 1.0.2
+  resolution: "truncate-utf8-bytes@npm:1.0.2"
+  dependencies:
+    utf8-byte-length: "npm:^1.0.1"
+  checksum: 10c0/af2b431fc4314f119b551e5fccfad49d4c0ef82e13ba9ca61be6567801195b08e732ce9643542e8ad1b3df44f3df2d7345b3dd34f723954b6bb43a14584d6b3c
+  languageName: node
+  linkType: hard
+
 "tsconfig-paths@npm:^3.15.0":
   version: 3.15.0
   resolution: "tsconfig-paths@npm:3.15.0"
@@ -14605,6 +14663,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"utf8-byte-length@npm:^1.0.1":
+  version: 1.0.5
+  resolution: "utf8-byte-length@npm:1.0.5"
+  checksum: 10c0/e69bda3299608f4cc75976da9fb74ac94801a58b9ca29fdad03a20ec952e7477d7f226c12716b5f36bd4cff8151d1d152d02ee1df3752f017d4b2c725ce3e47a
+  languageName: node
+  linkType: hard
+
 "util-deprecate@npm:^1.0.1, util-deprecate@npm:~1.0.1":
   version: 1.0.2
   resolution: "util-deprecate@npm:1.0.2"