diff --git a/.gitignore b/.gitignore index 0c7d22a87..3f577c55a 100644 --- a/.gitignore +++ b/.gitignore @@ -84,6 +84,7 @@ local_keycloak.json cypress.env.json cypress/videos/ cypress/screenshots/ +cypress/snapshots/ cypress/downloads/ package-lock.json ## Documentation Markdown files served locally for testing during development diff --git a/cypress.config.js b/cypress.config.js index 54210488f..671c028d8 100644 --- a/cypress.config.js +++ b/cypress.config.js @@ -1,5 +1,6 @@ const { defineConfig } = require('cypress') const { plugin: cypressGrepPlugin } = require('@cypress/grep/plugin') +const { configureVisualRegression } = require('cypress-visual-regression') const fs = require('fs') module.exports = defineConfig({ @@ -16,6 +17,9 @@ module.exports = defineConfig({ AUTH_REALM: 'hmda2', AUTH_CLIENT_ID: 'hmda2-api', preserveCookies: ['_login_gov_session'], + visualRegressionBaseDirectory: 'cypress/snapshots/base', + visualRegressionDiffDirectory: 'cypress/snapshots/diff', + visualRegressionGenerateDiff: 'fail', // Always enable spec filtering grepFilterSpecs: true, // Always omit filtered tests @@ -35,10 +39,8 @@ module.exports = defineConfig({ e2e: { experimentalRunAllSpecs: true, testIsolation: true, - setupNodeEvents(on, config) { - return require('./cypress/plugins/index.js')(on, config) - }, specPattern: 'cypress/e2e/**/*.{js,jsx,ts,tsx}', + screenshotsFolder: 'cypress/snapshots/actual', experimentalOriginDependencies: true, chromeWebSecurity: false, defaultCommandTimeout: 10000, @@ -50,6 +52,9 @@ module.exports = defineConfig({ // Delete videos for specs without failing or retried tests, see docs: // https://docs.cypress.io/app/guides/screenshots-and-videos#Delete-videos-for-specs-without-failing-or-retried-tests setupNodeEvents(on, config) { + require('./cypress/plugins/index.js')(on, config) + configureVisualRegression(on) + on('after:spec', (spec, results) => { if (results && results.video) { // Do we have failures for any retry attempts? diff --git a/cypress/README.md b/cypress/README.md index 0a4054481..b95e9d48e 100644 --- a/cypress/README.md +++ b/cypress/README.md @@ -67,6 +67,8 @@ We use [`@cypress/grep`](https://www.npmjs.com/package/@cypress/grep) to organiz - `yarn test-smoke`: Executes much quicker than the full suite of tests by running a subset of end-to-end tests that represent core functionality. - `yarn test-unauthenticated`: Runs all tests that don't require Keycloak authentication. - `yarn test-large`: Runs our large filer test which uploads a very large file and takes awhile. +- `yarn test-visual:baseline`: Generates baseline snapshots for [visual regression testing](https://github.com/cypress-visual-regression/cypress-visual-regression). Can be run against any environment but to test local changes you'll want to do `CYPRESS_HOST=http://localhost:3000 yarn test-visual:baseline --env grepTags='@localhost'`. +- `yarn test-visual`: Takes snapshots and compares them to the baseline images. Can be run against any environment but to test local changes you'll want to do `CYPRESS_HOST=http://localhost:3000 yarn test-visual --env grepTags='@localhost'`. When you add new tests, make sure to tag them appropriately so that they run with the above commands. You can view the commands and their tags in [`package.json`](https://github.com/cfpb/hmda-frontend/blob/master/package.json). For example, localhost tests have a `@localhost` tag. Tests that require Keycloak authentication have a `@auth-required` tag and the above `yarn test-unauthenticated` command runs all tests *without* this tag. diff --git a/cypress/support/e2e.js b/cypress/support/e2e.js index eb15cb8b8..41dfaa587 100644 --- a/cypress/support/e2e.js +++ b/cypress/support/e2e.js @@ -2,9 +2,39 @@ import { register as registerCypressGrep } from '@cypress/grep' import '@testing-library/cypress/add-commands' import 'cypress-file-upload' import 'cypress-keycloak' +import { addCompareSnapshotCommand } from 'cypress-visual-regression/dist/command' import { logEnv, urlExists } from './helpers' registerCypressGrep() +addCompareSnapshotCommand({ errorThreshold: 0.1 }) + +// Create a name for the screenshot based on the test title +const getScreenshotName = title => title.replace(/[^a-zA-Z0-9-_]/g, '-').replace(/-{2,}/g, '-').slice(0, 120) + +// When CYPRESS_visualRegressionType env var is set (to either `base` or `regression`), +// a screenshot will be taken after each test. +// See https://github.com/cypress-visual-regression/cypress-visual-regression#plugin-options +afterEach(function () { + if (!Cypress.env('visualRegressionType')) return + if (this.currentTest?.state !== 'passed') return + + const name = getScreenshotName(this.currentTest.fullTitle()) + + cy.get('body', { log: false }).then((body) => { + // Make sure the page is done loading + const isLoading = body.find('.LoadingIconWrapper').length > 0 + if (isLoading) return + + // Every page (I think?) has a #mainWrapper div that contains the content + const hasMainWrapper = body.find('#mainWrapper').length > 0 + const screenshotWrapper = hasMainWrapper ? '#mainWrapper' : 'body' + const screenshotElement = body.find(screenshotWrapper).first() + + if (!screenshotElement.length || !screenshotElement.is(':visible')) return + + cy.get(screenshotWrapper, { log: false }).compareSnapshot(name) + }) +}) // *********************************************************** // This example support/index.js is processed and diff --git a/cypress/support/helpers.js b/cypress/support/helpers.js index 069656659..d10ec3b98 100644 --- a/cypress/support/helpers.js +++ b/cypress/support/helpers.js @@ -24,17 +24,8 @@ export function withFormData(method, url, formData, done) { } export function urlExists(url) { - return new Promise((resolve) => { - const xhr = new XMLHttpRequest() - - xhr.onreadystatechange = () => { - if (xhr.readyState === 4) - resolve({ url, status: xhr.status < 400, statusCode: xhr.status }) - } - - xhr.open('HEAD', url) - xhr.send() - }) + return cy.request({ url, method: 'HEAD', failOnStatusCode: false, timeout: 30000 }) + .then((response) => ({ url, status: response.status < 400, statusCode: response.status })) } /* Data Browser Helpers */ diff --git a/package.json b/package.json index 17a4f5c20..53eb1fd02 100644 --- a/package.json +++ b/package.json @@ -11,6 +11,8 @@ "build": "vite build", "preview": "vite preview", "test": "yarn run cypress run", + "test-visual:baseline": "CYPRESS_visualRegressionType=base yarn run cypress run", + "test-visual": "CYPRESS_visualRegressionType=regression yarn run cypress run", "test-localhost": "CYPRESS_HOST=http://localhost:3000 yarn run cypress run --env grepTags='@localhost'", "test-unauthenticated": "yarn run cypress run --env grepTags='-@auth-required'", "test-smoke": "CYPRESS_YEARS=2024 yarn run cypress run --env grepTags='@smoke'", @@ -96,6 +98,7 @@ "cypress": "^14.0", "cypress-file-upload": "5.0.8", "cypress-keycloak": "2.0.2", + "cypress-visual-regression": "^5.3.0", "dotenv": "^16.3.1", "ejs": "3.1.10", "enzyme": "^3.11.0", diff --git a/yarn.lock b/yarn.lock index 0ed12b34e..8ad843831 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4941,7 +4941,7 @@ __metadata: languageName: node linkType: hard -"chalk@npm:^4.0.0, chalk@npm:^4.1.0, chalk@npm:^4.1.2": +"chalk@npm:4.1.2, chalk@npm:^4.0.0, chalk@npm:^4.1.0, chalk@npm:^4.1.2": version: 4.1.2 resolution: "chalk@npm:4.1.2" dependencies: @@ -5621,6 +5621,20 @@ __metadata: languageName: node linkType: hard +"cypress-visual-regression@npm:^5.3.0": + version: 5.3.0 + resolution: "cypress-visual-regression@npm:5.3.0" + dependencies: + chalk: "npm:4.1.2" + pixelmatch: "npm:5.3.0" + pngjs: "npm:7.0.0" + sanitize-filename: "npm:1.6.3" + peerDependencies: + cypress: ">=12" + checksum: 10c0/a38ee912f85dd444fcd327884449dbf2079216d6aa2e8e953b7dd389ebc9a6d7c6ee03a3dccb20b93957976bd536299b314c74d3c98aab9b8fb59d416afeacd0 + languageName: node + linkType: hard + "cypress@npm:^14.0": version: 14.5.4 resolution: "cypress@npm:14.5.4" @@ -8107,6 +8121,7 @@ __metadata: cypress: "npm:^14.0" cypress-file-upload: "npm:5.0.8" cypress-keycloak: "npm:2.0.2" + cypress-visual-regression: "npm:^5.3.0" date-fns: "npm:^4.1.0" detect-browser: "npm:4.8.0" dotenv: "npm:^16.3.1" @@ -11541,6 +11556,17 @@ __metadata: languageName: node linkType: hard +"pixelmatch@npm:5.3.0": + version: 5.3.0 + resolution: "pixelmatch@npm:5.3.0" + dependencies: + pngjs: "npm:^6.0.0" + bin: + pixelmatch: bin/pixelmatch + checksum: 10c0/30850661db29b57cefbe6cf36e930b7517aea4e0ed129e85fcc8ec04a7e6e7648a822a972f8e01d2d3db268ca3c735555caf6b8099a164d8b64d105986d682d2 + languageName: node + linkType: hard + "pkg-dir@npm:^4.2.0": version: 4.2.0 resolution: "pkg-dir@npm:4.2.0" @@ -11578,6 +11604,20 @@ __metadata: languageName: node linkType: hard +"pngjs@npm:7.0.0": + version: 7.0.0 + resolution: "pngjs@npm:7.0.0" + checksum: 10c0/0d4c7a0fd476a9c33df7d0a2a73e1d56537628a668841f6995c2bca070cf30819f9254a64363266bc14ef2fee47659dd3b4f2b18eec7ab65143015139f497b38 + languageName: node + linkType: hard + +"pngjs@npm:^6.0.0": + version: 6.0.0 + resolution: "pngjs@npm:6.0.0" + checksum: 10c0/ac23ea329b1881d1a10575aff58116dc27b894ec3f5b84ba15c7f527d21e609fbce7ba16d48f8ccb86c7ce45ceed622472765476ab2875949d4bec55e153f87a + languageName: node + linkType: hard + "possible-typed-array-names@npm:^1.0.0": version: 1.1.0 resolution: "possible-typed-array-names@npm:1.1.0" @@ -12993,6 +13033,15 @@ __metadata: languageName: node linkType: hard +"sanitize-filename@npm:1.6.3": + version: 1.6.3 + resolution: "sanitize-filename@npm:1.6.3" + dependencies: + truncate-utf8-bytes: "npm:^1.0.0" + checksum: 10c0/16ff47556a6e54e228c28db096bedd303da67b030d4bea4925fd71324932d6b02c7b0446f00ad33987b25b6414f24ae968e01a1a1679ce599542e82c4b07eb1f + languageName: node + linkType: hard + "sass-embedded-android-arm64@npm:1.83.4": version: 1.83.4 resolution: "sass-embedded-android-arm64@npm:1.83.4" @@ -14269,6 +14318,15 @@ __metadata: languageName: node linkType: hard +"truncate-utf8-bytes@npm:^1.0.0": + version: 1.0.2 + resolution: "truncate-utf8-bytes@npm:1.0.2" + dependencies: + utf8-byte-length: "npm:^1.0.1" + checksum: 10c0/af2b431fc4314f119b551e5fccfad49d4c0ef82e13ba9ca61be6567801195b08e732ce9643542e8ad1b3df44f3df2d7345b3dd34f723954b6bb43a14584d6b3c + languageName: node + linkType: hard + "tsconfig-paths@npm:^3.15.0": version: 3.15.0 resolution: "tsconfig-paths@npm:3.15.0" @@ -14605,6 +14663,13 @@ __metadata: languageName: node linkType: hard +"utf8-byte-length@npm:^1.0.1": + version: 1.0.5 + resolution: "utf8-byte-length@npm:1.0.5" + checksum: 10c0/e69bda3299608f4cc75976da9fb74ac94801a58b9ca29fdad03a20ec952e7477d7f226c12716b5f36bd4cff8151d1d152d02ee1df3752f017d4b2c725ce3e47a + languageName: node + linkType: hard + "util-deprecate@npm:^1.0.1, util-deprecate@npm:~1.0.1": version: 1.0.2 resolution: "util-deprecate@npm:1.0.2"