From 5da7a6f9231e467ee1852fda6f2e08a9cca6a9a1 Mon Sep 17 00:00:00 2001 From: Tonya Mork Date: Tue, 2 Nov 2021 21:03:10 +0000 Subject: [PATCH] Build/Test Tools: Introduce local visual regression testing. Adds the ability to ''locally'' run visual regression testing for wp-admin pages via `npm run test:visual`. Snapshots are stored on contributors' local machines. Note: Wiring to the CI is not included. Why? The challenges for the CI are storage of the artifacts and unreliability of testing these across different environments. This commit is a first step towards visual regression testing. Running it locally provides a learning opportunity which could help to craft how to build it into the automated CI process. Props isabel_brison, andraganescu, azaozz, danfarrow, desrosj, hellofromTonya, justinahinon, netweb, talldanwp. Fixes #49606. git-svn-id: https://develop.svn.wordpress.org/trunk@51989 602fd350-edb4-49c9-b593-d223f7449a82 --- .gitignore | 3 + package-lock.json | 87 +++++++ package.json | 2 + tests/e2e/config/bootstrap.js | 1 + tests/visual-regression/README.md | 11 + tests/visual-regression/config/bootstrap.js | 10 + tests/visual-regression/jest.config.js | 8 + tests/visual-regression/run-tests.js | 13 + .../specs/visual-snapshots.test.js | 222 ++++++++++++++++++ 9 files changed, 357 insertions(+) create mode 100644 tests/visual-regression/README.md create mode 100644 tests/visual-regression/config/bootstrap.js create mode 100644 tests/visual-regression/jest.config.js create mode 100644 tests/visual-regression/run-tests.js create mode 100644 tests/visual-regression/specs/visual-snapshots.test.js diff --git a/.gitignore b/.gitignore index 792c0c8ccb..fbae7cad27 100644 --- a/.gitignore +++ b/.gitignore @@ -90,3 +90,6 @@ wp-tests-config.php # Files for local environment config /docker-compose.override.yml + +# Visual regression test diffs +tests/visual-regression/specs/__image_snapshots__ \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 26f2fca201..6ab7785d46 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12535,6 +12535,12 @@ "minimatch": "~3.0.2" } }, + "glur": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/glur/-/glur-1.1.2.tgz", + "integrity": "sha1-8g6jbbEDv8KSNDkh8fkeg8NGdok=", + "dev": true + }, "gonzales-pe": { "version": "4.3.0", "resolved": "https://registry.npmjs.org/gonzales-pe/-/gonzales-pe-4.3.0.tgz", @@ -15607,6 +15613,64 @@ } } }, + "jest-image-snapshot": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/jest-image-snapshot/-/jest-image-snapshot-3.0.1.tgz", + "integrity": "sha512-bW8eYxgAVyO8cNLlTt15wd5YiWvRfzQyNQ4K8FKHUEPasQADEZ5NzaWmnOpSdh3/NLYoH++TMp6o/rRVLpOIkQ==", + "dev": true, + "requires": { + "chalk": "^1.1.3", + "get-stdin": "^5.0.1", + "glur": "^1.1.2", + "lodash": "^4.17.4", + "mkdirp": "^0.5.1", + "pixelmatch": "^5.1.0", + "pngjs": "^3.4.0", + "rimraf": "^2.6.2" + }, + "dependencies": { + "ansi-regex": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", + "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=", + "dev": true + }, + "ansi-styles": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz", + "integrity": "sha1-tDLdM1i2NM914eRmQ2gkBTPB3b4=", + "dev": true + }, + "chalk": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz", + "integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=", + "dev": true, + "requires": { + "ansi-styles": "^2.2.1", + "escape-string-regexp": "^1.0.2", + "has-ansi": "^2.0.0", + "strip-ansi": "^3.0.0", + "supports-color": "^2.0.0" + } + }, + "strip-ansi": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", + "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", + "dev": true, + "requires": { + "ansi-regex": "^2.0.0" + } + }, + "supports-color": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz", + "integrity": "sha1-U10EXOa2Nj+kARcIRimZXp3zJMc=", + "dev": true + } + } + }, "jest-jasmine2": { "version": "26.6.3", "resolved": "https://registry.npmjs.org/jest-jasmine2/-/jest-jasmine2-26.6.3.tgz", @@ -19785,6 +19849,23 @@ "node-modules-regexp": "^1.0.0" } }, + "pixelmatch": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/pixelmatch/-/pixelmatch-5.2.1.tgz", + "integrity": "sha512-WjcAdYSnKrrdDdqTcVEY7aB7UhhwjYQKYhHiBXdJef0MOaQeYpUdQ+iVyBLa5YBKS8MPVPPMX7rpOByISLpeEQ==", + "dev": true, + "requires": { + "pngjs": "^4.0.1" + }, + "dependencies": { + "pngjs": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/pngjs/-/pngjs-4.0.1.tgz", + "integrity": "sha512-rf5+2/ioHeQxR6IxuYNYGFytUyG3lma/WW1nsmjeHlWwtb2aByla6dkVc8pmJ9nplzkTA0q2xx7mMWrOTqT4Gg==", + "dev": true + } + } + }, "pkg-dir": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-3.0.0.tgz", @@ -19857,6 +19938,12 @@ "irregular-plurals": "^3.2.0" } }, + "pngjs": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/pngjs/-/pngjs-3.4.0.tgz", + "integrity": "sha512-NCrCHhWmnQklfH4MtJMRjZ2a8c80qXeMlQMv2uVp9ISJMTt562SbGd6n2oq0PaPgKm7Z6pL9E2UlLIhC+SHL3w==", + "dev": true + }, "polyfill-library": { "version": "3.105.0", "resolved": "https://registry.npmjs.org/polyfill-library/-/polyfill-library-3.105.0.tgz", diff --git a/package.json b/package.json index 172cb074be..d3b6998d60 100644 --- a/package.json +++ b/package.json @@ -61,6 +61,7 @@ "grunt-webpack": "^4.0.3", "ink-docstrap": "1.3.2", "install-changed": "1.1.0", + "jest-image-snapshot": "3.0.1", "matchdep": "~2.0.0", "prettier": "npm:wp-prettier@2.0.5", "qunit": "~2.16.0", @@ -172,6 +173,7 @@ "env:pull": "node ./tools/local-env/scripts/docker.js pull", "test:php": "node ./tools/local-env/scripts/docker.js run -T php composer update -W && node ./tools/local-env/scripts/docker.js run php ./vendor/bin/phpunit", "test:e2e": "node ./tests/e2e/run-tests.js", + "test:visual": "node ./tests/visual-regression/run-tests.js", "wp-packages-update": "wp-scripts packages-update" } } diff --git a/tests/e2e/config/bootstrap.js b/tests/e2e/config/bootstrap.js index 3a767b21b0..18fb153465 100644 --- a/tests/e2e/config/bootstrap.js +++ b/tests/e2e/config/bootstrap.js @@ -33,6 +33,7 @@ const pageEvents = []; // The Jest timeout is increased because these tests are a bit slow jest.setTimeout( PUPPETEER_TIMEOUT || 100000 ); + /** * Adds an event listener to the page to handle additions of page event * handlers, to assure that they are removed at test teardown. diff --git a/tests/visual-regression/README.md b/tests/visual-regression/README.md new file mode 100644 index 0000000000..fe32fc0688 --- /dev/null +++ b/tests/visual-regression/README.md @@ -0,0 +1,11 @@ +# Visual Regression Tests in WordPress Core + +These tests make use of Jest and Puppeteer, with a setup very similar to that of the e2e tests, together with [jest-image-snapshot](https://github.com/americanexpress/jest-image-snapshot) for generating the visual diffs. + +## How to Run the Tests Locally + +1. Check out trunk. +2. Run `npm run test:visual` to generate some base snapshots. +3. Check out the feature branch to be tested. +4. Run `npm run test:visual` again. If any tests fail, the diff images can be found in `tests/visual-regression/specs/__image_snapshots__/__diff_output__`. + diff --git a/tests/visual-regression/config/bootstrap.js b/tests/visual-regression/config/bootstrap.js new file mode 100644 index 0000000000..6130909e0c --- /dev/null +++ b/tests/visual-regression/config/bootstrap.js @@ -0,0 +1,10 @@ +import { configureToMatchImageSnapshot } from 'jest-image-snapshot'; + +// All available options: https://github.com/americanexpress/jest-image-snapshot#%EF%B8%8F-api +const toMatchImageSnapshot = configureToMatchImageSnapshot( { + // Maximum diff to allow in px. + failureThreshold: 1, +} ); + +// Extend Jest's "expect" with image snapshot functionality. +expect.extend( { toMatchImageSnapshot } ); diff --git a/tests/visual-regression/jest.config.js b/tests/visual-regression/jest.config.js new file mode 100644 index 0000000000..aa5d3d1fbd --- /dev/null +++ b/tests/visual-regression/jest.config.js @@ -0,0 +1,8 @@ +const config = require( '@wordpress/scripts/config/jest-e2e.config' ); + +const jestVisualRegressionConfig = { + ...config, + setupFilesAfterEnv: [ '/config/bootstrap.js' ], +}; + +module.exports = jestVisualRegressionConfig; diff --git a/tests/visual-regression/run-tests.js b/tests/visual-regression/run-tests.js new file mode 100644 index 0000000000..c5f1169434 --- /dev/null +++ b/tests/visual-regression/run-tests.js @@ -0,0 +1,13 @@ +const dotenv = require( 'dotenv' ); +const dotenv_expand = require( 'dotenv-expand' ); +const { execSync } = require( 'child_process' ); + +// WP_BASE_URL interpolates LOCAL_PORT, so needs to be parsed by dotenv_expand(). +dotenv_expand( dotenv.config() ); + +// Run the tests, passing additional arguments through to the test script. +execSync( + 'wp-scripts test-e2e --config tests/visual-regression/jest.config.js ' + + process.argv.slice( 2 ).join( ' ' ), + { stdio: 'inherit' } +); diff --git a/tests/visual-regression/specs/visual-snapshots.test.js b/tests/visual-regression/specs/visual-snapshots.test.js new file mode 100644 index 0000000000..458c40f86e --- /dev/null +++ b/tests/visual-regression/specs/visual-snapshots.test.js @@ -0,0 +1,222 @@ +import { visitAdminPage } from '@wordpress/e2e-test-utils'; + +// See https://github.com/puppeteer/puppeteer/blob/main/docs/api.md#pagescreenshotoptions for more available options. +const screenshotOptions = { + fullPage: true, +}; + +async function hideElementVisibility( elements ) { + for ( let i = 0; i < elements.length; i++ ) { + const elementOnPage = await page.$( elements[ i ] ); + if ( elementOnPage ) { + await elementOnPage.evaluate( ( el ) => { + el.style.visibility = 'hidden'; + } ); + } + } + await page.waitFor( 1000 ); +} + +async function removeElementFromLayout( elements ) { + for ( let i = 0; i < elements.length; i++ ) { + const elementOnPage = await page.$( elements[ i ] ); + if ( elementOnPage ) { + await elementOnPage.evaluate( ( el ) => { + el.style.visibility = 'hidden'; + } ); + } + } + await page.waitFor( 1000 ); +} + +const elementsToHide = [ '#footer-upgrade', '#wp-admin-bar-root-default' ]; + +const elementsToRemove = [ '#toplevel_page_gutenberg' ]; + +describe( 'Admin Visual Snapshots', () => { + beforeAll( async () => { + await page.setViewport( { + width: 1000, + height: 750, + } ); + } ); + + it( 'All Posts', async () => { + await visitAdminPage( '/edit.php' ); + await hideElementVisibility( elementsToHide ); + await removeElementFromLayout( elementsToRemove ); + const image = await page.screenshot( screenshotOptions ); + expect( image ).toMatchImageSnapshot(); + } ); + + it( 'Categories', async () => { + await visitAdminPage( '/edit-tags.php', 'taxonomy=category' ); + await hideElementVisibility( elementsToHide ); + await removeElementFromLayout( elementsToRemove ); + const image = await page.screenshot( screenshotOptions ); + expect( image ).toMatchImageSnapshot(); + } ); + + it( 'Tags', async () => { + await visitAdminPage( '/edit-tags.php', 'taxonomy=post_tag' ); + await hideElementVisibility( elementsToHide ); + await removeElementFromLayout( elementsToRemove ); + const image = await page.screenshot( screenshotOptions ); + expect( image ).toMatchImageSnapshot(); + } ); + + it( 'Media Library', async () => { + await visitAdminPage( '/upload.php' ); + await hideElementVisibility( elementsToHide ); + await removeElementFromLayout( elementsToRemove ); + const image = await page.screenshot( screenshotOptions ); + expect( image ).toMatchImageSnapshot(); + } ); + + it( 'Add New Media', async () => { + await visitAdminPage( '/media-new.php' ); + await hideElementVisibility( elementsToHide ); + await removeElementFromLayout( elementsToRemove ); + const image = await page.screenshot( screenshotOptions ); + expect( image ).toMatchImageSnapshot(); + } ); + + it( 'All Pages', async () => { + await visitAdminPage( '/edit.php', 'post_type=page' ); + await hideElementVisibility( elementsToHide ); + await removeElementFromLayout( elementsToRemove ); + const image = await page.screenshot( screenshotOptions ); + expect( image ).toMatchImageSnapshot(); + } ); + + it( 'Comments', async () => { + await visitAdminPage( '/edit-comments.php' ); + await hideElementVisibility( elementsToHide ); + await removeElementFromLayout( elementsToRemove ); + const image = await page.screenshot( screenshotOptions ); + expect( image ).toMatchImageSnapshot(); + } ); + + it( 'Widgets', async () => { + await visitAdminPage( '/widgets.php' ); + await hideElementVisibility( elementsToHide ); + await removeElementFromLayout( elementsToRemove ); + const image = await page.screenshot( screenshotOptions ); + expect( image ).toMatchImageSnapshot(); + } ); + + it( 'Menus', async () => { + await visitAdminPage( '/nav-menus.php' ); + await hideElementVisibility( elementsToHide ); + await removeElementFromLayout( elementsToRemove ); + const image = await page.screenshot( screenshotOptions ); + expect( image ).toMatchImageSnapshot(); + } ); + + it( 'Plugins', async () => { + await visitAdminPage( '/plugins.php' ); + await hideElementVisibility( elementsToHide ); + await removeElementFromLayout( elementsToRemove ); + const image = await page.screenshot( screenshotOptions ); + expect( image ).toMatchImageSnapshot(); + } ); + + it( 'All Users', async () => { + await visitAdminPage( '/users.php' ); + await hideElementVisibility( elementsToHide ); + await removeElementFromLayout( elementsToRemove ); + const image = await page.screenshot( screenshotOptions ); + expect( image ).toMatchImageSnapshot(); + } ); + + it( 'Add New User', async () => { + await visitAdminPage( '/user-new.php' ); + await hideElementVisibility( [ + ...elementsToHide, + '.password-input-wrapper', + ] ); + await removeElementFromLayout( elementsToRemove ); + const image = await page.screenshot( screenshotOptions ); + expect( image ).toMatchImageSnapshot(); + } ); + + it( 'Your Profile', async () => { + await visitAdminPage( '/profile.php' ); + await hideElementVisibility( elementsToHide ); + await removeElementFromLayout( elementsToRemove ); + const image = await page.screenshot( screenshotOptions ); + expect( image ).toMatchImageSnapshot(); + } ); + + it( 'Available Tools', async () => { + await visitAdminPage( '/tools.php' ); + await hideElementVisibility( elementsToHide ); + await removeElementFromLayout( elementsToRemove ); + const image = await page.screenshot( screenshotOptions ); + expect( image ).toMatchImageSnapshot(); + } ); + + it( 'Import', async () => { + await visitAdminPage( '/import.php' ); + await hideElementVisibility( elementsToHide ); + await removeElementFromLayout( elementsToRemove ); + const image = await page.screenshot( screenshotOptions ); + expect( image ).toMatchImageSnapshot(); + } ); + + it( 'Export', async () => { + await visitAdminPage( '/export.php' ); + await hideElementVisibility( elementsToHide ); + await removeElementFromLayout( elementsToRemove ); + const image = await page.screenshot( screenshotOptions ); + expect( image ).toMatchImageSnapshot(); + } ); + + it( 'Export Personal Data', async () => { + await visitAdminPage( '/export-personal-data.php' ); + await hideElementVisibility( elementsToHide ); + await removeElementFromLayout( elementsToRemove ); + const image = await page.screenshot( screenshotOptions ); + expect( image ).toMatchImageSnapshot(); + } ); + + it( 'Erase Personal Data', async () => { + await visitAdminPage( '/erase-personal-data.php' ); + await hideElementVisibility( elementsToHide ); + await removeElementFromLayout( elementsToRemove ); + const image = await page.screenshot( screenshotOptions ); + expect( image ).toMatchImageSnapshot(); + } ); + + it( 'Reading Settings', async () => { + await visitAdminPage( '/options-reading.php' ); + await hideElementVisibility( elementsToHide ); + await removeElementFromLayout( elementsToRemove ); + const image = await page.screenshot( screenshotOptions ); + expect( image ).toMatchImageSnapshot(); + } ); + + it( 'Discussion Settings', async () => { + await visitAdminPage( '/options-discussion.php' ); + await hideElementVisibility( elementsToHide ); + await removeElementFromLayout( elementsToRemove ); + const image = await page.screenshot( screenshotOptions ); + expect( image ).toMatchImageSnapshot(); + } ); + + it( 'Media Settings', async () => { + await visitAdminPage( '/options-media.php' ); + await hideElementVisibility( elementsToHide ); + await removeElementFromLayout( elementsToRemove ); + const image = await page.screenshot( screenshotOptions ); + expect( image ).toMatchImageSnapshot(); + } ); + + it( 'Privacy Settings', async () => { + await visitAdminPage( '/options-privacy.php' ); + await hideElementVisibility( elementsToHide ); + await removeElementFromLayout( elementsToRemove ); + const image = await page.screenshot( screenshotOptions ); + expect( image ).toMatchImageSnapshot(); + } ); +} );