Build/Test Tools: Introduce local visual regression testing.

Adds the ability to ''locally'' run visual regression testing for wp-admin pages via `npm run test:visual`. Snapshots are stored on contributors' local machines.

Note:
Wiring to the CI is not included. Why? The challenges for the CI are storage of the artifacts and unreliability of testing these across different environments.

This commit is a first step towards visual regression testing. Running it locally provides a learning opportunity which could help to craft how to build it into the automated CI process.

Props isabel_brison, andraganescu, azaozz, danfarrow, desrosj, hellofromTonya, justinahinon, netweb, talldanwp.
Fixes #49606.

git-svn-id: https://develop.svn.wordpress.org/trunk@51989 602fd350-edb4-49c9-b593-d223f7449a82

.gitignore

@@ -90,3 +90,6 @@ wp-tests-config.php
 
 # Files for local environment config
 /docker-compose.override.yml
+
+# Visual regression test diffs
+tests/visual-regression/specs/__image_snapshots__

package-lock.json

@@ -12535,6 +12535,12 @@
 				"minimatch": "~3.0.2"
 			}
 		},
+		"glur": {
+			"version": "1.1.2",
+			"resolved": "https://registry.npmjs.org/glur/-/glur-1.1.2.tgz",
+			"integrity": "sha1-8g6jbbEDv8KSNDkh8fkeg8NGdok=",
+			"dev": true
+		},
 		"gonzales-pe": {
 			"version": "4.3.0",
 			"resolved": "https://registry.npmjs.org/gonzales-pe/-/gonzales-pe-4.3.0.tgz",
@@ -15607,6 +15613,64 @@
 				}
 			}
 		},
+		"jest-image-snapshot": {
+			"version": "3.0.1",
+			"resolved": "https://registry.npmjs.org/jest-image-snapshot/-/jest-image-snapshot-3.0.1.tgz",
+			"integrity": "sha512-bW8eYxgAVyO8cNLlTt15wd5YiWvRfzQyNQ4K8FKHUEPasQADEZ5NzaWmnOpSdh3/NLYoH++TMp6o/rRVLpOIkQ==",
+			"dev": true,
+			"requires": {
+				"chalk": "^1.1.3",
+				"get-stdin": "^5.0.1",
+				"glur": "^1.1.2",
+				"lodash": "^4.17.4",
+				"mkdirp": "^0.5.1",
+				"pixelmatch": "^5.1.0",
+				"pngjs": "^3.4.0",
+				"rimraf": "^2.6.2"
+			},
+			"dependencies": {
+				"ansi-regex": {
+					"version": "2.1.1",
+					"resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz",
+					"integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=",
+					"dev": true
+				},
+				"ansi-styles": {
+					"version": "2.2.1",
+					"resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-2.2.1.tgz",
+					"integrity": "sha1-tDLdM1i2NM914eRmQ2gkBTPB3b4=",
+					"dev": true
+				},
+				"chalk": {
+					"version": "1.1.3",
+					"resolved": "https://registry.npmjs.org/chalk/-/chalk-1.1.3.tgz",
+					"integrity": "sha1-qBFcVeSnAv5NFQq9OHKCKn4J/Jg=",
+					"dev": true,
+					"requires": {
+						"ansi-styles": "^2.2.1",
+						"escape-string-regexp": "^1.0.2",
+						"has-ansi": "^2.0.0",
+						"strip-ansi": "^3.0.0",
+						"supports-color": "^2.0.0"
+					}
+				},
+				"strip-ansi": {
+					"version": "3.0.1",
+					"resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz",
+					"integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=",
+					"dev": true,
+					"requires": {
+						"ansi-regex": "^2.0.0"
+					}
+				},
+				"supports-color": {
+					"version": "2.0.0",
+					"resolved": "https://registry.npmjs.org/supports-color/-/supports-color-2.0.0.tgz",
+					"integrity": "sha1-U10EXOa2Nj+kARcIRimZXp3zJMc=",
+					"dev": true
+				}
+			}
+		},
 		"jest-jasmine2": {
 			"version": "26.6.3",
 			"resolved": "https://registry.npmjs.org/jest-jasmine2/-/jest-jasmine2-26.6.3.tgz",
@@ -19785,6 +19849,23 @@
 				"node-modules-regexp": "^1.0.0"
 			}
 		},
+		"pixelmatch": {
+			"version": "5.2.1",
+			"resolved": "https://registry.npmjs.org/pixelmatch/-/pixelmatch-5.2.1.tgz",
+			"integrity": "sha512-WjcAdYSnKrrdDdqTcVEY7aB7UhhwjYQKYhHiBXdJef0MOaQeYpUdQ+iVyBLa5YBKS8MPVPPMX7rpOByISLpeEQ==",
+			"dev": true,
+			"requires": {
+				"pngjs": "^4.0.1"
+			},
+			"dependencies": {
+				"pngjs": {
+					"version": "4.0.1",
+					"resolved": "https://registry.npmjs.org/pngjs/-/pngjs-4.0.1.tgz",
+					"integrity": "sha512-rf5+2/ioHeQxR6IxuYNYGFytUyG3lma/WW1nsmjeHlWwtb2aByla6dkVc8pmJ9nplzkTA0q2xx7mMWrOTqT4Gg==",
+					"dev": true
+				}
+			}
+		},
 		"pkg-dir": {
 			"version": "3.0.0",
 			"resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-3.0.0.tgz",
@@ -19857,6 +19938,12 @@
 				"irregular-plurals": "^3.2.0"
 			}
 		},
+		"pngjs": {
+			"version": "3.4.0",
+			"resolved": "https://registry.npmjs.org/pngjs/-/pngjs-3.4.0.tgz",
+			"integrity": "sha512-NCrCHhWmnQklfH4MtJMRjZ2a8c80qXeMlQMv2uVp9ISJMTt562SbGd6n2oq0PaPgKm7Z6pL9E2UlLIhC+SHL3w==",
+			"dev": true
+		},
 		"polyfill-library": {
 			"version": "3.105.0",
 			"resolved": "https://registry.npmjs.org/polyfill-library/-/polyfill-library-3.105.0.tgz",

package.json

@@ -61,6 +61,7 @@
 		"grunt-webpack": "^4.0.3",
 		"ink-docstrap": "1.3.2",
 		"install-changed": "1.1.0",
+		"jest-image-snapshot": "3.0.1",
 		"matchdep": "~2.0.0",
 		"prettier": "npm:wp-prettier@2.0.5",
 		"qunit": "~2.16.0",
@@ -172,6 +173,7 @@
 		"env:pull": "node ./tools/local-env/scripts/docker.js pull",
 		"test:php": "node ./tools/local-env/scripts/docker.js run -T php composer update -W && node ./tools/local-env/scripts/docker.js run php ./vendor/bin/phpunit",
 		"test:e2e": "node ./tests/e2e/run-tests.js",
+		"test:visual": "node ./tests/visual-regression/run-tests.js",
 		"wp-packages-update": "wp-scripts packages-update"
 	}
 }

tests/e2e/config/bootstrap.js

@@ -33,6 +33,7 @@ const pageEvents = [];
 // The Jest timeout is increased because these tests are a bit slow
 jest.setTimeout( PUPPETEER_TIMEOUT || 100000 );
 
+
 /**
  * Adds an event listener to the page to handle additions of page event
  * handlers, to assure that they are removed at test teardown.

tests/visual-regression/README.md

@@ -0,0 +1,11 @@
+# Visual Regression Tests in WordPress Core
+
+These tests make use of Jest and Puppeteer, with a setup very similar to that of the e2e tests, together with [jest-image-snapshot](https://github.com/americanexpress/jest-image-snapshot) for generating the visual diffs.
+
+## How to Run the Tests Locally
+
+1. Check out trunk.
+2. Run `npm run test:visual` to generate some base snapshots.
+3. Check out the feature branch to be tested.
+4. Run `npm run test:visual` again. If any tests fail, the diff images can be found in `tests/visual-regression/specs/__image_snapshots__/__diff_output__`.
+

tests/visual-regression/config/bootstrap.js

@@ -0,0 +1,10 @@
+import { configureToMatchImageSnapshot } from 'jest-image-snapshot';
+
+// All available options: https://github.com/americanexpress/jest-image-snapshot#%EF%B8%8F-api
+const toMatchImageSnapshot = configureToMatchImageSnapshot( {
+	// Maximum diff to allow in px.
+	failureThreshold: 1,
+} );
+
+// Extend Jest's "expect" with image snapshot functionality.
+expect.extend( { toMatchImageSnapshot } );

tests/visual-regression/jest.config.js

@@ -0,0 +1,8 @@
+const config = require( '@wordpress/scripts/config/jest-e2e.config' );
+
+const jestVisualRegressionConfig = {
+	...config,
+	setupFilesAfterEnv: [ '<rootDir>/config/bootstrap.js' ],
+};
+
+module.exports = jestVisualRegressionConfig;

tests/visual-regression/run-tests.js

@@ -0,0 +1,13 @@
+const dotenv = require( 'dotenv' );
+const dotenv_expand = require( 'dotenv-expand' );
+const { execSync } = require( 'child_process' );
+
+// WP_BASE_URL interpolates LOCAL_PORT, so needs to be parsed by dotenv_expand().
+dotenv_expand( dotenv.config() );
+
+// Run the tests, passing additional arguments through to the test script.
+execSync(
+	'wp-scripts test-e2e --config tests/visual-regression/jest.config.js ' +
+		process.argv.slice( 2 ).join( ' ' ),
+	{ stdio: 'inherit' }
+);

tests/visual-regression/specs/visual-snapshots.test.js

@@ -0,0 +1,222 @@
+import { visitAdminPage } from '@wordpress/e2e-test-utils';
+
+// See https://github.com/puppeteer/puppeteer/blob/main/docs/api.md#pagescreenshotoptions for more available options.
+const screenshotOptions = {
+	fullPage: true,
+};
+
+async function hideElementVisibility( elements ) {
+	for ( let i = 0; i < elements.length; i++ ) {
+		const elementOnPage = await page.$( elements[ i ] );
+		if ( elementOnPage ) {
+			await elementOnPage.evaluate( ( el ) => {
+				el.style.visibility = 'hidden';
+			} );
+		}
+	}
+	await page.waitFor( 1000 );
+}
+
+async function removeElementFromLayout( elements ) {
+	for ( let i = 0; i < elements.length; i++ ) {
+		const elementOnPage = await page.$( elements[ i ] );
+		if ( elementOnPage ) {
+			await elementOnPage.evaluate( ( el ) => {
+				el.style.visibility = 'hidden';
+			} );
+		}
+	}
+	await page.waitFor( 1000 );
+}
+
+const elementsToHide = [ '#footer-upgrade', '#wp-admin-bar-root-default' ];
+
+const elementsToRemove = [ '#toplevel_page_gutenberg' ];
+
+describe( 'Admin Visual Snapshots', () => {
+	beforeAll( async () => {
+		await page.setViewport( {
+			width: 1000,
+			height: 750,
+		} );
+	} );
+
+	it( 'All Posts', async () => {
+		await visitAdminPage( '/edit.php' );
+		await hideElementVisibility( elementsToHide );
+		await removeElementFromLayout( elementsToRemove );
+		const image = await page.screenshot( screenshotOptions );
+		expect( image ).toMatchImageSnapshot();
+	} );
+
+	it( 'Categories', async () => {
+		await visitAdminPage( '/edit-tags.php', 'taxonomy=category' );
+		await hideElementVisibility( elementsToHide );
+		await removeElementFromLayout( elementsToRemove );
+		const image = await page.screenshot( screenshotOptions );
+		expect( image ).toMatchImageSnapshot();
+	} );
+
+	it( 'Tags', async () => {
+		await visitAdminPage( '/edit-tags.php', 'taxonomy=post_tag' );
+		await hideElementVisibility( elementsToHide );
+		await removeElementFromLayout( elementsToRemove );
+		const image = await page.screenshot( screenshotOptions );
+		expect( image ).toMatchImageSnapshot();
+	} );
+
+	it( 'Media Library', async () => {
+		await visitAdminPage( '/upload.php' );
+		await hideElementVisibility( elementsToHide );
+		await removeElementFromLayout( elementsToRemove );
+		const image = await page.screenshot( screenshotOptions );
+		expect( image ).toMatchImageSnapshot();
+	} );
+
+	it( 'Add New Media', async () => {
+		await visitAdminPage( '/media-new.php' );
+		await hideElementVisibility( elementsToHide );
+		await removeElementFromLayout( elementsToRemove );
+		const image = await page.screenshot( screenshotOptions );
+		expect( image ).toMatchImageSnapshot();
+	} );
+
+	it( 'All Pages', async () => {
+		await visitAdminPage( '/edit.php', 'post_type=page' );
+		await hideElementVisibility( elementsToHide );
+		await removeElementFromLayout( elementsToRemove );
+		const image = await page.screenshot( screenshotOptions );
+		expect( image ).toMatchImageSnapshot();
+	} );
+
+	it( 'Comments', async () => {
+		await visitAdminPage( '/edit-comments.php' );
+		await hideElementVisibility( elementsToHide );
+		await removeElementFromLayout( elementsToRemove );
+		const image = await page.screenshot( screenshotOptions );
+		expect( image ).toMatchImageSnapshot();
+	} );
+
+	it( 'Widgets', async () => {
+		await visitAdminPage( '/widgets.php' );
+		await hideElementVisibility( elementsToHide );
+		await removeElementFromLayout( elementsToRemove );
+		const image = await page.screenshot( screenshotOptions );
+		expect( image ).toMatchImageSnapshot();
+	} );
+
+	it( 'Menus', async () => {
+		await visitAdminPage( '/nav-menus.php' );
+		await hideElementVisibility( elementsToHide );
+		await removeElementFromLayout( elementsToRemove );
+		const image = await page.screenshot( screenshotOptions );
+		expect( image ).toMatchImageSnapshot();
+	} );
+
+	it( 'Plugins', async () => {
+		await visitAdminPage( '/plugins.php' );
+		await hideElementVisibility( elementsToHide );
+		await removeElementFromLayout( elementsToRemove );
+		const image = await page.screenshot( screenshotOptions );
+		expect( image ).toMatchImageSnapshot();
+	} );
+
+	it( 'All Users', async () => {
+		await visitAdminPage( '/users.php' );
+		await hideElementVisibility( elementsToHide );
+		await removeElementFromLayout( elementsToRemove );
+		const image = await page.screenshot( screenshotOptions );
+		expect( image ).toMatchImageSnapshot();
+	} );
+
+	it( 'Add New User', async () => {
+		await visitAdminPage( '/user-new.php' );
+		await hideElementVisibility( [
+			...elementsToHide,
+			'.password-input-wrapper',
+		] );
+		await removeElementFromLayout( elementsToRemove );
+		const image = await page.screenshot( screenshotOptions );
+		expect( image ).toMatchImageSnapshot();
+	} );
+
+	it( 'Your Profile', async () => {
+		await visitAdminPage( '/profile.php' );
+		await hideElementVisibility( elementsToHide );
+		await removeElementFromLayout( elementsToRemove );
+		const image = await page.screenshot( screenshotOptions );
+		expect( image ).toMatchImageSnapshot();
+	} );
+
+	it( 'Available Tools', async () => {
+		await visitAdminPage( '/tools.php' );
+		await hideElementVisibility( elementsToHide );
+		await removeElementFromLayout( elementsToRemove );
+		const image = await page.screenshot( screenshotOptions );
+		expect( image ).toMatchImageSnapshot();
+	} );
+
+	it( 'Import', async () => {
+		await visitAdminPage( '/import.php' );
+		await hideElementVisibility( elementsToHide );
+		await removeElementFromLayout( elementsToRemove );
+		const image = await page.screenshot( screenshotOptions );
+		expect( image ).toMatchImageSnapshot();
+	} );
+
+	it( 'Export', async () => {
+		await visitAdminPage( '/export.php' );
+		await hideElementVisibility( elementsToHide );
+		await removeElementFromLayout( elementsToRemove );
+		const image = await page.screenshot( screenshotOptions );
+		expect( image ).toMatchImageSnapshot();
+	} );
+
+	it( 'Export Personal Data', async () => {
+		await visitAdminPage( '/export-personal-data.php' );
+		await hideElementVisibility( elementsToHide );
+		await removeElementFromLayout( elementsToRemove );
+		const image = await page.screenshot( screenshotOptions );
+		expect( image ).toMatchImageSnapshot();
+	} );
+
+	it( 'Erase Personal Data', async () => {
+		await visitAdminPage( '/erase-personal-data.php' );
+		await hideElementVisibility( elementsToHide );
+		await removeElementFromLayout( elementsToRemove );
+		const image = await page.screenshot( screenshotOptions );
+		expect( image ).toMatchImageSnapshot();
+	} );
+
+	it( 'Reading Settings', async () => {
+		await visitAdminPage( '/options-reading.php' );
+		await hideElementVisibility( elementsToHide );
+		await removeElementFromLayout( elementsToRemove );
+		const image = await page.screenshot( screenshotOptions );
+		expect( image ).toMatchImageSnapshot();
+	} );
+
+	it( 'Discussion Settings', async () => {
+		await visitAdminPage( '/options-discussion.php' );
+		await hideElementVisibility( elementsToHide );
+		await removeElementFromLayout( elementsToRemove );
+		const image = await page.screenshot( screenshotOptions );
+		expect( image ).toMatchImageSnapshot();
+	} );
+
+	it( 'Media Settings', async () => {
+		await visitAdminPage( '/options-media.php' );
+		await hideElementVisibility( elementsToHide );
+		await removeElementFromLayout( elementsToRemove );
+		const image = await page.screenshot( screenshotOptions );
+		expect( image ).toMatchImageSnapshot();
+	} );
+
+	it( 'Privacy Settings', async () => {
+		await visitAdminPage( '/options-privacy.php' );
+		await hideElementVisibility( elementsToHide );
+		await removeElementFromLayout( elementsToRemove );
+		const image = await page.screenshot( screenshotOptions );
+		expect( image ).toMatchImageSnapshot();
+	} );
+} );