From 74e06045081e8f1db32b3603733610751a16e8f8 Mon Sep 17 00:00:00 2001 From: Pascal Birchler Date: Wed, 8 Nov 2023 10:30:21 +0000 Subject: [PATCH] Build/Test Tools: Expand performance test scenarios. Adds new tests for localized sites as well as the dashboard. Also amends Server-Timing output to measure memory usage in all scenarios. Props swissspidy, joemcgill, flixos90, mukesh27, mamaduka. See #59656. Fixes #59815. git-svn-id: https://develop.svn.wordpress.org/trunk@57083 602fd350-edb4-49c9-b593-d223f7449a82 --- .github/workflows/performance.yml | 6 ++ tests/performance/compare-results.js | 36 +++++++++-- tests/performance/log-results.js | 4 ++ tests/performance/playwright.config.js | 4 +- tests/performance/results.js | 4 ++ tests/performance/specs/admin-l10n.test.js | 52 +++++++++++++++ tests/performance/specs/admin.test.js | 46 ++++++++++++++ .../specs/home-block-theme-l10n.test.js | 63 +++++++++++++++++++ .../specs/home-block-theme.test.js | 2 +- .../specs/home-classic-theme-l10n.test.js | 62 ++++++++++++++++++ .../specs/home-classic-theme.test.js | 2 +- .../wp-content/mu-plugins/server-timing.php | 7 +++ 12 files changed, 277 insertions(+), 11 deletions(-) create mode 100644 tests/performance/specs/admin-l10n.test.js create mode 100644 tests/performance/specs/admin.test.js create mode 100644 tests/performance/specs/home-block-theme-l10n.test.js create mode 100644 tests/performance/specs/home-classic-theme-l10n.test.js diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml index 6c2b5e1104..6c99e18b4f 100644 --- a/.github/workflows/performance.yml +++ b/.github/workflows/performance.yml @@ -162,6 +162,12 @@ jobs: run: | npm run env:cli -- rewrite structure '/%year%/%monthnum%/%postname%/' --path=/var/www/${{ env.LOCAL_DIR }} + - name: Install additional languages + run: | + npm run env:cli -- language core install de_DE --path=/var/www/${{ env.LOCAL_DIR }} + npm run env:cli -- language plugin install de_DE --all --path=/var/www/${{ env.LOCAL_DIR }} + npm run env:cli -- language theme install de_DE --all --path=/var/www/${{ env.LOCAL_DIR }} + - name: Install MU plugin run: | mkdir ./${{ env.LOCAL_DIR }}/wp-content/mu-plugins diff --git a/tests/performance/compare-results.js b/tests/performance/compare-results.js index 5682f08dc2..6af85a2f12 100644 --- a/tests/performance/compare-results.js +++ b/tests/performance/compare-results.js @@ -23,7 +23,14 @@ const parseFile = ( fileName ) => ); // The list of test suites to log. -const testSuites = [ 'home-block-theme', 'home-classic-theme' ]; +const testSuites = [ + 'admin', + 'admin-l10n', + 'home-block-theme', + 'home-block-theme-l10n', + 'home-classic-theme', + 'home-classic-theme-l10n', +]; // The current commit's results. const testResults = Object.fromEntries( @@ -128,6 +135,23 @@ console.log( 'Performance Test Results\n' ); console.log( 'Note: Due to the nature of how GitHub Actions work, some variance in the results is expected.\n' ); +/** + * Nicely formats a given value. + * + * @param {string} metric Metric. + * @param {number} value + */ +function formatValue( metric, value) { + if ( null === value ) { + return 'N/A'; + } + if ( 'wpMemoryUsage' === metric ) { + return `${ ( value / Math.pow( 10, 6 ) ).toFixed( 2 ) } MB`; + } + + return `${ value.toFixed( 2 ) } ms`; +} + for ( const key of testSuites ) { const current = testResults[ key ] || {}; const prev = prevResults[ key ] || {}; @@ -141,15 +165,15 @@ for ( const key of testSuites ) { for ( const [ metric, values ] of Object.entries( current ) ) { const value = median( values ); - const prevValue = median( prev[ metric ] ); + const prevValue = prev[ metric ] ? median( prev[ metric ] ) : null; - const delta = value - prevValue; + const delta = null !== prevValue ? value - prevValue : 0 const percentage = ( delta / value ) * 100; rows.push( { Metric: metric, - Before: `${ prevValue.toFixed( 2 ) } ms`, - After: `${ value.toFixed( 2 ) } ms`, - 'Diff abs.': `${ delta.toFixed( 2 ) } ms`, + Before: formatValue( metric, prevValue ), + After: formatValue( metric, value ), + 'Diff abs.': formatValue( metric, delta ), 'Diff %': `${ percentage.toFixed( 2 ) } %`, } ); } diff --git a/tests/performance/log-results.js b/tests/performance/log-results.js index 8a29366a94..14c836ff67 100644 --- a/tests/performance/log-results.js +++ b/tests/performance/log-results.js @@ -11,8 +11,12 @@ const { median } = require( './utils' ); // The list of test suites to log. const testSuites = [ + 'admin', + 'admin-l10n', 'home-block-theme', + 'home-block-theme-l10n', 'home-classic-theme', + 'home-classic-theme-l10n', ]; // A list of results to parse based on test suites. diff --git a/tests/performance/playwright.config.js b/tests/performance/playwright.config.js index 6c2ff45472..1d2781f73c 100644 --- a/tests/performance/playwright.config.js +++ b/tests/performance/playwright.config.js @@ -19,9 +19,7 @@ process.env.TEST_RUNS ??= '20'; const config = defineConfig( { ...baseConfig, globalSetup: require.resolve( './config/global-setup.js' ), - reporter: process.env.CI - ? './config/performance-reporter.js' - : [ [ 'list' ], [ './config/performance-reporter.js' ] ], + reporter: [ [ 'list' ], [ './config/performance-reporter.js' ] ], forbidOnly: !! process.env.CI, workers: 1, retries: 0, diff --git a/tests/performance/results.js b/tests/performance/results.js index c7a977181d..d9f981f5e7 100644 --- a/tests/performance/results.js +++ b/tests/performance/results.js @@ -8,8 +8,12 @@ const { join } = require( 'node:path' ); const { median, getResultsFilename } = require( './utils' ); const testSuites = [ + 'admin', + 'admin-l10n', 'home-classic-theme', + 'home-classic-theme-l10n', 'home-block-theme', + 'home-block-theme-l10n', ]; console.log( '\n>> 🎉 Results 🎉 \n' ); diff --git a/tests/performance/specs/admin-l10n.test.js b/tests/performance/specs/admin-l10n.test.js new file mode 100644 index 0000000000..a8c9be0997 --- /dev/null +++ b/tests/performance/specs/admin-l10n.test.js @@ -0,0 +1,52 @@ +/** + * WordPress dependencies + */ +import { test } from '@wordpress/e2e-test-utils-playwright'; + +/** + * Internal dependencies + */ +import { camelCaseDashes } from '../utils'; + +const results = { + timeToFirstByte: [], +}; + +test.describe( 'Admin (L10N)', () => { + test.beforeAll( async ( { requestUtils } ) => { + await requestUtils.activateTheme( 'twentytwentyone' ); + await requestUtils.updateSiteSettings( { + language: 'de_DE', + } ); + } ); + + test.afterAll( async ( { requestUtils }, testInfo ) => { + await testInfo.attach( 'results', { + body: JSON.stringify( results, null, 2 ), + contentType: 'application/json', + } ); + await requestUtils.updateSiteSettings( { + language: '', + } ); + } ); + + const iterations = Number( process.env.TEST_RUNS ); + for ( let i = 1; i <= iterations; i++ ) { + test( `Measure load time metrics (${ i } of ${ iterations })`, async ( { + admin, + metrics, + } ) => { + await admin.visitAdminPage( '/' ); + + const serverTiming = await metrics.getServerTiming(); + + for ( const [ key, value ] of Object.entries( serverTiming ) ) { + results[ camelCaseDashes( key ) ] ??= []; + results[ camelCaseDashes( key ) ].push( value ); + } + + const ttfb = await metrics.getTimeToFirstByte(); + results.timeToFirstByte.push( ttfb ); + } ); + } +} ); diff --git a/tests/performance/specs/admin.test.js b/tests/performance/specs/admin.test.js new file mode 100644 index 0000000000..9860229114 --- /dev/null +++ b/tests/performance/specs/admin.test.js @@ -0,0 +1,46 @@ +/** + * WordPress dependencies + */ +import { test } from '@wordpress/e2e-test-utils-playwright'; + +/** + * Internal dependencies + */ +import { camelCaseDashes } from '../utils'; + +const results = { + timeToFirstByte: [], +}; + +test.describe( 'Admin', () => { + test.beforeAll( async ( { requestUtils } ) => { + await requestUtils.activateTheme( 'twentytwentyone' ); + } ); + + test.afterAll( async ( {}, testInfo ) => { + await testInfo.attach( 'results', { + body: JSON.stringify( results, null, 2 ), + contentType: 'application/json', + } ); + } ); + + const iterations = Number( process.env.TEST_RUNS ); + for ( let i = 1; i <= iterations; i++ ) { + test( `Measure load time metrics (${ i } of ${ iterations })`, async ( { + admin, + metrics, + } ) => { + await admin.visitAdminPage( '/' ); + + const serverTiming = await metrics.getServerTiming(); + + for ( const [ key, value ] of Object.entries( serverTiming ) ) { + results[ camelCaseDashes( key ) ] ??= []; + results[ camelCaseDashes( key ) ].push( value ); + } + + const ttfb = await metrics.getTimeToFirstByte(); + results.timeToFirstByte.push( ttfb ); + } ); + } +} ); diff --git a/tests/performance/specs/home-block-theme-l10n.test.js b/tests/performance/specs/home-block-theme-l10n.test.js new file mode 100644 index 0000000000..591925056f --- /dev/null +++ b/tests/performance/specs/home-block-theme-l10n.test.js @@ -0,0 +1,63 @@ +/** + * WordPress dependencies + */ +import { test } from '@wordpress/e2e-test-utils-playwright'; + +/** + * Internal dependencies + */ +import { camelCaseDashes } from '../utils'; + +const results = { + timeToFirstByte: [], + largestContentfulPaint: [], + lcpMinusTtfb: [], +}; + +test.describe( 'Front End - Twenty Twenty Three (L10N)', () => { + test.use( { + storageState: {}, // User will be logged out. + } ); + + test.beforeAll( async ( { requestUtils } ) => { + await requestUtils.activateTheme( 'twentytwentythree' ); + await requestUtils.updateSiteSettings( { + language: 'de_DE', + } ); + } ); + + test.afterAll( async ( { requestUtils }, testInfo ) => { + await testInfo.attach( 'results', { + body: JSON.stringify( results, null, 2 ), + contentType: 'application/json', + } ); + await requestUtils.activateTheme( 'twentytwentyone' ); + await requestUtils.updateSiteSettings( { + language: '', + } ); + } ); + + const iterations = Number( process.env.TEST_RUNS ); + for ( let i = 1; i <= iterations; i++ ) { + test( `Measure load time metrics (${ i } of ${ iterations })`, async ( { + page, + metrics, + } ) => { + await page.goto( '/' ); + + const serverTiming = await metrics.getServerTiming(); + + for ( const [ key, value ] of Object.entries( serverTiming ) ) { + results[ camelCaseDashes( key ) ] ??= []; + results[ camelCaseDashes( key ) ].push( value ); + } + + const ttfb = await metrics.getTimeToFirstByte(); + const lcp = await metrics.getLargestContentfulPaint(); + + results.largestContentfulPaint.push( lcp ); + results.timeToFirstByte.push( ttfb ); + results.lcpMinusTtfb.push( lcp - ttfb ); + } ); + } +} ); diff --git a/tests/performance/specs/home-block-theme.test.js b/tests/performance/specs/home-block-theme.test.js index 496445ad0d..00bccc6996 100644 --- a/tests/performance/specs/home-block-theme.test.js +++ b/tests/performance/specs/home-block-theme.test.js @@ -41,7 +41,7 @@ test.describe( 'Front End - Twenty Twenty Three', () => { const serverTiming = await metrics.getServerTiming(); - for ( const [key, value] of Object.entries( serverTiming ) ) { + for ( const [ key, value ] of Object.entries( serverTiming ) ) { results[ camelCaseDashes( key ) ] ??= []; results[ camelCaseDashes( key ) ].push( value ); } diff --git a/tests/performance/specs/home-classic-theme-l10n.test.js b/tests/performance/specs/home-classic-theme-l10n.test.js new file mode 100644 index 0000000000..e6f6e1cbb9 --- /dev/null +++ b/tests/performance/specs/home-classic-theme-l10n.test.js @@ -0,0 +1,62 @@ +/** + * WordPress dependencies + */ +import { test } from '@wordpress/e2e-test-utils-playwright'; + +/** + * Internal dependencies + */ +import { camelCaseDashes } from '../utils'; + +const results = { + timeToFirstByte: [], + largestContentfulPaint: [], + lcpMinusTtfb: [], +}; + +test.describe( 'Front End - Twenty Twenty One (L10N)', () => { + test.use( { + storageState: {}, // User will be logged out. + } ); + + test.beforeAll( async ( { requestUtils } ) => { + await requestUtils.activateTheme( 'twentytwentyone' ); + await requestUtils.updateSiteSettings( { + language: 'de_DE', + } ); + } ); + + test.afterAll( async ( { requestUtils }, testInfo ) => { + await testInfo.attach( 'results', { + body: JSON.stringify( results, null, 2 ), + contentType: 'application/json', + } ); + await requestUtils.updateSiteSettings( { + language: '', + } ); + } ); + + const iterations = Number( process.env.TEST_RUNS ); + for ( let i = 1; i <= iterations; i++ ) { + test( `Measure load time metrics (${ i } of ${ iterations })`, async ( { + page, + metrics, + } ) => { + await page.goto( '/' ); + + const serverTiming = await metrics.getServerTiming(); + + for ( const [ key, value ] of Object.entries( serverTiming ) ) { + results[ camelCaseDashes( key ) ] ??= []; + results[ camelCaseDashes( key ) ].push( value ); + } + + const ttfb = await metrics.getTimeToFirstByte(); + const lcp = await metrics.getLargestContentfulPaint(); + + results.largestContentfulPaint.push( lcp ); + results.timeToFirstByte.push( ttfb ); + results.lcpMinusTtfb.push( lcp - ttfb ); + } ); + } +} ); diff --git a/tests/performance/specs/home-classic-theme.test.js b/tests/performance/specs/home-classic-theme.test.js index 32125c37a4..a95e50fa06 100644 --- a/tests/performance/specs/home-classic-theme.test.js +++ b/tests/performance/specs/home-classic-theme.test.js @@ -40,7 +40,7 @@ test.describe( 'Front End - Twenty Twenty One', () => { const serverTiming = await metrics.getServerTiming(); - for (const [key, value] of Object.entries( serverTiming ) ) { + for ( const [ key, value ] of Object.entries( serverTiming ) ) { results[ camelCaseDashes( key ) ] ??= []; results[ camelCaseDashes( key ) ].push( value ); } diff --git a/tests/performance/wp-content/mu-plugins/server-timing.php b/tests/performance/wp-content/mu-plugins/server-timing.php index 4f9a0d04f1..53f83fea79 100644 --- a/tests/performance/wp-content/mu-plugins/server-timing.php +++ b/tests/performance/wp-content/mu-plugins/server-timing.php @@ -25,6 +25,13 @@ add_filter( $server_timing_values['total'] = $server_timing_values['before-template'] + $server_timing_values['template']; + /* + * While values passed via Server-Timing are intended to be durations, + * any numeric value can actually be passed. + * This is a nice little trick as it allows to easily get this information in JS. + */ + $server_timing_values['memory-usage'] = memory_get_usage(); + $header_values = array(); foreach ( $server_timing_values as $slug => $value ) { if ( is_float( $value ) ) {