diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml new file mode 100644 index 0000000000..0325f950ee --- /dev/null +++ b/.github/workflows/performance.yml @@ -0,0 +1,231 @@ +name: Performance Tests + +on: + push: + branches: + - trunk + - '6.[2-9]' + - '[7-9].[0-9]' + tags: + - '[0-9]+.[0-9]' + - '[0-9]+.[0-9].[0-9]+' + - '![45].[0-9].[0-9]+' + - '!6.[01].[0-9]+' + pull_request: + branches: + - trunk + - '6.[2-9]' + - '[7-9].[0-9]' + +# Cancels all previous workflow runs for pull requests that have not completed. +concurrency: + # The concurrency group contains the workflow name and the branch name for pull requests + # or the commit hash for any other events. + group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.head_ref || github.sha }} + cancel-in-progress: true + +env: + # This workflow takes two sets of measurements — one for the current commit, + # and another against a consistent version that is used as a baseline measurement. + # This is done to isolate variance in measurements caused by the GitHub runners + # from differences caused by code changes between commits. The BASE_TAG value here + # represents the version being used for baseline measurements. It should only be + # changed if we want to normalize results against a different baseline. + BASE_TAG: '6.1.1' + LOCAL_DIR: build + +jobs: + # Runs the performance test suite. + # + # Performs the following steps: + # - Configure environment variables. + # - Checkout repository. + # - Set up Node.js. + # - Log debug information. + # - Install npm dependencies. + # - Build WordPress. + # - Start Docker environment. + # - Log running Docker containers. + # - Docker debug information. + # - Install WordPress. + # - Install WordPress Importer plugin. + # - Import mock data. + # - Update permalink structure. + # - Install MU plugin. + # - Run performance tests (current commit). + # - Print performance tests results. + # - Set the environment to the baseline version. + # - Run baseline performance tests. + # - Print base line performance tests results. + # - Set the base sha. + # - Set commit details. + # - Publish performance results. + # - Ensure version-controlled files are not modified or deleted. + # - Dispatch workflow run. + performance: + name: Run performance tests + runs-on: ubuntu-latest + if: ${{ github.repository == 'WordPress/wordpress-develop' || github.event_name == 'pull_request' }} + + steps: + - name: Configure environment variables + run: | + echo "PHP_FPM_UID=$(id -u)" >> $GITHUB_ENV + echo "PHP_FPM_GID=$(id -g)" >> $GITHUB_ENV + + - name: Checkout repository + uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0 + + - name: Set up Node.js + uses: actions/setup-node@64ed1c7eab4cce3362f8c340dee64e5eaeef8f7c # v3.6.0 + with: + node-version-file: '.nvmrc' + cache: npm + + - name: Log debug information + run: | + npm --version + node --version + curl --version + git --version + svn --version + locale -a + + - name: Install npm dependencies + run: npm ci + + - name: Build WordPress + run: npm run build + + - name: Start Docker environment + run: | + npm run env:start + + - name: Log running Docker containers + run: docker ps -a + + - name: Docker debug information + run: | + docker -v + docker-compose -v + docker-compose run --rm mysql mysql --version + docker-compose run --rm php php --version + docker-compose run --rm php php -m + docker-compose run --rm php php -i + docker-compose run --rm php locale -a + + - name: Install WordPress + run: npm run env:install + + - name: Install WordPress Importer plugin + run: npm run env:cli -- plugin install wordpress-importer --activate --path=/var/www/${{ env.LOCAL_DIR }} + + - name: Import mock data + run: | + curl -O https://raw.githubusercontent.com/WPTT/theme-test-data/b9752e0533a5acbb876951a8cbb5bcc69a56474c/themeunittestdata.wordpress.xml + npm run env:cli -- import themeunittestdata.wordpress.xml --authors=create --path=/var/www/${{ env.LOCAL_DIR }} + rm themeunittestdata.wordpress.xml + + - name: Update permalink structure + run: | + npm run env:cli -- rewrite structure '/%year%/%monthnum%/%postname%/' --path=/var/www/${{ env.LOCAL_DIR }} + + - name: Install MU plugin + run: | + mkdir ./${{ env.LOCAL_DIR }}/wp-content/mu-plugins + cp ./tests/performance/wp-content/mu-plugins/server-timing.php ./${{ env.LOCAL_DIR }}/wp-content/mu-plugins/server-timing.php + + - name: Run performance tests (current commit) + run: npm run test:performance + + - name: Print performance tests results + run: "node ./tests/performance/results.js" + + - name: Set the environment to the baseline version + run: | + npm run env:cli -- core update --version=${{ env.BASE_TAG }} --force --path=/var/www/${{ env.LOCAL_DIR }} + npm run env:cli -- core version --path=/var/www/${{ env.LOCAL_DIR }} + + - name: Run baseline performance tests + run: npm run test:performance -- --prefix=base + + - name: Print base line performance tests results + run: "node ./tests/performance/results.js --prefix=base" + + - name: Set the base sha + # Only needed when publishing results. + if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/trunk' }} + uses: actions/github-script@98814c53be79b1d30f795b907e553d8679345975 # v6.4.0 + id: base-sha + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const baseRef = await github.rest.git.getRef({ owner: context.repo.owner, repo: context.repo.repo, ref: 'tags/${{ env.BASE_TAG }}' }); + return baseRef.data.object.sha; + + - name: Set commit details + # Only needed when publishing results. + if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/trunk' }} + uses: actions/github-script@98814c53be79b1d30f795b907e553d8679345975 # v6.4.0 + id: commit-timestamp + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const commit_details = await github.rest.git.getCommit({ owner: context.repo.owner, repo: context.repo.repo, commit_sha: context.sha }); + return parseInt((new Date( commit_details.data.author.date ).getTime() / 1000).toFixed(0)) + + - name: Publish performance results + # Only publish results on pushes to trunk. + if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/trunk' }} + env: + BASE_SHA: ${{ steps.base-sha.outputs.result }} + COMMITTED_AT: ${{ steps.commit-timestamp.outputs.result }} + CODEVITALS_PROJECT_TOKEN: ${{ secrets.CODEVITALS_PROJECT_TOKEN }} + HOST_NAME: "codevitals.run" + run: node ./tests/performance/log-results.js $CODEVITALS_PROJECT_TOKEN trunk $GITHUB_SHA $BASE_SHA $COMMITTED_AT $HOST_NAME + + - name: Ensure version-controlled files are not modified or deleted + run: git diff --exit-code + + slack-notifications: + name: Slack Notifications + uses: WordPress/wordpress-develop/.github/workflows/slack-notifications.yml@trunk + needs: [ performance ] + if: ${{ github.repository == 'WordPress/wordpress-develop' && github.event_name != 'pull_request' && always() }} + with: + calling_status: ${{ needs.performance.result == 'success' && 'success' || needs.performance.result == 'cancelled' && 'cancelled' || 'failure' }} + secrets: + SLACK_GHA_SUCCESS_WEBHOOK: ${{ secrets.SLACK_GHA_SUCCESS_WEBHOOK }} + SLACK_GHA_CANCELLED_WEBHOOK: ${{ secrets.SLACK_GHA_CANCELLED_WEBHOOK }} + SLACK_GHA_FIXED_WEBHOOK: ${{ secrets.SLACK_GHA_FIXED_WEBHOOK }} + SLACK_GHA_FAILURE_WEBHOOK: ${{ secrets.SLACK_GHA_FAILURE_WEBHOOK }} + + failed-workflow: + name: Failed workflow tasks + runs-on: ubuntu-latest + needs: [ performance, slack-notifications ] + if: | + always() && + github.repository == 'WordPress/wordpress-develop' && + github.event_name != 'pull_request' && + github.run_attempt < 2 && + ( + needs.performance.result == 'cancelled' || needs.performance.result == 'failure' + ) + + steps: + - name: Dispatch workflow run + uses: actions/github-script@98814c53be79b1d30f795b907e553d8679345975 # v6.4.0 + with: + retries: 2 + retry-exempt-status-codes: 418 + script: | + github.rest.actions.createWorkflowDispatch({ + owner: context.repo.owner, + repo: context.repo.repo, + workflow_id: 'failed-workflow.yml', + ref: 'trunk', + inputs: { + run_id: '${{ github.run_id }}' + } + }); diff --git a/.gitignore b/.gitignore index 2a08b9f1d7..c0d8a2d79f 100644 --- a/.gitignore +++ b/.gitignore @@ -14,6 +14,7 @@ wp-tests-config.php /tests/phpunit/data/plugins/wordpress-importer /tests/phpunit/data/.trac-ticket-cache* /tests/qunit/compiled.html +/tests/performance/**/*.test.results.json /src/.wp-tests-version /node_modules /npm-debug.log diff --git a/package.json b/package.json index da9b506add..d386d95e79 100644 --- a/package.json +++ b/package.json @@ -176,6 +176,7 @@ "env:cli": "node ./tools/local-env/scripts/docker.js run cli", "env:logs": "node ./tools/local-env/scripts/docker.js logs", "env:pull": "node ./tools/local-env/scripts/docker.js pull", + "test:performance": "node ./tests/performance/run-tests.js", "test:php": "node ./tools/local-env/scripts/docker.js run -T php composer update -W && node ./tools/local-env/scripts/docker.js run php ./vendor/bin/phpunit", "test:e2e": "node ./tests/e2e/run-tests.js", "test:visual": "node ./tests/visual-regression/run-tests.js", diff --git a/tests/performance/config/bootstrap.js b/tests/performance/config/bootstrap.js new file mode 100644 index 0000000000..773d2c1d74 --- /dev/null +++ b/tests/performance/config/bootstrap.js @@ -0,0 +1,41 @@ +/** + * WordPress dependencies. + */ +import { + clearLocalStorage, + enablePageDialogAccept, + setBrowserViewport, +} from '@wordpress/e2e-test-utils'; + +/** + * Timeout, in seconds, that the test should be allowed to run. + * + * @type {string|undefined} + */ +const PUPPETEER_TIMEOUT = process.env.PUPPETEER_TIMEOUT; + +// The Jest timeout is increased because these tests are a bit slow. +jest.setTimeout( PUPPETEER_TIMEOUT || 100000 ); + +async function setupBrowser() { + await clearLocalStorage(); + await setBrowserViewport( 'large' ); +} + +/* + * Before every test suite run, delete all content created by the test. This ensures + * other posts/comments/etc. aren't dirtying tests and tests don't depend on + * each other's side-effects. + */ +beforeAll( async () => { + enablePageDialogAccept(); + + await setBrowserViewport( 'large' ); + await page.emulateMediaFeatures( [ + { name: 'prefers-reduced-motion', value: 'reduce' }, + ] ); +} ); + +afterEach( async () => { + await setupBrowser(); +} ); diff --git a/tests/performance/jest.config.js b/tests/performance/jest.config.js new file mode 100644 index 0000000000..b62bb016c3 --- /dev/null +++ b/tests/performance/jest.config.js @@ -0,0 +1,14 @@ +const config = require( '@wordpress/scripts/config/jest-e2e.config' ); + +const jestE2EConfig = { + ...config, + setupFilesAfterEnv: [ + '/config/bootstrap.js', + ], + globals: { + // Number of requests to run per test. + TEST_RUNS: 20, + } +}; + +module.exports = jestE2EConfig; diff --git a/tests/performance/log-results.js b/tests/performance/log-results.js new file mode 100644 index 0000000000..8a29366a94 --- /dev/null +++ b/tests/performance/log-results.js @@ -0,0 +1,102 @@ +#!/usr/bin/env node + +/** + * External dependencies. + */ +const fs = require( 'fs' ); +const path = require( 'path' ); +const https = require( 'https' ); +const [ token, branch, hash, baseHash, timestamp, host ] = process.argv.slice( 2 ); +const { median } = require( './utils' ); + +// The list of test suites to log. +const testSuites = [ + 'home-block-theme', + 'home-classic-theme', +]; + +// A list of results to parse based on test suites. +const testResults = testSuites.map(( key ) => ({ + key, + file: `${ key }.test.results.json`, +})); + +// A list of base results to parse based on test suites. +const baseResults = testSuites.map(( key ) => ({ + key, + file: `base-${ key }.test.results.json`, +})); + +/** + * Parse test files into JSON objects. + * + * @param {string} fileName The name of the file. + * @returns An array of parsed objects from each file. + */ +const parseFile = ( fileName ) => ( + JSON.parse( + fs.readFileSync( path.join( __dirname, '/specs/', fileName ), 'utf8' ) + ) +); + +/** + * Gets the array of metrics from a list of results. + * + * @param {Object[]} results A list of results to format. + * @return {Object[]} Metrics. + */ +const formatResults = ( results ) => { + return results.reduce( + ( result, { key, file } ) => { + return { + ...result, + ...Object.fromEntries( + Object.entries( + parseFile( file ) ?? {} + ).map( ( [ metric, value ] ) => [ + key + '-' + metric, + median ( value ), + ] ) + ), + }; + }, + {} + ); +}; + +const data = new TextEncoder().encode( + JSON.stringify( { + branch, + hash, + baseHash, + timestamp: parseInt( timestamp, 10 ), + metrics: formatResults( testResults ), + baseMetrics: formatResults( baseResults ), + } ) +); + +const options = { + hostname: host, + port: 443, + path: '/api/log?token=' + token, + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Content-Length': data.length, + }, +}; + +const req = https.request( options, ( res ) => { + console.log( `statusCode: ${ res.statusCode }` ); + + res.on( 'data', ( d ) => { + process.stdout.write( d ); + } ); +} ); + +req.on( 'error', ( error ) => { + console.error( error ); +} ); + +req.write( data ); +req.end(); diff --git a/tests/performance/results.js b/tests/performance/results.js new file mode 100644 index 0000000000..3ebf47edaa --- /dev/null +++ b/tests/performance/results.js @@ -0,0 +1,38 @@ +#!/usr/bin/env node + +/** + * External dependencies. + */ +const fs = require( 'fs' ); +const { join } = require( 'path' ); +const { median, getResultsFilename } = require( './utils' ); + +const testSuites = [ + 'home-classic-theme', + 'home-block-theme', +]; + +console.log( '\n>> 🎉 Results 🎉 \n' ); + +for ( const testSuite of testSuites ) { + const resultsFileName = getResultsFilename( testSuite + '.test' ); + const resultsPath = join( __dirname, '/specs/', resultsFileName ); + fs.readFile( resultsPath, "utf8", ( err, data ) => { + if ( err ) { + console.log( "File read failed:", err ); + return; + } + const convertString = testSuite.charAt( 0 ).toUpperCase() + testSuite.slice( 1 ); + console.log( convertString.replace( /[-]+/g, " " ) + ':' ); + + tableData = JSON.parse( data ); + const rawResults = []; + + for ( var key in tableData ) { + if ( tableData.hasOwnProperty( key ) ) { + rawResults[ key ] = median( tableData[ key ] ); + } + } + console.table( rawResults ); + }); +} diff --git a/tests/performance/run-tests.js b/tests/performance/run-tests.js new file mode 100644 index 0000000000..84e3c84784 --- /dev/null +++ b/tests/performance/run-tests.js @@ -0,0 +1,16 @@ +/** + * External dependencies. + */ +const dotenv = require( 'dotenv' ); +const dotenv_expand = require( 'dotenv-expand' ); +const { execSync } = require( 'child_process' ); + +// WP_BASE_URL interpolates LOCAL_PORT, so needs to be parsed by dotenv_expand(). +dotenv_expand.expand( dotenv.config() ); + +// Run the tests, passing additional arguments through to the test script. +execSync( + 'wp-scripts test-e2e --config tests/performance/jest.config.js ' + + process.argv.slice( 2 ).join( ' ' ), + { stdio: 'inherit' } +); diff --git a/tests/performance/specs/home-block-theme.test.js b/tests/performance/specs/home-block-theme.test.js new file mode 100644 index 0000000000..8811fee698 --- /dev/null +++ b/tests/performance/specs/home-block-theme.test.js @@ -0,0 +1,53 @@ +/** + * External dependencies. + */ +const { basename, join } = require( 'path' ); +const { writeFileSync } = require( 'fs' ); +const { getResultsFilename } = require( './../utils' ); + +/** + * WordPress dependencies. + */ +import { activateTheme, createURL } from '@wordpress/e2e-test-utils'; + +describe( 'Server Timing - Twenty Twenty Three', () => { + const results = { + wpBeforeTemplate: [], + wpTemplate: [], + wpTotal: [], + }; + + beforeAll( async () => { + await activateTheme( 'twentytwentythree' ); + } ); + + afterAll( async () => { + const resultsFilename = getResultsFilename( basename( __filename, '.js' ) ); + writeFileSync( + join( __dirname, resultsFilename ), + JSON.stringify( results, null, 2 ) + ); + } ); + + it( 'Server Timing Metrics', async () => { + let i = TEST_RUNS; + while ( i-- ) { + await page.goto( createURL( '/' ) ); + const navigationTimingJson = await page.evaluate( () => + JSON.stringify( performance.getEntriesByType( 'navigation' ) ) + ); + + const [ navigationTiming ] = JSON.parse( navigationTimingJson ); + + results.wpBeforeTemplate.push( + navigationTiming.serverTiming[0].duration + ); + results.wpTemplate.push( + navigationTiming.serverTiming[1].duration + ); + results.wpTotal.push( + navigationTiming.serverTiming[2].duration + ); + } + } ); +} ); diff --git a/tests/performance/specs/home-classic-theme.test.js b/tests/performance/specs/home-classic-theme.test.js new file mode 100644 index 0000000000..64a170207c --- /dev/null +++ b/tests/performance/specs/home-classic-theme.test.js @@ -0,0 +1,55 @@ +/** + * External dependencies. + */ +const { basename, join } = require( 'path' ); +const { writeFileSync } = require( 'fs' ); +const { exec } = require( 'child_process' ); +const { getResultsFilename } = require( './../utils' ); + +/** + * WordPress dependencies. + */ +import { activateTheme, createURL } from '@wordpress/e2e-test-utils'; + +describe( 'Server Timing - Twenty Twenty One', () => { + const results = { + wpBeforeTemplate: [], + wpTemplate: [], + wpTotal: [], + }; + + beforeAll( async () => { + await activateTheme( 'twentytwentyone' ); + await exec( 'npm run env:cli -- menu location assign all-pages primary' ); + } ); + + afterAll( async () => { + const resultsFilename = getResultsFilename( basename( __filename, '.js' ) ); + writeFileSync( + join( __dirname, resultsFilename ), + JSON.stringify( results, null, 2 ) + ); + } ); + + it( 'Server Timing Metrics', async () => { + let i = TEST_RUNS; + while ( i-- ) { + await page.goto( createURL( '/' ) ); + const navigationTimingJson = await page.evaluate( () => + JSON.stringify( performance.getEntriesByType( 'navigation' ) ) + ); + + const [ navigationTiming ] = JSON.parse( navigationTimingJson ); + + results.wpBeforeTemplate.push( + navigationTiming.serverTiming[0].duration + ); + results.wpTemplate.push( + navigationTiming.serverTiming[1].duration + ); + results.wpTotal.push( + navigationTiming.serverTiming[2].duration + ); + } + } ); +} ); diff --git a/tests/performance/utils.js b/tests/performance/utils.js new file mode 100644 index 0000000000..f85e708847 --- /dev/null +++ b/tests/performance/utils.js @@ -0,0 +1,33 @@ +/** + * Computes the median number from an array numbers. + * + * @param {number[]} array + * + * @return {number} Median. + */ +function median( array ) { + const mid = Math.floor( array.length / 2 ); + const numbers = [ ...array ].sort( ( a, b ) => a - b ); + return array.length % 2 !== 0 + ? numbers[ mid ] + : ( numbers[ mid - 1 ] + numbers[ mid ] ) / 2; +} + +/** + * Gets the result file name. + * + * @param {string} File name. + * + * @return {string} Result file name. + */ +function getResultsFilename( fileName ) { + const prefixArg = process.argv.find( ( arg ) => arg.startsWith( '--prefix' ) ); + const fileNamePrefix = prefixArg ? `${prefixArg.split( '=' )[1]}-` : ''; + const resultsFilename = fileNamePrefix + fileName + '.results.json'; + return resultsFilename; +} + +module.exports = { + median, + getResultsFilename, +}; diff --git a/tests/performance/wp-content/mu-plugins/server-timing.php b/tests/performance/wp-content/mu-plugins/server-timing.php new file mode 100644 index 0000000000..267ca164d3 --- /dev/null +++ b/tests/performance/wp-content/mu-plugins/server-timing.php @@ -0,0 +1,43 @@ + $value ) { + if ( is_float( $value ) ) { + $value = round( $value * 1000.0, 2 ); + } + $header_values[] = sprintf( 'wp-%1$s;dur=%2$s', $slug, $value ); + } + header( 'Server-Timing: ' . implode( ', ', $header_values ) ); + + echo $output; + }, + PHP_INT_MIN + ); + }, + PHP_INT_MAX +);