From 5523bd276b41fc67ba032c867c292bd7b7ab9ec9 Mon Sep 17 00:00:00 2001
From: Jonathan Desrosiers <desrosj@git.wordpress.org>
Date: Fri, 22 Sep 2023 17:41:11 +0000
Subject: [PATCH] Build/Test Tools: Introduce install and upgrade testing
 workflows.

This introduces GitHub Action workflows for testing the installation and upgrade process to ensure there are no unexpected issues. The goal of these workflows is to replicate some of the manual testing that happens during the release process. This does not replace that testing which is a fantastic, low barrier way for someone newer to contributing to get involved. Instead, it should serve as an additional sanity check to ensure no combinations are missed.

The installation workflow tests installing the latest or specified version of WordPress on all PHP and MySQL version combinations for both single and multisite.

The upgrade testing will test upgrading from each branch that still receives security updates to the latest or specified version on all PHP and MySQL version combinations for both single and multisite.

Both workflows confirm that no errors are encountered when using WP-CLI to install or upgrade WordPress. No test assertions of any type are currently performed, but adding some will be explored.

Props costdev, azaozz, ironprogrammer, desrosj.
See #58977.

git-svn-id: https://develop.svn.wordpress.org/trunk@56661 602fd350-edb4-49c9-b593-d223f7449a82
---
 .github/workflows/install-testing.yml     | 131 ++++++++++++++
 .github/workflows/upgrade-testing-run.yml |  89 ++++++++++
 .github/workflows/upgrade-testing.yml     | 207 ++++++++++++++++++++++
 3 files changed, 427 insertions(+)
 create mode 100644 .github/workflows/install-testing.yml
 create mode 100644 .github/workflows/upgrade-testing-run.yml
 create mode 100644 .github/workflows/upgrade-testing.yml

diff --git a/.github/workflows/install-testing.yml b/.github/workflows/install-testing.yml
new file mode 100644
index 0000000000..bd8dccc0b8
--- /dev/null
+++ b/.github/workflows/install-testing.yml
@@ -0,0 +1,131 @@
+name: Installation Tests
+
+on:
+  push:
+    branches:
+      - trunk
+    # Always test the workflow after it's updated.
+    paths:
+      - '.github/workflows/install-testing.yml'
+  pull_request:
+    # Always test the workflow when changes are suggested.
+    paths:
+      - '.github/workflows/install-testing.yml'
+  workflow_dispatch:
+    inputs:
+      wp-version:
+        description: 'The version to test installing. Accepts major and minor versions, "latest", or "nightly". Major releases must not end with ".0".'
+        type: string
+        default: 'latest'
+
+# Cancels all previous workflow runs for pull requests that have not completed.
+concurrency:
+  # The concurrency group contains the workflow name and the branch name for pull requests
+  # or the commit hash for any other events.
+  group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.head_ref || github.sha }}
+  cancel-in-progress: true
+
+# Disable permissions for all available scopes by default.
+# Any needed permissions should be configured at the job level.
+permissions: {}
+
+jobs:
+  # Test the WordPress installation process through WP-CLI.
+  #
+  # Performs the following steps:
+  # - Sets up PHP.
+  # - Starts the database server.
+  # - Downloads the specified version of WordPress.
+  # - Creates a `wp-config.php` file.
+  # - Installs WordPress.
+  install-tests-mysql:
+    name: WP ${{ inputs.new-version || 'latest' }} / PHP ${{ matrix.php }} / ${{ 'mariadb' == matrix.db-type && 'MariaDB' || 'MySQL' }} ${{ matrix.db-version }}${{ matrix.multisite && ' multisite' || '' }}
+    permissions:
+      contents: read
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ ubuntu-latest ]
+        php: [ '7.0', '7.1', '7.2', '7.3', '7.4', '8.0', '8.1', '8.2', '8.3' ]
+        db-type: [ 'mysql' ]
+        db-version: [ '5.7', '8.0' ]
+        multisite: [ false, true ]
+
+    services:
+      database:
+        image: ${{ matrix.db-type }}:${{ matrix.db-version }}
+        ports:
+          - 3306
+        options: --health-cmd="mysqladmin ping" --health-interval=30s --health-timeout=10s --health-retries=5 -e MYSQL_ROOT_PASSWORD=root -e MYSQL_DATABASE=test_db --entrypoint sh ${{ matrix.db-type }}:${{ matrix.db-version }} -c "exec docker-entrypoint.sh mysqld --default-authentication-plugin=mysql_native_password"
+
+    steps:
+      - name: Set up PHP ${{ matrix.php }}
+        uses: shivammathur/setup-php@7fdd3ece872ec7ec4c098ae5ab7637d5e0a96067 # v2.26.0
+        with:
+          php-version: '${{ matrix.php }}'
+          coverage: none
+          tools: wp-cli
+
+      - name: Start the database server
+        run: |
+          sudo systemctl start ${{ matrix.db-type }}
+
+      - name: Download WordPress
+        run: wp core download ${{ inputs.wp-version && 'latest' != inputs.wp-version && format( '--version={0}', inputs.wp-version ) || '' }}
+
+      - name: Create wp-config.php file
+        run: wp config create --dbname=test_db --dbuser=root --dbpass=root --dbhost=127.0.0.1:${{ job.services.database.ports['3306'] }}
+
+      - name: Install WordPress
+        run: wp core ${{ matrix.multisite && 'multisite-' || '' }}install --url=http://localhost/ --title="Upgrade Test" --admin_user=admin --admin_password=password --admin_email=me@example.org --skip-email
+
+  slack-notifications:
+    name: Slack Notifications
+    uses: WordPress/wordpress-develop/.github/workflows/slack-notifications.yml@trunk
+    permissions:
+      actions: read
+      contents: read
+    needs: [ install-tests-mysql ]
+    if: ${{ github.repository == 'WordPress/wordpress-develop' && github.event_name != 'pull_request' && always() }}
+    with:
+      calling_status: ${{ contains( needs.*.result, 'cancelled' ) && 'cancelled' || contains( needs.*.result, 'failure' ) && 'failure' || 'success' }}
+    secrets:
+      SLACK_GHA_SUCCESS_WEBHOOK: ${{ secrets.SLACK_GHA_SUCCESS_WEBHOOK }}
+      SLACK_GHA_CANCELLED_WEBHOOK: ${{ secrets.SLACK_GHA_CANCELLED_WEBHOOK }}
+      SLACK_GHA_FIXED_WEBHOOK: ${{ secrets.SLACK_GHA_FIXED_WEBHOOK }}
+      SLACK_GHA_FAILURE_WEBHOOK: ${{ secrets.SLACK_GHA_FAILURE_WEBHOOK }}
+
+  failed-workflow:
+    name: Failed workflow tasks
+    runs-on: ubuntu-latest
+    permissions:
+      actions: write
+    needs: [ slack-notifications ]
+    if: |
+      always() &&
+      github.repository == 'WordPress/wordpress-develop' &&
+      github.event_name != 'pull_request' &&
+      github.run_attempt < 2 &&
+      (
+        contains( needs.*.result, 'cancelled' ) ||
+        contains( needs.*.result, 'failure' )
+      )
+
+    steps:
+      - name: Dispatch workflow run
+        uses: actions/github-script@d7906e4ad0b1822421a7e6a35d5ca353c962f410 # v6.4.1
+        with:
+          retries: 2
+          retry-exempt-status-codes: 418
+          script: |
+            github.rest.actions.createWorkflowDispatch({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              workflow_id: 'failed-workflow.yml',
+              ref: 'trunk',
+              inputs: {
+                run_id: '${{ github.run_id }}'
+              }
+            });
diff --git a/.github/workflows/upgrade-testing-run.yml b/.github/workflows/upgrade-testing-run.yml
new file mode 100644
index 0000000000..b129818192
--- /dev/null
+++ b/.github/workflows/upgrade-testing-run.yml
@@ -0,0 +1,89 @@
+# A callable workflow that runs WordPress upgrade testing under the conditions provided.
+name: Upgrade Tests
+
+on:
+  workflow_call:
+    inputs:
+      os:
+        description: 'Operating system to run tests on.'
+        required: false
+        type: 'string'
+        default: 'ubuntu-latest'
+      wp:
+        description: 'The version of WordPress to start with.'
+        required: true
+        type: 'string'
+      new-version:
+        description: 'The version of WordPress to update to.'
+        type: 'string'
+        default: 'latest'
+      php:
+        description: 'The version of PHP to use. Expected format: X.Y.'
+        required: true
+        type: 'string'
+      multisite:
+        description: 'Whether to run tests as multisite.'
+        required: false
+        type: 'boolean'
+        default: false
+      db-type:
+        description: 'Database type. Valid types are mysql and mariadb.'
+        required: false
+        type: 'string'
+        default: 'mysql'
+      db-version:
+        description: 'Database version.'
+        required: false
+        type: 'string'
+        default: '5.7'
+
+jobs:
+  # Runs upgrade tests on a build of WordPress.
+  #
+  # Performs the following steps:
+  # - Sets up PHP.
+  # - Starts the database server.
+  # - Downloads the specified version of WordPress.
+  # - Creates a `wp-config.php` file.
+  # - Installs WordPress.
+  # - Updates to the latest minor version.
+  # - Updates to the version of WordPress being tested.
+  upgrade-tests:
+    name: ${{ inputs.wp }} to ${{ inputs.new-version }} / PHP ${{ inputs.php }} with ${{ 'mariadb' == inputs.db-type && 'MariaDB' || 'MySQL' }} ${{ inputs.db-version }}${{ inputs.multisite && ' multisite' || '' }}
+    runs-on: ${{ inputs.os }}
+    timeout-minutes: 20
+
+    services:
+      database:
+        image: ${{ inputs.db-type }}:${{ inputs.db-version }}
+        ports:
+          - 3306
+        options: --health-cmd="mysqladmin ping" --health-interval=30s --health-timeout=10s --health-retries=5 -e MYSQL_ROOT_PASSWORD=root -e MYSQL_DATABASE=test_db --entrypoint sh ${{ inputs.db-type }}:${{ inputs.db-version }} -c "exec docker-entrypoint.sh mysqld --default-authentication-plugin=mysql_native_password"
+
+    steps:
+      - name: Set up PHP ${{ inputs.php }}
+        uses: shivammathur/setup-php@7fdd3ece872ec7ec4c098ae5ab7637d5e0a96067 # v2.26.0
+        with:
+          php-version: '${{ inputs.php }}'
+          coverage: none
+          tools: wp-cli
+
+      - name: Start the database server
+        run: |
+          sudo systemctl start ${{ inputs.db-type }}
+
+      - name: Download WordPress ${{ inputs.wp }}
+        run: wp core download --version=${{ inputs.wp }}
+
+      - name: Create wp-config.php file
+        run: wp config create --dbname=test_db --dbuser=root --dbpass=root --dbhost=127.0.0.1:${{ job.services.database.ports['3306'] }}
+
+      - name: Install WordPress
+        run: wp core ${{ inputs.multisite && 'multisite-' || '' }}install --url=http://localhost/ --title="Upgrade Test" --admin_user=admin --admin_password=password --admin_email=me@example.org --skip-email
+
+      - name: Update to the latest minor version
+        run: wp core update --minor
+
+      - name: Upgrade to WordPress ${{ inputs.new-version }}
+        run: |
+          wp core update${{ 'latest' != inputs.new-version && format( ' --version={0}', inputs.new-version ) || '' }}
diff --git a/.github/workflows/upgrade-testing.yml b/.github/workflows/upgrade-testing.yml
new file mode 100644
index 0000000000..18a45d4bd6
--- /dev/null
+++ b/.github/workflows/upgrade-testing.yml
@@ -0,0 +1,207 @@
+name: Upgrade Tests
+
+on:
+  push:
+    branches:
+      - trunk
+    # Always test the workflow after it's updated.
+    paths:
+      - '.github/workflows/upgrade-testing.yml'
+      - '.github/workflows/upgrade-testing-run.yml'
+  pull_request:
+    # Always test the workflow when changes are suggested.
+    paths:
+      - '.github/workflows/upgrade-testing.yml'
+      - '.github/workflows/upgrade-testing-run.yml'
+  workflow_dispatch:
+    inputs:
+      new-version:
+        description: 'The version to test installing. Accepts major and minor versions, "latest", or "nightly". Major releases must not end with ".0".'
+        type: string
+        default: 'latest'
+
+# Cancels all previous workflow runs for pull requests that have not completed.
+concurrency:
+  # The concurrency group contains the workflow name and the branch name for pull requests
+  # or the commit hash for any other events.
+  group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.head_ref || github.sha }}
+  cancel-in-progress: true
+
+# Disable permissions for all available scopes by default.
+# Any needed permissions should be configured at the job level.
+permissions: {}
+
+jobs:
+  # Spawns upgrade testing from WordPress 6.x versions with MySQL.
+  upgrade-tests-wp-6x-mysql:
+    name: ${{ matrix.wp }} to ${{ inputs.new-version && inputs.new-version || 'latest' }}
+    uses: WordPress/wordpress-develop/.github/workflows/upgrade-testing-run.yml@trunk
+    if: ${{ github.repository == 'WordPress/wordpress-develop' || github.event_name == 'pull_request' }}
+    permissions:
+      contents: read
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ 'ubuntu-latest' ]
+        php: [ '7.0', '7.1', '7.2', '7.3', '7.4', '8.0', '8.1', '8.2', '8.3' ]
+        db-type: [ 'mysql' ]
+        db-version: [ '5.7', '8.0' ]
+        wp: [ '6.0', '6.1', '6.2' ]
+        multisite: [ false, true ]
+    with:
+      os: ${{ matrix.os }}
+      php: ${{ matrix.php }}
+      db-type: ${{ matrix.db-type }}
+      db-version: ${{ matrix.db-version }}
+      wp: ${{ matrix.wp }}
+      new-version: ${{ inputs.new-version && inputs.new-version || 'latest' }}
+      multisite: ${{ matrix.multisite }}
+
+  # Spawns upgrade testing from WordPress 5.x versions on PHP 7.x with MySQL.
+  upgrade-tests-wp-5x-php-7x-mysql:
+    name: ${{ matrix.wp }} to ${{ inputs.new-version && inputs.new-version || 'latest' }}
+    uses: WordPress/wordpress-develop/.github/workflows/upgrade-testing-run.yml@trunk
+    if: ${{ github.repository == 'WordPress/wordpress-develop' || github.event_name == 'pull_request' }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ 'ubuntu-latest' ]
+        php: [ '7.0', '7.1', '7.2', '7.3', '7.4' ]
+        db-type: [ 'mysql' ]
+        db-version: [ '5.7', '8.0' ]
+        wp: [ '5.0', '5.1', '5.2', '5.3', '5.4', '5.5', '5.6', '5.7', '5.8', '5.9' ]
+        multisite: [ false, true ]
+    with:
+      os: ${{ matrix.os }}
+      php: ${{ matrix.php }}
+      db-type: ${{ matrix.db-type }}
+      db-version: ${{ matrix.db-version }}
+      wp: ${{ matrix.wp }}
+      new-version: ${{ inputs.new-version && inputs.new-version || 'latest' }}
+      multisite: ${{ matrix.multisite }}
+
+  # Spawns upgrade testing from WordPress 5.x versions on PHP 8.x with MySQL.
+  #
+  # WordPress 5.0-5.2 are excluded from PHP 8+ testing because of the following fatal errors:
+  # - Use of __autoload().
+  # - array/string offset with curly braces.
+  upgrade-tests-wp-5x-php-8x-mysql:
+    name: ${{ matrix.wp }} to ${{ inputs.new-version && inputs.new-version || 'latest' }}
+    uses: WordPress/wordpress-develop/.github/workflows/upgrade-testing-run.yml@trunk
+    if: ${{ github.repository == 'WordPress/wordpress-develop' || github.event_name == 'pull_request' }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ 'ubuntu-latest' ]
+        php: [ '8.0', '8.1', '8.2', '8.3' ]
+        db-type: [ 'mysql' ]
+        db-version: [ '5.7', '8.0' ]
+        wp: [ '5.3', '5.4', '5.5', '5.6', '5.7', '5.8', '5.9' ]
+        multisite: [ false, true ]
+    with:
+      os: ${{ matrix.os }}
+      php: ${{ matrix.php }}
+      db-type: ${{ matrix.db-type }}
+      db-version: ${{ matrix.db-version }}
+      wp: ${{ matrix.wp }}
+      new-version: ${{ inputs.new-version && inputs.new-version || 'latest' }}
+      multisite: ${{ matrix.multisite }}
+
+  # Spawns upgrade testing from WordPress 4.x versions on PHP 7.x with MySQL.
+  upgrade-tests-wp-4x-php-7x-mysql:
+    name: ${{ matrix.wp }} to ${{ inputs.new-version && inputs.new-version || 'latest' }}
+    uses: WordPress/wordpress-develop/.github/workflows/upgrade-testing-run.yml@trunk
+    if: ${{ github.repository == 'WordPress/wordpress-develop' || github.event_name == 'pull_request' }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ 'ubuntu-latest' ]
+        php: [ '7.0', '7.1', '7.2', '7.3', '7.4' ]
+        db-type: [ 'mysql' ]
+        db-version: [ '5.7', '8.0' ]
+        wp: [ '4.1', '4.2', '4.3', '4.4', '4.5', '4.6', '4.7', '4.8', '4.9' ]
+        multisite: [ false, true ]
+    with:
+      os: ${{ matrix.os }}
+      php: ${{ matrix.php }}
+      db-type: ${{ matrix.db-type }}
+      db-version: ${{ matrix.db-version }}
+      wp: ${{ matrix.wp }}
+      new-version: ${{ inputs.new-version && inputs.new-version || 'latest' }}
+      multisite: ${{ matrix.multisite }}
+
+  # Spawns upgrade testing from WordPress 4.x versions on PHP 8.x with MySQL.
+  #
+  # WordPress 4.6-4.9 are excluded from PHP 8+ testing because of the following fatal errors:
+  # - Use of __autoload().
+  # - array/string offset with curly braces.
+  upgrade-tests-wp-4x-php-8x-mysql:
+    name: ${{ matrix.wp }} to ${{ inputs.new-version && inputs.new-version || 'latest' }}
+    uses: WordPress/wordpress-develop/.github/workflows/upgrade-testing-run.yml@trunk
+    if: ${{ github.repository == 'WordPress/wordpress-develop' || github.event_name == 'pull_request' }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ 'ubuntu-latest' ]
+        php: [ '8.0', '8.1', '8.2', '8.3' ]
+        db-type: [ 'mysql' ]
+        db-version: [ '5.7', '8.0' ]
+        wp: [ '4.1', '4.2', '4.3', '4.4', '4.5' ]
+        multisite: [ false, true ]
+    with:
+      os: ${{ matrix.os }}
+      php: ${{ matrix.php }}
+      db-type: ${{ matrix.db-type }}
+      db-version: ${{ matrix.db-version }}
+      wp: ${{ matrix.wp }}
+      new-version: ${{ inputs.new-version && inputs.new-version || 'latest' }}
+      multisite: ${{ matrix.multisite }}
+
+  slack-notifications:
+    name: Slack Notifications
+    uses: WordPress/wordpress-develop/.github/workflows/slack-notifications.yml@trunk
+    permissions:
+      actions: read
+      contents: read
+    needs: [ upgrade-tests-wp-6x-mysql, upgrade-tests-wp-5x-php-7x-mysql, upgrade-tests-wp-5x-php-8x-mysql, upgrade-tests-wp-4x-php-7x-mysql, upgrade-tests-wp-4x-php-8x-mysql ]
+    if: ${{ github.repository == 'WordPress/wordpress-develop' && github.event_name != 'pull_request' && always() }}
+    with:
+      calling_status: ${{ contains( needs.*.result, 'cancelled' ) && 'cancelled' || contains( needs.*.result, 'failure' ) && 'failure' || 'success' }}
+    secrets:
+      SLACK_GHA_SUCCESS_WEBHOOK: ${{ secrets.SLACK_GHA_SUCCESS_WEBHOOK }}
+      SLACK_GHA_CANCELLED_WEBHOOK: ${{ secrets.SLACK_GHA_CANCELLED_WEBHOOK }}
+      SLACK_GHA_FIXED_WEBHOOK: ${{ secrets.SLACK_GHA_FIXED_WEBHOOK }}
+      SLACK_GHA_FAILURE_WEBHOOK: ${{ secrets.SLACK_GHA_FAILURE_WEBHOOK }}
+
+  failed-workflow:
+    name: Failed workflow tasks
+    runs-on: ubuntu-latest
+    permissions:
+      actions: write
+    needs: [ slack-notifications ]
+    if: |
+      always() &&
+      github.repository == 'WordPress/wordpress-develop' &&
+      github.event_name != 'pull_request' &&
+      github.run_attempt < 2 &&
+      (
+        contains( needs.*.result, 'cancelled' ) ||
+        contains( needs.*.result, 'failure' )
+      )
+
+    steps:
+      - name: Dispatch workflow run
+        uses: actions/github-script@d7906e4ad0b1822421a7e6a35d5ca353c962f410 # v6.4.1
+        with:
+          retries: 2
+          retry-exempt-status-codes: 418
+          script: |
+            github.rest.actions.createWorkflowDispatch({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              workflow_id: 'failed-workflow.yml',
+              ref: 'trunk',
+              inputs: {
+                run_id: '${{ github.run_id }}'
+              }
+            });