Merge pull request #117 from stronk7/bisect_phpunit

Add support to run bisect sessions to the phpunit job

runner/main/jobtypes/phpunit/phpunit.sh

@@ -62,6 +62,10 @@ function phpunit_to_summary() {
     echo "== PHPUNIT_FILTER: ${PHPUNIT_FILTER}"
     echo "== PHPUNIT_TESTSUITE: ${PHPUNIT_TESTSUITE}"
     echo "== MOODLE_CONFIG: ${MOODLE_CONFIG}"
+    if [[ -n "${GOOD_COMMIT}" ]] || [[ -n "${BAD_COMMIT}" ]]; then
+        echo "== GOOD_COMMIT: ${GOOD_COMMIT}"
+        echo "== BAD_COMMIT: ${BAD_COMMIT}"
+    fi
 }
 
 # This job type defines the following env variables
@@ -106,7 +110,7 @@ function phpunit_check() {
     verify_modules $(phpunit_modules)
 
     # These env variables must be set for the job to work.
-    verify_env UUID ENVIROPATH WEBSERVER
+    verify_env UUID ENVIROPATH WEBSERVER GOOD_COMMIT BAD_COMMIT
 }
 
 # PHPUnit job type init.
@@ -116,23 +120,72 @@ function phpunit_config() {
     PHPUNIT_FILTER="${PHPUNIT_FILTER:-}"
     PHPUNIT_TESTSUITE="${PHPUNIT_TESTSUITE:-}"
     EXITCODE=0
+
+    # If GOOD_COMMIT and BAD_COMMIT are set, it means that we are going to run a bisect
+    # session, so we need to enable FULLGIT (to get access to complete repository clone).
+    if [[ -n "${GOOD_COMMIT}" ]] && [[ -n "${BAD_COMMIT}" ]]; then
+        FULLGIT="yes"
+        # Also, we don't want to allow repetitions in the bisect session.
+        RUNCOUNT=1
+    fi
 }
 
 # PHPUnit job type setup.
 function phpunit_setup() {
+    # If one of GOOD_COMMIT or BAD_COMMIT are not set, but the other is, error out.
+    if [[ -n "${GOOD_COMMIT}" ]] && [[ -z "${BAD_COMMIT}" ]]; then
+        exit_error "GOOD_COMMIT is set but BAD_COMMIT is not set."
+    fi
+    if [[ -z "${GOOD_COMMIT}" ]] && [[ -n "${BAD_COMMIT}" ]]; then
+        exit_error "BAD_COMMIT is set but GOOD_COMMIT is not set."
+    fi
+    # If both GOOD_COMMIT and BAD_COMMIT are set and they are the same, error out.
+    if [[ -n "${GOOD_COMMIT}" ]] && [[ -n "${BAD_COMMIT}" ]] && [[ "${GOOD_COMMIT}" == "${BAD_COMMIT}" ]]; then
+        exit_error "GOOD_COMMIT and BAD_COMMIT are set, but they are the same."
+    fi
+
+    # If both GOOD_COMMIT and BAD_COMMIT are not set, we are going to run a normal session.
+    # (for bisect sessions we don't have to setup the environment).
+    if [[ -z "${GOOD_COMMIT}" ]] && [[ -z "${BAD_COMMIT}" ]]; then
+        phpunit_setup_normal
+    fi
+}
+
+# PHPUnit job type setup for normal mode.
+function phpunit_setup_normal() {
     # Init the PHPUnit site.
     echo
     echo ">>> startsection Initialising PHPUnit environment at $(date)<<<"
     echo "============================================================================"
-    docker exec -t -u www-data "${WEBSERVER}" \
-        php admin/tool/phpunit/cli/init.php \
-            --force
+    local initcmd
+    phpunit_initcmd initcmd # By nameref.
+    docker exec -t -u www-data "${WEBSERVER}" "${initcmd[@]}"
     echo "============================================================================"
     echo ">>> stopsection <<<"
 }
 
+# Returns (by nameref) an array with the command needed to init the PHPUnit site.
+function phpunit_initcmd() {
+    local -n cmd=$1
+    cmd=(
+        php
+        admin/tool/phpunit/cli/init.php
+    )
+}
+
 # PHPUnit job type run.
 function phpunit_run() {
+    # If both GOOD_COMMIT and BAD_COMMIT are not set, we are going to run a normal session.
+    if [[ -z "${GOOD_COMMIT}" ]] && [[ -z "${BAD_COMMIT}" ]]; then
+        phpunit_run_normal
+    else
+        # If GOOD_COMMIT and BAD_COMMIT are set, we are going to run a bisect session.
+        phpunit_run_bisect
+    fi
+}
+
+# PHPUnit job tye run for normal mode.
+function phpunit_run_normal() {
     # Run the job type.
     echo
     if [[ RUNCOUNT -gt 1 ]]; then
@@ -142,32 +195,85 @@ function phpunit_run() {
     fi
     echo "============================================================================"
     # Build the complete command
-    local cmd=(
-        php vendor/bin/phpunit
-        --disallow-test-output
-        --fail-on-risky
-        --log-junit /shared/log.junit
-        --verbose
-    )
-    if [[ -n "${PHPUNIT_FILTER}" ]]; then
-        cmd+=(--filter "${PHPUNIT_FILTER}")
-    fi
-    if [[ -n "${PHPUNIT_TESTSUITE}" ]]; then
-        cmd+=(--testsuite "${PHPUNIT_TESTSUITE}")
-    fi
+    local runcmd
+    phpunit_runcmd runcmd # By nameref.
 
-    echo "Running: ${cmd[*]}"
+    echo "Running: ${runcmd[*]}"
 
     # Run the command RUNCOUNT times.
     local iter=1
     while [[ ${iter} -le ${RUNCOUNT} ]]; do
         echo
         echo ">>> PHPUnit run ${iter} at $(date) <<<"
-        docker exec -t "${WEBSERVER}" "${cmd[@]}"
+        docker exec -t -u www-data "${WEBSERVER}" "${runcmd[@]}"
         EXITCODE=$((EXITCODE + $?))
         iter=$((iter+1))
     done
 
     echo "============================================================================"
     echo ">>> stopsection <<<"
-}
+}
+
+# PHPUnit job tye run for bisect mode.
+function phpunit_run_bisect() {
+    # Run the job type.
+    echo
+    echo ">>> startsection Starting PHPUnit bisect session at $(date) <<<"
+    echo "=== Good commit: ${GOOD_COMMIT}"
+    echo "=== Bad commit: ${BAD_COMMIT}"
+    echo "============================================================================"
+    # Start the bisect session.
+    docker exec -t -u www-data "${WEBSERVER}" \
+        git bisect start "${BAD_COMMIT}" "${GOOD_COMMIT}"
+
+    # Build the int command.
+    local initcmd
+    phpunit_initcmd initcmd # By nameref.
+
+    # Build the run command.
+    local runcmd
+    phpunit_runcmd runcmd # By nameref.
+
+    # Generate the bisect.sh script that we are going to use to run the phpunit bisect session.
+    # (it runs both init and run commands together).
+    docker exec -i -u www-data "${WEBSERVER}" \
+        bash -c "cat > bisect.sh" <<- EOF
+			#!/bin/bash
+			${initcmd[@]} >/dev/null 2>&1; ${runcmd[@]}
+			exitcode=\$?
+			echo "============================================================================"
+			exit \$exitcode
+			EOF
+
+    # Run the bisect session.
+    echo "============================================================================"
+    docker exec -u www-data "${WEBSERVER}" \
+        git bisect run bash bisect.sh
+    EXITCODE=$?
+
+    # Finish the bisect session.
+    docker exec -u www-data "${WEBSERVER}" \
+        git bisect reset
+
+    echo "============================================================================"
+    echo ">>> stopsection <<<"
+}
+
+# Returns (by nameref) an array with the command needed to run the PHPUnit tests.
+function phpunit_runcmd() {
+    local -n cmd=$1
+    cmd=(
+        php
+        vendor/bin/phpunit
+        --disallow-test-output
+        --fail-on-risky
+        --log-junit /shared/log.junit
+        --verbose
+    )
+    if [[ -n "${PHPUNIT_FILTER}" ]]; then
+        cmd+=(--filter "${PHPUNIT_FILTER}")
+    fi
+    if [[ -n "${PHPUNIT_TESTSUITE}" ]]; then
+        cmd+=(--testsuite "${PHPUNIT_TESTSUITE}")
+    fi
+}

runner/main/modules/docker-php/docker-php.sh

@@ -58,6 +58,9 @@ function docker-php_setup() {
       -v "${SHAREDDIR}":/shared \
       "${DOCKER_PHP}"
 
+    # Ensure that the whole .composer directory is writable to all (www-data needs to write there).
+    docker exec "${WEBSERVER}" chmod -R go+rw /var/www/.composer
+
     echo
     echo "Webserver logs:"
     docker logs "${WEBSERVER}"

runner/main/modules/git/git.sh

@@ -21,6 +21,8 @@
 function git_env() {
     env=(
         GIT_COMMIT
+        GOOD_COMMIT
+        BAD_COMMIT
     )
     echo "${env[@]}"
 }

runner/main/modules/moodle-core-copy/moodle-core-copy.sh

@@ -74,7 +74,7 @@ function moodle-core-copy_setup() {
         fi
 
         # If the repository was cloned shallow (--depth), un-shallow it.
-        if (docker exec -u www-data "${WEBSERVER}" git rev-parse --is-shallow-repository); then
+        if (docker exec -u www-data "${WEBSERVER}" git rev-parse --is-shallow-repository | grep -q 'true'); then
             echo "== Unshallowing the repository."
             docker exec -u www-data "${WEBSERVER}" git fetch --unshallow
         fi
@@ -91,17 +91,20 @@ function moodle-core-copy_setup() {
     # Copy the config.php in place
     echo "== Copying configuration in place."
     docker cp "${BASEDIR}/modules/docker-php/config.template.php" "${WEBSERVER}":/var/www/html/config.php
+    docker exec "${WEBSERVER}" chown -R www-data:www-data /var/www/html/config.php
 
     # Copy the plugins in place.
     if [[ -n "$PLUGINSTOINSTALL" ]]; then
-      echo "== Copying external plugins in place."
-      docker cp "${PLUGINSDIR}"/. "${WEBSERVER}":/var/www/html
+        echo "== Copying external plugins in place."
+        docker cp "${PLUGINSDIR}"/. "${WEBSERVER}":/var/www/html
+        docker exec "${WEBSERVER}" chown -R www-data:www-data /var/www/html
     fi
 
     # Copy composer-phar if available in caches.
     if [[ -f "${COMPOSERCACHE}/composer.phar" ]]; then
-      echo "== Copying composer.phar in place."
-      docker cp "${COMPOSERCACHE}/composer.phar" "${WEBSERVER}":/var/www/html/composer.phar
+        echo "== Copying composer.phar in place."
+        docker cp "${COMPOSERCACHE}/composer.phar" "${WEBSERVER}":/var/www/html/composer.phar
+        docker exec "${WEBSERVER}" chown -R www-data:www-data /var/www/html/composer.phar
     fi
 
     echo "============================================================================"

test/phpunit_bisect_test.bats

@@ -0,0 +1,120 @@
+#!/usr/bin/env bash
+#
+# This file is part of the Moodle Continuous Integration Project.
+#
+# Moodle is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Moodle is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Moodle.  If not, see <https://www.gnu.org/licenses/>.
+
+# Run the tests for the phpunit job type (bisect mode)
+
+setup() {
+    load 'helpers/common'
+    _common_setup
+}
+
+teardown() {
+    load 'helpers/common'
+    _common_teardown
+}
+
+@test "PHPUnit bisect tests: run a known 4.4 regression (MDL-81386)" {
+    # Set all the required variables.
+    JOBTYPE="phpunit"
+    PHP_VERSION="8.3"
+    DBTYPE="pgsql"
+    CODEDIR="${MOODLE_CI_RUNNER_GITDIR}"
+    PHPUNIT_FILTER="test_enrol_user_sees_own_courses"
+    GOOD_COMMIT="b4c6ed36503c0d1e69efdb9b18e6846234706da7"
+    BAD_COMMIT="ecddfa6ccd8fa1390cf84a568baee78816b549aa"
+
+    # Checkout
+    run git_moodle_checkout v4.3.2
+    assert_success
+
+    # Run the job
+    run launch_runner
+    assert_success
+    assert_output --partial "== JOBTYPE: phpunit"
+    assert_output --partial "== Moodle branch (version.php): 403"
+    assert_output --partial "== PHP version: 8.3"
+    assert_output --partial "== DBTYPE: pgsql"
+    assert_output --partial "== PHPUNIT_FILTER: test_enrol_user_sees_own_courses"
+    assert_output --partial "== DBREPLICAS: 0"
+    assert_output --partial "== GOOD_COMMIT: b4c6ed36503c0d1e69efdb9b18e6846234706da7"
+    assert_output --partial "== BAD_COMMIT: ecddfa6ccd8fa1390cf84a568baee78816b549aa"
+    assert_output --partial "Setting up docker-caches module..."
+    assert_output --partial "Bisecting:"
+    assert_output --partial "52811000310e7c663fcb75d61b90756f9ded6c7a is the first bad commit"
+    assert_output --partial "MDL-67271 core: Add test to find missing SVG icons"
+    assert_output --partial "3 files changed, 70 insertions(+), 2 deletions(-)"
+    assert_output --partial "Exporting all docker logs for UUID"
+    assert_output --partial "Stopping and removing all docker containers"
+    assert_output --partial "== Exit code: 0"
+}
+
+@test "PHPUnit bisect tests: only GOOD_COMMIT specified" {
+    # Set all the required variables.
+    JOBTYPE="phpunit"
+    PHP_VERSION="8.3"
+    DBTYPE="pgsql"
+    CODEDIR="${MOODLE_CI_RUNNER_GITDIR}"
+    PHPUNIT_FILTER="test_enrol_user_sees_own_courses"
+    GOOD_COMMIT="b4c6ed36503c0d1e69efdb9b18e6846234706da7"
+
+    # Checkout
+    run git_moodle_checkout v4.3.2
+    assert_success
+
+    # Run the job
+    run launch_runner
+    assert_failure
+    assert_output --partial "ERROR: GOOD_COMMIT is set but BAD_COMMIT is not set."
+}
+
+@test "PHPUnit bisect tests: only BAD_COMMIT specified" {
+    # Set all the required variables.
+    JOBTYPE="phpunit"
+    PHP_VERSION="8.3"
+    DBTYPE="pgsql"
+    CODEDIR="${MOODLE_CI_RUNNER_GITDIR}"
+    PHPUNIT_FILTER="test_enrol_user_sees_own_courses"
+    BAD_COMMIT="b4c6ed36503c0d1e69efdb9b18e6846234706da7"
+
+    # Checkout
+    run git_moodle_checkout v4.3.2
+    assert_success
+
+    # Run the job
+    run launch_runner
+    assert_failure
+    assert_output --partial "ERROR: BAD_COMMIT is set but GOOD_COMMIT is not set."
+}
+
+@test "PHPUnit bisect tests: same GOOD and BAD commits specified" {
+    # Set all the required variables.
+    JOBTYPE="phpunit"
+    PHP_VERSION="8.3"
+    DBTYPE="pgsql"
+    CODEDIR="${MOODLE_CI_RUNNER_GITDIR}"
+    GOOD_COMMIT="b4c6ed36503c0d1e69efdb9b18e6846234706da7"
+    BAD_COMMIT="${GOOD_COMMIT}"
+
+    # Checkout
+    run git_moodle_checkout v4.3.2
+    assert_success
+
+    # Run the job
+    run launch_runner
+    assert_failure
+    assert_output --partial "ERROR: GOOD_COMMIT and BAD_COMMIT are set, but they are the same."
+}