diff --git a/runner/main/jobtypes/phpunit/phpunit.sh b/runner/main/jobtypes/phpunit/phpunit.sh index ac24bc8..1d82b6e 100644 --- a/runner/main/jobtypes/phpunit/phpunit.sh +++ b/runner/main/jobtypes/phpunit/phpunit.sh @@ -62,6 +62,10 @@ function phpunit_to_summary() { echo "== PHPUNIT_FILTER: ${PHPUNIT_FILTER}" echo "== PHPUNIT_TESTSUITE: ${PHPUNIT_TESTSUITE}" echo "== MOODLE_CONFIG: ${MOODLE_CONFIG}" + if [[ -n "${GOOD_COMMIT}" ]] || [[ -n "${BAD_COMMIT}" ]]; then + echo "== GOOD_COMMIT: ${GOOD_COMMIT}" + echo "== BAD_COMMIT: ${BAD_COMMIT}" + fi } # This job type defines the following env variables @@ -106,7 +110,7 @@ function phpunit_check() { verify_modules $(phpunit_modules) # These env variables must be set for the job to work. - verify_env UUID ENVIROPATH WEBSERVER + verify_env UUID ENVIROPATH WEBSERVER GOOD_COMMIT BAD_COMMIT } # PHPUnit job type init. @@ -116,23 +120,72 @@ function phpunit_config() { PHPUNIT_FILTER="${PHPUNIT_FILTER:-}" PHPUNIT_TESTSUITE="${PHPUNIT_TESTSUITE:-}" EXITCODE=0 + + # If GOOD_COMMIT and BAD_COMMIT are set, it means that we are going to run a bisect + # session, so we need to enable FULLGIT (to get access to complete repository clone). + if [[ -n "${GOOD_COMMIT}" ]] && [[ -n "${BAD_COMMIT}" ]]; then + FULLGIT="yes" + # Also, we don't want to allow repetitions in the bisect session. + RUNCOUNT=1 + fi } # PHPUnit job type setup. function phpunit_setup() { + # If one of GOOD_COMMIT or BAD_COMMIT are not set, but the other is, error out. + if [[ -n "${GOOD_COMMIT}" ]] && [[ -z "${BAD_COMMIT}" ]]; then + exit_error "GOOD_COMMIT is set but BAD_COMMIT is not set." + fi + if [[ -z "${GOOD_COMMIT}" ]] && [[ -n "${BAD_COMMIT}" ]]; then + exit_error "BAD_COMMIT is set but GOOD_COMMIT is not set." + fi + # If both GOOD_COMMIT and BAD_COMMIT are set and they are the same, error out. + if [[ -n "${GOOD_COMMIT}" ]] && [[ -n "${BAD_COMMIT}" ]] && [[ "${GOOD_COMMIT}" == "${BAD_COMMIT}" ]]; then + exit_error "GOOD_COMMIT and BAD_COMMIT are set, but they are the same." + fi + + # If both GOOD_COMMIT and BAD_COMMIT are not set, we are going to run a normal session. + # (for bisect sessions we don't have to setup the environment). + if [[ -z "${GOOD_COMMIT}" ]] && [[ -z "${BAD_COMMIT}" ]]; then + phpunit_setup_normal + fi +} + +# PHPUnit job type setup for normal mode. +function phpunit_setup_normal() { # Init the PHPUnit site. echo echo ">>> startsection Initialising PHPUnit environment at $(date)<<<" echo "============================================================================" - docker exec -t -u www-data "${WEBSERVER}" \ - php admin/tool/phpunit/cli/init.php \ - --force + local initcmd + phpunit_initcmd initcmd # By nameref. + docker exec -t -u www-data "${WEBSERVER}" "${initcmd[@]}" echo "============================================================================" echo ">>> stopsection <<<" } +# Returns (by nameref) an array with the command needed to init the PHPUnit site. +function phpunit_initcmd() { + local -n cmd=$1 + cmd=( + php + admin/tool/phpunit/cli/init.php + ) +} + # PHPUnit job type run. function phpunit_run() { + # If both GOOD_COMMIT and BAD_COMMIT are not set, we are going to run a normal session. + if [[ -z "${GOOD_COMMIT}" ]] && [[ -z "${BAD_COMMIT}" ]]; then + phpunit_run_normal + else + # If GOOD_COMMIT and BAD_COMMIT are set, we are going to run a bisect session. + phpunit_run_bisect + fi +} + +# PHPUnit job tye run for normal mode. +function phpunit_run_normal() { # Run the job type. echo if [[ RUNCOUNT -gt 1 ]]; then @@ -142,32 +195,85 @@ function phpunit_run() { fi echo "============================================================================" # Build the complete command - local cmd=( - php vendor/bin/phpunit - --disallow-test-output - --fail-on-risky - --log-junit /shared/log.junit - --verbose - ) - if [[ -n "${PHPUNIT_FILTER}" ]]; then - cmd+=(--filter "${PHPUNIT_FILTER}") - fi - if [[ -n "${PHPUNIT_TESTSUITE}" ]]; then - cmd+=(--testsuite "${PHPUNIT_TESTSUITE}") - fi + local runcmd + phpunit_runcmd runcmd # By nameref. - echo "Running: ${cmd[*]}" + echo "Running: ${runcmd[*]}" # Run the command RUNCOUNT times. local iter=1 while [[ ${iter} -le ${RUNCOUNT} ]]; do echo echo ">>> PHPUnit run ${iter} at $(date) <<<" - docker exec -t "${WEBSERVER}" "${cmd[@]}" + docker exec -t -u www-data "${WEBSERVER}" "${runcmd[@]}" EXITCODE=$((EXITCODE + $?)) iter=$((iter+1)) done echo "============================================================================" echo ">>> stopsection <<<" -} \ No newline at end of file +} + +# PHPUnit job tye run for bisect mode. +function phpunit_run_bisect() { + # Run the job type. + echo + echo ">>> startsection Starting PHPUnit bisect session at $(date) <<<" + echo "=== Good commit: ${GOOD_COMMIT}" + echo "=== Bad commit: ${BAD_COMMIT}" + echo "============================================================================" + # Start the bisect session. + docker exec -t -u www-data "${WEBSERVER}" \ + git bisect start "${BAD_COMMIT}" "${GOOD_COMMIT}" + + # Build the int command. + local initcmd + phpunit_initcmd initcmd # By nameref. + + # Build the run command. + local runcmd + phpunit_runcmd runcmd # By nameref. + + # Generate the bisect.sh script that we are going to use to run the phpunit bisect session. + # (it runs both init and run commands together). + docker exec -i -u www-data "${WEBSERVER}" \ + bash -c "cat > bisect.sh" <<- EOF + #!/bin/bash + ${initcmd[@]} >/dev/null 2>&1; ${runcmd[@]} + exitcode=\$? + echo "============================================================================" + exit \$exitcode + EOF + + # Run the bisect session. + echo "============================================================================" + docker exec -u www-data "${WEBSERVER}" \ + git bisect run bash bisect.sh + EXITCODE=$? + + # Finish the bisect session. + docker exec -u www-data "${WEBSERVER}" \ + git bisect reset + + echo "============================================================================" + echo ">>> stopsection <<<" +} + +# Returns (by nameref) an array with the command needed to run the PHPUnit tests. +function phpunit_runcmd() { + local -n cmd=$1 + cmd=( + php + vendor/bin/phpunit + --disallow-test-output + --fail-on-risky + --log-junit /shared/log.junit + --verbose + ) + if [[ -n "${PHPUNIT_FILTER}" ]]; then + cmd+=(--filter "${PHPUNIT_FILTER}") + fi + if [[ -n "${PHPUNIT_TESTSUITE}" ]]; then + cmd+=(--testsuite "${PHPUNIT_TESTSUITE}") + fi +} diff --git a/runner/main/modules/docker-php/docker-php.sh b/runner/main/modules/docker-php/docker-php.sh index 75e3d52..d438b79 100644 --- a/runner/main/modules/docker-php/docker-php.sh +++ b/runner/main/modules/docker-php/docker-php.sh @@ -58,6 +58,9 @@ function docker-php_setup() { -v "${SHAREDDIR}":/shared \ "${DOCKER_PHP}" + # Ensure that the whole .composer directory is writable to all (www-data needs to write there). + docker exec "${WEBSERVER}" chmod -R go+rw /var/www/.composer + echo echo "Webserver logs:" docker logs "${WEBSERVER}" diff --git a/runner/main/modules/git/git.sh b/runner/main/modules/git/git.sh index a2495eb..7ffeadf 100644 --- a/runner/main/modules/git/git.sh +++ b/runner/main/modules/git/git.sh @@ -21,6 +21,8 @@ function git_env() { env=( GIT_COMMIT + GOOD_COMMIT + BAD_COMMIT ) echo "${env[@]}" } diff --git a/runner/main/modules/moodle-core-copy/moodle-core-copy.sh b/runner/main/modules/moodle-core-copy/moodle-core-copy.sh index 1b774e3..c709637 100644 --- a/runner/main/modules/moodle-core-copy/moodle-core-copy.sh +++ b/runner/main/modules/moodle-core-copy/moodle-core-copy.sh @@ -74,7 +74,7 @@ function moodle-core-copy_setup() { fi # If the repository was cloned shallow (--depth), un-shallow it. - if (docker exec -u www-data "${WEBSERVER}" git rev-parse --is-shallow-repository); then + if (docker exec -u www-data "${WEBSERVER}" git rev-parse --is-shallow-repository | grep -q 'true'); then echo "== Unshallowing the repository." docker exec -u www-data "${WEBSERVER}" git fetch --unshallow fi @@ -91,17 +91,20 @@ function moodle-core-copy_setup() { # Copy the config.php in place echo "== Copying configuration in place." docker cp "${BASEDIR}/modules/docker-php/config.template.php" "${WEBSERVER}":/var/www/html/config.php + docker exec "${WEBSERVER}" chown -R www-data:www-data /var/www/html/config.php # Copy the plugins in place. if [[ -n "$PLUGINSTOINSTALL" ]]; then - echo "== Copying external plugins in place." - docker cp "${PLUGINSDIR}"/. "${WEBSERVER}":/var/www/html + echo "== Copying external plugins in place." + docker cp "${PLUGINSDIR}"/. "${WEBSERVER}":/var/www/html + docker exec "${WEBSERVER}" chown -R www-data:www-data /var/www/html fi # Copy composer-phar if available in caches. if [[ -f "${COMPOSERCACHE}/composer.phar" ]]; then - echo "== Copying composer.phar in place." - docker cp "${COMPOSERCACHE}/composer.phar" "${WEBSERVER}":/var/www/html/composer.phar + echo "== Copying composer.phar in place." + docker cp "${COMPOSERCACHE}/composer.phar" "${WEBSERVER}":/var/www/html/composer.phar + docker exec "${WEBSERVER}" chown -R www-data:www-data /var/www/html/composer.phar fi echo "============================================================================" diff --git a/test/phpunit_bisect_test.bats b/test/phpunit_bisect_test.bats new file mode 100644 index 0000000..2f81e60 --- /dev/null +++ b/test/phpunit_bisect_test.bats @@ -0,0 +1,120 @@ +#!/usr/bin/env bash +# +# This file is part of the Moodle Continuous Integration Project. +# +# Moodle is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Moodle is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Moodle. If not, see . + +# Run the tests for the phpunit job type (bisect mode) + +setup() { + load 'helpers/common' + _common_setup +} + +teardown() { + load 'helpers/common' + _common_teardown +} + +@test "PHPUnit bisect tests: run a known 4.4 regression (MDL-81386)" { + # Set all the required variables. + JOBTYPE="phpunit" + PHP_VERSION="8.3" + DBTYPE="pgsql" + CODEDIR="${MOODLE_CI_RUNNER_GITDIR}" + PHPUNIT_FILTER="test_enrol_user_sees_own_courses" + GOOD_COMMIT="b4c6ed36503c0d1e69efdb9b18e6846234706da7" + BAD_COMMIT="ecddfa6ccd8fa1390cf84a568baee78816b549aa" + + # Checkout + run git_moodle_checkout v4.3.2 + assert_success + + # Run the job + run launch_runner + assert_success + assert_output --partial "== JOBTYPE: phpunit" + assert_output --partial "== Moodle branch (version.php): 403" + assert_output --partial "== PHP version: 8.3" + assert_output --partial "== DBTYPE: pgsql" + assert_output --partial "== PHPUNIT_FILTER: test_enrol_user_sees_own_courses" + assert_output --partial "== DBREPLICAS: 0" + assert_output --partial "== GOOD_COMMIT: b4c6ed36503c0d1e69efdb9b18e6846234706da7" + assert_output --partial "== BAD_COMMIT: ecddfa6ccd8fa1390cf84a568baee78816b549aa" + assert_output --partial "Setting up docker-caches module..." + assert_output --partial "Bisecting:" + assert_output --partial "52811000310e7c663fcb75d61b90756f9ded6c7a is the first bad commit" + assert_output --partial "MDL-67271 core: Add test to find missing SVG icons" + assert_output --partial "3 files changed, 70 insertions(+), 2 deletions(-)" + assert_output --partial "Exporting all docker logs for UUID" + assert_output --partial "Stopping and removing all docker containers" + assert_output --partial "== Exit code: 0" +} + +@test "PHPUnit bisect tests: only GOOD_COMMIT specified" { + # Set all the required variables. + JOBTYPE="phpunit" + PHP_VERSION="8.3" + DBTYPE="pgsql" + CODEDIR="${MOODLE_CI_RUNNER_GITDIR}" + PHPUNIT_FILTER="test_enrol_user_sees_own_courses" + GOOD_COMMIT="b4c6ed36503c0d1e69efdb9b18e6846234706da7" + + # Checkout + run git_moodle_checkout v4.3.2 + assert_success + + # Run the job + run launch_runner + assert_failure + assert_output --partial "ERROR: GOOD_COMMIT is set but BAD_COMMIT is not set." +} + +@test "PHPUnit bisect tests: only BAD_COMMIT specified" { + # Set all the required variables. + JOBTYPE="phpunit" + PHP_VERSION="8.3" + DBTYPE="pgsql" + CODEDIR="${MOODLE_CI_RUNNER_GITDIR}" + PHPUNIT_FILTER="test_enrol_user_sees_own_courses" + BAD_COMMIT="b4c6ed36503c0d1e69efdb9b18e6846234706da7" + + # Checkout + run git_moodle_checkout v4.3.2 + assert_success + + # Run the job + run launch_runner + assert_failure + assert_output --partial "ERROR: BAD_COMMIT is set but GOOD_COMMIT is not set." +} + +@test "PHPUnit bisect tests: same GOOD and BAD commits specified" { + # Set all the required variables. + JOBTYPE="phpunit" + PHP_VERSION="8.3" + DBTYPE="pgsql" + CODEDIR="${MOODLE_CI_RUNNER_GITDIR}" + GOOD_COMMIT="b4c6ed36503c0d1e69efdb9b18e6846234706da7" + BAD_COMMIT="${GOOD_COMMIT}" + + # Checkout + run git_moodle_checkout v4.3.2 + assert_success + + # Run the job + run launch_runner + assert_failure + assert_output --partial "ERROR: GOOD_COMMIT and BAD_COMMIT are set, but they are the same." +}