PYTHON-4721 - Create individualized scripts for all shell.exec comman…

…ds (#1997)

Co-authored-by: Jib <[email protected]>

.evergreen/hatch.sh

@@ -29,7 +29,7 @@ else # Set up virtualenv before installing hatch
     # Ensure hatch does not write to user or global locations.
     touch hatch_config.toml
     HATCH_CONFIG=$(pwd)/hatch_config.toml
-    if [ "Windows_NT" = "$OS" ]; then # Magic variable in cygwin
+    if [ "Windows_NT" = "${OS:-}" ]; then # Magic variable in cygwin
         HATCH_CONFIG=$(cygpath -m "$HATCH_CONFIG")
     fi
     export HATCH_CONFIG

.evergreen/run-tests.sh

@@ -53,18 +53,18 @@ if [ -z "${NO_EXT:-}" ]; then
 fi
 
 if [ "$AUTH" != "noauth" ]; then
-    if [ ! -z "$TEST_DATA_LAKE" ]; then
+    if [ -n "$TEST_DATA_LAKE" ]; then
         export DB_USER="mhuser"
         export DB_PASSWORD="pencil"
-    elif [ ! -z "$TEST_SERVERLESS" ]; then
-        source ${DRIVERS_TOOLS}/.evergreen/serverless/secrets-export.sh
+    elif [ -n "$TEST_SERVERLESS" ]; then
+        source "${DRIVERS_TOOLS}"/.evergreen/serverless/secrets-export.sh
         export DB_USER=$SERVERLESS_ATLAS_USER
         export DB_PASSWORD=$SERVERLESS_ATLAS_PASSWORD
         export MONGODB_URI="$SERVERLESS_URI"
         echo "MONGODB_URI=$MONGODB_URI"
         export SINGLE_MONGOS_LB_URI=$MONGODB_URI
         export MULTI_MONGOS_LB_URI=$MONGODB_URI
-    elif [ ! -z "$TEST_AUTH_OIDC" ]; then
+    elif [ -n "$TEST_AUTH_OIDC" ]; then
         export DB_USER=$OIDC_ADMIN_USER
         export DB_PASSWORD=$OIDC_ADMIN_PWD
         export DB_IP="$MONGODB_URI"

.evergreen/scripts/archive-mongodb-logs.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set -o xtrace
+mkdir out_dir
+# shellcheck disable=SC2156
+find "$MONGO_ORCHESTRATION_HOME" -name \*.log -exec sh -c 'x="{}"; mv $x $PWD/out_dir/$(basename $(dirname $x))_$(basename $x)' \;
+tar zcvf mongodb-logs.tar.gz -C out_dir/ .
+rm -rf out_dir

.evergreen/scripts/bootstrap-mongo-orchestration.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+
+set -o xtrace
+
+# Enable core dumps if enabled on the machine
+# Copied from https://github.com/mongodb/mongo/blob/master/etc/evergreen.yml
+if [ -f /proc/self/coredump_filter ]; then
+    # Set the shell process (and its children processes) to dump ELF headers (bit 4),
+    # anonymous shared mappings (bit 1), and anonymous private mappings (bit 0).
+    echo 0x13 >/proc/self/coredump_filter
+
+    if [ -f /sbin/sysctl ]; then
+        # Check that the core pattern is set explicitly on our distro image instead
+        # of being the OS's default value. This ensures that coredump names are consistent
+        # across distros and can be picked up by Evergreen.
+        core_pattern=$(/sbin/sysctl -n "kernel.core_pattern")
+        if [ "$core_pattern" = "dump_%e.%p.core" ]; then
+            echo "Enabling coredumps"
+            ulimit -c unlimited
+        fi
+    fi
+fi
+
+if [ "$(uname -s)" = "Darwin" ]; then
+    core_pattern_mac=$(/usr/sbin/sysctl -n "kern.corefile")
+    if [ "$core_pattern_mac" = "dump_%N.%P.core" ]; then
+        echo "Enabling coredumps"
+        ulimit -c unlimited
+    fi
+fi
+
+if [ -n "${skip_crypt_shared}" ]; then
+    export SKIP_CRYPT_SHARED=1
+fi
+
+MONGODB_VERSION=${VERSION} \
+    TOPOLOGY=${TOPOLOGY} \
+    AUTH=${AUTH:-noauth} \
+    SSL=${SSL:-nossl} \
+    STORAGE_ENGINE=${STORAGE_ENGINE:-} \
+    DISABLE_TEST_COMMANDS=${DISABLE_TEST_COMMANDS:-} \
+    ORCHESTRATION_FILE=${ORCHESTRATION_FILE:-} \
+    REQUIRE_API_VERSION=${REQUIRE_API_VERSION:-} \
+    LOAD_BALANCER=${LOAD_BALANCER:-} \
+    bash ${DRIVERS_TOOLS}/.evergreen/run-orchestration.sh
+# run-orchestration generates expansion file with the MONGODB_URI for the cluster

.evergreen/scripts/check-import-time.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+. .evergreen/scripts/env.sh
+set -x
+export BASE_SHA="$1"
+export HEAD_SHA="$2"
+bash .evergreen/run-import-time-test.sh

.evergreen/scripts/cleanup.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+if [ -f "$DRIVERS_TOOLS"/.evergreen/csfle/secrets-export.sh ]; then
+  . .evergreen/hatch.sh encryption:teardown
+fi
+rm -rf "${DRIVERS_TOOLS}" || true
+rm -f ./secrets-export.sh || true

.evergreen/scripts/configure-env.sh

@@ -1,4 +1,4 @@
-#!/bin/bash -ex
+#!/bin/bash -eux
 
 # Get the current unique version of this checkout
 # shellcheck disable=SC2154
@@ -29,7 +29,7 @@ fi
 export MONGO_ORCHESTRATION_HOME="$DRIVERS_TOOLS/.evergreen/orchestration"
 export MONGODB_BINARIES="$DRIVERS_TOOLS/mongodb/bin"
 
-cat <<EOT > $SCRIPT_DIR/env.sh
+cat <<EOT > "$SCRIPT_DIR"/env.sh
 set -o errexit
 export PROJECT_DIRECTORY="$PROJECT_DIRECTORY"
 export CURRENT_VERSION="$CURRENT_VERSION"
@@ -38,6 +38,21 @@ export DRIVERS_TOOLS="$DRIVERS_TOOLS"
 export MONGO_ORCHESTRATION_HOME="$MONGO_ORCHESTRATION_HOME"
 export MONGODB_BINARIES="$MONGODB_BINARIES"
 export PROJECT_DIRECTORY="$PROJECT_DIRECTORY"
+export SETDEFAULTENCODING="${SETDEFAULTENCODING:-}"
+export SKIP_CSOT_TESTS="${SKIP_CSOT_TESTS:-}"
+export MONGODB_STARTED="${MONGODB_STARTED:-}"
+export DISABLE_TEST_COMMANDS="${DISABLE_TEST_COMMANDS:-}"
+export GREEN_FRAMEWORK="${GREEN_FRAMEWORK:-}"
+export NO_EXT="${NO_EXT:-}"
+export COVERAGE="${COVERAGE:-}"
+export COMPRESSORS="${COMPRESSORS:-}"
+export MONGODB_API_VERSION="${MONGODB_API_VERSION:-}"
+export SKIP_HATCH="${SKIP_HATCH:-}"
+export skip_crypt_shared="${skip_crypt_shared:-}"
+export STORAGE_ENGINE="${STORAGE_ENGINE:-}"
+export REQUIRE_API_VERSION="${REQUIRE_API_VERSION:-}"
+export skip_web_identity_auth_test="${skip_web_identity_auth_test:-}"
+export skip_ECS_auth_test="${skip_ECS_auth_test:-}"
 
 export TMPDIR="$MONGO_ORCHESTRATION_HOME/db"
 export PATH="$MONGODB_BINARIES:$PATH"

.evergreen/scripts/download-and-merge-coverage.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+# Download all the task coverage files.
+aws s3 cp --recursive s3://"$1"/coverage/"$2"/"$3"/coverage/ coverage/

.evergreen/scripts/fix-absolute-paths.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set +x
+. src/.evergreen/scripts/env.sh
+# shellcheck disable=SC2044
+for filename in $(find $DRIVERS_TOOLS -name \*.json); do
+    perl -p -i -e "s|ABSOLUTE_PATH_REPLACEMENT_TOKEN|$DRIVERS_TOOLS|g" $filename
+done

.evergreen/scripts/init-test-results.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+set +x
+. src/.evergreen/scripts/env.sh
+echo '{"results": [{ "status": "FAIL", "test_file": "Build", "log_raw": "No test-results.json found was created"  } ]}' >$PROJECT_DIRECTORY/test-results.json

.evergreen/scripts/install-dependencies.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+set -o xtrace
+file="$PROJECT_DIRECTORY/.evergreen/install-dependencies.sh"
+# Don't use ${file} syntax here because evergreen treats it as an empty expansion.
+[ -f "$file" ] && bash "$file" || echo "$file not available, skipping"

.evergreen/scripts/make-files-executable.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set +x
+. src/.evergreen/scripts/env.sh
+# shellcheck disable=SC2044
+for i in $(find "$DRIVERS_TOOLS"/.evergreen "$PROJECT_DIRECTORY"/.evergreen -name \*.sh); do
+    chmod +x "$i"
+done

.evergreen/scripts/prepare-resources.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+. src/.evergreen/scripts/env.sh
+set -o xtrace
+rm -rf $DRIVERS_TOOLS
+if [ "$PROJECT" = "drivers-tools" ]; then
+    # If this was a patch build, doing a fresh clone would not actually test the patch
+    cp -R $PROJECT_DIRECTORY/ $DRIVERS_TOOLS
+else
+    git clone https://github.com/mongodb-labs/drivers-evergreen-tools.git $DRIVERS_TOOLS
+fi
+echo "{ \"releases\": { \"default\": \"$MONGODB_BINARIES\" }}" >$MONGO_ORCHESTRATION_HOME/orchestration.config

.evergreen/scripts/run-atlas-tests.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+# Disable xtrace for security reasons (just in case it was accidentally set).
+set +x
+set -o errexit
+bash "${DRIVERS_TOOLS}"/.evergreen/auth_aws/setup_secrets.sh drivers/atlas_connect
+TEST_ATLAS=1 bash "${PROJECT_DIRECTORY}"/.evergreen/hatch.sh test:test-eg

.evergreen/scripts/run-aws-ecs-auth-test.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+# shellcheck disable=SC2154
+if [ "${skip_ECS_auth_test}" = "true" ]; then
+    echo "This platform does not support the ECS auth test, skipping..."
+    exit 0
+fi
+set -ex
+cd "$DRIVERS_TOOLS"/.evergreen/auth_aws
+. ./activate-authawsvenv.sh
+. aws_setup.sh ecs
+export MONGODB_BINARIES="$MONGODB_BINARIES"
+export PROJECT_DIRECTORY="$PROJECT_DIRECTORY"
+python aws_tester.py ecs
+cd -

.evergreen/scripts/run-doctests.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+set -o xtrace
+PYTHON_BINARY=${PYTHON_BINARY} bash "${PROJECT_DIRECTORY}"/.evergreen/hatch.sh doctest:test

.evergreen/scripts/run-enterprise-auth-tests.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+# Disable xtrace for security reasons (just in case it was accidentally set).
+set +x
+bash "${DRIVERS_TOOLS}"/.evergreen/auth_aws/setup_secrets.sh drivers/enterprise_auth
+TEST_ENTERPRISE_AUTH=1 AUTH=auth bash "${PROJECT_DIRECTORY}"/.evergreen/hatch.sh test:test-eg

.evergreen/scripts/run-gcpkms-fail-test.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+. .evergreen/scripts/env.sh
+export PYTHON_BINARY=/opt/mongodbtoolchain/v4/bin/python3
+export LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian11/master/latest/libmongocrypt.tar.gz
+SKIP_SERVERS=1 bash ./.evergreen/setup-encryption.sh
+SUCCESS=false TEST_FLE_GCP_AUTO=1 ./.evergreen/hatch.sh test:test-eg

.evergreen/scripts/run-getdata.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+set -o xtrace
+. ${DRIVERS_TOOLS}/.evergreen/download-mongodb.sh || true
+get_distro || true
+echo $DISTRO
+echo $MARCH
+echo $OS
+uname -a || true
+ls /etc/*release* || true
+cc --version || true
+gcc --version || true
+clang --version || true
+gcov --version || true
+lcov --version || true
+llvm-cov --version || true
+echo $PATH
+ls -la /usr/local/Cellar/llvm/*/bin/ || true
+ls -la /usr/local/Cellar/ || true
+scan-build --version || true
+genhtml --version || true
+valgrind --version || true

.evergreen/scripts/run-load-balancer.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+MONGODB_URI=${MONGODB_URI} bash "${DRIVERS_TOOLS}"/.evergreen/run-load-balancer.sh start

.evergreen/scripts/run-mockupdb-tests.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+set -o xtrace
+export PYTHON_BINARY=${PYTHON_BINARY}
+bash "${PROJECT_DIRECTORY}"/.evergreen/hatch.sh test:test-mockupdb

.evergreen/run-mod-wsgi-tests.sh → .evergreen/scripts/run-mod-wsgi-tests.sh

@@ -28,7 +28,7 @@ export MOD_WSGI_SO=/opt/python/mod_wsgi/python_version/$PYTHON_VERSION/mod_wsgi_
 export PYTHONHOME=/opt/python/$PYTHON_VERSION
 # If MOD_WSGI_EMBEDDED is set use the default embedded mode behavior instead
 # of daemon mode (WSGIDaemonProcess).
-if [ -n "$MOD_WSGI_EMBEDDED" ]; then
+if [ -n "${MOD_WSGI_EMBEDDED:-}" ]; then
     export MOD_WSGI_CONF=mod_wsgi_test_embedded.conf
 else
     export MOD_WSGI_CONF=mod_wsgi_test.conf

.evergreen/run-mongodb-aws-test.sh → .evergreen/scripts/run-mongodb-aws-test.sh

@@ -13,10 +13,16 @@ set -o errexit  # Exit the script with error if any of the commands fail
 #                 mechanism.
 #  PYTHON_BINARY  The Python version to use.
 
-echo "Running MONGODB-AWS authentication tests"
+# shellcheck disable=SC2154
+if [ "${skip_EC2_auth_test:-}" = "true" ] && { [ "$1" = "ec2" ] || [ "$1" = "web-identity" ]; }; then
+   echo "This platform does not support the EC2 auth test, skipping..."
+   exit 0
+fi
+
+echo "Running MONGODB-AWS authentication tests for $1"
 
 # Handle credentials and environment setup.
-. $DRIVERS_TOOLS/.evergreen/auth_aws/aws_setup.sh $1
+. "$DRIVERS_TOOLS"/.evergreen/auth_aws/aws_setup.sh "$1"
 
 # show test output
 set -x

.evergreen/scripts/run-ocsp-test.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+TEST_OCSP=1 \
+PYTHON_BINARY="${PYTHON_BINARY}" \
+CA_FILE="${DRIVERS_TOOLS}/.evergreen/ocsp/${OCSP_ALGORITHM}/ca.pem" \
+OCSP_TLS_SHOULD_SUCCEED="${OCSP_TLS_SHOULD_SUCCEED}" \
+bash "${PROJECT_DIRECTORY}"/.evergreen/hatch.sh test:test-eg
+bash "${DRIVERS_TOOLS}"/.evergreen/ocsp/teardown.sh

.evergreen/scripts/run-perf-tests.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+PROJECT_DIRECTORY=${PROJECT_DIRECTORY}
+bash "${PROJECT_DIRECTORY}"/.evergreen/run-perf-tests.sh

.evergreen/scripts/run-tests.sh

@@ -0,0 +1,55 @@
+#!/bin/bash
+
+# Disable xtrace
+set +x
+if [ -n "${MONGODB_STARTED}" ]; then
+  export PYMONGO_MUST_CONNECT=true
+fi
+if [ -n "${DISABLE_TEST_COMMANDS}" ]; then
+  export PYMONGO_DISABLE_TEST_COMMANDS=1
+fi
+if [ -n "${test_encryption}" ]; then
+  # Disable xtrace (just in case it was accidentally set).
+  set +x
+  bash "${DRIVERS_TOOLS}"/.evergreen/csfle/await-servers.sh
+  export TEST_ENCRYPTION=1
+  if [ -n "${test_encryption_pyopenssl}" ]; then
+    export TEST_ENCRYPTION_PYOPENSSL=1
+  fi
+fi
+if [ -n "${test_crypt_shared}" ]; then
+  export TEST_CRYPT_SHARED=1
+  export CRYPT_SHARED_LIB_PATH=${CRYPT_SHARED_LIB_PATH}
+fi
+if [ -n "${test_pyopenssl}" ]; then
+  export TEST_PYOPENSSL=1
+fi
+if [ -n "${SETDEFAULTENCODING}" ]; then
+  export SETDEFAULTENCODING="${SETDEFAULTENCODING}"
+fi
+if [ -n "${test_loadbalancer}" ]; then
+  export TEST_LOADBALANCER=1
+  export SINGLE_MONGOS_LB_URI="${SINGLE_MONGOS_LB_URI}"
+  export MULTI_MONGOS_LB_URI="${MULTI_MONGOS_LB_URI}"
+fi
+if [ -n "${test_serverless}" ]; then
+  export TEST_SERVERLESS=1
+fi
+if [ -n "${TEST_INDEX_MANAGEMENT:-}" ]; then
+  export TEST_INDEX_MANAGEMENT=1
+fi
+if [ -n "${SKIP_CSOT_TESTS}" ]; then
+  export SKIP_CSOT_TESTS=1
+fi
+GREEN_FRAMEWORK=${GREEN_FRAMEWORK} \
+  PYTHON_BINARY=${PYTHON_BINARY} \
+  NO_EXT=${NO_EXT} \
+  COVERAGE=${COVERAGE} \
+  COMPRESSORS=${COMPRESSORS} \
+  AUTH=${AUTH} \
+  SSL=${SSL} \
+  TEST_DATA_LAKE=${TEST_DATA_LAKE:-} \
+  TEST_SUITES=${TEST_SUITES:-} \
+  MONGODB_API_VERSION=${MONGODB_API_VERSION} \
+  SKIP_HATCH=${SKIP_HATCH} \
+  bash "${PROJECT_DIRECTORY}"/.evergreen/hatch.sh test:test-eg

.evergreen/scripts/run-with-env.sh

@@ -0,0 +1,21 @@
+#!/bin/bash -eu
+
+# Example use: bash run-with-env.sh run-tests.sh {args...}
+
+# Parameter expansion to get just the current directory's name
+if [ "${PWD##*/}" == "src" ]; then
+  . .evergreen/scripts/env.sh
+  if [ -f ".evergreen/scripts/test-env.sh" ]; then
+    . .evergreen/scripts/test-env.sh
+  fi
+else
+  . src/.evergreen/scripts/env.sh
+  if [ -f "src/.evergreen/scripts/test-env.sh" ]; then
+    . src/.evergreen/scripts/test-env.sh
+  fi
+fi
+
+set -eu
+
+# shellcheck source=/dev/null
+. "$@"

.evergreen/scripts/setup-encryption.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+if [ -n "${test_encryption}" ]; then
+  ./.evergreen/hatch.sh encryption:setup
+fi