From 4d2deba8db5f144c3ada733d31e4e126bfad9b17 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 23 Feb 2024 05:47:08 -0600 Subject: [PATCH] PYTHON-4210 Migrate Remaining Secrets to AWS Vaults (#1517) --- .evergreen/config.yml | 325 ++++++++------------------- .evergreen/run-azurekms-fail-test.sh | 10 + .evergreen/run-azurekms-test.sh | 21 ++ .evergreen/run-gcpkms-test.sh | 18 ++ .evergreen/run-mongodb-oidc-test.sh | 17 +- .evergreen/run-tests.sh | 10 +- tox.ini | 6 +- 7 files changed, 166 insertions(+), 241 deletions(-) create mode 100644 .evergreen/run-azurekms-fail-test.sh create mode 100644 .evergreen/run-azurekms-test.sh create mode 100644 .evergreen/run-gcpkms-test.sh diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 16ab9e2579..ddb25ab3fa 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -416,13 +416,9 @@ functions: fi if [ -n "${test_serverless}" ]; then export TEST_SERVERLESS=1 - export MONGODB_URI="${SERVERLESS_URI}" - export SINGLE_MONGOS_LB_URI="${MONGODB_URI}" - export MULTI_MONGOS_LB_URI="${MONGODB_URI}" fi if [ -n "${TEST_INDEX_MANAGEMENT}" ]; then export TEST_INDEX_MANAGEMENT=1 - export MONGODB_URI="${TEST_INDEX_URI}" fi GREEN_FRAMEWORK=${GREEN_FRAMEWORK} \ @@ -437,9 +433,6 @@ functions: bash ${PROJECT_DIRECTORY}/.evergreen/tox.sh -m test-eg "run enterprise auth tests": - - command: ec2.assume_role - params: - role_arn: ${aws_test_secrets_role} - command: shell.exec type: test params: @@ -456,9 +449,6 @@ functions: bash ${PROJECT_DIRECTORY}/.evergreen/tox.sh -m test-eg "run atlas tests": - - command: ec2.assume_role - params: - role_arn: ${aws_test_secrets_role} - command: shell.exec type: test params: @@ -475,18 +465,13 @@ functions: bash ${PROJECT_DIRECTORY}/.evergreen/tox.sh -m test-eg "get aws auth secrets": - - command: ec2.assume_role - params: - role_arn: ${aws_test_secrets_role} - - command: shell.exec + - command: subprocess.exec type: test params: - add_expansions_to_env: true - working_dir: "src" - script: | - ${PREPARE_SHELL} - cd $DRIVERS_TOOLS/.evergreen/auth_aws - ./setup_secrets.sh drivers/aws_auth + include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] + binary: bash + args: + - ${DRIVERS_TOOLS}/.evergreen/auth_aws/setup-secrets.sh "run aws auth test with regular aws credentials": - command: shell.exec @@ -540,18 +525,14 @@ functions: .evergreen/run-mongodb-aws-test.sh web-identity "run oidc auth test with aws credentials": - - command: ec2.assume_role - params: - role_arn: ${aws_test_secrets_role} - - command: shell.exec + - command: subprocess.exec type: test params: working_dir: "src" - shell: bash - include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] - script: | - ${PREPARE_SHELL} - bash .evergreen/run-mongodb-oidc-test.sh + binary: bash + include_expansions_in_env: ["DRIVERS_TOOLS", "AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"] + args: + - .evergreen/run-mongodb-oidc-test.sh "run aws auth test with aws credentials as environment variables": - command: shell.exec @@ -661,6 +642,20 @@ functions: params: role_arn: ${aws_test_secrets_role} + "setup atlas": + - command: subprocess.exec + params: + binary: bash + include_expansions_in_env: ["task_id", "execution"] + env: + MONGODB_VERSION: "7.0" + LAMBDA_STACK_NAME: dbx-python-lambda + args: + - ${DRIVERS_TOOLS}/.evergreen/atlas/setup-atlas-cluster.sh + - command: expansions.update + params: + file: atlas-expansion.yml + "run-ocsp-test": - command: shell.exec type: test @@ -769,6 +764,13 @@ functions: python ./lib/aws_assign_instance_profile.py fi + "teardown atlas": + - command: subprocess.exec + params: + binary: bash + args: + - ${DRIVERS_TOOLS}/.evergreen/atlas/teardown-atlas-cluster.sh + "build release": - command: shell.exec type: test @@ -896,25 +898,19 @@ task_groups: setup_group: - func: "fetch source" - func: "prepare resources" - - command: shell.exec - params: - shell: "bash" - working_dir: src - script: | - ${PREPARE_SHELL} - bash ${DRIVERS_TOOLS}/.evergreen/serverless/setup-secrets.sh ${VAULT_NAME} - cp ${DRIVERS_TOOLS}/.evergreen/serverless/secrets-export.sh . - bash ${DRIVERS_TOOLS}/.evergreen/serverless/create-instance.sh - - command: expansions.update + - command: subprocess.exec params: - file: src/serverless-expansion.yml + binary: bash + env: + VAULT_NAME: ${VAULT_NAME} + args: + - ${DRIVERS_TOOLS}/.evergreen/serverless/create-instance.sh teardown_task: - - command: shell.exec - working_dir: src + - command: subprocess.exec params: - script: | - ${PREPARE_SHELL} - bash ${DRIVERS_TOOLS}/.evergreen/serverless/delete-instance.sh ${VAULT_NAME} + binary: bash + args: + - ${DRIVERS_TOOLS}/.evergreen/serverless/delete-instance.sh - func: "upload test results" tasks: - ".serverless" @@ -927,32 +923,17 @@ task_groups: - func: prepare resources - func: fix absolute paths - func: make files executable - - command: shell.exec - params: - shell: "bash" - script: | - ${PREPARE_SHELL} - echo '${testgcpkms_key_file}' > /tmp/testgcpkms_key_file.json - export GCPKMS_KEYFILE=/tmp/testgcpkms_key_file.json - export GCPKMS_DRIVERS_TOOLS=$DRIVERS_TOOLS - export GCPKMS_SERVICEACCOUNT="${testgcpkms_service_account}" - export GCPKMS_MACHINETYPE="e2-standard-4" - $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/create-and-setup-instance.sh - # Load the GCPKMS_GCLOUD, GCPKMS_INSTANCE, GCPKMS_REGION, and GCPKMS_ZONE expansions. - - command: expansions.update + - command: subprocess.exec params: - file: testgcpkms-expansions.yml + binary: "bash" + args: + - ${DRIVERS_TOOLS}/.evergreen/csfle/gcpkms/create-and-setup-instance.sh teardown_task: - - command: shell.exec + - command: subprocess.exec params: - shell: "bash" - script: | - ${PREPARE_SHELL} - export GCPKMS_GCLOUD=${GCPKMS_GCLOUD} - export GCPKMS_PROJECT=${GCPKMS_PROJECT} - export GCPKMS_ZONE=${GCPKMS_ZONE} - export GCPKMS_INSTANCENAME=${GCPKMS_INSTANCENAME} - $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/delete-instance.sh + binary: "bash" + args: + - ${DRIVERS_TOOLS}/.evergreen/csfle/gcpkms/delete-instance.sh - func: "upload test results" tasks: - testgcpkms-task @@ -963,35 +944,19 @@ task_groups: - func: prepare resources - func: fix absolute paths - func: make files executable - - command: shell.exec - params: - shell: bash - script: |- - ${PREPARE_SHELL} - # Get azurekms credentials from the vault. - bash $DRIVERS_TOOLS/.evergreen/auth_aws/setup_secrets.sh drivers/azurekms - source ./secrets-export.sh - export AZUREKMS_VMNAME_PREFIX="PYTHON_DRIVER" - export AZUREKMS_DRIVERS_TOOLS="$DRIVERS_TOOLS" - $DRIVERS_TOOLS/.evergreen/csfle/azurekms/create-and-setup-vm.sh - - command: expansions.update + - command: subprocess.exec params: - file: testazurekms-expansions.yml + binary: bash + env: + AZUREKMS_VMNAME_PREFIX: "PYTHON_DRIVER" + args: + - ${DRIVERS_TOOLS}/.evergreen/csfle/azurekms/create-and-setup-vm.sh teardown_group: - # Load expansions again. The setup task may have failed before running `expansions.update`. - - command: expansions.update - params: - file: testazurekms-expansions.yml - - command: shell.exec + - command: subprocess.exec params: - shell: bash - script: |- - ${PREPARE_SHELL} - set -x - export AZUREKMS_VMNAME=${AZUREKMS_VMNAME} - export AZUREKMS_RESOURCEGROUP=${AZUREKMS_RESOURCEGROUP} - export AZUREKMS_SCOPE=${AZUREKMS_SCOPE} - $DRIVERS_TOOLS/.evergreen/csfle/azurekms/delete-vm.sh + binary: bash + args: + - ${DRIVERS_TOOLS}/.evergreen/csfle/azurekms/delete-vm.sh - func: "upload test results" setup_group_can_fail_task: true teardown_group_can_fail_task: true @@ -1005,21 +970,19 @@ task_groups: - func: prepare resources - func: fix absolute paths - func: make files executable - - command: shell.exec + - command: subprocess.exec params: - shell: bash - script: |- - set -o errexit - ${PREPARE_SHELL} - export AZUREOIDC_VMNAME_PREFIX="PYTHON_DRIVER" - $DRIVERS_TOOLS/.evergreen/auth_oidc/azure/create-and-setup-vm.sh + binary: bash + env: + AZUREOIDC_VMNAME_PREFIX: "PYTHON_DRIVER" + args: + - ${DRIVERS_TOOLS}/.evergreen/auth_oidc/azure/create-and-setup-vm.sh teardown_task: - - command: shell.exec + - command: subprocess.exec params: - shell: bash - script: |- - ${PREPARE_SHELL} - $DRIVERS_TOOLS/.evergreen/auth_oidc/azure/delete-vm.sh + binary: bash + args: + - ${DRIVERS_TOOLS}/.evergreen/auth_oidc/azure/delete-vm.sh setup_group_can_fail_task: true setup_group_timeout_secs: 1800 tasks: @@ -1029,24 +992,9 @@ task_groups: setup_group: - func: fetch source - func: prepare resources - - command: subprocess.exec - params: - working_dir: src - binary: bash - add_expansions_to_env: true - args: - - ${DRIVERS_TOOLS}/.evergreen/atlas/setup-atlas-cluster.sh - - command: expansions.update - params: - file: src/atlas-expansion.yml + - func: setup atlas teardown_task: - - command: subprocess.exec - params: - working_dir: src - binary: bash - add_expansions_to_env: true - args: - - ${DRIVERS_TOOLS}/.evergreen/atlas/teardown-atlas-cluster.sh + - func: teardown atlas setup_group_can_fail_task: true setup_group_timeout_secs: 1800 tasks: @@ -1058,35 +1006,9 @@ task_groups: - func: prepare resources - func: fix absolute paths - func: make files executable - - command: subprocess.exec - params: - working_dir: src - binary: bash - add_expansions_to_env: true - env: - MONGODB_VERSION: "7.0" - args: - - ${DRIVERS_TOOLS}/.evergreen/atlas/setup-atlas-cluster.sh - - command: expansions.update - params: - file: src/atlas-expansion.yml - - command: shell.exec - params: - working_dir: src - shell: bash - script: |- - echo "TEST_INDEX_URI: ${MONGODB_URI}" > atlas-expansion.yml - - command: expansions.update - params: - file: src/atlas-expansion.yml + - func: setup atlas teardown_task: - - command: subprocess.exec - params: - working_dir: src - binary: bash - add_expansions_to_env: true - args: - - ${DRIVERS_TOOLS}/.evergreen/atlas/teardown-atlas-cluster.sh + - func: teardown atlas setup_group_can_fail_task: true setup_group_timeout_secs: 1800 tasks: @@ -1468,6 +1390,7 @@ tasks: vars: VERSION: "latest" TOPOLOGY: "server" + - func: "assume ec2 role" - func: "run enterprise auth tests" - name: "test-search-index-helpers" @@ -1528,6 +1451,7 @@ tasks: - name: "atlas-connect" tags: ["atlas-connect"] commands: + - func: "assume ec2 role" - func: "run atlas tests" - name: atlas-data-lake-tests @@ -1886,6 +1810,7 @@ tasks: ORCHESTRATION_FILE: "auth-aws.json" TOPOLOGY: "server" VERSION: "4.4" + - func: "assume ec2 role" - func: "get aws auth secrets" - func: "run aws auth test with regular aws credentials" - func: "run aws auth test with assume role credentials" @@ -1903,6 +1828,7 @@ tasks: ORCHESTRATION_FILE: "auth-aws.json" TOPOLOGY: "server" VERSION: "5.0" + - func: "assume ec2 role" - func: "get aws auth secrets" - func: "run aws auth test with regular aws credentials" - func: "run aws auth test with assume role credentials" @@ -1920,6 +1846,7 @@ tasks: ORCHESTRATION_FILE: "auth-aws.json" TOPOLOGY: "server" VERSION: "6.0" + - func: "assume ec2 role" - func: "get aws auth secrets" - func: "run aws auth test with regular aws credentials" - func: "run aws auth test with assume role credentials" @@ -1937,6 +1864,7 @@ tasks: ORCHESTRATION_FILE: "auth-aws.json" TOPOLOGY: "server" VERSION: "7.0" + - func: "assume ec2 role" - func: "get aws auth secrets" - func: "run aws auth test with regular aws credentials" - func: "run aws auth test with assume role credentials" @@ -1954,6 +1882,7 @@ tasks: ORCHESTRATION_FILE: "auth-aws.json" TOPOLOGY: "server" VERSION: "rapid" + - func: "assume ec2 role" - func: "get aws auth secrets" - func: "run aws auth test with regular aws credentials" - func: "run aws auth test with assume role credentials" @@ -1971,6 +1900,7 @@ tasks: ORCHESTRATION_FILE: "auth-aws.json" TOPOLOGY: "server" VERSION: "latest" + - func: "assume ec2 role" - func: "get aws auth secrets" - func: "run aws auth test with regular aws credentials" - func: "run aws auth test with assume role credentials" @@ -1982,6 +1912,7 @@ tasks: - name: "oidc-auth-test-latest" commands: + - func: "assume ec2 role" - func: "run oidc auth test with aws credentials" - name: load-balancer-test @@ -2042,36 +1973,14 @@ tasks: - name: "testgcpkms-task" commands: - - command: shell.exec + - command: subprocess.exec type: setup params: working_dir: "src" - shell: "bash" - script: | - ${PREPARE_SHELL} - echo "Copying files ... begin" - export GCPKMS_GCLOUD=${GCPKMS_GCLOUD} - export GCPKMS_PROJECT=${GCPKMS_PROJECT} - export GCPKMS_ZONE=${GCPKMS_ZONE} - export GCPKMS_INSTANCENAME=${GCPKMS_INSTANCENAME} - tar czf /tmp/mongo-python-driver.tgz . - GCPKMS_SRC=/tmp/mongo-python-driver.tgz GCPKMS_DST=$GCPKMS_INSTANCENAME: $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/copy-file.sh - echo "Copying files ... end" - echo "Untarring file ... begin" - GCPKMS_CMD="tar xf mongo-python-driver.tgz" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh - echo "Untarring file ... end" - - command: shell.exec - type: test - params: - working_dir: "src" - shell: "bash" - script: | - ${PREPARE_SHELL} - export GCPKMS_GCLOUD=${GCPKMS_GCLOUD} - export GCPKMS_PROJECT=${GCPKMS_PROJECT} - export GCPKMS_ZONE=${GCPKMS_ZONE} - export GCPKMS_INSTANCENAME=${GCPKMS_INSTANCENAME} - GCPKMS_CMD="SUCCESS=true TEST_FLE_GCP_AUTO=1 LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian10/master/latest/libmongocrypt.tar.gz ./.evergreen/tox.sh -m test-eg" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh + binary: "bash" + include_expansions_in_env: ["DRIVERS_TOOLS"] + args: + - .evergreen/run-gcpkms-test.sh - name: "testgcpkms-fail-task" # testgcpkms-fail-task runs in a non-GCE environment. @@ -2094,40 +2003,13 @@ tasks: - name: testazurekms-task commands: - - command: shell.exec - params: - shell: bash - script: |- - set -o errexit - ${PREPARE_SHELL} - source ./secrets-export.sh - cd src - echo "Copying files ... begin" - export AZUREKMS_RESOURCEGROUP=${AZUREKMS_RESOURCEGROUP} - export AZUREKMS_VMNAME=${AZUREKMS_VMNAME} - export AZUREKMS_PRIVATEKEYPATH=/tmp/testazurekms_privatekey - tar czf /tmp/mongo-python-driver.tgz . - AZUREKMS_SRC="/tmp/mongo-python-driver.tgz" \ - AZUREKMS_DST="~/" \ - $DRIVERS_TOOLS/.evergreen/csfle/azurekms/copy-file.sh - echo "Copying files ... end" - echo "Untarring file ... begin" - AZUREKMS_CMD="tar xf mongo-python-driver.tgz" \ - $DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh - echo "Untarring file ... end" - - command: shell.exec - type: test + - command: subprocess.exec params: - shell: bash - script: |- - set -o errexit - ${PREPARE_SHELL} - source ./secrets-export.sh - export AZUREKMS_RESOURCEGROUP=${AZUREKMS_RESOURCEGROUP} - export AZUREKMS_VMNAME=${AZUREKMS_VMNAME} - export AZUREKMS_PRIVATEKEYPATH=/tmp/testazurekms_privatekey - AZUREKMS_CMD="KEY_NAME=\"$AZUREKMS_KEYNAME\" KEY_VAULT_ENDPOINT=\"$AZUREKMS_KEYVAULTENDPOINT\" LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian10/master/latest/libmongocrypt.tar.gz SUCCESS=true TEST_FLE_AZURE_AUTO=1 ./.evergreen/tox.sh -m test-eg" \ - $DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh + binary: bash + working_dir: src + include_expansions_in_env: ["DRIVERS_TOOLS"] + args: + - .evergreen/run-azurekms-test.sh - name: testazurekms-fail-task commands: @@ -2137,23 +2019,14 @@ tasks: vars: VERSION: "latest" TOPOLOGY: "server" - - command: shell.exec + - command: subprocess.exec type: test params: - shell: bash - script: |- - set -o errexit - ${PREPARE_SHELL} - # Get azurekms credentials from the vault. - bash $DRIVERS_TOOLS/.evergreen/auth_aws/setup_secrets.sh drivers/azurekms - source ./secrets-export.sh - cd src - PYTHON_BINARY=/opt/mongodbtoolchain/v4/bin/python3 \ - KEY_NAME="${AZUREKMS_KEYNAME}" \ - KEY_VAULT_ENDPOINT="${AZUREKMS_KEYVAULTENDPOINT}" \ - LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian10/master/latest/libmongocrypt.tar.gz \ - SUCCESS=false TEST_FLE_AZURE_AUTO=1 \ - ./.evergreen/tox.sh -m test-eg + binary: bash + working_dir: src + include_expansions_in_env: ["DRIVERS_TOOLS"] + args: + - .evergreen/run-azurekms-fail-test.sh - name: "perf-6.0-standalone" tags: ["perf"] diff --git a/.evergreen/run-azurekms-fail-test.sh b/.evergreen/run-azurekms-fail-test.sh new file mode 100644 index 0000000000..13b34d01e2 --- /dev/null +++ b/.evergreen/run-azurekms-fail-test.sh @@ -0,0 +1,10 @@ +#!/bin/bash +set -o errexit # Exit the script with error if any of the commands fail + +. $DRIVERS_TOOLS/.evergreen/csfle/azurekms/setup-secrets.sh +PYTHON_BINARY=/opt/mongodbtoolchain/v4/bin/python3 \ + KEY_NAME="${AZUREKMS_KEYNAME}" \ + KEY_VAULT_ENDPOINT="${AZUREKMS_KEYVAULTENDPOINT}" \ + LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian10/master/latest/libmongocrypt.tar.gz \ + SUCCESS=false TEST_FLE_AZURE_AUTO=1 \ + ./.evergreen/tox.sh -m test-eg diff --git a/.evergreen/run-azurekms-test.sh b/.evergreen/run-azurekms-test.sh new file mode 100644 index 0000000000..d8fb3449f4 --- /dev/null +++ b/.evergreen/run-azurekms-test.sh @@ -0,0 +1,21 @@ +#!/bin/bash +set -o errexit # Exit the script with error if any of the commands fail + +source ${DRIVERS_TOOLS}/.evergreen/csfle/azurekms/secrets-export.sh +echo "Copying files ... begin" +export AZUREKMS_RESOURCEGROUP=${AZUREKMS_RESOURCEGROUP} +export AZUREKMS_VMNAME=${AZUREKMS_VMNAME} +export AZUREKMS_PRIVATEKEYPATH=/tmp/testazurekms_privatekey +tar czf /tmp/mongo-python-driver.tgz . +# shellcheck disable=SC2088 +AZUREKMS_SRC="/tmp/mongo-python-driver.tgz" AZUREKMS_DST="~/" \ + $DRIVERS_TOOLS/.evergreen/csfle/azurekms/copy-file.sh +echo "Copying files ... end" +echo "Untarring file ... begin" +AZUREKMS_CMD="tar xf mongo-python-driver.tgz" \ + $DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh +echo "Untarring file ... end" +echo "Running test ... begin" +AZUREKMS_CMD="KEY_NAME=\"$AZUREKMS_KEYNAME\" KEY_VAULT_ENDPOINT=\"$AZUREKMS_KEYVAULTENDPOINT\" LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian10/master/latest/libmongocrypt.tar.gz SUCCESS=true TEST_FLE_AZURE_AUTO=1 ./.evergreen/tox.sh -m test-eg" \ + $DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh +echo "Running test ... end" diff --git a/.evergreen/run-gcpkms-test.sh b/.evergreen/run-gcpkms-test.sh new file mode 100644 index 0000000000..221100de8a --- /dev/null +++ b/.evergreen/run-gcpkms-test.sh @@ -0,0 +1,18 @@ +#!/bin/bash +set -o errexit # Exit the script with error if any of the commands fail + +source ${DRIVERS_TOOLS}/.evergreen/csfle/gcpkms/secrets-export.sh +echo "Copying files ... begin" +export GCPKMS_GCLOUD=${GCPKMS_GCLOUD} +export GCPKMS_PROJECT=${GCPKMS_PROJECT} +export GCPKMS_ZONE=${GCPKMS_ZONE} +export GCPKMS_INSTANCENAME=${GCPKMS_INSTANCENAME} +tar czf /tmp/mongo-python-driver.tgz . +GCPKMS_SRC=/tmp/mongo-python-driver.tgz GCPKMS_DST=$GCPKMS_INSTANCENAME: $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/copy-file.sh +echo "Copying files ... end" +echo "Untarring file ... begin" +GCPKMS_CMD="tar xf mongo-python-driver.tgz" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh +echo "Untarring file ... end" +echo "Running test ... begin" +GCPKMS_CMD="SUCCESS=true TEST_FLE_GCP_AUTO=1 LIBMONGOCRYPT_URL=https://s3.amazonaws.com/mciuploads/libmongocrypt/debian10/master/latest/libmongocrypt.tar.gz ./.evergreen/tox.sh -m test-eg" $DRIVERS_TOOLS/.evergreen/csfle/gcpkms/run-command.sh +echo "Running test ... end" diff --git a/.evergreen/run-mongodb-oidc-test.sh b/.evergreen/run-mongodb-oidc-test.sh index ac7d29e5e9..5a14748464 100755 --- a/.evergreen/run-mongodb-oidc-test.sh +++ b/.evergreen/run-mongodb-oidc-test.sh @@ -15,15 +15,10 @@ if [ $OIDC_PROVIDER_NAME == "aws" ]; then fi # Get the drivers secrets. Use an existing secrets file first. - if [ ! -f "./secrets-export.sh" ]; then - bash ${DRIVERS_TOOLS}/.evergreen/auth_aws/setup_secrets.sh drivers/oidc - fi - source ./secrets-export.sh - - # # If the file did not have our creds, get them from the vault. - if [ -z "$OIDC_ATLAS_URI_SINGLE" ]; then - bash ${DRIVERS_TOOLS}/.evergreen/auth_aws/setup_secrets.sh drivers/oidc - source ./secrets-export.sh + if [ ! -f "${DRIVERS_TOOLS}/.evergreen/auth_oidc/secrets-export.sh" ]; then + . ${DRIVERS_TOOLS}/.evergreen/auth_oidc/setup-secrets.sh + else + source "${DRIVERS_TOOLS}/.evergreen/auth_oidc/secrets-export.sh" fi # Make the OIDC tokens. @@ -33,7 +28,7 @@ if [ $OIDC_PROVIDER_NAME == "aws" ]; then popd # Set up variables and run the test. - if [ -n "$LOCAL_OIDC_SERVER" ]; then + if [ -n "${LOCAL_OIDC_SERVER:-}" ]; then export MONGODB_URI=${MONGODB_URI:-"mongodb://localhost"} export MONGODB_URI_SINGLE="${MONGODB_URI}/?authMechanism=MONGODB-OIDC" export MONGODB_URI_MULTI="${MONGODB_URI}:27018/?authMechanism=MONGODB-OIDC&directConnection=true" @@ -49,7 +44,7 @@ if [ $OIDC_PROVIDER_NAME == "aws" ]; then export OIDC_ADMIN_PWD=$OIDC_ATLAS_PASSWORD elif [ $OIDC_PROVIDER_NAME == "azure" ]; then - if [ -z "${AZUREOIDC_AUDIENCE}" ]; then + if [ -z "${AZUREOIDC_AUDIENCE:-}" ]; then echo "Must specify an AZUREOIDC_AUDIENCE" exit 1 fi diff --git a/.evergreen/run-tests.sh b/.evergreen/run-tests.sh index 75d68e9e1b..35d8c26c0a 100755 --- a/.evergreen/run-tests.sh +++ b/.evergreen/run-tests.sh @@ -36,7 +36,7 @@ export PIP_QUIET=1 # Quiet by default python -c "import sys; sys.exit(sys.prefix == sys.base_prefix)" || (echo "Not inside a virtual env!"; exit 1) -# Try to source exported AWS Secrets +# Try to source local Drivers Secrets if [ -f ./secrets-export.sh ]; then echo "Sourcing secrets" source ./secrets-export.sh @@ -50,8 +50,13 @@ if [ "$AUTH" != "noauth" ]; then export DB_USER="mhuser" export DB_PASSWORD="pencil" elif [ ! -z "$TEST_SERVERLESS" ]; then + source ${DRIVERS_TOOLS}/.evergreen/serverless/secrets-export.sh export DB_USER=$SERVERLESS_ATLAS_USER export DB_PASSWORD=$SERVERLESS_ATLAS_PASSWORD + export MONGODB_URI="$SERVERLESS_URI" + echo "MONGODB_URI=$MONGODB_URI" + export SINGLE_MONGOS_LB_URI=$MONGODB_URI + export MULTI_MONGOS_LB_URI=$MONGODB_URI elif [ ! -z "$TEST_AUTH_OIDC" ]; then export DB_USER=$OIDC_ADMIN_USER export DB_PASSWORD=$OIDC_ADMIN_PWD @@ -187,8 +192,11 @@ if [ -n "$TEST_FLE_AZURE_AUTO" ] || [ -n "$TEST_FLE_GCP_AUTO" ]; then fi if [ -n "$TEST_INDEX_MANAGEMENT" ]; then + source $DRIVERS_TOOLS/.evergreen/atlas/secrets-export.sh export DB_USER="${DRIVERS_ATLAS_LAMBDA_USER}" + set +x export DB_PASSWORD="${DRIVERS_ATLAS_LAMBDA_PASSWORD}" + set -x TEST_ARGS="test/test_index_management.py" fi diff --git a/tox.ini b/tox.ini index 2e7227b865..d411020122 100644 --- a/tox.ini +++ b/tox.ini @@ -200,8 +200,8 @@ allowlist_externals = bash commands = bash .evergreen/setup-libmongocrypt.sh - bash {env:DRIVERS_TOOLS}/.evergreen/csfle/setup_secrets.sh - bash {env:DRIVERS_TOOLS}/.evergreen/csfle/start_servers.sh + bash {env:DRIVERS_TOOLS}/.evergreen/csfle/setup-secrets.sh + bash {env:DRIVERS_TOOLS}/.evergreen/csfle/start-servers.sh [testenv:teardown-encryption] description = tear down encryption assets and servers @@ -211,5 +211,5 @@ allowlist_externals = bash rm commands = - bash {env:DRIVERS_TOOLS}/.evergreen/csfle/stop_servers.sh + bash {env:DRIVERS_TOOLS}/.evergreen/csfle/stop-servers.sh rm -rf libmongocrypt/ libmongocrypt_git/ libmongocrypt.tar.gz mongocryptd.pid