From 0c0a7ebbd8396d404d04c39cd8092adc466c1f6d Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 9 Jan 2025 09:19:49 -0600 Subject: [PATCH 1/6] test against older drivers-tools --- .evergreen/scripts/prepare-resources.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.evergreen/scripts/prepare-resources.sh b/.evergreen/scripts/prepare-resources.sh index 3cfa2c4efd..629ff7c57f 100755 --- a/.evergreen/scripts/prepare-resources.sh +++ b/.evergreen/scripts/prepare-resources.sh @@ -11,6 +11,9 @@ if [ "$PROJECT" = "drivers-tools" ]; then cp -R $PROJECT_DIRECTORY/ $DRIVERS_TOOLS else git clone https://github.com/mongodb-labs/drivers-evergreen-tools.git $DRIVERS_TOOLS + pushd $DRIVERS_TOOLS + git checkout 93b20d9660fa5ef82b63d541d5a6f86f80ba4503 + popd fi echo "{ \"releases\": { \"default\": \"$MONGODB_BINARIES\" }}" >$MONGO_ORCHESTRATION_HOME/orchestration.config From d696857b5a9e2324b5f8d137a79939e05f3561f1 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Thu, 9 Jan 2025 10:11:42 -0600 Subject: [PATCH 2/6] try with pymongocrypt 1.11 --- .evergreen/run-tests.sh | 58 ++++++++++++------------- .evergreen/scripts/prepare-resources.sh | 3 -- requirements/encryption.txt | 2 +- 3 files changed, 30 insertions(+), 33 deletions(-) diff --git a/.evergreen/run-tests.sh b/.evergreen/run-tests.sh index 95fe10a6c3..58e56ff8a6 100755 --- a/.evergreen/run-tests.sh +++ b/.evergreen/run-tests.sh @@ -136,35 +136,35 @@ if [ -n "$TEST_ENCRYPTION" ] || [ -n "$TEST_FLE_AZURE_AUTO" ] || [ -n "$TEST_FLE fi python -m pip install '.[encryption]' - - # Use the nocrypto build to avoid dependency issues with older windows/python versions. - BASE=$(pwd)/libmongocrypt/nocrypto - if [ -f "${BASE}/lib/libmongocrypt.so" ]; then - PYMONGOCRYPT_LIB=${BASE}/lib/libmongocrypt.so - elif [ -f "${BASE}/lib/libmongocrypt.dylib" ]; then - PYMONGOCRYPT_LIB=${BASE}/lib/libmongocrypt.dylib - elif [ -f "${BASE}/bin/mongocrypt.dll" ]; then - PYMONGOCRYPT_LIB=${BASE}/bin/mongocrypt.dll - # libmongocrypt's windows dll is not marked executable. - chmod +x $PYMONGOCRYPT_LIB - PYMONGOCRYPT_LIB=$(cygpath -m $PYMONGOCRYPT_LIB) - elif [ -f "${BASE}/lib64/libmongocrypt.so" ]; then - PYMONGOCRYPT_LIB=${BASE}/lib64/libmongocrypt.so - else - echo "Cannot find libmongocrypt shared object file" - exit 1 - fi - export PYMONGOCRYPT_LIB - - # TODO: Test with 'pip install pymongocrypt' - if [ ! -d "libmongocrypt_git" ]; then - git clone https://github.com/mongodb/libmongocrypt.git libmongocrypt_git - fi - python -m pip install -U setuptools - python -m pip install ./libmongocrypt_git/bindings/python - python -c "import pymongocrypt; print('pymongocrypt version: '+pymongocrypt.__version__)" - python -c "import pymongocrypt; print('libmongocrypt version: '+pymongocrypt.libmongocrypt_version())" - # PATH is updated by PREPARE_SHELL for access to mongocryptd. + pip install pymongocrypt==1.11 + # # Use the nocrypto build to avoid dependency issues with older windows/python versions. + # BASE=$(pwd)/libmongocrypt/nocrypto + # if [ -f "${BASE}/lib/libmongocrypt.so" ]; then + # PYMONGOCRYPT_LIB=${BASE}/lib/libmongocrypt.so + # elif [ -f "${BASE}/lib/libmongocrypt.dylib" ]; then + # PYMONGOCRYPT_LIB=${BASE}/lib/libmongocrypt.dylib + # elif [ -f "${BASE}/bin/mongocrypt.dll" ]; then + # PYMONGOCRYPT_LIB=${BASE}/bin/mongocrypt.dll + # # libmongocrypt's windows dll is not marked executable. + # chmod +x $PYMONGOCRYPT_LIB + # PYMONGOCRYPT_LIB=$(cygpath -m $PYMONGOCRYPT_LIB) + # elif [ -f "${BASE}/lib64/libmongocrypt.so" ]; then + # PYMONGOCRYPT_LIB=${BASE}/lib64/libmongocrypt.so + # else + # echo "Cannot find libmongocrypt shared object file" + # exit 1 + # fi + # export PYMONGOCRYPT_LIB + + # # TODO: Test with 'pip install pymongocrypt' + # if [ ! -d "libmongocrypt_git" ]; then + # git clone https://github.com/mongodb/libmongocrypt.git libmongocrypt_git + # fi + # python -m pip install -U setuptools + # python -m pip install ./libmongocrypt_git/bindings/python + # python -c "import pymongocrypt; print('pymongocrypt version: '+pymongocrypt.__version__)" + # python -c "import pymongocrypt; print('libmongocrypt version: '+pymongocrypt.libmongocrypt_version())" + # # PATH is updated by PREPARE_SHELL for access to mongocryptd. fi if [ -n "$TEST_ENCRYPTION" ]; then diff --git a/.evergreen/scripts/prepare-resources.sh b/.evergreen/scripts/prepare-resources.sh index 629ff7c57f..3cfa2c4efd 100755 --- a/.evergreen/scripts/prepare-resources.sh +++ b/.evergreen/scripts/prepare-resources.sh @@ -11,9 +11,6 @@ if [ "$PROJECT" = "drivers-tools" ]; then cp -R $PROJECT_DIRECTORY/ $DRIVERS_TOOLS else git clone https://github.com/mongodb-labs/drivers-evergreen-tools.git $DRIVERS_TOOLS - pushd $DRIVERS_TOOLS - git checkout 93b20d9660fa5ef82b63d541d5a6f86f80ba4503 - popd fi echo "{ \"releases\": { \"default\": \"$MONGODB_BINARIES\" }}" >$MONGO_ORCHESTRATION_HOME/orchestration.config diff --git a/requirements/encryption.txt b/requirements/encryption.txt index 5962f5028f..f6df6a6eeb 100644 --- a/requirements/encryption.txt +++ b/requirements/encryption.txt @@ -1,3 +1,3 @@ pymongo-auth-aws>=1.1.0,<2.0.0 -pymongocrypt>=1.12.0,<2.0.0 +pymongocrypt>=1.11.0,<2.0.0 certifi;os.name=='nt' or sys_platform=='darwin' From 8070b8d6956127cc107c43a34d9b34ab0e212c41 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 10 Jan 2025 08:57:07 -0600 Subject: [PATCH 3/6] PYTHON-5014 Fix handling of async socket errors in kms request --- .evergreen/run-tests.sh | 58 +++++++++++++++--------------- pymongo/asynchronous/encryption.py | 3 ++ pymongo/synchronous/encryption.py | 3 ++ 3 files changed, 35 insertions(+), 29 deletions(-) diff --git a/.evergreen/run-tests.sh b/.evergreen/run-tests.sh index 58e56ff8a6..95fe10a6c3 100755 --- a/.evergreen/run-tests.sh +++ b/.evergreen/run-tests.sh @@ -136,35 +136,35 @@ if [ -n "$TEST_ENCRYPTION" ] || [ -n "$TEST_FLE_AZURE_AUTO" ] || [ -n "$TEST_FLE fi python -m pip install '.[encryption]' - pip install pymongocrypt==1.11 - # # Use the nocrypto build to avoid dependency issues with older windows/python versions. - # BASE=$(pwd)/libmongocrypt/nocrypto - # if [ -f "${BASE}/lib/libmongocrypt.so" ]; then - # PYMONGOCRYPT_LIB=${BASE}/lib/libmongocrypt.so - # elif [ -f "${BASE}/lib/libmongocrypt.dylib" ]; then - # PYMONGOCRYPT_LIB=${BASE}/lib/libmongocrypt.dylib - # elif [ -f "${BASE}/bin/mongocrypt.dll" ]; then - # PYMONGOCRYPT_LIB=${BASE}/bin/mongocrypt.dll - # # libmongocrypt's windows dll is not marked executable. - # chmod +x $PYMONGOCRYPT_LIB - # PYMONGOCRYPT_LIB=$(cygpath -m $PYMONGOCRYPT_LIB) - # elif [ -f "${BASE}/lib64/libmongocrypt.so" ]; then - # PYMONGOCRYPT_LIB=${BASE}/lib64/libmongocrypt.so - # else - # echo "Cannot find libmongocrypt shared object file" - # exit 1 - # fi - # export PYMONGOCRYPT_LIB - - # # TODO: Test with 'pip install pymongocrypt' - # if [ ! -d "libmongocrypt_git" ]; then - # git clone https://github.com/mongodb/libmongocrypt.git libmongocrypt_git - # fi - # python -m pip install -U setuptools - # python -m pip install ./libmongocrypt_git/bindings/python - # python -c "import pymongocrypt; print('pymongocrypt version: '+pymongocrypt.__version__)" - # python -c "import pymongocrypt; print('libmongocrypt version: '+pymongocrypt.libmongocrypt_version())" - # # PATH is updated by PREPARE_SHELL for access to mongocryptd. + + # Use the nocrypto build to avoid dependency issues with older windows/python versions. + BASE=$(pwd)/libmongocrypt/nocrypto + if [ -f "${BASE}/lib/libmongocrypt.so" ]; then + PYMONGOCRYPT_LIB=${BASE}/lib/libmongocrypt.so + elif [ -f "${BASE}/lib/libmongocrypt.dylib" ]; then + PYMONGOCRYPT_LIB=${BASE}/lib/libmongocrypt.dylib + elif [ -f "${BASE}/bin/mongocrypt.dll" ]; then + PYMONGOCRYPT_LIB=${BASE}/bin/mongocrypt.dll + # libmongocrypt's windows dll is not marked executable. + chmod +x $PYMONGOCRYPT_LIB + PYMONGOCRYPT_LIB=$(cygpath -m $PYMONGOCRYPT_LIB) + elif [ -f "${BASE}/lib64/libmongocrypt.so" ]; then + PYMONGOCRYPT_LIB=${BASE}/lib64/libmongocrypt.so + else + echo "Cannot find libmongocrypt shared object file" + exit 1 + fi + export PYMONGOCRYPT_LIB + + # TODO: Test with 'pip install pymongocrypt' + if [ ! -d "libmongocrypt_git" ]; then + git clone https://github.com/mongodb/libmongocrypt.git libmongocrypt_git + fi + python -m pip install -U setuptools + python -m pip install ./libmongocrypt_git/bindings/python + python -c "import pymongocrypt; print('pymongocrypt version: '+pymongocrypt.__version__)" + python -c "import pymongocrypt; print('libmongocrypt version: '+pymongocrypt.libmongocrypt_version())" + # PATH is updated by PREPARE_SHELL for access to mongocryptd. fi if [ -n "$TEST_ENCRYPTION" ]; then diff --git a/pymongo/asynchronous/encryption.py b/pymongo/asynchronous/encryption.py index 1cf165e6a2..152ac07cb1 100644 --- a/pymongo/asynchronous/encryption.py +++ b/pymongo/asynchronous/encryption.py @@ -213,6 +213,9 @@ async def kms_request(self, kms_context: MongoCryptKmsContext) -> None: if not data: raise OSError("KMS connection closed") kms_context.feed(data) + # Async raises an OSError instead of returning empty bytes + except OSError as err: + raise OSError("KMS connection closed") from err except MongoCryptError: raise # Propagate MongoCryptError errors directly. except Exception as exc: diff --git a/pymongo/synchronous/encryption.py b/pymongo/synchronous/encryption.py index ef49855059..86e052303a 100644 --- a/pymongo/synchronous/encryption.py +++ b/pymongo/synchronous/encryption.py @@ -213,6 +213,9 @@ def kms_request(self, kms_context: MongoCryptKmsContext) -> None: if not data: raise OSError("KMS connection closed") kms_context.feed(data) + # Async raises an OSError instead of returning empty bytes + except OSError as err: + raise OSError("KMS connection closed") from err except MongoCryptError: raise # Propagate MongoCryptError errors directly. except Exception as exc: From 4f007c670ad49e277b064f82ee865f74eb37cace Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 10 Jan 2025 08:57:39 -0600 Subject: [PATCH 4/6] undo requirements change --- requirements/encryption.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/encryption.txt b/requirements/encryption.txt index f6df6a6eeb..5962f5028f 100644 --- a/requirements/encryption.txt +++ b/requirements/encryption.txt @@ -1,3 +1,3 @@ pymongo-auth-aws>=1.1.0,<2.0.0 -pymongocrypt>=1.11.0,<2.0.0 +pymongocrypt>=1.12.0,<2.0.0 certifi;os.name=='nt' or sys_platform=='darwin' From 8232f2672633048249d2af813e7a64818d9d572d Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 10 Jan 2025 11:05:44 -0600 Subject: [PATCH 5/6] Address rhel error --- test/asynchronous/test_encryption.py | 12 ++++++++---- test/test_encryption.py | 12 ++++++++---- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/test/asynchronous/test_encryption.py b/test/asynchronous/test_encryption.py index 48f791ac16..6a4af2fa0f 100644 --- a/test/asynchronous/test_encryption.py +++ b/test/asynchronous/test_encryption.py @@ -2163,7 +2163,8 @@ async def test_01_aws(self): # 127.0.0.1:9001: ('Certificate does not contain any `subjectAltName`s.',) key["endpoint"] = "127.0.0.1:9001" with self.assertRaisesRegex( - EncryptionError, "IP address mismatch|wronghost|IPAddressMismatch|Certificate" + EncryptionError, + "IP address mismatch|wronghost|IPAddressMismatch|Certificate|SSL handshake failed", ): await self.client_encryption_invalid_hostname.create_data_key("aws", key) @@ -2180,7 +2181,8 @@ async def test_02_azure(self): await self.client_encryption_expired.create_data_key("azure", key) # Invalid cert hostname error. with self.assertRaisesRegex( - EncryptionError, "IP address mismatch|wronghost|IPAddressMismatch|Certificate" + EncryptionError, + "IP address mismatch|wronghost|IPAddressMismatch|Certificate|SSL handshake failed", ): await self.client_encryption_invalid_hostname.create_data_key("azure", key) @@ -2197,7 +2199,8 @@ async def test_03_gcp(self): await self.client_encryption_expired.create_data_key("gcp", key) # Invalid cert hostname error. with self.assertRaisesRegex( - EncryptionError, "IP address mismatch|wronghost|IPAddressMismatch|Certificate" + EncryptionError, + "IP address mismatch|wronghost|IPAddressMismatch|Certificate|SSL handshake failed", ): await self.client_encryption_invalid_hostname.create_data_key("gcp", key) @@ -2211,7 +2214,8 @@ async def test_04_kmip(self): await self.client_encryption_expired.create_data_key("kmip") # Invalid cert hostname error. with self.assertRaisesRegex( - EncryptionError, "IP address mismatch|wronghost|IPAddressMismatch|Certificate" + EncryptionError, + "IP address mismatch|wronghost|IPAddressMismatch|Certificate|SSL handshake failed", ): await self.client_encryption_invalid_hostname.create_data_key("kmip") diff --git a/test/test_encryption.py b/test/test_encryption.py index daa5fd5d4c..4bfa10abe3 100644 --- a/test/test_encryption.py +++ b/test/test_encryption.py @@ -2155,7 +2155,8 @@ def test_01_aws(self): # 127.0.0.1:9001: ('Certificate does not contain any `subjectAltName`s.',) key["endpoint"] = "127.0.0.1:9001" with self.assertRaisesRegex( - EncryptionError, "IP address mismatch|wronghost|IPAddressMismatch|Certificate" + EncryptionError, + "IP address mismatch|wronghost|IPAddressMismatch|Certificate|SSL handshake failed", ): self.client_encryption_invalid_hostname.create_data_key("aws", key) @@ -2172,7 +2173,8 @@ def test_02_azure(self): self.client_encryption_expired.create_data_key("azure", key) # Invalid cert hostname error. with self.assertRaisesRegex( - EncryptionError, "IP address mismatch|wronghost|IPAddressMismatch|Certificate" + EncryptionError, + "IP address mismatch|wronghost|IPAddressMismatch|Certificate|SSL handshake failed", ): self.client_encryption_invalid_hostname.create_data_key("azure", key) @@ -2189,7 +2191,8 @@ def test_03_gcp(self): self.client_encryption_expired.create_data_key("gcp", key) # Invalid cert hostname error. with self.assertRaisesRegex( - EncryptionError, "IP address mismatch|wronghost|IPAddressMismatch|Certificate" + EncryptionError, + "IP address mismatch|wronghost|IPAddressMismatch|Certificate|SSL handshake failed", ): self.client_encryption_invalid_hostname.create_data_key("gcp", key) @@ -2203,7 +2206,8 @@ def test_04_kmip(self): self.client_encryption_expired.create_data_key("kmip") # Invalid cert hostname error. with self.assertRaisesRegex( - EncryptionError, "IP address mismatch|wronghost|IPAddressMismatch|Certificate" + EncryptionError, + "IP address mismatch|wronghost|IPAddressMismatch|Certificate|SSL handshake failed", ): self.client_encryption_invalid_hostname.create_data_key("kmip") From 321b3e36f4ded1dfdae04ef1d59c89b709f7ebe7 Mon Sep 17 00:00:00 2001 From: Steven Silvester Date: Fri, 10 Jan 2025 12:12:55 -0600 Subject: [PATCH 6/6] address review --- pymongo/asynchronous/encryption.py | 12 ++++++++---- pymongo/synchronous/encryption.py | 12 ++++++++---- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/pymongo/asynchronous/encryption.py b/pymongo/asynchronous/encryption.py index 152ac07cb1..98ab68527c 100644 --- a/pymongo/asynchronous/encryption.py +++ b/pymongo/asynchronous/encryption.py @@ -213,16 +213,20 @@ async def kms_request(self, kms_context: MongoCryptKmsContext) -> None: if not data: raise OSError("KMS connection closed") kms_context.feed(data) - # Async raises an OSError instead of returning empty bytes - except OSError as err: - raise OSError("KMS connection closed") from err except MongoCryptError: raise # Propagate MongoCryptError errors directly. except Exception as exc: # Wrap I/O errors in PyMongo exceptions. if isinstance(exc, BLOCKING_IO_ERRORS): exc = socket.timeout("timed out") - _raise_connection_failure(address, exc, timeout_details=_get_timeout_details(opts)) + # Async raises an OSError instead of returning empty bytes. + if isinstance(exc, OSError): + msg_prefix = "KMS connection closed" + else: + msg_prefix = None + _raise_connection_failure( + address, exc, msg_prefix=msg_prefix, timeout_details=_get_timeout_details(opts) + ) finally: conn.close() except MongoCryptError: diff --git a/pymongo/synchronous/encryption.py b/pymongo/synchronous/encryption.py index 86e052303a..d41169861f 100644 --- a/pymongo/synchronous/encryption.py +++ b/pymongo/synchronous/encryption.py @@ -213,16 +213,20 @@ def kms_request(self, kms_context: MongoCryptKmsContext) -> None: if not data: raise OSError("KMS connection closed") kms_context.feed(data) - # Async raises an OSError instead of returning empty bytes - except OSError as err: - raise OSError("KMS connection closed") from err except MongoCryptError: raise # Propagate MongoCryptError errors directly. except Exception as exc: # Wrap I/O errors in PyMongo exceptions. if isinstance(exc, BLOCKING_IO_ERRORS): exc = socket.timeout("timed out") - _raise_connection_failure(address, exc, timeout_details=_get_timeout_details(opts)) + # Async raises an OSError instead of returning empty bytes. + if isinstance(exc, OSError): + msg_prefix = "KMS connection closed" + else: + msg_prefix = None + _raise_connection_failure( + address, exc, msg_prefix=msg_prefix, timeout_details=_get_timeout_details(opts) + ) finally: conn.close() except MongoCryptError: